def walk_filesystem(self): candidates = {} rootpath = utils.from_unicode(self.ROOTPATH) for dirpath, dirnames, files in os.walk(rootpath): try: dirpath = utils.to_unicode(dirpath) except UnicodeDecodeError as e: continue rel_dirpath = os.path.relpath(dirpath, start=self.ROOTPATH) logger.debug("'%s' '%s'" % (dirpath, rel_dirpath)) if rel_dirpath != '.': objname = utils.to_standard_sep(rel_dirpath) candidates[objname] = self.none_info() for filename in files: try: filename = utils.to_unicode(filename) except UnicodeDecodeError as e: continue if rel_dirpath == '.': prefix = "" else: prefix = utils.to_standard_sep(rel_dirpath) objname = utils.join_objname(prefix, filename) candidates[objname] = self.none_info() db_cands = dict((name, self.none_info()) for name in self.list_files()) candidates.update(db_cands) logger.debug("Candidates: %s" % candidates) return candidates
def walk_filesystem(self): candidates = {} rootpath = utils.from_unicode(self.ROOTPATH) for dirpath, dirnames, files in os.walk(rootpath): try: dirpath = utils.to_unicode(dirpath) except UnicodeDecodeError as e: continue rel_dirpath = os.path.relpath(dirpath, start=self.ROOTPATH) logger.debug("'%s' '%s'" % (dirpath, rel_dirpath)) if rel_dirpath != ".": objname = utils.to_standard_sep(rel_dirpath) candidates[objname] = self.none_info() for filename in files: try: filename = utils.to_unicode(filename) except UnicodeDecodeError as e: continue if rel_dirpath == ".": prefix = "" else: prefix = utils.to_standard_sep(rel_dirpath) objname = utils.join_objname(prefix, filename) candidates[objname] = self.none_info() db_cands = dict((name, self.none_info()) for name in self.list_files()) candidates.update(db_cands) logger.debug("Candidates: %s" % candidates) return candidates
def file_is_open(path): for proc in psutil.process_iter(): try: flist = psutil_open_files(proc) for nt in flist: try: nt_path = utils.to_unicode(nt.path) except UnicodeDecodeError as e: continue if nt_path == path: return True except psutil.Error: pass return False
def handle_path(path, rec=False): try: path = utils.to_unicode(path) except UnicodeDecodeError as e: return if path.startswith(self.CACHEPATH): return rel_path = os.path.relpath(path, start=self.ROOTPATH) if rel_path == '.': return objname = utils.to_standard_sep(rel_path) leaves = self.get_dir_contents(objname) if rec else None with self.probe_candidates.lock() as d: d[objname] = self.none_info() if rec: for leaf in leaves: d[leaf] = self.none_info()
def handle_path(path, rec=False): try: path = utils.to_unicode(path) except UnicodeDecodeError as e: return if path.startswith(self.CACHEPATH): return rel_path = os.path.relpath(path, start=self.ROOTPATH) if rel_path == ".": return objname = utils.to_standard_sep(rel_path) leaves = self.get_dir_contents(objname) if rec else None with self.probe_candidates.lock() as d: d[objname] = self.none_info() if rec: for leaf in leaves: d[leaf] = self.none_info()
def __init__(self, auth_url, auth_token, container, local_root_path, *args, **kwargs): check_encoding() auth_url = utils.to_unicode(auth_url) auth_token = utils.to_unicode(auth_token) container = utils.to_unicode(container) local_root_path = utils.to_unicode(local_root_path) self.auth_url = utils.normalize_standard_suffix(auth_url) self.auth_token = auth_token self.container = utils.normalize_standard_suffix(container) self.ignore_ssl = kwargs.get("ignore_ssl", False) if self.ignore_ssl: https.patch_ignore_ssl() elif kwargs.get('ca_certs', None): https.patch_with_certs(kwargs['ca_certs']) self.endpoint = self._get_pithos_client( auth_url, auth_token, container) container_exists = self.check_container_exists(container) home_dir = utils.to_unicode(os.path.expanduser('~')) default_settings_path = join_path(home_dir, GLOBAL_SETTINGS_NAME) self.settings_path = utils.to_unicode( kwargs.get("agkyra_path", default_settings_path)) self.create_dir(self.settings_path, mode=stat.S_IRWXU) self.instances_path = join_path(self.settings_path, INSTANCES_NAME) self.create_dir(self.instances_path) self.local_root_path = utils.normalize_local_suffix(local_root_path) local_root_path_exists = os.path.isdir(self.local_root_path) self.cache_name = utils.to_unicode( kwargs.get("cache_name", DEFAULT_CACHE_NAME)) self.cache_path = join_path(self.local_root_path, self.cache_name) self.cache_hide_name = utils.to_unicode( kwargs.get("cache_hide_name", DEFAULT_CACHE_HIDE_NAME)) self.cache_hide_path = join_path(self.cache_path, self.cache_hide_name) self.cache_stage_name = utils.to_unicode( kwargs.get("cache_stage_name", DEFAULT_CACHE_STAGE_NAME)) self.cache_stage_path = join_path(self.cache_path, self.cache_stage_name) self.cache_fetch_name = utils.to_unicode( kwargs.get("cache_fetch_name", DEFAULT_CACHE_FETCH_NAME)) self.cache_fetch_path = join_path(self.cache_path, self.cache_fetch_name) self.user_id = self.endpoint.account self.instance = get_instance( [self.auth_url, self.user_id, self.container, self.local_root_path]) self.instance_path = join_path(self.instances_path, self.instance) self.create_dir(self.instance_path) self.dbname = utils.to_unicode(kwargs.get("dbname", DEFAULT_DBNAME)) self.full_dbname = join_path(self.instance_path, self.dbname) self.syncer_dbtuple = common.DBTuple( dbtype=database.SyncerDB, dbname=self.full_dbname) db_existed = os.path.isfile(self.full_dbname) if not db_existed: database.initialize(self.syncer_dbtuple) self.mtime_lag = 0 self.case_insensitive = False if not db_existed: self.set_localfs_enabled(True) self.create_local_dirs() self.set_pithos_enabled(True) if not container_exists: self.mk_container(container) else: if not local_root_path_exists: self.set_localfs_enabled(False) else: self.create_local_dirs() if not container_exists: self.set_pithos_enabled(False) self.heartbeat = ThreadSafeDict() self.action_max_wait = kwargs.get("action_max_wait", DEFAULT_ACTION_MAX_WAIT) self.pithos_list_interval = kwargs.get("pithos_list_interval", DEFAULT_PITHOS_LIST_INTERVAL) self.connection_retry_limit = kwargs.get( "connection_retry_limit", DEFAULT_CONNECTION_RETRY_LIMIT) self.endpoint.CONNECTION_RETRY_LIMIT = self.connection_retry_limit self.max_alive_sync_threads = kwargs.get( "max_alive_sync_threads", DEFAULT_MAX_ALIVE_SYNC_THREADS) self.messager = Messager()
def __init__(self, auth_url, auth_token, container, local_root_path, *args, **kwargs): check_encoding() auth_url = utils.to_unicode(auth_url) auth_token = utils.to_unicode(auth_token) container = utils.to_unicode(container) local_root_path = utils.to_unicode(local_root_path) self.auth_url = utils.normalize_standard_suffix(auth_url) self.auth_token = auth_token self.container = utils.normalize_standard_suffix(container) self.ignore_ssl = kwargs.get("ignore_ssl", False) if self.ignore_ssl: https.patch_ignore_ssl() elif kwargs.get('ca_certs', None): https.patch_with_certs(kwargs['ca_certs']) self.endpoint = self._get_pithos_client(auth_url, auth_token, container) container_exists = self.check_container_exists(container) home_dir = utils.to_unicode(os.path.expanduser('~')) default_settings_path = join_path(home_dir, GLOBAL_SETTINGS_NAME) self.settings_path = utils.to_unicode( kwargs.get("agkyra_path", default_settings_path)) self.create_dir(self.settings_path, mode=stat.S_IRWXU) self.instances_path = join_path(self.settings_path, INSTANCES_NAME) self.create_dir(self.instances_path) self.local_root_path = utils.normalize_local_suffix(local_root_path) local_root_path_exists = os.path.isdir(self.local_root_path) self.cache_name = utils.to_unicode( kwargs.get("cache_name", DEFAULT_CACHE_NAME)) self.cache_path = join_path(self.local_root_path, self.cache_name) self.cache_hide_name = utils.to_unicode( kwargs.get("cache_hide_name", DEFAULT_CACHE_HIDE_NAME)) self.cache_hide_path = join_path(self.cache_path, self.cache_hide_name) self.cache_stage_name = utils.to_unicode( kwargs.get("cache_stage_name", DEFAULT_CACHE_STAGE_NAME)) self.cache_stage_path = join_path(self.cache_path, self.cache_stage_name) self.cache_fetch_name = utils.to_unicode( kwargs.get("cache_fetch_name", DEFAULT_CACHE_FETCH_NAME)) self.cache_fetch_path = join_path(self.cache_path, self.cache_fetch_name) self.user_id = self.endpoint.account self.instance = get_instance([ self.auth_url, self.user_id, self.container, self.local_root_path ]) self.instance_path = join_path(self.instances_path, self.instance) self.create_dir(self.instance_path) self.dbname = utils.to_unicode(kwargs.get("dbname", DEFAULT_DBNAME)) self.full_dbname = join_path(self.instance_path, self.dbname) self.syncer_dbtuple = common.DBTuple(dbtype=database.SyncerDB, dbname=self.full_dbname) db_existed = os.path.isfile(self.full_dbname) if not db_existed: database.initialize(self.syncer_dbtuple) self.mtime_lag = 0 self.case_insensitive = False if not db_existed: self.set_localfs_enabled(True) self.create_local_dirs() self.set_pithos_enabled(True) if not container_exists: self.mk_container(container) else: if not local_root_path_exists: self.set_localfs_enabled(False) else: self.create_local_dirs() if not container_exists: self.set_pithos_enabled(False) self.heartbeat = ThreadSafeDict() self.action_max_wait = kwargs.get("action_max_wait", DEFAULT_ACTION_MAX_WAIT) self.pithos_list_interval = kwargs.get("pithos_list_interval", DEFAULT_PITHOS_LIST_INTERVAL) self.connection_retry_limit = kwargs.get( "connection_retry_limit", DEFAULT_CONNECTION_RETRY_LIMIT) self.endpoint.CONNECTION_RETRY_LIMIT = self.connection_retry_limit self.max_alive_sync_threads = kwargs.get( "max_alive_sync_threads", DEFAULT_MAX_ALIVE_SYNC_THREADS) self.messager = Messager()