Beispiel #1
0
    def walk_filesystem(self):
        candidates = {}
        rootpath = utils.from_unicode(self.ROOTPATH)
        for dirpath, dirnames, files in os.walk(rootpath):
            try:
                dirpath = utils.to_unicode(dirpath)
            except UnicodeDecodeError as e:
                continue
            rel_dirpath = os.path.relpath(dirpath, start=self.ROOTPATH)
            logger.debug("'%s' '%s'" % (dirpath, rel_dirpath))
            if rel_dirpath != '.':
                objname = utils.to_standard_sep(rel_dirpath)
                candidates[objname] = self.none_info()
            for filename in files:
                try:
                    filename = utils.to_unicode(filename)
                except UnicodeDecodeError as e:
                    continue
                if rel_dirpath == '.':
                    prefix = ""
                else:
                    prefix = utils.to_standard_sep(rel_dirpath)
                objname = utils.join_objname(prefix, filename)
                candidates[objname] = self.none_info()

        db_cands = dict((name, self.none_info()) for name in self.list_files())
        candidates.update(db_cands)
        logger.debug("Candidates: %s" % candidates)
        return candidates
Beispiel #2
0
    def walk_filesystem(self):
        candidates = {}
        rootpath = utils.from_unicode(self.ROOTPATH)
        for dirpath, dirnames, files in os.walk(rootpath):
            try:
                dirpath = utils.to_unicode(dirpath)
            except UnicodeDecodeError as e:
                continue
            rel_dirpath = os.path.relpath(dirpath, start=self.ROOTPATH)
            logger.debug("'%s' '%s'" % (dirpath, rel_dirpath))
            if rel_dirpath != ".":
                objname = utils.to_standard_sep(rel_dirpath)
                candidates[objname] = self.none_info()
            for filename in files:
                try:
                    filename = utils.to_unicode(filename)
                except UnicodeDecodeError as e:
                    continue
                if rel_dirpath == ".":
                    prefix = ""
                else:
                    prefix = utils.to_standard_sep(rel_dirpath)
                objname = utils.join_objname(prefix, filename)
                candidates[objname] = self.none_info()

        db_cands = dict((name, self.none_info()) for name in self.list_files())
        candidates.update(db_cands)
        logger.debug("Candidates: %s" % candidates)
        return candidates
Beispiel #3
0
def file_is_open(path):
    for proc in psutil.process_iter():
        try:
            flist = psutil_open_files(proc)
            for nt in flist:
                try:
                    nt_path = utils.to_unicode(nt.path)
                except UnicodeDecodeError as e:
                    continue
                if nt_path == path:
                    return True
        except psutil.Error:
            pass
    return False
Beispiel #4
0
def file_is_open(path):
    for proc in psutil.process_iter():
        try:
            flist = psutil_open_files(proc)
            for nt in flist:
                try:
                    nt_path = utils.to_unicode(nt.path)
                except UnicodeDecodeError as e:
                    continue
                if nt_path == path:
                    return True
        except psutil.Error:
            pass
    return False
Beispiel #5
0
 def handle_path(path, rec=False):
     try:
         path = utils.to_unicode(path)
     except UnicodeDecodeError as e:
         return
     if path.startswith(self.CACHEPATH):
         return
     rel_path = os.path.relpath(path, start=self.ROOTPATH)
     if rel_path == '.':
         return
     objname = utils.to_standard_sep(rel_path)
     leaves = self.get_dir_contents(objname) if rec else None
     with self.probe_candidates.lock() as d:
         d[objname] = self.none_info()
         if rec:
             for leaf in leaves:
                 d[leaf] = self.none_info()
Beispiel #6
0
 def handle_path(path, rec=False):
     try:
         path = utils.to_unicode(path)
     except UnicodeDecodeError as e:
         return
     if path.startswith(self.CACHEPATH):
         return
     rel_path = os.path.relpath(path, start=self.ROOTPATH)
     if rel_path == ".":
         return
     objname = utils.to_standard_sep(rel_path)
     leaves = self.get_dir_contents(objname) if rec else None
     with self.probe_candidates.lock() as d:
         d[objname] = self.none_info()
         if rec:
             for leaf in leaves:
                 d[leaf] = self.none_info()
Beispiel #7
0
    def __init__(self, auth_url, auth_token, container, local_root_path,
                 *args, **kwargs):
        check_encoding()
        auth_url = utils.to_unicode(auth_url)
        auth_token = utils.to_unicode(auth_token)
        container = utils.to_unicode(container)
        local_root_path = utils.to_unicode(local_root_path)
        self.auth_url = utils.normalize_standard_suffix(auth_url)
        self.auth_token = auth_token
        self.container = utils.normalize_standard_suffix(container)

        self.ignore_ssl = kwargs.get("ignore_ssl", False)
        if self.ignore_ssl:
            https.patch_ignore_ssl()
        elif kwargs.get('ca_certs', None):
            https.patch_with_certs(kwargs['ca_certs'])

        self.endpoint = self._get_pithos_client(
            auth_url, auth_token, container)

        container_exists = self.check_container_exists(container)

        home_dir = utils.to_unicode(os.path.expanduser('~'))
        default_settings_path = join_path(home_dir, GLOBAL_SETTINGS_NAME)
        self.settings_path = utils.to_unicode(
            kwargs.get("agkyra_path", default_settings_path))
        self.create_dir(self.settings_path, mode=stat.S_IRWXU)

        self.instances_path = join_path(self.settings_path, INSTANCES_NAME)
        self.create_dir(self.instances_path)

        self.local_root_path = utils.normalize_local_suffix(local_root_path)
        local_root_path_exists = os.path.isdir(self.local_root_path)

        self.cache_name = utils.to_unicode(
            kwargs.get("cache_name", DEFAULT_CACHE_NAME))
        self.cache_path = join_path(self.local_root_path, self.cache_name)

        self.cache_hide_name = utils.to_unicode(
            kwargs.get("cache_hide_name", DEFAULT_CACHE_HIDE_NAME))
        self.cache_hide_path = join_path(self.cache_path, self.cache_hide_name)

        self.cache_stage_name = utils.to_unicode(
            kwargs.get("cache_stage_name", DEFAULT_CACHE_STAGE_NAME))
        self.cache_stage_path = join_path(self.cache_path,
                                          self.cache_stage_name)

        self.cache_fetch_name = utils.to_unicode(
            kwargs.get("cache_fetch_name", DEFAULT_CACHE_FETCH_NAME))
        self.cache_fetch_path = join_path(self.cache_path,
                                          self.cache_fetch_name)

        self.user_id = self.endpoint.account
        self.instance = get_instance(
            [self.auth_url, self.user_id,
             self.container, self.local_root_path])
        self.instance_path = join_path(self.instances_path, self.instance)
        self.create_dir(self.instance_path)

        self.dbname = utils.to_unicode(kwargs.get("dbname", DEFAULT_DBNAME))
        self.full_dbname = join_path(self.instance_path, self.dbname)
        self.syncer_dbtuple = common.DBTuple(
            dbtype=database.SyncerDB,
            dbname=self.full_dbname)

        db_existed = os.path.isfile(self.full_dbname)
        if not db_existed:
            database.initialize(self.syncer_dbtuple)

        self.mtime_lag = 0
        self.case_insensitive = False

        if not db_existed:
            self.set_localfs_enabled(True)
            self.create_local_dirs()
            self.set_pithos_enabled(True)
            if not container_exists:
                self.mk_container(container)
        else:
            if not local_root_path_exists:
                self.set_localfs_enabled(False)
            else:
                self.create_local_dirs()
            if not container_exists:
                self.set_pithos_enabled(False)

        self.heartbeat = ThreadSafeDict()
        self.action_max_wait = kwargs.get("action_max_wait",
                                          DEFAULT_ACTION_MAX_WAIT)
        self.pithos_list_interval = kwargs.get("pithos_list_interval",
                                               DEFAULT_PITHOS_LIST_INTERVAL)

        self.connection_retry_limit = kwargs.get(
            "connection_retry_limit", DEFAULT_CONNECTION_RETRY_LIMIT)
        self.endpoint.CONNECTION_RETRY_LIMIT = self.connection_retry_limit
        self.max_alive_sync_threads = kwargs.get(
            "max_alive_sync_threads", DEFAULT_MAX_ALIVE_SYNC_THREADS)
        self.messager = Messager()
Beispiel #8
0
    def __init__(self, auth_url, auth_token, container, local_root_path, *args,
                 **kwargs):
        check_encoding()
        auth_url = utils.to_unicode(auth_url)
        auth_token = utils.to_unicode(auth_token)
        container = utils.to_unicode(container)
        local_root_path = utils.to_unicode(local_root_path)
        self.auth_url = utils.normalize_standard_suffix(auth_url)
        self.auth_token = auth_token
        self.container = utils.normalize_standard_suffix(container)

        self.ignore_ssl = kwargs.get("ignore_ssl", False)
        if self.ignore_ssl:
            https.patch_ignore_ssl()
        elif kwargs.get('ca_certs', None):
            https.patch_with_certs(kwargs['ca_certs'])

        self.endpoint = self._get_pithos_client(auth_url, auth_token,
                                                container)

        container_exists = self.check_container_exists(container)

        home_dir = utils.to_unicode(os.path.expanduser('~'))
        default_settings_path = join_path(home_dir, GLOBAL_SETTINGS_NAME)
        self.settings_path = utils.to_unicode(
            kwargs.get("agkyra_path", default_settings_path))
        self.create_dir(self.settings_path, mode=stat.S_IRWXU)

        self.instances_path = join_path(self.settings_path, INSTANCES_NAME)
        self.create_dir(self.instances_path)

        self.local_root_path = utils.normalize_local_suffix(local_root_path)
        local_root_path_exists = os.path.isdir(self.local_root_path)

        self.cache_name = utils.to_unicode(
            kwargs.get("cache_name", DEFAULT_CACHE_NAME))
        self.cache_path = join_path(self.local_root_path, self.cache_name)

        self.cache_hide_name = utils.to_unicode(
            kwargs.get("cache_hide_name", DEFAULT_CACHE_HIDE_NAME))
        self.cache_hide_path = join_path(self.cache_path, self.cache_hide_name)

        self.cache_stage_name = utils.to_unicode(
            kwargs.get("cache_stage_name", DEFAULT_CACHE_STAGE_NAME))
        self.cache_stage_path = join_path(self.cache_path,
                                          self.cache_stage_name)

        self.cache_fetch_name = utils.to_unicode(
            kwargs.get("cache_fetch_name", DEFAULT_CACHE_FETCH_NAME))
        self.cache_fetch_path = join_path(self.cache_path,
                                          self.cache_fetch_name)

        self.user_id = self.endpoint.account
        self.instance = get_instance([
            self.auth_url, self.user_id, self.container, self.local_root_path
        ])
        self.instance_path = join_path(self.instances_path, self.instance)
        self.create_dir(self.instance_path)

        self.dbname = utils.to_unicode(kwargs.get("dbname", DEFAULT_DBNAME))
        self.full_dbname = join_path(self.instance_path, self.dbname)
        self.syncer_dbtuple = common.DBTuple(dbtype=database.SyncerDB,
                                             dbname=self.full_dbname)

        db_existed = os.path.isfile(self.full_dbname)
        if not db_existed:
            database.initialize(self.syncer_dbtuple)

        self.mtime_lag = 0
        self.case_insensitive = False

        if not db_existed:
            self.set_localfs_enabled(True)
            self.create_local_dirs()
            self.set_pithos_enabled(True)
            if not container_exists:
                self.mk_container(container)
        else:
            if not local_root_path_exists:
                self.set_localfs_enabled(False)
            else:
                self.create_local_dirs()
            if not container_exists:
                self.set_pithos_enabled(False)

        self.heartbeat = ThreadSafeDict()
        self.action_max_wait = kwargs.get("action_max_wait",
                                          DEFAULT_ACTION_MAX_WAIT)
        self.pithos_list_interval = kwargs.get("pithos_list_interval",
                                               DEFAULT_PITHOS_LIST_INTERVAL)

        self.connection_retry_limit = kwargs.get(
            "connection_retry_limit", DEFAULT_CONNECTION_RETRY_LIMIT)
        self.endpoint.CONNECTION_RETRY_LIMIT = self.connection_retry_limit
        self.max_alive_sync_threads = kwargs.get(
            "max_alive_sync_threads", DEFAULT_MAX_ALIVE_SYNC_THREADS)
        self.messager = Messager()