def walk_filesystem(self):
candidates = {}
rootpath = utils.from_unicode(self.ROOTPATH)
for dirpath, dirnames, files in os.walk(rootpath):
try:
dirpath = utils.to_unicode(dirpath)
except UnicodeDecodeError as e:
continue
rel_dirpath = os.path.relpath(dirpath, start=self.ROOTPATH)
logger.debug("'%s' '%s'" % (dirpath, rel_dirpath))
if rel_dirpath != '.':
objname = utils.to_standard_sep(rel_dirpath)
candidates[objname] = self.none_info()
for filename in files:
try:
filename = utils.to_unicode(filename)
except UnicodeDecodeError as e:
continue
if rel_dirpath == '.':
prefix = ""
else:
prefix = utils.to_standard_sep(rel_dirpath)
objname = utils.join_objname(prefix, filename)
candidates[objname] = self.none_info()
db_cands = dict((name, self.none_info()) for name in self.list_files())
candidates.update(db_cands)
logger.debug("Candidates: %s" % candidates)
return candidates
def walk_filesystem(self):
candidates = {}
rootpath = utils.from_unicode(self.ROOTPATH)
for dirpath, dirnames, files in os.walk(rootpath):
try:
dirpath = utils.to_unicode(dirpath)
except UnicodeDecodeError as e:
continue
rel_dirpath = os.path.relpath(dirpath, start=self.ROOTPATH)
logger.debug("'%s' '%s'" % (dirpath, rel_dirpath))
if rel_dirpath != ".":
objname = utils.to_standard_sep(rel_dirpath)
candidates[objname] = self.none_info()
for filename in files:
try:
filename = utils.to_unicode(filename)
except UnicodeDecodeError as e:
continue
if rel_dirpath == ".":
prefix = ""
else:
prefix = utils.to_standard_sep(rel_dirpath)
objname = utils.join_objname(prefix, filename)
candidates[objname] = self.none_info()
db_cands = dict((name, self.none_info()) for name in self.list_files())
candidates.update(db_cands)
logger.debug("Candidates: %s" % candidates)
return candidates
def file_is_open(path):
for proc in psutil.process_iter():
try:
flist = psutil_open_files(proc)
for nt in flist:
try:
nt_path = utils.to_unicode(nt.path)
except UnicodeDecodeError as e:
continue
if nt_path == path:
return True
except psutil.Error:
pass
return False
def file_is_open(path):
for proc in psutil.process_iter():
try:
flist = psutil_open_files(proc)
for nt in flist:
try:
nt_path = utils.to_unicode(nt.path)
except UnicodeDecodeError as e:
continue
if nt_path == path:
return True
except psutil.Error:
pass
return False
def handle_path(path, rec=False):
try:
path = utils.to_unicode(path)
except UnicodeDecodeError as e:
return
if path.startswith(self.CACHEPATH):
return
rel_path = os.path.relpath(path, start=self.ROOTPATH)
if rel_path == '.':
return
objname = utils.to_standard_sep(rel_path)
leaves = self.get_dir_contents(objname) if rec else None
with self.probe_candidates.lock() as d:
d[objname] = self.none_info()
if rec:
for leaf in leaves:
d[leaf] = self.none_info()
def handle_path(path, rec=False):
try:
path = utils.to_unicode(path)
except UnicodeDecodeError as e:
return
if path.startswith(self.CACHEPATH):
return
rel_path = os.path.relpath(path, start=self.ROOTPATH)
if rel_path == ".":
return
objname = utils.to_standard_sep(rel_path)
leaves = self.get_dir_contents(objname) if rec else None
with self.probe_candidates.lock() as d:
d[objname] = self.none_info()
if rec:
for leaf in leaves:
d[leaf] = self.none_info()
def __init__(self, auth_url, auth_token, container, local_root_path,
*args, **kwargs):
check_encoding()
auth_url = utils.to_unicode(auth_url)
auth_token = utils.to_unicode(auth_token)
container = utils.to_unicode(container)
local_root_path = utils.to_unicode(local_root_path)
self.auth_url = utils.normalize_standard_suffix(auth_url)
self.auth_token = auth_token
self.container = utils.normalize_standard_suffix(container)
self.ignore_ssl = kwargs.get("ignore_ssl", False)
if self.ignore_ssl:
https.patch_ignore_ssl()
elif kwargs.get('ca_certs', None):
https.patch_with_certs(kwargs['ca_certs'])
self.endpoint = self._get_pithos_client(
auth_url, auth_token, container)
container_exists = self.check_container_exists(container)
home_dir = utils.to_unicode(os.path.expanduser('~'))
default_settings_path = join_path(home_dir, GLOBAL_SETTINGS_NAME)
self.settings_path = utils.to_unicode(
kwargs.get("agkyra_path", default_settings_path))
self.create_dir(self.settings_path, mode=stat.S_IRWXU)
self.instances_path = join_path(self.settings_path, INSTANCES_NAME)
self.create_dir(self.instances_path)
self.local_root_path = utils.normalize_local_suffix(local_root_path)
local_root_path_exists = os.path.isdir(self.local_root_path)
self.cache_name = utils.to_unicode(
kwargs.get("cache_name", DEFAULT_CACHE_NAME))
self.cache_path = join_path(self.local_root_path, self.cache_name)
self.cache_hide_name = utils.to_unicode(
kwargs.get("cache_hide_name", DEFAULT_CACHE_HIDE_NAME))
self.cache_hide_path = join_path(self.cache_path, self.cache_hide_name)
self.cache_stage_name = utils.to_unicode(
kwargs.get("cache_stage_name", DEFAULT_CACHE_STAGE_NAME))
self.cache_stage_path = join_path(self.cache_path,
self.cache_stage_name)
self.cache_fetch_name = utils.to_unicode(
kwargs.get("cache_fetch_name", DEFAULT_CACHE_FETCH_NAME))
self.cache_fetch_path = join_path(self.cache_path,
self.cache_fetch_name)
self.user_id = self.endpoint.account
self.instance = get_instance(
[self.auth_url, self.user_id,
self.container, self.local_root_path])
self.instance_path = join_path(self.instances_path, self.instance)
self.create_dir(self.instance_path)
self.dbname = utils.to_unicode(kwargs.get("dbname", DEFAULT_DBNAME))
self.full_dbname = join_path(self.instance_path, self.dbname)
self.syncer_dbtuple = common.DBTuple(
dbtype=database.SyncerDB,
dbname=self.full_dbname)
db_existed = os.path.isfile(self.full_dbname)
if not db_existed:
database.initialize(self.syncer_dbtuple)
self.mtime_lag = 0
self.case_insensitive = False
if not db_existed:
self.set_localfs_enabled(True)
self.create_local_dirs()
self.set_pithos_enabled(True)
if not container_exists:
self.mk_container(container)
else:
if not local_root_path_exists:
self.set_localfs_enabled(False)
else:
self.create_local_dirs()
if not container_exists:
self.set_pithos_enabled(False)
self.heartbeat = ThreadSafeDict()
self.action_max_wait = kwargs.get("action_max_wait",
DEFAULT_ACTION_MAX_WAIT)
self.pithos_list_interval = kwargs.get("pithos_list_interval",
DEFAULT_PITHOS_LIST_INTERVAL)
self.connection_retry_limit = kwargs.get(
"connection_retry_limit", DEFAULT_CONNECTION_RETRY_LIMIT)
self.endpoint.CONNECTION_RETRY_LIMIT = self.connection_retry_limit
self.max_alive_sync_threads = kwargs.get(
"max_alive_sync_threads", DEFAULT_MAX_ALIVE_SYNC_THREADS)
self.messager = Messager()
def __init__(self, auth_url, auth_token, container, local_root_path, *args,
**kwargs):
check_encoding()
auth_url = utils.to_unicode(auth_url)
auth_token = utils.to_unicode(auth_token)
container = utils.to_unicode(container)
local_root_path = utils.to_unicode(local_root_path)
self.auth_url = utils.normalize_standard_suffix(auth_url)
self.auth_token = auth_token
self.container = utils.normalize_standard_suffix(container)
self.ignore_ssl = kwargs.get("ignore_ssl", False)
if self.ignore_ssl:
https.patch_ignore_ssl()
elif kwargs.get('ca_certs', None):
https.patch_with_certs(kwargs['ca_certs'])
self.endpoint = self._get_pithos_client(auth_url, auth_token,
container)
container_exists = self.check_container_exists(container)
home_dir = utils.to_unicode(os.path.expanduser('~'))
default_settings_path = join_path(home_dir, GLOBAL_SETTINGS_NAME)
self.settings_path = utils.to_unicode(
kwargs.get("agkyra_path", default_settings_path))
self.create_dir(self.settings_path, mode=stat.S_IRWXU)
self.instances_path = join_path(self.settings_path, INSTANCES_NAME)
self.create_dir(self.instances_path)
self.local_root_path = utils.normalize_local_suffix(local_root_path)
local_root_path_exists = os.path.isdir(self.local_root_path)
self.cache_name = utils.to_unicode(
kwargs.get("cache_name", DEFAULT_CACHE_NAME))
self.cache_path = join_path(self.local_root_path, self.cache_name)
self.cache_hide_name = utils.to_unicode(
kwargs.get("cache_hide_name", DEFAULT_CACHE_HIDE_NAME))
self.cache_hide_path = join_path(self.cache_path, self.cache_hide_name)
self.cache_stage_name = utils.to_unicode(
kwargs.get("cache_stage_name", DEFAULT_CACHE_STAGE_NAME))
self.cache_stage_path = join_path(self.cache_path,
self.cache_stage_name)
self.cache_fetch_name = utils.to_unicode(
kwargs.get("cache_fetch_name", DEFAULT_CACHE_FETCH_NAME))
self.cache_fetch_path = join_path(self.cache_path,
self.cache_fetch_name)
self.user_id = self.endpoint.account
self.instance = get_instance([
self.auth_url, self.user_id, self.container, self.local_root_path
])
self.instance_path = join_path(self.instances_path, self.instance)
self.create_dir(self.instance_path)
self.dbname = utils.to_unicode(kwargs.get("dbname", DEFAULT_DBNAME))
self.full_dbname = join_path(self.instance_path, self.dbname)
self.syncer_dbtuple = common.DBTuple(dbtype=database.SyncerDB,
dbname=self.full_dbname)
db_existed = os.path.isfile(self.full_dbname)
if not db_existed:
database.initialize(self.syncer_dbtuple)
self.mtime_lag = 0
self.case_insensitive = False
if not db_existed:
self.set_localfs_enabled(True)
self.create_local_dirs()
self.set_pithos_enabled(True)
if not container_exists:
self.mk_container(container)
else:
if not local_root_path_exists:
self.set_localfs_enabled(False)
else:
self.create_local_dirs()
if not container_exists:
self.set_pithos_enabled(False)
self.heartbeat = ThreadSafeDict()
self.action_max_wait = kwargs.get("action_max_wait",
DEFAULT_ACTION_MAX_WAIT)
self.pithos_list_interval = kwargs.get("pithos_list_interval",
DEFAULT_PITHOS_LIST_INTERVAL)
self.connection_retry_limit = kwargs.get(
"connection_retry_limit", DEFAULT_CONNECTION_RETRY_LIMIT)
self.endpoint.CONNECTION_RETRY_LIMIT = self.connection_retry_limit
self.max_alive_sync_threads = kwargs.get(
"max_alive_sync_threads", DEFAULT_MAX_ALIVE_SYNC_THREADS)
self.messager = Messager()
