def _register_fetch_name(self, db, filename): f = utils.hash_string(filename) + "_" + \ utils.str_time_stamp() fetch_name = utils.join_path(self.cache_fetch_name, f) self.fetch_name = fetch_name db.insert_cachename(fetch_name, self.SIGNATURE, filename) return utils.join_path(self.cache_path, fetch_name)
def setUpClass(cls): cnf = AgkyraConfig() cloud_conf = cnf.get('cloud', 'test') if cloud_conf is None: print "Define a 'test' cloud in %s" % CONFIG_PATH exit() AUTHENTICATION_URL = cloud_conf['url'] TOKEN = cloud_conf['token'] cls.ID = "ΑΓΚΥΡΑTEST" + str(random.random()).split('.')[1] cls.LOCAL_ROOT_PATH = utils.join_path(TMP, cls.ID) cls.settings = SyncerSettings( auth_url=AUTHENTICATION_URL, auth_token=TOKEN, container=cls.ID, local_root_path=cls.LOCAL_ROOT_PATH, action_max_wait=5, ignore_ssl=True) cls.master = PithosFileClient(cls.settings) cls.slave = localfs_client.LocalfsFileClient(cls.settings) cls.s = FileSyncer(cls.settings, cls.master, cls.slave) cls.pithos = cls.master.endpoint cls.pithos.create_container(cls.ID) cls.db = database.get_db(cls.s.syncer_dbtuple) m = cls.s.get_next_message(block=True) assert isinstance(m, messaging.PithosSyncEnabled) m = cls.s.get_next_message(block=True) assert isinstance(m, messaging.LocalfsSyncEnabled)
def test_012_cachename(self): fil = "φ012" f_path = self.get_path(fil) with open(f_path, "w") as f: f.write("content") state = self.db.get_state(self.s.SLAVE, fil) handle = self.slave.prepare_target(state) hidden_filename = utils.join_path( handle.cache_hide_name, utils.hash_string(handle.objname)) hidden_path = handle.get_path_in_cache(hidden_filename) self.assertFalse(os.path.isfile(hidden_path)) client_db = database.get_db(self.slave.client_dbtuple) self.assertIsNone(client_db.get_cachename(hidden_filename)) handle.move_file() self.assertTrue(os.path.isfile(hidden_path)) self.assertIsNotNone(client_db.get_cachename(hidden_filename)) handle.move_file() self.assertTrue(os.path.isfile(hidden_path)) shutil.move(hidden_path, f_path) self.assertIsNotNone(client_db.get_cachename(hidden_filename)) handle.move_file() self.assertTrue(os.path.isfile(hidden_path)) # open file to cause busy error f = open(hidden_path, "r") with self.assertRaises(common.BusyError): handle.hide_file()
def stash_file(self): stash_name = mk_stash_name(self.objname) stash_path = utils.join_path(self.rootpath, stash_name) msg = messaging.ConflictStashMessage(objname=self.objname, stash_name=stash_name, logger=logger) self.settings.messager.put(msg) os.rename(self.hidden_path, stash_path)
def _register_hidden_name(self, db, filename): f = utils.hash_string(filename) hide_filename = utils.join_path(self.cache_hide_name, f) self.hidden_filename = hide_filename self.hidden_path = self.get_path_in_cache(self.hidden_filename) if db.get_cachename(hide_filename): return False db.insert_cachename(hide_filename, self.SIGNATURE, filename) return True
def _register_stage_name(self, db, filename): f = utils.hash_string(filename) stage_filename = utils.join_path(self.cache_stage_name, f) self.stage_filename = stage_filename stage_path = self.get_path_in_cache(stage_filename) self.staged_path = stage_path if db.get_cachename(stage_filename): return False db.insert_cachename(stage_filename, self.SIGNATURE, filename) return True
def __init__(self, settings): self.settings = settings self.SIGNATURE = "LocalfsFileClient" self.ROOTPATH = settings.local_root_path self.CACHEPATH = settings.cache_path self.syncer_dbtuple = settings.syncer_dbtuple client_dbname = self.SIGNATURE + ".db" self.client_dbtuple = common.DBTuple( dbtype=database.ClientDB, dbname=utils.join_path(settings.instance_path, client_dbname) ) database.initialize(self.client_dbtuple) self.probe_candidates = utils.ThreadSafeDict() self.check_enabled()
def __init__(self, settings): self.settings = settings self.SIGNATURE = "LocalfsFileClient" self.ROOTPATH = settings.local_root_path self.CACHEPATH = settings.cache_path self.syncer_dbtuple = settings.syncer_dbtuple client_dbname = self.SIGNATURE + '.db' self.client_dbtuple = common.DBTuple(dbtype=database.ClientDB, dbname=utils.join_path( settings.instance_path, client_dbname)) database.initialize(self.client_dbtuple) self.probe_candidates = utils.ThreadSafeDict() self.check_enabled()
def __init__(self, settings): self.settings = settings self.SIGNATURE = "PithosFileClient" self.auth_url = settings.auth_url self.auth_token = settings.auth_token self.container = settings.container self.syncer_dbtuple = settings.syncer_dbtuple client_dbname = self.SIGNATURE+'.db' self.client_dbtuple = common.DBTuple( dbtype=database.ClientDB, dbname=utils.join_path(settings.instance_path, client_dbname)) database.initialize(self.client_dbtuple) self.endpoint = settings.endpoint self.last_modification = "0000-00-00" self.probe_candidates = utils.ThreadSafeDict() self.check_enabled()
def __init__(self, settings): self.settings = settings self.SIGNATURE = "PithosFileClient" self.auth_url = settings.auth_url self.auth_token = settings.auth_token self.container = settings.container self.syncer_dbtuple = settings.syncer_dbtuple client_dbname = self.SIGNATURE + '.db' self.client_dbtuple = common.DBTuple(dbtype=database.ClientDB, dbname=utils.join_path( settings.instance_path, client_dbname)) database.initialize(self.client_dbtuple) self.endpoint = settings.endpoint self.last_modification = "0000-00-00" self.probe_candidates = utils.ThreadSafeDict() self.check_enabled()
def __init__(self, client, target_state): self.client = client settings = client.settings self.settings = settings self.SIGNATURE = "LocalfsTargetHandle" self.rootpath = settings.local_root_path self.cache_hide_name = settings.cache_hide_name self.cache_hide_path = settings.cache_hide_path self.cache_path = settings.cache_path self.syncer_dbtuple = settings.syncer_dbtuple self.client_dbtuple = client.client_dbtuple self.mtime_lag = settings.mtime_lag self.target_state = target_state self.objname = target_state.objname self.fspath = utils.join_path(self.rootpath, self.objname) self.hidden_filename = None self.hidden_path = None
def __init__(self, client, source_state): self.client = client settings = client.settings self.settings = settings self.SIGNATURE = "LocalfsSourceHandle" self.rootpath = settings.local_root_path self.cache_stage_name = settings.cache_stage_name self.cache_stage_path = settings.cache_stage_path self.cache_path = settings.cache_path self.syncer_dbtuple = settings.syncer_dbtuple self.client_dbtuple = client.client_dbtuple self.source_state = source_state self.objname = source_state.objname self.fspath = utils.join_path(self.rootpath, self.objname) self.stage_filename = None self.staged_path = None self.heartbeat = settings.heartbeat if info_of_regular_file(self.source_state.info): self.stage_file()
def _local_path_changes(self, name, state): local_path = utils.join_path(self.ROOTPATH, name) return local_path_changes(self.settings, local_path, state)
def get_path_in_cache(self, name): return utils.join_path(self.cache_path, name)
def test_0002_notifier_local(self): f_out = "φ0002out" f_cache = "φ0002cache" f_upd = "φ0002upd" f_ren = "φ0002ren" f_cached = "φ0002cached" dbefore = "δ0002before" f_out_path = self.get_path(f_out) f_cache_path = self.get_path(f_cache) f_upd_path = self.get_path(f_upd) f_ren_path = self.get_path(f_ren) f_cached_path = os.path.join( self.settings.cache_path, f_cached) dbefore_path = self.get_path(dbefore) open(f_out_path, "a").close() open(f_cache_path, "a").close() open(f_upd_path, "a").close() open(f_ren_path, "a").close() open(f_cached_path, "a").close() os.mkdir(dbefore_path) notifier = self.slave.notifier() candidates = self.slave.list_candidate_files() self.assertEqual(candidates, []) fafter = "φ0002after" fafter_path = self.get_path(fafter) dafter = "δ0002after" dafter_path = self.get_path(dafter) open(fafter_path, "a").close() os.mkdir(dafter_path) time.sleep(1) candidates = self.slave.list_candidate_files() self.assertEqual(sorted(candidates), sorted([fafter, dafter])) os.rename(f_cache_path, utils.join_path(self.settings.cache_path, f_cache)) os.rename(f_out_path, utils.join_path(TMP, f_out)) with open(f_upd_path, "a") as f: f.write("upd") f_in = "φ0002in" f_in_path = self.get_path(f_in) f_in_orig_path = utils.join_path(TMP, f_in) open(f_in_orig_path, "a").close() os.rename(f_in_orig_path, f_in_path) f_ren_new = "φ0002ren_new" f_ren_new_path = self.get_path(f_ren_new) os.rename(f_ren_path, f_ren_new_path) f_cached_out_path = self.get_path(f_cached) os.rename(f_cached_path, f_cached_out_path) time.sleep(1) candidates = self.slave.list_candidate_files() self.assertEqual(sorted(candidates), sorted([fafter, dafter, f_in, f_out, f_upd, f_ren, f_ren_new, f_cache, f_cached])) notifier.stop()
def __init__(self, auth_url, auth_token, container, local_root_path, *args, **kwargs): check_encoding() auth_url = utils.to_unicode(auth_url) auth_token = utils.to_unicode(auth_token) container = utils.to_unicode(container) local_root_path = utils.to_unicode(local_root_path) self.auth_url = utils.normalize_standard_suffix(auth_url) self.auth_token = auth_token self.container = utils.normalize_standard_suffix(container) self.ignore_ssl = kwargs.get("ignore_ssl", False) if self.ignore_ssl: https.patch_ignore_ssl() elif kwargs.get('ca_certs', None): https.patch_with_certs(kwargs['ca_certs']) self.endpoint = self._get_pithos_client( auth_url, auth_token, container) container_exists = self.check_container_exists(container) home_dir = utils.to_unicode(os.path.expanduser('~')) default_settings_path = join_path(home_dir, GLOBAL_SETTINGS_NAME) self.settings_path = utils.to_unicode( kwargs.get("agkyra_path", default_settings_path)) self.create_dir(self.settings_path, mode=stat.S_IRWXU) self.instances_path = join_path(self.settings_path, INSTANCES_NAME) self.create_dir(self.instances_path) self.local_root_path = utils.normalize_local_suffix(local_root_path) local_root_path_exists = os.path.isdir(self.local_root_path) self.cache_name = utils.to_unicode( kwargs.get("cache_name", DEFAULT_CACHE_NAME)) self.cache_path = join_path(self.local_root_path, self.cache_name) self.cache_hide_name = utils.to_unicode( kwargs.get("cache_hide_name", DEFAULT_CACHE_HIDE_NAME)) self.cache_hide_path = join_path(self.cache_path, self.cache_hide_name) self.cache_stage_name = utils.to_unicode( kwargs.get("cache_stage_name", DEFAULT_CACHE_STAGE_NAME)) self.cache_stage_path = join_path(self.cache_path, self.cache_stage_name) self.cache_fetch_name = utils.to_unicode( kwargs.get("cache_fetch_name", DEFAULT_CACHE_FETCH_NAME)) self.cache_fetch_path = join_path(self.cache_path, self.cache_fetch_name) self.user_id = self.endpoint.account self.instance = get_instance( [self.auth_url, self.user_id, self.container, self.local_root_path]) self.instance_path = join_path(self.instances_path, self.instance) self.create_dir(self.instance_path) self.dbname = utils.to_unicode(kwargs.get("dbname", DEFAULT_DBNAME)) self.full_dbname = join_path(self.instance_path, self.dbname) self.syncer_dbtuple = common.DBTuple( dbtype=database.SyncerDB, dbname=self.full_dbname) db_existed = os.path.isfile(self.full_dbname) if not db_existed: database.initialize(self.syncer_dbtuple) self.mtime_lag = 0 self.case_insensitive = False if not db_existed: self.set_localfs_enabled(True) self.create_local_dirs() self.set_pithos_enabled(True) if not container_exists: self.mk_container(container) else: if not local_root_path_exists: self.set_localfs_enabled(False) else: self.create_local_dirs() if not container_exists: self.set_pithos_enabled(False) self.heartbeat = ThreadSafeDict() self.action_max_wait = kwargs.get("action_max_wait", DEFAULT_ACTION_MAX_WAIT) self.pithos_list_interval = kwargs.get("pithos_list_interval", DEFAULT_PITHOS_LIST_INTERVAL) self.connection_retry_limit = kwargs.get( "connection_retry_limit", DEFAULT_CONNECTION_RETRY_LIMIT) self.endpoint.CONNECTION_RETRY_LIMIT = self.connection_retry_limit self.max_alive_sync_threads = kwargs.get( "max_alive_sync_threads", DEFAULT_MAX_ALIVE_SYNC_THREADS) self.messager = Messager()
def __init__(self, auth_url, auth_token, container, local_root_path, *args, **kwargs): check_encoding() auth_url = utils.to_unicode(auth_url) auth_token = utils.to_unicode(auth_token) container = utils.to_unicode(container) local_root_path = utils.to_unicode(local_root_path) self.auth_url = utils.normalize_standard_suffix(auth_url) self.auth_token = auth_token self.container = utils.normalize_standard_suffix(container) self.ignore_ssl = kwargs.get("ignore_ssl", False) if self.ignore_ssl: https.patch_ignore_ssl() elif kwargs.get('ca_certs', None): https.patch_with_certs(kwargs['ca_certs']) self.endpoint = self._get_pithos_client(auth_url, auth_token, container) container_exists = self.check_container_exists(container) home_dir = utils.to_unicode(os.path.expanduser('~')) default_settings_path = join_path(home_dir, GLOBAL_SETTINGS_NAME) self.settings_path = utils.to_unicode( kwargs.get("agkyra_path", default_settings_path)) self.create_dir(self.settings_path, mode=stat.S_IRWXU) self.instances_path = join_path(self.settings_path, INSTANCES_NAME) self.create_dir(self.instances_path) self.local_root_path = utils.normalize_local_suffix(local_root_path) local_root_path_exists = os.path.isdir(self.local_root_path) self.cache_name = utils.to_unicode( kwargs.get("cache_name", DEFAULT_CACHE_NAME)) self.cache_path = join_path(self.local_root_path, self.cache_name) self.cache_hide_name = utils.to_unicode( kwargs.get("cache_hide_name", DEFAULT_CACHE_HIDE_NAME)) self.cache_hide_path = join_path(self.cache_path, self.cache_hide_name) self.cache_stage_name = utils.to_unicode( kwargs.get("cache_stage_name", DEFAULT_CACHE_STAGE_NAME)) self.cache_stage_path = join_path(self.cache_path, self.cache_stage_name) self.cache_fetch_name = utils.to_unicode( kwargs.get("cache_fetch_name", DEFAULT_CACHE_FETCH_NAME)) self.cache_fetch_path = join_path(self.cache_path, self.cache_fetch_name) self.user_id = self.endpoint.account self.instance = get_instance([ self.auth_url, self.user_id, self.container, self.local_root_path ]) self.instance_path = join_path(self.instances_path, self.instance) self.create_dir(self.instance_path) self.dbname = utils.to_unicode(kwargs.get("dbname", DEFAULT_DBNAME)) self.full_dbname = join_path(self.instance_path, self.dbname) self.syncer_dbtuple = common.DBTuple(dbtype=database.SyncerDB, dbname=self.full_dbname) db_existed = os.path.isfile(self.full_dbname) if not db_existed: database.initialize(self.syncer_dbtuple) self.mtime_lag = 0 self.case_insensitive = False if not db_existed: self.set_localfs_enabled(True) self.create_local_dirs() self.set_pithos_enabled(True) if not container_exists: self.mk_container(container) else: if not local_root_path_exists: self.set_localfs_enabled(False) else: self.create_local_dirs() if not container_exists: self.set_pithos_enabled(False) self.heartbeat = ThreadSafeDict() self.action_max_wait = kwargs.get("action_max_wait", DEFAULT_ACTION_MAX_WAIT) self.pithos_list_interval = kwargs.get("pithos_list_interval", DEFAULT_PITHOS_LIST_INTERVAL) self.connection_retry_limit = kwargs.get( "connection_retry_limit", DEFAULT_CONNECTION_RETRY_LIMIT) self.endpoint.CONNECTION_RETRY_LIMIT = self.connection_retry_limit self.max_alive_sync_threads = kwargs.get( "max_alive_sync_threads", DEFAULT_MAX_ALIVE_SYNC_THREADS) self.messager = Messager()