def _register_fetch_name(self, db, filename):
f = utils.hash_string(filename) + "_" + \
utils.str_time_stamp()
fetch_name = utils.join_path(self.cache_fetch_name, f)
self.fetch_name = fetch_name
db.insert_cachename(fetch_name, self.SIGNATURE, filename)
return utils.join_path(self.cache_path, fetch_name)
def _register_fetch_name(self, db, filename):
f = utils.hash_string(filename) + "_" + \
utils.str_time_stamp()
fetch_name = utils.join_path(self.cache_fetch_name, f)
self.fetch_name = fetch_name
db.insert_cachename(fetch_name, self.SIGNATURE, filename)
return utils.join_path(self.cache_path, fetch_name)
def setUpClass(cls):
cnf = AgkyraConfig()
cloud_conf = cnf.get('cloud', 'test')
if cloud_conf is None:
print "Define a 'test' cloud in %s" % CONFIG_PATH
exit()
AUTHENTICATION_URL = cloud_conf['url']
TOKEN = cloud_conf['token']
cls.ID = "ΑΓΚΥΡΑTEST" + str(random.random()).split('.')[1]
cls.LOCAL_ROOT_PATH = utils.join_path(TMP, cls.ID)
cls.settings = SyncerSettings(
auth_url=AUTHENTICATION_URL,
auth_token=TOKEN,
container=cls.ID,
local_root_path=cls.LOCAL_ROOT_PATH,
action_max_wait=5,
ignore_ssl=True)
cls.master = PithosFileClient(cls.settings)
cls.slave = localfs_client.LocalfsFileClient(cls.settings)
cls.s = FileSyncer(cls.settings, cls.master, cls.slave)
cls.pithos = cls.master.endpoint
cls.pithos.create_container(cls.ID)
cls.db = database.get_db(cls.s.syncer_dbtuple)
m = cls.s.get_next_message(block=True)
assert isinstance(m, messaging.PithosSyncEnabled)
m = cls.s.get_next_message(block=True)
assert isinstance(m, messaging.LocalfsSyncEnabled)
def test_012_cachename(self):
fil = "φ012"
f_path = self.get_path(fil)
with open(f_path, "w") as f:
f.write("content")
state = self.db.get_state(self.s.SLAVE, fil)
handle = self.slave.prepare_target(state)
hidden_filename = utils.join_path(
handle.cache_hide_name, utils.hash_string(handle.objname))
hidden_path = handle.get_path_in_cache(hidden_filename)
self.assertFalse(os.path.isfile(hidden_path))
client_db = database.get_db(self.slave.client_dbtuple)
self.assertIsNone(client_db.get_cachename(hidden_filename))
handle.move_file()
self.assertTrue(os.path.isfile(hidden_path))
self.assertIsNotNone(client_db.get_cachename(hidden_filename))
handle.move_file()
self.assertTrue(os.path.isfile(hidden_path))
shutil.move(hidden_path, f_path)
self.assertIsNotNone(client_db.get_cachename(hidden_filename))
handle.move_file()
self.assertTrue(os.path.isfile(hidden_path))
# open file to cause busy error
f = open(hidden_path, "r")
with self.assertRaises(common.BusyError):
handle.hide_file()
def stash_file(self):
stash_name = mk_stash_name(self.objname)
stash_path = utils.join_path(self.rootpath, stash_name)
msg = messaging.ConflictStashMessage(objname=self.objname,
stash_name=stash_name,
logger=logger)
self.settings.messager.put(msg)
os.rename(self.hidden_path, stash_path)
def _register_hidden_name(self, db, filename):
f = utils.hash_string(filename)
hide_filename = utils.join_path(self.cache_hide_name, f)
self.hidden_filename = hide_filename
self.hidden_path = self.get_path_in_cache(self.hidden_filename)
if db.get_cachename(hide_filename):
return False
db.insert_cachename(hide_filename, self.SIGNATURE, filename)
return True
def _register_hidden_name(self, db, filename):
f = utils.hash_string(filename)
hide_filename = utils.join_path(self.cache_hide_name, f)
self.hidden_filename = hide_filename
self.hidden_path = self.get_path_in_cache(self.hidden_filename)
if db.get_cachename(hide_filename):
return False
db.insert_cachename(hide_filename, self.SIGNATURE, filename)
return True
def _register_stage_name(self, db, filename):
f = utils.hash_string(filename)
stage_filename = utils.join_path(self.cache_stage_name, f)
self.stage_filename = stage_filename
stage_path = self.get_path_in_cache(stage_filename)
self.staged_path = stage_path
if db.get_cachename(stage_filename):
return False
db.insert_cachename(stage_filename, self.SIGNATURE, filename)
return True
def _register_stage_name(self, db, filename):
f = utils.hash_string(filename)
stage_filename = utils.join_path(self.cache_stage_name, f)
self.stage_filename = stage_filename
stage_path = self.get_path_in_cache(stage_filename)
self.staged_path = stage_path
if db.get_cachename(stage_filename):
return False
db.insert_cachename(stage_filename, self.SIGNATURE, filename)
return True
def __init__(self, settings):
self.settings = settings
self.SIGNATURE = "LocalfsFileClient"
self.ROOTPATH = settings.local_root_path
self.CACHEPATH = settings.cache_path
self.syncer_dbtuple = settings.syncer_dbtuple
client_dbname = self.SIGNATURE + ".db"
self.client_dbtuple = common.DBTuple(
dbtype=database.ClientDB, dbname=utils.join_path(settings.instance_path, client_dbname)
)
database.initialize(self.client_dbtuple)
self.probe_candidates = utils.ThreadSafeDict()
self.check_enabled()
def __init__(self, settings):
self.settings = settings
self.SIGNATURE = "LocalfsFileClient"
self.ROOTPATH = settings.local_root_path
self.CACHEPATH = settings.cache_path
self.syncer_dbtuple = settings.syncer_dbtuple
client_dbname = self.SIGNATURE + '.db'
self.client_dbtuple = common.DBTuple(dbtype=database.ClientDB,
dbname=utils.join_path(
settings.instance_path,
client_dbname))
database.initialize(self.client_dbtuple)
self.probe_candidates = utils.ThreadSafeDict()
self.check_enabled()
def __init__(self, settings):
self.settings = settings
self.SIGNATURE = "PithosFileClient"
self.auth_url = settings.auth_url
self.auth_token = settings.auth_token
self.container = settings.container
self.syncer_dbtuple = settings.syncer_dbtuple
client_dbname = self.SIGNATURE+'.db'
self.client_dbtuple = common.DBTuple(
dbtype=database.ClientDB,
dbname=utils.join_path(settings.instance_path, client_dbname))
database.initialize(self.client_dbtuple)
self.endpoint = settings.endpoint
self.last_modification = "0000-00-00"
self.probe_candidates = utils.ThreadSafeDict()
self.check_enabled()
def __init__(self, settings):
self.settings = settings
self.SIGNATURE = "PithosFileClient"
self.auth_url = settings.auth_url
self.auth_token = settings.auth_token
self.container = settings.container
self.syncer_dbtuple = settings.syncer_dbtuple
client_dbname = self.SIGNATURE + '.db'
self.client_dbtuple = common.DBTuple(dbtype=database.ClientDB,
dbname=utils.join_path(
settings.instance_path,
client_dbname))
database.initialize(self.client_dbtuple)
self.endpoint = settings.endpoint
self.last_modification = "0000-00-00"
self.probe_candidates = utils.ThreadSafeDict()
self.check_enabled()
def __init__(self, client, target_state):
self.client = client
settings = client.settings
self.settings = settings
self.SIGNATURE = "LocalfsTargetHandle"
self.rootpath = settings.local_root_path
self.cache_hide_name = settings.cache_hide_name
self.cache_hide_path = settings.cache_hide_path
self.cache_path = settings.cache_path
self.syncer_dbtuple = settings.syncer_dbtuple
self.client_dbtuple = client.client_dbtuple
self.mtime_lag = settings.mtime_lag
self.target_state = target_state
self.objname = target_state.objname
self.fspath = utils.join_path(self.rootpath, self.objname)
self.hidden_filename = None
self.hidden_path = None
def __init__(self, client, target_state):
self.client = client
settings = client.settings
self.settings = settings
self.SIGNATURE = "LocalfsTargetHandle"
self.rootpath = settings.local_root_path
self.cache_hide_name = settings.cache_hide_name
self.cache_hide_path = settings.cache_hide_path
self.cache_path = settings.cache_path
self.syncer_dbtuple = settings.syncer_dbtuple
self.client_dbtuple = client.client_dbtuple
self.mtime_lag = settings.mtime_lag
self.target_state = target_state
self.objname = target_state.objname
self.fspath = utils.join_path(self.rootpath, self.objname)
self.hidden_filename = None
self.hidden_path = None
def __init__(self, client, source_state):
self.client = client
settings = client.settings
self.settings = settings
self.SIGNATURE = "LocalfsSourceHandle"
self.rootpath = settings.local_root_path
self.cache_stage_name = settings.cache_stage_name
self.cache_stage_path = settings.cache_stage_path
self.cache_path = settings.cache_path
self.syncer_dbtuple = settings.syncer_dbtuple
self.client_dbtuple = client.client_dbtuple
self.source_state = source_state
self.objname = source_state.objname
self.fspath = utils.join_path(self.rootpath, self.objname)
self.stage_filename = None
self.staged_path = None
self.heartbeat = settings.heartbeat
if info_of_regular_file(self.source_state.info):
self.stage_file()
def __init__(self, client, source_state):
self.client = client
settings = client.settings
self.settings = settings
self.SIGNATURE = "LocalfsSourceHandle"
self.rootpath = settings.local_root_path
self.cache_stage_name = settings.cache_stage_name
self.cache_stage_path = settings.cache_stage_path
self.cache_path = settings.cache_path
self.syncer_dbtuple = settings.syncer_dbtuple
self.client_dbtuple = client.client_dbtuple
self.source_state = source_state
self.objname = source_state.objname
self.fspath = utils.join_path(self.rootpath, self.objname)
self.stage_filename = None
self.staged_path = None
self.heartbeat = settings.heartbeat
if info_of_regular_file(self.source_state.info):
self.stage_file()
def _local_path_changes(self, name, state):
local_path = utils.join_path(self.ROOTPATH, name)
return local_path_changes(self.settings, local_path, state)
def get_path_in_cache(self, name):
return utils.join_path(self.cache_path, name)
def test_0002_notifier_local(self):
f_out = "φ0002out"
f_cache = "φ0002cache"
f_upd = "φ0002upd"
f_ren = "φ0002ren"
f_cached = "φ0002cached"
dbefore = "δ0002before"
f_out_path = self.get_path(f_out)
f_cache_path = self.get_path(f_cache)
f_upd_path = self.get_path(f_upd)
f_ren_path = self.get_path(f_ren)
f_cached_path = os.path.join(
self.settings.cache_path, f_cached)
dbefore_path = self.get_path(dbefore)
open(f_out_path, "a").close()
open(f_cache_path, "a").close()
open(f_upd_path, "a").close()
open(f_ren_path, "a").close()
open(f_cached_path, "a").close()
os.mkdir(dbefore_path)
notifier = self.slave.notifier()
candidates = self.slave.list_candidate_files()
self.assertEqual(candidates, [])
fafter = "φ0002after"
fafter_path = self.get_path(fafter)
dafter = "δ0002after"
dafter_path = self.get_path(dafter)
open(fafter_path, "a").close()
os.mkdir(dafter_path)
time.sleep(1)
candidates = self.slave.list_candidate_files()
self.assertEqual(sorted(candidates), sorted([fafter, dafter]))
os.rename(f_cache_path,
utils.join_path(self.settings.cache_path, f_cache))
os.rename(f_out_path,
utils.join_path(TMP, f_out))
with open(f_upd_path, "a") as f:
f.write("upd")
f_in = "φ0002in"
f_in_path = self.get_path(f_in)
f_in_orig_path = utils.join_path(TMP, f_in)
open(f_in_orig_path, "a").close()
os.rename(f_in_orig_path, f_in_path)
f_ren_new = "φ0002ren_new"
f_ren_new_path = self.get_path(f_ren_new)
os.rename(f_ren_path, f_ren_new_path)
f_cached_out_path = self.get_path(f_cached)
os.rename(f_cached_path, f_cached_out_path)
time.sleep(1)
candidates = self.slave.list_candidate_files()
self.assertEqual(sorted(candidates),
sorted([fafter, dafter,
f_in, f_out, f_upd,
f_ren, f_ren_new,
f_cache, f_cached]))
notifier.stop()
def stash_file(self):
stash_name = mk_stash_name(self.objname)
stash_path = utils.join_path(self.rootpath, stash_name)
msg = messaging.ConflictStashMessage(objname=self.objname, stash_name=stash_name, logger=logger)
self.settings.messager.put(msg)
os.rename(self.hidden_path, stash_path)
def get_path_in_cache(self, name):
return utils.join_path(self.cache_path, name)
def __init__(self, auth_url, auth_token, container, local_root_path,
*args, **kwargs):
check_encoding()
auth_url = utils.to_unicode(auth_url)
auth_token = utils.to_unicode(auth_token)
container = utils.to_unicode(container)
local_root_path = utils.to_unicode(local_root_path)
self.auth_url = utils.normalize_standard_suffix(auth_url)
self.auth_token = auth_token
self.container = utils.normalize_standard_suffix(container)
self.ignore_ssl = kwargs.get("ignore_ssl", False)
if self.ignore_ssl:
https.patch_ignore_ssl()
elif kwargs.get('ca_certs', None):
https.patch_with_certs(kwargs['ca_certs'])
self.endpoint = self._get_pithos_client(
auth_url, auth_token, container)
container_exists = self.check_container_exists(container)
home_dir = utils.to_unicode(os.path.expanduser('~'))
default_settings_path = join_path(home_dir, GLOBAL_SETTINGS_NAME)
self.settings_path = utils.to_unicode(
kwargs.get("agkyra_path", default_settings_path))
self.create_dir(self.settings_path, mode=stat.S_IRWXU)
self.instances_path = join_path(self.settings_path, INSTANCES_NAME)
self.create_dir(self.instances_path)
self.local_root_path = utils.normalize_local_suffix(local_root_path)
local_root_path_exists = os.path.isdir(self.local_root_path)
self.cache_name = utils.to_unicode(
kwargs.get("cache_name", DEFAULT_CACHE_NAME))
self.cache_path = join_path(self.local_root_path, self.cache_name)
self.cache_hide_name = utils.to_unicode(
kwargs.get("cache_hide_name", DEFAULT_CACHE_HIDE_NAME))
self.cache_hide_path = join_path(self.cache_path, self.cache_hide_name)
self.cache_stage_name = utils.to_unicode(
kwargs.get("cache_stage_name", DEFAULT_CACHE_STAGE_NAME))
self.cache_stage_path = join_path(self.cache_path,
self.cache_stage_name)
self.cache_fetch_name = utils.to_unicode(
kwargs.get("cache_fetch_name", DEFAULT_CACHE_FETCH_NAME))
self.cache_fetch_path = join_path(self.cache_path,
self.cache_fetch_name)
self.user_id = self.endpoint.account
self.instance = get_instance(
[self.auth_url, self.user_id,
self.container, self.local_root_path])
self.instance_path = join_path(self.instances_path, self.instance)
self.create_dir(self.instance_path)
self.dbname = utils.to_unicode(kwargs.get("dbname", DEFAULT_DBNAME))
self.full_dbname = join_path(self.instance_path, self.dbname)
self.syncer_dbtuple = common.DBTuple(
dbtype=database.SyncerDB,
dbname=self.full_dbname)
db_existed = os.path.isfile(self.full_dbname)
if not db_existed:
database.initialize(self.syncer_dbtuple)
self.mtime_lag = 0
self.case_insensitive = False
if not db_existed:
self.set_localfs_enabled(True)
self.create_local_dirs()
self.set_pithos_enabled(True)
if not container_exists:
self.mk_container(container)
else:
if not local_root_path_exists:
self.set_localfs_enabled(False)
else:
self.create_local_dirs()
if not container_exists:
self.set_pithos_enabled(False)
self.heartbeat = ThreadSafeDict()
self.action_max_wait = kwargs.get("action_max_wait",
DEFAULT_ACTION_MAX_WAIT)
self.pithos_list_interval = kwargs.get("pithos_list_interval",
DEFAULT_PITHOS_LIST_INTERVAL)
self.connection_retry_limit = kwargs.get(
"connection_retry_limit", DEFAULT_CONNECTION_RETRY_LIMIT)
self.endpoint.CONNECTION_RETRY_LIMIT = self.connection_retry_limit
self.max_alive_sync_threads = kwargs.get(
"max_alive_sync_threads", DEFAULT_MAX_ALIVE_SYNC_THREADS)
self.messager = Messager()
def _local_path_changes(self, name, state):
local_path = utils.join_path(self.ROOTPATH, name)
return local_path_changes(self.settings, local_path, state)
def __init__(self, auth_url, auth_token, container, local_root_path, *args,
**kwargs):
check_encoding()
auth_url = utils.to_unicode(auth_url)
auth_token = utils.to_unicode(auth_token)
container = utils.to_unicode(container)
local_root_path = utils.to_unicode(local_root_path)
self.auth_url = utils.normalize_standard_suffix(auth_url)
self.auth_token = auth_token
self.container = utils.normalize_standard_suffix(container)
self.ignore_ssl = kwargs.get("ignore_ssl", False)
if self.ignore_ssl:
https.patch_ignore_ssl()
elif kwargs.get('ca_certs', None):
https.patch_with_certs(kwargs['ca_certs'])
self.endpoint = self._get_pithos_client(auth_url, auth_token,
container)
container_exists = self.check_container_exists(container)
home_dir = utils.to_unicode(os.path.expanduser('~'))
default_settings_path = join_path(home_dir, GLOBAL_SETTINGS_NAME)
self.settings_path = utils.to_unicode(
kwargs.get("agkyra_path", default_settings_path))
self.create_dir(self.settings_path, mode=stat.S_IRWXU)
self.instances_path = join_path(self.settings_path, INSTANCES_NAME)
self.create_dir(self.instances_path)
self.local_root_path = utils.normalize_local_suffix(local_root_path)
local_root_path_exists = os.path.isdir(self.local_root_path)
self.cache_name = utils.to_unicode(
kwargs.get("cache_name", DEFAULT_CACHE_NAME))
self.cache_path = join_path(self.local_root_path, self.cache_name)
self.cache_hide_name = utils.to_unicode(
kwargs.get("cache_hide_name", DEFAULT_CACHE_HIDE_NAME))
self.cache_hide_path = join_path(self.cache_path, self.cache_hide_name)
self.cache_stage_name = utils.to_unicode(
kwargs.get("cache_stage_name", DEFAULT_CACHE_STAGE_NAME))
self.cache_stage_path = join_path(self.cache_path,
self.cache_stage_name)
self.cache_fetch_name = utils.to_unicode(
kwargs.get("cache_fetch_name", DEFAULT_CACHE_FETCH_NAME))
self.cache_fetch_path = join_path(self.cache_path,
self.cache_fetch_name)
self.user_id = self.endpoint.account
self.instance = get_instance([
self.auth_url, self.user_id, self.container, self.local_root_path
])
self.instance_path = join_path(self.instances_path, self.instance)
self.create_dir(self.instance_path)
self.dbname = utils.to_unicode(kwargs.get("dbname", DEFAULT_DBNAME))
self.full_dbname = join_path(self.instance_path, self.dbname)
self.syncer_dbtuple = common.DBTuple(dbtype=database.SyncerDB,
dbname=self.full_dbname)
db_existed = os.path.isfile(self.full_dbname)
if not db_existed:
database.initialize(self.syncer_dbtuple)
self.mtime_lag = 0
self.case_insensitive = False
if not db_existed:
self.set_localfs_enabled(True)
self.create_local_dirs()
self.set_pithos_enabled(True)
if not container_exists:
self.mk_container(container)
else:
if not local_root_path_exists:
self.set_localfs_enabled(False)
else:
self.create_local_dirs()
if not container_exists:
self.set_pithos_enabled(False)
self.heartbeat = ThreadSafeDict()
self.action_max_wait = kwargs.get("action_max_wait",
DEFAULT_ACTION_MAX_WAIT)
self.pithos_list_interval = kwargs.get("pithos_list_interval",
DEFAULT_PITHOS_LIST_INTERVAL)
self.connection_retry_limit = kwargs.get(
"connection_retry_limit", DEFAULT_CONNECTION_RETRY_LIMIT)
self.endpoint.CONNECTION_RETRY_LIMIT = self.connection_retry_limit
self.max_alive_sync_threads = kwargs.get(
"max_alive_sync_threads", DEFAULT_MAX_ALIVE_SYNC_THREADS)
self.messager = Messager()