def test_container_details_cached_miss_mtime(self):
mc = SimMemcache()
the_path = "/tmp/bar"
def mock_get_container_details_from_fs(cont_path, bu_p=5):
bu = bu_p
oc = 1
ol = ['foo',]
dl = [('a',100),]
return utils.ContainerDetails(bu, oc, ol, dl)
def mock_do_stat(path):
# Be sure we miss due to mtimes not matching
class MockStat(object):
def __init__(self, mtime):
self.st_mtime = mtime
return MockStat(200)
cd = mock_get_container_details_from_fs(the_path, bu_p=6)
mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), cd)
orig_gcdff = utils._get_container_details_from_fs
utils._get_container_details_from_fs = mock_get_container_details_from_fs
orig_ds = utils.do_stat
utils.do_stat = mock_do_stat
try:
retval = utils.get_container_details(the_path, memcache=mc)
cd = mock_get_container_details_from_fs(the_path)
assert retval == (cd.obj_list, cd.object_count, cd.bytes_used)
mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path)
assert mkey in mc._d
assert 5 == mc._d[mkey].bytes_used
finally:
utils._get_container_details_from_fs = orig_gcdff
utils.do_stat = orig_ds
def test_container_details_cached_miss_dir_list(self):
mc = SimMemcache()
the_path = "/tmp/bar"
def mock_get_container_details_from_fs(cont_path, bu_p=5):
bu = bu_p
oc = 1
ol = ['foo',]
dl = []
return utils.ContainerDetails(bu, oc, ol, dl)
def mock_do_stat(path):
# Be sure we don't miss due to mtimes not matching
self.fail("do_stat should not have been called")
cd = mock_get_container_details_from_fs(the_path, bu_p=6)
mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), cd)
orig_gcdff = utils._get_container_details_from_fs
utils._get_container_details_from_fs = mock_get_container_details_from_fs
orig_ds = utils.do_stat
utils.do_stat = mock_do_stat
try:
retval = utils.get_container_details(the_path, memcache=mc)
cd = mock_get_container_details_from_fs(the_path)
assert retval == (cd.obj_list, cd.object_count, cd.bytes_used)
mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path)
assert mkey in mc._d
assert 5 == mc._d[mkey].bytes_used
finally:
utils._get_container_details_from_fs = orig_gcdff
utils.do_stat = orig_ds
def test_account_details_cached_miss_mtime(self):
mc = SimMemcache()
the_path = "/tmp/bar"
def mock_get_account_details_from_fs(acc_path, acc_stats):
mt = 100
cc = 2
cl = ['a', 'b']
return utils.AccountDetails(mt, cc, cl)
def mock_do_stat(path):
class MockStat(object):
def __init__(self, mtime):
self.st_mtime = mtime
return MockStat(100)
ad = mock_get_account_details_from_fs(the_path, None)
ad.container_list = ['x', 'y']
ad.mtime = 200
mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), ad)
orig_gcdff = utils._get_account_details_from_fs
orig_ds = utils.do_stat
utils._get_account_details_from_fs = mock_get_account_details_from_fs
utils.do_stat = mock_do_stat
try:
retval = utils.get_account_details(the_path, memcache=mc)
correct_ad = mock_get_account_details_from_fs(the_path, None)
assert retval == (correct_ad.container_list, correct_ad.container_count)
assert correct_ad != ad
finally:
utils._get_account_details_from_fs = orig_gcdff
utils.do_stat = orig_ds
def test_container_details_cached_hit(self):
mc = SimMemcache()
the_path = "/tmp/bar"
def mock_get_container_details_from_fs(cont_path, bu_p=5):
bu = bu_p
oc = 1
ol = ['foo',]
dl = [('a',100),]
return utils.ContainerDetails(bu, oc, ol, dl)
def mock_do_stat(path):
class MockStat(object):
def __init__(self, mtime):
self.st_mtime = mtime
return MockStat(100)
cd = mock_get_container_details_from_fs(the_path, bu_p=6)
mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), cd)
orig_gcdff = utils._get_container_details_from_fs
utils._get_container_details_from_fs = mock_get_container_details_from_fs
orig_ds = utils.do_stat
utils.do_stat = mock_do_stat
try:
retval = utils.get_container_details(the_path, memcache=mc)
# If it did not properly use memcache, the default mocked version
# of get details from fs would return 5 bytes used instead of the
# 6 we specified above.
cd = mock_get_container_details_from_fs(the_path, bu_p=6)
assert retval == (cd.obj_list, cd.object_count, cd.bytes_used)
finally:
utils._get_container_details_from_fs = orig_gcdff
utils.do_stat = orig_ds
def _check_valid_account(account):
full_mount_path = os.path.join(Glusterfs.MOUNT_PATH, account)
if os.path.ismount(full_mount_path):
return True
if not Glusterfs.check_account_exists(Glusterfs.get_export_from_account_id(account)):
logging.error('Account not present %s', account)
return False
if not os.path.isdir(full_mount_path):
mkdirs(full_mount_path)
if not Glusterfs.mount(account):
return False
return True
def _update_list(path, const_path, src_list, reg_file=True, object_count=0,
bytes_used=0, obj_list=[]):
obj_path = Glusterfs.strip_obj_storage_path(path, const_path)
for i in src_list:
if obj_path:
obj_list.append(os.path.join(obj_path, i))
else:
obj_list.append(i)
object_count += 1
if reg_file:
bytes_used += os.path.getsize(path + '/' + i)
return object_count, bytes_used
def get_container_details(cont_path, memcache=None):
"""
Return object_list, object_count and bytes_used.
"""
mkey = ''
if memcache:
mkey = MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(cont_path)
cd = memcache.get(mkey)
if cd:
if not cd.dir_list:
cd = None
else:
for (path, mtime) in cd.dir_list:
if mtime != do_stat(path).st_mtime:
cd = None
else:
cd = None
if not cd:
cd = _get_container_details_from_fs(cont_path)
if memcache:
memcache.set(mkey, cd)
return cd.obj_list, cd.object_count, cd.bytes_used
def get_account_details(acc_path, memcache=None):
"""
Return container_list and container_count.
"""
acc_stats = None
mkey = ''
if memcache:
mkey = MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(acc_path)
ad = memcache.get(mkey)
if ad:
# FIXME: Do we really need to stat the file? If we are object
# only, then we can track the other Swift HTTP APIs that would
# modify the account and invalidate the cached entry there. If we
# are not object only, are we even called on this path?
acc_stats = do_stat(acc_path)
if ad.mtime != acc_stats.st_mtime:
ad = None
else:
ad = None
if not ad:
ad = _get_account_details_from_fs(acc_path, acc_stats)
if memcache:
memcache.set(mkey, ad)
return ad.container_list, ad.container_count
def get_account_list():
return Glusterfs.get_export_list()
