def test_container_details_cached_miss_mtime(self): mc = SimMemcache() the_path = "/tmp/bar" def mock_get_container_details_from_fs(cont_path, bu_p=5): bu = bu_p oc = 1 ol = ['foo',] dl = [('a',100),] return utils.ContainerDetails(bu, oc, ol, dl) def mock_do_stat(path): # Be sure we miss due to mtimes not matching class MockStat(object): def __init__(self, mtime): self.st_mtime = mtime return MockStat(200) cd = mock_get_container_details_from_fs(the_path, bu_p=6) mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), cd) orig_gcdff = utils._get_container_details_from_fs utils._get_container_details_from_fs = mock_get_container_details_from_fs orig_ds = utils.do_stat utils.do_stat = mock_do_stat try: retval = utils.get_container_details(the_path, memcache=mc) cd = mock_get_container_details_from_fs(the_path) assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path) assert mkey in mc._d assert 5 == mc._d[mkey].bytes_used finally: utils._get_container_details_from_fs = orig_gcdff utils.do_stat = orig_ds
def test_container_details_cached_miss_dir_list(self): mc = SimMemcache() the_path = "/tmp/bar" def mock_get_container_details_from_fs(cont_path, bu_p=5): bu = bu_p oc = 1 ol = ['foo',] dl = [] return utils.ContainerDetails(bu, oc, ol, dl) def mock_do_stat(path): # Be sure we don't miss due to mtimes not matching self.fail("do_stat should not have been called") cd = mock_get_container_details_from_fs(the_path, bu_p=6) mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), cd) orig_gcdff = utils._get_container_details_from_fs utils._get_container_details_from_fs = mock_get_container_details_from_fs orig_ds = utils.do_stat utils.do_stat = mock_do_stat try: retval = utils.get_container_details(the_path, memcache=mc) cd = mock_get_container_details_from_fs(the_path) assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path) assert mkey in mc._d assert 5 == mc._d[mkey].bytes_used finally: utils._get_container_details_from_fs = orig_gcdff utils.do_stat = orig_ds
def test_account_details_cached_miss_mtime(self): mc = SimMemcache() the_path = "/tmp/bar" def mock_get_account_details_from_fs(acc_path, acc_stats): mt = 100 cc = 2 cl = ['a', 'b'] return utils.AccountDetails(mt, cc, cl) def mock_do_stat(path): class MockStat(object): def __init__(self, mtime): self.st_mtime = mtime return MockStat(100) ad = mock_get_account_details_from_fs(the_path, None) ad.container_list = ['x', 'y'] ad.mtime = 200 mc.set(utils.MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), ad) orig_gcdff = utils._get_account_details_from_fs orig_ds = utils.do_stat utils._get_account_details_from_fs = mock_get_account_details_from_fs utils.do_stat = mock_do_stat try: retval = utils.get_account_details(the_path, memcache=mc) correct_ad = mock_get_account_details_from_fs(the_path, None) assert retval == (correct_ad.container_list, correct_ad.container_count) assert correct_ad != ad finally: utils._get_account_details_from_fs = orig_gcdff utils.do_stat = orig_ds
def test_container_details_cached_hit(self): mc = SimMemcache() the_path = "/tmp/bar" def mock_get_container_details_from_fs(cont_path, bu_p=5): bu = bu_p oc = 1 ol = ['foo',] dl = [('a',100),] return utils.ContainerDetails(bu, oc, ol, dl) def mock_do_stat(path): class MockStat(object): def __init__(self, mtime): self.st_mtime = mtime return MockStat(100) cd = mock_get_container_details_from_fs(the_path, bu_p=6) mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), cd) orig_gcdff = utils._get_container_details_from_fs utils._get_container_details_from_fs = mock_get_container_details_from_fs orig_ds = utils.do_stat utils.do_stat = mock_do_stat try: retval = utils.get_container_details(the_path, memcache=mc) # If it did not properly use memcache, the default mocked version # of get details from fs would return 5 bytes used instead of the # 6 we specified above. cd = mock_get_container_details_from_fs(the_path, bu_p=6) assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) finally: utils._get_container_details_from_fs = orig_gcdff utils.do_stat = orig_ds
def _check_valid_account(account): full_mount_path = os.path.join(Glusterfs.MOUNT_PATH, account) if os.path.ismount(full_mount_path): return True if not Glusterfs.check_account_exists(Glusterfs.get_export_from_account_id(account)): logging.error('Account not present %s', account) return False if not os.path.isdir(full_mount_path): mkdirs(full_mount_path) if not Glusterfs.mount(account): return False return True
def _update_list(path, const_path, src_list, reg_file=True, object_count=0, bytes_used=0, obj_list=[]): obj_path = Glusterfs.strip_obj_storage_path(path, const_path) for i in src_list: if obj_path: obj_list.append(os.path.join(obj_path, i)) else: obj_list.append(i) object_count += 1 if reg_file: bytes_used += os.path.getsize(path + '/' + i) return object_count, bytes_used
def get_container_details(cont_path, memcache=None): """ Return object_list, object_count and bytes_used. """ mkey = '' if memcache: mkey = MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(cont_path) cd = memcache.get(mkey) if cd: if not cd.dir_list: cd = None else: for (path, mtime) in cd.dir_list: if mtime != do_stat(path).st_mtime: cd = None else: cd = None if not cd: cd = _get_container_details_from_fs(cont_path) if memcache: memcache.set(mkey, cd) return cd.obj_list, cd.object_count, cd.bytes_used
def get_account_details(acc_path, memcache=None): """ Return container_list and container_count. """ acc_stats = None mkey = '' if memcache: mkey = MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(acc_path) ad = memcache.get(mkey) if ad: # FIXME: Do we really need to stat the file? If we are object # only, then we can track the other Swift HTTP APIs that would # modify the account and invalidate the cached entry there. If we # are not object only, are we even called on this path? acc_stats = do_stat(acc_path) if ad.mtime != acc_stats.st_mtime: ad = None else: ad = None if not ad: ad = _get_account_details_from_fs(acc_path, acc_stats) if memcache: memcache.set(mkey, ad) return ad.container_list, ad.container_count
def get_account_list(): return Glusterfs.get_export_list()