def test_container_details_cached_miss_dir_list(self): mc = SimMemcache() the_path = "/tmp/bar" def mock_get_container_details_from_fs(cont_path, bu_p=5): bu = bu_p oc = 1 ol = ['foo',] dl = [] return utils.ContainerDetails(bu, oc, ol, dl) def mock_do_stat(path): # Be sure we don't miss due to mtimes not matching self.fail("do_stat should not have been called") cd = mock_get_container_details_from_fs(the_path, bu_p=6) mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) orig_gcdff = utils._get_container_details_from_fs utils._get_container_details_from_fs = mock_get_container_details_from_fs orig_ds = utils.do_stat utils.do_stat = mock_do_stat try: retval = utils.get_container_details(the_path, memcache=mc) cd = mock_get_container_details_from_fs(the_path) assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path assert mkey in mc._d assert 5 == mc._d[mkey].bytes_used finally: utils._get_container_details_from_fs = orig_gcdff utils.do_stat = orig_ds
def test_container_details_cached_miss_mtime(self): mc = SimMemcache() the_path = "/tmp/bar" def mock_get_container_details_from_fs(cont_path, bu_p=5): bu = bu_p oc = 1 ol = ['foo',] dl = [('a',100),] return utils.ContainerDetails(bu, oc, ol, dl) def mock_do_stat(path): # Be sure we miss due to mtimes not matching class MockStat(object): def __init__(self, mtime): self.st_mtime = mtime return MockStat(200) cd = mock_get_container_details_from_fs(the_path, bu_p=6) mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) orig_gcdff = utils._get_container_details_from_fs utils._get_container_details_from_fs = mock_get_container_details_from_fs orig_ds = utils.do_stat utils.do_stat = mock_do_stat try: retval = utils.get_container_details(the_path, memcache=mc) cd = mock_get_container_details_from_fs(the_path) assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path assert mkey in mc._d assert 5 == mc._d[mkey].bytes_used finally: utils._get_container_details_from_fs = orig_gcdff utils.do_stat = orig_ds
def test_get_container_details_notadir(self): tf = tempfile.NamedTemporaryFile() obj_list, object_count, bytes_used = \ utils.get_container_details(tf.name) assert bytes_used == 0 assert object_count == 0 assert obj_list == []
def test_container_details_cached_hit(self): mc = SimMemcache() the_path = "/tmp/bar" def mock_get_container_details_from_fs(cont_path, bu_p=5): bu = bu_p oc = 1 ol = ['foo',] dl = [('a',100),] return utils.ContainerDetails(bu, oc, ol, dl) def mock_do_stat(path): class MockStat(object): def __init__(self, mtime): self.st_mtime = mtime return MockStat(100) cd = mock_get_container_details_from_fs(the_path, bu_p=6) mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd) orig_gcdff = utils._get_container_details_from_fs utils._get_container_details_from_fs = mock_get_container_details_from_fs orig_ds = utils.do_stat utils.do_stat = mock_do_stat try: retval = utils.get_container_details(the_path, memcache=mc) # If it did not properly use memcache, the default mocked version # of get details from fs would return 5 bytes used instead of the # 6 we specified above. cd = mock_get_container_details_from_fs(the_path, bu_p=6) assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) finally: utils._get_container_details_from_fs = orig_gcdff utils.do_stat = orig_ds
def test_get_container_details_from_fs_do_getsize_true(self): orig_cwd = os.getcwd() __do_getsize = Glusterfs._do_getsize td = tempfile.mkdtemp() try: tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2") os.chdir(td) tf.extractall() Glusterfs._do_getsize = True obj_list, object_count, bytes_used = \ utils.get_container_details(td) assert bytes_used == 30, repr(bytes_used) assert object_count == 5, repr(object_count) assert set(obj_list) == set(['file1', 'file3', 'file2', 'dir1/file1', 'dir1/file2' ]), repr(obj_list) full_dir1 = os.path.join(td, 'dir1') full_dir2 = os.path.join(td, 'dir2') full_dir3 = os.path.join(td, 'dir3') exp_dir_dict = { td: os.path.getmtime(td), full_dir1: os.path.getmtime(full_dir1), full_dir2: os.path.getmtime(full_dir2), full_dir3: os.path.getmtime(full_dir3), } finally: Glusterfs._do_getsize = __do_getsize os.chdir(orig_cwd) shutil.rmtree(td)
def test_get_container_details(self): orig_cwd = os.getcwd() td = tempfile.mkdtemp() try: tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2") os.chdir(td) tf.extractall() obj_list, object_count, bytes_used = \ utils.get_container_details(td) assert bytes_used == 0, repr(bytes_used) # Should not include the directories assert object_count == 5, repr(object_count) assert set(obj_list) == set(['file1', 'file3', 'file2', 'dir1/file1', 'dir1/file2' ]), repr(obj_list) full_dir1 = os.path.join(td, 'dir1') full_dir2 = os.path.join(td, 'dir2') full_dir3 = os.path.join(td, 'dir3') exp_dir_dict = { td: os.path.getmtime(td), full_dir1: os.path.getmtime(full_dir1), full_dir2: os.path.getmtime(full_dir2), full_dir3: os.path.getmtime(full_dir3), } finally: os.chdir(orig_cwd) shutil.rmtree(td)
def _update_object_count(self): objects, object_count, bytes_used = get_container_details(self.datadir) if X_OBJECTS_COUNT not in self.metadata \ or int(self.metadata[X_OBJECTS_COUNT][0]) != object_count \ or X_BYTES_USED not in self.metadata \ or int(self.metadata[X_BYTES_USED][0]) != bytes_used: self.metadata[X_OBJECTS_COUNT] = (object_count, 0) self.metadata[X_BYTES_USED] = (bytes_used, 0) write_metadata(self.datadir, self.metadata) return objects
def update_object_count(self): if not self.object_info: self.object_info = get_container_details(self.datadir) objects, object_count, bytes_used = self.object_info if ( X_OBJECTS_COUNT not in self.metadata or int(self.metadata[X_OBJECTS_COUNT][0]) != object_count or X_BYTES_USED not in self.metadata or int(self.metadata[X_BYTES_USED][0]) != bytes_used ): self.metadata[X_OBJECTS_COUNT] = (object_count, 0) self.metadata[X_BYTES_USED] = (bytes_used, 0) write_metadata(self.datadir, self.metadata)
def test_container_details_uncached(self): the_path = "/tmp/bar" def mock_get_container_details_from_fs(cont_path): bu = 5 oc = 1 ol = ['foo',] dl = [('a',100),] return utils.ContainerDetails(bu, oc, ol, dl) orig_gcdff = utils._get_container_details_from_fs utils._get_container_details_from_fs = mock_get_container_details_from_fs try: retval = utils.get_container_details(the_path) cd = mock_get_container_details_from_fs(the_path) assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) finally: utils._get_container_details_from_fs = orig_gcdff
def test_get_container_details_from_fs_do_getsize_true(self): orig_cwd = os.getcwd() __do_getsize = Glusterfs._do_getsize td = tempfile.mkdtemp() try: tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2") os.chdir(td) tf.extractall() Glusterfs._do_getsize = True obj_list, object_count, bytes_used = utils.get_container_details(td) assert bytes_used == 30, repr(bytes_used) assert object_count == 5, repr(object_count) assert set(obj_list) == set(["file1", "file3", "file2", "dir1/file1", "dir1/file2"]), repr(obj_list) finally: Glusterfs._do_getsize = __do_getsize os.chdir(orig_cwd) shutil.rmtree(td)
def test_get_container_details_from_fs_do_getsize_true(self): orig_cwd = os.getcwd() __do_getsize = Glusterfs._do_getsize td = tempfile.mkdtemp() try: tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2") os.chdir(td) tf.extractall() Glusterfs._do_getsize = True obj_list, object_count, bytes_used = \ utils.get_container_details(td) assert bytes_used == 30, repr(bytes_used) assert object_count == 5, repr(object_count) assert set(obj_list) == set( ['file1', 'file3', 'file2', 'dir1/file1', 'dir1/file2']), repr(obj_list) finally: Glusterfs._do_getsize = __do_getsize os.chdir(orig_cwd) shutil.rmtree(td)
def test_get_container_details(self): orig_cwd = os.getcwd() __do_getsize = Glusterfs._do_getsize td = tempfile.mkdtemp() try: tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2") os.chdir(td) tf.extractall() Glusterfs._do_getsize = False obj_list, object_count, bytes_used = \ utils.get_container_details(td) assert bytes_used == 0, repr(bytes_used) # Should not include the directories assert object_count == 5, repr(object_count) assert set(obj_list) == set(['file1', 'file3', 'file2', 'dir1/file1', 'dir1/file2' ]), repr(obj_list) finally: Glusterfs._do_getsize = __do_getsize os.chdir(orig_cwd) shutil.rmtree(td)
def update_object_count(self): if not self.object_info: self.object_info = get_container_details(self.datadir)