def test_container_details_cached_miss_mtime(self): mc = SimMemcache() the_path = "/tmp/bar" def mock_get_container_details_from_fs(cont_path, bu_p=5): bu = bu_p oc = 1 ol = ['foo',] dl = [('a',100),] return utils.ContainerDetails(bu, oc, ol, dl) def mock_do_stat(path): # Be sure we miss due to mtimes not matching class MockStat(object): def __init__(self, mtime): self.st_mtime = mtime return MockStat(200) cd = mock_get_container_details_from_fs(the_path, bu_p=6) mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), cd) orig_gcdff = utils._get_container_details_from_fs utils._get_container_details_from_fs = mock_get_container_details_from_fs orig_ds = utils.do_stat utils.do_stat = mock_do_stat try: retval = utils.get_container_details(the_path, memcache=mc) cd = mock_get_container_details_from_fs(the_path) assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path) assert mkey in mc._d assert 5 == mc._d[mkey].bytes_used finally: utils._get_container_details_from_fs = orig_gcdff utils.do_stat = orig_ds
def test_container_details_cached_miss_dir_list(self): mc = SimMemcache() the_path = "/tmp/bar" def mock_get_container_details_from_fs(cont_path, bu_p=5): bu = bu_p oc = 1 ol = ['foo',] dl = [] return utils.ContainerDetails(bu, oc, ol, dl) def mock_do_stat(path): # Be sure we don't miss due to mtimes not matching self.fail("do_stat should not have been called") cd = mock_get_container_details_from_fs(the_path, bu_p=6) mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), cd) orig_gcdff = utils._get_container_details_from_fs utils._get_container_details_from_fs = mock_get_container_details_from_fs orig_ds = utils.do_stat utils.do_stat = mock_do_stat try: retval = utils.get_container_details(the_path, memcache=mc) cd = mock_get_container_details_from_fs(the_path) assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path) assert mkey in mc._d assert 5 == mc._d[mkey].bytes_used finally: utils._get_container_details_from_fs = orig_gcdff utils.do_stat = orig_ds
def test_container_details_cached_hit(self): mc = SimMemcache() the_path = "/tmp/bar" def mock_get_container_details_from_fs(cont_path, bu_p=5): bu = bu_p oc = 1 ol = ['foo',] dl = [('a',100),] return utils.ContainerDetails(bu, oc, ol, dl) def mock_do_stat(path): class MockStat(object): def __init__(self, mtime): self.st_mtime = mtime return MockStat(100) cd = mock_get_container_details_from_fs(the_path, bu_p=6) mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), cd) orig_gcdff = utils._get_container_details_from_fs utils._get_container_details_from_fs = mock_get_container_details_from_fs orig_ds = utils.do_stat utils.do_stat = mock_do_stat try: retval = utils.get_container_details(the_path, memcache=mc) # If it did not properly use memcache, the default mocked version # of get details from fs would return 5 bytes used instead of the # 6 we specified above. cd = mock_get_container_details_from_fs(the_path, bu_p=6) assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) finally: utils._get_container_details_from_fs = orig_gcdff utils.do_stat = orig_ds
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, path): """ Returns tuple of name, created_at, size, content_type, etag. """ if path: prefix = path = path.rstrip('/') + '/' delimiter = '/' if delimiter and not prefix: prefix = '' objects = [] object_count = 0 bytes_used = 0 container_list = [] objects, object_count, bytes_used = get_container_details(self.datadir) if int(self.metadata[X_OBJECTS_COUNT][0]) != object_count or \ int(self.metadata[X_BYTES_USED][0]) != bytes_used: self.metadata[X_OBJECTS_COUNT] = (object_count, 0) self.metadata[X_BYTES_USED] = (bytes_used, 0) self.update_container(self.metadata) if objects: objects.sort() if objects and prefix: objects = self.filter_prefix(objects, prefix) if objects and delimiter: objects = self.filter_delimiter(objects, delimiter, prefix) if objects and marker: objects = self.filter_marker(objects, marker) if objects and end_marker: objects = self.filter_end_marker(objects, end_marker) if objects and limit: if len(objects) > limit: objects = self.filter_limit(objects, limit) if objects: for obj in objects: list_item = [] list_item.append(obj) obj_path = os.path.join(self.datadir, obj) metadata = read_metadata(obj_path) if not metadata or not validate_object(metadata): metadata = create_object_metadata(obj_path) if metadata: list_item.append(metadata[X_TIMESTAMP]) list_item.append(int(metadata[X_CONTENT_LENGTH])) list_item.append(metadata[X_CONTENT_TYPE]) list_item.append(metadata[X_ETAG]) container_list.append(list_item) return container_list
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, path): """ Returns tuple of name, created_at, size, content_type, etag. """ if path: prefix = path = path.rstrip('/') + '/' delimiter = '/' if delimiter and not prefix: prefix = '' objects = [] object_count = 0 bytes_used = 0 container_list = [] objects, object_count, bytes_used = get_container_details(self.datadir) if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \ int(self.metadata[X_BYTES_USED]) != bytes_used: self.metadata[X_OBJECTS_COUNT] = object_count self.metadata[X_BYTES_USED] = bytes_used self.update_container(self.metadata) if objects: objects.sort() if objects and prefix: objects = self.filter_prefix(objects, prefix) if objects and delimiter: objects = self.filter_delimiter(objects, delimiter, prefix) if objects and marker: objects = self.filter_marker(objects, marker) if objects and end_marker: objects = self.filter_end_marker(objects, end_marker) if objects and limit: if len(objects) > limit: objects = self.filter_limit(objects, limit) if objects: for obj in objects: list_item = [] list_item.append(obj) metadata = read_metadata(self.datadir + '/' + obj) if not metadata or not validate_object(metadata): metadata = create_object_metadata(self.datadir + '/' + obj) if metadata: list_item.append(metadata[X_TIMESTAMP]) list_item.append(int(metadata[X_CONTENT_LENGTH])) list_item.append(metadata[X_CONTENT_TYPE]) list_item.append(metadata[X_ETAG]) container_list.append(list_item) return container_list
def update_object_count(self): objects = [] object_count = 0 bytes_used = 0 objects, object_count, bytes_used = get_container_details(self.datadir) if int(self.metadata[X_OBJECTS_COUNT][0]) != object_count or \ int(self.metadata[X_BYTES_USED][0]) != bytes_used: self.metadata[X_OBJECTS_COUNT] = (object_count, 0) self.metadata[X_BYTES_USED] = (bytes_used, 0) self.update_container(self.metadata)
def update_object_count(self): objects = [] object_count = 0 bytes_used = 0 objects, object_count, bytes_used = get_container_details(self.datadir) if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \ int(self.metadata[X_BYTES_USED]) != bytes_used: self.metadata[X_OBJECTS_COUNT] = object_count self.metadata[X_BYTES_USED] = bytes_used self.update_container(self.metadata)
def test_container_details_uncached(self): the_path = "/tmp/bar" def mock_get_container_details_from_fs(cont_path): bu = 5 oc = 1 ol = ['foo',] dl = [('a',100),] return utils.ContainerDetails(bu, oc, ol, dl) orig_gcdff = utils._get_container_details_from_fs utils._get_container_details_from_fs = mock_get_container_details_from_fs try: retval = utils.get_container_details(the_path) cd = mock_get_container_details_from_fs(the_path) assert retval == (cd.obj_list, cd.object_count, cd.bytes_used) finally: utils._get_container_details_from_fs = orig_gcdff