def test_container_details_cached_miss_mtime(self):
mc = SimMemcache()
the_path = "/tmp/bar"
def mock_get_container_details_from_fs(cont_path, bu_p=5):
bu = bu_p
oc = 1
ol = ['foo',]
dl = [('a',100),]
return utils.ContainerDetails(bu, oc, ol, dl)
def mock_do_stat(path):
# Be sure we miss due to mtimes not matching
class MockStat(object):
def __init__(self, mtime):
self.st_mtime = mtime
return MockStat(200)
cd = mock_get_container_details_from_fs(the_path, bu_p=6)
mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), cd)
orig_gcdff = utils._get_container_details_from_fs
utils._get_container_details_from_fs = mock_get_container_details_from_fs
orig_ds = utils.do_stat
utils.do_stat = mock_do_stat
try:
retval = utils.get_container_details(the_path, memcache=mc)
cd = mock_get_container_details_from_fs(the_path)
assert retval == (cd.obj_list, cd.object_count, cd.bytes_used)
mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path)
assert mkey in mc._d
assert 5 == mc._d[mkey].bytes_used
finally:
utils._get_container_details_from_fs = orig_gcdff
utils.do_stat = orig_ds
def test_container_details_cached_miss_dir_list(self):
mc = SimMemcache()
the_path = "/tmp/bar"
def mock_get_container_details_from_fs(cont_path, bu_p=5):
bu = bu_p
oc = 1
ol = ['foo',]
dl = []
return utils.ContainerDetails(bu, oc, ol, dl)
def mock_do_stat(path):
# Be sure we don't miss due to mtimes not matching
self.fail("do_stat should not have been called")
cd = mock_get_container_details_from_fs(the_path, bu_p=6)
mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), cd)
orig_gcdff = utils._get_container_details_from_fs
utils._get_container_details_from_fs = mock_get_container_details_from_fs
orig_ds = utils.do_stat
utils.do_stat = mock_do_stat
try:
retval = utils.get_container_details(the_path, memcache=mc)
cd = mock_get_container_details_from_fs(the_path)
assert retval == (cd.obj_list, cd.object_count, cd.bytes_used)
mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path)
assert mkey in mc._d
assert 5 == mc._d[mkey].bytes_used
finally:
utils._get_container_details_from_fs = orig_gcdff
utils.do_stat = orig_ds
def test_container_details_cached_hit(self):
mc = SimMemcache()
the_path = "/tmp/bar"
def mock_get_container_details_from_fs(cont_path, bu_p=5):
bu = bu_p
oc = 1
ol = ['foo',]
dl = [('a',100),]
return utils.ContainerDetails(bu, oc, ol, dl)
def mock_do_stat(path):
class MockStat(object):
def __init__(self, mtime):
self.st_mtime = mtime
return MockStat(100)
cd = mock_get_container_details_from_fs(the_path, bu_p=6)
mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + Glusterfs.strip_obj_storage_path(the_path), cd)
orig_gcdff = utils._get_container_details_from_fs
utils._get_container_details_from_fs = mock_get_container_details_from_fs
orig_ds = utils.do_stat
utils.do_stat = mock_do_stat
try:
retval = utils.get_container_details(the_path, memcache=mc)
# If it did not properly use memcache, the default mocked version
# of get details from fs would return 5 bytes used instead of the
# 6 we specified above.
cd = mock_get_container_details_from_fs(the_path, bu_p=6)
assert retval == (cd.obj_list, cd.object_count, cd.bytes_used)
finally:
utils._get_container_details_from_fs = orig_gcdff
utils.do_stat = orig_ds
def list_objects_iter(self, limit, marker, end_marker,
prefix, delimiter, path):
"""
Returns tuple of name, created_at, size, content_type, etag.
"""
if path:
prefix = path = path.rstrip('/') + '/'
delimiter = '/'
if delimiter and not prefix:
prefix = ''
objects = []
object_count = 0
bytes_used = 0
container_list = []
objects, object_count, bytes_used = get_container_details(self.datadir)
if int(self.metadata[X_OBJECTS_COUNT][0]) != object_count or \
int(self.metadata[X_BYTES_USED][0]) != bytes_used:
self.metadata[X_OBJECTS_COUNT] = (object_count, 0)
self.metadata[X_BYTES_USED] = (bytes_used, 0)
self.update_container(self.metadata)
if objects:
objects.sort()
if objects and prefix:
objects = self.filter_prefix(objects, prefix)
if objects and delimiter:
objects = self.filter_delimiter(objects, delimiter, prefix)
if objects and marker:
objects = self.filter_marker(objects, marker)
if objects and end_marker:
objects = self.filter_end_marker(objects, end_marker)
if objects and limit:
if len(objects) > limit:
objects = self.filter_limit(objects, limit)
if objects:
for obj in objects:
list_item = []
list_item.append(obj)
obj_path = os.path.join(self.datadir, obj)
metadata = read_metadata(obj_path)
if not metadata or not validate_object(metadata):
metadata = create_object_metadata(obj_path)
if metadata:
list_item.append(metadata[X_TIMESTAMP])
list_item.append(int(metadata[X_CONTENT_LENGTH]))
list_item.append(metadata[X_CONTENT_TYPE])
list_item.append(metadata[X_ETAG])
container_list.append(list_item)
return container_list
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter,
path):
"""
Returns tuple of name, created_at, size, content_type, etag.
"""
if path:
prefix = path = path.rstrip('/') + '/'
delimiter = '/'
if delimiter and not prefix:
prefix = ''
objects = []
object_count = 0
bytes_used = 0
container_list = []
objects, object_count, bytes_used = get_container_details(self.datadir)
if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \
int(self.metadata[X_BYTES_USED]) != bytes_used:
self.metadata[X_OBJECTS_COUNT] = object_count
self.metadata[X_BYTES_USED] = bytes_used
self.update_container(self.metadata)
if objects:
objects.sort()
if objects and prefix:
objects = self.filter_prefix(objects, prefix)
if objects and delimiter:
objects = self.filter_delimiter(objects, delimiter, prefix)
if objects and marker:
objects = self.filter_marker(objects, marker)
if objects and end_marker:
objects = self.filter_end_marker(objects, end_marker)
if objects and limit:
if len(objects) > limit:
objects = self.filter_limit(objects, limit)
if objects:
for obj in objects:
list_item = []
list_item.append(obj)
metadata = read_metadata(self.datadir + '/' + obj)
if not metadata or not validate_object(metadata):
metadata = create_object_metadata(self.datadir + '/' + obj)
if metadata:
list_item.append(metadata[X_TIMESTAMP])
list_item.append(int(metadata[X_CONTENT_LENGTH]))
list_item.append(metadata[X_CONTENT_TYPE])
list_item.append(metadata[X_ETAG])
container_list.append(list_item)
return container_list
def update_object_count(self):
objects = []
object_count = 0
bytes_used = 0
objects, object_count, bytes_used = get_container_details(self.datadir)
if int(self.metadata[X_OBJECTS_COUNT][0]) != object_count or \
int(self.metadata[X_BYTES_USED][0]) != bytes_used:
self.metadata[X_OBJECTS_COUNT] = (object_count, 0)
self.metadata[X_BYTES_USED] = (bytes_used, 0)
self.update_container(self.metadata)
def update_object_count(self):
objects = []
object_count = 0
bytes_used = 0
objects, object_count, bytes_used = get_container_details(self.datadir)
if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \
int(self.metadata[X_BYTES_USED]) != bytes_used:
self.metadata[X_OBJECTS_COUNT] = object_count
self.metadata[X_BYTES_USED] = bytes_used
self.update_container(self.metadata)
def test_container_details_uncached(self):
the_path = "/tmp/bar"
def mock_get_container_details_from_fs(cont_path):
bu = 5
oc = 1
ol = ['foo',]
dl = [('a',100),]
return utils.ContainerDetails(bu, oc, ol, dl)
orig_gcdff = utils._get_container_details_from_fs
utils._get_container_details_from_fs = mock_get_container_details_from_fs
try:
retval = utils.get_container_details(the_path)
cd = mock_get_container_details_from_fs(the_path)
assert retval == (cd.obj_list, cd.object_count, cd.bytes_used)
finally:
utils._get_container_details_from_fs = orig_gcdff