def test_container_details_cached_miss_dir_list(self):
mc = SimMemcache()
the_path = "/tmp/bar"
def mock_get_container_details_from_fs(cont_path, bu_p=5):
bu = bu_p
oc = 1
ol = ['foo',]
dl = []
return utils.ContainerDetails(bu, oc, ol, dl)
def mock_do_stat(path):
# Be sure we don't miss due to mtimes not matching
self.fail("do_stat should not have been called")
cd = mock_get_container_details_from_fs(the_path, bu_p=6)
mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd)
orig_gcdff = utils._get_container_details_from_fs
utils._get_container_details_from_fs = mock_get_container_details_from_fs
orig_ds = utils.do_stat
utils.do_stat = mock_do_stat
try:
retval = utils.get_container_details(the_path, memcache=mc)
cd = mock_get_container_details_from_fs(the_path)
assert retval == (cd.obj_list, cd.object_count, cd.bytes_used)
mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path
assert mkey in mc._d
assert 5 == mc._d[mkey].bytes_used
finally:
utils._get_container_details_from_fs = orig_gcdff
utils.do_stat = orig_ds
def test_container_details_cached_miss_mtime(self):
mc = SimMemcache()
the_path = "/tmp/bar"
def mock_get_container_details_from_fs(cont_path, bu_p=5):
bu = bu_p
oc = 1
ol = ['foo',]
dl = [('a',100),]
return utils.ContainerDetails(bu, oc, ol, dl)
def mock_do_stat(path):
# Be sure we miss due to mtimes not matching
class MockStat(object):
def __init__(self, mtime):
self.st_mtime = mtime
return MockStat(200)
cd = mock_get_container_details_from_fs(the_path, bu_p=6)
mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd)
orig_gcdff = utils._get_container_details_from_fs
utils._get_container_details_from_fs = mock_get_container_details_from_fs
orig_ds = utils.do_stat
utils.do_stat = mock_do_stat
try:
retval = utils.get_container_details(the_path, memcache=mc)
cd = mock_get_container_details_from_fs(the_path)
assert retval == (cd.obj_list, cd.object_count, cd.bytes_used)
mkey = utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path
assert mkey in mc._d
assert 5 == mc._d[mkey].bytes_used
finally:
utils._get_container_details_from_fs = orig_gcdff
utils.do_stat = orig_ds
def test_get_container_details_notadir(self):
tf = tempfile.NamedTemporaryFile()
obj_list, object_count, bytes_used = \
utils.get_container_details(tf.name)
assert bytes_used == 0
assert object_count == 0
assert obj_list == []
def test_container_details_cached_hit(self):
mc = SimMemcache()
the_path = "/tmp/bar"
def mock_get_container_details_from_fs(cont_path, bu_p=5):
bu = bu_p
oc = 1
ol = ['foo',]
dl = [('a',100),]
return utils.ContainerDetails(bu, oc, ol, dl)
def mock_do_stat(path):
class MockStat(object):
def __init__(self, mtime):
self.st_mtime = mtime
return MockStat(100)
cd = mock_get_container_details_from_fs(the_path, bu_p=6)
mc.set(utils.MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + the_path, cd)
orig_gcdff = utils._get_container_details_from_fs
utils._get_container_details_from_fs = mock_get_container_details_from_fs
orig_ds = utils.do_stat
utils.do_stat = mock_do_stat
try:
retval = utils.get_container_details(the_path, memcache=mc)
# If it did not properly use memcache, the default mocked version
# of get details from fs would return 5 bytes used instead of the
# 6 we specified above.
cd = mock_get_container_details_from_fs(the_path, bu_p=6)
assert retval == (cd.obj_list, cd.object_count, cd.bytes_used)
finally:
utils._get_container_details_from_fs = orig_gcdff
utils.do_stat = orig_ds
def test_get_container_details_notadir(self):
tf = tempfile.NamedTemporaryFile()
obj_list, object_count, bytes_used = \
utils.get_container_details(tf.name)
assert bytes_used == 0
assert object_count == 0
assert obj_list == []
def test_get_container_details_from_fs_do_getsize_true(self):
orig_cwd = os.getcwd()
__do_getsize = Glusterfs._do_getsize
td = tempfile.mkdtemp()
try:
tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2")
os.chdir(td)
tf.extractall()
Glusterfs._do_getsize = True
obj_list, object_count, bytes_used = \
utils.get_container_details(td)
assert bytes_used == 30, repr(bytes_used)
assert object_count == 5, repr(object_count)
assert set(obj_list) == set(['file1', 'file3', 'file2',
'dir1/file1', 'dir1/file2'
]), repr(obj_list)
full_dir1 = os.path.join(td, 'dir1')
full_dir2 = os.path.join(td, 'dir2')
full_dir3 = os.path.join(td, 'dir3')
exp_dir_dict = { td: os.path.getmtime(td),
full_dir1: os.path.getmtime(full_dir1),
full_dir2: os.path.getmtime(full_dir2),
full_dir3: os.path.getmtime(full_dir3),
}
finally:
Glusterfs._do_getsize = __do_getsize
os.chdir(orig_cwd)
shutil.rmtree(td)
def test_get_container_details(self):
orig_cwd = os.getcwd()
td = tempfile.mkdtemp()
try:
tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2")
os.chdir(td)
tf.extractall()
obj_list, object_count, bytes_used = \
utils.get_container_details(td)
assert bytes_used == 0, repr(bytes_used)
# Should not include the directories
assert object_count == 5, repr(object_count)
assert set(obj_list) == set(['file1', 'file3', 'file2',
'dir1/file1', 'dir1/file2'
]), repr(obj_list)
full_dir1 = os.path.join(td, 'dir1')
full_dir2 = os.path.join(td, 'dir2')
full_dir3 = os.path.join(td, 'dir3')
exp_dir_dict = { td: os.path.getmtime(td),
full_dir1: os.path.getmtime(full_dir1),
full_dir2: os.path.getmtime(full_dir2),
full_dir3: os.path.getmtime(full_dir3),
}
finally:
os.chdir(orig_cwd)
shutil.rmtree(td)
def _update_object_count(self):
objects, object_count, bytes_used = get_container_details(self.datadir)
if X_OBJECTS_COUNT not in self.metadata \
or int(self.metadata[X_OBJECTS_COUNT][0]) != object_count \
or X_BYTES_USED not in self.metadata \
or int(self.metadata[X_BYTES_USED][0]) != bytes_used:
self.metadata[X_OBJECTS_COUNT] = (object_count, 0)
self.metadata[X_BYTES_USED] = (bytes_used, 0)
write_metadata(self.datadir, self.metadata)
return objects
def _update_object_count(self):
objects, object_count, bytes_used = get_container_details(self.datadir)
if X_OBJECTS_COUNT not in self.metadata \
or int(self.metadata[X_OBJECTS_COUNT][0]) != object_count \
or X_BYTES_USED not in self.metadata \
or int(self.metadata[X_BYTES_USED][0]) != bytes_used:
self.metadata[X_OBJECTS_COUNT] = (object_count, 0)
self.metadata[X_BYTES_USED] = (bytes_used, 0)
write_metadata(self.datadir, self.metadata)
return objects
def update_object_count(self):
if not self.object_info:
self.object_info = get_container_details(self.datadir)
objects, object_count, bytes_used = self.object_info
if (
X_OBJECTS_COUNT not in self.metadata
or int(self.metadata[X_OBJECTS_COUNT][0]) != object_count
or X_BYTES_USED not in self.metadata
or int(self.metadata[X_BYTES_USED][0]) != bytes_used
):
self.metadata[X_OBJECTS_COUNT] = (object_count, 0)
self.metadata[X_BYTES_USED] = (bytes_used, 0)
write_metadata(self.datadir, self.metadata)
def test_container_details_uncached(self):
the_path = "/tmp/bar"
def mock_get_container_details_from_fs(cont_path):
bu = 5
oc = 1
ol = ['foo',]
dl = [('a',100),]
return utils.ContainerDetails(bu, oc, ol, dl)
orig_gcdff = utils._get_container_details_from_fs
utils._get_container_details_from_fs = mock_get_container_details_from_fs
try:
retval = utils.get_container_details(the_path)
cd = mock_get_container_details_from_fs(the_path)
assert retval == (cd.obj_list, cd.object_count, cd.bytes_used)
finally:
utils._get_container_details_from_fs = orig_gcdff
def test_get_container_details_from_fs_do_getsize_true(self):
orig_cwd = os.getcwd()
__do_getsize = Glusterfs._do_getsize
td = tempfile.mkdtemp()
try:
tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2")
os.chdir(td)
tf.extractall()
Glusterfs._do_getsize = True
obj_list, object_count, bytes_used = utils.get_container_details(td)
assert bytes_used == 30, repr(bytes_used)
assert object_count == 5, repr(object_count)
assert set(obj_list) == set(["file1", "file3", "file2", "dir1/file1", "dir1/file2"]), repr(obj_list)
finally:
Glusterfs._do_getsize = __do_getsize
os.chdir(orig_cwd)
shutil.rmtree(td)
def test_get_container_details_from_fs_do_getsize_true(self):
orig_cwd = os.getcwd()
__do_getsize = Glusterfs._do_getsize
td = tempfile.mkdtemp()
try:
tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2")
os.chdir(td)
tf.extractall()
Glusterfs._do_getsize = True
obj_list, object_count, bytes_used = \
utils.get_container_details(td)
assert bytes_used == 30, repr(bytes_used)
assert object_count == 5, repr(object_count)
assert set(obj_list) == set(
['file1', 'file3', 'file2', 'dir1/file1',
'dir1/file2']), repr(obj_list)
finally:
Glusterfs._do_getsize = __do_getsize
os.chdir(orig_cwd)
shutil.rmtree(td)
def test_get_container_details(self):
orig_cwd = os.getcwd()
__do_getsize = Glusterfs._do_getsize
td = tempfile.mkdtemp()
try:
tf = tarfile.open("common/data/container_tree.tar.bz2", "r:bz2")
os.chdir(td)
tf.extractall()
Glusterfs._do_getsize = False
obj_list, object_count, bytes_used = \
utils.get_container_details(td)
assert bytes_used == 0, repr(bytes_used)
# Should not include the directories
assert object_count == 5, repr(object_count)
assert set(obj_list) == set(['file1', 'file3', 'file2',
'dir1/file1', 'dir1/file2'
]), repr(obj_list)
finally:
Glusterfs._do_getsize = __do_getsize
os.chdir(orig_cwd)
shutil.rmtree(td)
def update_object_count(self):
if not self.object_info:
self.object_info = get_container_details(self.datadir)
