class MetaRebuilder(Rebuilder):
"""
Abstract class for directory rebuilders.
"""
def __init__(self, conf, logger, volume, **kwargs):
super(MetaRebuilder, self).__init__(conf, logger, volume, **kwargs)
self.api = ObjectStorageApi(self.namespace,
logger=self.logger,
**kwargs)
def _fill_queue_from_file(self, queue, **kwargs):
if self.input_file is None:
return False
with open(self.input_file, 'r') as ifile:
for line in ifile:
stripped = line.strip()
if stripped and not stripped.startswith('#'):
queue.put(stripped)
if not self.running:
break
return True
def _full_container_list(self, account, **kwargs):
listing = self.api.container_list(account, **kwargs)
for element in listing:
yield element
while listing:
kwargs['marker'] = listing[-1][0]
listing = self.api.container_list(account, **kwargs)
if listing:
for element in listing:
yield element
def test_iter_container_list(self):
worker = StorageTiererWorker(self.gridconf, Mock())
api = ObjectStorageApi(self.namespace)
actual = [x[0] for x in api.container_list(self.test_account)]
if len(actual) < 3:
print "Slow event propagation!"
# account events have not yet propagated
time.sleep(3.0)
actual = [x[0] for x in api.container_list(self.test_account)[0]]
gen = worker._list_containers()
self.assertListEqual(list(gen), actual)
class TestStorageTierer(BaseTestCase):
def setUp(self):
super(TestStorageTierer, self).setUp()
self.namespace = self.conf['namespace']
self.test_account = "test_storage_tiering_%f" % time.time()
self.api = ObjectStorageApi(self.namespace)
self.gridconf = {
"namespace": self.namespace,
"container_fetch_limit": 2,
"content_fetch_limit": 2,
"account": self.test_account,
"outdated_threshold": 0,
"new_policy": "EC"
}
self.api.container = ContainerClient(self.gridconf)
self._populate()
def _populate(self):
self.container_0_name = 'container_empty'
self.container_0 = self._new_container(self.container_0_name)
self.container_1_name = 'container_with_1_content'
self.container_1 = self._new_container(self.container_1_name)
self.container_1_content_0_name = 'container_1_content_0'
self.container_1_content_0 = self._new_object(
self.container_1_name, self.container_1_content_0_name, 'SINGLE')
self.container_2_name = 'container_with_2_contents'
self.container_2 = self._new_container(self.container_1_name)
self.container_2_content_0_name = 'container_2_content_0'
self.container_2_content_0 = self._new_object(
self.container_2_name, self.container_2_content_0_name, 'SINGLE')
self.container_2_content_1_name = 'container_2_content_1'
self.container_2_content_1 = self._new_object(
self.container_2_name, self.container_2_content_1_name,
'TWOCOPIES')
def _new_container(self, container):
self.api.container_create(self.test_account, container)
cnt = self.api.container_get_properties(self.test_account, container)
return cnt
def _new_object(self, container, obj_name, stgpol):
data = random_data(10)
self.api.object_create(self.test_account,
container,
obj_name=obj_name,
policy=stgpol,
data=data)
obj = self.api.object_get_properties(self.test_account, container,
obj_name)
return obj
def tearDown(self):
super(TestStorageTierer, self).tearDown()
def test_iter_container_list(self):
worker = StorageTiererWorker(self.gridconf, Mock())
actual = [x[0] for x in self.api.container_list(self.test_account)]
if len(actual) < 3:
print("Slow event propagation!")
# account events have not yet propagated
time.sleep(3.0)
actual = [
x[0] for x in self.api.container_list(self.test_account)[0]
]
gen = worker._list_containers()
self.assertListEqual(list(gen), actual)
def test_iter_content_list_outdated_threshold_0(self):
self.gridconf["outdated_threshold"] = 0
worker = StorageTiererWorker(self.gridconf, Mock())
gen = worker._list_contents()
self.assertEqual((self.test_account, self.container_1_name,
self.container_1_content_0_name,
int(self.container_1_content_0['version'])),
next(gen))
self.assertEqual((self.test_account, self.container_2_name,
self.container_2_content_0_name,
int(self.container_2_content_0['version'])),
next(gen))
self.assertEqual((self.test_account, self.container_2_name,
self.container_2_content_1_name,
int(self.container_2_content_1['version'])),
next(gen))
self.assertRaises(StopIteration, next, gen)
def test_iter_content_list_outdated_threshold_9999999999(self):
self.gridconf["outdated_threshold"] = 9999999999
worker = StorageTiererWorker(self.gridconf, Mock())
gen = worker._list_contents()
self.assertRaises(StopIteration, next, gen)
def test_iter_content_list_outdated_threshold_2(self):
# add a new content created after the three previous contents
now = int(time.time())
time.sleep(2)
self._new_object(self.container_2_name, 'titi', 'TWOCOPIES')
self.gridconf["outdated_threshold"] = 2
worker = StorageTiererWorker(self.gridconf, Mock())
with mock.patch('oio.crawler.storage_tierer.time.time',
mock.MagicMock(return_value=now)):
gen = worker._list_contents()
self.assertEqual((self.test_account, self.container_1_name,
self.container_1_content_0_name,
int(self.container_1_content_0['version'])),
next(gen))
self.assertEqual((self.test_account, self.container_2_name,
self.container_2_content_0_name,
int(self.container_2_content_0['version'])),
next(gen))
self.assertEqual((self.test_account, self.container_2_name,
self.container_2_content_1_name,
int(self.container_2_content_1['version'])),
next(gen))
self.assertRaises(StopIteration, next, gen)
def test_iter_content_list_skip_good_policy(self):
self.gridconf["new_policy"] = "SINGLE"
worker = StorageTiererWorker(self.gridconf, Mock())
gen = worker._list_contents()
self.assertEqual((self.test_account, self.container_2_name,
self.container_2_content_1_name,
int(self.container_2_content_1['version'])),
next(gen))
self.assertRaises(StopIteration, next, gen)