def test_iter_container_list(self):
worker = StorageTiererWorker(self.gridconf, Mock())
gen = worker._list_containers()
self.assertEqual(gen.next(), self.container_0_name)
self.assertEqual(gen.next(), self.container_1_name)
self.assertEqual(gen.next(), self.container_2_name)
self.assertRaises(StopIteration, gen.next)
def test_iter_content_list_skip_good_policy(self):
self.gridconf["new_policy"] = "SINGLE"
worker = StorageTiererWorker(self.gridconf, Mock())
gen = worker._list_contents()
self.assertEqual(gen.next(), (
self.container_2_id, self.container_2_content_1.content_id))
self.assertRaises(StopIteration, gen.next)
def test_iter_container_list(self):
worker = StorageTiererWorker(self.gridconf, Mock())
gen = worker._list_containers()
self.assertEqual(gen.next(), self.container_0_name)
self.assertEqual(gen.next(), self.container_1_name)
self.assertEqual(gen.next(), self.container_2_name)
self.assertRaises(StopIteration, gen.next)
def test_iter_content_list_skip_good_policy(self):
self.gridconf["new_policy"] = "SINGLE"
worker = StorageTiererWorker(self.gridconf, Mock())
gen = worker._list_contents()
self.assertEqual(gen.next(), (
self.container_2_id, self.container_2_content_1.content_id))
self.assertRaises(StopIteration, gen.next)
def test_iter_content_list_skip_good_policy(self):
self.gridconf["new_policy"] = "SINGLE"
worker = StorageTiererWorker(self.gridconf, Mock())
gen = worker._list_contents()
self.assertEqual((self.test_account, self.container_2_name,
self.container_2_content_1_name,
int(self.container_2_content_1['version'])),
next(gen))
self.assertRaises(StopIteration, next, gen)
def test_iter_content_list_outdated_threshold_0(self):
self.gridconf["outdated_threshold"] = 0
worker = StorageTiererWorker(self.gridconf, Mock())
gen = worker._list_contents()
self.assertEqual(gen.next(), (
self.container_1_id, self.container_1_content_0.content_id))
self.assertEqual(gen.next(), (
self.container_2_id, self.container_2_content_0.content_id))
self.assertEqual(gen.next(), (
self.container_2_id, self.container_2_content_1.content_id))
self.assertRaises(StopIteration, gen.next)
def test_iter_container_list(self):
worker = StorageTiererWorker(self.gridconf, Mock())
api = ObjectStorageApi(self.namespace)
actual = [x[0] for x in api.container_list(self.test_account)]
if len(actual) < 3:
print "Slow event propagation!"
# account events have not yet propagated
time.sleep(3.0)
actual = [x[0] for x in api.container_list(self.test_account)[0]]
gen = worker._list_containers()
self.assertListEqual(list(gen), actual)
def test_iter_content_list_outdated_threshold_0(self):
self.gridconf["outdated_threshold"] = 0
worker = StorageTiererWorker(self.gridconf, Mock())
gen = worker._list_contents()
self.assertEqual(gen.next(), (
self.container_1_id, self.container_1_content_0.content_id))
self.assertEqual(gen.next(), (
self.container_2_id, self.container_2_content_0.content_id))
self.assertEqual(gen.next(), (
self.container_2_id, self.container_2_content_1.content_id))
self.assertRaises(StopIteration, gen.next)
def test_iter_content_list_outdated_threshold_0(self):
self.gridconf["outdated_threshold"] = 0
worker = StorageTiererWorker(self.gridconf, Mock())
gen = worker._list_contents()
self.assertEqual((self.test_account, self.container_1_name,
self.container_1_content_0_name,
int(self.container_1_content_0['version'])),
next(gen))
self.assertEqual((self.test_account, self.container_2_name,
self.container_2_content_0_name,
int(self.container_2_content_0['version'])),
next(gen))
self.assertEqual((self.test_account, self.container_2_name,
self.container_2_content_1_name,
int(self.container_2_content_1['version'])),
next(gen))
self.assertRaises(StopIteration, next, gen)
def test_iter_content_list_outdated_threshold_2(self):
# add a new content created after the three previous contents
now = int(time.time())
time.sleep(2)
self._new_content(self.container_2_id, "titi", "TWOCOPIES")
self.gridconf["outdated_threshold"] = 2
worker = StorageTiererWorker(self.gridconf, Mock())
with mock.patch('oio.crawler.storage_tierer.time.time',
mock.MagicMock(return_value=now+2)):
gen = worker._list_contents()
self.assertEqual(gen.next(), (
self.container_1_id, self.container_1_content_0.content_id))
self.assertEqual(gen.next(), (
self.container_2_id, self.container_2_content_0.content_id))
self.assertEqual(gen.next(), (
self.container_2_id, self.container_2_content_1.content_id))
self.assertRaises(StopIteration, gen.next)
def test_iter_content_list_outdated_threshold_2(self):
# add a new content created after the three previous contents
now = int(time.time())
time.sleep(2)
self._new_content(self.container_2_id, "titi", "TWOCOPIES")
self.gridconf["outdated_threshold"] = 2
worker = StorageTiererWorker(self.gridconf, Mock())
with mock.patch('oio.crawler.storage_tierer.time.time',
mock.MagicMock(return_value=now+2)):
gen = worker._list_contents()
self.assertEqual(gen.next(), (
self.container_1_id, self.container_1_content_0.content_id))
self.assertEqual(gen.next(), (
self.container_2_id, self.container_2_content_0.content_id))
self.assertEqual(gen.next(), (
self.container_2_id, self.container_2_content_1.content_id))
self.assertRaises(StopIteration, gen.next)
def test_iter_content_list_outdated_threshold_2(self):
# add a new content created after the three previous contents
now = int(time.time())
time.sleep(2)
self._new_object(self.container_2_name, 'titi', 'TWOCOPIES')
self.gridconf["outdated_threshold"] = 2
worker = StorageTiererWorker(self.gridconf, Mock())
with mock.patch('oio.crawler.storage_tierer.time.time',
mock.MagicMock(return_value=now)):
gen = worker._list_contents()
self.assertEqual((self.test_account, self.container_1_name,
self.container_1_content_0_name,
int(self.container_1_content_0['version'])),
next(gen))
self.assertEqual((self.test_account, self.container_2_name,
self.container_2_content_0_name,
int(self.container_2_content_0['version'])),
next(gen))
self.assertEqual((self.test_account, self.container_2_name,
self.container_2_content_1_name,
int(self.container_2_content_1['version'])),
next(gen))
self.assertRaises(StopIteration, next, gen)
def test_iter_content_list_outdated_threshold_9999999999(self):
self.gridconf["outdated_threshold"] = 9999999999
worker = StorageTiererWorker(self.gridconf, Mock())
gen = worker._list_contents()
self.assertRaises(StopIteration, gen.next)
def test_iter_content_list_outdated_threshold_9999999999(self):
self.gridconf["outdated_threshold"] = 9999999999
worker = StorageTiererWorker(self.gridconf, Mock())
gen = worker._list_contents()
self.assertRaises(StopIteration, gen.next)
