def test_success_gets_counted(self): class InternalClient(object): def __init__(self, containers, objects): self.containers = containers self.objects = objects def get_account_info(*a, **kw): return 1, 2 def iter_containers(self, *a, **kw): return self.containers def delete_container(*a, **kw): pass def delete_object(*a, **kw): pass def iter_objects(self, *a, **kw): return self.objects x = expirer.ObjectExpirer({}) x.logger = FakeLogger() x.delete_actual_object = lambda o, t: None self.assertEqual(x.report_objects, 0) x.swift = InternalClient([{ 'name': str(int(time() - 86400)) }], [{ 'name': '%d-actual-obj' % int(time() - 86400) }]) x.run_once() self.assertEqual(x.report_objects, 1) self.assertEqual(x.logger.log_dict['info'], [(('Pass beginning; 1 possible containers; ' '2 possible objects', ), {}), (('Pass completed in 0s; 1 objects expired', ), {})])
def test_run_forever_catches_usual_exceptions(self): raises = [0] def raise_exceptions(): raises[0] += 1 if raises[0] < 2: raise Exception('exception %d' % raises[0]) raise SystemExit('exiting exception %d' % raises[0]) x = expirer.ObjectExpirer({}, logger=self.logger) orig_sleep = expirer.sleep try: expirer.sleep = not_sleep x.run_once = raise_exceptions x.run_forever() except SystemExit as err: pass finally: expirer.sleep = orig_sleep self.assertEqual(str(err), 'exiting exception 2') self.assertEqual(x.logger.get_lines_for_level('error'), ['Unhandled exception: ']) log_args, log_kwargs = x.logger.log_dict['error'][0] self.assertEqual(str(log_kwargs['exc_info'][1]), 'exception 1')
def test_failed_delete_continues_on(self): class InternalClient(object): container_ring = None def __init__(self, containers, objects): self.containers = containers self.objects = objects def get_account_info(*a, **kw): return 1, 2 def iter_containers(self, *a, **kw): return self.containers def delete_container(*a, **kw): raise Exception('failed to delete container') def delete_object(*a, **kw): pass def iter_objects(self, *a, **kw): return self.objects def fail_delete_actual_object(actual_obj, timestamp): raise Exception('failed to delete actual object') x = expirer.ObjectExpirer(self.conf, logger=self.logger) cts = int(time() - 86400) ots = int(time() - 86400) containers = [ { 'name': str(cts) }, { 'name': str(cts + 1) }, ] objects = [{ 'name': '%d-actual-obj' % ots }, { 'name': '%d-next-obj' % ots }] x.swift = InternalClient(containers, objects) x.delete_actual_object = fail_delete_actual_object x.run_once() error_lines = x.logger.get_lines_for_level('error') self.assertEqual( sorted(error_lines), sorted([ 'Exception while deleting object %d %d-actual-obj failed to ' 'delete actual object: ' % (cts, ots), 'Exception while deleting object %d %d-next-obj failed to ' 'delete actual object: ' % (cts, ots), 'Exception while deleting object %d %d-actual-obj failed to ' 'delete actual object: ' % (cts + 1, ots), 'Exception while deleting object %d %d-next-obj failed to ' 'delete actual object: ' % (cts + 1, ots), 'Exception while deleting container %d failed to delete ' 'container: ' % (cts, ), 'Exception while deleting container %d failed to delete ' 'container: ' % (cts + 1, ) ])) self.assertEqual(x.logger.get_lines_for_level('info'), [ 'Pass beginning; 1 possible containers; 2 possible objects', 'Pass completed in 0s; 0 objects expired', ])
def test_failed_delete_keeps_entry(self): class InternalClient(object): container_ring = None def __init__(self, containers, objects): self.containers = containers self.objects = objects def get_account_info(*a, **kw): return 1, 2 def iter_containers(self, *a, **kw): return self.containers def delete_container(*a, **kw): pass def iter_objects(self, *a, **kw): return self.objects def deliberately_blow_up(actual_obj, timestamp): raise Exception('failed to delete actual object') def should_not_get_called(container, obj): raise Exception('This should not have been called') ts = int(time() - 86400) fake_swift = InternalClient([{ 'name': str(int(time() - 86400)) }], [{ 'name': '%d-actual-obj' % ts }]) x = expirer.ObjectExpirer(self.conf, logger=self.logger, swift=fake_swift) x.iter_containers = lambda: [str(int(time() - 86400))] x.delete_actual_object = deliberately_blow_up x.pop_queue = should_not_get_called x.run_once() error_lines = x.logger.get_lines_for_level('error') self.assertEqual(error_lines, [ 'Exception while deleting object %d %d-actual-obj ' 'failed to delete actual object: ' % (ts, ts) ]) self.assertEqual(x.logger.get_lines_for_level('info'), [ 'Pass beginning; 1 possible containers; 2 possible objects', 'Pass completed in 0s; 0 objects expired', ]) # Reverse test to be sure it still would blow up the way expected. ts = int(time() - 86400) fake_swift = InternalClient([{ 'name': str(int(time() - 86400)) }], [{ 'name': '%d-actual-obj' % ts }]) self.logger._clear() x = expirer.ObjectExpirer(self.conf, logger=self.logger, swift=fake_swift) x.delete_actual_object = lambda o, t: None x.pop_queue = should_not_get_called x.run_once() self.assertEqual(self.logger.get_lines_for_level('error'), [ 'Exception while deleting object %d %d-actual-obj This should ' 'not have been called: ' % (ts, ts) ])
def test_round_robin_order(self): def make_task(delete_at, target): return { 'task_container': delete_at, 'task_object': delete_at + '-' + target, 'delete_timestamp': Timestamp(delete_at), 'target_path': target, } x = expirer.ObjectExpirer(self.conf, logger=self.logger) task_con_obj_list = [ # objects in 0000 timestamp container make_task('0000', 'a/c0/o0'), make_task('0000', 'a/c0/o1'), # objects in 0001 timestamp container make_task('0001', 'a/c1/o0'), make_task('0001', 'a/c1/o1'), # objects in 0002 timestamp container make_task('0002', 'a/c2/o0'), make_task('0002', 'a/c2/o1'), ] result = list(x.round_robin_order(task_con_obj_list)) # sorted by popping one object to delete for each target_container expected = [ make_task('0000', 'a/c0/o0'), make_task('0001', 'a/c1/o0'), make_task('0002', 'a/c2/o0'), make_task('0000', 'a/c0/o1'), make_task('0001', 'a/c1/o1'), make_task('0002', 'a/c2/o1'), ] self.assertEqual(expected, result) # for a given target container, tasks won't necessarily all go in # the same timestamp container task_con_obj_list = [ # objects in 0000 timestamp container make_task('0000', 'a/c0/o0'), make_task('0000', 'a/c0/o1'), make_task('0000', 'a/c2/o2'), make_task('0000', 'a/c2/o3'), # objects in 0001 timestamp container make_task('0001', 'a/c0/o2'), make_task('0001', 'a/c0/o3'), make_task('0001', 'a/c1/o0'), make_task('0001', 'a/c1/o1'), # objects in 0002 timestamp container make_task('0002', 'a/c2/o0'), make_task('0002', 'a/c2/o1'), ] result = list(x.round_robin_order(task_con_obj_list)) # so we go around popping by *target* container, not *task* container expected = [ make_task('0000', 'a/c0/o0'), make_task('0001', 'a/c1/o0'), make_task('0000', 'a/c2/o2'), make_task('0000', 'a/c0/o1'), make_task('0001', 'a/c1/o1'), make_task('0000', 'a/c2/o3'), make_task('0001', 'a/c0/o2'), make_task('0002', 'a/c2/o0'), make_task('0001', 'a/c0/o3'), make_task('0002', 'a/c2/o1'), ] self.assertEqual(expected, result) # all of the work to be done could be for different target containers task_con_obj_list = [ # objects in 0000 timestamp container make_task('0000', 'a/c0/o'), make_task('0000', 'a/c1/o'), make_task('0000', 'a/c2/o'), make_task('0000', 'a/c3/o'), # objects in 0001 timestamp container make_task('0001', 'a/c4/o'), make_task('0001', 'a/c5/o'), make_task('0001', 'a/c6/o'), make_task('0001', 'a/c7/o'), # objects in 0002 timestamp container make_task('0002', 'a/c8/o'), make_task('0002', 'a/c9/o'), ] result = list(x.round_robin_order(task_con_obj_list)) # in which case, we kind of hammer the task containers self.assertEqual(task_con_obj_list, result)
def test_failed_delete_continues_on(self): class InternalClient(object): def __init__(self, containers, objects): self.containers = containers self.objects = objects def get_account_info(*a, **kw): return 1, 2 def iter_containers(self, *a, **kw): return self.containers def delete_container(*a, **kw): raise Exception('failed to delete container') def delete_object(*a, **kw): pass def iter_objects(self, *a, **kw): return self.objects def fail_delete_actual_object(actual_obj, timestamp): raise Exception('failed to delete actual object') x = expirer.ObjectExpirer({}) x.logger = FakeLogger() cts = int(time() - 86400) ots = int(time() - 86400) containers = [ {'name': str(cts)}, {'name': str(cts + 1)}, ] objects = [ {'name': '%d-actual-obj' % ots}, {'name': '%d-next-obj' % ots} ] x.swift = InternalClient(containers, objects) x.delete_actual_object = fail_delete_actual_object x.run_once() excswhiledeleting = [] for exccall in x.logger.log_dict['exception']: if exccall[0][0].startswith('Exception while deleting '): excswhiledeleting.append(exccall[0][0]) self.assertEquals(excswhiledeleting, [ 'Exception while deleting object %d %d-actual-obj failed to ' 'delete actual object' % (cts, ots), 'Exception while deleting object %d %d-next-obj failed to ' 'delete actual object' % (cts, ots), 'Exception while deleting container %d failed to delete ' 'container' % (cts,), 'Exception while deleting object %d %d-actual-obj failed to ' 'delete actual object' % (cts + 1, ots), 'Exception while deleting object %d %d-next-obj failed to ' 'delete actual object' % (cts + 1, ots), 'Exception while deleting container %d failed to delete ' 'container' % (cts + 1,)]) self.assertEquals(x.logger.log_dict['info'], [(('Pass beginning; 1 possible containers; ' '2 possible objects',), {}), (('Pass completed in 0s; 0 objects expired',), {})])
def test_failed_delete_keeps_entry(self): class InternalClient(object): def __init__(self, containers, objects): self.containers = containers self.objects = objects def get_account_info(*a, **kw): return 1, 2 def iter_containers(self, *a, **kw): return self.containers def delete_container(*a, **kw): pass def delete_object(*a, **kw): raise Exception('This should not have been called') def iter_objects(self, *a, **kw): return self.objects def deliberately_blow_up(actual_obj, timestamp): raise Exception('failed to delete actual object') def should_not_get_called(container, obj): raise Exception('This should not have been called') x = expirer.ObjectExpirer({}) x.logger = FakeLogger() x.iter_containers = lambda: [str(int(time() - 86400))] ts = int(time() - 86400) x.delete_actual_object = deliberately_blow_up x.swift = InternalClient([{'name': str(int(time() - 86400))}], [{'name': '%d-actual-obj' % ts}]) x.run_once() excswhiledeleting = [] for exccall in x.logger.log_dict['exception']: if exccall[0][0].startswith('Exception while deleting '): excswhiledeleting.append(exccall[0][0]) self.assertEquals(excswhiledeleting, ['Exception while deleting object %d %d-actual-obj ' 'failed to delete actual object' % (ts, ts)]) self.assertEquals(x.logger.log_dict['info'], [(('Pass beginning; 1 possible containers; ' '2 possible objects',), {}), (('Pass completed in 0s; 0 objects expired',), {})]) # Reverse test to be sure it still would blow up the way expected. x = expirer.ObjectExpirer({}) x.logger = FakeLogger() ts = int(time() - 86400) x.delete_actual_object = lambda o, t: None x.swift = InternalClient([{'name': str(int(time() - 86400))}], [{'name': '%d-actual-obj' % ts}]) x.run_once() excswhiledeleting = [] for exccall in x.logger.log_dict['exception']: if exccall[0][0].startswith('Exception while deleting '): excswhiledeleting.append(exccall[0][0]) self.assertEquals(excswhiledeleting, ['Exception while deleting object %d %d-actual-obj This should ' 'not have been called' % (ts, ts)])
def test_iter_task_to_expire(self): # In this test, all tasks are assigned to the tested expirer my_index = 0 divisor = 1 task_account_container_list = [('.expiring_objects', self.past_time)] expected = [ self.make_task(self.past_time, target_path) for target_path in self.expired_target_path_list ] self.assertEqual( list( self.expirer.iter_task_to_expire(task_account_container_list, my_index, divisor)), expected) # the task queue has invalid task object invalid_aco_dict = deepcopy(self.fake_swift.aco_dict) invalid_aco_dict['.expiring_objects'][self.past_time].insert( 0, self.past_time + '-invalid0') invalid_aco_dict['.expiring_objects'][self.past_time].insert( 5, self.past_time + '-invalid1') invalid_fake_swift = FakeInternalClient(invalid_aco_dict) x = expirer.ObjectExpirer(self.conf, logger=self.logger, swift=invalid_fake_swift) # but the invalid tasks are skipped self.assertEqual( list( x.iter_task_to_expire(task_account_container_list, my_index, divisor)), expected) # test some of that async delete async_delete_aco_dict = { '.expiring_objects': { # this task container will be checked self.past_time: [ # tasks ready for execution { 'name': self.past_time + '-a0/c0/o0', 'content_type': 'application/async-deleted' }, { 'name': self.past_time + '-a1/c1/o1', 'content_type': 'application/async-deleted' }, { 'name': self.past_time + '-a2/c2/o2', 'content_type': 'application/async-deleted' }, { 'name': self.past_time + '-a3/c3/o3', 'content_type': 'application/async-deleted' }, { 'name': self.past_time + '-a4/c4/o4', 'content_type': 'application/async-deleted' }, { 'name': self.past_time + '-a5/c5/o5', 'content_type': 'application/async-deleted' }, { 'name': self.past_time + '-a6/c6/o6', 'content_type': 'application/async-deleted' }, { 'name': self.past_time + '-a7/c7/o7', 'content_type': 'application/async-deleted' }, # task objects for unicode test { 'name': self.past_time + u'-a8/c8/o8\u2661', 'content_type': 'application/async-deleted' }, { 'name': self.past_time + u'-a9/c9/o9\xf8', 'content_type': 'application/async-deleted' }, ] } } async_delete_fake_swift = FakeInternalClient(async_delete_aco_dict) x = expirer.ObjectExpirer(self.conf, logger=self.logger, swift=async_delete_fake_swift) expected = [ self.make_task(self.past_time, target_path, is_async_delete=True) for target_path in self.expired_target_path_list ] self.assertEqual( list( x.iter_task_to_expire(task_account_container_list, my_index, divisor)), expected)
def test_delete_at_time_of_task_container(self): x = expirer.ObjectExpirer(self.conf, logger=self.logger) self.assertEqual(x.delete_at_time_of_task_container('0000'), 0) self.assertEqual(x.delete_at_time_of_task_container('0001'), 1) self.assertEqual(x.delete_at_time_of_task_container('1000'), 1000)
def test_round_robin_order(self): x = expirer.ObjectExpirer(self.conf, logger=self.logger) task_con_obj_list = [ # objects in 0000 timestamp container self.make_task('0000', 'a/c0/o0'), self.make_task('0000', 'a/c0/o1'), # objects in 0001 timestamp container self.make_task('0001', 'a/c1/o0'), self.make_task('0001', 'a/c1/o1'), # objects in 0002 timestamp container self.make_task('0002', 'a/c2/o0'), self.make_task('0002', 'a/c2/o1'), ] result = list(x.round_robin_order(task_con_obj_list)) # sorted by popping one object to delete for each target_container expected = [ self.make_task('0000', 'a/c0/o0'), self.make_task('0001', 'a/c1/o0'), self.make_task('0002', 'a/c2/o0'), self.make_task('0000', 'a/c0/o1'), self.make_task('0001', 'a/c1/o1'), self.make_task('0002', 'a/c2/o1'), ] self.assertEqual(expected, result) # task containers have some task objects with invalid target paths task_con_obj_list = [ # objects in 0000 timestamp container self.make_task('0000', 'invalid0'), self.make_task('0000', 'a/c0/o0'), self.make_task('0000', 'a/c0/o1'), # objects in 0001 timestamp container self.make_task('0001', 'a/c1/o0'), self.make_task('0001', 'invalid1'), self.make_task('0001', 'a/c1/o1'), # objects in 0002 timestamp container self.make_task('0002', 'a/c2/o0'), self.make_task('0002', 'a/c2/o1'), self.make_task('0002', 'invalid2'), ] result = list(x.round_robin_order(task_con_obj_list)) # the invalid task objects are ignored expected = [ self.make_task('0000', 'a/c0/o0'), self.make_task('0001', 'a/c1/o0'), self.make_task('0002', 'a/c2/o0'), self.make_task('0000', 'a/c0/o1'), self.make_task('0001', 'a/c1/o1'), self.make_task('0002', 'a/c2/o1'), ] self.assertEqual(expected, result) # for a given target container, tasks won't necessarily all go in # the same timestamp container task_con_obj_list = [ # objects in 0000 timestamp container self.make_task('0000', 'a/c0/o0'), self.make_task('0000', 'a/c0/o1'), self.make_task('0000', 'a/c2/o2'), self.make_task('0000', 'a/c2/o3'), # objects in 0001 timestamp container self.make_task('0001', 'a/c0/o2'), self.make_task('0001', 'a/c0/o3'), self.make_task('0001', 'a/c1/o0'), self.make_task('0001', 'a/c1/o1'), # objects in 0002 timestamp container self.make_task('0002', 'a/c2/o0'), self.make_task('0002', 'a/c2/o1'), ] result = list(x.round_robin_order(task_con_obj_list)) # so we go around popping by *target* container, not *task* container expected = [ self.make_task('0000', 'a/c0/o0'), self.make_task('0001', 'a/c1/o0'), self.make_task('0000', 'a/c2/o2'), self.make_task('0000', 'a/c0/o1'), self.make_task('0001', 'a/c1/o1'), self.make_task('0000', 'a/c2/o3'), self.make_task('0001', 'a/c0/o2'), self.make_task('0002', 'a/c2/o0'), self.make_task('0001', 'a/c0/o3'), self.make_task('0002', 'a/c2/o1'), ] self.assertEqual(expected, result) # all of the work to be done could be for different target containers task_con_obj_list = [ # objects in 0000 timestamp container self.make_task('0000', 'a/c0/o'), self.make_task('0000', 'a/c1/o'), self.make_task('0000', 'a/c2/o'), self.make_task('0000', 'a/c3/o'), # objects in 0001 timestamp container self.make_task('0001', 'a/c4/o'), self.make_task('0001', 'a/c5/o'), self.make_task('0001', 'a/c6/o'), self.make_task('0001', 'a/c7/o'), # objects in 0002 timestamp container self.make_task('0002', 'a/c8/o'), self.make_task('0002', 'a/c9/o'), ] result = list(x.round_robin_order(task_con_obj_list)) # in which case, we kind of hammer the task containers self.assertEqual(task_con_obj_list, result)
def test_iter_task_to_expire(self): fake_swift = FakeInternalClient({ '.expiring_objects': { u'1234': ['1234-a0/c0/o0', '1234-a1/c1/o1'], u'2000': ['2000-a2/c2/o2', '2000-a3/c3/o3'], } }) x = expirer.ObjectExpirer(self.conf, logger=self.logger, swift=fake_swift) # In this test, all tasks are assigned to the tested expirer my_index = 0 divisor = 1 task_account_container_list = [('.expiring_objects', u'1234'), ('.expiring_objects', u'2000')] expected = [{ 'task_account': '.expiring_objects', 'task_container': u'1234', 'task_object': '1234-a0/c0/o0', 'target_path': 'a0/c0/o0', 'delete_timestamp': Timestamp(1234), }, { 'task_account': '.expiring_objects', 'task_container': u'1234', 'task_object': '1234-a1/c1/o1', 'target_path': 'a1/c1/o1', 'delete_timestamp': Timestamp(1234), }, { 'task_account': '.expiring_objects', 'task_container': u'2000', 'task_object': '2000-a2/c2/o2', 'target_path': 'a2/c2/o2', 'delete_timestamp': Timestamp(2000), }, { 'task_account': '.expiring_objects', 'task_container': u'2000', 'task_object': '2000-a3/c3/o3', 'target_path': 'a3/c3/o3', 'delete_timestamp': Timestamp(2000), }] self.assertEqual( list( x.iter_task_to_expire(task_account_container_list, my_index, divisor)), expected) # the task queue has invalid task object fake_swift = FakeInternalClient({ '.expiring_objects': { u'1234': ['1234-invalid', '1234-a0/c0/o0', '1234-a1/c1/o1'], u'2000': ['2000-a2/c2/o2', '2000-invalid', '2000-a3/c3/o3'], } }) x = expirer.ObjectExpirer(self.conf, logger=self.logger, swift=fake_swift) # but the invalid tasks are skipped self.assertEqual( list( x.iter_task_to_expire(task_account_container_list, my_index, divisor)), expected)