def test_success_gets_counted(self):
class InternalClient(object):
def __init__(self, containers, objects):
self.containers = containers
self.objects = objects
def get_account_info(*a, **kw):
return 1, 2
def iter_containers(self, *a, **kw):
return self.containers
def delete_container(*a, **kw):
pass
def delete_object(*a, **kw):
pass
def iter_objects(self, *a, **kw):
return self.objects
x = expirer.ObjectExpirer({})
x.logger = FakeLogger()
x.delete_actual_object = lambda o, t: None
self.assertEqual(x.report_objects, 0)
x.swift = InternalClient([{
'name': str(int(time() - 86400))
}], [{
'name': '%d-actual-obj' % int(time() - 86400)
}])
x.run_once()
self.assertEqual(x.report_objects, 1)
self.assertEqual(x.logger.log_dict['info'],
[(('Pass beginning; 1 possible containers; '
'2 possible objects', ), {}),
(('Pass completed in 0s; 1 objects expired', ), {})])
def test_run_forever_catches_usual_exceptions(self):
raises = [0]
def raise_exceptions():
raises[0] += 1
if raises[0] < 2:
raise Exception('exception %d' % raises[0])
raise SystemExit('exiting exception %d' % raises[0])
x = expirer.ObjectExpirer({}, logger=self.logger)
orig_sleep = expirer.sleep
try:
expirer.sleep = not_sleep
x.run_once = raise_exceptions
x.run_forever()
except SystemExit as err:
pass
finally:
expirer.sleep = orig_sleep
self.assertEqual(str(err), 'exiting exception 2')
self.assertEqual(x.logger.get_lines_for_level('error'),
['Unhandled exception: '])
log_args, log_kwargs = x.logger.log_dict['error'][0]
self.assertEqual(str(log_kwargs['exc_info'][1]), 'exception 1')
def test_failed_delete_continues_on(self):
class InternalClient(object):
container_ring = None
def __init__(self, containers, objects):
self.containers = containers
self.objects = objects
def get_account_info(*a, **kw):
return 1, 2
def iter_containers(self, *a, **kw):
return self.containers
def delete_container(*a, **kw):
raise Exception('failed to delete container')
def delete_object(*a, **kw):
pass
def iter_objects(self, *a, **kw):
return self.objects
def fail_delete_actual_object(actual_obj, timestamp):
raise Exception('failed to delete actual object')
x = expirer.ObjectExpirer(self.conf, logger=self.logger)
cts = int(time() - 86400)
ots = int(time() - 86400)
containers = [
{
'name': str(cts)
},
{
'name': str(cts + 1)
},
]
objects = [{
'name': '%d-actual-obj' % ots
}, {
'name': '%d-next-obj' % ots
}]
x.swift = InternalClient(containers, objects)
x.delete_actual_object = fail_delete_actual_object
x.run_once()
error_lines = x.logger.get_lines_for_level('error')
self.assertEqual(
sorted(error_lines),
sorted([
'Exception while deleting object %d %d-actual-obj failed to '
'delete actual object: ' % (cts, ots),
'Exception while deleting object %d %d-next-obj failed to '
'delete actual object: ' % (cts, ots),
'Exception while deleting object %d %d-actual-obj failed to '
'delete actual object: ' % (cts + 1, ots),
'Exception while deleting object %d %d-next-obj failed to '
'delete actual object: ' % (cts + 1, ots),
'Exception while deleting container %d failed to delete '
'container: ' % (cts, ),
'Exception while deleting container %d failed to delete '
'container: ' % (cts + 1, )
]))
self.assertEqual(x.logger.get_lines_for_level('info'), [
'Pass beginning; 1 possible containers; 2 possible objects',
'Pass completed in 0s; 0 objects expired',
])
def test_failed_delete_keeps_entry(self):
class InternalClient(object):
container_ring = None
def __init__(self, containers, objects):
self.containers = containers
self.objects = objects
def get_account_info(*a, **kw):
return 1, 2
def iter_containers(self, *a, **kw):
return self.containers
def delete_container(*a, **kw):
pass
def iter_objects(self, *a, **kw):
return self.objects
def deliberately_blow_up(actual_obj, timestamp):
raise Exception('failed to delete actual object')
def should_not_get_called(container, obj):
raise Exception('This should not have been called')
ts = int(time() - 86400)
fake_swift = InternalClient([{
'name': str(int(time() - 86400))
}], [{
'name': '%d-actual-obj' % ts
}])
x = expirer.ObjectExpirer(self.conf,
logger=self.logger,
swift=fake_swift)
x.iter_containers = lambda: [str(int(time() - 86400))]
x.delete_actual_object = deliberately_blow_up
x.pop_queue = should_not_get_called
x.run_once()
error_lines = x.logger.get_lines_for_level('error')
self.assertEqual(error_lines, [
'Exception while deleting object %d %d-actual-obj '
'failed to delete actual object: ' % (ts, ts)
])
self.assertEqual(x.logger.get_lines_for_level('info'), [
'Pass beginning; 1 possible containers; 2 possible objects',
'Pass completed in 0s; 0 objects expired',
])
# Reverse test to be sure it still would blow up the way expected.
ts = int(time() - 86400)
fake_swift = InternalClient([{
'name': str(int(time() - 86400))
}], [{
'name': '%d-actual-obj' % ts
}])
self.logger._clear()
x = expirer.ObjectExpirer(self.conf,
logger=self.logger,
swift=fake_swift)
x.delete_actual_object = lambda o, t: None
x.pop_queue = should_not_get_called
x.run_once()
self.assertEqual(self.logger.get_lines_for_level('error'), [
'Exception while deleting object %d %d-actual-obj This should '
'not have been called: ' % (ts, ts)
])
def test_round_robin_order(self):
def make_task(delete_at, target):
return {
'task_container': delete_at,
'task_object': delete_at + '-' + target,
'delete_timestamp': Timestamp(delete_at),
'target_path': target,
}
x = expirer.ObjectExpirer(self.conf, logger=self.logger)
task_con_obj_list = [
# objects in 0000 timestamp container
make_task('0000', 'a/c0/o0'),
make_task('0000', 'a/c0/o1'),
# objects in 0001 timestamp container
make_task('0001', 'a/c1/o0'),
make_task('0001', 'a/c1/o1'),
# objects in 0002 timestamp container
make_task('0002', 'a/c2/o0'),
make_task('0002', 'a/c2/o1'),
]
result = list(x.round_robin_order(task_con_obj_list))
# sorted by popping one object to delete for each target_container
expected = [
make_task('0000', 'a/c0/o0'),
make_task('0001', 'a/c1/o0'),
make_task('0002', 'a/c2/o0'),
make_task('0000', 'a/c0/o1'),
make_task('0001', 'a/c1/o1'),
make_task('0002', 'a/c2/o1'),
]
self.assertEqual(expected, result)
# for a given target container, tasks won't necessarily all go in
# the same timestamp container
task_con_obj_list = [
# objects in 0000 timestamp container
make_task('0000', 'a/c0/o0'),
make_task('0000', 'a/c0/o1'),
make_task('0000', 'a/c2/o2'),
make_task('0000', 'a/c2/o3'),
# objects in 0001 timestamp container
make_task('0001', 'a/c0/o2'),
make_task('0001', 'a/c0/o3'),
make_task('0001', 'a/c1/o0'),
make_task('0001', 'a/c1/o1'),
# objects in 0002 timestamp container
make_task('0002', 'a/c2/o0'),
make_task('0002', 'a/c2/o1'),
]
result = list(x.round_robin_order(task_con_obj_list))
# so we go around popping by *target* container, not *task* container
expected = [
make_task('0000', 'a/c0/o0'),
make_task('0001', 'a/c1/o0'),
make_task('0000', 'a/c2/o2'),
make_task('0000', 'a/c0/o1'),
make_task('0001', 'a/c1/o1'),
make_task('0000', 'a/c2/o3'),
make_task('0001', 'a/c0/o2'),
make_task('0002', 'a/c2/o0'),
make_task('0001', 'a/c0/o3'),
make_task('0002', 'a/c2/o1'),
]
self.assertEqual(expected, result)
# all of the work to be done could be for different target containers
task_con_obj_list = [
# objects in 0000 timestamp container
make_task('0000', 'a/c0/o'),
make_task('0000', 'a/c1/o'),
make_task('0000', 'a/c2/o'),
make_task('0000', 'a/c3/o'),
# objects in 0001 timestamp container
make_task('0001', 'a/c4/o'),
make_task('0001', 'a/c5/o'),
make_task('0001', 'a/c6/o'),
make_task('0001', 'a/c7/o'),
# objects in 0002 timestamp container
make_task('0002', 'a/c8/o'),
make_task('0002', 'a/c9/o'),
]
result = list(x.round_robin_order(task_con_obj_list))
# in which case, we kind of hammer the task containers
self.assertEqual(task_con_obj_list, result)
def test_failed_delete_continues_on(self):
class InternalClient(object):
def __init__(self, containers, objects):
self.containers = containers
self.objects = objects
def get_account_info(*a, **kw):
return 1, 2
def iter_containers(self, *a, **kw):
return self.containers
def delete_container(*a, **kw):
raise Exception('failed to delete container')
def delete_object(*a, **kw):
pass
def iter_objects(self, *a, **kw):
return self.objects
def fail_delete_actual_object(actual_obj, timestamp):
raise Exception('failed to delete actual object')
x = expirer.ObjectExpirer({})
x.logger = FakeLogger()
cts = int(time() - 86400)
ots = int(time() - 86400)
containers = [
{'name': str(cts)},
{'name': str(cts + 1)},
]
objects = [
{'name': '%d-actual-obj' % ots},
{'name': '%d-next-obj' % ots}
]
x.swift = InternalClient(containers, objects)
x.delete_actual_object = fail_delete_actual_object
x.run_once()
excswhiledeleting = []
for exccall in x.logger.log_dict['exception']:
if exccall[0][0].startswith('Exception while deleting '):
excswhiledeleting.append(exccall[0][0])
self.assertEquals(excswhiledeleting, [
'Exception while deleting object %d %d-actual-obj failed to '
'delete actual object' % (cts, ots),
'Exception while deleting object %d %d-next-obj failed to '
'delete actual object' % (cts, ots),
'Exception while deleting container %d failed to delete '
'container' % (cts,),
'Exception while deleting object %d %d-actual-obj failed to '
'delete actual object' % (cts + 1, ots),
'Exception while deleting object %d %d-next-obj failed to '
'delete actual object' % (cts + 1, ots),
'Exception while deleting container %d failed to delete '
'container' % (cts + 1,)])
self.assertEquals(x.logger.log_dict['info'],
[(('Pass beginning; 1 possible containers; '
'2 possible objects',), {}),
(('Pass completed in 0s; 0 objects expired',), {})])
def test_failed_delete_keeps_entry(self):
class InternalClient(object):
def __init__(self, containers, objects):
self.containers = containers
self.objects = objects
def get_account_info(*a, **kw):
return 1, 2
def iter_containers(self, *a, **kw):
return self.containers
def delete_container(*a, **kw):
pass
def delete_object(*a, **kw):
raise Exception('This should not have been called')
def iter_objects(self, *a, **kw):
return self.objects
def deliberately_blow_up(actual_obj, timestamp):
raise Exception('failed to delete actual object')
def should_not_get_called(container, obj):
raise Exception('This should not have been called')
x = expirer.ObjectExpirer({})
x.logger = FakeLogger()
x.iter_containers = lambda: [str(int(time() - 86400))]
ts = int(time() - 86400)
x.delete_actual_object = deliberately_blow_up
x.swift = InternalClient([{'name': str(int(time() - 86400))}],
[{'name': '%d-actual-obj' % ts}])
x.run_once()
excswhiledeleting = []
for exccall in x.logger.log_dict['exception']:
if exccall[0][0].startswith('Exception while deleting '):
excswhiledeleting.append(exccall[0][0])
self.assertEquals(excswhiledeleting,
['Exception while deleting object %d %d-actual-obj '
'failed to delete actual object' % (ts, ts)])
self.assertEquals(x.logger.log_dict['info'],
[(('Pass beginning; 1 possible containers; '
'2 possible objects',), {}),
(('Pass completed in 0s; 0 objects expired',), {})])
# Reverse test to be sure it still would blow up the way expected.
x = expirer.ObjectExpirer({})
x.logger = FakeLogger()
ts = int(time() - 86400)
x.delete_actual_object = lambda o, t: None
x.swift = InternalClient([{'name': str(int(time() - 86400))}],
[{'name': '%d-actual-obj' % ts}])
x.run_once()
excswhiledeleting = []
for exccall in x.logger.log_dict['exception']:
if exccall[0][0].startswith('Exception while deleting '):
excswhiledeleting.append(exccall[0][0])
self.assertEquals(excswhiledeleting,
['Exception while deleting object %d %d-actual-obj This should '
'not have been called' % (ts, ts)])
def test_iter_task_to_expire(self):
# In this test, all tasks are assigned to the tested expirer
my_index = 0
divisor = 1
task_account_container_list = [('.expiring_objects', self.past_time)]
expected = [
self.make_task(self.past_time, target_path)
for target_path in self.expired_target_path_list
]
self.assertEqual(
list(
self.expirer.iter_task_to_expire(task_account_container_list,
my_index, divisor)), expected)
# the task queue has invalid task object
invalid_aco_dict = deepcopy(self.fake_swift.aco_dict)
invalid_aco_dict['.expiring_objects'][self.past_time].insert(
0, self.past_time + '-invalid0')
invalid_aco_dict['.expiring_objects'][self.past_time].insert(
5, self.past_time + '-invalid1')
invalid_fake_swift = FakeInternalClient(invalid_aco_dict)
x = expirer.ObjectExpirer(self.conf,
logger=self.logger,
swift=invalid_fake_swift)
# but the invalid tasks are skipped
self.assertEqual(
list(
x.iter_task_to_expire(task_account_container_list, my_index,
divisor)), expected)
# test some of that async delete
async_delete_aco_dict = {
'.expiring_objects': {
# this task container will be checked
self.past_time: [
# tasks ready for execution
{
'name': self.past_time + '-a0/c0/o0',
'content_type': 'application/async-deleted'
},
{
'name': self.past_time + '-a1/c1/o1',
'content_type': 'application/async-deleted'
},
{
'name': self.past_time + '-a2/c2/o2',
'content_type': 'application/async-deleted'
},
{
'name': self.past_time + '-a3/c3/o3',
'content_type': 'application/async-deleted'
},
{
'name': self.past_time + '-a4/c4/o4',
'content_type': 'application/async-deleted'
},
{
'name': self.past_time + '-a5/c5/o5',
'content_type': 'application/async-deleted'
},
{
'name': self.past_time + '-a6/c6/o6',
'content_type': 'application/async-deleted'
},
{
'name': self.past_time + '-a7/c7/o7',
'content_type': 'application/async-deleted'
},
# task objects for unicode test
{
'name': self.past_time + u'-a8/c8/o8\u2661',
'content_type': 'application/async-deleted'
},
{
'name': self.past_time + u'-a9/c9/o9\xf8',
'content_type': 'application/async-deleted'
},
]
}
}
async_delete_fake_swift = FakeInternalClient(async_delete_aco_dict)
x = expirer.ObjectExpirer(self.conf,
logger=self.logger,
swift=async_delete_fake_swift)
expected = [
self.make_task(self.past_time, target_path, is_async_delete=True)
for target_path in self.expired_target_path_list
]
self.assertEqual(
list(
x.iter_task_to_expire(task_account_container_list, my_index,
divisor)), expected)
def test_delete_at_time_of_task_container(self):
x = expirer.ObjectExpirer(self.conf, logger=self.logger)
self.assertEqual(x.delete_at_time_of_task_container('0000'), 0)
self.assertEqual(x.delete_at_time_of_task_container('0001'), 1)
self.assertEqual(x.delete_at_time_of_task_container('1000'), 1000)
def test_round_robin_order(self):
x = expirer.ObjectExpirer(self.conf, logger=self.logger)
task_con_obj_list = [
# objects in 0000 timestamp container
self.make_task('0000', 'a/c0/o0'),
self.make_task('0000', 'a/c0/o1'),
# objects in 0001 timestamp container
self.make_task('0001', 'a/c1/o0'),
self.make_task('0001', 'a/c1/o1'),
# objects in 0002 timestamp container
self.make_task('0002', 'a/c2/o0'),
self.make_task('0002', 'a/c2/o1'),
]
result = list(x.round_robin_order(task_con_obj_list))
# sorted by popping one object to delete for each target_container
expected = [
self.make_task('0000', 'a/c0/o0'),
self.make_task('0001', 'a/c1/o0'),
self.make_task('0002', 'a/c2/o0'),
self.make_task('0000', 'a/c0/o1'),
self.make_task('0001', 'a/c1/o1'),
self.make_task('0002', 'a/c2/o1'),
]
self.assertEqual(expected, result)
# task containers have some task objects with invalid target paths
task_con_obj_list = [
# objects in 0000 timestamp container
self.make_task('0000', 'invalid0'),
self.make_task('0000', 'a/c0/o0'),
self.make_task('0000', 'a/c0/o1'),
# objects in 0001 timestamp container
self.make_task('0001', 'a/c1/o0'),
self.make_task('0001', 'invalid1'),
self.make_task('0001', 'a/c1/o1'),
# objects in 0002 timestamp container
self.make_task('0002', 'a/c2/o0'),
self.make_task('0002', 'a/c2/o1'),
self.make_task('0002', 'invalid2'),
]
result = list(x.round_robin_order(task_con_obj_list))
# the invalid task objects are ignored
expected = [
self.make_task('0000', 'a/c0/o0'),
self.make_task('0001', 'a/c1/o0'),
self.make_task('0002', 'a/c2/o0'),
self.make_task('0000', 'a/c0/o1'),
self.make_task('0001', 'a/c1/o1'),
self.make_task('0002', 'a/c2/o1'),
]
self.assertEqual(expected, result)
# for a given target container, tasks won't necessarily all go in
# the same timestamp container
task_con_obj_list = [
# objects in 0000 timestamp container
self.make_task('0000', 'a/c0/o0'),
self.make_task('0000', 'a/c0/o1'),
self.make_task('0000', 'a/c2/o2'),
self.make_task('0000', 'a/c2/o3'),
# objects in 0001 timestamp container
self.make_task('0001', 'a/c0/o2'),
self.make_task('0001', 'a/c0/o3'),
self.make_task('0001', 'a/c1/o0'),
self.make_task('0001', 'a/c1/o1'),
# objects in 0002 timestamp container
self.make_task('0002', 'a/c2/o0'),
self.make_task('0002', 'a/c2/o1'),
]
result = list(x.round_robin_order(task_con_obj_list))
# so we go around popping by *target* container, not *task* container
expected = [
self.make_task('0000', 'a/c0/o0'),
self.make_task('0001', 'a/c1/o0'),
self.make_task('0000', 'a/c2/o2'),
self.make_task('0000', 'a/c0/o1'),
self.make_task('0001', 'a/c1/o1'),
self.make_task('0000', 'a/c2/o3'),
self.make_task('0001', 'a/c0/o2'),
self.make_task('0002', 'a/c2/o0'),
self.make_task('0001', 'a/c0/o3'),
self.make_task('0002', 'a/c2/o1'),
]
self.assertEqual(expected, result)
# all of the work to be done could be for different target containers
task_con_obj_list = [
# objects in 0000 timestamp container
self.make_task('0000', 'a/c0/o'),
self.make_task('0000', 'a/c1/o'),
self.make_task('0000', 'a/c2/o'),
self.make_task('0000', 'a/c3/o'),
# objects in 0001 timestamp container
self.make_task('0001', 'a/c4/o'),
self.make_task('0001', 'a/c5/o'),
self.make_task('0001', 'a/c6/o'),
self.make_task('0001', 'a/c7/o'),
# objects in 0002 timestamp container
self.make_task('0002', 'a/c8/o'),
self.make_task('0002', 'a/c9/o'),
]
result = list(x.round_robin_order(task_con_obj_list))
# in which case, we kind of hammer the task containers
self.assertEqual(task_con_obj_list, result)
def test_iter_task_to_expire(self):
fake_swift = FakeInternalClient({
'.expiring_objects': {
u'1234': ['1234-a0/c0/o0', '1234-a1/c1/o1'],
u'2000': ['2000-a2/c2/o2', '2000-a3/c3/o3'],
}
})
x = expirer.ObjectExpirer(self.conf,
logger=self.logger,
swift=fake_swift)
# In this test, all tasks are assigned to the tested expirer
my_index = 0
divisor = 1
task_account_container_list = [('.expiring_objects', u'1234'),
('.expiring_objects', u'2000')]
expected = [{
'task_account': '.expiring_objects',
'task_container': u'1234',
'task_object': '1234-a0/c0/o0',
'target_path': 'a0/c0/o0',
'delete_timestamp': Timestamp(1234),
}, {
'task_account': '.expiring_objects',
'task_container': u'1234',
'task_object': '1234-a1/c1/o1',
'target_path': 'a1/c1/o1',
'delete_timestamp': Timestamp(1234),
}, {
'task_account': '.expiring_objects',
'task_container': u'2000',
'task_object': '2000-a2/c2/o2',
'target_path': 'a2/c2/o2',
'delete_timestamp': Timestamp(2000),
}, {
'task_account': '.expiring_objects',
'task_container': u'2000',
'task_object': '2000-a3/c3/o3',
'target_path': 'a3/c3/o3',
'delete_timestamp': Timestamp(2000),
}]
self.assertEqual(
list(
x.iter_task_to_expire(task_account_container_list, my_index,
divisor)), expected)
# the task queue has invalid task object
fake_swift = FakeInternalClient({
'.expiring_objects': {
u'1234': ['1234-invalid', '1234-a0/c0/o0', '1234-a1/c1/o1'],
u'2000': ['2000-a2/c2/o2', '2000-invalid', '2000-a3/c3/o3'],
}
})
x = expirer.ObjectExpirer(self.conf,
logger=self.logger,
swift=fake_swift)
# but the invalid tasks are skipped
self.assertEqual(
list(
x.iter_task_to_expire(task_account_container_list, my_index,
divisor)), expected)
