def test_restore_machine_removing_locking_on_failure(self, log, nginx):
redis_lock = redis.StrictRedis().lock('restore_lock',
timeout=600,
blocking_timeout=1)
self.assertTrue(redis_lock.acquire(blocking=False))
redis_lock.release()
FakeManager.fail_ids = [2]
restorer = healing.RestoreMachine(self.config)
nginx_manager = nginx.Nginx.return_value
restorer = healing.RestoreMachine(self.config)
restorer.start()
time.sleep(1)
restorer.stop()
tasks = [
task['_id'] for task in self.storage.find_task(
{"_id": {
"$regex": "restore_.+"
}})
]
self.assertListEqual([
'restore_10.2.2.2', 'restore_10.3.3.3', 'restore_10.4.4.4',
'restore_10.5.5.5'
], tasks)
self.assertEqual(log.info.call_args_list, [call("Machine 0 restored")])
self.assertEqual(nginx_manager.wait_healthcheck.call_args_list,
[call('10.1.1.1', timeout=600)])
self.assertTrue(redis_lock.acquire(blocking=False))
redis_lock.release()
def test_restore_machine_removing_locking_on_success(self, log, nginx):
redis_lock = redis.StrictRedis().lock('restore_lock',
timeout=600,
blocking_timeout=1)
self.assertTrue(redis_lock.acquire(blocking=False))
redis_lock.release()
restorer = healing.RestoreMachine(self.config)
nginx_manager = nginx.Nginx.return_value
restorer = healing.RestoreMachine(self.config)
restorer.start()
time.sleep(1)
restorer.stop()
nginx_expected_calls = [
call('10.1.1.1', timeout=600),
call('10.3.3.3', timeout=600),
call('10.4.4.4', timeout=600),
call('10.5.5.5', timeout=600)
]
self.assertEqual(nginx_expected_calls,
nginx_manager.wait_healthcheck.call_args_list)
tasks = [
task['_id'] for task in self.storage.find_task(
{"_id": {
"$regex": "restore_.+"
}})
]
self.assertListEqual(tasks, ['restore_10.2.2.2'])
self.assertTrue(redis_lock.acquire(blocking=False))
redis_lock.release()
def test_restore_machine_store_healing_events(self, log, nginx,
extend_lock):
time_now = datetime.datetime.utcnow()
def healing_time(x):
return time_now + datetime.timedelta(seconds=x)
with patch.object(tasks.datetime.datetime, 'utcnow') as mock_time_now:
time_side_effect = []
for x in range(18):
time_side_effect.append(time_now +
datetime.timedelta(seconds=10 * x))
mock_time_now.side_effect = time_side_effect
FakeManager.fail_ids = [4]
restorer = healing.RestoreMachine(self.config)
restorer.start()
time.sleep(1)
restorer.stop()
expected_healings = [{
'end_time': healing_time(40),
'machine': '10.1.1.1',
'start_time': healing_time(30),
'status': 'success'
}, {
'end_time': healing_time(80),
'machine': '10.3.3.3',
'start_time': healing_time(70),
'status': 'success'
}, {
'end_time': healing_time(120),
'machine': '10.4.4.4',
'start_time': healing_time(110),
'status': 'success'
}]
foo_instances_healings = []
filter_fields = {
"status": 1,
"start_time": 1,
"machine": 1,
"end_time": 1
}
healing_collection = self.storage.db[
self.storage.healing_collection]
for event in healing_collection.find({"instance": "foo"},
filter_fields):
del event['_id']
foo_instances_healings.append(event)
self.assertListEqual(foo_instances_healings, expected_healings)
expected_healings = [{
'end_time': healing_time(160),
'machine': '10.5.5.5',
'start_time': healing_time(150),
'status': 'iaas restore error'
}]
bar_instances_healings = []
for event in healing_collection.find({"instance": "bar"},
filter_fields):
del event['_id']
bar_instances_healings.append(event)
self.assertListEqual(bar_instances_healings, expected_healings)
def test_restore_machine_extending_time_btw_restores(
self, log, nginx, extend_lock, update_healing, store_healing):
time_now = datetime.datetime.utcnow()
with patch.object(tasks.datetime.datetime, 'utcnow') as mock_time_now:
time_side_effect = []
for x in range(9):
time_side_effect.append(time_now +
datetime.timedelta(seconds=10 * x))
mock_time_now.side_effect = time_side_effect
nginx_manager = nginx.Nginx.return_value
restorer = healing.RestoreMachine(self.config)
restorer.start()
time.sleep(1)
restorer.stop()
self.assertEqual(extend_lock.call_args_list, [
call(extra_time=10),
call(extra_time=10),
call(extra_time=10)
])
nginx_expected_calls = [
call('10.1.1.1', timeout=600),
call('10.3.3.3', timeout=600),
call('10.4.4.4', timeout=600),
call('10.5.5.5', timeout=600)
]
self.assertEqual(nginx_expected_calls,
nginx_manager.wait_healthcheck.call_args_list)
tasks_restore = []
for task in self.storage.find_task(
{"_id": {
"$regex": "restore_.+"
}}):
tasks_restore.append(task['_id'])
self.assertListEqual(tasks_restore, ['restore_10.2.2.2'])
def test_restore_machine_success(self, log, nginx):
nginx_manager = nginx.Nginx.return_value
restorer = healing.RestoreMachine(self.config)
restorer.start()
time.sleep(1)
restorer.stop()
self.assertEqual(log.info.call_args_list, [
call("Machine 0 restored"),
call("Machine 2 restored"),
call("Machine 3 restored"),
call("Machine 4 restored")
])
nginx_expected_calls = [
call('10.1.1.1', timeout=600),
call('10.3.3.3', timeout=600),
call('10.4.4.4', timeout=600),
call('10.5.5.5', timeout=600)
]
self.assertEqual(nginx_expected_calls,
nginx_manager.wait_healthcheck.call_args_list)
tasks = [
task['_id'] for task in self.storage.find_task(
{"_id": {
"$regex": "restore_.+"
}})
]
self.assertListEqual(tasks, ['restore_10.2.2.2'])
def test_restore_machine_dry_mode(self, log, nginx):
self.config['RESTORE_MACHINE_DRY_MODE'] = "1"
nginx_manager = nginx.Nginx.return_value
restorer = healing.RestoreMachine(self.config)
restorer.start()
time.sleep(1)
restorer.stop()
self.assertListEqual(log.info.call_args_list, [])
self.assertListEqual(nginx_manager.wait_healthcheck.call_args_list, [])
tasks = [
task['_id'] for task in self.storage.find_task(
{"_id": {
"$regex": "restore_.+"
}})
]
self.assertListEqual(tasks, ['restore_10.2.2.2'])
def test_restore_machine_iaas_fail(self, log, nginx):
FakeManager.fail_ids = [2]
restorer = healing.RestoreMachine(self.config)
nginx_manager = nginx.Nginx.return_value
restorer.start()
time.sleep(1)
restorer.stop()
tasks = [
task['_id'] for task in self.storage.find_task(
{"_id": {
"$regex": "restore_.+"
}})
]
self.assertListEqual([
'restore_10.2.2.2', 'restore_10.3.3.3', 'restore_10.4.4.4',
'restore_10.5.5.5'
], tasks)
self.assertEqual(log.info.call_args_list, [call("Machine 0 restored")])
self.assertEqual(nginx_manager.wait_healthcheck.call_args_list,
[call('10.1.1.1', timeout=600)])
log.reset_mock()
nginx.reset_mock()
redis.StrictRedis().delete("restore_machine:last_run")
restorer = healing.RestoreMachine(self.config)
restorer.start()
time.sleep(1)
restorer.stop()
tasks = [
task['_id'] for task in self.storage.find_task(
{"_id": {
"$regex": "restore_.+"
}})
]
self.assertEqual(log.info.call_args_list, [call("Machine 4 restored")])
self.assertEqual(nginx_manager.wait_healthcheck.call_args_list,
[call('10.5.5.5', timeout=600)])
self.assertListEqual(
['restore_10.2.2.2', 'restore_10.3.3.3', 'restore_10.4.4.4'],
tasks)
log.reset_mock()
nginx.reset_mock()
redis.StrictRedis().delete("restore_machine:last_run")
FakeManager.fail_ids = []
with freeze_time("2016-02-03 12:06:00"):
restorer = healing.RestoreMachine(self.config)
restorer.start()
time.sleep(1)
restorer.stop()
tasks = [
task['_id'] for task in self.storage.find_task(
{"_id": {
"$regex": "restore_.+"
}})
]
self.assertEqual(log.info.call_args_list, [
call("Machine 1 restored"),
call("Machine 2 restored"),
call("Machine 3 restored")
])
nginx_expected_calls = [
call('10.2.2.2', timeout=600),
call('10.3.3.3', timeout=600),
call('10.4.4.4', timeout=600)
]
self.assertEqual(nginx_expected_calls,
nginx_manager.wait_healthcheck.call_args_list)
self.assertListEqual(tasks, [])