def test_retry_deletion(self):
delete_age = self.configuration.get("delete_age")
delete_delta = timezone.timedelta(days=(delete_age + 1))
delete_timestamp = timezone.now() - delete_delta
state = 'RESUME_COMPLETE'
stack_name = 'bogus_stack'
stack = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack_name,
suspend_timestamp=delete_timestamp)
stack.status = state
stack.save()
mock_heat_client = Mock()
mock_heat_client.stacks.get.side_effect = [
self.stacks[state], self.stacks[DELETE_FAILED_STATE],
self.stacks[DELETE_FAILED_STATE], self.stacks[DELETE_FAILED_STATE]
]
job = ReaperJob(self.configuration)
with patch.multiple(
job, get_heat_client=Mock(return_value=mock_heat_client)):
job.run()
mock_heat_client.stacks.delete.assert_has_calls([
call(stack_id=stack_name),
call(stack_id=stack_name),
call(stack_id=stack_name)
])
stack = Stack.objects.get(name=stack_name)
self.assertEqual(stack.status, DELETE_FAILED_STATE)
def test_dont_suspend_stack_with_no_provider(self):
# Setup
suspend_timeout = self.settings.get("suspend_timeout")
timedelta = timezone.timedelta(seconds=(suspend_timeout + 1))
suspend_timestamp = timezone.now() - timedelta
state = "RESUME_COMPLETE"
stack = Stack(student_id=self.student_id,
course_id=self.course_id,
suspend_timestamp=suspend_timestamp,
name=self.stack_name,
status=state)
stack.save()
mock_suspend_task = self.get_suspend_task_mock()
# Run
job = SuspenderJob(self.settings)
job.run()
# Assert
mock_suspend_task.apply_async.assert_not_called()
stack = Stack.objects.get(name=self.stack_name)
self.assertEqual(stack.status, state)
def test_dont_delete_if_age_is_zero(self):
self.configuration["delete_age"] = 0
delete_delta = timezone.timedelta(days=15)
delete_timestamp = timezone.now() - delete_delta
state = 'RESUME_COMPLETE'
stack_name = 'bogus_stack'
stack = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack_name,
suspend_timestamp=delete_timestamp,
status=state)
stack.save()
mock_heat_client = Mock()
job = ReaperJob(self.configuration)
with patch.multiple(
job, get_heat_client=Mock(return_value=mock_heat_client)):
job.run()
mock_heat_client.stacks.delete.assert_not_called()
stack = Stack.objects.get(name=stack_name)
self.assertEqual(stack.status, state)
def test_dont_suspend_deleted_stack(self):
suspend_timeout = self.configuration.get("suspend_timeout")
timedelta = timezone.timedelta(seconds=(suspend_timeout + 1))
suspend_timestamp = timezone.now() - timedelta
state = 'RESUME_COMPLETE'
stack = Stack(student_id=self.student_id,
course_id=self.course_id,
suspend_timestamp=suspend_timestamp,
name=self.stack_name,
status=state)
stack.save()
mock_heat_client = Mock()
mock_heat_client.stacks.get.side_effect = [HTTPNotFound]
job = SuspenderJob(self.configuration)
with patch.multiple(
job, get_heat_client=Mock(return_value=mock_heat_client)):
job.run()
mock_heat_client.actions.suspend.assert_not_called()
stack = Stack.objects.get(name=self.stack_name)
self.assertEqual(stack.status, DELETED_STATE)
def test_stack_log(self):
suspend_timeout = self.configuration.get("suspend_timeout")
timedelta = timezone.timedelta(seconds=(suspend_timeout + 1))
suspend_timestamp = timezone.now() - timedelta
state = 'CREATE_COMPLETE'
stack = Stack(student_id=self.student_id,
course_id=self.course_id,
suspend_timestamp=suspend_timestamp,
name=self.stack_name)
stack.status = state
stack.save()
mock_heat_client = Mock()
mock_heat_client.stacks.get.side_effect = [self.stacks[state]]
job = SuspenderJob(self.configuration)
with patch.multiple(
job, get_heat_client=Mock(return_value=mock_heat_client)):
job.run()
stacklog = StackLog.objects.filter(stack_id=stack.id)
states = [l.status for l in stacklog]
expected_states = [state, 'SUSPEND_PENDING', 'SUSPEND_ISSUED']
self.assertEqual(states, expected_states)
def test_suspend_stack_for_the_first_time(self):
# Setup
suspend_timeout = self.settings.get("suspend_timeout")
timedelta = timezone.timedelta(seconds=(suspend_timeout + 1))
suspend_timestamp = timezone.now() - timedelta
state = "CREATE_COMPLETE"
stack = Stack(student_id=self.student_id,
course_id=self.course_id,
suspend_timestamp=suspend_timestamp,
name=self.stack_name,
provider="provider1",
status=state)
stack.save()
mock_suspend_task = self.get_suspend_task_mock()
# Run
job = SuspenderJob(self.settings)
job.run()
# Assert
mock_suspend_task.apply_async.assert_called()
stack = Stack.objects.get(name=self.stack_name)
self.assertEqual(stack.status, SUSPEND_PENDING)
def test_stack_log(self):
# Setup
suspend_timeout = self.settings.get("suspend_timeout")
timedelta = timezone.timedelta(seconds=(suspend_timeout + 1))
suspend_timestamp = timezone.now() - timedelta
state = 'CREATE_COMPLETE'
stack = Stack(student_id=self.student_id,
course_id=self.course_id,
suspend_timestamp=suspend_timestamp,
provider='provider1',
name=self.stack_name)
stack.status = state
stack.save()
# Run
job = SuspenderJob(self.settings)
job.run()
# Assert
stacklog = StackLog.objects.filter(stack_id=stack.id)
states = [l.status for l in stacklog]
expected_states = [state, SUSPEND_PENDING]
self.assertEqual(states, expected_states)
def test_dont_delete_if_age_is_zero(self):
# Setup
self.settings["delete_age"] = 0
delete_delta = timezone.timedelta(days=15)
delete_timestamp = timezone.now() - delete_delta
state = 'RESUME_COMPLETE'
stack_name = 'bogus_stack'
stack = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack_name,
suspend_timestamp=delete_timestamp,
provider='provider1',
status=state)
stack.save()
mock_delete_task = self.get_delete_task_mock()
# Run
job = ReaperJob(self.settings)
job.run()
mock_delete_task.apply_async.assert_not_called()
stack = Stack.objects.get(name=stack_name)
self.assertEqual(stack.status, state)
def test_dont_try_to_delete_certain_stack_states(self):
delete_age = self.configuration.get("delete_age")
delete_delta = timezone.timedelta(days=(delete_age + 1))
delete_timestamp = timezone.now() - delete_delta
stack1_name = 'bogus_stack_1'
stack1 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack1_name,
suspend_timestamp=delete_timestamp,
status=DELETE_STATE)
stack1.save()
stack2_name = 'bogus_stack_2'
stack2 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack2_name,
suspend_timestamp=delete_timestamp,
status=DELETE_IN_PROGRESS_STATE)
stack2.save()
stack3_name = 'bogus_stack_3'
stack3 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack3_name,
suspend_timestamp=delete_timestamp,
status=DELETED_STATE)
stack3.save()
mock_heat_client = Mock()
job = ReaperJob(self.configuration)
with patch.multiple(
job, get_heat_client=Mock(return_value=mock_heat_client)):
job.run()
mock_heat_client.stacks.delete.assert_not_called()
stack1 = Stack.objects.get(name=stack1_name)
self.assertEqual(stack1.status, DELETE_STATE)
stack2 = Stack.objects.get(name=stack2_name)
self.assertEqual(stack2.status, DELETE_IN_PROGRESS_STATE)
stack3 = Stack.objects.get(name=stack3_name)
self.assertEqual(stack3.status, DELETED_STATE)
def test_delete_old_stacks(self):
delete_age = self.configuration.get("delete_age")
delete_delta = timezone.timedelta(days=(delete_age + 1))
delete_timestamp = timezone.now() - delete_delta
dont_delete_delta = timezone.timedelta(days=(delete_age - 1))
dont_delete_timestamp = timezone.now() - dont_delete_delta
state = 'RESUME_COMPLETE'
stack1_name = 'bogus_stack_1'
stack1 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack1_name,
suspend_timestamp=delete_timestamp,
status=state)
stack1.save()
stack2_name = 'bogus_stack_2'
stack2 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack2_name,
suspend_timestamp=delete_timestamp,
status=state)
stack2.save()
stack3_name = 'bogus_stack_3'
stack3 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack3_name,
suspend_timestamp=dont_delete_timestamp,
status=state)
stack3.save()
mock_heat_client = Mock()
mock_heat_client.stacks.get.side_effect = [
self.stacks[state], self.stacks[DELETE_IN_PROGRESS_STATE],
HTTPNotFound, self.stacks[state],
self.stacks[DELETE_IN_PROGRESS_STATE], HTTPNotFound
]
job = ReaperJob(self.configuration)
with patch.multiple(
job, get_heat_client=Mock(return_value=mock_heat_client)):
job.run()
mock_heat_client.stacks.delete.assert_has_calls(
[call(stack_id=stack1_name),
call(stack_id=stack2_name)])
self.assertNotIn(call(stack_id=stack3_name),
mock_heat_client.stacks.delete.mock_calls)
stack1 = Stack.objects.get(name=stack1_name)
self.assertEqual(stack1.status, DELETED_STATE)
stack2 = Stack.objects.get(name=stack2_name)
self.assertEqual(stack2.status, DELETED_STATE)
stack3 = Stack.objects.get(name=stack3_name)
self.assertEqual(stack3.status, state)
def test_suspend_concurrency(self):
self.configuration["suspend_concurrency"] = 2
suspend_timeout = self.configuration.get("suspend_timeout")
timedelta = timezone.timedelta(seconds=(suspend_timeout + 1))
suspend_timestamp = timezone.now() - timedelta
state = 'CREATE_COMPLETE'
stack1_name = 'bogus_stack_1'
stack1 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack1_name,
suspend_timestamp=suspend_timestamp,
status=state)
stack1.save()
stack2_name = 'bogus_stack_2'
stack2 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack2_name,
suspend_timestamp=suspend_timestamp,
status=state)
stack2.save()
stack3_name = 'bogus_stack_3'
stack3 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack3_name,
suspend_timestamp=suspend_timestamp,
status=state)
stack3.save()
mock_heat_client = Mock()
mock_heat_client.stacks.get.side_effect = [
self.stacks[state], self.stacks[state]
]
job = SuspenderJob(self.configuration)
with patch.multiple(
job, get_heat_client=Mock(return_value=mock_heat_client)):
job.run()
mock_heat_client.actions.suspend.assert_has_calls(
[call(stack_id=stack1_name),
call(stack_id=stack2_name)])
self.assertNotIn(call(stack_id=stack3_name),
mock_heat_client.actions.suspend.mock_calls)
stack1 = Stack.objects.get(name=stack1_name)
self.assertEqual(stack1.status, SUSPEND_ISSUED_STATE)
stack2 = Stack.objects.get(name=stack2_name)
self.assertEqual(stack2.status, SUSPEND_ISSUED_STATE)
stack3 = Stack.objects.get(name=stack3_name)
self.assertEqual(stack3.status, state)
def test_destroy_zombies(self):
# Setup
delete_age = self.settings.get("delete_age")
delete_delta = timezone.timedelta(days=(delete_age + 1))
delete_timestamp = timezone.now() - delete_delta
dont_delete_delta = timezone.timedelta(days=(delete_age - 1))
dont_delete_timestamp = timezone.now() - dont_delete_delta
stack_names = ('zombie_stack_1', 'zombie_stack_2', 'zombie_stack_3',
'zombie_stack_4', 'not_a_zombie_stack')
# Create zombie stacks
for i in range(0, 4):
_stack = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack_names[i],
suspend_timestamp=delete_timestamp,
status=DELETE_COMPLETE)
_stack.save()
# Create living stack
_stack = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack_names[4],
suspend_timestamp=dont_delete_timestamp,
status=CREATE_COMPLETE)
_stack.save()
mock_provider = self.mocks["Provider"].init.return_value
provider1_stacks = []
for i in range(0, 3):
provider1_stacks.append({
"name": stack_names[i],
"status": CREATE_COMPLETE
})
provider2_stacks = []
for i in range(3, 5):
provider2_stacks.append({
"name": stack_names[i],
"status": CREATE_COMPLETE
})
provider3_stacks = [{"name": "unknown", "status": CREATE_COMPLETE}]
mock_provider.get_stacks.side_effect = [
provider1_stacks, provider2_stacks, provider3_stacks
]
self.settings["providers"] = {
"provider1": {},
"provider2": {},
"provider3": {}
}
mock_delete_task = self.get_delete_task_mock()
# Run
job = ReaperJob(self.settings)
job.run()
# Assert
self.assertEqual(4, len(mock_delete_task.apply_async.mock_calls))
stack = Stack.objects.get(name=stack_names[0])
self.assertEqual(stack.status, DELETE_PENDING)
stack = Stack.objects.get(name=stack_names[1])
self.assertEqual(stack.status, DELETE_PENDING)
stack = Stack.objects.get(name=stack_names[2])
self.assertEqual(stack.status, DELETE_PENDING)
stack = Stack.objects.get(name=stack_names[3])
self.assertEqual(stack.status, DELETE_PENDING)
stack = Stack.objects.get(name=stack_names[4])
self.assertEqual(stack.status, CREATE_COMPLETE)
def test_dont_try_to_delete_certain_stack_states(self):
# Setup
delete_age = self.settings.get("delete_age")
delete_delta = timezone.timedelta(days=(delete_age + 1))
delete_timestamp = timezone.now() - delete_delta
stack1_name = "bogus_stack_1"
stack1 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack1_name,
suspend_timestamp=delete_timestamp,
provider="provider1",
status=DELETE_PENDING)
stack1.save()
stack2_name = "bogus_stack_2"
stack2 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack2_name,
suspend_timestamp=delete_timestamp,
provider="provider2",
status=DELETE_IN_PROGRESS)
stack2.save()
stack3_name = "bogus_stack_3"
stack3 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack3_name,
suspend_timestamp=delete_timestamp,
provider="provider3",
status=DELETE_COMPLETE)
stack3.save()
stack4_name = "bogus_stack_4"
stack4 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack4_name,
suspend_timestamp=delete_timestamp,
status="CREATE_FAILED")
stack4.save()
mock_delete_task = self.get_delete_task_mock()
# Run
job = ReaperJob(self.settings)
job.run()
# Assert
mock_delete_task.apply_async.assert_not_called()
stack1 = Stack.objects.get(name=stack1_name)
self.assertEqual(stack1.status, DELETE_PENDING)
stack2 = Stack.objects.get(name=stack2_name)
self.assertEqual(stack2.status, DELETE_IN_PROGRESS)
stack3 = Stack.objects.get(name=stack3_name)
self.assertEqual(stack3.status, DELETE_COMPLETE)
stack4 = Stack.objects.get(name=stack4_name)
self.assertEqual(stack4.status, "CREATE_FAILED")
def test_delete_old_stacks(self):
# Setup
delete_age = self.settings.get("delete_age")
delete_delta = timezone.timedelta(days=(delete_age + 1))
delete_timestamp = timezone.now() - delete_delta
dont_delete_delta = timezone.timedelta(days=(delete_age - 1))
dont_delete_timestamp = timezone.now() - dont_delete_delta
state = "RESUME_COMPLETE"
stack1_name = "bogus_stack_1"
stack1 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack1_name,
suspend_timestamp=delete_timestamp,
provider="provider1",
status=state)
stack1.save()
stack2_name = "bogus_stack_2"
stack2 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack2_name,
suspend_timestamp=delete_timestamp,
provider="provider2",
status=state)
stack2.save()
stack3_name = "bogus_stack_3"
stack3 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack3_name,
suspend_timestamp=dont_delete_timestamp,
provider='provider3',
status=state)
stack3.save()
mock_delete_task = self.get_delete_task_mock()
# Run
job = ReaperJob(self.settings)
job.run()
# Assert
self.assertEqual(2, len(mock_delete_task.apply_async.mock_calls))
stack1 = Stack.objects.get(name=stack1_name)
self.assertEqual(stack1.status, DELETE_PENDING)
stack2 = Stack.objects.get(name=stack2_name)
self.assertEqual(stack2.status, DELETE_PENDING)
stack3 = Stack.objects.get(name=stack3_name)
self.assertEqual(stack3.status, state)
def test_suspend_concurrency(self):
# Setup
self.settings["suspend_concurrency"] = 2
suspend_timeout = self.settings.get("suspend_timeout")
timedelta = timezone.timedelta(seconds=(suspend_timeout + 1))
suspend_timestamp = timezone.now() - timedelta
state = "CREATE_COMPLETE"
stack1_name = "bogus_stack_1"
stack1 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack1_name,
suspend_timestamp=suspend_timestamp,
provider="provider1",
status=state)
stack1.save()
stack2_name = "bogus_stack_2"
stack2 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack2_name,
suspend_timestamp=suspend_timestamp,
provider="provider2",
status=state)
stack2.save()
stack3_name = "bogus_stack_3"
stack3 = Stack(student_id=self.student_id,
course_id=self.course_id,
name=stack3_name,
suspend_timestamp=suspend_timestamp,
provider="provider3",
status=state)
stack3.save()
mock_suspend_task = self.get_suspend_task_mock()
# Run
job = SuspenderJob(self.settings)
job.run()
# Assert
self.assertEqual(2, len(mock_suspend_task.apply_async.mock_calls))
stack1 = Stack.objects.get(name=stack1_name)
self.assertEqual(stack1.status, SUSPEND_PENDING)
stack2 = Stack.objects.get(name=stack2_name)
self.assertEqual(stack2.status, SUSPEND_PENDING)
stack3 = Stack.objects.get(name=stack3_name)
self.assertEqual(stack3.status, state)