def assert_machine_status(self, from_status: str,
accepted_status: List[str]):
for status in accepted_status:
routine = factories.RoutineWithoutSignalFactory(status=from_status)
routine.status = status
routine.save()
accepted_status.append(from_status)
for status in self._status_list(ignore_items=accepted_status):
msg_error = f"Status update from '{from_status}' to '{status}' is not allowed"
with self.assertRaises(ValidationError, msg=msg_error):
routine = factories.RoutineWithoutSignalFactory(
status=from_status)
routine.status = status
routine.save()
def tests_dont_process_completed_routine(self):
routine = factories.RoutineWithoutSignalFactory(
status="completed",
task_name="SayHelloTask",
)
with self.assertLogs(level="INFO") as context:
PipelineRoutineTask().delay(routine_id=routine.pk)
self.assert_routine_lock(routine_id=routine.pk)
self.assertEqual(context.output, [f"INFO:root:Routine #{routine.pk} is already completed"])
def tests_revert_completed_routine(self):
routine = factories.RoutineWithoutSignalFactory(status="completed",
output="{'id': 42}")
with patch("django_cloud_tasks.tasks.PipelineRoutineRevertTask.delay"
) as revert_task:
routine.revert()
routine.refresh_from_db()
self.assertEqual("reverting", routine.status)
revert_task.assert_called_once_with(routine_id=routine.pk)
def tests_complete(self):
routine = factories.RoutineWithoutSignalFactory(status="running",
output=None,
ends_at=None)
output = {"id": 42}
routine.complete(output=output)
routine.refresh_from_db()
self.assertEqual("completed", routine.status)
self.assertEqual(output, routine.output)
self.assertEqual(datetime.now(), routine.ends_at.replace(tzinfo=None))
def tests_fail(self):
routine = factories.RoutineWithoutSignalFactory(status="running",
output=None,
ends_at=None)
error = {"error": "something went wrong"}
routine.fail(output=error)
routine.refresh_from_db()
self.assertEqual("failed", routine.status)
self.assertEqual(error, routine.output)
self.assertEqual(datetime.now(), routine.ends_at.replace(tzinfo=None))
def test_process_revert_and_update_routine_to_reverted(self):
routine = factories.RoutineWithoutSignalFactory(
status="reverting",
task_name="SayHelloTask",
output={"spell": "Obliviate"},
)
with patch("sample_project.sample_app.tasks.SayHelloTask.revert") as revert:
PipelineRoutineRevertTask().delay(routine_id=routine.pk)
revert.assert_called_once_with(data=routine.output)
routine.refresh_from_db()
self.assertEqual(routine.status, "reverted")
def tests_revert_pipeline(self):
pipeline = factories.PipelineFactory()
leaf_already_reverted = factories.RoutineWithoutSignalFactory(
status="reverted")
pipeline.routines.add(leaf_already_reverted)
with patch("django_cloud_tasks.tasks.PipelineRoutineRevertTask.delay"
) as task:
pipeline.revert()
task.assert_not_called()
second_routine = factories.RoutineFactory()
pipeline.routines.add(second_routine)
third_routine = factories.RoutineWithoutSignalFactory(
status="completed")
pipeline.routines.add(third_routine)
first_routine = factories.RoutineFactory()
pipeline.routines.add(first_routine)
fourth_routine = factories.RoutineWithoutSignalFactory(
status="completed")
pipeline.routines.add(fourth_routine)
factories.RoutineVertexFactory(routine=second_routine,
next_routine=third_routine)
factories.RoutineVertexFactory(routine=first_routine,
next_routine=second_routine)
with patch("django_cloud_tasks.tasks.PipelineRoutineRevertTask.delay"
) as task:
pipeline.revert()
calls = [
call(routine_id=fourth_routine.pk),
call(routine_id=third_routine.pk),
]
task.assert_has_calls(calls, any_order=True)
def tests_enqueue_previously_routines_after_reverted(self):
pipeline = factories.PipelineFactory()
first_routine = factories.RoutineWithoutSignalFactory(
status="completed")
pipeline.routines.add(first_routine)
second_routine = factories.RoutineFactory()
pipeline.routines.add(second_routine)
third_routine = factories.RoutineWithoutSignalFactory(
status="reverting")
pipeline.routines.add(third_routine)
factories.RoutineVertexFactory(routine=first_routine,
next_routine=second_routine)
factories.RoutineVertexFactory(routine=first_routine,
next_routine=third_routine)
with patch("django_cloud_tasks.tasks.PipelineRoutineRevertTask.delay"
) as task:
third_routine.status = "reverted"
third_routine.save()
task.assert_called_once_with(routine_id=first_routine.pk)
def tests_start_pipeline_revert_flow_if_exceeded_retries(self):
routine = factories.RoutineWithoutSignalFactory(
status="running",
task_name="SayHelloTask",
max_retries=1,
attempt_count=2,
)
with patch("django_cloud_tasks.models.Pipeline.revert") as revert:
with self.assertLogs(level="INFO") as context:
PipelineRoutineTask().delay(routine_id=routine.pk)
self.assertEqual(
context.output,
[
f"INFO:root:Routine #{routine.id} has exhausted retries and is being reverted",
],
)
self.assert_routine_lock(routine_id=routine.pk)
revert.assert_called_once()
def tests_store_task_output_into_routine(self):
routine = factories.RoutineWithoutSignalFactory(
status="running",
task_name="SayHelloTask",
body={"attributes": [1, 2, 3]},
attempt_count=1,
)
with self.assertLogs(level="INFO") as context:
PipelineRoutineTask().run(routine_id=routine.pk)
self.assert_routine_lock(routine_id=routine.pk)
routine.refresh_from_db()
self.assertEqual(
context.output,
[
f"INFO:root:Routine #{routine.id} is running",
f"INFO:root:Routine #{routine.id} just completed",
],
)
self.assertEqual("completed", routine.status)
self.assertEqual(2, routine.attempt_count)
def tests_dont_enqueue_next_routines_after_completed_when_status_dont_change(
self):
pipeline = factories.PipelineFactory()
first_routine = factories.RoutineWithoutSignalFactory(
status="completed")
pipeline.routines.add(first_routine)
second_routine = factories.RoutineFactory()
pipeline.routines.add(second_routine)
third_routine = factories.RoutineFactory()
pipeline.routines.add(third_routine)
factories.RoutineVertexFactory(routine=first_routine,
next_routine=second_routine)
factories.RoutineVertexFactory(routine=first_routine,
next_routine=third_routine)
with patch(
"django_cloud_tasks.tasks.PipelineRoutineTask.delay") as task:
first_routine.status = "completed"
first_routine.save()
task.assert_not_called()
def tests_enqueue_next_routines_after_completed(self):
pipeline = factories.PipelineFactory()
first_routine = factories.RoutineWithoutSignalFactory(status="running")
pipeline.routines.add(first_routine)
second_routine = factories.RoutineFactory()
pipeline.routines.add(second_routine)
third_routine = factories.RoutineFactory()
pipeline.routines.add(third_routine)
factories.RoutineVertexFactory(routine=first_routine,
next_routine=second_routine)
factories.RoutineVertexFactory(routine=first_routine,
next_routine=third_routine)
with patch(
"django_cloud_tasks.tasks.PipelineRoutineTask.delay") as task:
first_routine.status = "completed"
first_routine.save()
calls = [
call(routine_id=second_routine.pk),
call(routine_id=third_routine.pk)
]
task.assert_has_calls(calls, any_order=True)
def tests_fail_routine_if_task_has_failed(self):
routine = factories.RoutineWithoutSignalFactory(
status="running",
task_name="SayHelloTask",
body={"attributes": [1, 2, 3]},
attempt_count=1,
)
with self.assertLogs(level="INFO") as context:
with patch("sample_project.sample_app.tasks.SayHelloTask.run", side_effect=Exception("any error")):
with patch("django_cloud_tasks.models.Routine.enqueue") as enqueue:
PipelineRoutineTask().run(routine_id=routine.pk)
self.assert_routine_lock(routine_id=routine.pk)
routine.refresh_from_db()
self.assertEqual(
context.output,
[
f"INFO:root:Routine #{routine.id} is running",
f"INFO:root:Routine #{routine.id} has failed",
f"INFO:root:Routine #{routine.id} has been enqueued for retry",
],
)
self.assertEqual("failed", routine.status)
enqueue.assert_called_once()
self.assertEqual(2, routine.attempt_count)
