def assert_machine_status(self, from_status: str, accepted_status: List[str]): for status in accepted_status: routine = factories.RoutineWithoutSignalFactory(status=from_status) routine.status = status routine.save() accepted_status.append(from_status) for status in self._status_list(ignore_items=accepted_status): msg_error = f"Status update from '{from_status}' to '{status}' is not allowed" with self.assertRaises(ValidationError, msg=msg_error): routine = factories.RoutineWithoutSignalFactory( status=from_status) routine.status = status routine.save()
def tests_dont_process_completed_routine(self): routine = factories.RoutineWithoutSignalFactory( status="completed", task_name="SayHelloTask", ) with self.assertLogs(level="INFO") as context: PipelineRoutineTask().delay(routine_id=routine.pk) self.assert_routine_lock(routine_id=routine.pk) self.assertEqual(context.output, [f"INFO:root:Routine #{routine.pk} is already completed"])
def tests_revert_completed_routine(self): routine = factories.RoutineWithoutSignalFactory(status="completed", output="{'id': 42}") with patch("django_cloud_tasks.tasks.PipelineRoutineRevertTask.delay" ) as revert_task: routine.revert() routine.refresh_from_db() self.assertEqual("reverting", routine.status) revert_task.assert_called_once_with(routine_id=routine.pk)
def tests_complete(self): routine = factories.RoutineWithoutSignalFactory(status="running", output=None, ends_at=None) output = {"id": 42} routine.complete(output=output) routine.refresh_from_db() self.assertEqual("completed", routine.status) self.assertEqual(output, routine.output) self.assertEqual(datetime.now(), routine.ends_at.replace(tzinfo=None))
def tests_fail(self): routine = factories.RoutineWithoutSignalFactory(status="running", output=None, ends_at=None) error = {"error": "something went wrong"} routine.fail(output=error) routine.refresh_from_db() self.assertEqual("failed", routine.status) self.assertEqual(error, routine.output) self.assertEqual(datetime.now(), routine.ends_at.replace(tzinfo=None))
def test_process_revert_and_update_routine_to_reverted(self): routine = factories.RoutineWithoutSignalFactory( status="reverting", task_name="SayHelloTask", output={"spell": "Obliviate"}, ) with patch("sample_project.sample_app.tasks.SayHelloTask.revert") as revert: PipelineRoutineRevertTask().delay(routine_id=routine.pk) revert.assert_called_once_with(data=routine.output) routine.refresh_from_db() self.assertEqual(routine.status, "reverted")
def tests_revert_pipeline(self): pipeline = factories.PipelineFactory() leaf_already_reverted = factories.RoutineWithoutSignalFactory( status="reverted") pipeline.routines.add(leaf_already_reverted) with patch("django_cloud_tasks.tasks.PipelineRoutineRevertTask.delay" ) as task: pipeline.revert() task.assert_not_called() second_routine = factories.RoutineFactory() pipeline.routines.add(second_routine) third_routine = factories.RoutineWithoutSignalFactory( status="completed") pipeline.routines.add(third_routine) first_routine = factories.RoutineFactory() pipeline.routines.add(first_routine) fourth_routine = factories.RoutineWithoutSignalFactory( status="completed") pipeline.routines.add(fourth_routine) factories.RoutineVertexFactory(routine=second_routine, next_routine=third_routine) factories.RoutineVertexFactory(routine=first_routine, next_routine=second_routine) with patch("django_cloud_tasks.tasks.PipelineRoutineRevertTask.delay" ) as task: pipeline.revert() calls = [ call(routine_id=fourth_routine.pk), call(routine_id=third_routine.pk), ] task.assert_has_calls(calls, any_order=True)
def tests_enqueue_previously_routines_after_reverted(self): pipeline = factories.PipelineFactory() first_routine = factories.RoutineWithoutSignalFactory( status="completed") pipeline.routines.add(first_routine) second_routine = factories.RoutineFactory() pipeline.routines.add(second_routine) third_routine = factories.RoutineWithoutSignalFactory( status="reverting") pipeline.routines.add(third_routine) factories.RoutineVertexFactory(routine=first_routine, next_routine=second_routine) factories.RoutineVertexFactory(routine=first_routine, next_routine=third_routine) with patch("django_cloud_tasks.tasks.PipelineRoutineRevertTask.delay" ) as task: third_routine.status = "reverted" third_routine.save() task.assert_called_once_with(routine_id=first_routine.pk)
def tests_start_pipeline_revert_flow_if_exceeded_retries(self): routine = factories.RoutineWithoutSignalFactory( status="running", task_name="SayHelloTask", max_retries=1, attempt_count=2, ) with patch("django_cloud_tasks.models.Pipeline.revert") as revert: with self.assertLogs(level="INFO") as context: PipelineRoutineTask().delay(routine_id=routine.pk) self.assertEqual( context.output, [ f"INFO:root:Routine #{routine.id} has exhausted retries and is being reverted", ], ) self.assert_routine_lock(routine_id=routine.pk) revert.assert_called_once()
def tests_store_task_output_into_routine(self): routine = factories.RoutineWithoutSignalFactory( status="running", task_name="SayHelloTask", body={"attributes": [1, 2, 3]}, attempt_count=1, ) with self.assertLogs(level="INFO") as context: PipelineRoutineTask().run(routine_id=routine.pk) self.assert_routine_lock(routine_id=routine.pk) routine.refresh_from_db() self.assertEqual( context.output, [ f"INFO:root:Routine #{routine.id} is running", f"INFO:root:Routine #{routine.id} just completed", ], ) self.assertEqual("completed", routine.status) self.assertEqual(2, routine.attempt_count)
def tests_dont_enqueue_next_routines_after_completed_when_status_dont_change( self): pipeline = factories.PipelineFactory() first_routine = factories.RoutineWithoutSignalFactory( status="completed") pipeline.routines.add(first_routine) second_routine = factories.RoutineFactory() pipeline.routines.add(second_routine) third_routine = factories.RoutineFactory() pipeline.routines.add(third_routine) factories.RoutineVertexFactory(routine=first_routine, next_routine=second_routine) factories.RoutineVertexFactory(routine=first_routine, next_routine=third_routine) with patch( "django_cloud_tasks.tasks.PipelineRoutineTask.delay") as task: first_routine.status = "completed" first_routine.save() task.assert_not_called()
def tests_enqueue_next_routines_after_completed(self): pipeline = factories.PipelineFactory() first_routine = factories.RoutineWithoutSignalFactory(status="running") pipeline.routines.add(first_routine) second_routine = factories.RoutineFactory() pipeline.routines.add(second_routine) third_routine = factories.RoutineFactory() pipeline.routines.add(third_routine) factories.RoutineVertexFactory(routine=first_routine, next_routine=second_routine) factories.RoutineVertexFactory(routine=first_routine, next_routine=third_routine) with patch( "django_cloud_tasks.tasks.PipelineRoutineTask.delay") as task: first_routine.status = "completed" first_routine.save() calls = [ call(routine_id=second_routine.pk), call(routine_id=third_routine.pk) ] task.assert_has_calls(calls, any_order=True)
def tests_fail_routine_if_task_has_failed(self): routine = factories.RoutineWithoutSignalFactory( status="running", task_name="SayHelloTask", body={"attributes": [1, 2, 3]}, attempt_count=1, ) with self.assertLogs(level="INFO") as context: with patch("sample_project.sample_app.tasks.SayHelloTask.run", side_effect=Exception("any error")): with patch("django_cloud_tasks.models.Routine.enqueue") as enqueue: PipelineRoutineTask().run(routine_id=routine.pk) self.assert_routine_lock(routine_id=routine.pk) routine.refresh_from_db() self.assertEqual( context.output, [ f"INFO:root:Routine #{routine.id} is running", f"INFO:root:Routine #{routine.id} has failed", f"INFO:root:Routine #{routine.id} has been enqueued for retry", ], ) self.assertEqual("failed", routine.status) enqueue.assert_called_once() self.assertEqual(2, routine.attempt_count)