def test_schedule_start_works_with_pipeline_concurrency(self):
operation_run = OperationRunFactory()
operation_run.operation.trigger_policy = TriggerPolicy.ONE_DONE
operation_run.operation.save()
pipeline_run = operation_run.pipeline_run
# Set pipeline concurrency to 1
pipeline_run.pipeline.concurrency = 1
pipeline_run.pipeline.save()
# Add a failed upstream
upstream_run1 = OperationRunFactory(pipeline_run=pipeline_run)
upstream_run2 = OperationRunFactory(pipeline_run=pipeline_run)
operation_run.upstream_runs.set([upstream_run1, upstream_run2])
with patch('pipelines.tasks.start_operation_run.delay'
) as start_operation_run:
OperationRunStatus.objects.create(status=OperationStatuses.FAILED,
operation_run=upstream_run1)
OperationRunStatus.objects.create(status=OperationStatuses.RUNNING,
operation_run=upstream_run2)
assert start_operation_run.call_count == 1
with patch(
'pipelines.models.OperationRun.start') as start_operation_run:
assert operation_run.schedule_start() is True
assert start_operation_run.call_count == 0
operation_run.refresh_from_db()
assert operation_run.last_status == OperationStatuses.CREATED
def test_schedule_start_with_failed_upstream(self):
operation_run = OperationRunFactory()
operation_run.operation.trigger_policy = TriggerPolicy.ALL_SUCCEEDED
operation_run.operation.save()
# Add a failed upstream
upstream_run1 = OperationRunFactory()
operation_run.upstream_runs.set([upstream_run1])
with patch('pipelines.tasks.start_operation_run.delay'
) as start_operation_run:
OperationRunStatus.objects.create(status=OperationStatuses.FAILED,
operation_run=upstream_run1)
assert start_operation_run.call_count == 1
assert operation_run.schedule_start() is False
# Check also that the task is marked as UPSTREAM_FAILED
# Since this operation cannot be started anymore
operation_run.refresh_from_db()
assert operation_run.last_status == OperationStatuses.UPSTREAM_FAILED
def test_schedule_start_works_when_conditions_are_met(self):
operation_run = OperationRunFactory()
operation_run.operation.trigger_policy = TriggerPolicy.ONE_DONE
operation_run.operation.save()
pipeline_run = operation_run.pipeline_run
# Add a failed upstream
upstream_run1 = OperationRunFactory(pipeline_run=pipeline_run)
operation_run.upstream_runs.set([upstream_run1])
with patch('pipelines.tasks.start_operation_run.delay'
) as start_operation_run:
OperationRunStatus.objects.create(status=OperationStatuses.FAILED,
operation_run=upstream_run1)
assert start_operation_run.call_count == 1
with patch(
'pipelines.models.OperationRun.start') as start_operation_run:
assert operation_run.schedule_start() is False
assert start_operation_run.call_count == 1
operation_run.refresh_from_db()
assert operation_run.last_status == OperationStatuses.SCHEDULED
def test_schedule_start_works_with_operation_concurrency(self):
operation_run = OperationRunFactory()
operation_run.operation.trigger_policy = TriggerPolicy.ONE_DONE
operation_run.operation.save()
pipeline_run = operation_run.pipeline_run
# Set operation concurrency to 1
operation_run.operation.concurrency = 1
operation_run.operation.save()
# Add a failed upstream
upstream_run1 = OperationRunFactory(pipeline_run=pipeline_run)
upstream_run2 = OperationRunFactory(pipeline_run=pipeline_run)
operation_run.upstream_runs.set([upstream_run1, upstream_run2])
with patch('pipelines.tasks.start_operation_run.delay'
) as start_operation_run:
OperationRunStatus.objects.create(status=OperationStatuses.FAILED,
operation_run=upstream_run1)
OperationRunStatus.objects.create(status=OperationStatuses.RUNNING,
operation_run=upstream_run2)
assert start_operation_run.call_count == 1
# Add another operation run for this operation with scheduled
new_operation_run = OperationRunFactory(
operation=operation_run.operation)
new_operation_run.upstream_runs.set([upstream_run1, upstream_run2])
with patch(
'pipelines.models.OperationRun.start') as start_operation_run:
assert operation_run.schedule_start() is False
assert start_operation_run.call_count == 1
operation_run.refresh_from_db()
assert operation_run.last_status == OperationStatuses.SCHEDULED
# Check if we can start another instance
new_operation_run.refresh_from_db()
assert new_operation_run.last_status == OperationStatuses.CREATED
with patch(
'pipelines.models.OperationRun.start') as start_operation_run:
assert new_operation_run.schedule_start() is True
assert start_operation_run.call_count == 0
new_operation_run.refresh_from_db()
assert new_operation_run.last_status == OperationStatuses.CREATED
class TestOperationTask(BaseTest):
def setUp(self):
self.operation_run = OperationRunFactory()
self.pipeline_run = self.operation_run.pipeline_run
# Manually set status to scheduled
self.operation_run.on_scheduled()
return super().setUp()
def test_task_without_operation_run_raises(self):
@celery_app.task(base=OperationTask, shared=False)
def dummy_task():
return
with self.assertRaises(TypeError):
dummy_task.apply_async()
def test_task_with_operation_run_succeeds(self):
@celery_app.task(base=OperationTask, shared=False)
def dummy_task(operation_run_id):
return
kwargs = {'operation_run_id': self.operation_run.id}
dummy_task.apply_async(kwargs=kwargs)
self.operation_run.refresh_from_db()
assert self.operation_run.succeeded is True
assert set(self.operation_run.statuses.values_list(
'status', flat=True)) == {
OperationStatuses.CREATED,
OperationStatuses.SCHEDULED,
OperationStatuses.RUNNING,
OperationStatuses.SUCCEEDED,
}
self.pipeline_run.refresh_from_db()
assert self.operation_run.pipeline_run.last_status == PipelineStatuses.FINISHED
assert set(
self.operation_run.pipeline_run.statuses.values_list(
'status', flat=True)) == {
PipelineStatuses.CREATED,
PipelineStatuses.SCHEDULED,
PipelineStatuses.RUNNING,
PipelineStatuses.FINISHED,
}
def test_task_with_error_fails(self):
@celery_app.task(base=OperationTask, shared=False)
def raising_task(operation_run_id):
raise KeyError
kwargs = {'operation_run_id': self.operation_run.id}
raising_task.apply_async(kwargs=kwargs)
self.operation_run.refresh_from_db()
assert self.operation_run.failed is True
assert set(self.operation_run.statuses.values_list(
'status', flat=True)) == {
OperationStatuses.CREATED,
OperationStatuses.SCHEDULED,
OperationStatuses.RUNNING,
OperationStatuses.FAILED,
}
self.pipeline_run.refresh_from_db()
assert self.operation_run.pipeline_run.last_status == PipelineStatuses.FINISHED
assert set(
self.operation_run.pipeline_run.statuses.values_list(
'status', flat=True)) == {
PipelineStatuses.CREATED,
PipelineStatuses.SCHEDULED,
PipelineStatuses.RUNNING,
PipelineStatuses.FINISHED,
}
def test_task_retries_for_specified_exception(self):
class RetryTask(ClassBasedTask):
retry_for = (KeyError, )
@staticmethod
def _run(task_bind, *args, **kwargs):
raise KeyError
@celery_app.task(base=OperationTask, bind=True, shared=False)
def retry_task(task_bind, operation_run_id):
assert task_bind.max_retries == 2
assert task_bind.countdown == 0
RetryTask.run(task_bind=task_bind,
operation_run_id=operation_run_id)
# Add retries and count to the operation
self.operation_run.operation.max_retries = 2
self.operation_run.operation.retry_delay = 0
self.operation_run.operation.save()
kwargs = {'operation_run_id': self.operation_run.id}
retry_task.apply_async(kwargs=kwargs)
self.operation_run.refresh_from_db()
assert self.operation_run.last_status == OperationStatuses.RETRYING
assert set(self.operation_run.statuses.values_list(
'status', flat=True)) == {
OperationStatuses.CREATED,
OperationStatuses.SCHEDULED,
OperationStatuses.RUNNING,
OperationStatuses.RETRYING,
}
self.pipeline_run.refresh_from_db()
assert self.operation_run.pipeline_run.last_status == PipelineStatuses.RUNNING
assert set(
self.operation_run.pipeline_run.statuses.values_list(
'status', flat=True)) == {
PipelineStatuses.CREATED,
PipelineStatuses.SCHEDULED,
PipelineStatuses.RUNNING,
}