def test_check_concurrency(self):
# Operation without concurrency defaults to infinite concurrency
operation = OperationFactory()
operation_run = OperationRunFactory(operation=operation)
assert operation_run.check_concurrency() is True
# Operation with concurrency and operation run with operation runs
operation.concurrency = 2
operation.save()
# No running operation runs
assert operation_run.check_concurrency() is True
# One operation run
operation_run1 = OperationRunFactory(operation=operation)
assert operation_run.check_concurrency() is True
# One operation run with RUNNING status
operation_run1.status = OperationStatuses.RUNNING
operation_run1.save()
assert operation_run.check_concurrency() is True
# Second operation run
operation_run2 = OperationRunFactory(operation=operation)
assert operation_run.check_concurrency() is True
# Second operation run with RUNNING status
operation_run2.status = OperationStatuses.RUNNING
operation_run2.save()
assert operation_run.check_concurrency() is False
def test_check_concurrency(self):
# Pipeline without concurrency defaults to infinite concurrency
pipeline = PipelineFactory()
pipeline_run = PipelineRunFactory(pipeline=pipeline)
assert pipeline_run.check_concurrency() is True
# Pipeline with concurrency and pipeline run with operation runs
pipeline.concurrency = 2
pipeline.save()
# No running operation runs
assert pipeline_run.check_concurrency() is True
# One operation run
operation_run1 = OperationRunFactory(pipeline_run=pipeline_run)
assert pipeline_run.check_concurrency() is True
# One operation run with RUNNING status
operation_run1.status = OperationStatuses.RUNNING
operation_run1.save()
assert pipeline_run.check_concurrency() is True
# Second operation run
operation_run2 = OperationRunFactory(pipeline_run=pipeline_run)
assert pipeline_run.check_concurrency() is True
# Second operation run with RUNNING status
operation_run2.status = OperationStatuses.RUNNING
operation_run2.save()
assert pipeline_run.check_concurrency() is False
def test_schedule_start_with_operation_run_already_scheduled_operation_run(
self):
operation_run = OperationRunFactory()
operation_run.status = OperationStatuses.FAILED
operation_run.save()
assert start_operation_run(operation_run) is False
operation_run = OperationRunFactory()
operation_run.status = OperationStatuses.SCHEDULED
operation_run.save()
assert start_operation_run(operation_run) is False
def test_trigger_policy_all_failed(self):
operation_run = OperationRunFactory()
operation = operation_run.operation
operation.trigger_policy = TriggerPolicy.ALL_FAILED
operation.save()
# No upstream
assert operation_run.check_upstream_trigger() is True
# Add non done upstream
upstream_run1 = OperationRunFactory()
operation_run.upstream_runs.set([upstream_run1])
assert operation_run.check_upstream_trigger() is False
# A running upstream
upstream_run1.status = OperationStatuses.RUNNING
upstream_run1.save()
assert operation_run.check_upstream_trigger() is False
# A failed upstream
upstream_run1.status = OperationStatuses.FAILED
upstream_run1.save()
assert operation_run.check_upstream_trigger() is True
# Add skipped upstream
upstream_run2 = OperationRunFactory()
operation_run.upstream_runs.set([upstream_run2])
upstream_run2.status = OperationStatuses.SKIPPED
upstream_run2.save()
assert operation_run.check_upstream_trigger() is False
# Add succeeded upstream
upstream_run3 = OperationRunFactory()
operation_run.upstream_runs.set([upstream_run3])
upstream_run3.status = OperationStatuses.SUCCEEDED
upstream_run3.save()
assert operation_run.check_upstream_trigger() is False
# Add many failed upstream
upstream_run4 = OperationRunFactory()
operation_run.upstream_runs.set([upstream_run1, upstream_run4])
upstream_run4.status = OperationStatuses.FAILED
upstream_run4.save()
assert operation_run.check_upstream_trigger() is True
# Many done upstreams
operation_run.upstream_runs.set(
[upstream_run1, upstream_run2, upstream_run3])
assert operation_run.check_upstream_trigger() is False
def test_is_upstream_done(self):
operation_run = OperationRunFactory()
# No upstream
assert operation_run.is_upstream_done is True
# Add non done upstream
upstream_run1 = OperationRunFactory()
operation_run.upstream_runs.set([upstream_run1])
assert operation_run.is_upstream_done is False
# A running upstream
upstream_run1.status = OperationStatuses.RUNNING
upstream_run1.save()
assert operation_run.is_upstream_done is False
# A failed upstream
upstream_run1.status = OperationStatuses.FAILED
upstream_run1.save()
assert operation_run.is_upstream_done is True
# Add skipped upstream
upstream_run2 = OperationRunFactory()
operation_run.upstream_runs.set([upstream_run2])
upstream_run2.status = OperationStatuses.SKIPPED
upstream_run2.save()
assert operation_run.is_upstream_done is True
# Add succeeded upstream
upstream_run3 = OperationRunFactory()
operation_run.upstream_runs.set([upstream_run3])
upstream_run3.status = OperationStatuses.SUCCEEDED
upstream_run3.save()
assert operation_run.is_upstream_done is True
# Many done upstreams
operation_run.upstream_runs.set(
[upstream_run1, upstream_run2, upstream_run3])
assert operation_run.is_upstream_done is True
# Add another upstream
upstream_run4 = OperationRunFactory()
operation_run.upstream_runs.add(upstream_run4)
assert operation_run.is_upstream_done is False
def test_trigger_policy_one_done(self):
operation_run = OperationRunFactory()
start_operation_run(operation_run)
operation_run.refresh_from_db()
operation = operation_run.operation
operation.trigger_policy = TriggerPolicy.ONE_DONE
operation.save()
# No upstream
assert operation_run.check_upstream_trigger() is False
# Add non done upstream
upstream_run1 = OperationRunFactory()
operation_run.upstream_runs.set([upstream_run1])
assert operation_run.check_upstream_trigger() is False
# A running upstream
upstream_run1.status = OperationStatuses.RUNNING
upstream_run1.save()
assert operation_run.check_upstream_trigger() is False
# A failed upstream
upstream_run1.status = OperationStatuses.FAILED
upstream_run1.save()
assert operation_run.check_upstream_trigger() is True
# Add skipped upstream
upstream_run2 = OperationRunFactory()
operation_run.upstream_runs.set([upstream_run2])
upstream_run2.status = OperationStatuses.SKIPPED
upstream_run2.save()
assert operation_run.check_upstream_trigger() is True
# Add succeeded upstream
upstream_run3 = OperationRunFactory()
operation_run.upstream_runs.set([upstream_run3])
upstream_run3.status = OperationStatuses.SUCCEEDED
upstream_run3.save()
assert operation_run.check_upstream_trigger() is True
# Add another upstream still True
upstream_run4 = OperationRunFactory()
operation_run.upstream_runs.add(upstream_run4)
assert operation_run.check_upstream_trigger() is True
