def test_dag_property(self):
pipeline = PipelineFactory()
operations = [OperationFactory(pipeline=pipeline) for _ in range(4)]
operations[0].upstream_operations.set(operations[2:])
operations[1].upstream_operations.set(operations[2:])
operation_by_ids = {op.id: op for op in operations}
assert pipeline.dag == ({
operations[0].id: set(),
operations[1].id: set(),
operations[2].id: {operations[0].id, operations[1].id},
operations[3].id: {operations[0].id, operations[1].id},
}, operation_by_ids)
# Add operations outside the dag
operation1 = OperationFactory()
operation1.downstream_operations.set(
[operations[1], operations[2], operations[3]])
operation2 = OperationFactory()
operation2.upstream_operations.set([operations[0], operations[2]])
assert pipeline.dag == ({
operations[0].id: {
operation2.id,
},
operations[1].id: set(),
operations[2].id:
{operations[0].id, operations[1].id, operation2.id},
operations[3].id: {operations[0].id, operations[1].id},
}, operation_by_ids)
def test_check_concurrency(self):
# Operation without concurrency defaults to infinite concurrency
operation = OperationFactory()
operation_run = OperationRunFactory(operation=operation)
assert operation_run.check_concurrency() is True
# Operation with concurrency and operation run with operation runs
operation.concurrency = 2
operation.save()
# No running operation runs
assert operation_run.check_concurrency() is True
# One operation run
operation_run1 = OperationRunFactory(operation=operation)
assert operation_run.check_concurrency() is True
# One operation run with RUNNING status
operation_run1.status = OperationStatuses.RUNNING
operation_run1.save()
assert operation_run.check_concurrency() is True
# Second operation run
operation_run2 = OperationRunFactory(operation=operation)
assert operation_run.check_concurrency() is True
# Second operation run with RUNNING status
operation_run2.status = OperationStatuses.RUNNING
operation_run2.save()
assert operation_run.check_concurrency() is False
def test_get_dag(self):
operations = [OperationFactory() for _ in range(4)]
operations[0].upstream_operations.set(operations[2:])
operations[1].upstream_operations.set(operations[2:])
operation_by_ids = {op.id: op for op in operations}
def get_downstream(op):
return op.downstream_operations.values_list('id', flat=True)
assert dags.get_dag(nodes=operations, downstream_fn=get_downstream) == (
{
operations[0].id: set(),
operations[1].id: set(),
operations[2].id: {operations[0].id, operations[1].id},
operations[3].id: {operations[0].id, operations[1].id},
},
operation_by_ids
)
# Add operations outside the dag
operation1 = OperationFactory()
operation1.downstream_operations.set([operations[1], operations[2], operations[3]])
operation2 = OperationFactory()
operation2.upstream_operations.set([operations[0], operations[2]])
assert dags.get_dag(nodes=operations, downstream_fn=get_downstream) == (
{
operations[0].id: {operation2.id, },
operations[1].id: set(),
operations[2].id: {operations[0].id, operations[1].id, operation2.id},
operations[3].id: {operations[0].id, operations[1].id},
},
operation_by_ids
)
def test_check_concurrency(self):
# Operation without concurrency defaults to infinite concurrency
operation = OperationFactory()
operation_run = OperationRunFactory(operation=operation)
assert operation_run.check_concurrency() is True
# Operation with concurrency and operation run with operation runs
operation.concurrency = 2
operation.save()
# No running operation runs
assert operation_run.check_concurrency() is True
# One operation run
operation_run1 = OperationRunFactory(operation=operation)
assert operation_run.check_concurrency() is True
# One operation run with RUNNING status
OperationRunStatus.objects.create(status=OperationStatuses.RUNNING,
operation_run=operation_run1)
assert operation_run.check_concurrency() is True
# Second operation run
operation_run2 = OperationRunFactory(operation=operation)
assert operation_run.check_concurrency() is True
# Second operation run with RUNNING status
OperationRunStatus.objects.create(status=OperationStatuses.RUNNING,
operation_run=operation_run2)
assert operation_run.check_concurrency() is False
def test_get_countdown(self):
operation = OperationFactory(retry_delay=5)
assert operation.get_countdown(1) == 5
assert operation.get_countdown(2) == 5
# Test exponential backoff
operation.retry_exponential_backoff = True
operation.max_retry_delay = 24
operation.save()
assert operation.get_countdown(1) == 5
assert operation.get_countdown(2) == 5
assert operation.get_countdown(3) == 8
assert operation.get_countdown(4) == 16
assert operation.get_countdown(5) == 24 # Max retry delay
def test_get_countdown(self):
operation = OperationFactory(retry_delay=5)
assert operation.get_countdown(1) == 5
assert operation.get_countdown(2) == 5
# Test exponential backoff
operation.retry_exponential_backoff = True
operation.max_retry_delay = 24
operation.save()
assert operation.get_countdown(1) == 5
assert operation.get_countdown(2) == 5
assert operation.get_countdown(3) == 8
assert operation.get_countdown(4) == 16
assert operation.get_countdown(5) == 24 # Max retry delay
def setUp(self):
super().setUp()
self.operation = OperationFactory()
self.tested_events = {
operation_events.OPERATION_CREATED,
operation_events.OPERATION_UPDATED,
operation_events.OPERATION_DELETED,
operation_events.OPERATION_CLEANED_TRIGGERED,
operation_events.OPERATION_VIEWED,
operation_events.OPERATION_ARCHIVED,
operation_events.OPERATION_RESTORED,
operation_events.OPERATION_DELETED_TRIGGERED,
}
def test_get_run_params(self):
operation = OperationFactory()
assert operation.get_run_params() == {}
operation.queue = 'dummy_queue'
operation.save()
assert operation.get_run_params() == {'queue': 'dummy_queue'}
operation.timeout = 10
operation.save()
assert operation.get_run_params() == {
'queue': 'dummy_queue',
'soft_time_limit': 10,
'time_limit': settings.CELERY_HARD_TIME_LIMIT_DELAY + 10,
}
operation.execute_at = timezone.now() + timedelta(hours=1)
operation.save()
assert operation.get_run_params() == {
'queue': 'dummy_queue',
'soft_time_limit': 10,
'time_limit': settings.CELERY_HARD_TIME_LIMIT_DELAY + 10,
'eta': operation.execute_at
}
def test_get_run_params(self):
operation = OperationFactory()
assert operation.get_run_params() == {}
operation.celery_queue = 'dummy_queue'
operation.save()
assert operation.get_run_params() == {'queue': 'dummy_queue'}
operation.timeout = 10
operation.save()
assert operation.get_run_params() == {
'queue': 'dummy_queue',
'soft_time_limit': 10,
'time_limit': settings.CELERY_HARD_TIME_LIMIT_DELAY + 10,
}
operation.execute_at = timezone.now() + timedelta(hours=1)
operation.save()
assert operation.get_run_params() == {
'queue': 'dummy_queue',
'soft_time_limit': 10,
'time_limit': settings.CELERY_HARD_TIME_LIMIT_DELAY + 10,
'eta': operation.execute_at
}
