def test_dag_property(self): pipeline = PipelineFactory() operations = [OperationFactory(pipeline=pipeline) for _ in range(4)] operations[0].upstream_operations.set(operations[2:]) operations[1].upstream_operations.set(operations[2:]) operation_by_ids = {op.id: op for op in operations} assert pipeline.dag == ({ operations[0].id: set(), operations[1].id: set(), operations[2].id: {operations[0].id, operations[1].id}, operations[3].id: {operations[0].id, operations[1].id}, }, operation_by_ids) # Add operations outside the dag operation1 = OperationFactory() operation1.downstream_operations.set( [operations[1], operations[2], operations[3]]) operation2 = OperationFactory() operation2.upstream_operations.set([operations[0], operations[2]]) assert pipeline.dag == ({ operations[0].id: { operation2.id, }, operations[1].id: set(), operations[2].id: {operations[0].id, operations[1].id, operation2.id}, operations[3].id: {operations[0].id, operations[1].id}, }, operation_by_ids)
def test_check_concurrency(self): # Operation without concurrency defaults to infinite concurrency operation = OperationFactory() operation_run = OperationRunFactory(operation=operation) assert operation_run.check_concurrency() is True # Operation with concurrency and operation run with operation runs operation.concurrency = 2 operation.save() # No running operation runs assert operation_run.check_concurrency() is True # One operation run operation_run1 = OperationRunFactory(operation=operation) assert operation_run.check_concurrency() is True # One operation run with RUNNING status operation_run1.status = OperationStatuses.RUNNING operation_run1.save() assert operation_run.check_concurrency() is True # Second operation run operation_run2 = OperationRunFactory(operation=operation) assert operation_run.check_concurrency() is True # Second operation run with RUNNING status operation_run2.status = OperationStatuses.RUNNING operation_run2.save() assert operation_run.check_concurrency() is False
def test_get_dag(self): operations = [OperationFactory() for _ in range(4)] operations[0].upstream_operations.set(operations[2:]) operations[1].upstream_operations.set(operations[2:]) operation_by_ids = {op.id: op for op in operations} def get_downstream(op): return op.downstream_operations.values_list('id', flat=True) assert dags.get_dag(nodes=operations, downstream_fn=get_downstream) == ( { operations[0].id: set(), operations[1].id: set(), operations[2].id: {operations[0].id, operations[1].id}, operations[3].id: {operations[0].id, operations[1].id}, }, operation_by_ids ) # Add operations outside the dag operation1 = OperationFactory() operation1.downstream_operations.set([operations[1], operations[2], operations[3]]) operation2 = OperationFactory() operation2.upstream_operations.set([operations[0], operations[2]]) assert dags.get_dag(nodes=operations, downstream_fn=get_downstream) == ( { operations[0].id: {operation2.id, }, operations[1].id: set(), operations[2].id: {operations[0].id, operations[1].id, operation2.id}, operations[3].id: {operations[0].id, operations[1].id}, }, operation_by_ids )
def test_check_concurrency(self): # Operation without concurrency defaults to infinite concurrency operation = OperationFactory() operation_run = OperationRunFactory(operation=operation) assert operation_run.check_concurrency() is True # Operation with concurrency and operation run with operation runs operation.concurrency = 2 operation.save() # No running operation runs assert operation_run.check_concurrency() is True # One operation run operation_run1 = OperationRunFactory(operation=operation) assert operation_run.check_concurrency() is True # One operation run with RUNNING status OperationRunStatus.objects.create(status=OperationStatuses.RUNNING, operation_run=operation_run1) assert operation_run.check_concurrency() is True # Second operation run operation_run2 = OperationRunFactory(operation=operation) assert operation_run.check_concurrency() is True # Second operation run with RUNNING status OperationRunStatus.objects.create(status=OperationStatuses.RUNNING, operation_run=operation_run2) assert operation_run.check_concurrency() is False
def test_get_countdown(self): operation = OperationFactory(retry_delay=5) assert operation.get_countdown(1) == 5 assert operation.get_countdown(2) == 5 # Test exponential backoff operation.retry_exponential_backoff = True operation.max_retry_delay = 24 operation.save() assert operation.get_countdown(1) == 5 assert operation.get_countdown(2) == 5 assert operation.get_countdown(3) == 8 assert operation.get_countdown(4) == 16 assert operation.get_countdown(5) == 24 # Max retry delay
def setUp(self): super().setUp() self.operation = OperationFactory() self.tested_events = { operation_events.OPERATION_CREATED, operation_events.OPERATION_UPDATED, operation_events.OPERATION_DELETED, operation_events.OPERATION_CLEANED_TRIGGERED, operation_events.OPERATION_VIEWED, operation_events.OPERATION_ARCHIVED, operation_events.OPERATION_RESTORED, operation_events.OPERATION_DELETED_TRIGGERED, }
def test_get_run_params(self): operation = OperationFactory() assert operation.get_run_params() == {} operation.queue = 'dummy_queue' operation.save() assert operation.get_run_params() == {'queue': 'dummy_queue'} operation.timeout = 10 operation.save() assert operation.get_run_params() == { 'queue': 'dummy_queue', 'soft_time_limit': 10, 'time_limit': settings.CELERY_HARD_TIME_LIMIT_DELAY + 10, } operation.execute_at = timezone.now() + timedelta(hours=1) operation.save() assert operation.get_run_params() == { 'queue': 'dummy_queue', 'soft_time_limit': 10, 'time_limit': settings.CELERY_HARD_TIME_LIMIT_DELAY + 10, 'eta': operation.execute_at }
def test_get_run_params(self): operation = OperationFactory() assert operation.get_run_params() == {} operation.celery_queue = 'dummy_queue' operation.save() assert operation.get_run_params() == {'queue': 'dummy_queue'} operation.timeout = 10 operation.save() assert operation.get_run_params() == { 'queue': 'dummy_queue', 'soft_time_limit': 10, 'time_limit': settings.CELERY_HARD_TIME_LIMIT_DELAY + 10, } operation.execute_at = timezone.now() + timedelta(hours=1) operation.save() assert operation.get_run_params() == { 'queue': 'dummy_queue', 'soft_time_limit': 10, 'time_limit': settings.CELERY_HARD_TIME_LIMIT_DELAY + 10, 'eta': operation.execute_at }