def test_all_met(self):
"""
Test to make sure all of the conditions for the dep are met
"""
task = FakeTask(depends_on_past=True,
start_date=datetime(2016, 1, 1),
wait_for_downstream=True)
prev_ti = FakeTI(state=State.SUCCESS, dependents_done=True)
ti = FakeTI(task=task,
previous_ti=prev_ti,
execution_date=datetime(2016, 1, 2))
dep_context = FakeContext(ignore_depends_on_past=False)
self.assertTrue(PrevDagrunDep().is_met(ti=ti, dep_context=dep_context))
def test_prev_ti_bad_state(self):
"""
If the previous TI did not complete execution this dep should fail.
"""
task = FakeTask(depends_on_past=True,
start_date=datetime(2016, 1, 1),
wait_for_downstream=False)
prev_ti = FakeTI(state=State.NONE, dependents_done=True)
ti = FakeTI(task=task,
previous_ti=prev_ti,
execution_date=datetime(2016, 1, 2))
dep_context = FakeContext(ignore_depends_on_past=False)
self.assertFalse(PrevDagrunDep().is_met(ti=ti,
dep_context=dep_context))
def test_pool_full(self):
"""
Full pools should fail this dep
"""
ti = FakeTI(pool="fake_pool", pool_filled=True)
self.assertFalse(PoolHasSpaceDep().is_met(ti=ti, dep_context=None))
def test_not_skipped(self):
"""
Non-skipped task instances should pass this dep
"""
ti = FakeTI(state=State.RUNNING)
self.assertTrue(NotSkippedDep().is_met(ti=ti, dep_context=None))
def test_skipped(self):
"""
Skipped task instances should fail this dep
"""
ti = FakeTI(state=State.SKIPPED)
self.assertFalse(NotSkippedDep().is_met(ti=ti, dep_context=None))
def test_ti_running(self):
"""
Running task instances should fail this dep
"""
ti = FakeTI(state=State.RUNNING, start_date=datetime(2016, 1, 1))
self.assertFalse(NotRunningDep().is_met(ti=ti, dep_context=None))
def test_ti_not_running(self):
"""
Non-running task instances should pass this dep
"""
ti = FakeTI(state=State.NONE, start_date=datetime(2016, 1, 1))
self.assertTrue(NotRunningDep().is_met(ti=ti, dep_context=None))
def test_one_failure_tr_success(self):
"""
One-failure trigger rule success
"""
task = FakeTask(trigger_rule=TriggerRule.ONE_FAILED,
upstream_task_ids=[])
ti = FakeTI(task=task)
dep_statuses = tuple(TriggerRuleDep()._evaluate_trigger_rule(
ti=ti,
successes=0,
skipped=2,
failed=2,
upstream_failed=0,
done=2,
flag_upstream_failed=False,
session="Fake Session"))
self.assertEqual(len(dep_statuses), 0)
dep_statuses = tuple(TriggerRuleDep()._evaluate_trigger_rule(
ti=ti,
successes=0,
skipped=2,
failed=0,
upstream_failed=2,
done=2,
flag_upstream_failed=False,
session="Fake Session"))
self.assertEqual(len(dep_statuses), 0)
def test_not_skipped(self):
"""
Pools with room should pass this dep
"""
ti = FakeTI(pool="fake_pool", pool_filled=False)
self.assertTrue(PoolHasSpaceDep().is_met(ti=ti, dep_context=None))
def test_valid_state(self):
"""
Valid state should pass this dep
"""
ti = FakeTI(state=State.QUEUED, end_date=datetime(2016, 1, 1))
self.assertTrue(
ValidStateDep({State.QUEUED}).is_met(ti=ti, dep_context=None))
def test_failed_wait_for_downstream(self):
"""
If the previous TI specified to wait for the downstream tasks of the previous
dagrun then it should fail this dep if the downstream TIs of the previous TI are
not done.
"""
task = FakeTask(depends_on_past=True,
start_date=datetime(2016, 1, 1),
wait_for_downstream=True)
prev_ti = FakeTI(state=State.SUCCESS, dependents_done=False)
ti = FakeTI(task=task,
previous_ti=prev_ti,
execution_date=datetime(2016, 1, 2))
dep_context = FakeContext(ignore_depends_on_past=False)
self.assertFalse(PrevDagrunDep().is_met(ti=ti,
dep_context=dep_context))
def test_dummy_tr(self):
"""
The dummy trigger rule should always pass this dep
"""
task = FakeTask(trigger_rule=TriggerRule.DUMMY, upstream_list=[])
ti = FakeTI(task=task, state=State.UP_FOR_RETRY)
self.assertTrue(TriggerRuleDep().is_met(ti=ti, dep_context=None))
def test_invalid_state(self):
"""
Invalid state should fail this dep
"""
ti = FakeTI(state=State.SUCCESS, end_date=datetime(2016, 1, 1))
self.assertFalse(
ValidStateDep({State.FAILURE}).is_met(ti=ti, dep_context=None))
def test_no_valid_states(self):
"""
If there are no valid states the dependency should throw
"""
ti = FakeTI(state=State.SUCCESS, end_date=datetime(2016, 1, 1))
with self.assertRaises(AirflowException):
ValidStateDep({}).is_met(ti=ti, dep_context=None)
def test_no_upstream_tasks(self):
"""
If the TI has no upstream TIs then there is nothing to check and the dep is passed
"""
task = FakeTask(trigger_rule=TriggerRule.ALL_DONE, upstream_list=[])
ti = FakeTI(task=task, state=State.UP_FOR_RETRY)
self.assertTrue(TriggerRuleDep().is_met(ti=ti, dep_context=None))
def test_dagrun_exists(self):
"""
Task instances with a dagrun should pass this dep
"""
dag = FakeDag(running_dagruns=[], max_active_runs=1)
task = FakeTask(dag=dag)
ti = FakeTI(dagrun="Fake Dagrun", task=task, dag_id="fake_dag")
self.assertTrue(DagrunRunningDep().is_met(ti=ti, dep_context=None))
def test_all_conditions_met(self):
"""
Test all conditions met should pass dep
"""
dag = FakeDag(is_paused=False)
task = FakeTask(dag=dag)
ti = FakeTI(task=task, dag_id="fake_dag")
self.assertTrue(DagUnpausedDep().is_met(ti=ti, dep_context=None))
def test_concurrency_reached(self):
"""
Test paused DAG should fail dependency
"""
dag = FakeDag(is_paused=True)
task = FakeTask(dag=dag)
ti = FakeTI(task=task, dag_id="fake_dag")
self.assertFalse(DagUnpausedDep().is_met(ti=ti, dep_context=None))
def test_dagrun_doesnt_exist(self):
"""
Task instances without dagruns should fail this dep
"""
dag = FakeDag(running_dagruns=[], max_active_runs=1)
task = FakeTask(dag=dag)
ti = FakeTI(dagrun=None, task=task, dag_id="fake_dag")
self.assertFalse(DagrunRunningDep().is_met(ti=ti, dep_context=None))
def test_not_in_retry_period(self):
"""
Task instance's that are not up for retry can not be in their retry period
"""
dag = FakeDag()
task = FakeTask(dag=dag)
ti = FakeTI(task=task, state=State.SUCCESS)
self.assertTrue(NotInRetryPeriodDep().is_met(ti=ti, dep_context=None))
def test_context_ignore_depends_on_past(self):
"""
If the context overrides depends_on_past then the dep should be met, even though
there is no previous_ti which would normally fail the dep
"""
task = FakeTask(depends_on_past=True,
start_date=datetime(2016, 1, 1),
wait_for_downstream=False)
prev_ti = FakeTI(task=task,
execution_date=datetime(2016, 1, 2),
state=State.SUCCESS,
dependents_done=True)
ti = FakeTI(task=task,
previous_ti=prev_ti,
execution_date=datetime(2016, 1, 3))
dep_context = FakeContext(ignore_depends_on_past=True)
self.assertTrue(PrevDagrunDep().is_met(ti=ti, dep_context=dep_context))
def test_concurrency_reached(self):
"""
Test concurrency reached should fail dep
"""
dag = FakeDag(concurrency=1, concurrency_reached=True)
task = FakeTask(dag=dag)
ti = FakeTI(task=task, dag_id="fake_dag")
self.assertFalse(DagTISlotsAvailableDep().is_met(ti=ti,
dep_context=None))
def test_all_conditions_met(self):
"""
Test all conditions met should pass dep
"""
dag = FakeDag(concurrency=1, concurrency_reached=False)
task = FakeTask(dag=dag)
ti = FakeTI(task=task, dag_id="fake_dag")
self.assertTrue(DagTISlotsAvailableDep().is_met(ti=ti,
dep_context=None))
def test_still_in_retry_period(self):
"""
Task instances that are in their retry period should fail this dep
"""
dag = FakeDag()
task = FakeTask(dag=dag, retry_delay=timedelta(minutes=1))
ti = FakeTI(
task=task,
state=State.UP_FOR_RETRY,
end_date=datetime(2016, 1, 1),
is_premature=True)
self.assertFalse(NotInRetryPeriodDep().is_met(ti=ti, dep_context=None))
def test_retry_period_finished(self):
"""
Task instance's that have had their retry period elapse should pass this dep
"""
dag = FakeDag()
task = FakeTask(dag=dag, retry_delay=timedelta(minutes=1))
ti = FakeTI(
task=task,
state=State.UP_FOR_RETRY,
end_date=datetime(2016, 1, 1),
is_premature=False)
self.assertTrue(NotInRetryPeriodDep().is_met(ti=ti, dep_context=None))
def test_first_task_run(self):
"""
The first task run for a TI should pass since it has no previous dagrun.
"""
task = FakeTask(depends_on_past=True,
start_date=datetime(2016, 1, 1),
wait_for_downstream=False)
prev_ti = None
ti = FakeTI(task=task,
previous_ti=prev_ti,
execution_date=datetime(2016, 1, 1))
dep_context = FakeContext(ignore_depends_on_past=False)
self.assertTrue(PrevDagrunDep().is_met(ti=ti, dep_context=dep_context))
def test_all_done_tr_success(self):
"""
All-done trigger rule success
"""
task = FakeTask(trigger_rule=TriggerRule.ALL_DONE,
upstream_task_ids=["FakeTaskID", "OtherFakeTaskID"])
ti = FakeTI(task=task)
dep_statuses = tuple(TriggerRuleDep()._evaluate_trigger_rule(
ti=ti,
successes=2,
skipped=0,
failed=0,
upstream_failed=0,
done=2,
flag_upstream_failed=False,
session="Fake Session"))
self.assertEqual(len(dep_statuses), 0)
def test_one_success_tr_failure(self):
"""
One-success trigger rule failure
"""
task = FakeTask(trigger_rule=TriggerRule.ONE_SUCCESS,
upstream_task_ids=[])
ti = FakeTI(task=task)
dep_statuses = tuple(TriggerRuleDep()._evaluate_trigger_rule(
ti=ti,
successes=0,
skipped=2,
failed=2,
upstream_failed=2,
done=2,
flag_upstream_failed=False,
session="Fake Session"))
self.assertEqual(len(dep_statuses), 1)
self.assertFalse(dep_statuses[0].passed)
def test_unknown_tr(self):
"""
Unknown trigger rules should cause this dep to fail
"""
task = FakeTask(trigger_rule="Unknown Trigger Rule",
upstream_task_ids=[])
ti = FakeTI(task=task)
dep_statuses = tuple(TriggerRuleDep()._evaluate_trigger_rule(
ti=ti,
successes=1,
skipped=0,
failed=0,
upstream_failed=0,
done=1,
flag_upstream_failed=False,
session="Fake Session"))
self.assertEqual(len(dep_statuses), 1)
self.assertFalse(dep_statuses[0].passed)
def test_all_failed_tr_failure(self):
"""
All-failed trigger rule failure
"""
task = FakeTask(trigger_rule=TriggerRule.ALL_FAILED,
upstream_task_ids=["FakeTaskID", "OtherFakeTaskID"])
ti = FakeTI(task=task)
dep_statuses = tuple(TriggerRuleDep()._evaluate_trigger_rule(
ti=ti,
successes=2,
skipped=0,
failed=0,
upstream_failed=0,
done=2,
flag_upstream_failed=False,
session="Fake Session"))
self.assertEqual(len(dep_statuses), 1)
self.assertFalse(dep_statuses[0].passed)