def test_concurrency_reached(self):
"""
Test paused DAG should fail dependency
"""
dag = Mock(is_paused=True)
task = Mock(dag=dag)
ti = TaskInstance(task=task, execution_date=None)
self.assertFalse(DagUnpausedDep().is_met(ti=ti))
def test_all_conditions_met(self):
"""
Test all conditions met should pass dep
"""
dag = Mock(is_paused=False)
task = Mock(dag=dag)
ti = TaskInstance(task=task, execution_date=None)
self.assertTrue(DagUnpausedDep().is_met(ti=ti))
def test_all_conditions_met(self):
"""
Test all conditions met should pass dep
"""
dag = Mock(**{'get_is_paused.return_value': False})
task = Mock(dag=dag)
ti = TaskInstance(task=task, execution_date=None)
assert DagUnpausedDep().is_met(ti=ti)
def test_concurrency_reached(self):
"""
Test paused DAG should fail dependency
"""
dag = Mock(**{'get_is_paused.return_value': True})
task = Mock(dag=dag)
ti = TaskInstance(task=task, execution_date=None)
assert not DagUnpausedDep().is_met(ti=ti)
def test_all_conditions_met(self):
"""
Test all conditions met should pass dep
"""
dag = FakeDag(is_paused=False)
task = FakeTask(dag=dag)
ti = FakeTI(task=task, dag_id="fake_dag")
self.assertTrue(DagUnpausedDep().is_met(ti=ti, dep_context=None))
def test_concurrency_reached(self):
"""
Test paused DAG should fail dependency
"""
dag = FakeDag(is_paused=True)
task = FakeTask(dag=dag)
ti = FakeTI(task=task, dag_id="fake_dag")
self.assertFalse(DagUnpausedDep().is_met(ti=ti, dep_context=None))
# Context to get the dependencies that need to be met in order for a task instance to
# be backfilled.
QUEUE_DEPS = {
NotRunningDep(),
NotSkippedDep(),
RunnableExecDateDep(),
ValidStateDep(QUEUEABLE_STATES),
}
# Dependencies that need to be met for a given task instance to be able to get run by an
# executor. This class just extends QueueContext by adding dependencies for resources.
RUN_DEPS = QUEUE_DEPS | {
DagTISlotsAvailableDep(),
TaskConcurrencyDep(),
}
# TODO(aoen): SCHEDULER_DEPS is not coupled to actual execution in any way and
# could easily be modified or removed from the scheduler causing this dependency to become
# outdated and incorrect. This coupling should be created (e.g. via a dag_deps analog of
# ti_deps that will be used in the scheduler code) to ensure that the logic here is
# equivalent to the logic in the scheduler.
# Dependencies that need to be met for a given task instance to get scheduled by the
# scheduler, then queued by the scheduler, then run by an executor.
SCHEDULER_DEPS = RUN_DEPS | {
DagrunRunningDep(),
DagUnpausedDep(),
ExecDateAfterStartDateDep(),
}
# Context to get the dependencies that need to be met in order for a task instance to
# be backfilled.
QUEUE_DEPS = {
NotRunningDep(), # 任务实例没有运行
NotSkippedDep(), # 任务实例没有被标记为跳过
RunnableExecDateDep(), # 判断任务执行时间 必须小于等于当前时间 且 小于等于结束时间
ValidStateDep(QUEUEABLE_STATES), # 验证任务的状态必须在队列状态中
}
# Dependencies that need to be met for a given task instance to be able to get run by an
# executor. This class just extends QueueContext by adding dependencies for resources.
RUN_DEPS = QUEUE_DEPS | {
DagTISlotsAvailableDep(), # 每个dag能并发执行的最大任务数依赖
TaskConcurrencyDep(), # 每个任务的任务实例有最大限制
}
# TODO(aoen): SCHEDULER_DEPS is not coupled to actual execution in any way and
# could easily be modified or removed from the scheduler causing this dependency to become
# outdated and incorrect. This coupling should be created (e.g. via a dag_deps analog of
# ti_deps that will be used in the scheduler code) to ensure that the logic here is
# equivalent to the logic in the scheduler.
# Dependencies that need to be met for a given task instance to get scheduled by the
# scheduler, then queued by the scheduler, then run by an executor.
SCHEDULER_DEPS = RUN_DEPS | {
DagrunRunningDep(), # 验证Dagrun必须是RUNNING的状态
DagUnpausedDep(), # DAG不能是暂停状态
ExecDateAfterStartDateDep(), # 任务的执行时间必须大于等于任务的开始时间 ，大于等于dag的开始时间
}
