def test_all_conditions_met(self): """ Test all conditions met should pass dep """ dag = Mock(concurrency=1, get_concurrency_reached=Mock(return_value=False)) task = Mock(dag=dag, pool_slots=1) ti = TaskInstance(task, execution_date=None) assert DagTISlotsAvailableDep().is_met(ti=ti)
def test_all_conditions_met(self): """ Test all conditions met should pass dep """ dag = Mock(concurrency=1, concurrency_reached=False) task = Mock(dag=dag) ti = TaskInstance(task, execution_date=None) self.assertTrue(DagTISlotsAvailableDep().is_met(ti=ti))
def test_concurrency_reached(self): """ Test concurrency reached should fail dep """ dag = Mock(concurrency=1, get_concurrency_reached=Mock(return_value=True)) task = Mock(dag=dag, pool_slots=1) ti = TaskInstance(task, execution_date=None) assert not DagTISlotsAvailableDep().is_met(ti=ti)
def test_concurrency_reached(self): """ Test concurrency reached should fail dep """ dag = Mock(concurrency=1, concurrency_reached=True) task = Mock(dag=dag) ti = TaskInstance(task, execution_date=None) self.assertFalse(DagTISlotsAvailableDep().is_met(ti=ti))
def test_all_conditions_met(self): """ Test all conditions met should pass dep """ dag = FakeDag(concurrency=1, concurrency_reached=False) task = FakeTask(dag=dag) ti = FakeTI(task=task, dag_id="fake_dag") self.assertTrue(DagTISlotsAvailableDep().is_met(ti=ti, dep_context=None))
def test_concurrency_reached(self): """ Test concurrency reached should fail dep """ dag = FakeDag(concurrency=1, concurrency_reached=True) task = FakeTask(dag=dag) ti = FakeTI(task=task, dag_id="fake_dag") self.assertFalse(DagTISlotsAvailableDep().is_met(ti=ti, dep_context=None))
State.UP_FOR_RESCHEDULE, } # Context to get the dependencies that need to be met in order for a task instance to # be backfilled. QUEUE_DEPS = { NotRunningDep(), NotSkippedDep(), RunnableExecDateDep(), ValidStateDep(QUEUEABLE_STATES), } # Dependencies that need to be met for a given task instance to be able to get run by an # executor. This class just extends QueueContext by adding dependencies for resources. RUN_DEPS = QUEUE_DEPS | { DagTISlotsAvailableDep(), TaskConcurrencyDep(), } # TODO(aoen): SCHEDULER_DEPS is not coupled to actual execution in any way and # could easily be modified or removed from the scheduler causing this dependency to become # outdated and incorrect. This coupling should be created (e.g. via a dag_deps analog of # ti_deps that will be used in the scheduler code) to ensure that the logic here is # equivalent to the logic in the scheduler. # Dependencies that need to be met for a given task instance to get scheduled by the # scheduler, then queued by the scheduler, then run by an executor. SCHEDULER_DEPS = RUN_DEPS | { DagrunRunningDep(), DagUnpausedDep(), ExecDateAfterStartDateDep(),
State.UP_FOR_RETRY, } # Context to get the dependencies that need to be met in order for a task instance to # be backfilled. QUEUE_DEPS = { NotRunningDep(), # 任务实例没有运行 NotSkippedDep(), # 任务实例没有被标记为跳过 RunnableExecDateDep(), # 判断任务执行时间 必须小于等于当前时间 且 小于等于结束时间 ValidStateDep(QUEUEABLE_STATES), # 验证任务的状态必须在队列状态中 } # Dependencies that need to be met for a given task instance to be able to get run by an # executor. This class just extends QueueContext by adding dependencies for resources. RUN_DEPS = QUEUE_DEPS | { DagTISlotsAvailableDep(), # 每个dag能并发执行的最大任务数依赖 TaskConcurrencyDep(), # 每个任务的任务实例有最大限制 } # TODO(aoen): SCHEDULER_DEPS is not coupled to actual execution in any way and # could easily be modified or removed from the scheduler causing this dependency to become # outdated and incorrect. This coupling should be created (e.g. via a dag_deps analog of # ti_deps that will be used in the scheduler code) to ensure that the logic here is # equivalent to the logic in the scheduler. # Dependencies that need to be met for a given task instance to get scheduled by the # scheduler, then queued by the scheduler, then run by an executor. SCHEDULER_DEPS = RUN_DEPS | { DagrunRunningDep(), # 验证Dagrun必须是RUNNING的状态 DagUnpausedDep(), # DAG不能是暂停状态 ExecDateAfterStartDateDep(), # 任务的执行时间必须大于等于任务的开始时间 ,大于等于dag的开始时间
State.UPSTREAM_FAILED, State.UP_FOR_RETRY, } # Context to get the dependencies that need to be met in order for a task instance to # be backfilled. QUEUE_DEPS = { NotRunningDep(), NotSkippedDep(), RunnableExecDateDep(), ValidStateDep(QUEUEABLE_STATES), } # Dependencies that need to be met for a given task instance to be able to get run by an # executor. This class just extends QueueContext by adding dependencies for resources. RUN_DEPS = QUEUE_DEPS | {DagTISlotsAvailableDep()} # TODO(aoen): SCHEDULER_DEPS is not coupled to actual execution in any way and # could easily be modified or removed from the scheduler causing this dependency to become # outdated and incorrect. This coupling should be created (e.g. via a dag_deps analog of # ti_deps that will be used in the scheduler code) to ensure that the logic here is # equivalent to the logic in the scheduler. # Dependencies that need to be met for a given task instance to get scheduled by the # scheduler, then queued by the scheduler, then run by an executor. SCHEDULER_DEPS = RUN_DEPS | { DagrunRunningDep(), DagUnpausedDep(), ExecDateAfterStartDateDep(), }