Python TaskConcurrencyDep Examples

Example #1

File: test_task_concurrency.py Project: bomboradata/bombora-incubator-airflow

 def test_reached_concurrency(self):
     task = self._get_task(start_date=datetime(2016, 1, 1), task_concurrency=2)
     dep_context = DepContext()
     ti = Mock(task=task, execution_date=datetime(2016, 1, 1))
     ti.get_num_running_task_instances = lambda x: 1
     self.assertTrue(TaskConcurrencyDep().is_met(ti=ti, dep_context=dep_context))
     ti.get_num_running_task_instances = lambda x: 2
     self.assertFalse(TaskConcurrencyDep().is_met(ti=ti, dep_context=dep_context))

Example #2

File: test_task_concurrency.py Project: PreethamMadupuri91/AirflowWorkflow

 def test_not_task_concurrency(self):
     task = self._get_task(start_date=datetime(2016, 1, 1))
     dep_context = DepContext()
     ti = Mock(task=task, execution_date=datetime(2016, 1, 1))
     self.assertTrue(TaskConcurrencyDep().is_met(ti=ti,
                                                 dep_context=dep_context))

Example #3

File: dep_context.py Project: FlyrInc/apache-airflow

}

# Context to get the dependencies that need to be met in order for a task instance to
# be backfilled.
QUEUE_DEPS = {
    NotRunningDep(),
    NotSkippedDep(),
    RunnableExecDateDep(),
    ValidStateDep(QUEUEABLE_STATES),
}

# Dependencies that need to be met for a given task instance to be able to get run by an
# executor. This class just extends QueueContext by adding dependencies for resources.
RUN_DEPS = QUEUE_DEPS | {
    DagTISlotsAvailableDep(),
    TaskConcurrencyDep(),
}

# TODO(aoen): SCHEDULER_DEPS is not coupled to actual execution in any way and
# could easily be modified or removed from the scheduler causing this dependency to become
# outdated and incorrect. This coupling should be created (e.g. via a dag_deps analog of
# ti_deps that will be used in the scheduler code) to ensure that the logic here is
# equivalent to the logic in the scheduler.

# Dependencies that need to be met for a given task instance to get scheduled by the
# scheduler, then queued by the scheduler, then run by an executor.
SCHEDULER_DEPS = RUN_DEPS | {
    DagrunRunningDep(),
    DagUnpausedDep(),
    ExecDateAfterStartDateDep(),
}

Example #4

}

# Context to get the dependencies that need to be met in order for a task instance to
# be backfilled.
QUEUE_DEPS = {
    NotRunningDep(),  # 任务实例没有运行
    NotSkippedDep(),  # 任务实例没有被标记为跳过
    RunnableExecDateDep(),  # 判断任务执行时间 必须小于等于当前时间  且 小于等于结束时间
    ValidStateDep(QUEUEABLE_STATES),  # 验证任务的状态必须在队列状态中
}

# Dependencies that need to be met for a given task instance to be able to get run by an
# executor. This class just extends QueueContext by adding dependencies for resources.
RUN_DEPS = QUEUE_DEPS | {
    DagTISlotsAvailableDep(),  # 每个dag能并发执行的最大任务数依赖
    TaskConcurrencyDep(),  # 每个任务的任务实例有最大限制
}

# TODO(aoen): SCHEDULER_DEPS is not coupled to actual execution in any way and
# could easily be modified or removed from the scheduler causing this dependency to become
# outdated and incorrect. This coupling should be created (e.g. via a dag_deps analog of
# ti_deps that will be used in the scheduler code) to ensure that the logic here is
# equivalent to the logic in the scheduler.

# Dependencies that need to be met for a given task instance to get scheduled by the
# scheduler, then queued by the scheduler, then run by an executor.
SCHEDULER_DEPS = RUN_DEPS | {
    DagrunRunningDep(),  # 验证Dagrun必须是RUNNING的状态
    DagUnpausedDep(),  # DAG不能是暂停状态
    ExecDateAfterStartDateDep(),  # 任务的执行时间必须大于等于任务的开始时间 ，大于等于dag的开始时间
}