def _process_finished_ti(session, ti):
"""
Process the TaskInstance object which already finished.
:param session:
:param ti:
:return:
"""
from airflow.ti_deps.deps.runnable_exec_date_dep import RunnableExecDateDep
from airflow.ti_deps.deps.valid_state_dep import ValidStateDep
from airflow.ti_deps.deps.events_dep import EventTIDep
EVENT_SCHEDULED_DEPS = {
RunnableExecDateDep(),
ValidStateDep(FINISHED_STATES),
EventTIDep(),
}
dep_context = DepContext(deps=EVENT_SCHEDULED_DEPS)
if ti.are_dependencies_met(dep_context=dep_context, session=session):
ts = TaskState.query_task_state(ti, session=session)
if ts.action is None or TaskAction(ts.action) == TaskAction.NONE:
return
if TaskAction(ts.action) == TaskAction.RESTART:
log.debug('Queuing Finished task: %s', ti)
ti.state = State.SCHEDULED
log.info("Creating / updating %s in ORM", ti)
session.merge(ti)
ts.action = None
session.merge(ts)
session.commit()
def test_no_valid_states(self):
"""
If there are no valid states the dependency should throw
"""
ti = Mock(state=State.SUCCESS, end_date=datetime(2016, 1, 1))
with self.assertRaises(AirflowException):
ValidStateDep({}).is_met(ti=ti)
def test_invalid_state(self):
"""
Invalid state should fail this dep
"""
ti = FakeTI(state=State.SUCCESS, end_date=datetime(2016, 1, 1))
self.assertFalse(
ValidStateDep({State.FAILURE}).is_met(ti=ti, dep_context=None))
def test_valid_state(self):
"""
Valid state should pass this dep
"""
ti = FakeTI(state=State.QUEUED, end_date=datetime(2016, 1, 1))
self.assertTrue(
ValidStateDep({State.QUEUED}).is_met(ti=ti, dep_context=None))
def _process_running_ti(session, ti):
"""
Process the TaskInstance object which are running.
:param session:
:param ti:
:return:
"""
from airflow.ti_deps.deps.runnable_exec_date_dep import RunnableExecDateDep
from airflow.ti_deps.deps.valid_state_dep import ValidStateDep
from airflow.ti_deps.deps.events_dep import EventTIDep
EVENT_SCHEDULED_DEPS = {
RunnableExecDateDep(),
ValidStateDep(RUNNING_STATES),
EventTIDep(),
}
dep_context = DepContext(deps=EVENT_SCHEDULED_DEPS)
if ti.are_dependencies_met(dep_context=dep_context, session=session):
if action_is_stop_or_restart(ti, session):
log.info("stop or restart task %s ", ti)
State.NONE,
State.QUEUED,
State.SCHEDULED,
State.SKIPPED,
State.UPSTREAM_FAILED,
State.UP_FOR_RETRY,
State.UP_FOR_RESCHEDULE,
}
# Context to get the dependencies that need to be met in order for a task instance to
# be backfilled.
QUEUE_DEPS = {
NotRunningDep(),
NotSkippedDep(),
RunnableExecDateDep(),
ValidStateDep(QUEUEABLE_STATES),
}
# Dependencies that need to be met for a given task instance to be able to get run by an
# executor. This class just extends QueueContext by adding dependencies for resources.
RUN_DEPS = QUEUE_DEPS | {
DagTISlotsAvailableDep(),
TaskConcurrencyDep(),
}
# TODO(aoen): SCHEDULER_DEPS is not coupled to actual execution in any way and
# could easily be modified or removed from the scheduler causing this dependency to become
# outdated and incorrect. This coupling should be created (e.g. via a dag_deps analog of
# ti_deps that will be used in the scheduler code) to ensure that the logic here is
# equivalent to the logic in the scheduler.
def test_invalid_state(self):
"""
Invalid state should fail this dep
"""
ti = Mock(state=State.SUCCESS, end_date=datetime(2016, 1, 1))
self.assertFalse(ValidStateDep({State.FAILED}).is_met(ti=ti))
def test_valid_state(self):
"""
Valid state should pass this dep
"""
ti = Mock(state=State.QUEUED, end_date=datetime(2016, 1, 1))
self.assertTrue(ValidStateDep({State.QUEUED}).is_met(ti=ti))
def _process_dag_task_instances(self, ti_status, executor, pickle_id, session=None):
"""
Process a set of task instances from a set of dag runs. Special handling is done
to account for different task instance states that could be present when running
them in a backfill process.
:param ti_status: the internal status of the job
:type ti_status: DagRunJob._DagRunTaskStatus
:param executor: the executor to run the task instances
:type executor: BaseExecutor
:param pickle_id: the pickle_id if dag is pickled, None otherwise
:type pickle_id: int
:param start_date: the start date of the backfill job
:type start_date: datetime
:param session: the current session object
:type session: Session
:return: the list of execution_dates for the finished dag runs
:rtype: list
"""
executed_run_dates = []
# values() returns a view so we copy to maintain a full list of the TIs to run
all_ti = list(ti_status.to_run.values())
waiting_for_executor_result = {}
while (len(ti_status.to_run) > 0 or len(ti_status.running) > 0) and len(
ti_status.deadlocked
) == 0:
if current.is_killed():
raise friendly_error.task_execution.databand_context_killed(
"SingleDagRunJob scheduling main loop"
)
self.log.debug("*** Clearing out not_ready list ***")
ti_status.not_ready.clear()
self.ti_state_manager.refresh_task_instances_state(
all_ti, self.dag.dag_id, self.execution_date, session=session
)
# we need to execute the tasks bottom to top
# or leaf to root, as otherwise tasks might be
# determined deadlocked while they are actually
# waiting for their upstream to finish
for task in self.dag.topological_sort():
# TODO: too complicated mechanism,
# it's not possible that we have multiple tasks with the same id in to run
for key, ti in list(ti_status.to_run.items()):
if task.task_id != ti.task_id:
continue
if not self._optimize:
ti.refresh_from_db()
task = self.dag.get_task(ti.task_id)
ti.task = task
# TODO : do we need that?
# ignore_depends_on_past = (
# self.ignore_first_depends_on_past and
# ti.execution_date == (start_date or ti.start_date))
ignore_depends_on_past = False
self.log.debug("Task instance to run %s state %s", ti, ti.state)
# guard against externally modified tasks instances or
# in case max concurrency has been reached at task runtime
if ti.state == State.NONE:
self.log.warning(
"FIXME: task instance {} state was set to None "
"externally. This should not happen"
)
ti.set_state(State.SCHEDULED, session=session)
# The task was already marked successful or skipped by a
# different Job. Don't rerun it.
if ti.state == State.SUCCESS:
ti_status.succeeded.add(key)
self.log.debug("Task instance %s succeeded. Don't rerun.", ti)
ti_status.to_run.pop(key)
if key in ti_status.running:
ti_status.running.pop(key)
continue
elif ti.state == State.SKIPPED:
ti_status.skipped.add(key)
self.log.debug("Task instance %s skipped. Don't rerun.", ti)
ti_status.to_run.pop(key)
if key in ti_status.running:
ti_status.running.pop(key)
continue
elif ti.state == State.FAILED:
self.log.error("Task instance %s failed", ti)
ti_status.failed.add(key)
ti_status.to_run.pop(key)
if key in ti_status.running:
ti_status.running.pop(key)
continue
elif ti.state == State.UPSTREAM_FAILED:
self.log.error("Task instance %s upstream failed", ti)
ti_status.failed.add(key)
ti_status.to_run.pop(key)
if key in ti_status.running:
ti_status.running.pop(key)
continue
runtime_deps = []
if self.airflow_config.disable_dag_concurrency_rules:
# RUN Deps validate dag and task concurrency
# It's less relevant when we run in stand along mode with SingleDagRunJob
# from airflow.ti_deps.deps.runnable_exec_date_dep import RunnableExecDateDep
from airflow.ti_deps.deps.valid_state_dep import ValidStateDep
# from airflow.ti_deps.deps.dag_ti_slots_available_dep import DagTISlotsAvailableDep
# from airflow.ti_deps.deps.task_concurrency_dep import TaskConcurrencyDep
# from airflow.ti_deps.deps.pool_slots_available_dep import PoolSlotsAvailableDep
runtime_deps = {
# RunnableExecDateDep(),
ValidStateDep(SCHEDUALED_OR_RUNNABLE),
# DagTISlotsAvailableDep(),
# TaskConcurrencyDep(),
# PoolSlotsAvailableDep(),
}
else:
runtime_deps = RUNNING_DEPS
dagrun_dep_context = DepContext(
deps=runtime_deps,
ignore_depends_on_past=ignore_depends_on_past,
ignore_task_deps=self.ignore_task_deps,
flag_upstream_failed=True,
)
# Is the task runnable? -- then run it
# the dependency checker can change states of tis
if ti.are_dependencies_met(
dep_context=dagrun_dep_context,
session=session,
verbose=self.verbose,
):
ti.refresh_from_db(lock_for_update=True, session=session)
if (
ti.state == State.SCHEDULED
or ti.state == State.UP_FOR_RETRY
):
if executor.has_task(ti):
self.log.debug(
"Task Instance %s already in executor "
"waiting for queue to clear",
ti,
)
else:
self.log.debug("Sending %s to executor", ti)
# if ti.state == State.UP_FOR_RETRY:
# ti._try_number += 1
# Skip scheduled state, we are executing immediately
ti.state = State.QUEUED
session.merge(ti)
cfg_path = None
if executor.__class__ in (
executors.LocalExecutor,
executors.SequentialExecutor,
):
cfg_path = tmp_configuration_copy()
executor.queue_task_instance(
ti,
mark_success=self.mark_success,
pickle_id=pickle_id,
ignore_task_deps=self.ignore_task_deps,
ignore_depends_on_past=ignore_depends_on_past,
pool=self.pool,
cfg_path=cfg_path,
)
ti_status.to_run.pop(key)
ti_status.running[key] = ti
waiting_for_executor_result[key] = ti
session.commit()
continue
if ti.state == State.UPSTREAM_FAILED:
self.log.error("Task instance %s upstream failed", ti)
ti_status.failed.add(key)
ti_status.to_run.pop(key)
if key in ti_status.running:
ti_status.running.pop(key)
continue
# special case
if ti.state == State.UP_FOR_RETRY:
self.log.debug(
"Task instance %s retry period not " "expired yet", ti
)
if key in ti_status.running:
ti_status.running.pop(key)
ti_status.to_run[key] = ti
continue
# all remaining tasks
self.log.debug("Adding %s to not_ready", ti)
ti_status.not_ready.add(key)
# execute the tasks in the queue
self.heartbeat()
executor.heartbeat()
# If the set of tasks that aren't ready ever equals the set of
# tasks to run and there are no running tasks then the backfill
# is deadlocked
if (
ti_status.not_ready
and ti_status.not_ready == set(ti_status.to_run)
and len(ti_status.running) == 0
):
self.log.warning(
"scheduler: Deadlock discovered for ti_status.to_run=%s",
ti_status.to_run.values(),
)
ti_status.deadlocked.update(ti_status.to_run.values())
ti_status.to_run.clear()
self.ti_state_manager.refresh_task_instances_state(
all_ti, self.dag.dag_id, self.execution_date, session=session
)
# check executor state
self._manage_executor_state(ti_status.running, waiting_for_executor_result)
if self._zombie_cleaner:
# this code exists in airflow original scheduler
# clean zombies ( we don't need multiple runs here actually
self._zombie_cleaner.find_and_clean_dag_zombies(
dag=self.dag, execution_date=self.execution_date
)
# update the task counters
self._update_counters(ti_status, waiting_for_executor_result)
# update dag run state
_dag_runs = ti_status.active_runs[:]
for run in _dag_runs:
run.update_state(session=session)
self._update_databand_task_run_states(run)
if run.state in State.finished():
ti_status.finished_runs += 1
ti_status.active_runs.remove(run)
executed_run_dates.append(run.execution_date)
self._log_progress(ti_status)
if self.fail_fast and ti_status.failed:
msg = ",".join([t[1] for t in ti_status.failed])
logger.error(
"scheduler: Terminating executor because a task failed and fail_fast mode is enabled %s",
msg,
)
raise DatabandFailFastError(
"Failing whole pipeline as it has failed/canceled tasks %s" % msg,
)
# return updated status
return executed_run_dates
}
BACKFILL_QUEUEABLE_STATES = {
# For cases like unit tests and run manually
State.NONE,
State.UP_FOR_RESCHEDULE,
State.UP_FOR_RETRY,
# For normal backfill cases
State.SCHEDULED,
}
# Context to get the dependencies that need to be met in order for a task instance to be
# set to 'scheduled' state.
SCHEDULED_DEPS = {
RunnableExecDateDep(),
ValidStateDep(SCHEDULEABLE_STATES),
TaskNotRunningDep(),
}
# Dependencies that if met, task instance should be re-queued.
REQUEUEABLE_DEPS = {
DagTISlotsAvailableDep(),
TaskConcurrencyDep(),
PoolSlotsAvailableDep(),
}
# Dependencies that need to be met for a given task instance to be set to 'RUNNING' state.
RUNNING_DEPS = {
RunnableExecDateDep(),
ValidStateDep(RUNNABLE_STATES),
DagTISlotsAvailableDep(),
State.FAILED,
State.NONE,
State.QUEUED,
State.SCHEDULED,
State.SKIPPED,
State.UPSTREAM_FAILED,
State.UP_FOR_RETRY,
}
# Context to get the dependencies that need to be met in order for a task instance to
# be backfilled.
QUEUE_DEPS = {
NotRunningDep(), # 任务实例没有运行
NotSkippedDep(), # 任务实例没有被标记为跳过
RunnableExecDateDep(), # 判断任务执行时间 必须小于等于当前时间 且 小于等于结束时间
ValidStateDep(QUEUEABLE_STATES), # 验证任务的状态必须在队列状态中
}
# Dependencies that need to be met for a given task instance to be able to get run by an
# executor. This class just extends QueueContext by adding dependencies for resources.
RUN_DEPS = QUEUE_DEPS | {
DagTISlotsAvailableDep(), # 每个dag能并发执行的最大任务数依赖
TaskConcurrencyDep(), # 每个任务的任务实例有最大限制
}
# TODO(aoen): SCHEDULER_DEPS is not coupled to actual execution in any way and
# could easily be modified or removed from the scheduler causing this dependency to become
# outdated and incorrect. This coupling should be created (e.g. via a dag_deps analog of
# ti_deps that will be used in the scheduler code) to ensure that the logic here is
# equivalent to the logic in the scheduler.
State.SCHEDULED,
State.SKIPPED,
State.UPSTREAM_FAILED,
State.UP_FOR_RETRY,
}
# The minimum execution context for task instances to be executed.
MIN_EXEC_DEPS = {
NotRunningDep(),
NotSkippedDep(),
RunnableExecDateDep(),
}
# Context to get the dependencies that need to be met in order for a task instance to
# be backfilled.
QUEUE_DEPS = MIN_EXEC_DEPS | {ValidStateDep(QUEUEABLE_STATES)}
# Dependencies that need to be met for a given task instance to be able to get run by an
# executor. This class just extends QueueContext by adding dependencies for resources.
RUN_DEPS = QUEUE_DEPS | {DagTISlotsAvailableDep()}
# TODO(aoen): SCHEDULER_DEPS is not coupled to actual execution in any way and
# could easily be modified or removed from the scheduler causing this dependency to become
# outdated and incorrect. This coupling should be created (e.g. via a dag_deps analog of
# ti_deps that will be used in the scheduler code) to ensure that the logic here is
# equivalent to the logic in the scheduler.
# Dependencies that need to be met for a given task instance to get scheduled by the
# scheduler, then queued by the scheduler, then run by an executor.
SCHEDULER_DEPS = RUN_DEPS | {
DagrunRunningDep(),
