def _run_event_loop(self, session=None):
"""
The main process event loop
:param session: the connection of db session.
:return: None
"""
while self.running:
event: Event = self.mail_box.get_message()
self.log.debug('EVENT: {0}'.format(event))
if not self.use_local_nf:
self._sync_event_to_db(session)
try:
dag_runs = self._get_dag_runs(event)
if dag_runs is None or len(dag_runs) == 0:
continue
# create SimpleDagBag
simple_dags = []
for dag_run in dag_runs:
simple_dags.append(
self.dagrun_route.find_simple_dag(
dagrun_id=dag_run.id))
simple_dag_bag = SimpleDagBag(simple_dags)
if not self._validate_and_run_task_instances(
simple_dag_bag=simple_dag_bag):
continue
except Exception as e:
self.log.exception(str(e))
# scheduler end
self.log.debug("_run_event_loop end")
def test_scheduler_verify_pool_full(self, mock_pool_full):
"""
Test task instances not queued when pool is full
"""
mock_pool_full.return_value = False
dag = DAG(
dag_id='test_scheduler_verify_pool_full',
start_date=DEFAULT_DATE)
DummyOperator(
task_id='dummy',
dag=dag,
owner='airflow',
pool='test_scheduler_verify_pool_full')
session = settings.Session()
pool = Pool(pool='test_scheduler_verify_pool_full', slots=1)
session.add(pool)
orm_dag = DagModel(dag_id=dag.dag_id)
orm_dag.is_paused = False
session.merge(orm_dag)
session.commit()
scheduler = SchedulerJob()
dag.clear()
# Create 2 dagruns, which will create 2 task instances.
dr = scheduler.create_dag_run(dag)
self.assertIsNotNone(dr)
self.assertEquals(dr.execution_date, DEFAULT_DATE)
dr = scheduler.create_dag_run(dag)
self.assertIsNotNone(dr)
queue = []
scheduler._process_task_instances(dag, queue=queue)
self.assertEquals(len(queue), 2)
dagbag = SimpleDagBag([dag])
# Recreated part of the scheduler here, to kick off tasks -> executor
for ti_key in queue:
task = dag.get_task(ti_key[1])
ti = models.TaskInstance(task, ti_key[2])
# Task starts out in the scheduled state. All tasks in the
# scheduled state will be sent to the executor
ti.state = State.SCHEDULED
# Also save this task instance to the DB.
session.merge(ti)
session.commit()
scheduler._execute_task_instances(dagbag,
(State.SCHEDULED,
State.UP_FOR_RETRY))
self.assertEquals(len(scheduler.executor.queued_tasks), 1)