def set_is_paused(is_paused, args):
"""Sets is_paused for DAG by a given dag_id"""
dag = DagModel.get_dagmodel(args.dag_id)
if not dag:
raise SystemExit(f"DAG: {args.dag_id} does not exist in 'dag' table")
dag.set_is_paused(is_paused=is_paused)
print(f"Dag: {args.dag_id}, paused: {is_paused}")
def test_dag_trigger_parse_dag(self):
mailbox = Mailbox()
dag_trigger = DagTrigger("../../dags/test_scheduler_dags.py", -1, [],
False, mailbox)
dag_trigger.start()
message = mailbox.get_message()
message = SchedulerInnerEventUtil.to_inner_event(message)
# only one dag is executable
assert "test_task_start_date_scheduling" == message.dag_id
assert DagModel.get_dagmodel(
dag_id="test_task_start_date_scheduling") is not None
assert DagModel.get_dagmodel(
dag_id="test_start_date_scheduling") is not None
assert SerializedDagModel.get(
dag_id="test_task_start_date_scheduling") is not None
assert SerializedDagModel.get(
dag_id="test_start_date_scheduling") is not None
dag_trigger.end()
def get_dags(self): # type: () -> List[DAG]
if not config.get("core", "databand_url"):
self.scheduled_jobs = []
return []
logger.debug("about to get scheduler job dags from dbnd db")
self.refresh_scheduled_jobs()
dags = []
for job in self.scheduled_jobs:
if "schedule_interval" not in job:
continue
dag = self.job_to_dag(job)
dag.sync_to_db()
DagModel.get_dagmodel(dag.dag_id).set_is_paused(
is_paused=not job["active"]
or (job.get("validation_errors", None) is not None
and len(job.get("validation_errors", None)) > 0),
including_subdags=False,
)
dags.append(dag)
return dags
def test_get_paused_dag_ids(self):
dag_id = "test_get_paused_dag_ids"
dag = DAG(dag_id, is_paused_upon_creation=True)
dag.sync_to_db()
self.assertIsNotNone(DagModel.get_dagmodel(dag_id))
paused_dag_ids = DagModel.get_paused_dag_ids([dag_id])
self.assertEqual(paused_dag_ids, {dag_id})
with create_session() as session:
session.query(DagModel).filter(DagModel.dag_id == dag_id).delete(
synchronize_session=False)
def test_is_paused_subdag(self, mock_dag_bag):
subdag_id = 'dag.subdag'
subdag = DAG(
subdag_id,
start_date=DEFAULT_DATE,
)
with subdag:
DummyOperator(task_id='dummy_task', )
dag_id = 'dag'
dag = DAG(
dag_id,
start_date=DEFAULT_DATE,
)
with dag:
SubDagOperator(task_id='subdag', subdag=subdag)
mock_dag_bag.return_value.get_dag.return_value = dag
session = settings.Session()
dag.sync_to_db(session=session)
unpaused_dags = session.query(DagModel).filter(
DagModel.dag_id.in_([subdag_id, dag_id]), ).filter(
DagModel.is_paused.is_(False)).count()
self.assertEqual(2, unpaused_dags)
DagModel.get_dagmodel(dag.dag_id).set_is_paused(is_paused=True)
paused_dags = session.query(DagModel).filter(
DagModel.dag_id.in_([subdag_id, dag_id]), ).filter(
DagModel.is_paused.is_(True)).count()
self.assertEqual(2, paused_dags)
session.close()
def test_stop_dag(self):
t = threading.Thread(target=self.stop_dag_function)
t.setDaemon(True)
t.start()
self.start_scheduler('../../dags/test_event_based_scheduler.py')
with create_session() as session:
from airflow.models import DagModel
dag_model: DagModel = DagModel.get_dagmodel(
EVENT_BASED_SCHEDULER_DAG)
self.assertTrue(dag_model.is_paused)
self.assertEqual(dag_model.get_last_dagrun().state, "killed")
for ti in session.query(TaskInstance).filter(
TaskInstance.dag_id == EVENT_BASED_SCHEDULER_DAG):
self.assertTrue(ti.state in [State.SUCCESS, State.KILLED])
for te in session.query(TaskExecution).filter(
TaskExecution.dag_id == EVENT_BASED_SCHEDULER_DAG):
self.assertTrue(te.state in [State.SUCCESS, State.KILLED])
def dag_state(args, session=NEW_SESSION):
"""
Returns the state (and conf if exists) of a DagRun at the command line.
>>> airflow dags state tutorial 2015-01-01T00:00:00.000000
running
>>> airflow dags state a_dag_with_conf_passed 2015-01-01T00:00:00.000000
failed, {"name": "bob", "age": "42"}
"""
dag = DagModel.get_dagmodel(args.dag_id, session=session)
if not dag:
raise SystemExit(f"DAG: {args.dag_id} does not exist in 'dag' table")
dr = session.query(DagRun).filter_by(dag_id=args.dag_id, execution_date=args.execution_date).one_or_none()
out = dr.state if dr else None
conf_out = ''
if out and dr.conf:
conf_out = ', ' + json.dumps(dr.conf)
print(str(out) + conf_out)
def test_deactivate_unknown_dags(self):
"""
Test that dag_ids not passed into deactivate_unknown_dags
are deactivated when function is invoked
"""
dagbag = DagBag(include_examples=True)
dag_id = "test_deactivate_unknown_dags"
expected_active_dags = dagbag.dags.keys()
model_before = DagModel(dag_id=dag_id, is_active=True)
with create_session() as session:
session.merge(model_before)
models.DAG.deactivate_unknown_dags(expected_active_dags)
after_model = DagModel.get_dagmodel(dag_id)
self.assertTrue(model_before.is_active)
self.assertFalse(after_model.is_active)
# clean up
with create_session() as session:
session.query(DagModel).filter(DagModel.dag_id == 'test_deactivate_unknown_dags').delete()
def test_deactivate_unknown_dags(self):
"""
Test that dag_ids not passed into deactivate_unknown_dags
are deactivated when function is invoked
"""
dagbag = DagBag(include_examples=True)
dag_id = "test_deactivate_unknown_dags"
expected_active_dags = dagbag.dags.keys()
model_before = DagModel(dag_id=dag_id, is_active=True)
with create_session() as session:
session.merge(model_before)
models.DAG.deactivate_unknown_dags(expected_active_dags)
after_model = DagModel.get_dagmodel(dag_id)
self.assertTrue(model_before.is_active)
self.assertFalse(after_model.is_active)
# clean up
with create_session() as session:
session.query(DagModel).filter(DagModel.dag_id == 'test_deactivate_unknown_dags').delete()
def set_is_paused(is_paused, args):
"""Sets is_paused for DAG by a given dag_id"""
DagModel.get_dagmodel(args.dag_id).set_is_paused(is_paused=is_paused, )
print("Dag: {}, paused: {}".format(args.dag_id, str(is_paused)))
