def set_is_paused(is_paused, args): """Sets is_paused for DAG by a given dag_id""" dag = DagModel.get_dagmodel(args.dag_id) if not dag: raise SystemExit(f"DAG: {args.dag_id} does not exist in 'dag' table") dag.set_is_paused(is_paused=is_paused) print(f"Dag: {args.dag_id}, paused: {is_paused}")
def test_dag_trigger_parse_dag(self): mailbox = Mailbox() dag_trigger = DagTrigger("../../dags/test_scheduler_dags.py", -1, [], False, mailbox) dag_trigger.start() message = mailbox.get_message() message = SchedulerInnerEventUtil.to_inner_event(message) # only one dag is executable assert "test_task_start_date_scheduling" == message.dag_id assert DagModel.get_dagmodel( dag_id="test_task_start_date_scheduling") is not None assert DagModel.get_dagmodel( dag_id="test_start_date_scheduling") is not None assert SerializedDagModel.get( dag_id="test_task_start_date_scheduling") is not None assert SerializedDagModel.get( dag_id="test_start_date_scheduling") is not None dag_trigger.end()
def get_dags(self): # type: () -> List[DAG] if not config.get("core", "databand_url"): self.scheduled_jobs = [] return [] logger.debug("about to get scheduler job dags from dbnd db") self.refresh_scheduled_jobs() dags = [] for job in self.scheduled_jobs: if "schedule_interval" not in job: continue dag = self.job_to_dag(job) dag.sync_to_db() DagModel.get_dagmodel(dag.dag_id).set_is_paused( is_paused=not job["active"] or (job.get("validation_errors", None) is not None and len(job.get("validation_errors", None)) > 0), including_subdags=False, ) dags.append(dag) return dags
def test_get_paused_dag_ids(self): dag_id = "test_get_paused_dag_ids" dag = DAG(dag_id, is_paused_upon_creation=True) dag.sync_to_db() self.assertIsNotNone(DagModel.get_dagmodel(dag_id)) paused_dag_ids = DagModel.get_paused_dag_ids([dag_id]) self.assertEqual(paused_dag_ids, {dag_id}) with create_session() as session: session.query(DagModel).filter(DagModel.dag_id == dag_id).delete( synchronize_session=False)
def test_is_paused_subdag(self, mock_dag_bag): subdag_id = 'dag.subdag' subdag = DAG( subdag_id, start_date=DEFAULT_DATE, ) with subdag: DummyOperator(task_id='dummy_task', ) dag_id = 'dag' dag = DAG( dag_id, start_date=DEFAULT_DATE, ) with dag: SubDagOperator(task_id='subdag', subdag=subdag) mock_dag_bag.return_value.get_dag.return_value = dag session = settings.Session() dag.sync_to_db(session=session) unpaused_dags = session.query(DagModel).filter( DagModel.dag_id.in_([subdag_id, dag_id]), ).filter( DagModel.is_paused.is_(False)).count() self.assertEqual(2, unpaused_dags) DagModel.get_dagmodel(dag.dag_id).set_is_paused(is_paused=True) paused_dags = session.query(DagModel).filter( DagModel.dag_id.in_([subdag_id, dag_id]), ).filter( DagModel.is_paused.is_(True)).count() self.assertEqual(2, paused_dags) session.close()
def test_stop_dag(self): t = threading.Thread(target=self.stop_dag_function) t.setDaemon(True) t.start() self.start_scheduler('../../dags/test_event_based_scheduler.py') with create_session() as session: from airflow.models import DagModel dag_model: DagModel = DagModel.get_dagmodel( EVENT_BASED_SCHEDULER_DAG) self.assertTrue(dag_model.is_paused) self.assertEqual(dag_model.get_last_dagrun().state, "killed") for ti in session.query(TaskInstance).filter( TaskInstance.dag_id == EVENT_BASED_SCHEDULER_DAG): self.assertTrue(ti.state in [State.SUCCESS, State.KILLED]) for te in session.query(TaskExecution).filter( TaskExecution.dag_id == EVENT_BASED_SCHEDULER_DAG): self.assertTrue(te.state in [State.SUCCESS, State.KILLED])
def dag_state(args, session=NEW_SESSION): """ Returns the state (and conf if exists) of a DagRun at the command line. >>> airflow dags state tutorial 2015-01-01T00:00:00.000000 running >>> airflow dags state a_dag_with_conf_passed 2015-01-01T00:00:00.000000 failed, {"name": "bob", "age": "42"} """ dag = DagModel.get_dagmodel(args.dag_id, session=session) if not dag: raise SystemExit(f"DAG: {args.dag_id} does not exist in 'dag' table") dr = session.query(DagRun).filter_by(dag_id=args.dag_id, execution_date=args.execution_date).one_or_none() out = dr.state if dr else None conf_out = '' if out and dr.conf: conf_out = ', ' + json.dumps(dr.conf) print(str(out) + conf_out)
def test_deactivate_unknown_dags(self): """ Test that dag_ids not passed into deactivate_unknown_dags are deactivated when function is invoked """ dagbag = DagBag(include_examples=True) dag_id = "test_deactivate_unknown_dags" expected_active_dags = dagbag.dags.keys() model_before = DagModel(dag_id=dag_id, is_active=True) with create_session() as session: session.merge(model_before) models.DAG.deactivate_unknown_dags(expected_active_dags) after_model = DagModel.get_dagmodel(dag_id) self.assertTrue(model_before.is_active) self.assertFalse(after_model.is_active) # clean up with create_session() as session: session.query(DagModel).filter(DagModel.dag_id == 'test_deactivate_unknown_dags').delete()
def test_deactivate_unknown_dags(self): """ Test that dag_ids not passed into deactivate_unknown_dags are deactivated when function is invoked """ dagbag = DagBag(include_examples=True) dag_id = "test_deactivate_unknown_dags" expected_active_dags = dagbag.dags.keys() model_before = DagModel(dag_id=dag_id, is_active=True) with create_session() as session: session.merge(model_before) models.DAG.deactivate_unknown_dags(expected_active_dags) after_model = DagModel.get_dagmodel(dag_id) self.assertTrue(model_before.is_active) self.assertFalse(after_model.is_active) # clean up with create_session() as session: session.query(DagModel).filter(DagModel.dag_id == 'test_deactivate_unknown_dags').delete()
def set_is_paused(is_paused, args): """Sets is_paused for DAG by a given dag_id""" DagModel.get_dagmodel(args.dag_id).set_is_paused(is_paused=is_paused, ) print("Dag: {}, paused: {}".format(args.dag_id, str(is_paused)))