def test_subdag_pools(self):
"""
Subdags and subdag tasks can't both have a pool with 1 slot
"""
dag = DAG('parent', default_args=default_args)
subdag = DAG('parent.child', default_args=default_args)
session = airflow.settings.Session()
pool_1 = airflow.models.Pool(pool='test_pool_1', slots=1)
pool_10 = airflow.models.Pool(pool='test_pool_10', slots=10)
session.add(pool_1)
session.add(pool_10)
session.commit()
DummyOperator(task_id='dummy', dag=subdag, pool='test_pool_1')
with pytest.raises(AirflowException):
SubDagOperator(task_id='child',
dag=dag,
subdag=subdag,
pool='test_pool_1')
# recreate dag because failed subdagoperator was already added
dag = DAG('parent', default_args=default_args)
SubDagOperator(task_id='child',
dag=dag,
subdag=subdag,
pool='test_pool_10')
session.delete(pool_1)
session.delete(pool_10)
session.commit()
def subdag_0():
subdag_0 = DAG('nested_cycle.op_subdag_0',
default_args=default_args)
SubDagOperator(task_id='opSubdag_A',
dag=subdag_0,
subdag=subdag_a())
SubDagOperator(task_id='opSubdag_B',
dag=subdag_0,
subdag=subdag_b())
return subdag_0
def subdag_1():
subdag_1 = DAG('nested_cycle.op_subdag_1',
default_args=default_args)
SubDagOperator(task_id='opSubdag_C',
dag=subdag_1,
subdag=subdag_c())
SubDagOperator(task_id='opSubdag_D',
dag=subdag_1,
subdag=subdag_d())
return subdag_1
def test_execute_create_dagrun_with_conf(self):
"""
When SubDagOperator executes, it creates a DagRun if there is no existing one
and wait until the DagRun succeeds.
"""
conf = {"key": "value"}
dag = DAG('parent', default_args=default_args)
subdag = DAG('parent.test', default_args=default_args)
subdag_task = SubDagOperator(task_id='test',
subdag=subdag,
dag=dag,
poke_interval=1,
conf=conf)
subdag.create_dagrun = Mock()
subdag.create_dagrun.return_value = self.dag_run_running
subdag_task._get_dagrun = Mock()
subdag_task._get_dagrun.side_effect = [
None, self.dag_run_success, self.dag_run_success
]
subdag_task.pre_execute(context={'execution_date': DEFAULT_DATE})
subdag_task.execute(context={'execution_date': DEFAULT_DATE})
subdag_task.post_execute(context={'execution_date': DEFAULT_DATE})
subdag.create_dagrun.assert_called_once_with(
run_type=DagRunType.SCHEDULED,
execution_date=DEFAULT_DATE,
conf=conf,
state=State.RUNNING,
external_trigger=True,
)
assert 3 == len(subdag_task._get_dagrun.mock_calls)
def test_subdag_name(self):
"""
Subdag names must be {parent_dag}.{subdag task}
"""
dag = DAG('parent', default_args=default_args)
subdag_good = DAG('parent.test', default_args=default_args)
subdag_bad1 = DAG('parent.bad', default_args=default_args)
subdag_bad2 = DAG('bad.test', default_args=default_args)
subdag_bad3 = DAG('bad.bad', default_args=default_args)
SubDagOperator(task_id='test', dag=dag, subdag=subdag_good)
self.assertRaises(AirflowException,
SubDagOperator,
task_id='test',
dag=dag,
subdag=subdag_bad1)
self.assertRaises(AirflowException,
SubDagOperator,
task_id='test',
dag=dag,
subdag=subdag_bad2)
self.assertRaises(AirflowException,
SubDagOperator,
task_id='test',
dag=dag,
subdag=subdag_bad3)
def test_subdag_pools_no_possible_conflict(self):
"""
Subdags and subdag tasks with no pool overlap, should not to query
pools
"""
dag = DAG('parent', default_args=default_args)
subdag = DAG('parent.child', default_args=default_args)
session = airflow.settings.Session()
pool_1 = airflow.models.Pool(pool='test_pool_1', slots=1)
pool_10 = airflow.models.Pool(pool='test_pool_10', slots=10)
session.add(pool_1)
session.add(pool_10)
session.commit()
DummyOperator(task_id='dummy', dag=subdag, pool='test_pool_10')
mock_session = Mock()
SubDagOperator(task_id='child',
dag=dag,
subdag=subdag,
pool='test_pool_1',
session=mock_session)
assert not mock_session.query.called
session.delete(pool_1)
session.delete(pool_10)
session.commit()
def standard_subdag():
import datetime # pylint: disable=redefined-outer-name,reimported
from airflow.models import DAG
from airflow.operators.dummy import DummyOperator
from airflow.operators.subdag import SubDagOperator
dag_name = 'parent'
default_args = {
'owner': 'owner1',
'start_date': datetime.datetime(2016, 1, 1)
}
dag = DAG(dag_name, default_args=default_args)
# parent:
# A -> opSubDag_0
# parent.opsubdag_0:
# -> subdag_0.task
# A -> opSubDag_1
# parent.opsubdag_1:
# -> subdag_1.task
with dag:
def subdag_0():
subdag_0 = DAG('parent.op_subdag_0',
default_args=default_args)
DummyOperator(task_id='subdag_0.task', dag=subdag_0)
return subdag_0
def subdag_1():
subdag_1 = DAG('parent.op_subdag_1',
default_args=default_args)
DummyOperator(task_id='subdag_1.task', dag=subdag_1)
return subdag_1
op_subdag_0 = SubDagOperator(task_id='op_subdag_0',
dag=dag,
subdag=subdag_0())
op_subdag_1 = SubDagOperator(task_id='op_subdag_1',
dag=dag,
subdag=subdag_1())
op_a = DummyOperator(task_id='A')
op_a.set_downstream(op_subdag_0)
op_a.set_downstream(op_subdag_1)
return dag
def test_subdag_in_context_manager(self):
"""
Creating a sub DAG within a main DAG's context manager
"""
with DAG('parent', default_args=default_args) as dag:
subdag = DAG('parent.test', default_args=default_args)
op = SubDagOperator(task_id='test', subdag=subdag)
assert op.dag == dag
assert op.subdag == subdag
def test_subdag_with_propagate_skipped_state(self, propagate_option,
states, skip_parent,
mock_get_task_instance,
mock_skip):
"""
Tests that skipped state of leaf tasks propagates to the parent dag.
Note that the skipped state propagation only takes affect when the dagrun's state is SUCCESS.
"""
dag = DAG('parent', default_args=default_args)
subdag = DAG('parent.test', default_args=default_args)
subdag_task = SubDagOperator(task_id='test',
subdag=subdag,
dag=dag,
poke_interval=1,
propagate_skipped_state=propagate_option)
dummy_subdag_tasks = [
DummyOperator(task_id=f'dummy_subdag_{i}', dag=subdag)
for i in range(len(states))
]
dummy_dag_task = DummyOperator(task_id='dummy_dag', dag=dag)
subdag_task >> dummy_dag_task
subdag_task._get_dagrun = Mock()
subdag_task._get_dagrun.return_value = self.dag_run_success
mock_get_task_instance.side_effect = [
TaskInstance(task=task, execution_date=DEFAULT_DATE, state=state)
for task, state in zip(dummy_subdag_tasks, states)
]
context = {
'execution_date': DEFAULT_DATE,
'dag_run': DagRun(),
'task': subdag_task
}
subdag_task.post_execute(context)
if skip_parent:
mock_skip.assert_called_once_with(context['dag_run'],
context['execution_date'],
[dummy_dag_task])
else:
mock_skip.assert_not_called()
def test_rerun_failed_subdag(self):
"""
When there is an existing DagRun with failed state, reset the DagRun and the
corresponding TaskInstances
"""
dag = DAG('parent', default_args=default_args)
subdag = DAG('parent.test', default_args=default_args)
subdag_task = SubDagOperator(task_id='test',
subdag=subdag,
dag=dag,
poke_interval=1)
dummy_task = DummyOperator(task_id='dummy', dag=subdag)
with create_session() as session:
dummy_task_instance = TaskInstance(
task=dummy_task,
execution_date=DEFAULT_DATE,
state=State.FAILED,
)
session.add(dummy_task_instance)
session.commit()
sub_dagrun = subdag.create_dagrun(
run_type=DagRunType.SCHEDULED,
execution_date=DEFAULT_DATE,
state=State.FAILED,
external_trigger=True,
)
subdag_task._reset_dag_run_and_task_instances(
sub_dagrun, execution_date=DEFAULT_DATE)
dummy_task_instance.refresh_from_db()
assert dummy_task_instance.state == State.NONE
sub_dagrun.refresh_from_db()
assert sub_dagrun.state == State.RUNNING
def test_execute_skip_if_dagrun_success(self):
"""
When there is an existing DagRun in SUCCESS state, skip the execution.
"""
dag = DAG('parent', default_args=default_args)
subdag = DAG('parent.test', default_args=default_args)
subdag.create_dagrun = Mock()
subdag_task = SubDagOperator(task_id='test',
subdag=subdag,
dag=dag,
poke_interval=1)
subdag_task._get_dagrun = Mock()
subdag_task._get_dagrun.return_value = self.dag_run_success
subdag_task.pre_execute(context={'execution_date': DEFAULT_DATE})
subdag_task.execute(context={'execution_date': DEFAULT_DATE})
subdag_task.post_execute(context={'execution_date': DEFAULT_DATE})
subdag.create_dagrun.assert_not_called()
assert 3 == len(subdag_task._get_dagrun.mock_calls)
def create_subdag_opt(main_dag):
subdag_name = "daily_job"
subdag = DAG(
dag_id='.'.join([dag_name, subdag_name]),
start_date=start_date,
schedule_interval=None,
concurrency=2,
)
BashOperator(bash_command="echo 1",
task_id="daily_job_subdag_task",
dag=subdag)
return SubDagOperator(
task_id=subdag_name,
subdag=subdag,
dag=main_dag,
)
def subDag():
"""
subDag에 다른 argument를 제공할 때, 사용 하는 듯?
"""
DAG_NAME = "subdag_test"
dag = makeDag("subdag_test")
start = DummyOperator(
task_id='start',
dag=dag,
)
section_1 = SubDagOperator(
task_id='section-1',
subdag=subdag(DAG_NAME, 'section-1', {'owner': 'airflow'}),
dag=dag,
)
start >> section_1
def test_execute_dagrun_failed(self):
"""
When the DagRun failed during the execution, it raises an Airflow Exception.
"""
dag = DAG('parent', default_args=default_args)
subdag = DAG('parent.test', default_args=default_args)
subdag_task = SubDagOperator(task_id='test',
subdag=subdag,
dag=dag,
poke_interval=1)
subdag.create_dagrun = Mock()
subdag.create_dagrun.return_value = self.dag_run_running
subdag_task._get_dagrun = Mock()
subdag_task._get_dagrun.side_effect = [
None, self.dag_run_failed, self.dag_run_failed
]
with pytest.raises(AirflowException):
subdag_task.pre_execute(context={'execution_date': DEFAULT_DATE})
subdag_task.execute(context={'execution_date': DEFAULT_DATE})
subdag_task.post_execute(context={'execution_date': DEFAULT_DATE})
}
dag = DAG(dag_id=DAG_NAME,
default_args=args,
start_date=days_ago(2),
schedule_interval="@once",
tags=['example'])
start = DummyOperator(
task_id='start',
dag=dag,
)
section_1 = SubDagOperator(
task_id='section-1',
subdag=subdag(DAG_NAME, 'section-1', args),
dag=dag,
)
some_other_task = DummyOperator(
task_id='some-other-task',
dag=dag,
)
section_2 = SubDagOperator(
task_id='section-2',
subdag=subdag(DAG_NAME, 'section-2', args),
dag=dag,
)
end = DummyOperator(
start_date=CONFIGS[DAG_ID]["start_date"],
tags=["example"],
)
with dag:
""" main DAG:
smart_sensor (looking for run file) ->
trigger_external_dag (dag_id_DB_1) ->
SubDAG (external_sensor -> print_logs -> remove_file -> print_finish_log | example TaskGroup) ->
send_message (into Slack chanell)
"""
@task()
def slack_send_message():
client = WebClient(token=SLACK_TOKEN)
try:
response = client.chat_postMessage(channel="airflowtask33", text="Hello from your app! :tada:")
except SlackApiError as e:
assert e.response["error"] # str like 'invalid_auth', 'channel_not_found'
sens = SmartFileSensor(task_id="checking_file", filepath=TRIGGER_DIR, fs_conn_id='fs_default')
task_trigger = TriggerDagRunOperator(
task_id="trigger_database_update", trigger_dag_id="dag_id_DB_1", wait_for_completion=True, poke_interval=15,
)
sub_dag = SubDagOperator(task_id='XCOM_sub_dag', subdag = sub_dag_processing(), default_args=DEFAULT_ARGS)
task_slack = slack_send_message()
sens >> task_trigger >> sub_dag >> task_slack
def nested_subdag_cycle():
import datetime # pylint: disable=redefined-outer-name,reimported
from airflow.models import DAG
from airflow.operators.dummy import DummyOperator
from airflow.operators.subdag import SubDagOperator
dag_name = 'nested_cycle'
default_args = {
'owner': 'owner1',
'start_date': datetime.datetime(2016, 1, 1)
}
dag = DAG(dag_name, default_args=default_args)
# cycle:
# A -> op_subdag_0
# cycle.op_subdag_0:
# -> opSubDag_A
# cycle.op_subdag_0.opSubdag_A:
# -> subdag_a.task
# -> opSubdag_B
# cycle.op_subdag_0.opSubdag_B:
# -> subdag_b.task
# A -> op_subdag_1
# cycle.op_subdag_1:
# -> opSubdag_C
# cycle.op_subdag_1.opSubdag_C:
# -> subdag_c.task -> subdag_c.task >Invalid Loop<
# -> opSubDag_D
# cycle.op_subdag_1.opSubdag_D:
# -> subdag_d.task
with dag:
def subdag_a():
subdag_a = DAG('nested_cycle.op_subdag_0.opSubdag_A',
default_args=default_args)
DummyOperator(task_id='subdag_a.task', dag=subdag_a)
return subdag_a
def subdag_b():
subdag_b = DAG('nested_cycle.op_subdag_0.opSubdag_B',
default_args=default_args)
DummyOperator(task_id='subdag_b.task', dag=subdag_b)
return subdag_b
def subdag_c():
subdag_c = DAG('nested_cycle.op_subdag_1.opSubdag_C',
default_args=default_args)
op_subdag_c_task = DummyOperator(task_id='subdag_c.task',
dag=subdag_c)
# introduce a loop in opSubdag_C
op_subdag_c_task.set_downstream(op_subdag_c_task)
return subdag_c
def subdag_d():
subdag_d = DAG('nested_cycle.op_subdag_1.opSubdag_D',
default_args=default_args)
DummyOperator(task_id='subdag_d.task', dag=subdag_d)
return subdag_d
def subdag_0():
subdag_0 = DAG('nested_cycle.op_subdag_0',
default_args=default_args)
SubDagOperator(task_id='opSubdag_A',
dag=subdag_0,
subdag=subdag_a())
SubDagOperator(task_id='opSubdag_B',
dag=subdag_0,
subdag=subdag_b())
return subdag_0
def subdag_1():
subdag_1 = DAG('nested_cycle.op_subdag_1',
default_args=default_args)
SubDagOperator(task_id='opSubdag_C',
dag=subdag_1,
subdag=subdag_c())
SubDagOperator(task_id='opSubdag_D',
dag=subdag_1,
subdag=subdag_d())
return subdag_1
op_subdag_0 = SubDagOperator(task_id='op_subdag_0',
dag=dag,
subdag=subdag_0())
op_subdag_1 = SubDagOperator(task_id='op_subdag_1',
dag=dag,
subdag=subdag_1())
op_a = DummyOperator(task_id='A')
op_a.set_downstream(op_subdag_0)
op_a.set_downstream(op_subdag_1)
return dag
"depends_on_past": False,
"owner": "demo",
"retries": 2,
"retry_delay": timedelta(minutes=1),
"execution_timeout": timedelta(minutes=10),
"start_date": datetime(2021, 1, 1),
}
with DAG(
dag_id=DAG_ID,
description="This is a DAG demo for how to use sub DAG",
catchup=False,
max_active_runs=1,
schedule_interval=timedelta(days=1),
default_args=default_args,
) as dag:
task_1 = BashOperator(dag=dag,
task_id="task_1",
bash_command="sleep 2; echo This is Task 1")
processing = SubDagOperator(
task_id="processing_tasks",
subdag=sample_subdag.main(DAG_ID, "processing_tasks", default_args),
)
task_4 = BashOperator(dag=dag,
task_id="task_4",
bash_command="sleep 2; echo This is Task 4")
task_1 >> processing >> task_4
tags=['HOM', 'Movimientos', 'Cuentas']) as dag:
sensor_cuenta_trn = ExternalTaskSensor(
task_id='sensor_trn_cuenta',
external_dag_id='dag-sii-bch-ing-ab-trn-cue-mov',
external_task_id='trn_cuenta',
)
sensor_movimientos_trn = ExternalTaskSensor(
task_id='sensor_trn_movimientos',
external_dag_id='dag-sii-bch-ing-ab-trn-cue-mov',
external_task_id='trn_movimientos',
)
start = DummyOperator(task_id='start', )
hom_cuenta = SubDagOperator(
task_id='hom_cuenta',
subdag=subdag(
DAG_NAME, 'hom_cuenta', args,
'gs://yas-sii-int-des-dev/AB/config/PAR_SII_BCH_ELT_AB_TRN_HOM_CUENTA.json'
),
)
hom_movimientos = SubDagOperator(
task_id='hom_movimientos',
subdag=subdag(
DAG_NAME, 'hom_movimientos', args,
'gs://yas-sii-int-des-dev/AB/config/PAR_SII_BCH_ELT_AB_TRN_HOM_MOVIMIENTOS.json'
),
)
end = DummyOperator(task_id='end', )
start >> sensor_cuenta_trn >> hom_cuenta >> end
start >> sensor_movimientos_trn >> hom_movimientos >> end
from datetime import datetime
from airflow.models import DAG
from airflow.operators.bash import BashOperator
from airflow.operators.python import PythonOperator
from airflow.operators.subdag import SubDagOperator
DEFAULT_DATE = datetime(2016, 1, 1)
default_args = {'owner': 'airflow', 'start_date': DEFAULT_DATE, 'run_as_user': '******'}
dag = DAG(dag_id='impersonation_subdag', default_args=default_args)
def print_today():
print(f'Today is {datetime.utcnow()}')
subdag = DAG('impersonation_subdag.test_subdag_operation', default_args=default_args)
PythonOperator(python_callable=print_today, task_id='exec_python_fn', dag=subdag)
BashOperator(task_id='exec_bash_operator', bash_command='echo "Running within SubDag"', dag=subdag)
subdag_operator = SubDagOperator(
task_id='test_subdag_operation', subdag=subdag, mode='reschedule', poke_interval=1, dag=dag
)
for i in range(2):
DummyOperator(
task_id='{}-task-{}'.format(child_dag_name, i + 1),
default_args=args,
dag=dag_subdag,
)
return dag_subdag
with DAG(
dag_id=DAG_NAME,
start_date=datetime(2019, 1, 1),
max_active_runs=1,
default_args=DEFAULT_TASK_ARGS,
schedule_interval=timedelta(minutes=1),
) as dag:
start = DummyOperator(task_id='start', )
section_1 = SubDagOperator(
task_id='section-1',
subdag=subdag(DAG_NAME, 'section-1', DEFAULT_TASK_ARGS),
default_args=DEFAULT_TASK_ARGS,
)
some_other_task = DummyOperator(task_id='some-other-task', )
start >> section_1 >> some_other_task # pylint: disable=W0104
start = DummyOperator(task_id="start", dag=subdag)
end = DummyOperator(task_id="end", dag=subdag)
start >> end
return subdag
with DAG(dag_id="tms_practice_subdag_operator",
default_args=args,
start_date=days_ago(2),
schedule_interval="@once",
tags=['tms_practice']) as outer_dag:
start = DummyOperator(
task_id='start',
dag=outer_dag,
)
end = DummyOperator(
task_id='end',
dag=outer_dag,
)
# Create subdag Operator
subdag1 = SubDagOperator(
task_id='subdag1',
subdag=create_subdag("tms_practice_subdag_operator"),
dag=outer_dag)
# Define Dependencies
start >> subdag1 >> end
redshift_conn_id='redshift',
aws_credentials_id='aws_credentials')
load_songplays_table = LoadFactOperator(task_id='Load_songplays_fact_table',
dag=dag,
redshift_conn_id='redshift',
table='songplays',
append_data='False',
sql=SqlQueries.songplay_table_insert)
user_table_task_id = "Load_user_dim_table"
load_user_dim_task = SubDagOperator(subdag=load_dimension_tables_dag(
"sparkify_etl_dag",
user_table_task_id,
"redshift",
"users",
"False",
SqlQueries.user_table_insert,
start_date=start_date),
task_id=user_table_task_id,
dag=dag)
song_table_task_id = "Load_song_dim_table"
load_song_dim_task = SubDagOperator(subdag=load_dimension_tables_dag(
"sparkify_etl_dag",
song_table_task_id,
"redshift",
"songs",
"False",
SqlQueries.song_table_insert,
start_date=start_date),
task_id=song_table_task_id,
from airflow.models import DAG
from airflow.operators.bash import BashOperator
from airflow.operators.subdag import SubDagOperator
from datetime import datetime
from pandas import json_normalize
import json
from subdags.subdag_parallel_dag import subdag_parallel_dag
default_args = {'start_date': datetime(2020, 1, 1)}
with DAG('parallel_sub_dag',
schedule_interval='@daily',
default_args=default_args,
catchup=False) as dag:
task_1 = BashOperator(task_id='task_1', bash_command="sleep 3")
processing = SubDagOperator(task_id="processing_task",
subdag=subdag_parallel_dag(
'parallel_sub_dag', 'processing_task',
default_args))
task_4 = BashOperator(task_id='task_4', bash_command="sleep 3")
task_1 >> processing >> task_4
'email_on_failure': False,
'email_on_retry': False,
# 'schedule_interval': '@once',
'retries': 0,
'retry_delay': timedelta(minutes=30),
}
with DAG(dag_id=DAG_NAME,
default_args=args,
start_date=days_ago(2),
schedule_interval="@once",
tags=['TRN']) as dag:
start = DummyOperator(task_id='start', )
trn_cuenta = SubDagOperator(
task_id='trn_cuenta',
subdag=subdag(
DAG_NAME, 'trn_cuenta', args,
'gs://yas-sii-int-des-dev/AB/config/PAR_SII_BCH_ELT_AB_TRN_HOM_CUENTA.json'
),
)
trn_estatus_cuenta = SubDagOperator(
task_id='trn_estatus_cuenta',
subdag=subdag(
DAG_NAME, 'trn_estatus_cuenta', args,
'gs://yas-sii-int-des-dev/AB/config/PAR_SII_BCH_ELT_AB_TRN_HOM_ESTATUS_CUENTA.json'
),
)
trn_movimientos = SubDagOperator(
task_id='trn_movimientos',
subdag=subdag(
DAG_NAME, 'trn_movimientos', args,
'gs://yas-sii-int-des-dev/AB/config/PAR_SII_BCH_ELT_AB_TRN_HOM_MOVIMIENTOS.json'
from covid_dag import covid_subdag
from popgroup_dag import covid_per_popgroup_subdag
from weather_dag import daily_weather_subdag
args = {
'owner': 'Airflow',
'schedule_interval': '@once',
'start_date': datetime(2021, 5, 2),
'max_active_runs': 1,
}
PARENT_DAG_NAME = "main_dag"
SUBDAG_COUNTY_NAME = "covid_county_subdag"
SUBDAG_WEATHER_NAME = "weather_subdag"
with DAG(
dag_id=PARENT_DAG_NAME,
default_args=args,
start_date=datetime(
2021, 5, 2), #days_ago(2), #datetime.datetime.now(), #days_ago(2),
schedule_interval='@once',
tags=["covid"],
) as dag:
sub_dag_covid_county = SubDagOperator(subdag=covid_subdag(
PARENT_DAG_NAME, SUBDAG_COUNTY_NAME, args),
task_id=SUBDAG_COUNTY_NAME)
sub_dag_weather = SubDagOperator(subdag=daily_weather_subdag(
PARENT_DAG_NAME, SUBDAG_WEATHER_NAME, args),
task_id=SUBDAG_WEATHER_NAME)
DAG_NAME = 'example_subdag_operator'
with DAG(
dag_id=DAG_NAME,
default_args={"retries": 2},
start_date=days_ago(2),
schedule_interval="@once",
tags=['example'],
) as dag:
start = DummyOperator(
task_id='start',
)
section_1 = SubDagOperator(
task_id='section-1',
subdag=subdag(DAG_NAME, 'section-1', dag.default_args),
)
some_other_task = DummyOperator(
task_id='some-other-task',
)
section_2 = SubDagOperator(
task_id='section-2',
subdag=subdag(DAG_NAME, 'section-2', dag.default_args),
)
end = DummyOperator(
task_id='end',
)
create_timestamp = BashOperator(
task_id='create_timestamp',
bash_command='touch ~/timestamp_{{ ts_nodash }}',
)
task_sensor >> print_results >> remove_file >> create_timestamp
return dag
# creates a dag
with DAG(dag_id='trigger_run',
start_date=datetime(2021, 1, 26),
schedule_interval='@once') as dag:
# checks if a file exists
check_for_file = SmartFileSensor(task_id='check_for_file', filepath=path)
# triggers another dag
trigger_dag = TriggerDagRunOperator(task_id='trigger_dag',
trigger_dag_id=external_dag,
execution_date='{{ execution_date }}')
# creates a sub_dag that processes results
process_results = SubDagOperator(task_id='process_results_dag',
subdag=create_sub_dag(
dag.dag_id,
'process_results_dag',
start_date=datetime(2021, 1, 26),
schedule_interval='@once')),
# sends a slack message
alert_slack = PythonOperator(task_id='alert_slack',
python_callable=slack_message)
check_for_file >> trigger_dag >> process_results >> alert_slack
globals()[dag.dag_id] = dag
return dag_subdag
with DAG(
dag_id=set_dag_id(__file__), start_date=days_ago(2), schedule_interval="@once", tags=['example']
) as dag:
start = DummyOperator(
task_id='start',
dag=dag,
)
section_1 = SubDagOperator(
task_id='section-1',
subdag=subdag(set_dag_id(__file__), 'section-1', {}),
dag=dag,
)
some_other_task = DummyOperator(
task_id='some-other-task',
dag=dag,
)
section_2 = SubDagOperator(
task_id='section-2',
subdag=subdag(set_dag_id(__file__), 'section-2', {}),
dag=dag,
)
end = DummyOperator(
