def subDag(): """ subDag에 다른 argument를 제공할 때, 사용 하는 듯? """ DAG_NAME = "subdag_test" dag = makeDag("subdag_test") start = DummyOperator( task_id='start', dag=dag, ) section_1 = SubDagOperator( task_id='section-1', subdag=subdag(DAG_NAME, 'section-1', {'owner': 'airflow'}), dag=dag, ) start >> section_1
t_analysis = BashOperator(task_id='clean_data', bash_command=a1, dag=dag) t_scrap_data = BashOperator(task_id='scrap_data', bash_command=a1, dag=dag) t_run_main_PROJECTNAME = BashOperator(task_id='run_main_PROJECTNAME', bash_command=a1, dag=dag) t_clean_data = BashOperator(task_id='clean_data', bash_command=a1, dag=dag) t_download_data = BashOperator(task_id='download_data', bash_command=a1, dag=dag) t_to_hive = BashOperator(task_id='to_hive', bash_command=a1, dag=dag) feature_analysis = SubDagOperator( task_id='feature_analysis', subdag=subdag(DAG_NAME, 'feature_analysis', default_args), dag=dag, ) feature_slope = SubDagOperator( task_id='feature_slope', subdag=subdag(DAG_NAME, 'feature_slope', default_args), dag=dag, ) kdj_rsi = SubDagOperator( task_id='kdj_rsi_stockstats', subdag=subdag(DAG_NAME, 'kdj_rsi_stockstats', default_args), dag=dag, ) download_main = DummyOperator(task_id='run_main_PROJECTNAME') to_hive = DummyOperator(task_id='to_hive') #t_mv_daily_report.set_upstream(t_dazongjiaoyi)
} dag = DAG(dag_id=DAG_NAME, default_args=args, start_date=days_ago(2), schedule_interval="@once", tags=['example']) start = DummyOperator( task_id='start', dag=dag, ) section_1 = SubDagOperator( task_id='section-1', subdag=subdag(DAG_NAME, 'section-1', args), dag=dag, ) some_other_task = DummyOperator( task_id='some-other-task', dag=dag, ) section_2 = SubDagOperator( task_id='section-2', subdag=subdag(DAG_NAME, 'section-2', args), dag=dag, ) end = DummyOperator(
dag = DAG( dag_id=DAG_NAME, default_args=args, schedule_interval="@once", ) start = DummyOperator( task_id='start', default_args=args, dag=dag, ) section_1 = SubDagOperator( task_id='section-1', subdag=subdag(DAG_NAME, 'section-1', args), default_args=args, dag=dag, ) some_other_task = DummyOperator( task_id='some-other-task', default_args=args, dag=dag, ) section_2 = SubDagOperator( task_id='section-2', subdag=subdag(DAG_NAME, 'section-2', args), default_args=args, dag=dag,
rescheduling loop. """ from datetime import datetime from airflow.models import DAG from airflow.operators import SubDagOperator from airflow.example_dags.subdags.subdag import subdag args = { 'owner': 'airflow', 'start_date': datetime(2016, 1, 1), } dag = DAG( dag_id='test_raise_executor_error', default_args=args, schedule_interval="@daily", ) section_1 = SubDagOperator( task_id='subdag_op', subdag=subdag('test_raise_executor_error', 'subdag_op', args), default_args=args, dag=dag, ) # change the subdag name -- this creates an error because the subdag # won't be found, but it'll do it in a way that causes the executor to report # success section_1.subdag.dag_id = 'bad_id'
dag = DAG( dag_id=DAG_NAME, default_args=args, schedule_interval="@once", ) start = DummyOperator( task_id='start', default_args=args, dag=dag, ) onboard = SubDagOperator( task_id='onboard_dag', subdag=subdag(DAG_NAME, 'onboard_dag', args), default_args=args, dag=dag, ) ongoing = SubDagOperator( task_id='ongoing_dag', subdag=subdag(DAG_NAME, 'ongoing_dag', args), default_args=args, dag=dag, ) end = DummyOperator( task_id='end', default_args=args, dag=dag,
with DAG( dag_id=DAG_NAME, default_args={"retries": 2}, start_date=days_ago(2), schedule_interval="@once", tags=['example'], ) as dag: start = DummyOperator( task_id='start', ) section_1 = SubDagOperator( task_id='section-1', subdag=subdag(DAG_NAME, 'section-1', dag.default_args), ) some_other_task = DummyOperator( task_id='some-other-task', ) section_2 = SubDagOperator( task_id='section-2', subdag=subdag(DAG_NAME, 'section-2', dag.default_args), ) end = DummyOperator( task_id='end', )