Python subdag Examples

Example #1

def subDag():
    """
        subDag에 다른 argument를 제공할 때, 사용 하는 듯?
    """
    DAG_NAME = "subdag_test"
    dag = makeDag("subdag_test")
    start = DummyOperator(
        task_id='start',
        dag=dag,
    )

    section_1 = SubDagOperator(
        task_id='section-1',
        subdag=subdag(DAG_NAME, 'section-1', {'owner': 'airflow'}),
        dag=dag,
    )
    start >> section_1

Example #2

File: shiny_design.py Project: NAMEs/davidyu_stock

t_analysis = BashOperator(task_id='clean_data', bash_command=a1, dag=dag)

t_scrap_data = BashOperator(task_id='scrap_data', bash_command=a1, dag=dag)

t_run_main_PROJECTNAME = BashOperator(task_id='run_main_PROJECTNAME',
                                      bash_command=a1,
                                      dag=dag)
t_clean_data = BashOperator(task_id='clean_data', bash_command=a1, dag=dag)
t_download_data = BashOperator(task_id='download_data',
                               bash_command=a1,
                               dag=dag)
t_to_hive = BashOperator(task_id='to_hive', bash_command=a1, dag=dag)
feature_analysis = SubDagOperator(
    task_id='feature_analysis',
    subdag=subdag(DAG_NAME, 'feature_analysis', default_args),
    dag=dag,
)
feature_slope = SubDagOperator(
    task_id='feature_slope',
    subdag=subdag(DAG_NAME, 'feature_slope', default_args),
    dag=dag,
)
kdj_rsi = SubDagOperator(
    task_id='kdj_rsi_stockstats',
    subdag=subdag(DAG_NAME, 'kdj_rsi_stockstats', default_args),
    dag=dag,
)
download_main = DummyOperator(task_id='run_main_PROJECTNAME')
to_hive = DummyOperator(task_id='to_hive')
#t_mv_daily_report.set_upstream(t_dazongjiaoyi)

Example #3

File: example_subdag_operator.py Project: mayrop/docker-airflow

}

dag = DAG(dag_id=DAG_NAME,
          default_args=args,
          start_date=days_ago(2),
          schedule_interval="@once",
          tags=['example'])

start = DummyOperator(
    task_id='start',
    dag=dag,
)

section_1 = SubDagOperator(
    task_id='section-1',
    subdag=subdag(DAG_NAME, 'section-1', args),
    dag=dag,
)

some_other_task = DummyOperator(
    task_id='some-other-task',
    dag=dag,
)

section_2 = SubDagOperator(
    task_id='section-2',
    subdag=subdag(DAG_NAME, 'section-2', args),
    dag=dag,
)

end = DummyOperator(

Example #4

File: example_subdag_operator.py Project: AndreiDev/incubator-airflow

dag = DAG(
    dag_id=DAG_NAME,
    default_args=args,
    schedule_interval="@once",
)

start = DummyOperator(
    task_id='start',
    default_args=args,
    dag=dag,
)

section_1 = SubDagOperator(
    task_id='section-1',
    subdag=subdag(DAG_NAME, 'section-1', args),
    default_args=args,
    dag=dag,
)

some_other_task = DummyOperator(
    task_id='some-other-task',
    default_args=args,
    dag=dag,
)

section_2 = SubDagOperator(
    task_id='section-2',
    subdag=subdag(DAG_NAME, 'section-2', args),
    default_args=args,
    dag=dag,

Example #5

File: test_raise_executor_error.py Project: yiqideren/airflow

rescheduling loop.
"""
from datetime import datetime

from airflow.models import DAG
from airflow.operators import SubDagOperator
from airflow.example_dags.subdags.subdag import subdag

args = {
    'owner': 'airflow',
    'start_date': datetime(2016, 1, 1),
}

dag = DAG(
    dag_id='test_raise_executor_error',
    default_args=args,
    schedule_interval="@daily",
)

section_1 = SubDagOperator(
    task_id='subdag_op',
    subdag=subdag('test_raise_executor_error', 'subdag_op', args),
    default_args=args,
    dag=dag,
)

# change the subdag name -- this creates an error because the subdag
# won't be found, but it'll do it in a way that causes the executor to report
# success
section_1.subdag.dag_id = 'bad_id'

Example #6

dag = DAG(
    dag_id=DAG_NAME,
    default_args=args,
    schedule_interval="@once",
)

start = DummyOperator(
    task_id='start',
    default_args=args,
    dag=dag,
)

onboard = SubDagOperator(
    task_id='onboard_dag',
    subdag=subdag(DAG_NAME, 'onboard_dag', args),
    default_args=args,
    dag=dag,
)

ongoing = SubDagOperator(
    task_id='ongoing_dag',
    subdag=subdag(DAG_NAME, 'ongoing_dag', args),
    default_args=args,
    dag=dag,
)

end = DummyOperator(
    task_id='end',
    default_args=args,
    dag=dag,

Example #7

File: example_subdag_operator.py Project: youngyjd/incubator-airflow

with DAG(
    dag_id=DAG_NAME,
    default_args={"retries": 2},
    start_date=days_ago(2),
    schedule_interval="@once",
    tags=['example'],
) as dag:

    start = DummyOperator(
        task_id='start',
    )

    section_1 = SubDagOperator(
        task_id='section-1',
        subdag=subdag(DAG_NAME, 'section-1', dag.default_args),
    )

    some_other_task = DummyOperator(
        task_id='some-other-task',
    )

    section_2 = SubDagOperator(
        task_id='section-2',
        subdag=subdag(DAG_NAME, 'section-2', dag.default_args),
    )

    end = DummyOperator(
        task_id='end',
    )