default_args=default_args, description='A simple tutorial DAG', schedule_interval=timedelta(minutes=20), ) # t1, t2 and t3 are examples of tasks created by instantiating operators t1 = DockerOperator( task_id='identities', command= 'python ./etl_src/entry_point.py --job_and_func=ETLs.IEEE.identities_etl.identities_etl --start_date=2017-04-25 --end_date=2017-05-15', dag=dag, ) t2 = DockerOperator( task_id='transactions', command= 'python ./etl_src/entry_point.py --job_and_func=ETLs.IEEE.transactions_etl.transactions_etl --start_date=2017-04-25 --end_date=2017-05-15', dag=dag) dag.doc_md = __doc__ t1.doc_md = """\ #### Identities etl see etl_src/ETLs/IEEE/identities_etl.py """ t2.doc_md = """\ #### Transactions etl see etl_src/ETLs/IEEE/transactions_etl.py """ t1 >> t2
#command="echo 'starting docker' && /bin/sleep 30", docker_url="unix://var/run/docker.sock", network_mode="bridge", sla=timedelta(seconds=1), dag=dag, ) ''' t3 = BashOperator( task_id='print_hello', bash_command='echo "hello world"', dag=dag, ) ''' #t1 >> dockertask >> t3 dockertask.doc_md = """\ #### Task Documentation You can document your task using the attributes `doc_md` (markdown), `doc` (plain text), `doc_rst`, `doc_json`, `doc_yaml` which gets rendered in the UI's Task Instance Details page. ![img](http://montcs.bloomu.edu/~bobmon/Semesters/2012-01/491/import%20soul.png) """ dag.doc_md = __doc__ ''' finisher = BashOperator( task_id="finish_task", bash_command='echo "{{ ti.xcom_pull(key="k1") }}" "{{ ti.xcom_pull(key="k2") }}" "{{ ti.xcom_pull(task_ids="execute_docker_container") }}"', dag=dag, ) '''