default_args=default_args,
description='A simple tutorial DAG',
schedule_interval=timedelta(minutes=20),
)
# t1, t2 and t3 are examples of tasks created by instantiating operators
t1 = DockerOperator(
task_id='identities',
command=
'python ./etl_src/entry_point.py --job_and_func=ETLs.IEEE.identities_etl.identities_etl --start_date=2017-04-25 --end_date=2017-05-15',
dag=dag,
)
t2 = DockerOperator(
task_id='transactions',
command=
'python ./etl_src/entry_point.py --job_and_func=ETLs.IEEE.transactions_etl.transactions_etl --start_date=2017-04-25 --end_date=2017-05-15',
dag=dag)
dag.doc_md = __doc__
t1.doc_md = """\
#### Identities etl
see etl_src/ETLs/IEEE/identities_etl.py
"""
t2.doc_md = """\
#### Transactions etl
see etl_src/ETLs/IEEE/transactions_etl.py
"""
t1 >> t2
#command="echo 'starting docker' && /bin/sleep 30",
docker_url="unix://var/run/docker.sock",
network_mode="bridge",
sla=timedelta(seconds=1),
dag=dag,
)
'''
t3 = BashOperator(
task_id='print_hello',
bash_command='echo "hello world"',
dag=dag,
)
'''
#t1 >> dockertask >> t3
dockertask.doc_md = """\
#### Task Documentation
You can document your task using the attributes `doc_md` (markdown),
`doc` (plain text), `doc_rst`, `doc_json`, `doc_yaml` which gets
rendered in the UI's Task Instance Details page.
"""
dag.doc_md = __doc__
'''
finisher = BashOperator(
task_id="finish_task",
bash_command='echo "{{ ti.xcom_pull(key="k1") }}" "{{ ti.xcom_pull(key="k2") }}" "{{ ti.xcom_pull(task_ids="execute_docker_container") }}"',
dag=dag,
)
'''