dag=dag,
)
t2 = BashOperator(
task_id='sleep',
depends_on_past=False,
bash_command='sleep 5',
retries=3,
dag=dag,
)
# [END basic_task]
# [START documentation]
dag.doc_md = __doc__
t1.doc_md = """\
#### Task Documentation
You can document your task using the attributes `doc_md` (markdown),
`doc` (plain text), `doc_rst`, `doc_json`, `doc_yaml` which gets
rendered in the UI's Task Instance Details page.
"""
# [END documentation]
# [START jinja_template]
templated_command = """
{% for i in range(5) %}
echo "{{ ds }}"
echo "{{ macros.ds_add(ds, 7)}}"
echo "{{ params.my_param }}"
{% endfor %}
) as dag:
t1 = BashOperator(
task_id='print_date',
bash_command='date',
)
t2 = BashOperator(
task_id='sleep',
depends_on_past=False,
bash_command='sleep 5',
retries=3,
)
t1.doc_md = dedent("""\
#### Task Documentation
You can document your task using the attributes `doc_md` (markdown),
`doc` (plain text), `doc_rst`, `doc_json`, `doc_yaml` which gets
rendered in the UI's Task Instance Details page.
""")
dag.doc_md = __doc__ # providing that you have a docstring at the beggining of the DAG
dag.doc_md = """
This is a documentation placed anywhere
""" # otherwise, type it like this
templated_command = dedent("""
{% for i in range(5) %}
echo "{{ ds }}"
echo "{{ macros.ds_add(ds, 7)}}"
echo "{{ params.my_param }}"
{% endfor %}
""")
import_tsv = BashOperator(
task_id='import_tsv',
bash_command=
'for x in $(ls {{ params.import_dir }}/*.tsv); do node {{ params.working_dir }}/js/submit-tsv-to-mongo.js $x; done;',
dag=dag,
)
update_mongo_with_sequences = BashOperator(
task_id='update_with_sequences',
bash_command=
'for x in $(ls {{ params.import_dir }}/*.fasta); do python3 {{ params.working_dir }}/python/update_with_sequence_name.py -i $x; done;',
dag=dag,
)
mv_files = BashOperator(
task_id='move_files',
bash_command=
'mv {{ params.import_dir }}/*.tsv {{ params.import_dir }}/*.fasta {{ params.imported_dir }}',
dag=dag,
)
dag.doc_md = __doc__
import_tsv.doc_md = """\
#### Task Documentation
IMPORT TSV FROM GISAID
"""
[retrieve_meta_from_gisaid, retrieve_fasta_from_gisaid
] >> gunzip_files >> import_tsv >> update_mongo_with_sequences >> mv_files
with open(dag.params["region_cfg"], 'r') as stream:
regions = yaml.safe_load(stream)
PREMSA = """
mpirun -np {{ params.num_procs }} {{ params.hyphy_mpi }} LIBPATH={{ params.hyphy_lib_path}} {{ params.pre_msa }} --input {{ params.input}}.{{ params.gene }}.tmp --reference {{ params.working_dir }}/{{ params.regions[params["gene"]]["reference"] }} --trim-from {{ params.regions[params.gene]["trim_from"] }} --trim-to {{ params.regions[params.gene]["trim_to"] }} --E 0.01 --N-fraction {{ params.regions[params["gene"]]["fraction"] }} --remove-stop-codons Yes
"""
# t1, t2 and t3 are examples of tasks created by instantiating operators
mk_tmp = BashOperator(
task_id='mk_tmp',
bash_command=
'cp {{ params.input }} {{ params.input }}.{{ params.gene }}.tmp',
dag=dag,
)
pre_msa = BashOperator(
task_id='pre_msa',
bash_command=PREMSA,
params={'regions': regions},
dag=dag,
)
dag.doc_md = __doc__
pre_msa.doc_md = """\
#### Task Documentation
PREMSA
"""
mk_tmp >> pre_msa
# [START basic_task]
t1 = BashOperator(
task_id='print_date',
bash_command='date',
)
t2 = BashOperator(
task_id='sleep',
depends_on_past=False,
bash_command='sleep 5',
retries=3,
)
# [END basic_task]
# [START documentation]
t1.doc_md = __doc__
dag.doc_md = dedent("""\
#### Task Documentation
You can document your task using the attributes `doc_md` (markdown),
`doc` (plain text), `doc_rst`, `doc_json`, `doc_yaml` which gets
rendered in the UI's Task Instance Details page.
""")
# [END documentation]
# [START jinja_template]
templated_command = dedent("""
{% for i in range(5) %}
echo "{{ ds }}"
echo "{{ macros.ds_add(ds, i)}}"
