object_name='%s/%s_%s.csv' %
(rpt_name, rpt_name, "{{ next_ds_nodash }}"),
entity='allUsers',
role='READER',
bucket='la_reporting_external',
google_cloud_storage_conn_id='gcp_project_deng',
dag=dag)
task_send_report_email = EmailOperator(
task_id='send_report_email_' + rpt_name,
to=rpt_rcpnt,
cc=rpt_pic,
subject='Report ' + rpt_name + "_" + "{{ next_ds_nodash }}",
params={
'report_name':
rpt_name,
'report_period':
'{{ next_ds_nodash }}',
'report_link':
"http://la_reporting_external.storage.googleapis.com/%s/%s" %
(rpt_name, rpt_name)
},
html_content=EmailTemplate,
dag=dag)
task_start.set_downstream(task_queryoracle_to_gcs)
task_queryoracle_to_gcs.set_downstream(task_gcs_give_acl)
task_gcs_give_acl.set_downstream(task_send_report_email)
task_send_report_email.set_downstream(task_finish)
rpt_cnt = rpt_cnt + 1
dag=dag,
)
sub = SubDagOperator(subdag=subdag,
task_id="insert_and_id_pop",
trigger_rule="one_success",
dag=dag)
clear_latest = BashOperator(
bash_command="rm -rf {}/latest_links.txt".format(RAW_TWEET_DIR),
task_id="clear_latest",
dag=dag,
)
gen_search_terms.set_upstream(fill_search_terms)
for term in SEARCH_TERMS:
term_without_punctuation = re.sub(r"\W+", "", term)
simple_search = PythonOperator(
task_id="search_{}_twitter".format(term_without_punctuation),
provide_context=True,
python_callable=search_twitter,
dag=dag,
params={"query": term},
)
simple_search.set_upstream(gen_search_terms)
simple_search.set_downstream(sub)
sub.set_downstream(email_links)
email_links.set_downstream(clear_latest)
