checkforfile = ShortCircuitOperator(task_id='checkforfile',
provide_context=False,
python_callable=checkforfile)
gcs_to_bq = StorageToBQ.StorageToBigQuery(
task_id='gcs_to_bq',
dataset_name=dataset_name_dag,
bigquery_table_name=bigquery_table_name_dag,
write_mode=write_mode_dag)
check_dups = CheckDupBQ.CheckBQDuplication(
task_id='check_dups',
dataset_name=dataset_name_dag,
bigquery_table_name=bigquery_table_name_dag,
bigquery_table_key=bigquery_table_key_dag,
date_column=date_column_dag)
write_logs = WriteLogs.WriteLogToBigQuery(
task_id='write_logs', bigquery_table_name=bigquery_table_name_dag)
SendEmail = SendEmail.SendNotificationEmail(
task_id='send_email',
dataset_name=dataset_name_dag,
bigquery_table_name=bigquery_table_name_dag,
date_column=date_column_dag)
email_to_gcs
email_to_gcs.set_downstream([checkforfile])
checkforfile.set_downstream([gcs_to_bq])
gcs_to_bq >> check_dups >> write_logs >> SendEmail
'email': ['*****@*****.**'],
'email_on_failure': True,
'email_on_retry': True,
'retries': 0,
}
with DAG(
'PriceAlerter',
default_args=default_args,
schedule_interval='*/5 9-17 * * *',
catchup=False,
) as dag:
price_listener = PythonOperator(
task_id='listener',
python_callable=listener.listen,
)
email_trigger = ShortCircuitOperator(
task_id='email_trigger',
python_callable=lambda: True if listener.trigger else False,
trigger_rule=TriggerRule.NONE_FAILED,
)
email = EmailOperator(
task_id='email',
to=email_service.email_list,
subject=email_service.get_email_subject(listener.summary),
html_content=email_service.get_html_content(listener.summary),
)
price_listener.set_downstream(email_trigger)
email_trigger.set_downstream(email)