bash_command=pytest_cmd.format(
environment=ENVIRONMENT,
directory=TESTS_DIRECTORY,
spark_directory=SPARK_DIRECTORY,
script='test_filter_countries.py',
spark_home=SPARK_HOME),
dag=dag)
# Trigger the next environment based on current environment
if ENVIRONMENT != 'prd':
if ENVIRONMENT == 'dev':
trigger_next_environment_deploy = TriggerDagRunOperator(
task_id='trigger_next_environment_deploy',
python_callable=lambda context, dag_run: dag_run,
trigger_dag_id="awesome_dag_tst",
dag=dag)
trigger_next_environment_deploy.set_upstream(test_filter_countries)
elif ENVIRONMENT == 'tst':
trigger_next_environment_deploy = TriggerDagRunOperator(
task_id='trigger_next_environment_deploy',
python_callable=lambda context, dag_run: dag_run,
trigger_dag_id="awesome_dag_acc",
dag=dag)
trigger_next_environment_deploy.set_upstream(test_filter_countries)
# Set order of tasks
union_transactions.set_downstream(test_union_transactions)
test_union_transactions.set_downstream(enrich_transactions)
enrich_transactions.set_downstream(test_enrich_transactions)
test_enrich_transactions.set_downstream(filter_countries)
["/Users/ravimuthyala/AirflowSparkTestCode/receipts.csv"],
'driver_memory': '1g',
'executor_cores': 1,
'num_executors': 1,
'executor_memory': '1g'
}
spark_submit_operator = SparkSubmitOperator(task_id='Spark_Scala_Submit_Job',
dag=dag,
**spark_config)
emailNotify = EmailOperator(task_id='email_notification',
to='*****@*****.**',
subject='Spark Submit Job Alert',
html_content='Airflow Spark Submit Job Done',
dag=dag)
t1Failed = EmailOperator(dag=dag,
trigger_rule=TriggerRule.ONE_FAILED,
task_id="SparkJobFailed",
to=["*****@*****.**"],
subject="Spark job Failed",
html_content='<h3>Spark job has failed</h3>')
python_operator.set_downstream(spark_submit_operator)
spark_submit_operator.set_downstream(emailNotify)
t1Failed.set_upstream([spark_submit_operator])
if __name__ == '__main__':
dag.cli()
directory=TESTS_DIRECTORY,
spark_directory=SPARK_DIRECTORY,
script='test_filter_countries.py',
spark_home=SPARK_HOME),
dag=dag)
# Trigger the next environment based on current environment
if ENVIRONMENT != 'prd':
if ENVIRONMENT == 'dev':
trigger_next_environment_deploy = TriggerDagRunOperator(
task_id='trigger_next_environment_deploy',
python_callable=lambda context, dag_run: dag_run,
trigger_dag_id="awesome_dag_tst",
dag=dag)
trigger_next_environment_deploy.set_upstream(test_filter_countries)
elif ENVIRONMENT == 'tst':
trigger_next_environment_deploy = TriggerDagRunOperator(
task_id='trigger_next_environment_deploy',
python_callable=lambda context, dag_run: dag_run,
trigger_dag_id="awesome_dag_acc",
dag=dag)
trigger_next_environment_deploy.set_upstream(test_filter_countries)
# Set order of tasks
union_transactions.set_downstream(test_union_transactions)
test_union_transactions.set_downstream(enrich_transactions)
enrich_transactions.set_downstream(test_enrich_transactions)
test_enrich_transactions.set_downstream(filter_countries)
filter_countries.set_downstream(test_filter_countries)