bash_command=pytest_cmd.format( environment=ENVIRONMENT, directory=TESTS_DIRECTORY, spark_directory=SPARK_DIRECTORY, script='test_filter_countries.py', spark_home=SPARK_HOME), dag=dag) # Trigger the next environment based on current environment if ENVIRONMENT != 'prd': if ENVIRONMENT == 'dev': trigger_next_environment_deploy = TriggerDagRunOperator( task_id='trigger_next_environment_deploy', python_callable=lambda context, dag_run: dag_run, trigger_dag_id="awesome_dag_tst", dag=dag) trigger_next_environment_deploy.set_upstream(test_filter_countries) elif ENVIRONMENT == 'tst': trigger_next_environment_deploy = TriggerDagRunOperator( task_id='trigger_next_environment_deploy', python_callable=lambda context, dag_run: dag_run, trigger_dag_id="awesome_dag_acc", dag=dag) trigger_next_environment_deploy.set_upstream(test_filter_countries) # Set order of tasks union_transactions.set_downstream(test_union_transactions) test_union_transactions.set_downstream(enrich_transactions) enrich_transactions.set_downstream(test_enrich_transactions) test_enrich_transactions.set_downstream(filter_countries)
["/Users/ravimuthyala/AirflowSparkTestCode/receipts.csv"], 'driver_memory': '1g', 'executor_cores': 1, 'num_executors': 1, 'executor_memory': '1g' } spark_submit_operator = SparkSubmitOperator(task_id='Spark_Scala_Submit_Job', dag=dag, **spark_config) emailNotify = EmailOperator(task_id='email_notification', to='*****@*****.**', subject='Spark Submit Job Alert', html_content='Airflow Spark Submit Job Done', dag=dag) t1Failed = EmailOperator(dag=dag, trigger_rule=TriggerRule.ONE_FAILED, task_id="SparkJobFailed", to=["*****@*****.**"], subject="Spark job Failed", html_content='<h3>Spark job has failed</h3>') python_operator.set_downstream(spark_submit_operator) spark_submit_operator.set_downstream(emailNotify) t1Failed.set_upstream([spark_submit_operator]) if __name__ == '__main__': dag.cli()
directory=TESTS_DIRECTORY, spark_directory=SPARK_DIRECTORY, script='test_filter_countries.py', spark_home=SPARK_HOME), dag=dag) # Trigger the next environment based on current environment if ENVIRONMENT != 'prd': if ENVIRONMENT == 'dev': trigger_next_environment_deploy = TriggerDagRunOperator( task_id='trigger_next_environment_deploy', python_callable=lambda context, dag_run: dag_run, trigger_dag_id="awesome_dag_tst", dag=dag) trigger_next_environment_deploy.set_upstream(test_filter_countries) elif ENVIRONMENT == 'tst': trigger_next_environment_deploy = TriggerDagRunOperator( task_id='trigger_next_environment_deploy', python_callable=lambda context, dag_run: dag_run, trigger_dag_id="awesome_dag_acc", dag=dag) trigger_next_environment_deploy.set_upstream(test_filter_countries) # Set order of tasks union_transactions.set_downstream(test_union_transactions) test_union_transactions.set_downstream(enrich_transactions) enrich_transactions.set_downstream(test_enrich_transactions) test_enrich_transactions.set_downstream(filter_countries) filter_countries.set_downstream(test_filter_countries)