schedule_interval="0 * * * *"
)
wait_on_data_model_creation = ExternalTaskSensor(
task_id="wait_on_data_model_creation",
dag=dag,
external_dag_id="02_data_etl_dag",
external_task_id="end_task",
mode="reschedule",
poke_interval=120
)
run_data_quality_checks = AWSRedshiftDataQualityOperator(
task_id='run_data_quality_checks',
dag=dag,
conn_id=AWS_REDSHIFT_CONN_ID,
checks=data_quality_checks
)
end_task = DummyOperator(
task_id="end_task",
dag=dag
)
# set dependencies
wait_on_data_model_creation.set_downstream(run_data_quality_checks)
run_data_quality_checks.set_downstream(end_task)
cluster_type="redshift",
time_zone=local_tz)
wait_on_tasks_completed = ExternalTaskSensor(
task_id="wait_on_tasks_completed",
dag=dag,
external_dag_id="03_data_quality_check_dag",
external_task_id="end_task",
mode="reschedule",
poke_interval=120)
terminate_emr_cluster = AWSTerminateClusterOperator(
task_id="terminate_emr_cluster",
dag=dag,
conn_id=AWS_CONN_ID,
cluster_creation_task=DAG_NAME + ".create_emr_cluster",
cluster_type="emr")
terminate_redshift_cluster = AWSTerminateClusterOperator(
task_id="terminate_redshift_cluster",
dag=dag,
conn_id=AWS_CONN_ID,
cluster_creation_task=DAG_NAME + ".create_redshift_cluster",
cluster_type="redshift")
# set dependencies
wait_on_tasks_completed.set_upstream(
[create_emr_cluster, create_redshift_cluster])
wait_on_tasks_completed.set_downstream(
[terminate_emr_cluster, terminate_redshift_cluster])
