submit_sqoop = BashOperator(
task_id="sqoop_full_table_import",
bash_command=
'bash /home/airflow/gcs/plugins/sqoop_simple_table_imports_for_airflow.sh ephemeral-spark-cluster-{{ds_nodash}}'
)
bq_load_flight_delays = GoogleCloudStorageToBigQueryOperator(
task_id="bq_load_flight_delays",
bucket="spark-etl-1",
source_objects=["sqoop-output/flights/*.avro"],
destination_project_dataset_table=
"bigdata-etl-20201027.data_analysis.flight_delays",
autodetect=True,
source_format="AVRO",
create_disposition="CREATE_IF_NEEDED",
skip_leading_rows=0,
write_disposition="WRITE_APPEND",
max_bad_records=0)
delete_cluster = DataprocClusterDeleteOperator(
task_id="delete_dataproc_cluster",
cluster_name="ephemeral-spark-cluster-{{ds_nodash}}",
region="us-central1",
trigger_rule=TriggerRule.ALL_DONE)
create_cluster.dag = dag
create_cluster.set_downstream(submit_sqoop)
submit_sqoop.set_downstream(bq_load_flight_delays)
bq_load_flight_delays.set_downstream(delete_cluster)
# tasks.
trigger_rule=TriggerRule.ALL_DONE)
# Delete gcs files in the timestamped transformed folder.
delete_transformed_files = BashOperator(
task_id='delete_transformed_files',
bash_command="gsutil -m rm -r gs://{{ var.value.gcs_bucket }}" +
"/{{ dag_run.conf['transformed_path'] }}/")
# If the spark job or BQ Load fails we rename the timestamped raw path to
# a timestamped failed path.
move_failed_files = BashOperator(
task_id='move_failed_files',
bash_command="gsutil mv gs://{{ var.value.gcs_bucket }}" +
"/{{ dag_run.conf['raw_path'] }}/ " +
"gs://{{ var.value.gcs_bucket}}" +
"/{{ dag_run.conf['failed_path'] }}/",
trigger_rule=TriggerRule.ONE_FAILED)
# Set the dag property of the first Operators, this will be inherited by
# downstream Operators.
create_cluster.dag = dag
create_cluster.set_downstream(submit_pyspark)
submit_pyspark.set_downstream([delete_cluster, bq_load])
bq_load.set_downstream(delete_transformed_files)
move_failed_files.set_upstream([bq_load, submit_pyspark])
},
{
'name': 'retention',
'type': 'INTEGER',
'mode': 'NULLABLE'
},
],
create_disposition='CREATE_IF_NEEDED',
write_disposition='WRITE_APPEND',
dag=dag)
# =================
# == tasks flow ===
# =================
# dataproc upstream & downstream for both create and delete dataproc
create_cluster_1.set_upstream(push_unique_cluster_name)
create_cluster_1.set_upstream(sensor_task)
create_cluster_2.set_upstream(push_unique_cluster_name)
create_cluster_2.set_upstream(delete_cluster_1)
# create job upstream & downstream
calc_unique_users.set_upstream(create_cluster_1)
calc_unique_users.set_downstream(bq_load_user)
calc_unique_users.set_downstream(calc_agg)
calc_agg.set_downstream(bq_load_agg)
calc_retention_day1.set_upstream(create_cluster_2)
calc_retention_day1.set_downstream(bq_load_retention)
# create cfs_to_bq upstream & downstream
bq_load_agg.set_downstream(delete_cluster_1)
bq_load_retention.set_downstream(delete_cluster_2)
create_disposition="CREATE_IF_NEEDED",
skip_leading_rows=0,
write_disposition="WRITE_APPEND",
max_bad_records=0,
)
delete_cluster = DataprocClusterDeleteOperator(
task_id ="delete_dataproc_cluster",
cluster_name="ephemeral-spark-cluster-{{ds_nodash}}",
region="asia-southeast2",
trigger_rule = TriggerRule.ALL_DONE
)
delete_tranformed_files = BashOperator(
task_id = "delete_tranformed_files",
bash_command = "gsutil -m rm -r " +BUCKET + "/series_data_output/*"
)
create_cluster.dag = dag
create_cluster.set_downstream(submit_pyspark)
submit_pyspark.set_downstream(bq_load_series_1)
bq_load_series_1.set_downstream(bq_load_series_2)
bq_load_series_2.set_downstream(delete_cluster)
delete_cluster.set_downstream(delete_tranformed_files)
autodetect=True,
source_format="NEWLINE_DELIMITED_JSON",
create_disposition="CREATE_IF_NEEDED",
skip_leading_rows=0,
write_disposition="WRITE_APPEND",
max_bad_records=0,
)
delete_cluster = DataprocClusterDeleteOperator(
task_id="delete_dataproc_cluster",
cluster_name="ephemeral-spark-cluster-{{ds_nodash}}",
region="asia-southeast2",
trigger_rule=TriggerRule.ALL_DONE)
delete_tranformed_files = BashOperator(task_id="delete_tranformed_files",
bash_command="gsutil -m rm -r " +
BUCKET + "/movies_data_output/*")
create_cluster.dag = dag
create_cluster.set_downstream(submit_pyspark)
submit_pyspark.set_downstream(bq_load_movies_1)
bq_load_movies_1.set_downstream(bq_load_movies_2)
bq_load_movies_2.set_downstream(bq_load_movies_3)
bq_load_movies_3.set_downstream(delete_cluster)
delete_cluster.set_downstream(delete_tranformed_files)
"type": "INTEGER"
}, {
"mode": "NULLABLE",
"name": "upvote_ratio",
"type": "FLOAT"
}, {
"mode": "NULLABLE",
"name": "upvote_category",
"type": "INTEGER"
}, {
"mode": "NULLABLE",
"name": "num_comments",
"type": "INTEGER"
}, {
"mode": "NULLABLE",
"name": "comments_per_upvote",
"type": "NUMERIC"
}, {
"mode": "NULLABLE",
"name": "permalink",
"type": "STRING"
}])
submit_reddit.dag = dag
submit_reddit.set_downstream(bq_load_submissions)
bq_load_submissions.set_downstream(submit_pyspark)
submit_pyspark.set_downstream(bq_load_analysis)