,bucket = gcs_bucket_id
,source_objects = ['data/data_files/{0}/{1}_{2}.csv'.format(current_date_nodash, file, current_date)]
,destination_project_dataset_table = "canopy-advertising.reporting_raw.report_{0}".format(file)
# ,google_cloud_storage_conn_id = google_cloud_storage_default
# ,bigquery_conn_id = SOURCE_CONNECTION_ID
# ,write_disposition = 'WRITE_TRUNCATE'
,create_disposition = 'CREATE_IF_NEEDED'
#,schema_object = 'data/table_definitions/report_{0}_table_definition.json'.format(file)
,schema_object = 'data/table_definitions/report_{0}_table_definition.json'.format(file)
,field_delimiter = ','
,skip_leading_rows = 1
,max_bad_records = 0
,quote_character = '"'
,allow_jagged_rows = False
,on_retry_callback = lambda context: download.clear(
start_date=context['execution_date'],
end_date=context['execution_date'])
)
finished_downloading >> import_to_gcs >> finished_uploading_to_gbq
success_email = EmailOperator(
task_id = 'success_email'
,to = dag.default_args.get('email', [])
,subject = 'SUCCESS - DAG_ID = {{ dag.dag_id }}'
,html_content = "DAG_ID = {{ dag.dag_id }} has completed successfully on {{ ti.hostname }}"
#,provide_context = True
)
import_to_bq = GoogleCloudStorageToBigQueryOperator(
task_id = 'task_import_to_bq'
,bucket = config['gcp_client_gcs_bucket']
,source_objects = ['data/rtbiq_data/{0}*'.format(execution_date)]
,destination_project_dataset_table = config['gcp_client_project_id'] + ':' + config['gcp_client_bq_dataset'] + '.' + config['gcp_client_bq_tablename']
,create_disposition = 'CREATE_IF_NEEDED'
,schema_object = config['rtbiq_table_schema']
,write_disposition = 'WRITE_APPEND'
,field_delimiter = ','
,skip_leading_rows = 1
,max_bad_records = 0
,quote_character = '"'
,allow_jagged_rows = False
,on_retry_callback = lambda context: download_data.clear(
start_date=execution_date,
end_date=execution_date)
)
download_data >> finish_download >> move_to_client_gcs >> finish_move_to_client_gcs >> import_to_bq >> finish_upload_to_bq
send_success_email = EmailOperator(
task_id = 'task_send_success_email'
,to = dag.default_args.get('email', [])
,subject = 'SUCCESS - DAG_ID = {{ dag.dag_id }}'
,html_content = "DAG_ID = {{ dag.dag_id }} has completed successfully on {{ ti.hostname }}"
)
finish_upload_to_bq >> send_success_email >> complete
