task_id='computronix_domi_permits_bq', destination_project_dataset_table='{}:computronix.domi_permits_raw'.format(os.environ['GCLOUD_PROJECT']), bucket='{}_computronix'.format(os.environ['GCS_PREFIX']), source_objects=["domi_permits/avro_output/{{ ds|get_ds_year }}/{{ ds|get_ds_month }}/{{ ds }}/*.avro"], write_disposition='WRITE_TRUNCATE', create_disposition='CREATE_IF_NEEDED', time_partitioning={'type': 'DAY'}, source_format='AVRO', autodetect=True, dag=dag ) computronix_domi_permits_geojoin = BigQueryOperator( task_id='computronix_domi_permits_geojoin', sql=build_revgeo_query('computronix', 'domi_permits_raw', 'JOBID'), use_legacy_sql=False, destination_dataset_table='{}:computronix.domi_permits'.format(os.environ['GCLOUD_PROJECT']), write_disposition='WRITE_TRUNCATE', time_partitioning={'type': 'DAY'}, dag=dag ) computronix_domi_permits_beam_cleanup = BashOperator( task_id='computronix_domi_permits_beam_cleanup', bash_command=airflow_utils.beam_cleanup_statement('{}_computronix'.format(os.environ['GCS_PREFIX'])), dag=dag ) computronix_domi_permits_gcs >> computronix_domi_permits_dataflow >> computronix_domi_permits_bq >> \ (computronix_domi_permits_geojoin, computronix_domi_permits_beam_cleanup)
# task_id='firearms_dataflow', # job_name='firearms-dataflow', # py_file=os.getcwd() + '/airflow_scripts/dags/dependencies/dataflow_scripts/firearms_dataflow.py'), # dag=dag # ) dataflow_task = BashOperator( task_id='firearms_dataflow', bash_command='python {}'.format( os.getcwd() + '/airflow_scripts/dags/dependencies/dataflow_scripts' '/firearms_dataflow.py'), dag=dag) beam_cleanup = BashOperator(task_id='firearms_beam_cleanup', bash_command=airflow_utils.beam_cleanup_statement( '{}_firearm_seizures'.format( os.environ['GCS_PREFIX'])), dag=dag) bq_insert_temp = GoogleCloudStorageToBigQueryOperator( task_id='firearms_bq_insert', destination_project_dataset_table='{}:firearm_seizures.seizures_temp'. format(os.environ['GCP_PROJECT']), bucket='{}_firearm_seizures'.format(os.environ['GCS_PREFIX']), source_objects=[ "avro_output/{}/{}/{}/*.avro".format(dt.strftime('%Y'), dt.strftime('%m').lower(), dt.strftime("%Y-%m-%d")) ], write_disposition='WRITE_APPEND', source_format='AVRO',
"gs://{}_community_centers/attendance/".format(os.environ['DAGS_PATH'], os.environ['GCS_PREFIX']) + "{{ ds|get_ds_year }}/{{ ds|get_ds_month }}/{{ ds }}_attendance.json --avro_output " + "gs://{}_community_centers/attendance/avro_output/".format( os.environ['GCS_PREFIX']) + "{{ ds|get_ds_year }}/{{ ds|get_ds_month }}/{{ ds }}/", dag=dag) comm_ctrs_bq = GoogleCloudStorageToBigQueryOperator( task_id='comm_ctrs_bq', destination_project_dataset_table='{}:community_centers.attendance'.format( os.environ['GCLOUD_PROJECT']), bucket='{}_community_centers'.format(os.environ['GCS_PREFIX']), source_objects=[ "attendance/avro_output/{{ ds|get_ds_year }}/{{ ds|get_ds_month }}/{{ ds }}/*.avro" ], write_disposition='WRITE_TRUNCATE', create_disposition='CREATE_IF_NEEDED', source_format='AVRO', autodetect=True, time_partitioning={'type': 'DAY'}, dag=dag) comm_ctrs_beam_cleanup = BashOperator( task_id='comm_ctrs_beam_cleanup', bash_command=airflow_utils.beam_cleanup_statement( '{}_community_center'.format(os.environ['GCS_PREFIX'])), dag=dag) comm_ctrs_gcs >> comm_ctrs_dataflow >> (comm_ctrs_bq, comm_ctrs_beam_cleanup)