default_args = {
'start_date': days_ago(0),
}
data_ingestion_dag = DAG(
'acs_hhi_ingestion_dag',
default_args=default_args,
schedule_interval='@yearly',
description='Ingestion configuration for ACS Household Income')
acs_hhi_gcs_task_id = 'acs_hhi_to_gcs'
acs_hhi_gcs_payload = util.generate_gcs_payload(_ACS_WORKFLOW_ID,
url=_ACS_BASE_URL)
acs_hhi_gcs_operator = util.create_gcs_ingest_operator(acs_hhi_gcs_task_id,
acs_hhi_gcs_payload,
data_ingestion_dag)
acs_hhi_bq_payload = util.generate_bq_payload(_ACS_WORKFLOW_ID,
_ACS_DATASET_NAME,
url=_ACS_BASE_URL)
acs_hhi_bq_operator = util.create_bq_ingest_operator('acs_hhi_to_bq',
acs_hhi_bq_payload,
data_ingestion_dag)
acs_hhi_aggregator_payload = {'dataset_name': _ACS_DATASET_NAME}
acs_hhi_aggregator_operator = util.create_aggregator_operator(
'acs_hhi_aggregator', acs_hhi_aggregator_payload, data_ingestion_dag)
acs_hhi_exporter_payload = {'dataset_name': _ACS_DATASET_NAME}
acs_hhi_exporter_operator = util.create_exporter_operator(
'start_date': days_ago(0),
}
data_ingestion_dag = DAG(
'covid_tracking_project_ingestion_dag',
default_args=default_args,
schedule_interval='@daily', # Run once a day at midnight
description='Ingestion configuration for Covid Tracking Project')
# Ingest to GCS
ctp_gcs_task_id = 'covid_tracking_project_to_gcs'
ctp_gcs_payload = util.generate_gcs_payload(_CTP_WORKFLOW_ID,
filename=_CTP_GCS_FILENAME,
url=_CTP_DOWNLOAD_URL)
ctp_gcs_operator = util.create_gcs_ingest_operator(ctp_gcs_task_id,
ctp_gcs_payload,
data_ingestion_dag)
ctp_gcs_short_op = util.create_gcs_short_circuit_operator(
'did_ctp_files_download', ctp_gcs_task_id, data_ingestion_dag)
# Standardize and write to BQ
ctp_bq_payload = util.generate_bq_payload(_CTP_WORKFLOW_ID,
_CTP_DATASET,
filename=_CTP_GCS_FILENAME)
ctp_bq_op = util.create_bq_ingest_operator('ctp_standardize', ctp_bq_payload,
data_ingestion_dag)
# Covid Tracking Project Ingestion DAG
# TODO(jenniebrown): Add the rest of the steps
(ctp_gcs_operator >> ctp_gcs_short_op >> ctp_bq_op)
}
data_ingestion_dag = DAG(
'data_ingestion_dag',
default_args=default_args,
# TODO(https://github.com/SatcherInstitute/health-equity-tracker/issues/30)
# schedule_interval='@daily', # Run once a day at midnight
description='The data ingestion pipeline.')
# CDC Covid Deaths
cdc_covid_deaths_task_id = 'cdc_covid_deaths_to_gcs'
cdc_covid_deaths_gcs_payload = util.generate_gcs_payload(
_CDC_WORKFLOW_ID,
filename=_CDC_GCS_FILENAME,
url=_CDC_COVID_DEATHS_DOWNLOAD_URL)
cdc_covid_deaths_gcs_operator = util.create_gcs_ingest_operator(
cdc_covid_deaths_task_id, cdc_covid_deaths_gcs_payload, data_ingestion_dag)
cdc_covid_deaths_gcs_short_op = util.create_gcs_short_circuit_operator(
'did_cdc_covid_deaths_gcs_file_download', cdc_covid_deaths_task_id,
data_ingestion_dag)
cdc_covid_deaths_bq_payload = util.generate_bq_payload(
_CDC_WORKFLOW_ID, _CDC_DATASET_NAME, filename=_CDC_GCS_FILENAME)
cdc_covid_deaths_bq_operator = util.create_bq_ingest_operator(
'cdc_covid_deaths_to_bq', cdc_covid_deaths_bq_payload, data_ingestion_dag)
cdc_covid_deaths_exporter_payload = {'dataset_name': _CDC_DATASET_NAME}
cdc_covid_deaths_exporter_operator = util.create_exporter_operator(
'cdc_covid_deaths_exporter', cdc_covid_deaths_exporter_payload,
data_ingestion_dag)
# Ingestion DAG
(cdc_covid_deaths_gcs_operator >> cdc_covid_deaths_gcs_short_op >>
cdc_covid_deaths_bq_operator >> cdc_covid_deaths_exporter_operator)