dim_time_task_id = 'load_time_dim_table'
dim_time_task = SubDagOperator(
subdag=get_staging_to_dim(
dag_name,
dim_time_task_id,
db_conn_name,
'dim_time',
1,
SqlQueries.dim_time_insert,
SqlQueries.row_count,
equals=366 * 24, # we expect to find hourly data for one year
start_date=start_date),
task_id=dim_time_task_id,
dag=dag,
)
dim_time_task.doc_md = """\
### Load data to time dimension
This task populates the `dim_time` table. \
Dimension table is truncated before inserting data.
We run data validation that expects to find a total of \
24 rows for 366 days (2016 was a leap year).
"""
# Add task to Load data into facts table
fact_weather_task = LoadFactOperator(task_id='load_weather_fact_table',
dag=dag,
redshift_conn_id=db_conn_name,
dest_table="fact_weather",
sql_query=SqlQueries.fact_weather_insert,
provide_context=True)
dim_payment_types_task = SubDagOperator(
subdag=get_staging_to_dim(
dag_name,
dim_payment_types_task_id,
db_conn_name,
'dim_payment_types',
0,
SqlQueries.dim_payment_type_insert,
SqlQueries.row_count,
source_table=
'staging_trips_{{ macros.ds_format(yesterday_ds, "%Y-%m-%d", "%Y_%m") }}', # noqa
start_date=start_date),
task_id=dim_payment_types_task_id,
dag=dag,
)
dim_payment_types_task.doc_md = """\
### Load data to payment types dimension
This task populates the `dim_payment_types` table. \
Each task execution appends any rows that are not \
previously found in the table.
"""
# Add data to facts table
fact_trips_task = LoadFactOperator(
task_id='load_trips_fact_table',
dag=dag,
redshift_conn_id=db_conn_name,
dest_table="fact_trips",
source_table=
'staging_trips_{{ macros.ds_format(yesterday_ds, "%Y-%m-%d", "%Y_%m") }}', # noqa
sql_query=SqlQueries.fact_trips_insert,
