dim_time_task_id = 'load_time_dim_table' dim_time_task = SubDagOperator( subdag=get_staging_to_dim( dag_name, dim_time_task_id, db_conn_name, 'dim_time', 1, SqlQueries.dim_time_insert, SqlQueries.row_count, equals=366 * 24, # we expect to find hourly data for one year start_date=start_date), task_id=dim_time_task_id, dag=dag, ) dim_time_task.doc_md = """\ ### Load data to time dimension This task populates the `dim_time` table. \ Dimension table is truncated before inserting data. We run data validation that expects to find a total of \ 24 rows for 366 days (2016 was a leap year). """ # Add task to Load data into facts table fact_weather_task = LoadFactOperator(task_id='load_weather_fact_table', dag=dag, redshift_conn_id=db_conn_name, dest_table="fact_weather", sql_query=SqlQueries.fact_weather_insert, provide_context=True)
dim_payment_types_task = SubDagOperator( subdag=get_staging_to_dim( dag_name, dim_payment_types_task_id, db_conn_name, 'dim_payment_types', 0, SqlQueries.dim_payment_type_insert, SqlQueries.row_count, source_table= 'staging_trips_{{ macros.ds_format(yesterday_ds, "%Y-%m-%d", "%Y_%m") }}', # noqa start_date=start_date), task_id=dim_payment_types_task_id, dag=dag, ) dim_payment_types_task.doc_md = """\ ### Load data to payment types dimension This task populates the `dim_payment_types` table. \ Each task execution appends any rows that are not \ previously found in the table. """ # Add data to facts table fact_trips_task = LoadFactOperator( task_id='load_trips_fact_table', dag=dag, redshift_conn_id=db_conn_name, dest_table="fact_trips", source_table= 'staging_trips_{{ macros.ds_format(yesterday_ds, "%Y-%m-%d", "%Y_%m") }}', # noqa sql_query=SqlQueries.fact_trips_insert,