def selected_data_types_to_cols(selected_data_types, data_params): if selected_data_types: filtered_data_params = {sdt: data_params[sdt] for sdt in selected_data_types} else: filtered_data_params = data_params filtered_data_cols = config.col_params_to_col_list(filtered_data_params) return filtered_data_cols
def aggregate_and_append(data_df, params): reg_params = params['load']['regions'] if 'aggregate_by' in reg_params: agg_by = reg_params['aggregate_by'] columns_to_sum = config.col_params_to_col_list(params['data']) agg_dict = { columns_to_sum[i]: 'sum' for i in range(len(columns_to_sum)) } agg_df = data_df.groupby(['date', agg_by]).agg(agg_dict).reset_index() agg_df = agg_df.rename(columns={agg_by: 'region_code'}) data_df = data_df.append(agg_df, ignore_index=True) data_df = data_df.drop_duplicates(subset=['date', 'region_code']) return data_df