def fetch_data(model): """ Reads df_train and df_forecast from db df_train contains ids, outcomes and features for the training period. df_forecast contains only ids for the forecasting period. """ def get_columns_train(model, ids): columns = [] columns += ids columns.append(model['outcome']) columns += model['features'] return columns def get_columns_forecast(model, ids): columns = ids return columns connectstring = model['table']['connectstring'] schema = model['table']['schema'] table = model['table']['table'] timevar = model['table']['timevar'] groupvar = model['table']['groupvar'] ids = [timevar, groupvar] df_train = dbutils.db_to_df_limited( connectstring = connectstring, schema = schema, table = table, columns = get_columns_train(model, ids), timevar = timevar, groupvar = groupvar, tmin = model['train_start'], tmax = model['train_end'] ) df_forecast = dbutils.db_to_df_limited( connectstring = connectstring, schema = schema, table = table, columns = get_columns_forecast(model, ids), timevar = timevar, groupvar = groupvar, tmin = model['forecast_start'], tmax = model['forecast_end'] ) return df_train, df_forecast
def plot_world_average_with_actuals(df, connectstring, directory, timevar, groupvar): directory = directory + "/world_avg_w_actuals/" create_dirs([directory]) columns = get_numeric_cols(df) t_start_actuals, t_end_actuals = get_times_actuals(df) cols_actuals = [ "ged_dummy_sb", "ged_dummy_ns", "ged_dummy_os", "acled_dummy_pr" ] schema_actuals = "preflight" if groupvar == "pg_id": table_actuals = "flight_pgm" elif groupvar == "country_id": table_actuals = "flight_cm" df_actuals = dbutils.db_to_df_limited(connectstring, schema_actuals, table_actuals, columns=cols_actuals, timevar=timevar, groupvar=groupvar, tmin=t_start_actuals, tmax=t_end_actuals) df_months = fetch_df_months(connectstring) df = df.merge(df_actuals, left_index=True, right_index=True, how='outer') df_mean = df.groupby(level=0).mean() for c in columns: actual = maputils.match_plotvar_actual(c) path = directory + "world_" + str(c) + ".png" plt.figure() plt.plot(df_mean[actual], label='History') plt.plot(df_mean[c], linestyle='--', label=c) plt.xticks(*make_ticks(df, df_months), rotation=90) #title = "{}\n{}".format("World", c) #plt.title(title, loc='left') plt.legend() plt.tight_layout() plt.savefig(path) print("wrote", path) plt.close()
def get_predictions_eval_test(): schema_pred = "landed" schema_actuals = "preflight" table_ds = "ds_pgm_eval_test" table_osa = "osa_pgm_eval_test" table_ebma = "ebma_pgm_eval_test" table_actuals = "flight_pgm" timevar = "month_id" groupvar = "pg_id" ids = [timevar, groupvar] outcomes = [ "ged_dummy_sb", "ged_dummy_ns", "ged_dummy_os", "acled_dummy_pr" ] df_ds = dbutils.db_to_df(connectstring, schema_pred, table_ds) df_osa = dbutils.db_to_df(connectstring, schema_pred, table_osa) df_ebma = dbutils.db_to_df(connectstring, schema_pred, table_ebma) df_ds.set_index(ids, inplace=True) df_osa.set_index(ids, inplace=True) df_ebma.set_index(ids, inplace=True) t_start_ds = df_ds.index.get_level_values(timevar).min() t_start_osa = df_osa.index.get_level_values(timevar).min() t_start_ebma = df_ebma.index.get_level_values(timevar).min() t_end_ds = df_ds.index.get_level_values(timevar).max() t_end_osa = df_osa.index.get_level_values(timevar).max() t_end_ebma = df_ebma.index.get_level_values(timevar).max() start_same = t_start_ds == t_start_osa == t_start_ebma end_same = t_end_ds == t_end_osa == t_end_ebma if not start_same and end_same: raise RuntimeError("The time indexes for ds, osa and ebma don't match") df = dbutils.db_to_df_limited(connectstring, schema_actuals, table_actuals, columns=outcomes + ids, timevar=timevar, groupvar=groupvar, tmin=t_start_ds, tmax=t_end_ds) df = df.merge(df_ds, left_index=True, right_index=True) df = df.merge(df_osa, left_index=True, right_index=True) df = df.merge(df_ebma, left_index=True, right_index=True) return df
def get_data(connectstring, level="cm", runtype="fcast"): """ Get actuals and calibration period and testing period predictions Args: connectstring: level: cm or pgm runtype: fcast or eval """ schema_actuals = "preflight" schema_predictions = "landed" table_actuals = "flight_" + level cols_actual = ["ged_dummy_" + t for t in ["sb", "ns", "os"]] cols_actual.append("acled_dummy_pr") timevar = "month_id" if level == "cm": groupvar = "country_id" elif level == "pgm": groupvar = "pg_id" ids = [timevar, groupvar] table_osa_calib = "_".join(["osa", level, runtype, "calib"]) table_osa_test = "_".join(["osa", level, runtype, "test"]) table_ds_calib = "_".join(["ds", level, runtype, "calib"]) table_ds_test = "_".join(["ds", level, runtype, "test"]) table_cl_calib = "_".join(["cl", level, runtype, "calib"]) table_cl_test = "_".join(["cl", level, runtype, "test"]) df_osa_calib = dbutils.db_to_df(connectstring, schema_predictions, table_osa_calib, ids=ids) df_osa_test = dbutils.db_to_df(connectstring, schema_predictions, table_osa_test, ids=ids) df_ds_calib = dbutils.db_to_df(connectstring, schema_predictions, table_ds_calib, ids=ids) df_ds_test = dbutils.db_to_df(connectstring, schema_predictions, table_ds_test, ids=ids) # only include cl for pgm level if level=="pgm": df_cl_calib = dbutils.db_to_df(connectstring, schema_predictions, table_cl_calib, ids=ids) df_cl_test = dbutils.db_to_df(connectstring, schema_predictions, table_cl_test, ids=ids) assert_equal_times(df_osa_calib, df_ds_calib, timevar) assert_equal_times(df_osa_test, df_ds_test, timevar) t_start_calib = df_osa_calib.index.get_level_values(timevar).min() t_end_calib = df_osa_calib.index.get_level_values(timevar).max() df_pred_calib = df_osa_calib.merge(df_ds_calib, left_index=True, right_index=True) df_pred_test = df_osa_test.merge(df_ds_test, left_index=True, right_index=True) # only include cl for pgm level if level == "pgm": df_pred_calib = df_pred_calib.merge(df_cl_calib, left_index=True, right_index=True) df_pred_test = df_pred_test.merge(df_cl_test, left_index=True, right_index=True) df_actuals = dbutils.db_to_df_limited(connectstring, schema_actuals, table_actuals, columns=cols_actual, timevar=timevar, groupvar=groupvar, tmin=t_start_calib, tmax=t_end_calib) for df in [df_actuals, df_pred_calib, df_pred_test]: df.sort_index(inplace=True) return df_actuals, df_pred_calib, df_pred_test
df_cpgm = dbutils.db_to_df(connectstring, schema="staging_test", table="cpgm", ids=[timevar, groupvar]) df = df_pgm.merge(df_cpgm, left_index=True, right_index=True) df.reset_index(inplace=True) df = df.merge(df_c, on=["country_id"]) df.set_index([timevar, groupvar], inplace=True) tmin = df.index.get_level_values(timevar).min() tmax = df.index.get_level_values(timevar).max() df_actuals = dbutils.db_to_df_limited(connectstring, schema="preflight", table="flight_pgm", timevar="month_id", groupvar=groupvar, tmin=tmin, tmax=tmax, columns=outcomes.copy()) df_actuals.fillna(0, inplace=True) print(outcomes) df = df.merge(df_actuals, left_index=True, right_index=True) df.reset_index(inplace=True) df_months = fetch_df_months(connectstring) df.set_index(['month_id'], inplace=True) df = df.merge(df_months, left_index=True, right_index=True) df.sort_index(inplace=True) df['month'] = df['month'].astype(str)
def plot_map_worker(local_settings, plotjob): connectstring = local_settings['connectstring'] dir_plots = local_settings['dir_plots'] dir_spatial_pgm = local_settings['dir_spatial_pgm'] dir_spatial_cm = local_settings['dir_spatial_cm'] plotvar = plotjob['plotvar'] varname_actual = plotjob['varname_actual'] schema_plotvar = plotjob['schema_plotvar'] schema_actual = plotjob['schema_actual'] table_plotvar = plotjob['table_plotvar'] table_actual = plotjob['table_actual'] timevar = plotjob['timevar'] groupvar = plotjob['groupvar'] variable_scale = plotjob['variable_scale'] projection = plotjob['projection'] crop = plotjob['crop'] run_id = plotjob['run_id'] path_shape_pg = dir_spatial_pgm + "/priogrid" path_shape_c = dir_spatial_cm + "/country" ids = [timevar, groupvar] print("Plotting variable {} from table {}.{}".format(plotvar, schema_plotvar, table_plotvar)) print(json.dumps(plotjob, indent=4)) df_plotvar = dbutils.db_to_df(connectstring, schema_plotvar, table_plotvar, [plotvar], ids) df_plotvar = restrict_prob_lower_bound(df_plotvar, 0.001) time_start, time_end = get_time_limits(df_plotvar, timevar) print("time_start:", time_start) print("time_end:", time_end) have_actual=False if varname_actual is not None: have_actual=True if groupvar == "pg_id": df_geo = fetch_df_geo_pgm(connectstring) df_geo = prune_priogrid(df_geo) size = get_figure_size(df_geo, scale=0.6) if have_actual and groupvar == "pg_id": df_actuals = dbutils.db_to_df_limited(connectstring, schema_actual, table_actual, [varname_actual], timevar, groupvar, time_start, time_end) df_event_coords = get_df_actuals_event_coords(df_actuals, df_geo, varname_actual) elif groupvar == "country_id": df_geo = fetch_df_geo_c(crop) size = get_figure_size(df_geo, scale=0.6) if timevar == 'month_id': df_months = fetch_df_months(connectstring) plotvar_bounds = get_var_bounds(df_plotvar, plotvar) print("Bounds: ", plotvar_bounds) times = range(time_start, time_end+1) cmap = get_cmap(variable_scale) ticks = make_ticks(variable_scale) dir_schema = dir_plots + schema_plotvar + "/" dir_table = dir_schema + table_plotvar + "/" dir_plotvar = dir_table + plotvar + "/" create_dirs([dir_plots, dir_schema, dir_table, dir_plotvar]) for t in times: print("Plotting for {}".format(t)) df_plotvar_t = df_plotvar.loc[t] #fig = plt.figure(figsize = size) fig, ax = plt.subplots(figsize = size) print("Making basemap") map = get_basemap(projection, df_geo) print("Reading shape") if groupvar == "pg_id": map.readshapefile(path_shape_pg, 'GID', drawbounds=False) if groupvar == "country_id": map.readshapefile(path_shape_c, 'ID', drawbounds=False) if groupvar == "pg_id" and have_actual: events_t = get_events_t(df_event_coords, t) map = plot_events_on_map(map, events_t) print("Making collection") # Plot the probs collection = make_collection(map, df_plotvar_t, cmap, ticks['values'], variable_scale, plotvar_bounds, groupvar) ax.add_collection(collection) if groupvar == "pg_id": # The Africa limited shapefile map.readshapefile(path_shape_pg, 'GID', drawbounds=True) cbar_fontsize = size[1]/2 if variable_scale in ["logodds", "prob"]: # if we're plotting logodds or probs set custom ticks cbar = plt.colorbar(collection, ticks=ticks['values'], fraction=0.046, pad=0.04) cbar.ax.set_yticklabels(ticks['labels'], size=cbar_fontsize) else: # else use default colorbar for interval variables cbar = plt.colorbar(collection) map.drawmapboundary() if groupvar == "pg_id": map.readshapefile(path_shape_pg, 'GID', drawbounds=True) map.readshapefile(path_shape_c, 'ID', drawbounds=True, color='w', linewidth = 2) map.readshapefile(path_shape_c, 'ID', drawbounds=True, color='k', linewidth = 1) elif groupvar == "country_id": map.readshapefile(path_shape_c, 'ID', drawbounds=True) s_t_plotvar = "{}.{}".format(schema_plotvar, table_plotvar) text_box = "Modelname: {}\nRun: {}\nTable: {}".format(plotvar, run_id, s_t_plotvar) bbox = {'boxstyle' : 'square', 'facecolor' : "white"} lon_min = df_geo['longitude'].min() lon_max = df_geo['longitude'].max() lat_min = df_geo['latitude'].min() lat_max = df_geo['latitude'].max() w_eu = 6.705 h_eu = 4.5 w_erc = 4.7109375 h_erc = 4.5 w_views = w_eu + w_erc + 1 h_views = 4.024 lon_start_eu = lon_min + 1 lon_end_eu = lon_start_eu + w_eu lon_start_erc = lon_end_eu + 1 lon_end_erc = lon_start_erc + w_erc lon_start_views = lon_start_eu lon_end_views = lon_end_erc lat_start_eu = lat_min + 1 lat_end_eu = lat_start_eu + h_eu lat_start_erc = lat_start_eu lat_end_erc = lat_start_erc + h_erc lat_start_views = lat_end_eu + 1 lat_end_views = lat_start_views + h_views lon_min_textbox = lon_start_views lat_min_textbox = lat_end_views + 1 box_logo_eu = (lon_start_eu, lon_end_eu, lat_start_eu, lat_end_eu) box_logo_erc = (lon_start_erc, lon_end_erc, lat_start_erc, lat_end_erc) box_logo_views = (lon_start_views, lon_end_views, lat_start_views, lat_end_views) plt.text(lon_min_textbox, lat_min_textbox, text_box, bbox=bbox, fontsize=size[1]*0.5) logo_eu = mpimg.imread("/storage/static/logos/eu.png") logo_erc = mpimg.imread("/storage/static/logos/erc.png") logo_views = mpimg.imread("/storage/static/logos/views.png") plt.imshow(logo_erc, extent = box_logo_erc) plt.imshow(logo_eu, extent = box_logo_eu) plt.imshow(logo_views, extent = box_logo_views) text_title = "TITLE" if timevar == 'month_id': text_title = month_id_to_datestr(df_months, t) plt.figtext(0.5, 0.85, text_title, fontsize=size[1], ha='center') path = dir_plotvar + str(t) + ".png" plt.savefig(path, bbox_inches="tight") print("wrote", path) plt.close()
t_end_ds = df_ds.index.get_level_values(timevar).max() t_end_osa = df_osa.index.get_level_values(timevar).max() t_end_ebma = df_ebma.index.get_level_values(timevar).max() start_same = t_start_ds == t_start_osa == t_start_ebma end_same = t_end_ds == t_end_osa == t_end_ebma except: print("Error") if not start_same and end_same: raise RuntimeError("The time indexes for ds, osa and ebma don't match") df = dbutils.db_to_df_limited(connectstring, schema_actuals, table_actuals, columns=outcomes + ids, timevar=timevar, groupvar=groupvar, tmin=t_start_ds, tmax=t_end_ds) df = df.merge(df_ds, left_index=True, right_index=True) df = df.merge(df_osa, left_index=True, right_index=True) df = df.merge(df_ebma, left_index=True, right_index=True) df.to_hdf("temp.hdf5", key='data') cols = df.columns cols = [col.replace("_eval_test", "") for col in cols] cols = [col.replace("_pgm", "") for col in cols] cols = [col.replace("_downsampled", "_dsmp") for col in cols] cols = [col.replace("_fullsample", "_fsmp") for col in cols]