def test_show_multi_exps_results(self): periods = [["1980-01-01", "1990-01-01"], ["1990-01-01", "2000-01-01"], ["2000-01-01", "2010-01-01"], ["2010-01-01", "2020-01-01"]] train_test_period_pairs = list(itertools.permutations(periods, 2)) sub_lst = [ "basic/exp6", "basic/exp46", "basic/exp47", "basic/exp7", "basic/exp37", "basic/exp1", "basic/exp48", "basic/exp8", "basic/exp5", "basic/exp49", "basic/exp50", "basic/exp9" ] exp_lst = [["basic_exp6"], ["basic_exp46"], ["basic_exp47"], ["basic_exp7"], ["basic_exp37"], ["basic_exp1"], ["basic_exp48"], ["basic_exp8"], ["basic_exp5"], ["basic_exp49"], ["basic_exp50"], ["basic_exp9"]] for i in range(len(exp_lst)): config_file = copy.deepcopy(cfg) args = cmd( sub=sub_lst[i], train_period=train_test_period_pairs[i][0], train_mode=0, test_period=train_test_period_pairs[i][1], quick_data=0, cache_state=1, flow_screen={ 'missing_data_ratio': 1, 'zero_value_ratio': 1 }, te=300, gage_id_file= "/mnt/data/owen411/code/hydro-anthropogenic-lstm/example/output/gages/basic/exp37/3557basins_ID_NSE_DOR.csv" ) update_cfg(config_file, args) config_data = GagesConfig(config_file) test_epoch = config_data.config_file.TEST_EPOCH inds_df, pred_mean, obs_mean = load_ensemble_result( config_file, exp_lst[i], test_epoch, return_value=True) hydro_logger.info("the median NSE of %s is %s", sub_lst[i], inds_df["NSE"].median())
var_dict_file_name='test_dictAttribute.json', t_s_dict_file_name='test_dictTimeSpace.json') hydro_logger.info("read and save camels 531 data model") # plot data_model = GagesModel.load_datamodel( config_data.data_path["Temp"], data_source_file_name='test_data_source.txt', stat_file_name='test_Statistics.json', flow_file_name='test_flow.npy', forcing_file_name='test_forcing.npy', attr_file_name='test_attr.npy', f_dict_file_name='test_dictFactorize.json', var_dict_file_name='test_dictAttribute.json', t_s_dict_file_name='test_dictTimeSpace.json') inds_df_camels, pred_mean, obs_mean = load_ensemble_result(cfg, exp_lst, test_epoch, return_value=True) data_model_conus = GagesModel.load_datamodel( all_config_Data.data_path["Temp"], data_source_file_name='test_data_source.txt', stat_file_name='test_Statistics.json', flow_file_name='test_flow.npy', forcing_file_name='test_forcing.npy', attr_file_name='test_attr.npy', f_dict_file_name='test_dictFactorize.json', var_dict_file_name='test_dictAttribute.json', t_s_dict_file_name='test_dictTimeSpace.json') all_sites = data_model_conus.t_s_dict["sites_id"] idx_lst_camels = [ i for i in range(len(all_sites))
def test_plot_map_cartopy_multi_vars(self): conus_exps = ["basic_exp37"] config_data = load_dataconfig_case_exp(cfg, conus_exps[0]) dor_1 = -0.02 source_data_dor1 = GagesSource.choose_some_basins( config_data, config_data.model_dict["data"]["tRangeTrain"], screen_basin_area_huc4=False, DOR=dor_1) # basins with dams source_data_withdams = GagesSource.choose_some_basins( config_data, config_data.model_dict["data"]["tRangeTrain"], screen_basin_area_huc4=False, dam_num=[1, 10000]) # basins without dams source_data_withoutdams = GagesSource.choose_some_basins( config_data, config_data.model_dict["data"]["tRangeTrain"], screen_basin_area_huc4=False, dam_num=0) sites_id_dor1 = source_data_dor1.all_configs['flow_screen_gage_id'] sites_id_withdams = source_data_withdams.all_configs[ 'flow_screen_gage_id'] sites_id_nodam = source_data_withoutdams.all_configs[ 'flow_screen_gage_id'] sites_id_smalldam = np.intersect1d( np.array(sites_id_dor1), np.array(sites_id_withdams)).tolist() data_model = GagesModel.load_datamodel( config_data.data_path["Temp"], data_source_file_name='test_data_source.txt', stat_file_name='test_Statistics.json', flow_file_name='test_flow.npy', forcing_file_name='test_forcing.npy', attr_file_name='test_attr.npy', f_dict_file_name='test_dictFactorize.json', var_dict_file_name='test_dictAttribute.json', t_s_dict_file_name='test_dictTimeSpace.json') all_lat = data_model.data_source.gage_dict["LAT_GAGE"] all_lon = data_model.data_source.gage_dict["LNG_GAGE"] conus_sites = data_model.t_s_dict["sites_id"] idx_lst_nodam_in_conus = [ i for i in range(len(conus_sites)) if conus_sites[i] in sites_id_nodam ] idx_lst_smalldam_in_conus = [ i for i in range(len(conus_sites)) if conus_sites[i] in sites_id_smalldam ] attr_lst = ["SLOPE_PCT", "ELEV_MEAN_M_BASIN"] attrs = data_model.data_source.read_attr(conus_sites, attr_lst, is_return_dict=False) test_epoch = 300 inds_df, pred, obs = load_ensemble_result(cfg, conus_exps, test_epoch, return_value=True) show_ind_key = "NSE" nse_range = [0, 1] idx_lst_nse = inds_df[(inds_df[show_ind_key] >= nse_range[0]) & ( inds_df[show_ind_key] < nse_range[1])].index.tolist() type_1_index_lst = np.intersect1d(idx_lst_nodam_in_conus, idx_lst_nse).tolist() type_2_index_lst = np.intersect1d(idx_lst_smalldam_in_conus, idx_lst_nse).tolist() frame = [] df_type1 = pd.DataFrame({ "type": np.full(len(type_1_index_lst), "zero-dor"), show_ind_key: inds_df[show_ind_key].values[type_1_index_lst], "lat": all_lat[type_1_index_lst], "lon": all_lon[type_1_index_lst], "slope": attrs[type_1_index_lst, 0], "elevation": attrs[type_1_index_lst, 1] }) frame.append(df_type1) df_type2 = pd.DataFrame({ "type": np.full(len(type_2_index_lst), "small-dor"), show_ind_key: inds_df[show_ind_key].values[type_2_index_lst], "lat": all_lat[type_2_index_lst], "lon": all_lon[type_2_index_lst], "slope": attrs[type_2_index_lst, 0], "elevation": attrs[type_2_index_lst, 1] }) frame.append(df_type2) data_df = pd.concat(frame) idx_lst = [ np.arange(len(type_1_index_lst)), np.arange(len(type_1_index_lst), len(type_1_index_lst) + len(type_2_index_lst)) ] plot_gages_map_and_scatter(data_df, [show_ind_key, "lat", "lon", "elevation"], idx_lst, cmap_strs=["Reds", "Blues"], labels=["zero-dor", "small-dor"], scatter_label=[attr_lst[1], show_ind_key]) # matplotlib.rcParams.update({'font.size': 12}) plt.tight_layout() plt.show()
idx_lst_largedam_in_pair3 = [ i for i in range(len(pair3_sites)) if pair3_sites[i] in sites_id_largedam ] idx_lst_largedam_in_conus = [ i for i in range(len(conus_sites)) if conus_sites[i] in sites_id_largedam ] compare_item = 2 # 0: plot the map for comparing NSE of small-dor and zero-dor # 1: ecdf plots of different cases # 2: box plots pf different cases # 3: show the median NSE values of different cases # 4: see if the differences between different cases are significant if compare_item == 0: inds_df, pred, obs = load_ensemble_result(cfg, conus_exps, test_epoch, return_value=True) all_lat = conus_data_model.data_source.gage_dict["LAT_GAGE"] all_lon = conus_data_model.data_source.gage_dict["LNG_GAGE"] show_ind_key = "NSE" attr_lst = ["SLOPE_PCT", "ELEV_MEAN_M_BASIN"] attrs = conus_data_model.data_source.read_attr(conus_sites, attr_lst, is_return_dict=False) western_lon_idx = [i for i in range(all_lon.size) if all_lon[i] < -100] nse_range = [0, 1] idx_lst_nse = inds_df[(inds_df[show_ind_key] >= nse_range[0]) & ( inds_df[show_ind_key] < nse_range[1])].index.tolist()
def test_zero_small_dor_basins_locations(self): conus_exps = self.exp_lst test_epoch = self.test_epoch inds_df, pred, obs = load_ensemble_result(self.config_file, conus_exps, test_epoch, return_value=True) conus_config_data = load_dataconfig_case_exp(self.config_file, conus_exps[0]) conus_data_model = GagesModel.load_datamodel( conus_config_data.data_path["Temp"], data_source_file_name='test_data_source.txt', stat_file_name='test_Statistics.json', flow_file_name='test_flow.npy', forcing_file_name='test_forcing.npy', attr_file_name='test_attr.npy', f_dict_file_name='test_dictFactorize.json', var_dict_file_name='test_dictAttribute.json', t_s_dict_file_name='test_dictTimeSpace.json') conus_sites = conus_data_model.t_s_dict["sites_id"] all_lat = conus_data_model.data_source.gage_dict["LAT_GAGE"] all_lon = conus_data_model.data_source.gage_dict["LNG_GAGE"] show_ind_key = "NSE" attr_lst = ["SLOPE_PCT", "ELEV_MEAN_M_BASIN"] attrs = conus_data_model.data_source.read_attr(conus_sites, attr_lst, is_return_dict=False) western_lon_idx = [i for i in range(all_lon.size) if all_lon[i] < -100] nse_range = [0, 1] idx_lst_nse = inds_df[(inds_df[show_ind_key] >= nse_range[0]) & ( inds_df[show_ind_key] < nse_range[1])].index.tolist() idx_lst_nse = np.intersect1d(western_lon_idx, idx_lst_nse) # small dor source_data_dor1 = GagesSource.choose_some_basins( conus_config_data, conus_config_data.model_dict["data"]["tRangeTrain"], screen_basin_area_huc4=False, DOR=-self.dor) # basins with dams source_data_withdams = GagesSource.choose_some_basins( conus_config_data, conus_config_data.model_dict["data"]["tRangeTrain"], screen_basin_area_huc4=False, dam_num=[1, 10000]) # basins without dams source_data_withoutdams = GagesSource.choose_some_basins( conus_config_data, conus_config_data.model_dict["data"]["tRangeTrain"], screen_basin_area_huc4=False, dam_num=0) sites_id_dor1 = source_data_dor1.all_configs['flow_screen_gage_id'] sites_id_withdams = source_data_withdams.all_configs[ 'flow_screen_gage_id'] sites_id_nodam = source_data_withoutdams.all_configs[ 'flow_screen_gage_id'] sites_id_smalldam = np.intersect1d( np.array(sites_id_dor1), np.array(sites_id_withdams)).tolist() idx_lst_nodam_in_conus = [ i for i in range(len(conus_sites)) if conus_sites[i] in sites_id_nodam ] idx_lst_smalldam_in_conus = [ i for i in range(len(conus_sites)) if conus_sites[i] in sites_id_smalldam ] type_1_index_lst = np.intersect1d(idx_lst_nodam_in_conus, idx_lst_nse).tolist() type_2_index_lst = np.intersect1d(idx_lst_smalldam_in_conus, idx_lst_nse).tolist() pd.DataFrame({ "GAGE_ID": np.array(conus_sites)[type_1_index_lst] }).to_csv( os.path.join(conus_config_data.data_path["Out"], "western-zero-dor-sites.csv")) pd.DataFrame({ "GAGE_ID": np.array(conus_sites)[type_2_index_lst] }).to_csv( os.path.join(conus_config_data.data_path["Out"], "western-small-dor-sites.csv")) frame = [] df_type1 = pd.DataFrame({ "type": np.full(len(type_1_index_lst), "zero-dor"), show_ind_key: inds_df[show_ind_key].values[type_1_index_lst], "lat": all_lat[type_1_index_lst], "lon": all_lon[type_1_index_lst], "slope": attrs[type_1_index_lst, 0], "elevation": attrs[type_1_index_lst, 1] }) frame.append(df_type1) df_type2 = pd.DataFrame({ "type": np.full(len(type_2_index_lst), "small-dor"), show_ind_key: inds_df[show_ind_key].values[type_2_index_lst], "lat": all_lat[type_2_index_lst], "lon": all_lon[type_2_index_lst], "slope": attrs[type_2_index_lst, 0], "elevation": attrs[type_2_index_lst, 1] }) frame.append(df_type2) data_df = pd.concat(frame) idx_lst = [ np.arange(len(type_1_index_lst)), np.arange(len(type_1_index_lst), len(type_1_index_lst) + len(type_2_index_lst)) ] plot_gages_map_and_scatter(data_df, [show_ind_key, "lat", "lon", "slope"], idx_lst, cmap_strs=["Reds", "Blues"], labels=["zero-dor", "small-dor"], scatter_label=[attr_lst[0], show_ind_key], wspace=2, hspace=1.5, legend_y=.8, sub_fig_ratio=[6, 4, 1]) plt.tight_layout() plt.show()
def test_diff_dor(self): dor_1 = -self.dor dor_2 = self.dor test_epoch = self.test_epoch config_file = self.config_file conus_exps = ["basic_exp37"] pair1_exps = ["dam_exp1"] pair2_exps = ["nodam_exp7"] pair3_exps = ["dam_exp27"] nodam_exp_lst = ["nodam_exp1"] smalldam_exp_lst = [ "dam_exp17" ] # -0.003["dam_exp11"] -0.08["dam_exp17"] -1["dam_exp32"] largedam_exp_lst = [ "dam_exp4" ] # 0.003["dam_exp12"] 0.08["dam_exp18"] 1["dam_exp33"] pair1_config_data = load_dataconfig_case_exp(config_file, pair1_exps[0]) pair2_config_data = load_dataconfig_case_exp(config_file, pair2_exps[0]) pair3_config_data = load_dataconfig_case_exp(config_file, pair3_exps[0]) conus_config_data = load_dataconfig_case_exp(config_file, conus_exps[0]) conus_data_model = GagesModel.load_datamodel( conus_config_data.data_path["Temp"], data_source_file_name='test_data_source.txt', stat_file_name='test_Statistics.json', flow_file_name='test_flow.npy', forcing_file_name='test_forcing.npy', attr_file_name='test_attr.npy', f_dict_file_name='test_dictFactorize.json', var_dict_file_name='test_dictAttribute.json', t_s_dict_file_name='test_dictTimeSpace.json') conus_sites = conus_data_model.t_s_dict["sites_id"] source_data_dor1 = GagesSource.choose_some_basins( conus_config_data, conus_config_data.model_dict["data"]["tRangeTrain"], screen_basin_area_huc4=False, DOR=dor_1) source_data_dor2 = GagesSource.choose_some_basins( conus_config_data, conus_config_data.model_dict["data"]["tRangeTrain"], screen_basin_area_huc4=False, DOR=dor_2) # basins with dams source_data_withdams = GagesSource.choose_some_basins( conus_config_data, conus_config_data.model_dict["data"]["tRangeTrain"], screen_basin_area_huc4=False, dam_num=[1, 10000]) # basins without dams source_data_withoutdams = GagesSource.choose_some_basins( conus_config_data, conus_config_data.model_dict["data"]["tRangeTrain"], screen_basin_area_huc4=False, dam_num=0) sites_id_dor1 = source_data_dor1.all_configs['flow_screen_gage_id'] sites_id_withdams = source_data_withdams.all_configs[ 'flow_screen_gage_id'] sites_id_nodam = source_data_withoutdams.all_configs[ 'flow_screen_gage_id'] sites_id_smalldam = np.intersect1d( np.array(sites_id_dor1), np.array(sites_id_withdams)).tolist() sites_id_largedam = source_data_dor2.all_configs['flow_screen_gage_id'] # sites_id_nolargedam = np.sort(np.union1d(np.array(sites_id_nodam), np.array(sites_id_largedam))).tolist() # pair1_sites = np.sort(np.intersect1d(np.array(sites_id_dor1), np.array(conus_sites))).tolist() # pair2_sites = np.sort(np.intersect1d(np.array(sites_id_nolargedam), np.array(conus_sites))).tolist() # pair3_sites = np.sort(np.intersect1d(np.array(sites_id_withdams), np.array(conus_sites))).tolist() pair1_data_model = GagesModel.load_datamodel( pair1_config_data.data_path["Temp"], data_source_file_name='test_data_source.txt', stat_file_name='test_Statistics.json', flow_file_name='test_flow.npy', forcing_file_name='test_forcing.npy', attr_file_name='test_attr.npy', f_dict_file_name='test_dictFactorize.json', var_dict_file_name='test_dictAttribute.json', t_s_dict_file_name='test_dictTimeSpace.json') pair1_sites = pair1_data_model.t_s_dict["sites_id"] pair2_data_model = GagesModel.load_datamodel( pair2_config_data.data_path["Temp"], data_source_file_name='test_data_source.txt', stat_file_name='test_Statistics.json', flow_file_name='test_flow.npy', forcing_file_name='test_forcing.npy', attr_file_name='test_attr.npy', f_dict_file_name='test_dictFactorize.json', var_dict_file_name='test_dictAttribute.json', t_s_dict_file_name='test_dictTimeSpace.json') pair2_sites = pair2_data_model.t_s_dict["sites_id"] pair3_data_model = GagesModel.load_datamodel( pair3_config_data.data_path["Temp"], data_source_file_name='test_data_source.txt', stat_file_name='test_Statistics.json', flow_file_name='test_flow.npy', forcing_file_name='test_forcing.npy', attr_file_name='test_attr.npy', f_dict_file_name='test_dictFactorize.json', var_dict_file_name='test_dictAttribute.json', t_s_dict_file_name='test_dictTimeSpace.json') pair3_sites = pair3_data_model.t_s_dict["sites_id"] idx_lst_nodam_in_pair1 = [ i for i in range(len(pair1_sites)) if pair1_sites[i] in sites_id_nodam ] idx_lst_nodam_in_pair2 = [ i for i in range(len(pair2_sites)) if pair2_sites[i] in sites_id_nodam ] idx_lst_nodam_in_pair3 = [ i for i in range(len(pair3_sites)) if pair3_sites[i] in sites_id_nodam ] idx_lst_nodam_in_conus = [ i for i in range(len(conus_sites)) if conus_sites[i] in sites_id_nodam ] idx_lst_smalldam_in_pair1 = [ i for i in range(len(pair1_sites)) if pair1_sites[i] in sites_id_smalldam ] idx_lst_smalldam_in_pair2 = [ i for i in range(len(pair2_sites)) if pair2_sites[i] in sites_id_smalldam ] idx_lst_smalldam_in_pair3 = [ i for i in range(len(pair3_sites)) if pair3_sites[i] in sites_id_smalldam ] idx_lst_smalldam_in_conus = [ i for i in range(len(conus_sites)) if conus_sites[i] in sites_id_smalldam ] idx_lst_largedam_in_pair1 = [ i for i in range(len(pair1_sites)) if pair1_sites[i] in sites_id_largedam ] idx_lst_largedam_in_pair2 = [ i for i in range(len(pair2_sites)) if pair2_sites[i] in sites_id_largedam ] idx_lst_largedam_in_pair3 = [ i for i in range(len(pair3_sites)) if pair3_sites[i] in sites_id_largedam ] idx_lst_largedam_in_conus = [ i for i in range(len(conus_sites)) if conus_sites[i] in sites_id_largedam ] print("multi box") inds_df_pair1 = load_ensemble_result(config_file, pair1_exps, test_epoch) inds_df_pair2 = load_ensemble_result(config_file, pair2_exps, test_epoch) inds_df_pair3 = load_ensemble_result(config_file, pair3_exps, test_epoch) inds_df_conus = load_ensemble_result(config_file, conus_exps, test_epoch) fig = plt.figure(figsize=(15, 8)) gs = gridspec.GridSpec(1, 3) keys_nse = "NSE" color_chosen = ["Greens", "Blues", "Reds"] median_loc = 0.015 decimal_places = 2 sns.despine() sns.set(font_scale=1.5) attr_nodam = "zero_dor" cases_exps_legends_nodam = [ "LSTM-Z", "LSTM-ZS", "LSTM-ZL", "LSTM-CONUS" ] frames_nodam = [] inds_df_nodam = load_ensemble_result(config_file, nodam_exp_lst, test_epoch) df_nodam_alone = pd.DataFrame({ attr_nodam: np.full([inds_df_nodam.shape[0]], cases_exps_legends_nodam[0]), keys_nse: inds_df_nodam[keys_nse] }) frames_nodam.append(df_nodam_alone) df_nodam_in_pair1 = pd.DataFrame({ attr_nodam: np.full([ inds_df_pair1[keys_nse].iloc[idx_lst_nodam_in_pair1].shape[0] ], cases_exps_legends_nodam[1]), keys_nse: inds_df_pair1[keys_nse].iloc[idx_lst_nodam_in_pair1] }) frames_nodam.append(df_nodam_in_pair1) df_nodam_in_pair2 = pd.DataFrame({ attr_nodam: np.full([ inds_df_pair2[keys_nse].iloc[idx_lst_nodam_in_pair2].shape[0] ], cases_exps_legends_nodam[2]), keys_nse: inds_df_pair2[keys_nse].iloc[idx_lst_nodam_in_pair2] }) frames_nodam.append(df_nodam_in_pair2) df_nodam_in_conus = pd.DataFrame({ attr_nodam: np.full([ inds_df_conus[keys_nse].iloc[idx_lst_nodam_in_conus].shape[0] ], cases_exps_legends_nodam[3]), keys_nse: inds_df_conus[keys_nse].iloc[idx_lst_nodam_in_conus] }) frames_nodam.append(df_nodam_in_conus) result_nodam = pd.concat(frames_nodam) ax1 = plt.subplot(gs[0]) # ax1.set_title("(a)") ax1.set_xticklabels(ax1.get_xticklabels(), rotation=30) ax1.set_ylim([0, 1]) sns.boxplot(ax=ax1, x=attr_nodam, y=keys_nse, data=result_nodam, showfliers=False, palette=color_chosen[0]) medians_nodam = result_nodam.groupby( [attr_nodam], sort=False)[keys_nse].median().values median_labels_nodam = [ str(np.round(s, decimal_places)) for s in medians_nodam ] pos1 = range(len(medians_nodam)) for tick, label in zip(pos1, ax1.get_xticklabels()): ax1.text(pos1[tick], medians_nodam[tick] + median_loc, median_labels_nodam[tick], horizontalalignment='center', size='x-small', weight='semibold') attr_smalldam = "small_dor" cases_exps_legends_smalldam = [ "LSTM-S", "LSTM-ZS", "LSTM-SL", "LSTM-CONUS" ] frames_smalldam = [] inds_df_smalldam = load_ensemble_result(config_file, smalldam_exp_lst, test_epoch) df_smalldam_alone = pd.DataFrame({ attr_smalldam: np.full([inds_df_smalldam.shape[0]], cases_exps_legends_smalldam[0]), keys_nse: inds_df_smalldam[keys_nse] }) frames_smalldam.append(df_smalldam_alone) df_smalldam_in_pair1 = pd.DataFrame({ attr_smalldam: np.full([ inds_df_pair1[keys_nse].iloc[idx_lst_smalldam_in_pair1]. shape[0] ], cases_exps_legends_smalldam[1]), keys_nse: inds_df_pair1[keys_nse].iloc[idx_lst_smalldam_in_pair1] }) frames_smalldam.append(df_smalldam_in_pair1) df_smalldam_in_pair3 = pd.DataFrame({ attr_smalldam: np.full([ inds_df_pair3[keys_nse].iloc[idx_lst_smalldam_in_pair3]. shape[0] ], cases_exps_legends_smalldam[2]), keys_nse: inds_df_pair3[keys_nse].iloc[idx_lst_smalldam_in_pair3] }) frames_smalldam.append(df_smalldam_in_pair3) df_smalldam_in_conus = pd.DataFrame({ attr_smalldam: np.full([ inds_df_conus[keys_nse].iloc[idx_lst_smalldam_in_conus]. shape[0] ], cases_exps_legends_smalldam[3]), keys_nse: inds_df_conus[keys_nse].iloc[idx_lst_smalldam_in_conus] }) frames_smalldam.append(df_smalldam_in_conus) result_smalldam = pd.concat(frames_smalldam) ax2 = plt.subplot(gs[1]) # ax2.set_title("(b)") ax2.set_xticklabels(ax2.get_xticklabels(), rotation=30) ax2.set_ylim([0, 1]) ax2.set(ylabel=None) sns.boxplot(ax=ax2, x=attr_smalldam, y=keys_nse, data=result_smalldam, showfliers=False, palette=color_chosen[1]) medians_smalldam = result_smalldam.groupby( [attr_smalldam], sort=False)[keys_nse].median().values median_labels_smalldam = [ str(np.round(s, decimal_places)) for s in medians_smalldam ] pos2 = range(len(medians_smalldam)) for tick, label in zip(pos2, ax2.get_xticklabels()): ax2.text(pos2[tick], medians_smalldam[tick] + median_loc, median_labels_smalldam[tick], horizontalalignment='center', size='x-small', weight='semibold') attr_largedam = "large_dor" cases_exps_legends_largedam = [ "LSTM-L", "LSTM-ZL", "LSTM-SL", "LSTM-CONUS" ] frames_largedam = [] inds_df_largedam = load_ensemble_result(config_file, largedam_exp_lst, test_epoch) df_largedam_alone = pd.DataFrame({ attr_largedam: np.full([inds_df_largedam.shape[0]], cases_exps_legends_largedam[0]), keys_nse: inds_df_largedam[keys_nse] }) frames_largedam.append(df_largedam_alone) df_largedam_in_pair2 = pd.DataFrame({ attr_largedam: np.full([ inds_df_pair2[keys_nse].iloc[idx_lst_largedam_in_pair2]. shape[0] ], cases_exps_legends_largedam[1]), keys_nse: inds_df_pair2[keys_nse].iloc[idx_lst_largedam_in_pair2] }) frames_largedam.append(df_largedam_in_pair2) df_largedam_in_pair3 = pd.DataFrame({ attr_largedam: np.full([ inds_df_pair3[keys_nse].iloc[idx_lst_largedam_in_pair3]. shape[0] ], cases_exps_legends_largedam[2]), keys_nse: inds_df_pair3[keys_nse].iloc[idx_lst_largedam_in_pair3] }) frames_largedam.append(df_largedam_in_pair3) df_largedam_in_conus = pd.DataFrame({ attr_largedam: np.full([ inds_df_conus[keys_nse].iloc[idx_lst_largedam_in_conus]. shape[0] ], cases_exps_legends_largedam[3]), keys_nse: inds_df_conus[keys_nse].iloc[idx_lst_largedam_in_conus] }) frames_largedam.append(df_largedam_in_conus) result_largedam = pd.concat(frames_largedam) ax3 = plt.subplot(gs[2]) # ax3.set_title("(c)") ax3.set_xticklabels(ax3.get_xticklabels(), rotation=30) ax3.set_ylim([0, 1]) ax3.set(ylabel=None) sns.boxplot(ax=ax3, x=attr_largedam, y=keys_nse, data=result_largedam, showfliers=False, palette=color_chosen[2]) medians_largedam = result_largedam.groupby( [attr_largedam], sort=False)[keys_nse].median().values median_labels_largedam = [ str(np.round(s, decimal_places)) for s in medians_largedam ] pos3 = range(len(medians_largedam)) for tick, label in zip(pos3, ax3.get_xticklabels()): ax3.text(pos3[tick], medians_largedam[tick] + median_loc, median_labels_largedam[tick], horizontalalignment='center', size='x-small', weight='semibold') # sns.despine() plt.tight_layout() plt.show()
def test_diff_dor_fig2_in_the_paper(self): data_model = GagesModel.load_datamodel( self.config_data.data_path["Temp"], data_source_file_name='data_source.txt', stat_file_name='Statistics.json', flow_file_name='flow.npy', forcing_file_name='forcing.npy', attr_file_name='attr.npy', f_dict_file_name='dictFactorize.json', var_dict_file_name='dictAttribute.json', t_s_dict_file_name='dictTimeSpace.json') config_data = self.config_data config_file = self.config_file test_epoch = self.test_epoch exp_lst = self.exp_lst figure_dpi = self.FIGURE_DPI inds_df, pred_mean, obs_mean = load_ensemble_result(config_file, exp_lst, test_epoch, return_value=True) diversion_yes = True diversion_no = False source_data_diversion = GagesSource.choose_some_basins( config_data, config_data.model_dict["data"]["tRangeTrain"], screen_basin_area_huc4=False, diversion=diversion_yes) source_data_nodivert = GagesSource.choose_some_basins( config_data, config_data.model_dict["data"]["tRangeTrain"], screen_basin_area_huc4=False, diversion=diversion_no) sites_id_nodivert = source_data_nodivert.all_configs[ 'flow_screen_gage_id'] sites_id_diversion = source_data_diversion.all_configs[ 'flow_screen_gage_id'] dor_1 = -self.dor dor_2 = self.dor source_data_dor1 = GagesSource.choose_some_basins( config_data, config_data.model_dict["data"]["tRangeTrain"], screen_basin_area_huc4=False, DOR=dor_1) source_data_dor2 = GagesSource.choose_some_basins( config_data, config_data.model_dict["data"]["tRangeTrain"], screen_basin_area_huc4=False, DOR=dor_2) sites_id_dor1 = source_data_dor1.all_configs['flow_screen_gage_id'] sites_id_dor2 = source_data_dor2.all_configs['flow_screen_gage_id'] # basins with dams source_data_withdams = GagesSource.choose_some_basins( config_data, config_data.model_dict["data"]["tRangeTrain"], screen_basin_area_huc4=False, dam_num=[1, 100000]) sites_id_withdams = source_data_withdams.all_configs[ 'flow_screen_gage_id'] sites_id_dor1 = np.intersect1d(np.array(sites_id_dor1), np.array(sites_id_withdams)).tolist() no_divert_small_dor = np.intersect1d(sites_id_nodivert, sites_id_dor1) no_divert_large_dor = np.intersect1d(sites_id_nodivert, sites_id_dor2) diversion_small_dor = np.intersect1d(sites_id_diversion, sites_id_dor1) diversion_large_dor = np.intersect1d(sites_id_diversion, sites_id_dor2) all_sites = data_model.t_s_dict["sites_id"] idx_lst_nodivert_smalldor = [ i for i in range(len(all_sites)) if all_sites[i] in no_divert_small_dor ] idx_lst_nodivert_largedor = [ i for i in range(len(all_sites)) if all_sites[i] in no_divert_large_dor ] idx_lst_diversion_smalldor = [ i for i in range(len(all_sites)) if all_sites[i] in diversion_small_dor ] idx_lst_diversion_largedor = [ i for i in range(len(all_sites)) if all_sites[i] in diversion_large_dor ] keys_nse = "NSE" xs = [] ys = [] cases_exps_legends_together = [ "not_diverted_small_dor", "not_diverted_large_dor", "diversion_small_dor", "diversion_large_dor", "CONUS" ] x1, y1 = ecdf(inds_df[keys_nse].iloc[idx_lst_nodivert_smalldor]) xs.append(x1) ys.append(y1) x2, y2 = ecdf(inds_df[keys_nse].iloc[idx_lst_nodivert_largedor]) xs.append(x2) ys.append(y2) x3, y3 = ecdf(inds_df[keys_nse].iloc[idx_lst_diversion_smalldor]) xs.append(x3) ys.append(y3) x4, y4 = ecdf(inds_df[keys_nse].iloc[idx_lst_diversion_largedor]) xs.append(x4) ys.append(y4) x_conus, y_conus = ecdf(inds_df[keys_nse]) xs.append(x_conus) ys.append(y_conus) hydro_logger.info( "The median NSEs of all five curves (%s) are \n %.2f, %.2f, %.2f, %.2f, %.2f", cases_exps_legends_together, np.median(x1), np.median(x2), np.median(x3), np.median(x4), np.median(x_conus)) # plot_ecdfs_matplot(xs, ys, cases_exps_legends_together, # colors=["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "grey"], # dash_lines=[False, False, False, False, True], x_str="NSE", y_str="CDF") # plot using two linestyles and two colors for dor and diversion. # plot_ecdfs(xs, ys, cases_exps_legends_together, x_str="NSE", y_str="CDF") # define color scheme and line style colors = ["#1f77b4", "#d62728"] linestyles = ['-', "--"] markers = ["", "."] fig = plt.figure(figsize=(8, 6)) axes = fig.add_axes([0.1, 0.1, 0.8, 0.8]) # for i, marker in enumerate(markers): for i, linestyle in enumerate(linestyles): for j, color in enumerate(colors): plt.plot( xs[i * 2 + j], ys[i * 2 + j], color=color, ls=linestyle, # marker=marker, label=cases_exps_legends_together[i * 2 + j]) line_i, = axes.plot(x_conus, y_conus, color="grey", label=cases_exps_legends_together[4]) line_i.set_dashes([2, 2, 10, 2]) x_str = "NSE" y_str = "CDF" x_lim = (0, 1) y_lim = (0, 1) x_interval = 0.1 y_interval = 0.1 plt.xlabel(x_str, fontsize=18) plt.ylabel(y_str, fontsize=18) axes.set_xlim(x_lim[0], x_lim[1]) axes.set_ylim(y_lim[0], y_lim[1]) # set x y number font size plt.xticks(np.arange(x_lim[0], x_lim[1] + x_lim[1] / 100, x_interval), fontsize=16) plt.yticks(np.arange(y_lim[0], y_lim[1] + y_lim[1] / 100, y_interval), fontsize=16) plt.grid() # Hide the right and top spines axes.spines['right'].set_visible(False) axes.spines['top'].set_visible(False) axes.legend() plt.legend(prop={'size': 16}) plt.savefig(os.path.join(config_data.data_path["Out"], 'new_dor_divert_comp_matplotlib.png'), dpi=figure_dpi, bbox_inches="tight") plt.show()
def test_gages_nse_dam_attr(self): figure_dpi = 600 config_data = self.config_data data_dir = config_data.data_path["Temp"] data_model = GagesModel.load_datamodel( data_dir, data_source_file_name='test_data_source.txt', stat_file_name='test_Statistics.json', flow_file_name='test_flow.npy', forcing_file_name='test_forcing.npy', attr_file_name='test_attr.npy', f_dict_file_name='test_dictFactorize.json', var_dict_file_name='test_dictAttribute.json', t_s_dict_file_name='test_dictTimeSpace.json') gages_id = data_model.t_s_dict["sites_id"] exp_lst = [ "basic_exp37", "basic_exp39", "basic_exp40", "basic_exp41", "basic_exp42", "basic_exp43" ] self.inds_df, pred_mean, obs_mean = load_ensemble_result( config_data.config_file, exp_lst, config_data.config_file.TEST_EPOCH, return_value=True) show_ind_key = 'NSE' plt.rcParams['font.family'] = 'serif' plt.rcParams['font.serif'] = ['Times New Roman' ] + plt.rcParams['font.serif'] # plot NSE-DOR attr_lst = ["RUNAVE7100", "STOR_NOR_2009"] attrs_runavg_stor = data_model.data_source.read_attr( gages_id, attr_lst, is_return_dict=False) run_avg = attrs_runavg_stor[:, 0] * (10**(-3)) * (10**6 ) # m^3 per year nor_storage = attrs_runavg_stor[:, 1] * 1000 # m^3 dors = nor_storage / run_avg # dor = 0 is not totally same with dam_num=0 (some dammed basins' dor is about 0.00), # here for zero-dor we mainly rely on dam_num = 0 attr_dam_num = ["NDAMS_2009"] attrs_dam_num = data_model.data_source.read_attr(gages_id, attr_dam_num, is_return_dict=False) df = pd.DataFrame({ "DOR": dors, "DAM_NUM": attrs_dam_num[:, 0], show_ind_key: self.inds_df[show_ind_key].values }) hydro_logger.info("statistics of dors:\n %s", df.describe()) hydro_logger.info("percentiles of dors:\n %s", df.quantile(q=0.95)) hydro_logger.info("ecdf of dors:\n %s", ecdf(dors)) # boxplot # add a column to represent the dor range for the df dor_value_range_lst = [[0, 0], [0, 0.02], [0.02, 0.05], [0.05, 0.1], [0.1, 0.2], [0.2, 0.4], [0.4, 0.8], [0.8, 10000]] dor_range_lst = ["0"] + [ str(dor_value_range_lst[i][0]) + "-" + str(dor_value_range_lst[i][1]) for i in range(1, len(dor_value_range_lst) - 1) ] + [">" + str(dor_value_range_lst[-1][0])] # add a column to represent the dam_num range for the df dam_num_value_range_lst = [[0, 0], [0, 1], [1, 3], [3, 5], [5, 10], [10, 20], [20, 50], [50, 10000]] dam_num_range_lst = ["0", "1"] + [ str(dam_num_value_range_lst[i][0]) + "-" + str(dam_num_value_range_lst[i][1]) for i in range(2, len(dam_num_value_range_lst) - 1) ] + [">" + str(dam_num_value_range_lst[-1][0])] def in_which_range(value_temp): if value_temp == 0: return "0" the_range = [ a_range for a_range in dor_value_range_lst if a_range[0] < value_temp <= a_range[1] ] if the_range[0][0] == dor_value_range_lst[-1][0]: the_range_str = ">" + str(the_range[0][0]) else: the_range_str = str(the_range[0][0]) + "-" + str( the_range[0][1]) return the_range_str def in_which_dam_num_range(value_tmp): if value_tmp == 0: return "0" if value_tmp == 1: return "1" the_ran = [ a_ran for a_ran in dam_num_value_range_lst if a_ran[0] < value_tmp <= a_ran[1] ] if the_ran[0][0] == dam_num_value_range_lst[-1][0]: the_ran_str = ">" + str(the_ran[0][0]) else: the_ran_str = str(the_ran[0][0]) + "-" + str(the_ran[0][1]) return the_ran_str df["DOR_RANGE"] = df["DOR"].apply(in_which_range) df["DAM_NUM_RANGE"] = df["DAM_NUM"].apply(in_which_dam_num_range) df.loc[(df["DAM_NUM"] > 0) & (df["DOR_RANGE"] == "0"), "DOR_RANGE"] = dor_range_lst[1] shown_nse_range_boxplots = [-0.5, 1.0] sns.set(font="serif", font_scale=1.5, color_codes=True) plot_boxs(df, "DOR_RANGE", show_ind_key, ylim=shown_nse_range_boxplots, order=dor_range_lst) plt.savefig(os.path.join( config_data.data_path["Out"], 'NSE~DOR-boxplots-' + str(shown_nse_range_boxplots) + '.png'), dpi=figure_dpi, bbox_inches="tight") plt.figure() shown_nse_range_boxplots = [0, 1.0] sns.set(font="serif", font_scale=1.5, color_codes=True) plot_boxs(df, "DAM_NUM_RANGE", show_ind_key, ylim=shown_nse_range_boxplots, order=dam_num_range_lst) plt.savefig(os.path.join( config_data.data_path["Out"], 'NSE~DAM_NUM-boxplots-' + str(shown_nse_range_boxplots) + '.png'), dpi=figure_dpi, bbox_inches="tight") nums_in_dor_range = [ df[df["DOR_RANGE"] == a_range_rmp].shape[0] for a_range_rmp in dor_range_lst ] ratios_in_dor_range = [ a_num / df.shape[0] for a_num in nums_in_dor_range ] hydro_logger.info( "the number and ratio of basins in each dor range\n: %s \n %s", nums_in_dor_range, ratios_in_dor_range) nums_in_dam_num_range = [ df[df["DAM_NUM_RANGE"] == a_range_rmp].shape[0] for a_range_rmp in dam_num_range_lst ] ratios_in_dam_num_range = [ a_num / df.shape[0] for a_num in nums_in_dam_num_range ] hydro_logger.info( "the number and ratio of basins in each dam_num range\n: %s \n %s", nums_in_dam_num_range, ratios_in_dam_num_range) # regplot plt.figure() sns.set(font="serif", font_scale=1.5, color_codes=True) sr = sns.regplot(x="DOR", y=show_ind_key, data=df[df[show_ind_key] >= 0], scatter_kws={'s': 10}) show_dor_max = df.quantile( q=0.95)["DOR"] # 30 # max(dors) # 0.8 # 10 show_dor_min = min(dors) plt.ylim(0, 1) plt.xlim(show_dor_min, show_dor_max) plt.savefig(os.path.join( config_data.data_path["Out"], 'NSE~DOR-shown-max-' + str(show_dor_max) + '.png'), dpi=figure_dpi, bbox_inches="tight") # jointplot # dor_range = [0.2, 0.9] dor_range = [0.002, 0.2] # plt.figure() sns.set(font="serif", font_scale=1.5, color_codes=True) # g = sns.jointplot(x="DOR", y=show_ind_key, data=df[(df["DOR"] < 1) & (df[show_ind_key] >= 0)], kind="reg", # marginal_kws=dict(bins=25)) # g = sns.jointplot(x="DOR", y=show_ind_key, data=df[(df["DOR"] < 1) & (df[show_ind_key] >= 0)], kind="hex", # color="b", marginal_kws=dict(bins=50)) g = sns.jointplot( x="DOR", y=show_ind_key, data=df[(df["DOR"] < dor_range[1]) & (df["DOR"] > dor_range[0]) & (df[show_ind_key] >= 0)], kind="hex", color="b") g.ax_marg_x.set_xlim(dor_range[0], dor_range[1]) # g.ax_marg_y.set_ylim(-0.5, 1) plt.savefig(os.path.join( config_data.data_path["Out"], 'NSE~DOR(range-)' + str(dor_range) + '-jointplot.png'), dpi=figure_dpi, bbox_inches="tight") nid_dir = os.path.join( "/".join(self.config_data.data_path["DB"].split("/")[:-1]), "nid", "test") nid_input = NidModel.load_nidmodel( nid_dir, nid_source_file_name='nid_source.txt', nid_data_file_name='nid_data.shp') gage_main_dam_purpose = unserialize_json( os.path.join(nid_dir, "dam_main_purpose_dict.json")) data_input = GagesDamDataModel(data_model, nid_input, gage_main_dam_purpose) dam_coords = unserialize_json_ordered( os.path.join(nid_dir, "dam_points_dict.json")) dam_storages = unserialize_json_ordered( os.path.join(nid_dir, "dam_storages_dict.json")) dam_ids_1 = list(gage_main_dam_purpose.keys()) dam_ids_2 = list(dam_coords.keys()) dam_ids_3 = list(dam_storages.keys()) assert (all(x < y for x, y in zip(dam_ids_1, dam_ids_1[1:]))) assert (all(x < y for x, y in zip(dam_ids_2, dam_ids_2[1:]))) assert (all(x < y for x, y in zip(dam_ids_3, dam_ids_3[1:]))) sites = list(dam_coords.keys()) c, ind1, idx_lst_nse_range = np.intersect1d(sites, gages_id, return_indices=True) std_storage_in_a_basin = list(map(np.std, dam_storages.values())) log_std_storage_in_a_basin = list( map(np.log, np.array(std_storage_in_a_basin) + 1)) nse_values = self.inds_df["NSE"].values[idx_lst_nse_range] df = pd.DataFrame({ "DAM_STORAGE_STD": log_std_storage_in_a_basin, show_ind_key: nse_values }) plt.figure() sns.set(font="serif", font_scale=1.5, color_codes=True) g = sns.regplot(x="DAM_STORAGE_STD", y=show_ind_key, data=df[df[show_ind_key] >= 0], scatter_kws={'s': 10}) show_max = max(log_std_storage_in_a_basin) show_min = min(log_std_storage_in_a_basin) if show_min < 0: show_min = 0 # g.ax_marg_x.set_xlim(show_min, show_max) # g.ax_marg_y.set_ylim(0, 1) plt.ylim(0, 1) plt.xlim(show_min, show_max) plt.savefig(os.path.join(config_data.data_path["Out"], 'NSE~' + "DAM_STORAGE_STD" + '.png'), dpi=figure_dpi, bbox_inches="tight") gages_loc_lat = data_model.data_source.gage_dict["LAT_GAGE"] gages_loc_lon = data_model.data_source.gage_dict["LNG_GAGE"] gages_loc = [[gages_loc_lat[i], gages_loc_lon[i]] for i in range(len(gages_id))] # calculate index of dispersion, then plot the NSE-dispersion scatterplot # Geo coord system of gages_loc and dam_coords are both NAD83 coefficient_of_var = list( map(coefficient_of_variation, gages_loc, dam_coords.values())) coefficient_of_var_min = min(coefficient_of_var) coefficient_of_var_max = max(coefficient_of_var) dispersion_var = "DAM_GAGE_DIS_VAR" nse_values = self.inds_df["NSE"].values[idx_lst_nse_range] df = pd.DataFrame({ dispersion_var: coefficient_of_var, show_ind_key: nse_values }) plt.figure() sns.set(font="serif", font_scale=1.5, color_codes=True) g = sns.regplot(x=dispersion_var, y=show_ind_key, data=df[df[show_ind_key] >= 0], scatter_kws={'s': 10}) show_max = coefficient_of_var_max show_min = coefficient_of_var_min if show_min < 0: show_min = 0 # g.ax_marg_x.set_xlim(show_min, show_max) # g.ax_marg_y.set_ylim(0, 1) plt.ylim(0, 1) plt.xlim(show_min, show_max) plt.savefig(os.path.join(config_data.data_path["Out"], 'NSE~' + dispersion_var + '.png'), dpi=figure_dpi, bbox_inches="tight") idx_dispersions = list( map(ind_of_dispersion, gages_loc, dam_coords.values())) idx_dispersion_min = min(idx_dispersions) idx_dispersion_max = max(idx_dispersions) dispersion_var = "DAM_DISPERSION_BASIN" # nse_range = [0, 1] # idx_lst_nse_range = inds_df_now[(inds_df_now[show_ind_key] >= nse_range[0]) & (inds_df_now[show_ind_key] < nse_range[1])].index.tolist() nse_values = self.inds_df["NSE"].values[idx_lst_nse_range] df = pd.DataFrame({ dispersion_var: idx_dispersions, show_ind_key: nse_values }) # g = sns.regplot(x=dispersion_var, y=show_ind_key, data=df[df[show_ind_key] >= 0], scatter_kws={'s': 10}) if idx_dispersion_min < 0: idx_dispersion_min = 0 plt.ylim(0, 1) plt.xlim(idx_dispersion_min, idx_dispersion_max) # plt.figure() sns.set(font="serif", font_scale=1.5, color_codes=True) g = sns.jointplot(x=dispersion_var, y=show_ind_key, data=df[df[show_ind_key] >= 0], kind="reg") g.ax_marg_x.set_xlim(idx_dispersion_min, idx_dispersion_max) g.ax_marg_y.set_ylim(0, 1) plt.show()
var_dict_file_name='test_dictAttribute.json', t_s_dict_file_name='test_dictTimeSpace.json') gages_model_train = GagesModel.update_data_model( config_data, data_model_train, data_attr_update=True, screen_basin_area_huc4=False) data_model = GagesModel.update_data_model( config_data, data_model_test, data_attr_update=True, train_stat_dict=gages_model_train.stat_dict, screen_basin_area_huc4=False) inds_df, pred_mean, obs_mean = load_ensemble_result(cfg, exp_lst, test_epoch, return_value=True) plt.rcParams['font.family'] = 'serif' plt.rcParams['font.serif'] = ['Times New Roman' ] + plt.rcParams['font.serif'] ########################### plot diversion dor ecdf ########################### diversion_yes = True diversion_no = False source_data_diversion = GagesSource.choose_some_basins( config_data, config_data.model_dict["data"]["tRangeTrain"], screen_basin_area_huc4=False, diversion=diversion_yes) source_data_nodivert = GagesSource.choose_some_basins( config_data, config_data.model_dict["data"]["tRangeTrain"],
paper_dor_tmp = (capacity_reservoir * 1000000 / (watershed_area * 1000000)) * 1000 / mean_runoff paper_dors.append(paper_dor_tmp) paper_dors = np.array(paper_dors) hydro_logger.info("The dor values of those basins: %s", paper_dors) hydro_logger.info("Are dor values of those basins bigger than 0.02: %s", paper_dors > 0.02) hydro_logger.info("Are dor values of those basins bigger than 0.1: %s", paper_dors > 0.1) hydro_logger.info("Are dor values of those basins bigger than 1: %s", paper_dors > 1) test_epoch = 300 all_exps_lst = ["basic_exp39", "basic_exp37", "basic_exp40", "basic_exp41", "basic_exp42", "basic_exp43", "basic_exp32", "basic_exp31", "basic_exp33", "basic_exp34", "basic_exp35", "basic_exp36"] all_exps_random_seeds = ["123", "1234", "12345", "111", "1111", "11111", "123", "1234", "12345", "111", "1111", "11111"] idx_tmp_now = 0 for exp_tmp in all_exps_lst: exp_tmp_lst = [exp_tmp] inds_df_tmp = load_ensemble_result(cfg, exp_tmp_lst, test_epoch) hydro_logger.info("The median NSE value of %s is %.2f (random seed: %s)", exp_tmp, inds_df_tmp.median()["NSE"], all_exps_random_seeds[idx_tmp_now]) idx_tmp_now = idx_tmp_now + 1 exp_lst = ["basic_exp37"] config_data = load_dataconfig_case_exp(cfg, exp_lst[0]) data_model = GagesModel.load_datamodel(config_data.data_path["Temp"], data_source_file_name='test_data_source.txt', stat_file_name='test_Statistics.json', flow_file_name='test_flow.npy', forcing_file_name='test_forcing.npy', attr_file_name='test_attr.npy', f_dict_file_name='test_dictFactorize.json', var_dict_file_name='test_dictAttribute.json', t_s_dict_file_name='test_dictTimeSpace.json') camels_gageid_file = os.path.join(config_data.data_path["DB"], "camels_attributes_v2.0", "camels_attributes_v2.0", "camels_name.txt")