def plot_we_need(data_model_test,
obs,
pred,
show_me_num=5,
point_file=None,
**kwargs):
pred = pred.reshape(pred.shape[0], pred.shape[1])
obs = obs.reshape(pred.shape[0], pred.shape[1])
inds = statError(obs, pred)
# plot box
keys = ["Bias", "RMSE", "NSE"]
inds_test = subset_of_dict(inds, keys)
box_fig = plot_diff_boxes(inds_test)
box_fig.savefig(
os.path.join(data_model_test.data_source.data_config.data_path["Out"],
"box_fig.png"))
# plot ts
t_s_dict = data_model_test.t_s_dict
sites = np.array(t_s_dict["sites_id"])
t_range = np.array(t_s_dict["t_final_range"])
ts_fig = plot_ts_obs_pred(obs, pred, sites, t_range, show_me_num)
ts_fig.savefig(
os.path.join(data_model_test.data_source.data_config.data_path["Out"],
"ts_fig.png"))
# plot nse ecdf
sites_df_nse = pd.DataFrame({"sites": sites, keys[2]: inds_test[keys[2]]})
plot_ecdf(sites_df_nse, keys[2])
# plot map
if point_file is None:
gauge_dict = data_model_test.data_source.gage_dict
plot_map(gauge_dict, sites_df_nse, **kwargs)
else:
plot_ind_map(point_file, sites_df_nse, percentile=25)
def test_inv_plot(self):
data_model = GagesModel.load_datamodel(
self.config_data_2.data_path["Temp"],
"2",
data_source_file_name='test_data_source.txt',
stat_file_name='test_Statistics.json',
flow_file_name='test_flow.npy',
forcing_file_name='test_forcing.npy',
attr_file_name='test_attr.npy',
f_dict_file_name='test_dictFactorize.json',
var_dict_file_name='test_dictAttribute.json',
t_s_dict_file_name='test_dictTimeSpace.json')
flow_pred_file = os.path.join(
data_model.data_source.data_config.data_path['Temp'],
'epoch' + str(self.test_epoch) + 'flow_pred.npy')
flow_obs_file = os.path.join(
data_model.data_source.data_config.data_path['Temp'],
'epoch' + str(self.test_epoch) + 'flow_obs.npy')
pred = unserialize_numpy(flow_pred_file)
obs = unserialize_numpy(flow_obs_file)
pred = pred.reshape(pred.shape[0], pred.shape[1])
obs = obs.reshape(obs.shape[0], obs.shape[1])
inds = statError(obs, pred)
inds['STAID'] = data_model.t_s_dict["sites_id"]
inds_df = pd.DataFrame(inds)
inds_df.to_csv(
os.path.join(self.config_data_2.data_path["Out"], 'data_df.csv'))
# plot box，使用seaborn库
keys = ["Bias", "RMSE", "NSE"]
inds_test = subset_of_dict(inds, keys)
box_fig = plot_diff_boxes(inds_test)
box_fig.savefig(
os.path.join(self.config_data_2.data_path["Out"], "box_fig.png"))
# plot ts
show_me_num = 5
t_s_dict = data_model.t_s_dict
sites = np.array(t_s_dict["sites_id"])
t_range = np.array(t_s_dict["t_final_range"])
time_seq_length = self.config_data_1.model_dict['model']['seqLength']
time_start = np.datetime64(t_range[0]) + np.timedelta64(
time_seq_length - 1, 'D')
t_range[0] = np.datetime_as_string(time_start, unit='D')
ts_fig = plot_ts_obs_pred(obs, pred, sites, t_range, show_me_num)
ts_fig.savefig(
os.path.join(self.config_data_2.data_path["Out"], "ts_fig.png"))
# plot nse ecdf
sites_df_nse = pd.DataFrame({
"sites": sites,
keys[2]: inds_test[keys[2]]
})
plot_ecdf(sites_df_nse, keys[2])
# plot map
gauge_dict = data_model.data_source.gage_dict
plot_map(gauge_dict,
sites_df_nse,
id_col="STAID",
lon_col="LNG_GAGE",
lat_col="LAT_GAGE")
def test_plot_pretrained_model_test(self):
data_model_test = GagesModel.load_datamodel(
self.majordam_config_data.data_path["Temp"],
data_source_file_name='test_data_source.txt',
stat_file_name='test_Statistics.json',
flow_file_name='test_flow.npy',
forcing_file_name='test_forcing.npy',
attr_file_name='test_attr.npy',
f_dict_file_name='test_dictFactorize.json',
var_dict_file_name='test_dictAttribute.json',
t_s_dict_file_name='test_dictTimeSpace.json')
pretrained_model_name = "nodam" + self.nomajordam_subdir.split(
"/")[1] + "_pretrained_model"
save_dir = os.path.join(
data_model_test.data_source.data_config.data_path['Out'],
pretrained_model_name)
pred, obs = load_result(save_dir, self.test_epoch)
pred = pred.reshape(pred.shape[0], pred.shape[1])
obs = obs.reshape(pred.shape[0], pred.shape[1])
inds = statError(obs, pred)
# plot box，使用seaborn库
keys = ["Bias", "RMSE", "NSE"]
inds_test = subset_of_dict(inds, keys)
box_fig = plot_diff_boxes(inds_test)
box_fig.savefig(os.path.join(save_dir, "box_fig.png"))
# plot ts
show_me_num = 5
t_s_dict = data_model_test.t_s_dict
sites = np.array(t_s_dict["sites_id"])
t_range = np.array(t_s_dict["t_final_range"])
ts_fig = plot_ts_obs_pred(obs, pred, sites, t_range, show_me_num)
ts_fig.savefig(os.path.join(save_dir, "ts_fig.png"))
# plot nse ecdf
sites_df_nse = pd.DataFrame({
"sites": sites,
keys[2]: inds_test[keys[2]]
})
plot_ecdf(sites_df_nse, keys[2])
# plot map
gauge_dict = data_model_test.data_source.gage_dict
plot_map(gauge_dict,
sites_df_nse,
id_col="STAID",
lon_col="LNG_GAGE",
lat_col="LAT_GAGE")
def plot_region_seperately(gages_data_model, epoch, id_regions_idx, preds,
obss, inds_dfs):
df_id_region = np.array(gages_data_model.t_s_dict["sites_id"])
regions_name = gages_data_model.data_source.all_configs.get("regions")
for i in range(len(id_regions_idx)):
# plot box
keys = ["Bias", "RMSE", "NSE"]
inds_test = subset_of_dict(inds_dfs[i], keys)
box_fig = plot_diff_boxes(inds_test)
box_fig.savefig(
os.path.join(
gages_data_model.data_source.data_config.data_path["Out"],
regions_name[i] + "epoch" + str(epoch) + "box_fig.png"))
# plot ts
sites = np.array(df_id_region[id_regions_idx[i]])
t_range = np.array(gages_data_model.t_s_dict["t_final_range"])
show_me_num = 5
ts_fig = plot_ts_obs_pred(obss[i], preds[i], sites, t_range,
show_me_num)
ts_fig.savefig(
os.path.join(
gages_data_model.data_source.data_config.data_path["Out"],
regions_name[i] + "epoch" + str(epoch) + "ts_fig.png"))
# plot nse ecdf
sites_df_nse = pd.DataFrame({
"sites": sites,
keys[2]: inds_test[keys[2]]
})
plot_ecdf(
sites_df_nse, keys[2],
os.path.join(
gages_data_model.data_source.data_config.data_path["Out"],
regions_name[i] + "epoch" + str(epoch) + "ecdf_fig.png"))
# plot map
gauge_dict = gages_data_model.data_source.gage_dict
save_map_file = os.path.join(
gages_data_model.data_source.data_config.data_path["Out"],
regions_name[i] + "epoch" + str(epoch) + "map_fig.png")
plot_map(gauge_dict,
sites_df_nse,
save_file=save_map_file,
id_col="STAID",
lon_col="LNG_GAGE",
lat_col="LAT_GAGE")
def test_plot_pretrained_model_test(self):
data_model_test = GagesModel.load_datamodel(
self.config_data.data_path["Temp"],
data_source_file_name='test_data_source.txt',
stat_file_name='test_Statistics.json',
flow_file_name='test_flow.npy',
forcing_file_name='test_forcing.npy',
attr_file_name='test_attr.npy',
f_dict_file_name='test_dictFactorize.json',
var_dict_file_name='test_dictAttribute.json',
t_s_dict_file_name='test_dictTimeSpace.json')
save_dir = data_model_test.data_source.data_config.data_path['Temp']
pred, obs = load_result(save_dir,
self.config_data.config_file.TEST_EPOCH)
pred = pred.reshape(pred.shape[0], pred.shape[1])
obs = obs.reshape(pred.shape[0], pred.shape[1])
inds = statError(obs, pred)
keys = ["Bias", "RMSE", "NSE"]
t_s_dict = data_model_test.t_s_dict
sites = np.array(t_s_dict["sites_id"])
t_range = np.array(t_s_dict["t_final_range"])
# plot nse ecdf
sites_df_nse = pd.DataFrame({"sites": sites, keys[2]: inds[keys[2]]})
plot_ecdf(sites_df_nse, keys[2])
# plot map
gauge_dict = data_model_test.data_source.gage_dict
plot_map(gauge_dict,
sites_df_nse,
id_col="STAID",
lon_col="LNG_GAGE",
lat_col="LAT_GAGE")
# plot box，使用seaborn库
inds_test = subset_of_dict(inds, keys)
box_fig = plot_diff_boxes(inds_test)
# plot ts
show_me_num = 5
ts_fig = plot_ts_obs_pred(obs, pred, sites, t_range, show_me_num)
plt.show()
def test_plot_ecdf(self):
plot_ecdf(self.inds_test, self.keys[0])
def test_purposes_seperate(self):
quick_data_dir = os.path.join(self.config_data.data_path["DB"],
"quickdata")
data_dir = os.path.join(quick_data_dir,
"allnonref-dam_95-05_nan-0.1_00-1.0")
data_model_test = GagesModel.load_datamodel(
data_dir,
data_source_file_name='test_data_source.txt',
stat_file_name='test_Statistics.json',
flow_file_name='test_flow.npy',
forcing_file_name='test_forcing.npy',
attr_file_name='test_attr.npy',
f_dict_file_name='test_dictFactorize.json',
var_dict_file_name='test_dictAttribute.json',
t_s_dict_file_name='test_dictTimeSpace.json')
data_model = GagesModel.update_data_model(self.config_data,
data_model_test)
nid_dir = os.path.join(
"/".join(self.config_data.data_path["DB"].split("/")[:-1]), "nid",
"quickdata")
gage_main_dam_purpose = unserialize_json(
os.path.join(nid_dir, "dam_main_purpose_dict.json"))
gage_main_dam_purpose_lst = list(gage_main_dam_purpose.values())
gage_main_dam_purpose_unique = np.unique(gage_main_dam_purpose_lst)
purpose_regions = {}
for i in range(gage_main_dam_purpose_unique.size):
sites_id = []
for key, value in gage_main_dam_purpose.items():
if value == gage_main_dam_purpose_unique[i]:
sites_id.append(key)
assert (all(x < y for x, y in zip(sites_id, sites_id[1:])))
purpose_regions[gage_main_dam_purpose_unique[i]] = sites_id
id_regions_idx = []
id_regions_sites_ids = []
df_id_region = np.array(data_model.t_s_dict["sites_id"])
for key, value in purpose_regions.items():
gages_id = value
c, ind1, ind2 = np.intersect1d(df_id_region,
gages_id,
return_indices=True)
assert (all(x < y for x, y in zip(ind1, ind1[1:])))
assert (all(x < y for x, y in zip(c, c[1:])))
id_regions_idx.append(ind1)
id_regions_sites_ids.append(c)
pred_all, obs_all = load_result(self.config_data.data_path["Temp"],
self.test_epoch)
pred_all = pred_all.reshape(pred_all.shape[0], pred_all.shape[1])
obs_all = obs_all.reshape(obs_all.shape[0], obs_all.shape[1])
for i in range(9, len(gage_main_dam_purpose_unique)):
pred = pred_all[id_regions_idx[i], :]
obs = obs_all[id_regions_idx[i], :]
inds = statError(obs, pred)
inds['STAID'] = id_regions_sites_ids[i]
inds_df = pd.DataFrame(inds)
inds_df.to_csv(
os.path.join(
self.config_data.data_path["Out"],
gage_main_dam_purpose_unique[i] + "epoch" +
str(self.test_epoch) + 'data_df.csv'))
# plot box，使用seaborn库
keys = ["Bias", "RMSE", "NSE"]
inds_test = subset_of_dict(inds, keys)
box_fig = plot_diff_boxes(inds_test)
box_fig.savefig(
os.path.join(
self.config_data.data_path["Out"],
gage_main_dam_purpose_unique[i] + "epoch" +
str(self.test_epoch) + "box_fig.png"))
# plot ts
sites = np.array(df_id_region[id_regions_idx[i]])
t_range = np.array(data_model.t_s_dict["t_final_range"])
show_me_num = 1
ts_fig = plot_ts_obs_pred(obs, pred, sites, t_range, show_me_num)
ts_fig.savefig(
os.path.join(
self.config_data.data_path["Out"],
gage_main_dam_purpose_unique[i] + "epoch" +
str(self.test_epoch) + "ts_fig.png"))
# plot nse ecdf
sites_df_nse = pd.DataFrame({
"sites": sites,
keys[2]: inds_test[keys[2]]
})
plot_ecdf(
sites_df_nse, keys[2],
os.path.join(
self.config_data.data_path["Out"],
gage_main_dam_purpose_unique[i] + "epoch" +
str(self.test_epoch) + "ecdf_fig.png"))
# plot map
gauge_dict = data_model.data_source.gage_dict
save_map_file = os.path.join(
self.config_data.data_path["Out"],
gage_main_dam_purpose_unique[i] + "epoch" +
str(self.test_epoch) + "map_fig.png")
plot_map(gauge_dict,
sites_df_nse,
save_file=save_map_file,
id_col="STAID",
lon_col="LNG_GAGE",
lat_col="LAT_GAGE")
