def test_test_gages_iter(self): data_model = GagesModel.load_datamodel(self.config_data.data_path["Temp"], data_source_file_name='test_data_source.txt', stat_file_name='test_Statistics.json', flow_file_name='test_flow.npy', forcing_file_name='test_forcing.npy', attr_file_name='test_attr.npy', f_dict_file_name='test_dictFactorize.json', var_dict_file_name='test_dictAttribute.json', t_s_dict_file_name='test_dictTimeSpace.json') with torch.cuda.device(1): obs_lst = [] pred_lst = [] for i in range(0, data_model.data_flow.shape[0]): print("\n", "Testing model", str(i + 1), ":\n") data_models_i = GagesModel.which_data_model(data_model, i) pred, obs = master_test_1by1(data_models_i) basin_area = data_models_i.data_source.read_attr(data_models_i.t_s_dict["sites_id"], ['DRAIN_SQKM'], is_return_dict=False) mean_prep = data_models_i.data_source.read_attr(data_models_i.t_s_dict["sites_id"], ['PPTAVG_BASIN'], is_return_dict=False) mean_prep = mean_prep / 365 * 10 pred = _basin_norm(pred, basin_area, mean_prep, to_norm=False) obs = _basin_norm(obs, basin_area, mean_prep, to_norm=False) obs_lst.append(obs.flatten()) pred_lst.append(pred.flatten()) preds = np.array(pred_lst) obss = np.array(obs_lst) flow_pred_file = os.path.join(data_model.data_source.data_config.data_path['Temp'], 'flow_pred') flow_obs_file = os.path.join(data_model.data_source.data_config.data_path['Temp'], 'flow_obs') serialize_numpy(preds, flow_pred_file) serialize_numpy(obss, flow_obs_file)
def test_compact_data_model(self): data_dir_temp = '/'.join( self.config_data.data_path['Temp'].split('/')[:-1]) data_dir = os.path.join(data_dir_temp, "exp" + str(1)) data_model = GagesModel.load_datamodel( data_dir, data_source_file_name='test_data_source.txt', stat_file_name='test_Statistics.json', flow_file_name='test_flow.npy', forcing_file_name='test_forcing.npy', attr_file_name='test_attr.npy', f_dict_file_name='test_dictFactorize.json', var_dict_file_name='test_dictAttribute.json', t_s_dict_file_name='test_dictTimeSpace.json') data_model_lst = [] for j in range(0, 2): # data_model_i.data_flow.shape[0] data_models_j = GagesModel.which_data_model(data_model, j) data_model_lst.append(data_models_j) gages_model_test = GagesModel.load_datamodel( self.config_data.data_path["Temp"], data_source_file_name='test_data_source.txt', stat_file_name='test_Statistics.json', flow_file_name='test_flow.npy', forcing_file_name='test_forcing.npy', attr_file_name='test_attr.npy', f_dict_file_name='test_dictFactorize.json', var_dict_file_name='test_dictAttribute.json', t_s_dict_file_name='test_dictTimeSpace.json') data_model_test = GagesModel.compact_data_model( data_model_lst, gages_model_test.data_source) print(data_model_test)
def test_test_gages_iter(self): data_config = self.config_data.read_data_config() regions = data_config["regions"] data_model_test_lst = [] with torch.cuda.device(1): obs_lsts = [] pred_lsts = [] for i in range(1, len(regions) + 1): data_dir_i_temp = '/'.join( self.config_data.data_path['Temp'].split('/')[:-1]) data_dir_i = os.path.join(data_dir_i_temp, "exp" + str(i)) data_model_i = GagesModel.load_datamodel( data_dir_i, data_source_file_name='test_data_source.txt', stat_file_name='test_Statistics.json', flow_file_name='test_flow.npy', forcing_file_name='test_forcing.npy', attr_file_name='test_attr.npy', f_dict_file_name='test_dictFactorize.json', var_dict_file_name='test_dictAttribute.json', t_s_dict_file_name='test_dictTimeSpace.json') data_model_test_lst.append(data_model_i) obs_lst = [] pred_lst = [] for j in range(0, data_model_i.data_flow.shape[0]): print("\n", "Testing model", str(j + 1), "of", regions[i - 1], "region", ":\n") data_models_j = GagesModel.which_data_model( data_model_i, j) pred, obs = master_test_1by1(data_models_j) basin_area = data_models_j.data_source.read_attr( data_models_j.t_s_dict["sites_id"], ['DRAIN_SQKM'], is_return_dict=False) mean_prep = data_models_j.data_source.read_attr( data_models_j.t_s_dict["sites_id"], ['PPTAVG_BASIN'], is_return_dict=False) mean_prep = mean_prep / 365 * 10 pred = _basin_norm(pred, basin_area, mean_prep, to_norm=False) obs = _basin_norm(obs, basin_area, mean_prep, to_norm=False) obs_lst.append(obs.flatten()) pred_lst.append(pred.flatten()) preds = np.array(pred_lst) obss = np.array(obs_lst) obs_lsts.append(obss) pred_lsts.append(preds) obs_final = reduce(lambda a, b: np.vstack((a, b)), obs_lsts) pred_final = reduce(lambda a, b: np.vstack((a, b)), pred_lsts) serialize_numpy(pred_final, self.flow_pred_file) serialize_numpy(obs_final, self.flow_obs_file)
def test_train_gages_iter(self): data_model = GagesModel.load_datamodel(self.config_data.data_path["Temp"], data_source_file_name='data_source.txt', stat_file_name='Statistics.json', flow_file_name='flow.npy', forcing_file_name='forcing.npy', attr_file_name='attr.npy', f_dict_file_name='dictFactorize.json', var_dict_file_name='dictAttribute.json', t_s_dict_file_name='dictTimeSpace.json') valid_size = 0.2 with torch.cuda.device(1): for i in range(145, data_model.data_flow.shape[0]): print("\n", "Training model", str(i + 1), ":\n") data_models_i = GagesModel.which_data_model(data_model, i) model, train_loss, valid_loss = master_train_1by1(data_models_i, valid_size=valid_size) fig = plot_loss_early_stop(train_loss, valid_loss) out_dir = data_models_i.data_source.data_config.data_path["Out"] fig.savefig(os.path.join(out_dir, 'loss_plot.png'), bbox_inches='tight')