def test_inv_train(self):
with torch.cuda.device(2):
df1 = GagesModel.load_datamodel(
self.config_data_1.data_path["Temp"],
"1",
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json',
flow_file_name='flow.npy',
forcing_file_name='forcing.npy',
attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
df2 = GagesModel.load_datamodel(
self.config_data_2.data_path["Temp"],
"2",
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json',
flow_file_name='flow.npy',
forcing_file_name='forcing.npy',
attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
data_model = GagesInvDataModel(df1, df2)
pre_trained_model_epoch = 285
# train_lstm_inv(data_model)
train_lstm_inv(data_model,
pre_trained_model_epoch=pre_trained_model_epoch)
def test_dam_train(self):
quick_data_dir = os.path.join(self.config_data_1.data_path["DB"],
"quickdata")
data_dir = os.path.join(quick_data_dir,
"allnonref_85-05_nan-0.1_00-1.0")
# for inv model, datamodel of train and test are same
data_model_8595 = GagesModel.load_datamodel(
data_dir,
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json',
flow_file_name='flow.npy',
forcing_file_name='forcing.npy',
attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
t_range1_train = self.config_data_1.model_dict["data"]["tRangeTrain"]
gages_model1_train = GagesModel.update_data_model(
self.config_data_1,
data_model_8595,
t_range_update=t_range1_train,
data_attr_update=True)
t_range2_train = self.config_data_2.model_dict["data"]["tRangeTrain"]
gages_model2_train = GagesModel.update_data_model(
self.config_data_2,
data_model_8595,
t_range_update=t_range2_train,
data_attr_update=True)
nid_dir = os.path.join(
"/".join(self.config_data_1.data_path["DB"].split("/")[:-1]),
"nid", "quickdata")
nid_input = NidModel.load_nidmodel(
nid_dir,
nid_file=self.nid_file,
nid_source_file_name='nid_source.txt',
nid_data_file_name='nid_data.shp')
gage_main_dam_purpose = unserialize_json(
os.path.join(nid_dir, "dam_main_purpose_dict.json"))
gage_main_dam_purpose_lst = list(gage_main_dam_purpose.values())
gage_main_dam_purpose_unique = np.unique(gage_main_dam_purpose_lst)
with torch.cuda.device(1):
for i in range(0, gage_main_dam_purpose_unique.size):
data_input1 = GagesDamDataModel(gages_model1_train, nid_input,
True, gage_main_dam_purpose)
gages_input1 = choose_which_purpose(
data_input1, purpose=gage_main_dam_purpose_unique[i])
data_input2 = GagesDamDataModel(gages_model2_train, nid_input,
True, gage_main_dam_purpose)
gages_input2 = choose_which_purpose(
data_input2, purpose=gage_main_dam_purpose_unique[i])
data_model = GagesInvDataModel(gages_input1, gages_input2)
# pre_trained_model_epoch = 165
train_lstm_inv(data_model)
def test_inv_test(self):
with torch.cuda.device(2):
df1 = GagesModel.load_datamodel(
self.config_data_1.data_path["Temp"],
"1",
data_source_file_name='test_data_source.txt',
stat_file_name='test_Statistics.json',
flow_file_name='test_flow.npy',
forcing_file_name='test_forcing.npy',
attr_file_name='test_attr.npy',
f_dict_file_name='test_dictFactorize.json',
var_dict_file_name='test_dictAttribute.json',
t_s_dict_file_name='test_dictTimeSpace.json')
df2 = GagesModel.load_datamodel(
self.config_data_2.data_path["Temp"],
"2",
data_source_file_name='test_data_source.txt',
stat_file_name='test_Statistics.json',
flow_file_name='test_flow.npy',
forcing_file_name='test_forcing.npy',
attr_file_name='test_attr.npy',
f_dict_file_name='test_dictFactorize.json',
var_dict_file_name='test_dictAttribute.json',
t_s_dict_file_name='test_dictTimeSpace.json')
data_model = GagesInvDataModel(df1, df2)
pred, obs = test_lstm_inv(data_model, epoch=self.test_epoch)
basin_area = df2.data_source.read_attr(df2.t_s_dict["sites_id"],
['DRAIN_SQKM'],
is_return_dict=False)
mean_prep = df2.data_source.read_attr(df2.t_s_dict["sites_id"],
['PPTAVG_BASIN'],
is_return_dict=False)
mean_prep = mean_prep / 365 * 10
pred = _basin_norm(pred, basin_area, mean_prep, to_norm=False)
obs = _basin_norm(obs, basin_area, mean_prep, to_norm=False)
flow_pred_file = os.path.join(
df2.data_source.data_config.data_path['Temp'],
'epoch' + str(self.test_epoch) + 'flow_pred')
flow_obs_file = os.path.join(
df2.data_source.data_config.data_path['Temp'],
'epoch' + str(self.test_epoch) + 'flow_obs')
serialize_numpy(pred, flow_pred_file)
serialize_numpy(obs, flow_obs_file)
def test_dam_test(self):
quick_data_dir = os.path.join(self.config_data_1.data_path["DB"],
"quickdata")
data_dir = os.path.join(quick_data_dir,
"allnonref_85-05_nan-0.1_00-1.0")
# for inv model, datamodel of train and test are same
data_model_8595 = GagesModel.load_datamodel(
data_dir,
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json',
flow_file_name='flow.npy',
forcing_file_name='forcing.npy',
attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
# for 2nd model, datamodel of train and test belong to parts of the test time
data_model_9505 = GagesModel.load_datamodel(
data_dir,
data_source_file_name='test_data_source.txt',
stat_file_name='test_Statistics.json',
flow_file_name='test_flow.npy',
forcing_file_name='test_forcing.npy',
attr_file_name='test_attr.npy',
f_dict_file_name='test_dictFactorize.json',
var_dict_file_name='test_dictAttribute.json',
t_s_dict_file_name='test_dictTimeSpace.json')
t_range1_test = self.config_data_1.model_dict["data"]["tRangeTest"]
# Because we know data of period "90-95", so that we can get its statistics according to this period
gages_model1_test = GagesModel.update_data_model(
self.config_data_1,
data_model_8595,
t_range_update=t_range1_test,
data_attr_update=True)
t_range2_train = self.config_data_2.model_dict["data"]["tRangeTrain"]
t_range2_test = self.config_data_2.model_dict["data"]["tRangeTest"]
gages_model2_train = GagesModel.update_data_model(
self.config_data_2,
data_model_8595,
t_range_update=t_range2_train,
data_attr_update=True)
gages_model2_test = GagesModel.update_data_model(
self.config_data_2,
data_model_9505,
t_range_update=t_range2_test,
data_attr_update=True,
train_stat_dict=gages_model2_train.stat_dict)
nid_dir = os.path.join(
"/".join(self.config_data_2.data_path["DB"].split("/")[:-1]),
"nid", "quickdata")
nid_input = NidModel.load_nidmodel(
nid_dir,
nid_file=self.nid_file,
nid_source_file_name='nid_source.txt',
nid_data_file_name='nid_data.shp')
gage_main_dam_purpose = unserialize_json(
os.path.join(nid_dir, "dam_main_purpose_dict.json"))
data_input1 = GagesDamDataModel(gages_model1_test, nid_input, True,
gage_main_dam_purpose)
df1 = choose_which_purpose(data_input1)
data_input2 = GagesDamDataModel(gages_model2_test, nid_input, True,
gage_main_dam_purpose)
df2 = choose_which_purpose(data_input2)
with torch.cuda.device(2):
data_model = GagesInvDataModel(df1, df2)
pred, obs = test_lstm_inv(data_model, epoch=self.test_epoch)
basin_area = df2.data_source.read_attr(df2.t_s_dict["sites_id"],
['DRAIN_SQKM'],
is_return_dict=False)
mean_prep = df2.data_source.read_attr(df2.t_s_dict["sites_id"],
['PPTAVG_BASIN'],
is_return_dict=False)
mean_prep = mean_prep / 365 * 10
pred = _basin_norm(pred, basin_area, mean_prep, to_norm=False)
obs = _basin_norm(obs, basin_area, mean_prep, to_norm=False)
save_result(df2.data_source.data_config.data_path['Temp'],
self.test_epoch, pred, obs)