def test_dam_train(self):
quick_data_dir = os.path.join(self.config_data_1.data_path["DB"],
"quickdata")
data_dir = os.path.join(quick_data_dir,
"allnonref_85-05_nan-0.1_00-1.0")
# for inv model, datamodel of train and test are same
data_model_8595 = GagesModel.load_datamodel(
data_dir,
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json',
flow_file_name='flow.npy',
forcing_file_name='forcing.npy',
attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
t_range1_train = self.config_data_1.model_dict["data"]["tRangeTrain"]
gages_model1_train = GagesModel.update_data_model(
self.config_data_1,
data_model_8595,
t_range_update=t_range1_train,
data_attr_update=True)
t_range2_train = self.config_data_2.model_dict["data"]["tRangeTrain"]
gages_model2_train = GagesModel.update_data_model(
self.config_data_2,
data_model_8595,
t_range_update=t_range2_train,
data_attr_update=True)
nid_dir = os.path.join(
"/".join(self.config_data_1.data_path["DB"].split("/")[:-1]),
"nid", "quickdata")
nid_input = NidModel.load_nidmodel(
nid_dir,
nid_file=self.nid_file,
nid_source_file_name='nid_source.txt',
nid_data_file_name='nid_data.shp')
gage_main_dam_purpose = unserialize_json(
os.path.join(nid_dir, "dam_main_purpose_dict.json"))
gage_main_dam_purpose_lst = list(gage_main_dam_purpose.values())
gage_main_dam_purpose_unique = np.unique(gage_main_dam_purpose_lst)
with torch.cuda.device(1):
for i in range(0, gage_main_dam_purpose_unique.size):
data_input1 = GagesDamDataModel(gages_model1_train, nid_input,
True, gage_main_dam_purpose)
gages_input1 = choose_which_purpose(
data_input1, purpose=gage_main_dam_purpose_unique[i])
data_input2 = GagesDamDataModel(gages_model2_train, nid_input,
True, gage_main_dam_purpose)
gages_input2 = choose_which_purpose(
data_input2, purpose=gage_main_dam_purpose_unique[i])
data_model = GagesInvDataModel(gages_input1, gages_input2)
# pre_trained_model_epoch = 165
train_lstm_inv(data_model)
def test_dam_train(self):
"""just test for one purpose as a case"""
with torch.cuda.device(2):
quick_data_dir = os.path.join(self.config_data.data_path["DB"],
"quickdata")
data_dir = os.path.join(quick_data_dir,
"conus-all_90-10_nan-0.0_00-1.0")
df = GagesModel.load_datamodel(
data_dir,
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json',
flow_file_name='flow.npy',
forcing_file_name='forcing.npy',
attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
nid_dir = os.path.join(
"/".join(self.config_data.data_path["DB"].split("/")[:-1]),
"nid", "quickdata")
nid_input = NidModel.load_nidmodel(
nid_dir,
nid_file=self.nid_file,
nid_source_file_name='nid_source.txt',
nid_data_file_name='nid_data.shp')
gage_main_dam_purpose = unserialize_json(
os.path.join(nid_dir, "dam_main_purpose_dict.json"))
data_input = GagesDamDataModel(df, nid_input, True,
gage_main_dam_purpose)
purpose_chosen = 'C'
gages_input = choose_which_purpose(data_input,
purpose=purpose_chosen)
master_train(gages_input)
def test_dam_train(self):
with torch.cuda.device(0):
quick_data_dir = os.path.join(self.config_data.data_path["DB"],
"quickdata")
data_dir = os.path.join(quick_data_dir,
"allnonref_85-05_nan-0.1_00-1.0")
data_model_8595 = GagesModel.load_datamodel(
data_dir,
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json',
flow_file_name='flow.npy',
forcing_file_name='forcing.npy',
attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
gages_model_train = GagesModel.update_data_model(
self.config_data, data_model_8595)
nid_dir = os.path.join(
"/".join(self.config_data.data_path["DB"].split("/")[:-1]),
"nid", "quickdata")
nid_input = NidModel.load_nidmodel(
nid_dir,
nid_file=self.nid_file,
nid_source_file_name='nid_source.txt',
nid_data_file_name='nid_data.shp')
gage_main_dam_purpose = unserialize_json(
os.path.join(nid_dir, "dam_main_purpose_dict.json"))
data_input = GagesDamDataModel(gages_model_train, nid_input, True,
gage_main_dam_purpose)
gages_input = choose_which_purpose(data_input)
master_train(gages_input)
def test_dam_test(self):
quick_data_dir = os.path.join(self.config_data.data_path["DB"],
"quickdata")
data_dir = os.path.join(quick_data_dir,
"conus-all_90-10_nan-0.0_00-1.0")
data_model_train = GagesModel.load_datamodel(
data_dir,
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json',
flow_file_name='flow.npy',
forcing_file_name='forcing.npy',
attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
data_model_test = GagesModel.load_datamodel(
data_dir,
data_source_file_name='test_data_source.txt',
stat_file_name='test_Statistics.json',
flow_file_name='test_flow.npy',
forcing_file_name='test_forcing.npy',
attr_file_name='test_attr.npy',
f_dict_file_name='test_dictFactorize.json',
var_dict_file_name='test_dictAttribute.json',
t_s_dict_file_name='test_dictTimeSpace.json')
gages_model_train = GagesModel.update_data_model(
self.config_data, data_model_train)
gages_model_test = GagesModel.update_data_model(
self.config_data,
data_model_test,
train_stat_dict=gages_model_train.stat_dict)
nid_dir = os.path.join(
"/".join(self.config_data.data_path["DB"].split("/")[:-1]), "nid",
"quickdata")
nid_input = NidModel.load_nidmodel(
nid_dir,
nid_file=self.nid_file,
nid_source_file_name='nid_source.txt',
nid_data_file_name='nid_data.shp')
gage_main_dam_purpose = unserialize_json(
os.path.join(nid_dir, "dam_main_purpose_dict.json"))
data_input = GagesDamDataModel(gages_model_test, nid_input, True,
gage_main_dam_purpose)
gages_input = choose_which_purpose(data_input)
pred, obs = master_test(gages_input)
basin_area = gages_input.data_source.read_attr(
gages_input.t_s_dict["sites_id"], ['DRAIN_SQKM'],
is_return_dict=False)
mean_prep = gages_input.data_source.read_attr(
gages_input.t_s_dict["sites_id"], ['PPTAVG_BASIN'],
is_return_dict=False)
mean_prep = mean_prep / 365 * 10
pred = _basin_norm(pred, basin_area, mean_prep, to_norm=False)
obs = _basin_norm(obs, basin_area, mean_prep, to_norm=False)
save_result(gages_input.data_source.data_config.data_path['Temp'],
self.test_epoch, pred, obs)
def test_gages_dam_all_save(self):
quick_data_dir = os.path.join(self.config_data.data_path["DB"],
"quickdata")
data_dir = os.path.join(quick_data_dir,
"conus-all_90-10_nan-0.0_00-1.0")
data_model_train = GagesModel.load_datamodel(
data_dir,
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json',
flow_file_name='flow.npy',
forcing_file_name='forcing.npy',
attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
gages_model_train = GagesModel.update_data_model(
self.config_data, data_model_train)
data_model_test = GagesModel.load_datamodel(
data_dir,
data_source_file_name='test_data_source.txt',
stat_file_name='test_Statistics.json',
flow_file_name='test_flow.npy',
forcing_file_name='test_forcing.npy',
attr_file_name='test_attr.npy',
f_dict_file_name='test_dictFactorize.json',
var_dict_file_name='test_dictAttribute.json',
t_s_dict_file_name='test_dictTimeSpace.json')
gages_model_test = GagesModel.update_data_model(
self.config_data,
data_model_test,
train_stat_dict=gages_model_train.stat_dict)
nid_dir = os.path.join(
"/".join(self.config_data.data_path["DB"].split("/")[:-1]), "nid",
"test")
nid_input = NidModel.load_nidmodel(
nid_dir,
nid_source_file_name='nid_source.txt',
nid_data_file_name='nid_data.shp')
gage_main_dam_purpose = unserialize_json(
os.path.join(nid_dir, "dam_main_purpose_dict.json"))
data_input = GagesDamDataModel(gages_model_test, nid_input,
gage_main_dam_purpose)
data_model_dam = choose_which_purpose(data_input)
save_datamodel(data_model_dam,
data_source_file_name='test_data_source.txt',
stat_file_name='test_Statistics.json',
flow_file_name='test_flow',
forcing_file_name='test_forcing',
attr_file_name='test_attr',
f_dict_file_name='test_dictFactorize.json',
var_dict_file_name='test_dictAttribute.json',
t_s_dict_file_name='test_dictTimeSpace.json')
def test_damcls_test_datamodel(self):
quick_data_dir = os.path.join(self.config_data.data_path["DB"], "quickdata")
data_dir = os.path.join(quick_data_dir, "allnonref_85-05_nan-0.1_00-1.0")
data_model_train = GagesModel.load_datamodel(data_dir,
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json', flow_file_name='flow.npy',
forcing_file_name='forcing.npy', attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
data_model_test = GagesModel.load_datamodel(data_dir,
data_source_file_name='test_data_source.txt',
stat_file_name='test_Statistics.json',
flow_file_name='test_flow.npy',
forcing_file_name='test_forcing.npy',
attr_file_name='test_attr.npy',
f_dict_file_name='test_dictFactorize.json',
var_dict_file_name='test_dictAttribute.json',
t_s_dict_file_name='test_dictTimeSpace.json')
gages_model_train = GagesModel.update_data_model(self.config_data, data_model_train)
df = GagesModel.update_data_model(self.config_data, data_model_test,
train_stat_dict=gages_model_train.stat_dict)
nid_dir = os.path.join("/".join(self.config_data.data_path["DB"].split("/")[:-1]), "nid", "quickdata")
nid_input = NidModel.load_nidmodel(nid_dir, nid_file=self.nid_file,
nid_source_file_name='nid_source.txt', nid_data_file_name='nid_data.shp')
gage_main_dam_purpose = unserialize_json(os.path.join(nid_dir, "dam_main_purpose_dict.json"))
gage_main_dam_purpose_lst = list(gage_main_dam_purpose.values())
gage_main_dam_purpose_unique = np.unique(gage_main_dam_purpose_lst)
data_input = GagesDamDataModel(df, nid_input, True, gage_main_dam_purpose)
for i in range(gage_main_dam_purpose_unique.size):
gages_input = choose_which_purpose(data_input, purpose=gage_main_dam_purpose_unique[i])
save_datamodel(gages_input, gage_main_dam_purpose_unique[i], data_source_file_name='test_data_source.txt',
stat_file_name='test_Statistics.json', flow_file_name='test_flow',
forcing_file_name='test_forcing', attr_file_name='test_attr',
f_dict_file_name='test_dictFactorize.json', var_dict_file_name='test_dictAttribute.json',
t_s_dict_file_name='test_dictTimeSpace.json')
def test_dam_test(self):
quick_data_dir = os.path.join(self.config_data_1.data_path["DB"],
"quickdata")
data_dir = os.path.join(quick_data_dir,
"allnonref_85-05_nan-0.1_00-1.0")
# for inv model, datamodel of train and test are same
data_model_8595 = GagesModel.load_datamodel(
data_dir,
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json',
flow_file_name='flow.npy',
forcing_file_name='forcing.npy',
attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
# for 2nd model, datamodel of train and test belong to parts of the test time
data_model_9505 = GagesModel.load_datamodel(
data_dir,
data_source_file_name='test_data_source.txt',
stat_file_name='test_Statistics.json',
flow_file_name='test_flow.npy',
forcing_file_name='test_forcing.npy',
attr_file_name='test_attr.npy',
f_dict_file_name='test_dictFactorize.json',
var_dict_file_name='test_dictAttribute.json',
t_s_dict_file_name='test_dictTimeSpace.json')
t_range1_test = self.config_data_1.model_dict["data"]["tRangeTest"]
# Because we know data of period "90-95", so that we can get its statistics according to this period
gages_model1_test = GagesModel.update_data_model(
self.config_data_1,
data_model_8595,
t_range_update=t_range1_test,
data_attr_update=True)
t_range2_train = self.config_data_2.model_dict["data"]["tRangeTrain"]
t_range2_test = self.config_data_2.model_dict["data"]["tRangeTest"]
gages_model2_train = GagesModel.update_data_model(
self.config_data_2,
data_model_8595,
t_range_update=t_range2_train,
data_attr_update=True)
gages_model2_test = GagesModel.update_data_model(
self.config_data_2,
data_model_9505,
t_range_update=t_range2_test,
data_attr_update=True,
train_stat_dict=gages_model2_train.stat_dict)
nid_dir = os.path.join(
"/".join(self.config_data_2.data_path["DB"].split("/")[:-1]),
"nid", "quickdata")
nid_input = NidModel.load_nidmodel(
nid_dir,
nid_file=self.nid_file,
nid_source_file_name='nid_source.txt',
nid_data_file_name='nid_data.shp')
gage_main_dam_purpose = unserialize_json(
os.path.join(nid_dir, "dam_main_purpose_dict.json"))
data_input1 = GagesDamDataModel(gages_model1_test, nid_input, True,
gage_main_dam_purpose)
df1 = choose_which_purpose(data_input1)
data_input2 = GagesDamDataModel(gages_model2_test, nid_input, True,
gage_main_dam_purpose)
df2 = choose_which_purpose(data_input2)
with torch.cuda.device(2):
data_model = GagesInvDataModel(df1, df2)
pred, obs = test_lstm_inv(data_model, epoch=self.test_epoch)
basin_area = df2.data_source.read_attr(df2.t_s_dict["sites_id"],
['DRAIN_SQKM'],
is_return_dict=False)
mean_prep = df2.data_source.read_attr(df2.t_s_dict["sites_id"],
['PPTAVG_BASIN'],
is_return_dict=False)
mean_prep = mean_prep / 365 * 10
pred = _basin_norm(pred, basin_area, mean_prep, to_norm=False)
obs = _basin_norm(obs, basin_area, mean_prep, to_norm=False)
save_result(df2.data_source.data_config.data_path['Temp'],
self.test_epoch, pred, obs)
def test_dam_train(self):
quick_data_dir = os.path.join(self.config_data.data_path["DB"],
"quickdata")
sim_data_dir = os.path.join(quick_data_dir,
"allref_85-05_nan-0.1_00-1.0")
data_dir = os.path.join(quick_data_dir,
"allnonref_85-05_nan-0.1_00-1.0")
data_model_sim8595 = GagesModel.load_datamodel(
sim_data_dir,
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json',
flow_file_name='flow.npy',
forcing_file_name='forcing.npy',
attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
data_model_8595 = GagesModel.load_datamodel(
data_dir,
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json',
flow_file_name='flow.npy',
forcing_file_name='forcing.npy',
attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
sim_gages_model_train = GagesModel.update_data_model(
self.sim_config_data, data_model_sim8595, data_attr_update=True)
gages_model_train = GagesModel.update_data_model(self.config_data,
data_model_8595,
data_attr_update=True)
nid_dir = os.path.join(
"/".join(self.config_data.data_path["DB"].split("/")[:-1]), "nid",
"quickdata")
gage_main_dam_purpose = unserialize_json(
os.path.join(nid_dir, "dam_main_purpose_dict.json"))
gage_main_dam_purpose_lst = list(gage_main_dam_purpose.values())
gage_main_dam_purpose_unique = np.unique(gage_main_dam_purpose_lst)
nid_dir = os.path.join(
"/".join(self.config_data.data_path["DB"].split("/")[:-1]), "nid",
"quickdata")
nid_input = NidModel.load_nidmodel(
nid_dir,
nid_file=self.nid_file,
nid_source_file_name='nid_source.txt',
nid_data_file_name='nid_data.shp')
gage_main_dam_purpose = unserialize_json(
os.path.join(nid_dir, "dam_main_purpose_dict.json"))
data_input = GagesDamDataModel(gages_model_train, nid_input, True,
gage_main_dam_purpose)
with torch.cuda.device(0):
for i in range(0, gage_main_dam_purpose_unique.size):
sim_gages_model_train.update_model_param('train', nEpoch=300)
gages_input = choose_which_purpose(
data_input, purpose=gage_main_dam_purpose_unique[i])
new_temp_dir = os.path.join(
gages_input.data_source.data_config.model_dict["dir"]
["Temp"], gage_main_dam_purpose_unique[i])
new_out_dir = os.path.join(
gages_input.data_source.data_config.model_dict["dir"]
["Out"], gage_main_dam_purpose_unique[i])
gages_input.update_datamodel_dir(new_temp_dir, new_out_dir)
data_model = GagesSimDataModel(sim_gages_model_train,
gages_input)
# pre_trained_model_epoch = 25
# master_train_natural_flow(data_model, pre_trained_model_epoch=pre_trained_model_epoch)
master_train_natural_flow(data_model)
def test_dam_test(self):
quick_data_dir = os.path.join(self.config_data.data_path["DB"],
"quickdata")
sim_data_dir = os.path.join(quick_data_dir,
"allref_85-05_nan-0.1_00-1.0")
data_dir = os.path.join(quick_data_dir,
"allnonref_85-05_nan-0.1_00-1.0")
data_model_sim8595 = GagesModel.load_datamodel(
sim_data_dir,
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json',
flow_file_name='flow.npy',
forcing_file_name='forcing.npy',
attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
data_model_8595 = GagesModel.load_datamodel(
data_dir,
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json',
flow_file_name='flow.npy',
forcing_file_name='forcing.npy',
attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
data_model_sim9505 = GagesModel.load_datamodel(
sim_data_dir,
data_source_file_name='test_data_source.txt',
stat_file_name='test_Statistics.json',
flow_file_name='test_flow.npy',
forcing_file_name='test_forcing.npy',
attr_file_name='test_attr.npy',
f_dict_file_name='test_dictFactorize.json',
var_dict_file_name='test_dictAttribute.json',
t_s_dict_file_name='test_dictTimeSpace.json')
data_model_9505 = GagesModel.load_datamodel(
data_dir,
data_source_file_name='test_data_source.txt',
stat_file_name='test_Statistics.json',
flow_file_name='test_flow.npy',
forcing_file_name='test_forcing.npy',
attr_file_name='test_attr.npy',
f_dict_file_name='test_dictFactorize.json',
var_dict_file_name='test_dictAttribute.json',
t_s_dict_file_name='test_dictTimeSpace.json')
sim_gages_model_train = GagesModel.update_data_model(
self.sim_config_data, data_model_sim8595, data_attr_update=True)
gages_model_train = GagesModel.update_data_model(self.config_data,
data_model_8595,
data_attr_update=True)
sim_gages_model_test = GagesModel.update_data_model(
self.sim_config_data,
data_model_sim9505,
data_attr_update=True,
train_stat_dict=sim_gages_model_train.stat_dict)
gages_model_test = GagesModel.update_data_model(
self.config_data,
data_model_9505,
data_attr_update=True,
train_stat_dict=gages_model_train.stat_dict)
nid_dir = os.path.join(
"/".join(self.config_data.data_path["DB"].split("/")[:-1]), "nid",
"quickdata")
nid_input = NidModel.load_nidmodel(
nid_dir,
nid_file=self.nid_file,
nid_source_file_name='nid_source.txt',
nid_data_file_name='nid_data.shp')
gage_main_dam_purpose = unserialize_json(
os.path.join(nid_dir, "dam_main_purpose_dict.json"))
gage_main_dam_purpose_lst = list(gage_main_dam_purpose.values())
gage_main_dam_purpose_unique = np.unique(gage_main_dam_purpose_lst)
data_input = GagesDamDataModel(gages_model_test, nid_input, True,
gage_main_dam_purpose)
for i in range(0, gage_main_dam_purpose_unique.size):
sim_gages_model_test.update_model_param('train', nEpoch=300)
gages_input = choose_which_purpose(
data_input, purpose=gage_main_dam_purpose_unique[i])
new_temp_dir = os.path.join(
gages_input.data_source.data_config.model_dict["dir"]["Temp"],
gage_main_dam_purpose_unique[i])
new_out_dir = os.path.join(
gages_input.data_source.data_config.model_dict["dir"]["Out"],
gage_main_dam_purpose_unique[i])
gages_input.update_datamodel_dir(new_temp_dir, new_out_dir)
model_input = GagesSimDataModel(sim_gages_model_test, gages_input)
pred, obs = master_test_natural_flow(model_input,
epoch=self.test_epoch)
basin_area = model_input.data_model2.data_source.read_attr(
model_input.data_model2.t_s_dict["sites_id"], ['DRAIN_SQKM'],
is_return_dict=False)
mean_prep = model_input.data_model2.data_source.read_attr(
model_input.data_model2.t_s_dict["sites_id"], ['PPTAVG_BASIN'],
is_return_dict=False)
mean_prep = mean_prep / 365 * 10
pred = _basin_norm(pred, basin_area, mean_prep, to_norm=False)
obs = _basin_norm(obs, basin_area, mean_prep, to_norm=False)
save_result(
model_input.data_model2.data_source.data_config.
data_path['Temp'], str(self.test_epoch), pred, obs)
plot_we_need(gages_input,
obs,
pred,
id_col="STAID",
lon_col="LNG_GAGE",
lat_col="LAT_GAGE")
