def __init__(self, config_data):
# prepare training data
t_train = config_data.model_dict["data"]["tRangeTrain"]
t_test = config_data.model_dict["data"]["tRangeTest"]
t_train_test = [t_train[0], t_test[1]]
source_data = CamelsSource(config_data, t_train_test)
# create datamodel as input for the model
data_model = CamelsModel(source_data)
self.data_model_train, self.data_model_test = CamelsModel.data_models_of_train_test(
data_model, t_train, t_test)
def test_test_camels(self):
data_model = CamelsModel.load_datamodel(
self.config_data.data_path["Temp"],
data_source_file_name='test_data_source.txt',
stat_file_name='test_Statistics.json',
flow_file_name='test_flow.npy',
forcing_file_name='test_forcing.npy',
attr_file_name='test_attr.npy',
f_dict_file_name='test_dictFactorize.json',
var_dict_file_name='test_dictAttribute.json',
t_s_dict_file_name='test_dictTimeSpace.json')
with torch.cuda.device(2):
pred, obs = master_test(data_model)
basin_area = data_model.data_source.read_attr(
data_model.t_s_dict["sites_id"], ['area_gages2'],
is_return_dict=False)
mean_prep = data_model.data_source.read_attr(
data_model.t_s_dict["sites_id"], ['p_mean'],
is_return_dict=False)
pred = _basin_norm(pred, basin_area, mean_prep, to_norm=False)
obs = _basin_norm(obs, basin_area, mean_prep, to_norm=False)
plot_we_need(data_model,
obs,
pred,
id_col="id",
lon_col="lon",
lat_col="lat")
def test_train_camels(self):
data_model = CamelsModel.load_datamodel(
self.config_data.data_path["Temp"],
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json',
flow_file_name='flow.npy',
forcing_file_name='forcing.npy',
attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
with torch.cuda.device(1):
master_train(data_model)
def test_test_camels_easy_lstm(self):
data_model = CamelsModel.load_datamodel(
self.config_data.data_path["Temp"],
data_source_file_name='test_data_source.txt',
stat_file_name='test_Statistics.json',
flow_file_name='test_flow.npy',
forcing_file_name='test_forcing.npy',
attr_file_name='test_attr.npy',
f_dict_file_name='test_dictFactorize.json',
var_dict_file_name='test_dictAttribute.json',
t_s_dict_file_name='test_dictTimeSpace.json')
data_model_train = CamelsModel.load_datamodel(
self.config_data.data_path["Temp"],
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json',
flow_file_name='flow.npy',
forcing_file_name='forcing.npy',
attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
# pred, obs = master_test(data_model)
with torch.cuda.device(0):
data_set = StreamflowInputDataset(data_model, train_mode=False)
# data_set = StreamflowInputDataset(data_model_train, train_mode=False)
pred, obs = master_test_easier_lstm(data_set, load_epoch=1)
# pred, obs = master_test_better_lstm(data_set)
basin_area = data_model.data_source.read_attr(
data_model.t_s_dict["sites_id"], ['area_gages2'],
is_return_dict=False)
mean_prep = data_model.data_source.read_attr(
data_model.t_s_dict["sites_id"], ['p_mean'],
is_return_dict=False)
pred = _basin_norm(pred, basin_area, mean_prep, to_norm=False)
obs = _basin_norm(obs, basin_area, mean_prep, to_norm=False)
plot_we_need(data_set.data_model, obs, pred)
def test_train_valid_camels(self):
data_model = CamelsModel.load_datamodel(
self.config_data.data_path["Temp"],
data_source_file_name='data_source.txt',
stat_file_name='Statistics.json',
flow_file_name='flow.npy',
forcing_file_name='forcing.npy',
attr_file_name='attr.npy',
f_dict_file_name='dictFactorize.json',
var_dict_file_name='dictAttribute.json',
t_s_dict_file_name='dictTimeSpace.json')
with torch.cuda.device(2):
model, train_loss, valid_loss = master_train(data_model,
valid_size=0.2)
fig = plot_loss_early_stop(train_loss, valid_loss)
out_dir = self.config_data.data_path["Out"]
fig.savefig(os.path.join(out_dir, 'loss_plot.png'),
bbox_inches='tight')