def test_snow_and_ground_water_response_calibration(self):
"""
Test dual calibration strategy:
* First fit the three Kirchner parameters for
ground water response during July, August, and
September.
* Then fit two snow routine parameters (tx and max_water)
from November to April.
"""
# Simulation time axis
year, month, day, hour = 2010, 9, 1, 0
dt = api.deltahours(24)
n_steps = 400
utc = api.Calendar() # No offset gives Utc
t0 = utc.time(api.YMDhms(year, month, day, hour))
time_axis = api.Timeaxis(t0, dt, n_steps)
# Some fake ids
region_id = 0
interpolation_id = 0
# Simulation coordinate system
epsg = "32633"
# Model
model_t = pt_gs_k.PTGSKOptModel
# Configs and repositories
dataset_config_file = path.join(path.dirname(__file__), "netcdf",
"atnsjoen_datasets.yaml")
region_config_file = path.join(path.dirname(__file__), "netcdf",
"atnsjoen_calibration_region.yaml")
region_config = RegionConfig(region_config_file)
model_config = ModelConfig(self.model_config_file)
dataset_config = YamlContent(dataset_config_file)
region_model_repository = RegionModelRepository(region_config, model_config, model_t, epsg)
interp_repos = InterpolationParameterRepository(model_config)
netcdf_geo_ts_repos = []
for source in dataset_config.sources:
station_file = source["params"]["stations_met"]
netcdf_geo_ts_repos.append(GeoTsRepository(source["params"], station_file, ""))
geo_ts_repository = GeoTsRepositoryCollection(netcdf_geo_ts_repos)
# Construct target discharge series
simulator = DefaultSimulator(region_id, interpolation_id, region_model_repository,
geo_ts_repository, interp_repos, None)
n_cells = simulator.region_model.size()
state_repos = DefaultStateRepository(model_t, n_cells)
simulator.run(time_axis, state_repos.get_state(0))
cid = 1
target_discharge = simulator.region_model.statistics.discharge([cid])
# Construct kirchner parameters
param = simulator.region_model.parameter_t(simulator.region_model.get_region_parameter())
print_param("True solution", param)
kirchner_param_min = simulator.region_model.parameter_t(param)
kirchner_param_max = simulator.region_model.parameter_t(param)
# Kichner parameters are quite abstract (no physical meaning), so simply scale them
kirchner_param_min.kirchner.c1 *= 0.8
kirchner_param_min.kirchner.c2 *= 0.8
kirchner_param_min.kirchner.c3 *= 0.8
kirchner_param_max.kirchner.c1 *= 1.2
kirchner_param_max.kirchner.c2 *= 1.2
kirchner_param_max.kirchner.c3 *= 1.2
# kirchner_t_start = utc.time(api.YMDhms(2011, 4, 1, 0))
# kirchner_time_axis = api.Timeaxis(kirchner_t_start, dt, 150)
kirchner_time_axis = time_axis
# Construct gamma snow parameters (realistic tx and max_lwc)
gamma_snow_param_min = simulator.region_model.parameter_t(param)
gamma_snow_param_max = simulator.region_model.parameter_t(param)
gamma_snow_param_min.gs.tx = -1.0 # Min snow/rain temperature threshold
gamma_snow_param_min.gs.max_water = 0.05 # Min 8% max water in snow in costal regions
gamma_snow_param_max.gs.tx = 1.0
gamma_snow_param_max.gs.max_water = 0.25 # Max 35% max water content, or we get too little melt
gs_t_start = utc.time(api.YMDhms(2010, 11, 1, 0))
gs_time_axis = api.Timeaxis(gs_t_start, dt, 250)
# gs_time_axis = time_axis
# Find parameters
target_spec = api.TargetSpecificationPts(target_discharge, api.IntVector([cid]),
1.0, api.KLING_GUPTA)
target_spec_vec = api.TargetSpecificationVector() # TODO: We currently dont fix list initializer for vectors
target_spec_vec.append(target_spec)
# Construct a fake, perturbed starting point for calibration
p_vec = [param.get(i) for i in range(param.size())]
for i, name in enumerate([param.get_name(i) for i in range(len(p_vec))]):
if name not in ("c1" "c2", "c3", "TX", "max_water"):
next
if name in ("c1", "c2", "c3"):
p_vec[i] = random.uniform(0.8*p_vec[i], 1.2*p_vec[i])
elif name == "TX":
p_vec[i] = random.uniform(gamma_snow_param_min.gs.tx, gamma_snow_param_max.gs.tx)
elif name == "max_water":
p_vec[i] = random.uniform(gamma_snow_param_min.gs.max_water, gamma_snow_param_max.gs.max_water)
param.set(p_vec)
print_param("Initial guess", param)
# Two pass optimization, once for the ground water response, and second time for
kirchner_p_opt = simulator.optimize(kirchner_time_axis, state_repos.get_state(0), target_spec_vec, param,
kirchner_param_min, kirchner_param_max)
gamma_snow_p_opt = simulator.optimize(gs_time_axis, state_repos.get_state(0), target_spec_vec, kirchner_p_opt,
gamma_snow_param_min, gamma_snow_param_max)
print_param("Half way result", kirchner_p_opt)
print_param("Result", gamma_snow_p_opt)
simulator.region_model.set_catchment_parameter(cid, gamma_snow_p_opt)
simulator.run(time_axis, state_repos.get_state(0))
found_discharge = simulator.region_model.statistics.discharge([cid])
t_vs = np.array(target_discharge.v)
t_ts = np.array([target_discharge.time(i) for i in range(target_discharge.size())])
f_vs = np.array(found_discharge.v)
f_ts = np.array([found_discharge.time(i) for i in range(found_discharge.size())])
def test_snow_and_ground_water_response_calibration(self):
"""
Test dual calibration strategy:
* First fit the three Kirchner parameters for
ground water response during July, August, and
September.
* Then fit two snow routine parameters (tx and max_water)
from November to April.
"""
# Simulation time axis
dt = api.deltahours(24)
n_steps = 364
utc = api.Calendar() # No offset gives Utc
t0 = utc.time(2013, 9, 1, 0)
time_axis = api.TimeAxisFixedDeltaT(t0, dt, n_steps)
# Some fake ids
region_id = 0
interpolation_id = 0
opt_model_t = self.model_config.model_type().opt_model_t
model_config = ModelConfig(self.model_config_file,
overrides={'model_t': opt_model_t})
region_model_repository = CFRegionModelRepository(
self.region_config, model_config)
interp_repos = InterpolationParameterRepository(
self.interpolation_config)
netcdf_geo_ts_repos = [
CFDataRepository(32633,
source["params"]["filename"],
padding=source["params"]['padding'])
for source in self.dataset_config.sources
]
geo_ts_repository = GeoTsRepositoryCollection(netcdf_geo_ts_repos)
# Construct target discharge series
simulator = DefaultSimulator(region_id, interpolation_id,
region_model_repository,
geo_ts_repository, interp_repos, None)
state_repos = DefaultStateRepository(simulator.region_model)
simulator.run(time_axis, state_repos.get_state(0))
cid = 1228
target_discharge = api.TsTransform().to_average(
t0, dt, n_steps,
simulator.region_model.statistics.discharge([cid]))
# Construct kirchner parameters
param = simulator.region_model.parameter_t(
simulator.region_model.get_region_parameter())
print_param("True solution", param)
kirchner_param_min = simulator.region_model.parameter_t(param)
kirchner_param_max = simulator.region_model.parameter_t(param)
# Kichner parameters are quite abstract (no physical meaning), so simply scale them
kirchner_param_min.kirchner.c1 *= 0.8
kirchner_param_min.kirchner.c2 *= 0.8
kirchner_param_min.kirchner.c3 *= 0.8
kirchner_param_max.kirchner.c1 *= 1.2
kirchner_param_max.kirchner.c2 *= 1.2
kirchner_param_max.kirchner.c3 *= 1.2
# kirchner_t_start = utc.time(2011, 4, 1, 0)
# kirchner_time_axis = api.TimeAxisFixedDeltaT(kirchner_t_start, dt, 150)
kirchner_time_axis = time_axis
# Construct gamma snow parameters (realistic tx and max_lwc)
gamma_snow_param_min = simulator.region_model.parameter_t(param)
gamma_snow_param_max = simulator.region_model.parameter_t(param)
gamma_snow_param_min.gs.tx = -1.0 # Min snow/rain temperature threshold
gamma_snow_param_min.gs.max_water = 0.05 # Min 8% max water in snow in costal regions
gamma_snow_param_max.gs.tx = 1.0
gamma_snow_param_max.gs.max_water = 0.25 # Max 35% max water content, or we get too little melt
gs_t_start = utc.time(2013, 11, 1, 0)
gs_time_axis = api.TimeAxisFixedDeltaT(gs_t_start, dt, 250)
# gs_time_axis = time_axis
# Find parameters
target_spec = api.TargetSpecificationPts(target_discharge,
api.IntVector([cid]), 1.0,
api.KLING_GUPTA)
target_spec_vec = api.TargetSpecificationVector(
) # TODO: We currently dont fix list initializer for vectors
target_spec_vec.append(target_spec)
# Construct a fake, perturbed starting point for calibration
p_vec = [param.get(i) for i in range(param.size())]
for i, name in enumerate(
[param.get_name(i) for i in range(len(p_vec))]):
if name not in ("c1" "c2", "c3", "TX", "max_water"):
next
if name in ("c1", "c2", "c3"):
p_vec[i] = random.uniform(0.8 * p_vec[i], 1.2 * p_vec[i])
elif name == "TX":
p_vec[i] = random.uniform(gamma_snow_param_min.gs.tx,
gamma_snow_param_max.gs.tx)
elif name == "max_water":
p_vec[i] = random.uniform(gamma_snow_param_min.gs.max_water,
gamma_snow_param_max.gs.max_water)
param.set(p_vec)
print_param("Initial guess", param)
# Two pass optimization, once for the ground water response, and second time for
kirchner_p_opt = simulator.optimize(kirchner_time_axis,
state_repos.get_state(0),
target_spec_vec, param,
kirchner_param_min,
kirchner_param_max)
gamma_snow_p_opt = simulator.optimize(gs_time_axis,
state_repos.get_state(0),
target_spec_vec, kirchner_p_opt,
gamma_snow_param_min,
gamma_snow_param_max)
print_param("Half way result", kirchner_p_opt)
print_param("Result", gamma_snow_p_opt)
simulator.region_model.set_catchment_parameter(cid, gamma_snow_p_opt)
simulator.run(time_axis, state_repos.get_state(0))
found_discharge = simulator.region_model.statistics.discharge([cid])
def test_snow_and_ground_water_response_calibration(self):
"""
Test dual calibration strategy:
* First fit the three Kirchner parameters for
ground water response during July, August, and
September.
* Then fit two snow routine parameters (tx and max_water)
from November to April.
"""
# Simulation time axis
year, month, day, hour = 2010, 9, 1, 0
dt = api.deltahours(24)
n_steps = 400
utc = api.Calendar() # No offset gives Utc
t0 = utc.time(api.YMDhms(year, month, day, hour))
time_axis = api.Timeaxis(t0, dt, n_steps)
# Some fake ids
region_id = 0
interpolation_id = 0
# Simulation coordinate system
epsg = "32633"
# Model
model_t = pt_gs_k.PTGSKOptModel
# Configs and repositories
dataset_config_file = path.join(path.dirname(__file__), "netcdf",
"atnsjoen_datasets.yaml")
region_config_file = path.join(path.dirname(__file__), "netcdf",
"atnsjoen_calibration_region.yaml")
region_config = RegionConfig(region_config_file)
model_config = ModelConfig(self.model_config_file)
dataset_config = YamlContent(dataset_config_file)
region_model_repository = RegionModelRepository(
region_config, model_config, model_t, epsg)
interp_repos = InterpolationParameterRepository(model_config)
netcdf_geo_ts_repos = []
for source in dataset_config.sources:
station_file = source["params"]["stations_met"]
netcdf_geo_ts_repos.append(
GeoTsRepository(source["params"], station_file, ""))
geo_ts_repository = GeoTsRepositoryCollection(netcdf_geo_ts_repos)
# Construct target discharge series
simulator = DefaultSimulator(region_id, interpolation_id,
region_model_repository,
geo_ts_repository, interp_repos, None)
n_cells = simulator.region_model.size()
state_repos = DefaultStateRepository(model_t, n_cells)
simulator.run(time_axis, state_repos.get_state(0))
cid = 1
target_discharge = simulator.region_model.statistics.discharge([cid])
# Construct kirchner parameters
param = simulator.region_model.parameter_t(
simulator.region_model.get_region_parameter())
print_param("True solution", param)
kirchner_param_min = simulator.region_model.parameter_t(param)
kirchner_param_max = simulator.region_model.parameter_t(param)
# Kichner parameters are quite abstract (no physical meaning), so simply scale them
kirchner_param_min.kirchner.c1 *= 0.8
kirchner_param_min.kirchner.c2 *= 0.8
kirchner_param_min.kirchner.c3 *= 0.8
kirchner_param_max.kirchner.c1 *= 1.2
kirchner_param_max.kirchner.c2 *= 1.2
kirchner_param_max.kirchner.c3 *= 1.2
# kirchner_t_start = utc.time(api.YMDhms(2011, 4, 1, 0))
# kirchner_time_axis = api.Timeaxis(kirchner_t_start, dt, 150)
kirchner_time_axis = time_axis
# Construct gamma snow parameters (realistic tx and max_lwc)
gamma_snow_param_min = simulator.region_model.parameter_t(param)
gamma_snow_param_max = simulator.region_model.parameter_t(param)
gamma_snow_param_min.gs.tx = -1.0 # Min snow/rain temperature threshold
gamma_snow_param_min.gs.max_water = 0.05 # Min 8% max water in snow in costal regions
gamma_snow_param_max.gs.tx = 1.0
gamma_snow_param_max.gs.max_water = 0.25 # Max 35% max water content, or we get too little melt
gs_t_start = utc.time(api.YMDhms(2010, 11, 1, 0))
gs_time_axis = api.Timeaxis(gs_t_start, dt, 250)
# gs_time_axis = time_axis
# Find parameters
target_spec = api.TargetSpecificationPts(target_discharge,
api.IntVector([cid]), 1.0,
api.KLING_GUPTA)
target_spec_vec = api.TargetSpecificationVector(
) # TODO: We currently dont fix list initializer for vectors
target_spec_vec.append(target_spec)
# Construct a fake, perturbed starting point for calibration
p_vec = [param.get(i) for i in range(param.size())]
for i, name in enumerate(
[param.get_name(i) for i in range(len(p_vec))]):
if name not in ("c1" "c2", "c3", "TX", "max_water"):
next
if name in ("c1", "c2", "c3"):
p_vec[i] = random.uniform(0.8 * p_vec[i], 1.2 * p_vec[i])
elif name == "TX":
p_vec[i] = random.uniform(gamma_snow_param_min.gs.tx,
gamma_snow_param_max.gs.tx)
elif name == "max_water":
p_vec[i] = random.uniform(gamma_snow_param_min.gs.max_water,
gamma_snow_param_max.gs.max_water)
param.set(p_vec)
print_param("Initial guess", param)
# Two pass optimization, once for the ground water response, and second time for
kirchner_p_opt = simulator.optimize(kirchner_time_axis,
state_repos.get_state(0),
target_spec_vec, param,
kirchner_param_min,
kirchner_param_max)
gamma_snow_p_opt = simulator.optimize(gs_time_axis,
state_repos.get_state(0),
target_spec_vec, kirchner_p_opt,
gamma_snow_param_min,
gamma_snow_param_max)
print_param("Half way result", kirchner_p_opt)
print_param("Result", gamma_snow_p_opt)
simulator.region_model.set_catchment_parameter(cid, gamma_snow_p_opt)
simulator.run(time_axis, state_repos.get_state(0))
found_discharge = simulator.region_model.statistics.discharge([cid])
t_vs = np.array(target_discharge.v)
t_ts = np.array(
[target_discharge.time(i) for i in range(target_discharge.size())])
f_vs = np.array(found_discharge.v)
f_ts = np.array(
[found_discharge.time(i) for i in range(found_discharge.size())])
def run_calibration(self, model_t):
def param_obj_2_dict(p_obj):
p_dict = {}
[
p_dict[r].update({p: getattr(getattr(p_obj, r), p)})
if r in p_dict else
p_dict.update({r: {
p: getattr(getattr(p_obj, r), p)
}}) for r, p in [
nm.split('.')
for nm in [p_obj.get_name(i) for i in range(p_obj.size())]
]
]
return p_dict
# set up configuration
cfg = YAMLSimConfig(self.sim_config_file,
"neanidelva",
overrides={
'model': {
'model_t':
model_t,
'model_parameters':
param_obj_2_dict(model_t.parameter_t())
}
})
# create a simulator
simulator = DefaultSimulator(cfg.region_model_id,
cfg.interpolation_id,
cfg.get_region_model_repo(),
cfg.get_geots_repo(),
cfg.get_interp_repo(),
initial_state_repository=None,
catchments=None)
time_axis = cfg.time_axis.__class__(cfg.time_axis.start,
cfg.time_axis.delta_t, 2000)
state_repos = DefaultStateRepository(simulator.region_model)
s0 = state_repos.get_state(0)
param = simulator.region_model.get_region_parameter()
cid = 1228
simulator.region_model.set_calculation_filter(api.IntVector(
[cid]), api.IntVector()) # only this sub-catchment
# not needed, we auto initialize to default if not done explicitely
# if model_t in [pt_hs_k.PTHSKOptModel]:
# for i in range(len(s0)):
# s0[i].snow.distribute(param.hs)
simulator.run(time_axis=time_axis, state=s0)
target_discharge_ts = simulator.region_model.statistics.discharge(
[cid])
cell_charge = simulator.region_model.get_cells(
)[603].rc.avg_charge # in m3s for this cell
assert cell_charge.values.to_numpy().max(
) > 0.001, 'some charge expected here'
target_discharge_ts.set_point_interpretation(
api.point_interpretation_policy.POINT_AVERAGE_VALUE)
target_discharge = target_discharge_ts.average(
target_discharge_ts.time_axis)
# Perturb parameters
p_vec_orig = [param.get(i) for i in range(param.size())]
p_vec_min = p_vec_orig[:]
p_vec_max = p_vec_orig[:]
p_vec_guess = p_vec_orig[:]
random.seed(0)
p_names = []
for i in range(2):
p_names.append(param.get_name(i))
p_vec_min[i] *= 0.9
p_vec_max[i] *= 1.1
p_vec_guess[i] = random.uniform(p_vec_min[i], p_vec_max[i])
if p_vec_min[i] > p_vec_max[i]:
p_vec_min[i], p_vec_max[i] = p_vec_max[i], p_vec_min[i]
p_min = simulator.region_model.parameter_t()
p_max = simulator.region_model.parameter_t()
p_guess = simulator.region_model.parameter_t()
p_min.set(p_vec_min)
p_max.set(p_vec_max)
p_guess.set(p_vec_guess)
# Find parameters
target_spec = api.TargetSpecificationPts(target_discharge,
api.IntVector([cid]), 1.0,
api.NASH_SUTCLIFFE)
target_spec_vec = api.TargetSpecificationVector(
) # ([target_spec]) does not yet work
target_spec_vec.append(target_spec)
self.assertEqual(simulator.optimizer.trace_size,
0) # before optmize, trace_size should be 0
p_opt = simulator.optimize(time_axis, s0, target_spec_vec, p_guess,
p_min, p_max)
self.assertGreater(simulator.optimizer.trace_size,
0) # after opt, some trace values should be there
# the trace values are in the order of appearance 0...trace_size-1
#
goal_fn_values = simulator.optimizer.trace_goal_function_values.to_numpy(
) # all of them, as np array
self.assertEqual(len(goal_fn_values), simulator.optimizer.trace_size)
p_last = simulator.optimizer.trace_parameter(
simulator.optimizer.trace_size -
1) # get out the last (not neccessary the best)
self.assertIsNotNone(p_last)
simulator.region_model.set_catchment_parameter(cid, p_opt)
simulator.run(time_axis, s0)
found_discharge = simulator.region_model.statistics.discharge([cid])
t_vs = np.array([
target_discharge.value(i) for i in range(target_discharge.size())
])
t_ts = np.array([
int(target_discharge.time(i))
for i in range(target_discharge.size())
])
f_vs = np.array(
[found_discharge.value(i) for i in range(found_discharge.size())])
f_ts = np.array([
int(found_discharge.time(i)) for i in range(found_discharge.size())
])
self.assertTrue(np.linalg.norm(t_ts - f_ts) < 1.0e-10)
print(np.linalg.norm(t_vs - f_vs), np.abs(t_vs - f_vs).max())
self.assertTrue(np.linalg.norm(t_vs - f_vs) < 1.0e-3)