def test_snow_and_ground_water_response_calibration(self): """ Test dual calibration strategy: * First fit the three Kirchner parameters for ground water response during July, August, and September. * Then fit two snow routine parameters (tx and max_water) from November to April. """ # Simulation time axis year, month, day, hour = 2010, 9, 1, 0 dt = api.deltahours(24) n_steps = 400 utc = api.Calendar() # No offset gives Utc t0 = utc.time(api.YMDhms(year, month, day, hour)) time_axis = api.Timeaxis(t0, dt, n_steps) # Some fake ids region_id = 0 interpolation_id = 0 # Simulation coordinate system epsg = "32633" # Model model_t = pt_gs_k.PTGSKOptModel # Configs and repositories dataset_config_file = path.join(path.dirname(__file__), "netcdf", "atnsjoen_datasets.yaml") region_config_file = path.join(path.dirname(__file__), "netcdf", "atnsjoen_calibration_region.yaml") region_config = RegionConfig(region_config_file) model_config = ModelConfig(self.model_config_file) dataset_config = YamlContent(dataset_config_file) region_model_repository = RegionModelRepository(region_config, model_config, model_t, epsg) interp_repos = InterpolationParameterRepository(model_config) netcdf_geo_ts_repos = [] for source in dataset_config.sources: station_file = source["params"]["stations_met"] netcdf_geo_ts_repos.append(GeoTsRepository(source["params"], station_file, "")) geo_ts_repository = GeoTsRepositoryCollection(netcdf_geo_ts_repos) # Construct target discharge series simulator = DefaultSimulator(region_id, interpolation_id, region_model_repository, geo_ts_repository, interp_repos, None) n_cells = simulator.region_model.size() state_repos = DefaultStateRepository(model_t, n_cells) simulator.run(time_axis, state_repos.get_state(0)) cid = 1 target_discharge = simulator.region_model.statistics.discharge([cid]) # Construct kirchner parameters param = simulator.region_model.parameter_t(simulator.region_model.get_region_parameter()) print_param("True solution", param) kirchner_param_min = simulator.region_model.parameter_t(param) kirchner_param_max = simulator.region_model.parameter_t(param) # Kichner parameters are quite abstract (no physical meaning), so simply scale them kirchner_param_min.kirchner.c1 *= 0.8 kirchner_param_min.kirchner.c2 *= 0.8 kirchner_param_min.kirchner.c3 *= 0.8 kirchner_param_max.kirchner.c1 *= 1.2 kirchner_param_max.kirchner.c2 *= 1.2 kirchner_param_max.kirchner.c3 *= 1.2 # kirchner_t_start = utc.time(api.YMDhms(2011, 4, 1, 0)) # kirchner_time_axis = api.Timeaxis(kirchner_t_start, dt, 150) kirchner_time_axis = time_axis # Construct gamma snow parameters (realistic tx and max_lwc) gamma_snow_param_min = simulator.region_model.parameter_t(param) gamma_snow_param_max = simulator.region_model.parameter_t(param) gamma_snow_param_min.gs.tx = -1.0 # Min snow/rain temperature threshold gamma_snow_param_min.gs.max_water = 0.05 # Min 8% max water in snow in costal regions gamma_snow_param_max.gs.tx = 1.0 gamma_snow_param_max.gs.max_water = 0.25 # Max 35% max water content, or we get too little melt gs_t_start = utc.time(api.YMDhms(2010, 11, 1, 0)) gs_time_axis = api.Timeaxis(gs_t_start, dt, 250) # gs_time_axis = time_axis # Find parameters target_spec = api.TargetSpecificationPts(target_discharge, api.IntVector([cid]), 1.0, api.KLING_GUPTA) target_spec_vec = api.TargetSpecificationVector() # TODO: We currently dont fix list initializer for vectors target_spec_vec.append(target_spec) # Construct a fake, perturbed starting point for calibration p_vec = [param.get(i) for i in range(param.size())] for i, name in enumerate([param.get_name(i) for i in range(len(p_vec))]): if name not in ("c1" "c2", "c3", "TX", "max_water"): next if name in ("c1", "c2", "c3"): p_vec[i] = random.uniform(0.8*p_vec[i], 1.2*p_vec[i]) elif name == "TX": p_vec[i] = random.uniform(gamma_snow_param_min.gs.tx, gamma_snow_param_max.gs.tx) elif name == "max_water": p_vec[i] = random.uniform(gamma_snow_param_min.gs.max_water, gamma_snow_param_max.gs.max_water) param.set(p_vec) print_param("Initial guess", param) # Two pass optimization, once for the ground water response, and second time for kirchner_p_opt = simulator.optimize(kirchner_time_axis, state_repos.get_state(0), target_spec_vec, param, kirchner_param_min, kirchner_param_max) gamma_snow_p_opt = simulator.optimize(gs_time_axis, state_repos.get_state(0), target_spec_vec, kirchner_p_opt, gamma_snow_param_min, gamma_snow_param_max) print_param("Half way result", kirchner_p_opt) print_param("Result", gamma_snow_p_opt) simulator.region_model.set_catchment_parameter(cid, gamma_snow_p_opt) simulator.run(time_axis, state_repos.get_state(0)) found_discharge = simulator.region_model.statistics.discharge([cid]) t_vs = np.array(target_discharge.v) t_ts = np.array([target_discharge.time(i) for i in range(target_discharge.size())]) f_vs = np.array(found_discharge.v) f_ts = np.array([found_discharge.time(i) for i in range(found_discharge.size())])
def test_snow_and_ground_water_response_calibration(self): """ Test dual calibration strategy: * First fit the three Kirchner parameters for ground water response during July, August, and September. * Then fit two snow routine parameters (tx and max_water) from November to April. """ # Simulation time axis dt = api.deltahours(24) n_steps = 364 utc = api.Calendar() # No offset gives Utc t0 = utc.time(2013, 9, 1, 0) time_axis = api.TimeAxisFixedDeltaT(t0, dt, n_steps) # Some fake ids region_id = 0 interpolation_id = 0 opt_model_t = self.model_config.model_type().opt_model_t model_config = ModelConfig(self.model_config_file, overrides={'model_t': opt_model_t}) region_model_repository = CFRegionModelRepository( self.region_config, model_config) interp_repos = InterpolationParameterRepository( self.interpolation_config) netcdf_geo_ts_repos = [ CFDataRepository(32633, source["params"]["filename"], padding=source["params"]['padding']) for source in self.dataset_config.sources ] geo_ts_repository = GeoTsRepositoryCollection(netcdf_geo_ts_repos) # Construct target discharge series simulator = DefaultSimulator(region_id, interpolation_id, region_model_repository, geo_ts_repository, interp_repos, None) state_repos = DefaultStateRepository(simulator.region_model) simulator.run(time_axis, state_repos.get_state(0)) cid = 1228 target_discharge = api.TsTransform().to_average( t0, dt, n_steps, simulator.region_model.statistics.discharge([cid])) # Construct kirchner parameters param = simulator.region_model.parameter_t( simulator.region_model.get_region_parameter()) print_param("True solution", param) kirchner_param_min = simulator.region_model.parameter_t(param) kirchner_param_max = simulator.region_model.parameter_t(param) # Kichner parameters are quite abstract (no physical meaning), so simply scale them kirchner_param_min.kirchner.c1 *= 0.8 kirchner_param_min.kirchner.c2 *= 0.8 kirchner_param_min.kirchner.c3 *= 0.8 kirchner_param_max.kirchner.c1 *= 1.2 kirchner_param_max.kirchner.c2 *= 1.2 kirchner_param_max.kirchner.c3 *= 1.2 # kirchner_t_start = utc.time(2011, 4, 1, 0) # kirchner_time_axis = api.TimeAxisFixedDeltaT(kirchner_t_start, dt, 150) kirchner_time_axis = time_axis # Construct gamma snow parameters (realistic tx and max_lwc) gamma_snow_param_min = simulator.region_model.parameter_t(param) gamma_snow_param_max = simulator.region_model.parameter_t(param) gamma_snow_param_min.gs.tx = -1.0 # Min snow/rain temperature threshold gamma_snow_param_min.gs.max_water = 0.05 # Min 8% max water in snow in costal regions gamma_snow_param_max.gs.tx = 1.0 gamma_snow_param_max.gs.max_water = 0.25 # Max 35% max water content, or we get too little melt gs_t_start = utc.time(2013, 11, 1, 0) gs_time_axis = api.TimeAxisFixedDeltaT(gs_t_start, dt, 250) # gs_time_axis = time_axis # Find parameters target_spec = api.TargetSpecificationPts(target_discharge, api.IntVector([cid]), 1.0, api.KLING_GUPTA) target_spec_vec = api.TargetSpecificationVector( ) # TODO: We currently dont fix list initializer for vectors target_spec_vec.append(target_spec) # Construct a fake, perturbed starting point for calibration p_vec = [param.get(i) for i in range(param.size())] for i, name in enumerate( [param.get_name(i) for i in range(len(p_vec))]): if name not in ("c1" "c2", "c3", "TX", "max_water"): next if name in ("c1", "c2", "c3"): p_vec[i] = random.uniform(0.8 * p_vec[i], 1.2 * p_vec[i]) elif name == "TX": p_vec[i] = random.uniform(gamma_snow_param_min.gs.tx, gamma_snow_param_max.gs.tx) elif name == "max_water": p_vec[i] = random.uniform(gamma_snow_param_min.gs.max_water, gamma_snow_param_max.gs.max_water) param.set(p_vec) print_param("Initial guess", param) # Two pass optimization, once for the ground water response, and second time for kirchner_p_opt = simulator.optimize(kirchner_time_axis, state_repos.get_state(0), target_spec_vec, param, kirchner_param_min, kirchner_param_max) gamma_snow_p_opt = simulator.optimize(gs_time_axis, state_repos.get_state(0), target_spec_vec, kirchner_p_opt, gamma_snow_param_min, gamma_snow_param_max) print_param("Half way result", kirchner_p_opt) print_param("Result", gamma_snow_p_opt) simulator.region_model.set_catchment_parameter(cid, gamma_snow_p_opt) simulator.run(time_axis, state_repos.get_state(0)) found_discharge = simulator.region_model.statistics.discharge([cid])
def test_snow_and_ground_water_response_calibration(self): """ Test dual calibration strategy: * First fit the three Kirchner parameters for ground water response during July, August, and September. * Then fit two snow routine parameters (tx and max_water) from November to April. """ # Simulation time axis year, month, day, hour = 2010, 9, 1, 0 dt = api.deltahours(24) n_steps = 400 utc = api.Calendar() # No offset gives Utc t0 = utc.time(api.YMDhms(year, month, day, hour)) time_axis = api.Timeaxis(t0, dt, n_steps) # Some fake ids region_id = 0 interpolation_id = 0 # Simulation coordinate system epsg = "32633" # Model model_t = pt_gs_k.PTGSKOptModel # Configs and repositories dataset_config_file = path.join(path.dirname(__file__), "netcdf", "atnsjoen_datasets.yaml") region_config_file = path.join(path.dirname(__file__), "netcdf", "atnsjoen_calibration_region.yaml") region_config = RegionConfig(region_config_file) model_config = ModelConfig(self.model_config_file) dataset_config = YamlContent(dataset_config_file) region_model_repository = RegionModelRepository( region_config, model_config, model_t, epsg) interp_repos = InterpolationParameterRepository(model_config) netcdf_geo_ts_repos = [] for source in dataset_config.sources: station_file = source["params"]["stations_met"] netcdf_geo_ts_repos.append( GeoTsRepository(source["params"], station_file, "")) geo_ts_repository = GeoTsRepositoryCollection(netcdf_geo_ts_repos) # Construct target discharge series simulator = DefaultSimulator(region_id, interpolation_id, region_model_repository, geo_ts_repository, interp_repos, None) n_cells = simulator.region_model.size() state_repos = DefaultStateRepository(model_t, n_cells) simulator.run(time_axis, state_repos.get_state(0)) cid = 1 target_discharge = simulator.region_model.statistics.discharge([cid]) # Construct kirchner parameters param = simulator.region_model.parameter_t( simulator.region_model.get_region_parameter()) print_param("True solution", param) kirchner_param_min = simulator.region_model.parameter_t(param) kirchner_param_max = simulator.region_model.parameter_t(param) # Kichner parameters are quite abstract (no physical meaning), so simply scale them kirchner_param_min.kirchner.c1 *= 0.8 kirchner_param_min.kirchner.c2 *= 0.8 kirchner_param_min.kirchner.c3 *= 0.8 kirchner_param_max.kirchner.c1 *= 1.2 kirchner_param_max.kirchner.c2 *= 1.2 kirchner_param_max.kirchner.c3 *= 1.2 # kirchner_t_start = utc.time(api.YMDhms(2011, 4, 1, 0)) # kirchner_time_axis = api.Timeaxis(kirchner_t_start, dt, 150) kirchner_time_axis = time_axis # Construct gamma snow parameters (realistic tx and max_lwc) gamma_snow_param_min = simulator.region_model.parameter_t(param) gamma_snow_param_max = simulator.region_model.parameter_t(param) gamma_snow_param_min.gs.tx = -1.0 # Min snow/rain temperature threshold gamma_snow_param_min.gs.max_water = 0.05 # Min 8% max water in snow in costal regions gamma_snow_param_max.gs.tx = 1.0 gamma_snow_param_max.gs.max_water = 0.25 # Max 35% max water content, or we get too little melt gs_t_start = utc.time(api.YMDhms(2010, 11, 1, 0)) gs_time_axis = api.Timeaxis(gs_t_start, dt, 250) # gs_time_axis = time_axis # Find parameters target_spec = api.TargetSpecificationPts(target_discharge, api.IntVector([cid]), 1.0, api.KLING_GUPTA) target_spec_vec = api.TargetSpecificationVector( ) # TODO: We currently dont fix list initializer for vectors target_spec_vec.append(target_spec) # Construct a fake, perturbed starting point for calibration p_vec = [param.get(i) for i in range(param.size())] for i, name in enumerate( [param.get_name(i) for i in range(len(p_vec))]): if name not in ("c1" "c2", "c3", "TX", "max_water"): next if name in ("c1", "c2", "c3"): p_vec[i] = random.uniform(0.8 * p_vec[i], 1.2 * p_vec[i]) elif name == "TX": p_vec[i] = random.uniform(gamma_snow_param_min.gs.tx, gamma_snow_param_max.gs.tx) elif name == "max_water": p_vec[i] = random.uniform(gamma_snow_param_min.gs.max_water, gamma_snow_param_max.gs.max_water) param.set(p_vec) print_param("Initial guess", param) # Two pass optimization, once for the ground water response, and second time for kirchner_p_opt = simulator.optimize(kirchner_time_axis, state_repos.get_state(0), target_spec_vec, param, kirchner_param_min, kirchner_param_max) gamma_snow_p_opt = simulator.optimize(gs_time_axis, state_repos.get_state(0), target_spec_vec, kirchner_p_opt, gamma_snow_param_min, gamma_snow_param_max) print_param("Half way result", kirchner_p_opt) print_param("Result", gamma_snow_p_opt) simulator.region_model.set_catchment_parameter(cid, gamma_snow_p_opt) simulator.run(time_axis, state_repos.get_state(0)) found_discharge = simulator.region_model.statistics.discharge([cid]) t_vs = np.array(target_discharge.v) t_ts = np.array( [target_discharge.time(i) for i in range(target_discharge.size())]) f_vs = np.array(found_discharge.v) f_ts = np.array( [found_discharge.time(i) for i in range(found_discharge.size())])
def run_calibration(self, model_t): def param_obj_2_dict(p_obj): p_dict = {} [ p_dict[r].update({p: getattr(getattr(p_obj, r), p)}) if r in p_dict else p_dict.update({r: { p: getattr(getattr(p_obj, r), p) }}) for r, p in [ nm.split('.') for nm in [p_obj.get_name(i) for i in range(p_obj.size())] ] ] return p_dict # set up configuration cfg = YAMLSimConfig(self.sim_config_file, "neanidelva", overrides={ 'model': { 'model_t': model_t, 'model_parameters': param_obj_2_dict(model_t.parameter_t()) } }) # create a simulator simulator = DefaultSimulator(cfg.region_model_id, cfg.interpolation_id, cfg.get_region_model_repo(), cfg.get_geots_repo(), cfg.get_interp_repo(), initial_state_repository=None, catchments=None) time_axis = cfg.time_axis.__class__(cfg.time_axis.start, cfg.time_axis.delta_t, 2000) state_repos = DefaultStateRepository(simulator.region_model) s0 = state_repos.get_state(0) param = simulator.region_model.get_region_parameter() cid = 1228 simulator.region_model.set_calculation_filter(api.IntVector( [cid]), api.IntVector()) # only this sub-catchment # not needed, we auto initialize to default if not done explicitely # if model_t in [pt_hs_k.PTHSKOptModel]: # for i in range(len(s0)): # s0[i].snow.distribute(param.hs) simulator.run(time_axis=time_axis, state=s0) target_discharge_ts = simulator.region_model.statistics.discharge( [cid]) cell_charge = simulator.region_model.get_cells( )[603].rc.avg_charge # in m3s for this cell assert cell_charge.values.to_numpy().max( ) > 0.001, 'some charge expected here' target_discharge_ts.set_point_interpretation( api.point_interpretation_policy.POINT_AVERAGE_VALUE) target_discharge = target_discharge_ts.average( target_discharge_ts.time_axis) # Perturb parameters p_vec_orig = [param.get(i) for i in range(param.size())] p_vec_min = p_vec_orig[:] p_vec_max = p_vec_orig[:] p_vec_guess = p_vec_orig[:] random.seed(0) p_names = [] for i in range(2): p_names.append(param.get_name(i)) p_vec_min[i] *= 0.9 p_vec_max[i] *= 1.1 p_vec_guess[i] = random.uniform(p_vec_min[i], p_vec_max[i]) if p_vec_min[i] > p_vec_max[i]: p_vec_min[i], p_vec_max[i] = p_vec_max[i], p_vec_min[i] p_min = simulator.region_model.parameter_t() p_max = simulator.region_model.parameter_t() p_guess = simulator.region_model.parameter_t() p_min.set(p_vec_min) p_max.set(p_vec_max) p_guess.set(p_vec_guess) # Find parameters target_spec = api.TargetSpecificationPts(target_discharge, api.IntVector([cid]), 1.0, api.NASH_SUTCLIFFE) target_spec_vec = api.TargetSpecificationVector( ) # ([target_spec]) does not yet work target_spec_vec.append(target_spec) self.assertEqual(simulator.optimizer.trace_size, 0) # before optmize, trace_size should be 0 p_opt = simulator.optimize(time_axis, s0, target_spec_vec, p_guess, p_min, p_max) self.assertGreater(simulator.optimizer.trace_size, 0) # after opt, some trace values should be there # the trace values are in the order of appearance 0...trace_size-1 # goal_fn_values = simulator.optimizer.trace_goal_function_values.to_numpy( ) # all of them, as np array self.assertEqual(len(goal_fn_values), simulator.optimizer.trace_size) p_last = simulator.optimizer.trace_parameter( simulator.optimizer.trace_size - 1) # get out the last (not neccessary the best) self.assertIsNotNone(p_last) simulator.region_model.set_catchment_parameter(cid, p_opt) simulator.run(time_axis, s0) found_discharge = simulator.region_model.statistics.discharge([cid]) t_vs = np.array([ target_discharge.value(i) for i in range(target_discharge.size()) ]) t_ts = np.array([ int(target_discharge.time(i)) for i in range(target_discharge.size()) ]) f_vs = np.array( [found_discharge.value(i) for i in range(found_discharge.size())]) f_ts = np.array([ int(found_discharge.time(i)) for i in range(found_discharge.size()) ]) self.assertTrue(np.linalg.norm(t_ts - f_ts) < 1.0e-10) print(np.linalg.norm(t_vs - f_vs), np.abs(t_vs - f_vs).max()) self.assertTrue(np.linalg.norm(t_vs - f_vs) < 1.0e-3)