def test_run_geo_ts_data_config_simulator(self):
# These config files are versioned in shyft git
config_dir = path.join(path.dirname(__file__), "netcdf")
config_file = path.join(config_dir, "neanidelva_simulation.yaml")
config_section = "neanidelva"
cfg = YAMLSimConfig(config_file, config_section, overrides={'config': {'number_of_steps': 168}})
# These config files are versioned in shyft-data git. Read from ${SHYFTDATA}/netcdf/orchestration-testdata/
# TODO: Put all config files needed to run this test under the same versioning system (shyft git)
simulator = ConfigSimulator(cfg)
n_cells = simulator.region_model.size()
state_repos = DefaultStateRepository(simulator.region_model.__class__, n_cells)
simulator.run(cfg.time_axis, state_repos.get_state(0))
cids = IntVector()
discharge = simulator.region_model.statistics.discharge(cids)
# Regression tests on discharge values
self.assertAlmostEqual(discharge.values[0], 0.0957723, 3)
self.assertAlmostEqual(discharge.values[3], 3.9098, 3) #
# x self.assertAlmostEqual(discharge.values[6400], 58.8385, 3) # was 58.9381,3 before glacier&fractions adjustments
# x self.assertAlmostEqual(discharge.values[3578],5.5069,3)
# glacier_melt, not much, but enough to test
# x self.assertAlmostEqual(simulator.region_model.gamma_snow_response.glacier_melt(cids).values.to_numpy().max(),0.201625547258,4)
self.assertAlmostEqual(simulator.region_model.gamma_snow_response.glacier_melt(cids).values.to_numpy().max(), 0.12393672891230645, 4)
# Regression tests on geo fractions
self.assertAlmostEqual(simulator.region_model.cells[0].geo.land_type_fractions_info().unspecified(), 1.0, 3)
self.assertAlmostEqual(simulator.region_model.cells[2].geo.land_type_fractions_info().unspecified(), 0.1433, 3)
self.assertAlmostEqual(simulator.region_model.cells[2].geo.land_type_fractions_info().forest(), 0.0, 3)
self.assertAlmostEqual(simulator.region_model.cells[2].geo.land_type_fractions_info().reservoir(), 0.8566, 3)
def test_set_observed_state(self):
# set up configuration
cfg = YAMLSimConfig(self.sim_config_file, "neanidelva")
# create a simulator
simulator = DefaultSimulator(cfg.region_model_id,
cfg.interpolation_id,
cfg.get_region_model_repo(),
cfg.get_geots_repo(),
cfg.get_interp_repo(),
initial_state_repository=None,
catchments=None)
state_repos = DefaultStateRepository(simulator.region_model)
state = state_repos.get_state(0)
small_time_axis = cfg.time_axis.__class__(cfg.time_axis.start,
cfg.time_axis.delta_t, 100)
simulator.run(
time_axis=small_time_axis, state=state
) # Here we have already checked if StateIDs match with model cell ID. Further check is redundant.
# simulator.region_model.get_states(state.state_vector)
state = simulator.reg_model_state
obs_discharge = 0.0
state = simulator.discharge_adjusted_state(obs_discharge, state)
self.assertAlmostEqual(
0.0,
reduce(operator.add,
(state[i].state.kirchner.q for i in range(len(state)))))
# simulator.region_model.get_states(state.state_vector)
state = simulator.reg_model_state
obs_discharge = 10.0 # m3/s
state = simulator.discharge_adjusted_state(obs_discharge, state)
# Convert from l/h to m3/s by dividing by 3.6e6
adj_discharge = reduce(
operator.add,
(state[i].state.kirchner.q * cell.geo.area()
for (i, cell) in enumerate(simulator.region_model.get_cells())
)) / (3.6e6)
self.assertAlmostEqual(obs_discharge, adj_discharge)
def test_run_geo_ts_data_simulator(self):
# set up configuration
cfg = YAMLSimConfig(self.sim_config_file, "neanidelva")
# create a simulator
simulator = DefaultSimulator(cfg.region_model_id,
cfg.interpolation_id,
cfg.get_region_model_repo(),
cfg.get_geots_repo(),
cfg.get_interp_repo(),
initial_state_repository=None,
catchments=None)
state_repos = DefaultStateRepository(simulator.region_model)
simulator.region_model.set_calculation_filter(api.IntVector([1228]),
api.IntVector())
simulator.run(time_axis=cfg.time_axis, state=state_repos.get_state(0))
sim_copy = simulator.copy()
sim_copy.region_model.set_calculation_filter(api.IntVector([1228]),
api.IntVector())
sim_copy.run(cfg.time_axis, state_repos.get_state(0))
def test_run_geo_ts_data_config_simulator(self):
# set up configuration
config_dir = path.join(path.dirname(__file__), "netcdf")
config_file = path.join(config_dir,"neanidelva_simulation.yaml")
config_section = "neanidelva"
cfg = YAMLSimConfig(config_file, config_section)
# get a simulator
simulator = ConfigSimulator(cfg)
n_cells = simulator.region_model.size()
state_repos = DefaultStateRepository(simulator.region_model.__class__, n_cells)
simulator.run(cfg.time_axis, state_repos.get_state(0))
def test_run_geo_ts_data_config_simulator(self):
# These config files are versioned in shyft-data git
config_dir = path.join(shyftdata_dir, "neanidelv", "yaml_config")
config_file = path.join(config_dir, "neanidelva_simulation.yaml")
config_section = "neanidelva"
cfg = YAMLSimConfig(config_file,
config_section,
overrides={'config': {
'number_of_steps': 168
}})
# These config files are versioned in shyft-data git. Read from ${SHYFTDATA}/netcdf/orchestration-testdata/
simulator = ConfigSimulator(cfg)
# Regression tests on interpolation parameters
self.assertAlmostEqual(
simulator.region_model.interpolation_parameter.precipitation.
scale_factor, 1.01, 3)
#n_cells = simulator.region_model.size()
state_repos = DefaultStateRepository(simulator.region_model)
simulator.run(cfg.time_axis, state_repos.get_state(0))
cids = IntVector()
discharge = simulator.region_model.statistics.discharge(cids)
# Regression tests on discharge values
self.assertAlmostEqual(discharge.values[0], 80.23843199, 3)
self.assertAlmostEqual(discharge.values[3], 82.50344985, 3)
# Regression tests on geo fractions
self.assertAlmostEqual(
simulator.region_model.cells[0].geo.land_type_fractions_info().
unspecified(), 1.0, 3)
self.assertAlmostEqual(
simulator.region_model.cells[2].geo.land_type_fractions_info().
unspecified(), 0.1433, 3)
self.assertAlmostEqual(
simulator.region_model.cells[2].geo.land_type_fractions_info().
forest(), 0.0, 3)
self.assertAlmostEqual(
simulator.region_model.cells[2].geo.land_type_fractions_info().
reservoir(), 0.8566, 3)
def test_run_geo_ts_data_config_simulator(self):
# set up configuration
config_dir = path.join(path.dirname(__file__), "netcdf")
config_file = path.join(config_dir, "neanidelva_simulation.yaml")
config_section = "neanidelva"
cfg = YAMLSimConfig(config_file, config_section)
# get a simulator
simulator = ConfigSimulator(cfg)
n_cells = simulator.region_model.size()
state_repos = DefaultStateRepository(simulator.region_model.__class__,
n_cells)
simulator.run(cfg.time_axis, state_repos.get_state(0))
cids = IntVector()
discharge = simulator.region_model.statistics.discharge(cids)
# regression test on discharge values
self.assertAlmostEqual(discharge.values[0], 0.1961, 3)
self.assertAlmostEqual(discharge.values[3], 2.7582, 3)
self.assertAlmostEqual(discharge.values[6400], 58.9381, 3)
self.assertAlmostEqual(discharge.values[3578], 5.5069, 3)
# regression test on geo fractions
self.assertAlmostEqual(
simulator.region_model.cells[0].geo.land_type_fractions_info().
unspecified(), 1.0, 3)
self.assertAlmostEqual(
simulator.region_model.cells[2].geo.land_type_fractions_info().
unspecified(), 0.1433, 3)
self.assertAlmostEqual(
simulator.region_model.cells[2].geo.land_type_fractions_info().
forest(), 0.0, 3)
self.assertAlmostEqual(
simulator.region_model.cells[2].geo.land_type_fractions_info().
reservoir(), 0.8566, 3)
self.assertAlmostEqual(
simulator.region_model.cells[3383].geo.land_type_fractions_info().
lake(), 0.7432, 3)
self.assertAlmostEqual(
simulator.region_model.cells[652].geo.land_type_fractions_info().
glacier(), 0.1351, 3)
def test_run_geo_ts_data_config_simulator(self):
# These config files are versioned in shyft git
config_dir = path.join(path.dirname(__file__), "netcdf")
config_file = path.join(config_dir, "neanidelva_simulation.yaml")
config_section = "neanidelva"
cfg = YAMLSimConfig(config_file,
config_section,
overrides={'config': {
'number_of_steps': 168
}})
# These config files are versioned in shyft-data git. Read from ${SHYFTDATA}/netcdf/orchestration-testdata/
# TODO: Put all config files needed to run this test under the same versioning system (shyft git)
simulator = ConfigSimulator(cfg)
#n_cells = simulator.region_model.size()
state_repos = DefaultStateRepository(simulator.region_model)
simulator.run(cfg.time_axis, state_repos.get_state(0))
cids = IntVector()
discharge = simulator.region_model.statistics.discharge(cids)
# Regression tests on discharge values
self.assertAlmostEqual(discharge.values[0], 0.1001063, 3)
self.assertAlmostEqual(discharge.values[3], 3.9141928, 3)
# Regression tests on geo fractions
self.assertAlmostEqual(
simulator.region_model.cells[0].geo.land_type_fractions_info().
unspecified(), 1.0, 3)
self.assertAlmostEqual(
simulator.region_model.cells[2].geo.land_type_fractions_info().
unspecified(), 0.1433, 3)
self.assertAlmostEqual(
simulator.region_model.cells[2].geo.land_type_fractions_info().
forest(), 0.0, 3)
self.assertAlmostEqual(
simulator.region_model.cells[2].geo.land_type_fractions_info().
reservoir(), 0.8566, 3)
import datetime as dt #import pandas as pd import numpy as np import math from matplotlib import pyplot as plt from shyft import api # importing the shyft modules needed for running a calibration from shyft.repository.default_state_repository import DefaultStateRepository from shyft.orchestration.configuration.yaml_configs import YAMLCalibConfig, YAMLSimConfig from shyft.orchestration.simulators.config_simulator import ConfigCalibrator, ConfigSimulator # conduct a configured simulation first. config_file_path = '/home/olga/workspace/shyft-data/narayani/yaml_config-rpmgsk/narayani_simulation.yaml' # config_file_path = '/home/olga/workspace/shyft-data/narayani/yaml_config-ptgsk/narayani_simulation.yaml' cfg = YAMLSimConfig(config_file_path, "narayani") simulator = ConfigSimulator(cfg) # run the model, and we'll just pull the `api.model` from the `simulator` simulator.run() state = simulator.region_model.state config_file_path = '/home/olga/workspace/shyft-data/narayani/yaml_config-rpmgsk/narayani_calibration.yaml' # here is the *.yaml file # config_file_path = '/home/olga/workspace/shyft-data/narayani/yaml_config-ptgsk/narayani_calibration.yaml' # here is the *.yaml file cfg = YAMLCalibConfig(config_file_path, "narayani") # config_file_path = '/home/olga/workspace/shyft-data/neanidelv/yaml_config/neanidelva_simulation.yaml' # here is the *.yaml file # cfg = YAMLSimConfig(config_file_path, "neanidelva") # to run a calibration using the above initiated configuration
all_results, good_results = [], []
counter = -1
while counter < 0:
shyft_data_path = path.abspath(r"C:\shyft_workspace\shyft-data")
if path.exists(shyft_data_path) and 'SHYFT_DATA' not in os.environ:
os.environ['SHYFT_DATA']=shyft_data_path
from shyft.repository.default_state_repository import DefaultStateRepository
from shyft.orchestration.configuration.yaml_configs import YAMLCalibConfig, YAMLSimConfig
from shyft.orchestration.simulators.config_simulator import ConfigCalibrator, ConfigSimulator
counter += 1
t1 = time.time()
config_file_path = os.path.abspath(r"D:\Dropbox\Thesis\SHyFT\Yaml_files\Skaugen\neanidelva_simulation.yaml")
cfg = YAMLSimConfig(config_file_path, "neanidelva")
simulator = ConfigSimulator(cfg)
simulator.run()
state = simulator.region_model.state
region_model = simulator.region_model
config_file_path = os.path.abspath(r"D:\Dropbox\Thesis\SHyFT\Yaml_files\Skaugen\neanidelva_simulation.yaml")
cfg = YAMLCalibConfig(config_file_path, "neanidelva")
calib = ConfigCalibrator(cfg)
cfg.optimization_method['params']['tr_start'] = random.randrange(1,2000)/10000
state_repos = DefaultStateRepository(calib.region_model)
results = calib.calibrate(cfg.sim_config.time_axis, state_repos.get_state(0).state_vector,
cfg.optimization_method['name'], cfg.optimization_method['params'])
t2 = time.time()
# print(f" velocity: {my_data.iloc[16][column]}", file=parameters2)
time.sleep(5)
shyft_data_path = path.abspath(r"C:\shyft_workspace\shyft-data")
if path.exists(shyft_data_path) and 'SHYFT_DATA' not in os.environ:
os.environ['SHYFT_DATA'] = shyft_data_path
import shyft
from shyft import api
from shyft.repository.default_state_repository import DefaultStateRepository
from shyft.orchestration.configuration.yaml_configs import YAMLSimConfig
from shyft.orchestration.simulators.config_simulator import ConfigSimulator
config_file_path = r'D:\Dropbox\Thesis\SHyFT\neanidelva_simulation.yaml'
cfg = YAMLSimConfig(config_file_path, "neanidelva")
simulator = ConfigSimulator(cfg)
region_model = simulator.region_model
simulator.region_model.set_snow_sca_swe_collection(-1, True)
simulator.region_model.set_state_collection(-1, True)
simulator.run()
cells = region_model.get_cells()
x = np.array([cell.geo.mid_point().x for cell in cells])
y = np.array([cell.geo.mid_point().y for cell in cells])
z = np.array([cell.geo.mid_point().z for cell in cells])
area = np.array([cell.geo.area() for cell in cells])
catch_ids = np.array([cell.geo.catchment_id() for cell in cells])
def run_calibration(self, model_t):
def param_obj_2_dict(p_obj):
p_dict = {}
[
p_dict[r].update({p: getattr(getattr(p_obj, r), p)})
if r in p_dict else
p_dict.update({r: {
p: getattr(getattr(p_obj, r), p)
}}) for r, p in [
nm.split('.')
for nm in [p_obj.get_name(i) for i in range(p_obj.size())]
]
]
return p_dict
# set up configuration
cfg = YAMLSimConfig(self.sim_config_file,
"neanidelva",
overrides={
'model': {
'model_t':
model_t,
'model_parameters':
param_obj_2_dict(model_t.parameter_t())
}
})
# create a simulator
simulator = DefaultSimulator(cfg.region_model_id,
cfg.interpolation_id,
cfg.get_region_model_repo(),
cfg.get_geots_repo(),
cfg.get_interp_repo(),
initial_state_repository=None,
catchments=None)
time_axis = cfg.time_axis.__class__(cfg.time_axis.start,
cfg.time_axis.delta_t, 2000)
state_repos = DefaultStateRepository(simulator.region_model)
s0 = state_repos.get_state(0)
param = simulator.region_model.get_region_parameter()
cid = 1228
simulator.region_model.set_calculation_filter(api.IntVector(
[cid]), api.IntVector()) # only this sub-catchment
# not needed, we auto initialize to default if not done explicitely
# if model_t in [pt_hs_k.PTHSKOptModel]:
# for i in range(len(s0)):
# s0[i].snow.distribute(param.hs)
simulator.run(time_axis=time_axis, state=s0)
target_discharge_ts = simulator.region_model.statistics.discharge(
[cid])
cell_charge = simulator.region_model.get_cells(
)[603].rc.avg_charge # in m3s for this cell
assert cell_charge.values.to_numpy().max(
) > 0.001, 'some charge expected here'
target_discharge_ts.set_point_interpretation(
api.point_interpretation_policy.POINT_AVERAGE_VALUE)
target_discharge = target_discharge_ts.average(
target_discharge_ts.time_axis)
# Perturb parameters
p_vec_orig = [param.get(i) for i in range(param.size())]
p_vec_min = p_vec_orig[:]
p_vec_max = p_vec_orig[:]
p_vec_guess = p_vec_orig[:]
random.seed(0)
p_names = []
for i in range(2):
p_names.append(param.get_name(i))
p_vec_min[i] *= 0.9
p_vec_max[i] *= 1.1
p_vec_guess[i] = random.uniform(p_vec_min[i], p_vec_max[i])
if p_vec_min[i] > p_vec_max[i]:
p_vec_min[i], p_vec_max[i] = p_vec_max[i], p_vec_min[i]
p_min = simulator.region_model.parameter_t()
p_max = simulator.region_model.parameter_t()
p_guess = simulator.region_model.parameter_t()
p_min.set(p_vec_min)
p_max.set(p_vec_max)
p_guess.set(p_vec_guess)
# Find parameters
target_spec = api.TargetSpecificationPts(target_discharge,
api.IntVector([cid]), 1.0,
api.NASH_SUTCLIFFE)
target_spec_vec = api.TargetSpecificationVector(
) # ([target_spec]) does not yet work
target_spec_vec.append(target_spec)
self.assertEqual(simulator.optimizer.trace_size,
0) # before optmize, trace_size should be 0
p_opt = simulator.optimize(time_axis, s0, target_spec_vec, p_guess,
p_min, p_max)
self.assertGreater(simulator.optimizer.trace_size,
0) # after opt, some trace values should be there
# the trace values are in the order of appearance 0...trace_size-1
#
goal_fn_values = simulator.optimizer.trace_goal_function_values.to_numpy(
) # all of them, as np array
self.assertEqual(len(goal_fn_values), simulator.optimizer.trace_size)
p_last = simulator.optimizer.trace_parameter(
simulator.optimizer.trace_size -
1) # get out the last (not neccessary the best)
self.assertIsNotNone(p_last)
simulator.region_model.set_catchment_parameter(cid, p_opt)
simulator.run(time_axis, s0)
found_discharge = simulator.region_model.statistics.discharge([cid])
t_vs = np.array([
target_discharge.value(i) for i in range(target_discharge.size())
])
t_ts = np.array([
int(target_discharge.time(i))
for i in range(target_discharge.size())
])
f_vs = np.array(
[found_discharge.value(i) for i in range(found_discharge.size())])
f_ts = np.array([
int(found_discharge.time(i)) for i in range(found_discharge.size())
])
self.assertTrue(np.linalg.norm(t_ts - f_ts) < 1.0e-10)
print(np.linalg.norm(t_vs - f_vs), np.abs(t_vs - f_vs).max())
self.assertTrue(np.linalg.norm(t_vs - f_vs) < 1.0e-3)