def __init__(self, config):
self._config = config
self._model_config = config.model_config
self.runner = ShyftRunner(self._model_config)
self.calibrator = None
self.target_ts = {}
self.ts_minfo = {}
self.tv=None
self.obj_funcs = {'NSE':api.NASH_SUTCLIFFE,'KGE':api.KLING_GUPTA}
def _main_runner(config_file):
print 'Starting runner'
from shyft.orchestration.utils.ShyftConfig import ShyftConfig
config = ShyftConfig(config_file)
simulator = ShyftRunner(config)
simulator.build_model(config.t_start, config.dt, config.n_steps)
simulator.run_model()
if "shapes" in simulator.cell_data_types():
extractor = {'Total discharge': lambda x: x.response.total_discharge,
'Snow storage': lambda x: x.response.gs.storage*(1.0 - (x.lake_frac + x.reservoir_frac)),
'Temperature': lambda x: x.temperature[len(x.temperature)-1],
'Precipitation': lambda x: x.precipitation[len(x.precipitation)-1]}
simulator.plot_distributed_data(simulator.cell_data("shapes"), simulator.model.get_cells(), extractor)
print "Exit.."
def make_fake_target(config, time_axis, catchment_index):
print "Fake target Catchment index = {}".format(catchment_index)
tv=api.TargetSpecificationVector()
t=api.TargetSpecificationPts()
simulator = ShyftRunner(config)
simulator.build_model(time_axis.start(), time_axis.delta(), time_axis.size())
simulator.run_model()
ts = simulator.get_sum_catchment_result('SimDischarge',catchment_index)
mapped_indx = [i for i,j in enumerate(simulator.catchment_map) if j in catchment_index]
catch_indx = api.IntVector(mapped_indx)
t.catchment_indexes=catch_indx
t.scale_factor=1.0
t.calc_mode=api.NASH_SUTCLIFFE
t.ts=ts
tv.push_back(t)
return tv
class ShyftCalibrator(object):
def __init__(self, config):
self._config = config
self._model_config = config.model_config
self.runner = ShyftRunner(self._model_config)
self.calibrator = None
self.target_ts = {}
self.ts_minfo = {}
self.tv=None
self.obj_funcs = {'NSE':api.NASH_SUTCLIFFE,'KGE':api.KLING_GUPTA}
def init(self, time_axis):
self._load_target_spec_input()
self._fetch_target_timeseries()
print self.target_ts
self.runner.build_model(time_axis.start(), time_axis.delta(), time_axis.size())
self.calib_param_names = [self.param_accessor.get_name(i) for i in range(self.param_accessor.size())]
if self.tv is None:
self._create_target_SpecVect()
self.calibrator = self._config.calibration_type(self.runner.model, self.tv, api.DoubleVector(self.p_min), api.DoubleVector(self.p_max))
self.calibrator.set_verbose_level(1) # To control console print out during calibration
print "Calibrator catchment index = {}".format(self._config.catchment_index)
@property
def param_accessor(self):
return self.runner.model.get_region_parameter()
@property
def p_min(self):
return [self._config.calibration_parameters[name]['min'] for name in self.calib_param_names]
@property
def p_max(self):
return [self._config.calibration_parameters[name]['max'] for name in self.calib_param_names]
def calibrate(self, p_init=None, tol=1.0e-5):
print "Calibrating"
if p_init is None:
#p_init=api.DoubleVector([(a + b)*0.5 for a,b in zip(self.p_min, self.p_max)])
p_init=api.DoubleVector([a+(b-a)*0.5 for a,b in zip(self.p_min, self.p_max)])
n_iterations = 1500
return [p for p in self.calibrator.optimize(p_init,n_iterations, 0.1, tol)]
def calculate_goal_function(self,optim_param_list):
""" calls calibrator with parameter vector"""
self.calibrator.set_verbose_level(0)
return self.calibrator.calculate_goal_function(api.DoubleVector(optim_param_list))
def _load_target_spec_input(self):
catch_indices = {catch['internal_id']:catch['catch_id'] for catch in self._config.catchment_index}
for repository in self._config.target:
for target in repository['1D_timeseries']:
ID = target['internal_id']
if ID in catch_indices.keys():
spec={}
#self.target_ts_minfo[target['internal_id']]={k: target[k] for k in (target.keys()) if k != 'internal_id'}
self.ts_minfo[ID]={k: target[k] for k in (target.keys()) if k in ['uid','weight','obj_func']}
# Do some checks here on whether each target-period is within the run-period
spec['start_t'] = int(round((target['start_datetime'] - datetime.utcfromtimestamp(0)).total_seconds()))
spec['dt'] = target['time_step_length']
spec['nsteps'] = target['number_of_steps']
spec['catch_indx'] = catch_indices[ID]
self.ts_minfo[target['internal_id']].update(spec)
print self.ts_minfo
def _fetch_target_timeseries(self):
targets = self._config.target
for repository in targets:
constructor = repository["repository"][0]
arg = repository["repository"][1]
ts_repository = constructor(arg)
for target in repository['1D_timeseries']:
ID = target['internal_id']
if ID in self.ts_minfo.keys():
ts_info=self.ts_minfo[ID]
period = api.UtcPeriod(ts_info['start_t'],
ts_info['start_t']+ts_info['nsteps']*ts_info['dt'])
self.target_ts.update(ts_repository.read([target['uid']],period))
def _create_target_SpecVect(self):
self.tv=api.TargetSpecificationVector()
tst=api.TsTransform()
for ID,ts_info in self.ts_minfo.items():
mapped_indx = [i for i,j in enumerate(self.runner.catchment_map) if j in ts_info['catch_indx']]
catch_indx = api.IntVector(mapped_indx)
tsp = self.target_ts[ts_info['uid']]
#t=api.TargetOptSpecification()
t=api.TargetSpecificationPts()
t.catchment_indexes=catch_indx
t.scale_factor=ts_info['weight']
#t.calc_mode=api.NASH_SUTCLIFFE
t.calc_mode=self.obj_funcs[ts_info['obj_func']['name']]
t.s_r=ts_info['obj_func']['scaling_factors']['s_corr']
t.s_a=ts_info['obj_func']['scaling_factors']['s_var']
t.s_b=ts_info['obj_func']['scaling_factors']['s_bias']
tsa= tst.to_average(ts_info['start_t'],ts_info['dt'],ts_info['nsteps'],tsp)
#tsa= tst.to_average_staircase(ts_info['start_t'],ts_info['dt'],ts_info['nsteps'],tsp)
t.ts=tsa
#-To avoid any kind of averaging-----
#for i in range(len(t.ts)):
# t.ts.set(i,tsp.value(i))
#------------------------------------
self.tv.push_back(t)
print ID,ts_info['uid'],mapped_indx
# time_axis = api.FixedIntervalTimeAxis(t_start, delta_t, n_steps)
time_axis = api.TimeAxis(t_start, delta_t, n_steps)
config._target = make_fake_target(config.model_config, time_axis, config.catchment_index[0]['catch_id'])
calibrator = ShyftCalibrator(config)
calibrator.init(time_axis)
print calibrator.calibrate(tol=1.0e-5)
print "Exit.."
if __name__ == "__main__":
import sys
import os
from shyft.orchestration.utils.ShyftRunner import ShyftRunner
from shyft.orchestration.utils.ShyftConfig import ShyftConfig
enki_root=os.path.join("D:\\","Users","sih","enki_config_for_test")
config_file = os.path.join(enki_root, "runner_configurations.yaml")
config= ShyftConfig(config_file,'NeaNidelva')
simulator = ShyftRunner(config)
simulator.build_model(config.t_start, config.dt, config.n_steps)
simulator.run_model()
discharge_0=simulator.get_calculated_discharge(38)
if "shapes" in simulator.cell_data_types():
extractor = {'Total discharge': lambda x: x.response.total_discharge,
'Snow storage': lambda x: x.response.gs.storage*(1.0 - (x.lake_frac + x.reservoir_frac)),
'Temperature': lambda x: x.temperature[len(x.temperature)-1],
'Precipitation': lambda x: x.precipitation[len(x.precipitation)-1]}
simulator.plot_distributed_data(simulator.cell_data("shapes"), simulator.model.get_cells(), extractor)
print "Exit.."
#default_config_file = os.path.join(os.path.dirname(__file__), "config\NeaNidelva_calibration.yaml")
#filename = sys.argv[1] if len(sys.argv) == 2 else default_config_file
#_main_calibration_runner(filename)
d=val['start_datetime']
init_state_timestamp=cal.time(api.YMDhms(d.year,d.month,d.day,d.hour,d.minute,d.second)) # enki calendar
n_steps=int((current_d_timestamp-init_state_timestamp)/val['run_time_step'])
val['number_of_steps']=n_steps
with open(config_file, "w") as f:
yaml.dump(dct, f)
if(run_mode=='forecast'):
config_name=model_name+'_forecast'
with open(config_file) as f:
dct = yaml.load(f)
val=dct[config_name]
val['start_datetime']=current_d
n_steps=int(forecast_period/val['run_time_step'])
val['number_of_steps']=n_steps
with open(config_file, "w") as f:
yaml.dump(dct, f)
config= ShyftConfig(config_file,config_name)
simulator = ShyftRunner(config)
simulator.build_model(config.t_start, config.dt, config.n_steps)
simulator.run_model()
#if(run_mode=='forecast'):
# obs_state=fetch_smg_data.get_obs_inflow(model_name,obs_series_f,config.t_start-config.dt, n_steps=24,dt=config.dt)
# state_update.update_q(simulator,obs_state)
# misc.change_uid(simulator._config)
# simulator.run_model()