def ensemble_demo():
utc = Calendar()
t_start = utc.time(YMDhms(2011, 9, 1))
t_fc_ens_start = utc.time(YMDhms(2015, 7, 26))
disp_start = utc.time(YMDhms(2015, 7, 20))
dt = deltahours(1)
n_obs = int(round((t_fc_ens_start - t_start)/dt))
n_fc_ens = 30
n_disp = int(round(t_fc_ens_start - disp_start)/dt) + n_fc_ens + 24*7
obs_time_axis = Timeaxis(t_start, dt, n_obs + 1)
fc_ens_time_axis = Timeaxis(t_fc_ens_start, dt, n_fc_ens)
display_time_axis = Timeaxis(disp_start, dt, n_disp)
q_obs_m3s_ts = observed_tistel_discharge(obs_time_axis.total_period())
ptgsk = create_tistel_simulator(PTGSKOptModel, tistel.geo_ts_repository(tistel.grid_spec.epsg()))
initial_state = burn_in_state(ptgsk, t_start, utc.time(YMDhms(2012, 9, 1)), q_obs_m3s_ts)
ptgsk.run(obs_time_axis, initial_state)
current_state = adjust_simulator_state(ptgsk, t_fc_ens_start, q_obs_m3s_ts)
q_obs_m3s_ts = observed_tistel_discharge(display_time_axis.total_period())
ens_repos = tistel.arome_ensemble_repository(tistel.grid_spec)
ptgsk_fc_ens = create_tistel_simulator(PTGSKModel, ens_repos)
sims = ptgsk_fc_ens.create_ensembles(fc_ens_time_axis, t_fc_ens_start, current_state)
for sim in sims:
sim.simulate()
plt.hold(1)
percentiles = [10, 25, 50, 75, 90]
plot_percentiles(sims, percentiles, obs=q_obs_m3s_ts)
#plt.interactive(1)
plt.show()
def simple_run_demo():
"""Simple demo using HBV time-series and similar model-values
"""
# 1. Setup the time-axis for our simulation
normal_calendar = Calendar(3600) # we need UTC+1, since day-boundaries in day series in SmG is at UTC+1
t_start = normal_calendar.time(2010, 9, 1) # we start at
t_end = normal_calendar.add(t_start,Calendar.YEAR,5) # 5 years period for simulation
time_axis = Timeaxis(t_start, model_dt, normal_calendar.diff_units(t_start,t_end,model_dt))
# 2. Create the shyft model from the HBV model-repository
shyft_model = create_kjela_model(PTHSKModel, kjela.geo_ts_repository)
# 3. establish the initial state
# using the *pattern* of distribution after one year (so hbv-zone 1..10 get approximate distribution of discharge)
# *and* the observed discharge at the start time t_start
#
t_burnin = normal_calendar.add(t_start,Calendar.YEAR,1) # use one year to get distribution between hbvzones
burnin_time_axis = Timeaxis(t_start, model_dt, normal_calendar.diff_units(t_start, t_burnin, model_dt))
q_obs_m3s_ts = observed_kjela_discharge(time_axis.total_period()) # get out the observation ts
q_obs_m3s_at_t_start= q_obs_m3s_ts(t_start) # get the m3/s at t_start
initial_state = burn_in_state(shyft_model,burnin_time_axis, q_obs_m3s_at_t_start)
# 4. now run the model with the established state
# that will start out with the burn in state
shyft_model.run(time_axis, initial_state)
# 5. display results etc. goes here
plot_results(shyft_model, q_obs_m3s_ts)
plt.show()
def continuous_calibration():
utc = Calendar()
t_start = utc.time(YMDhms(2011, 9, 1))
t_fc_start = utc.time(YMDhms(2015, 10, 1))
dt = deltahours(1)
n_obs = int(round((t_fc_start - t_start)/dt))
obs_time_axis = Timeaxis(t_start, dt, n_obs + 1)
q_obs_m3s_ts = observed_tistel_discharge(obs_time_axis.total_period())
ptgsk = create_tistel_simulator(PTGSKOptModel, tistel.geo_ts_repository(tistel.grid_spec.epsg()))
initial_state = burn_in_state(ptgsk, t_start, utc.time(YMDhms(2012, 9, 1)), q_obs_m3s_ts)
num_opt_days = 30
# Step forward num_opt_days days and store the state for each day:
recal_start = t_start + deltahours(num_opt_days*24)
t = t_start
state = initial_state
opt_states = {t: state}
while t < recal_start:
ptgsk.run(Timeaxis(t, dt, 24), state)
t += deltahours(24)
state = ptgsk.reg_model_state
opt_states[t] = state
recal_stop = utc.time(YMDhms(2011, 10, 30))
recal_stop = utc.time(YMDhms(2012, 5, 30))
curr_time = recal_start
q_obs_avg = TsTransform().to_average(t_start, dt, n_obs + 1, q_obs_m3s_ts)
target_spec = TargetSpecificationPts(q_obs_avg, IntVector([0]), 1.0, KLING_GUPTA)
target_spec_vec = TargetSpecificationVector([target_spec])
i = 0
times = []
values = []
p, p_min, p_max = construct_calibration_parameters(ptgsk)
while curr_time < recal_stop:
print(i)
i += 1
opt_start = curr_time - deltahours(24*num_opt_days)
opt_state = opt_states.pop(opt_start)
p = ptgsk.region_model.get_region_parameter()
p_opt = ptgsk.optimize(Timeaxis(opt_start, dt, 24*num_opt_days), opt_state, target_spec_vec,
p, p_min, p_max, tr_stop=1.0e-5)
ptgsk.region_model.set_region_parameter(p_opt)
corr_state = adjust_simulator_state(ptgsk, curr_time, q_obs_m3s_ts)
ptgsk.run(Timeaxis(curr_time, dt, 24), corr_state)
curr_time += deltahours(24)
opt_states[curr_time] = ptgsk.reg_model_state
discharge = ptgsk.region_model.statistics.discharge([0])
times.extend(discharge.time(i) for i in range(discharge.size()))
values.extend(list(np.array(discharge.v)))
plt.plot(utc_to_greg(times), values)
plot_results(None, q_obs=observed_tistel_discharge(UtcPeriod(recal_start, recal_stop)))
set_calendar_formatter(Calendar())
#plt.interactive(1)
plt.title("Continuously recalibrated discharge vs observed")
plt.xlabel("Time in UTC")
plt.ylabel(r"Discharge in $\mathbf{m^3s^{-1}}$", verticalalignment="top", rotation="horizontal")
plt.gca().yaxis.set_label_coords(0, 1.1)
def ensemble_demo():
utc = Calendar()
t_start = utc.time(YMDhms(2011, 9, 1))
t_fc_ens_start = utc.time(YMDhms(2015, 7, 26))
disp_start = utc.time(YMDhms(2015, 7, 20))
dt = deltahours(1)
n_obs = int(round((t_fc_ens_start - t_start) / dt))
n_fc_ens = 30
n_disp = int(round(t_fc_ens_start - disp_start) / dt) + n_fc_ens + 24 * 7
obs_time_axis = Timeaxis(t_start, dt, n_obs + 1)
fc_ens_time_axis = Timeaxis(t_fc_ens_start, dt, n_fc_ens)
display_time_axis = Timeaxis(disp_start, dt, n_disp)
q_obs_m3s_ts = observed_tistel_discharge(obs_time_axis.total_period())
ptgsk = create_tistel_simulator(
PTGSKOptModel, tistel.geo_ts_repository(tistel.grid_spec.epsg()))
initial_state = burn_in_state(ptgsk, t_start, utc.time(YMDhms(2012, 9, 1)),
q_obs_m3s_ts)
ptgsk.run(obs_time_axis, initial_state)
current_state = adjust_simulator_state(ptgsk, t_fc_ens_start, q_obs_m3s_ts)
q_obs_m3s_ts = observed_tistel_discharge(display_time_axis.total_period())
ens_repos = tistel.arome_ensemble_repository(tistel.grid_spec)
ptgsk_fc_ens = create_tistel_simulator(PTGSKModel, ens_repos)
sims = ptgsk_fc_ens.create_ensembles(fc_ens_time_axis, t_fc_ens_start,
current_state)
for sim in sims:
sim.simulate()
plt.hold(1)
percentiles = [10, 25, 50, 75, 90]
plot_percentiles(sims, percentiles, obs=q_obs_m3s_ts)
plt.interactive(1)
plt.show()
def burn_in_state(simulator, t_start, t_stop, q_obs_m3s_ts):
dt = deltahours(1)
n = int(round((t_stop - t_start) / dt))
time_axis = Timeaxis(t_start, dt, n)
n_cells = simulator.region_model.size()
state_repos = DefaultStateRepository(simulator.region_model.__class__,
n_cells)
simulator.run(time_axis, state_repos.get_state(0))
# Go back in time (to t_start) and adjust q with observed discharge at that time.
# This will give us a good initial state at t_start
return adjust_simulator_state(simulator, t_start, q_obs_m3s_ts)
def forecast_demo():
"""Simple forecast demo using arome data from met.no. Initial state
is bootstrapped by simulating one hydrological year (starting
Sept 1. 2011), and then calculating the state August 31. 2012. This
state is then used as initial state for simulating Sept 1, 2011,
after scaling with observed discharge. The validity of this approach
is limited by the temporal variation of the spatial distribution of
the discharge state, q, in the Kirchner method. The model is then
stepped forward until Oct 1, 2015, and then used to compute the
discharge for 65 hours using Arome data. At last, the results
are plotted as simple timeseries.
"""
utc = Calendar()
t_start = utc.time(YMDhms(2011, 9, 1))
t_fc_start = utc.time(YMDhms(2015, 10, 1))
dt = deltahours(1)
n_obs = int(round((t_fc_start - t_start)/dt))
n_fc = 65
obs_time_axis = Timeaxis(t_start, dt, n_obs)
fc_time_axis = Timeaxis(t_fc_start, dt, n_fc)
total_time_axis = Timeaxis(t_start, dt, n_obs + n_fc)
q_obs_m3s_ts = observed_tistel_discharge(total_time_axis.total_period())
ptgsk = create_tistel_simulator(PTGSKOptModel, tistel.geo_ts_repository(tistel.grid_spec.epsg()))
initial_state = burn_in_state(ptgsk, t_start, utc.time(YMDhms(2012, 9, 1)), q_obs_m3s_ts)
ptgsk.run(obs_time_axis, initial_state)
plot_results(ptgsk, q_obs_m3s_ts)
current_state = adjust_simulator_state(ptgsk, t_fc_start, q_obs_m3s_ts)
ptgsk_fc = create_tistel_simulator(PTGSKModel, tistel.arome_repository(tistel.grid_spec, t_fc_start))
ptgsk_fc.run(fc_time_axis, current_state)
plt.figure()
q_obs_m3s_ts = observed_tistel_discharge(fc_time_axis.total_period())
plot_results(ptgsk_fc, q_obs_m3s_ts)
#plt.interactive(1)
plt.show()
def forecast_demo():
"""Simple forecast demo using arome data from met.no. Initial state
is bootstrapped by simulating one hydrological year (starting
Sept 1. 2011), and then calculating the state August 31. 2012. This
state is then used as initial state for simulating Sept 1, 2011,
after scaling with observed discharge. The validity of this approach
is limited by the temporal variation of the spatial distribution of
the discharge state, q, in the Kirchner method. The model is then
stepped forward until Oct 1, 2015, and then used to compute the
discharge for 65 hours using Arome data. At last, the results
are plotted as simple timeseries.
"""
utc = Calendar()
t_start = utc.time(YMDhms(2011, 9, 1))
t_fc_start = utc.time(YMDhms(2015, 10, 1))
dt = deltahours(1)
n_obs = int(round((t_fc_start - t_start) / dt))
n_fc = 65
obs_time_axis = Timeaxis(t_start, dt, n_obs)
fc_time_axis = Timeaxis(t_fc_start, dt, n_fc)
total_time_axis = Timeaxis(t_start, dt, n_obs + n_fc)
q_obs_m3s_ts = observed_tistel_discharge(total_time_axis.total_period())
ptgsk = create_tistel_simulator(
PTGSKOptModel, tistel.geo_ts_repository(tistel.grid_spec.epsg()))
initial_state = burn_in_state(ptgsk, t_start, utc.time(YMDhms(2012, 9, 1)),
q_obs_m3s_ts)
ptgsk.run(obs_time_axis, initial_state)
plot_results(ptgsk, q_obs_m3s_ts)
current_state = adjust_simulator_state(ptgsk, t_fc_start, q_obs_m3s_ts)
ptgsk_fc = create_tistel_simulator(
PTGSKModel, tistel.arome_repository(tistel.grid_spec, t_fc_start))
ptgsk_fc.run(fc_time_axis, current_state)
plt.figure()
q_obs_m3s_ts = observed_tistel_discharge(fc_time_axis.total_period())
plot_results(ptgsk_fc, q_obs_m3s_ts)
plt.interactive(1)
plt.show()
def test_run_observed_then_arome_and_store(self):
"""
Start Tistel 2015.09.01, dummy state with some kirchner water
use observations around Tistel (geo_ts_repository)
and simulate forwared to 2015.10.01 (store discharge and catchment level precip/temp)
then use arome forecast for 65 hours (needs arome for this period in arome-directory)
finally store the arome results.
"""
utc = Calendar() # No offset gives Utc
time_axis = Timeaxis(utc.time(YMDhms(2015, 9, 1, 0)), deltahours(1), 30 * 24)
fc_time_axis = Timeaxis(utc.time(YMDhms(2015, 10, 1, 0)), deltahours(1), 65)
interpolation_id = 0
ptgsk = DefaultSimulator("Tistel-ptgsk",
interpolation_id,
self.region_model_repository,
self.geo_ts_repository,
self.interpolation_repository, None)
n_cells = ptgsk.region_model.size()
ptgsk_state = DefaultStateRepository(ptgsk.region_model.__class__, n_cells)
ptgsk.region_model.set_state_collection(-1, True) # collect state so we can inspect it
s0 = ptgsk_state.get_state(0)
for i in range(s0.size()): # add some juice to get started
s0[i].kirchner.q = 0.5
ptgsk.run(time_axis, s0)
print("Done simulation, testing that we can extract data from model")
cids = api.IntVector() # we pull out for all the catchments-id if it's empty
model = ptgsk.region_model # fetch out the model
sum_discharge = model.statistics.discharge(cids)
self.assertIsNotNone(sum_discharge)
avg_temperature = model.statistics.temperature(cids)
avg_precipitation = model.statistics.precipitation(cids)
self.assertIsNotNone(avg_precipitation)
self.assertIsNotNone(avg_temperature)
for time_step in range(time_axis.size()):
precip_raster = model.statistics.precipitation(cids, time_step) # example raster output
self.assertEqual(precip_raster.size(), n_cells)
avg_gs_lwc = model.gamma_snow_state.lwc(cids) # sca skaugen|gamma
self.assertIsNotNone(avg_gs_lwc)
# lwc surface_heat alpha melt_mean melt iso_pot_energy temp_sw
avg_gs_output = model.gamma_snow_response.outflow(cids)
self.assertIsNotNone(avg_gs_output)
print("done. now save to db")
# SmGTsRepository(PROD,FC_PROD)
save_list = [
TsStoreItem(u'/test/x/shyft/tistel/discharge_m3s', lambda m: m.statistics.discharge(cids)),
TsStoreItem(u'/test/x/shyft/tistel/temperature', lambda m: m.statistics.temperature(cids)),
TsStoreItem(u'/test/x/shyft/tistel/precipitation', lambda m: m.statistics.precipitation(cids)),
]
tss = TimeseriesStore(SmGTsRepository(PREPROD, FC_PREPROD), save_list)
self.assertTrue(tss.store_ts(ptgsk.region_model))
print("Run forecast arome")
endstate = ptgsk.region_model.state_t.vector_t()
ptgsk.region_model.get_states(endstate) # get the state at end of obs
ptgsk.geo_ts_repository = self.arome_repository # switch to arome here
ptgsk.run_forecast(fc_time_axis, fc_time_axis.start, endstate) # now forecast
print("Done forecast")
fc_save_list = [
TsStoreItem(u'/test/x/shyft/tistel/fc_discharge_m3s', lambda m: m.statistics.discharge(cids)),
TsStoreItem(u'/test/x/shyft/tistel/fc_temperature', lambda m: m.statistics.temperature(cids)),
TsStoreItem(u'/test/x/shyft/tistel/fc_precipitation', lambda m: m.statistics.precipitation(cids)),
TsStoreItem(u'/test/x/shyft/tistel/fc_radiation', lambda m: m.statistics.radiation(cids)),
TsStoreItem(u'/test/x/shyft/tistel/fc_rel_hum', lambda m: m.statistics.rel_hum(cids)),
TsStoreItem(u'/test/x/shyft/tistel/fc_wind_speed', lambda m: m.statistics.wind_speed(cids)),
]
TimeseriesStore(SmGTsRepository(PREPROD, FC_PREPROD), fc_save_list).store_ts(ptgsk.region_model)
print("Done save to db")
def continuous_calibration():
utc = Calendar()
t_start = utc.time(YMDhms(2011, 9, 1))
t_fc_start = utc.time(YMDhms(2015, 10, 1))
dt = deltahours(1)
n_obs = int(round((t_fc_start - t_start) / dt))
obs_time_axis = Timeaxis(t_start, dt, n_obs + 1)
q_obs_m3s_ts = observed_tistel_discharge(obs_time_axis.total_period())
ptgsk = create_tistel_simulator(
PTGSKOptModel, tistel.geo_ts_repository(tistel.grid_spec.epsg()))
initial_state = burn_in_state(ptgsk, t_start, utc.time(YMDhms(2012, 9, 1)),
q_obs_m3s_ts)
num_opt_days = 30
# Step forward num_opt_days days and store the state for each day:
recal_start = t_start + deltahours(num_opt_days * 24)
t = t_start
state = initial_state
opt_states = {t: state}
while t < recal_start:
ptgsk.run(Timeaxis(t, dt, 24), state)
t += deltahours(24)
state = ptgsk.reg_model_state
opt_states[t] = state
recal_stop = utc.time(YMDhms(2011, 10, 30))
recal_stop = utc.time(YMDhms(2012, 5, 30))
curr_time = recal_start
q_obs_avg = TsTransform().to_average(t_start, dt, n_obs + 1, q_obs_m3s_ts)
target_spec = TargetSpecificationPts(q_obs_avg, IntVector([0]), 1.0,
KLING_GUPTA)
target_spec_vec = TargetSpecificationVector([target_spec])
i = 0
times = []
values = []
p, p_min, p_max = construct_calibration_parameters(ptgsk)
while curr_time < recal_stop:
print(i)
i += 1
opt_start = curr_time - deltahours(24 * num_opt_days)
opt_state = opt_states.pop(opt_start)
p = ptgsk.region_model.get_region_parameter()
p_opt = ptgsk.optimize(Timeaxis(opt_start, dt, 24 * num_opt_days),
opt_state,
target_spec_vec,
p,
p_min,
p_max,
tr_stop=1.0e-5)
ptgsk.region_model.set_region_parameter(p_opt)
corr_state = adjust_simulator_state(ptgsk, curr_time, q_obs_m3s_ts)
ptgsk.run(Timeaxis(curr_time, dt, 24), corr_state)
curr_time += deltahours(24)
opt_states[curr_time] = ptgsk.reg_model_state
discharge = ptgsk.region_model.statistics.discharge([0])
times.extend(discharge.time(i) for i in range(discharge.size()))
values.extend(list(np.array(discharge.v)))
plt.plot(utc_to_greg(times), values)
plot_results(None,
q_obs=observed_tistel_discharge(
UtcPeriod(recal_start, recal_stop)))
set_calendar_formatter(Calendar())
plt.interactive(1)
plt.title("Continuously recalibrated discharge vs observed")
plt.xlabel("Time in UTC")
plt.ylabel(r"Discharge in $\mathbf{m^3s^{-1}}$",
verticalalignment="top",
rotation="horizontal")
plt.gca().yaxis.set_label_coords(0, 1.1)
def test_run_observed_then_arome_and_store(self):
"""
Start Tistel 2015.09.01, dummy state with some kirchner water
use observations around Tistel (geo_ts_repository)
and simulate forwared to 2015.10.01 (store discharge and catchment level precip/temp)
then use arome forecast for 65 hours (needs arome for this period in arome-directory)
finally store the arome results.
"""
utc = Calendar() # No offset gives Utc
time_axis = Timeaxis(utc.time(YMDhms(2015, 9, 1, 0)),
deltahours(1), 30 * 24)
fc_time_axis = Timeaxis(utc.time(YMDhms(2015, 10, 1, 0)),
deltahours(1), 65)
interpolation_id = 0
ptgsk = DefaultSimulator("Tistel-ptgsk", interpolation_id,
self.region_model_repository,
self.geo_ts_repository,
self.interpolation_repository, None)
n_cells = ptgsk.region_model.size()
ptgsk_state = DefaultStateRepository(ptgsk.region_model.__class__,
n_cells)
ptgsk.region_model.set_state_collection(
-1, True) # collect state so we can inspect it
s0 = ptgsk_state.get_state(0)
for i in range(s0.size()): # add some juice to get started
s0[i].kirchner.q = 0.5
ptgsk.run(time_axis, s0)
print(
"Done simulation, testing that we can extract data from model")
cids = api.IntVector(
) # we pull out for all the catchments-id if it's empty
model = ptgsk.region_model # fetch out the model
sum_discharge = model.statistics.discharge(cids)
self.assertIsNotNone(sum_discharge)
avg_temperature = model.statistics.temperature(cids)
avg_precipitation = model.statistics.precipitation(cids)
self.assertIsNotNone(avg_precipitation)
self.assertIsNotNone(avg_temperature)
for time_step in range(time_axis.size()):
precip_raster = model.statistics.precipitation(
cids, time_step) # example raster output
self.assertEqual(precip_raster.size(), n_cells)
avg_gs_lwc = model.gamma_snow_state.lwc(cids) # sca skaugen|gamma
self.assertIsNotNone(avg_gs_lwc)
# lwc surface_heat alpha melt_mean melt iso_pot_energy temp_sw
avg_gs_output = model.gamma_snow_response.outflow(cids)
self.assertIsNotNone(avg_gs_output)
print("done. now save to db")
# SmGTsRepository(PROD,FC_PROD)
save_list = [
TsStoreItem(u'/test/x/shyft/tistel/discharge_m3s',
lambda m: m.statistics.discharge(cids)),
TsStoreItem(u'/test/x/shyft/tistel/temperature',
lambda m: m.statistics.temperature(cids)),
TsStoreItem(u'/test/x/shyft/tistel/precipitation',
lambda m: m.statistics.precipitation(cids)),
]
tss = TimeseriesStore(SmGTsRepository(PREPROD, FC_PREPROD),
save_list)
self.assertTrue(tss.store_ts(ptgsk.region_model))
print("Run forecast arome")
endstate = ptgsk.region_model.state_t.vector_t()
ptgsk.region_model.get_states(
endstate) # get the state at end of obs
ptgsk.geo_ts_repository = self.arome_repository # switch to arome here
ptgsk.run_forecast(fc_time_axis, fc_time_axis.start,
endstate) # now forecast
print("Done forecast")
fc_save_list = [
TsStoreItem(u'/test/x/shyft/tistel/fc_discharge_m3s',
lambda m: m.statistics.discharge(cids)),
TsStoreItem(u'/test/x/shyft/tistel/fc_temperature',
lambda m: m.statistics.temperature(cids)),
TsStoreItem(u'/test/x/shyft/tistel/fc_precipitation',
lambda m: m.statistics.precipitation(cids)),
TsStoreItem(u'/test/x/shyft/tistel/fc_radiation',
lambda m: m.statistics.radiation(cids)),
TsStoreItem(u'/test/x/shyft/tistel/fc_rel_hum',
lambda m: m.statistics.rel_hum(cids)),
TsStoreItem(u'/test/x/shyft/tistel/fc_wind_speed',
lambda m: m.statistics.wind_speed(cids)),
]
TimeseriesStore(SmGTsRepository(PREPROD, FC_PREPROD),
fc_save_list).store_ts(ptgsk.region_model)
print("Done save to db")
