def ensemble_demo(): utc = Calendar() t_start = utc.time(YMDhms(2011, 9, 1)) t_fc_ens_start = utc.time(YMDhms(2015, 7, 26)) disp_start = utc.time(YMDhms(2015, 7, 20)) dt = deltahours(1) n_obs = int(round((t_fc_ens_start - t_start)/dt)) n_fc_ens = 30 n_disp = int(round(t_fc_ens_start - disp_start)/dt) + n_fc_ens + 24*7 obs_time_axis = Timeaxis(t_start, dt, n_obs + 1) fc_ens_time_axis = Timeaxis(t_fc_ens_start, dt, n_fc_ens) display_time_axis = Timeaxis(disp_start, dt, n_disp) q_obs_m3s_ts = observed_tistel_discharge(obs_time_axis.total_period()) ptgsk = create_tistel_simulator(PTGSKOptModel, tistel.geo_ts_repository(tistel.grid_spec.epsg())) initial_state = burn_in_state(ptgsk, t_start, utc.time(YMDhms(2012, 9, 1)), q_obs_m3s_ts) ptgsk.run(obs_time_axis, initial_state) current_state = adjust_simulator_state(ptgsk, t_fc_ens_start, q_obs_m3s_ts) q_obs_m3s_ts = observed_tistel_discharge(display_time_axis.total_period()) ens_repos = tistel.arome_ensemble_repository(tistel.grid_spec) ptgsk_fc_ens = create_tistel_simulator(PTGSKModel, ens_repos) sims = ptgsk_fc_ens.create_ensembles(fc_ens_time_axis, t_fc_ens_start, current_state) for sim in sims: sim.simulate() plt.hold(1) percentiles = [10, 25, 50, 75, 90] plot_percentiles(sims, percentiles, obs=q_obs_m3s_ts) #plt.interactive(1) plt.show()
def simple_run_demo(): """Simple demo using HBV time-series and similar model-values """ # 1. Setup the time-axis for our simulation normal_calendar = Calendar(3600) # we need UTC+1, since day-boundaries in day series in SmG is at UTC+1 t_start = normal_calendar.time(2010, 9, 1) # we start at t_end = normal_calendar.add(t_start,Calendar.YEAR,5) # 5 years period for simulation time_axis = Timeaxis(t_start, model_dt, normal_calendar.diff_units(t_start,t_end,model_dt)) # 2. Create the shyft model from the HBV model-repository shyft_model = create_kjela_model(PTHSKModel, kjela.geo_ts_repository) # 3. establish the initial state # using the *pattern* of distribution after one year (so hbv-zone 1..10 get approximate distribution of discharge) # *and* the observed discharge at the start time t_start # t_burnin = normal_calendar.add(t_start,Calendar.YEAR,1) # use one year to get distribution between hbvzones burnin_time_axis = Timeaxis(t_start, model_dt, normal_calendar.diff_units(t_start, t_burnin, model_dt)) q_obs_m3s_ts = observed_kjela_discharge(time_axis.total_period()) # get out the observation ts q_obs_m3s_at_t_start= q_obs_m3s_ts(t_start) # get the m3/s at t_start initial_state = burn_in_state(shyft_model,burnin_time_axis, q_obs_m3s_at_t_start) # 4. now run the model with the established state # that will start out with the burn in state shyft_model.run(time_axis, initial_state) # 5. display results etc. goes here plot_results(shyft_model, q_obs_m3s_ts) plt.show()
def continuous_calibration(): utc = Calendar() t_start = utc.time(YMDhms(2011, 9, 1)) t_fc_start = utc.time(YMDhms(2015, 10, 1)) dt = deltahours(1) n_obs = int(round((t_fc_start - t_start)/dt)) obs_time_axis = Timeaxis(t_start, dt, n_obs + 1) q_obs_m3s_ts = observed_tistel_discharge(obs_time_axis.total_period()) ptgsk = create_tistel_simulator(PTGSKOptModel, tistel.geo_ts_repository(tistel.grid_spec.epsg())) initial_state = burn_in_state(ptgsk, t_start, utc.time(YMDhms(2012, 9, 1)), q_obs_m3s_ts) num_opt_days = 30 # Step forward num_opt_days days and store the state for each day: recal_start = t_start + deltahours(num_opt_days*24) t = t_start state = initial_state opt_states = {t: state} while t < recal_start: ptgsk.run(Timeaxis(t, dt, 24), state) t += deltahours(24) state = ptgsk.reg_model_state opt_states[t] = state recal_stop = utc.time(YMDhms(2011, 10, 30)) recal_stop = utc.time(YMDhms(2012, 5, 30)) curr_time = recal_start q_obs_avg = TsTransform().to_average(t_start, dt, n_obs + 1, q_obs_m3s_ts) target_spec = TargetSpecificationPts(q_obs_avg, IntVector([0]), 1.0, KLING_GUPTA) target_spec_vec = TargetSpecificationVector([target_spec]) i = 0 times = [] values = [] p, p_min, p_max = construct_calibration_parameters(ptgsk) while curr_time < recal_stop: print(i) i += 1 opt_start = curr_time - deltahours(24*num_opt_days) opt_state = opt_states.pop(opt_start) p = ptgsk.region_model.get_region_parameter() p_opt = ptgsk.optimize(Timeaxis(opt_start, dt, 24*num_opt_days), opt_state, target_spec_vec, p, p_min, p_max, tr_stop=1.0e-5) ptgsk.region_model.set_region_parameter(p_opt) corr_state = adjust_simulator_state(ptgsk, curr_time, q_obs_m3s_ts) ptgsk.run(Timeaxis(curr_time, dt, 24), corr_state) curr_time += deltahours(24) opt_states[curr_time] = ptgsk.reg_model_state discharge = ptgsk.region_model.statistics.discharge([0]) times.extend(discharge.time(i) for i in range(discharge.size())) values.extend(list(np.array(discharge.v))) plt.plot(utc_to_greg(times), values) plot_results(None, q_obs=observed_tistel_discharge(UtcPeriod(recal_start, recal_stop))) set_calendar_formatter(Calendar()) #plt.interactive(1) plt.title("Continuously recalibrated discharge vs observed") plt.xlabel("Time in UTC") plt.ylabel(r"Discharge in $\mathbf{m^3s^{-1}}$", verticalalignment="top", rotation="horizontal") plt.gca().yaxis.set_label_coords(0, 1.1)
def ensemble_demo(): utc = Calendar() t_start = utc.time(YMDhms(2011, 9, 1)) t_fc_ens_start = utc.time(YMDhms(2015, 7, 26)) disp_start = utc.time(YMDhms(2015, 7, 20)) dt = deltahours(1) n_obs = int(round((t_fc_ens_start - t_start) / dt)) n_fc_ens = 30 n_disp = int(round(t_fc_ens_start - disp_start) / dt) + n_fc_ens + 24 * 7 obs_time_axis = Timeaxis(t_start, dt, n_obs + 1) fc_ens_time_axis = Timeaxis(t_fc_ens_start, dt, n_fc_ens) display_time_axis = Timeaxis(disp_start, dt, n_disp) q_obs_m3s_ts = observed_tistel_discharge(obs_time_axis.total_period()) ptgsk = create_tistel_simulator( PTGSKOptModel, tistel.geo_ts_repository(tistel.grid_spec.epsg())) initial_state = burn_in_state(ptgsk, t_start, utc.time(YMDhms(2012, 9, 1)), q_obs_m3s_ts) ptgsk.run(obs_time_axis, initial_state) current_state = adjust_simulator_state(ptgsk, t_fc_ens_start, q_obs_m3s_ts) q_obs_m3s_ts = observed_tistel_discharge(display_time_axis.total_period()) ens_repos = tistel.arome_ensemble_repository(tistel.grid_spec) ptgsk_fc_ens = create_tistel_simulator(PTGSKModel, ens_repos) sims = ptgsk_fc_ens.create_ensembles(fc_ens_time_axis, t_fc_ens_start, current_state) for sim in sims: sim.simulate() plt.hold(1) percentiles = [10, 25, 50, 75, 90] plot_percentiles(sims, percentiles, obs=q_obs_m3s_ts) plt.interactive(1) plt.show()
def burn_in_state(simulator, t_start, t_stop, q_obs_m3s_ts): dt = deltahours(1) n = int(round((t_stop - t_start) / dt)) time_axis = Timeaxis(t_start, dt, n) n_cells = simulator.region_model.size() state_repos = DefaultStateRepository(simulator.region_model.__class__, n_cells) simulator.run(time_axis, state_repos.get_state(0)) # Go back in time (to t_start) and adjust q with observed discharge at that time. # This will give us a good initial state at t_start return adjust_simulator_state(simulator, t_start, q_obs_m3s_ts)
def forecast_demo(): """Simple forecast demo using arome data from met.no. Initial state is bootstrapped by simulating one hydrological year (starting Sept 1. 2011), and then calculating the state August 31. 2012. This state is then used as initial state for simulating Sept 1, 2011, after scaling with observed discharge. The validity of this approach is limited by the temporal variation of the spatial distribution of the discharge state, q, in the Kirchner method. The model is then stepped forward until Oct 1, 2015, and then used to compute the discharge for 65 hours using Arome data. At last, the results are plotted as simple timeseries. """ utc = Calendar() t_start = utc.time(YMDhms(2011, 9, 1)) t_fc_start = utc.time(YMDhms(2015, 10, 1)) dt = deltahours(1) n_obs = int(round((t_fc_start - t_start)/dt)) n_fc = 65 obs_time_axis = Timeaxis(t_start, dt, n_obs) fc_time_axis = Timeaxis(t_fc_start, dt, n_fc) total_time_axis = Timeaxis(t_start, dt, n_obs + n_fc) q_obs_m3s_ts = observed_tistel_discharge(total_time_axis.total_period()) ptgsk = create_tistel_simulator(PTGSKOptModel, tistel.geo_ts_repository(tistel.grid_spec.epsg())) initial_state = burn_in_state(ptgsk, t_start, utc.time(YMDhms(2012, 9, 1)), q_obs_m3s_ts) ptgsk.run(obs_time_axis, initial_state) plot_results(ptgsk, q_obs_m3s_ts) current_state = adjust_simulator_state(ptgsk, t_fc_start, q_obs_m3s_ts) ptgsk_fc = create_tistel_simulator(PTGSKModel, tistel.arome_repository(tistel.grid_spec, t_fc_start)) ptgsk_fc.run(fc_time_axis, current_state) plt.figure() q_obs_m3s_ts = observed_tistel_discharge(fc_time_axis.total_period()) plot_results(ptgsk_fc, q_obs_m3s_ts) #plt.interactive(1) plt.show()
def forecast_demo(): """Simple forecast demo using arome data from met.no. Initial state is bootstrapped by simulating one hydrological year (starting Sept 1. 2011), and then calculating the state August 31. 2012. This state is then used as initial state for simulating Sept 1, 2011, after scaling with observed discharge. The validity of this approach is limited by the temporal variation of the spatial distribution of the discharge state, q, in the Kirchner method. The model is then stepped forward until Oct 1, 2015, and then used to compute the discharge for 65 hours using Arome data. At last, the results are plotted as simple timeseries. """ utc = Calendar() t_start = utc.time(YMDhms(2011, 9, 1)) t_fc_start = utc.time(YMDhms(2015, 10, 1)) dt = deltahours(1) n_obs = int(round((t_fc_start - t_start) / dt)) n_fc = 65 obs_time_axis = Timeaxis(t_start, dt, n_obs) fc_time_axis = Timeaxis(t_fc_start, dt, n_fc) total_time_axis = Timeaxis(t_start, dt, n_obs + n_fc) q_obs_m3s_ts = observed_tistel_discharge(total_time_axis.total_period()) ptgsk = create_tistel_simulator( PTGSKOptModel, tistel.geo_ts_repository(tistel.grid_spec.epsg())) initial_state = burn_in_state(ptgsk, t_start, utc.time(YMDhms(2012, 9, 1)), q_obs_m3s_ts) ptgsk.run(obs_time_axis, initial_state) plot_results(ptgsk, q_obs_m3s_ts) current_state = adjust_simulator_state(ptgsk, t_fc_start, q_obs_m3s_ts) ptgsk_fc = create_tistel_simulator( PTGSKModel, tistel.arome_repository(tistel.grid_spec, t_fc_start)) ptgsk_fc.run(fc_time_axis, current_state) plt.figure() q_obs_m3s_ts = observed_tistel_discharge(fc_time_axis.total_period()) plot_results(ptgsk_fc, q_obs_m3s_ts) plt.interactive(1) plt.show()
def test_run_observed_then_arome_and_store(self): """ Start Tistel 2015.09.01, dummy state with some kirchner water use observations around Tistel (geo_ts_repository) and simulate forwared to 2015.10.01 (store discharge and catchment level precip/temp) then use arome forecast for 65 hours (needs arome for this period in arome-directory) finally store the arome results. """ utc = Calendar() # No offset gives Utc time_axis = Timeaxis(utc.time(YMDhms(2015, 9, 1, 0)), deltahours(1), 30 * 24) fc_time_axis = Timeaxis(utc.time(YMDhms(2015, 10, 1, 0)), deltahours(1), 65) interpolation_id = 0 ptgsk = DefaultSimulator("Tistel-ptgsk", interpolation_id, self.region_model_repository, self.geo_ts_repository, self.interpolation_repository, None) n_cells = ptgsk.region_model.size() ptgsk_state = DefaultStateRepository(ptgsk.region_model.__class__, n_cells) ptgsk.region_model.set_state_collection(-1, True) # collect state so we can inspect it s0 = ptgsk_state.get_state(0) for i in range(s0.size()): # add some juice to get started s0[i].kirchner.q = 0.5 ptgsk.run(time_axis, s0) print("Done simulation, testing that we can extract data from model") cids = api.IntVector() # we pull out for all the catchments-id if it's empty model = ptgsk.region_model # fetch out the model sum_discharge = model.statistics.discharge(cids) self.assertIsNotNone(sum_discharge) avg_temperature = model.statistics.temperature(cids) avg_precipitation = model.statistics.precipitation(cids) self.assertIsNotNone(avg_precipitation) self.assertIsNotNone(avg_temperature) for time_step in range(time_axis.size()): precip_raster = model.statistics.precipitation(cids, time_step) # example raster output self.assertEqual(precip_raster.size(), n_cells) avg_gs_lwc = model.gamma_snow_state.lwc(cids) # sca skaugen|gamma self.assertIsNotNone(avg_gs_lwc) # lwc surface_heat alpha melt_mean melt iso_pot_energy temp_sw avg_gs_output = model.gamma_snow_response.outflow(cids) self.assertIsNotNone(avg_gs_output) print("done. now save to db") # SmGTsRepository(PROD,FC_PROD) save_list = [ TsStoreItem(u'/test/x/shyft/tistel/discharge_m3s', lambda m: m.statistics.discharge(cids)), TsStoreItem(u'/test/x/shyft/tistel/temperature', lambda m: m.statistics.temperature(cids)), TsStoreItem(u'/test/x/shyft/tistel/precipitation', lambda m: m.statistics.precipitation(cids)), ] tss = TimeseriesStore(SmGTsRepository(PREPROD, FC_PREPROD), save_list) self.assertTrue(tss.store_ts(ptgsk.region_model)) print("Run forecast arome") endstate = ptgsk.region_model.state_t.vector_t() ptgsk.region_model.get_states(endstate) # get the state at end of obs ptgsk.geo_ts_repository = self.arome_repository # switch to arome here ptgsk.run_forecast(fc_time_axis, fc_time_axis.start, endstate) # now forecast print("Done forecast") fc_save_list = [ TsStoreItem(u'/test/x/shyft/tistel/fc_discharge_m3s', lambda m: m.statistics.discharge(cids)), TsStoreItem(u'/test/x/shyft/tistel/fc_temperature', lambda m: m.statistics.temperature(cids)), TsStoreItem(u'/test/x/shyft/tistel/fc_precipitation', lambda m: m.statistics.precipitation(cids)), TsStoreItem(u'/test/x/shyft/tistel/fc_radiation', lambda m: m.statistics.radiation(cids)), TsStoreItem(u'/test/x/shyft/tistel/fc_rel_hum', lambda m: m.statistics.rel_hum(cids)), TsStoreItem(u'/test/x/shyft/tistel/fc_wind_speed', lambda m: m.statistics.wind_speed(cids)), ] TimeseriesStore(SmGTsRepository(PREPROD, FC_PREPROD), fc_save_list).store_ts(ptgsk.region_model) print("Done save to db")
def continuous_calibration(): utc = Calendar() t_start = utc.time(YMDhms(2011, 9, 1)) t_fc_start = utc.time(YMDhms(2015, 10, 1)) dt = deltahours(1) n_obs = int(round((t_fc_start - t_start) / dt)) obs_time_axis = Timeaxis(t_start, dt, n_obs + 1) q_obs_m3s_ts = observed_tistel_discharge(obs_time_axis.total_period()) ptgsk = create_tistel_simulator( PTGSKOptModel, tistel.geo_ts_repository(tistel.grid_spec.epsg())) initial_state = burn_in_state(ptgsk, t_start, utc.time(YMDhms(2012, 9, 1)), q_obs_m3s_ts) num_opt_days = 30 # Step forward num_opt_days days and store the state for each day: recal_start = t_start + deltahours(num_opt_days * 24) t = t_start state = initial_state opt_states = {t: state} while t < recal_start: ptgsk.run(Timeaxis(t, dt, 24), state) t += deltahours(24) state = ptgsk.reg_model_state opt_states[t] = state recal_stop = utc.time(YMDhms(2011, 10, 30)) recal_stop = utc.time(YMDhms(2012, 5, 30)) curr_time = recal_start q_obs_avg = TsTransform().to_average(t_start, dt, n_obs + 1, q_obs_m3s_ts) target_spec = TargetSpecificationPts(q_obs_avg, IntVector([0]), 1.0, KLING_GUPTA) target_spec_vec = TargetSpecificationVector([target_spec]) i = 0 times = [] values = [] p, p_min, p_max = construct_calibration_parameters(ptgsk) while curr_time < recal_stop: print(i) i += 1 opt_start = curr_time - deltahours(24 * num_opt_days) opt_state = opt_states.pop(opt_start) p = ptgsk.region_model.get_region_parameter() p_opt = ptgsk.optimize(Timeaxis(opt_start, dt, 24 * num_opt_days), opt_state, target_spec_vec, p, p_min, p_max, tr_stop=1.0e-5) ptgsk.region_model.set_region_parameter(p_opt) corr_state = adjust_simulator_state(ptgsk, curr_time, q_obs_m3s_ts) ptgsk.run(Timeaxis(curr_time, dt, 24), corr_state) curr_time += deltahours(24) opt_states[curr_time] = ptgsk.reg_model_state discharge = ptgsk.region_model.statistics.discharge([0]) times.extend(discharge.time(i) for i in range(discharge.size())) values.extend(list(np.array(discharge.v))) plt.plot(utc_to_greg(times), values) plot_results(None, q_obs=observed_tistel_discharge( UtcPeriod(recal_start, recal_stop))) set_calendar_formatter(Calendar()) plt.interactive(1) plt.title("Continuously recalibrated discharge vs observed") plt.xlabel("Time in UTC") plt.ylabel(r"Discharge in $\mathbf{m^3s^{-1}}$", verticalalignment="top", rotation="horizontal") plt.gca().yaxis.set_label_coords(0, 1.1)
def test_run_observed_then_arome_and_store(self): """ Start Tistel 2015.09.01, dummy state with some kirchner water use observations around Tistel (geo_ts_repository) and simulate forwared to 2015.10.01 (store discharge and catchment level precip/temp) then use arome forecast for 65 hours (needs arome for this period in arome-directory) finally store the arome results. """ utc = Calendar() # No offset gives Utc time_axis = Timeaxis(utc.time(YMDhms(2015, 9, 1, 0)), deltahours(1), 30 * 24) fc_time_axis = Timeaxis(utc.time(YMDhms(2015, 10, 1, 0)), deltahours(1), 65) interpolation_id = 0 ptgsk = DefaultSimulator("Tistel-ptgsk", interpolation_id, self.region_model_repository, self.geo_ts_repository, self.interpolation_repository, None) n_cells = ptgsk.region_model.size() ptgsk_state = DefaultStateRepository(ptgsk.region_model.__class__, n_cells) ptgsk.region_model.set_state_collection( -1, True) # collect state so we can inspect it s0 = ptgsk_state.get_state(0) for i in range(s0.size()): # add some juice to get started s0[i].kirchner.q = 0.5 ptgsk.run(time_axis, s0) print( "Done simulation, testing that we can extract data from model") cids = api.IntVector( ) # we pull out for all the catchments-id if it's empty model = ptgsk.region_model # fetch out the model sum_discharge = model.statistics.discharge(cids) self.assertIsNotNone(sum_discharge) avg_temperature = model.statistics.temperature(cids) avg_precipitation = model.statistics.precipitation(cids) self.assertIsNotNone(avg_precipitation) self.assertIsNotNone(avg_temperature) for time_step in range(time_axis.size()): precip_raster = model.statistics.precipitation( cids, time_step) # example raster output self.assertEqual(precip_raster.size(), n_cells) avg_gs_lwc = model.gamma_snow_state.lwc(cids) # sca skaugen|gamma self.assertIsNotNone(avg_gs_lwc) # lwc surface_heat alpha melt_mean melt iso_pot_energy temp_sw avg_gs_output = model.gamma_snow_response.outflow(cids) self.assertIsNotNone(avg_gs_output) print("done. now save to db") # SmGTsRepository(PROD,FC_PROD) save_list = [ TsStoreItem(u'/test/x/shyft/tistel/discharge_m3s', lambda m: m.statistics.discharge(cids)), TsStoreItem(u'/test/x/shyft/tistel/temperature', lambda m: m.statistics.temperature(cids)), TsStoreItem(u'/test/x/shyft/tistel/precipitation', lambda m: m.statistics.precipitation(cids)), ] tss = TimeseriesStore(SmGTsRepository(PREPROD, FC_PREPROD), save_list) self.assertTrue(tss.store_ts(ptgsk.region_model)) print("Run forecast arome") endstate = ptgsk.region_model.state_t.vector_t() ptgsk.region_model.get_states( endstate) # get the state at end of obs ptgsk.geo_ts_repository = self.arome_repository # switch to arome here ptgsk.run_forecast(fc_time_axis, fc_time_axis.start, endstate) # now forecast print("Done forecast") fc_save_list = [ TsStoreItem(u'/test/x/shyft/tistel/fc_discharge_m3s', lambda m: m.statistics.discharge(cids)), TsStoreItem(u'/test/x/shyft/tistel/fc_temperature', lambda m: m.statistics.temperature(cids)), TsStoreItem(u'/test/x/shyft/tistel/fc_precipitation', lambda m: m.statistics.precipitation(cids)), TsStoreItem(u'/test/x/shyft/tistel/fc_radiation', lambda m: m.statistics.radiation(cids)), TsStoreItem(u'/test/x/shyft/tistel/fc_rel_hum', lambda m: m.statistics.rel_hum(cids)), TsStoreItem(u'/test/x/shyft/tistel/fc_wind_speed', lambda m: m.statistics.wind_speed(cids)), ] TimeseriesStore(SmGTsRepository(PREPROD, FC_PREPROD), fc_save_list).store_ts(ptgsk.region_model) print("Done save to db")