def test_forecast_average_slice(self): """ Demo and test TsVector.average_slice(lead_time,dt,n) """ utc = Calendar() t0 = utc.time(2017, 1, 1) dt = deltahours(1) n = 66 # typical arome fc_dt_n_hours = 6 fc_dt = deltahours(fc_dt_n_hours) fc_n = 4 * 10 # 4 each day 10 days fc_v = self._create_forecasts(t0, dt, n, fc_dt, fc_n) for lead_time_hours in range(12): for slice_length_units in [1, 2, 3, 4, 6, 12]: for dt_hours in [1, 2, 3]: slice_v = fc_v.average_slice(deltahours(lead_time_hours), deltahours(dt_hours), slice_length_units) self.assertEqual(len(slice_v), len(fc_v)) # then loop over the slice_v and prove it's equal # to the average of the same portion on the originalj for s, f in zip(slice_v, fc_v): ta = TimeAxis( f.time_axis.time(0) + deltahours(lead_time_hours), deltahours(dt_hours), slice_length_units) ts_expected = f.average(ta) self.assertTrue(s.time_axis == ts_expected.time_axis) self.assertTrue( np.allclose(s.values.to_numpy(), ts_expected.values.to_numpy())) pass
def test_raise_exception_when_no_data_in_request_period(self): utc_calendar = Calendar() netcdf_repository = self._construct_from_test_data() netcdf_repository.raise_if_no_data = True # yes, for now, just imagine this could work. self.assertIsNotNone(netcdf_repository) utc_period = UtcPeriod( utc_calendar.time(YMDhms(2017, 1, 1, 0, 0, 0)), # a period where there is no data in utc_calendar.time(YMDhms(2020, 12, 31, 0, 0, 0))) # the file supplied type_source_map = dict() type_source_map['temperature'] = TemperatureSource #def test_function(): # # return netcdf_repository.get_timeseries( # type_source_map, # geo_location_criteria=None, # utc_period=utc_period) #self.assertRaises(RuntimeError, test_function) self.assertRaises( RuntimeError, netcdf_repository.get_timeseries, type_source_map, **{ 'geo_location_criteria': None, 'utc_period': utc_period })
def test_fixed_tsv_empty(self) -> None: """Test that an empty TsVector is generated by fixed_tsv when given an empty sequence of values.""" cal = Calendar() period = UtcPeriod(cal.time(2017, 1, 1), cal.time(2018, 1, 1)) tsv = fixed_tsv(period, []) self.assertEqual(len(tsv), 0)
def test_glacier_melt_ts_m3s(self): utc = Calendar() t0 = utc.time(2016, 10, 1) dt = deltahours(1) n = 240 ta = TimeAxis(t0, dt, n) area_m2 = 487 * 1000 * 1000 # Jostedalsbreen, largest i Europe temperature = TimeSeries(ta=ta, fill_value=10.0, point_fx=fx_policy.POINT_AVERAGE_VALUE) sca_values = dv.from_numpy(np.linspace(area_m2 * 1.0, 0.0, num=n)) sca = TimeSeries(ta=ta, values=sca_values, point_fx=fx_policy.POINT_AVERAGE_VALUE) gf = 1.0 * area_m2 dtf = 6.0 melt_m3s = create_glacier_melt_ts_m3s( temperature, sca, gf, dtf) # Here we get back a melt_ts, that we can do ts-stuff with self.assertIsNotNone(melt_m3s) full_melt_m3s = glacier_melt_step(dtf, 10.0, 0.0, gf) expected_melt_m3s = np.linspace(0.0, full_melt_m3s, num=n) assert_array_almost_equal(expected_melt_m3s, melt_m3s.values.to_numpy(), 4) # Just to check we can work with the result as a ts in all aspects mx2 = melt_m3s * 2.0 emx2 = expected_melt_m3s * 2.0 assert_array_almost_equal(emx2, mx2.values.to_numpy(), 4)
def test_windowed_percentiles_tsv_values(self) -> None: """Test that a TsVector is generated by windowed_percentiles_tsv with time-series fulfilling some properties of being percentiles of the data ts.""" cal = Calendar() period = UtcPeriod(cal.time(2017, 1, 1), cal.time(2018, 1, 1)) data = np.linspace(-2, 2, 24 * 7) data_ts = TimeSeries(TimeAxis(0, Calendar.HOUR, len(data)), data, POINT_INSTANT_VALUE) # compute percentiles = [0, 10, 50, 90, 100] tsv = windowed_percentiles_tsv(data_ts, period, 3 * Calendar.HOUR, 12 * Calendar.HOUR, percentiles, self.client, cal) self.assertEqual(len(tsv), 5) # assert that the time-series have the correct properties for being percentile series for i in range(len(tsv[0])): prev_v = tsv[0].values[i] for j in range(len(percentiles) - 1): v = tsv[j + 1].values[i] # both values will be NaN at the end - that is ok if math.isnan(prev_v) and math.isnan(v): continue # check that no larger percentile have values greater than lower percentiles self.assertLessEqual(prev_v, v) prev_v = v
def test_error_handling(self): utc = Calendar() t0 = utc.time(2018, 1, 1) dt = deltahours(1) dv = DoubleVector() dv[:] = [1.0, 2.0, 2.5, 1.9, 3.0, 3.1, float('nan')] # also verify nan-handling ts = TimeSeries(TimeAxis(t0, dt, len(dv)), dv, POINT_AVERAGE_VALUE) try: ts.decode(start_bit=0, n_bits=0) self.assertTrue(False, 'This should throw, n_bits >0') except RuntimeError as re: pass try: ts.decode(start_bit=41, n_bits=12) self.assertTrue(False, 'This should throw, start_bit + n_bits >52') except RuntimeError as re: pass try: ts.decode(start_bit=-1, n_bits=12) self.assertTrue(False, 'This should throw, start_bit >=0') except RuntimeError as re: pass
def ensemble_demo(): utc = Calendar() t_start = utc.time(YMDhms(2011, 9, 1)) t_fc_ens_start = utc.time(YMDhms(2015, 7, 26)) disp_start = utc.time(YMDhms(2015, 7, 20)) dt = deltahours(1) n_obs = int(round((t_fc_ens_start - t_start)/dt)) n_fc_ens = 30 n_disp = int(round(t_fc_ens_start - disp_start)/dt) + n_fc_ens + 24*7 obs_time_axis = Timeaxis(t_start, dt, n_obs + 1) fc_ens_time_axis = Timeaxis(t_fc_ens_start, dt, n_fc_ens) display_time_axis = Timeaxis(disp_start, dt, n_disp) q_obs_m3s_ts = observed_tistel_discharge(obs_time_axis.total_period()) ptgsk = create_tistel_simulator(PTGSKOptModel, tistel.geo_ts_repository(tistel.grid_spec.epsg())) initial_state = burn_in_state(ptgsk, t_start, utc.time(YMDhms(2012, 9, 1)), q_obs_m3s_ts) ptgsk.run(obs_time_axis, initial_state) current_state = adjust_simulator_state(ptgsk, t_fc_ens_start, q_obs_m3s_ts) q_obs_m3s_ts = observed_tistel_discharge(display_time_axis.total_period()) ens_repos = tistel.arome_ensemble_repository(tistel.grid_spec) ptgsk_fc_ens = create_tistel_simulator(PTGSKModel, ens_repos) sims = ptgsk_fc_ens.create_ensembles(fc_ens_time_axis, t_fc_ens_start, current_state) for sim in sims: sim.simulate() plt.hold(1) percentiles = [10, 25, 50, 75, 90] plot_percentiles(sims, percentiles, obs=q_obs_m3s_ts) #plt.interactive(1) plt.show()
def plot_results(ptgsk, q_obs=None): h_obs = None if ptgsk is not None: plt.subplot(3, 1, 1) discharge = ptgsk.region_model.statistics.discharge([0]) temp = ptgsk.region_model.statistics.temperature([0]) precip = ptgsk.region_model.statistics.precipitation([0]) # Results on same time axis, so we only need one times = utc_to_greg( [discharge.time(i) for i in range(discharge.size())]) plt.plot(times, np.array(discharge.v)) plt.gca().set_xlim(times[0], times[-1]) plt.ylabel(r"Discharge in $\mathbf{m^3s^{-1}}$") set_calendar_formatter(Calendar()) if q_obs is not None: obs_times = utc_to_greg([q_obs.time(i) for i in range(q_obs.size())]) ovs = [q_obs.value(i) for i in range(q_obs.size())] h_obs, = plt.plot(obs_times, ovs, linewidth=2, color='k') ax = plt.gca() ax.set_xlim(obs_times[0], obs_times[-1]) if ptgsk is not None: plt.subplot(3, 1, 2) plt.plot(times, np.array(temp.v)) set_calendar_formatter(Calendar()) plt.gca().set_xlim(times[0], times[-1]) plt.ylabel(r"Temperature in C") plt.subplot(3, 1, 3) plt.plot(times, np.array(precip.v)) set_calendar_formatter(Calendar()) plt.gca().set_xlim(times[0], times[-1]) plt.ylabel(r"Precipitation in mm") return h_obs
def test_inverted_values(self): utc = Calendar() t0 = utc.time(2018, 1, 1) dt = deltahours(1) dv = DoubleVector() dv[:] = [1.0, 2.0, 2.5, 1.9, 3.0, 3.1, float('nan')] # also verify nan-handling i1_ex = [1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0] ts = TimeSeries(TimeAxis(t0, dt, len(dv)), dv, POINT_AVERAGE_VALUE) i2 = ts.inside(min_v=2.0, max_v=3.0, nan_v=1.0, inside_v=0.0, outside_v=1.0) assert_array_almost_equal(i2.values.to_numpy(), np.array(i1_ex))
def test_simple_case(self): utc = Calendar() t0 = utc.time(2018, 1, 1) dt = deltahours(1) dv = DoubleVector() dv[:] = [1.0, 2.0, 2.5, 1.9, 3.0, 3.1, -1.0] i1_ex = [0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0] ts = TimeSeries(TimeAxis(t0, dt, len(dv)), dv, POINT_AVERAGE_VALUE) i1 = ts.inside(2.0, 3.0) assert_array_almost_equal(i1.values.to_numpy(), np.array(i1_ex))
def test_construct_repository(self): utc_calendar = Calendar() netcdf_repository = self._construct_from_test_data() self.assertIsNotNone(netcdf_repository) utc_period = UtcPeriod( utc_calendar.time(YMDhms(2005, 1, 1, 0, 0, 0)), utc_calendar.time(YMDhms(2014, 12, 31, 0, 0, 0))) type_source_map = dict() type_source_map['temperature'] = TemperatureSource geo_ts_dict = netcdf_repository.get_timeseries( type_source_map, geo_location_criteria=None, utc_period=utc_period) self.assertIsNotNone(geo_ts_dict)
def test_fixed_tsv_values(self) -> None: """Test that a TsVector with fixed constant values is generated by fixed_tsv when given a sequence of values.""" cal = Calendar() period = UtcPeriod(cal.time(2017, 1, 1), cal.time(2018, 1, 1)) values = [12, 15.5] tsv = fixed_tsv(period, values) self.assertEqual(len(tsv), 2) for v, ts in zip(values, tsv): for ts_v in ts.values: self.assertEqual(ts_v, v)
def test_dtss_remove_series(self): with tempfile.TemporaryDirectory() as c_dir: # start the server dtss = DtsServer() port_no = find_free_port() host_port = 'localhost:{0}'.format(port_no) dtss.set_listening_port(port_no) dtss.set_container( "test", c_dir ) # notice we set container 'test' to point to c_dir directory dtss.start_async( ) # the internal shyft time-series will be stored to that container # setup some data utc = Calendar() d = deltahours(1) n = 365 * 24 // 3 t = utc.time(2016, 1, 1) ta = TimeAxis(t, d, n) tsv = TsVector() pts = TimeSeries(ta, np.linspace(start=0, stop=1.0, num=ta.size()), point_fx.POINT_AVERAGE_VALUE) tsv.append(TimeSeries("cache://test/foo", pts)) # get a client client = DtsClient(host_port) client.store_ts(tsv) # start with no removing dtss.set_can_remove(False) # we should be disallowed to remove now try: client.remove("shyft://test/foo") except Exception as err: self.assertEqual( str(err), "dtss::server: server does not support removing") # then try with allowing remove dtss.set_can_remove(True) # we only support removing shyft-url style data try: client.remove("protocol://test/foo") except Exception as err: self.assertEqual( str(err), "dtss::server: server does not allow removing for non shyft-url type data" ) # now it should work client.remove("shyft://test/foo")
def test_crudf_cycle(self): """ Verify we can create, store, read, find and delete state in the state repository """ # arrange, by creating one State cal = Calendar() utc_timestamp = cal.time(2001, 1, 1) region_model_id = "neanidelv-ptgsk" n_cells = 10 tags = ["initial", "unverified"] state_vector = self._create_state_vector(n_cells) self.assertIsNotNone( state_vector, "we should have a valid state vector object at this spot") # now start state_repository test state_repository = YamlStateRepository( directory_path=self._test_state_directory, state_serializer=StateSerializer(PTGSKStateWithIdVector)) # put in two states, record the unique state_id.. state_id_1 = state_repository.put_state(region_model_id, utc_timestamp, state_vector, tags) state_id_2 = state_repository.put_state(region_model_id, utc_timestamp, state_vector, tags) # assert that we got two unique state_id self.assertIsNotNone(state_id_1, "We expect back a unique id") self.assertIsNotNone(state_id_2, "We expect back a unique id") self.assertNotEqual( state_id_1, state_id_2, "storing two state, same model, same time, each state should be stored with a unique id" ) # now we should have two states in the repository state_infos = state_repository.find_state() self.assertEqual(2, len(state_infos), "We just stored two, expect two back..") # extra test, verify that we really stored the state (using kirchner q) state_1 = state_repository.get_state(state_id_1) self.assertEqual(n_cells, len(state_1), "expect to get back state with same number of cells") for i in range(n_cells): self.assertAlmostEqual( state_1[i].state.kirchner.q, state_vector[i].state.kirchner.q, 3, "state repository should preserve state...") # now remove state state_repository.delete_state(state_id_1) # check that we got just one left, and that it is the correct one.. state_list = state_repository.find_state() self.assertEqual(1, len(state_list)) self.assertEqual(state_list[0].region_model_id, region_model_id) self.assertEqual(state_list[0].utc_timestamp, utc_timestamp) self.assertEqual(state_list[0].state_id, state_id_2)
def test_returns_empty_ts_when_no_data_in_request_period(self): utc_calendar = Calendar() netcdf_repository = self._construct_from_test_data() self.assertIsNotNone(netcdf_repository) utc_period = UtcPeriod( utc_calendar.time(2017, 1, 1, 0, 0, 0), # a period where there is no data in utc_calendar.time(2020, 12, 31, 0, 0, 0)) # the file supplied type_source_map = dict() type_source_map['temperature'] = TemperatureSource geo_ts_dict = netcdf_repository.get_timeseries( type_source_map, geo_location_criteria=None, utc_period=utc_period) self.assertIsNotNone(geo_ts_dict)
def test_returns_empty_ts_when_no_data_in_request_period(self): utc_calendar = Calendar() netcdf_repository = self._construct_from_test_data() self.assertIsNotNone(netcdf_repository) utc_period = UtcPeriod(utc_calendar.time(YMDhms(2017, 1, 1, 0, 0, 0)),# a period where there is no data in utc_calendar.time(YMDhms(2020, 12, 31, 0, 0, 0)))# the file supplied type_source_map = dict() type_source_map['temperature'] = TemperatureSource geo_ts_dict = netcdf_repository.get_timeseries( type_source_map, geo_location_criteria=None, utc_period=utc_period) self.assertIsNotNone(geo_ts_dict)
def test_construct_repository(self): utc_calendar = Calendar() netcdf_repository = self._construct_from_test_data() self.assertIsNotNone(netcdf_repository) utc_period = UtcPeriod(utc_calendar.time(YMDhms(2005, 1, 1, 0, 0, 0)), utc_calendar.time(YMDhms(2014, 12, 31, 0, 0, 0))) type_source_map = dict() type_source_map['temperature'] = TemperatureSource geo_ts_dict = netcdf_repository.get_timeseries( type_source_map, geo_location_criteria=None, utc_period=utc_period) self.assertIsNotNone(geo_ts_dict)
def test_convolve_policy(self): utc = Calendar() ts = TimeSeries(ta=TimeAxisFixedDeltaT(utc.time(2001, 1, 1), deltahours(1), 24), fill_value=10.0, point_fx=point_fx.POINT_AVERAGE_VALUE) w = DoubleVector.from_numpy([0.05, 0.15, 0.6, 0.15, 0.05]) cts = ts.convolve_w(w, convolve_policy.USE_FIRST ) # ensure mass-balance between source and cts self.assertIsNotNone(cts) self.assertEquals(len(cts), len(ts)) self.assertEquals(cts.values.to_numpy().sum(), ts.values.to_numpy().sum())
def test_hbv_snow_step(self): utc = Calendar() s = HbvSnowState() p = HbvSnowParameter() s.distribute(p) r = HbvSnowResponse() calc = HbvSnowCalculator(p) t0 = utc.time(2016, 10, 1) t1 = utc.time(2016, 10, 2) dt = deltahours(1) temp = 0.4 prec_mm_h = 0.3 # Just check that we don't get an error when stepping calc.step(s, r, t0, t1, prec_mm_h, temp)
def test_windowed_percentiles_tsv_empty(self) -> None: """Test that an empty TsVector is generated by windowed_percentiles_tsv when given an empty sequence of percentiles.""" cal = Calendar() period = UtcPeriod(cal.time(2017, 1, 1), cal.time(2018, 1, 1)) data = np.linspace(-2, 2, 24 * 7) data_ts = TimeSeries(TimeAxis(0, Calendar.HOUR, len(data)), data, POINT_INSTANT_VALUE) # compute tsv = windowed_percentiles_tsv(data_ts, period, Calendar.HOUR, Calendar.HOUR, [], self.client, cal) self.assertEqual(len(tsv), 0)
def test_failures(self): """ Verify that dtss client server connections are auto-magically restored and fixed """ with tempfile.TemporaryDirectory() as c_dir: # start the server dtss = DtsServer() port_no = find_free_port() host_port = 'localhost:{0}'.format(port_no) dtss.set_listening_port(port_no) dtss.set_container( "test", c_dir ) # notice we set container 'test' to point to c_dir directory dtss.start_async( ) # the internal shyft time-series will be stored to that container # setup some data utc = Calendar() d = deltahours(1) n = 365 * 24 // 3 t = utc.time(2016, 1, 1) ta = TimeAxis(t, d, n) tsv = TsVector() pts = TimeSeries(ta, np.linspace(start=0, stop=1.0, num=ta.size()), point_fx.POINT_AVERAGE_VALUE) tsv.append(TimeSeries("cache://test/foo", pts)) # get a client client = DtsClient(host_port, auto_connect=False) client.store_ts(tsv) client.close() client.store_ts(tsv) # should just work, it re-open automagically dtss.clear( ) # the server is out and away, no chance this would work try: client.store_ts(tsv) self.assertTrue( False, 'This should throw, because there is no dtss server to help you' ) except Exception as ee: self.assertFalse(False, f'expected {ee} here') dtss.set_listening_port(port_no) dtss.start_async() client.store_ts( tsv) # this should just work, automagically reconnect
def continuous_calibration(): utc = Calendar() t_start = utc.time(YMDhms(2011, 9, 1)) t_fc_start = utc.time(YMDhms(2015, 10, 1)) dt = deltahours(1) n_obs = int(round((t_fc_start - t_start)/dt)) obs_time_axis = TimeAxisFixedDeltaT(t_start, dt, n_obs + 1) q_obs_m3s_ts = observed_tistel_discharge(obs_time_axis.total_period()) ptgsk = create_tistel_simulator(PTGSKOptModel, tistel.geo_ts_repository(tistel.grid_spec.epsg())) initial_state = burn_in_state(ptgsk, t_start, utc.time(YMDhms(2012, 9, 1)), q_obs_m3s_ts) num_opt_days = 30 # Step forward num_opt_days days and store the state for each day: recal_start = t_start + deltahours(num_opt_days*24) t = t_start state = initial_state opt_states = {t: state} while t < recal_start: ptgsk.run(TimeAxisFixedDeltaT(t, dt, 24), state) t += deltahours(24) state = ptgsk.reg_model_state opt_states[t] = state recal_stop = utc.time(YMDhms(2011, 10, 30)) recal_stop = utc.time(YMDhms(2012, 5, 30)) curr_time = recal_start q_obs_avg = TsTransform().to_average(t_start, dt, n_obs + 1, q_obs_m3s_ts) target_spec = TargetSpecificationPts(q_obs_avg, IntVector([0]), 1.0, KLING_GUPTA) target_spec_vec = TargetSpecificationVector([target_spec]) i = 0 times = [] values = [] p, p_min, p_max = construct_calibration_parameters(ptgsk) while curr_time < recal_stop: print(i) i += 1 opt_start = curr_time - deltahours(24*num_opt_days) opt_state = opt_states.pop(opt_start) p = ptgsk.region_model.get_region_parameter() p_opt = ptgsk.optimize(TimeAxisFixedDeltaT(opt_start, dt, 24*num_opt_days), opt_state, target_spec_vec, p, p_min, p_max, tr_stop=1.0e-5) ptgsk.region_model.set_region_parameter(p_opt) corr_state = adjust_simulator_state(ptgsk, curr_time, q_obs_m3s_ts) ptgsk.run(TimeAxisFixedDeltaT(curr_time, dt, 24), corr_state) curr_time += deltahours(24) opt_states[curr_time] = ptgsk.reg_model_state discharge = ptgsk.region_model.statistics.discharge([0]) times.extend(discharge.time(i) for i in range(discharge.size())) values.extend(list(np.array(discharge.v))) plt.plot(utc_to_greg(times), values) plot_results(None, q_obs=observed_tistel_discharge(UtcPeriod(recal_start, recal_stop))) set_calendar_formatter(Calendar()) #plt.interactive(1) plt.title("Continuously recalibrated discharge vs observed") plt.xlabel("Time in UTC") plt.ylabel(r"Discharge in $\mathbf{m^3s^{-1}}$", verticalalignment="top", rotation="horizontal") plt.gca().yaxis.set_label_coords(0, 1.1)
def plot_percentiles(sim, percentiles, obs=None): discharges = [s.region_model.statistics.discharge([0]) for s in sim] times = utc_to_greg(np.array([discharges[0].time(i) for i in range(discharges[0].size())], dtype='d')) all_discharges = np.array([d.v for d in discharges]) perc_arrs = [a for a in np.percentile(all_discharges, percentiles, 0)] h, fill_handles = plot_np_percentiles(times, perc_arrs, base_color=(51/256, 102/256, 193/256)) percentile_texts = ["{} - {}".format(percentiles[i], percentiles[-(i + 1)]) for i in range(len(percentiles)//2)] ax = plt.gca() maj_loc = AutoDateLocator(tz=pytz.UTC, interval_multiples=True) ax.xaxis.set_major_locator(maj_loc) set_calendar_formatter(Calendar()) if len(percentiles) % 2: fill_handles.append(h[0]) percentile_texts.append("{}".format(percentiles[len(percentiles)//2])) if obs is not None: h_obs = plot_results(None, obs) fill_handles.append(h_obs) percentile_texts.append("Observed") ax.legend(fill_handles, percentile_texts) ax.grid(b=True, color=(51/256, 102/256, 193/256), linewidth=0.1, linestyle='-', axis='y') plt.xlabel("Time in UTC") plt.ylabel(r"Discharge in $\mathbf{m^3s^{-1}}$", verticalalignment="top", rotation="horizontal") ax.yaxis.set_label_coords(0, 1.1) return h, ax
def test_hbv_physical_snow_step(self): utc = Calendar() s = HbvPhysicalSnowState() p = HbvPhysicalSnowParameter() r = HbvPhysicalSnowResponse() s.distribute(p) calc = HbvPhysicalSnowCalculator(p) t = utc.time(2016, 10, 1) dt = deltahours(1) temp = 0.4 rad = 12.0 prec_mm_h = 0.3 wind_speed = 1.3 rel_hum = 0.4 # Just check that we don't get an error when stepping calc.step(s, r, t, dt, temp, rad, prec_mm_h, wind_speed, rel_hum)
def test_find_with_region_model_filter(self): cal = Calendar() utc_timestamp = cal.time(YMDhms(2001, 1, 1)) region_model_id = "neanidelv-ptgsk" n_cells = 10 tags = ["initial", "unverified"] state_vector = self._create_state_vector(n_cells) # now start state_repository test state_repository = YamlStateRepository(self._test_state_directory) # put in two states, record the unique state_id.. state_repository.put_state(region_model_id, utc_timestamp, state_vector, tags) state_repository.put_state("tokke-ptgsk", utc_timestamp, state_vector, tags) all_states = state_repository.find_state() neanidelv_states = state_repository.find_state(region_model_id) self.assertEqual(2, len(all_states)) self.assertEqual(1, len(neanidelv_states)) self.assertEqual(neanidelv_states[0].region_model_id, region_model_id)
def test_construct_repository(self): utc_calendar = Calendar() netcdf_repository = self._construct_from_test_data() self.assertIsNotNone(netcdf_repository) utc_period = UtcPeriod(utc_calendar.time(2005, 1, 1, 0, 20, 0), # make it a challenge! ensure we get the first hour utc_calendar.time(2014, 12, 30, 0, 20, 0)) # and also, we should have the last hour! type_source_map = dict() type_source_map['temperature'] = TemperatureSource geo_ts_dict = netcdf_repository.get_timeseries( type_source_map, geo_location_criteria=None, utc_period=utc_period) self.assertIsNotNone(geo_ts_dict) temperature_source = geo_ts_dict['temperature'] self.assertIsNotNone(temperature_source) self.assertLessEqual(temperature_source[0].ts.time_axis.time(0),utc_period.start,'expect returned time-axis to cover the requested period') self.assertGreaterEqual(temperature_source[0].ts.time_axis.total_period().end,utc_period.end,'expected returned time-axis to cover requested period')
def test_inside_of_derivative(self): """Created in response to https://github.com/statkraft/shyft/issues/352""" values = [1, 1, 1, 1] utc = Calendar() data = np.array(values, dtype='float64') data_ta = TimeAxis(utc.time(2015, 1, 1), 3600, len(data)) orig = TimeSeries(data_ta, data, POINT_AVERAGE_VALUE) orig_derivative_inside_inf = orig.derivative(derivative_method.BACKWARD).inside(-float('inf'), float('inf'), 0, 1, 0) def check_nan(ts): """Method that returns 1 for all timesteps that contain nan.""" return ts.inside(-float('inf'), float('inf'), 1, 0, 0).values.to_numpy() np.testing.assert_equal(check_nan(orig + orig_derivative_inside_inf), [0., 0., 0., 0.], 'TimeSeries.inside() should not find any NaN-values in this TimeSeries')
def test_gamma_snow_step(self): utc = Calendar() s = GammaSnowState() p = GammaSnowParameter() r = GammaSnowResponse() calc = GammaSnowCalculator() t = utc.time(2016,10,1) dt = deltahours(1) temp = 0.4 rad = 12.0 prec_mm_h = 0.3 wind_speed=1.3 rel_hum=0.4 forest_fraction = 0.2 altitude = 100.0 # Just check that we don't get an error when stepping calc.step(s, r, t, dt, p, temp, rad, prec_mm_h, wind_speed, rel_hum, forest_fraction, altitude)
def test_get_ts_info(self): """ Verify we can get specific TsInfo objects for time-series from the server backend. """ with tempfile.TemporaryDirectory() as c_dir: # start the server dtss = DtsServer() port_no = find_free_port() host_adr = 'localhost:{0}'.format(port_no) dtss.set_listening_port(port_no) dtss.set_container( "testing", c_dir ) # notice we set container 'test' to point to c_dir directory dtss.start_async( ) # the internal shyft time-series will be stored to that container # get a client client = DtsClient(host_adr) try: client.get_ts_info(r'shyft://testing/data') except Exception as e: pass else: # only end up here if no exceptions self.fail('Could fetch info for non-existing ts info') # setup some data utc = Calendar() d = deltahours(1) n = 365 * 24 // 3 t = utc.time(2016, 1, 1) ta = TimeAxis(t, d, n) tsv = TsVector() pts = TimeSeries(ta, np.linspace(start=0, stop=1.0, num=ta.size()), point_fx.POINT_AVERAGE_VALUE) tsv.append(TimeSeries(r'shyft://testing/data', pts)) client.store_ts(tsv) info: TsInfo = client.get_ts_info(r'shyft://testing/data') self.assertEqual(info.name, r'data') self.assertEqual(info.point_fx, point_fx.POINT_AVERAGE_VALUE) self.assertEqual(info.data_period, ta.total_period())
def simple_run_demo(): """Simple demo using HBV time-series and similar model-values """ # 1. Setup the time-axis for our simulation normal_calendar = Calendar(3600) # we need UTC+1, since day-boundaries in day series in SmG is at UTC+1 t_start = normal_calendar.time(2010, 9, 1) # we start at t_end = normal_calendar.add(t_start,Calendar.YEAR,5) # 5 years period for simulation time_axis = Timeaxis(t_start, model_dt, normal_calendar.diff_units(t_start,t_end,model_dt)) # 2. Create the shyft model from the HBV model-repository shyft_model = create_kjela_model(PTHSKModel, kjela.geo_ts_repository) # 3. establish the initial state # using the *pattern* of distribution after one year (so hbv-zone 1..10 get approximate distribution of discharge) # *and* the observed discharge at the start time t_start # t_burnin = normal_calendar.add(t_start,Calendar.YEAR,1) # use one year to get distribution between hbvzones burnin_time_axis = Timeaxis(t_start, model_dt, normal_calendar.diff_units(t_start, t_burnin, model_dt)) q_obs_m3s_ts = observed_kjela_discharge(time_axis.total_period()) # get out the observation ts q_obs_m3s_at_t_start= q_obs_m3s_ts(t_start) # get the m3/s at t_start initial_state = burn_in_state(shyft_model,burnin_time_axis, q_obs_m3s_at_t_start) # 4. now run the model with the established state # that will start out with the burn in state shyft_model.run(time_axis, initial_state) # 5. display results etc. goes here plot_results(shyft_model, q_obs_m3s_ts) plt.show()
def test_find_with_region_model_and_time_filter(self): cal = Calendar() region_model_id = "neanidelv-ptgsk" n_cells = 10 tags = ["initial", "unverified"] state_vector = self._create_state_vector(n_cells) # now start state_repository test state_repository = YamlStateRepository(self._test_state_directory) # put in two states, record the unique state_id.. state_id_1 = state_repository.put_state(region_model_id, cal.time(YMDhms(2001, 1, 1, 0, 0, 0)), state_vector, tags) state_id_2 = state_repository.put_state(region_model_id, cal.time(YMDhms(2001, 1, 2, 0, 0, 0)), state_vector, tags) all_states = state_repository.find_state() neanidelv_states = state_repository.find_state(region_model_id) self.assertEqual(2, len(all_states)) self.assertEqual(2, len(neanidelv_states)) most_recent_state_before_time = state_repository.find_state(region_model_id, cal.time(YMDhms(2001, 1, 1, 0, 0, 0))) self.assertEqual(1, len(most_recent_state_before_time)) self.assertEqual(state_id_1, most_recent_state_before_time[0].state_id) self.assertEqual(0, len(state_repository.find_state(region_model_id, cal.time(YMDhms(2000, 12, 31, 23, 59, 59))))) self.assertEqual(state_id_2, state_repository.find_state(region_model_id, cal.time(YMDhms(2002, 1, 1, 0, 0, 0)))[ 0].state_id)
def test_easy_rating_curve_construct(self): utc = Calendar() rating_curve = RatingCurveParameters(RatingCurveTimeFunctions([ RatingCurveTimeFunction( utc.time(1950, 3, 27), RatingCurveFunction(RatingCurveSegments([ RatingCurveSegment(lower=0.474, a=5.97489, b=-0.4745, c=2.36997) ]))), RatingCurveTimeFunction( utc.time(1968, 7, 29), RatingCurveFunction(RatingCurveSegments([ RatingCurveSegment(lower=0.25, a=2.9822, b=-0.45, c=1.5078), RatingCurveSegment(lower=0.79, a=3.9513, b=-0.45, c=2.8087), RatingCurveSegment(lower=1.38, a=5.7071, b=-0.45, c=2.3503), RatingCurveSegment(lower=2.55, a=8.2672, b=-0.45, c=2.052) ]))) ])) self.assertIsNotNone(rating_curve) flow = rating_curve.flow(utc.time(2018, 1, 1), 3.2) self.assertAlmostEqual(flow, 117.8103380205204)
def test_get_ensemble_forecast_using_known_service_and_db_content(self): utc = Calendar() # always use Calendar() stuff met_stations=[ # this is the list of MetStations, the gis_id tells the position, the remaining tells us what properties we observe/forecast/calculate at the metstation (smg-ts) MetStationConfig(gis_id=598,temperature=u'/LTM5-Nea...........-T0017A3P_EC00_ENS',precipitation=u'/LTM5-Nea...........-T0000A5P_EC00_ENS') ] #note: the MetStationConfig can be constructed from yaml-config gis_location_repository=GisLocationService() # this provides the gis locations for my stations smg_ts_repository = SmGTsRepository(PROD,FC_PROD) # this provide the read function for my time-series n_ensembles=51 ens_station_list=[ EnsembleStation(598,n_ensembles, temperature_ens=lambda i:u'/LTM5-Nea...........-T0017A3P_EC00_E{0:02}'.format(i), precipitation_ens=lambda i:u'/LTM5-Nea...........-T0000A5P_EC00_E{0:02}'.format(i), wind_speed_ens=None, radiation_ens=None, relative_humidity_ens=None ), EnsembleStation(574,n_ensembles, temperature_ens=lambda i:u'/LTM5-Tya...........-T0017A3P_EC00_E{0:02}'.format(i), precipitation_ens=lambda i:u'/LTM5-Tya...........-T0000A5P_EC00_E{0:02}'.format(i), wind_speed_ens=None, radiation_ens=None, relative_humidity_ens=None ) ] ens_config=EnsembleConfig(n_ensembles,ens_station_list) geo_ts_repository = GeoTsRepository( epsg_id=32633, geo_location_repository=gis_location_repository, ts_repository=smg_ts_repository, met_station_list=met_stations, ens_config=ens_config) #pass service info and met_stations self.assertIsNotNone(geo_ts_repository) utc_period = UtcPeriod(utc.time(YMDhms(2015, 10, 1, 0, 0, 0)),utc.time(YMDhms(2015, 10, 10, 0, 0, 0))) ts_types= ['temperature','precipitation'] ens_geo_ts_dict = geo_ts_repository.get_forecast_ensemble(ts_types,utc_period=utc_period,t_c=None,geo_location_criteria=None) self.assertIsNotNone(ens_geo_ts_dict) self.assertEqual(ens_config.n_ensembles,len(ens_geo_ts_dict)) for i in range(ens_config.n_ensembles): for ts_type in ts_types: self.assertTrue(ts_type in ens_geo_ts_dict[i].keys(),"we ecpect to find an entry for each requested type (it could be empty list though") self.assertTrue(len(ens_geo_ts_dict[i][ts_type])>0,"we expect to find the series that we pass in, given they have not changed the name in SmG PROD")
def test_get_timeseries_using_known_service_and_db_content(self): utc = Calendar() # always use Calendar() stuff met_stations = [ # this is the list of MetStations, the gis_id tells the position, the remaining tells us what properties we observe/forecast/calculate at the metstation (smg-ts) MetStationConfig( gis_id=598, temperature=u'/NeNi-Sylsjøen......-T0017V3KI0114', precipitation=u'/NeNi-Sylsjøen-2....-T0000D9BI0124'), MetStationConfig( gis_id=574, temperature=u'/NeNi-Stuggusjøen...-T0017V3KI0114', precipitation=u'/NeNi-Stuggusjøen...-T0000D9BI0124', radiation= u'/ENKI/STS/Radiation/Sim.-Stuggusjøen...-T0006A0B-0119') ] #note: the MetStationConfig can be constructed from yaml-config gis_location_repository = GisLocationService( ) # this provides the gis locations for my stations smg_ts_repository = SmGTsRepository( PROD, FC_PROD) # this provide the read function for my time-series geo_ts_repository = GeoTsRepository( epsg_id=32633, geo_location_repository=gis_location_repository, ts_repository=smg_ts_repository, met_station_list=met_stations, ens_config=None) #pass service info and met_stations self.assertIsNotNone(geo_ts_repository) utc_period = UtcPeriod(utc.time(YMDhms(2010, 1, 1, 0, 0, 0)), utc.time(YMDhms(2010, 1, 2, 0, 0, 0))) ts_types = ['temperature', 'precipitation', 'radiation'] geo_ts_dict = geo_ts_repository.get_timeseries( ts_types, utc_period=utc_period, geo_location_criteria=None) self.assertIsNotNone(geo_ts_dict) for ts_type in ts_types: self.assertTrue( ts_type in geo_ts_dict.keys(), "we ecpect to find an entry for each requested type (it could be empty list though" ) self.assertTrue( len(geo_ts_dict[ts_type]) > 0, "we expect to find the series that we pass in, given they have not changed the name in SmG PROD" )
def test_raise_exception_when_no_data_in_request_period(self): utc_calendar = Calendar() netcdf_repository = self._construct_from_test_data() netcdf_repository.raise_if_no_data=True # yes, for now, just imagine this could work. self.assertIsNotNone(netcdf_repository) utc_period = UtcPeriod(utc_calendar.time(YMDhms(2017, 1, 1, 0, 0, 0)),# a period where there is no data in utc_calendar.time(YMDhms(2020, 12, 31, 0, 0, 0)))# the file supplied type_source_map = dict() type_source_map['temperature'] = TemperatureSource #def test_function(): # # return netcdf_repository.get_timeseries( # type_source_map, # geo_location_criteria=None, # utc_period=utc_period) #self.assertRaises(RuntimeError, test_function) self.assertRaises(RuntimeError, netcdf_repository.get_timeseries, type_source_map, **{'geo_location_criteria':None, 'utc_period':utc_period})
def test_simple_case(self): utc = Calendar() t0 = utc.time(2018, 1, 1) dt = deltahours(1) dv = DoubleVector() dv[:] = [1.2, 0.0, 2.0, 5.0, 15.0, float('nan'), -1.0] # these are bit-encoded values, note 1.2 -> 1.0 i0_1_e = [1.0, 0.0, 0.0, 1.0, 1.0, float('nan'), float('nan')] # expected values for bit 0 1-bit i1_3_e = [0.0, 0.0, 1.0, 2.0, 7.0, float('nan'), float('nan')] # expected values for bit 3 3-bits ts = TimeSeries(TimeAxis(t0, dt, len(dv)), dv, POINT_AVERAGE_VALUE) i0_1 = ts.decode(start_bit=0, n_bits=1) i1_3 = ts.decode(start_bit=1, n_bits=3) assert_array_almost_equal(i0_1.values.to_numpy(), np.array(i0_1_e)) assert_array_almost_equal(i1_3.values.to_numpy(), np.array(i1_3_e))
def test_crudf_cycle(self): """ Verify we can create, store, read, find and delete state in the state repository """ # arrange, by creating one State cal = Calendar() utc_timestamp = cal.time(YMDhms(2001, 1, 1)) region_model_id = "neanidelv-ptgsk" n_cells = 10 tags = ["initial", "unverified"] state_vector = self._create_state_vector(n_cells) self.assertIsNotNone(state_vector, "we should have a valid state vector object at this spot") # now start state_repository test state_repository = YamlStateRepository(self._test_state_directory) # put in two states, record the unique state_id.. state_id_1 = state_repository.put_state(region_model_id, utc_timestamp, state_vector, tags) state_id_2 = state_repository.put_state(region_model_id, utc_timestamp, state_vector, tags) # assert that we got two unique state_id self.assertIsNotNone(state_id_1, "We expect back a unique id") self.assertIsNotNone(state_id_2, "We expect back a unique id") self.assertNotEqual(state_id_1, state_id_2, "storing two state, same model, same time, each state should be stored with a unique id") # now we should have two states in the repository state_infos = state_repository.find_state() self.assertEqual(2, len(state_infos), "We just stored two, expect two back..") # extra test, verify that we really stored the state (using kirchner q) state_1 = state_repository.get_state(state_id_1) self.assertEqual(n_cells, state_1.size(), "expect to get back state with same number of cells") for i in range(n_cells): self.assertAlmostEqual(state_1[i].kirchner.q, state_vector[i].kirchner.q, 3, "state repository should preserve state...") # now remove state state_repository.delete_state(state_id_1) # check that we got just one left, and that it is the correct one.. state_list = state_repository.find_state() self.assertEqual(1, len(state_list)) self.assertEqual(state_list[0].region_model_id, region_model_id) self.assertEqual(state_list[0].utc_timestamp, utc_timestamp) self.assertEqual(state_list[0].state_id, state_id_2)
def test_glacier_melt_ts_m3s(self): utc = Calendar() t0 = utc.time(2016,10,1) dt = deltahours(1) n = 240 ta = Timeaxis(t0, dt, n) area_m2 = 487*1000*1000 # Jostedalsbreen, largest i Europe temperature = Timeseries(ta=ta, fill_value=10.0, point_fx=fx_policy.POINT_AVERAGE_VALUE) sca_values = dv.from_numpy(np.linspace(area_m2*1.0,0.0,num=n)) sca = Timeseries(ta=ta, values=sca_values, point_fx=fx_policy.POINT_AVERAGE_VALUE) gf = 1.0 *area_m2 dtf = 6.0 melt_m3s = create_glacier_melt_ts_m3s(temperature, sca, gf, dtf) # Here we get back a melt_ts, that we can do ts-stuff with self.assertIsNotNone(melt_m3s) full_melt_m3s = glacier_melt_step(dtf, 10.0, 0.0, gf) expected_melt_m3s = np.linspace(0.0,full_melt_m3s,num=n) assert_array_almost_equal(expected_melt_m3s,melt_m3s.values.to_numpy(),4) # Just to check we can work with the result as a ts in all aspects mx2 = melt_m3s*2.0 emx2 = expected_melt_m3s * 2.0; assert_array_almost_equal(emx2, mx2.values.to_numpy(), 4)
def test_get_ensemble_forecast_using_known_service_and_db_content(self): utc = Calendar() met_stations = [MetStationConfig(gis_id=598, temperature=u'/LTM5-Nea...........-T0017A3P_EC00_ENS', precipitation=u'/LTM5-Nea...........-T0000A5P_EC00_ENS')] gis_location_repository = GisLocationService() smg_ts_repository = SmGTsRepository(PREPROD, FC_PREPROD) n_ensembles = 51 ens_station_list = [ EnsembleStation(598, n_ensembles, temperature_ens = lambda i:u'/LTM5-Nea...........-T0017A3P_EC00_E{0:02}'.format(i), precipitation_ens = lambda i:u'/LTM5-Nea...........-T0000A5P_EC00_E{0:02}'.format(i), wind_speed_ens = None, radiation_ens = None, relative_humidity_ens = None ), EnsembleStation(574, n_ensembles, temperature_ens = lambda i:u'/LTM5-Tya...........-T0017A3P_EC00_E{0:02}'.format(i), precipitation_ens = lambda i:u'/LTM5-Tya...........-T0000A5P_EC00_E{0:02}'.format(i), wind_speed_ens = None, radiation_ens = None, relative_humidity_ens = None ) ] ens_config = EnsembleConfig(n_ensembles, ens_station_list) geo_ts_repository = GeoTsRepository( epsg_id = 32633, geo_location_repository = gis_location_repository, ts_repository = smg_ts_repository, met_station_list = met_stations, ens_config = ens_config) self.assertIsNotNone(geo_ts_repository) utc_period = UtcPeriod(utc.time(YMDhms(2015, 10, 1, 0, 0, 0)),utc.time(YMDhms(2015, 10, 10, 0, 0, 0))) ts_types= ['temperature', 'precipitation'] ens_geo_ts_dict = geo_ts_repository.get_forecast_ensemble(ts_types, utc_period=utc_period, t_c=None, geo_location_criteria=None) self.assertIsNotNone(ens_geo_ts_dict) self.assertEqual(ens_config.n_ensembles, len(ens_geo_ts_dict)) for i in range(ens_config.n_ensembles): for ts_type in ts_types: self.assertTrue(ts_type in ens_geo_ts_dict[i].keys(), "we expect to find an entry for each requested type (it could be empty list though)") self.assertTrue(len(ens_geo_ts_dict[i][ts_type])>0, "we expect to find the series that we pass in by name in SmG PREPROD")
def test_get_forecast_using_known_service_and_db_content(self): utc = Calendar() met_stations=[ # this is the list of MetStations, the gis_id tells the position, the remaining tells us what properties we observe/forecast/calculate at the metstation (smg-ts) MetStationConfig(gis_id=598, temperature=u'/LTM5-Nea...........-T0017A3P_EC00_ENS', precipitation=u'/LTM5-Nea...........-T0000A5P_EC00_ENS') ] gis_location_repository = GisLocationService() smg_ts_repository = SmGTsRepository(PREPROD, FC_PREPROD) geo_ts_repository = GeoTsRepository( epsg_id = 32633, geo_location_repository = gis_location_repository, ts_repository = smg_ts_repository, met_station_list = met_stations, ens_config = None) self.assertIsNotNone(geo_ts_repository) utc_period = UtcPeriod(utc.time(YMDhms(2015, 10, 1, 0, 0, 0)), utc.time(YMDhms(2015, 10, 10, 0, 0, 0))) ts_types= ['temperature', 'precipitation'] geo_ts_dict = geo_ts_repository.get_forecast(ts_types, utc_period=utc_period, t_c=None, geo_location_criteria=None) self.assertIsNotNone(geo_ts_dict) for ts_type in ts_types: self.assertTrue(ts_type in geo_ts_dict.keys(), "we expect to find an entry for each requested type (it could be empty list though)") self.assertTrue(len(geo_ts_dict[ts_type])>0, "we expect to find the series that we pass in by name in SmG PREPROD")
def continuous_calibration(): utc = Calendar() t_start = utc.time(YMDhms(2011, 9, 1)) t_fc_start = utc.time(YMDhms(2015, 10, 1)) dt = deltahours(1) n_obs = int(round((t_fc_start - t_start)/dt)) obs_time_axis = Timeaxis(t_start, dt, n_obs + 1) q_obs_m3s_ts = observed_tistel_discharge(obs_time_axis.total_period()) ptgsk = create_tistel_simulator(PTGSKOptModel, tistel.geo_ts_repository(tistel.grid_spec.epsg())) initial_state = burn_in_state(ptgsk, t_start, utc.time(YMDhms(2012, 9, 1)), q_obs_m3s_ts) num_opt_days = 30 # Step forward num_opt_days days and store the state for each day: recal_start = t_start + deltahours(num_opt_days*24) t = t_start state = initial_state opt_states = {t: state} while t < recal_start: ptgsk.run(Timeaxis(t, dt, 24), state) t += deltahours(24) state = ptgsk.reg_model_state opt_states[t] = state recal_stop = utc.time(YMDhms(2011, 10, 30)) recal_stop = utc.time(YMDhms(2012, 5, 30)) curr_time = recal_start q_obs_avg = TsTransform().to_average(t_start, dt, n_obs + 1, q_obs_m3s_ts) target_spec = TargetSpecificationPts(q_obs_avg, IntVector([0]), 1.0, KLING_GUPTA) target_spec_vec = TargetSpecificationVector([target_spec]) i = 0 times = [] values = [] p, p_min, p_max = construct_calibration_parameters(ptgsk) while curr_time < recal_stop: print(i) i += 1 opt_start = curr_time - deltahours(24*num_opt_days) opt_state = opt_states.pop(opt_start) p = ptgsk.region_model.get_region_parameter() p_opt = ptgsk.optimize(Timeaxis(opt_start, dt, 24*num_opt_days), opt_state, target_spec_vec, p, p_min, p_max, tr_stop=1.0e-5) ptgsk.region_model.set_region_parameter(p_opt) corr_state = adjust_simulator_state(ptgsk, curr_time, q_obs_m3s_ts) ptgsk.run(Timeaxis(curr_time, dt, 24), corr_state) curr_time += deltahours(24) opt_states[curr_time] = ptgsk.reg_model_state discharge = ptgsk.region_model.statistics.discharge([0]) times.extend(discharge.time(i) for i in range(discharge.size())) values.extend(list(np.array(discharge.v))) plt.plot(utc_to_greg(times), values) plot_results(None, q_obs=observed_tistel_discharge(UtcPeriod(recal_start, recal_stop))) set_calendar_formatter(Calendar()) #plt.interactive(1) plt.title("Continuously recalibrated discharge vs observed") plt.xlabel("Time in UTC") plt.ylabel(r"Discharge in $\mathbf{m^3s^{-1}}$", verticalalignment="top", rotation="horizontal") plt.gca().yaxis.set_label_coords(0, 1.1)
def forecast_demo(): """Simple forecast demo using arome data from met.no. Initial state is bootstrapped by simulating one hydrological year (starting Sept 1. 2011), and then calculating the state August 31. 2012. This state is then used as initial state for simulating Sept 1, 2011, after scaling with observed discharge. The validity of this approach is limited by the temporal variation of the spatial distribution of the discharge state, q, in the Kirchner method. The model is then stepped forward until Oct 1, 2015, and then used to compute the discharge for 65 hours using Arome data. At last, the results are plotted as simple timeseries. """ utc = Calendar() t_start = utc.time(YMDhms(2011, 9, 1)) t_fc_start = utc.time(YMDhms(2015, 10, 1)) dt = deltahours(1) n_obs = int(round((t_fc_start - t_start)/dt)) n_fc = 65 obs_time_axis = Timeaxis(t_start, dt, n_obs) fc_time_axis = Timeaxis(t_fc_start, dt, n_fc) total_time_axis = Timeaxis(t_start, dt, n_obs + n_fc) q_obs_m3s_ts = observed_tistel_discharge(total_time_axis.total_period()) ptgsk = create_tistel_simulator(PTGSKOptModel, tistel.geo_ts_repository(tistel.grid_spec.epsg())) initial_state = burn_in_state(ptgsk, t_start, utc.time(YMDhms(2012, 9, 1)), q_obs_m3s_ts) ptgsk.run(obs_time_axis, initial_state) plot_results(ptgsk, q_obs_m3s_ts) current_state = adjust_simulator_state(ptgsk, t_fc_start, q_obs_m3s_ts) ptgsk_fc = create_tistel_simulator(PTGSKModel, tistel.arome_repository(tistel.grid_spec, t_fc_start)) ptgsk_fc.run(fc_time_axis, current_state) plt.figure() q_obs_m3s_ts = observed_tistel_discharge(fc_time_axis.total_period()) plot_results(ptgsk_fc, q_obs_m3s_ts) #plt.interactive(1) plt.show()
def test_get_timeseries_using_known_service_and_db_content(self): utc = Calendar() # always use Calendar() stuff met_stations=[ # this is the list of MetStations, the gis_id tells the position, the remaining tells us what properties we observe/forecast/calculate at the metstation (smg-ts) MetStationConfig(gis_id=598,temperature=u'/NeNi-Sylsjøen......-T0017V3KI0114',precipitation=u'/NeNi-Sylsjøen-2....-T0000D9BI0124'), MetStationConfig(gis_id=574,temperature=u'/NeNi-Stuggusjøen...-T0017V3KI0114',precipitation=u'/NeNi-Stuggusjøen...-T0000D9BI0124',radiation=u'/ENKI/STS/Radiation/Sim.-Stuggusjøen...-T0006A0B-0119') ] #note: the MetStationConfig can be constructed from yaml-config gis_location_repository=GisLocationService() # this provides the gis locations for my stations smg_ts_repository = SmGTsRepository(PROD,FC_PROD) # this provide the read function for my time-series geo_ts_repository = GeoTsRepository(epsg_id=32633, geo_location_repository=gis_location_repository, ts_repository=smg_ts_repository, met_station_list=met_stations, ens_config=None) #pass service info and met_stations self.assertIsNotNone(geo_ts_repository) utc_period = UtcPeriod(utc.time(YMDhms(2010, 1, 1, 0, 0, 0)),utc.time(YMDhms(2010, 1, 2, 0, 0, 0))) ts_types= ['temperature','precipitation','radiation'] geo_ts_dict = geo_ts_repository.get_timeseries(ts_types,utc_period=utc_period,geo_location_criteria=None) self.assertIsNotNone(geo_ts_dict) for ts_type in ts_types: self.assertTrue(ts_type in geo_ts_dict.keys(),"we ecpect to find an entry for each requested type (it could be empty list though") self.assertTrue(len(geo_ts_dict[ts_type])>0,"we expect to find the series that we pass in, given they have not changed the name in SmG PROD")
def test_run_observed_then_arome_and_store(self): """ Start Tistel 2015.09.01, dummy state with some kirchner water use observations around Tistel (geo_ts_repository) and simulate forwared to 2015.10.01 (store discharge and catchment level precip/temp) then use arome forecast for 65 hours (needs arome for this period in arome-directory) finally store the arome results. """ utc = Calendar() # No offset gives Utc time_axis = Timeaxis(utc.time(YMDhms(2015, 9, 1, 0)), deltahours(1), 30 * 24) fc_time_axis = Timeaxis(utc.time(YMDhms(2015, 10, 1, 0)), deltahours(1), 65) interpolation_id = 0 ptgsk = DefaultSimulator("Tistel-ptgsk", interpolation_id, self.region_model_repository, self.geo_ts_repository, self.interpolation_repository, None) n_cells = ptgsk.region_model.size() ptgsk_state = DefaultStateRepository(ptgsk.region_model.__class__, n_cells) ptgsk.region_model.set_state_collection(-1, True) # collect state so we can inspect it s0 = ptgsk_state.get_state(0) for i in range(s0.size()): # add some juice to get started s0[i].kirchner.q = 0.5 ptgsk.run(time_axis, s0) print("Done simulation, testing that we can extract data from model") cids = api.IntVector() # we pull out for all the catchments-id if it's empty model = ptgsk.region_model # fetch out the model sum_discharge = model.statistics.discharge(cids) self.assertIsNotNone(sum_discharge) avg_temperature = model.statistics.temperature(cids) avg_precipitation = model.statistics.precipitation(cids) self.assertIsNotNone(avg_precipitation) self.assertIsNotNone(avg_temperature) for time_step in range(time_axis.size()): precip_raster = model.statistics.precipitation(cids, time_step) # example raster output self.assertEqual(precip_raster.size(), n_cells) avg_gs_lwc = model.gamma_snow_state.lwc(cids) # sca skaugen|gamma self.assertIsNotNone(avg_gs_lwc) # lwc surface_heat alpha melt_mean melt iso_pot_energy temp_sw avg_gs_output = model.gamma_snow_response.outflow(cids) self.assertIsNotNone(avg_gs_output) print("done. now save to db") # SmGTsRepository(PROD,FC_PROD) save_list = [ TsStoreItem(u'/test/x/shyft/tistel/discharge_m3s', lambda m: m.statistics.discharge(cids)), TsStoreItem(u'/test/x/shyft/tistel/temperature', lambda m: m.statistics.temperature(cids)), TsStoreItem(u'/test/x/shyft/tistel/precipitation', lambda m: m.statistics.precipitation(cids)), ] tss = TimeseriesStore(SmGTsRepository(PREPROD, FC_PREPROD), save_list) self.assertTrue(tss.store_ts(ptgsk.region_model)) print("Run forecast arome") endstate = ptgsk.region_model.state_t.vector_t() ptgsk.region_model.get_states(endstate) # get the state at end of obs ptgsk.geo_ts_repository = self.arome_repository # switch to arome here ptgsk.run_forecast(fc_time_axis, fc_time_axis.start, endstate) # now forecast print("Done forecast") fc_save_list = [ TsStoreItem(u'/test/x/shyft/tistel/fc_discharge_m3s', lambda m: m.statistics.discharge(cids)), TsStoreItem(u'/test/x/shyft/tistel/fc_temperature', lambda m: m.statistics.temperature(cids)), TsStoreItem(u'/test/x/shyft/tistel/fc_precipitation', lambda m: m.statistics.precipitation(cids)), TsStoreItem(u'/test/x/shyft/tistel/fc_radiation', lambda m: m.statistics.radiation(cids)), TsStoreItem(u'/test/x/shyft/tistel/fc_rel_hum', lambda m: m.statistics.rel_hum(cids)), TsStoreItem(u'/test/x/shyft/tistel/fc_wind_speed', lambda m: m.statistics.wind_speed(cids)), ] TimeseriesStore(SmGTsRepository(PREPROD, FC_PREPROD), fc_save_list).store_ts(ptgsk.region_model) print("Done save to db")
def test_utctime_from_datetime(self): utc = Calendar() dt1 = dt.datetime(2015, 6, 1, 2, 3, 4) t1 = utctime_from_datetime(dt1) self.assertEqual(t1, utc.time(2015, 6, 1, 2, 3, 4))