def test_percentiles(self): c = api.Calendar() t0 = c.time(2016, 1, 1) dt = api.deltahours(1) n = 240 ta = api.Timeaxis(t0, dt, n) timeseries = api.TsVector() for i in range(10): timeseries.append( api.Timeseries(ta=ta, fill_value=i, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE)) wanted_percentiles = api.IntVector([0, 10, 50, -1, 70, 100]) ta_day = api.Timeaxis(t0, dt * 24, n // 24) ta_day2 = api.Timeaxis2(t0, dt * 24, n // 24) percentiles = api.percentiles(timeseries, ta_day, wanted_percentiles) percentiles2 = timeseries.percentiles(ta_day2, wanted_percentiles) # just to verify it works with alt. syntax self.assertEqual(len(percentiles2), len(percentiles)) for i in range(len(ta_day)): self.assertAlmostEqual(0.0, percentiles[0].value(i), 3, " 0-percentile") self.assertAlmostEqual(0.9, percentiles[1].value(i), 3, " 10-percentile") self.assertAlmostEqual(4.5, percentiles[2].value(i), 3, " 50-percentile") self.assertAlmostEqual(4.5, percentiles[3].value(i), 3, " -average") self.assertAlmostEqual(6.3, percentiles[4].value(i), 3, " 70-percentile") self.assertAlmostEqual(9.0, percentiles[5].value(i), 3, "100-percentile")
def setUp(self): self.c=api.Calendar() self.d=api.deltahours(1) self.n=24 #self.t= self.c.trim(api.utctime_now(),self.d) self.t= self.c.trim(self.c.time(api.YMDhms(1969,12,31,0,0,0)),self.d) self.ta=api.Timeaxis(self.t,self.d,self.n)
def create_mock_station_data(t0, dt, n_steps, **kwargs): time_axis = api.Timeaxis(t0, dt, n_steps) return {"temperature": create_mock_time_series_data("temperature", time_axis, **kwargs), "precipitation": create_mock_time_series_data("precipitation", time_axis, **kwargs), "relative_humidity": create_mock_time_series_data("relative_humidity", time_axis, **kwargs), "wind_speed": create_mock_time_series_data("wind_speed", time_axis, **kwargs), "radiation": create_mock_time_series_data("radiation", time_axis, **kwargs)}
def test_can_run_bayesian_kriging_from_arome25_to_1km(self): """ Verify that if we run btk interpolation, we do get updated time-series according to time-axis and range specified. """ # arrange the test with a btk_parameter, a source grid and a destination grid btk_parameter = api.BTKParameter(temperature_gradient=-0.6, temperature_gradient_sd=0.25, sill=25.0, nugget=0.5, range=20000.0, zscale=20.0) fx = lambda z: api.DoubleVector.from_numpy( (20.0 - 0.6 * z / 100) + 3.0 * np.sin( np.arange(start=0, stop=self.n, step=1) * 2 * np.pi / 24.0 - np .pi / 2.0)) arome_grid = self._create_geo_ts_grid(self.nx, self.ny, self.dx_arome, fx) destination_grid = self._create_geo_point_grid(self.mnx, self.mny, self.dx_model) ta = api.Timeaxis(self.t, self.d * 3, int(self.n / 3)) # act, - run the bayesian_kriging_temperature algoritm. r = api.bayesian_kriging_temperature(arome_grid, destination_grid, ta, btk_parameter) # assert self.assertIsNotNone(r) self.assertEqual(len(r), self.mnx * self.mny) for gts in r: # do some sanity checks for the btk. Note that full-range checking is already done elsewhere self.assertEqual(gts.ts.size(), ta.size()) self.assertLess(np.max(gts.ts.values.to_numpy()), 23.0) # all values less than ~max self.assertGreater(np.min(gts.ts.values.to_numpy()), 7.0) # all values larger than ~ min
def test_create_region_environment(self): cal = api.Calendar() time_axis = api.Timeaxis(cal.time(api.YMDhms(2015, 1, 1, 0, 0, 0)), api.deltahours(1), 240) re = self.create_dummy_region_environment(time_axis, api.GeoPoint(1000, 1000, 100)) self.assertIsNotNone(re) self.assertEqual(len(re.radiation), 1) self.assertAlmostEqual(re.radiation[0].ts.value(0), 300.0)
def _transform_raw(self, data, time): #def noop_time(t): # return api.Timeaxis(api.utctime(t[0]), api.timespan(t[1] - t[0]), len(t)) def noop_space(x): return x def air_temp_conv(x): return x - 273.15 def prec_conv(x): return x * 3600 convert_map = { "wind_speed": lambda x, ta: (noop_space(x), ta), "radiation": lambda x, ta: (noop_space(x), ta), "temperature": lambda x, ta: (air_temp_conv(x), ta), "precipitation": lambda x, ta: (prec_conv(x), ta), "relative_humidity": lambda x, ta: (noop_space(x), ta) } ta = api.Timeaxis(int(time[0]), int(time[1] - time[0]), len(time)) res = {} for k, v in data.items(): res[k] = convert_map[k](v, ta) return res
def __init__(self, config_file, config_section, **kwargs): """ Setup a config instance for a netcdf orchestration from a YAML file. Parameters ---------- config_file : string Path to the YAML configuration file config_section : string Section in YAML file for simulation parameters. Returns ------- YAMLConfig instance """ # The config_file needs to be an absolute path or have 'config_dir' if os.path.isabs(config_file): self._config_file = config_file self.config_dir = os.path.dirname(config_file) elif "config_dir" in kwargs: self._config_file = os.path.join(kwargs["config_dir"], config_file) else: raise ConfigError("'config_file' must be an absolute path " "or 'config_dir' passed as an argument") self._config_section = config_section # Load main configuration file with open(self._config_file) as cfg: config = yaml.load(cfg)[config_section] # Expose all keys in yaml file as attributes self.__dict__.update(config) # Override the parameters with kwargs self.__dict__.update(kwargs) # Check validity of some attributes if not hasattr(self, "config_dir"): raise ConfigError("'config_dir' must be present in config section " "or passed as an argument") if not (os.path.isdir(self.config_dir) and os.path.isabs(self.config_dir)): raise ConfigError( "'config_dir' must exist and be an absolute path") if not hasattr(self, "data_dir"): raise ConfigError("'data_dir' must be present in config section " "or passed as an argument") if not (os.path.isdir(self.data_dir) and os.path.isabs(self.data_dir)): raise ConfigError("'data_dir' must exist and be an absolute path") # Create a time axis self.start_time = utctime_from_datetime(self.start_datetime) self.time_axis = api.Timeaxis(self.start_time, self.run_time_step, self.number_of_steps) # Get the region model in API (already an object if in kwargs) if 'model_t' not in kwargs: module, model_t = self.model_t.split(".") self.model_t = getattr(globals()[module], model_t)
def test_compute_lwc_percentiles(self): # Simulation time axis year, month, day, hour = 2010, 9, 1, 0 dt = api.deltahours(24) n_steps = 400 utc = api.Calendar() # No offset gives Utc t0 = utc.time(api.YMDhms(year, month, day, hour)) time_axis = api.Timeaxis(t0, dt, n_steps) # Some fake ids region_id = 0 interpolation_id = 0 # Simulation coordinate system epsg = "32633" # Model model_t = pt_gs_k.PTGSKModel # Configs and repositories dataset_config_file = path.join(path.dirname(__file__), "netcdf", "atnsjoen_datasets.yaml") region_config_file = path.join(path.dirname(__file__), "netcdf", "atnsjoen_calibration_region.yaml") region_config = RegionConfig(region_config_file) model_config = ModelConfig(self.model_config_file) dataset_config = YamlContent(dataset_config_file) region_model_repository = RegionModelRepository( region_config, model_config, model_t, epsg) interp_repos = InterpolationParameterRepository(model_config) netcdf_geo_ts_repos = [] for source in dataset_config.sources: station_file = source["params"]["stations_met"] netcdf_geo_ts_repos.append( GeoTsRepository(source["params"], station_file, "")) geo_ts_repository = GeoTsRepositoryCollection(netcdf_geo_ts_repos) # Construct target discharge series simulator = DefaultSimulator(region_id, interpolation_id, region_model_repository, geo_ts_repository, interp_repos, None) n_cells = simulator.region_model.size() state_repos = DefaultStateRepository(model_t, n_cells) cid = 1 simulator.region_model.set_state_collection(cid, True) simulator.run(time_axis, state_repos.get_state(0)) percentile_list = [10, 25, 50, 75, 90] # From here, things could be calculated without copies (except for 't') # TODO: Graham optimize with numba :-) cells = simulator.region_model.get_cells() lwcs = [np.array(cell.sc.gs_lwc.v) for cell in cells] # Contiguous t = np.array([ cells[0].sc.gs_lwc.time(i) for i in range(cells[0].sc.gs_lwc.size()) ]) percentiles = np.percentile(np.array(lwcs), percentile_list, 0)
def __init__(self, config_file, config_section, overrides=None): """ Setup a config instance for a netcdf orchestration from a YAML file. Parameters ---------- config_file : string Path to the YAML configuration file config_section : string Section in YAML file for simulation parameters. Returns ------- YAMLConfig instance """ if overrides is None: overrides = {} # The config_file needs to be an absolute path if os.path.isabs(config_file): self._config_file = config_file self.config_dir = os.path.dirname(config_file) else: raise ConfigError("'config_file' must be an absolute path ") self._config_section = config_section # Load main configuration file with open(self._config_file, encoding='utf8') as cfg: config = yaml.load(cfg)[config_section] # Expose all keys in yaml file as attributes self.__dict__.update(config) # Override the parameters with kwargs #self.__dict__.update(kwargs) self.__dict__.update(overrides.get("config", {})) self.validate() # Create a time axis # It is assumed that the time specified in the config file is in UTC self.start_time = utctime_from_datetime(self.start_datetime) self.time_axis = api.Timeaxis(self.start_time, self.run_time_step, self.number_of_steps) # Get the region model in API (already an object if in kwargs) #if 'model_t' not in kwargs: # module, model_t = self.model_t.split(".") # self.model_t = getattr(globals()[module], model_t) # If region and interpolation ids are not present, just use fake ones # self.region_id = 0 if not hasattr(self, "region_id") else int(self.region_id) self.region_model_id = str(self.region_model_id) self.interpolation_id = 0 if not hasattr(self, "interpolation_id") \ else int(self.interpolation_id) self.initial_state_repo = None self.end_state_repo = None self.construct_repos(overrides)
def _convert_to_timeseries(self, data, t, ts_id): ta = api.Timeaxis(int(t[0]), int(t[1]) - int(t[0]), len(t)) tsc = api.TsFactory().create_point_ts def construct(d): return tsc(ta.size(), ta.start, ta.delta_t, api.DoubleVector.FromNdArray(d)) ts = [construct(data[:, j]) for j in range(data.shape[-1])] return {k: v for k, v in zip(ts_id, ts)}
def test_create_timeaxis(self): self.assertEqual(self.ta.size(),self.n) self.assertEqual(len(self.ta),self.n) self.assertEqual(self.ta(0).start,self.t) self.assertEqual(self.ta(0).end,self.t+self.d) self.assertEqual(self.ta(1).start,self.t+self.d) self.assertEqual(self.ta.total_period().start,self.t) va=np.array([86400,3600,3],dtype=np.int64) xta = api.Timeaxis(int(va[0]), int(va[1]), int(va[2])) #xta = api.Timeaxis(va[0], va[1], va[2])# TODO: boost.python require this to be int, needs overload for np.int64 types.. #xta = api.Timeaxis(86400,3600,3) self.assertEqual(xta.size(),3)
def setUp(self): self.cal = api.Calendar() self.dt = api.deltahours(1) self.nt = 24 * 10 self.t0 = self.cal.time(2016, 1, 1) self.ta = api.Timeaxis2(self.t0, self.dt, self.nt) self.ta1 = api.Timeaxis(self.t0, self.dt, self.nt) self.geo_points = api.GeoPointVector() self.geo_points.append(api.GeoPoint(100, 100, 1000)) self.geo_points.append(api.GeoPoint(5100, 100, 1150)) self.geo_points.append(api.GeoPoint(100, 5100, 850))
def test_unit_conversion(self): utc = api.Calendar() t_num = np.arange( -24, 24, 1, dtype=np.float64 ) # we use both before and after epoch to ensure sign is ok t_converted = convert_netcdf_time('hours since 1970-01-01 00:00:00', t_num) t_axis = api.Timeaxis(utc.time(api.YMDhms(1969, 12, 31, 0, 0, 0)), api.deltahours(1), 2 * 24) [ self.assertEqual(t_converted[i], t_axis(i).start) for i in range(t_axis.size()) ]
def test_average_accessor(self): dv = np.arange(self.ta.size()) v = api.DoubleVector.from_numpy(dv) t = api.UtcTimeVector() for i in range(self.ta.size()): t.push_back(self.ta(i).start) t.push_back( self.ta(self.ta.size() - 1).end) # important! needs n+1 points to determine n periods in the timeaxis tsf = api.TsFactory() ts1 = tsf.create_point_ts(self.ta.size(), self.t, self.d, v) ts2 = tsf.create_time_point_ts(self.ta.total_period(), t, v) tax = api.Timeaxis(self.ta.total_period().start + api.deltaminutes(30), api.deltahours(1), self.ta.size()) avg1 = api.AverageAccessorTs(ts1, tax) self.assertEqual(avg1.size(), tax.size()) self.assertIsNotNone(ts2)
def test_run_arome_ensemble(self): # Simulation time axis year, month, day, hour = 2015, 7, 26, 0 n_hours = 30 dt = api.deltahours(1) utc = api.Calendar() # No offset gives Utc t0 = utc.time(api.YMDhms(year, month, day, hour)) time_axis = api.Timeaxis(t0, dt, n_hours) # Some dummy ids not needed for the netcdf based repositories region_id = 0 interpolation_id = 0 # Simulation coordinate system epsg = "32633" # Model model_t = pt_gs_k.PTGSKOptModel # Configs and repositories region_config = RegionConfig(self.region_config_file) model_config = ModelConfig(self.model_config_file) region_model_repository = RegionModelRepository( region_config, model_config, model_t, epsg) interp_repos = InterpolationParameterRepository(model_config) base_dir = path.join(shyftdata_dir, "netcdf", "arome") pattern = "fc*.nc" try: geo_ts_repository = AromeDataRepository(epsg, base_dir, filename=pattern, allow_subset=True) except Exception as e: print("**** test_run_arome_ensemble: Arome data missing or" " wrong, test inconclusive ****") print("****{}****".format(e)) self.skipTest( "**** test_run_arome_ensemble: Arome data missing or wrong, test " "inconclusive ****\n\t exception:{}".format(e)) simulator = DefaultSimulator(region_id, interpolation_id, region_model_repository, geo_ts_repository, interp_repos, None) n_cells = simulator.region_model.size() state_repos = DefaultStateRepository(model_t, n_cells) simulators = simulator.create_ensembles(time_axis, t0, state_repos.get_state(0)) for s in simulators: s.simulate()
def test_time_shift(self): c = api.Calendar() t0 = c.time(2016, 1, 1) t1 = c.time(2017, 1, 1) dt = api.deltahours(1) n = 240 ta = api.Timeaxis(t0, dt, n) ts0 = api.Timeseries(ta=ta, fill_value=3.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) ts1 = api.time_shift(ts0, t1 - t0) ts2 = 2.0 * ts1.time_shift(t0 - t1) # just to verify it still can take part in an expression for i in range(ts0.size()): self.assertAlmostEqual(ts0.value(i), ts1.value(i), 3, "expect values to be equal") self.assertAlmostEqual(ts0.value(i) * 2.0, ts2.value(i), 3, "expect values to be double value") self.assertEqual(ts0.time(i) + (t1 - t0), ts1.time(i), "expect time to be offset delta_t different") self.assertEqual(ts0.time(i), ts2.time(i), "expect time to be equal")
def run_simulator(self, model_t): # Simulation time axis dt0 = api.YMDhms(2015, 8, 24, 6) n_hours = 30 dt = api.deltahours(1) utc = api.Calendar() # No offset gives Utc t0 = utc.time(dt0) time_axis = api.Timeaxis(t0, dt, n_hours) # Some dummy ids not needed for the netcdf based repositories region_id = 0 interpolation_id = 0 # Simulation coordinate system epsg = "32633" # Configs and repositories region_config = RegionConfig(self.region_config_file) model_config = ModelConfig(self.model_config_file) region_model_repository = RegionModelRepository( region_config, model_config, model_t, epsg) interp_repos = InterpolationParameterRepository(model_config) date_str = "{}{:02}{:02}_{:02}".format(dt0.year, dt0.month, dt0.day, dt0.hour) base_dir = path.join(shyftdata_dir, "repository", "arome_data_repository") f1 = "arome_metcoop_red_default2_5km_{}.nc".format(date_str) f2 = "arome_metcoop_red_test2_5km_{}.nc".format(date_str) ar1 = AromeDataRepository(epsg, base_dir, filename=f1, allow_subset=True) ar2 = AromeDataRepository(epsg, base_dir, filename=f2, elevation_file=f1, allow_subset=True) geo_ts_repository = GeoTsRepositoryCollection([ar1, ar2]) simulator = DefaultSimulator(region_id, interpolation_id, region_model_repository, geo_ts_repository, interp_repos, None) n_cells = simulator.region_model.size() state_repos = DefaultStateRepository(model_t, n_cells) simulator.run(time_axis, state_repos.get_state(0))
def test_compute_lwc_percentiles(self): # Simulation time axis year, month, day, hour = 2010, 9, 1, 0 dt = api.deltahours(24) n_steps = 400 utc = api.Calendar() # No offset gives Utc t0 = utc.time(api.YMDhms(year, month, day, hour)) time_axis = api.Timeaxis(t0, dt, n_steps) # Some fake ids region_id = 0 interpolation_id = 0 # Simulation coordinate system epsg = "32633" # Model model_t = pt_gs_k.PTGSKModel # Configs and repositories dataset_config_file = path.join(path.dirname(__file__), "netcdf", "atnsjoen_datasets.yaml") region_config_file = path.join(path.dirname(__file__), "netcdf", "atnsjoen_calibration_region.yaml") region_config = RegionConfig(region_config_file) model_config = ModelConfig(self.model_config_file) dataset_config = YamlContent(dataset_config_file) region_model_repository = RegionModelRepository(region_config, model_config, model_t, epsg) interp_repos = InterpolationParameterRepository(model_config) netcdf_geo_ts_repos = [] for source in dataset_config.sources: station_file = source["params"]["stations_met"] netcdf_geo_ts_repos.append(GeoTsRepository(source["params"], station_file, "")) geo_ts_repository = GeoTsRepositoryCollection(netcdf_geo_ts_repos) # Construct target discharge series simulator = DefaultSimulator(region_id, interpolation_id, region_model_repository, geo_ts_repository, interp_repos, None) n_cells = simulator.region_model.size() state_repos = DefaultStateRepository(model_t, n_cells) cid = 1 simulator.region_model.set_state_collection(cid, True) simulator.run(time_axis, state_repos.get_state(0)) self.assertAlmostEqual(simulator.region_model.cells[0].rc.pe_output.values[0], 0.039768354, 5) # just to verify pot.evap by regression, mm/h percentile_list = [10, 25, 50, 75, 90]
def test_idw_precipitation_transform_from_set_to_grid(self): """ Test IDW interpolation transforms precipitation time-series according to time-axis and range. """ idw_p = api.IDWPrecipitationParameter() self.assertEqual(idw_p.max_distance, 200000) self.assertEqual(idw_p.max_members, 20) fx = lambda z: [15 for x in range(self.n)] arome_grid = self._create_geo_precipitation_grid( self.nx, self.ny, self.dx_arome, fx) dest_grid_points = self._create_geo_point_grid(self.mnx, self.mny, self.dx_model) ta = api.Timeaxis(self.t, self.d * 3, int(self.n / 3)) dest_grid = api.idw_precipitation(arome_grid, dest_grid_points, ta, idw_p) self.assertIsNotNone(dest_grid) self.assertEqual(len(dest_grid), self.mnx * self.mny)
def test_timeseries_vector(self): c = api.Calendar() t0 = api.utctime_now() dt = api.deltahours(1) n = 240 ta = api.Timeaxis(t0, dt, n) a = api.Timeseries(ta=ta, fill_value=3.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) b = api.Timeseries(ta=ta, fill_value=2.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) v = api.TsVector() v.append(a) v.append(b) self.assertEqual(len(v), 2) self.assertAlmostEqual(v[0].value(0), 3.0, "expect first ts to be 3.0") aa = api.Timeseries(ta=a.time_axis, values=a.values, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE) # copy construct (really copy the values!) a.fill(1.0) self.assertAlmostEqual(v[0].value(0), 1.0, "expect first ts to be 1.0, because the vector keeps a reference ") self.assertAlmostEqual(aa.value(0), 3.0)
def test_ts_transform(self): dv = np.arange(self.ta.size()) v = api.DoubleVector.from_numpy(dv) t = api.UtcTimeVector(); for i in range(self.ta.size()): t.push_back(self.ta(i).start) # t.push_back(self.ta(self.ta.size()-1).end) #important! needs n+1 points to determine n periods in the timeaxis t_start = self.ta.total_period().start dt = api.deltahours(1) tax = api.Timeaxis(t_start + api.deltaminutes(30), dt, self.ta.size()) tsf = api.TsFactory() ts1 = tsf.create_point_ts(self.ta.size(), self.t, self.d, v) ts2 = tsf.create_time_point_ts(self.ta.total_period(), t, v) ts3 = api.TsFixed(tax, v) tst = api.TsTransform() tt1 = tst.to_average(t_start, dt, tax.size(), ts1) tt2 = tst.to_average(t_start, dt, tax.size(), ts2) tt3 = tst.to_average(t_start, dt, tax.size(), ts3) self.assertEqual(tt1.size(), tax.size()) self.assertEqual(tt2.size(), tax.size()) self.assertEqual(tt3.size(), tax.size())
def calc_alpha(cls,T): alpha=np.zeros(T.shape,dtype='float') #alpha[T<=Tice]=0. alpha[T>=cls.__T0]=1. indx=(T<cls.__T0)&(T>cls.__Tice) alpha[indx]=np.square((T[indx]-cls.__Tice)/(cls.__T0-cls.__Tice)) return alpha @classmethod def calc_RH(cls,T,Td,p): alpha = cls.calc_alpha(T) qsat = cls.calc_q(T,p,alpha) q = cls.calc_q(Td,p,alpha) return q/qsat if __name__ == "__main__": rconf = {'EPSG':32633} params = {'data_dir':'D:/users/ysa/shyft_main/shyft-data', 'stations_met':'netcdf/orchestration-testdata/All_variables_z_201408_5-31E_58-71N_0.75.nc'} utc = api.Calendar() # No offset gives Utc time_axis = api.Timeaxis(utc.time(api.YMDhms(2014, 8,1, 2)), api.deltahours(3), 10*8) #grid_spec = (xll, yll, delta x, delta y, nx, ny) xll, yll, dx, dy, nx, ny = 266000, 6960000, 1000.0, 1000.0, 109, 80 #bbox =[[x_min, x_max, x_max, x_min], # [y_min, y_min, y_max, y_max]] bbox =[[xll, xll+dx*nx, xll+dx*nx, xll], [yll, yll, yll+dy*ny, yll+dy*ny]] repo = ERAInterimDataRepository(params,rconf) res = repo.get_timeseries(['temperature','wind_speed','relative_humidity','radiation','precipitation'], time_axis.total_period(), geo_location_criteria=bbox)
def test_model_initialize_and_run(self): num_cells = 20 model_type = pt_gs_k.PTGSKModel model = self.build_model(model_type, pt_gs_k.PTGSKParameter, num_cells) self.assertEqual(model.size(), num_cells) # now modify snow_cv forest_factor to 0.1 region_parameter = model.get_region_parameter() region_parameter.gs.snow_cv_forest_factor = 0.1 region_parameter.gs.snow_cv_altitude_factor = 0.0001 self.assertEqual(region_parameter.gs.snow_cv_forest_factor, 0.1) self.assertEqual(region_parameter.gs.snow_cv_altitude_factor, 0.0001) self.assertAlmostEqual(region_parameter.gs.effective_snow_cv(1.0, 0.0), region_parameter.gs.snow_cv + 0.1) self.assertAlmostEqual( region_parameter.gs.effective_snow_cv(1.0, 1000.0), region_parameter.gs.snow_cv + 0.1 + 0.1) cal = api.Calendar() time_axis = api.Timeaxis(cal.time(api.YMDhms(2015, 1, 1, 0, 0, 0)), api.deltahours(1), 240) model_interpolation_parameter = api.InterpolationParameter() # degC/m, so -0.5 degC/100m model_interpolation_parameter.temperature_idw.default_temp_gradient = -0.005 # if possible use closest neighbor points and solve gradient using equation,(otherwise default min/max height) model_interpolation_parameter.temperature_idw.gradient_by_equation = True # Max number of temperature sources used for one interpolation model_interpolation_parameter.temperature_idw.max_members = 6 # 20 km is max distance model_interpolation_parameter.temperature_idw.max_distance = 20000 # Pure linear interpolation model_interpolation_parameter.temperature_idw.distance_measure_factor = 1.0 # This enables IDW with default temperature gradient. model_interpolation_parameter.use_idw_for_temperature = True self.assertAlmostEqual( model_interpolation_parameter.precipitation.scale_factor, 1.02) # just verify this one is as before change to scale_factor model.run_interpolation( model_interpolation_parameter, time_axis, self.create_dummy_region_environment( time_axis, model.get_cells()[int(num_cells / 2)].geo.mid_point())) s0 = pt_gs_k.PTGSKStateVector() for i in range(num_cells): si = pt_gs_k.PTGSKState() si.kirchner.q = 40.0 s0.append(si) model.set_states(s0) model.set_state_collection( -1, True) # enable state collection for all cells model.run_cells() cids = api.IntVector( ) # optional, we can add selective catchment_ids here sum_discharge = model.statistics.discharge(cids) self.assertIsNotNone(sum_discharge) avg_temperature = model.statistics.temperature(cids) avg_precipitation = model.statistics.precipitation(cids) self.assertIsNotNone(avg_precipitation) for time_step in range(time_axis.size()): precip_raster = model.statistics.precipitation( cids, time_step) # example raster output self.assertEqual(precip_raster.size(), num_cells) avg_gs_sca = model.gamma_snow_response.sca(cids) # swe output self.assertIsNotNone(avg_gs_sca) # lwc surface_heat alpha melt_mean melt iso_pot_energy temp_sw avg_gs_albedo = model.gamma_snow_state.albedo(cids) self.assertIsNotNone(avg_gs_albedo) self.assertEqual(avg_temperature.size(), time_axis.size(), "expect results equal to time-axis size") copy_region_model = model.__class__(model) self.assertIsNotNone(copy_region_model) copy_region_model.run_cells( ) #just to verify we can copy and run the new model
def test_snow_and_ground_water_response_calibration(self): """ Test dual calibration strategy: * First fit the three Kirchner parameters for ground water response during July, August, and September. * Then fit two snow routine parameters (tx and max_water) from November to April. """ # Simulation time axis year, month, day, hour = 2010, 9, 1, 0 dt = api.deltahours(24) n_steps = 400 utc = api.Calendar() # No offset gives Utc t0 = utc.time(api.YMDhms(year, month, day, hour)) time_axis = api.Timeaxis(t0, dt, n_steps) # Some fake ids region_id = 0 interpolation_id = 0 # Simulation coordinate system epsg = "32633" # Model model_t = pt_gs_k.PTGSKOptModel # Configs and repositories dataset_config_file = path.join(path.dirname(__file__), "netcdf", "atnsjoen_datasets.yaml") region_config_file = path.join(path.dirname(__file__), "netcdf", "atnsjoen_calibration_region.yaml") region_config = RegionConfig(region_config_file) model_config = ModelConfig(self.model_config_file) dataset_config = YamlContent(dataset_config_file) region_model_repository = RegionModelRepository( region_config, model_config, model_t, epsg) interp_repos = InterpolationParameterRepository(model_config) netcdf_geo_ts_repos = [] for source in dataset_config.sources: station_file = source["params"]["stations_met"] netcdf_geo_ts_repos.append( GeoTsRepository(source["params"], station_file, "")) geo_ts_repository = GeoTsRepositoryCollection(netcdf_geo_ts_repos) # Construct target discharge series simulator = DefaultSimulator(region_id, interpolation_id, region_model_repository, geo_ts_repository, interp_repos, None) n_cells = simulator.region_model.size() state_repos = DefaultStateRepository(model_t, n_cells) simulator.run(time_axis, state_repos.get_state(0)) cid = 1 target_discharge = simulator.region_model.statistics.discharge([cid]) # Construct kirchner parameters param = simulator.region_model.parameter_t( simulator.region_model.get_region_parameter()) print_param("True solution", param) kirchner_param_min = simulator.region_model.parameter_t(param) kirchner_param_max = simulator.region_model.parameter_t(param) # Kichner parameters are quite abstract (no physical meaning), so simply scale them kirchner_param_min.kirchner.c1 *= 0.8 kirchner_param_min.kirchner.c2 *= 0.8 kirchner_param_min.kirchner.c3 *= 0.8 kirchner_param_max.kirchner.c1 *= 1.2 kirchner_param_max.kirchner.c2 *= 1.2 kirchner_param_max.kirchner.c3 *= 1.2 # kirchner_t_start = utc.time(api.YMDhms(2011, 4, 1, 0)) # kirchner_time_axis = api.Timeaxis(kirchner_t_start, dt, 150) kirchner_time_axis = time_axis # Construct gamma snow parameters (realistic tx and max_lwc) gamma_snow_param_min = simulator.region_model.parameter_t(param) gamma_snow_param_max = simulator.region_model.parameter_t(param) gamma_snow_param_min.gs.tx = -1.0 # Min snow/rain temperature threshold gamma_snow_param_min.gs.max_water = 0.05 # Min 8% max water in snow in costal regions gamma_snow_param_max.gs.tx = 1.0 gamma_snow_param_max.gs.max_water = 0.25 # Max 35% max water content, or we get too little melt gs_t_start = utc.time(api.YMDhms(2010, 11, 1, 0)) gs_time_axis = api.Timeaxis(gs_t_start, dt, 250) # gs_time_axis = time_axis # Find parameters target_spec = api.TargetSpecificationPts(target_discharge, api.IntVector([cid]), 1.0, api.KLING_GUPTA) target_spec_vec = api.TargetSpecificationVector( ) # TODO: We currently dont fix list initializer for vectors target_spec_vec.append(target_spec) # Construct a fake, perturbed starting point for calibration p_vec = [param.get(i) for i in range(param.size())] for i, name in enumerate( [param.get_name(i) for i in range(len(p_vec))]): if name not in ("c1" "c2", "c3", "TX", "max_water"): next if name in ("c1", "c2", "c3"): p_vec[i] = random.uniform(0.8 * p_vec[i], 1.2 * p_vec[i]) elif name == "TX": p_vec[i] = random.uniform(gamma_snow_param_min.gs.tx, gamma_snow_param_max.gs.tx) elif name == "max_water": p_vec[i] = random.uniform(gamma_snow_param_min.gs.max_water, gamma_snow_param_max.gs.max_water) param.set(p_vec) print_param("Initial guess", param) # Two pass optimization, once for the ground water response, and second time for kirchner_p_opt = simulator.optimize(kirchner_time_axis, state_repos.get_state(0), target_spec_vec, param, kirchner_param_min, kirchner_param_max) gamma_snow_p_opt = simulator.optimize(gs_time_axis, state_repos.get_state(0), target_spec_vec, kirchner_p_opt, gamma_snow_param_min, gamma_snow_param_max) print_param("Half way result", kirchner_p_opt) print_param("Result", gamma_snow_p_opt) simulator.region_model.set_catchment_parameter(cid, gamma_snow_p_opt) simulator.run(time_axis, state_repos.get_state(0)) found_discharge = simulator.region_model.statistics.discharge([cid]) t_vs = np.array(target_discharge.v) t_ts = np.array( [target_discharge.time(i) for i in range(target_discharge.size())]) f_vs = np.array(found_discharge.v) f_ts = np.array( [found_discharge.time(i) for i in range(found_discharge.size())])
def setUp(self): self.c = api.Calendar() self.d = api.deltahours(1) self.n = 24 self.t = self.c.trim(api.utctime_now(), self.d) self.ta = api.Timeaxis(self.t, self.d, self.n)
def test_can_run_bayesian_kriging_from_observation_sites_to_1km_grid(self): """ Somewhat more complex test, first do kriging of 1 timeseries out to grid (expect same values flat) then do kriging of 3 time-series out to the grid (expect different values, no real verification here since this is done elsewhere """ # arrange the test with a btk_parameter, a source grid and a destination grid btk_parameter = api.BTKParameter(temperature_gradient=-0.6, temperature_gradient_sd=0.25, sill=25.0, nugget=0.5, range=20000.0, zscale=20.0) fx = lambda z: api.DoubleVector.from_numpy(np.zeros(self.n)) grid_1km_1 = self._create_geo_point_grid(self.mnx, self.mny, self.dx_model) grid_1km_3 = self._create_geo_point_grid(self.mnx, self.mny, self.dx_model) observation_sites = api.TemperatureSourceVector() ta_obs = api.Timeaxis(self.t, self.d * 3, int(self.n / 3)) ta_grid = api.Timeaxis(self.t, self.d, self.n) ts_site_1 = api.Timeseries( ta_obs, values=api.DoubleVector.from_numpy( (20.0 - 0.6 * 5.0 / 100) + 3.0 * np.sin( np.arange(start=0, stop=ta_obs.size(), step=1) * 2 * np.pi / 8.0 - np.pi / 2.0))) ts_site_2 = api.Timeseries( ta_obs, values=api.DoubleVector.from_numpy( (20.0 - 0.6 * 500.0 / 100) + 3.0 * np.sin( np.arange(start=0, stop=ta_obs.size(), step=1) * 2 * np.pi / 8.0 - np.pi / 2.0))) ts_site_3 = api.Timeseries( ta_obs, values=api.DoubleVector.from_numpy( (20.0 - 0.6 * 1050.0 / 100) + 3.0 * np.sin( np.arange(start=0, stop=ta_obs.size(), step=1) * 2 * np.pi / 8.0 - np.pi / 2.0))) observation_sites.append( api.TemperatureSource(api.GeoPoint(50.0, 50.0, 5.0), ts_site_1)) # act 1: just one time-series put into the system, should give same ts (true-averaged) in all the grid-1km_ts (which can be improved using std.gradient..) grid_1km_1ts = api.bayesian_kriging_temperature( observation_sites, grid_1km_1, ta_grid, btk_parameter) # assert 1: self.assertEqual(len(grid_1km_1ts), self.mnx * self.mny) expected_grid_1ts_values = ts_site_1.average( api.Timeaxis2(ta_grid)).values.to_numpy() for gts in grid_1km_1ts: self.assertEqual(gts.ts.size(), ta_grid.size()) self.assertTrue( np.allclose(expected_grid_1ts_values, gts.ts.values.to_numpy())) observation_sites.append( api.TemperatureSource(api.GeoPoint(9000.0, 500.0, 500), ts_site_2)) observation_sites.append( api.TemperatureSource(api.GeoPoint(9000.0, 12000.0, 1050.0), ts_site_3)) grid_1km_3ts = api.bayesian_kriging_temperature( observation_sites, grid_1km_3, ta_grid, btk_parameter) self.assertEqual(len(grid_1km_3ts), self.mnx * self.mny) for gts in grid_1km_3ts: self.assertEqual(gts.ts.size(), ta_grid.size()) self.assertFalse( np.allclose(expected_grid_1ts_values, gts.ts.values.to_numpy()))
def construct_repos(self, overrides): """ Construct repositories """ # Read region, model and datasets config files region_config_file = os.path.join(self.config_dir, self.region_config_file) self.region_config = RegionConfig(region_config_file) self.model_config_file = os.path.join(self.config_dir, self.model_config_file) model_config = ModelConfig(self.model_config_file, overrides=overrides.get("model", {})) datasets_config_file = os.path.join(self.config_dir, self.datasets_config_file) datasets_config = YamlContent(datasets_config_file) interpolation_config_file = os.path.join( self.config_dir, self.interpolation_config_file) interpolation_config = InterpolationConfig(interpolation_config_file) # Construct RegionModelRepository self.region_model = region_model_repo_constructor( self.region_config.repository()['class'], self.region_config, model_config, self.region_model_id) # Construct InterpolationParameterRepository self.interp_repos = InterpolationParameterRepository( interpolation_config) # Construct GeoTsRepository self.geo_ts = geots_repo_from_config( datasets_config_file, self.region_config.domain()["EPSG"]) # Construct destination repository self.dst_repo = [] if hasattr(datasets_config, 'destinations'): for repo in datasets_config.destinations: repo['repository'] = target_repo_constructor( repo['repository'], repo['params']) #[dst['time_axis'].update({'start_datetime': utctime_from_datetime(dst['time_axis']['start_datetime'])}) # for dst in repo['1D_timeseries'] if dst['time_axis'] is not None] [ dst.update({'time_axis': self.time_axis}) if dst['time_axis'] is None else dst.update({ 'time_axis': api.Timeaxis( utctime_from_datetime( dst['time_axis']['start_datetime']), dst['time_axis']['time_step_length'], dst['time_axis']['number_of_steps']) }) for dst in repo['1D_timeseries'] ] self.dst_repo.append(repo) # Construct reference data repository self.ref_repo = [] if hasattr(self, 'references'): for repo in self.references: repo_ = target_repo_constructor(repo['repository'], repo['params']) [dst.update({'repo': repo_}) for dst in repo['1D_timeseries']] self.ref_repo.extend(repo['1D_timeseries']) # Construct StateRepository if hasattr(self, 'initial_state'): self.initial_state_repo = self.initial_state['repository'][ 'class'](**self.initial_state['repository']['params']) if hasattr(self, 'end_state'): self.end_state_repo = self.end_state['repository']['class']( **self.end_state['repository']['params'])
def dacc_time(t): t0 = int(t[0]) t1 = int(t[1]) return noop_time(t) if issubset else api.Timeaxis( t0, t1 - t0, len(t) - 1)
def noop_time(t): t0 = int(t[0]) t1 = int(t[1]) return api.Timeaxis(t0, t1 - t0, len(t))