def test_generic_timeaxis(self):
c = api.Calendar()
dt = api.deltahours(1)
n = 240
t0 = c.time(2016, 4, 10)
tag1 = api.Timeaxis2(t0, dt, n)
self.assertEqual(len(tag1), n)
self.assertEqual(tag1.time(0), t0)
tag2 = api.Timeaxis2(c, t0, dt, n)
self.assertEqual(len(tag2), n)
self.assertEqual(tag2.time(0), t0)
def _create_forecast_set(self, n_fc, t0, dt, n_steps, dt_fc, fx):
"""
Parameters
----------
n_fc : int number of forecasts, e.g. 8
t0 : utctime start of first forecast
dt : utctimespan delta t for forecast-ts
n_steps : number of steps in one forecast-ts
dt_fc : utctimespan delta t between each forecast, like deltahours(6)
fx : lambda time_axis: a function returning a DoubleVector with values for the supplied time-axis
Returns
-------
api.TsVector()
"""
fc_set = api.TsVector()
for i in range(n_fc):
ta = api.Timeaxis2(t0 + i * dt_fc, dt, n_steps)
ts = api.Timeseries(
ta=ta,
values=fx(ta),
point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE)
fc_set.append(ts)
return fc_set
def test_kling_gupta_and_nash_sutcliffe(self):
"""
Test/verify exposure of the kling_gupta and nash_sutcliffe correlation functions
"""
def np_nash_sutcliffe(o, p):
return 1 - (np.sum((o - p) ** 2)) / (np.sum((o - np.mean(o)) ** 2))
c = api.Calendar()
t0 = c.time(2016, 1, 1)
dt = api.deltahours(1)
n = 240
ta = api.Timeaxis2(t0, dt, n)
from math import sin, pi
rad_max = 10 * 2 * pi
obs_values = api.DoubleVector.from_numpy(np.array([sin(i * rad_max / n) for i in range(n)]))
mod_values = api.DoubleVector.from_numpy(np.array([0.1 + sin(pi / 10.0 + i * rad_max / n) for i in range(n)]))
obs_ts = api.Timeseries(ta=ta, values=obs_values, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE)
mod_ts = api.Timeseries(ta=ta, values=mod_values, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE)
self.assertAlmostEqual(api.kling_gupta(obs_ts, obs_ts, ta, 1.0, 1.0, 1.0), 1.0, None, "1.0 for perfect match")
self.assertAlmostEqual(api.nash_sutcliffe(obs_ts, obs_ts, ta), 1.0, None, "1.0 for perfect match")
# verify some non trivial cases, and compare to numpy version of ns
mod_inv = obs_ts * -1.0
kge_inv = obs_ts.kling_gupta(mod_inv) # also show how to use time-series.method itself to ease use
ns_inv = obs_ts.nash_sutcliffe(mod_inv) # similar for nash_sutcliffe, you can reach it directly from a ts
ns_inv2 = np_nash_sutcliffe(obs_ts.values.to_numpy(), mod_inv.values.to_numpy())
self.assertLessEqual(kge_inv, 1.0, "should be less than 1")
self.assertLessEqual(ns_inv, 1.0, "should be less than 1")
self.assertAlmostEqual(ns_inv, ns_inv2, 4, "should equal numpy calculated value")
kge_obs_mod = api.kling_gupta(obs_ts, mod_ts, ta, 1.0, 1.0, 1.0)
self.assertLessEqual(kge_obs_mod, 1.0)
self.assertAlmostEqual(obs_ts.nash_sutcliffe( mod_ts), np_nash_sutcliffe(obs_ts.values.to_numpy(), mod_ts.values.to_numpy()))
def test_percentiles(self):
c = api.Calendar()
t0 = c.time(2016, 1, 1)
dt = api.deltahours(1)
n = 240
ta = api.Timeaxis(t0, dt, n)
timeseries = api.TsVector()
for i in range(10):
timeseries.append(
api.Timeseries(ta=ta, fill_value=i, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE))
wanted_percentiles = api.IntVector([0, 10, 50, -1, 70, 100])
ta_day = api.Timeaxis(t0, dt * 24, n // 24)
ta_day2 = api.Timeaxis2(t0, dt * 24, n // 24)
percentiles = api.percentiles(timeseries, ta_day, wanted_percentiles)
percentiles2 = timeseries.percentiles(ta_day2, wanted_percentiles) # just to verify it works with alt. syntax
self.assertEqual(len(percentiles2), len(percentiles))
for i in range(len(ta_day)):
self.assertAlmostEqual(0.0, percentiles[0].value(i), 3, " 0-percentile")
self.assertAlmostEqual(0.9, percentiles[1].value(i), 3, " 10-percentile")
self.assertAlmostEqual(4.5, percentiles[2].value(i), 3, " 50-percentile")
self.assertAlmostEqual(4.5, percentiles[3].value(i), 3, " -average")
self.assertAlmostEqual(6.3, percentiles[4].value(i), 3, " 70-percentile")
self.assertAlmostEqual(9.0, percentiles[5].value(i), 3, "100-percentile")
def setUp(self):
self.c = api.Calendar()
self.d = api.deltahours(1)
self.n = 24
#self.t= self.c.trim(api.utctime_now(),self.d)
self.t = self.c.trim(self.c.time(api.YMDhms(1969, 12, 31, 0, 0, 0)),
self.d)
self.ta = api.Timeaxis2(self.t, self.d, self.n)
def setUp(self):
self.cal = api.Calendar()
self.dt = api.deltahours(1)
self.nt = 24 * 10
self.t0 = self.cal.time(2016, 1, 1)
self.ta = api.Timeaxis2(self.t0, self.dt, self.nt)
self.ta1 = api.Timeaxis(self.t0, self.dt, self.nt)
self.geo_points = api.GeoPointVector()
self.geo_points.append(api.GeoPoint(100, 100, 1000))
self.geo_points.append(api.GeoPoint(5100, 100, 1150))
self.geo_points.append(api.GeoPoint(100, 5100, 850))
def test_bias_predictor(self):
"""
Verify that if we feed forecast[n] and observation into the bias-predictor
it will create the estimated bias offsets
"""
f = api.KalmanFilter()
bp = api.KalmanBiasPredictor(f)
self.assertIsNotNone(bp)
self.assertEqual(bp.filter.parameter.n_daily_observations, 8)
n_fc = 8
utc = api.Calendar()
t0 = utc.time(2016, 1, 1)
dt = api.deltahours(1)
n_fc_steps = 36 # e.g. like arome 36 hours
fc_dt = api.deltahours(6)
fc_fx = lambda time_axis: self._create_fc_values(
time_axis, 2.0) # just return a constant 2.0 deg C for now
fc_set = self._create_geo_forecast_set(n_fc, t0, dt, n_fc_steps, fc_dt,
fc_fx)
n_obs = 24
obs_ta = api.Timeaxis2(t0, dt, n_obs)
obs_ts = api.Timeseries(obs_ta, fill_value=0.0)
kalman_dt = api.deltahours(
3) # suitable average for prediction temperature
kalman_ta = api.Timeaxis2(t0, kalman_dt, 8)
bp.update_with_forecast(
fc_set, obs_ts, kalman_ta
) # here we feed in forecast-set and observation into kalman
fc_setv = self._create_forecast_set(n_fc, t0, dt, n_fc_steps, fc_dt,
fc_fx)
bp.update_with_forecast(
fc_setv, obs_ts,
kalman_ta) # also verify we can feed in a pure TsVector
bias_pattern = bp.state.x # the bp.state.x is now the best estimates fo the bias between fc and observation
self.assertEqual(len(bias_pattern), 8)
for i in range(len(bias_pattern)):
self.assertLess(abs(bias_pattern[i] - 2.0),
0.2) # bias should iterate to approx 2.0 degC now.
def test_accumulate(self):
c = api.Calendar()
t0 = c.time(2016, 1, 1)
dt = api.deltahours(1)
n = 240
ta = api.Timeaxis2(t0, dt, n)
ts0 = api.Timeseries(ta=ta, fill_value=1.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE)
ts1 = ts0.accumulate(ts0.get_time_axis()) # ok, maybe we should make method that does time-axis implicit ?
ts1_values = ts1.values
for i in range(n):
expected_value = i * dt * 1.0
self.assertAlmostEqual(expected_value, ts1.value(i), 3, "expect integral f(t)*dt")
self.assertAlmostEqual(expected_value, ts1_values[i], 3, "expect value vector equal as well")
def setUp(self):
self.c = api.Calendar()
self.d = api.deltahours(1)
self.n = 24
self.t = self.c.trim(self.c.time(2016, 9, 1), self.d)
self.ta = api.Timeaxis2(self.t, self.d, self.n)
self.dx_arome = 2500
self.dx_model = 1000
self.nx = 2
self.ny = 2
self.mnx = 5
self.mny = 5
self.max_elevation = 1000
def test_partition_by(self):
"""
verify/demo exposure of the .partition_by function that can
be used to produce yearly percentiles statistics for long historical
time-series
"""
c = api.Calendar()
t0 = c.time(1930, 9, 1)
dt = api.deltahours(1)
n = c.diff_units(t0, c.time(2016, 9, 1), dt)
ta = api.Timeaxis2(t0, dt, n)
pattern_values = api.DoubleVector.from_numpy(np.arange(len(ta))) # increasing values
src_ts = api.Timeseries(ta=ta, values=pattern_values, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE)
partition_t0 = c.time(2016, 9, 1)
n_partitions = 80
partition_interval = api.Calendar.YEAR
# get back TsVector,
# where all TsVector[i].index_of(partition_t0)
# is equal to the index ix for which the TsVector[i].value(ix) correspond to start value of that particular partition.
ts_partitions = src_ts.partition_by(c, t0, partition_interval, n_partitions, partition_t0)
self.assertEqual(len(ts_partitions),n_partitions)
ty = t0
for ts in ts_partitions:
ix = ts.index_of(partition_t0)
vix = ts.value(ix)
expected_value = c.diff_units(t0, ty, dt)
self.assertEqual(vix, expected_value)
ty = c.add(ty, partition_interval, 1)
# Now finally, try percentiles on the partitions
wanted_percentiles = [0, 10, 25, -1, 50, 75, 90, 100]
ta_percentiles = api.Timeaxis2(partition_t0, api.deltahours(24), 365)
percentiles = api.percentiles(ts_partitions,ta_percentiles,wanted_percentiles)
self.assertEqual(len(percentiles), len(wanted_percentiles))
def test_periodic_pattern_ts(self):
c = api.Calendar()
t0 = c.time(2016, 1, 1)
dt = api.deltahours(1)
n = 240
ta = api.Timeaxis2(t0, dt, n)
pattern_values = api.DoubleVector.from_numpy(np.arange(8))
pattern_dt = api.deltahours(3)
pattern_t0 = c.time(2015,6,1)
pattern_ts = api.create_periodic_pattern_ts(pattern_values, pattern_dt, pattern_t0, ta) # this is how to create a periodic pattern ts (used in gridpp/kalman bias handling)
self.assertAlmostEqual(pattern_ts.value(0), 0.0)
self.assertAlmostEqual(pattern_ts.value(1), 0.0)
self.assertAlmostEqual(pattern_ts.value(2), 0.0)
self.assertAlmostEqual(pattern_ts.value(3), 1.0) # next step in pattern starts here
self.assertAlmostEqual(pattern_ts.value(24), 0.0) # next day repeats the pattern
def _predict_bias(self, obs_set, fc_set):
# Return a set of bias_ts per observation geo_point
bias_set = api.TemperatureSourceVector()
kf = api.KalmanFilter()
kbp = api.KalmanBiasPredictor(kf)
kta = api.Timeaxis2(self.t0, api.deltahours(3), 8)
for obs in obs_set:
kbp.update_with_forecast(fc_set, obs.ts, kta)
pattern = api.KalmanState.get_x(kbp.state)
# a_ts = api.Timeseries(pattern, api.deltahours(3), self.ta) # can do using ct of Timeseries, or:
b_ts = api.create_periodic_pattern_ts(pattern, api.deltahours(3),
self.ta.time(0),
self.ta) # function
bias_set.append(api.TemperatureSource(obs.mid_point(), b_ts))
return bias_set
def test_filter(self):
f = api.KalmanFilter()
self.assertEqual(f.parameter.n_daily_observations, 8)
s = f.create_initial_state()
self.assertEqual(s.size(), 8)
utc = api.Calendar()
t0 = utc.time(2015, 1, 1)
dt = api.deltahours(3)
n = 8
ta = api.Timeaxis2(t0, dt, n)
for i in range(ta.size()):
f.update(2.0, ta.time(i), s)
x = s.x
self.assertEqual(len(x), 8)
self.assertEqual(len(s.k), 8)
self.assertEqual(s.P.shape[0], 8)
self.assertEqual(s.P.shape[1], 8)
self.assertEqual(s.W.shape[0], 8)
self.assertEqual(s.W.shape[1], 8)
def test_basic_timeseries_math_operations(self):
"""
Test that timeseries functionality is exposed, and briefly verify correctness
of operators (the shyft core do the rest of the test job, not repeated here).
"""
c = api.Calendar()
t0 = api.utctime_now()
dt = api.deltahours(1)
n = 240
ta = api.Timeaxis2(t0, dt, n)
a = api.Timeseries(ta=ta, fill_value=3.0, point_fx=api.point_interpretation_policy.POINT_AVERAGE_VALUE)
b = api.Timeseries(ta=ta, fill_value=1.0)
b.fill(2.0) # demo how to fill a point ts
c = a + b * 3.0 - a / 2.0 # operator + * - /
d = -a # unary minus
e = a.average(ta) # average
f = api.max(c, 300.0)
g = api.min(c, -300.0)
h = a.max(c, 300)
k = a.min(c, -300)
self.assertEqual(a.size(), n)
self.assertEqual(b.size(), n)
self.assertEqual(c.size(), n)
self.assertAlmostEqual(c.value(0), 3.0 + 2.0 * 3.0 - 3.0 / 2.0) # 7.5
for i in range(n):
self.assertAlmostEqual(c.value(i), a.value(i) + b.value(i) * 3.0 - a.value(i) / 2.0, delta=0.0001)
self.assertAlmostEqual(d.value(i), - a.value(i), delta=0.0001)
self.assertAlmostEqual(e.value(i), a.value(i), delta=0.00001)
self.assertAlmostEqual(f.value(i), +300.0, delta=0.00001)
self.assertAlmostEqual(h.value(i), +300.0, delta=0.00001)
self.assertAlmostEqual(g.value(i), -300.0, delta=0.00001)
self.assertAlmostEqual(k.value(i), -300.0, delta=0.00001)
# now some more detailed tests for setting values
b.set(0, 3.0)
self.assertAlmostEqual(b.value(0), 3.0)
# 3.0 + 3 * 3 - 3.0/2.0
self.assertAlmostEqual(c.value(1), 7.5, delta=0.0001) # 3 + 3*3 - 1.5 = 10.5
self.assertAlmostEqual(c.value(0), 10.5, delta=0.0001) # 3 + 3*3 - 1.5 = 10.5
def test_can_run_bayesian_kriging_from_observation_sites_to_1km_grid(self):
"""
Somewhat more complex test, first do kriging of 1 timeseries out to grid (expect same values flat)
then do kriging of 3 time-series out to the grid (expect different values, no real verification here since this is done elsewhere
"""
# arrange the test with a btk_parameter, a source grid and a destination grid
btk_parameter = api.BTKParameter(temperature_gradient=-0.6,
temperature_gradient_sd=0.25,
sill=25.0,
nugget=0.5,
range=20000.0,
zscale=20.0)
fx = lambda z: api.DoubleVector.from_numpy(np.zeros(self.n))
grid_1km_1 = self._create_geo_point_grid(self.mnx, self.mny,
self.dx_model)
grid_1km_3 = self._create_geo_point_grid(self.mnx, self.mny,
self.dx_model)
observation_sites = api.TemperatureSourceVector()
ta_obs = api.Timeaxis(self.t, self.d * 3, int(self.n / 3))
ta_grid = api.Timeaxis(self.t, self.d, self.n)
ts_site_1 = api.Timeseries(
ta_obs,
values=api.DoubleVector.from_numpy(
(20.0 - 0.6 * 5.0 / 100) + 3.0 * np.sin(
np.arange(start=0, stop=ta_obs.size(), step=1) * 2 *
np.pi / 8.0 - np.pi / 2.0)))
ts_site_2 = api.Timeseries(
ta_obs,
values=api.DoubleVector.from_numpy(
(20.0 - 0.6 * 500.0 / 100) + 3.0 * np.sin(
np.arange(start=0, stop=ta_obs.size(), step=1) * 2 *
np.pi / 8.0 - np.pi / 2.0)))
ts_site_3 = api.Timeseries(
ta_obs,
values=api.DoubleVector.from_numpy(
(20.0 - 0.6 * 1050.0 / 100) + 3.0 * np.sin(
np.arange(start=0, stop=ta_obs.size(), step=1) * 2 *
np.pi / 8.0 - np.pi / 2.0)))
observation_sites.append(
api.TemperatureSource(api.GeoPoint(50.0, 50.0, 5.0), ts_site_1))
# act 1: just one time-series put into the system, should give same ts (true-averaged) in all the grid-1km_ts (which can be improved using std.gradient..)
grid_1km_1ts = api.bayesian_kriging_temperature(
observation_sites, grid_1km_1, ta_grid, btk_parameter)
# assert 1:
self.assertEqual(len(grid_1km_1ts), self.mnx * self.mny)
expected_grid_1ts_values = ts_site_1.average(
api.Timeaxis2(ta_grid)).values.to_numpy()
for gts in grid_1km_1ts:
self.assertEqual(gts.ts.size(), ta_grid.size())
self.assertTrue(
np.allclose(expected_grid_1ts_values,
gts.ts.values.to_numpy()))
observation_sites.append(
api.TemperatureSource(api.GeoPoint(9000.0, 500.0, 500), ts_site_2))
observation_sites.append(
api.TemperatureSource(api.GeoPoint(9000.0, 12000.0, 1050.0),
ts_site_3))
grid_1km_3ts = api.bayesian_kriging_temperature(
observation_sites, grid_1km_3, ta_grid, btk_parameter)
self.assertEqual(len(grid_1km_3ts), self.mnx * self.mny)
for gts in grid_1km_3ts:
self.assertEqual(gts.ts.size(), ta_grid.size())
self.assertFalse(
np.allclose(expected_grid_1ts_values,
gts.ts.values.to_numpy()))
