def test_create_basic(self):
a = np.array([[1.1, 1.2, 1.3], [2.1, 2.2, 2.3]], dtype=np.float64)
ta = TimeAxis(0, deltahours(1), 3)
tsv = create_ts_vector_from_np_array(ta, a, ts_point_fx.POINT_AVERAGE_VALUE)
self.assertIsNotNone(tsv)
for i in range(2):
self.assertTrue(np.allclose(tsv[i].values.to_numpy(), a[i]))
self.assertTrue(ta == tsv[i].time_axis)
self.assertEqual(tsv[i].point_interpretation(), ts_point_fx.POINT_AVERAGE_VALUE)
# create missmatch throws
b = np.array([[], []], dtype=np.float64)
try:
create_ts_vector_from_np_array(ta, b, ts_point_fx.POINT_AVERAGE_VALUE)
self.assertTrue(False, "Should throw for missmatch time-axis")
except RuntimeError as e:
pass
# create empty ts works
tb = TimeAxis(0, 0, 0)
r = create_ts_vector_from_np_array(tb, b, ts_point_fx.POINT_AVERAGE_VALUE)
self.assertEqual(len(r), 2)
for ts in r:
self.assertFalse(ts)
# create empty returns empty
c = np.empty(shape=(0, 0), dtype=np.float64)
z = create_ts_vector_from_np_array(tb, c, ts_point_fx.POINT_AVERAGE_VALUE)
self.assertEqual(len(z), 0)
def test_dtss_partition_by_average(self):
"""
This test illustrates use of partition_by client and server-side.
The main point here is to ensure that the evaluate period covers
both the historical and evaluation peri
"""
with tempfile.TemporaryDirectory() as c_dir:
# setup data to be calculated
utc = Calendar()
d = deltahours(1)
t = utc.time(2000, 1, 1)
n = utc.diff_units(t, utc.add(t, Calendar.YEAR, 10), d)
ta = TimeAxis(t, d, n)
td = TimeAxis(t, d * 24, n // 24)
n_ts = 1
store_tsv = TsVector() # something we store at server side
for i in range(n_ts):
pts = TimeSeries(
ta,
np.sin(
np.linspace(start=0, stop=1.0 * (i + 1),
num=ta.size())),
point_fx.POINT_AVERAGE_VALUE)
ts_id = shyft_store_url(f"{i}")
store_tsv.append(TimeSeries(
ts_id, pts)) # generate a bound pts to store
# start dtss server
dtss = DtsServer()
cache_on_write = True
port_no = find_free_port()
host_port = 'localhost:{0}'.format(port_no)
dtss.set_auto_cache(True)
dtss.set_listening_port(port_no)
dtss.set_container(
"test", c_dir
) # notice we set container 'test' to point to c_dir directory
dtss.start_async(
) # the internal shyft time-series will be stored to that container
# create dts client
c = DtsClient(
host_port,
auto_connect=False) # demonstrate object life-time connection
c.store_ts(store_tsv,
overwrite_on_write=True,
cache_on_write=cache_on_write)
t_0 = utc.time(2018, 1, 1)
tax = TimeAxis(t_0, Calendar.DAY, 365)
ts_h1 = TimeSeries(shyft_store_url(f'{0}'))
ts_h2 = store_tsv[0]
ts_p1 = ts_h1.partition_by(utc, t, Calendar.YEAR, 10,
t_0).average(tax)
ts_p2 = ts_h2.partition_by(utc, t, Calendar.YEAR, 10,
t_0).average(tax)
def test_pow(self):
""" verify pow(ts,num) pow(num,ts) pow(ts,ts) """
a = TimeSeries(TimeAxis(time('2018-01-01T00:00:00Z'), time(3600), 3),
DoubleVector([1.0, 2.0, 3.0]), stair_case)
b = TimeSeries(TimeAxis(time('2018-01-01T00:00:00Z'), time(3600), 3),
DoubleVector([2.0, 2.0, 2.0]), stair_case)
assert_array_almost_equal([1, 4, 9], a.pow(2.0).values.to_numpy())
assert_array_almost_equal([1, 4, 9], pow(a, 2.0).values.to_numpy())
assert_array_almost_equal([2, 4, 8], pow(2.0, a).values.to_numpy())
assert_array_almost_equal([2, 4, 8], pow(b, a).values.to_numpy())
assert_array_almost_equal([2, 4, 8], b.pow(a).values.to_numpy())
def test_dtss_remove_series(self):
with tempfile.TemporaryDirectory() as c_dir:
# start the server
dtss = DtsServer()
port_no = find_free_port()
host_port = 'localhost:{0}'.format(port_no)
dtss.set_listening_port(port_no)
dtss.set_container(
"test", c_dir
) # notice we set container 'test' to point to c_dir directory
dtss.start_async(
) # the internal shyft time-series will be stored to that container
# setup some data
utc = Calendar()
d = deltahours(1)
n = 365 * 24 // 3
t = utc.time(2016, 1, 1)
ta = TimeAxis(t, d, n)
tsv = TsVector()
pts = TimeSeries(ta, np.linspace(start=0, stop=1.0, num=ta.size()),
point_fx.POINT_AVERAGE_VALUE)
tsv.append(TimeSeries("cache://test/foo", pts))
# get a client
client = DtsClient(host_port)
client.store_ts(tsv)
# start with no removing
dtss.set_can_remove(False)
# we should be disallowed to remove now
try:
client.remove("shyft://test/foo")
except Exception as err:
self.assertEqual(
str(err), "dtss::server: server does not support removing")
# then try with allowing remove
dtss.set_can_remove(True)
# we only support removing shyft-url style data
try:
client.remove("protocol://test/foo")
except Exception as err:
self.assertEqual(
str(err),
"dtss::server: server does not allow removing for non shyft-url type data"
)
# now it should work
client.remove("shyft://test/foo")
def test_failures(self):
"""
Verify that dtss client server connections are auto-magically
restored and fixed
"""
with tempfile.TemporaryDirectory() as c_dir:
# start the server
dtss = DtsServer()
port_no = find_free_port()
host_port = 'localhost:{0}'.format(port_no)
dtss.set_listening_port(port_no)
dtss.set_container(
"test", c_dir
) # notice we set container 'test' to point to c_dir directory
dtss.start_async(
) # the internal shyft time-series will be stored to that container
# setup some data
utc = Calendar()
d = deltahours(1)
n = 365 * 24 // 3
t = utc.time(2016, 1, 1)
ta = TimeAxis(t, d, n)
tsv = TsVector()
pts = TimeSeries(ta, np.linspace(start=0, stop=1.0, num=ta.size()),
point_fx.POINT_AVERAGE_VALUE)
tsv.append(TimeSeries("cache://test/foo", pts))
# get a client
client = DtsClient(host_port, auto_connect=False)
client.store_ts(tsv)
client.close()
client.store_ts(tsv) # should just work, it re-open automagically
dtss.clear(
) # the server is out and away, no chance this would work
try:
client.store_ts(tsv)
self.assertTrue(
False,
'This should throw, because there is no dtss server to help you'
)
except Exception as ee:
self.assertFalse(False, f'expected {ee} here')
dtss.set_listening_port(port_no)
dtss.start_async()
client.store_ts(
tsv) # this should just work, automagically reconnect
def test_glacier_melt_ts_m3s(self):
utc = Calendar()
t0 = utc.time(2016, 10, 1)
dt = deltahours(1)
n = 240
ta = TimeAxis(t0, dt, n)
area_m2 = 487 * 1000 * 1000 # Jostedalsbreen, largest i Europe
temperature = TimeSeries(ta=ta,
fill_value=10.0,
point_fx=fx_policy.POINT_AVERAGE_VALUE)
sca_values = dv.from_numpy(np.linspace(area_m2 * 1.0, 0.0, num=n))
sca = TimeSeries(ta=ta,
values=sca_values,
point_fx=fx_policy.POINT_AVERAGE_VALUE)
gf = 1.0 * area_m2
dtf = 6.0
melt_m3s = create_glacier_melt_ts_m3s(
temperature, sca, gf,
dtf) # Here we get back a melt_ts, that we can do ts-stuff with
self.assertIsNotNone(melt_m3s)
full_melt_m3s = glacier_melt_step(dtf, 10.0, 0.0, gf)
expected_melt_m3s = np.linspace(0.0, full_melt_m3s, num=n)
assert_array_almost_equal(expected_melt_m3s,
melt_m3s.values.to_numpy(), 4)
# Just to check we can work with the result as a ts in all aspects
mx2 = melt_m3s * 2.0
emx2 = expected_melt_m3s * 2.0
assert_array_almost_equal(emx2, mx2.values.to_numpy(), 4)
def test_windowed_percentiles_tsv_values(self) -> None:
"""Test that a TsVector is generated by windowed_percentiles_tsv with time-series
fulfilling some properties of being percentiles of the data ts."""
cal = Calendar()
period = UtcPeriod(cal.time(2017, 1, 1), cal.time(2018, 1, 1))
data = np.linspace(-2, 2, 24 * 7)
data_ts = TimeSeries(TimeAxis(0, Calendar.HOUR, len(data)), data,
POINT_INSTANT_VALUE)
# compute
percentiles = [0, 10, 50, 90, 100]
tsv = windowed_percentiles_tsv(data_ts, period, 3 * Calendar.HOUR,
12 * Calendar.HOUR, percentiles,
self.client, cal)
self.assertEqual(len(tsv), 5)
# assert that the time-series have the correct properties for being percentile series
for i in range(len(tsv[0])):
prev_v = tsv[0].values[i]
for j in range(len(percentiles) - 1):
v = tsv[j + 1].values[i]
# both values will be NaN at the end - that is ok
if math.isnan(prev_v) and math.isnan(v):
continue
# check that no larger percentile have values greater than lower percentiles
self.assertLessEqual(prev_v, v)
prev_v = v
def test_linear_vector(self):
ta = TimeAxis(0, 10, 6)
tsv = DoubleVector([1, 1, 2, 3, -1.0, 5.0])
fv = TsVector([TimeSeries(ta, tsv, POINT_INSTANT_VALUE)])
d_fv = fv.derivative()
f = fv[0]
d_f = d_fv[0]
self.assertEqual(len(f), len(d_f))
self.assertAlmostEqual(d_f.value(0), 0.0)
self.assertAlmostEqual(d_f.value(1), 0.1)
self.assertAlmostEqual(d_f.value(2), 0.1)
self.assertAlmostEqual(d_f.value(3), -0.4)
self.assertAlmostEqual(d_f(f.time(3) + 5), -0.4)
self.assertAlmostEqual(d_f.value(4), 0.6)
self.assertFalse(math.isfinite(d_f.value(5)))
self.assertFalse(math.isfinite(d_f(f.time(5))))
self.assertTrue(math.isfinite(d_f(f.time(5) - 1)))
v = d_f.values
self.assertAlmostEqual(len(v), len(f))
self.assertAlmostEqual(v[0], 0.0)
self.assertAlmostEqual(v[1], 0.1)
self.assertAlmostEqual(v[2], 0.1)
self.assertAlmostEqual(v[3], -0.4)
self.assertAlmostEqual(v[4], 0.6)
self.assertFalse(math.isfinite(v[5]))
def test_create_xx_source_vector(self):
# arrange the setup
a = np.array([[1.1, 1.2, 1.3], [2.1, 2.2, 2.3]], dtype=np.float64)
ta = TimeAxis(deltahours(0), deltahours(1), 3)
gpv = GeoPointVector()
gpv[:] = [GeoPoint(1, 2, 3), GeoPoint(4, 5, 6)]
cfs = [(create_precipitation_source_vector_from_np_array,
PrecipitationSourceVector),
(create_temperature_source_vector_from_np_array,
TemperatureSourceVector),
(create_radiation_source_vector_from_np_array,
RadiationSourceVector),
(create_rel_hum_source_vector_from_np_array,
RelHumSourceVector),
(create_radiation_source_vector_from_np_array,
RadiationSourceVector)]
# test all creation types:
for cf in cfs:
r = cf[0](ta, gpv, a, ts_point_fx.POINT_AVERAGE_VALUE) # act here
self.assertTrue(isinstance(r, cf[1])) # then the asserts
self.assertEqual(len(r), len(gpv))
for i in range(len(gpv)):
self.assertEqual(r[i].mid_point(), gpv[i])
self.assertTrue(np.allclose(r[i].ts.values.to_numpy(), a[i]))
self.assertEqual(r[i].ts.point_interpretation(),
ts_point_fx.POINT_AVERAGE_VALUE)
def test_error_handling(self):
utc = Calendar()
t0 = utc.time(2018, 1, 1)
dt = deltahours(1)
dv = DoubleVector()
dv[:] = [1.0, 2.0, 2.5, 1.9, 3.0, 3.1,
float('nan')] # also verify nan-handling
ts = TimeSeries(TimeAxis(t0, dt, len(dv)), dv, POINT_AVERAGE_VALUE)
try:
ts.decode(start_bit=0, n_bits=0)
self.assertTrue(False, 'This should throw, n_bits >0')
except RuntimeError as re:
pass
try:
ts.decode(start_bit=41, n_bits=12)
self.assertTrue(False, 'This should throw, start_bit + n_bits >52')
except RuntimeError as re:
pass
try:
ts.decode(start_bit=-1, n_bits=12)
self.assertTrue(False, 'This should throw, start_bit >=0')
except RuntimeError as re:
pass
def test_forecast_average_slice(self):
"""
Demo and test TsVector.average_slice(lead_time,dt,n)
"""
utc = Calendar()
t0 = utc.time(2017, 1, 1)
dt = deltahours(1)
n = 66 # typical arome
fc_dt_n_hours = 6
fc_dt = deltahours(fc_dt_n_hours)
fc_n = 4 * 10 # 4 each day 10 days
fc_v = self._create_forecasts(t0, dt, n, fc_dt, fc_n)
for lead_time_hours in range(12):
for slice_length_units in [1, 2, 3, 4, 6, 12]:
for dt_hours in [1, 2, 3]:
slice_v = fc_v.average_slice(deltahours(lead_time_hours),
deltahours(dt_hours),
slice_length_units)
self.assertEqual(len(slice_v), len(fc_v))
# then loop over the slice_v and prove it's equal
# to the average of the same portion on the originalj
for s, f in zip(slice_v, fc_v):
ta = TimeAxis(
f.time_axis.time(0) + deltahours(lead_time_hours),
deltahours(dt_hours), slice_length_units)
ts_expected = f.average(ta)
self.assertTrue(s.time_axis == ts_expected.time_axis)
self.assertTrue(
np.allclose(s.values.to_numpy(),
ts_expected.values.to_numpy()))
pass
def remove_tp_data(self, period: UtcPeriod):
"""
delete data given within the time period
:param period:
:return:
"""
time_series_cropped = None
with Dataset(self.file_path, 'a') as ds:
# 1. load the data
time_variable = 'time'
time = ds.variables.get(time_variable, None)
if time is None:
raise TimeSeriesStoreError(
'Something is wrong with the dataset. time not found.')
var = ds.variables.get(self.ts_meta_info.variable_name, None)
if var is None:
raise TimeSeriesStoreError(
'Something is wrong with the dataset. variable {0} not found.'
.format(self.ts_meta_info.variable_name))
if len(time):
# 2. get indices of the data to delete
time_utc = convert_netcdf_time(time.units, time)
idx_min = np.searchsorted(time_utc, period.start, side='left')
idx_max = np.searchsorted(time_utc, period.end, side='right')
# check if there is data outside the range
if idx_max - idx_min != len(time):
# print('indices ', idx_min, idx_max, len(time))
# 3. crop the data array
if idx_max < len(time):
time_cropped = np.append(time[0:idx_min],
time[idx_max:])
var_cropped = np.append(var[0:idx_min], var[idx_max:])
else:
time_cropped = np.append(time[0:idx_min], [])
var_cropped = np.append(var[0:idx_min], [])
last_time_point = 2 * time_cropped[-1] - time_cropped[-2]
# print(type(time_cropped[0]))
# print(UtcTimeVector.from_numpy(time_cropped.astype(np.int64)).to_numpy())
ta = TimeAxis(
UtcTimeVector.from_numpy(time_cropped.astype(
np.int64)), int(last_time_point))
# print(var_cropped)
# print(type(var_cropped))
time_series_cropped = TimeSeries(
ta, dv.from_numpy(var_cropped),
point_fx.POINT_INSTANT_VALUE
) # TODO: is this correct point policy?
# 4. save the cropped data
self.create_new_file()
if time_series_cropped:
self.append_ts_data(time_series_cropped)
def test_get_ts_info(self):
"""
Verify we can get specific TsInfo objects for time-series from the server backend.
"""
with tempfile.TemporaryDirectory() as c_dir:
# start the server
dtss = DtsServer()
port_no = find_free_port()
host_adr = 'localhost:{0}'.format(port_no)
dtss.set_listening_port(port_no)
dtss.set_container(
"testing", c_dir
) # notice we set container 'test' to point to c_dir directory
dtss.start_async(
) # the internal shyft time-series will be stored to that container
# get a client
client = DtsClient(host_adr)
try:
client.get_ts_info(r'shyft://testing/data')
except Exception as e:
pass
else:
# only end up here if no exceptions
self.fail('Could fetch info for non-existing ts info')
# setup some data
utc = Calendar()
d = deltahours(1)
n = 365 * 24 // 3
t = utc.time(2016, 1, 1)
ta = TimeAxis(t, d, n)
tsv = TsVector()
pts = TimeSeries(ta, np.linspace(start=0, stop=1.0, num=ta.size()),
point_fx.POINT_AVERAGE_VALUE)
tsv.append(TimeSeries(r'shyft://testing/data', pts))
client.store_ts(tsv)
info: TsInfo = client.get_ts_info(r'shyft://testing/data')
self.assertEqual(info.name, r'data')
self.assertEqual(info.point_fx, point_fx.POINT_AVERAGE_VALUE)
self.assertEqual(info.data_period, ta.total_period())
def test_simple_case(self):
utc = Calendar()
t0 = utc.time(2018, 1, 1)
dt = deltahours(1)
dv = DoubleVector()
dv[:] = [1.0, 2.0, 2.5, 1.9, 3.0, 3.1, -1.0]
i1_ex = [0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0]
ts = TimeSeries(TimeAxis(t0, dt, len(dv)), dv, POINT_AVERAGE_VALUE)
i1 = ts.inside(2.0, 3.0)
assert_array_almost_equal(i1.values.to_numpy(), np.array(i1_ex))
def period_percentiles_tsv(
ts: TimeSeries,
period: UtcPeriod,
average_dt: int,
percentile_period: UtcPeriod,
percentiles: Sequence[int],
client: DtsClient, calendar: Calendar
) -> TsVector:
"""Compute percentiles from a part of a time-series and generate a TsVector of
percentile time-series spanning a possibly different time period.
Args:
ts: TimeSeries to compute percentile time-series for.
period: Time period the output time-series should span.
average_dt: Period to average values by when partitioning the input time-series.
percentile_period: Period from ts to compute percentiles for.
percentiles: Percentiles to compute. Values should be in the range ``0..100``.
client: DtsClient to use for performing the partitioning.
calendar: Calendar to to for interpreting time and partitioning the input
time-series.
Returns:
A TsVector with one TimeSeries for each value in percentiles, all spanning period.
"""
periods = int((percentile_period.end - percentile_period.start)//average_dt)
n_avg = calendar.diff_units(period.start, period.end, average_dt)
if n_avg*average_dt < period.end - period.start:
n_avg += 1
tsv = client.evaluate(
ts.average(TimeAxis(period.start, average_dt, n_avg))
.partition_by(calendar, period.start, average_dt, periods, period.start),
period
)
tsv_p = tsv.percentiles(TimeAxis(period.start, average_dt, 1), percentiles)
tsv = TsVector()
ta = TimeAxis(period.start, period.end - period.start, 1)
for ts_p in tsv_p:
tsv.append(TimeSeries(ta, [ts_p.values[0]], POINT_AVERAGE_VALUE))
return tsv
def test_inverted_values(self):
utc = Calendar()
t0 = utc.time(2018, 1, 1)
dt = deltahours(1)
dv = DoubleVector()
dv[:] = [1.0, 2.0, 2.5, 1.9, 3.0, 3.1, float('nan')] # also verify nan-handling
i1_ex = [1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0]
ts = TimeSeries(TimeAxis(t0, dt, len(dv)), dv, POINT_AVERAGE_VALUE)
i2 = ts.inside(min_v=2.0, max_v=3.0, nan_v=1.0, inside_v=0.0, outside_v=1.0)
assert_array_almost_equal(i2.values.to_numpy(), np.array(i1_ex))
def windowed_percentiles_tsv(
ts: TimeSeries,
period: UtcPeriod,
average_dt: int,
percentile_dt: int,
percentiles: Sequence[int],
client: DtsClient, calendar: Calendar
) -> TsVector:
"""Compute percentiles for a time-series in a gliding window fashion.
The input time-series is partitioned and averaged using ``ts.partition_by`` such
that each value is included in percentile computations spanning percentile_dt time
from the value occurrence.
Args:
ts: TimeSeries to compute percentile time-series for.
period: Period to generate percentile time-series for.
average_dt: Period to average values by when partitioning the input time-series.
This is the time-step of the time-series in the output TsVector.
percentile_dt: Time span for a value to contribute to percentile computations.
percentiles: Percentiles to compute. Values should be in the range ``0..100``.
client: DtsClient to use for performing the partitioning.
calendar: Calendar to to for interpreting time and partitioning the input
time-series.
Returns:
A TsVector with one TimeSeries for each value in percentiles, all spanning period.
"""
periods = int(percentile_dt//average_dt)
n_avg = calendar.diff_units(period.start, period.end, average_dt)
if n_avg*average_dt < period.end - period.start:
n_avg += 1
tsv = client.evaluate(
ts.average(TimeAxis(period.start, average_dt, n_avg))
.partition_by(calendar, period.start, average_dt, periods, period.start)
.time_shift(periods*average_dt),
period
)
return tsv.percentiles(TimeAxis(period.start, average_dt, n_avg), percentiles)
def test_basic_case(self):
ta = TimeAxis(0, 10, 6)
to = TimeAxis(10, 20, 3)
va = DoubleVector([0, 10, 20, 30, 40.0, 50.0])
a = TimeSeries(ta, va, POINT_INSTANT_VALUE)
o = TimeSeries(
to, fill_value=0.0,
point_fx=POINT_INSTANT_VALUE) # point-ip should not matter
r = a.use_time_axis_from(o)
self.assertIsNotNone(r)
self.assertEquals(r.time_axis, o.time_axis)
ev = DoubleVector([10.0, 30.0, 50.0]).to_numpy()
self.assertTrue(np.allclose(r.values.to_numpy(), ev))
self.assertEquals(r.point_interpretation(), a.point_interpretation())
tsv = TsVector([a, 2.0 * a])
rv = tsv.use_time_axis_from(o)
for x in rv:
self.assertEquals(x.time_axis, o.time_axis)
self.assertTrue(np.allclose(rv[0].values.to_numpy(), ev))
self.assertTrue(np.allclose(rv[1].values.to_numpy(), 2.0 * ev))
def test_create_xx_vector_from_list(self):
""" verify we can construct from list of xxSource"""
ts = TemperatureSource(
GeoPoint(1.0, 2.0, 3.0),
TimeSeries(TimeAxis(0, 3600, 10),
fill_value=1.0,
point_fx=ts_point_fx.POINT_AVERAGE_VALUE))
tsv = TemperatureSourceVector([ts])
self.assertEqual(len(tsv), 1)
self.assertEqual(len(TemperatureSourceVector([])), 0)
self.assertEqual(len(TemperatureSourceVector([ts, ts])), 2)
def _create_forecasts(self, t0: int, dt: int, n: int, fc_dt: int,
fc_n: int) -> TsVector:
tsv = TsVector()
stair_case = ts_point_fx.POINT_AVERAGE_VALUE
for i in range(fc_n):
ta = TimeAxis(t0 + i * fc_dt, dt, n)
mrk = (i + 1) / 100.0
v = dv.from_numpy(
np.linspace(1 + mrk, 1 + n + mrk, n, endpoint=False))
tsv.append(TimeSeries(ta, v, stair_case))
return tsv
def _create_observation(self, t0: int, dt: int, fc_lead_steps: int,
fc_n: int) -> TimeSeries:
ta = TimeAxis(t0, dt, fc_n * fc_lead_steps)
ts = TimeSeries(ta,
fill_value=0.0,
point_fx=ts_point_fx.POINT_AVERAGE_VALUE)
for i in range(len(ta)):
ts.set(
i,
math.sin(0.314 +
3.14 * i / 240.0)) # sin-wave at specified 'time' i
return ts
def _create_forecasts(self, t0: int, dt: int, n: int, fc_dt: int,
fc_n: int) -> TsVector:
tsv = TsVector()
stair_case = ts_point_fx.POINT_AVERAGE_VALUE
for i in range(fc_n):
ta = TimeAxis(t0 + i * fc_dt, dt, n)
ts = TimeSeries(ta, fill_value=0.0, point_fx=stair_case)
for t in range(len(ta)):
tt = i * fc_dt / dt + t
ts.set(t, math.sin(0.314 + 3.14 * tt /
240.0)) # make it a sin-wave at time tt
tsv.append(ts)
return tsv
def test_create_tsvector_from_ts_list(self):
ts_list = [
TimeSeries(TimeAxis(0, 3600, 10),
fill_value=float(i),
point_fx=ts_point_fx.POINT_AVERAGE_VALUE)
for i in range(3)
]
tsv = TsVector(ts_list)
assert tsv
assert len(tsv) == 3
assert tsv[0].value(0) == 0.0
assert tsv[1].value(0) == 1.0
assert tsv[2].value(0) == 2.0
def test_windowed_percentiles_tsv_empty(self) -> None:
"""Test that an empty TsVector is generated by windowed_percentiles_tsv
when given an empty sequence of percentiles."""
cal = Calendar()
period = UtcPeriod(cal.time(2017, 1, 1), cal.time(2018, 1, 1))
data = np.linspace(-2, 2, 24 * 7)
data_ts = TimeSeries(TimeAxis(0, Calendar.HOUR, len(data)), data,
POINT_INSTANT_VALUE)
# compute
tsv = windowed_percentiles_tsv(data_ts, period, Calendar.HOUR,
Calendar.HOUR, [], self.client, cal)
self.assertEqual(len(tsv), 0)
def test_inside_of_derivative(self):
"""Created in response to https://github.com/statkraft/shyft/issues/352"""
values = [1, 1, 1, 1]
utc = Calendar()
data = np.array(values, dtype='float64')
data_ta = TimeAxis(utc.time(2015, 1, 1), 3600, len(data))
orig = TimeSeries(data_ta, data, POINT_AVERAGE_VALUE)
orig_derivative_inside_inf = orig.derivative(derivative_method.BACKWARD).inside(-float('inf'), float('inf'), 0,
1, 0)
def check_nan(ts):
"""Method that returns 1 for all timesteps that contain nan."""
return ts.inside(-float('inf'), float('inf'), 1, 0, 0).values.to_numpy()
np.testing.assert_equal(check_nan(orig + orig_derivative_inside_inf), [0., 0., 0., 0.],
'TimeSeries.inside() should not find any NaN-values in this TimeSeries')
def dtss_read_callback(self, ts_ids: StringVector,
read_period: UtcPeriod) -> TsVector:
self.callback_count += 1
r = TsVector()
ta = TimeAxis(read_period.start, deltahours(1),
int(read_period.timespan() // deltahours(1)))
if self.rd_throws:
self.rd_throws = False
raise RuntimeError("read-ts-problem")
for ts_id in ts_ids:
r.append(
TimeSeries(ta,
fill_value=1.0,
point_fx=point_fx.POINT_AVERAGE_VALUE))
if self.cache_reads and self.cache_dtss: # illustrate how the read-callback can ask the dtss to cache it's reads
self.cache_dtss.cache(ts_ids, r)
return r
def test_simple_case(self):
utc = Calendar()
t0 = utc.time(2018, 1, 1)
dt = deltahours(1)
dv = DoubleVector()
dv[:] = [1.2, 0.0, 2.0, 5.0, 15.0,
float('nan'),
-1.0] # these are bit-encoded values, note 1.2 -> 1.0
i0_1_e = [1.0, 0.0, 0.0, 1.0, 1.0,
float('nan'),
float('nan')] # expected values for bit 0 1-bit
i1_3_e = [0.0, 0.0, 1.0, 2.0, 7.0,
float('nan'),
float('nan')] # expected values for bit 3 3-bits
ts = TimeSeries(TimeAxis(t0, dt, len(dv)), dv, POINT_AVERAGE_VALUE)
i0_1 = ts.decode(start_bit=0, n_bits=1)
i1_3 = ts.decode(start_bit=1, n_bits=3)
assert_array_almost_equal(i0_1.values.to_numpy(), np.array(i0_1_e))
assert_array_almost_equal(i1_3.values.to_numpy(), np.array(i1_3_e))
def fixed_tsv(
period: UtcPeriod,
fixed_values: Sequence[float]
) -> TsVector:
"""Create a TsVector with TimeSeries with fixed values spanning the given period.
Args:
period: Time period the generated TimeSeries should span.
fixed_values: A sequence of numbers to generate constant TimeSeries for.
Returns:
A TsVector with one TimeSeries for each value in fixed_values, all spanning period.
"""
tsv = TsVector()
for fv in fixed_values:
tsv.append(TimeSeries(
TimeAxis(UtcTimeVector([period.start, period.end])),
[fv], POINT_AVERAGE_VALUE
))
return tsv
def test_selector_ts(self) -> None:
"""Test that selector_ts constructs a time-series selects data from different time-series correctly."""
n = 24
cal = Calendar()
period = UtcPeriod(0, n * Calendar.HOUR)
data_ts = TimeSeries(TimeAxis(0, Calendar.HOUR, n),
np.linspace(-10, 10, n), POINT_INSTANT_VALUE)
source_tss = [
TimeSeries(TimeAxis(0, Calendar.HOUR, n), 1.00 * np.ones(n),
POINT_INSTANT_VALUE),
TimeSeries(TimeAxis(0, Calendar.HOUR, n), 10.0 * np.ones(n),
POINT_INSTANT_VALUE),
TimeSeries(TimeAxis(0, Calendar.HOUR, n), 100. * np.ones(n),
POINT_INSTANT_VALUE),
]
threshold_1 = -5
threshold_2 = 5
threshold_tss = [
TimeSeries(TimeAxis(0, Calendar.HOUR, n), threshold_1 * np.ones(n),
POINT_INSTANT_VALUE),
TimeSeries(TimeAxis(0, Calendar.HOUR, n), threshold_2 * np.ones(n),
POINT_INSTANT_VALUE),
]
ts = selector_ts(data_ts, period, 2 * Calendar.HOUR, threshold_tss,
source_tss, POINT_AVERAGE_VALUE, self.client, cal)
self.assertEqual(len(data_ts), len(ts))
for dv, rv in zip(data_ts.values, ts.values):
if dv < threshold_1:
self.assertEqual(rv, source_tss[0].values[0])
elif threshold_1 <= dv < threshold_2:
self.assertEqual(rv, source_tss[1].values[0])
else:
self.assertEqual(rv, source_tss[2].values[0])
def test_can_create_cf_compliant_file(self):
# create files
test_file = path.join(path.abspath(os.curdir), 'shyft_test.nc')
if path.exists(test_file):
os.remove(test_file)
# create meta info
epsg_id = 32633
x0 = 100000
x1 = 200000
y0 = 100000
y1 = 200000
x = 101000
y = 101000
z = 1200
temperature = TimeSeriesMetaInfo('temperature',
'/observed/at_stn_abc/temperature',
'observed air temperature', x, y, z,
epsg_id)
# create time axis
utc = Calendar()
ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 24)
data = np.arange(0, ta.size(), dtype=np.float64)
ts = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the first batch
t_ds = TimeSeriesStore(test_file, temperature)
t_ds.create_new_file()
t_ds.append_ts_data(ts)
# expected result
ts_exp = ts
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# Append data
print("\n\n append at the end data")
# create time axis
ta = TimeAxis(utc.time(2016, 1, 2), deltahours(1), 48)
ts = TimeSeries(ta,
dv.from_numpy(np.arange(0, ta.size(),
dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the data
t_ds.append_ts_data(ts)
# expected result
ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 72)
data = np.empty(72)
data[:24] = np.arange(0, 24, dtype=np.float64)
data[24:72] = np.arange(0, 48, dtype=np.float64) # <-- new data
ts_exp = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# Append with overlap
print("\n\n append with overlap")
# create time axis
ta = TimeAxis(utc.time(2016, 1, 3), deltahours(1), 48)
ts = TimeSeries(ta,
dv.from_numpy(np.arange(0, ta.size(),
dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the data
t_ds.append_ts_data(ts)
# expected result
ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 96)
data = np.empty(96)
data[:24] = np.arange(0, 24, dtype=np.float64)
data[24:48] = np.arange(0, 24, dtype=np.float64) # <-- new data
data[48:96] = np.arange(0, 48, dtype=np.float64) # <-- new data
ts_exp = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# Append with gap in time axis
print("\n\n Append with gap in time axis")
# create time axis
ta = TimeAxis(utc.time(2016, 1, 6), deltahours(1), 24)
ts = TimeSeries(ta,
dv.from_numpy(np.arange(0, ta.size(),
dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the data
t_ds.append_ts_data(ts)
# expected result
time_vals = np.append(
TimeAxis(utc.time(2016, 1, 1), deltahours(1), 96).time_points[:-1],
ta.time_points)
# print(time_vals)
ta = TimeAxis(UtcTimeVector.from_numpy(time_vals.astype(np.int64)))
data = np.empty(120)
data[:24] = np.arange(0, 24, dtype=np.float64)
data[24:48] = np.arange(0, 24, dtype=np.float64)
data[48:96] = np.arange(0, 48, dtype=np.float64)
data[96:120] = np.arange(0, 24, dtype=np.float64) # <-- new data
ts_exp = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
# print(ts_exp.total_period())
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
# print(geo_temperature[0].ts.time_axis.time_points - ts_exp.time_axis.time_points)
# print(geo_temperature[0].ts.time_axis.time_points - time_vals)
# print(ts_exp.time_axis.time_points - time_vals)
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# Add new data in the middle where nothing was defined (no moving)
print(
"\n\n Add new data in the middle where nothing was defined (no moving)"
)
# create time axis
ta = TimeAxis(utc.time(2016, 1, 2), deltahours(1), 24)
ts = TimeSeries(ta,
dv.from_numpy(
np.arange(100, 100 + ta.size(), dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the data
t_ds.append_ts_data(ts)
# expected result
time_vals = np.append(
TimeAxis(utc.time(2016, 1, 1), deltahours(1), 96).time_points[:-1],
TimeAxis(utc.time(2016, 1, 6), deltahours(1), 24).time_points)
ta = TimeAxis(UtcTimeVector.from_numpy(time_vals.astype(np.int64)))
data = np.empty(120)
data[:24] = np.arange(0, 24, dtype=np.float64)
data[24:48] = np.arange(100, 124, dtype=np.float64) # <-- new data
data[48:96] = np.arange(0, 48, dtype=np.float64)
data[96:120] = np.arange(0, 24, dtype=np.float64)
ts_exp = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# print(ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# Insert new data in the middle and move rest
print("\n\n insert new data and move rest")
# create time axis
ta = TimeAxis(utc.time(2016, 1, 5), deltahours(1), 36)
ts = TimeSeries(ta,
dv.from_numpy(
np.arange(200, 200 + ta.size(), dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the data
t_ds.append_ts_data(ts)
# expected result
ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 144)
data = np.empty(144)
data[:24] = np.arange(0, 24, dtype=np.float64)
data[24:48] = np.arange(100, 124, dtype=np.float64)
data[48:96] = np.arange(0, 48, dtype=np.float64)
data[96:132] = np.arange(200, 236, dtype=np.float64) # <-- new data
data[132:144] = np.arange(12, 24, dtype=np.float64)
ts_exp = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# Add new data before existing data without overlap
print("\n\n add new data before existing data without overlap")
# create time axis
ta = TimeAxis(utc.time(2015, 12, 31), deltahours(1), 24)
ts = TimeSeries(ta,
dv.from_numpy(
np.arange(300, 300 + ta.size(), dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the first batch
t_ds.append_ts_data(ts)
# expected result
ta = TimeAxis(utc.time(2015, 12, 31), deltahours(1), 168)
data = np.empty(168)
data[:24] = np.arange(300, 324, dtype=np.float64) # <-- new data
data[24:48] = np.arange(0, 24, dtype=np.float64)
data[48:72] = np.arange(100, 124, dtype=np.float64)
data[72:120] = np.arange(0, 48, dtype=np.float64)
data[120:156] = np.arange(200, 236, dtype=np.float64)
data[156:168] = np.arange(12, 24, dtype=np.float64)
ts_exp = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# add new data before existing data with overlap
print("\n\n add new data before existing data with overlap")
# create time axis
ta = TimeAxis(utc.time(2015, 12, 30), deltahours(1), 36)
ts = TimeSeries(ta,
dv.from_numpy(
np.arange(400, 400 + ta.size(), dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the first batch
# t_ds = TimeSeriesStore(test_file, temperature)
t_ds.append_ts_data(ts)
# expected result
ta = TimeAxis(utc.time(2015, 12, 30), deltahours(1), 192)
data = np.empty(192)
data[:36] = np.arange(400, 436, dtype=np.float64) # <-- new data
data[36:48] = np.arange(312, 324, dtype=np.float64)
data[48:72] = np.arange(0, 24, dtype=np.float64)
data[72:96] = np.arange(100, 124, dtype=np.float64)
data[96:144] = np.arange(0, 48, dtype=np.float64)
data[144:180] = np.arange(200, 236, dtype=np.float64)
data[180:192] = np.arange(12, 24, dtype=np.float64)
ts_exp = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# Overwrite everything with less data points
# create time axis
print('\n\n Overwrite everything with less data points')
ta = TimeAxis(utc.time(2015, 12, 30), deltahours(24), 9)
ts = TimeSeries(ta,
dv.from_numpy(
np.arange(1000, 1000 + ta.size(),
dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# write the time series
t_ds.append_ts_data(ts)
# expected result
ts_exp = ts
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# Insert data with different dt
# create time axis
print('\n\n Insert data with different dt')
ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 24)
ts = TimeSeries(ta,
dv.from_numpy(np.arange(0, 24, dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# write the time series
t_ds.append_ts_data(ts)
# expected result
time_points = np.empty(33, dtype=np.int)
time_points[0:2] = TimeAxis(utc.time(2015, 12, 30), deltahours(24),
1).time_points
time_points[2:26] = TimeAxis(utc.time(2016, 1, 1), deltahours(1),
23).time_points
time_points[26:] = TimeAxis(utc.time(2016, 1, 2), deltahours(24),
6).time_points
ta = TimeAxis(UtcTimeVector.from_numpy(time_points))
data = np.empty(32)
data[0:2] = np.array([1000, 1001])
data[2:26] = np.arange(0, 24) # <-- new data
data[26:] = np.arange(1003, 1009)
ts_exp = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# delete data with range UtcPeriod in the middle
print('\n\n delete data with range UtcPeriod')
tp = UtcPeriod(utc.time(2015, 12, 31), utc.time(2016, 1, 1, 12))
# ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 24)
# ts = TimeSeries(ta, dv.from_numpy(np.arange(0, 24, dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE)
# write the time series
t_ds.remove_tp_data(tp)
# expected result
time_points = np.array([
1451433600, 1451653200, 1451656800, 1451660400, 1451664000,
1451667600, 1451671200, 1451674800, 1451678400, 1451682000,
1451685600, 1451689200, 1451692800, 1451779200, 1451865600,
1451952000, 1452038400, 1452124800, 1452211200
])
ta = TimeAxis(UtcTimeVector.from_numpy(time_points))
data = np.array([
1000, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 1003, 1004, 1005,
1006, 1007, 1008
])
ts_exp = TimeSeries(ta, dv.from_numpy(data),
point_fx.POINT_INSTANT_VALUE
) # TODO: is this correct policy to use
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# delete data with range UtcPeriod at the start
print('\n\n delete data with range UtcPeriod at the start')
tp = UtcPeriod(1451433600, 1451667600)
# ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 24)
# ts = TimeSeries(ta, dv.from_numpy(np.arange(0, 24, dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE)
# write the time series
t_ds.remove_tp_data(tp)
# expected result
time_points = np.array([
1451671200, 1451674800, 1451678400, 1451682000, 1451685600,
1451689200, 1451692800, 1451779200, 1451865600, 1451952000,
1452038400, 1452124800, 1452211200
])
ta = TimeAxis(UtcTimeVector.from_numpy(time_points))
data = np.array(
[18, 19, 20, 21, 22, 23, 1003, 1004, 1005, 1006, 1007, 1008])
ts_exp = TimeSeries(
ta, dv.from_numpy(data), point_fx.POINT_INSTANT_VALUE
) # TODO: is this correct policy to use for this test
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# delete data with range UtcPeriod at the end
print('\n\n delete data with range UtcPeriod at the end')
tp = UtcPeriod(1451952000, utc.time(2016, 1, 10))
# ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 24)
# ts = TimeSeries(ta, dv.from_numpy(np.arange(0, 24, dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE)
# write the time series
t_ds.remove_tp_data(tp)
# expected result
time_points = np.array([
1451671200, 1451674800, 1451678400, 1451682000, 1451685600,
1451689200, 1451692800, 1451779200, 1451865600, 1451952000
])
ta = TimeAxis(UtcTimeVector.from_numpy(time_points))
data = np.array([18, 19, 20, 21, 22, 23, 1003, 1004, 1005])
ts_exp = TimeSeries(ta, dv.from_numpy(data),
point_fx.POINT_INSTANT_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
try:
rts_map = ts_dr.get_timeseries(['temperature'],
ts_exp.total_period())
except CFDataRepositoryError:
pass
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# delete data with range UtcPeriod everything
print('\n\n delete data with range UtcPeriod everything')
tp = UtcPeriod(utc.time(2016, 1, 1), utc.time(2016, 1, 10))
# write the time series
t_ds.remove_tp_data(tp)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
self.assertRaises(CFDataRepositoryError, ts_dr.get_timeseries,
['temperature'], tp)
# --------------------------------------
# insert data in between time saved data points
print('\n\n insert data in between time saved data points')
# insert first data in which we want to insert the second batch
utc = Calendar()
ta = TimeAxis(utc.time(2016, 1, 1), deltahours(24), 2)
data = np.arange(0, ta.size(), dtype=np.float64)
ts = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the first batch
t_ds.append_ts_data(ts)
# insert first data for every hour in between
utc = Calendar()
ta = TimeAxis(utc.time(2016, 1, 1) + deltahours(1), deltahours(1), 23)
data = np.arange(10, 10 + ta.size(), dtype=np.float64)
ts = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the first batch
t_ds.append_ts_data(ts)
# expected result
time_points = np.array([
1451606400, 1451610000, 1451613600, 1451617200, 1451620800,
1451624400, 1451628000, 1451631600, 1451635200, 1451638800,
1451642400, 1451646000, 1451649600, 1451653200, 1451656800,
1451660400, 1451664000, 1451667600, 1451671200, 1451674800,
1451678400, 1451682000, 1451685600, 1451689200, 1451692800, 0
])
time_points[-1] = 2 * time_points[-2] - time_points[
-3] # last time point calc
data = np.array([
0, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
26, 27, 28, 29, 30, 31, 32, 1
])
ta = TimeAxis(UtcTimeVector.from_numpy(time_points))
ts_exp = TimeSeries(
ta, dv.from_numpy(data), point_fx.POINT_INSTANT_VALUE
) # TODO: is this correct policy value for this case
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# insert data including nan
print('\n\n insert data including nan')
utc = Calendar()
ta = TimeAxis(utc.time(2016, 1, 1) + deltahours(1), deltahours(1), 23)
data = np.arange(10, 10 + ta.size(), dtype=np.float64)
data[4] = np.nan
data[
6] = np.nan # np.inf, but trouble getting inf trough all version of numpy/netcdf
data[8] = np.nan # -np.inf, --"--
ts = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the first batch
t_ds.append_ts_data(ts)
# expected result
time_points = np.array([
1451606400, 1451610000, 1451613600, 1451617200, 1451620800,
1451624400, 1451628000, 1451631600, 1451635200, 1451638800,
1451642400, 1451646000, 1451649600, 1451653200, 1451656800,
1451660400, 1451664000, 1451667600, 1451671200, 1451674800,
1451678400, 1451682000, 1451685600, 1451689200, 1451692800, 0
])
time_points[-1] = 2 * time_points[-2] - time_points[
-3] # last time point calc
data = np.array([
0,
10,
11,
12,
13,
np.nan,
15,
# np.inf,
np.
nan, # TODO: figure out how to unmask restoring 'used' mask-values
17,
#-np.inf,
np.nan,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
1
])
ta = TimeAxis(UtcTimeVector.from_numpy(time_points))
ts_exp = TimeSeries(
ta, dv.from_numpy(data),
point_fx.POINT_INSTANT_VALUE) # TODO: policy right ?
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy(),
equal_nan=True))
if path.exists(test_file):
os.remove(test_file)
