def test_create_basic(self): a = np.array([[1.1, 1.2, 1.3], [2.1, 2.2, 2.3]], dtype=np.float64) ta = TimeAxis(0, deltahours(1), 3) tsv = create_ts_vector_from_np_array(ta, a, ts_point_fx.POINT_AVERAGE_VALUE) self.assertIsNotNone(tsv) for i in range(2): self.assertTrue(np.allclose(tsv[i].values.to_numpy(), a[i])) self.assertTrue(ta == tsv[i].time_axis) self.assertEqual(tsv[i].point_interpretation(), ts_point_fx.POINT_AVERAGE_VALUE) # create missmatch throws b = np.array([[], []], dtype=np.float64) try: create_ts_vector_from_np_array(ta, b, ts_point_fx.POINT_AVERAGE_VALUE) self.assertTrue(False, "Should throw for missmatch time-axis") except RuntimeError as e: pass # create empty ts works tb = TimeAxis(0, 0, 0) r = create_ts_vector_from_np_array(tb, b, ts_point_fx.POINT_AVERAGE_VALUE) self.assertEqual(len(r), 2) for ts in r: self.assertFalse(ts) # create empty returns empty c = np.empty(shape=(0, 0), dtype=np.float64) z = create_ts_vector_from_np_array(tb, c, ts_point_fx.POINT_AVERAGE_VALUE) self.assertEqual(len(z), 0)
def test_dtss_partition_by_average(self): """ This test illustrates use of partition_by client and server-side. The main point here is to ensure that the evaluate period covers both the historical and evaluation peri """ with tempfile.TemporaryDirectory() as c_dir: # setup data to be calculated utc = Calendar() d = deltahours(1) t = utc.time(2000, 1, 1) n = utc.diff_units(t, utc.add(t, Calendar.YEAR, 10), d) ta = TimeAxis(t, d, n) td = TimeAxis(t, d * 24, n // 24) n_ts = 1 store_tsv = TsVector() # something we store at server side for i in range(n_ts): pts = TimeSeries( ta, np.sin( np.linspace(start=0, stop=1.0 * (i + 1), num=ta.size())), point_fx.POINT_AVERAGE_VALUE) ts_id = shyft_store_url(f"{i}") store_tsv.append(TimeSeries( ts_id, pts)) # generate a bound pts to store # start dtss server dtss = DtsServer() cache_on_write = True port_no = find_free_port() host_port = 'localhost:{0}'.format(port_no) dtss.set_auto_cache(True) dtss.set_listening_port(port_no) dtss.set_container( "test", c_dir ) # notice we set container 'test' to point to c_dir directory dtss.start_async( ) # the internal shyft time-series will be stored to that container # create dts client c = DtsClient( host_port, auto_connect=False) # demonstrate object life-time connection c.store_ts(store_tsv, overwrite_on_write=True, cache_on_write=cache_on_write) t_0 = utc.time(2018, 1, 1) tax = TimeAxis(t_0, Calendar.DAY, 365) ts_h1 = TimeSeries(shyft_store_url(f'{0}')) ts_h2 = store_tsv[0] ts_p1 = ts_h1.partition_by(utc, t, Calendar.YEAR, 10, t_0).average(tax) ts_p2 = ts_h2.partition_by(utc, t, Calendar.YEAR, 10, t_0).average(tax)
def test_pow(self): """ verify pow(ts,num) pow(num,ts) pow(ts,ts) """ a = TimeSeries(TimeAxis(time('2018-01-01T00:00:00Z'), time(3600), 3), DoubleVector([1.0, 2.0, 3.0]), stair_case) b = TimeSeries(TimeAxis(time('2018-01-01T00:00:00Z'), time(3600), 3), DoubleVector([2.0, 2.0, 2.0]), stair_case) assert_array_almost_equal([1, 4, 9], a.pow(2.0).values.to_numpy()) assert_array_almost_equal([1, 4, 9], pow(a, 2.0).values.to_numpy()) assert_array_almost_equal([2, 4, 8], pow(2.0, a).values.to_numpy()) assert_array_almost_equal([2, 4, 8], pow(b, a).values.to_numpy()) assert_array_almost_equal([2, 4, 8], b.pow(a).values.to_numpy())
def test_dtss_remove_series(self): with tempfile.TemporaryDirectory() as c_dir: # start the server dtss = DtsServer() port_no = find_free_port() host_port = 'localhost:{0}'.format(port_no) dtss.set_listening_port(port_no) dtss.set_container( "test", c_dir ) # notice we set container 'test' to point to c_dir directory dtss.start_async( ) # the internal shyft time-series will be stored to that container # setup some data utc = Calendar() d = deltahours(1) n = 365 * 24 // 3 t = utc.time(2016, 1, 1) ta = TimeAxis(t, d, n) tsv = TsVector() pts = TimeSeries(ta, np.linspace(start=0, stop=1.0, num=ta.size()), point_fx.POINT_AVERAGE_VALUE) tsv.append(TimeSeries("cache://test/foo", pts)) # get a client client = DtsClient(host_port) client.store_ts(tsv) # start with no removing dtss.set_can_remove(False) # we should be disallowed to remove now try: client.remove("shyft://test/foo") except Exception as err: self.assertEqual( str(err), "dtss::server: server does not support removing") # then try with allowing remove dtss.set_can_remove(True) # we only support removing shyft-url style data try: client.remove("protocol://test/foo") except Exception as err: self.assertEqual( str(err), "dtss::server: server does not allow removing for non shyft-url type data" ) # now it should work client.remove("shyft://test/foo")
def test_failures(self): """ Verify that dtss client server connections are auto-magically restored and fixed """ with tempfile.TemporaryDirectory() as c_dir: # start the server dtss = DtsServer() port_no = find_free_port() host_port = 'localhost:{0}'.format(port_no) dtss.set_listening_port(port_no) dtss.set_container( "test", c_dir ) # notice we set container 'test' to point to c_dir directory dtss.start_async( ) # the internal shyft time-series will be stored to that container # setup some data utc = Calendar() d = deltahours(1) n = 365 * 24 // 3 t = utc.time(2016, 1, 1) ta = TimeAxis(t, d, n) tsv = TsVector() pts = TimeSeries(ta, np.linspace(start=0, stop=1.0, num=ta.size()), point_fx.POINT_AVERAGE_VALUE) tsv.append(TimeSeries("cache://test/foo", pts)) # get a client client = DtsClient(host_port, auto_connect=False) client.store_ts(tsv) client.close() client.store_ts(tsv) # should just work, it re-open automagically dtss.clear( ) # the server is out and away, no chance this would work try: client.store_ts(tsv) self.assertTrue( False, 'This should throw, because there is no dtss server to help you' ) except Exception as ee: self.assertFalse(False, f'expected {ee} here') dtss.set_listening_port(port_no) dtss.start_async() client.store_ts( tsv) # this should just work, automagically reconnect
def test_glacier_melt_ts_m3s(self): utc = Calendar() t0 = utc.time(2016, 10, 1) dt = deltahours(1) n = 240 ta = TimeAxis(t0, dt, n) area_m2 = 487 * 1000 * 1000 # Jostedalsbreen, largest i Europe temperature = TimeSeries(ta=ta, fill_value=10.0, point_fx=fx_policy.POINT_AVERAGE_VALUE) sca_values = dv.from_numpy(np.linspace(area_m2 * 1.0, 0.0, num=n)) sca = TimeSeries(ta=ta, values=sca_values, point_fx=fx_policy.POINT_AVERAGE_VALUE) gf = 1.0 * area_m2 dtf = 6.0 melt_m3s = create_glacier_melt_ts_m3s( temperature, sca, gf, dtf) # Here we get back a melt_ts, that we can do ts-stuff with self.assertIsNotNone(melt_m3s) full_melt_m3s = glacier_melt_step(dtf, 10.0, 0.0, gf) expected_melt_m3s = np.linspace(0.0, full_melt_m3s, num=n) assert_array_almost_equal(expected_melt_m3s, melt_m3s.values.to_numpy(), 4) # Just to check we can work with the result as a ts in all aspects mx2 = melt_m3s * 2.0 emx2 = expected_melt_m3s * 2.0 assert_array_almost_equal(emx2, mx2.values.to_numpy(), 4)
def test_windowed_percentiles_tsv_values(self) -> None: """Test that a TsVector is generated by windowed_percentiles_tsv with time-series fulfilling some properties of being percentiles of the data ts.""" cal = Calendar() period = UtcPeriod(cal.time(2017, 1, 1), cal.time(2018, 1, 1)) data = np.linspace(-2, 2, 24 * 7) data_ts = TimeSeries(TimeAxis(0, Calendar.HOUR, len(data)), data, POINT_INSTANT_VALUE) # compute percentiles = [0, 10, 50, 90, 100] tsv = windowed_percentiles_tsv(data_ts, period, 3 * Calendar.HOUR, 12 * Calendar.HOUR, percentiles, self.client, cal) self.assertEqual(len(tsv), 5) # assert that the time-series have the correct properties for being percentile series for i in range(len(tsv[0])): prev_v = tsv[0].values[i] for j in range(len(percentiles) - 1): v = tsv[j + 1].values[i] # both values will be NaN at the end - that is ok if math.isnan(prev_v) and math.isnan(v): continue # check that no larger percentile have values greater than lower percentiles self.assertLessEqual(prev_v, v) prev_v = v
def test_linear_vector(self): ta = TimeAxis(0, 10, 6) tsv = DoubleVector([1, 1, 2, 3, -1.0, 5.0]) fv = TsVector([TimeSeries(ta, tsv, POINT_INSTANT_VALUE)]) d_fv = fv.derivative() f = fv[0] d_f = d_fv[0] self.assertEqual(len(f), len(d_f)) self.assertAlmostEqual(d_f.value(0), 0.0) self.assertAlmostEqual(d_f.value(1), 0.1) self.assertAlmostEqual(d_f.value(2), 0.1) self.assertAlmostEqual(d_f.value(3), -0.4) self.assertAlmostEqual(d_f(f.time(3) + 5), -0.4) self.assertAlmostEqual(d_f.value(4), 0.6) self.assertFalse(math.isfinite(d_f.value(5))) self.assertFalse(math.isfinite(d_f(f.time(5)))) self.assertTrue(math.isfinite(d_f(f.time(5) - 1))) v = d_f.values self.assertAlmostEqual(len(v), len(f)) self.assertAlmostEqual(v[0], 0.0) self.assertAlmostEqual(v[1], 0.1) self.assertAlmostEqual(v[2], 0.1) self.assertAlmostEqual(v[3], -0.4) self.assertAlmostEqual(v[4], 0.6) self.assertFalse(math.isfinite(v[5]))
def test_create_xx_source_vector(self): # arrange the setup a = np.array([[1.1, 1.2, 1.3], [2.1, 2.2, 2.3]], dtype=np.float64) ta = TimeAxis(deltahours(0), deltahours(1), 3) gpv = GeoPointVector() gpv[:] = [GeoPoint(1, 2, 3), GeoPoint(4, 5, 6)] cfs = [(create_precipitation_source_vector_from_np_array, PrecipitationSourceVector), (create_temperature_source_vector_from_np_array, TemperatureSourceVector), (create_radiation_source_vector_from_np_array, RadiationSourceVector), (create_rel_hum_source_vector_from_np_array, RelHumSourceVector), (create_radiation_source_vector_from_np_array, RadiationSourceVector)] # test all creation types: for cf in cfs: r = cf[0](ta, gpv, a, ts_point_fx.POINT_AVERAGE_VALUE) # act here self.assertTrue(isinstance(r, cf[1])) # then the asserts self.assertEqual(len(r), len(gpv)) for i in range(len(gpv)): self.assertEqual(r[i].mid_point(), gpv[i]) self.assertTrue(np.allclose(r[i].ts.values.to_numpy(), a[i])) self.assertEqual(r[i].ts.point_interpretation(), ts_point_fx.POINT_AVERAGE_VALUE)
def test_error_handling(self): utc = Calendar() t0 = utc.time(2018, 1, 1) dt = deltahours(1) dv = DoubleVector() dv[:] = [1.0, 2.0, 2.5, 1.9, 3.0, 3.1, float('nan')] # also verify nan-handling ts = TimeSeries(TimeAxis(t0, dt, len(dv)), dv, POINT_AVERAGE_VALUE) try: ts.decode(start_bit=0, n_bits=0) self.assertTrue(False, 'This should throw, n_bits >0') except RuntimeError as re: pass try: ts.decode(start_bit=41, n_bits=12) self.assertTrue(False, 'This should throw, start_bit + n_bits >52') except RuntimeError as re: pass try: ts.decode(start_bit=-1, n_bits=12) self.assertTrue(False, 'This should throw, start_bit >=0') except RuntimeError as re: pass
def test_forecast_average_slice(self): """ Demo and test TsVector.average_slice(lead_time,dt,n) """ utc = Calendar() t0 = utc.time(2017, 1, 1) dt = deltahours(1) n = 66 # typical arome fc_dt_n_hours = 6 fc_dt = deltahours(fc_dt_n_hours) fc_n = 4 * 10 # 4 each day 10 days fc_v = self._create_forecasts(t0, dt, n, fc_dt, fc_n) for lead_time_hours in range(12): for slice_length_units in [1, 2, 3, 4, 6, 12]: for dt_hours in [1, 2, 3]: slice_v = fc_v.average_slice(deltahours(lead_time_hours), deltahours(dt_hours), slice_length_units) self.assertEqual(len(slice_v), len(fc_v)) # then loop over the slice_v and prove it's equal # to the average of the same portion on the originalj for s, f in zip(slice_v, fc_v): ta = TimeAxis( f.time_axis.time(0) + deltahours(lead_time_hours), deltahours(dt_hours), slice_length_units) ts_expected = f.average(ta) self.assertTrue(s.time_axis == ts_expected.time_axis) self.assertTrue( np.allclose(s.values.to_numpy(), ts_expected.values.to_numpy())) pass
def remove_tp_data(self, period: UtcPeriod): """ delete data given within the time period :param period: :return: """ time_series_cropped = None with Dataset(self.file_path, 'a') as ds: # 1. load the data time_variable = 'time' time = ds.variables.get(time_variable, None) if time is None: raise TimeSeriesStoreError( 'Something is wrong with the dataset. time not found.') var = ds.variables.get(self.ts_meta_info.variable_name, None) if var is None: raise TimeSeriesStoreError( 'Something is wrong with the dataset. variable {0} not found.' .format(self.ts_meta_info.variable_name)) if len(time): # 2. get indices of the data to delete time_utc = convert_netcdf_time(time.units, time) idx_min = np.searchsorted(time_utc, period.start, side='left') idx_max = np.searchsorted(time_utc, period.end, side='right') # check if there is data outside the range if idx_max - idx_min != len(time): # print('indices ', idx_min, idx_max, len(time)) # 3. crop the data array if idx_max < len(time): time_cropped = np.append(time[0:idx_min], time[idx_max:]) var_cropped = np.append(var[0:idx_min], var[idx_max:]) else: time_cropped = np.append(time[0:idx_min], []) var_cropped = np.append(var[0:idx_min], []) last_time_point = 2 * time_cropped[-1] - time_cropped[-2] # print(type(time_cropped[0])) # print(UtcTimeVector.from_numpy(time_cropped.astype(np.int64)).to_numpy()) ta = TimeAxis( UtcTimeVector.from_numpy(time_cropped.astype( np.int64)), int(last_time_point)) # print(var_cropped) # print(type(var_cropped)) time_series_cropped = TimeSeries( ta, dv.from_numpy(var_cropped), point_fx.POINT_INSTANT_VALUE ) # TODO: is this correct point policy? # 4. save the cropped data self.create_new_file() if time_series_cropped: self.append_ts_data(time_series_cropped)
def test_get_ts_info(self): """ Verify we can get specific TsInfo objects for time-series from the server backend. """ with tempfile.TemporaryDirectory() as c_dir: # start the server dtss = DtsServer() port_no = find_free_port() host_adr = 'localhost:{0}'.format(port_no) dtss.set_listening_port(port_no) dtss.set_container( "testing", c_dir ) # notice we set container 'test' to point to c_dir directory dtss.start_async( ) # the internal shyft time-series will be stored to that container # get a client client = DtsClient(host_adr) try: client.get_ts_info(r'shyft://testing/data') except Exception as e: pass else: # only end up here if no exceptions self.fail('Could fetch info for non-existing ts info') # setup some data utc = Calendar() d = deltahours(1) n = 365 * 24 // 3 t = utc.time(2016, 1, 1) ta = TimeAxis(t, d, n) tsv = TsVector() pts = TimeSeries(ta, np.linspace(start=0, stop=1.0, num=ta.size()), point_fx.POINT_AVERAGE_VALUE) tsv.append(TimeSeries(r'shyft://testing/data', pts)) client.store_ts(tsv) info: TsInfo = client.get_ts_info(r'shyft://testing/data') self.assertEqual(info.name, r'data') self.assertEqual(info.point_fx, point_fx.POINT_AVERAGE_VALUE) self.assertEqual(info.data_period, ta.total_period())
def test_simple_case(self): utc = Calendar() t0 = utc.time(2018, 1, 1) dt = deltahours(1) dv = DoubleVector() dv[:] = [1.0, 2.0, 2.5, 1.9, 3.0, 3.1, -1.0] i1_ex = [0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0] ts = TimeSeries(TimeAxis(t0, dt, len(dv)), dv, POINT_AVERAGE_VALUE) i1 = ts.inside(2.0, 3.0) assert_array_almost_equal(i1.values.to_numpy(), np.array(i1_ex))
def period_percentiles_tsv( ts: TimeSeries, period: UtcPeriod, average_dt: int, percentile_period: UtcPeriod, percentiles: Sequence[int], client: DtsClient, calendar: Calendar ) -> TsVector: """Compute percentiles from a part of a time-series and generate a TsVector of percentile time-series spanning a possibly different time period. Args: ts: TimeSeries to compute percentile time-series for. period: Time period the output time-series should span. average_dt: Period to average values by when partitioning the input time-series. percentile_period: Period from ts to compute percentiles for. percentiles: Percentiles to compute. Values should be in the range ``0..100``. client: DtsClient to use for performing the partitioning. calendar: Calendar to to for interpreting time and partitioning the input time-series. Returns: A TsVector with one TimeSeries for each value in percentiles, all spanning period. """ periods = int((percentile_period.end - percentile_period.start)//average_dt) n_avg = calendar.diff_units(period.start, period.end, average_dt) if n_avg*average_dt < period.end - period.start: n_avg += 1 tsv = client.evaluate( ts.average(TimeAxis(period.start, average_dt, n_avg)) .partition_by(calendar, period.start, average_dt, periods, period.start), period ) tsv_p = tsv.percentiles(TimeAxis(period.start, average_dt, 1), percentiles) tsv = TsVector() ta = TimeAxis(period.start, period.end - period.start, 1) for ts_p in tsv_p: tsv.append(TimeSeries(ta, [ts_p.values[0]], POINT_AVERAGE_VALUE)) return tsv
def test_inverted_values(self): utc = Calendar() t0 = utc.time(2018, 1, 1) dt = deltahours(1) dv = DoubleVector() dv[:] = [1.0, 2.0, 2.5, 1.9, 3.0, 3.1, float('nan')] # also verify nan-handling i1_ex = [1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0] ts = TimeSeries(TimeAxis(t0, dt, len(dv)), dv, POINT_AVERAGE_VALUE) i2 = ts.inside(min_v=2.0, max_v=3.0, nan_v=1.0, inside_v=0.0, outside_v=1.0) assert_array_almost_equal(i2.values.to_numpy(), np.array(i1_ex))
def windowed_percentiles_tsv( ts: TimeSeries, period: UtcPeriod, average_dt: int, percentile_dt: int, percentiles: Sequence[int], client: DtsClient, calendar: Calendar ) -> TsVector: """Compute percentiles for a time-series in a gliding window fashion. The input time-series is partitioned and averaged using ``ts.partition_by`` such that each value is included in percentile computations spanning percentile_dt time from the value occurrence. Args: ts: TimeSeries to compute percentile time-series for. period: Period to generate percentile time-series for. average_dt: Period to average values by when partitioning the input time-series. This is the time-step of the time-series in the output TsVector. percentile_dt: Time span for a value to contribute to percentile computations. percentiles: Percentiles to compute. Values should be in the range ``0..100``. client: DtsClient to use for performing the partitioning. calendar: Calendar to to for interpreting time and partitioning the input time-series. Returns: A TsVector with one TimeSeries for each value in percentiles, all spanning period. """ periods = int(percentile_dt//average_dt) n_avg = calendar.diff_units(period.start, period.end, average_dt) if n_avg*average_dt < period.end - period.start: n_avg += 1 tsv = client.evaluate( ts.average(TimeAxis(period.start, average_dt, n_avg)) .partition_by(calendar, period.start, average_dt, periods, period.start) .time_shift(periods*average_dt), period ) return tsv.percentiles(TimeAxis(period.start, average_dt, n_avg), percentiles)
def test_basic_case(self): ta = TimeAxis(0, 10, 6) to = TimeAxis(10, 20, 3) va = DoubleVector([0, 10, 20, 30, 40.0, 50.0]) a = TimeSeries(ta, va, POINT_INSTANT_VALUE) o = TimeSeries( to, fill_value=0.0, point_fx=POINT_INSTANT_VALUE) # point-ip should not matter r = a.use_time_axis_from(o) self.assertIsNotNone(r) self.assertEquals(r.time_axis, o.time_axis) ev = DoubleVector([10.0, 30.0, 50.0]).to_numpy() self.assertTrue(np.allclose(r.values.to_numpy(), ev)) self.assertEquals(r.point_interpretation(), a.point_interpretation()) tsv = TsVector([a, 2.0 * a]) rv = tsv.use_time_axis_from(o) for x in rv: self.assertEquals(x.time_axis, o.time_axis) self.assertTrue(np.allclose(rv[0].values.to_numpy(), ev)) self.assertTrue(np.allclose(rv[1].values.to_numpy(), 2.0 * ev))
def test_create_xx_vector_from_list(self): """ verify we can construct from list of xxSource""" ts = TemperatureSource( GeoPoint(1.0, 2.0, 3.0), TimeSeries(TimeAxis(0, 3600, 10), fill_value=1.0, point_fx=ts_point_fx.POINT_AVERAGE_VALUE)) tsv = TemperatureSourceVector([ts]) self.assertEqual(len(tsv), 1) self.assertEqual(len(TemperatureSourceVector([])), 0) self.assertEqual(len(TemperatureSourceVector([ts, ts])), 2)
def _create_forecasts(self, t0: int, dt: int, n: int, fc_dt: int, fc_n: int) -> TsVector: tsv = TsVector() stair_case = ts_point_fx.POINT_AVERAGE_VALUE for i in range(fc_n): ta = TimeAxis(t0 + i * fc_dt, dt, n) mrk = (i + 1) / 100.0 v = dv.from_numpy( np.linspace(1 + mrk, 1 + n + mrk, n, endpoint=False)) tsv.append(TimeSeries(ta, v, stair_case)) return tsv
def _create_observation(self, t0: int, dt: int, fc_lead_steps: int, fc_n: int) -> TimeSeries: ta = TimeAxis(t0, dt, fc_n * fc_lead_steps) ts = TimeSeries(ta, fill_value=0.0, point_fx=ts_point_fx.POINT_AVERAGE_VALUE) for i in range(len(ta)): ts.set( i, math.sin(0.314 + 3.14 * i / 240.0)) # sin-wave at specified 'time' i return ts
def _create_forecasts(self, t0: int, dt: int, n: int, fc_dt: int, fc_n: int) -> TsVector: tsv = TsVector() stair_case = ts_point_fx.POINT_AVERAGE_VALUE for i in range(fc_n): ta = TimeAxis(t0 + i * fc_dt, dt, n) ts = TimeSeries(ta, fill_value=0.0, point_fx=stair_case) for t in range(len(ta)): tt = i * fc_dt / dt + t ts.set(t, math.sin(0.314 + 3.14 * tt / 240.0)) # make it a sin-wave at time tt tsv.append(ts) return tsv
def test_create_tsvector_from_ts_list(self): ts_list = [ TimeSeries(TimeAxis(0, 3600, 10), fill_value=float(i), point_fx=ts_point_fx.POINT_AVERAGE_VALUE) for i in range(3) ] tsv = TsVector(ts_list) assert tsv assert len(tsv) == 3 assert tsv[0].value(0) == 0.0 assert tsv[1].value(0) == 1.0 assert tsv[2].value(0) == 2.0
def test_windowed_percentiles_tsv_empty(self) -> None: """Test that an empty TsVector is generated by windowed_percentiles_tsv when given an empty sequence of percentiles.""" cal = Calendar() period = UtcPeriod(cal.time(2017, 1, 1), cal.time(2018, 1, 1)) data = np.linspace(-2, 2, 24 * 7) data_ts = TimeSeries(TimeAxis(0, Calendar.HOUR, len(data)), data, POINT_INSTANT_VALUE) # compute tsv = windowed_percentiles_tsv(data_ts, period, Calendar.HOUR, Calendar.HOUR, [], self.client, cal) self.assertEqual(len(tsv), 0)
def test_inside_of_derivative(self): """Created in response to https://github.com/statkraft/shyft/issues/352""" values = [1, 1, 1, 1] utc = Calendar() data = np.array(values, dtype='float64') data_ta = TimeAxis(utc.time(2015, 1, 1), 3600, len(data)) orig = TimeSeries(data_ta, data, POINT_AVERAGE_VALUE) orig_derivative_inside_inf = orig.derivative(derivative_method.BACKWARD).inside(-float('inf'), float('inf'), 0, 1, 0) def check_nan(ts): """Method that returns 1 for all timesteps that contain nan.""" return ts.inside(-float('inf'), float('inf'), 1, 0, 0).values.to_numpy() np.testing.assert_equal(check_nan(orig + orig_derivative_inside_inf), [0., 0., 0., 0.], 'TimeSeries.inside() should not find any NaN-values in this TimeSeries')
def dtss_read_callback(self, ts_ids: StringVector, read_period: UtcPeriod) -> TsVector: self.callback_count += 1 r = TsVector() ta = TimeAxis(read_period.start, deltahours(1), int(read_period.timespan() // deltahours(1))) if self.rd_throws: self.rd_throws = False raise RuntimeError("read-ts-problem") for ts_id in ts_ids: r.append( TimeSeries(ta, fill_value=1.0, point_fx=point_fx.POINT_AVERAGE_VALUE)) if self.cache_reads and self.cache_dtss: # illustrate how the read-callback can ask the dtss to cache it's reads self.cache_dtss.cache(ts_ids, r) return r
def test_simple_case(self): utc = Calendar() t0 = utc.time(2018, 1, 1) dt = deltahours(1) dv = DoubleVector() dv[:] = [1.2, 0.0, 2.0, 5.0, 15.0, float('nan'), -1.0] # these are bit-encoded values, note 1.2 -> 1.0 i0_1_e = [1.0, 0.0, 0.0, 1.0, 1.0, float('nan'), float('nan')] # expected values for bit 0 1-bit i1_3_e = [0.0, 0.0, 1.0, 2.0, 7.0, float('nan'), float('nan')] # expected values for bit 3 3-bits ts = TimeSeries(TimeAxis(t0, dt, len(dv)), dv, POINT_AVERAGE_VALUE) i0_1 = ts.decode(start_bit=0, n_bits=1) i1_3 = ts.decode(start_bit=1, n_bits=3) assert_array_almost_equal(i0_1.values.to_numpy(), np.array(i0_1_e)) assert_array_almost_equal(i1_3.values.to_numpy(), np.array(i1_3_e))
def fixed_tsv( period: UtcPeriod, fixed_values: Sequence[float] ) -> TsVector: """Create a TsVector with TimeSeries with fixed values spanning the given period. Args: period: Time period the generated TimeSeries should span. fixed_values: A sequence of numbers to generate constant TimeSeries for. Returns: A TsVector with one TimeSeries for each value in fixed_values, all spanning period. """ tsv = TsVector() for fv in fixed_values: tsv.append(TimeSeries( TimeAxis(UtcTimeVector([period.start, period.end])), [fv], POINT_AVERAGE_VALUE )) return tsv
def test_selector_ts(self) -> None: """Test that selector_ts constructs a time-series selects data from different time-series correctly.""" n = 24 cal = Calendar() period = UtcPeriod(0, n * Calendar.HOUR) data_ts = TimeSeries(TimeAxis(0, Calendar.HOUR, n), np.linspace(-10, 10, n), POINT_INSTANT_VALUE) source_tss = [ TimeSeries(TimeAxis(0, Calendar.HOUR, n), 1.00 * np.ones(n), POINT_INSTANT_VALUE), TimeSeries(TimeAxis(0, Calendar.HOUR, n), 10.0 * np.ones(n), POINT_INSTANT_VALUE), TimeSeries(TimeAxis(0, Calendar.HOUR, n), 100. * np.ones(n), POINT_INSTANT_VALUE), ] threshold_1 = -5 threshold_2 = 5 threshold_tss = [ TimeSeries(TimeAxis(0, Calendar.HOUR, n), threshold_1 * np.ones(n), POINT_INSTANT_VALUE), TimeSeries(TimeAxis(0, Calendar.HOUR, n), threshold_2 * np.ones(n), POINT_INSTANT_VALUE), ] ts = selector_ts(data_ts, period, 2 * Calendar.HOUR, threshold_tss, source_tss, POINT_AVERAGE_VALUE, self.client, cal) self.assertEqual(len(data_ts), len(ts)) for dv, rv in zip(data_ts.values, ts.values): if dv < threshold_1: self.assertEqual(rv, source_tss[0].values[0]) elif threshold_1 <= dv < threshold_2: self.assertEqual(rv, source_tss[1].values[0]) else: self.assertEqual(rv, source_tss[2].values[0])
def test_can_create_cf_compliant_file(self): # create files test_file = path.join(path.abspath(os.curdir), 'shyft_test.nc') if path.exists(test_file): os.remove(test_file) # create meta info epsg_id = 32633 x0 = 100000 x1 = 200000 y0 = 100000 y1 = 200000 x = 101000 y = 101000 z = 1200 temperature = TimeSeriesMetaInfo('temperature', '/observed/at_stn_abc/temperature', 'observed air temperature', x, y, z, epsg_id) # create time axis utc = Calendar() ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 24) data = np.arange(0, ta.size(), dtype=np.float64) ts = TimeSeries(ta, dv.from_numpy(data), point_fx=point_fx.POINT_AVERAGE_VALUE) # save the first batch t_ds = TimeSeriesStore(test_file, temperature) t_ds.create_new_file() t_ds.append_ts_data(ts) # expected result ts_exp = ts # now read back the result using a *standard* shyft cf geo repository selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]} ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria) # now read back 'temperature' that we know should be there rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period()) # and verify that we get exactly back what we wanted. self.assertIsNotNone(rts_map) self.assertTrue('temperature' in rts_map) geo_temperature = rts_map['temperature'] self.assertEqual(len(geo_temperature), 1) self.assertLessEqual( GeoPoint.distance2(geo_temperature[0].mid_point(), GeoPoint(x, y, z)), 1.0) # check if time axis is as expected self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis) self.assertTrue( np.allclose(geo_temperature[0].ts.time_axis.time_points, ts_exp.time_axis.time_points)) self.assertEqual(geo_temperature[0].ts.point_interpretation(), point_fx.POINT_AVERAGE_VALUE) # check if variable data is as expected self.assertTrue( np.allclose(geo_temperature[0].ts.values.to_numpy(), ts_exp.values.to_numpy())) # -------------------------------------- # Append data print("\n\n append at the end data") # create time axis ta = TimeAxis(utc.time(2016, 1, 2), deltahours(1), 48) ts = TimeSeries(ta, dv.from_numpy(np.arange(0, ta.size(), dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE) # save the data t_ds.append_ts_data(ts) # expected result ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 72) data = np.empty(72) data[:24] = np.arange(0, 24, dtype=np.float64) data[24:72] = np.arange(0, 48, dtype=np.float64) # <-- new data ts_exp = TimeSeries(ta, dv.from_numpy(data), point_fx=point_fx.POINT_AVERAGE_VALUE) # now read back the result using a *standard* shyft cf geo repository selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]} ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria) # now read back 'temperature' that we know should be there rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period()) # and verify that we get exactly back what we wanted. self.assertIsNotNone(rts_map) self.assertTrue('temperature' in rts_map) geo_temperature = rts_map['temperature'] self.assertEqual(len(geo_temperature), 1) self.assertLessEqual( GeoPoint.distance2(geo_temperature[0].mid_point(), GeoPoint(x, y, z)), 1.0) # check if time axis is as expected self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis) self.assertTrue( np.allclose(geo_temperature[0].ts.time_axis.time_points, ts_exp.time_axis.time_points)) self.assertEqual(geo_temperature[0].ts.point_interpretation(), point_fx.POINT_AVERAGE_VALUE) # check if variable data is as expected self.assertTrue( np.allclose(geo_temperature[0].ts.values.to_numpy(), ts_exp.values.to_numpy())) # -------------------------------------- # Append with overlap print("\n\n append with overlap") # create time axis ta = TimeAxis(utc.time(2016, 1, 3), deltahours(1), 48) ts = TimeSeries(ta, dv.from_numpy(np.arange(0, ta.size(), dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE) # save the data t_ds.append_ts_data(ts) # expected result ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 96) data = np.empty(96) data[:24] = np.arange(0, 24, dtype=np.float64) data[24:48] = np.arange(0, 24, dtype=np.float64) # <-- new data data[48:96] = np.arange(0, 48, dtype=np.float64) # <-- new data ts_exp = TimeSeries(ta, dv.from_numpy(data), point_fx=point_fx.POINT_AVERAGE_VALUE) # now read back the result using a *standard* shyft cf geo repository selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]} ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria) # now read back 'temperature' that we know should be there rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period()) # and verify that we get exactly back what we wanted. self.assertIsNotNone(rts_map) self.assertTrue('temperature' in rts_map) geo_temperature = rts_map['temperature'] self.assertEqual(len(geo_temperature), 1) self.assertLessEqual( GeoPoint.distance2(geo_temperature[0].mid_point(), GeoPoint(x, y, z)), 1.0) # check if time axis is as expected self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis) self.assertTrue( np.allclose(geo_temperature[0].ts.time_axis.time_points, ts_exp.time_axis.time_points)) self.assertEqual(geo_temperature[0].ts.point_interpretation(), point_fx.POINT_AVERAGE_VALUE) # check if variable data is as expected self.assertTrue( np.allclose(geo_temperature[0].ts.values.to_numpy(), ts_exp.values.to_numpy())) # -------------------------------------- # Append with gap in time axis print("\n\n Append with gap in time axis") # create time axis ta = TimeAxis(utc.time(2016, 1, 6), deltahours(1), 24) ts = TimeSeries(ta, dv.from_numpy(np.arange(0, ta.size(), dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE) # save the data t_ds.append_ts_data(ts) # expected result time_vals = np.append( TimeAxis(utc.time(2016, 1, 1), deltahours(1), 96).time_points[:-1], ta.time_points) # print(time_vals) ta = TimeAxis(UtcTimeVector.from_numpy(time_vals.astype(np.int64))) data = np.empty(120) data[:24] = np.arange(0, 24, dtype=np.float64) data[24:48] = np.arange(0, 24, dtype=np.float64) data[48:96] = np.arange(0, 48, dtype=np.float64) data[96:120] = np.arange(0, 24, dtype=np.float64) # <-- new data ts_exp = TimeSeries(ta, dv.from_numpy(data), point_fx=point_fx.POINT_AVERAGE_VALUE) # now read back the result using a *standard* shyft cf geo repository selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]} ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria) # now read back 'temperature' that we know should be there # print(ts_exp.total_period()) rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period()) # and verify that we get exactly back what we wanted. self.assertIsNotNone(rts_map) self.assertTrue('temperature' in rts_map) geo_temperature = rts_map['temperature'] self.assertEqual(len(geo_temperature), 1) self.assertLessEqual( GeoPoint.distance2(geo_temperature[0].mid_point(), GeoPoint(x, y, z)), 1.0) # check if time axis is as expected # print(geo_temperature[0].ts.time_axis.time_points - ts_exp.time_axis.time_points) # print(geo_temperature[0].ts.time_axis.time_points - time_vals) # print(ts_exp.time_axis.time_points - time_vals) self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis) self.assertTrue( np.allclose(geo_temperature[0].ts.time_axis.time_points, ts_exp.time_axis.time_points)) self.assertEqual(geo_temperature[0].ts.point_interpretation(), point_fx.POINT_AVERAGE_VALUE) # check if variable data is as expected self.assertTrue( np.allclose(geo_temperature[0].ts.values.to_numpy(), ts_exp.values.to_numpy())) # -------------------------------------- # Add new data in the middle where nothing was defined (no moving) print( "\n\n Add new data in the middle where nothing was defined (no moving)" ) # create time axis ta = TimeAxis(utc.time(2016, 1, 2), deltahours(1), 24) ts = TimeSeries(ta, dv.from_numpy( np.arange(100, 100 + ta.size(), dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE) # save the data t_ds.append_ts_data(ts) # expected result time_vals = np.append( TimeAxis(utc.time(2016, 1, 1), deltahours(1), 96).time_points[:-1], TimeAxis(utc.time(2016, 1, 6), deltahours(1), 24).time_points) ta = TimeAxis(UtcTimeVector.from_numpy(time_vals.astype(np.int64))) data = np.empty(120) data[:24] = np.arange(0, 24, dtype=np.float64) data[24:48] = np.arange(100, 124, dtype=np.float64) # <-- new data data[48:96] = np.arange(0, 48, dtype=np.float64) data[96:120] = np.arange(0, 24, dtype=np.float64) ts_exp = TimeSeries(ta, dv.from_numpy(data), point_fx=point_fx.POINT_AVERAGE_VALUE) # now read back the result using a *standard* shyft cf geo repository selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]} ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria) # now read back 'temperature' that we know should be there rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period()) # print(ts_exp.total_period()) # and verify that we get exactly back what we wanted. self.assertIsNotNone(rts_map) self.assertTrue('temperature' in rts_map) geo_temperature = rts_map['temperature'] self.assertEqual(len(geo_temperature), 1) self.assertLessEqual( GeoPoint.distance2(geo_temperature[0].mid_point(), GeoPoint(x, y, z)), 1.0) # check if time axis is as expected self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis) self.assertTrue( np.allclose(geo_temperature[0].ts.time_axis.time_points, ts_exp.time_axis.time_points)) self.assertEqual(geo_temperature[0].ts.point_interpretation(), point_fx.POINT_AVERAGE_VALUE) # check if variable data is as expected self.assertTrue( np.allclose(geo_temperature[0].ts.values.to_numpy(), ts_exp.values.to_numpy())) # -------------------------------------- # Insert new data in the middle and move rest print("\n\n insert new data and move rest") # create time axis ta = TimeAxis(utc.time(2016, 1, 5), deltahours(1), 36) ts = TimeSeries(ta, dv.from_numpy( np.arange(200, 200 + ta.size(), dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE) # save the data t_ds.append_ts_data(ts) # expected result ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 144) data = np.empty(144) data[:24] = np.arange(0, 24, dtype=np.float64) data[24:48] = np.arange(100, 124, dtype=np.float64) data[48:96] = np.arange(0, 48, dtype=np.float64) data[96:132] = np.arange(200, 236, dtype=np.float64) # <-- new data data[132:144] = np.arange(12, 24, dtype=np.float64) ts_exp = TimeSeries(ta, dv.from_numpy(data), point_fx=point_fx.POINT_AVERAGE_VALUE) # now read back the result using a *standard* shyft cf geo repository selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]} ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria) # now read back 'temperature' that we know should be there rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period()) # and verify that we get exactly back what we wanted. self.assertIsNotNone(rts_map) self.assertTrue('temperature' in rts_map) geo_temperature = rts_map['temperature'] self.assertEqual(len(geo_temperature), 1) self.assertLessEqual( GeoPoint.distance2(geo_temperature[0].mid_point(), GeoPoint(x, y, z)), 1.0) # check if time axis is as expected self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis) self.assertTrue( np.allclose(geo_temperature[0].ts.time_axis.time_points, ts_exp.time_axis.time_points)) self.assertEqual(geo_temperature[0].ts.point_interpretation(), point_fx.POINT_AVERAGE_VALUE) # check if variable data is as expected self.assertTrue( np.allclose(geo_temperature[0].ts.values.to_numpy(), ts_exp.values.to_numpy())) # -------------------------------------- # Add new data before existing data without overlap print("\n\n add new data before existing data without overlap") # create time axis ta = TimeAxis(utc.time(2015, 12, 31), deltahours(1), 24) ts = TimeSeries(ta, dv.from_numpy( np.arange(300, 300 + ta.size(), dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE) # save the first batch t_ds.append_ts_data(ts) # expected result ta = TimeAxis(utc.time(2015, 12, 31), deltahours(1), 168) data = np.empty(168) data[:24] = np.arange(300, 324, dtype=np.float64) # <-- new data data[24:48] = np.arange(0, 24, dtype=np.float64) data[48:72] = np.arange(100, 124, dtype=np.float64) data[72:120] = np.arange(0, 48, dtype=np.float64) data[120:156] = np.arange(200, 236, dtype=np.float64) data[156:168] = np.arange(12, 24, dtype=np.float64) ts_exp = TimeSeries(ta, dv.from_numpy(data), point_fx=point_fx.POINT_AVERAGE_VALUE) # now read back the result using a *standard* shyft cf geo repository selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]} ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria) # now read back 'temperature' that we know should be there rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period()) # and verify that we get exactly back what we wanted. self.assertIsNotNone(rts_map) self.assertTrue('temperature' in rts_map) geo_temperature = rts_map['temperature'] self.assertEqual(len(geo_temperature), 1) self.assertLessEqual( GeoPoint.distance2(geo_temperature[0].mid_point(), GeoPoint(x, y, z)), 1.0) # check if time axis is as expected self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis) self.assertTrue( np.allclose(geo_temperature[0].ts.time_axis.time_points, ts_exp.time_axis.time_points)) self.assertEqual(geo_temperature[0].ts.point_interpretation(), point_fx.POINT_AVERAGE_VALUE) # check if variable data is as expected self.assertTrue( np.allclose(geo_temperature[0].ts.values.to_numpy(), ts_exp.values.to_numpy())) # -------------------------------------- # add new data before existing data with overlap print("\n\n add new data before existing data with overlap") # create time axis ta = TimeAxis(utc.time(2015, 12, 30), deltahours(1), 36) ts = TimeSeries(ta, dv.from_numpy( np.arange(400, 400 + ta.size(), dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE) # save the first batch # t_ds = TimeSeriesStore(test_file, temperature) t_ds.append_ts_data(ts) # expected result ta = TimeAxis(utc.time(2015, 12, 30), deltahours(1), 192) data = np.empty(192) data[:36] = np.arange(400, 436, dtype=np.float64) # <-- new data data[36:48] = np.arange(312, 324, dtype=np.float64) data[48:72] = np.arange(0, 24, dtype=np.float64) data[72:96] = np.arange(100, 124, dtype=np.float64) data[96:144] = np.arange(0, 48, dtype=np.float64) data[144:180] = np.arange(200, 236, dtype=np.float64) data[180:192] = np.arange(12, 24, dtype=np.float64) ts_exp = TimeSeries(ta, dv.from_numpy(data), point_fx=point_fx.POINT_AVERAGE_VALUE) # now read back the result using a *standard* shyft cf geo repository selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]} ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria) # now read back 'temperature' that we know should be there rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period()) # and verify that we get exactly back what we wanted. self.assertIsNotNone(rts_map) self.assertTrue('temperature' in rts_map) geo_temperature = rts_map['temperature'] self.assertEqual(len(geo_temperature), 1) self.assertLessEqual( GeoPoint.distance2(geo_temperature[0].mid_point(), GeoPoint(x, y, z)), 1.0) # check if time axis is as expected self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis) self.assertTrue( np.allclose(geo_temperature[0].ts.time_axis.time_points, ts_exp.time_axis.time_points)) self.assertEqual(geo_temperature[0].ts.point_interpretation(), point_fx.POINT_AVERAGE_VALUE) # check if variable data is as expected self.assertTrue( np.allclose(geo_temperature[0].ts.values.to_numpy(), ts_exp.values.to_numpy())) # -------------------------------------- # Overwrite everything with less data points # create time axis print('\n\n Overwrite everything with less data points') ta = TimeAxis(utc.time(2015, 12, 30), deltahours(24), 9) ts = TimeSeries(ta, dv.from_numpy( np.arange(1000, 1000 + ta.size(), dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE) # write the time series t_ds.append_ts_data(ts) # expected result ts_exp = ts # now read back the result using a *standard* shyft cf geo repository selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]} ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria) # now read back 'temperature' that we know should be there rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period()) # and verify that we get exactly back what we wanted. self.assertIsNotNone(rts_map) self.assertTrue('temperature' in rts_map) geo_temperature = rts_map['temperature'] self.assertEqual(len(geo_temperature), 1) self.assertLessEqual( GeoPoint.distance2(geo_temperature[0].mid_point(), GeoPoint(x, y, z)), 1.0) # check if time axis is as expected self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis) self.assertTrue( np.allclose(geo_temperature[0].ts.time_axis.time_points, ts_exp.time_axis.time_points)) self.assertEqual(geo_temperature[0].ts.point_interpretation(), point_fx.POINT_AVERAGE_VALUE) # check if variable data is as expected self.assertTrue( np.allclose(geo_temperature[0].ts.values.to_numpy(), ts_exp.values.to_numpy())) # -------------------------------------- # Insert data with different dt # create time axis print('\n\n Insert data with different dt') ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 24) ts = TimeSeries(ta, dv.from_numpy(np.arange(0, 24, dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE) # write the time series t_ds.append_ts_data(ts) # expected result time_points = np.empty(33, dtype=np.int) time_points[0:2] = TimeAxis(utc.time(2015, 12, 30), deltahours(24), 1).time_points time_points[2:26] = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 23).time_points time_points[26:] = TimeAxis(utc.time(2016, 1, 2), deltahours(24), 6).time_points ta = TimeAxis(UtcTimeVector.from_numpy(time_points)) data = np.empty(32) data[0:2] = np.array([1000, 1001]) data[2:26] = np.arange(0, 24) # <-- new data data[26:] = np.arange(1003, 1009) ts_exp = TimeSeries(ta, dv.from_numpy(data), point_fx=point_fx.POINT_AVERAGE_VALUE) # now read back the result using a *standard* shyft cf geo repository selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]} ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria) # now read back 'temperature' that we know should be there rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period()) # and verify that we get exactly back what we wanted. self.assertIsNotNone(rts_map) self.assertTrue('temperature' in rts_map) geo_temperature = rts_map['temperature'] self.assertEqual(len(geo_temperature), 1) self.assertLessEqual( GeoPoint.distance2(geo_temperature[0].mid_point(), GeoPoint(x, y, z)), 1.0) # check if time axis is as expected self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis) self.assertTrue( np.allclose(geo_temperature[0].ts.time_axis.time_points, ts_exp.time_axis.time_points)) self.assertEqual(geo_temperature[0].ts.point_interpretation(), point_fx.POINT_AVERAGE_VALUE) # check if variable data is as expected self.assertTrue( np.allclose(geo_temperature[0].ts.values.to_numpy(), ts_exp.values.to_numpy())) # -------------------------------------- # delete data with range UtcPeriod in the middle print('\n\n delete data with range UtcPeriod') tp = UtcPeriod(utc.time(2015, 12, 31), utc.time(2016, 1, 1, 12)) # ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 24) # ts = TimeSeries(ta, dv.from_numpy(np.arange(0, 24, dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE) # write the time series t_ds.remove_tp_data(tp) # expected result time_points = np.array([ 1451433600, 1451653200, 1451656800, 1451660400, 1451664000, 1451667600, 1451671200, 1451674800, 1451678400, 1451682000, 1451685600, 1451689200, 1451692800, 1451779200, 1451865600, 1451952000, 1452038400, 1452124800, 1452211200 ]) ta = TimeAxis(UtcTimeVector.from_numpy(time_points)) data = np.array([ 1000, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 1003, 1004, 1005, 1006, 1007, 1008 ]) ts_exp = TimeSeries(ta, dv.from_numpy(data), point_fx.POINT_INSTANT_VALUE ) # TODO: is this correct policy to use # now read back the result using a *standard* shyft cf geo repository selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]} ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria) # now read back 'temperature' that we know should be there rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period()) # and verify that we get exactly back what we wanted. self.assertIsNotNone(rts_map) self.assertTrue('temperature' in rts_map) geo_temperature = rts_map['temperature'] self.assertEqual(len(geo_temperature), 1) self.assertLessEqual( GeoPoint.distance2(geo_temperature[0].mid_point(), GeoPoint(x, y, z)), 1.0) # check if time axis is as expected self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis) self.assertTrue( np.allclose(geo_temperature[0].ts.time_axis.time_points, ts_exp.time_axis.time_points)) self.assertEqual(geo_temperature[0].ts.point_interpretation(), point_fx.POINT_AVERAGE_VALUE) # check if variable data is as expected self.assertTrue( np.allclose(geo_temperature[0].ts.values.to_numpy(), ts_exp.values.to_numpy())) # -------------------------------------- # delete data with range UtcPeriod at the start print('\n\n delete data with range UtcPeriod at the start') tp = UtcPeriod(1451433600, 1451667600) # ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 24) # ts = TimeSeries(ta, dv.from_numpy(np.arange(0, 24, dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE) # write the time series t_ds.remove_tp_data(tp) # expected result time_points = np.array([ 1451671200, 1451674800, 1451678400, 1451682000, 1451685600, 1451689200, 1451692800, 1451779200, 1451865600, 1451952000, 1452038400, 1452124800, 1452211200 ]) ta = TimeAxis(UtcTimeVector.from_numpy(time_points)) data = np.array( [18, 19, 20, 21, 22, 23, 1003, 1004, 1005, 1006, 1007, 1008]) ts_exp = TimeSeries( ta, dv.from_numpy(data), point_fx.POINT_INSTANT_VALUE ) # TODO: is this correct policy to use for this test # now read back the result using a *standard* shyft cf geo repository selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]} ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria) # now read back 'temperature' that we know should be there rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period()) # and verify that we get exactly back what we wanted. self.assertIsNotNone(rts_map) self.assertTrue('temperature' in rts_map) geo_temperature = rts_map['temperature'] self.assertEqual(len(geo_temperature), 1) self.assertLessEqual( GeoPoint.distance2(geo_temperature[0].mid_point(), GeoPoint(x, y, z)), 1.0) # check if time axis is as expected self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis) self.assertTrue( np.allclose(geo_temperature[0].ts.time_axis.time_points, ts_exp.time_axis.time_points)) self.assertEqual(geo_temperature[0].ts.point_interpretation(), point_fx.POINT_AVERAGE_VALUE) # check if variable data is as expected self.assertTrue( np.allclose(geo_temperature[0].ts.values.to_numpy(), ts_exp.values.to_numpy())) # -------------------------------------- # delete data with range UtcPeriod at the end print('\n\n delete data with range UtcPeriod at the end') tp = UtcPeriod(1451952000, utc.time(2016, 1, 10)) # ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 24) # ts = TimeSeries(ta, dv.from_numpy(np.arange(0, 24, dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE) # write the time series t_ds.remove_tp_data(tp) # expected result time_points = np.array([ 1451671200, 1451674800, 1451678400, 1451682000, 1451685600, 1451689200, 1451692800, 1451779200, 1451865600, 1451952000 ]) ta = TimeAxis(UtcTimeVector.from_numpy(time_points)) data = np.array([18, 19, 20, 21, 22, 23, 1003, 1004, 1005]) ts_exp = TimeSeries(ta, dv.from_numpy(data), point_fx.POINT_INSTANT_VALUE) # now read back the result using a *standard* shyft cf geo repository selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]} ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria) # now read back 'temperature' that we know should be there try: rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period()) except CFDataRepositoryError: pass # and verify that we get exactly back what we wanted. self.assertIsNotNone(rts_map) self.assertTrue('temperature' in rts_map) geo_temperature = rts_map['temperature'] self.assertEqual(len(geo_temperature), 1) self.assertLessEqual( GeoPoint.distance2(geo_temperature[0].mid_point(), GeoPoint(x, y, z)), 1.0) # check if time axis is as expected self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis) self.assertTrue( np.allclose(geo_temperature[0].ts.time_axis.time_points, ts_exp.time_axis.time_points)) self.assertEqual(geo_temperature[0].ts.point_interpretation(), point_fx.POINT_AVERAGE_VALUE) # check if variable data is as expected self.assertTrue( np.allclose(geo_temperature[0].ts.values.to_numpy(), ts_exp.values.to_numpy())) # -------------------------------------- # delete data with range UtcPeriod everything print('\n\n delete data with range UtcPeriod everything') tp = UtcPeriod(utc.time(2016, 1, 1), utc.time(2016, 1, 10)) # write the time series t_ds.remove_tp_data(tp) # now read back the result using a *standard* shyft cf geo repository selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]} ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria) # now read back 'temperature' that we know should be there self.assertRaises(CFDataRepositoryError, ts_dr.get_timeseries, ['temperature'], tp) # -------------------------------------- # insert data in between time saved data points print('\n\n insert data in between time saved data points') # insert first data in which we want to insert the second batch utc = Calendar() ta = TimeAxis(utc.time(2016, 1, 1), deltahours(24), 2) data = np.arange(0, ta.size(), dtype=np.float64) ts = TimeSeries(ta, dv.from_numpy(data), point_fx=point_fx.POINT_AVERAGE_VALUE) # save the first batch t_ds.append_ts_data(ts) # insert first data for every hour in between utc = Calendar() ta = TimeAxis(utc.time(2016, 1, 1) + deltahours(1), deltahours(1), 23) data = np.arange(10, 10 + ta.size(), dtype=np.float64) ts = TimeSeries(ta, dv.from_numpy(data), point_fx=point_fx.POINT_AVERAGE_VALUE) # save the first batch t_ds.append_ts_data(ts) # expected result time_points = np.array([ 1451606400, 1451610000, 1451613600, 1451617200, 1451620800, 1451624400, 1451628000, 1451631600, 1451635200, 1451638800, 1451642400, 1451646000, 1451649600, 1451653200, 1451656800, 1451660400, 1451664000, 1451667600, 1451671200, 1451674800, 1451678400, 1451682000, 1451685600, 1451689200, 1451692800, 0 ]) time_points[-1] = 2 * time_points[-2] - time_points[ -3] # last time point calc data = np.array([ 0, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1 ]) ta = TimeAxis(UtcTimeVector.from_numpy(time_points)) ts_exp = TimeSeries( ta, dv.from_numpy(data), point_fx.POINT_INSTANT_VALUE ) # TODO: is this correct policy value for this case # now read back the result using a *standard* shyft cf geo repository selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]} ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria) # now read back 'temperature' that we know should be there rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period()) # and verify that we get exactly back what we wanted. self.assertIsNotNone(rts_map) self.assertTrue('temperature' in rts_map) geo_temperature = rts_map['temperature'] self.assertEqual(len(geo_temperature), 1) self.assertLessEqual( GeoPoint.distance2(geo_temperature[0].mid_point(), GeoPoint(x, y, z)), 1.0) # check if time axis is as expected self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis) self.assertTrue( np.allclose(geo_temperature[0].ts.time_axis.time_points, ts_exp.time_axis.time_points)) self.assertEqual(geo_temperature[0].ts.point_interpretation(), point_fx.POINT_AVERAGE_VALUE) # check if variable data is as expected self.assertTrue( np.allclose(geo_temperature[0].ts.values.to_numpy(), ts_exp.values.to_numpy())) # -------------------------------------- # insert data including nan print('\n\n insert data including nan') utc = Calendar() ta = TimeAxis(utc.time(2016, 1, 1) + deltahours(1), deltahours(1), 23) data = np.arange(10, 10 + ta.size(), dtype=np.float64) data[4] = np.nan data[ 6] = np.nan # np.inf, but trouble getting inf trough all version of numpy/netcdf data[8] = np.nan # -np.inf, --"-- ts = TimeSeries(ta, dv.from_numpy(data), point_fx=point_fx.POINT_AVERAGE_VALUE) # save the first batch t_ds.append_ts_data(ts) # expected result time_points = np.array([ 1451606400, 1451610000, 1451613600, 1451617200, 1451620800, 1451624400, 1451628000, 1451631600, 1451635200, 1451638800, 1451642400, 1451646000, 1451649600, 1451653200, 1451656800, 1451660400, 1451664000, 1451667600, 1451671200, 1451674800, 1451678400, 1451682000, 1451685600, 1451689200, 1451692800, 0 ]) time_points[-1] = 2 * time_points[-2] - time_points[ -3] # last time point calc data = np.array([ 0, 10, 11, 12, 13, np.nan, 15, # np.inf, np. nan, # TODO: figure out how to unmask restoring 'used' mask-values 17, #-np.inf, np.nan, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 1 ]) ta = TimeAxis(UtcTimeVector.from_numpy(time_points)) ts_exp = TimeSeries( ta, dv.from_numpy(data), point_fx.POINT_INSTANT_VALUE) # TODO: policy right ? # now read back the result using a *standard* shyft cf geo repository selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]} ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria) # now read back 'temperature' that we know should be there rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period()) # and verify that we get exactly back what we wanted. self.assertIsNotNone(rts_map) self.assertTrue('temperature' in rts_map) geo_temperature = rts_map['temperature'] self.assertEqual(len(geo_temperature), 1) self.assertLessEqual( GeoPoint.distance2(geo_temperature[0].mid_point(), GeoPoint(x, y, z)), 1.0) # check if time axis is as expected self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis) self.assertTrue( np.allclose(geo_temperature[0].ts.time_axis.time_points, ts_exp.time_axis.time_points)) self.assertEqual(geo_temperature[0].ts.point_interpretation(), point_fx.POINT_AVERAGE_VALUE) # check if variable data is as expected self.assertTrue( np.allclose(geo_temperature[0].ts.values.to_numpy(), ts_exp.values.to_numpy(), equal_nan=True)) if path.exists(test_file): os.remove(test_file)