def test_fixed_tsv_empty(self) -> None:
"""Test that an empty TsVector is generated by fixed_tsv when given an empty sequence of values."""
cal = Calendar()
period = UtcPeriod(cal.time(2017, 1, 1), cal.time(2018, 1, 1))
tsv = fixed_tsv(period, [])
self.assertEqual(len(tsv), 0)
def test_windowed_percentiles_tsv_values(self) -> None:
"""Test that a TsVector is generated by windowed_percentiles_tsv with time-series
fulfilling some properties of being percentiles of the data ts."""
cal = Calendar()
period = UtcPeriod(cal.time(2017, 1, 1), cal.time(2018, 1, 1))
data = np.linspace(-2, 2, 24 * 7)
data_ts = TimeSeries(TimeAxis(0, Calendar.HOUR, len(data)), data,
POINT_INSTANT_VALUE)
# compute
percentiles = [0, 10, 50, 90, 100]
tsv = windowed_percentiles_tsv(data_ts, period, 3 * Calendar.HOUR,
12 * Calendar.HOUR, percentiles,
self.client, cal)
self.assertEqual(len(tsv), 5)
# assert that the time-series have the correct properties for being percentile series
for i in range(len(tsv[0])):
prev_v = tsv[0].values[i]
for j in range(len(percentiles) - 1):
v = tsv[j + 1].values[i]
# both values will be NaN at the end - that is ok
if math.isnan(prev_v) and math.isnan(v):
continue
# check that no larger percentile have values greater than lower percentiles
self.assertLessEqual(prev_v, v)
prev_v = v
def test_forecasts_that_cover_period(self):
test_period = UtcPeriod(0, 3600)
fc = ForecastSelectionCriteria(forecasts_that_cover_period=test_period)
self.assertTrue(fc.criterion[0] == 'forecasts_that_cover_period')
self.assertTrue(fc.criterion[1] == test_period)
with self.assertRaises(ForecastSelectionCriteriaError):
ForecastSelectionCriteria(forecasts_that_cover_period='hmm throw')
def test_raise_exception_when_no_data_in_request_period(self):
utc_calendar = Calendar()
netcdf_repository = self._construct_from_test_data()
netcdf_repository.raise_if_no_data = True # yes, for now, just imagine this could work.
self.assertIsNotNone(netcdf_repository)
utc_period = UtcPeriod(
utc_calendar.time(YMDhms(2017, 1, 1, 0, 0,
0)), # a period where there is no data in
utc_calendar.time(YMDhms(2020, 12, 31, 0, 0,
0))) # the file supplied
type_source_map = dict()
type_source_map['temperature'] = TemperatureSource
#def test_function():
#
# return netcdf_repository.get_timeseries(
# type_source_map,
# geo_location_criteria=None,
# utc_period=utc_period)
#self.assertRaises(RuntimeError, test_function)
self.assertRaises(
RuntimeError, netcdf_repository.get_timeseries, type_source_map,
**{
'geo_location_criteria': None,
'utc_period': utc_period
})
def continuous_calibration():
utc = Calendar()
t_start = utc.time(YMDhms(2011, 9, 1))
t_fc_start = utc.time(YMDhms(2015, 10, 1))
dt = deltahours(1)
n_obs = int(round((t_fc_start - t_start)/dt))
obs_time_axis = TimeAxisFixedDeltaT(t_start, dt, n_obs + 1)
q_obs_m3s_ts = observed_tistel_discharge(obs_time_axis.total_period())
ptgsk = create_tistel_simulator(PTGSKOptModel, tistel.geo_ts_repository(tistel.grid_spec.epsg()))
initial_state = burn_in_state(ptgsk, t_start, utc.time(YMDhms(2012, 9, 1)), q_obs_m3s_ts)
num_opt_days = 30
# Step forward num_opt_days days and store the state for each day:
recal_start = t_start + deltahours(num_opt_days*24)
t = t_start
state = initial_state
opt_states = {t: state}
while t < recal_start:
ptgsk.run(TimeAxisFixedDeltaT(t, dt, 24), state)
t += deltahours(24)
state = ptgsk.reg_model_state
opt_states[t] = state
recal_stop = utc.time(YMDhms(2011, 10, 30))
recal_stop = utc.time(YMDhms(2012, 5, 30))
curr_time = recal_start
q_obs_avg = TsTransform().to_average(t_start, dt, n_obs + 1, q_obs_m3s_ts)
target_spec = TargetSpecificationPts(q_obs_avg, IntVector([0]), 1.0, KLING_GUPTA)
target_spec_vec = TargetSpecificationVector([target_spec])
i = 0
times = []
values = []
p, p_min, p_max = construct_calibration_parameters(ptgsk)
while curr_time < recal_stop:
print(i)
i += 1
opt_start = curr_time - deltahours(24*num_opt_days)
opt_state = opt_states.pop(opt_start)
p = ptgsk.region_model.get_region_parameter()
p_opt = ptgsk.optimize(TimeAxisFixedDeltaT(opt_start, dt, 24*num_opt_days), opt_state, target_spec_vec,
p, p_min, p_max, tr_stop=1.0e-5)
ptgsk.region_model.set_region_parameter(p_opt)
corr_state = adjust_simulator_state(ptgsk, curr_time, q_obs_m3s_ts)
ptgsk.run(TimeAxisFixedDeltaT(curr_time, dt, 24), corr_state)
curr_time += deltahours(24)
opt_states[curr_time] = ptgsk.reg_model_state
discharge = ptgsk.region_model.statistics.discharge([0])
times.extend(discharge.time(i) for i in range(discharge.size()))
values.extend(list(np.array(discharge.v)))
plt.plot(utc_to_greg(times), values)
plot_results(None, q_obs=observed_tistel_discharge(UtcPeriod(recal_start, recal_stop)))
set_calendar_formatter(Calendar())
#plt.interactive(1)
plt.title("Continuously recalibrated discharge vs observed")
plt.xlabel("Time in UTC")
plt.ylabel(r"Discharge in $\mathbf{m^3s^{-1}}$", verticalalignment="top", rotation="horizontal")
plt.gca().yaxis.set_label_coords(0, 1.1)
def test_utcperiod_criteria(self):
test_period = UtcPeriod(0, 3600)
fc = ForecastSelectionCriteria(
forecasts_created_within_period=test_period)
self.assertTrue(fc.criterion[0] == 'forecasts_created_within_period')
self.assertTrue(fc.criterion[1] == test_period)
with self.assertRaises(ForecastSelectionCriteriaError):
ForecastSelectionCriteria(
forecasts_created_within_period='should throw')
def test_fixed_tsv_values(self) -> None:
"""Test that a TsVector with fixed constant values is generated by fixed_tsv when given
a sequence of values."""
cal = Calendar()
period = UtcPeriod(cal.time(2017, 1, 1), cal.time(2018, 1, 1))
values = [12, 15.5]
tsv = fixed_tsv(period, values)
self.assertEqual(len(tsv), 2)
for v, ts in zip(values, tsv):
for ts_v in ts.values:
self.assertEqual(ts_v, v)
def test_construct_repository(self):
utc_calendar = Calendar()
netcdf_repository = self._construct_from_test_data()
self.assertIsNotNone(netcdf_repository)
utc_period = UtcPeriod(
utc_calendar.time(YMDhms(2005, 1, 1, 0, 0, 0)),
utc_calendar.time(YMDhms(2014, 12, 31, 0, 0, 0)))
type_source_map = dict()
type_source_map['temperature'] = TemperatureSource
geo_ts_dict = netcdf_repository.get_timeseries(
type_source_map, geo_location_criteria=None, utc_period=utc_period)
self.assertIsNotNone(geo_ts_dict)
def test_returns_empty_ts_when_no_data_in_request_period(self):
utc_calendar = Calendar()
netcdf_repository = self._construct_from_test_data()
self.assertIsNotNone(netcdf_repository)
utc_period = UtcPeriod(
utc_calendar.time(2017, 1, 1, 0, 0,
0), # a period where there is no data in
utc_calendar.time(2020, 12, 31, 0, 0, 0)) # the file supplied
type_source_map = dict()
type_source_map['temperature'] = TemperatureSource
geo_ts_dict = netcdf_repository.get_timeseries(
type_source_map, geo_location_criteria=None, utc_period=utc_period)
self.assertIsNotNone(geo_ts_dict)
def test_windowed_percentiles_tsv_empty(self) -> None:
"""Test that an empty TsVector is generated by windowed_percentiles_tsv
when given an empty sequence of percentiles."""
cal = Calendar()
period = UtcPeriod(cal.time(2017, 1, 1), cal.time(2018, 1, 1))
data = np.linspace(-2, 2, 24 * 7)
data_ts = TimeSeries(TimeAxis(0, Calendar.HOUR, len(data)), data,
POINT_INSTANT_VALUE)
# compute
tsv = windowed_percentiles_tsv(data_ts, period, Calendar.HOUR,
Calendar.HOUR, [], self.client, cal)
self.assertEqual(len(tsv), 0)
def test_get_timeseries_using_known_service_and_db_content(self):
utc = Calendar() # always use Calendar() stuff
met_stations = [ # this is the list of MetStations, the gis_id tells the position, the remaining tells us what properties we observe/forecast/calculate at the metstation (smg-ts)
MetStationConfig(
gis_id=598,
temperature=u'/NeNi-Sylsjøen......-T0017V3KI0114',
precipitation=u'/NeNi-Sylsjøen-2....-T0000D9BI0124'),
MetStationConfig(
gis_id=574,
temperature=u'/NeNi-Stuggusjøen...-T0017V3KI0114',
precipitation=u'/NeNi-Stuggusjøen...-T0000D9BI0124',
radiation=
u'/ENKI/STS/Radiation/Sim.-Stuggusjøen...-T0006A0B-0119')
]
#note: the MetStationConfig can be constructed from yaml-config
gis_location_repository = GisLocationService(
) # this provides the gis locations for my stations
smg_ts_repository = SmGTsRepository(
PROD,
FC_PROD) # this provide the read function for my time-series
geo_ts_repository = GeoTsRepository(
epsg_id=32633,
geo_location_repository=gis_location_repository,
ts_repository=smg_ts_repository,
met_station_list=met_stations,
ens_config=None) #pass service info and met_stations
self.assertIsNotNone(geo_ts_repository)
utc_period = UtcPeriod(utc.time(YMDhms(2010, 1, 1, 0, 0, 0)),
utc.time(YMDhms(2010, 1, 2, 0, 0, 0)))
ts_types = ['temperature', 'precipitation', 'radiation']
geo_ts_dict = geo_ts_repository.get_timeseries(
ts_types, utc_period=utc_period, geo_location_criteria=None)
self.assertIsNotNone(geo_ts_dict)
for ts_type in ts_types:
self.assertTrue(
ts_type in geo_ts_dict.keys(),
"we ecpect to find an entry for each requested type (it could be empty list though"
)
self.assertTrue(
len(geo_ts_dict[ts_type]) > 0,
"we expect to find the series that we pass in, given they have not changed the name in SmG PROD"
)
def dtss_read_callback(self, ts_ids: StringVector,
read_period: UtcPeriod) -> TsVector:
self.callback_count += 1
r = TsVector()
ta = TimeAxis(read_period.start, deltahours(1),
int(read_period.timespan() // deltahours(1)))
if self.rd_throws:
self.rd_throws = False
raise RuntimeError("read-ts-problem")
for ts_id in ts_ids:
r.append(
TimeSeries(ta,
fill_value=1.0,
point_fx=point_fx.POINT_AVERAGE_VALUE))
if self.cache_reads and self.cache_dtss: # illustrate how the read-callback can ask the dtss to cache it's reads
self.cache_dtss.cache(ts_ids, r)
return r
def test_get_ensemble_forecast_using_known_service_and_db_content(self):
utc = Calendar()
met_stations = [MetStationConfig(gis_id=402, temperature=u'/LTM5-Nea...........-T0017A3P_EC00_ENS', precipitation=u'/LTM5-Nea...........-T0000A5P_EC00_ENS')]
gis_location_repository = GisLocationService()
smg_ts_repository = SmGTsRepository(PREPROD, FC_PREPROD)
n_ensembles = 51
ens_station_list = [
EnsembleStation(402, n_ensembles,
temperature_ens = lambda i:u'/LTM5-Nea...........-T0017A3P_EC00_E{0:02}'.format(i),
precipitation_ens = lambda i:u'/LTM5-Nea...........-T0000A5P_EC00_E{0:02}'.format(i),
wind_speed_ens = None,
radiation_ens = None,
relative_humidity_ens = None
),
EnsembleStation(460, n_ensembles,
temperature_ens = lambda i:u'/LTM5-Tya...........-T0017A3P_EC00_E{0:02}'.format(i),
precipitation_ens = lambda i:u'/LTM5-Tya...........-T0000A5P_EC00_E{0:02}'.format(i),
wind_speed_ens = None,
radiation_ens = None,
relative_humidity_ens = None
)
]
ens_config = EnsembleConfig(n_ensembles, ens_station_list)
geo_ts_repository = GeoTsRepository(
epsg_id = 32633,
geo_location_repository = gis_location_repository,
ts_repository = smg_ts_repository,
met_station_list = met_stations,
ens_config = ens_config)
self.assertIsNotNone(geo_ts_repository)
utc_period = UtcPeriod(utc.time(YMDhms(2015, 10, 1, 0, 0, 0)),utc.time(YMDhms(2015, 10, 10, 0, 0, 0)))
ts_types= ['temperature', 'precipitation']
ens_geo_ts_dict = geo_ts_repository.get_forecast_ensemble(ts_types, utc_period=utc_period, t_c=None, geo_location_criteria=None)
self.assertIsNotNone(ens_geo_ts_dict)
self.assertEqual(ens_config.n_ensembles, len(ens_geo_ts_dict))
for i in range(ens_config.n_ensembles):
for ts_type in ts_types:
self.assertTrue(ts_type in ens_geo_ts_dict[i].keys(), "we expect to find an entry for each requested type (it could be empty list though)")
self.assertTrue(len(ens_geo_ts_dict[i][ts_type])>0, "we expect to find the series that we pass in by name in SmG PREPROD")
def __init__(self, *args, **kwargs):
super(DtssTestCase, self).__init__(*args, **kwargs)
self.callback_count = 0
self.find_count = 0
self.ts_infos = TsInfoVector()
self.rd_throws = False
self.cache_reads = False
self.cache_dtss = None
utc = Calendar()
t_now = utctime_now()
self.stored_tsv = list()
for i in range(30):
self.ts_infos.append(
TsInfo(name=fake_store_url('{0}'.format(i)),
point_fx=point_fx.POINT_AVERAGE_VALUE,
delta_t=deltahours(1),
olson_tz_id='',
data_period=UtcPeriod(utc.time(2017, 1, 1),
utc.time(2018, 1, 1)),
created=t_now,
modified=t_now))
def test_get_forecast_using_known_service_and_db_content(self):
utc = Calendar()
met_stations=[ # this is the list of MetStations, the gis_id tells the position, the remaining tells us what properties we observe/forecast/calculate at the metstation (smg-ts)
MetStationConfig(gis_id=402, temperature=u'/LTM5-Nea...........-T0017A3P_EC00_ENS', precipitation=u'/LTM5-Nea...........-T0000A5P_EC00_ENS')
]
gis_location_repository = GisLocationService()
smg_ts_repository = SmGTsRepository(PREPROD, FC_PREPROD)
geo_ts_repository = GeoTsRepository(
epsg_id = 32633,
geo_location_repository = gis_location_repository,
ts_repository = smg_ts_repository,
met_station_list = met_stations,
ens_config = None)
self.assertIsNotNone(geo_ts_repository)
utc_period = UtcPeriod(utc.time(YMDhms(2015, 10, 1, 0, 0, 0)), utc.time(YMDhms(2015, 10, 10, 0, 0, 0)))
ts_types= ['temperature', 'precipitation']
geo_ts_dict = geo_ts_repository.get_forecast(ts_types, utc_period=utc_period, t_c=None, geo_location_criteria=None)
self.assertIsNotNone(geo_ts_dict)
for ts_type in ts_types:
self.assertTrue(ts_type in geo_ts_dict.keys(), "we expect to find an entry for each requested type (it could be empty list though)")
self.assertTrue(len(geo_ts_dict[ts_type])>0, "we expect to find the series that we pass in by name in SmG PREPROD")
def test_selector_ts(self) -> None:
"""Test that selector_ts constructs a time-series selects data from different time-series correctly."""
n = 24
cal = Calendar()
period = UtcPeriod(0, n * Calendar.HOUR)
data_ts = TimeSeries(TimeAxis(0, Calendar.HOUR, n),
np.linspace(-10, 10, n), POINT_INSTANT_VALUE)
source_tss = [
TimeSeries(TimeAxis(0, Calendar.HOUR, n), 1.00 * np.ones(n),
POINT_INSTANT_VALUE),
TimeSeries(TimeAxis(0, Calendar.HOUR, n), 10.0 * np.ones(n),
POINT_INSTANT_VALUE),
TimeSeries(TimeAxis(0, Calendar.HOUR, n), 100. * np.ones(n),
POINT_INSTANT_VALUE),
]
threshold_1 = -5
threshold_2 = 5
threshold_tss = [
TimeSeries(TimeAxis(0, Calendar.HOUR, n), threshold_1 * np.ones(n),
POINT_INSTANT_VALUE),
TimeSeries(TimeAxis(0, Calendar.HOUR, n), threshold_2 * np.ones(n),
POINT_INSTANT_VALUE),
]
ts = selector_ts(data_ts, period, 2 * Calendar.HOUR, threshold_tss,
source_tss, POINT_AVERAGE_VALUE, self.client, cal)
self.assertEqual(len(data_ts), len(ts))
for dv, rv in zip(data_ts.values, ts.values):
if dv < threshold_1:
self.assertEqual(rv, source_tss[0].values[0])
elif threshold_1 <= dv < threshold_2:
self.assertEqual(rv, source_tss[1].values[0])
else:
self.assertEqual(rv, source_tss[2].values[0])
def test_construct_repository(self):
utc_calendar = Calendar()
netcdf_repository = self._construct_from_test_data()
self.assertIsNotNone(netcdf_repository)
utc_period = UtcPeriod(
utc_calendar.time(
2005, 1, 1, 0, 20,
0), # make it a challenge! ensure we get the first hour
utc_calendar.time(2014, 12, 30, 0, 20,
0)) # and also, we should have the last hour!
type_source_map = dict()
type_source_map['temperature'] = TemperatureSource
geo_ts_dict = netcdf_repository.get_timeseries(
type_source_map, geo_location_criteria=None, utc_period=utc_period)
self.assertIsNotNone(geo_ts_dict)
temperature_source = geo_ts_dict['temperature']
self.assertIsNotNone(temperature_source)
self.assertLessEqual(
temperature_source[0].ts.time_axis.time(0), utc_period.start,
'expect returned time-axis to cover the requested period')
self.assertGreaterEqual(
temperature_source[0].ts.time_axis.total_period().end,
utc_period.end,
'expected returned time-axis to cover requested period')
def test_construct_wrong_keyword_raises(self):
""" force user to supply at least one criteria """
with self.assertRaises(ForecastSelectionCriteriaError):
fc = ForecastSelectionCriteria(
any_key_word_that_is_wrong=UtcPeriod())
def test_can_create_cf_compliant_file(self):
# create files
test_file = path.join(path.abspath(os.curdir), 'shyft_test.nc')
if path.exists(test_file):
os.remove(test_file)
# create meta info
epsg_id = 32633
x0 = 100000
x1 = 200000
y0 = 100000
y1 = 200000
x = 101000
y = 101000
z = 1200
temperature = TimeSeriesMetaInfo('temperature',
'/observed/at_stn_abc/temperature',
'observed air temperature', x, y, z,
epsg_id)
# create time axis
utc = Calendar()
ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 24)
data = np.arange(0, ta.size(), dtype=np.float64)
ts = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the first batch
t_ds = TimeSeriesStore(test_file, temperature)
t_ds.create_new_file()
t_ds.append_ts_data(ts)
# expected result
ts_exp = ts
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# Append data
print("\n\n append at the end data")
# create time axis
ta = TimeAxis(utc.time(2016, 1, 2), deltahours(1), 48)
ts = TimeSeries(ta,
dv.from_numpy(np.arange(0, ta.size(),
dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the data
t_ds.append_ts_data(ts)
# expected result
ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 72)
data = np.empty(72)
data[:24] = np.arange(0, 24, dtype=np.float64)
data[24:72] = np.arange(0, 48, dtype=np.float64) # <-- new data
ts_exp = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# Append with overlap
print("\n\n append with overlap")
# create time axis
ta = TimeAxis(utc.time(2016, 1, 3), deltahours(1), 48)
ts = TimeSeries(ta,
dv.from_numpy(np.arange(0, ta.size(),
dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the data
t_ds.append_ts_data(ts)
# expected result
ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 96)
data = np.empty(96)
data[:24] = np.arange(0, 24, dtype=np.float64)
data[24:48] = np.arange(0, 24, dtype=np.float64) # <-- new data
data[48:96] = np.arange(0, 48, dtype=np.float64) # <-- new data
ts_exp = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# Append with gap in time axis
print("\n\n Append with gap in time axis")
# create time axis
ta = TimeAxis(utc.time(2016, 1, 6), deltahours(1), 24)
ts = TimeSeries(ta,
dv.from_numpy(np.arange(0, ta.size(),
dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the data
t_ds.append_ts_data(ts)
# expected result
time_vals = np.append(
TimeAxis(utc.time(2016, 1, 1), deltahours(1), 96).time_points[:-1],
ta.time_points)
# print(time_vals)
ta = TimeAxis(UtcTimeVector.from_numpy(time_vals.astype(np.int64)))
data = np.empty(120)
data[:24] = np.arange(0, 24, dtype=np.float64)
data[24:48] = np.arange(0, 24, dtype=np.float64)
data[48:96] = np.arange(0, 48, dtype=np.float64)
data[96:120] = np.arange(0, 24, dtype=np.float64) # <-- new data
ts_exp = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
# print(ts_exp.total_period())
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
# print(geo_temperature[0].ts.time_axis.time_points - ts_exp.time_axis.time_points)
# print(geo_temperature[0].ts.time_axis.time_points - time_vals)
# print(ts_exp.time_axis.time_points - time_vals)
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# Add new data in the middle where nothing was defined (no moving)
print(
"\n\n Add new data in the middle where nothing was defined (no moving)"
)
# create time axis
ta = TimeAxis(utc.time(2016, 1, 2), deltahours(1), 24)
ts = TimeSeries(ta,
dv.from_numpy(
np.arange(100, 100 + ta.size(), dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the data
t_ds.append_ts_data(ts)
# expected result
time_vals = np.append(
TimeAxis(utc.time(2016, 1, 1), deltahours(1), 96).time_points[:-1],
TimeAxis(utc.time(2016, 1, 6), deltahours(1), 24).time_points)
ta = TimeAxis(UtcTimeVector.from_numpy(time_vals.astype(np.int64)))
data = np.empty(120)
data[:24] = np.arange(0, 24, dtype=np.float64)
data[24:48] = np.arange(100, 124, dtype=np.float64) # <-- new data
data[48:96] = np.arange(0, 48, dtype=np.float64)
data[96:120] = np.arange(0, 24, dtype=np.float64)
ts_exp = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# print(ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# Insert new data in the middle and move rest
print("\n\n insert new data and move rest")
# create time axis
ta = TimeAxis(utc.time(2016, 1, 5), deltahours(1), 36)
ts = TimeSeries(ta,
dv.from_numpy(
np.arange(200, 200 + ta.size(), dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the data
t_ds.append_ts_data(ts)
# expected result
ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 144)
data = np.empty(144)
data[:24] = np.arange(0, 24, dtype=np.float64)
data[24:48] = np.arange(100, 124, dtype=np.float64)
data[48:96] = np.arange(0, 48, dtype=np.float64)
data[96:132] = np.arange(200, 236, dtype=np.float64) # <-- new data
data[132:144] = np.arange(12, 24, dtype=np.float64)
ts_exp = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# Add new data before existing data without overlap
print("\n\n add new data before existing data without overlap")
# create time axis
ta = TimeAxis(utc.time(2015, 12, 31), deltahours(1), 24)
ts = TimeSeries(ta,
dv.from_numpy(
np.arange(300, 300 + ta.size(), dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the first batch
t_ds.append_ts_data(ts)
# expected result
ta = TimeAxis(utc.time(2015, 12, 31), deltahours(1), 168)
data = np.empty(168)
data[:24] = np.arange(300, 324, dtype=np.float64) # <-- new data
data[24:48] = np.arange(0, 24, dtype=np.float64)
data[48:72] = np.arange(100, 124, dtype=np.float64)
data[72:120] = np.arange(0, 48, dtype=np.float64)
data[120:156] = np.arange(200, 236, dtype=np.float64)
data[156:168] = np.arange(12, 24, dtype=np.float64)
ts_exp = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# add new data before existing data with overlap
print("\n\n add new data before existing data with overlap")
# create time axis
ta = TimeAxis(utc.time(2015, 12, 30), deltahours(1), 36)
ts = TimeSeries(ta,
dv.from_numpy(
np.arange(400, 400 + ta.size(), dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the first batch
# t_ds = TimeSeriesStore(test_file, temperature)
t_ds.append_ts_data(ts)
# expected result
ta = TimeAxis(utc.time(2015, 12, 30), deltahours(1), 192)
data = np.empty(192)
data[:36] = np.arange(400, 436, dtype=np.float64) # <-- new data
data[36:48] = np.arange(312, 324, dtype=np.float64)
data[48:72] = np.arange(0, 24, dtype=np.float64)
data[72:96] = np.arange(100, 124, dtype=np.float64)
data[96:144] = np.arange(0, 48, dtype=np.float64)
data[144:180] = np.arange(200, 236, dtype=np.float64)
data[180:192] = np.arange(12, 24, dtype=np.float64)
ts_exp = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# Overwrite everything with less data points
# create time axis
print('\n\n Overwrite everything with less data points')
ta = TimeAxis(utc.time(2015, 12, 30), deltahours(24), 9)
ts = TimeSeries(ta,
dv.from_numpy(
np.arange(1000, 1000 + ta.size(),
dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# write the time series
t_ds.append_ts_data(ts)
# expected result
ts_exp = ts
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# Insert data with different dt
# create time axis
print('\n\n Insert data with different dt')
ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 24)
ts = TimeSeries(ta,
dv.from_numpy(np.arange(0, 24, dtype=np.float64)),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# write the time series
t_ds.append_ts_data(ts)
# expected result
time_points = np.empty(33, dtype=np.int)
time_points[0:2] = TimeAxis(utc.time(2015, 12, 30), deltahours(24),
1).time_points
time_points[2:26] = TimeAxis(utc.time(2016, 1, 1), deltahours(1),
23).time_points
time_points[26:] = TimeAxis(utc.time(2016, 1, 2), deltahours(24),
6).time_points
ta = TimeAxis(UtcTimeVector.from_numpy(time_points))
data = np.empty(32)
data[0:2] = np.array([1000, 1001])
data[2:26] = np.arange(0, 24) # <-- new data
data[26:] = np.arange(1003, 1009)
ts_exp = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# delete data with range UtcPeriod in the middle
print('\n\n delete data with range UtcPeriod')
tp = UtcPeriod(utc.time(2015, 12, 31), utc.time(2016, 1, 1, 12))
# ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 24)
# ts = TimeSeries(ta, dv.from_numpy(np.arange(0, 24, dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE)
# write the time series
t_ds.remove_tp_data(tp)
# expected result
time_points = np.array([
1451433600, 1451653200, 1451656800, 1451660400, 1451664000,
1451667600, 1451671200, 1451674800, 1451678400, 1451682000,
1451685600, 1451689200, 1451692800, 1451779200, 1451865600,
1451952000, 1452038400, 1452124800, 1452211200
])
ta = TimeAxis(UtcTimeVector.from_numpy(time_points))
data = np.array([
1000, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 1003, 1004, 1005,
1006, 1007, 1008
])
ts_exp = TimeSeries(ta, dv.from_numpy(data),
point_fx.POINT_INSTANT_VALUE
) # TODO: is this correct policy to use
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# delete data with range UtcPeriod at the start
print('\n\n delete data with range UtcPeriod at the start')
tp = UtcPeriod(1451433600, 1451667600)
# ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 24)
# ts = TimeSeries(ta, dv.from_numpy(np.arange(0, 24, dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE)
# write the time series
t_ds.remove_tp_data(tp)
# expected result
time_points = np.array([
1451671200, 1451674800, 1451678400, 1451682000, 1451685600,
1451689200, 1451692800, 1451779200, 1451865600, 1451952000,
1452038400, 1452124800, 1452211200
])
ta = TimeAxis(UtcTimeVector.from_numpy(time_points))
data = np.array(
[18, 19, 20, 21, 22, 23, 1003, 1004, 1005, 1006, 1007, 1008])
ts_exp = TimeSeries(
ta, dv.from_numpy(data), point_fx.POINT_INSTANT_VALUE
) # TODO: is this correct policy to use for this test
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# delete data with range UtcPeriod at the end
print('\n\n delete data with range UtcPeriod at the end')
tp = UtcPeriod(1451952000, utc.time(2016, 1, 10))
# ta = TimeAxis(utc.time(2016, 1, 1), deltahours(1), 24)
# ts = TimeSeries(ta, dv.from_numpy(np.arange(0, 24, dtype=np.float64)), point_fx=point_fx.POINT_AVERAGE_VALUE)
# write the time series
t_ds.remove_tp_data(tp)
# expected result
time_points = np.array([
1451671200, 1451674800, 1451678400, 1451682000, 1451685600,
1451689200, 1451692800, 1451779200, 1451865600, 1451952000
])
ta = TimeAxis(UtcTimeVector.from_numpy(time_points))
data = np.array([18, 19, 20, 21, 22, 23, 1003, 1004, 1005])
ts_exp = TimeSeries(ta, dv.from_numpy(data),
point_fx.POINT_INSTANT_VALUE)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
try:
rts_map = ts_dr.get_timeseries(['temperature'],
ts_exp.total_period())
except CFDataRepositoryError:
pass
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# delete data with range UtcPeriod everything
print('\n\n delete data with range UtcPeriod everything')
tp = UtcPeriod(utc.time(2016, 1, 1), utc.time(2016, 1, 10))
# write the time series
t_ds.remove_tp_data(tp)
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
self.assertRaises(CFDataRepositoryError, ts_dr.get_timeseries,
['temperature'], tp)
# --------------------------------------
# insert data in between time saved data points
print('\n\n insert data in between time saved data points')
# insert first data in which we want to insert the second batch
utc = Calendar()
ta = TimeAxis(utc.time(2016, 1, 1), deltahours(24), 2)
data = np.arange(0, ta.size(), dtype=np.float64)
ts = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the first batch
t_ds.append_ts_data(ts)
# insert first data for every hour in between
utc = Calendar()
ta = TimeAxis(utc.time(2016, 1, 1) + deltahours(1), deltahours(1), 23)
data = np.arange(10, 10 + ta.size(), dtype=np.float64)
ts = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the first batch
t_ds.append_ts_data(ts)
# expected result
time_points = np.array([
1451606400, 1451610000, 1451613600, 1451617200, 1451620800,
1451624400, 1451628000, 1451631600, 1451635200, 1451638800,
1451642400, 1451646000, 1451649600, 1451653200, 1451656800,
1451660400, 1451664000, 1451667600, 1451671200, 1451674800,
1451678400, 1451682000, 1451685600, 1451689200, 1451692800, 0
])
time_points[-1] = 2 * time_points[-2] - time_points[
-3] # last time point calc
data = np.array([
0, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
26, 27, 28, 29, 30, 31, 32, 1
])
ta = TimeAxis(UtcTimeVector.from_numpy(time_points))
ts_exp = TimeSeries(
ta, dv.from_numpy(data), point_fx.POINT_INSTANT_VALUE
) # TODO: is this correct policy value for this case
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy()))
# --------------------------------------
# insert data including nan
print('\n\n insert data including nan')
utc = Calendar()
ta = TimeAxis(utc.time(2016, 1, 1) + deltahours(1), deltahours(1), 23)
data = np.arange(10, 10 + ta.size(), dtype=np.float64)
data[4] = np.nan
data[
6] = np.nan # np.inf, but trouble getting inf trough all version of numpy/netcdf
data[8] = np.nan # -np.inf, --"--
ts = TimeSeries(ta,
dv.from_numpy(data),
point_fx=point_fx.POINT_AVERAGE_VALUE)
# save the first batch
t_ds.append_ts_data(ts)
# expected result
time_points = np.array([
1451606400, 1451610000, 1451613600, 1451617200, 1451620800,
1451624400, 1451628000, 1451631600, 1451635200, 1451638800,
1451642400, 1451646000, 1451649600, 1451653200, 1451656800,
1451660400, 1451664000, 1451667600, 1451671200, 1451674800,
1451678400, 1451682000, 1451685600, 1451689200, 1451692800, 0
])
time_points[-1] = 2 * time_points[-2] - time_points[
-3] # last time point calc
data = np.array([
0,
10,
11,
12,
13,
np.nan,
15,
# np.inf,
np.
nan, # TODO: figure out how to unmask restoring 'used' mask-values
17,
#-np.inf,
np.nan,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
1
])
ta = TimeAxis(UtcTimeVector.from_numpy(time_points))
ts_exp = TimeSeries(
ta, dv.from_numpy(data),
point_fx.POINT_INSTANT_VALUE) # TODO: policy right ?
# now read back the result using a *standard* shyft cf geo repository
selection_criteria = {'bbox': [[x0, x1, x1, x0], [y0, y0, y1, y1]]}
ts_dr = CFDataRepository(epsg_id, test_file, selection_criteria)
# now read back 'temperature' that we know should be there
rts_map = ts_dr.get_timeseries(['temperature'], ts_exp.total_period())
# and verify that we get exactly back what we wanted.
self.assertIsNotNone(rts_map)
self.assertTrue('temperature' in rts_map)
geo_temperature = rts_map['temperature']
self.assertEqual(len(geo_temperature), 1)
self.assertLessEqual(
GeoPoint.distance2(geo_temperature[0].mid_point(),
GeoPoint(x, y, z)), 1.0)
# check if time axis is as expected
self.assertEqual(geo_temperature[0].ts.time_axis, ts_exp.time_axis)
self.assertTrue(
np.allclose(geo_temperature[0].ts.time_axis.time_points,
ts_exp.time_axis.time_points))
self.assertEqual(geo_temperature[0].ts.point_interpretation(),
point_fx.POINT_AVERAGE_VALUE)
# check if variable data is as expected
self.assertTrue(
np.allclose(geo_temperature[0].ts.values.to_numpy(),
ts_exp.values.to_numpy(),
equal_nan=True))
if path.exists(test_file):
os.remove(test_file)
def test_get_ensemble_forecast_using_known_service_and_db_content(
self):
utc = Calendar() # always use Calendar() stuff
met_stations = [ # this is the list of MetStations, the gis_id tells the position, the remaining tells us what properties we observe/forecast/calculate at the metstation (smg-ts)
MetStationConfig(
gis_id=598,
temperature=u'/LTM5-Nea...........-T0017A3P_EC00_ENS',
precipitation=u'/LTM5-Nea...........-T0000A5P_EC00_ENS')
]
#note: the MetStationConfig can be constructed from yaml-config
gis_location_repository = GisLocationService(
) # this provides the gis locations for my stations
smg_ts_repository = SmGTsRepository(
PROD,
FC_PROD) # this provide the read function for my time-series
n_ensembles = 51
ens_station_list = [
EnsembleStation(
598,
n_ensembles,
temperature_ens=lambda i:
u'/LTM5-Nea...........-T0017A3P_EC00_E{0:02}'.format(i),
precipitation_ens=lambda i:
u'/LTM5-Nea...........-T0000A5P_EC00_E{0:02}'.format(i),
wind_speed_ens=None,
radiation_ens=None,
relative_humidity_ens=None),
EnsembleStation(
574,
n_ensembles,
temperature_ens=lambda i:
u'/LTM5-Tya...........-T0017A3P_EC00_E{0:02}'.format(i),
precipitation_ens=lambda i:
u'/LTM5-Tya...........-T0000A5P_EC00_E{0:02}'.format(i),
wind_speed_ens=None,
radiation_ens=None,
relative_humidity_ens=None)
]
ens_config = EnsembleConfig(n_ensembles, ens_station_list)
geo_ts_repository = GeoTsRepository(
epsg_id=32633,
geo_location_repository=gis_location_repository,
ts_repository=smg_ts_repository,
met_station_list=met_stations,
ens_config=ens_config) #pass service info and met_stations
self.assertIsNotNone(geo_ts_repository)
utc_period = UtcPeriod(utc.time(YMDhms(2015, 10, 1, 0, 0, 0)),
utc.time(YMDhms(2015, 10, 10, 0, 0, 0)))
ts_types = ['temperature', 'precipitation']
ens_geo_ts_dict = geo_ts_repository.get_forecast_ensemble(
ts_types,
utc_period=utc_period,
t_c=None,
geo_location_criteria=None)
self.assertIsNotNone(ens_geo_ts_dict)
self.assertEqual(ens_config.n_ensembles, len(ens_geo_ts_dict))
for i in range(ens_config.n_ensembles):
for ts_type in ts_types:
self.assertTrue(
ts_type in ens_geo_ts_dict[i].keys(),
"we ecpect to find an entry for each requested type (it could be empty list though"
)
self.assertTrue(
len(ens_geo_ts_dict[i][ts_type]) > 0,
"we expect to find the series that we pass in, given they have not changed the name in SmG PROD"
)