def _convert_to_timeseries(self, data):
"""Convert timeseries from numpy structures to shyft.api timeseries.
We assume the time axis is regular, and that we can use a point time
series with a parametrized time axis definition and corresponding
vector of values. If the time series is missing on the data, we insert
it into non_time_series.
Returns
-------
timeseries: dict
Time series arrays keyed by type
"""
tsc = api.TsFactory().create_point_ts
time_series = {}
for key, (data, ta) in data.items():
def construct(d):
if ta.size() != d.size:
raise WRFDataRepositoryError(
"Time axis size {} not equal to the number of "
"data points ({}) for {}"
"".format(ta.size(), d.size, key))
return tsc(ta.size(), ta.start, ta.delta_t,
api.DoubleVector_FromNdArray(d),
api.point_interpretation_policy.POINT_AVERAGE_VALUE)
time_series[key] = np.array(
[construct(data[:, i]) for i in range(data.shape[1])])
return time_series
def _create_shyft_ts(self):
b = 946684800 # 2000.01.01 00:00:00
h = 3600 #one hour in seconds
values = np.array([1.0, 2.0, 3.0])
shyft_ts_factory = api.TsFactory()
return shyft_ts_factory.create_point_ts(len(values), b, h,
api.DoubleVector(values))
def test_store(self):
ds = SmGTsRepository(PREPROD)
nl = [u'/shyft/test/a', u'/shyft/test/b',
u'/shyft/test/c'] #[u'/ICC-test-v9.2']
t0 = 946684800 # time_t/unixtime 2000.01.01 00:00:00
dt = 3600 #one hour in seconds
values = np.array([1.0, 2.0, 3.0])
shyft_ts_factory = api.TsFactory()
shyft_result_ts = shyft_ts_factory.create_point_ts(
len(values), t0, dt, api.DoubleVector(values))
shyft_catchment_result = dict()
shyft_catchment_result[nl[0]] = shyft_result_ts
shyft_catchment_result[nl[1]] = shyft_result_ts
shyft_catchment_result[nl[2]] = shyft_result_ts
r = ds.store(shyft_catchment_result)
self.assertEqual(r, True)
# now read back the ts.. and verify it's there..
read_period = api.UtcPeriod(t0, t0 + 3 * dt)
rts_list = ds.read(nl, read_period)
self.assertIsNotNone(rts_list)
c2 = rts_list[nl[-1]]
[
self.assertAlmostEqual(c2.value(i), values[i])
for i in range(len(values))
]
def _convert_to_timeseries(self, data):
"""Convert timeseries from numpy structures to shyft.api timeseries.
We assume the time axis is regular, and that we can use a point time
series with a parametrized time axis definition and corresponding
vector of values. If the time series is missing on the data, we insert
it into non_time_series.
Returns
-------
timeseries: dict
Time series arrays keyed by type
"""
tsc = api.TsFactory().create_point_ts
time_series = {}
for key, (data, ta) in data.items():
fslice = (len(data.shape) - 2) * [slice(None)]
I, J = data.shape[-2:]
def construct(d):
if ta.size() != d.size:
raise EcDataRepositoryError(
"Time axis size {} not equal to the number of "
"data points ({}) for {}"
"".format(ta.size(), d.size, key))
return api.TimeSeries(
ta, api.DoubleVector_FromNdArray(d.flatten()),
self.series_type[key])
time_series[key] = np.array(
[[construct(data[fslice + [i, j]]) for j in range(J)]
for i in range(I)])
return time_series
def _convert_to_timeseries(self, data, concat):
"""Convert timeseries from numpy structures to shyft.api timeseries.
Returns
-------
timeseries: dict
Time series arrays keyed by type
"""
tsf = api.TsFactory().create_point_ts
tsc = api.TimeSeries
time_series = {}
if concat:
for key, (data, ta) in data.items():
nb_timesteps, nb_pts = data.shape
def construct(d):
if ta.size() != d.size:
raise ECConcatDataRepositoryError("Time axis size {} not equal to the number of "
"data points ({}) for {}"
"".format(ta.size(), d.size, key))
return tsf(ta.size(), ta.start, ta.delta_t,
api.DoubleVector.FromNdArray(d.flatten()), self.series_type[key])
time_series[key] = np.array([construct(data[:, j]) for j in range(nb_pts)])
else:
def construct(d, tax):
if tax.size() != d.size:
raise ECConcatDataRepositoryError("Time axis size {} not equal to the number of "
"data points ({}) for {}"
"".format(tax.size(), d.size, key))
return tsc(tax, api.DoubleVector.FromNdArray(d.flatten()), self.series_type[key])
for key, (data, ta) in data.items():
nb_forecasts, nb_timesteps, nb_pts = data.shape
time_series[key] = np.array([construct(data[i, :, j], ta[i]) for i in range(nb_forecasts) for j in range(nb_pts)])
return time_series
def _convert_to_timeseries(self, data, t, ts_id):
ta = api.TimeAxisFixedDeltaT(int(t[0]), int(t[1]) - int(t[0]), len(t))
tsc = api.TsFactory().create_point_ts
def construct(d):
return tsc(ta.size(), ta.start, ta.delta_t,
api.DoubleVector.FromNdArray(d))
ts = [construct(data[:,j]) for j in range(data.shape[-1])]
return {k:v for k, v in zip(ts_id,ts)}
def test_vector_of_timeseries(self):
dv = np.arange(self.ta.size())
v = api.DoubleVector.from_numpy(dv)
tsf = api.TsFactory();
tsa = tsf.create_point_ts(self.n, self.t, self.d, v)
tsvector = api.TsVector()
self.assertEqual(len(tsvector), 0)
tsvector.push_back(tsa)
self.assertEqual(len(tsvector), 1)
def _create_constant_geo_ts(self, geo_ts_type, geo_point, utc_period, value):
"""Create a time point ts, with one value at the start
of the supplied utc_period."""
tv = api.UtcTimeVector()
tv.push_back(utc_period.start)
vv = api.DoubleVector()
vv.push_back(value)
cts = api.TsFactory().create_time_point_ts(utc_period, tv, vv, api.POINT_AVERAGE_VALUE)
return geo_ts_type(geo_point, cts)
def _make_shyft_ts_from_xts(xts):
if not isinstance(xts, ITimeSeries):
raise SmgDataError("Supplied xts should be of type ITimeSeries")
t = api.UtcTimeVector()
v = api.DoubleVector()
for i in range(xts.Count):
t.push_back(xts.Time(i).ToUnixTime())
v.push_back(xts.Value(i).V)
shyft_ts = api.TsFactory().create_time_point_ts(api.UtcPeriod(t[0], t[-1]), t, v)
return shyft_ts
def test_create_TargetSpecificationPts(self):
t = api.TargetSpecificationPts();
t.scale_factor = 1.0
t.calc_mode = api.NASH_SUTCLIFFE
t.calc_mode = api.KLING_GUPTA;
t.s_r = 1.0 # KGEs scale-factors
t.s_a = 2.0
t.s_b = 3.0
self.assertAlmostEqual(t.scale_factor, 1.0)
# create a ts with some points
cal = api.Calendar();
start = cal.time(api.YMDhms(2015, 1, 1, 0, 0, 0))
dt = api.deltahours(1)
tsf = api.TsFactory();
times = api.UtcTimeVector()
times.push_back(start + 1 * dt);
times.push_back(start + 3 * dt);
times.push_back(start + 4 * dt)
values = api.DoubleVector()
values.push_back(1.0)
values.push_back(3.0)
values.push_back(np.nan)
tsp = tsf.create_time_point_ts(api.UtcPeriod(start, start + 24 * dt), times, values)
# convert it from a time-point ts( as returned from current smgrepository) to a fixed interval with timeaxis, needed by calibration
tst = api.TsTransform()
tsa = tst.to_average(start, dt, 24, tsp)
# tsa2 = tst.to_average(start,dt,24,tsp,False)
# tsa_staircase = tst.to_average_staircase(start,dt,24,tsp,False) # nans infects the complete interval to nan
# tsa_staircase2 = tst.to_average_staircase(start,dt,24,tsp,True) # skip nans, nans are 0
# stuff it into the target spec.
# also show how to specify snow-calibration
cids = api.IntVector([0, 2, 3])
t2 = api.TargetSpecificationPts(tsa,cids, 0.7, api.KLING_GUPTA, 1.0, 1.0, 1.0, api.SNOW_COVERED_AREA)
t2.catchment_property = api.SNOW_WATER_EQUIVALENT
self.assertEqual(t2.catchment_property, api.SNOW_WATER_EQUIVALENT)
self.assertIsNotNone(t2.catchment_indexes)
for i in range(len(cids)):
self.assertEqual(cids[i],t2.catchment_indexes[i])
t.ts = tsa
#TODO: Does not work, list of objects are not yet convertible tv = api.TargetSpecificationVector([t, t2])
tv=api.TargetSpecificationVector()
tv.append(t)
tv.append(t2)
# now verify we got something ok
self.assertEqual(2, tv.size())
self.assertAlmostEqual(tv[0].ts.value(1), 1.5) # average value 0..1 ->0.5
self.assertAlmostEqual(tv[0].ts.value(2), 2.5) # average value 0..1 ->0.5
self.assertAlmostEqual(tv[0].ts.value(3), 3.0) # average value 0..1 ->0.5
# and that the target vector now have its own copy of ts
tsa.set(1, 3.0)
self.assertAlmostEqual(tv[0].ts.value(1), 1.5) # make sure the ts passed onto target spec, is a copy
self.assertAlmostEqual(tsa.value(1), 3.0) # and that we really did change the source
def test_ts_factory(self):
dv=np.arange(self.ta.size())
v=api.DoubleVector.from_numpy(dv)
t=api.UtcTimeVector();
for i in range(self.ta.size()):
t.push_back(self.ta(i).start)
t.push_back(self.ta(self.ta.size() - 1).end)
tsf=api.TsFactory()
ts1=tsf.create_point_ts(self.ta.size(), self.t, self.d, v)
ts2=tsf.create_time_point_ts(self.ta.total_period(), t, v)
tslist=api.TsVector()
tslist.push_back(ts1)
tslist.push_back(ts2)
self.assertEqual(tslist.size(), 2)
def _make_shyft_ts_from_ssa_ts(ssa_ts):
if (not isinstance(ssa_ts, SsaTimeSeries)):
raise SmgDataError(
"supplied ssa_ts should be of type SsaTimeSeries")
tsv = ssa_ts.GetTsAsVector(TimeSystemReference.Unix1970Utc)
tsfactory = api.TsFactory()
#todo: this can be done much faster using clr direct accesss, https://mail.python.org/pipermail/pythondotnet/2014-May/001526.html
times = api.UtcTimeVector.FromNdArray(
np.fromiter(tsv.Time, dtype=np.long))
values = api.DoubleVector.FromNdArray(
np.fromiter(tsv.Value, dtype=np.float))
ts_period = api.UtcPeriod(tsv.TotalPeriod.Start.ToUnixTime(),
tsv.TotalPeriod.End.ToUnixTime())
shyft_ts = tsfactory.create_time_point_ts(ts_period, times, values)
return shyft_ts
def test_average_accessor(self):
dv = np.arange(self.ta.size())
v = api.DoubleVector.from_numpy(dv)
t = api.UtcTimeVector()
for i in range(self.ta.size()):
t.push_back(self.ta(i).start)
t.push_back(
self.ta(self.ta.size() - 1).end) # important! needs n+1 points to determine n periods in the timeaxis
tsf = api.TsFactory()
ts1 = tsf.create_point_ts(self.ta.size(), self.t, self.d, v)
ts2 = tsf.create_time_point_ts(self.ta.total_period(), t, v)
tax = api.Timeaxis(self.ta.total_period().start + api.deltaminutes(30), api.deltahours(1), self.ta.size())
avg1 = api.AverageAccessorTs(ts1, tax)
self.assertEqual(avg1.size(), tax.size())
self.assertIsNotNone(ts2)
def get_timeseries(self,
input_source_types,
utc_period,
geo_location_criteria=None):
"""Method for fetching the sources in NetCDF files.
Parameters
----------
input_source_types: list
List of source types to retrieve (precipitation, temperature..)
geo_location_criteria: bbox + proj.ref ?
utc_period : of type UtcPeriod
Returns
-------
data: dict
Shyft.api container for geo-located time series. Types are found from the
input_source_type.vector_t attribute.
"""
data = dict()
# Fill the data with actual values
for input_source in input_source_types:
api_source_type = self.source_type_map[input_source]
ts = self._fetch_station_tseries(input_source,
self._params['types'], utc_period)
assert type(ts) is list
tsf = api.TsFactory()
acc_data = api_source_type.vector_t()
for station in ts:
times = station['time']
assert type(times) is list
dt = times[1] - times[0] if len(times) > 1 else api.deltahours(
1)
total_period = api.UtcPeriod(times[0], times[-1] + dt)
time_points = api.UtcTimeVector(times)
time_points.push_back(total_period.end)
values = station['values']
value_points = api.DoubleVector.FromNdArray(values)
api_ts = tsf.create_time_point_ts(total_period, time_points,
value_points)
data_source = api_source_type(
api.GeoPoint(*station['location']), api_ts)
acc_data.append(data_source)
data[input_source] = acc_data
return data
def _convert_to_timeseries(self, data):
tsc = api.TsFactory().create_point_ts
time_series = {}
for key, (data, ta) in data.items():
fslice = (len(data.shape) - 2)*[slice(None)]
I, J = data.shape[-2:]
def construct(d):
if ta.size() != d.size:
raise ERAInterimDataRepositoryError("Time axis size {} not equal to the number of "
"data points ({}) for {}"
"".format(ta.size(), d.size, key))
return tsc(ta.size(), ta.start, ta.delta_t,
api.DoubleVector_FromNdArray(d.flatten()), self.series_type[key])
time_series[key] = np.array([[construct(data[fslice + [i, j]])
for j in range(J)] for i in range(I)])
return time_series
def test_vector_of_timeseries(self):
dv=np.arange(self.ta.size())
v=api.DoubleVector.from_numpy(dv)
tsf=api.TsFactory()
tsa=tsf.create_point_ts(self.n, self.t, self.d, v)
tsvector=api.TsVector()
self.assertEqual(len(tsvector), 0)
tsvector.push_back(tsa)
self.assertEqual(len(tsvector), 1)
tsvector.push_back(tsa)
vv=tsvector.values_at_time(self.ta.time(3)) # verify it's easy to get out vectorized results at time t
self.assertEqual(len(vv), len(tsvector))
self.assertAlmostEqual(vv[0], 3.0)
self.assertAlmostEqual(vv[1], 3.0)
ts_list=[tsa, tsa]
vv=api.ts_vector_values_at_time(ts_list, self.ta.time(4)) # also check it work with list(TimeSeries)
self.assertEqual(len(vv), len(tsvector))
self.assertAlmostEqual(vv[0], 4.0)
self.assertAlmostEqual(vv[1], 4.0)
def test_ts_transform(self):
dv=np.arange(self.ta.size())
v=api.DoubleVector.from_numpy(dv)
t=api.UtcTimeVector();
for i in range(self.ta.size()):
t.push_back(self.ta(i).start)
# t.push_back(self.ta(self.ta.size()-1).end) #important! needs n+1 points to determine n periods in the timeaxis
t_start=self.ta.total_period().start
dt=api.deltahours(1)
tax=api.TimeAxisFixedDeltaT(t_start + api.deltaminutes(30), dt, self.ta.size())
tsf=api.TsFactory()
ts1=tsf.create_point_ts(self.ta.size(), self.t, self.d, v)
ts2=tsf.create_time_point_ts(self.ta.total_period(), t, v)
ts3=api.TsFixed(tax, v, api.POINT_INSTANT_VALUE)
tst=api.TsTransform()
tt1=tst.to_average(t_start, dt, tax.size(), ts1)
tt2=tst.to_average(t_start, dt, tax.size(), ts2)
tt3=tst.to_average(t_start, dt, tax.size(), ts3)
self.assertEqual(tt1.size(), tax.size())
self.assertEqual(tt2.size(), tax.size())
self.assertEqual(tt3.size(), tax.size())
for var, (file_name, source_type, source_vec) in source_map.items():
nci = Dataset(
os.path.join(shyftdata_dir,
'netcdf/orchestration-testdata/' + file_name))
time = api.UtcTimeVector([int(t) for t in nci.variables['time'][:]])
delta_t = time[1] - time[0] if len(time) > 1 else api.deltahours(1)
for i in range(nci.dimensions['station'].size):
x = nci.variables['x'][i]
y = nci.variables['y'][i]
z = nci.variables['z'][i]
gp = api.GeoPoint(float(x), float(y), float(z))
data = nci.variables[var][:, i]
time_axis = api.TimeAxis(int(time[0]), delta_t, len(time))
dts = api.TsFactory().create_time_point_ts(time_axis.total_period(),
time, data,
api.POINT_AVERAGE_VALUE)
# add it to the variable source vector
source_vec.append(source_type(gp, dts))
nci.close()
# source data, which will be fed to the interpolation
region_environment = re
def plot_station_data(region_environment):
"""plot the data within each source vector of the 'ARegionEnvironment'
"""
for fv, sv in region_environment.variables:
n_stn = len(sv)
fig, ax = plt.subplots(figsize=(15, 5))
def test_create_TargetSpecificationPts(self):
t = api.TargetSpecificationPts()
t.scale_factor = 1.0
t.calc_mode = api.NASH_SUTCLIFFE
t.calc_mode = api.KLING_GUPTA
t.calc_mode = api.ABS_DIFF
t.calc_mode = api.RMSE
t.s_r = 1.0 # KGEs scale-factors
t.s_a = 2.0
t.s_b = 3.0
self.assertIsNotNone(t.uid)
t.uid = 'test'
self.assertEqual(t.uid, 'test')
self.assertAlmostEqual(t.scale_factor, 1.0)
# create a ts with some points
cal = api.Calendar()
start = cal.time(2015, 1, 1, 0, 0, 0)
dt = api.deltahours(1)
tsf = api.TsFactory()
times = api.UtcTimeVector()
times.push_back(start + 1 * dt)
times.push_back(start + 3 * dt)
times.push_back(start + 4 * dt)
values = api.DoubleVector()
values.push_back(1.0)
values.push_back(3.0)
values.push_back(np.nan)
tsp = tsf.create_time_point_ts(api.UtcPeriod(start, start + 24 * dt),
times, values)
# convert it from a time-point ts( as returned from current smgrepository) to a fixed interval with timeaxis, needed by calibration
tst = api.TsTransform()
tsa = tst.to_average(start, dt, 24, tsp)
# tsa2 = tst.to_average(start,dt,24,tsp,False)
# tsa_staircase = tst.to_average_staircase(start,dt,24,tsp,False) # nans infects the complete interval to nan
# tsa_staircase2 = tst.to_average_staircase(start,dt,24,tsp,True) # skip nans, nans are 0
# stuff it into the target spec.
# also show how to specify snow-calibration
cids = api.IntVector([0, 2, 3])
t2 = api.TargetSpecificationPts(tsa, cids, 0.7, api.KLING_GUPTA, 1.0,
1.0, 1.0, api.SNOW_COVERED_AREA,
'test_uid')
self.assertEqual(t2.uid, 'test_uid')
t2.catchment_property = api.SNOW_WATER_EQUIVALENT
self.assertEqual(t2.catchment_property, api.SNOW_WATER_EQUIVALENT)
t2.catchment_property = api.CELL_CHARGE
self.assertEqual(t2.catchment_property, api.CELL_CHARGE)
self.assertIsNotNone(t2.catchment_indexes)
for i in range(len(cids)):
self.assertEqual(cids[i], t2.catchment_indexes[i])
t.ts = api.TimeSeries(tsa) # target spec is now a regular TimeSeries
tv = api.TargetSpecificationVector()
tv[:] = [t, t2]
# now verify we got something ok
self.assertEqual(2, tv.size())
self.assertAlmostEqual(tv[0].ts.value(1),
1.5) # average value 0..1 ->0.5
self.assertAlmostEqual(tv[0].ts.value(2),
2.5) # average value 0..1 ->0.5
# self.assertAlmostEqual(tv[0].ts.value(3), 3.0) # original flat out at end, but now:
self.assertTrue(math.isnan(
tv[0].ts.value(3))) # strictly linear between points.
# and that the target vector now have its own copy of ts
tsa.set(1, 3.0)
self.assertAlmostEqual(
tv[0].ts.value(1),
1.5) # make sure the ts passed onto target spec, is a copy
self.assertAlmostEqual(tsa.value(1),
3.0) # and that we really did change the source
# Create a clone of target specification vector
tv2 = api.TargetSpecificationVector(tv)
self.assertEqual(2, tv2.size())
self.assertAlmostEqual(tv2[0].ts.value(1),
1.5) # average value 0..1 ->0.5
self.assertAlmostEqual(tv2[0].ts.value(2),
2.5) # average value 0..1 ->0.5
self.assertTrue(math.isnan(
tv2[0].ts.value(3))) # average value 0..1 ->0.5
tv2[0].scale_factor = 10.0
self.assertAlmostEqual(tv[0].scale_factor, 1.0)
self.assertAlmostEqual(tv2[0].scale_factor, 10.0)
# test we can create from breakpoint time-series
ts_bp = api.TimeSeries(api.TimeAxis(api.UtcTimeVector([0, 25, 20]),
30),
fill_value=2.0,
point_fx=api.POINT_AVERAGE_VALUE)
tspec_bp = api.TargetSpecificationPts(ts_bp, cids, 0.7,
api.KLING_GUPTA, 1.0, 1.0, 1.0,
api.CELL_CHARGE, 'test_uid')
self.assertIsNotNone(tspec_bp)
def _create_shyft_ts(self):
b = 946684800 # 2000.01.01 00:00:00
h = 3600 # One hour in seconds
v = np.array([1.0, 2.0, 3.0])
return api.TsFactory().create_point_ts(len(v), b, h,
api.DoubleVector(v))