def test_ts_store(self):
"""
This test verifies the shyft internal time-series store,
that allow identified time-series to be stored
in the backend using a directory container specified for the
location.
All time-series of the form shyft://<container>/<ts-name>
is mapped to the configured <container> (aka a directory on the server)
This applies to expressions, as well as the new
.store_ts(ts_vector) function that allows the user to
stash away time-series into the configured back-end container.
All find-operations of the form shyft://<container>/<regular-expression>
is mapped to a search in the corresponding directory for the <container>
:return:
"""
with tempfile.TemporaryDirectory() as c_dir:
# setup data to be calculated
utc = Calendar()
d = deltahours(1)
n = 365 * 24 // 3
t = utc.time(2016, 1, 1)
ta = TimeAxis(t, d, n)
n_ts = 10
store_tsv = TsVector() # something we store at server side
tsv = TsVector(
) # something we put an expression into, refering to stored ts-symbols
for i in range(n_ts):
pts = TimeSeries(
ta,
np.sin(np.linspace(start=0, stop=1.0 * i, num=ta.size())),
point_fx.POINT_AVERAGE_VALUE)
ts_id = shyft_store_url("{0}".format(i))
tsv.append(float(1.0) * TimeSeries(ts_id)
) # make an expression that returns what we store
store_tsv.append(TimeSeries(
ts_id, pts)) # generate a bound pts to store
# krls with some extra challenges related to serialization
tsv_krls = TsVector()
krls_ts = TimeSeries(shyft_store_url("9")).krls_interpolation(
dt=d, gamma=1e-3, tolerance=0.001, size=ta.size())
tsv_krls.append(krls_ts)
# min_max_check_ts_fill also needs a serial check
# create a trivial-case
ts9 = TimeSeries(shyft_store_url("9"))
ts_qac = ts9.min_max_check_linear_fill(v_min=-10.0 * n_ts,
v_max=10.0 * n_ts)
tsv_krls.append(ts_qac)
tsv_krls.append(ts9)
tsv_krls.append(ts9.inside(min_v=-0.5, max_v=0.5))
# then start the server
dtss = DtsServer()
port_no = find_free_port()
host_port = 'localhost:{0}'.format(port_no)
dtss.set_auto_cache(True)
std_max_items = dtss.cache_max_items
dtss.cache_max_items = 3000
tst_max_items = dtss.cache_max_items
dtss.set_listening_port(port_no)
dtss.set_container(
"test", c_dir
) # notice we set container 'test' to point to c_dir directory
dtss.start_async(
) # the internal shyft time-series will be stored to that container
# also notice that we dont have to setup callbacks in this case (but we could, and they would work)
#
# finally start the action
dts = DtsClient(host_port)
# then try something that should work
dts.store_ts(store_tsv)
r1 = dts.evaluate(tsv, ta.total_period())
f1 = dts.find(
r"shyft://test/\d") # find all ts with one digit, 0..9
r2 = dts.evaluate(tsv_krls, ta.total_period())
url_x = shyft_store_url(r'does not exists')
tsvx = TsVector()
tsvx.append(TimeSeries(url_x))
try:
rx = dts.evaluate(tsvx, ta.total_period())
self.assertFalse(True, 'This did not work out')
except RuntimeError as rex:
self.assertIsNotNone(rex)
dts.close() # close connection (will use context manager later)
dtss.clear() # close server
# now the moment of truth:
self.assertEqual(len(r1), len(tsv))
for i in range(n_ts - 1):
self.assertEqual(r1[i].time_axis, store_tsv[i].time_axis)
assert_array_almost_equal(r1[i].values.to_numpy(),
store_tsv[i].values.to_numpy(),
decimal=4)
self.assertEqual(len(f1), 10)
self.assertEqual(len(r2), len(tsv_krls))
assert_array_almost_equal(r2[1].values.to_numpy(),
r2[2].values.to_numpy(),
decimal=4)
self.assertEqual(1000000, std_max_items)
self.assertEqual(3000, tst_max_items)
def test_functionality_hosting_localhost(self):
# setup data to be calculated
utc = Calendar()
d = deltahours(1)
d24 = deltahours(24)
n = 240
n24 = 10
t = utc.time(2016, 1, 1)
ta = TimeAxis(t, d, n)
ta24 = TimeAxis(t, d24, n24)
n_ts = 100
percentile_list = IntVector([0, 35, 50, 65, 100])
tsv = TsVector()
store_tsv = TsVector() # something we store at server side
for i in range(n_ts):
pts = TimeSeries(ta, np.linspace(start=0, stop=1.0, num=ta.size()),
point_fx.POINT_AVERAGE_VALUE)
tsv.append(float(1 + i / 10) * pts)
store_tsv.append(TimeSeries("cache://test/{0}".format(i),
pts)) # generate a bound pts to store
dummy_ts = TimeSeries('dummy://a')
tsv.append(dummy_ts.integral(ta))
self.assertGreater(len(ts_stringify(tsv[0])),
10) # just ensure ts_stringify work on expr.
# then start the server
dtss = DtsServer()
port_no = find_free_port()
host_port = 'localhost:{0}'.format(port_no)
dtss.set_listening_port(port_no)
dtss.cb = self.dtss_read_callback
dtss.find_cb = self.dtss_find_callback
dtss.store_ts_cb = self.dtss_store_callback
dtss.start_async()
dts = DtsClient(StringVector([host_port]), True,
1000) # as number of hosts
# then try something that should work
dts.store_ts(store_tsv)
r1 = dts.evaluate(tsv, ta.total_period())
tsv1x = tsv.inside(-0.5, 0.5)
tsv1x.append(tsv1x[-1].decode(
start_bit=1, n_bits=1)) # just to verify serialization/bind
tsv1x.append(store_tsv[1].derivative())
tsv1x.append(store_tsv[1].pow(
2.0)) # just for verify pow serialization(well, it's a bin-op..)
r1x = dts.evaluate(tsv1x, ta.total_period())
r2 = dts.percentiles(tsv, ta.total_period(), ta24, percentile_list)
r3 = dts.find('netcdf://dummy\.nc/ts\d')
self.rd_throws = True
ex_count = 0
try:
rx = dts.evaluate(tsv, ta.total_period())
except RuntimeError as e:
ex_count = 1
pass
self.rd_throws = True
try:
fx = dts.find('should throw')
except RuntimeError as e:
ex_count += 1
pass
dts.close() # close connection (will use context manager later)
dtss.clear() # close server
self.assertEqual(ex_count, 2)
self.assertEqual(len(r1), len(tsv))
self.assertEqual(self.callback_count, 4)
for i in range(n_ts - 1):
self.assertEqual(r1[i].time_axis, tsv[i].time_axis)
assert_array_almost_equal(r1[i].values.to_numpy(),
tsv[i].values.to_numpy(),
decimal=4)
self.assertEqual(len(r2), len(percentile_list))
dummy_ts.bind(
TimeSeries(ta,
fill_value=1.0,
point_fx=point_fx.POINT_AVERAGE_VALUE))
p2 = tsv.percentiles(ta24, percentile_list)
# r2 = tsv.percentiles(ta24,percentile_list)
for i in range(len(p2)):
self.assertEqual(r2[i].time_axis, p2[i].time_axis)
assert_array_almost_equal(r2[i].values.to_numpy(),
p2[i].values.to_numpy(),
decimal=1)
self.assertEqual(self.find_count, 2)
self.assertEqual(len(r3), 10) # 0..9
for i in range(len(r3)):
self.assertEqual(r3[i], self.ts_infos[i])
self.assertIsNotNone(r1x)
self.assertEqual(1, len(self.stored_tsv))
self.assertEqual(len(store_tsv), len(self.stored_tsv[0]))
for i in range(len(store_tsv)):
self.assertEqual(self.stored_tsv[0][i].ts_id(),
store_tsv[i].ts_id())