def test_pack_unpack_history(tsh, engine):
for numserie in (1, 2, 3):
with engine.begin() as cn:
tsh.update(cn,
genserie(datetime(2021, 1, 1), 'D', numserie),
'small-hist-naive',
'*****@*****.**',
insertion_date=utcdt(2021, 2, numserie))
hist = tsh.history(engine, 'small-hist-naive')
meta = tsh.metadata(engine, 'small-hist-naive')
packed = pack_history(meta, hist)
meta2, hist2 = unpack_history(packed)
assert meta2 == meta
for idate, series in hist.items():
assert hist[idate].equals(series)
for numserie in (1, 2, 3):
with engine.begin() as cn:
tsh.update(cn,
genserie(utcdt(2021, 1, 1), 'D', numserie),
'small-hist-tzaware',
'*****@*****.**',
insertion_date=utcdt(2021, 2, numserie))
hist = tsh.history(engine, 'small-hist-tzaware')
meta = tsh.metadata(engine, 'small-hist-tzaware')
packed = pack_history(meta, hist)
meta2, hist2 = unpack_history(packed)
assert meta2 == meta
for idate, series in hist.items():
assert hist[idate].equals(series)
def test_complicated_thing(mapi):
groundzero = pd.Series([0, 0, 0],
index=pd.date_range(utcdt(2019, 1, 1),
periods=3,
freq='D'))
one = pd.Series([1, 1, 1],
index=pd.date_range(utcdt(2019, 1, 1), periods=3,
freq='D'))
mapi.update('groundzero-b', groundzero, 'Babar')
mapi.update('one-b', one, 'Celeste')
mapi.register_formula(
'complicated',
'(add (* 3.1416 (series "groundzero-b" #:fill "bfill" #:prune 1))'
' (* 2 (min (+ 1 (series "groundzero-b")) (series "one-b"))))',
)
presenter = fancypresenter(mapi, 'complicated', {})
info = [{k: v
for k, v in info.items() if k != 'ts'}
for info in presenter.infos]
assert info == [{
'name': 'complicated',
'type': 'formula'
}, {
'name': 'groundzero-b',
'type': 'primary'
}, {
'name': 'one-b',
'type': 'primary'
}]
def test_staircase_history_naive(client, tsh):
# each days we insert 7 data points
from datetime import datetime
for idx, idate in enumerate(
pd.date_range(start=utcdt(2015, 1, 1),
end=utcdt(2015, 1, 4),
freq='D')):
series = genserie(start=idate.tz_convert(None), freq='H', repeat=7)
client.update('staircase-naive', series, 'Babar', insertion_date=idate)
series = client.staircase('staircase-naive',
pd.Timedelta(hours=3),
from_value_date=datetime(2015, 1, 1, 4),
to_value_date=datetime(2015, 1, 2, 5))
assert series.name == 'staircase-naive'
assert_df(
"""
2015-01-01 04:00:00 4.0
2015-01-01 05:00:00 5.0
2015-01-01 06:00:00 6.0
2015-01-02 03:00:00 3.0
2015-01-02 04:00:00 4.0
2015-01-02 05:00:00 5.0
""", series)
# series = client.staircase(
# 'staircase-naive',
# pd.Timedelta(hours=3),
# from_value_date=datetime(2015, 1, 1, 4),
# to_value_date=datetime(2015, 1, 2, 5)
# )
hist = client.history('staircase-naive')
assert len(hist) == 4
hist = client.history('staircase-naive',
from_insertion_date=datetime(2015, 1, 2),
to_insertion_date=datetime(2015, 1, 3))
assert len(hist) == 2
hist = client.history('staircase-naive',
from_value_date=datetime(2015, 1, 1, 3),
to_value_date=datetime(2015, 1, 2, 1))
assert all(series.name == 'staircase-naive' for series in hist.values())
assert_hist(
"""
insertion_date value_date
2015-01-01 00:00:00+00:00 2015-01-01 03:00:00 3.0
2015-01-01 04:00:00 4.0
2015-01-01 05:00:00 5.0
2015-01-01 06:00:00 6.0
2015-01-02 00:00:00+00:00 2015-01-01 03:00:00 3.0
2015-01-01 04:00:00 4.0
2015-01-01 05:00:00 5.0
2015-01-01 06:00:00 6.0
2015-01-02 00:00:00 0.0
2015-01-02 01:00:00 1.0
""", hist)
def test_alternative_handler(pgapi):
api = pgapi
sapi = timeseries(api.uri, api.namespace, formula_class())
sapi.update(
'test-features',
genserie(utcdt(2020, 1, 1), 'D', 3),
'Babar',
)
sapi.tsh.register_formula(sapi.engine, 'test-formula',
'(+ 1 (series "test-features"))')
tsa = sapi.get('test-features')
assert_df(
"""
2020-01-01 00:00:00+00:00 0.0
2020-01-02 00:00:00+00:00 1.0
2020-01-03 00:00:00+00:00 2.0
""", tsa)
tsb = sapi.get('test-formula')
assert_df(
"""
2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
""", tsb)
class supervision_and_formula(supervision_class(), formula_class()):
pass
sapi = timeseries(api.uri, api.namespace, supervision_and_formula)
tsa = sapi.get('test-features')
assert_df(
"""
2020-01-01 00:00:00+00:00 0.0
2020-01-02 00:00:00+00:00 1.0
2020-01-03 00:00:00+00:00 2.0
""", tsa)
tsb = sapi.get('test-formula')
assert_df(
"""
2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
""", tsb)
sapi.update('test-features',
genserie(utcdt(2020, 1, 2), 'D', 3),
'Babar',
manual=True)
tsb = sapi.get('test-formula')
assert_df(
"""
2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 1.0
2020-01-03 00:00:00+00:00 2.0
2020-01-04 00:00:00+00:00 3.0
""", tsb)
def test_naive(client):
series_in = genserie(pd.Timestamp('2018-1-1'), 'H', 3)
res = client.patch('/series/state',
params={
'name': 'test-naive',
'series': util.tojson(series_in),
'author': 'Babar',
'insertion_date': utcdt(2018, 1, 1, 10),
'tzaware': util.tzaware_serie(series_in)
})
assert res.status_code == 201
res = client.get('/series/metadata?name=test-naive&all=1')
meta = res.json
assert meta == {
'index_dtype': '<M8[ns]',
'index_type': 'datetime64[ns]',
'tzaware': False,
'value_dtype': '<f8',
'value_type': 'float64'
}
res = client.get('/series/state?name=test-naive')
series = util.fromjson(res.body, 'test', meta['tzaware'])
assert_df(
"""
2018-01-01 00:00:00 0.0
2018-01-01 01:00:00 1.0
2018-01-01 02:00:00 2.0
""", series)
def test_multisources(client, engine):
series = genserie(utcdt(2020, 1, 1), 'D', 3)
tsh = tsio.timeseries('other')
tsh.update(engine, series, 'test-other', 'Babar')
client.update('test-mainsource', series, 'Babar')
with pytest.raises(ValueError) as err:
client.update('test-other', series, 'Babar')
assert err.value.args[0] == 'not allowed to update to a secondary source'
with pytest.raises(ValueError) as err:
client.replace('test-other', series, 'Babar')
assert err.value.args[0] == 'not allowed to replace to a secondary source'
cat = client.catalog()
assert cat == {
('db://localhost:5433/postgres', 'other'): [['test-other', 'primary']],
('db://localhost:5433/postgres', 'tsh'):
[['test-naive', 'primary'], ['test', 'primary'],
['staircase', 'primary'], ['staircase-naive', 'primary'],
['in-a-formula', 'primary'], ['test-mainsource', 'primary'],
['new-formula', 'formula']]
}
cat = client.catalog(allsources=False)
assert ('db://localhost:5433/postgres', 'tsh') in cat
assert ('db://localhost:5433/postgres', 'other') not in cat
def test_mul(engine, tsh):
base = pd.Series([1, 2, 3],
index=pd.date_range(utcdt(2019, 1, 1),
periods=3,
freq='D'))
tsh.update(engine, base, 'mul-a', 'Babar')
tsh.update(engine, base, 'mul-b', 'Babar')
tsh.update(engine, base, 'mul-c', 'Babar')
tsh.register_formula(
engine,
'multiply-aligned',
'(mul (series "mul-a") (series "mul-b") (series "mul-c"))',
)
ts = tsh.get(engine, 'multiply-aligned')
assert_df(
"""
2019-01-01 00:00:00+00:00 1.0
2019-01-02 00:00:00+00:00 8.0
2019-01-03 00:00:00+00:00 27.0
""", ts)
base = pd.Series([1, 2, np.nan],
index=pd.date_range(utcdt(2019, 1, 1),
periods=3,
freq='D'))
tsh.update(engine, base, 'mul-b', 'Babar')
ts = tsh.get(engine, 'multiply-aligned')
assert_df(
"""
2019-01-01 00:00:00+00:00 1.0
2019-01-02 00:00:00+00:00 8.0
""", ts)
tsh.register_formula(
engine,
'multiply-aligned',
'(mul (series "mul-a") (series "mul-b" #:fill 1) (series "mul-c"))',
update=True)
ts = tsh.get(engine, 'multiply-aligned')
assert_df(
"""
2019-01-01 00:00:00+00:00 1.0
2019-01-02 00:00:00+00:00 8.0
2019-01-03 00:00:00+00:00 9.0
""", ts)
def test_editor_table_callback(mapi):
groundzero = pd.Series([0, 0, 0],
index=pd.date_range(utcdt(2019, 1, 1),
periods=3,
freq='D'))
one = pd.Series([1, 1, 1],
index=pd.date_range(utcdt(2019, 1, 1), periods=3,
freq='D'))
mapi.update('groundzero-a', groundzero, 'Babar')
mapi.update('one-a', one, 'Celeste')
mapi.register_formula(
'editor-1',
'(add (* 3.1416 (series "groundzero-a" #:fill "bfill" #:prune 1)) (series "one-a"))',
)
presenter = fancypresenter(mapi, 'editor-1', {})
info = [{k: v
for k, v in info.items() if k != 'ts'}
for info in presenter.infos]
assert info == [{
'name': 'editor-1',
'type': 'formula'
}, {
'name': 'groundzero-a',
'type': 'primary'
}, {
'name': 'one-a',
'type': 'primary'
}]
# trigger an empty series
presenter = fancypresenter(mapi, 'editor-1',
{'from_value_date': utcdt(2019, 1, 4)})
info = [{k: v
for k, v in info.items() if k != 'ts'}
for info in presenter.infos]
assert info == [{
'name': 'editor-1',
'type': 'formula'
}, {
'name': 'groundzero-a',
'type': 'primary'
}, {
'name': 'one-a',
'type': 'primary'
}]
def test_delete(client):
series_in = genserie(utcdt(2018, 1, 1), 'H', 3)
res = client.patch('/series/state',
params={
'name': 'test',
'series': util.tojson(series_in),
'author': 'Babar',
'insertion_date': utcdt(2018, 1, 1, 10),
'tzaware': util.tzaware_serie(series_in)
})
res = client.delete('/series/state', params={'name': 'no-such-series'})
assert res.status_code == 404
res = client.delete('/series/state', params={'name': 'test'})
assert res.status_code == 204
res = client.get('/series/catalog')
assert 'test' not in res.json
def test_rename(client):
series_in = genserie(utcdt(2018, 1, 1), 'H', 3)
res = client.patch('/series/state',
params={
'name': 'test',
'series': util.tojson(series_in),
'author': 'Babar',
'insertion_date': utcdt(2018, 1, 1, 10),
'tzaware': util.tzaware_serie(series_in)
})
res = client.put('/series/state',
params={
'name': 'no-such-series',
'newname': 'no-better'
})
assert res.status_code == 404
res = client.put('/series/state',
params={
'name': 'test',
'newname': 'test2'
})
assert res.status_code == 204
res = client.get('/series/catalog')
assert res.json == {
'db://localhost:5433/postgres!tsh': [['test-naive', 'primary'],
['test2', 'primary']]
}
assert 'test' not in res.json
res = client.patch('/series/state',
params={
'name': 'test3',
'series': util.tojson(series_in),
'author': 'Babar',
'insertion_date': utcdt(2018, 1, 1, 10),
'tzaware': util.tzaware_serie(series_in)
})
res = client.put('/series/state',
params={
'name': 'test2',
'newname': 'test3'
})
assert res.status_code == 409
assert res.json == {'message': '`test3` does exists'}
def test_resample(engine, tsh):
hourly = pd.Series(list(range(36)),
index=pd.date_range(utcdt(2020, 1, 1),
periods=36,
freq='H'))
gasday = pd.Series([1, 2, 3],
index=pd.date_range(utcdt(2020, 1, 1, 5),
periods=3,
freq='D'))
tsh.update(engine, hourly, 'hourly', 'Babar')
tsh.update(engine, gasday, 'gasday', 'Celeste')
tsh.register_formula(engine, 'hourly2daily',
'(resample (series "hourly") "D")')
tsh.register_formula(engine, 'hourly2dailysum',
'(resample (series "hourly") "D" "sum")')
tsh.register_formula(engine, 'gasdaytoday',
'(resample (series "gasday") "D")')
tsh.register_formula(engine, 'badmethod',
'(resample (series "gasday") "D" "NO-SUCH-METHOD")')
assert_df(
"""
2020-01-01 00:00:00+00:00 11.5
2020-01-02 00:00:00+00:00 29.5
""", tsh.get(engine, 'hourly2daily'))
assert_df(
"""
2020-01-01 00:00:00+00:00 276.0
2020-01-02 00:00:00+00:00 354.0
""", tsh.get(engine, 'hourly2dailysum'))
assert_df(
"""
2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
""", tsh.get(engine, 'gasdaytoday'))
with pytest.raises(ValueError) as err:
tsh.get(engine, 'badmethod')
assert err.value.args[0] == 'bad resampling method `NO-SUCH-METHOD`'
def test_div(engine, tsh):
base = pd.Series([1, 2, 3],
index=pd.date_range(utcdt(2019, 1, 1),
periods=3,
freq='D'))
tsh.update(engine, base, 'div-a', 'Babar')
tsh.update(engine, base, 'div-b', 'Babar')
tsh.register_formula(
engine,
'divide',
'(div (series "div-a") (series "div-b"))',
)
ts = tsh.get(engine, 'divide')
assert_df(
"""
2019-01-01 00:00:00+00:00 1.0
2019-01-02 00:00:00+00:00 1.0
2019-01-03 00:00:00+00:00 1.0
""", ts)
base = pd.Series([2, 1, np.nan],
index=pd.date_range(utcdt(2019, 1, 1),
periods=3,
freq='D'))
tsh.update(engine, base, 'div-b', 'Babar')
ts = tsh.get(engine, 'divide')
assert_df(
"""
2019-01-01 00:00:00+00:00 0.5
2019-01-02 00:00:00+00:00 2.0
""", ts)
tsh.register_formula(engine,
'divide',
'(div (series "div-a") (series "div-b" #:fill 3))',
update=True)
ts = tsh.get(engine, 'divide')
assert_df(
"""
2019-01-01 00:00:00+00:00 0.5
2019-01-02 00:00:00+00:00 2.0
2019-01-03 00:00:00+00:00 1.0
""", ts)
def test_editor_pure_scalar_op(mapi):
ts = pd.Series([1, 2, 3],
index=pd.date_range(utcdt(2020, 1, 1), periods=3, freq='D'))
mapi.update('pure-scalar-ops', ts, 'Babar')
mapi.register_formula('formula-pure-scalar-ops',
'(+ (* 3 (/ 6 2)) (series "pure-scalar-ops"))')
presenter = fancypresenter(mapi, 'formula-pure-scalar-ops',
{'from_value_date': utcdt(2019, 1, 4)})
info = [{k: v
for k, v in info.items() if k != 'ts'}
for info in presenter.infos]
assert info == [{
'name': 'formula-pure-scalar-ops',
'type': 'formula'
}, {
'name': 'pure-scalar-ops',
'type': 'primary'
}]
def test_history(engine, cli, tsh):
serie = genserie(datetime(2020, 1, 1), 'D', 3)
tsh.update(engine,
serie,
'some_history',
'Babar',
insertion_date=utcdt(2019, 1, 1))
serie = genserie(datetime(2020, 1, 2), 'D', 3)
tsh.update(engine,
serie,
'some_history',
'Babar',
insertion_date=utcdt(2019, 1, 2))
r = cli('history', engine.url, 'some_history', json=True)
assert r.output == (
'{"2019-01-01 00:00:00+00:00": {"2020-01-01 00:00:00": 0.0, "2020-01-02 00:00:00": 1.0, "2020-01-03 00:00:00": 2.0}, "2019-01-02 00:00:00+00:00": {"2020-01-02 00:00:00": 0.0, "2020-01-03 00:00:00": 1.0, "2020-01-04 00:00:00": 2.0}}\n'
)
def test_staircase(engine, tsh):
tsh.register_formula(engine, 's-addition',
'(add (series "sa") (series "sb"))', False)
for day in (1, 2, 3, 4, 5):
idate = utcdt(2018, 1, day)
for name in 'ab':
ts = pd.Series([day / 2.] * 5,
index=pd.date_range(dt(2018, 1, day),
periods=5,
freq='D'))
tsh.update(engine, ts, 's' + name, 'Babar', insertion_date=idate)
ts = tsh.staircase(engine, 's-addition', delta=pd.Timedelta(hours=12))
assert_df(
"""
2018-01-02 1.0
2018-01-03 2.0
2018-01-04 3.0
2018-01-05 4.0
2018-01-06 5.0
2018-01-07 5.0
2018-01-08 5.0
2018-01-09 5.0
""", ts)
# this is not allowed in the staircase fast-path
# hence we will take the slow path
@func('identity')
def identity(series: pd.Series) -> pd.Series:
return series
tsh.register_formula(engine, 'slow-down', '(identity (series "sa"))',
False)
tsh.register_formula(engine, 's-addition-not-fast',
'(add (series "slow-down") (series "sb"))', False)
ts = tsh.staircase(engine,
's-addition-not-fast',
delta=pd.Timedelta(hours=12))
assert_df(
"""
2018-01-02 1.0
2018-01-03 2.0
2018-01-04 3.0
2018-01-05 4.0
2018-01-06 5.0
2018-01-07 5.0
2018-01-08 5.0
2018-01-09 5.0
""", ts)
# cleanup
FUNCS.pop('identity')
def create_data():
# 4 years of sub-hourly points
for year in range(2015, 2020):
date = utcdt(year, 1, 1)
serie = genserie(date, '10Min', 6 * 24 * 365)
with engine.begin() as cn:
tsh.update(cn,
serie,
'big',
'*****@*****.**',
insertion_date=date)
def create(uri, ns, name):
api = timeseries(uri, ns)
series = pd.Series([1, 2, 3],
index=pd.date_range(utcdt(2020, 1, 1),
periods=3,
freq='D'))
api.update(name,
series,
'Babar',
insertion_date=utcdt(2019, 1, 1),
metadata={'about': 'test'})
out = api.get(name)
assert_df(
"""
2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
""", out)
series[utcdt(2020, 1, 4)] = 4
api.update(name, series, 'Babar', insertion_date=utcdt(2019, 1, 2))
out = api.get(name,
from_value_date=utcdt(2020, 1, 2),
to_value_date=utcdt(2020, 1, 3))
assert_df(
"""
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
""", out)
def create_data():
# one insert per day for 4 months
for month in range(1, 4):
days = calendar.monthrange(2017, month)[1]
for day in range(1, days + 1):
date = utcdt(2017, month, day)
serie = genserie(date, '10Min', 6 * 24)
with engine.begin() as cn:
tsh.update(cn,
serie,
'manydiffs',
'*****@*****.**',
insertion_date=date.replace(year=2018))
def test_slice(engine, tsh):
base = pd.Series([1, 2, 3],
index=pd.date_range(utcdt(2019, 1, 1),
periods=3,
freq='D'))
tsh.update(engine, base, 'test-slice', 'Babar')
tsh.register_formula(
engine,
'slicing-id',
'(slice (series "test-slice"))',
)
tsh.register_formula(
engine,
'slicing-from',
'(slice (series "test-slice") #:fromdate (date "2019-1-2"))',
)
tsh.register_formula(
engine,
'slicing-fromto',
'(slice (series "test-slice") '
' #:fromdate (date "2019-1-2") '
' #:todate (date "2019-1-2")'
')',
)
tsh.register_formula(
engine,
'slicing-empty',
'(slice (series "test-slice") '
' #:fromdate (date "2018-1-2") '
' #:todate (date "2018-1-2")'
')',
)
assert_df(
"""
2019-01-01 00:00:00+00:00 1.0
2019-01-02 00:00:00+00:00 2.0
2019-01-03 00:00:00+00:00 3.0
""", tsh.get(engine, 'slicing-id'))
assert_df(
"""
2019-01-02 00:00:00+00:00 2.0
2019-01-03 00:00:00+00:00 3.0
""", tsh.get(engine, 'slicing-from'))
assert_df("""
2019-01-02 00:00:00+00:00 2.0
""", tsh.get(engine, 'slicing-fromto'))
assert len(tsh.get(engine, 'slicing-empty')) == 0
def test_get_fast_path(client):
series_in = genserie(utcdt(2018, 1, 1), 'H', 3)
res = client.patch('/series/state',
params={
'name': 'test_fast',
'series': util.tojson(series_in),
'author': 'Babar',
'insertion_date': utcdt(2018, 1, 1, 10),
'tzaware': util.tzaware_serie(series_in)
})
assert res.status_code == 201
out = client.get('/series/state',
params={
'name': 'test_fast',
'format': 'tshpack'
})
meta, index, values = util.nary_unpack(zlib.decompress(out.body))
meta = json.loads(meta)
index, values = util.numpy_deserialize(index, values, meta)
series = pd.Series(values, index=index)
series = series.tz_localize('UTC')
assert_df(
"""
2018-01-01 00:00:00+00:00 0.0
2018-01-01 01:00:00+00:00 1.0
2018-01-01 02:00:00+00:00 2.0
""", series)
assert meta == {
'tzaware': True,
'index_type': 'datetime64[ns, UTC]',
'value_type': 'float64',
'index_dtype': '|M8[ns]',
'value_dtype': '<f8'
}
def test_lots_of_diffs(engine, tracker, ptsh):
tsh = ptsh
def create_data():
# one insert per day for 4 months
for month in range(1, 4):
days = calendar.monthrange(2017, month)[1]
for day in range(1, days + 1):
date = utcdt(2017, month, day)
serie = genserie(date, '10Min', 6 * 24)
with engine.begin() as cn:
tsh.update(cn,
serie,
'manydiffs',
'*****@*****.**',
insertion_date=date.replace(year=2018))
t0 = time()
create_data()
t1 = time() - t0
tshclass = tsh.__class__.__name__
tracker.append({'test': 'manydiffs_insert', 'class': tshclass, 'time': t1})
t0 = time()
with engine.begin() as cn:
tsh.history(cn, 'manydiffs')
t1 = time() - t0
tracker.append({
'test': 'manydiffs_history_all',
'class': tshclass,
'time': t1
})
t0 = time()
for month in range(1, 3):
for day in range(1, 5):
date = utcdt(2018, month, day)
with engine.begin() as cn:
ts = tsh.history(cn,
'manydiffs',
from_insertion_date=date,
to_insertion_date=date + timedelta(days=31))
assert ts is not None
t1 = time() - t0
tracker.append({
'test': 'manydiffs_history_chunks',
'class': tshclass,
'time': t1
})
def test_staircase(client):
# each days we insert 7 data points
for idate in pd.date_range(start=utcdt(2015, 1, 1),
end=utcdt(2015, 1, 4),
freq='D'):
series = genserie(start=idate, freq='H', repeat=7)
client.patch('/series/state',
params={
'name': 'staircase',
'series': util.tojson(series),
'author': 'Babar',
'insertion_date': idate,
'tzaware': util.tzaware_serie(series)
})
res = client.get('/series/staircase',
params={
'name': 'staircase',
'delta': pd.Timedelta(hours=3),
'from_value_date': utcdt(2015, 1, 1, 4),
'to_value_date': utcdt(2015, 1, 2, 5),
})
series = util.fromjson(res.body, 'test', True)
assert_df(
"""
2015-01-01 04:00:00+00:00 4.0
2015-01-01 05:00:00+00:00 5.0
2015-01-01 06:00:00+00:00 6.0
2015-01-02 03:00:00+00:00 3.0
2015-01-02 04:00:00+00:00 4.0
2015-01-02 05:00:00+00:00 5.0
""", series)
res = client.get('/series/staircase',
params={
'name': 'staircase',
'delta': pd.Timedelta(hours=3),
'from_value_date': utcdt(2015, 1, 1, 4),
'to_value_date': utcdt(2015, 1, 2, 5),
'format': 'tshpack'
})
meta, index, values = util.nary_unpack(zlib.decompress(res.body))
meta = json.loads(meta)
index, values = util.numpy_deserialize(index, values, meta)
series = pd.Series(values, index=index)
series = series.tz_localize('UTC')
assert_df(
"""
2015-01-01 04:00:00+00:00 4.0
2015-01-01 05:00:00+00:00 5.0
2015-01-01 06:00:00+00:00 6.0
2015-01-02 03:00:00+00:00 3.0
2015-01-02 04:00:00+00:00 4.0
2015-01-02 05:00:00+00:00 5.0
""", series)
def test_naive(client, engine, tsh):
series_in = genserie(pd.Timestamp('2018-1-1'), 'H', 3)
client.update('test-naive',
series_in,
'Babar',
insertion_date=utcdt(2019, 1, 1))
# now let's get it back
ts = client.get('test-naive')
assert_df(
"""
2018-01-01 00:00:00 0.0
2018-01-01 01:00:00 1.0
2018-01-01 02:00:00 2.0
""", ts)
assert not getattr(ts.index.dtype, 'tz', False)
def test_bigdata(engine, tracker, ptsh):
tsh = ptsh
def create_data():
# 4 years of sub-hourly points
for year in range(2015, 2020):
date = utcdt(year, 1, 1)
serie = genserie(date, '10Min', 6 * 24 * 365)
with engine.begin() as cn:
tsh.update(cn,
serie,
'big',
'*****@*****.**',
insertion_date=date)
t0 = time()
create_data()
t1 = time() - t0
tshclass = tsh.__class__.__name__
tracker.append({'test': 'bigdata_insert', 'class': tshclass, 'time': t1})
t0 = time()
with engine.begin() as cn:
tsh.history(cn, 'big')
t1 = time() - t0
tracker.append({
'test': 'bigdata_history_all',
'class': tshclass,
'time': t1
})
t0 = time()
for year in (2015, 2017, 2019):
for month in (1, 5, 9, 12):
date = utcdt(year, month, 1)
with engine.begin() as cn:
tsh.history(cn,
'big',
from_insertion_date=date,
to_insertion_date=date + timedelta(days=31))
t1 = time() - t0
tracker.append({
'test': 'bigdata_history_chunks',
'class': tshclass,
'time': t1
})
def test_slice_options(engine, tsh):
base = pd.Series([1, 2, 3],
index=pd.date_range(utcdt(2019, 1, 1),
periods=3,
freq='D'))
tsh.update(engine, base, 'test-slice', 'Babar')
# options transmissions
ts = tsh.eval_formula(
engine,
'(add (series "test-slice") '
' (slice (series "test-slice" #:fill 0) #:fromdate (date "2019-1-2")))',
)
assert_df(
"""
2019-01-01 00:00:00+00:00 1.0
2019-01-02 00:00:00+00:00 4.0
2019-01-03 00:00:00+00:00 6.0
""", ts)
def test_expanded(engine, tsh):
@func('customseries')
def customseries() -> pd.Series:
return pd.Series([1.0, 2.0, 3.0],
index=pd.date_range(dt(2019, 1, 1),
periods=3,
freq='D'))
@finder('customseries')
def find(cn, tsh, tree):
return {
tree[0]: {
'tzaware': True,
'index_type': 'datetime64[ns, UTC]',
'value_type': 'float64',
'index_dtype': '|M8[ns]',
'value_dtype': '<f8'
}
}
base = pd.Series([1, 2, 3],
index=pd.date_range(utcdt(2019, 1, 1),
periods=3,
freq='D'))
tsh.update(engine, base, 'exp-a', 'Babar')
tsh.update(engine, base, 'exp-b', 'Celeste')
tsh.register_formula(engine, 'expandmebase1',
'(+ 3 (priority (series "exp-a") (customseries)))',
False)
tsh.register_formula(engine, 'expandmebase2',
'(priority (series "exp-a") (series "exp-b"))', False)
tsh.register_formula(
engine, 'expandme',
'(add (series "expandmebase1") (series "exp-b") (series "expandmebase2"))',
False)
exp = tsh.expanded_formula(engine, 'expandme')
assert exp == ('(add '
'(+ 3 (priority (series "exp-a") (customseries))) '
'(series "exp-b") '
'(priority (series "exp-a") (series "exp-b")))')
def test_naive_tzone(engine, tsh):
x = pd.Series([1, 2, 3],
index=pd.date_range(utcdt(2020, 1, 1), periods=3, freq='D'))
tsh.update(engine, x, 'non-naive', 'Babar')
tsh.register_formula(
engine,
'to-naive',
'(naive (series "non-naive") "Europe/Moscow")',
)
ts = tsh.get(engine, 'to-naive')
assert_df(
"""
2020-01-01 03:00:00 1.0
2020-01-02 03:00:00 2.0
2020-01-03 03:00:00 3.0
""", ts)
meta = tsh.metadata(engine, 'to-naive')
assert meta['tzaware'] == False
def test_append(engine, tsh):
if tsh.namespace == 'z-z':
return
for x, dt in enumerate(pd.date_range(start=utcdt(2018, 1, 1),
freq='D', periods=10)):
ts = genserie(dt, 'D', 1, [x], name='daily')
tsh.update(engine, ts, 'append', '*****@*****.**',
insertion_date=dt)
sql = 'select id, parent, chunk from "{}.snapshot".append order by id'.format(
tsh.namespace
)
chunks = engine.execute(sql).fetchall()
c = {
chunk.id: chunk.parent for chunk in chunks
}
# nice linked list
assert c == {
1: None, 2: 1, 3: 2, 4: 3, 5: 4, 6: 5, 7: 6, 8: 7, 9: 8, 10: 9
}
def test_strip(engine, tsh):
ts = genserie(datetime(2019, 1, 1), 'D', 3)
tsh.update(engine,
ts,
'strip-unsupervised',
'test',
insertion_date=utcdt(2019, 1, 1))
csid = tsh.changeset_at(engine, 'strip-unsupervised', utcdt(2019, 1, 1))
tsh.strip(engine, 'strip-unsupervised', csid)
ts = genserie(datetime(2019, 1, 1), 'D', 3)
tsh.update(engine,
ts,
'strip-handcrafted',
'test',
manual=True,
insertion_date=utcdt(2019, 1, 1))
csid = tsh.changeset_at(engine, 'strip-handcrafted', utcdt(2019, 1, 1))
tsh.strip(engine, 'strip-handcrafted', csid)
ts = genserie(datetime(2019, 1, 1), 'D', 3)
tsh.update(engine,
ts,
'strip-supervised',
'test',
insertion_date=utcdt(2019, 1, 1))
ts = genserie(datetime(2019, 1, 2), 'D', 3)
tsh.update(engine,
ts,
'strip-supervised',
'test',
manual=True,
insertion_date=utcdt(2019, 1, 2))
with pytest.raises(ValueError) as err:
csid = tsh.changeset_at(engine, 'strip-supervised', utcdt(2019, 1, 1))
tsh.strip(engine, 'strip-supervised', csid)
def test_historical(engine, tsh):
for insertion_date in pd.date_range(start=datetime(2015, 1, 1),
end=datetime(2015, 1, 1, 6),
freq='H'):
ts = genserie(start=insertion_date, freq='H', repeat=7, tz='UTC')
tsh.insert(engine, ts, 'republication', 'test',
insertion_date=pd.Timestamp(insertion_date, tz='UTC'))
# this is the end state
# initially the first 1 was at 01:00 then at each insertion it gets
# moved one hour later
ts = tsh.get(engine, 'republication')
assert_df("""
2015-01-01 00:00:00+00:00 0.0
2015-01-01 01:00:00+00:00 0.0
2015-01-01 02:00:00+00:00 0.0
2015-01-01 03:00:00+00:00 0.0
2015-01-01 04:00:00+00:00 0.0
2015-01-01 05:00:00+00:00 0.0
2015-01-01 06:00:00+00:00 0.0
2015-01-01 07:00:00+00:00 1.0
2015-01-01 08:00:00+00:00 2.0
2015-01-01 09:00:00+00:00 3.0
2015-01-01 10:00:00+00:00 4.0
2015-01-01 11:00:00+00:00 5.0
2015-01-01 12:00:00+00:00 6.0
""", ts)
tsh.insert(engine, genserie(datetime(2015, 1, 1), 'H', 15, [10], tz='UTC'),
'stable', 'test',
insertion_date=pd.Timestamp(datetime(2015, 1, 1), tz='UTC'))
# tens all over the place inserted at hour zero
# there are two more tstamps than in republication
assert_df("""
2015-01-01 00:00:00+00:00 10.0
2015-01-01 01:00:00+00:00 10.0
2015-01-01 02:00:00+00:00 10.0
2015-01-01 03:00:00+00:00 10.0
2015-01-01 04:00:00+00:00 10.0
2015-01-01 05:00:00+00:00 10.0
2015-01-01 06:00:00+00:00 10.0
2015-01-01 07:00:00+00:00 10.0
2015-01-01 08:00:00+00:00 10.0
2015-01-01 09:00:00+00:00 10.0
2015-01-01 10:00:00+00:00 10.0
2015-01-01 11:00:00+00:00 10.0
2015-01-01 12:00:00+00:00 10.0
2015-01-01 13:00:00+00:00 10.0
2015-01-01 14:00:00+00:00 10.0
""", tsh.get(engine, 'stable'))
tsh.build_arithmetic(engine, 'operation_past',
{'republication': 1,
'stable': 1}
)
tsh.build_priority(engine, 'compo_past', ['republication', 'stable'])
ts = tsh.get(engine, 'operation_past')
assert_df("""
2015-01-01 00:00:00+00:00 10.0
2015-01-01 01:00:00+00:00 10.0
2015-01-01 02:00:00+00:00 10.0
2015-01-01 03:00:00+00:00 10.0
2015-01-01 04:00:00+00:00 10.0
2015-01-01 05:00:00+00:00 10.0
2015-01-01 06:00:00+00:00 10.0
2015-01-01 07:00:00+00:00 11.0
2015-01-01 08:00:00+00:00 12.0
2015-01-01 09:00:00+00:00 13.0
2015-01-01 10:00:00+00:00 14.0
2015-01-01 11:00:00+00:00 15.0
2015-01-01 12:00:00+00:00 16.0
""", ts)
ts = tsh.get(engine, 'compo_past')
assert_df("""
2015-01-01 00:00:00+00:00 0.0
2015-01-01 01:00:00+00:00 0.0
2015-01-01 02:00:00+00:00 0.0
2015-01-01 03:00:00+00:00 0.0
2015-01-01 04:00:00+00:00 0.0
2015-01-01 05:00:00+00:00 0.0
2015-01-01 06:00:00+00:00 0.0
2015-01-01 07:00:00+00:00 1.0
2015-01-01 08:00:00+00:00 2.0
2015-01-01 09:00:00+00:00 3.0
2015-01-01 10:00:00+00:00 4.0
2015-01-01 11:00:00+00:00 5.0
2015-01-01 12:00:00+00:00 6.0
2015-01-01 13:00:00+00:00 10.0
2015-01-01 14:00:00+00:00 10.0
""", ts)
assert_df("""
2015-01-01 00:00:00+00:00 10.0
2015-01-01 01:00:00+00:00 10.0
2015-01-01 02:00:00+00:00 10.0
2015-01-01 03:00:00+00:00 10.0
2015-01-01 04:00:00+00:00 11.0
2015-01-01 05:00:00+00:00 12.0
2015-01-01 06:00:00+00:00 13.0
2015-01-01 07:00:00+00:00 14.0
2015-01-01 08:00:00+00:00 15.0
2015-01-01 09:00:00+00:00 16.0
""", tsh.get(engine, 'operation_past',
revision_date=utcdt(2015, 1, 1, 3))
)
assert_df("""
2015-01-01 00:00:00+00:00 0.0
2015-01-01 01:00:00+00:00 0.0
2015-01-01 02:00:00+00:00 0.0
2015-01-01 03:00:00+00:00 0.0
2015-01-01 04:00:00+00:00 1.0
2015-01-01 05:00:00+00:00 2.0
2015-01-01 06:00:00+00:00 3.0
2015-01-01 07:00:00+00:00 4.0
2015-01-01 08:00:00+00:00 5.0
2015-01-01 09:00:00+00:00 6.0
2015-01-01 10:00:00+00:00 10.0
2015-01-01 11:00:00+00:00 10.0
2015-01-01 12:00:00+00:00 10.0
2015-01-01 13:00:00+00:00 10.0
2015-01-01 14:00:00+00:00 10.0
""", tsh.get(engine, 'compo_past',
revision_date=utcdt(2015, 1, 1, 3))
)
assert_df("""
2015-01-01 03:00:00+00:00 13.0
2015-01-01 04:00:00+00:00 13.0
2015-01-01 05:00:00+00:00 13.0
2015-01-01 06:00:00+00:00 13.0
2015-01-01 07:00:00+00:00 13.0
2015-01-01 08:00:00+00:00 13.0
2015-01-01 09:00:00+00:00 13.0
2015-01-01 10:00:00+00:00 14.0
2015-01-01 11:00:00+00:00 15.0
2015-01-01 12:00:00+00:00 16.0
""", tsh.get(engine, 'operation_past',
delta=timedelta(hours=2.5))
)
assert_df("""
2015-01-01 03:00:00+00:00 3.0
2015-01-01 04:00:00+00:00 3.0
2015-01-01 05:00:00+00:00 3.0
2015-01-01 06:00:00+00:00 3.0
2015-01-01 07:00:00+00:00 3.0
2015-01-01 08:00:00+00:00 3.0
2015-01-01 09:00:00+00:00 3.0
2015-01-01 10:00:00+00:00 4.0
2015-01-01 11:00:00+00:00 5.0
2015-01-01 12:00:00+00:00 6.0
2015-01-01 13:00:00+00:00 10.0
2015-01-01 14:00:00+00:00 10.0
""", tsh.get(engine, 'compo_past',
delta=timedelta(hours=2.5))
)
assert_df("""
2015-01-01 06:00:00+00:00 3.0
2015-01-01 07:00:00+00:00 3.0
2015-01-01 08:00:00+00:00 3.0
2015-01-01 09:00:00+00:00 3.0
2015-01-01 10:00:00+00:00 4.0
2015-01-01 11:00:00+00:00 5.0
""", tsh.get(engine, 'compo_past',
delta=timedelta(hours=2.5),
from_value_date=datetime(2015, 1, 1, 6),
to_value_date=datetime(2015, 1, 1, 11))
)
