def test_pack_unpack_history(tsh, engine):
for numserie in (1, 2, 3):
with engine.begin() as cn:
tsh.update(cn,
genserie(datetime(2021, 1, 1), 'D', numserie),
'small-hist-naive',
'*****@*****.**',
insertion_date=utcdt(2021, 2, numserie))
hist = tsh.history(engine, 'small-hist-naive')
meta = tsh.metadata(engine, 'small-hist-naive')
packed = pack_history(meta, hist)
meta2, hist2 = unpack_history(packed)
assert meta2 == meta
for idate, series in hist.items():
assert hist[idate].equals(series)
for numserie in (1, 2, 3):
with engine.begin() as cn:
tsh.update(cn,
genserie(utcdt(2021, 1, 1), 'D', numserie),
'small-hist-tzaware',
'*****@*****.**',
insertion_date=utcdt(2021, 2, numserie))
hist = tsh.history(engine, 'small-hist-tzaware')
meta = tsh.metadata(engine, 'small-hist-tzaware')
packed = pack_history(meta, hist)
meta2, hist2 = unpack_history(packed)
assert meta2 == meta
for idate, series in hist.items():
assert hist[idate].equals(series)
def test_alternative_handler(pgapi):
api = pgapi
sapi = timeseries(api.uri, api.namespace, formula_class())
sapi.update(
'test-features',
genserie(utcdt(2020, 1, 1), 'D', 3),
'Babar',
)
sapi.tsh.register_formula(sapi.engine, 'test-formula',
'(+ 1 (series "test-features"))')
tsa = sapi.get('test-features')
assert_df(
"""
2020-01-01 00:00:00+00:00 0.0
2020-01-02 00:00:00+00:00 1.0
2020-01-03 00:00:00+00:00 2.0
""", tsa)
tsb = sapi.get('test-formula')
assert_df(
"""
2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
""", tsb)
class supervision_and_formula(supervision_class(), formula_class()):
pass
sapi = timeseries(api.uri, api.namespace, supervision_and_formula)
tsa = sapi.get('test-features')
assert_df(
"""
2020-01-01 00:00:00+00:00 0.0
2020-01-02 00:00:00+00:00 1.0
2020-01-03 00:00:00+00:00 2.0
""", tsa)
tsb = sapi.get('test-formula')
assert_df(
"""
2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
""", tsb)
sapi.update('test-features',
genserie(utcdt(2020, 1, 2), 'D', 3),
'Babar',
manual=True)
tsb = sapi.get('test-formula')
assert_df(
"""
2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 1.0
2020-01-03 00:00:00+00:00 2.0
2020-01-04 00:00:00+00:00 3.0
""", tsb)
def test_log(engine, cli, tsh):
serie = genserie(datetime(2020, 1, 1), 'D', 3)
tsh.update(engine, serie, 'log_me', 'Babar')
serie = genserie(datetime(2020, 1, 2), 'D', 3)
tsh.update(engine, serie, 'log_me', 'Babar')
r = cli('log', engine.url, 'log_me', namespace=tsh.namespace)
assert r.output.count('revision:') == 2
def test_delete(engine, cli, datadir, tsh):
serie = genserie(datetime(2020, 1, 1), 'D', 3)
tsh.update(engine, serie, 'bfoo', 'Babar')
tsh.update(engine, serie, 'bbar', 'Babar')
tsh.update(engine, serie, 'bquux', 'Babar')
r = cli('delete',
engine.url,
deletefile=datadir / 'delete.csv',
namespace=tsh.namespace)
tsh = timeseries(tsh.namespace)
tsh._testing = True
assert not tsh.exists(engine, 'bfoo')
assert tsh.get(engine, 'bfoo') is None
assert tsh.get(engine, 'bbar') is None
assert tsh.get(engine, 'bquux') is not None
tsh.update(engine, serie, 'bbq', 'Babar')
tsh = timeseries(tsh.namespace)
tsh._testing = True
r = cli('delete', engine.url, series='bbq', namespace=tsh.namespace)
assert not tsh.exists(engine, 'bbq')
def test_multisources(client, engine):
series = genserie(utcdt(2020, 1, 1), 'D', 3)
tsh = tsio.timeseries('other')
tsh.update(engine, series, 'test-other', 'Babar')
client.update('test-mainsource', series, 'Babar')
with pytest.raises(ValueError) as err:
client.update('test-other', series, 'Babar')
assert err.value.args[0] == 'not allowed to update to a secondary source'
with pytest.raises(ValueError) as err:
client.replace('test-other', series, 'Babar')
assert err.value.args[0] == 'not allowed to replace to a secondary source'
cat = client.catalog()
assert cat == {
('db://localhost:5433/postgres', 'other'): [['test-other', 'primary']],
('db://localhost:5433/postgres', 'tsh'):
[['test-naive', 'primary'], ['test', 'primary'],
['staircase', 'primary'], ['staircase-naive', 'primary'],
['in-a-formula', 'primary'], ['test-mainsource', 'primary'],
['new-formula', 'formula']]
}
cat = client.catalog(allsources=False)
assert ('db://localhost:5433/postgres', 'tsh') in cat
assert ('db://localhost:5433/postgres', 'other') not in cat
def test_naive(client):
series_in = genserie(pd.Timestamp('2018-1-1'), 'H', 3)
res = client.patch('/series/state',
params={
'name': 'test-naive',
'series': util.tojson(series_in),
'author': 'Babar',
'insertion_date': utcdt(2018, 1, 1, 10),
'tzaware': util.tzaware_serie(series_in)
})
assert res.status_code == 201
res = client.get('/series/metadata?name=test-naive&all=1')
meta = res.json
assert meta == {
'index_dtype': '<M8[ns]',
'index_type': 'datetime64[ns]',
'tzaware': False,
'value_dtype': '<f8',
'value_type': 'float64'
}
res = client.get('/series/state?name=test-naive')
series = util.fromjson(res.body, 'test', meta['tzaware'])
assert_df(
"""
2018-01-01 00:00:00 0.0
2018-01-01 01:00:00 1.0
2018-01-01 02:00:00 2.0
""", series)
def test_staircase_history_naive(client, tsh):
# each days we insert 7 data points
from datetime import datetime
for idx, idate in enumerate(
pd.date_range(start=utcdt(2015, 1, 1),
end=utcdt(2015, 1, 4),
freq='D')):
series = genserie(start=idate.tz_convert(None), freq='H', repeat=7)
client.update('staircase-naive', series, 'Babar', insertion_date=idate)
series = client.staircase('staircase-naive',
pd.Timedelta(hours=3),
from_value_date=datetime(2015, 1, 1, 4),
to_value_date=datetime(2015, 1, 2, 5))
assert series.name == 'staircase-naive'
assert_df(
"""
2015-01-01 04:00:00 4.0
2015-01-01 05:00:00 5.0
2015-01-01 06:00:00 6.0
2015-01-02 03:00:00 3.0
2015-01-02 04:00:00 4.0
2015-01-02 05:00:00 5.0
""", series)
# series = client.staircase(
# 'staircase-naive',
# pd.Timedelta(hours=3),
# from_value_date=datetime(2015, 1, 1, 4),
# to_value_date=datetime(2015, 1, 2, 5)
# )
hist = client.history('staircase-naive')
assert len(hist) == 4
hist = client.history('staircase-naive',
from_insertion_date=datetime(2015, 1, 2),
to_insertion_date=datetime(2015, 1, 3))
assert len(hist) == 2
hist = client.history('staircase-naive',
from_value_date=datetime(2015, 1, 1, 3),
to_value_date=datetime(2015, 1, 2, 1))
assert all(series.name == 'staircase-naive' for series in hist.values())
assert_hist(
"""
insertion_date value_date
2015-01-01 00:00:00+00:00 2015-01-01 03:00:00 3.0
2015-01-01 04:00:00 4.0
2015-01-01 05:00:00 5.0
2015-01-01 06:00:00 6.0
2015-01-02 00:00:00+00:00 2015-01-01 03:00:00 3.0
2015-01-01 04:00:00 4.0
2015-01-01 05:00:00 5.0
2015-01-01 06:00:00 6.0
2015-01-02 00:00:00 0.0
2015-01-02 01:00:00 1.0
""", hist)
def test_check(engine, cli, tsh):
serie = genserie(datetime(2020, 1, 1), 'D', 3)
tsh.update(engine, serie, 'things', 'Babar')
tsh.update(engine, serie, 'other_things', 'Babar')
r = cli('check', engine.url, namespace=tsh.namespace)
assert 'things inserts=1, read-time=' in r.output
assert 'other_things inserts=1, read-time=' in r.output
def test_dispatch_get(engine, tsh):
ts_real = genserie(datetime(2010, 1, 1), 'D', 5, [1])
ts_nomination = genserie(datetime(2010, 1, 1), 'D', 6, [2])
ts_forecast = genserie(datetime(2010, 1, 1), 'D', 7, [3])
tsh.insert(engine, ts_real, 'realised0', 'test')
tsh.insert(engine, ts_nomination, 'nominated0', 'test')
tsh.insert(engine, ts_forecast, 'forecasted0', 'test')
tsh.build_priority(engine, 'composite_serie',
['realised0', 'nominated0', 'forecasted0'])
tsh.build_arithmetic(engine, 'sum_serie', {'realised0':1,
'forecasted0':1})
assert 'primary' == tsh.type(engine, 'realised0')
assert 'priority' == tsh.type(engine, 'composite_serie')
assert 'arithmetic' == tsh.type(engine, 'sum_serie')
assert tsh.type(engine, 'serie_not_defined') == 'primary'
assert_df("""
2010-01-01 2.0
2010-01-02 2.0
2010-01-03 2.0
2010-01-04 2.0
2010-01-05 2.0
2010-01-06 2.0
""", tsh.get(engine, 'nominated0'))
assert_df("""
2010-01-01 1.0
2010-01-02 1.0
2010-01-03 1.0
2010-01-04 1.0
2010-01-05 1.0
2010-01-06 2.0
2010-01-07 3.0
""", tsh.get(engine, 'composite_serie'))
assert_df("""
2010-01-01 4.0
2010-01-02 4.0
2010-01-03 4.0
2010-01-04 4.0
2010-01-05 4.0
""", tsh.get(engine,'sum_serie'))
def test_staircase(client):
# each days we insert 7 data points
for idate in pd.date_range(start=utcdt(2015, 1, 1),
end=utcdt(2015, 1, 4),
freq='D'):
series = genserie(start=idate, freq='H', repeat=7)
client.patch('/series/state',
params={
'name': 'staircase',
'series': util.tojson(series),
'author': 'Babar',
'insertion_date': idate,
'tzaware': util.tzaware_serie(series)
})
res = client.get('/series/staircase',
params={
'name': 'staircase',
'delta': pd.Timedelta(hours=3),
'from_value_date': utcdt(2015, 1, 1, 4),
'to_value_date': utcdt(2015, 1, 2, 5),
})
series = util.fromjson(res.body, 'test', True)
assert_df(
"""
2015-01-01 04:00:00+00:00 4.0
2015-01-01 05:00:00+00:00 5.0
2015-01-01 06:00:00+00:00 6.0
2015-01-02 03:00:00+00:00 3.0
2015-01-02 04:00:00+00:00 4.0
2015-01-02 05:00:00+00:00 5.0
""", series)
res = client.get('/series/staircase',
params={
'name': 'staircase',
'delta': pd.Timedelta(hours=3),
'from_value_date': utcdt(2015, 1, 1, 4),
'to_value_date': utcdt(2015, 1, 2, 5),
'format': 'tshpack'
})
meta, index, values = util.nary_unpack(zlib.decompress(res.body))
meta = json.loads(meta)
index, values = util.numpy_deserialize(index, values, meta)
series = pd.Series(values, index=index)
series = series.tz_localize('UTC')
assert_df(
"""
2015-01-01 04:00:00+00:00 4.0
2015-01-01 05:00:00+00:00 5.0
2015-01-01 06:00:00+00:00 6.0
2015-01-02 03:00:00+00:00 3.0
2015-01-02 04:00:00+00:00 4.0
2015-01-02 05:00:00+00:00 5.0
""", series)
def test_history(engine, cli, tsh):
serie = genserie(datetime(2020, 1, 1), 'D', 3)
tsh.update(engine,
serie,
'some_history',
'Babar',
insertion_date=utcdt(2019, 1, 1))
serie = genserie(datetime(2020, 1, 2), 'D', 3)
tsh.update(engine,
serie,
'some_history',
'Babar',
insertion_date=utcdt(2019, 1, 2))
r = cli('history', engine.url, 'some_history', json=True)
assert r.output == (
'{"2019-01-01 00:00:00+00:00": {"2020-01-01 00:00:00": 0.0, "2020-01-02 00:00:00": 1.0, "2020-01-03 00:00:00": 2.0}, "2019-01-02 00:00:00+00:00": {"2020-01-02 00:00:00": 0.0, "2020-01-03 00:00:00": 1.0, "2020-01-04 00:00:00": 2.0}}\n'
)
def test_info(engine, cli, tsh):
serie = genserie(datetime(2020, 1, 1), 'D', 3)
tsh.update(engine, serie, 'someseries', 'Babar')
r = cli('info', engine.url, namespace=tsh.namespace)
out = r.output.strip().split('\n')
assert len(out) == 2
assert out[0].startswith('series count:')
assert out[1].startswith('series names:')
def create_data():
# 4 years of sub-hourly points
for year in range(2015, 2020):
date = utcdt(year, 1, 1)
serie = genserie(date, '10Min', 6 * 24 * 365)
with engine.begin() as cn:
tsh.update(cn,
serie,
'big',
'*****@*****.**',
insertion_date=date)
def test_in_tx(tsh, engine):
assert tsh.type(engine, 'foo') == 'primary'
with pytest.raises(TypeError) as err:
tsh.update(engine.connect(), 0, 0, 0)
assert err.value.args[0] == 'You must use a transaction object'
ts = genserie(datetime(2017, 10, 28, 23), 'H', 4, tz='UTC')
with engine.begin() as cn:
tsh.update(cn, ts, 'test_tx', 'Babar')
# that old api is still there
tsh.insert(cn, ts, 'test_tx', 'Babar')
def test_get_from_to(engine, tsh):
ts = genserie(datetime(2015, 1, 1), 'D', 365)
tsh.update(engine, ts, 'quitelong', '*****@*****.**')
snap = Snapshot(engine, tsh, 'quitelong')
if tsh.namespace == 'z-z':
sql = 'select id, parent from "z-z.snapshot".quitelong order by id'
chunks = engine.execute(sql).fetchall()
# should be perfectly chained
chunks = {
chunk.id: chunk.parent for chunk in chunks
}
chunks.pop(1)
assert all(k == v+1 for k, v in chunks.items())
snap = Snapshot(engine, tsh, 'quitelong')
chunks = chunksize(snap, 73)
assert chunks == {None: 5, 1: 5, 2: 5, 3: 5, 4: 5, 5: 5, 6: 5, 7: 5,
8: 5, 9: 5, 10: 5, 11: 5, 12: 5, 13: 5, 14: 5, 15: 5,
16: 5, 17: 5, 18: 5, 19: 5, 20: 5, 21: 5, 22: 5,
23: 5, 24: 5, 25: 5, 26: 5, 27: 5, 28: 5, 29: 5,
30: 5, 31: 5, 32: 5, 33: 5, 34: 5, 35: 5, 36: 5,
37: 5, 38: 5, 39: 5, 40: 5, 41: 5, 42: 5, 43: 5, 44: 5,
45: 5, 46: 5, 47: 5, 48: 5, 49: 5, 50: 5, 51: 5, 52: 5,
53: 5, 54: 5, 55: 5, 56: 5, 57: 5, 58: 5, 59: 5, 60: 5,
61: 5, 62: 5, 63: 5, 64: 5, 65: 5, 66: 5, 67: 5, 68: 5,
69: 5, 70: 5, 71: 5, 72: 5}
chunks = chunksize(snap, 73, datetime(2015, 5, 1))
assert chunks == {24: 5, 25: 5, 26: 5, 27: 5, 28: 5, 29: 5, 30: 5, 31: 5,
32: 5, 33: 5, 34: 5, 35: 5, 36: 5, 37: 5, 38: 5, 39: 5,
40: 5, 41: 5, 42: 5, 43: 5, 44: 5, 45: 5, 46: 5, 47: 5,
48: 5, 49: 5, 50: 5, 51: 5, 52: 5, 53: 5, 54: 5, 55: 5,
56: 5, 57: 5, 58: 5, 59: 5, 60: 5, 61: 5, 62: 5, 63: 5,
64: 5, 65: 5, 66: 5, 67: 5, 68: 5, 69: 5, 70: 5, 71: 5,
72: 5}
serie = tsh.get(engine, 'quitelong')
assert serie.index[0] == pd.Timestamp('2015-01-01 00:00:00')
assert serie.index[-1] == pd.Timestamp('2015-12-31 00:00:00')
serie = tsh.get(engine, 'quitelong',
from_value_date=datetime(2015, 5, 1),
to_value_date=datetime(2015, 6, 1))
assert serie.index[0] == pd.Timestamp('2015-05-01 00:00:00')
assert serie.index[-1] == pd.Timestamp('2015-06-01 00:00:00')
serie = tsh.get(engine, 'quitelong',
from_value_date=datetime(2016, 5, 1),
to_value_date=datetime(2016, 6, 1))
assert serie.dtypes == np.dtype('float64')
assert len(serie) == 0
assert isinstance(serie.index, pd.DatetimeIndex)
assert serie.index.freq is None
def create_data():
# one insert per day for 4 months
for month in range(1, 4):
days = calendar.monthrange(2017, month)[1]
for day in range(1, days + 1):
date = utcdt(2017, month, day)
serie = genserie(date, '10Min', 6 * 24)
with engine.begin() as cn:
tsh.update(cn,
serie,
'manydiffs',
'*****@*****.**',
insertion_date=date.replace(year=2018))
def test_naive(client, engine, tsh):
series_in = genserie(pd.Timestamp('2018-1-1'), 'H', 3)
client.update('test-naive',
series_in,
'Babar',
insertion_date=utcdt(2019, 1, 1))
# now let's get it back
ts = client.get('test-naive')
assert_df(
"""
2018-01-01 00:00:00 0.0
2018-01-01 01:00:00 1.0
2018-01-01 02:00:00 2.0
""", ts)
assert not getattr(ts.index.dtype, 'tz', False)
def test_delete(client):
series_in = genserie(utcdt(2018, 1, 1), 'H', 3)
res = client.patch('/series/state',
params={
'name': 'test',
'series': util.tojson(series_in),
'author': 'Babar',
'insertion_date': utcdt(2018, 1, 1, 10),
'tzaware': util.tzaware_serie(series_in)
})
res = client.delete('/series/state', params={'name': 'no-such-series'})
assert res.status_code == 404
res = client.delete('/series/state', params={'name': 'test'})
assert res.status_code == 204
res = client.get('/series/catalog')
assert 'test' not in res.json
def test_rename(engine, cli, datadir, tsh):
serie = genserie(datetime(2020, 1, 1), 'D', 3)
tsh.update(engine, serie, 'afoo', 'Babar')
tsh.update(engine, serie, 'abar', 'Babar')
tsh.update(engine, serie, 'aquux', 'Babar')
r = cli('rename',
engine.url,
datadir / 'rename.csv',
namespace=tsh.namespace)
assert tsh.get(engine, 'afoo') is None
assert tsh.get(engine, 'abar') is None
for name in ('aquux', 'anew-foo', 'anew-bar'):
assert tsh.get(engine, name) is not None
def test_rename(client):
series_in = genserie(utcdt(2018, 1, 1), 'H', 3)
res = client.patch('/series/state',
params={
'name': 'test',
'series': util.tojson(series_in),
'author': 'Babar',
'insertion_date': utcdt(2018, 1, 1, 10),
'tzaware': util.tzaware_serie(series_in)
})
res = client.put('/series/state',
params={
'name': 'no-such-series',
'newname': 'no-better'
})
assert res.status_code == 404
res = client.put('/series/state',
params={
'name': 'test',
'newname': 'test2'
})
assert res.status_code == 204
res = client.get('/series/catalog')
assert res.json == {
'db://localhost:5433/postgres!tsh': [['test-naive', 'primary'],
['test2', 'primary']]
}
assert 'test' not in res.json
res = client.patch('/series/state',
params={
'name': 'test3',
'series': util.tojson(series_in),
'author': 'Babar',
'insertion_date': utcdt(2018, 1, 1, 10),
'tzaware': util.tzaware_serie(series_in)
})
res = client.put('/series/state',
params={
'name': 'test2',
'newname': 'test3'
})
assert res.status_code == 409
assert res.json == {'message': '`test3` does exists'}
def test_outliers(engine, tsh):
tsh.add_bounds(engine, 'serie1', min=5, max=10)
tsh.add_bounds(engine, 'serie2', max=10)
tsh.add_bounds(engine, 'serie3', min=5)
tsh.add_bounds(engine, 'serie4', min=50)
ts = genserie(datetime(2015,1,1), 'D', 15)
tsh.insert(engine, ts, 'serie1', 'test')
tsh.insert(engine, ts, 'serie2', 'test')
tsh.insert(engine, ts, 'serie3', 'test')
tsh.insert(engine, ts, 'serie4', 'test')
ts1 = tsh.get(engine, 'serie1')
ts2 = tsh.get(engine, 'serie2',
from_value_date=datetime(2015,1,2),
to_value_date=datetime(2015, 1, 13))
ts3 = tsh.get(engine, 'serie3')
ts4 = tsh.get(engine, 'serie4')
assert_df("""
2015-01-06 5.0
2015-01-07 6.0
2015-01-08 7.0
2015-01-09 8.0
2015-01-10 9.0
2015-01-11 10.0
""", ts1)
assert 10 == max(ts2)
assert 1 == min(ts2)
assert 14 == max(ts3)
assert 5 == min(ts3)
assert 0 == len(ts4)
# upsert:
tsh.add_bounds(engine, 'serie4', min=-50)
ts4 = tsh.get(engine, 'serie4')
assert 15 == len(ts4)
def test_append(engine, tsh):
if tsh.namespace == 'z-z':
return
for x, dt in enumerate(pd.date_range(start=utcdt(2018, 1, 1),
freq='D', periods=10)):
ts = genserie(dt, 'D', 1, [x], name='daily')
tsh.update(engine, ts, 'append', '*****@*****.**',
insertion_date=dt)
sql = 'select id, parent, chunk from "{}.snapshot".append order by id'.format(
tsh.namespace
)
chunks = engine.execute(sql).fetchall()
c = {
chunk.id: chunk.parent for chunk in chunks
}
# nice linked list
assert c == {
1: None, 2: 1, 3: 2, 4: 3, 5: 4, 6: 5, 7: 6, 8: 7, 9: 8, 10: 9
}
def test_get_fast_path(client):
series_in = genserie(utcdt(2018, 1, 1), 'H', 3)
res = client.patch('/series/state',
params={
'name': 'test_fast',
'series': util.tojson(series_in),
'author': 'Babar',
'insertion_date': utcdt(2018, 1, 1, 10),
'tzaware': util.tzaware_serie(series_in)
})
assert res.status_code == 201
out = client.get('/series/state',
params={
'name': 'test_fast',
'format': 'tshpack'
})
meta, index, values = util.nary_unpack(zlib.decompress(out.body))
meta = json.loads(meta)
index, values = util.numpy_deserialize(index, values, meta)
series = pd.Series(values, index=index)
series = series.tz_localize('UTC')
assert_df(
"""
2018-01-01 00:00:00+00:00 0.0
2018-01-01 01:00:00+00:00 1.0
2018-01-01 02:00:00+00:00 2.0
""", series)
assert meta == {
'tzaware': True,
'index_type': 'datetime64[ns, UTC]',
'value_type': 'float64',
'index_dtype': '|M8[ns]',
'value_dtype': '<f8'
}
def test_prepend(engine, tsh):
if tsh.namespace == 'z-z':
return
serie = genserie(datetime(2010, 1, 1), 'D', 40)
assert 40 == len(serie)
ts_insert = serie[2:]
tsh.update(engine, ts_insert, 'prepend', 'test')
assert 38 == len(tsh.get(engine, 'prepend'))
tsh.update(engine, serie, 'prepend', 'test')
assert 40 == len(tsh.get(engine, 'prepend'))
sql = 'select id, parent, chunk from "{}.snapshot".prepend order by id'.format(
tsh.namespace
)
chunks = engine.execute(sql).fetchall()
c = {
chunk.id: chunk.parent for chunk in chunks
}
# ugly un-linked list
assert c == {1: None, 2: None}
def test_combine(engine, tsh):
ts_real = genserie(datetime(2010, 1, 1), 'D', 10, [1])
ts_nomination = genserie(datetime(2010, 1, 1), 'D', 12, [2])
ts_forecast = genserie(datetime(2010, 1, 1), 'D', 20, [3])
tsh.insert(engine, ts_real, 'realised', 'test')
tsh.insert(engine, ts_nomination, 'nominated', 'test')
tsh.insert(engine, ts_forecast, 'forecasted', 'test')
tsh.build_priority(engine, 'serie5',
['realised', 'nominated', 'forecasted'])
values, origin = tsh.get_priority(engine,'serie5')
assert_df("""
2010-01-01 1.0
2010-01-02 1.0
2010-01-03 1.0
2010-01-04 1.0
2010-01-05 1.0
2010-01-06 1.0
2010-01-07 1.0
2010-01-08 1.0
2010-01-09 1.0
2010-01-10 1.0
2010-01-11 2.0
2010-01-12 2.0
2010-01-13 3.0
2010-01-14 3.0
2010-01-15 3.0
2010-01-16 3.0
2010-01-17 3.0
2010-01-18 3.0
2010-01-19 3.0
2010-01-20 3.0
""", values)
assert_df("""
2010-01-01 realised
2010-01-02 realised
2010-01-03 realised
2010-01-04 realised
2010-01-05 realised
2010-01-06 realised
2010-01-07 realised
2010-01-08 realised
2010-01-09 realised
2010-01-10 realised
2010-01-11 nominated
2010-01-12 nominated
2010-01-13 forecasted
2010-01-14 forecasted
2010-01-15 forecasted
2010-01-16 forecasted
2010-01-17 forecasted
2010-01-18 forecasted
2010-01-19 forecasted
2010-01-20 forecasted
""", origin)
# we remove the last value of the 2 first series which are considered as bogus
tsh.build_priority(engine, 'serie6',
['realised', 'nominated', 'forecasted'],
map_prune={'realised': 1, 'nominated': 1, 'forecasted': 0})
values, origin = tsh.get_priority(engine,'serie6')
assert_df("""
2010-01-01 1.0
2010-01-02 1.0
2010-01-03 1.0
2010-01-04 1.0
2010-01-05 1.0
2010-01-06 1.0
2010-01-07 1.0
2010-01-08 1.0
2010-01-09 1.0
2010-01-10 2.0
2010-01-11 2.0
2010-01-12 3.0
2010-01-13 3.0
2010-01-14 3.0
2010-01-15 3.0
2010-01-16 3.0
2010-01-17 3.0
2010-01-18 3.0
2010-01-19 3.0
2010-01-20 3.0
""", values)
assert_df("""
2010-01-01 realised
2010-01-02 realised
2010-01-03 realised
2010-01-04 realised
2010-01-05 realised
2010-01-06 realised
2010-01-07 realised
2010-01-08 realised
2010-01-09 realised
2010-01-10 nominated
2010-01-11 nominated
2010-01-12 forecasted
2010-01-13 forecasted
2010-01-14 forecasted
2010-01-15 forecasted
2010-01-16 forecasted
2010-01-17 forecasted
2010-01-18 forecasted
2010-01-19 forecasted
2010-01-20 forecasted
""", origin)
tsh.build_priority(engine, 'serie7',
['realised', 'nominated', 'forecasted'],
map_prune={'realised': 1, 'nominated': 3, 'forecasted': 0})
values, origin = tsh.get_priority(engine,'serie7')
assert_df("""
2010-01-01 realised
2010-01-02 realised
2010-01-03 realised
2010-01-04 realised
2010-01-05 realised
2010-01-06 realised
2010-01-07 realised
2010-01-08 realised
2010-01-09 realised
2010-01-10 forecasted
2010-01-11 forecasted
2010-01-12 forecasted
2010-01-13 forecasted
2010-01-14 forecasted
2010-01-15 forecasted
2010-01-16 forecasted
2010-01-17 forecasted
2010-01-18 forecasted
2010-01-19 forecasted
2010-01-20 forecasted
""", origin)
def test_historical(engine, tsh):
for insertion_date in pd.date_range(start=datetime(2015, 1, 1),
end=datetime(2015, 1, 1, 6),
freq='H'):
ts = genserie(start=insertion_date, freq='H', repeat=7, tz='UTC')
tsh.insert(engine, ts, 'republication', 'test',
insertion_date=pd.Timestamp(insertion_date, tz='UTC'))
# this is the end state
# initially the first 1 was at 01:00 then at each insertion it gets
# moved one hour later
ts = tsh.get(engine, 'republication')
assert_df("""
2015-01-01 00:00:00+00:00 0.0
2015-01-01 01:00:00+00:00 0.0
2015-01-01 02:00:00+00:00 0.0
2015-01-01 03:00:00+00:00 0.0
2015-01-01 04:00:00+00:00 0.0
2015-01-01 05:00:00+00:00 0.0
2015-01-01 06:00:00+00:00 0.0
2015-01-01 07:00:00+00:00 1.0
2015-01-01 08:00:00+00:00 2.0
2015-01-01 09:00:00+00:00 3.0
2015-01-01 10:00:00+00:00 4.0
2015-01-01 11:00:00+00:00 5.0
2015-01-01 12:00:00+00:00 6.0
""", ts)
tsh.insert(engine, genserie(datetime(2015, 1, 1), 'H', 15, [10], tz='UTC'),
'stable', 'test',
insertion_date=pd.Timestamp(datetime(2015, 1, 1), tz='UTC'))
# tens all over the place inserted at hour zero
# there are two more tstamps than in republication
assert_df("""
2015-01-01 00:00:00+00:00 10.0
2015-01-01 01:00:00+00:00 10.0
2015-01-01 02:00:00+00:00 10.0
2015-01-01 03:00:00+00:00 10.0
2015-01-01 04:00:00+00:00 10.0
2015-01-01 05:00:00+00:00 10.0
2015-01-01 06:00:00+00:00 10.0
2015-01-01 07:00:00+00:00 10.0
2015-01-01 08:00:00+00:00 10.0
2015-01-01 09:00:00+00:00 10.0
2015-01-01 10:00:00+00:00 10.0
2015-01-01 11:00:00+00:00 10.0
2015-01-01 12:00:00+00:00 10.0
2015-01-01 13:00:00+00:00 10.0
2015-01-01 14:00:00+00:00 10.0
""", tsh.get(engine, 'stable'))
tsh.build_arithmetic(engine, 'operation_past',
{'republication': 1,
'stable': 1}
)
tsh.build_priority(engine, 'compo_past', ['republication', 'stable'])
ts = tsh.get(engine, 'operation_past')
assert_df("""
2015-01-01 00:00:00+00:00 10.0
2015-01-01 01:00:00+00:00 10.0
2015-01-01 02:00:00+00:00 10.0
2015-01-01 03:00:00+00:00 10.0
2015-01-01 04:00:00+00:00 10.0
2015-01-01 05:00:00+00:00 10.0
2015-01-01 06:00:00+00:00 10.0
2015-01-01 07:00:00+00:00 11.0
2015-01-01 08:00:00+00:00 12.0
2015-01-01 09:00:00+00:00 13.0
2015-01-01 10:00:00+00:00 14.0
2015-01-01 11:00:00+00:00 15.0
2015-01-01 12:00:00+00:00 16.0
""", ts)
ts = tsh.get(engine, 'compo_past')
assert_df("""
2015-01-01 00:00:00+00:00 0.0
2015-01-01 01:00:00+00:00 0.0
2015-01-01 02:00:00+00:00 0.0
2015-01-01 03:00:00+00:00 0.0
2015-01-01 04:00:00+00:00 0.0
2015-01-01 05:00:00+00:00 0.0
2015-01-01 06:00:00+00:00 0.0
2015-01-01 07:00:00+00:00 1.0
2015-01-01 08:00:00+00:00 2.0
2015-01-01 09:00:00+00:00 3.0
2015-01-01 10:00:00+00:00 4.0
2015-01-01 11:00:00+00:00 5.0
2015-01-01 12:00:00+00:00 6.0
2015-01-01 13:00:00+00:00 10.0
2015-01-01 14:00:00+00:00 10.0
""", ts)
assert_df("""
2015-01-01 00:00:00+00:00 10.0
2015-01-01 01:00:00+00:00 10.0
2015-01-01 02:00:00+00:00 10.0
2015-01-01 03:00:00+00:00 10.0
2015-01-01 04:00:00+00:00 11.0
2015-01-01 05:00:00+00:00 12.0
2015-01-01 06:00:00+00:00 13.0
2015-01-01 07:00:00+00:00 14.0
2015-01-01 08:00:00+00:00 15.0
2015-01-01 09:00:00+00:00 16.0
""", tsh.get(engine, 'operation_past',
revision_date=utcdt(2015, 1, 1, 3))
)
assert_df("""
2015-01-01 00:00:00+00:00 0.0
2015-01-01 01:00:00+00:00 0.0
2015-01-01 02:00:00+00:00 0.0
2015-01-01 03:00:00+00:00 0.0
2015-01-01 04:00:00+00:00 1.0
2015-01-01 05:00:00+00:00 2.0
2015-01-01 06:00:00+00:00 3.0
2015-01-01 07:00:00+00:00 4.0
2015-01-01 08:00:00+00:00 5.0
2015-01-01 09:00:00+00:00 6.0
2015-01-01 10:00:00+00:00 10.0
2015-01-01 11:00:00+00:00 10.0
2015-01-01 12:00:00+00:00 10.0
2015-01-01 13:00:00+00:00 10.0
2015-01-01 14:00:00+00:00 10.0
""", tsh.get(engine, 'compo_past',
revision_date=utcdt(2015, 1, 1, 3))
)
assert_df("""
2015-01-01 03:00:00+00:00 13.0
2015-01-01 04:00:00+00:00 13.0
2015-01-01 05:00:00+00:00 13.0
2015-01-01 06:00:00+00:00 13.0
2015-01-01 07:00:00+00:00 13.0
2015-01-01 08:00:00+00:00 13.0
2015-01-01 09:00:00+00:00 13.0
2015-01-01 10:00:00+00:00 14.0
2015-01-01 11:00:00+00:00 15.0
2015-01-01 12:00:00+00:00 16.0
""", tsh.get(engine, 'operation_past',
delta=timedelta(hours=2.5))
)
assert_df("""
2015-01-01 03:00:00+00:00 3.0
2015-01-01 04:00:00+00:00 3.0
2015-01-01 05:00:00+00:00 3.0
2015-01-01 06:00:00+00:00 3.0
2015-01-01 07:00:00+00:00 3.0
2015-01-01 08:00:00+00:00 3.0
2015-01-01 09:00:00+00:00 3.0
2015-01-01 10:00:00+00:00 4.0
2015-01-01 11:00:00+00:00 5.0
2015-01-01 12:00:00+00:00 6.0
2015-01-01 13:00:00+00:00 10.0
2015-01-01 14:00:00+00:00 10.0
""", tsh.get(engine, 'compo_past',
delta=timedelta(hours=2.5))
)
assert_df("""
2015-01-01 06:00:00+00:00 3.0
2015-01-01 07:00:00+00:00 3.0
2015-01-01 08:00:00+00:00 3.0
2015-01-01 09:00:00+00:00 3.0
2015-01-01 10:00:00+00:00 4.0
2015-01-01 11:00:00+00:00 5.0
""", tsh.get(engine, 'compo_past',
delta=timedelta(hours=2.5),
from_value_date=datetime(2015, 1, 1, 6),
to_value_date=datetime(2015, 1, 1, 11))
)
def test_errors(engine, tsh):
tsh.insert(engine, genserie(datetime(2010, 1, 1), 'D', 5, [1]), 'primary_series', 'test')
tsh.build_arithmetic(engine, 'arithmetic2', {'toto': 1, 'tata': 1})
tsh.build_priority(engine, 'priority2', ['toto', 'tata'])
def test_micmac(engine, tsh, refresh):
tsh.insert(engine, genserie(datetime(2010, 1, 1), 'D', 5, [1]), 'micmac1', 'test')
tsh.insert(engine, genserie(datetime(2010, 1, 2), 'D', 1, [1000]), 'micmac1', 'test') # bogus data
tsh.insert(engine, genserie(datetime(2010, 1, 1), 'D', 6, [2]), 'micmac2', 'test')
tsh.insert(engine, genserie(datetime(2010, 1, 1), 'D', 6, [3]), 'micmac3', 'test')
tsh.insert(engine, genserie(datetime(2010, 1, 1), 'D', 7, [15]), 'micmac4', 'test')
tsh.build_priority(engine, 'prio1',
['micmac1', 'micmac2'],
map_prune={'micmac1': 2, 'micmac2': 1})
tsh.build_arithmetic(engine, 'arithmetic1',
{'prio1':1,
'micmac3':1})
tsh.build_priority(engine, 'final',
['arithmetic1', 'micmac4'],
map_prune={'arithmetic1': 1, 'micmac4': 0})
tsh.add_bounds(engine, 'micmac1', min=0, max=100)
tsh.get(engine, 'micmac1')
tsh.get(engine, 'micmac2')
tsh.get(engine, 'prio1')
tsh.get(engine, 'micmac3')
tsh.get(engine, 'arithmetic1')
tsh.get(engine, 'micmac4')
assert_df("""
2010-01-01 4.0
2010-01-02 5.0
2010-01-03 4.0
2010-01-04 5.0
2010-01-05 15.0
2010-01-06 15.0
2010-01-07 15.0
""", tsh.get(engine, 'final'))
# test get with bounds
assert_df("""
2010-01-03 4.0
2010-01-04 5.0
2010-01-05 15.0
2010-01-06 15.0
""", tsh.get(engine, 'final',
from_value_date = datetime(2010, 1, 3),
to_value_date=datetime(2010, 1, 6)
))
# test get with values out of bounds
ts_out_of_bounds = tsh.get(engine, 'final',
from_value_date=datetime(2013, 1, 1),
to_value_date=datetime(2013, 1, 6)
)
assert 0 == len(ts_out_of_bounds)
# the result is an empty pd.Series
dash_html_table = alias_table(engine, tsh, 'final')
inside = dash_html_table.children
assert 8 == len(inside)
assert 4 == len(inside[0].children)
refpath = DATADIR / 'dash-table.html'
out = str(dash_html_table).encode('utf-8')
if refresh:
refpath.write_bytes(out)
expected = refpath.read_bytes()
assert expected == out
tree = buildtree(engine, tsh, 'final', [])
assert tree == {
('final', 'priority'): [
{('arithmetic1', 'arithmetic'): [
{
('prio1', 'priority'): [
'micmac1',
'micmac2']
},
'micmac3'
]},
'micmac4'
]
}
out = []
showtree(tree, printer=lambda *x: out.append(''.join(x)))
assert out == [
'* priority `final`',
' * arithmetic `arithmetic1`',
' * priority `prio1`',
' -micmac1',
' -micmac2',
' -micmac3',
' -micmac4'
]
tsh.build_priority(engine, 'bogus2',
['prio1', 'no-such-series'])
tree = buildtree(engine, tsh, 'bogus2', [])
out = []
showtree(tree, printer=lambda *x: out.append(''.join(x)))
assert out == [
'* priority `bogus2`',
' * priority `prio1`',
' -micmac1',
' -micmac2',
' -unknown `no-such-series`'
]
# test .exists
assert tsh.exists(engine, 'micmac1')
assert tsh.exists(engine, 'final')
assert tsh.exists(engine, 'arithmetic1')
assert not tsh.exists(engine, 'no-such-series')
def test_arithmetic(engine, tsh):
ts_toto = genserie(datetime(2010, 1, 1), 'D', 7, [1])
ts_tata = genserie(datetime(2010, 1, 3), 'D', 7, [2])
ts_titi = genserie(datetime(2010, 1, 1), 'D', 5, [2])
ts_short = genserie(datetime(2010, 1, 3), 'D', 3, [2])
tsh.insert(engine, ts_toto, 'toto', 'test')
tsh.insert(engine, ts_tata, 'tata', 'test')
tsh.insert(engine, ts_titi, 'titi', 'test')
tsh.insert(engine, ts_short, 'short', 'test')
tsh.build_arithmetic(engine, 'sum', {'toto': 1,
'tata': 1})
tsh.build_arithmetic(engine, 'difference', {'toto': 1,
'tata': -1})
tsh.build_arithmetic(engine, 'mean', {'toto': 0.5,
'tata': 0.5})
tsh.build_arithmetic(engine, 'bogus', {'toto': 0.5,
'unknown': 0.5})
tsh.build_arithmetic(engine, 'forwardfill', {'toto': 1,
'tata': 1}, {'toto': 'ffill'})
tsh.build_arithmetic(engine, 'backwardfill', {'toto': 1,
'tata': 1}, {'tata': 'bfill'})
tsh.build_arithmetic(engine, 'allfill', {'toto': 1,
'short': 1}, {'short': 'bfill,ffill'})
tsh.build_arithmetic(engine, 'storagelike', {'toto': 1,
'titi': 1}, {'titi': 'fill=0'})
values = tsh.get_arithmetic(engine, 'sum')
assert_df("""
2010-01-03 3.0
2010-01-04 3.0
2010-01-05 3.0
2010-01-06 3.0
2010-01-07 3.0
""", values)
# NB: there are only data at the intersection of the index
values = tsh.get_arithmetic(engine, 'difference')
assert_df("""
2010-01-03 -1.0
2010-01-04 -1.0
2010-01-05 -1.0
2010-01-06 -1.0
2010-01-07 -1.0
""", values)
tsh.get_arithmetic(engine, 'mean')
assert_df("""
2010-01-03 -1.0
2010-01-04 -1.0
2010-01-05 -1.0
2010-01-06 -1.0
2010-01-07 -1.0
""", values)
values = tsh.get_arithmetic(engine, 'forwardfill')
assert_df("""
2010-01-03 3.0
2010-01-04 3.0
2010-01-05 3.0
2010-01-06 3.0
2010-01-07 3.0
2010-01-08 3.0
2010-01-09 3.0
""", values)
values = tsh.get_arithmetic(engine, 'backwardfill')
assert_df("""
2010-01-01 3.0
2010-01-02 3.0
2010-01-03 3.0
2010-01-04 3.0
2010-01-05 3.0
2010-01-06 3.0
2010-01-07 3.0
""", values)
values = tsh.get_arithmetic(engine, 'allfill')
assert_df("""
2010-01-01 3.0
2010-01-02 3.0
2010-01-03 3.0
2010-01-04 3.0
2010-01-05 3.0
2010-01-06 3.0
2010-01-07 3.0
""", values)
values = tsh.get_arithmetic(engine, 'storagelike')
assert_df("""
2010-01-01 3.0
2010-01-02 3.0
2010-01-03 3.0
2010-01-04 3.0
2010-01-05 3.0
2010-01-06 1.0
2010-01-07 1.0
""", values)
with pytest.raises(Exception) as err:
tsh.get_arithmetic(engine, 'bogus')
assert 'unknown is needed to calculate bogus and does not exist' == str(err.value)
def test_base(client, engine, tsh):
assert repr(client) == "tshistory-http-client(uri='http://test-uri')"
ts = client.get('no-such-series')
assert ts is None
assert not client.exists('no-such-series')
meta = client.metadata('no-such-series')
assert meta == {'message': '`no-such-series` does not exists'}
series_in = genserie(utcdt(2018, 1, 1), 'H', 3)
client.update('test', series_in, 'Babar', insertion_date=utcdt(2019, 1, 1))
assert client.exists('test')
# now let's get it back
ts = client.get('test')
assert_df(
"""
2018-01-01 00:00:00+00:00 0.0
2018-01-01 01:00:00+00:00 1.0
2018-01-01 02:00:00+00:00 2.0
""", ts)
ts = client.get('test', from_value_date=utcdt(2018, 1, 1, 2))
assert_df("""
2018-01-01 02:00:00+00:00 2.0
""", ts)
ts = client.get('test', to_value_date=utcdt(2018, 1, 1, 0))
assert_df("""
2018-01-01 00:00:00+00:00 0.0
""", ts)
# out of range
ts = client.get('test',
from_value_date=utcdt(2020, 1, 1, 2),
to_value_date=utcdt(2020, 1, 1, 2))
assert len(ts) == 0
assert ts.name == 'test'
meta = client.metadata('test', all=True)
assert meta == {
'tzaware': True,
'index_type': 'datetime64[ns, UTC]',
'value_type': 'float64',
'index_dtype': '|M8[ns]',
'value_dtype': '<f8'
}
# update
client.update_metadata('test', {'desc': 'banana spot price'})
meta = client.metadata('test', all=False)
assert meta == {
'desc': 'banana spot price',
}
# check the insertion_date
series_in = genserie(utcdt(2018, 1, 2), 'H', 3)
client.update('test',
series_in,
'Babar',
metadata={'event': 'hello'},
insertion_date=utcdt(2019, 1, 2))
v1 = client.get('test', revision_date=utcdt(2019, 1, 1))
assert_df(
"""
2018-01-01 00:00:00+00:00 0.0
2018-01-01 01:00:00+00:00 1.0
2018-01-01 02:00:00+00:00 2.0
""", v1)
d1, d2 = tsh.insertion_dates(engine, 'test')
assert d1 == utcdt(2019, 1, 1)
assert d2 > d1
client.update('test2', series_in, 'Babar')
series = client.catalog()
assert ['test',
'primary'] in series[('db://localhost:5433/postgres', 'tsh')]
assert ['test2',
'primary'] in series[('db://localhost:5433/postgres', 'tsh')]
client.replace('test2', genserie(utcdt(2020, 1, 1), 'D', 3), 'Babar')
series = client.get('test2')
assert_df(
"""
2020-01-01 00:00:00+00:00 0.0
2020-01-02 00:00:00+00:00 1.0
2020-01-03 00:00:00+00:00 2.0
""", series)
type = client.type('test2')
assert type == 'primary'
ival = client.interval('test2')
assert ival.left == pd.Timestamp('2020-01-01 00:00:00+0000', tz='UTC')
assert ival.right == pd.Timestamp('2020-01-03 00:00:00+0000', tz='UTC')
client.rename('test2', 'test3')
assert not client.exists('test2')
assert client.exists('test3')
client.delete('test3')
assert not client.exists('test3')
