def create(uri, ns, name):
api = timeseries(uri, ns)
series = pd.Series([1, 2, 3],
index=pd.date_range(utcdt(2020, 1, 1),
periods=3,
freq='D'))
api.update(name,
series,
'Babar',
insertion_date=utcdt(2019, 1, 1),
metadata={'about': 'test'})
out = api.get(name)
assert_df(
"""
2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
""", out)
series[utcdt(2020, 1, 4)] = 4
api.update(name, series, 'Babar', insertion_date=utcdt(2019, 1, 2))
out = api.get(name,
from_value_date=utcdt(2020, 1, 2),
to_value_date=utcdt(2020, 1, 3))
assert_df(
"""
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
""", out)
def test_naive(client):
series_in = genserie(pd.Timestamp('2018-1-1'), 'H', 3)
res = client.patch('/series/state',
params={
'name': 'test-naive',
'series': util.tojson(series_in),
'author': 'Babar',
'insertion_date': utcdt(2018, 1, 1, 10),
'tzaware': util.tzaware_serie(series_in)
})
assert res.status_code == 201
res = client.get('/series/metadata?name=test-naive&all=1')
meta = res.json
assert meta == {
'index_dtype': '<M8[ns]',
'index_type': 'datetime64[ns]',
'tzaware': False,
'value_dtype': '<f8',
'value_type': 'float64'
}
res = client.get('/series/state?name=test-naive')
series = util.fromjson(res.body, 'test', meta['tzaware'])
assert_df(
"""
2018-01-01 00:00:00 0.0
2018-01-01 01:00:00 1.0
2018-01-01 02:00:00 2.0
""", series)
def test_boolean_support(engine, tsh):
@func('op-with-boolean-kw')
def customseries(zeroes: bool = False) -> pd.Series:
return pd.Series(np.array([1.0, 2.0, 3.0]) * zeroes,
index=pd.date_range(dt(2019, 1, 1),
periods=3,
freq='D'))
tsh.register_formula(engine, 'no-zeroes', '(op-with-boolean-kw)')
tsh.register_formula(engine, 'zeroes', '(op-with-boolean-kw #:zeroes #t)')
ts1 = tsh.get(engine, 'no-zeroes')
assert_df("""
2019-01-01 0.0
2019-01-02 0.0
2019-01-03 0.0
""", ts1)
ts2 = tsh.get(engine, 'zeroes')
assert_df("""
2019-01-01 1.0
2019-01-02 2.0
2019-01-03 3.0
""", ts2)
FUNCS.pop('op-with-boolean-kw')
def test_priority2(engine, tsh):
tsh.register_formula(
engine, 'test_prio2',
'(priority (series "real") (series "nom") (series "fcst"))', False)
real = pd.Series([1, 1, 1],
index=pd.date_range(dt(2019, 1, 1), periods=3, freq='D'))
nom = pd.Series([10, 10, 10, 10],
index=pd.date_range(dt(2019, 1, 1), periods=4, freq='D'))
fcst = pd.Series([100, 100, 100, 100, 100],
index=pd.date_range(dt(2019, 1, 1), periods=5, freq='D'))
tsh.update(engine, real, 'real', 'Babar')
tsh.update(engine, nom, 'nom', 'Celeste')
tsh.update(engine, fcst, 'fcst', 'Arthur')
prio = tsh.get(engine, 'test_prio2')
assert_df(
"""
2019-01-01 1.0
2019-01-02 1.0
2019-01-03 1.0
2019-01-04 10.0
2019-01-05 100.0
""", prio)
def test_staircase_history_naive(client, tsh):
# each days we insert 7 data points
from datetime import datetime
for idx, idate in enumerate(
pd.date_range(start=utcdt(2015, 1, 1),
end=utcdt(2015, 1, 4),
freq='D')):
series = genserie(start=idate.tz_convert(None), freq='H', repeat=7)
client.update('staircase-naive', series, 'Babar', insertion_date=idate)
series = client.staircase('staircase-naive',
pd.Timedelta(hours=3),
from_value_date=datetime(2015, 1, 1, 4),
to_value_date=datetime(2015, 1, 2, 5))
assert series.name == 'staircase-naive'
assert_df(
"""
2015-01-01 04:00:00 4.0
2015-01-01 05:00:00 5.0
2015-01-01 06:00:00 6.0
2015-01-02 03:00:00 3.0
2015-01-02 04:00:00 4.0
2015-01-02 05:00:00 5.0
""", series)
# series = client.staircase(
# 'staircase-naive',
# pd.Timedelta(hours=3),
# from_value_date=datetime(2015, 1, 1, 4),
# to_value_date=datetime(2015, 1, 2, 5)
# )
hist = client.history('staircase-naive')
assert len(hist) == 4
hist = client.history('staircase-naive',
from_insertion_date=datetime(2015, 1, 2),
to_insertion_date=datetime(2015, 1, 3))
assert len(hist) == 2
hist = client.history('staircase-naive',
from_value_date=datetime(2015, 1, 1, 3),
to_value_date=datetime(2015, 1, 2, 1))
assert all(series.name == 'staircase-naive' for series in hist.values())
assert_hist(
"""
insertion_date value_date
2015-01-01 00:00:00+00:00 2015-01-01 03:00:00 3.0
2015-01-01 04:00:00 4.0
2015-01-01 05:00:00 5.0
2015-01-01 06:00:00 6.0
2015-01-02 00:00:00+00:00 2015-01-01 03:00:00 3.0
2015-01-01 04:00:00 4.0
2015-01-01 05:00:00 5.0
2015-01-01 06:00:00 6.0
2015-01-02 00:00:00 0.0
2015-01-02 01:00:00 1.0
""", hist)
def test_alternative_handler(pgapi):
api = pgapi
sapi = timeseries(api.uri, api.namespace, formula_class())
sapi.update(
'test-features',
genserie(utcdt(2020, 1, 1), 'D', 3),
'Babar',
)
sapi.tsh.register_formula(sapi.engine, 'test-formula',
'(+ 1 (series "test-features"))')
tsa = sapi.get('test-features')
assert_df(
"""
2020-01-01 00:00:00+00:00 0.0
2020-01-02 00:00:00+00:00 1.0
2020-01-03 00:00:00+00:00 2.0
""", tsa)
tsb = sapi.get('test-formula')
assert_df(
"""
2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
""", tsb)
class supervision_and_formula(supervision_class(), formula_class()):
pass
sapi = timeseries(api.uri, api.namespace, supervision_and_formula)
tsa = sapi.get('test-features')
assert_df(
"""
2020-01-01 00:00:00+00:00 0.0
2020-01-02 00:00:00+00:00 1.0
2020-01-03 00:00:00+00:00 2.0
""", tsa)
tsb = sapi.get('test-formula')
assert_df(
"""
2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
""", tsb)
sapi.update('test-features',
genserie(utcdt(2020, 1, 2), 'D', 3),
'Babar',
manual=True)
tsb = sapi.get('test-formula')
assert_df(
"""
2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 1.0
2020-01-03 00:00:00+00:00 2.0
2020-01-04 00:00:00+00:00 3.0
""", tsb)
def test_staircase(client):
# each days we insert 7 data points
for idate in pd.date_range(start=utcdt(2015, 1, 1),
end=utcdt(2015, 1, 4),
freq='D'):
series = genserie(start=idate, freq='H', repeat=7)
client.patch('/series/state',
params={
'name': 'staircase',
'series': util.tojson(series),
'author': 'Babar',
'insertion_date': idate,
'tzaware': util.tzaware_serie(series)
})
res = client.get('/series/staircase',
params={
'name': 'staircase',
'delta': pd.Timedelta(hours=3),
'from_value_date': utcdt(2015, 1, 1, 4),
'to_value_date': utcdt(2015, 1, 2, 5),
})
series = util.fromjson(res.body, 'test', True)
assert_df(
"""
2015-01-01 04:00:00+00:00 4.0
2015-01-01 05:00:00+00:00 5.0
2015-01-01 06:00:00+00:00 6.0
2015-01-02 03:00:00+00:00 3.0
2015-01-02 04:00:00+00:00 4.0
2015-01-02 05:00:00+00:00 5.0
""", series)
res = client.get('/series/staircase',
params={
'name': 'staircase',
'delta': pd.Timedelta(hours=3),
'from_value_date': utcdt(2015, 1, 1, 4),
'to_value_date': utcdt(2015, 1, 2, 5),
'format': 'tshpack'
})
meta, index, values = util.nary_unpack(zlib.decompress(res.body))
meta = json.loads(meta)
index, values = util.numpy_deserialize(index, values, meta)
series = pd.Series(values, index=index)
series = series.tz_localize('UTC')
assert_df(
"""
2015-01-01 04:00:00+00:00 4.0
2015-01-01 05:00:00+00:00 5.0
2015-01-01 06:00:00+00:00 6.0
2015-01-02 03:00:00+00:00 3.0
2015-01-02 04:00:00+00:00 4.0
2015-01-02 05:00:00+00:00 5.0
""", series)
def test_scalar_div(engine, tsh):
a = pd.Series([1, 2, 3],
index=pd.date_range(dt(2019, 1, 1), periods=3, freq='D'))
tsh.update(engine, a, 'div-me', 'Babar')
ts = tsh.eval_formula(engine, '(/ (series "div-me") (/ 3 2))')
assert_df(
"""
2019-01-01 0.666667
2019-01-02 1.333333
2019-01-03 2.000000
""", ts)
def test_staircase(engine, tsh):
tsh.register_formula(engine, 's-addition',
'(add (series "sa") (series "sb"))', False)
for day in (1, 2, 3, 4, 5):
idate = utcdt(2018, 1, day)
for name in 'ab':
ts = pd.Series([day / 2.] * 5,
index=pd.date_range(dt(2018, 1, day),
periods=5,
freq='D'))
tsh.update(engine, ts, 's' + name, 'Babar', insertion_date=idate)
ts = tsh.staircase(engine, 's-addition', delta=pd.Timedelta(hours=12))
assert_df(
"""
2018-01-02 1.0
2018-01-03 2.0
2018-01-04 3.0
2018-01-05 4.0
2018-01-06 5.0
2018-01-07 5.0
2018-01-08 5.0
2018-01-09 5.0
""", ts)
# this is not allowed in the staircase fast-path
# hence we will take the slow path
@func('identity')
def identity(series: pd.Series) -> pd.Series:
return series
tsh.register_formula(engine, 'slow-down', '(identity (series "sa"))',
False)
tsh.register_formula(engine, 's-addition-not-fast',
'(add (series "slow-down") (series "sb"))', False)
ts = tsh.staircase(engine,
's-addition-not-fast',
delta=pd.Timedelta(hours=12))
assert_df(
"""
2018-01-02 1.0
2018-01-03 2.0
2018-01-04 3.0
2018-01-05 4.0
2018-01-06 5.0
2018-01-07 5.0
2018-01-08 5.0
2018-01-09 5.0
""", ts)
# cleanup
FUNCS.pop('identity')
def test_local_formula_remote_series(mapi):
rtsh = timeseries('test-mapi-2')
rtsh.update(
mapi.engine,
pd.Series(
[1, 2, 3],
index=pd.date_range(pd.Timestamp('2020-1-1'), periods=3, freq='H'),
),
'remote-series',
'Babar',
insertion_date=pd.Timestamp('2020-1-1', tz='UTC')
)
mapi.register_formula(
'test-localformula-remoteseries',
'(+ 1 (series "remote-series"))'
)
ts = mapi.get('test-localformula-remoteseries')
assert_df("""
2020-01-01 00:00:00 2.0
2020-01-01 01:00:00 3.0
2020-01-01 02:00:00 4.0
""", ts)
hist = mapi.history('test-localformula-remoteseries')
assert_hist("""
insertion_date value_date
2020-01-01 00:00:00+00:00 2020-01-01 00:00:00 2.0
2020-01-01 01:00:00 3.0
2020-01-01 02:00:00 4.0
""", hist)
f = mapi.formula('test-localformula-remoteseries')
assert f == '(+ 1 (series "remote-series"))'
none = mapi.formula('nosuchformula')
assert none is None
# altsource formula
rtsh.register_formula(
mapi.engine,
'remote-formula-remote-series',
'(+ 2 (series "remote-series"))'
)
f = mapi.formula('remote-formula-remote-series')
assert f == '(+ 2 (series "remote-series"))'
assert_df("""
2020-01-01 00:00:00 3.0
2020-01-01 01:00:00 4.0
2020-01-01 02:00:00 5.0
""", mapi.get('remote-formula-remote-series'))
def test_custom_history(engine, tsh):
@func('made-up-series')
def madeup(base: int, coeff: float = 1.) -> pd.Series:
return pd.Series(np.array([base, base + 1, base + 2]) * coeff,
index=pd.date_range(dt(2019, 1, 1),
periods=3,
freq='D'))
@finder('made-up-series')
def find(cn, tsh, tree):
return {
tree[0]: {
'index_type': 'datetime64[ns]',
'index_dtype': '|M8[ns]',
'tzaware': False,
'value_type': 'float64',
'value_dtype': '<f8'
}
}
@history('made-up-series')
def madeup_history(__interpreter__, base, coeff):
hist = {}
for i in (1, 2, 3):
hist[pd.Timestamp(f'2020-1-{i}', tz='utc')] = pd.Series(
np.array([base, base + 1, base + 2]) * coeff,
index=pd.date_range(dt(2019, 1, i), periods=3, freq='D'))
return hist
tsh.register_formula(
engine, 'made-up',
'(+ 3 (add (made-up-series 1 #:coeff 2.) (made-up-series 2 #:coeff .5)))',
False)
assert_df("""
2019-01-01 6.0
2019-01-02 8.5
2019-01-03 11.0
""", tsh.get(engine, 'made-up'))
hist = tsh.history(engine, 'made-up')
assert_hist(
"""
insertion_date value_date
2020-01-01 00:00:00+00:00 2019-01-01 6.0
2019-01-02 8.5
2019-01-03 11.0
2020-01-02 00:00:00+00:00 2019-01-01 6.0
2019-01-02 8.5
2019-01-03 11.0
2020-01-03 00:00:00+00:00 2019-01-01 6.0
2019-01-02 8.5
2019-01-03 11.0
""", hist)
def test_naive(client, engine, tsh):
series_in = genserie(pd.Timestamp('2018-1-1'), 'H', 3)
client.update('test-naive',
series_in,
'Babar',
insertion_date=utcdt(2019, 1, 1))
# now let's get it back
ts = client.get('test-naive')
assert_df(
"""
2018-01-01 00:00:00 0.0
2018-01-01 01:00:00 1.0
2018-01-01 02:00:00 2.0
""", ts)
assert not getattr(ts.index.dtype, 'tz', False)
def test_scalar_ops(engine, tsh):
x = pd.Series([1, 2, 3],
index=pd.date_range(dt(2020, 1, 1), periods=3, freq='D'))
tsh.update(engine, x, 'scalar-ops', 'Babar')
tsh.register_formula(
engine,
'scalar-formula',
'(+ (+ (/ 20 (* 2 5)) 1) (series "scalar-ops"))',
)
ts = tsh.get(engine, 'scalar-formula')
assert_df("""
2020-01-01 4.0
2020-01-02 5.0
2020-01-03 6.0
""", ts)
def test_rename(engine, tsh):
ts = pd.Series([1, 2, 3],
index=pd.date_range(dt(2019, 1, 1), periods=3, freq='D'))
tsh.update(engine, ts, 'rename-a', 'Babar')
tsh.register_formula(engine, 'survive-renaming',
'(+ 1 (series "rename-a" #:fill 0))')
tsh.register_formula(
engine, 'survive-renaming-2',
'(add (series "survive-renaming") (series "rename-a" #:fill 0))')
ts = tsh.get(engine, 'survive-renaming')
assert_df("""
2019-01-01 2.0
2019-01-02 3.0
2019-01-03 4.0
""", ts)
ts = tsh.get(engine, 'survive-renaming-2')
assert_df("""
2019-01-01 3.0
2019-01-02 5.0
2019-01-03 7.0
""", ts)
with engine.begin() as cn:
tsh.rename(cn, 'rename-a', 'a-renamed')
ts = tsh.get(engine, 'survive-renaming')
assert_df("""
2019-01-01 2.0
2019-01-02 3.0
2019-01-03 4.0
""", ts)
ts = tsh.get(engine, 'survive-renaming-2')
assert_df("""
2019-01-01 3.0
2019-01-02 5.0
2019-01-03 7.0
""", ts)
with engine.begin() as cn:
with pytest.raises(ValueError) as err:
tsh.rename(cn, 'a-renamed', 'survive-renaming')
assert err.value.args[
0] == 'new name is already referenced by `survive-renaming-2`'
# rename a formula !
with engine.begin() as cn:
tsh.rename(cn, 'survive-renaming', 'survived')
assert tsh.formula(
engine, 'survive-renaming-2'
) == '(add (series "survived") (series "a-renamed" #:fill 0))'
def test_slice_options(engine, tsh):
base = pd.Series([1, 2, 3],
index=pd.date_range(utcdt(2019, 1, 1),
periods=3,
freq='D'))
tsh.update(engine, base, 'test-slice', 'Babar')
# options transmissions
ts = tsh.eval_formula(
engine,
'(add (series "test-slice") '
' (slice (series "test-slice" #:fill 0) #:fromdate (date "2019-1-2")))',
)
assert_df(
"""
2019-01-01 00:00:00+00:00 1.0
2019-01-02 00:00:00+00:00 4.0
2019-01-03 00:00:00+00:00 6.0
""", ts)
def test_priority(engine, tsh):
tsh.register_formula(
engine, 'test_prio',
'(priority (series "c" #:prune 1) (series "b") (series "a"))', False)
a = pd.Series([1, 2, 3],
index=pd.date_range(dt(2019, 1, 1), periods=3, freq='D'))
b = pd.Series([10, 20, 30],
index=pd.date_range(dt(2019, 1, 2), periods=3, freq='D'))
c = pd.Series([100, 200, 300],
index=pd.date_range(dt(2019, 1, 3), periods=3, freq='D'))
tsh.update(engine, a, 'a', 'Babar')
tsh.update(engine, b, 'b', 'Celeste')
tsh.update(engine, c, 'c', 'Arthur')
prio = tsh.get(engine, 'test_prio')
assert_df(
"""
2019-01-01 1.0
2019-01-02 10.0
2019-01-03 100.0
2019-01-04 200.0
""", prio)
limited = tsh.get(engine,
'test_prio',
from_value_date=dt(2019, 1, 2),
to_value_date=dt(2019, 1, 3))
# NOTE that the 1-3 point is now 20 because the 100 (series c)
# point has been pruned
assert_df("""
2019-01-02 10.0
2019-01-03 20.0
""", limited)
# type
assert tsh.type(engine, 'no-such-series') == 'primary'
assert tsh.type(engine, 'test_prio') == 'formula'
assert tsh.type(engine, 'a') == 'primary'
assert not tsh.exists(engine, 'no-such-series')
assert tsh.type(engine, 'test_prio')
assert tsh.type(engine, 'a')
def test_naive_tzone(engine, tsh):
x = pd.Series([1, 2, 3],
index=pd.date_range(utcdt(2020, 1, 1), periods=3, freq='D'))
tsh.update(engine, x, 'non-naive', 'Babar')
tsh.register_formula(
engine,
'to-naive',
'(naive (series "non-naive") "Europe/Moscow")',
)
ts = tsh.get(engine, 'to-naive')
assert_df(
"""
2020-01-01 03:00:00 1.0
2020-01-02 03:00:00 2.0
2020-01-03 03:00:00 3.0
""", ts)
meta = tsh.metadata(engine, 'to-naive')
assert meta['tzaware'] == False
def test_new_func(engine, tsh):
@func('identity')
def identity(series: pd.Series) -> pd.Series:
return series
tsh.register_formula(engine, 'identity', '(identity (series "id-a"))',
False)
ts = pd.Series([1, 2, 3],
index=pd.date_range(dt(2019, 1, 1), periods=3, freq='D'))
tsh.update(engine, ts, 'id-a', 'Babar')
ts = tsh.get(engine, 'identity')
assert_df("""
2019-01-01 1.0
2019-01-02 2.0
2019-01-03 3.0
""", ts)
# cleanup
FUNCS.pop('identity')
def test_multi_source_handcrafted(mapi):
series = pd.Series([1, 2, 3],
index=pd.date_range(pd.Timestamp('2020-1-1'),
freq='D',
periods=3))
mapi.update('multi-local', series, 'test', manual=True)
assert_df("""
2020-01-01 1.0
2020-01-02 2.0
2020-01-03 3.0
""", mapi.get('multi-local'))
edited = series.copy()
edited.iloc[1] = 42
mapi.update('multi-local', edited, 'test')
assert_df("""
2020-01-01 1.0
2020-01-02 42.0
2020-01-03 3.0
""", mapi.get('multi-local'))
# should be a noop
mapi.update('multi-local', edited, 'test', manual=True)
_, marker = mapi.edited('multi-local')
assert_df(
"""
2020-01-01 False
2020-01-02 False
2020-01-03 False
""", marker)
def test_clip(engine, tsh):
tsh.register_formula(engine, 'test_clip',
'(clip (series "a") #:min 2 #:max 4)')
a = pd.Series([1, 2, 3, 4, 5],
index=pd.date_range(dt(2019, 1, 1), periods=5, freq='D'))
tsh.update(engine, a, 'a', 'Babar')
cleaned = tsh.get(engine, 'test_clip')
assert_df("""
2019-01-02 2.0
2019-01-03 3.0
2019-01-04 4.0
""", cleaned)
restricted = tsh.get(engine,
'test_clip',
from_value_date=dt(2019, 1, 3),
to_value_date=dt(2019, 1, 3))
assert_df("""
2019-01-03 3.0
""", restricted)
def test_options(engine, tsh):
@func('dummy')
def dummy(option: int = None) -> pd.Series:
series = pd.Series([1, 2, 3],
index=pd.date_range(dt(2019, 1, 1),
periods=3,
freq='D'))
series.options = {'option': option}
return series
tsh.register_formula(engine, 'test_options', '(* 3 (dummy #:option 42))',
False)
ts = tsh.get(engine, 'test_options')
assert_df("""
2019-01-01 3
2019-01-02 6
2019-01-03 9
""", ts)
assert ts.options == {'option': 42}
FUNCS.pop('dummy')
def test_get_fast_path(client):
series_in = genserie(utcdt(2018, 1, 1), 'H', 3)
res = client.patch('/series/state',
params={
'name': 'test_fast',
'series': util.tojson(series_in),
'author': 'Babar',
'insertion_date': utcdt(2018, 1, 1, 10),
'tzaware': util.tzaware_serie(series_in)
})
assert res.status_code == 201
out = client.get('/series/state',
params={
'name': 'test_fast',
'format': 'tshpack'
})
meta, index, values = util.nary_unpack(zlib.decompress(out.body))
meta = json.loads(meta)
index, values = util.numpy_deserialize(index, values, meta)
series = pd.Series(values, index=index)
series = series.tz_localize('UTC')
assert_df(
"""
2018-01-01 00:00:00+00:00 0.0
2018-01-01 01:00:00+00:00 1.0
2018-01-01 02:00:00+00:00 2.0
""", series)
assert meta == {
'tzaware': True,
'index_type': 'datetime64[ns, UTC]',
'value_type': 'float64',
'index_dtype': '|M8[ns]',
'value_dtype': '<f8'
}
def test_formula(client, engine, tsh):
tsh.update(
engine,
pd.Series([1, 2, 3],
index=pd.date_range(pd.Timestamp('2020-1-1', tz='UTC'),
freq='D',
periods=3)), 'in-a-formula', 'Babar')
with pytest.raises(SyntaxError):
client.register_formula('new-formula', '(+ 3')
with pytest.raises(ValueError):
client.register_formula('new-formula', '(+ 3 (series "lol"))')
client.register_formula('new-formula',
'(+ 3 (series "lol"))',
reject_unknown=False)
with pytest.raises(AssertionError):
client.register_formula(
'new-formula',
'(+ 3 (series "in-a-formula"))',
)
client.register_formula('new-formula',
'(+ 3 (series "in-a-formula"))',
update=True)
series = client.get('new-formula')
assert_df(
"""
2020-01-01 00:00:00+00:00 4.0
2020-01-02 00:00:00+00:00 5.0
2020-01-03 00:00:00+00:00 6.0
""", series)
assert client.formula('new-formula') == '(+ 3 (series "in-a-formula"))'
assert client.formula('lol') is None
def test_slice(engine, tsh):
base = pd.Series([1, 2, 3],
index=pd.date_range(utcdt(2019, 1, 1),
periods=3,
freq='D'))
tsh.update(engine, base, 'test-slice', 'Babar')
tsh.register_formula(
engine,
'slicing-id',
'(slice (series "test-slice"))',
)
tsh.register_formula(
engine,
'slicing-from',
'(slice (series "test-slice") #:fromdate (date "2019-1-2"))',
)
tsh.register_formula(
engine,
'slicing-fromto',
'(slice (series "test-slice") '
' #:fromdate (date "2019-1-2") '
' #:todate (date "2019-1-2")'
')',
)
tsh.register_formula(
engine,
'slicing-empty',
'(slice (series "test-slice") '
' #:fromdate (date "2018-1-2") '
' #:todate (date "2018-1-2")'
')',
)
assert_df(
"""
2019-01-01 00:00:00+00:00 1.0
2019-01-02 00:00:00+00:00 2.0
2019-01-03 00:00:00+00:00 3.0
""", tsh.get(engine, 'slicing-id'))
assert_df(
"""
2019-01-02 00:00:00+00:00 2.0
2019-01-03 00:00:00+00:00 3.0
""", tsh.get(engine, 'slicing-from'))
assert_df("""
2019-01-02 00:00:00+00:00 2.0
""", tsh.get(engine, 'slicing-fromto'))
assert len(tsh.get(engine, 'slicing-empty')) == 0
def test_mul(engine, tsh):
base = pd.Series([1, 2, 3],
index=pd.date_range(utcdt(2019, 1, 1),
periods=3,
freq='D'))
tsh.update(engine, base, 'mul-a', 'Babar')
tsh.update(engine, base, 'mul-b', 'Babar')
tsh.update(engine, base, 'mul-c', 'Babar')
tsh.register_formula(
engine,
'multiply-aligned',
'(mul (series "mul-a") (series "mul-b") (series "mul-c"))',
)
ts = tsh.get(engine, 'multiply-aligned')
assert_df(
"""
2019-01-01 00:00:00+00:00 1.0
2019-01-02 00:00:00+00:00 8.0
2019-01-03 00:00:00+00:00 27.0
""", ts)
base = pd.Series([1, 2, np.nan],
index=pd.date_range(utcdt(2019, 1, 1),
periods=3,
freq='D'))
tsh.update(engine, base, 'mul-b', 'Babar')
ts = tsh.get(engine, 'multiply-aligned')
assert_df(
"""
2019-01-01 00:00:00+00:00 1.0
2019-01-02 00:00:00+00:00 8.0
""", ts)
tsh.register_formula(
engine,
'multiply-aligned',
'(mul (series "mul-a") (series "mul-b" #:fill 1) (series "mul-c"))',
update=True)
ts = tsh.get(engine, 'multiply-aligned')
assert_df(
"""
2019-01-01 00:00:00+00:00 1.0
2019-01-02 00:00:00+00:00 8.0
2019-01-03 00:00:00+00:00 9.0
""", ts)
def test_div(engine, tsh):
base = pd.Series([1, 2, 3],
index=pd.date_range(utcdt(2019, 1, 1),
periods=3,
freq='D'))
tsh.update(engine, base, 'div-a', 'Babar')
tsh.update(engine, base, 'div-b', 'Babar')
tsh.register_formula(
engine,
'divide',
'(div (series "div-a") (series "div-b"))',
)
ts = tsh.get(engine, 'divide')
assert_df(
"""
2019-01-01 00:00:00+00:00 1.0
2019-01-02 00:00:00+00:00 1.0
2019-01-03 00:00:00+00:00 1.0
""", ts)
base = pd.Series([2, 1, np.nan],
index=pd.date_range(utcdt(2019, 1, 1),
periods=3,
freq='D'))
tsh.update(engine, base, 'div-b', 'Babar')
ts = tsh.get(engine, 'divide')
assert_df(
"""
2019-01-01 00:00:00+00:00 0.5
2019-01-02 00:00:00+00:00 2.0
""", ts)
tsh.register_formula(engine,
'divide',
'(div (series "div-a") (series "div-b" #:fill 3))',
update=True)
ts = tsh.get(engine, 'divide')
assert_df(
"""
2019-01-01 00:00:00+00:00 0.5
2019-01-02 00:00:00+00:00 2.0
2019-01-03 00:00:00+00:00 1.0
""", ts)
def test_resample(engine, tsh):
hourly = pd.Series(list(range(36)),
index=pd.date_range(utcdt(2020, 1, 1),
periods=36,
freq='H'))
gasday = pd.Series([1, 2, 3],
index=pd.date_range(utcdt(2020, 1, 1, 5),
periods=3,
freq='D'))
tsh.update(engine, hourly, 'hourly', 'Babar')
tsh.update(engine, gasday, 'gasday', 'Celeste')
tsh.register_formula(engine, 'hourly2daily',
'(resample (series "hourly") "D")')
tsh.register_formula(engine, 'hourly2dailysum',
'(resample (series "hourly") "D" "sum")')
tsh.register_formula(engine, 'gasdaytoday',
'(resample (series "gasday") "D")')
tsh.register_formula(engine, 'badmethod',
'(resample (series "gasday") "D" "NO-SUCH-METHOD")')
assert_df(
"""
2020-01-01 00:00:00+00:00 11.5
2020-01-02 00:00:00+00:00 29.5
""", tsh.get(engine, 'hourly2daily'))
assert_df(
"""
2020-01-01 00:00:00+00:00 276.0
2020-01-02 00:00:00+00:00 354.0
""", tsh.get(engine, 'hourly2dailysum'))
assert_df(
"""
2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
""", tsh.get(engine, 'gasdaytoday'))
with pytest.raises(ValueError) as err:
tsh.get(engine, 'badmethod')
assert err.value.args[0] == 'bad resampling method `NO-SUCH-METHOD`'
def test_local_formula_remote_series(mapihttp, engine):
from tshistory_formula.tsio import timeseries as pgseries
mapi = mapihttp
assert repr(mapi) == (
'timeseries(uri=postgresql://localhost:5433/postgres,'
'ns=ns-test-local,'
'sources=[source(uri=http://test-uri2,ns=ns-test-remote), '
'source(uri=http://unavailable,ns=ns-test-unavailable-remote)])')
assert len(mapi.othersources.sources) == 2
assert mapi.namespace == 'ns-test-local'
assert mapi.uri == 'postgresql://localhost:5433/postgres'
assert mapi.othersources.sources[0].namespace == 'ns-test-remote'
assert mapi.othersources.sources[0].uri == 'http://test-uri2'
series = pd.Series(
[1, 2, 3],
index=pd.date_range(pd.Timestamp('2020-1-1'), periods=3, freq='H'),
)
mapi.update('local-series', series, 'Babar')
rtsh = pgseries('ns-test-remote')
rtsh.update(engine,
series,
'remote-series',
'Celeste',
insertion_date=pd.Timestamp('2020-1-1', tz='UTC'))
cat = mapi.catalog(allsources=True)
assert dict(cat) == {
('db://localhost:5433/postgres', 'ns-test-local'):
[('local-series', 'primary')],
('db://localhost:5433/postgres', 'ns-test-remote'):
[['remote-series', 'primary']]
}
mapi.register_formula('test-localformula-remoteseries',
'(+ 1 (series "remote-series"))')
ts = mapi.get('test-localformula-remoteseries')
assert_df(
"""
2020-01-01 00:00:00 2.0
2020-01-01 01:00:00 3.0
2020-01-01 02:00:00 4.0
""", ts)
hist = mapi.history('test-localformula-remoteseries')
assert_hist(
"""
insertion_date value_date
2020-01-01 00:00:00+00:00 2020-01-01 00:00:00 2.0
2020-01-01 01:00:00 3.0
2020-01-01 02:00:00 4.0
""", hist)
ival = mapi.interval('remote-series')
assert (ival.left, ival.right) == (pd.Timestamp('2020-01-01 00:00:00'),
pd.Timestamp('2020-01-01 02:00:00'))
with pytest.raises(ValueError) as err:
mapi.interval('test-localformula-remoteseries')
assert err.value.args[
0] == 'no interval for series: test-localformula-remoteseries'
meta = mapi.metadata('remote-series', all=True)
assert meta == {
'index_dtype': '<M8[ns]',
'index_type': 'datetime64[ns]',
'tzaware': False,
'value_dtype': '<f8',
'value_type': 'float64'
}
meta = mapi.metadata('test-localformula-remoteseries', all=True)
assert meta == {
'index_dtype': '<M8[ns]',
'index_type': 'datetime64[ns]',
'tzaware': False,
'value_dtype': '<f8',
'value_type': 'float64'
}
def test_base_universal_api(pgapi, httpapi):
series = pd.Series([1, 2, 3],
index=pd.date_range(utcdt(2020, 1, 1),
periods=3,
freq='D'))
pgapi.update('api-test',
series,
'Babar',
insertion_date=utcdt(2019, 1, 1),
metadata={'about': 'test'})
out = httpapi.get('api-test')
assert_df(
"""
2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
""", out)
series[utcdt(2020, 1, 4)] = 4
pgapi.update('api-test', series, 'Babar', insertion_date=utcdt(2019, 1, 2))
out = pgapi.get('api-test',
from_value_date=utcdt(2020, 1, 2),
to_value_date=utcdt(2020, 1, 3))
assert_df(
"""
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
""", out)
series[utcdt(2019, 12, 31)] = 0
pgapi.replace('api-test',
series,
'Babar',
insertion_date=utcdt(2019, 1, 3))
out = httpapi.get('api-test')
assert_df(
"""
2019-12-31 00:00:00+00:00 0.0
2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
2020-01-04 00:00:00+00:00 4.0
""", out)
assert httpapi.type('api-test') == pgapi.type('api-test')
assert httpapi.interval('api-test') == pgapi.interval('api-test')
out = httpapi.get('api-test', revision_date=utcdt(2019, 1, 1))
assert_df(
"""
2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
""", out)
hist = httpapi.history('api-test')
assert_hist(
"""
insertion_date value_date
2019-01-01 00:00:00+00:00 2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
2019-01-02 00:00:00+00:00 2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
2020-01-04 00:00:00+00:00 4.0
2019-01-03 00:00:00+00:00 2019-12-31 00:00:00+00:00 0.0
2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
2020-01-04 00:00:00+00:00 4.0
""", hist)
hist = pgapi.history('api-test', diffmode=True)
assert_hist(
"""
insertion_date value_date
2019-01-01 00:00:00+00:00 2020-01-01 00:00:00+00:00 1.0
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
2019-01-02 00:00:00+00:00 2020-01-04 00:00:00+00:00 4.0
2019-01-03 00:00:00+00:00 2019-12-31 00:00:00+00:00 0.0
""", hist)
assert pgapi.exists('api-test')
assert not pgapi.exists('i-dont-exist')
assert httpapi.exists('api-test')
assert not httpapi.exists('i-dont-exist')
ival = pgapi.interval('api-test')
assert ival.left == pd.Timestamp('2019-12-31 00:00:00+0000', tz='UTC')
assert ival.right == pd.Timestamp('2020-01-04 00:00:00+0000', tz='UTC')
meta = pgapi.metadata('api-test', all=True)
assert meta == {
'tzaware': True,
'index_type': 'datetime64[ns, UTC]',
'value_type': 'float64',
'index_dtype': '|M8[ns]',
'value_dtype': '<f8'
}
meta = httpapi.metadata('api-test')
assert meta == {}
pgapi.update_metadata('api-test', {'desc': 'a metadata test'})
meta = pgapi.metadata('api-test')
assert meta == {'desc': 'a metadata test'}
assert pgapi.type('api-test') == 'primary'
st = pgapi.staircase('api-test', delta=timedelta(days=366))
assert_df(
"""
2020-01-02 00:00:00+00:00 2.0
2020-01-03 00:00:00+00:00 3.0
2020-01-04 00:00:00+00:00 4.0
""", st)
pgapi.rename('api-test', 'api-test2')
assert pgapi.exists('api-test2')
assert not pgapi.exists('api-test')
assert httpapi.exists('api-test2')
assert not httpapi.exists('api-test')
pgapi.delete('api-test2')
assert not pgapi.exists('api-test')
assert not httpapi.exists('api-test')
pgapi.delete('api-test2')
def test_base(client, engine, tsh):
assert repr(client) == "tshistory-http-client(uri='http://test-uri')"
ts = client.get('no-such-series')
assert ts is None
assert not client.exists('no-such-series')
meta = client.metadata('no-such-series')
assert meta == {'message': '`no-such-series` does not exists'}
series_in = genserie(utcdt(2018, 1, 1), 'H', 3)
client.update('test', series_in, 'Babar', insertion_date=utcdt(2019, 1, 1))
assert client.exists('test')
# now let's get it back
ts = client.get('test')
assert_df(
"""
2018-01-01 00:00:00+00:00 0.0
2018-01-01 01:00:00+00:00 1.0
2018-01-01 02:00:00+00:00 2.0
""", ts)
ts = client.get('test', from_value_date=utcdt(2018, 1, 1, 2))
assert_df("""
2018-01-01 02:00:00+00:00 2.0
""", ts)
ts = client.get('test', to_value_date=utcdt(2018, 1, 1, 0))
assert_df("""
2018-01-01 00:00:00+00:00 0.0
""", ts)
# out of range
ts = client.get('test',
from_value_date=utcdt(2020, 1, 1, 2),
to_value_date=utcdt(2020, 1, 1, 2))
assert len(ts) == 0
assert ts.name == 'test'
meta = client.metadata('test', all=True)
assert meta == {
'tzaware': True,
'index_type': 'datetime64[ns, UTC]',
'value_type': 'float64',
'index_dtype': '|M8[ns]',
'value_dtype': '<f8'
}
# update
client.update_metadata('test', {'desc': 'banana spot price'})
meta = client.metadata('test', all=False)
assert meta == {
'desc': 'banana spot price',
}
# check the insertion_date
series_in = genserie(utcdt(2018, 1, 2), 'H', 3)
client.update('test',
series_in,
'Babar',
metadata={'event': 'hello'},
insertion_date=utcdt(2019, 1, 2))
v1 = client.get('test', revision_date=utcdt(2019, 1, 1))
assert_df(
"""
2018-01-01 00:00:00+00:00 0.0
2018-01-01 01:00:00+00:00 1.0
2018-01-01 02:00:00+00:00 2.0
""", v1)
d1, d2 = tsh.insertion_dates(engine, 'test')
assert d1 == utcdt(2019, 1, 1)
assert d2 > d1
client.update('test2', series_in, 'Babar')
series = client.catalog()
assert ['test',
'primary'] in series[('db://localhost:5433/postgres', 'tsh')]
assert ['test2',
'primary'] in series[('db://localhost:5433/postgres', 'tsh')]
client.replace('test2', genserie(utcdt(2020, 1, 1), 'D', 3), 'Babar')
series = client.get('test2')
assert_df(
"""
2020-01-01 00:00:00+00:00 0.0
2020-01-02 00:00:00+00:00 1.0
2020-01-03 00:00:00+00:00 2.0
""", series)
type = client.type('test2')
assert type == 'primary'
ival = client.interval('test2')
assert ival.left == pd.Timestamp('2020-01-01 00:00:00+0000', tz='UTC')
assert ival.right == pd.Timestamp('2020-01-03 00:00:00+0000', tz='UTC')
client.rename('test2', 'test3')
assert not client.exists('test2')
assert client.exists('test3')
client.delete('test3')
assert not client.exists('test3')
