def create(uri, ns, name): api = timeseries(uri, ns) series = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2020, 1, 1), periods=3, freq='D')) api.update(name, series, 'Babar', insertion_date=utcdt(2019, 1, 1), metadata={'about': 'test'}) out = api.get(name) assert_df( """ 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, out) series[utcdt(2020, 1, 4)] = 4 api.update(name, series, 'Babar', insertion_date=utcdt(2019, 1, 2)) out = api.get(name, from_value_date=utcdt(2020, 1, 2), to_value_date=utcdt(2020, 1, 3)) assert_df( """ 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, out)
def test_naive(client): series_in = genserie(pd.Timestamp('2018-1-1'), 'H', 3) res = client.patch('/series/state', params={ 'name': 'test-naive', 'series': util.tojson(series_in), 'author': 'Babar', 'insertion_date': utcdt(2018, 1, 1, 10), 'tzaware': util.tzaware_serie(series_in) }) assert res.status_code == 201 res = client.get('/series/metadata?name=test-naive&all=1') meta = res.json assert meta == { 'index_dtype': '<M8[ns]', 'index_type': 'datetime64[ns]', 'tzaware': False, 'value_dtype': '<f8', 'value_type': 'float64' } res = client.get('/series/state?name=test-naive') series = util.fromjson(res.body, 'test', meta['tzaware']) assert_df( """ 2018-01-01 00:00:00 0.0 2018-01-01 01:00:00 1.0 2018-01-01 02:00:00 2.0 """, series)
def test_boolean_support(engine, tsh): @func('op-with-boolean-kw') def customseries(zeroes: bool = False) -> pd.Series: return pd.Series(np.array([1.0, 2.0, 3.0]) * zeroes, index=pd.date_range(dt(2019, 1, 1), periods=3, freq='D')) tsh.register_formula(engine, 'no-zeroes', '(op-with-boolean-kw)') tsh.register_formula(engine, 'zeroes', '(op-with-boolean-kw #:zeroes #t)') ts1 = tsh.get(engine, 'no-zeroes') assert_df(""" 2019-01-01 0.0 2019-01-02 0.0 2019-01-03 0.0 """, ts1) ts2 = tsh.get(engine, 'zeroes') assert_df(""" 2019-01-01 1.0 2019-01-02 2.0 2019-01-03 3.0 """, ts2) FUNCS.pop('op-with-boolean-kw')
def test_priority2(engine, tsh): tsh.register_formula( engine, 'test_prio2', '(priority (series "real") (series "nom") (series "fcst"))', False) real = pd.Series([1, 1, 1], index=pd.date_range(dt(2019, 1, 1), periods=3, freq='D')) nom = pd.Series([10, 10, 10, 10], index=pd.date_range(dt(2019, 1, 1), periods=4, freq='D')) fcst = pd.Series([100, 100, 100, 100, 100], index=pd.date_range(dt(2019, 1, 1), periods=5, freq='D')) tsh.update(engine, real, 'real', 'Babar') tsh.update(engine, nom, 'nom', 'Celeste') tsh.update(engine, fcst, 'fcst', 'Arthur') prio = tsh.get(engine, 'test_prio2') assert_df( """ 2019-01-01 1.0 2019-01-02 1.0 2019-01-03 1.0 2019-01-04 10.0 2019-01-05 100.0 """, prio)
def test_staircase_history_naive(client, tsh): # each days we insert 7 data points from datetime import datetime for idx, idate in enumerate( pd.date_range(start=utcdt(2015, 1, 1), end=utcdt(2015, 1, 4), freq='D')): series = genserie(start=idate.tz_convert(None), freq='H', repeat=7) client.update('staircase-naive', series, 'Babar', insertion_date=idate) series = client.staircase('staircase-naive', pd.Timedelta(hours=3), from_value_date=datetime(2015, 1, 1, 4), to_value_date=datetime(2015, 1, 2, 5)) assert series.name == 'staircase-naive' assert_df( """ 2015-01-01 04:00:00 4.0 2015-01-01 05:00:00 5.0 2015-01-01 06:00:00 6.0 2015-01-02 03:00:00 3.0 2015-01-02 04:00:00 4.0 2015-01-02 05:00:00 5.0 """, series) # series = client.staircase( # 'staircase-naive', # pd.Timedelta(hours=3), # from_value_date=datetime(2015, 1, 1, 4), # to_value_date=datetime(2015, 1, 2, 5) # ) hist = client.history('staircase-naive') assert len(hist) == 4 hist = client.history('staircase-naive', from_insertion_date=datetime(2015, 1, 2), to_insertion_date=datetime(2015, 1, 3)) assert len(hist) == 2 hist = client.history('staircase-naive', from_value_date=datetime(2015, 1, 1, 3), to_value_date=datetime(2015, 1, 2, 1)) assert all(series.name == 'staircase-naive' for series in hist.values()) assert_hist( """ insertion_date value_date 2015-01-01 00:00:00+00:00 2015-01-01 03:00:00 3.0 2015-01-01 04:00:00 4.0 2015-01-01 05:00:00 5.0 2015-01-01 06:00:00 6.0 2015-01-02 00:00:00+00:00 2015-01-01 03:00:00 3.0 2015-01-01 04:00:00 4.0 2015-01-01 05:00:00 5.0 2015-01-01 06:00:00 6.0 2015-01-02 00:00:00 0.0 2015-01-02 01:00:00 1.0 """, hist)
def test_alternative_handler(pgapi): api = pgapi sapi = timeseries(api.uri, api.namespace, formula_class()) sapi.update( 'test-features', genserie(utcdt(2020, 1, 1), 'D', 3), 'Babar', ) sapi.tsh.register_formula(sapi.engine, 'test-formula', '(+ 1 (series "test-features"))') tsa = sapi.get('test-features') assert_df( """ 2020-01-01 00:00:00+00:00 0.0 2020-01-02 00:00:00+00:00 1.0 2020-01-03 00:00:00+00:00 2.0 """, tsa) tsb = sapi.get('test-formula') assert_df( """ 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, tsb) class supervision_and_formula(supervision_class(), formula_class()): pass sapi = timeseries(api.uri, api.namespace, supervision_and_formula) tsa = sapi.get('test-features') assert_df( """ 2020-01-01 00:00:00+00:00 0.0 2020-01-02 00:00:00+00:00 1.0 2020-01-03 00:00:00+00:00 2.0 """, tsa) tsb = sapi.get('test-formula') assert_df( """ 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, tsb) sapi.update('test-features', genserie(utcdt(2020, 1, 2), 'D', 3), 'Babar', manual=True) tsb = sapi.get('test-formula') assert_df( """ 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 1.0 2020-01-03 00:00:00+00:00 2.0 2020-01-04 00:00:00+00:00 3.0 """, tsb)
def test_staircase(client): # each days we insert 7 data points for idate in pd.date_range(start=utcdt(2015, 1, 1), end=utcdt(2015, 1, 4), freq='D'): series = genserie(start=idate, freq='H', repeat=7) client.patch('/series/state', params={ 'name': 'staircase', 'series': util.tojson(series), 'author': 'Babar', 'insertion_date': idate, 'tzaware': util.tzaware_serie(series) }) res = client.get('/series/staircase', params={ 'name': 'staircase', 'delta': pd.Timedelta(hours=3), 'from_value_date': utcdt(2015, 1, 1, 4), 'to_value_date': utcdt(2015, 1, 2, 5), }) series = util.fromjson(res.body, 'test', True) assert_df( """ 2015-01-01 04:00:00+00:00 4.0 2015-01-01 05:00:00+00:00 5.0 2015-01-01 06:00:00+00:00 6.0 2015-01-02 03:00:00+00:00 3.0 2015-01-02 04:00:00+00:00 4.0 2015-01-02 05:00:00+00:00 5.0 """, series) res = client.get('/series/staircase', params={ 'name': 'staircase', 'delta': pd.Timedelta(hours=3), 'from_value_date': utcdt(2015, 1, 1, 4), 'to_value_date': utcdt(2015, 1, 2, 5), 'format': 'tshpack' }) meta, index, values = util.nary_unpack(zlib.decompress(res.body)) meta = json.loads(meta) index, values = util.numpy_deserialize(index, values, meta) series = pd.Series(values, index=index) series = series.tz_localize('UTC') assert_df( """ 2015-01-01 04:00:00+00:00 4.0 2015-01-01 05:00:00+00:00 5.0 2015-01-01 06:00:00+00:00 6.0 2015-01-02 03:00:00+00:00 3.0 2015-01-02 04:00:00+00:00 4.0 2015-01-02 05:00:00+00:00 5.0 """, series)
def test_scalar_div(engine, tsh): a = pd.Series([1, 2, 3], index=pd.date_range(dt(2019, 1, 1), periods=3, freq='D')) tsh.update(engine, a, 'div-me', 'Babar') ts = tsh.eval_formula(engine, '(/ (series "div-me") (/ 3 2))') assert_df( """ 2019-01-01 0.666667 2019-01-02 1.333333 2019-01-03 2.000000 """, ts)
def test_staircase(engine, tsh): tsh.register_formula(engine, 's-addition', '(add (series "sa") (series "sb"))', False) for day in (1, 2, 3, 4, 5): idate = utcdt(2018, 1, day) for name in 'ab': ts = pd.Series([day / 2.] * 5, index=pd.date_range(dt(2018, 1, day), periods=5, freq='D')) tsh.update(engine, ts, 's' + name, 'Babar', insertion_date=idate) ts = tsh.staircase(engine, 's-addition', delta=pd.Timedelta(hours=12)) assert_df( """ 2018-01-02 1.0 2018-01-03 2.0 2018-01-04 3.0 2018-01-05 4.0 2018-01-06 5.0 2018-01-07 5.0 2018-01-08 5.0 2018-01-09 5.0 """, ts) # this is not allowed in the staircase fast-path # hence we will take the slow path @func('identity') def identity(series: pd.Series) -> pd.Series: return series tsh.register_formula(engine, 'slow-down', '(identity (series "sa"))', False) tsh.register_formula(engine, 's-addition-not-fast', '(add (series "slow-down") (series "sb"))', False) ts = tsh.staircase(engine, 's-addition-not-fast', delta=pd.Timedelta(hours=12)) assert_df( """ 2018-01-02 1.0 2018-01-03 2.0 2018-01-04 3.0 2018-01-05 4.0 2018-01-06 5.0 2018-01-07 5.0 2018-01-08 5.0 2018-01-09 5.0 """, ts) # cleanup FUNCS.pop('identity')
def test_local_formula_remote_series(mapi): rtsh = timeseries('test-mapi-2') rtsh.update( mapi.engine, pd.Series( [1, 2, 3], index=pd.date_range(pd.Timestamp('2020-1-1'), periods=3, freq='H'), ), 'remote-series', 'Babar', insertion_date=pd.Timestamp('2020-1-1', tz='UTC') ) mapi.register_formula( 'test-localformula-remoteseries', '(+ 1 (series "remote-series"))' ) ts = mapi.get('test-localformula-remoteseries') assert_df(""" 2020-01-01 00:00:00 2.0 2020-01-01 01:00:00 3.0 2020-01-01 02:00:00 4.0 """, ts) hist = mapi.history('test-localformula-remoteseries') assert_hist(""" insertion_date value_date 2020-01-01 00:00:00+00:00 2020-01-01 00:00:00 2.0 2020-01-01 01:00:00 3.0 2020-01-01 02:00:00 4.0 """, hist) f = mapi.formula('test-localformula-remoteseries') assert f == '(+ 1 (series "remote-series"))' none = mapi.formula('nosuchformula') assert none is None # altsource formula rtsh.register_formula( mapi.engine, 'remote-formula-remote-series', '(+ 2 (series "remote-series"))' ) f = mapi.formula('remote-formula-remote-series') assert f == '(+ 2 (series "remote-series"))' assert_df(""" 2020-01-01 00:00:00 3.0 2020-01-01 01:00:00 4.0 2020-01-01 02:00:00 5.0 """, mapi.get('remote-formula-remote-series'))
def test_custom_history(engine, tsh): @func('made-up-series') def madeup(base: int, coeff: float = 1.) -> pd.Series: return pd.Series(np.array([base, base + 1, base + 2]) * coeff, index=pd.date_range(dt(2019, 1, 1), periods=3, freq='D')) @finder('made-up-series') def find(cn, tsh, tree): return { tree[0]: { 'index_type': 'datetime64[ns]', 'index_dtype': '|M8[ns]', 'tzaware': False, 'value_type': 'float64', 'value_dtype': '<f8' } } @history('made-up-series') def madeup_history(__interpreter__, base, coeff): hist = {} for i in (1, 2, 3): hist[pd.Timestamp(f'2020-1-{i}', tz='utc')] = pd.Series( np.array([base, base + 1, base + 2]) * coeff, index=pd.date_range(dt(2019, 1, i), periods=3, freq='D')) return hist tsh.register_formula( engine, 'made-up', '(+ 3 (add (made-up-series 1 #:coeff 2.) (made-up-series 2 #:coeff .5)))', False) assert_df(""" 2019-01-01 6.0 2019-01-02 8.5 2019-01-03 11.0 """, tsh.get(engine, 'made-up')) hist = tsh.history(engine, 'made-up') assert_hist( """ insertion_date value_date 2020-01-01 00:00:00+00:00 2019-01-01 6.0 2019-01-02 8.5 2019-01-03 11.0 2020-01-02 00:00:00+00:00 2019-01-01 6.0 2019-01-02 8.5 2019-01-03 11.0 2020-01-03 00:00:00+00:00 2019-01-01 6.0 2019-01-02 8.5 2019-01-03 11.0 """, hist)
def test_naive(client, engine, tsh): series_in = genserie(pd.Timestamp('2018-1-1'), 'H', 3) client.update('test-naive', series_in, 'Babar', insertion_date=utcdt(2019, 1, 1)) # now let's get it back ts = client.get('test-naive') assert_df( """ 2018-01-01 00:00:00 0.0 2018-01-01 01:00:00 1.0 2018-01-01 02:00:00 2.0 """, ts) assert not getattr(ts.index.dtype, 'tz', False)
def test_scalar_ops(engine, tsh): x = pd.Series([1, 2, 3], index=pd.date_range(dt(2020, 1, 1), periods=3, freq='D')) tsh.update(engine, x, 'scalar-ops', 'Babar') tsh.register_formula( engine, 'scalar-formula', '(+ (+ (/ 20 (* 2 5)) 1) (series "scalar-ops"))', ) ts = tsh.get(engine, 'scalar-formula') assert_df(""" 2020-01-01 4.0 2020-01-02 5.0 2020-01-03 6.0 """, ts)
def test_rename(engine, tsh): ts = pd.Series([1, 2, 3], index=pd.date_range(dt(2019, 1, 1), periods=3, freq='D')) tsh.update(engine, ts, 'rename-a', 'Babar') tsh.register_formula(engine, 'survive-renaming', '(+ 1 (series "rename-a" #:fill 0))') tsh.register_formula( engine, 'survive-renaming-2', '(add (series "survive-renaming") (series "rename-a" #:fill 0))') ts = tsh.get(engine, 'survive-renaming') assert_df(""" 2019-01-01 2.0 2019-01-02 3.0 2019-01-03 4.0 """, ts) ts = tsh.get(engine, 'survive-renaming-2') assert_df(""" 2019-01-01 3.0 2019-01-02 5.0 2019-01-03 7.0 """, ts) with engine.begin() as cn: tsh.rename(cn, 'rename-a', 'a-renamed') ts = tsh.get(engine, 'survive-renaming') assert_df(""" 2019-01-01 2.0 2019-01-02 3.0 2019-01-03 4.0 """, ts) ts = tsh.get(engine, 'survive-renaming-2') assert_df(""" 2019-01-01 3.0 2019-01-02 5.0 2019-01-03 7.0 """, ts) with engine.begin() as cn: with pytest.raises(ValueError) as err: tsh.rename(cn, 'a-renamed', 'survive-renaming') assert err.value.args[ 0] == 'new name is already referenced by `survive-renaming-2`' # rename a formula ! with engine.begin() as cn: tsh.rename(cn, 'survive-renaming', 'survived') assert tsh.formula( engine, 'survive-renaming-2' ) == '(add (series "survived") (series "a-renamed" #:fill 0))'
def test_slice_options(engine, tsh): base = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2019, 1, 1), periods=3, freq='D')) tsh.update(engine, base, 'test-slice', 'Babar') # options transmissions ts = tsh.eval_formula( engine, '(add (series "test-slice") ' ' (slice (series "test-slice" #:fill 0) #:fromdate (date "2019-1-2")))', ) assert_df( """ 2019-01-01 00:00:00+00:00 1.0 2019-01-02 00:00:00+00:00 4.0 2019-01-03 00:00:00+00:00 6.0 """, ts)
def test_priority(engine, tsh): tsh.register_formula( engine, 'test_prio', '(priority (series "c" #:prune 1) (series "b") (series "a"))', False) a = pd.Series([1, 2, 3], index=pd.date_range(dt(2019, 1, 1), periods=3, freq='D')) b = pd.Series([10, 20, 30], index=pd.date_range(dt(2019, 1, 2), periods=3, freq='D')) c = pd.Series([100, 200, 300], index=pd.date_range(dt(2019, 1, 3), periods=3, freq='D')) tsh.update(engine, a, 'a', 'Babar') tsh.update(engine, b, 'b', 'Celeste') tsh.update(engine, c, 'c', 'Arthur') prio = tsh.get(engine, 'test_prio') assert_df( """ 2019-01-01 1.0 2019-01-02 10.0 2019-01-03 100.0 2019-01-04 200.0 """, prio) limited = tsh.get(engine, 'test_prio', from_value_date=dt(2019, 1, 2), to_value_date=dt(2019, 1, 3)) # NOTE that the 1-3 point is now 20 because the 100 (series c) # point has been pruned assert_df(""" 2019-01-02 10.0 2019-01-03 20.0 """, limited) # type assert tsh.type(engine, 'no-such-series') == 'primary' assert tsh.type(engine, 'test_prio') == 'formula' assert tsh.type(engine, 'a') == 'primary' assert not tsh.exists(engine, 'no-such-series') assert tsh.type(engine, 'test_prio') assert tsh.type(engine, 'a')
def test_naive_tzone(engine, tsh): x = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2020, 1, 1), periods=3, freq='D')) tsh.update(engine, x, 'non-naive', 'Babar') tsh.register_formula( engine, 'to-naive', '(naive (series "non-naive") "Europe/Moscow")', ) ts = tsh.get(engine, 'to-naive') assert_df( """ 2020-01-01 03:00:00 1.0 2020-01-02 03:00:00 2.0 2020-01-03 03:00:00 3.0 """, ts) meta = tsh.metadata(engine, 'to-naive') assert meta['tzaware'] == False
def test_new_func(engine, tsh): @func('identity') def identity(series: pd.Series) -> pd.Series: return series tsh.register_formula(engine, 'identity', '(identity (series "id-a"))', False) ts = pd.Series([1, 2, 3], index=pd.date_range(dt(2019, 1, 1), periods=3, freq='D')) tsh.update(engine, ts, 'id-a', 'Babar') ts = tsh.get(engine, 'identity') assert_df(""" 2019-01-01 1.0 2019-01-02 2.0 2019-01-03 3.0 """, ts) # cleanup FUNCS.pop('identity')
def test_multi_source_handcrafted(mapi): series = pd.Series([1, 2, 3], index=pd.date_range(pd.Timestamp('2020-1-1'), freq='D', periods=3)) mapi.update('multi-local', series, 'test', manual=True) assert_df(""" 2020-01-01 1.0 2020-01-02 2.0 2020-01-03 3.0 """, mapi.get('multi-local')) edited = series.copy() edited.iloc[1] = 42 mapi.update('multi-local', edited, 'test') assert_df(""" 2020-01-01 1.0 2020-01-02 42.0 2020-01-03 3.0 """, mapi.get('multi-local')) # should be a noop mapi.update('multi-local', edited, 'test', manual=True) _, marker = mapi.edited('multi-local') assert_df( """ 2020-01-01 False 2020-01-02 False 2020-01-03 False """, marker)
def test_clip(engine, tsh): tsh.register_formula(engine, 'test_clip', '(clip (series "a") #:min 2 #:max 4)') a = pd.Series([1, 2, 3, 4, 5], index=pd.date_range(dt(2019, 1, 1), periods=5, freq='D')) tsh.update(engine, a, 'a', 'Babar') cleaned = tsh.get(engine, 'test_clip') assert_df(""" 2019-01-02 2.0 2019-01-03 3.0 2019-01-04 4.0 """, cleaned) restricted = tsh.get(engine, 'test_clip', from_value_date=dt(2019, 1, 3), to_value_date=dt(2019, 1, 3)) assert_df(""" 2019-01-03 3.0 """, restricted)
def test_options(engine, tsh): @func('dummy') def dummy(option: int = None) -> pd.Series: series = pd.Series([1, 2, 3], index=pd.date_range(dt(2019, 1, 1), periods=3, freq='D')) series.options = {'option': option} return series tsh.register_formula(engine, 'test_options', '(* 3 (dummy #:option 42))', False) ts = tsh.get(engine, 'test_options') assert_df(""" 2019-01-01 3 2019-01-02 6 2019-01-03 9 """, ts) assert ts.options == {'option': 42} FUNCS.pop('dummy')
def test_get_fast_path(client): series_in = genserie(utcdt(2018, 1, 1), 'H', 3) res = client.patch('/series/state', params={ 'name': 'test_fast', 'series': util.tojson(series_in), 'author': 'Babar', 'insertion_date': utcdt(2018, 1, 1, 10), 'tzaware': util.tzaware_serie(series_in) }) assert res.status_code == 201 out = client.get('/series/state', params={ 'name': 'test_fast', 'format': 'tshpack' }) meta, index, values = util.nary_unpack(zlib.decompress(out.body)) meta = json.loads(meta) index, values = util.numpy_deserialize(index, values, meta) series = pd.Series(values, index=index) series = series.tz_localize('UTC') assert_df( """ 2018-01-01 00:00:00+00:00 0.0 2018-01-01 01:00:00+00:00 1.0 2018-01-01 02:00:00+00:00 2.0 """, series) assert meta == { 'tzaware': True, 'index_type': 'datetime64[ns, UTC]', 'value_type': 'float64', 'index_dtype': '|M8[ns]', 'value_dtype': '<f8' }
def test_formula(client, engine, tsh): tsh.update( engine, pd.Series([1, 2, 3], index=pd.date_range(pd.Timestamp('2020-1-1', tz='UTC'), freq='D', periods=3)), 'in-a-formula', 'Babar') with pytest.raises(SyntaxError): client.register_formula('new-formula', '(+ 3') with pytest.raises(ValueError): client.register_formula('new-formula', '(+ 3 (series "lol"))') client.register_formula('new-formula', '(+ 3 (series "lol"))', reject_unknown=False) with pytest.raises(AssertionError): client.register_formula( 'new-formula', '(+ 3 (series "in-a-formula"))', ) client.register_formula('new-formula', '(+ 3 (series "in-a-formula"))', update=True) series = client.get('new-formula') assert_df( """ 2020-01-01 00:00:00+00:00 4.0 2020-01-02 00:00:00+00:00 5.0 2020-01-03 00:00:00+00:00 6.0 """, series) assert client.formula('new-formula') == '(+ 3 (series "in-a-formula"))' assert client.formula('lol') is None
def test_slice(engine, tsh): base = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2019, 1, 1), periods=3, freq='D')) tsh.update(engine, base, 'test-slice', 'Babar') tsh.register_formula( engine, 'slicing-id', '(slice (series "test-slice"))', ) tsh.register_formula( engine, 'slicing-from', '(slice (series "test-slice") #:fromdate (date "2019-1-2"))', ) tsh.register_formula( engine, 'slicing-fromto', '(slice (series "test-slice") ' ' #:fromdate (date "2019-1-2") ' ' #:todate (date "2019-1-2")' ')', ) tsh.register_formula( engine, 'slicing-empty', '(slice (series "test-slice") ' ' #:fromdate (date "2018-1-2") ' ' #:todate (date "2018-1-2")' ')', ) assert_df( """ 2019-01-01 00:00:00+00:00 1.0 2019-01-02 00:00:00+00:00 2.0 2019-01-03 00:00:00+00:00 3.0 """, tsh.get(engine, 'slicing-id')) assert_df( """ 2019-01-02 00:00:00+00:00 2.0 2019-01-03 00:00:00+00:00 3.0 """, tsh.get(engine, 'slicing-from')) assert_df(""" 2019-01-02 00:00:00+00:00 2.0 """, tsh.get(engine, 'slicing-fromto')) assert len(tsh.get(engine, 'slicing-empty')) == 0
def test_mul(engine, tsh): base = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2019, 1, 1), periods=3, freq='D')) tsh.update(engine, base, 'mul-a', 'Babar') tsh.update(engine, base, 'mul-b', 'Babar') tsh.update(engine, base, 'mul-c', 'Babar') tsh.register_formula( engine, 'multiply-aligned', '(mul (series "mul-a") (series "mul-b") (series "mul-c"))', ) ts = tsh.get(engine, 'multiply-aligned') assert_df( """ 2019-01-01 00:00:00+00:00 1.0 2019-01-02 00:00:00+00:00 8.0 2019-01-03 00:00:00+00:00 27.0 """, ts) base = pd.Series([1, 2, np.nan], index=pd.date_range(utcdt(2019, 1, 1), periods=3, freq='D')) tsh.update(engine, base, 'mul-b', 'Babar') ts = tsh.get(engine, 'multiply-aligned') assert_df( """ 2019-01-01 00:00:00+00:00 1.0 2019-01-02 00:00:00+00:00 8.0 """, ts) tsh.register_formula( engine, 'multiply-aligned', '(mul (series "mul-a") (series "mul-b" #:fill 1) (series "mul-c"))', update=True) ts = tsh.get(engine, 'multiply-aligned') assert_df( """ 2019-01-01 00:00:00+00:00 1.0 2019-01-02 00:00:00+00:00 8.0 2019-01-03 00:00:00+00:00 9.0 """, ts)
def test_div(engine, tsh): base = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2019, 1, 1), periods=3, freq='D')) tsh.update(engine, base, 'div-a', 'Babar') tsh.update(engine, base, 'div-b', 'Babar') tsh.register_formula( engine, 'divide', '(div (series "div-a") (series "div-b"))', ) ts = tsh.get(engine, 'divide') assert_df( """ 2019-01-01 00:00:00+00:00 1.0 2019-01-02 00:00:00+00:00 1.0 2019-01-03 00:00:00+00:00 1.0 """, ts) base = pd.Series([2, 1, np.nan], index=pd.date_range(utcdt(2019, 1, 1), periods=3, freq='D')) tsh.update(engine, base, 'div-b', 'Babar') ts = tsh.get(engine, 'divide') assert_df( """ 2019-01-01 00:00:00+00:00 0.5 2019-01-02 00:00:00+00:00 2.0 """, ts) tsh.register_formula(engine, 'divide', '(div (series "div-a") (series "div-b" #:fill 3))', update=True) ts = tsh.get(engine, 'divide') assert_df( """ 2019-01-01 00:00:00+00:00 0.5 2019-01-02 00:00:00+00:00 2.0 2019-01-03 00:00:00+00:00 1.0 """, ts)
def test_resample(engine, tsh): hourly = pd.Series(list(range(36)), index=pd.date_range(utcdt(2020, 1, 1), periods=36, freq='H')) gasday = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2020, 1, 1, 5), periods=3, freq='D')) tsh.update(engine, hourly, 'hourly', 'Babar') tsh.update(engine, gasday, 'gasday', 'Celeste') tsh.register_formula(engine, 'hourly2daily', '(resample (series "hourly") "D")') tsh.register_formula(engine, 'hourly2dailysum', '(resample (series "hourly") "D" "sum")') tsh.register_formula(engine, 'gasdaytoday', '(resample (series "gasday") "D")') tsh.register_formula(engine, 'badmethod', '(resample (series "gasday") "D" "NO-SUCH-METHOD")') assert_df( """ 2020-01-01 00:00:00+00:00 11.5 2020-01-02 00:00:00+00:00 29.5 """, tsh.get(engine, 'hourly2daily')) assert_df( """ 2020-01-01 00:00:00+00:00 276.0 2020-01-02 00:00:00+00:00 354.0 """, tsh.get(engine, 'hourly2dailysum')) assert_df( """ 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, tsh.get(engine, 'gasdaytoday')) with pytest.raises(ValueError) as err: tsh.get(engine, 'badmethod') assert err.value.args[0] == 'bad resampling method `NO-SUCH-METHOD`'
def test_local_formula_remote_series(mapihttp, engine): from tshistory_formula.tsio import timeseries as pgseries mapi = mapihttp assert repr(mapi) == ( 'timeseries(uri=postgresql://localhost:5433/postgres,' 'ns=ns-test-local,' 'sources=[source(uri=http://test-uri2,ns=ns-test-remote), ' 'source(uri=http://unavailable,ns=ns-test-unavailable-remote)])') assert len(mapi.othersources.sources) == 2 assert mapi.namespace == 'ns-test-local' assert mapi.uri == 'postgresql://localhost:5433/postgres' assert mapi.othersources.sources[0].namespace == 'ns-test-remote' assert mapi.othersources.sources[0].uri == 'http://test-uri2' series = pd.Series( [1, 2, 3], index=pd.date_range(pd.Timestamp('2020-1-1'), periods=3, freq='H'), ) mapi.update('local-series', series, 'Babar') rtsh = pgseries('ns-test-remote') rtsh.update(engine, series, 'remote-series', 'Celeste', insertion_date=pd.Timestamp('2020-1-1', tz='UTC')) cat = mapi.catalog(allsources=True) assert dict(cat) == { ('db://localhost:5433/postgres', 'ns-test-local'): [('local-series', 'primary')], ('db://localhost:5433/postgres', 'ns-test-remote'): [['remote-series', 'primary']] } mapi.register_formula('test-localformula-remoteseries', '(+ 1 (series "remote-series"))') ts = mapi.get('test-localformula-remoteseries') assert_df( """ 2020-01-01 00:00:00 2.0 2020-01-01 01:00:00 3.0 2020-01-01 02:00:00 4.0 """, ts) hist = mapi.history('test-localformula-remoteseries') assert_hist( """ insertion_date value_date 2020-01-01 00:00:00+00:00 2020-01-01 00:00:00 2.0 2020-01-01 01:00:00 3.0 2020-01-01 02:00:00 4.0 """, hist) ival = mapi.interval('remote-series') assert (ival.left, ival.right) == (pd.Timestamp('2020-01-01 00:00:00'), pd.Timestamp('2020-01-01 02:00:00')) with pytest.raises(ValueError) as err: mapi.interval('test-localformula-remoteseries') assert err.value.args[ 0] == 'no interval for series: test-localformula-remoteseries' meta = mapi.metadata('remote-series', all=True) assert meta == { 'index_dtype': '<M8[ns]', 'index_type': 'datetime64[ns]', 'tzaware': False, 'value_dtype': '<f8', 'value_type': 'float64' } meta = mapi.metadata('test-localformula-remoteseries', all=True) assert meta == { 'index_dtype': '<M8[ns]', 'index_type': 'datetime64[ns]', 'tzaware': False, 'value_dtype': '<f8', 'value_type': 'float64' }
def test_base_universal_api(pgapi, httpapi): series = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2020, 1, 1), periods=3, freq='D')) pgapi.update('api-test', series, 'Babar', insertion_date=utcdt(2019, 1, 1), metadata={'about': 'test'}) out = httpapi.get('api-test') assert_df( """ 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, out) series[utcdt(2020, 1, 4)] = 4 pgapi.update('api-test', series, 'Babar', insertion_date=utcdt(2019, 1, 2)) out = pgapi.get('api-test', from_value_date=utcdt(2020, 1, 2), to_value_date=utcdt(2020, 1, 3)) assert_df( """ 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, out) series[utcdt(2019, 12, 31)] = 0 pgapi.replace('api-test', series, 'Babar', insertion_date=utcdt(2019, 1, 3)) out = httpapi.get('api-test') assert_df( """ 2019-12-31 00:00:00+00:00 0.0 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 2020-01-04 00:00:00+00:00 4.0 """, out) assert httpapi.type('api-test') == pgapi.type('api-test') assert httpapi.interval('api-test') == pgapi.interval('api-test') out = httpapi.get('api-test', revision_date=utcdt(2019, 1, 1)) assert_df( """ 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, out) hist = httpapi.history('api-test') assert_hist( """ insertion_date value_date 2019-01-01 00:00:00+00:00 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 2019-01-02 00:00:00+00:00 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 2020-01-04 00:00:00+00:00 4.0 2019-01-03 00:00:00+00:00 2019-12-31 00:00:00+00:00 0.0 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 2020-01-04 00:00:00+00:00 4.0 """, hist) hist = pgapi.history('api-test', diffmode=True) assert_hist( """ insertion_date value_date 2019-01-01 00:00:00+00:00 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 2019-01-02 00:00:00+00:00 2020-01-04 00:00:00+00:00 4.0 2019-01-03 00:00:00+00:00 2019-12-31 00:00:00+00:00 0.0 """, hist) assert pgapi.exists('api-test') assert not pgapi.exists('i-dont-exist') assert httpapi.exists('api-test') assert not httpapi.exists('i-dont-exist') ival = pgapi.interval('api-test') assert ival.left == pd.Timestamp('2019-12-31 00:00:00+0000', tz='UTC') assert ival.right == pd.Timestamp('2020-01-04 00:00:00+0000', tz='UTC') meta = pgapi.metadata('api-test', all=True) assert meta == { 'tzaware': True, 'index_type': 'datetime64[ns, UTC]', 'value_type': 'float64', 'index_dtype': '|M8[ns]', 'value_dtype': '<f8' } meta = httpapi.metadata('api-test') assert meta == {} pgapi.update_metadata('api-test', {'desc': 'a metadata test'}) meta = pgapi.metadata('api-test') assert meta == {'desc': 'a metadata test'} assert pgapi.type('api-test') == 'primary' st = pgapi.staircase('api-test', delta=timedelta(days=366)) assert_df( """ 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 2020-01-04 00:00:00+00:00 4.0 """, st) pgapi.rename('api-test', 'api-test2') assert pgapi.exists('api-test2') assert not pgapi.exists('api-test') assert httpapi.exists('api-test2') assert not httpapi.exists('api-test') pgapi.delete('api-test2') assert not pgapi.exists('api-test') assert not httpapi.exists('api-test') pgapi.delete('api-test2')
def test_base(client, engine, tsh): assert repr(client) == "tshistory-http-client(uri='http://test-uri')" ts = client.get('no-such-series') assert ts is None assert not client.exists('no-such-series') meta = client.metadata('no-such-series') assert meta == {'message': '`no-such-series` does not exists'} series_in = genserie(utcdt(2018, 1, 1), 'H', 3) client.update('test', series_in, 'Babar', insertion_date=utcdt(2019, 1, 1)) assert client.exists('test') # now let's get it back ts = client.get('test') assert_df( """ 2018-01-01 00:00:00+00:00 0.0 2018-01-01 01:00:00+00:00 1.0 2018-01-01 02:00:00+00:00 2.0 """, ts) ts = client.get('test', from_value_date=utcdt(2018, 1, 1, 2)) assert_df(""" 2018-01-01 02:00:00+00:00 2.0 """, ts) ts = client.get('test', to_value_date=utcdt(2018, 1, 1, 0)) assert_df(""" 2018-01-01 00:00:00+00:00 0.0 """, ts) # out of range ts = client.get('test', from_value_date=utcdt(2020, 1, 1, 2), to_value_date=utcdt(2020, 1, 1, 2)) assert len(ts) == 0 assert ts.name == 'test' meta = client.metadata('test', all=True) assert meta == { 'tzaware': True, 'index_type': 'datetime64[ns, UTC]', 'value_type': 'float64', 'index_dtype': '|M8[ns]', 'value_dtype': '<f8' } # update client.update_metadata('test', {'desc': 'banana spot price'}) meta = client.metadata('test', all=False) assert meta == { 'desc': 'banana spot price', } # check the insertion_date series_in = genserie(utcdt(2018, 1, 2), 'H', 3) client.update('test', series_in, 'Babar', metadata={'event': 'hello'}, insertion_date=utcdt(2019, 1, 2)) v1 = client.get('test', revision_date=utcdt(2019, 1, 1)) assert_df( """ 2018-01-01 00:00:00+00:00 0.0 2018-01-01 01:00:00+00:00 1.0 2018-01-01 02:00:00+00:00 2.0 """, v1) d1, d2 = tsh.insertion_dates(engine, 'test') assert d1 == utcdt(2019, 1, 1) assert d2 > d1 client.update('test2', series_in, 'Babar') series = client.catalog() assert ['test', 'primary'] in series[('db://localhost:5433/postgres', 'tsh')] assert ['test2', 'primary'] in series[('db://localhost:5433/postgres', 'tsh')] client.replace('test2', genserie(utcdt(2020, 1, 1), 'D', 3), 'Babar') series = client.get('test2') assert_df( """ 2020-01-01 00:00:00+00:00 0.0 2020-01-02 00:00:00+00:00 1.0 2020-01-03 00:00:00+00:00 2.0 """, series) type = client.type('test2') assert type == 'primary' ival = client.interval('test2') assert ival.left == pd.Timestamp('2020-01-01 00:00:00+0000', tz='UTC') assert ival.right == pd.Timestamp('2020-01-03 00:00:00+0000', tz='UTC') client.rename('test2', 'test3') assert not client.exists('test2') assert client.exists('test3') client.delete('test3') assert not client.exists('test3')