def test_staircase_history_naive(client, tsh): # each days we insert 7 data points from datetime import datetime for idx, idate in enumerate( pd.date_range(start=utcdt(2015, 1, 1), end=utcdt(2015, 1, 4), freq='D')): series = genserie(start=idate.tz_convert(None), freq='H', repeat=7) client.update('staircase-naive', series, 'Babar', insertion_date=idate) series = client.staircase('staircase-naive', pd.Timedelta(hours=3), from_value_date=datetime(2015, 1, 1, 4), to_value_date=datetime(2015, 1, 2, 5)) assert series.name == 'staircase-naive' assert_df( """ 2015-01-01 04:00:00 4.0 2015-01-01 05:00:00 5.0 2015-01-01 06:00:00 6.0 2015-01-02 03:00:00 3.0 2015-01-02 04:00:00 4.0 2015-01-02 05:00:00 5.0 """, series) # series = client.staircase( # 'staircase-naive', # pd.Timedelta(hours=3), # from_value_date=datetime(2015, 1, 1, 4), # to_value_date=datetime(2015, 1, 2, 5) # ) hist = client.history('staircase-naive') assert len(hist) == 4 hist = client.history('staircase-naive', from_insertion_date=datetime(2015, 1, 2), to_insertion_date=datetime(2015, 1, 3)) assert len(hist) == 2 hist = client.history('staircase-naive', from_value_date=datetime(2015, 1, 1, 3), to_value_date=datetime(2015, 1, 2, 1)) assert all(series.name == 'staircase-naive' for series in hist.values()) assert_hist( """ insertion_date value_date 2015-01-01 00:00:00+00:00 2015-01-01 03:00:00 3.0 2015-01-01 04:00:00 4.0 2015-01-01 05:00:00 5.0 2015-01-01 06:00:00 6.0 2015-01-02 00:00:00+00:00 2015-01-01 03:00:00 3.0 2015-01-01 04:00:00 4.0 2015-01-01 05:00:00 5.0 2015-01-01 06:00:00 6.0 2015-01-02 00:00:00 0.0 2015-01-02 01:00:00 1.0 """, hist)
def test_custom_history(engine, tsh): @func('made-up-series') def madeup(base: int, coeff: float = 1.) -> pd.Series: return pd.Series(np.array([base, base + 1, base + 2]) * coeff, index=pd.date_range(dt(2019, 1, 1), periods=3, freq='D')) @finder('made-up-series') def find(cn, tsh, tree): return { tree[0]: { 'index_type': 'datetime64[ns]', 'index_dtype': '|M8[ns]', 'tzaware': False, 'value_type': 'float64', 'value_dtype': '<f8' } } @history('made-up-series') def madeup_history(__interpreter__, base, coeff): hist = {} for i in (1, 2, 3): hist[pd.Timestamp(f'2020-1-{i}', tz='utc')] = pd.Series( np.array([base, base + 1, base + 2]) * coeff, index=pd.date_range(dt(2019, 1, i), periods=3, freq='D')) return hist tsh.register_formula( engine, 'made-up', '(+ 3 (add (made-up-series 1 #:coeff 2.) (made-up-series 2 #:coeff .5)))', False) assert_df(""" 2019-01-01 6.0 2019-01-02 8.5 2019-01-03 11.0 """, tsh.get(engine, 'made-up')) hist = tsh.history(engine, 'made-up') assert_hist( """ insertion_date value_date 2020-01-01 00:00:00+00:00 2019-01-01 6.0 2019-01-02 8.5 2019-01-03 11.0 2020-01-02 00:00:00+00:00 2019-01-01 6.0 2019-01-02 8.5 2019-01-03 11.0 2020-01-03 00:00:00+00:00 2019-01-01 6.0 2019-01-02 8.5 2019-01-03 11.0 """, hist)
def test_local_formula_remote_series(mapi): rtsh = timeseries('test-mapi-2') rtsh.update( mapi.engine, pd.Series( [1, 2, 3], index=pd.date_range(pd.Timestamp('2020-1-1'), periods=3, freq='H'), ), 'remote-series', 'Babar', insertion_date=pd.Timestamp('2020-1-1', tz='UTC') ) mapi.register_formula( 'test-localformula-remoteseries', '(+ 1 (series "remote-series"))' ) ts = mapi.get('test-localformula-remoteseries') assert_df(""" 2020-01-01 00:00:00 2.0 2020-01-01 01:00:00 3.0 2020-01-01 02:00:00 4.0 """, ts) hist = mapi.history('test-localformula-remoteseries') assert_hist(""" insertion_date value_date 2020-01-01 00:00:00+00:00 2020-01-01 00:00:00 2.0 2020-01-01 01:00:00 3.0 2020-01-01 02:00:00 4.0 """, hist) f = mapi.formula('test-localformula-remoteseries') assert f == '(+ 1 (series "remote-series"))' none = mapi.formula('nosuchformula') assert none is None # altsource formula rtsh.register_formula( mapi.engine, 'remote-formula-remote-series', '(+ 2 (series "remote-series"))' ) f = mapi.formula('remote-formula-remote-series') assert f == '(+ 2 (series "remote-series"))' assert_df(""" 2020-01-01 00:00:00 3.0 2020-01-01 01:00:00 4.0 2020-01-01 02:00:00 5.0 """, mapi.get('remote-formula-remote-series'))
def test_base_universal_api(pgapi, httpapi): series = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2020, 1, 1), periods=3, freq='D')) pgapi.update('api-test', series, 'Babar', insertion_date=utcdt(2019, 1, 1), metadata={'about': 'test'}) out = httpapi.get('api-test') assert_df( """ 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, out) series[utcdt(2020, 1, 4)] = 4 pgapi.update('api-test', series, 'Babar', insertion_date=utcdt(2019, 1, 2)) out = pgapi.get('api-test', from_value_date=utcdt(2020, 1, 2), to_value_date=utcdt(2020, 1, 3)) assert_df( """ 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, out) series[utcdt(2019, 12, 31)] = 0 pgapi.replace('api-test', series, 'Babar', insertion_date=utcdt(2019, 1, 3)) out = httpapi.get('api-test') assert_df( """ 2019-12-31 00:00:00+00:00 0.0 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 2020-01-04 00:00:00+00:00 4.0 """, out) assert httpapi.type('api-test') == pgapi.type('api-test') assert httpapi.interval('api-test') == pgapi.interval('api-test') out = httpapi.get('api-test', revision_date=utcdt(2019, 1, 1)) assert_df( """ 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, out) hist = httpapi.history('api-test') assert_hist( """ insertion_date value_date 2019-01-01 00:00:00+00:00 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 2019-01-02 00:00:00+00:00 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 2020-01-04 00:00:00+00:00 4.0 2019-01-03 00:00:00+00:00 2019-12-31 00:00:00+00:00 0.0 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 2020-01-04 00:00:00+00:00 4.0 """, hist) hist = pgapi.history('api-test', diffmode=True) assert_hist( """ insertion_date value_date 2019-01-01 00:00:00+00:00 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 2019-01-02 00:00:00+00:00 2020-01-04 00:00:00+00:00 4.0 2019-01-03 00:00:00+00:00 2019-12-31 00:00:00+00:00 0.0 """, hist) assert pgapi.exists('api-test') assert not pgapi.exists('i-dont-exist') assert httpapi.exists('api-test') assert not httpapi.exists('i-dont-exist') ival = pgapi.interval('api-test') assert ival.left == pd.Timestamp('2019-12-31 00:00:00+0000', tz='UTC') assert ival.right == pd.Timestamp('2020-01-04 00:00:00+0000', tz='UTC') meta = pgapi.metadata('api-test', all=True) assert meta == { 'tzaware': True, 'index_type': 'datetime64[ns, UTC]', 'value_type': 'float64', 'index_dtype': '|M8[ns]', 'value_dtype': '<f8' } meta = httpapi.metadata('api-test') assert meta == {} pgapi.update_metadata('api-test', {'desc': 'a metadata test'}) meta = pgapi.metadata('api-test') assert meta == {'desc': 'a metadata test'} assert pgapi.type('api-test') == 'primary' st = pgapi.staircase('api-test', delta=timedelta(days=366)) assert_df( """ 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 2020-01-04 00:00:00+00:00 4.0 """, st) pgapi.rename('api-test', 'api-test2') assert pgapi.exists('api-test2') assert not pgapi.exists('api-test') assert httpapi.exists('api-test2') assert not httpapi.exists('api-test') pgapi.delete('api-test2') assert not pgapi.exists('api-test') assert not httpapi.exists('api-test') pgapi.delete('api-test2')
def test_local_formula_remote_series(mapihttp, engine): from tshistory_formula.tsio import timeseries as pgseries mapi = mapihttp assert repr(mapi) == ( 'timeseries(uri=postgresql://localhost:5433/postgres,' 'ns=ns-test-local,' 'sources=[source(uri=http://test-uri2,ns=ns-test-remote), ' 'source(uri=http://unavailable,ns=ns-test-unavailable-remote)])') assert len(mapi.othersources.sources) == 2 assert mapi.namespace == 'ns-test-local' assert mapi.uri == 'postgresql://localhost:5433/postgres' assert mapi.othersources.sources[0].namespace == 'ns-test-remote' assert mapi.othersources.sources[0].uri == 'http://test-uri2' series = pd.Series( [1, 2, 3], index=pd.date_range(pd.Timestamp('2020-1-1'), periods=3, freq='H'), ) mapi.update('local-series', series, 'Babar') rtsh = pgseries('ns-test-remote') rtsh.update(engine, series, 'remote-series', 'Celeste', insertion_date=pd.Timestamp('2020-1-1', tz='UTC')) cat = mapi.catalog(allsources=True) assert dict(cat) == { ('db://localhost:5433/postgres', 'ns-test-local'): [('local-series', 'primary')], ('db://localhost:5433/postgres', 'ns-test-remote'): [['remote-series', 'primary']] } mapi.register_formula('test-localformula-remoteseries', '(+ 1 (series "remote-series"))') ts = mapi.get('test-localformula-remoteseries') assert_df( """ 2020-01-01 00:00:00 2.0 2020-01-01 01:00:00 3.0 2020-01-01 02:00:00 4.0 """, ts) hist = mapi.history('test-localformula-remoteseries') assert_hist( """ insertion_date value_date 2020-01-01 00:00:00+00:00 2020-01-01 00:00:00 2.0 2020-01-01 01:00:00 3.0 2020-01-01 02:00:00 4.0 """, hist) ival = mapi.interval('remote-series') assert (ival.left, ival.right) == (pd.Timestamp('2020-01-01 00:00:00'), pd.Timestamp('2020-01-01 02:00:00')) with pytest.raises(ValueError) as err: mapi.interval('test-localformula-remoteseries') assert err.value.args[ 0] == 'no interval for series: test-localformula-remoteseries' meta = mapi.metadata('remote-series', all=True) assert meta == { 'index_dtype': '<M8[ns]', 'index_type': 'datetime64[ns]', 'tzaware': False, 'value_dtype': '<f8', 'value_type': 'float64' } meta = mapi.metadata('test-localformula-remoteseries', all=True) assert meta == { 'index_dtype': '<M8[ns]', 'index_type': 'datetime64[ns]', 'tzaware': False, 'value_dtype': '<f8', 'value_type': 'float64' }
def test_ifunc(engine, tsh): @func('shifted') def shifted(__interpreter__, name: str, days: int = 0) -> pd.Series: args = __interpreter__.getargs.copy() fromdate = args.get('from_value_date') todate = args.get('to_value_date') if fromdate: args['from_value_date'] = fromdate + timedelta(days=days) if todate: args['to_value_date'] = todate + timedelta(days=days) return __interpreter__.get(name, args) @finder('shifted') def find_series(cn, tsh, stree): return {stree[1]: tsh.metadata(cn, stree[1])} tsh.register_formula(engine, 'shifting', '(+ 0 (shifted "shiftme" #:days -1))', False) ts = pd.Series([1, 2, 3, 4, 5], index=pd.date_range(dt(2019, 1, 1), periods=5, freq='D')) tsh.update(engine, ts, 'shiftme', 'Babar', insertion_date=utcdt(2019, 1, 1)) ts = tsh.get(engine, 'shifting') assert_df( """ 2019-01-01 1.0 2019-01-02 2.0 2019-01-03 3.0 2019-01-04 4.0 2019-01-05 5.0 """, ts) ts = tsh.get(engine, 'shifting', from_value_date=dt(2019, 1, 3), to_value_date=dt(2019, 1, 4)) assert_df(""" 2019-01-02 2.0 2019-01-03 3.0 """, ts) # now, history ts = pd.Series([1, 2, 3, 4, 5], index=pd.date_range(dt(2019, 1, 2), periods=5, freq='D')) tsh.update(engine, ts, 'shiftme', 'Babar', insertion_date=utcdt(2019, 1, 2)) hist = tsh.history(engine, 'shifting') assert_hist( """ insertion_date value_date 2019-01-01 00:00:00+00:00 2019-01-01 1.0 2019-01-02 2.0 2019-01-03 3.0 2019-01-04 4.0 2019-01-05 5.0 2019-01-02 00:00:00+00:00 2019-01-01 1.0 2019-01-02 1.0 2019-01-03 2.0 2019-01-04 3.0 2019-01-05 4.0 2019-01-06 5.0 """, hist) hist = tsh.history(engine, 'shifting', from_value_date=dt(2019, 1, 3), to_value_date=dt(2019, 1, 4)) assert_hist( """ insertion_date value_date 2019-01-01 00:00:00+00:00 2019-01-03 3.0 2019-01-04 4.0 2019-01-02 00:00:00+00:00 2019-01-03 2.0 2019-01-04 3.0 """, hist) # cleanup FUNCS.pop('shifted')
def test_history(engine, tsh): tsh.register_formula(engine, 'h-addition', '(add (series "ha") (series "hb"))', False) for day in (1, 2, 3): idate = utcdt(2019, 1, day) for name in 'ab': ts = pd.Series([day] * 3, index=pd.date_range(dt(2018, 1, 1), periods=3, freq='D')) tsh.update(engine, ts, 'h' + name, 'Babar', insertion_date=idate) h = tsh.history(engine, 'h-addition') assert_hist( """ insertion_date value_date 2019-01-01 00:00:00+00:00 2018-01-01 2.0 2018-01-02 2.0 2018-01-03 2.0 2019-01-02 00:00:00+00:00 2018-01-01 4.0 2018-01-02 4.0 2018-01-03 4.0 2019-01-03 00:00:00+00:00 2018-01-01 6.0 2018-01-02 6.0 2018-01-03 6.0 """, h) dates = tsh.insertion_dates(engine, 'h-addition') assert dates == [(1, pd.Timestamp('2019-01-01 00:00:00+0000', tz='UTC')), (2, pd.Timestamp('2019-01-02 00:00:00+0000', tz='UTC')), (3, pd.Timestamp('2019-01-03 00:00:00+0000', tz='UTC'))] h = tsh.history(engine, 'h-addition', from_insertion_date=utcdt(2019, 1, 2), to_insertion_date=utcdt(2019, 1, 2), from_value_date=dt(2018, 1, 2), to_value_date=dt(2018, 1, 2)) assert_hist( """ insertion_date value_date 2019-01-02 00:00:00+00:00 2018-01-02 4.0 """, h) # let's add a priority tsh.register_formula(engine, 'h-priority', '(priority (series "hz") (series "h-addition"))', False) for day in (1, 2, 3): idate = utcdt(2019, 1, day) ts = pd.Series([41 + day] * 3, index=pd.date_range(dt(2018, 1, 3), periods=3, freq='D')) tsh.update(engine, ts, 'hz', 'Babar', insertion_date=idate) h = tsh.history(engine, 'h-priority') assert_hist( """ insertion_date value_date 2019-01-01 00:00:00+00:00 2018-01-01 2.0 2018-01-02 2.0 2018-01-03 42.0 2018-01-04 42.0 2018-01-05 42.0 2019-01-02 00:00:00+00:00 2018-01-01 4.0 2018-01-02 4.0 2018-01-03 43.0 2018-01-04 43.0 2018-01-05 43.0 2019-01-03 00:00:00+00:00 2018-01-01 6.0 2018-01-02 6.0 2018-01-03 44.0 2018-01-04 44.0 2018-01-05 44.0 """, h)
def test_base(client): # insert series_in = genserie(utcdt(2018, 1, 1), 'H', 3) res = client.patch('/series/state', params={ 'name': 'test', 'series': util.tojson(series_in), 'author': 'Babar', 'insertion_date': utcdt(2018, 1, 1, 10), 'tzaware': util.tzaware_serie(series_in) }) assert res.status_code == 201 # catalog res = client.get('/series/catalog') assert res.status_code == 200 assert res.json == { 'db://localhost:5433/postgres!tsh': [['test-naive', 'primary'], ['test', 'primary']] } # metadata res = client.get('/series/metadata?name=test') meta = res.json assert meta == {} res = client.get('/series/metadata?name=test&all=1') meta = res.json assert meta == { 'index_dtype': '|M8[ns]', 'index_type': 'datetime64[ns, UTC]', 'tzaware': True, 'value_dtype': '<f8', 'value_type': 'float64' } res = client.put('/series/metadata', params={ 'metadata': json.dumps({ 'freq': 'D', 'description': 'banana spot price' }), 'name': 'test' }) assert res.status_code == 200 res = client.get('/series/metadata?name=test') meta2 = res.json assert meta2 == {'freq': 'D', 'description': 'banana spot price'} # metadata: delete by uploading an empty dict res = client.put('/series/metadata', params={ 'metadata': json.dumps({}), 'name': 'test' }) assert res.status_code == 200 res = client.get('/series/metadata?name=test') meta2 = res.json assert meta2 == {} # get res = client.get('/series/state?name=test') series = util.fromjson(res.body, 'test', meta['tzaware']) assert_df( """ 2018-01-01 00:00:00+00:00 0.0 2018-01-01 01:00:00+00:00 1.0 2018-01-01 02:00:00+00:00 2.0 """, series) # reinsert series_in = genserie(utcdt(2018, 1, 1, 3), 'H', 1, [3]) res = client.patch('/series/state', params={ 'name': 'test', 'series': util.tojson(series_in), 'author': 'Babar', 'insertion_date': utcdt(2018, 1, 1, 13), 'tzaware': util.tzaware_serie(series_in) }) assert res.status_code == 200 res = client.get('/series/state?name=test') series = util.fromjson(res.body, 'test', meta['tzaware']) assert_df( """ 2018-01-01 00:00:00+00:00 0.0 2018-01-01 01:00:00+00:00 1.0 2018-01-01 02:00:00+00:00 2.0 2018-01-01 03:00:00+00:00 3.0 """, series) # checkout a past state res = client.get('/series/state', params={ 'name': 'test', 'insertion_date': utcdt(2018, 1, 1, 10) }) series = util.fromjson(res.body, 'test', meta['tzaware']) assert_df( """ 2018-01-01 00:00:00+00:00 0.0 2018-01-01 01:00:00+00:00 1.0 2018-01-01 02:00:00+00:00 2.0 """, series) # checkout too far in the past res = client.get('/series/state', params={ 'name': 'test', 'insertion_date': utcdt(2018, 1, 1, 0) }) assert res.json is None # history res = client.get('/series/history?name=test') df = pd.read_json(res.body) # we real client would need to handle timestamp # tz-awareness assert_df( """ 2018-01-01 10:00:00 2018-01-01 13:00:00 2018-01-01 00:00:00 0.0 0 2018-01-01 01:00:00 1.0 1 2018-01-01 02:00:00 2.0 2 2018-01-01 03:00:00 NaN 3 """, df) res = client.get('/series/history?name=test&format=tshpack') meta, hist = util.unpack_history(res.body) assert_hist( """ insertion_date value_date 2018-01-01 10:00:00+00:00 2018-01-01 00:00:00+00:00 0.0 2018-01-01 01:00:00+00:00 1.0 2018-01-01 02:00:00+00:00 2.0 2018-01-01 13:00:00+00:00 2018-01-01 00:00:00+00:00 0.0 2018-01-01 01:00:00+00:00 1.0 2018-01-01 02:00:00+00:00 2.0 2018-01-01 03:00:00+00:00 3.0 """, hist) # diff mode res = client.get('/series/history', params={ 'name': 'test', 'diffmode': True }) df = pd.read_json(res.body) assert_df( """ 2018-01-01 10:00:00 2018-01-01 13:00:00 2018-01-01 00:00:00 0.0 NaN 2018-01-01 01:00:00 1.0 NaN 2018-01-01 02:00:00 2.0 NaN 2018-01-01 03:00:00 NaN 3.0 """, df) # empty range res = client.get('/series/history', params={ 'name': 'test', 'from_insertion_date': utcdt(2018, 1, 1, 11), 'to_insertion_date': utcdt(2018, 1, 1, 12), }) df = pd.read_json(res.body) assert len(df) == 0 # insertion dates subset res = client.get('/series/history', params={ 'name': 'test', 'from_insertion_date': utcdt(2018, 1, 1, 10), 'to_insertion_date': utcdt(2018, 1, 1, 12), }) df = pd.read_json(res.body) assert_df( """ 2018-01-01 10:00:00 2018-01-01 00:00:00 0 2018-01-01 01:00:00 1 2018-01-01 02:00:00 2 """, df) # value dates subset res = client.get('/series/history', params={ 'name': 'test', 'from_value_date': utcdt(2018, 1, 1, 2), 'to_value_date': utcdt(2018, 1, 1, 3), }) df = pd.read_json(res.body) assert_df( """ 2018-01-01 10:00:00 2018-01-01 13:00:00 2018-01-01 02:00:00 2.0 2 2018-01-01 03:00:00 NaN 3 """, df) # state/get from from/to value date restriction res = client.get('/series/state', params={ 'name': 'test', 'from_value_date': utcdt(2018, 1, 1, 1), 'to_value_date': utcdt(2018, 1, 1, 2) }) assert res.json == { '2018-01-01T01:00:00.000Z': 1.0, '2018-01-01T02:00:00.000Z': 2.0 } series_in = genserie(utcdt(2019, 1, 1), 'H', 3) res = client.patch('/series/state', params={ 'name': 'test', 'series': util.tojson(series_in), 'author': 'Babar', 'insertion_date': utcdt(2018, 1, 2), 'tzaware': util.tzaware_serie(series_in), 'replace': True }) assert res.status_code == 200 res = client.get('/series/state', params={'name': 'test'}) assert res.json == { '2019-01-01T00:00:00.000Z': 0.0, '2019-01-01T01:00:00.000Z': 1.0, '2019-01-01T02:00:00.000Z': 2.0 } res = client.get('/series/metadata', params={ 'name': 'test', 'type': 'interval' }) assert res.json == [ True, '2019-01-01T00:00:00+00:00', '2019-01-01T02:00:00+00:00' ] res = client.get('/series/metadata', params={ 'name': 'test', 'type': 'type' }) assert res.json == 'primary'