def test_pack_unpack_history(tsh, engine): for numserie in (1, 2, 3): with engine.begin() as cn: tsh.update(cn, genserie(datetime(2021, 1, 1), 'D', numserie), 'small-hist-naive', '*****@*****.**', insertion_date=utcdt(2021, 2, numserie)) hist = tsh.history(engine, 'small-hist-naive') meta = tsh.metadata(engine, 'small-hist-naive') packed = pack_history(meta, hist) meta2, hist2 = unpack_history(packed) assert meta2 == meta for idate, series in hist.items(): assert hist[idate].equals(series) for numserie in (1, 2, 3): with engine.begin() as cn: tsh.update(cn, genserie(utcdt(2021, 1, 1), 'D', numserie), 'small-hist-tzaware', '*****@*****.**', insertion_date=utcdt(2021, 2, numserie)) hist = tsh.history(engine, 'small-hist-tzaware') meta = tsh.metadata(engine, 'small-hist-tzaware') packed = pack_history(meta, hist) meta2, hist2 = unpack_history(packed) assert meta2 == meta for idate, series in hist.items(): assert hist[idate].equals(series)
def test_complicated_thing(mapi): groundzero = pd.Series([0, 0, 0], index=pd.date_range(utcdt(2019, 1, 1), periods=3, freq='D')) one = pd.Series([1, 1, 1], index=pd.date_range(utcdt(2019, 1, 1), periods=3, freq='D')) mapi.update('groundzero-b', groundzero, 'Babar') mapi.update('one-b', one, 'Celeste') mapi.register_formula( 'complicated', '(add (* 3.1416 (series "groundzero-b" #:fill "bfill" #:prune 1))' ' (* 2 (min (+ 1 (series "groundzero-b")) (series "one-b"))))', ) presenter = fancypresenter(mapi, 'complicated', {}) info = [{k: v for k, v in info.items() if k != 'ts'} for info in presenter.infos] assert info == [{ 'name': 'complicated', 'type': 'formula' }, { 'name': 'groundzero-b', 'type': 'primary' }, { 'name': 'one-b', 'type': 'primary' }]
def test_staircase_history_naive(client, tsh): # each days we insert 7 data points from datetime import datetime for idx, idate in enumerate( pd.date_range(start=utcdt(2015, 1, 1), end=utcdt(2015, 1, 4), freq='D')): series = genserie(start=idate.tz_convert(None), freq='H', repeat=7) client.update('staircase-naive', series, 'Babar', insertion_date=idate) series = client.staircase('staircase-naive', pd.Timedelta(hours=3), from_value_date=datetime(2015, 1, 1, 4), to_value_date=datetime(2015, 1, 2, 5)) assert series.name == 'staircase-naive' assert_df( """ 2015-01-01 04:00:00 4.0 2015-01-01 05:00:00 5.0 2015-01-01 06:00:00 6.0 2015-01-02 03:00:00 3.0 2015-01-02 04:00:00 4.0 2015-01-02 05:00:00 5.0 """, series) # series = client.staircase( # 'staircase-naive', # pd.Timedelta(hours=3), # from_value_date=datetime(2015, 1, 1, 4), # to_value_date=datetime(2015, 1, 2, 5) # ) hist = client.history('staircase-naive') assert len(hist) == 4 hist = client.history('staircase-naive', from_insertion_date=datetime(2015, 1, 2), to_insertion_date=datetime(2015, 1, 3)) assert len(hist) == 2 hist = client.history('staircase-naive', from_value_date=datetime(2015, 1, 1, 3), to_value_date=datetime(2015, 1, 2, 1)) assert all(series.name == 'staircase-naive' for series in hist.values()) assert_hist( """ insertion_date value_date 2015-01-01 00:00:00+00:00 2015-01-01 03:00:00 3.0 2015-01-01 04:00:00 4.0 2015-01-01 05:00:00 5.0 2015-01-01 06:00:00 6.0 2015-01-02 00:00:00+00:00 2015-01-01 03:00:00 3.0 2015-01-01 04:00:00 4.0 2015-01-01 05:00:00 5.0 2015-01-01 06:00:00 6.0 2015-01-02 00:00:00 0.0 2015-01-02 01:00:00 1.0 """, hist)
def test_alternative_handler(pgapi): api = pgapi sapi = timeseries(api.uri, api.namespace, formula_class()) sapi.update( 'test-features', genserie(utcdt(2020, 1, 1), 'D', 3), 'Babar', ) sapi.tsh.register_formula(sapi.engine, 'test-formula', '(+ 1 (series "test-features"))') tsa = sapi.get('test-features') assert_df( """ 2020-01-01 00:00:00+00:00 0.0 2020-01-02 00:00:00+00:00 1.0 2020-01-03 00:00:00+00:00 2.0 """, tsa) tsb = sapi.get('test-formula') assert_df( """ 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, tsb) class supervision_and_formula(supervision_class(), formula_class()): pass sapi = timeseries(api.uri, api.namespace, supervision_and_formula) tsa = sapi.get('test-features') assert_df( """ 2020-01-01 00:00:00+00:00 0.0 2020-01-02 00:00:00+00:00 1.0 2020-01-03 00:00:00+00:00 2.0 """, tsa) tsb = sapi.get('test-formula') assert_df( """ 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, tsb) sapi.update('test-features', genserie(utcdt(2020, 1, 2), 'D', 3), 'Babar', manual=True) tsb = sapi.get('test-formula') assert_df( """ 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 1.0 2020-01-03 00:00:00+00:00 2.0 2020-01-04 00:00:00+00:00 3.0 """, tsb)
def test_naive(client): series_in = genserie(pd.Timestamp('2018-1-1'), 'H', 3) res = client.patch('/series/state', params={ 'name': 'test-naive', 'series': util.tojson(series_in), 'author': 'Babar', 'insertion_date': utcdt(2018, 1, 1, 10), 'tzaware': util.tzaware_serie(series_in) }) assert res.status_code == 201 res = client.get('/series/metadata?name=test-naive&all=1') meta = res.json assert meta == { 'index_dtype': '<M8[ns]', 'index_type': 'datetime64[ns]', 'tzaware': False, 'value_dtype': '<f8', 'value_type': 'float64' } res = client.get('/series/state?name=test-naive') series = util.fromjson(res.body, 'test', meta['tzaware']) assert_df( """ 2018-01-01 00:00:00 0.0 2018-01-01 01:00:00 1.0 2018-01-01 02:00:00 2.0 """, series)
def test_multisources(client, engine): series = genserie(utcdt(2020, 1, 1), 'D', 3) tsh = tsio.timeseries('other') tsh.update(engine, series, 'test-other', 'Babar') client.update('test-mainsource', series, 'Babar') with pytest.raises(ValueError) as err: client.update('test-other', series, 'Babar') assert err.value.args[0] == 'not allowed to update to a secondary source' with pytest.raises(ValueError) as err: client.replace('test-other', series, 'Babar') assert err.value.args[0] == 'not allowed to replace to a secondary source' cat = client.catalog() assert cat == { ('db://localhost:5433/postgres', 'other'): [['test-other', 'primary']], ('db://localhost:5433/postgres', 'tsh'): [['test-naive', 'primary'], ['test', 'primary'], ['staircase', 'primary'], ['staircase-naive', 'primary'], ['in-a-formula', 'primary'], ['test-mainsource', 'primary'], ['new-formula', 'formula']] } cat = client.catalog(allsources=False) assert ('db://localhost:5433/postgres', 'tsh') in cat assert ('db://localhost:5433/postgres', 'other') not in cat
def test_mul(engine, tsh): base = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2019, 1, 1), periods=3, freq='D')) tsh.update(engine, base, 'mul-a', 'Babar') tsh.update(engine, base, 'mul-b', 'Babar') tsh.update(engine, base, 'mul-c', 'Babar') tsh.register_formula( engine, 'multiply-aligned', '(mul (series "mul-a") (series "mul-b") (series "mul-c"))', ) ts = tsh.get(engine, 'multiply-aligned') assert_df( """ 2019-01-01 00:00:00+00:00 1.0 2019-01-02 00:00:00+00:00 8.0 2019-01-03 00:00:00+00:00 27.0 """, ts) base = pd.Series([1, 2, np.nan], index=pd.date_range(utcdt(2019, 1, 1), periods=3, freq='D')) tsh.update(engine, base, 'mul-b', 'Babar') ts = tsh.get(engine, 'multiply-aligned') assert_df( """ 2019-01-01 00:00:00+00:00 1.0 2019-01-02 00:00:00+00:00 8.0 """, ts) tsh.register_formula( engine, 'multiply-aligned', '(mul (series "mul-a") (series "mul-b" #:fill 1) (series "mul-c"))', update=True) ts = tsh.get(engine, 'multiply-aligned') assert_df( """ 2019-01-01 00:00:00+00:00 1.0 2019-01-02 00:00:00+00:00 8.0 2019-01-03 00:00:00+00:00 9.0 """, ts)
def test_editor_table_callback(mapi): groundzero = pd.Series([0, 0, 0], index=pd.date_range(utcdt(2019, 1, 1), periods=3, freq='D')) one = pd.Series([1, 1, 1], index=pd.date_range(utcdt(2019, 1, 1), periods=3, freq='D')) mapi.update('groundzero-a', groundzero, 'Babar') mapi.update('one-a', one, 'Celeste') mapi.register_formula( 'editor-1', '(add (* 3.1416 (series "groundzero-a" #:fill "bfill" #:prune 1)) (series "one-a"))', ) presenter = fancypresenter(mapi, 'editor-1', {}) info = [{k: v for k, v in info.items() if k != 'ts'} for info in presenter.infos] assert info == [{ 'name': 'editor-1', 'type': 'formula' }, { 'name': 'groundzero-a', 'type': 'primary' }, { 'name': 'one-a', 'type': 'primary' }] # trigger an empty series presenter = fancypresenter(mapi, 'editor-1', {'from_value_date': utcdt(2019, 1, 4)}) info = [{k: v for k, v in info.items() if k != 'ts'} for info in presenter.infos] assert info == [{ 'name': 'editor-1', 'type': 'formula' }, { 'name': 'groundzero-a', 'type': 'primary' }, { 'name': 'one-a', 'type': 'primary' }]
def test_delete(client): series_in = genserie(utcdt(2018, 1, 1), 'H', 3) res = client.patch('/series/state', params={ 'name': 'test', 'series': util.tojson(series_in), 'author': 'Babar', 'insertion_date': utcdt(2018, 1, 1, 10), 'tzaware': util.tzaware_serie(series_in) }) res = client.delete('/series/state', params={'name': 'no-such-series'}) assert res.status_code == 404 res = client.delete('/series/state', params={'name': 'test'}) assert res.status_code == 204 res = client.get('/series/catalog') assert 'test' not in res.json
def test_rename(client): series_in = genserie(utcdt(2018, 1, 1), 'H', 3) res = client.patch('/series/state', params={ 'name': 'test', 'series': util.tojson(series_in), 'author': 'Babar', 'insertion_date': utcdt(2018, 1, 1, 10), 'tzaware': util.tzaware_serie(series_in) }) res = client.put('/series/state', params={ 'name': 'no-such-series', 'newname': 'no-better' }) assert res.status_code == 404 res = client.put('/series/state', params={ 'name': 'test', 'newname': 'test2' }) assert res.status_code == 204 res = client.get('/series/catalog') assert res.json == { 'db://localhost:5433/postgres!tsh': [['test-naive', 'primary'], ['test2', 'primary']] } assert 'test' not in res.json res = client.patch('/series/state', params={ 'name': 'test3', 'series': util.tojson(series_in), 'author': 'Babar', 'insertion_date': utcdt(2018, 1, 1, 10), 'tzaware': util.tzaware_serie(series_in) }) res = client.put('/series/state', params={ 'name': 'test2', 'newname': 'test3' }) assert res.status_code == 409 assert res.json == {'message': '`test3` does exists'}
def test_resample(engine, tsh): hourly = pd.Series(list(range(36)), index=pd.date_range(utcdt(2020, 1, 1), periods=36, freq='H')) gasday = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2020, 1, 1, 5), periods=3, freq='D')) tsh.update(engine, hourly, 'hourly', 'Babar') tsh.update(engine, gasday, 'gasday', 'Celeste') tsh.register_formula(engine, 'hourly2daily', '(resample (series "hourly") "D")') tsh.register_formula(engine, 'hourly2dailysum', '(resample (series "hourly") "D" "sum")') tsh.register_formula(engine, 'gasdaytoday', '(resample (series "gasday") "D")') tsh.register_formula(engine, 'badmethod', '(resample (series "gasday") "D" "NO-SUCH-METHOD")') assert_df( """ 2020-01-01 00:00:00+00:00 11.5 2020-01-02 00:00:00+00:00 29.5 """, tsh.get(engine, 'hourly2daily')) assert_df( """ 2020-01-01 00:00:00+00:00 276.0 2020-01-02 00:00:00+00:00 354.0 """, tsh.get(engine, 'hourly2dailysum')) assert_df( """ 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, tsh.get(engine, 'gasdaytoday')) with pytest.raises(ValueError) as err: tsh.get(engine, 'badmethod') assert err.value.args[0] == 'bad resampling method `NO-SUCH-METHOD`'
def test_div(engine, tsh): base = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2019, 1, 1), periods=3, freq='D')) tsh.update(engine, base, 'div-a', 'Babar') tsh.update(engine, base, 'div-b', 'Babar') tsh.register_formula( engine, 'divide', '(div (series "div-a") (series "div-b"))', ) ts = tsh.get(engine, 'divide') assert_df( """ 2019-01-01 00:00:00+00:00 1.0 2019-01-02 00:00:00+00:00 1.0 2019-01-03 00:00:00+00:00 1.0 """, ts) base = pd.Series([2, 1, np.nan], index=pd.date_range(utcdt(2019, 1, 1), periods=3, freq='D')) tsh.update(engine, base, 'div-b', 'Babar') ts = tsh.get(engine, 'divide') assert_df( """ 2019-01-01 00:00:00+00:00 0.5 2019-01-02 00:00:00+00:00 2.0 """, ts) tsh.register_formula(engine, 'divide', '(div (series "div-a") (series "div-b" #:fill 3))', update=True) ts = tsh.get(engine, 'divide') assert_df( """ 2019-01-01 00:00:00+00:00 0.5 2019-01-02 00:00:00+00:00 2.0 2019-01-03 00:00:00+00:00 1.0 """, ts)
def test_editor_pure_scalar_op(mapi): ts = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2020, 1, 1), periods=3, freq='D')) mapi.update('pure-scalar-ops', ts, 'Babar') mapi.register_formula('formula-pure-scalar-ops', '(+ (* 3 (/ 6 2)) (series "pure-scalar-ops"))') presenter = fancypresenter(mapi, 'formula-pure-scalar-ops', {'from_value_date': utcdt(2019, 1, 4)}) info = [{k: v for k, v in info.items() if k != 'ts'} for info in presenter.infos] assert info == [{ 'name': 'formula-pure-scalar-ops', 'type': 'formula' }, { 'name': 'pure-scalar-ops', 'type': 'primary' }]
def test_history(engine, cli, tsh): serie = genserie(datetime(2020, 1, 1), 'D', 3) tsh.update(engine, serie, 'some_history', 'Babar', insertion_date=utcdt(2019, 1, 1)) serie = genserie(datetime(2020, 1, 2), 'D', 3) tsh.update(engine, serie, 'some_history', 'Babar', insertion_date=utcdt(2019, 1, 2)) r = cli('history', engine.url, 'some_history', json=True) assert r.output == ( '{"2019-01-01 00:00:00+00:00": {"2020-01-01 00:00:00": 0.0, "2020-01-02 00:00:00": 1.0, "2020-01-03 00:00:00": 2.0}, "2019-01-02 00:00:00+00:00": {"2020-01-02 00:00:00": 0.0, "2020-01-03 00:00:00": 1.0, "2020-01-04 00:00:00": 2.0}}\n' )
def test_staircase(engine, tsh): tsh.register_formula(engine, 's-addition', '(add (series "sa") (series "sb"))', False) for day in (1, 2, 3, 4, 5): idate = utcdt(2018, 1, day) for name in 'ab': ts = pd.Series([day / 2.] * 5, index=pd.date_range(dt(2018, 1, day), periods=5, freq='D')) tsh.update(engine, ts, 's' + name, 'Babar', insertion_date=idate) ts = tsh.staircase(engine, 's-addition', delta=pd.Timedelta(hours=12)) assert_df( """ 2018-01-02 1.0 2018-01-03 2.0 2018-01-04 3.0 2018-01-05 4.0 2018-01-06 5.0 2018-01-07 5.0 2018-01-08 5.0 2018-01-09 5.0 """, ts) # this is not allowed in the staircase fast-path # hence we will take the slow path @func('identity') def identity(series: pd.Series) -> pd.Series: return series tsh.register_formula(engine, 'slow-down', '(identity (series "sa"))', False) tsh.register_formula(engine, 's-addition-not-fast', '(add (series "slow-down") (series "sb"))', False) ts = tsh.staircase(engine, 's-addition-not-fast', delta=pd.Timedelta(hours=12)) assert_df( """ 2018-01-02 1.0 2018-01-03 2.0 2018-01-04 3.0 2018-01-05 4.0 2018-01-06 5.0 2018-01-07 5.0 2018-01-08 5.0 2018-01-09 5.0 """, ts) # cleanup FUNCS.pop('identity')
def create_data(): # 4 years of sub-hourly points for year in range(2015, 2020): date = utcdt(year, 1, 1) serie = genserie(date, '10Min', 6 * 24 * 365) with engine.begin() as cn: tsh.update(cn, serie, 'big', '*****@*****.**', insertion_date=date)
def create(uri, ns, name): api = timeseries(uri, ns) series = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2020, 1, 1), periods=3, freq='D')) api.update(name, series, 'Babar', insertion_date=utcdt(2019, 1, 1), metadata={'about': 'test'}) out = api.get(name) assert_df( """ 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, out) series[utcdt(2020, 1, 4)] = 4 api.update(name, series, 'Babar', insertion_date=utcdt(2019, 1, 2)) out = api.get(name, from_value_date=utcdt(2020, 1, 2), to_value_date=utcdt(2020, 1, 3)) assert_df( """ 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, out)
def create_data(): # one insert per day for 4 months for month in range(1, 4): days = calendar.monthrange(2017, month)[1] for day in range(1, days + 1): date = utcdt(2017, month, day) serie = genserie(date, '10Min', 6 * 24) with engine.begin() as cn: tsh.update(cn, serie, 'manydiffs', '*****@*****.**', insertion_date=date.replace(year=2018))
def test_slice(engine, tsh): base = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2019, 1, 1), periods=3, freq='D')) tsh.update(engine, base, 'test-slice', 'Babar') tsh.register_formula( engine, 'slicing-id', '(slice (series "test-slice"))', ) tsh.register_formula( engine, 'slicing-from', '(slice (series "test-slice") #:fromdate (date "2019-1-2"))', ) tsh.register_formula( engine, 'slicing-fromto', '(slice (series "test-slice") ' ' #:fromdate (date "2019-1-2") ' ' #:todate (date "2019-1-2")' ')', ) tsh.register_formula( engine, 'slicing-empty', '(slice (series "test-slice") ' ' #:fromdate (date "2018-1-2") ' ' #:todate (date "2018-1-2")' ')', ) assert_df( """ 2019-01-01 00:00:00+00:00 1.0 2019-01-02 00:00:00+00:00 2.0 2019-01-03 00:00:00+00:00 3.0 """, tsh.get(engine, 'slicing-id')) assert_df( """ 2019-01-02 00:00:00+00:00 2.0 2019-01-03 00:00:00+00:00 3.0 """, tsh.get(engine, 'slicing-from')) assert_df(""" 2019-01-02 00:00:00+00:00 2.0 """, tsh.get(engine, 'slicing-fromto')) assert len(tsh.get(engine, 'slicing-empty')) == 0
def test_get_fast_path(client): series_in = genserie(utcdt(2018, 1, 1), 'H', 3) res = client.patch('/series/state', params={ 'name': 'test_fast', 'series': util.tojson(series_in), 'author': 'Babar', 'insertion_date': utcdt(2018, 1, 1, 10), 'tzaware': util.tzaware_serie(series_in) }) assert res.status_code == 201 out = client.get('/series/state', params={ 'name': 'test_fast', 'format': 'tshpack' }) meta, index, values = util.nary_unpack(zlib.decompress(out.body)) meta = json.loads(meta) index, values = util.numpy_deserialize(index, values, meta) series = pd.Series(values, index=index) series = series.tz_localize('UTC') assert_df( """ 2018-01-01 00:00:00+00:00 0.0 2018-01-01 01:00:00+00:00 1.0 2018-01-01 02:00:00+00:00 2.0 """, series) assert meta == { 'tzaware': True, 'index_type': 'datetime64[ns, UTC]', 'value_type': 'float64', 'index_dtype': '|M8[ns]', 'value_dtype': '<f8' }
def test_lots_of_diffs(engine, tracker, ptsh): tsh = ptsh def create_data(): # one insert per day for 4 months for month in range(1, 4): days = calendar.monthrange(2017, month)[1] for day in range(1, days + 1): date = utcdt(2017, month, day) serie = genserie(date, '10Min', 6 * 24) with engine.begin() as cn: tsh.update(cn, serie, 'manydiffs', '*****@*****.**', insertion_date=date.replace(year=2018)) t0 = time() create_data() t1 = time() - t0 tshclass = tsh.__class__.__name__ tracker.append({'test': 'manydiffs_insert', 'class': tshclass, 'time': t1}) t0 = time() with engine.begin() as cn: tsh.history(cn, 'manydiffs') t1 = time() - t0 tracker.append({ 'test': 'manydiffs_history_all', 'class': tshclass, 'time': t1 }) t0 = time() for month in range(1, 3): for day in range(1, 5): date = utcdt(2018, month, day) with engine.begin() as cn: ts = tsh.history(cn, 'manydiffs', from_insertion_date=date, to_insertion_date=date + timedelta(days=31)) assert ts is not None t1 = time() - t0 tracker.append({ 'test': 'manydiffs_history_chunks', 'class': tshclass, 'time': t1 })
def test_staircase(client): # each days we insert 7 data points for idate in pd.date_range(start=utcdt(2015, 1, 1), end=utcdt(2015, 1, 4), freq='D'): series = genserie(start=idate, freq='H', repeat=7) client.patch('/series/state', params={ 'name': 'staircase', 'series': util.tojson(series), 'author': 'Babar', 'insertion_date': idate, 'tzaware': util.tzaware_serie(series) }) res = client.get('/series/staircase', params={ 'name': 'staircase', 'delta': pd.Timedelta(hours=3), 'from_value_date': utcdt(2015, 1, 1, 4), 'to_value_date': utcdt(2015, 1, 2, 5), }) series = util.fromjson(res.body, 'test', True) assert_df( """ 2015-01-01 04:00:00+00:00 4.0 2015-01-01 05:00:00+00:00 5.0 2015-01-01 06:00:00+00:00 6.0 2015-01-02 03:00:00+00:00 3.0 2015-01-02 04:00:00+00:00 4.0 2015-01-02 05:00:00+00:00 5.0 """, series) res = client.get('/series/staircase', params={ 'name': 'staircase', 'delta': pd.Timedelta(hours=3), 'from_value_date': utcdt(2015, 1, 1, 4), 'to_value_date': utcdt(2015, 1, 2, 5), 'format': 'tshpack' }) meta, index, values = util.nary_unpack(zlib.decompress(res.body)) meta = json.loads(meta) index, values = util.numpy_deserialize(index, values, meta) series = pd.Series(values, index=index) series = series.tz_localize('UTC') assert_df( """ 2015-01-01 04:00:00+00:00 4.0 2015-01-01 05:00:00+00:00 5.0 2015-01-01 06:00:00+00:00 6.0 2015-01-02 03:00:00+00:00 3.0 2015-01-02 04:00:00+00:00 4.0 2015-01-02 05:00:00+00:00 5.0 """, series)
def test_naive(client, engine, tsh): series_in = genserie(pd.Timestamp('2018-1-1'), 'H', 3) client.update('test-naive', series_in, 'Babar', insertion_date=utcdt(2019, 1, 1)) # now let's get it back ts = client.get('test-naive') assert_df( """ 2018-01-01 00:00:00 0.0 2018-01-01 01:00:00 1.0 2018-01-01 02:00:00 2.0 """, ts) assert not getattr(ts.index.dtype, 'tz', False)
def test_bigdata(engine, tracker, ptsh): tsh = ptsh def create_data(): # 4 years of sub-hourly points for year in range(2015, 2020): date = utcdt(year, 1, 1) serie = genserie(date, '10Min', 6 * 24 * 365) with engine.begin() as cn: tsh.update(cn, serie, 'big', '*****@*****.**', insertion_date=date) t0 = time() create_data() t1 = time() - t0 tshclass = tsh.__class__.__name__ tracker.append({'test': 'bigdata_insert', 'class': tshclass, 'time': t1}) t0 = time() with engine.begin() as cn: tsh.history(cn, 'big') t1 = time() - t0 tracker.append({ 'test': 'bigdata_history_all', 'class': tshclass, 'time': t1 }) t0 = time() for year in (2015, 2017, 2019): for month in (1, 5, 9, 12): date = utcdt(year, month, 1) with engine.begin() as cn: tsh.history(cn, 'big', from_insertion_date=date, to_insertion_date=date + timedelta(days=31)) t1 = time() - t0 tracker.append({ 'test': 'bigdata_history_chunks', 'class': tshclass, 'time': t1 })
def test_slice_options(engine, tsh): base = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2019, 1, 1), periods=3, freq='D')) tsh.update(engine, base, 'test-slice', 'Babar') # options transmissions ts = tsh.eval_formula( engine, '(add (series "test-slice") ' ' (slice (series "test-slice" #:fill 0) #:fromdate (date "2019-1-2")))', ) assert_df( """ 2019-01-01 00:00:00+00:00 1.0 2019-01-02 00:00:00+00:00 4.0 2019-01-03 00:00:00+00:00 6.0 """, ts)
def test_expanded(engine, tsh): @func('customseries') def customseries() -> pd.Series: return pd.Series([1.0, 2.0, 3.0], index=pd.date_range(dt(2019, 1, 1), periods=3, freq='D')) @finder('customseries') def find(cn, tsh, tree): return { tree[0]: { 'tzaware': True, 'index_type': 'datetime64[ns, UTC]', 'value_type': 'float64', 'index_dtype': '|M8[ns]', 'value_dtype': '<f8' } } base = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2019, 1, 1), periods=3, freq='D')) tsh.update(engine, base, 'exp-a', 'Babar') tsh.update(engine, base, 'exp-b', 'Celeste') tsh.register_formula(engine, 'expandmebase1', '(+ 3 (priority (series "exp-a") (customseries)))', False) tsh.register_formula(engine, 'expandmebase2', '(priority (series "exp-a") (series "exp-b"))', False) tsh.register_formula( engine, 'expandme', '(add (series "expandmebase1") (series "exp-b") (series "expandmebase2"))', False) exp = tsh.expanded_formula(engine, 'expandme') assert exp == ('(add ' '(+ 3 (priority (series "exp-a") (customseries))) ' '(series "exp-b") ' '(priority (series "exp-a") (series "exp-b")))')
def test_naive_tzone(engine, tsh): x = pd.Series([1, 2, 3], index=pd.date_range(utcdt(2020, 1, 1), periods=3, freq='D')) tsh.update(engine, x, 'non-naive', 'Babar') tsh.register_formula( engine, 'to-naive', '(naive (series "non-naive") "Europe/Moscow")', ) ts = tsh.get(engine, 'to-naive') assert_df( """ 2020-01-01 03:00:00 1.0 2020-01-02 03:00:00 2.0 2020-01-03 03:00:00 3.0 """, ts) meta = tsh.metadata(engine, 'to-naive') assert meta['tzaware'] == False
def test_append(engine, tsh): if tsh.namespace == 'z-z': return for x, dt in enumerate(pd.date_range(start=utcdt(2018, 1, 1), freq='D', periods=10)): ts = genserie(dt, 'D', 1, [x], name='daily') tsh.update(engine, ts, 'append', '*****@*****.**', insertion_date=dt) sql = 'select id, parent, chunk from "{}.snapshot".append order by id'.format( tsh.namespace ) chunks = engine.execute(sql).fetchall() c = { chunk.id: chunk.parent for chunk in chunks } # nice linked list assert c == { 1: None, 2: 1, 3: 2, 4: 3, 5: 4, 6: 5, 7: 6, 8: 7, 9: 8, 10: 9 }
def test_strip(engine, tsh): ts = genserie(datetime(2019, 1, 1), 'D', 3) tsh.update(engine, ts, 'strip-unsupervised', 'test', insertion_date=utcdt(2019, 1, 1)) csid = tsh.changeset_at(engine, 'strip-unsupervised', utcdt(2019, 1, 1)) tsh.strip(engine, 'strip-unsupervised', csid) ts = genserie(datetime(2019, 1, 1), 'D', 3) tsh.update(engine, ts, 'strip-handcrafted', 'test', manual=True, insertion_date=utcdt(2019, 1, 1)) csid = tsh.changeset_at(engine, 'strip-handcrafted', utcdt(2019, 1, 1)) tsh.strip(engine, 'strip-handcrafted', csid) ts = genserie(datetime(2019, 1, 1), 'D', 3) tsh.update(engine, ts, 'strip-supervised', 'test', insertion_date=utcdt(2019, 1, 1)) ts = genserie(datetime(2019, 1, 2), 'D', 3) tsh.update(engine, ts, 'strip-supervised', 'test', manual=True, insertion_date=utcdt(2019, 1, 2)) with pytest.raises(ValueError) as err: csid = tsh.changeset_at(engine, 'strip-supervised', utcdt(2019, 1, 1)) tsh.strip(engine, 'strip-supervised', csid)
def test_historical(engine, tsh): for insertion_date in pd.date_range(start=datetime(2015, 1, 1), end=datetime(2015, 1, 1, 6), freq='H'): ts = genserie(start=insertion_date, freq='H', repeat=7, tz='UTC') tsh.insert(engine, ts, 'republication', 'test', insertion_date=pd.Timestamp(insertion_date, tz='UTC')) # this is the end state # initially the first 1 was at 01:00 then at each insertion it gets # moved one hour later ts = tsh.get(engine, 'republication') assert_df(""" 2015-01-01 00:00:00+00:00 0.0 2015-01-01 01:00:00+00:00 0.0 2015-01-01 02:00:00+00:00 0.0 2015-01-01 03:00:00+00:00 0.0 2015-01-01 04:00:00+00:00 0.0 2015-01-01 05:00:00+00:00 0.0 2015-01-01 06:00:00+00:00 0.0 2015-01-01 07:00:00+00:00 1.0 2015-01-01 08:00:00+00:00 2.0 2015-01-01 09:00:00+00:00 3.0 2015-01-01 10:00:00+00:00 4.0 2015-01-01 11:00:00+00:00 5.0 2015-01-01 12:00:00+00:00 6.0 """, ts) tsh.insert(engine, genserie(datetime(2015, 1, 1), 'H', 15, [10], tz='UTC'), 'stable', 'test', insertion_date=pd.Timestamp(datetime(2015, 1, 1), tz='UTC')) # tens all over the place inserted at hour zero # there are two more tstamps than in republication assert_df(""" 2015-01-01 00:00:00+00:00 10.0 2015-01-01 01:00:00+00:00 10.0 2015-01-01 02:00:00+00:00 10.0 2015-01-01 03:00:00+00:00 10.0 2015-01-01 04:00:00+00:00 10.0 2015-01-01 05:00:00+00:00 10.0 2015-01-01 06:00:00+00:00 10.0 2015-01-01 07:00:00+00:00 10.0 2015-01-01 08:00:00+00:00 10.0 2015-01-01 09:00:00+00:00 10.0 2015-01-01 10:00:00+00:00 10.0 2015-01-01 11:00:00+00:00 10.0 2015-01-01 12:00:00+00:00 10.0 2015-01-01 13:00:00+00:00 10.0 2015-01-01 14:00:00+00:00 10.0 """, tsh.get(engine, 'stable')) tsh.build_arithmetic(engine, 'operation_past', {'republication': 1, 'stable': 1} ) tsh.build_priority(engine, 'compo_past', ['republication', 'stable']) ts = tsh.get(engine, 'operation_past') assert_df(""" 2015-01-01 00:00:00+00:00 10.0 2015-01-01 01:00:00+00:00 10.0 2015-01-01 02:00:00+00:00 10.0 2015-01-01 03:00:00+00:00 10.0 2015-01-01 04:00:00+00:00 10.0 2015-01-01 05:00:00+00:00 10.0 2015-01-01 06:00:00+00:00 10.0 2015-01-01 07:00:00+00:00 11.0 2015-01-01 08:00:00+00:00 12.0 2015-01-01 09:00:00+00:00 13.0 2015-01-01 10:00:00+00:00 14.0 2015-01-01 11:00:00+00:00 15.0 2015-01-01 12:00:00+00:00 16.0 """, ts) ts = tsh.get(engine, 'compo_past') assert_df(""" 2015-01-01 00:00:00+00:00 0.0 2015-01-01 01:00:00+00:00 0.0 2015-01-01 02:00:00+00:00 0.0 2015-01-01 03:00:00+00:00 0.0 2015-01-01 04:00:00+00:00 0.0 2015-01-01 05:00:00+00:00 0.0 2015-01-01 06:00:00+00:00 0.0 2015-01-01 07:00:00+00:00 1.0 2015-01-01 08:00:00+00:00 2.0 2015-01-01 09:00:00+00:00 3.0 2015-01-01 10:00:00+00:00 4.0 2015-01-01 11:00:00+00:00 5.0 2015-01-01 12:00:00+00:00 6.0 2015-01-01 13:00:00+00:00 10.0 2015-01-01 14:00:00+00:00 10.0 """, ts) assert_df(""" 2015-01-01 00:00:00+00:00 10.0 2015-01-01 01:00:00+00:00 10.0 2015-01-01 02:00:00+00:00 10.0 2015-01-01 03:00:00+00:00 10.0 2015-01-01 04:00:00+00:00 11.0 2015-01-01 05:00:00+00:00 12.0 2015-01-01 06:00:00+00:00 13.0 2015-01-01 07:00:00+00:00 14.0 2015-01-01 08:00:00+00:00 15.0 2015-01-01 09:00:00+00:00 16.0 """, tsh.get(engine, 'operation_past', revision_date=utcdt(2015, 1, 1, 3)) ) assert_df(""" 2015-01-01 00:00:00+00:00 0.0 2015-01-01 01:00:00+00:00 0.0 2015-01-01 02:00:00+00:00 0.0 2015-01-01 03:00:00+00:00 0.0 2015-01-01 04:00:00+00:00 1.0 2015-01-01 05:00:00+00:00 2.0 2015-01-01 06:00:00+00:00 3.0 2015-01-01 07:00:00+00:00 4.0 2015-01-01 08:00:00+00:00 5.0 2015-01-01 09:00:00+00:00 6.0 2015-01-01 10:00:00+00:00 10.0 2015-01-01 11:00:00+00:00 10.0 2015-01-01 12:00:00+00:00 10.0 2015-01-01 13:00:00+00:00 10.0 2015-01-01 14:00:00+00:00 10.0 """, tsh.get(engine, 'compo_past', revision_date=utcdt(2015, 1, 1, 3)) ) assert_df(""" 2015-01-01 03:00:00+00:00 13.0 2015-01-01 04:00:00+00:00 13.0 2015-01-01 05:00:00+00:00 13.0 2015-01-01 06:00:00+00:00 13.0 2015-01-01 07:00:00+00:00 13.0 2015-01-01 08:00:00+00:00 13.0 2015-01-01 09:00:00+00:00 13.0 2015-01-01 10:00:00+00:00 14.0 2015-01-01 11:00:00+00:00 15.0 2015-01-01 12:00:00+00:00 16.0 """, tsh.get(engine, 'operation_past', delta=timedelta(hours=2.5)) ) assert_df(""" 2015-01-01 03:00:00+00:00 3.0 2015-01-01 04:00:00+00:00 3.0 2015-01-01 05:00:00+00:00 3.0 2015-01-01 06:00:00+00:00 3.0 2015-01-01 07:00:00+00:00 3.0 2015-01-01 08:00:00+00:00 3.0 2015-01-01 09:00:00+00:00 3.0 2015-01-01 10:00:00+00:00 4.0 2015-01-01 11:00:00+00:00 5.0 2015-01-01 12:00:00+00:00 6.0 2015-01-01 13:00:00+00:00 10.0 2015-01-01 14:00:00+00:00 10.0 """, tsh.get(engine, 'compo_past', delta=timedelta(hours=2.5)) ) assert_df(""" 2015-01-01 06:00:00+00:00 3.0 2015-01-01 07:00:00+00:00 3.0 2015-01-01 08:00:00+00:00 3.0 2015-01-01 09:00:00+00:00 3.0 2015-01-01 10:00:00+00:00 4.0 2015-01-01 11:00:00+00:00 5.0 """, tsh.get(engine, 'compo_past', delta=timedelta(hours=2.5), from_value_date=datetime(2015, 1, 1, 6), to_value_date=datetime(2015, 1, 1, 11)) )