def test_pack_unpack_history(tsh, engine): for numserie in (1, 2, 3): with engine.begin() as cn: tsh.update(cn, genserie(datetime(2021, 1, 1), 'D', numserie), 'small-hist-naive', '*****@*****.**', insertion_date=utcdt(2021, 2, numserie)) hist = tsh.history(engine, 'small-hist-naive') meta = tsh.metadata(engine, 'small-hist-naive') packed = pack_history(meta, hist) meta2, hist2 = unpack_history(packed) assert meta2 == meta for idate, series in hist.items(): assert hist[idate].equals(series) for numserie in (1, 2, 3): with engine.begin() as cn: tsh.update(cn, genserie(utcdt(2021, 1, 1), 'D', numserie), 'small-hist-tzaware', '*****@*****.**', insertion_date=utcdt(2021, 2, numserie)) hist = tsh.history(engine, 'small-hist-tzaware') meta = tsh.metadata(engine, 'small-hist-tzaware') packed = pack_history(meta, hist) meta2, hist2 = unpack_history(packed) assert meta2 == meta for idate, series in hist.items(): assert hist[idate].equals(series)
def test_alternative_handler(pgapi): api = pgapi sapi = timeseries(api.uri, api.namespace, formula_class()) sapi.update( 'test-features', genserie(utcdt(2020, 1, 1), 'D', 3), 'Babar', ) sapi.tsh.register_formula(sapi.engine, 'test-formula', '(+ 1 (series "test-features"))') tsa = sapi.get('test-features') assert_df( """ 2020-01-01 00:00:00+00:00 0.0 2020-01-02 00:00:00+00:00 1.0 2020-01-03 00:00:00+00:00 2.0 """, tsa) tsb = sapi.get('test-formula') assert_df( """ 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, tsb) class supervision_and_formula(supervision_class(), formula_class()): pass sapi = timeseries(api.uri, api.namespace, supervision_and_formula) tsa = sapi.get('test-features') assert_df( """ 2020-01-01 00:00:00+00:00 0.0 2020-01-02 00:00:00+00:00 1.0 2020-01-03 00:00:00+00:00 2.0 """, tsa) tsb = sapi.get('test-formula') assert_df( """ 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 2.0 2020-01-03 00:00:00+00:00 3.0 """, tsb) sapi.update('test-features', genserie(utcdt(2020, 1, 2), 'D', 3), 'Babar', manual=True) tsb = sapi.get('test-formula') assert_df( """ 2020-01-01 00:00:00+00:00 1.0 2020-01-02 00:00:00+00:00 1.0 2020-01-03 00:00:00+00:00 2.0 2020-01-04 00:00:00+00:00 3.0 """, tsb)
def test_log(engine, cli, tsh): serie = genserie(datetime(2020, 1, 1), 'D', 3) tsh.update(engine, serie, 'log_me', 'Babar') serie = genserie(datetime(2020, 1, 2), 'D', 3) tsh.update(engine, serie, 'log_me', 'Babar') r = cli('log', engine.url, 'log_me', namespace=tsh.namespace) assert r.output.count('revision:') == 2
def test_delete(engine, cli, datadir, tsh): serie = genserie(datetime(2020, 1, 1), 'D', 3) tsh.update(engine, serie, 'bfoo', 'Babar') tsh.update(engine, serie, 'bbar', 'Babar') tsh.update(engine, serie, 'bquux', 'Babar') r = cli('delete', engine.url, deletefile=datadir / 'delete.csv', namespace=tsh.namespace) tsh = timeseries(tsh.namespace) tsh._testing = True assert not tsh.exists(engine, 'bfoo') assert tsh.get(engine, 'bfoo') is None assert tsh.get(engine, 'bbar') is None assert tsh.get(engine, 'bquux') is not None tsh.update(engine, serie, 'bbq', 'Babar') tsh = timeseries(tsh.namespace) tsh._testing = True r = cli('delete', engine.url, series='bbq', namespace=tsh.namespace) assert not tsh.exists(engine, 'bbq')
def test_multisources(client, engine): series = genserie(utcdt(2020, 1, 1), 'D', 3) tsh = tsio.timeseries('other') tsh.update(engine, series, 'test-other', 'Babar') client.update('test-mainsource', series, 'Babar') with pytest.raises(ValueError) as err: client.update('test-other', series, 'Babar') assert err.value.args[0] == 'not allowed to update to a secondary source' with pytest.raises(ValueError) as err: client.replace('test-other', series, 'Babar') assert err.value.args[0] == 'not allowed to replace to a secondary source' cat = client.catalog() assert cat == { ('db://localhost:5433/postgres', 'other'): [['test-other', 'primary']], ('db://localhost:5433/postgres', 'tsh'): [['test-naive', 'primary'], ['test', 'primary'], ['staircase', 'primary'], ['staircase-naive', 'primary'], ['in-a-formula', 'primary'], ['test-mainsource', 'primary'], ['new-formula', 'formula']] } cat = client.catalog(allsources=False) assert ('db://localhost:5433/postgres', 'tsh') in cat assert ('db://localhost:5433/postgres', 'other') not in cat
def test_naive(client): series_in = genserie(pd.Timestamp('2018-1-1'), 'H', 3) res = client.patch('/series/state', params={ 'name': 'test-naive', 'series': util.tojson(series_in), 'author': 'Babar', 'insertion_date': utcdt(2018, 1, 1, 10), 'tzaware': util.tzaware_serie(series_in) }) assert res.status_code == 201 res = client.get('/series/metadata?name=test-naive&all=1') meta = res.json assert meta == { 'index_dtype': '<M8[ns]', 'index_type': 'datetime64[ns]', 'tzaware': False, 'value_dtype': '<f8', 'value_type': 'float64' } res = client.get('/series/state?name=test-naive') series = util.fromjson(res.body, 'test', meta['tzaware']) assert_df( """ 2018-01-01 00:00:00 0.0 2018-01-01 01:00:00 1.0 2018-01-01 02:00:00 2.0 """, series)
def test_staircase_history_naive(client, tsh): # each days we insert 7 data points from datetime import datetime for idx, idate in enumerate( pd.date_range(start=utcdt(2015, 1, 1), end=utcdt(2015, 1, 4), freq='D')): series = genserie(start=idate.tz_convert(None), freq='H', repeat=7) client.update('staircase-naive', series, 'Babar', insertion_date=idate) series = client.staircase('staircase-naive', pd.Timedelta(hours=3), from_value_date=datetime(2015, 1, 1, 4), to_value_date=datetime(2015, 1, 2, 5)) assert series.name == 'staircase-naive' assert_df( """ 2015-01-01 04:00:00 4.0 2015-01-01 05:00:00 5.0 2015-01-01 06:00:00 6.0 2015-01-02 03:00:00 3.0 2015-01-02 04:00:00 4.0 2015-01-02 05:00:00 5.0 """, series) # series = client.staircase( # 'staircase-naive', # pd.Timedelta(hours=3), # from_value_date=datetime(2015, 1, 1, 4), # to_value_date=datetime(2015, 1, 2, 5) # ) hist = client.history('staircase-naive') assert len(hist) == 4 hist = client.history('staircase-naive', from_insertion_date=datetime(2015, 1, 2), to_insertion_date=datetime(2015, 1, 3)) assert len(hist) == 2 hist = client.history('staircase-naive', from_value_date=datetime(2015, 1, 1, 3), to_value_date=datetime(2015, 1, 2, 1)) assert all(series.name == 'staircase-naive' for series in hist.values()) assert_hist( """ insertion_date value_date 2015-01-01 00:00:00+00:00 2015-01-01 03:00:00 3.0 2015-01-01 04:00:00 4.0 2015-01-01 05:00:00 5.0 2015-01-01 06:00:00 6.0 2015-01-02 00:00:00+00:00 2015-01-01 03:00:00 3.0 2015-01-01 04:00:00 4.0 2015-01-01 05:00:00 5.0 2015-01-01 06:00:00 6.0 2015-01-02 00:00:00 0.0 2015-01-02 01:00:00 1.0 """, hist)
def test_check(engine, cli, tsh): serie = genserie(datetime(2020, 1, 1), 'D', 3) tsh.update(engine, serie, 'things', 'Babar') tsh.update(engine, serie, 'other_things', 'Babar') r = cli('check', engine.url, namespace=tsh.namespace) assert 'things inserts=1, read-time=' in r.output assert 'other_things inserts=1, read-time=' in r.output
def test_dispatch_get(engine, tsh): ts_real = genserie(datetime(2010, 1, 1), 'D', 5, [1]) ts_nomination = genserie(datetime(2010, 1, 1), 'D', 6, [2]) ts_forecast = genserie(datetime(2010, 1, 1), 'D', 7, [3]) tsh.insert(engine, ts_real, 'realised0', 'test') tsh.insert(engine, ts_nomination, 'nominated0', 'test') tsh.insert(engine, ts_forecast, 'forecasted0', 'test') tsh.build_priority(engine, 'composite_serie', ['realised0', 'nominated0', 'forecasted0']) tsh.build_arithmetic(engine, 'sum_serie', {'realised0':1, 'forecasted0':1}) assert 'primary' == tsh.type(engine, 'realised0') assert 'priority' == tsh.type(engine, 'composite_serie') assert 'arithmetic' == tsh.type(engine, 'sum_serie') assert tsh.type(engine, 'serie_not_defined') == 'primary' assert_df(""" 2010-01-01 2.0 2010-01-02 2.0 2010-01-03 2.0 2010-01-04 2.0 2010-01-05 2.0 2010-01-06 2.0 """, tsh.get(engine, 'nominated0')) assert_df(""" 2010-01-01 1.0 2010-01-02 1.0 2010-01-03 1.0 2010-01-04 1.0 2010-01-05 1.0 2010-01-06 2.0 2010-01-07 3.0 """, tsh.get(engine, 'composite_serie')) assert_df(""" 2010-01-01 4.0 2010-01-02 4.0 2010-01-03 4.0 2010-01-04 4.0 2010-01-05 4.0 """, tsh.get(engine,'sum_serie'))
def test_staircase(client): # each days we insert 7 data points for idate in pd.date_range(start=utcdt(2015, 1, 1), end=utcdt(2015, 1, 4), freq='D'): series = genserie(start=idate, freq='H', repeat=7) client.patch('/series/state', params={ 'name': 'staircase', 'series': util.tojson(series), 'author': 'Babar', 'insertion_date': idate, 'tzaware': util.tzaware_serie(series) }) res = client.get('/series/staircase', params={ 'name': 'staircase', 'delta': pd.Timedelta(hours=3), 'from_value_date': utcdt(2015, 1, 1, 4), 'to_value_date': utcdt(2015, 1, 2, 5), }) series = util.fromjson(res.body, 'test', True) assert_df( """ 2015-01-01 04:00:00+00:00 4.0 2015-01-01 05:00:00+00:00 5.0 2015-01-01 06:00:00+00:00 6.0 2015-01-02 03:00:00+00:00 3.0 2015-01-02 04:00:00+00:00 4.0 2015-01-02 05:00:00+00:00 5.0 """, series) res = client.get('/series/staircase', params={ 'name': 'staircase', 'delta': pd.Timedelta(hours=3), 'from_value_date': utcdt(2015, 1, 1, 4), 'to_value_date': utcdt(2015, 1, 2, 5), 'format': 'tshpack' }) meta, index, values = util.nary_unpack(zlib.decompress(res.body)) meta = json.loads(meta) index, values = util.numpy_deserialize(index, values, meta) series = pd.Series(values, index=index) series = series.tz_localize('UTC') assert_df( """ 2015-01-01 04:00:00+00:00 4.0 2015-01-01 05:00:00+00:00 5.0 2015-01-01 06:00:00+00:00 6.0 2015-01-02 03:00:00+00:00 3.0 2015-01-02 04:00:00+00:00 4.0 2015-01-02 05:00:00+00:00 5.0 """, series)
def test_history(engine, cli, tsh): serie = genserie(datetime(2020, 1, 1), 'D', 3) tsh.update(engine, serie, 'some_history', 'Babar', insertion_date=utcdt(2019, 1, 1)) serie = genserie(datetime(2020, 1, 2), 'D', 3) tsh.update(engine, serie, 'some_history', 'Babar', insertion_date=utcdt(2019, 1, 2)) r = cli('history', engine.url, 'some_history', json=True) assert r.output == ( '{"2019-01-01 00:00:00+00:00": {"2020-01-01 00:00:00": 0.0, "2020-01-02 00:00:00": 1.0, "2020-01-03 00:00:00": 2.0}, "2019-01-02 00:00:00+00:00": {"2020-01-02 00:00:00": 0.0, "2020-01-03 00:00:00": 1.0, "2020-01-04 00:00:00": 2.0}}\n' )
def test_info(engine, cli, tsh): serie = genserie(datetime(2020, 1, 1), 'D', 3) tsh.update(engine, serie, 'someseries', 'Babar') r = cli('info', engine.url, namespace=tsh.namespace) out = r.output.strip().split('\n') assert len(out) == 2 assert out[0].startswith('series count:') assert out[1].startswith('series names:')
def create_data(): # 4 years of sub-hourly points for year in range(2015, 2020): date = utcdt(year, 1, 1) serie = genserie(date, '10Min', 6 * 24 * 365) with engine.begin() as cn: tsh.update(cn, serie, 'big', '*****@*****.**', insertion_date=date)
def test_in_tx(tsh, engine): assert tsh.type(engine, 'foo') == 'primary' with pytest.raises(TypeError) as err: tsh.update(engine.connect(), 0, 0, 0) assert err.value.args[0] == 'You must use a transaction object' ts = genserie(datetime(2017, 10, 28, 23), 'H', 4, tz='UTC') with engine.begin() as cn: tsh.update(cn, ts, 'test_tx', 'Babar') # that old api is still there tsh.insert(cn, ts, 'test_tx', 'Babar')
def test_get_from_to(engine, tsh): ts = genserie(datetime(2015, 1, 1), 'D', 365) tsh.update(engine, ts, 'quitelong', '*****@*****.**') snap = Snapshot(engine, tsh, 'quitelong') if tsh.namespace == 'z-z': sql = 'select id, parent from "z-z.snapshot".quitelong order by id' chunks = engine.execute(sql).fetchall() # should be perfectly chained chunks = { chunk.id: chunk.parent for chunk in chunks } chunks.pop(1) assert all(k == v+1 for k, v in chunks.items()) snap = Snapshot(engine, tsh, 'quitelong') chunks = chunksize(snap, 73) assert chunks == {None: 5, 1: 5, 2: 5, 3: 5, 4: 5, 5: 5, 6: 5, 7: 5, 8: 5, 9: 5, 10: 5, 11: 5, 12: 5, 13: 5, 14: 5, 15: 5, 16: 5, 17: 5, 18: 5, 19: 5, 20: 5, 21: 5, 22: 5, 23: 5, 24: 5, 25: 5, 26: 5, 27: 5, 28: 5, 29: 5, 30: 5, 31: 5, 32: 5, 33: 5, 34: 5, 35: 5, 36: 5, 37: 5, 38: 5, 39: 5, 40: 5, 41: 5, 42: 5, 43: 5, 44: 5, 45: 5, 46: 5, 47: 5, 48: 5, 49: 5, 50: 5, 51: 5, 52: 5, 53: 5, 54: 5, 55: 5, 56: 5, 57: 5, 58: 5, 59: 5, 60: 5, 61: 5, 62: 5, 63: 5, 64: 5, 65: 5, 66: 5, 67: 5, 68: 5, 69: 5, 70: 5, 71: 5, 72: 5} chunks = chunksize(snap, 73, datetime(2015, 5, 1)) assert chunks == {24: 5, 25: 5, 26: 5, 27: 5, 28: 5, 29: 5, 30: 5, 31: 5, 32: 5, 33: 5, 34: 5, 35: 5, 36: 5, 37: 5, 38: 5, 39: 5, 40: 5, 41: 5, 42: 5, 43: 5, 44: 5, 45: 5, 46: 5, 47: 5, 48: 5, 49: 5, 50: 5, 51: 5, 52: 5, 53: 5, 54: 5, 55: 5, 56: 5, 57: 5, 58: 5, 59: 5, 60: 5, 61: 5, 62: 5, 63: 5, 64: 5, 65: 5, 66: 5, 67: 5, 68: 5, 69: 5, 70: 5, 71: 5, 72: 5} serie = tsh.get(engine, 'quitelong') assert serie.index[0] == pd.Timestamp('2015-01-01 00:00:00') assert serie.index[-1] == pd.Timestamp('2015-12-31 00:00:00') serie = tsh.get(engine, 'quitelong', from_value_date=datetime(2015, 5, 1), to_value_date=datetime(2015, 6, 1)) assert serie.index[0] == pd.Timestamp('2015-05-01 00:00:00') assert serie.index[-1] == pd.Timestamp('2015-06-01 00:00:00') serie = tsh.get(engine, 'quitelong', from_value_date=datetime(2016, 5, 1), to_value_date=datetime(2016, 6, 1)) assert serie.dtypes == np.dtype('float64') assert len(serie) == 0 assert isinstance(serie.index, pd.DatetimeIndex) assert serie.index.freq is None
def create_data(): # one insert per day for 4 months for month in range(1, 4): days = calendar.monthrange(2017, month)[1] for day in range(1, days + 1): date = utcdt(2017, month, day) serie = genserie(date, '10Min', 6 * 24) with engine.begin() as cn: tsh.update(cn, serie, 'manydiffs', '*****@*****.**', insertion_date=date.replace(year=2018))
def test_naive(client, engine, tsh): series_in = genserie(pd.Timestamp('2018-1-1'), 'H', 3) client.update('test-naive', series_in, 'Babar', insertion_date=utcdt(2019, 1, 1)) # now let's get it back ts = client.get('test-naive') assert_df( """ 2018-01-01 00:00:00 0.0 2018-01-01 01:00:00 1.0 2018-01-01 02:00:00 2.0 """, ts) assert not getattr(ts.index.dtype, 'tz', False)
def test_delete(client): series_in = genserie(utcdt(2018, 1, 1), 'H', 3) res = client.patch('/series/state', params={ 'name': 'test', 'series': util.tojson(series_in), 'author': 'Babar', 'insertion_date': utcdt(2018, 1, 1, 10), 'tzaware': util.tzaware_serie(series_in) }) res = client.delete('/series/state', params={'name': 'no-such-series'}) assert res.status_code == 404 res = client.delete('/series/state', params={'name': 'test'}) assert res.status_code == 204 res = client.get('/series/catalog') assert 'test' not in res.json
def test_rename(engine, cli, datadir, tsh): serie = genserie(datetime(2020, 1, 1), 'D', 3) tsh.update(engine, serie, 'afoo', 'Babar') tsh.update(engine, serie, 'abar', 'Babar') tsh.update(engine, serie, 'aquux', 'Babar') r = cli('rename', engine.url, datadir / 'rename.csv', namespace=tsh.namespace) assert tsh.get(engine, 'afoo') is None assert tsh.get(engine, 'abar') is None for name in ('aquux', 'anew-foo', 'anew-bar'): assert tsh.get(engine, name) is not None
def test_rename(client): series_in = genserie(utcdt(2018, 1, 1), 'H', 3) res = client.patch('/series/state', params={ 'name': 'test', 'series': util.tojson(series_in), 'author': 'Babar', 'insertion_date': utcdt(2018, 1, 1, 10), 'tzaware': util.tzaware_serie(series_in) }) res = client.put('/series/state', params={ 'name': 'no-such-series', 'newname': 'no-better' }) assert res.status_code == 404 res = client.put('/series/state', params={ 'name': 'test', 'newname': 'test2' }) assert res.status_code == 204 res = client.get('/series/catalog') assert res.json == { 'db://localhost:5433/postgres!tsh': [['test-naive', 'primary'], ['test2', 'primary']] } assert 'test' not in res.json res = client.patch('/series/state', params={ 'name': 'test3', 'series': util.tojson(series_in), 'author': 'Babar', 'insertion_date': utcdt(2018, 1, 1, 10), 'tzaware': util.tzaware_serie(series_in) }) res = client.put('/series/state', params={ 'name': 'test2', 'newname': 'test3' }) assert res.status_code == 409 assert res.json == {'message': '`test3` does exists'}
def test_outliers(engine, tsh): tsh.add_bounds(engine, 'serie1', min=5, max=10) tsh.add_bounds(engine, 'serie2', max=10) tsh.add_bounds(engine, 'serie3', min=5) tsh.add_bounds(engine, 'serie4', min=50) ts = genserie(datetime(2015,1,1), 'D', 15) tsh.insert(engine, ts, 'serie1', 'test') tsh.insert(engine, ts, 'serie2', 'test') tsh.insert(engine, ts, 'serie3', 'test') tsh.insert(engine, ts, 'serie4', 'test') ts1 = tsh.get(engine, 'serie1') ts2 = tsh.get(engine, 'serie2', from_value_date=datetime(2015,1,2), to_value_date=datetime(2015, 1, 13)) ts3 = tsh.get(engine, 'serie3') ts4 = tsh.get(engine, 'serie4') assert_df(""" 2015-01-06 5.0 2015-01-07 6.0 2015-01-08 7.0 2015-01-09 8.0 2015-01-10 9.0 2015-01-11 10.0 """, ts1) assert 10 == max(ts2) assert 1 == min(ts2) assert 14 == max(ts3) assert 5 == min(ts3) assert 0 == len(ts4) # upsert: tsh.add_bounds(engine, 'serie4', min=-50) ts4 = tsh.get(engine, 'serie4') assert 15 == len(ts4)
def test_append(engine, tsh): if tsh.namespace == 'z-z': return for x, dt in enumerate(pd.date_range(start=utcdt(2018, 1, 1), freq='D', periods=10)): ts = genserie(dt, 'D', 1, [x], name='daily') tsh.update(engine, ts, 'append', '*****@*****.**', insertion_date=dt) sql = 'select id, parent, chunk from "{}.snapshot".append order by id'.format( tsh.namespace ) chunks = engine.execute(sql).fetchall() c = { chunk.id: chunk.parent for chunk in chunks } # nice linked list assert c == { 1: None, 2: 1, 3: 2, 4: 3, 5: 4, 6: 5, 7: 6, 8: 7, 9: 8, 10: 9 }
def test_get_fast_path(client): series_in = genserie(utcdt(2018, 1, 1), 'H', 3) res = client.patch('/series/state', params={ 'name': 'test_fast', 'series': util.tojson(series_in), 'author': 'Babar', 'insertion_date': utcdt(2018, 1, 1, 10), 'tzaware': util.tzaware_serie(series_in) }) assert res.status_code == 201 out = client.get('/series/state', params={ 'name': 'test_fast', 'format': 'tshpack' }) meta, index, values = util.nary_unpack(zlib.decompress(out.body)) meta = json.loads(meta) index, values = util.numpy_deserialize(index, values, meta) series = pd.Series(values, index=index) series = series.tz_localize('UTC') assert_df( """ 2018-01-01 00:00:00+00:00 0.0 2018-01-01 01:00:00+00:00 1.0 2018-01-01 02:00:00+00:00 2.0 """, series) assert meta == { 'tzaware': True, 'index_type': 'datetime64[ns, UTC]', 'value_type': 'float64', 'index_dtype': '|M8[ns]', 'value_dtype': '<f8' }
def test_prepend(engine, tsh): if tsh.namespace == 'z-z': return serie = genserie(datetime(2010, 1, 1), 'D', 40) assert 40 == len(serie) ts_insert = serie[2:] tsh.update(engine, ts_insert, 'prepend', 'test') assert 38 == len(tsh.get(engine, 'prepend')) tsh.update(engine, serie, 'prepend', 'test') assert 40 == len(tsh.get(engine, 'prepend')) sql = 'select id, parent, chunk from "{}.snapshot".prepend order by id'.format( tsh.namespace ) chunks = engine.execute(sql).fetchall() c = { chunk.id: chunk.parent for chunk in chunks } # ugly un-linked list assert c == {1: None, 2: None}
def test_combine(engine, tsh): ts_real = genserie(datetime(2010, 1, 1), 'D', 10, [1]) ts_nomination = genserie(datetime(2010, 1, 1), 'D', 12, [2]) ts_forecast = genserie(datetime(2010, 1, 1), 'D', 20, [3]) tsh.insert(engine, ts_real, 'realised', 'test') tsh.insert(engine, ts_nomination, 'nominated', 'test') tsh.insert(engine, ts_forecast, 'forecasted', 'test') tsh.build_priority(engine, 'serie5', ['realised', 'nominated', 'forecasted']) values, origin = tsh.get_priority(engine,'serie5') assert_df(""" 2010-01-01 1.0 2010-01-02 1.0 2010-01-03 1.0 2010-01-04 1.0 2010-01-05 1.0 2010-01-06 1.0 2010-01-07 1.0 2010-01-08 1.0 2010-01-09 1.0 2010-01-10 1.0 2010-01-11 2.0 2010-01-12 2.0 2010-01-13 3.0 2010-01-14 3.0 2010-01-15 3.0 2010-01-16 3.0 2010-01-17 3.0 2010-01-18 3.0 2010-01-19 3.0 2010-01-20 3.0 """, values) assert_df(""" 2010-01-01 realised 2010-01-02 realised 2010-01-03 realised 2010-01-04 realised 2010-01-05 realised 2010-01-06 realised 2010-01-07 realised 2010-01-08 realised 2010-01-09 realised 2010-01-10 realised 2010-01-11 nominated 2010-01-12 nominated 2010-01-13 forecasted 2010-01-14 forecasted 2010-01-15 forecasted 2010-01-16 forecasted 2010-01-17 forecasted 2010-01-18 forecasted 2010-01-19 forecasted 2010-01-20 forecasted """, origin) # we remove the last value of the 2 first series which are considered as bogus tsh.build_priority(engine, 'serie6', ['realised', 'nominated', 'forecasted'], map_prune={'realised': 1, 'nominated': 1, 'forecasted': 0}) values, origin = tsh.get_priority(engine,'serie6') assert_df(""" 2010-01-01 1.0 2010-01-02 1.0 2010-01-03 1.0 2010-01-04 1.0 2010-01-05 1.0 2010-01-06 1.0 2010-01-07 1.0 2010-01-08 1.0 2010-01-09 1.0 2010-01-10 2.0 2010-01-11 2.0 2010-01-12 3.0 2010-01-13 3.0 2010-01-14 3.0 2010-01-15 3.0 2010-01-16 3.0 2010-01-17 3.0 2010-01-18 3.0 2010-01-19 3.0 2010-01-20 3.0 """, values) assert_df(""" 2010-01-01 realised 2010-01-02 realised 2010-01-03 realised 2010-01-04 realised 2010-01-05 realised 2010-01-06 realised 2010-01-07 realised 2010-01-08 realised 2010-01-09 realised 2010-01-10 nominated 2010-01-11 nominated 2010-01-12 forecasted 2010-01-13 forecasted 2010-01-14 forecasted 2010-01-15 forecasted 2010-01-16 forecasted 2010-01-17 forecasted 2010-01-18 forecasted 2010-01-19 forecasted 2010-01-20 forecasted """, origin) tsh.build_priority(engine, 'serie7', ['realised', 'nominated', 'forecasted'], map_prune={'realised': 1, 'nominated': 3, 'forecasted': 0}) values, origin = tsh.get_priority(engine,'serie7') assert_df(""" 2010-01-01 realised 2010-01-02 realised 2010-01-03 realised 2010-01-04 realised 2010-01-05 realised 2010-01-06 realised 2010-01-07 realised 2010-01-08 realised 2010-01-09 realised 2010-01-10 forecasted 2010-01-11 forecasted 2010-01-12 forecasted 2010-01-13 forecasted 2010-01-14 forecasted 2010-01-15 forecasted 2010-01-16 forecasted 2010-01-17 forecasted 2010-01-18 forecasted 2010-01-19 forecasted 2010-01-20 forecasted """, origin)
def test_historical(engine, tsh): for insertion_date in pd.date_range(start=datetime(2015, 1, 1), end=datetime(2015, 1, 1, 6), freq='H'): ts = genserie(start=insertion_date, freq='H', repeat=7, tz='UTC') tsh.insert(engine, ts, 'republication', 'test', insertion_date=pd.Timestamp(insertion_date, tz='UTC')) # this is the end state # initially the first 1 was at 01:00 then at each insertion it gets # moved one hour later ts = tsh.get(engine, 'republication') assert_df(""" 2015-01-01 00:00:00+00:00 0.0 2015-01-01 01:00:00+00:00 0.0 2015-01-01 02:00:00+00:00 0.0 2015-01-01 03:00:00+00:00 0.0 2015-01-01 04:00:00+00:00 0.0 2015-01-01 05:00:00+00:00 0.0 2015-01-01 06:00:00+00:00 0.0 2015-01-01 07:00:00+00:00 1.0 2015-01-01 08:00:00+00:00 2.0 2015-01-01 09:00:00+00:00 3.0 2015-01-01 10:00:00+00:00 4.0 2015-01-01 11:00:00+00:00 5.0 2015-01-01 12:00:00+00:00 6.0 """, ts) tsh.insert(engine, genserie(datetime(2015, 1, 1), 'H', 15, [10], tz='UTC'), 'stable', 'test', insertion_date=pd.Timestamp(datetime(2015, 1, 1), tz='UTC')) # tens all over the place inserted at hour zero # there are two more tstamps than in republication assert_df(""" 2015-01-01 00:00:00+00:00 10.0 2015-01-01 01:00:00+00:00 10.0 2015-01-01 02:00:00+00:00 10.0 2015-01-01 03:00:00+00:00 10.0 2015-01-01 04:00:00+00:00 10.0 2015-01-01 05:00:00+00:00 10.0 2015-01-01 06:00:00+00:00 10.0 2015-01-01 07:00:00+00:00 10.0 2015-01-01 08:00:00+00:00 10.0 2015-01-01 09:00:00+00:00 10.0 2015-01-01 10:00:00+00:00 10.0 2015-01-01 11:00:00+00:00 10.0 2015-01-01 12:00:00+00:00 10.0 2015-01-01 13:00:00+00:00 10.0 2015-01-01 14:00:00+00:00 10.0 """, tsh.get(engine, 'stable')) tsh.build_arithmetic(engine, 'operation_past', {'republication': 1, 'stable': 1} ) tsh.build_priority(engine, 'compo_past', ['republication', 'stable']) ts = tsh.get(engine, 'operation_past') assert_df(""" 2015-01-01 00:00:00+00:00 10.0 2015-01-01 01:00:00+00:00 10.0 2015-01-01 02:00:00+00:00 10.0 2015-01-01 03:00:00+00:00 10.0 2015-01-01 04:00:00+00:00 10.0 2015-01-01 05:00:00+00:00 10.0 2015-01-01 06:00:00+00:00 10.0 2015-01-01 07:00:00+00:00 11.0 2015-01-01 08:00:00+00:00 12.0 2015-01-01 09:00:00+00:00 13.0 2015-01-01 10:00:00+00:00 14.0 2015-01-01 11:00:00+00:00 15.0 2015-01-01 12:00:00+00:00 16.0 """, ts) ts = tsh.get(engine, 'compo_past') assert_df(""" 2015-01-01 00:00:00+00:00 0.0 2015-01-01 01:00:00+00:00 0.0 2015-01-01 02:00:00+00:00 0.0 2015-01-01 03:00:00+00:00 0.0 2015-01-01 04:00:00+00:00 0.0 2015-01-01 05:00:00+00:00 0.0 2015-01-01 06:00:00+00:00 0.0 2015-01-01 07:00:00+00:00 1.0 2015-01-01 08:00:00+00:00 2.0 2015-01-01 09:00:00+00:00 3.0 2015-01-01 10:00:00+00:00 4.0 2015-01-01 11:00:00+00:00 5.0 2015-01-01 12:00:00+00:00 6.0 2015-01-01 13:00:00+00:00 10.0 2015-01-01 14:00:00+00:00 10.0 """, ts) assert_df(""" 2015-01-01 00:00:00+00:00 10.0 2015-01-01 01:00:00+00:00 10.0 2015-01-01 02:00:00+00:00 10.0 2015-01-01 03:00:00+00:00 10.0 2015-01-01 04:00:00+00:00 11.0 2015-01-01 05:00:00+00:00 12.0 2015-01-01 06:00:00+00:00 13.0 2015-01-01 07:00:00+00:00 14.0 2015-01-01 08:00:00+00:00 15.0 2015-01-01 09:00:00+00:00 16.0 """, tsh.get(engine, 'operation_past', revision_date=utcdt(2015, 1, 1, 3)) ) assert_df(""" 2015-01-01 00:00:00+00:00 0.0 2015-01-01 01:00:00+00:00 0.0 2015-01-01 02:00:00+00:00 0.0 2015-01-01 03:00:00+00:00 0.0 2015-01-01 04:00:00+00:00 1.0 2015-01-01 05:00:00+00:00 2.0 2015-01-01 06:00:00+00:00 3.0 2015-01-01 07:00:00+00:00 4.0 2015-01-01 08:00:00+00:00 5.0 2015-01-01 09:00:00+00:00 6.0 2015-01-01 10:00:00+00:00 10.0 2015-01-01 11:00:00+00:00 10.0 2015-01-01 12:00:00+00:00 10.0 2015-01-01 13:00:00+00:00 10.0 2015-01-01 14:00:00+00:00 10.0 """, tsh.get(engine, 'compo_past', revision_date=utcdt(2015, 1, 1, 3)) ) assert_df(""" 2015-01-01 03:00:00+00:00 13.0 2015-01-01 04:00:00+00:00 13.0 2015-01-01 05:00:00+00:00 13.0 2015-01-01 06:00:00+00:00 13.0 2015-01-01 07:00:00+00:00 13.0 2015-01-01 08:00:00+00:00 13.0 2015-01-01 09:00:00+00:00 13.0 2015-01-01 10:00:00+00:00 14.0 2015-01-01 11:00:00+00:00 15.0 2015-01-01 12:00:00+00:00 16.0 """, tsh.get(engine, 'operation_past', delta=timedelta(hours=2.5)) ) assert_df(""" 2015-01-01 03:00:00+00:00 3.0 2015-01-01 04:00:00+00:00 3.0 2015-01-01 05:00:00+00:00 3.0 2015-01-01 06:00:00+00:00 3.0 2015-01-01 07:00:00+00:00 3.0 2015-01-01 08:00:00+00:00 3.0 2015-01-01 09:00:00+00:00 3.0 2015-01-01 10:00:00+00:00 4.0 2015-01-01 11:00:00+00:00 5.0 2015-01-01 12:00:00+00:00 6.0 2015-01-01 13:00:00+00:00 10.0 2015-01-01 14:00:00+00:00 10.0 """, tsh.get(engine, 'compo_past', delta=timedelta(hours=2.5)) ) assert_df(""" 2015-01-01 06:00:00+00:00 3.0 2015-01-01 07:00:00+00:00 3.0 2015-01-01 08:00:00+00:00 3.0 2015-01-01 09:00:00+00:00 3.0 2015-01-01 10:00:00+00:00 4.0 2015-01-01 11:00:00+00:00 5.0 """, tsh.get(engine, 'compo_past', delta=timedelta(hours=2.5), from_value_date=datetime(2015, 1, 1, 6), to_value_date=datetime(2015, 1, 1, 11)) )
def test_errors(engine, tsh): tsh.insert(engine, genserie(datetime(2010, 1, 1), 'D', 5, [1]), 'primary_series', 'test') tsh.build_arithmetic(engine, 'arithmetic2', {'toto': 1, 'tata': 1}) tsh.build_priority(engine, 'priority2', ['toto', 'tata'])
def test_micmac(engine, tsh, refresh): tsh.insert(engine, genserie(datetime(2010, 1, 1), 'D', 5, [1]), 'micmac1', 'test') tsh.insert(engine, genserie(datetime(2010, 1, 2), 'D', 1, [1000]), 'micmac1', 'test') # bogus data tsh.insert(engine, genserie(datetime(2010, 1, 1), 'D', 6, [2]), 'micmac2', 'test') tsh.insert(engine, genserie(datetime(2010, 1, 1), 'D', 6, [3]), 'micmac3', 'test') tsh.insert(engine, genserie(datetime(2010, 1, 1), 'D', 7, [15]), 'micmac4', 'test') tsh.build_priority(engine, 'prio1', ['micmac1', 'micmac2'], map_prune={'micmac1': 2, 'micmac2': 1}) tsh.build_arithmetic(engine, 'arithmetic1', {'prio1':1, 'micmac3':1}) tsh.build_priority(engine, 'final', ['arithmetic1', 'micmac4'], map_prune={'arithmetic1': 1, 'micmac4': 0}) tsh.add_bounds(engine, 'micmac1', min=0, max=100) tsh.get(engine, 'micmac1') tsh.get(engine, 'micmac2') tsh.get(engine, 'prio1') tsh.get(engine, 'micmac3') tsh.get(engine, 'arithmetic1') tsh.get(engine, 'micmac4') assert_df(""" 2010-01-01 4.0 2010-01-02 5.0 2010-01-03 4.0 2010-01-04 5.0 2010-01-05 15.0 2010-01-06 15.0 2010-01-07 15.0 """, tsh.get(engine, 'final')) # test get with bounds assert_df(""" 2010-01-03 4.0 2010-01-04 5.0 2010-01-05 15.0 2010-01-06 15.0 """, tsh.get(engine, 'final', from_value_date = datetime(2010, 1, 3), to_value_date=datetime(2010, 1, 6) )) # test get with values out of bounds ts_out_of_bounds = tsh.get(engine, 'final', from_value_date=datetime(2013, 1, 1), to_value_date=datetime(2013, 1, 6) ) assert 0 == len(ts_out_of_bounds) # the result is an empty pd.Series dash_html_table = alias_table(engine, tsh, 'final') inside = dash_html_table.children assert 8 == len(inside) assert 4 == len(inside[0].children) refpath = DATADIR / 'dash-table.html' out = str(dash_html_table).encode('utf-8') if refresh: refpath.write_bytes(out) expected = refpath.read_bytes() assert expected == out tree = buildtree(engine, tsh, 'final', []) assert tree == { ('final', 'priority'): [ {('arithmetic1', 'arithmetic'): [ { ('prio1', 'priority'): [ 'micmac1', 'micmac2'] }, 'micmac3' ]}, 'micmac4' ] } out = [] showtree(tree, printer=lambda *x: out.append(''.join(x))) assert out == [ '* priority `final`', ' * arithmetic `arithmetic1`', ' * priority `prio1`', ' -micmac1', ' -micmac2', ' -micmac3', ' -micmac4' ] tsh.build_priority(engine, 'bogus2', ['prio1', 'no-such-series']) tree = buildtree(engine, tsh, 'bogus2', []) out = [] showtree(tree, printer=lambda *x: out.append(''.join(x))) assert out == [ '* priority `bogus2`', ' * priority `prio1`', ' -micmac1', ' -micmac2', ' -unknown `no-such-series`' ] # test .exists assert tsh.exists(engine, 'micmac1') assert tsh.exists(engine, 'final') assert tsh.exists(engine, 'arithmetic1') assert not tsh.exists(engine, 'no-such-series')
def test_arithmetic(engine, tsh): ts_toto = genserie(datetime(2010, 1, 1), 'D', 7, [1]) ts_tata = genserie(datetime(2010, 1, 3), 'D', 7, [2]) ts_titi = genserie(datetime(2010, 1, 1), 'D', 5, [2]) ts_short = genserie(datetime(2010, 1, 3), 'D', 3, [2]) tsh.insert(engine, ts_toto, 'toto', 'test') tsh.insert(engine, ts_tata, 'tata', 'test') tsh.insert(engine, ts_titi, 'titi', 'test') tsh.insert(engine, ts_short, 'short', 'test') tsh.build_arithmetic(engine, 'sum', {'toto': 1, 'tata': 1}) tsh.build_arithmetic(engine, 'difference', {'toto': 1, 'tata': -1}) tsh.build_arithmetic(engine, 'mean', {'toto': 0.5, 'tata': 0.5}) tsh.build_arithmetic(engine, 'bogus', {'toto': 0.5, 'unknown': 0.5}) tsh.build_arithmetic(engine, 'forwardfill', {'toto': 1, 'tata': 1}, {'toto': 'ffill'}) tsh.build_arithmetic(engine, 'backwardfill', {'toto': 1, 'tata': 1}, {'tata': 'bfill'}) tsh.build_arithmetic(engine, 'allfill', {'toto': 1, 'short': 1}, {'short': 'bfill,ffill'}) tsh.build_arithmetic(engine, 'storagelike', {'toto': 1, 'titi': 1}, {'titi': 'fill=0'}) values = tsh.get_arithmetic(engine, 'sum') assert_df(""" 2010-01-03 3.0 2010-01-04 3.0 2010-01-05 3.0 2010-01-06 3.0 2010-01-07 3.0 """, values) # NB: there are only data at the intersection of the index values = tsh.get_arithmetic(engine, 'difference') assert_df(""" 2010-01-03 -1.0 2010-01-04 -1.0 2010-01-05 -1.0 2010-01-06 -1.0 2010-01-07 -1.0 """, values) tsh.get_arithmetic(engine, 'mean') assert_df(""" 2010-01-03 -1.0 2010-01-04 -1.0 2010-01-05 -1.0 2010-01-06 -1.0 2010-01-07 -1.0 """, values) values = tsh.get_arithmetic(engine, 'forwardfill') assert_df(""" 2010-01-03 3.0 2010-01-04 3.0 2010-01-05 3.0 2010-01-06 3.0 2010-01-07 3.0 2010-01-08 3.0 2010-01-09 3.0 """, values) values = tsh.get_arithmetic(engine, 'backwardfill') assert_df(""" 2010-01-01 3.0 2010-01-02 3.0 2010-01-03 3.0 2010-01-04 3.0 2010-01-05 3.0 2010-01-06 3.0 2010-01-07 3.0 """, values) values = tsh.get_arithmetic(engine, 'allfill') assert_df(""" 2010-01-01 3.0 2010-01-02 3.0 2010-01-03 3.0 2010-01-04 3.0 2010-01-05 3.0 2010-01-06 3.0 2010-01-07 3.0 """, values) values = tsh.get_arithmetic(engine, 'storagelike') assert_df(""" 2010-01-01 3.0 2010-01-02 3.0 2010-01-03 3.0 2010-01-04 3.0 2010-01-05 3.0 2010-01-06 1.0 2010-01-07 1.0 """, values) with pytest.raises(Exception) as err: tsh.get_arithmetic(engine, 'bogus') assert 'unknown is needed to calculate bogus and does not exist' == str(err.value)
def test_base(client, engine, tsh): assert repr(client) == "tshistory-http-client(uri='http://test-uri')" ts = client.get('no-such-series') assert ts is None assert not client.exists('no-such-series') meta = client.metadata('no-such-series') assert meta == {'message': '`no-such-series` does not exists'} series_in = genserie(utcdt(2018, 1, 1), 'H', 3) client.update('test', series_in, 'Babar', insertion_date=utcdt(2019, 1, 1)) assert client.exists('test') # now let's get it back ts = client.get('test') assert_df( """ 2018-01-01 00:00:00+00:00 0.0 2018-01-01 01:00:00+00:00 1.0 2018-01-01 02:00:00+00:00 2.0 """, ts) ts = client.get('test', from_value_date=utcdt(2018, 1, 1, 2)) assert_df(""" 2018-01-01 02:00:00+00:00 2.0 """, ts) ts = client.get('test', to_value_date=utcdt(2018, 1, 1, 0)) assert_df(""" 2018-01-01 00:00:00+00:00 0.0 """, ts) # out of range ts = client.get('test', from_value_date=utcdt(2020, 1, 1, 2), to_value_date=utcdt(2020, 1, 1, 2)) assert len(ts) == 0 assert ts.name == 'test' meta = client.metadata('test', all=True) assert meta == { 'tzaware': True, 'index_type': 'datetime64[ns, UTC]', 'value_type': 'float64', 'index_dtype': '|M8[ns]', 'value_dtype': '<f8' } # update client.update_metadata('test', {'desc': 'banana spot price'}) meta = client.metadata('test', all=False) assert meta == { 'desc': 'banana spot price', } # check the insertion_date series_in = genserie(utcdt(2018, 1, 2), 'H', 3) client.update('test', series_in, 'Babar', metadata={'event': 'hello'}, insertion_date=utcdt(2019, 1, 2)) v1 = client.get('test', revision_date=utcdt(2019, 1, 1)) assert_df( """ 2018-01-01 00:00:00+00:00 0.0 2018-01-01 01:00:00+00:00 1.0 2018-01-01 02:00:00+00:00 2.0 """, v1) d1, d2 = tsh.insertion_dates(engine, 'test') assert d1 == utcdt(2019, 1, 1) assert d2 > d1 client.update('test2', series_in, 'Babar') series = client.catalog() assert ['test', 'primary'] in series[('db://localhost:5433/postgres', 'tsh')] assert ['test2', 'primary'] in series[('db://localhost:5433/postgres', 'tsh')] client.replace('test2', genserie(utcdt(2020, 1, 1), 'D', 3), 'Babar') series = client.get('test2') assert_df( """ 2020-01-01 00:00:00+00:00 0.0 2020-01-02 00:00:00+00:00 1.0 2020-01-03 00:00:00+00:00 2.0 """, series) type = client.type('test2') assert type == 'primary' ival = client.interval('test2') assert ival.left == pd.Timestamp('2020-01-01 00:00:00+0000', tz='UTC') assert ival.right == pd.Timestamp('2020-01-03 00:00:00+0000', tz='UTC') client.rename('test2', 'test3') assert not client.exists('test2') assert client.exists('test3') client.delete('test3') assert not client.exists('test3')