def test_formula(db_uri, formula, pdbshell=False, namespace='tsh'):
engine = create_engine(find_dburi(db_uri))
tsh = timeseries(namespace)
ts = tsh.eval_formula(engine, formula)
print(ts)
if pdbshell:
import ipdb
ipdb.set_trace()
def ingest_formulas(dburi, formula_file, strict=False, namespace='tsh'):
"""ingest a csv file of formulas
Must be a two-columns file with a header "name,text"
"""
engine = create_engine(find_dburi(dburi))
df = pd.read_csv(formula_file)
tsh = timeseries(namespace)
with engine.begin() as cn:
for row in df.itertuples():
print('ingesting', row.name)
tsh.register_formula(cn, row.name, row.text, strict)
def test_local_formula_remote_series(mapi):
rtsh = timeseries('test-mapi-2')
rtsh.update(
mapi.engine,
pd.Series(
[1, 2, 3],
index=pd.date_range(pd.Timestamp('2020-1-1'), periods=3, freq='H'),
),
'remote-series',
'Babar',
insertion_date=pd.Timestamp('2020-1-1', tz='UTC')
)
mapi.register_formula(
'test-localformula-remoteseries',
'(+ 1 (series "remote-series"))'
)
ts = mapi.get('test-localformula-remoteseries')
assert_df("""
2020-01-01 00:00:00 2.0
2020-01-01 01:00:00 3.0
2020-01-01 02:00:00 4.0
""", ts)
hist = mapi.history('test-localformula-remoteseries')
assert_hist("""
insertion_date value_date
2020-01-01 00:00:00+00:00 2020-01-01 00:00:00 2.0
2020-01-01 01:00:00 3.0
2020-01-01 02:00:00 4.0
""", hist)
f = mapi.formula('test-localformula-remoteseries')
assert f == '(+ 1 (series "remote-series"))'
none = mapi.formula('nosuchformula')
assert none is None
# altsource formula
rtsh.register_formula(
mapi.engine,
'remote-formula-remote-series',
'(+ 2 (series "remote-series"))'
)
f = mapi.formula('remote-formula-remote-series')
assert f == '(+ 2 (series "remote-series"))'
assert_df("""
2020-01-01 00:00:00 3.0
2020-01-01 01:00:00 4.0
2020-01-01 02:00:00 5.0
""", mapi.get('remote-formula-remote-series'))
def typecheck_formula(db_uri, pdbshell=False, namespace='tsh'):
engine = create_engine(find_dburi(db_uri))
tsh = timeseries(namespace)
i = Interpreter(engine, tsh, {})
for name, kind in tsh.list_series(engine).items():
if kind != 'formula':
continue
formula = tsh.formula(engine, name)
parsed = parse(formula)
print(name, f'`{parsed[0]}`')
typecheck(parsed, env=i.env)
def update_metadata(dburi, reset=False, namespace='tsh'):
engine = create_engine(find_dburi(dburi))
tsh = timeseries(namespace)
if reset:
for name, kind in tsh.list_series(engine).items():
if kind != 'formula':
continue
# reset
meta = tsh.metadata(engine, name)
if meta:
meta = {k: v for k, v in meta.items() if k not in tsh.metakeys}
else:
meta = {}
sql = (f'update "{namespace}".formula '
'set metadata = %(metadata)s '
'where name = %(name)s')
print('reset', name, 'to', meta)
with engine.begin() as cn:
cn.execute(sql, metadata=json.dumps(meta), name=name)
todo = []
errors = []
def justdoit():
for name, kind in tsh.list_series(engine).items():
if kind != 'formula':
continue
print(name)
tree = parse(tsh.formula(engine, name))
smap = tsh.find_series(engine, tree)
try:
meta = tsh.filter_metadata(smap)
except ValueError as err:
errors.append((name, err))
continue
if not meta or 'index_dtype' not in meta:
todo.append(name)
print(' -> todo')
continue
tsh.update_metadata(engine, name, meta)
justdoit()
print('TODO', todo)
print('FAIL', errors)
def drop_alias_tables(db_uri, drop=False, namespace='tsh'):
engine = create_engine(find_dburi(db_uri))
# convert outliers to clip operator
elts = {
k: (min, max)
for k, min, max in engine.execute(
'select serie, min, max from tsh.outliers').fetchall()
}
tsh = timeseries(namespace)
rewriteme = []
for name, kind in tsh.list_series(engine).items():
if kind != 'formula':
continue
tree = parse(tsh.formula(engine, name))
smap = tsh.find_series(engine, tree)
for sname in smap:
if sname in elts:
rewriteme.append((name, tree))
break
for name, tree in rewriteme:
tree2 = rewrite(tree, elts)
print(name)
print(serialize(tree))
print('->')
print(serialize(tree2))
print()
tsh.register_formula(engine, name, serialize(tree2), update=True)
if not drop:
print('DID NOT DROP the tables')
print('pass --drop to really drop them')
return
with engine.begin() as cn:
cn.execute(f'drop table if exists "{namespace}".arithmetic')
cn.execute(f'drop table if exists "{namespace}".priority')
cn.execute(f'drop table if exists "{namespace}".outliers')
def fix_slice(db_uri, really=False, namespace='tsh'):
e = create_engine(find_dburi(db_uri))
tsh = timeseries(namespace)
for name, kind in tsh.list_series(e).items():
if kind != 'formula':
continue
# parse+serialize -> normalization step
form = serialize(parse(tsh.formula(e, name)))
tree = parse(form)
newtree = rewrite_slice(tree)
newform = serialize(newtree)
if form != newform:
print('rewritten', name)
print(' was', form)
print(' ->', newform)
if not really:
continue
tsh.register_formula(e, name, newform, update=True)
if not really:
print('UNCHANGED. To apply changes, pass --really')
def shell(db_uri, namespace='tsh'):
e = create_engine(find_dburi(db_uri))
tsh = timeseries(namespace)
import pdb
pdb.set_trace()
def tsh(request, engine):
return timeseries()