def test_parse_serialize():
expr = parse('(fun 1 "foo" nil #f #:c 5 (zogzog 42 #t))')
assert [
node.__class__.__name__
for node in expr
] == [
'Symbol', 'int', 'str', 'NoneType', 'bool', 'Keyword', 'int', 'list'
]
assert serialize(parse('(fun 1 #t #f #:c "babar")')) == '(fun 1 #t #f #:c "babar")'
assert serialize(parse('(fun 1 nil #:c (+ 5.0 7))')) == '(fun 1 nil #:c (+ 5.0 7))'
def drop_alias_tables(db_uri, drop=False, namespace='tsh'):
engine = create_engine(find_dburi(db_uri))
# convert outliers to clip operator
elts = {
k: (min, max)
for k, min, max in engine.execute(
'select serie, min, max from tsh.outliers').fetchall()
}
tsh = timeseries(namespace)
rewriteme = []
for name, kind in tsh.list_series(engine).items():
if kind != 'formula':
continue
tree = parse(tsh.formula(engine, name))
smap = tsh.find_series(engine, tree)
for sname in smap:
if sname in elts:
rewriteme.append((name, tree))
break
for name, tree in rewriteme:
tree2 = rewrite(tree, elts)
print(name)
print(serialize(tree))
print('->')
print(serialize(tree2))
print()
tsh.register_formula(engine, name, serialize(tree2), update=True)
if not drop:
print('DID NOT DROP the tables')
print('pass --drop to really drop them')
return
with engine.begin() as cn:
cn.execute(f'drop table if exists "{namespace}".arithmetic')
cn.execute(f'drop table if exists "{namespace}".priority')
cn.execute(f'drop table if exists "{namespace}".outliers')
def constant_fold(tree):
op = tree[0]
if op in '+*/':
# immediately foldable
if (isinstance(tree[1], (int, float))
and isinstance(tree[2], (int, float))):
return evaluate(serialize(tree), _CFOLDENV)
newtree = [op]
for arg in tree[1:]:
if isinstance(arg, list):
newtree.append(constant_fold(arg))
else:
newtree.append(arg)
if op in '+*/':
# maybe foldable after arguments rewrite
if (isinstance(newtree[1], (int, float))
and isinstance(newtree[2], (int, float))):
return evaluate(serialize(newtree), _CFOLDENV)
return newtree
def rename(self, cn, oldname, newname):
# read all formulas and parse them ...
formulas = cn.execute(
f'select name, text from "{self.namespace}".formula').fetchall()
errors = []
def edit(tree, oldname, newname):
newtree = []
series = False
for node in tree:
if isinstance(node, list):
newtree.append(edit(node, oldname, newname))
continue
if node == 'series':
series = True
newtree.append(node)
continue
elif node == oldname and series:
node = newname
newtree.append(node)
series = False
return newtree
for fname, text in formulas:
tree = parse(text)
seriesmeta = self.find_series(cn, tree)
if newname in seriesmeta:
errors.append(fname)
if oldname not in seriesmeta or errors:
continue
newtree = edit(tree, oldname, newname)
newtext = serialize(newtree)
sql = (f'update "{self.namespace}".formula '
'set text = %(text)s '
'where name = %(name)s')
cn.execute(sql, text=newtext, name=fname)
if errors:
raise ValueError(
f'new name is already referenced by `{",".join(errors)}`')
if self.type(cn, oldname) == 'formula':
cn.execute(
f'update "{self.namespace}".formula '
'set name = %(newname)s '
'where name = %(oldname)s',
oldname=oldname,
newname=newname)
else:
super().rename(cn, oldname, newname)
def fix_slice(db_uri, really=False, namespace='tsh'):
e = create_engine(find_dburi(db_uri))
tsh = timeseries(namespace)
for name, kind in tsh.list_series(e).items():
if kind != 'formula':
continue
# parse+serialize -> normalization step
form = serialize(parse(tsh.formula(e, name)))
tree = parse(form)
newtree = rewrite_slice(tree)
newform = serialize(newtree)
if form != newform:
print('rewritten', name)
print(' was', form)
print(' ->', newform)
if not really:
continue
tsh.register_formula(e, name, newform, update=True)
if not really:
print('UNCHANGED. To apply changes, pass --really')
def register_formula(self,
cn,
name,
formula,
reject_unknown=True,
update=False):
if not update:
assert not self.formula(cn, name), f'`{name}` already exists'
if self.exists(cn, name) and self.type(cn, name) == 'primary':
raise TypeError(
f'primary series `{name}` cannot be overriden by a formula')
# basic syntax check
tree = parse(formula)
formula = serialize(tree)
# build metadata & check compat
seriesmeta = self.find_series(cn, tree)
if not all(seriesmeta.values()) and reject_unknown:
badseries = [k for k, v in seriesmeta.items() if not v]
raise ValueError(f'Formula `{name}` refers to unknown series '
f'{", ".join("`%s`" % s for s in badseries)}')
# bad operators
operators = self.find_operators(cn, tree)
badoperators = [op for op, func in operators.items() if func is None]
if badoperators:
raise ValueError(f'Formula `{name}` refers to unknown operators '
f'{", ".join("`%s`" % o for o in badoperators)}')
# type checking
i = interpreter.Interpreter(cn, self, {})
helper.typecheck(tree, env=i.env)
meta = self.filter_metadata(seriesmeta)
sql = (f'insert into "{self.namespace}".formula '
'(name, text) '
'values (%(name)s, %(text)s) '
'on conflict (name) do update '
'set text = %(text)s')
cn.execute(sql, name=name, text=formula)
if meta:
self.update_metadata(cn, name, meta, internal=True)
def expanded_formula(self, cn, name):
formula = self.formula(cn, name)
tree = parse(formula)
return serialize(helper.expanded(self, cn, tree))
def test_rewrite_slice():
form = '(* 3 (slice (series "42") #:fromdate "2020-1-1" #:todate (today)))'
newform = serialize(rewrite_slice(parse(form)))
assert newform == (
'(* 3 (slice (series "42") #:fromdate (date "2020-1-1") #:todate (today)))'
)