def as_module(mod_dict):
d = {}
for k, v in mod_dict.items():
if not isinstance(k, str):
k = DISPATCH_SEP.join(map(str, k))
if isinstance(v, list):
v = Leaf(TT.SPECIAL, v[0])
else:
v = Leaf(TT.BUILTIN, v)
d[k] = v
return d
def bakevars(x, vars):
if isinstance(x, Tree):
L, H, R = bakevars(x.L, vars), bakevars(x.H, vars), bakevars(x.R, vars)
x = Tree(L, H, R)
elif x.tt in (TT.THUNK, TT.FUNTHUNK):
x = Leaf(x.tt, bakevars(unwrap(x), vars))
elif x.tt == TT.FUNCTION:
body = bakevars(x.w.body, vars)
x = Leaf(x.tt, x.w.clone(body))
elif x.tt == TT.SYMBOL and x.w in vars:
# NOTE: only bake on symbol, not on string. Otherwise you couldn't do assignments
x = Tree(Leaf(TT.SYMBOL, ".", debug=x.debug),
Leaf(TT.PUNCTUATION, "$"), x)
return x
def num_fold_by(a, b):
if b.tt != TT.TREE:
raise TypecheckError(f"num fold by Expected tree. Got {b.tt}")
arr = a.w
key = b.L.w
op = b.R.w
op = {
"+": lambda a, b: a + b,
"-": lambda a, b: a - b,
"*": lambda a, b: a * b,
"/": lambda a, b: a // b,
}[op]
acc = [None] * (max(key) + 1)
for i, y in enumerate(zip(arr, key)):
x, slot = y
slot_value = acc[slot]
if slot_value is None:
acc[slot] = x
else:
acc[slot] = op(slot_value, x)
return Leaf("num_vec", acc)
def makefunc_(a, env):
if a.tt not in (TT.THUNK, TT.FUNTHUNK):
raise Exception(f"Can't create function out of '{a.tt}'")
body = a.w
left_name, right_name = "x", "y"
if body.tt == TT.TREE and body.H.tt == TT.SEPARATOR:
header, body = body.L, body.R
if header.tt == TT.SYMBOL:
left_name, right_name = header.w, "_"
elif header.tt == TT.TREE and iscons(header.H):
left_name, right_name = header.L, header.R
if not (left_name.tt == right_name.tt == TT.SYMBOL):
raise TypecheckError(
f"Function parameter names need to by symbols."
f" Given '{left_name.tt}' : '{right_name.tt}'")
left_name, right_name = left_name.w, right_name.w
else:
header, body = None, a.w
#raise TypecheckError(f"Fn header expected cons TREE | SYMBOL. Got {header.tt}")
body = bakevars(body, [left_name, right_name])
return Leaf(TT.FUNCTION,
Function(left_name, right_name, body, env),
debug=body.debug)
def scan(a, b):
if isinstance(b, Tree):
zero = b.R.w
op = b.L.w
else:
op = b.w
zero = {
"+": 0,
"-": 0,
"*": 1,
"/": 1,
}[op]
op = {
"+": lambda a, b: a + b,
"-": lambda a, b: a - b,
"*": lambda a, b: a * b,
"/": lambda a, b: a // b,
}[op]
acc = zero
r = []
for x in a.w:
acc = op(acc, x)
r += [acc]
return Leaf("vec", r)
def asmod_vec(a, b, env, cstack):
d = {}
for item in a.w:
assert item.tt == TT.TREE
assert item.L.tt in (TT.SYMBOL, TT.STRING)
d[item.L.w] = item.R
return Leaf(TT.OBJECT, Env(env, from_dict=d)), None, env, cstack
def load(a, b, env, cstack):
with open(b.w) as f:
code = f.read()
# print(f"CODE: '{code}'", file=sys.stderr)
_, _, module, _ = Execute(code, Env(env), cstack)
if module is None:
raise TypecheckError("Module can't be NULL")
return Leaf(TT.OBJECT, module), None, env, cstack
def add_type(x):
if isinstance(x, (Leaf, Tree)):
return x
elif isinstance(x, str):
t = TT.STRING
elif isinstance(x, int):
t = TT.NUM
else:
raise TypecheckError(
f"Can't add type to native type {type(x).__name__} ")
return Leaf(t, x)
def num_fold(a, b):
op = b.w
op = {
"+": lambda a, b: a + b,
"-": lambda a, b: a - b,
"*": lambda a, b: a * b,
"/": lambda a, b: a // b,
}[op]
acc = a.w[0]
for x in a.w[1:]:
acc = op(acc, x)
return Leaf(TT.NUM, acc)
def prepare_env():
modules_ = mod_merge(modules, matrix.modules)
mods = {
k: Leaf(TT.OBJECT, Env(None, from_dict=as_module(mod)))
for k, mod in modules_.items()
}
rootenv = Env(None, from_dict={
**as_module(BUILTINS),
**mods,
})
# rootenv = Env(rootenv) # dummy env
return rootenv
def json_each(filename, fn):
import json
with open(filename, "r") as f:
for item in f:
item = json.loads(item)
item = Leaf(TT.NATIVE_OBJECT, Env(None, from_dict=item))
item = Tree(item, fn, Unit)
env = prepare_env()
cstack = Cactus(ROOT_TAG)
item, _, _, _ = Eval(item, env, cstack)
print(item)
return Unit
def shift(a, b, env, cstack):
assert a.tt in (TT.SYMBOL, TT.STRING
) # TODO keep only symbol as continuation tags?
tag = a.w
try:
cc = cstack.spop(tag)
except Cactus.Empty:
raise
# # Don't let the continuation binding propagate to parent environment
# env = Env(env)
# So far continuation is just a pair of st and env
continuation = Leaf(TT.CONTINUATION, (cc, env))
#env.bind("cc", continuation)
# New: let the cc binding take place in function object
if isinstance(b, Leaf) and b.tt in (TT.THUNK, TT.FUNTHUNK, TT.FUNCTION):
b = Tree(continuation, b, Unit)
return b, None, env, cstack
def Execute_(code, env, cstack):
x = code
x = Lex(x)
# print("LEX", y)
x = Parse(x)
# Wrap in error reset
x = Tree(Leaf(TT.SYMBOL, "error", debug=Unit.debug),
Leaf(TT.SYMBOL, "reset"), Leaf(TT.THUNK, x, debug=x.debug))
# Wrap in global reset
x = Tree(Leaf(TT.SYMBOL, ROOT_TAG, debug=Unit.debug),
Leaf(TT.SYMBOL, "reset"), Leaf(TT.THUNK, x, debug=x.debug))
x, err, env, cstack = Eval(x, env, cstack)
return x, err, env, cstack
def app(a, b):
if a.tt == "vec":
return Leaf("vec", a.w + [b.w])
return Leaf("vec", [a.w, b.w])
def num_each(a, b, env, cstack):
f, R = each_prep(b)
v = [Eval(Tree(Leaf(TT.NUM, x), f, R), env, cstack)[0] for x in a.w]
return Leaf("vec", v), None, env, cstack
def each(a, b, env, cstack):
f, R = each_prep(b)
v = [Eval(Tree(x, f, R), env, cstack)[0] for x in a.w]
return Leaf("vec", v), None, env, cstack
def safe_variable(a, b, env, cstack):
value, err = env.lookup(b.w, None), None
if value is None:
err = Shift("error", Leaf(TT.STRING, f"Variable {b.w} not defined"))
return value, err, env, cstack
BUILTINS = {
"jsoneach":
lambda a, b: json_each(a.w, b),
"=":
eq,
"==":
eq,
"!=":
lambda a, b: Leaf(TT.NUM, 1 - eq(a, b).w),
"T":
get_type,
"type":
get_type,
"retype":
lambda a, b: Leaf(b.w, a.w),
"sametype":
lambda a, b: Leaf(TT.NUM, 1 if a.tt == b.tt else 0),
"dispatch":
set_dispatch,
"til":
lambda a, b: Leaf("range", mkrange(a.w, 1, b.w)),
# "enumerate": lambda a, b: Leaf("range", (0, 1, a.w)),
"if":
lambda a, b: unwrap(a.L),
def mod_merge(a, b):
return Leaf(a.tt, Env(a.w.parent, from_dict={
**a.w.e,
**b.w.e,
}))
def order(a, b):
return Leaf(a.tt,
[i for i, _ in sorted(enumerate(a.w), key=lambda x: x[1])])
def asmod_tree(a, b, env, cstack):
assert a.L.tt in (TT.SYMBOL, TT.STRING)
d = {a.L.w: a.R}
return Leaf(TT.OBJECT, Env(env, from_dict=d)), None, env, cstack
def get_type(a, b):
if isinstance(a, Tree):
return Leaf(TT.SYMBOL, TT.TREE)
return Leaf(TT.SYMBOL, a.tt)
def zip_(a, b):
return Leaf(
a.tt,
[Tree(x, Leaf(TT.PUNCTUATION, ":"), y) for x, y in zip(a.w, b.w)])
def new_object(a, b):
return Leaf(TT.OBJECT, Env(None))
def choose(a, b):
new = [0] * len(b.w)
for i, pos in enumerate(b.w):
# TODO handle out of bounds
new[i] = a.w[pos]
return Leaf(a.tt, new)
def eq(a, b):
result = 1 if a.tt == b.tt and a.w == b.w else 0
return Leaf(TT.NUM, result)
def hb_shift(tag, value):
tag = Leaf(TT.SYMBOL, tag, debug=value.debug)
return Tree(tag, Leaf(TT.SYMBOL, "shift"), value)
def Eval(x, env, cstack):
# Stack of continuations
cstack.push(Frame(CT.Return, None, None, None, env))
# Stored instruction pointer
ins = next_ins(x)
while True:
if ins >= CT.Tree:
if ins == CT.Tree:
L, H, R = x.L, x.H, x.R
if ins < CT.Left and isinstance(L, Tree):
cstack.push(Frame(CT.Left, L, H, R, env))
x, ins = L, next_ins(L)
continue
if ins < CT.Head and isinstance(H, Tree):
cstack.push(Frame(CT.Head, L, H, R, env))
x, ins = H, next_ins(H)
continue
# print("H", type(H), H)
if H.tt == TT.SEPARATOR:
# Tail recurse on separator '|' before R gets evaluated
x, ins = R, next_ins(R)
continue
if ins < CT.Right and isinstance(R, Tree):
cstack.push(Frame(CT.Right, L, H, R, env))
x, ins = R, next_ins(R)
continue
# print("EVAL", x, file=sys.stderr)
# print("L", L, file=sys.stderr)
# print("R", R, R.debug, file=sys.stderr)
new_debug = DebugInfo(L.debug.start, R.debug.end, L.debug.lineno) \
if L.debug and R.debug else None
#new_debug = DebugInfo(L.debug.start, R.debug.end, L.debug.lineno)
# TODO reorder by frequency of invocation. BUILTIN to top?
if H.tt == TT.UNIT:
x = H
x.debug = new_debug
elif H.tt == TT.CONTINUATION:
cc, env = H.w # unwrap continuation and captured environment
cstack.scopy(
cc) # Copy over its stack onto newly created stack
x, ins = L, next_ins(L)
x.debug = new_debug
elif iscons(H):
x = Tree(L, H, R)
x.debug = new_debug
elif H.tt == TT.BUILTIN:
try:
x = H.w(L, R)
except Exception as exc:
# NOTE temporary solution. Wrap exception's string into
# error and shift it in. Better solution would be to return
# (x, err) pair from builtins instead of catching arbitrary
# error. In this case, at least create error inheritance
# hierarchy
if isinstance(exc, AssertionError):
raise
#print("RROR", type(exc), str(exc), file=sys.stderr)
x = hb_shift("error",
Leaf(TT.ERROR, str(exc), debug=new_debug))
ins = next_ins(x)
x.debug = new_debug
continue
elif H.tt == TT.SPECIAL:
x, shift, env, cstack = H.w(L, R, env, cstack)
# Capture shift tag and value from shift channel, wrap it with
# TT.ERROR and continue with this error continuation to next
# eval iteration
if shift is not None:
assert isinstance(shift, Shift)
x = hb_shift(shift.tag, shift.value)
ins = next_ins(x)
x.debug = new_debug
continue
elif H.tt == TT.FUNTHUNK:
x = Tree(L, Tree(H, Leaf(TT.SYMBOL, "func"), Unit), R)
ins = next_ins(x)
x.debug = H.debug
continue
elif H.tt == TT.THUNK:
x = unwrap(H)
ins = next_ins(x)
x.debug = new_debug # TODO really this way? Or just grab debug from H itself?
continue
elif H.tt == TT.FUNCTION:
func = H.w
# Tail optimize cstack and env if the last frame would
# be effectively the same as the new one
self_h = env.lookup(SELF_F, None)
last_frame = cstack.peek()
if (last_frame and last_frame.ct != CT.Function) \
or self_h is not H:
# TODO compare self_h == H or self_h.func == func?
cstack.push(Frame(CT.Function, L, H, R, env))
# Set up func's original env -> lexical scoping
env = func.env
env = Env(env)
# print(TT.OBJECT, id(env))
env.bind(func.left_name, L)
env.bind(SELF_F, H)
env.bind(func.right_name, R)
x = func.body
ins = next_ins(x)
x.debug = new_debug
continue
elif H.tt == TT.TREE and iscons(H.H):
path, fn = tree2env(H, env)
fn_env = path2env(path, env)
op = fn_env.lookup(fn, None)
if op is None:
raise NoDispatch(
f"Can't find module function {H} on L: {L.tt}", H)
assert op.tt in (TT.CONTINUATION, TT.SPECIAL, TT.FUNCTION,
TT.BUILTIN, TT.THUNK, TT.FUNTHUNK)
H = op
ins = CT.Right
x.debug = new_debug
continue
elif H.tt == TT.OBJECT:
# If module given for dispatch,
# lookup a constructor "." function on it.
# "." reserved for constructors
# because it can't be overriden in the module
constructor = H.w.lookup(".", None)
if not constructor:
raise AssertionError("Constructor not found")
H = constructor
assert H.tt in (
TT.CONTINUATION,
TT.SPECIAL,
TT.FUNCTION, # TT.CLOSURE,
TT.BUILTIN,
TT.THUNK,
TT.FUNTHUNK,
TT.SYMBOL)
ins = CT.Right
x.debug = new_debug
continue
elif H.tt in (TT.PUNCTUATION, TT.SYMBOL, TT.STRING, TT.SEPARATOR):
H = dispatch(H, L.tt, R.tt, env)
ins = CT.Right
x.debug = new_debug
continue
#elif H.tt == TT.CLOSURE:
# cstack.push(Frame(CT.Function, L, H, R, env))
# env, H = H.w
# ins = CT.Right
# continue
else:
raise CantReduce(f"Can't reduce node: {H} of {H.tt}", H)
#
# # Capture current environment and close over it
# if isinstance(x, Leaf) and x.tt == TT.FUNCTION:
# x = Leaf(TT.CLOSURE, (env, x))
# Restore stack frame and apply continuation
c = cstack.pop()
ins = c.ct
if ins == CT.Return:
return x, None, env, cstack # TODO None as error?
L, H, R, env = c.L, c.H, c.R, c.env
# print("Restore", L, H, R, c.ct.name, id(env), env)
if c.ct == CT.Function:
ins = next_ins(x)
elif c.ct == CT.Left:
L = x
elif c.ct == CT.Head:
H = x
elif c.ct == CT.Right:
R = x
else:
assert False
def safe_variable(a, b, env, cstack):
value, err = env.lookup(b.w, None), None
if value is None:
err = Shift("error", Leaf(TT.STRING, f"Variable {b.w} not defined"))
return value, err, env, cstack
rest,
ar,
start + i * stride,
stride * cur)
for i in range(cur)
)
return x + "\n"
def __str__(self):
return Matrix.print_dim(self._shape, self._ar, 0, 1).rstrip("\n")
def tomatrix(vec):
return Matrix([len(vec.w)], vec.w)
modules = {
"num_vec": {
"tomatrix": lambda a, b: Leaf("matrix", tomatrix(a)),
},
"matrix": {
("reshape", "num_vec"): lambda a, b: Leaf(a.tt, a.w.reshape(b.w)),
("+", TT.NUM): lambda a, b: Leaf(a.tt, a.w.apply((lambda a, b: a + b), b.w)),
("-", TT.NUM): lambda a, b: Leaf(a.tt, a.w.apply((lambda a, b: a - b), b.w)),
("*", TT.NUM): lambda a, b: Leaf(a.tt, a.w.apply((lambda a, b: a * b), b.w)),
("/", TT.NUM): lambda a, b: Leaf(a.tt, a.w.apply((lambda a, b: a // b), b.w)),
"shape": lambda a, b: Leaf("num_vec", a.w.shape()),
"rank": lambda a, b: Leaf(TT.NUM, a.w.rank()),
}
}