def test_quote_unquote_ast(self):
a = q(x + y)
b = q(ast(a) + z)
x, y, z = 1, 2, 3
assert(eval(unparse_ast(b)) == 6)
x, y, z = 1, 3, 9
assert(eval(unparse_ast(b)) == 13)
def test_simple(self):
a = 10
b = 2
data1 = q(1 + u(a + b))
data2 = q(1 + (a + b))
assert eval(unparse_ast(data1)) == 13
assert eval(unparse_ast(data2)) == 13
a = 1
assert eval(unparse_ast(data1)) == 13
assert eval(unparse_ast(data2)) == 4
def peg(tree, **kw):
for statement in tree:
if type(statement) is Assign:
new_tree, bindings = _PegWalker.recurse_real(statement.value)
statement.value = q(Parser.Lazy(lambda: ast(new_tree), [u%statement.targets[0].id]))
return tree
def trace_walk(tree, ctx, stop, **kw):
if isinstance(tree, expr) and \
tree._fields != () and \
type(tree) is not Num and \
type(tree) is not Str and \
type(tree) is not Name:
try:
literal_eval(tree)
stop()
return tree
except ValueError:
txt = ctx(tree)
trace_walk.walk_children(tree, ctx)
wrapped = q(wrap(log, u(txt), ast % tree))
stop()
return wrapped
elif isinstance(tree, stmt):
txt = ctx(tree)
trace_walk.walk_children(tree, ctx)
with q as code:
log(u(txt))
stop()
return [code, tree]
def trace_walk(tree, ctx, stop, **kw):
if isinstance(tree, expr) and \
tree._fields != () and \
type(tree) is not Num and \
type(tree) is not Str and \
type(tree) is not Name:
try:
literal_eval(tree)
stop()
return tree
except ValueError:
txt = ctx(tree)
trace_walk.walk_children(tree, ctx)
wrapped = q(wrap(log, u(txt), ast%tree))
stop()
return wrapped
elif isinstance(tree, stmt):
txt = ctx(tree)
trace_walk.walk_children(tree , ctx)
with q as code:
log(u(txt))
stop()
return [code, tree]
def func(tree, **kw):
if _is_pattern_match_stmt(tree):
modified = set()
matcher = build_matcher(tree.value.left, modified)
# lol random names for hax
with q as assignment:
xsfvdy = ast(matcher)
statements = [assignment, Expr(q(xsfvdy._match_value(ast(tree.value.right))))]
for var_name in modified:
statements.append(Assign([Name(var_name, Store())],
q(xsfvdy.get_var(u(var_name)))))
return statements
else:
return tree
def _recurse(tree, **kw):
if type(tree) is Compare and type(tree.ops[0]) is In:
return q((ast(tree.left)).in_(ast(tree.comparators[0])))
elif type(tree) is GeneratorExp:
aliases = map(f(_.target), tree.generators)
tables = map(f(_.iter), tree.generators)
aliased_tables = map(lambda x: q((ast(x)).alias().c), tables)
elt = tree.elt
if type(elt) is Tuple:
sel = q(ast_list(elt.elts))
else:
sel = q([ast(elt)])
out = q(select(ast(sel)))
for gen in tree.generators:
for cond in gen.ifs:
out = q(ast(out).where(ast(cond)))
out = q((lambda x: ast(out))())
out.func.args.args = aliases
out.args = aliased_tables
return out
def peg(tree, **kw):
for statement in tree:
if type(statement) is Assign:
new_tree, bindings = _PegWalker.recurse_real(statement.value)
statement.value = q(
Parser.Lazy(lambda: ast(new_tree),
[u % statement.targets[0].id]))
return tree
def test_structured(self):
a = [1, 2, "omg"]
b = ["wtf", "bbq"]
data1 = q([x for x in u(a + b)])
assert(eval(unparse_ast(data1)) == [1, 2, "omg", "wtf", "bbq"])
b = []
assert(eval(unparse_ast(data1)) == [1, 2, "omg", "wtf", "bbq"])
def test_quote_unquote(self):
x = 1
y = 2
a = q(u(x + y))
assert(eval(unparse_ast(a)) == 3)
x = 0
y = 0
assert(eval(unparse_ast(a)) == 3)
def func(tree, **kw):
if _is_pattern_match_stmt(tree):
modified = set()
matcher = build_matcher(tree.value.left, modified)
# lol random names for hax
with q as assignment:
xsfvdy = ast(matcher)
statements = [
assignment,
Expr(q(xsfvdy._match_value(ast(tree.value.right))))
]
for var_name in modified:
statements.append(
Assign([Name(var_name, Store())],
q(xsfvdy.get_var(u(var_name)))))
return statements
else:
return tree
def build_matcher(tree, modified):
if isinstance(tree, Num):
return q(LiteralMatcher(u(tree.n)))
if isinstance(tree, Str):
return q(LiteralMatcher(u(tree.s)))
if isinstance(tree, Name):
if tree.id in ['True', 'False', 'None']:
return q(LiteralMatcher(ast(tree)))
elif tree.id in ['_']:
return q(WildcardMatcher())
modified.add(tree.id)
return q(NameMatcher(u(tree.id)))
if isinstance(tree, List):
sub_matchers = []
for child in tree.elts:
sub_matchers.append(build_matcher(child, modified))
return Call(Name('ListMatcher', Load()), sub_matchers, [], None, None)
if isinstance(tree, Tuple):
sub_matchers = []
for child in tree.elts:
sub_matchers.append(build_matcher(child, modified))
return Call(Name('TupleMatcher', Load()), sub_matchers, [], None, None)
if isinstance(tree, Call):
sub_matchers = []
for child in tree.args:
sub_matchers.append(build_matcher(child, modified))
positional_matchers = List(sub_matchers, Load())
kw_matchers = []
for kw in tree.keywords:
kw_matchers.append(
keyword(kw.arg, build_matcher(kw.value, modified)))
return Call(Name('ClassMatcher', Load()),
[tree.func, positional_matchers], kw_matchers, None, None)
if (isinstance(tree, BinOp) and isinstance(tree.op, BitAnd)):
sub1 = build_matcher(tree.left, modified)
sub2 = build_matcher(tree.right, modified)
return Call(Name('ParallelMatcher', Load()), [sub1, sub2], [], None,
None)
raise Exception("Unrecognized tree " + repr(tree))
def test_expand_lets(self):
"""
This tests the sorta knotty logic involved in making the for-
comprehension variable available *outside* of the comprehension
when used in PINQ
"""
tree = q(lambda x: x + (lambda y: y + 1)(3))(5)
goal = q(lambda x: (lambda y: (x + (y + 1)))(3))(5)
new_tree = expand_let_bindings.recurse(tree)
assert ast.dump(new_tree) == ast.dump(goal)
tree = q(lambda x: x + (lambda y: y + 1)(3) + (lambda z: z + 2)(4))(5)
goal = q(lambda x: (lambda z: (lambda y: ((x + (y + 1)) + (z + 2)))(3))
(4))(5)
new_tree = expand_let_bindings.recurse(tree)
assert ast.dump(new_tree) == ast.dump(goal)
tree = q(lambda x: (x, lambda w: (lambda y: y + 1)(3) +
(lambda z: z + 2)(4)))(5)
goal = q(lambda x: (x, (lambda w: (lambda z: (lambda y: (
(y + 1) + (z + 2)))(3))(4))))(5)
new_tree = expand_let_bindings.recurse(tree)
assert ast.dump(new_tree) == ast.dump(goal)
def build_matcher(tree, modified):
if isinstance(tree, Num):
return q(LiteralMatcher(u(tree.n)))
if isinstance(tree, Str):
return q(LiteralMatcher(u(tree.s)))
if isinstance(tree, Name):
if tree.id in ['True', 'False', 'None']:
return q(LiteralMatcher(ast(tree)))
elif tree.id in ['_']:
return q(WildcardMatcher())
modified.add(tree.id)
return q(NameMatcher(u(tree.id)))
if isinstance(tree, List):
sub_matchers = []
for child in tree.elts:
sub_matchers.append(build_matcher(child, modified))
return Call(Name('ListMatcher', Load()), sub_matchers, [], None, None)
if isinstance(tree, Tuple):
sub_matchers = []
for child in tree.elts:
sub_matchers.append(build_matcher(child, modified))
return Call(Name('TupleMatcher', Load()), sub_matchers, [], None, None)
if isinstance(tree, Call):
sub_matchers = []
for child in tree.args:
sub_matchers.append(build_matcher(child, modified))
positional_matchers = List(sub_matchers, Load())
kw_matchers = []
for kw in tree.keywords:
kw_matchers.append(
keyword(kw.arg, build_matcher(kw.value, modified)))
return Call(Name('ClassMatcher', Load()), [tree.func,
positional_matchers], kw_matchers, None, None)
if (isinstance(tree, BinOp) and isinstance(tree.op, BitAnd)):
sub1 = build_matcher(tree.left, modified)
sub2 = build_matcher(tree.right, modified)
return Call(Name('ParallelMatcher', Load()), [sub1, sub2], [], None,
None)
raise Exception("Unrecognized tree " + repr(tree))
def test_show_expanded(self):
show_expanded(q(1 + 2))
assert result[-1] == "BinOp(left=Num(n=1), op=Add(), right=Num(n=2))"
with show_expanded:
a = 1
b = 2
with q as code:
print a + u(b + 1)
assert result[-3:] == [
'\na = 1', '\nb = 2',
"\ncode = [Print(dest=None, values=[BinOp(left=Name(id='a', ctx=Load()), op=Add(), right=ast_repr((b + 1)))], nl=True)]"
]
def f(tree, gen_sym, **kw):
@Walker
def underscore_search(tree, collect, **kw):
if isinstance(tree, Name) and tree.id == "_":
name = gen_sym()
tree.id = name
collect(name)
return tree
tree, used_names = underscore_search.recurse_real(tree)
new_tree = q(lambda: ast(tree))
new_tree.args.args = [Name(id=x) for x in used_names]
return new_tree
def s(tree, **kw):
captured = []
new_string = ""
chunks = re.split("{(.*?)}", tree.s)
for i in range(0, len(chunks)):
if i % 2 == 0:
new_string += chunks[i]
else:
new_string += "%s"
captured += [chunks[i]]
result = q(u(new_string) % tuple(ast_list(map(parse_expr, captured))))
return result
def s(tree, **kw):
captured = []
new_string = ""
chunks = re.split("{(.*?)}", tree.s)
for i in range(0, len(chunks)):
if i % 2 == 0:
new_string += chunks[i]
else:
new_string += "%s"
captured += [chunks[i]]
result = q(u(new_string) % tuple(ast_list(map(parse_expr, captured))))
return result
def _PegWalker(tree, ctx, stop, collect, **kw):
if type(tree) is Str:
stop()
return q(Parser.Raw(ast(tree)))
if type(tree) is BinOp and type(tree.op) is RShift:
tree.left, b_left = _PegWalker.recurse_real(tree.left)
tree.right = q(lambda bindings: ast(tree.right))
tree.right.args.args = map(f(Name(id=_)), flatten(b_left))
stop()
return tree
if type(tree) is BinOp and type(tree.op) is FloorDiv:
tree.left, b_left = _PegWalker.recurse_real(tree.left)
stop()
collect(b_left)
return tree
if type(tree) is Tuple:
result = q(Parser.Seq([]))
result.args[0].elts = tree.elts
all_bindings = []
for i, elt in enumerate(tree.elts):
result.args[0].elts[i], bindings = _PegWalker.recurse_real(
tree.elts[i])
all_bindings.append(bindings)
stop()
collect(all_bindings)
return result
if type(tree) is Compare and type(tree.ops[0]) is Is:
left_tree, bindings = _PegWalker.recurse_real(tree.left)
new_tree = q(ast(left_tree).bind_to(u(tree.comparators[0].id)))
stop()
collect(bindings + [tree.comparators[0].id])
return new_tree
def tco(tree, **kw):
@Walker
# Replace returns of calls
def return_replacer(tree, **kw):
with switch(tree):
if Return(value=Call(
func=func,
args=args,
starargs=starargs,
kwargs=kwargs)):
with q as code:
return (tco.CALL,
ast(func),
ast(List(args, Load())),
ast(starargs or List([], Load())),
ast(kwargs or Dict([],[])))
return code
else:
return tree
# Replace calls (that aren't returned) which happen to be in a tail-call
# position
def replace_tc_pos(node):
with switch(node):
if Expr(value=Call(
func=func,
args=args,
starargs=starargs,
kwargs=kwargs)):
with q as code:
return (tco.IGNORE,
ast(func),
ast(List(args, Load())),
ast(starargs or List([], Load())),
ast(kwargs or Dict([], [])))
return code
elif If(test=test, body=body, orelse=orelse):
body[-1] = replace_tc_pos(body[-1])
if orelse:
orelse[-1] = replace_tc_pos(orelse[-1])
return If(test, body, orelse)
else:
return node
tree = return_replacer.recurse(tree)
tree.decorator_list = ([q(tco.trampoline_decorator)] +
tree.decorator_list)
tree.body[-1] = replace_tc_pos(tree.body[-1])
return tree
def _PegWalker(tree, ctx, stop, collect, **kw):
if type(tree) is Str:
stop()
return q(Parser.Raw(ast(tree)))
if type(tree) is BinOp and type(tree.op) is RShift:
tree.left, b_left = _PegWalker.recurse_real(tree.left)
tree.right = q(lambda bindings: ast(tree.right))
tree.right.args.args = map(f(Name(id = _)), flatten(b_left))
stop()
return tree
if type(tree) is BinOp and type(tree.op) is FloorDiv:
tree.left, b_left = _PegWalker.recurse_real(tree.left)
stop()
collect(b_left)
return tree
if type(tree) is Tuple:
result = q(Parser.Seq([]))
result.args[0].elts = tree.elts
all_bindings = []
for i, elt in enumerate(tree.elts):
result.args[0].elts[i], bindings = _PegWalker.recurse_real(tree.elts[i])
all_bindings.append(bindings)
stop()
collect(all_bindings)
return result
if type(tree) is Compare and type(tree.ops[0]) is Is:
left_tree, bindings = _PegWalker.recurse_real(tree.left)
new_tree = q(ast(left_tree).bind_to(u(tree.comparators[0].id)))
stop()
collect(bindings + [tree.comparators[0].id])
return new_tree
def test_show_expanded(self):
show_expanded(q(1 + 2))
assert result[-1] == "BinOp(left=Num(n=1), op=Add(), right=Num(n=2))"
with show_expanded:
a = 1
b = 2
with q as code:
print a + u(b + 1)
assert result[-3:] == [
'\na = 1',
'\nb = 2',
"\ncode = [Print(dest=None, values=[BinOp(left=Name(id='a', ctx=Load()), op=Add(), right=ast_repr((b + 1)))], nl=True)]"
]
def f(tree, **kw):
names = ('arg' + str(i) for i in xrange(100))
@Walker
def underscore_search(tree, collect, **kw):
if isinstance(tree, Name) and tree.id == "_":
name = names.next()
tree.id = name
collect(name)
return tree
tree, used_names = underscore_search.recurse_real(tree)
new_tree = q(lambda: ast(tree))
new_tree.args.args = [Name(id=x) for x in used_names]
return new_tree
def f(tree, **kw):
names = ('arg' + str(i) for i in xrange(100))
@Walker
def underscore_search(tree, collect, **kw):
if isinstance(tree, Name) and tree.id == "_":
name = names.next()
tree.id = name
collect(name)
return tree
tree, used_names = underscore_search.recurse_real(tree)
new_tree = q(lambda: ast(tree))
new_tree.args.args = [Name(id = x) for x in used_names]
return new_tree
def tco(tree, **kw):
@Walker
# Replace returns of calls
def return_replacer(tree, **kw):
with switch(tree):
if Return(value=Call(
func=func, args=args, starargs=starargs, kwargs=kwargs)):
with q as code:
return (tco.CALL, ast(func), ast(List(args, Load())),
ast(starargs or List([], Load())),
ast(kwargs or Dict([], [])))
return code
else:
return tree
# Replace calls (that aren't returned) which happen to be in a tail-call
# position
def replace_tc_pos(node):
with switch(node):
if Expr(value=Call(
func=func, args=args, starargs=starargs, kwargs=kwargs)):
with q as code:
return (tco.IGNORE, ast(func), ast(List(args, Load())),
ast(starargs or List([], Load())),
ast(kwargs or Dict([], [])))
return code
elif If(test=test, body=body, orelse=orelse):
body[-1] = replace_tc_pos(body[-1])
if orelse:
orelse[-1] = replace_tc_pos(orelse[-1])
return If(test, body, orelse)
else:
return node
tree = return_replacer.recurse(tree)
tree.decorator_list = ([q(tco.trampoline_decorator)] + tree.decorator_list)
tree.body[-1] = replace_tc_pos(tree.body[-1])
return tree
def test_expand_lets(self):
"""
This tests the sorta knotty logic involved in making the for-
comprehension variable available *outside* of the comprehension
when used in PINQ
"""
tree = q(lambda x: x + (lambda y: y + 1)(3))(5)
goal = q(lambda x: (lambda y: (x + (y + 1)))(3))(5)
new_tree = expand_let_bindings.recurse(tree)
assert ast.dump(new_tree) == ast.dump(goal)
tree = q(lambda x: x + (lambda y: y + 1)(3) + (lambda z: z + 2)(4))(5)
goal = q(lambda x: (lambda z: (lambda y: ((x + (y + 1)) + (z + 2)))(3))(4))(5)
new_tree = expand_let_bindings.recurse(tree)
assert ast.dump(new_tree) == ast.dump(goal)
tree = q(lambda x: (x, lambda w: (lambda y: y + 1)(3) + (lambda z: z + 2)(4)))(5)
goal = q(lambda x: (x, (lambda w: (lambda z: (lambda y: ((y + 1) + (z + 2)))(3))(4))))(5)
new_tree = expand_let_bindings.recurse(tree)
assert ast.dump(new_tree) == ast.dump(goal)
def show_expanded(tree, expand_macros, **kw):
expanded_tree = expand_macros(tree)
new_tree = q(wrap_simple(log, u(unparse_ast(expanded_tree)), ast(expanded_tree)))
return new_tree
def log(tree, exact_src, **kw):
new_tree = q(wrap(log, u(exact_src(tree)), ast(tree)))
return new_tree
def _require_transform(tree, exact_src):
ret = trace_walk.recurse(copy.deepcopy(tree), exact_src)
trace_walk.recurse(copy.deepcopy(tree), exact_src)
new = q(ast(tree) or handle(lambda log: ast(ret)))
return new
def test_unquote_name(self):
n = "x"
x = 1
y = q(name(n) + name(n))
assert(eval(unparse_ast(y)) == 2)
def pyjs(tree, **kw):
javascript = pjs.converter.Converter("").convert_node(tree, Scope())
return q((ast(tree), u(javascript)))
# (f, a1, ..., an) --> f(a1, ..., an)
posargs = [x for x in data if not iskwargs(x)]
# TODO: tag *args and **kwargs in a kw() as invalid, too (currently just ignored)
invalids = list(flatmap(lambda x: x.args, filter(iskwargs, data)))
if invalids:
assert False, "kw(...) may only specify named args"
kwargs = flatmap(lambda x: x.keywords, filter(iskwargs, data))
kwargs = list(rev(uniqify(rev(kwargs), key=lambda x: x.arg))) # latest wins, but keep original ordering
return Call(func=op, args=posargs, keywords=list(kwargs))
# This is a first-pass macro. Any nested macros should get clean standard Python,
# not having to worry about tuples possibly denoting function calls.
yield transform.recurse(block_body, quotelevel=0)
# note the exported "q" is ours, but the q we use in this module is a macro.
class q:
"""[syntax] Quote operator. Only meaningful in a tuple in a prefix block."""
def __repr__(self): # in case one of these ends up somewhere at runtime
return "<quote>"
q = q()
class u:
"""[syntax] Unquote operator. Only meaningful in a tuple in a prefix block."""
def __repr__(self): # in case one of these ends up somewhere at runtime
return "<unquote>"
u = u()
# not a @macro_stub; it only raises a run-time error on foo[...], not foo(...)
def kw(**kwargs):
"""[syntax] Pass-named-args operator. Only meaningful in a tuple in a prefix block."""
raise RuntimeError("kw only meaningful inside a tuple in a prefix block")
def _require_transform(tree, exact_src):
ret = trace_walk.recurse(copy.deepcopy(tree), exact_src)
trace_walk.recurse(copy.deepcopy(tree), exact_src)
new = q(ast(tree) or handle(lambda log: ast(ret)))
return new
def log(tree, exact_src, **kw):
new_tree = q(wrap(log, u(exact_src(tree)), ast(tree)))
return new_tree
def show_expanded(tree, expand_macros, **kw):
expanded_tree = expand_macros(tree)
new_tree = q(
wrap_simple(log, u(unparse_ast(expanded_tree)), ast(expanded_tree)))
return new_tree
def expand(tree, **kw):
addition = 10
return q(lambda x: x * ast(tree) + u(addition))
def expand(tree, **kw):
addition = 10
return q(lambda x: x * ast(tree) + u(addition))
def query(tree, **kw):
x = _recurse.recurse(tree)
x = expand_let_bindings.recurse(x)
return q((lambda query: query.bind.execute(query).fetchall())(ast(x)))
