def test_quote_unquote_ast(self): a = q(x + y) b = q(ast(a) + z) x, y, z = 1, 2, 3 assert(eval(unparse_ast(b)) == 6) x, y, z = 1, 3, 9 assert(eval(unparse_ast(b)) == 13)
def test_simple(self): a = 10 b = 2 data1 = q(1 + u(a + b)) data2 = q(1 + (a + b)) assert eval(unparse_ast(data1)) == 13 assert eval(unparse_ast(data2)) == 13 a = 1 assert eval(unparse_ast(data1)) == 13 assert eval(unparse_ast(data2)) == 4
def peg(tree, **kw): for statement in tree: if type(statement) is Assign: new_tree, bindings = _PegWalker.recurse_real(statement.value) statement.value = q(Parser.Lazy(lambda: ast(new_tree), [u%statement.targets[0].id])) return tree
def trace_walk(tree, ctx, stop, **kw): if isinstance(tree, expr) and \ tree._fields != () and \ type(tree) is not Num and \ type(tree) is not Str and \ type(tree) is not Name: try: literal_eval(tree) stop() return tree except ValueError: txt = ctx(tree) trace_walk.walk_children(tree, ctx) wrapped = q(wrap(log, u(txt), ast % tree)) stop() return wrapped elif isinstance(tree, stmt): txt = ctx(tree) trace_walk.walk_children(tree, ctx) with q as code: log(u(txt)) stop() return [code, tree]
def trace_walk(tree, ctx, stop, **kw): if isinstance(tree, expr) and \ tree._fields != () and \ type(tree) is not Num and \ type(tree) is not Str and \ type(tree) is not Name: try: literal_eval(tree) stop() return tree except ValueError: txt = ctx(tree) trace_walk.walk_children(tree, ctx) wrapped = q(wrap(log, u(txt), ast%tree)) stop() return wrapped elif isinstance(tree, stmt): txt = ctx(tree) trace_walk.walk_children(tree , ctx) with q as code: log(u(txt)) stop() return [code, tree]
def func(tree, **kw): if _is_pattern_match_stmt(tree): modified = set() matcher = build_matcher(tree.value.left, modified) # lol random names for hax with q as assignment: xsfvdy = ast(matcher) statements = [assignment, Expr(q(xsfvdy._match_value(ast(tree.value.right))))] for var_name in modified: statements.append(Assign([Name(var_name, Store())], q(xsfvdy.get_var(u(var_name))))) return statements else: return tree
def _recurse(tree, **kw): if type(tree) is Compare and type(tree.ops[0]) is In: return q((ast(tree.left)).in_(ast(tree.comparators[0]))) elif type(tree) is GeneratorExp: aliases = map(f(_.target), tree.generators) tables = map(f(_.iter), tree.generators) aliased_tables = map(lambda x: q((ast(x)).alias().c), tables) elt = tree.elt if type(elt) is Tuple: sel = q(ast_list(elt.elts)) else: sel = q([ast(elt)]) out = q(select(ast(sel))) for gen in tree.generators: for cond in gen.ifs: out = q(ast(out).where(ast(cond))) out = q((lambda x: ast(out))()) out.func.args.args = aliases out.args = aliased_tables return out
def peg(tree, **kw): for statement in tree: if type(statement) is Assign: new_tree, bindings = _PegWalker.recurse_real(statement.value) statement.value = q( Parser.Lazy(lambda: ast(new_tree), [u % statement.targets[0].id])) return tree
def test_structured(self): a = [1, 2, "omg"] b = ["wtf", "bbq"] data1 = q([x for x in u(a + b)]) assert(eval(unparse_ast(data1)) == [1, 2, "omg", "wtf", "bbq"]) b = [] assert(eval(unparse_ast(data1)) == [1, 2, "omg", "wtf", "bbq"])
def test_quote_unquote(self): x = 1 y = 2 a = q(u(x + y)) assert(eval(unparse_ast(a)) == 3) x = 0 y = 0 assert(eval(unparse_ast(a)) == 3)
def func(tree, **kw): if _is_pattern_match_stmt(tree): modified = set() matcher = build_matcher(tree.value.left, modified) # lol random names for hax with q as assignment: xsfvdy = ast(matcher) statements = [ assignment, Expr(q(xsfvdy._match_value(ast(tree.value.right)))) ] for var_name in modified: statements.append( Assign([Name(var_name, Store())], q(xsfvdy.get_var(u(var_name))))) return statements else: return tree
def build_matcher(tree, modified): if isinstance(tree, Num): return q(LiteralMatcher(u(tree.n))) if isinstance(tree, Str): return q(LiteralMatcher(u(tree.s))) if isinstance(tree, Name): if tree.id in ['True', 'False', 'None']: return q(LiteralMatcher(ast(tree))) elif tree.id in ['_']: return q(WildcardMatcher()) modified.add(tree.id) return q(NameMatcher(u(tree.id))) if isinstance(tree, List): sub_matchers = [] for child in tree.elts: sub_matchers.append(build_matcher(child, modified)) return Call(Name('ListMatcher', Load()), sub_matchers, [], None, None) if isinstance(tree, Tuple): sub_matchers = [] for child in tree.elts: sub_matchers.append(build_matcher(child, modified)) return Call(Name('TupleMatcher', Load()), sub_matchers, [], None, None) if isinstance(tree, Call): sub_matchers = [] for child in tree.args: sub_matchers.append(build_matcher(child, modified)) positional_matchers = List(sub_matchers, Load()) kw_matchers = [] for kw in tree.keywords: kw_matchers.append( keyword(kw.arg, build_matcher(kw.value, modified))) return Call(Name('ClassMatcher', Load()), [tree.func, positional_matchers], kw_matchers, None, None) if (isinstance(tree, BinOp) and isinstance(tree.op, BitAnd)): sub1 = build_matcher(tree.left, modified) sub2 = build_matcher(tree.right, modified) return Call(Name('ParallelMatcher', Load()), [sub1, sub2], [], None, None) raise Exception("Unrecognized tree " + repr(tree))
def test_expand_lets(self): """ This tests the sorta knotty logic involved in making the for- comprehension variable available *outside* of the comprehension when used in PINQ """ tree = q(lambda x: x + (lambda y: y + 1)(3))(5) goal = q(lambda x: (lambda y: (x + (y + 1)))(3))(5) new_tree = expand_let_bindings.recurse(tree) assert ast.dump(new_tree) == ast.dump(goal) tree = q(lambda x: x + (lambda y: y + 1)(3) + (lambda z: z + 2)(4))(5) goal = q(lambda x: (lambda z: (lambda y: ((x + (y + 1)) + (z + 2)))(3)) (4))(5) new_tree = expand_let_bindings.recurse(tree) assert ast.dump(new_tree) == ast.dump(goal) tree = q(lambda x: (x, lambda w: (lambda y: y + 1)(3) + (lambda z: z + 2)(4)))(5) goal = q(lambda x: (x, (lambda w: (lambda z: (lambda y: ( (y + 1) + (z + 2)))(3))(4))))(5) new_tree = expand_let_bindings.recurse(tree) assert ast.dump(new_tree) == ast.dump(goal)
def test_show_expanded(self): show_expanded(q(1 + 2)) assert result[-1] == "BinOp(left=Num(n=1), op=Add(), right=Num(n=2))" with show_expanded: a = 1 b = 2 with q as code: print a + u(b + 1) assert result[-3:] == [ '\na = 1', '\nb = 2', "\ncode = [Print(dest=None, values=[BinOp(left=Name(id='a', ctx=Load()), op=Add(), right=ast_repr((b + 1)))], nl=True)]" ]
def f(tree, gen_sym, **kw): @Walker def underscore_search(tree, collect, **kw): if isinstance(tree, Name) and tree.id == "_": name = gen_sym() tree.id = name collect(name) return tree tree, used_names = underscore_search.recurse_real(tree) new_tree = q(lambda: ast(tree)) new_tree.args.args = [Name(id=x) for x in used_names] return new_tree
def s(tree, **kw): captured = [] new_string = "" chunks = re.split("{(.*?)}", tree.s) for i in range(0, len(chunks)): if i % 2 == 0: new_string += chunks[i] else: new_string += "%s" captured += [chunks[i]] result = q(u(new_string) % tuple(ast_list(map(parse_expr, captured)))) return result
def _PegWalker(tree, ctx, stop, collect, **kw): if type(tree) is Str: stop() return q(Parser.Raw(ast(tree))) if type(tree) is BinOp and type(tree.op) is RShift: tree.left, b_left = _PegWalker.recurse_real(tree.left) tree.right = q(lambda bindings: ast(tree.right)) tree.right.args.args = map(f(Name(id=_)), flatten(b_left)) stop() return tree if type(tree) is BinOp and type(tree.op) is FloorDiv: tree.left, b_left = _PegWalker.recurse_real(tree.left) stop() collect(b_left) return tree if type(tree) is Tuple: result = q(Parser.Seq([])) result.args[0].elts = tree.elts all_bindings = [] for i, elt in enumerate(tree.elts): result.args[0].elts[i], bindings = _PegWalker.recurse_real( tree.elts[i]) all_bindings.append(bindings) stop() collect(all_bindings) return result if type(tree) is Compare and type(tree.ops[0]) is Is: left_tree, bindings = _PegWalker.recurse_real(tree.left) new_tree = q(ast(left_tree).bind_to(u(tree.comparators[0].id))) stop() collect(bindings + [tree.comparators[0].id]) return new_tree
def tco(tree, **kw): @Walker # Replace returns of calls def return_replacer(tree, **kw): with switch(tree): if Return(value=Call( func=func, args=args, starargs=starargs, kwargs=kwargs)): with q as code: return (tco.CALL, ast(func), ast(List(args, Load())), ast(starargs or List([], Load())), ast(kwargs or Dict([],[]))) return code else: return tree # Replace calls (that aren't returned) which happen to be in a tail-call # position def replace_tc_pos(node): with switch(node): if Expr(value=Call( func=func, args=args, starargs=starargs, kwargs=kwargs)): with q as code: return (tco.IGNORE, ast(func), ast(List(args, Load())), ast(starargs or List([], Load())), ast(kwargs or Dict([], []))) return code elif If(test=test, body=body, orelse=orelse): body[-1] = replace_tc_pos(body[-1]) if orelse: orelse[-1] = replace_tc_pos(orelse[-1]) return If(test, body, orelse) else: return node tree = return_replacer.recurse(tree) tree.decorator_list = ([q(tco.trampoline_decorator)] + tree.decorator_list) tree.body[-1] = replace_tc_pos(tree.body[-1]) return tree
def _PegWalker(tree, ctx, stop, collect, **kw): if type(tree) is Str: stop() return q(Parser.Raw(ast(tree))) if type(tree) is BinOp and type(tree.op) is RShift: tree.left, b_left = _PegWalker.recurse_real(tree.left) tree.right = q(lambda bindings: ast(tree.right)) tree.right.args.args = map(f(Name(id = _)), flatten(b_left)) stop() return tree if type(tree) is BinOp and type(tree.op) is FloorDiv: tree.left, b_left = _PegWalker.recurse_real(tree.left) stop() collect(b_left) return tree if type(tree) is Tuple: result = q(Parser.Seq([])) result.args[0].elts = tree.elts all_bindings = [] for i, elt in enumerate(tree.elts): result.args[0].elts[i], bindings = _PegWalker.recurse_real(tree.elts[i]) all_bindings.append(bindings) stop() collect(all_bindings) return result if type(tree) is Compare and type(tree.ops[0]) is Is: left_tree, bindings = _PegWalker.recurse_real(tree.left) new_tree = q(ast(left_tree).bind_to(u(tree.comparators[0].id))) stop() collect(bindings + [tree.comparators[0].id]) return new_tree
def f(tree, **kw): names = ('arg' + str(i) for i in xrange(100)) @Walker def underscore_search(tree, collect, **kw): if isinstance(tree, Name) and tree.id == "_": name = names.next() tree.id = name collect(name) return tree tree, used_names = underscore_search.recurse_real(tree) new_tree = q(lambda: ast(tree)) new_tree.args.args = [Name(id=x) for x in used_names] return new_tree
def f(tree, **kw): names = ('arg' + str(i) for i in xrange(100)) @Walker def underscore_search(tree, collect, **kw): if isinstance(tree, Name) and tree.id == "_": name = names.next() tree.id = name collect(name) return tree tree, used_names = underscore_search.recurse_real(tree) new_tree = q(lambda: ast(tree)) new_tree.args.args = [Name(id = x) for x in used_names] return new_tree
def tco(tree, **kw): @Walker # Replace returns of calls def return_replacer(tree, **kw): with switch(tree): if Return(value=Call( func=func, args=args, starargs=starargs, kwargs=kwargs)): with q as code: return (tco.CALL, ast(func), ast(List(args, Load())), ast(starargs or List([], Load())), ast(kwargs or Dict([], []))) return code else: return tree # Replace calls (that aren't returned) which happen to be in a tail-call # position def replace_tc_pos(node): with switch(node): if Expr(value=Call( func=func, args=args, starargs=starargs, kwargs=kwargs)): with q as code: return (tco.IGNORE, ast(func), ast(List(args, Load())), ast(starargs or List([], Load())), ast(kwargs or Dict([], []))) return code elif If(test=test, body=body, orelse=orelse): body[-1] = replace_tc_pos(body[-1]) if orelse: orelse[-1] = replace_tc_pos(orelse[-1]) return If(test, body, orelse) else: return node tree = return_replacer.recurse(tree) tree.decorator_list = ([q(tco.trampoline_decorator)] + tree.decorator_list) tree.body[-1] = replace_tc_pos(tree.body[-1]) return tree
def test_expand_lets(self): """ This tests the sorta knotty logic involved in making the for- comprehension variable available *outside* of the comprehension when used in PINQ """ tree = q(lambda x: x + (lambda y: y + 1)(3))(5) goal = q(lambda x: (lambda y: (x + (y + 1)))(3))(5) new_tree = expand_let_bindings.recurse(tree) assert ast.dump(new_tree) == ast.dump(goal) tree = q(lambda x: x + (lambda y: y + 1)(3) + (lambda z: z + 2)(4))(5) goal = q(lambda x: (lambda z: (lambda y: ((x + (y + 1)) + (z + 2)))(3))(4))(5) new_tree = expand_let_bindings.recurse(tree) assert ast.dump(new_tree) == ast.dump(goal) tree = q(lambda x: (x, lambda w: (lambda y: y + 1)(3) + (lambda z: z + 2)(4)))(5) goal = q(lambda x: (x, (lambda w: (lambda z: (lambda y: ((y + 1) + (z + 2)))(3))(4))))(5) new_tree = expand_let_bindings.recurse(tree) assert ast.dump(new_tree) == ast.dump(goal)
def show_expanded(tree, expand_macros, **kw): expanded_tree = expand_macros(tree) new_tree = q(wrap_simple(log, u(unparse_ast(expanded_tree)), ast(expanded_tree))) return new_tree
def log(tree, exact_src, **kw): new_tree = q(wrap(log, u(exact_src(tree)), ast(tree))) return new_tree
def _require_transform(tree, exact_src): ret = trace_walk.recurse(copy.deepcopy(tree), exact_src) trace_walk.recurse(copy.deepcopy(tree), exact_src) new = q(ast(tree) or handle(lambda log: ast(ret))) return new
def test_unquote_name(self): n = "x" x = 1 y = q(name(n) + name(n)) assert(eval(unparse_ast(y)) == 2)
def pyjs(tree, **kw): javascript = pjs.converter.Converter("").convert_node(tree, Scope()) return q((ast(tree), u(javascript)))
# (f, a1, ..., an) --> f(a1, ..., an) posargs = [x for x in data if not iskwargs(x)] # TODO: tag *args and **kwargs in a kw() as invalid, too (currently just ignored) invalids = list(flatmap(lambda x: x.args, filter(iskwargs, data))) if invalids: assert False, "kw(...) may only specify named args" kwargs = flatmap(lambda x: x.keywords, filter(iskwargs, data)) kwargs = list(rev(uniqify(rev(kwargs), key=lambda x: x.arg))) # latest wins, but keep original ordering return Call(func=op, args=posargs, keywords=list(kwargs)) # This is a first-pass macro. Any nested macros should get clean standard Python, # not having to worry about tuples possibly denoting function calls. yield transform.recurse(block_body, quotelevel=0) # note the exported "q" is ours, but the q we use in this module is a macro. class q: """[syntax] Quote operator. Only meaningful in a tuple in a prefix block.""" def __repr__(self): # in case one of these ends up somewhere at runtime return "<quote>" q = q() class u: """[syntax] Unquote operator. Only meaningful in a tuple in a prefix block.""" def __repr__(self): # in case one of these ends up somewhere at runtime return "<unquote>" u = u() # not a @macro_stub; it only raises a run-time error on foo[...], not foo(...) def kw(**kwargs): """[syntax] Pass-named-args operator. Only meaningful in a tuple in a prefix block.""" raise RuntimeError("kw only meaningful inside a tuple in a prefix block")
def show_expanded(tree, expand_macros, **kw): expanded_tree = expand_macros(tree) new_tree = q( wrap_simple(log, u(unparse_ast(expanded_tree)), ast(expanded_tree))) return new_tree
def expand(tree, **kw): addition = 10 return q(lambda x: x * ast(tree) + u(addition))
def query(tree, **kw): x = _recurse.recurse(tree) x = expand_let_bindings.recurse(x) return q((lambda query: query.bind.execute(query).fetchall())(ast(x)))