def rewrite_tg(env, tg_name, code): """Re-write a transform generating function pipe specification by extracting the transform generating part, and replacing it with the generated transform. so: tgen(a,b,c).foo.bar becomes: tg = tgen(a,b,c) tg.foo.bar """ visitor = ReplaceTG(env, tg_name) assert visitor.tg_name try: tree = visitor.visit(ast.parse(code)) except SyntaxError as e: raise SyntaxError(str(e)+"\nIn code: \n"+code) if visitor.loc: loc = ' #' + visitor.loc else: loc = file_loc() # The AST visitor didn't match a call node if visitor.trans_gen: tg = meta.dump_python_source(visitor.trans_gen).strip() else: tg = None return meta.dump_python_source(tree).strip(), tg, loc
def _debug(cls, method=None): import meta def recurse(x, readers): if isinstance(x, ast.Name) and x.id in readers: return ast.Name(repr(readers[x.id]), ast.Load()) elif isinstance(x, ast.AST): for field in x._fields: setattr(x, field, recurse(getattr(x, field), readers)) return x elif isinstance(x, list): return [recurse(y, readers) for y in x] else: return x if method is not None: source, readers = cls._source[method] recurse(source, readers) source.body[0].name = "{0}.{1}".format(cls.__name__, method) return meta.dump_python_source(source) else: out = "class {0}(Cursor):".format(cls.__name__) for method, (source, readers) in cls._source.items(): recurse(source, readers) source.body[0].name = method if method != "__init__": out += "\n @property" out += meta.dump_python_source(source).replace( "\n", "\n ").rstrip() + "\n" return out
def transform_function(func_tree): assert all(isinstance(t, If) for t in func_tree.body), \ 'Patterns function should only have if statements' # Adjust arglist and decorators func_tree.args.args.append(A('value')) func_tree.decorator_list = [] # Transform tests to pattern matching for test in func_tree.body: cond = test.test if isinstance(cond, (Num, Str, List, Tuple, Dict)) and not has_vars(cond): test.test = make_eq(N('value'), cond) elif isinstance(cond, (Num, Str, Name, Compare, List, Tuple, Dict, BinOp)): tests, assigns = destruct_to_tests_and_assigns(N('value'), cond) test.test = BoolOp(op=And(), values=tests) if len(tests) > 1 else \ tests[0] if tests else V(1) test.body = assigns + test.body else: raise TypeError("Don't know how to match %s" % meta.dump_python_source(cond).strip()) func_tree.body = lmap(wrap_tail_expr, func_tree.body) func_tree.body.append(make_raise(N('Mismatch'))) # Set raise Mismatch lineno just after function end func_tree.body[-1].lineno = last_lineno(func_tree) + 1
def destruct_to_tests_and_assigns(topic, pattern): if isinstance(pattern, (Num, Str)): return [make_eq(topic, pattern)], [] elif isinstance(pattern, Name): return [], [make_assign(pattern.id, topic)] elif isinstance(pattern, Compare) and len(pattern.ops) == 1 and isinstance(pattern.ops[0], Is): return [make_call('isinstance', topic, pattern.comparators[0])], \ [make_assign(pattern.left.id, topic)] elif isinstance(pattern, (List, Tuple, Dict)): elts = getattr(pattern, 'elts', []) or getattr(pattern, 'values', []) coll_tests = [ make_call('isinstance', topic, N(pattern.__class__.__name__.lower())), make_eq(make_call('len', topic), len(elts)) ] tests, assigns = subscript_tests_and_assigns(topic, pattern) return coll_tests + tests, assigns elif isinstance(pattern, BinOp) and isinstance(pattern.op, Add) \ and isinstance(pattern.left, (List, Tuple)) and isinstance(pattern.right, Name): coll_tests = [ make_call('isinstance', topic, N(pattern.left.__class__.__name__.lower())), make_op(GtE, make_call('len', topic), len(pattern.left.elts)), ] coll_assigns = [ make_assign(pattern.right.id, Subscript(ctx=Load(), value=topic, slice=Slice(lower=V(len(pattern.left.elts)), upper=None, step=None))) ] tests, assigns = subscript_tests_and_assigns(topic, pattern.left) return coll_tests + tests, assigns + coll_assigns else: raise TypeError("Don't know how to match %s" % meta.dump_python_source(pattern).strip())
def transform_function(func_tree): assert all(isinstance(t, If) for t in func_tree.body), \ 'Patterns function should only have if statements' # Adjust arglist and decorators func_tree.args.args.append(Name(ctx=Param(), id='value')) func_tree.decorator_list = [] # Transform tests to pattern matching for test in func_tree.body: cond = test.test if isinstance(cond, (Num, Str, List, Tuple)) and not has_vars(cond): test.test = make_eq(N('value'), cond) if isinstance(cond, (Num, Str, Name, Compare, List, Tuple, Dict, BinOp)): tests, assigns = destruct_to_tests_and_assigns(N('value'), cond) test.test = BoolOp(op=And(), values=tests) if tests else V(1) test.body = assigns + test.body else: raise TypeError("Don't know how to match %s" % meta.dump_python_source(cond).strip()) func_tree.body = map(wrap_tail_expr, func_tree.body) func_tree.body.append(Raise(type=N('Mismatch'), inst=None, tback=None))
def resolve(node): if isinstance(node, ast.Attribute): return getattr(resolve(node.value), node.attr, None) elif isinstance(node, ast.Name): return env.get(node.id, None) else: raise ExpressionError( "functions must be named, not constructed: {0}".format( meta.dump_python_source(node).strip()))
def parse_ast(cls, t): # This is done very naively result = '' for __x in t: frame = inspect.currentframe() if isinstance(__x, ast.Expr): result += cls.parse_node(__x, frame=frame) else: exec(meta.dump_python_source(__x)) return result
def decompile(code, version="2.7"): import meta from uncompyle2 import uncompyle try: source = StringIO.StringIO() uncompyle(version, code, source) source = source.getvalue() except: try: code_obj = meta.decompile(code) source = meta.dump_python_source(code_obj) except: return None return source.lstrip(' \n')
def toSource(x): return meta.dump_python_source(x).strip()
def translate_toplevel(self, commands): main = filter(lambda x: isinstance(x, ast.FunctionDef) and x.name == 'main', self.template.body)[0] main.body.extend(flatten(self.translate_commands(commands))) substassign = filter(lambda x: isinstance(x, ast.Assign) and x.targets[0].id == 'SUBSTS', self.template.body)[0] substassign.value.args = [ast.List([ast.Str(s) for s in self.macro_handler.substs], ast.Load())] make_arg_parser = filter(lambda x: isinstance(x, ast.FunctionDef) and x.name == 'make_arg_parser', self.template.body)[0] make_arg_parser.body[-1:-1] = self.make_argparse_arguments() p = Parser(sys.stdin.read()) p.changequote('[',']') macro_handler = MacroHandler() macro_handler.add_macros(MACROS, p) stream = StringIO() # Parse m4 p.parse(stream=stream) shell = stream.getvalue() if len(sys.argv) > 1: sys.stdout.write(shell) sys.exit(0) # now translate shell to Python template_file = os.path.join(os.path.dirname(__file__), 'template.py') template = ast.parse(open(template_file, 'r').read()) translator = ShellTranslator(macro_handler, template) translator.translate(shell, toplevel=True) sys.stdout.write(meta.dump_python_source(template))
def pyc_source(pyc_contents): code_section = pyc_contents[8:] code = marshal.load(code_section) return meta.dump_python_source(meta.decompile(code))
def __repr__(self): return self.complexity() + dump_python_source(self.ast)
def parse(expression, defs=None, returnlabel=False): """ Convert a lambda or string ``expression`` into a :py:class:`Expr <histbook.expr.Expr>`. Parameters ---------- expression : lambda or string expression involving constants (literals and ``pi``, ``e``, ``inf``, and ``nan``), field variables, and functions in ``Expr.recognized``; if a lambda, the fields are given by function parameters, if a string, any unrecognized names will be identified as fields defs : ``None`` or dict if not ``None``, provides names to recognize in ``expression`` (to avoid repetitive code) returnlabel : bool if ``True``, return value is a 2-tuple: :py:class:`Expr <histbook.expr.Expr>` and string representing the expression; otherwise *(default)*, just the :py:class:`Expr <histbook.expr.Expr>` """ _defs = { "pi": Const(math.pi), "e": Const(math.e), "inf": Const(float("inf")), "nan": Const(float("nan")) } if defs is not None: for n, x in defs.items(): if isinstance(x, Expr): _defs[n] = x elif sys.version_info[0] < 3 and isinstance(x, (str, unicode)): _defs[n] = Expr.parse(x) elif sys.version_info[0] >= 3 and isinstance(x, str): _defs[n] = Expr.parse(x) else: try: _defs[n] = Const(pickle.loads(pickle.dumps(x))) except: raise ExpressionError( "object can't be included in an expression as symbol {0} because it refers to an unserializable object" .format(repr(n))) pyast = None label = None params = None if isinstance(expression, types.FunctionType): # more specific than callable(...) fcn = meta.decompiler.decompile_func(expression) if isinstance(fcn, ast.FunctionDef) and len( fcn.body) == 1 and isinstance(fcn.body[0], ast.Return): pyast = fcn.body[0].value label = expression.__name__ elif isinstance(fcn, ast.Lambda): pyast = fcn.body.value label = meta.dump_python_source(pyast).strip() params = expression.__code__.co_varnames[:expression.__code__. co_argcount] elif (sys.version_info[0] < 3 and isinstance( expression, basestring)) or (sys.version_info[0] >= 3 and isinstance(expression, str)): mod = ast.parse(expression) if len(mod.body) == 1 and isinstance(mod.body[0], ast.Expr): pyast = mod.body[0].value label = expression if pyast is None: raise TypeError( "expression must be a one-line string, one-line function, or lambda expression, not {0}" .format(repr(expression))) calculate = { "+": lambda x, y: x + y, "-": lambda x, y: x - y, "*": lambda x, y: x * y, "/": lambda x, y: float(x) / float(y), "//": lambda x, y: int(x // y), "%": lambda x, y: x % y, "**": lambda x, y: x**y, "|": lambda x, y: numpy.uint64(x) | numpy.uint64(y), "&": lambda x, y: numpy.uint64(x) & numpy.uint64(y), "^": lambda x, y: numpy.uint64(x) ^ numpy.uint64(y) } env = dict(globals()) def resolve(node): if isinstance(node, ast.Attribute): return getattr(resolve(node.value), node.attr, None) elif isinstance(node, ast.Name): return env.get(node.id, None) else: raise ExpressionError( "functions must be named, not constructed: {0}".format( meta.dump_python_source(node).strip())) def recurse(node, relations=False): if isinstance(node, ast.Num): return Const(node.n) elif isinstance(node, ast.Str): return Const(node.s) elif isinstance(node, ast.Dict) and len(node.keys) == 0: return Const(set()) elif isinstance(node, ast.Set): content = [recurse(x) for x in node.elts] if all(isinstance(x, Const) for x in content): return Const(set(x.value for x in content)) else: raise ExpressionError( "sets in expressions may not contain variable contents: {0}" .format(meta.dump_python_source(node).strip())) elif isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load): if node.id == "None": return Const(None) elif node.id == "True": return Const(True) elif node.id == "False": return Const(False) elif node.id in _defs: return _defs[node.id] elif params is not None and node.id not in params: if node.id in env: try: return Const( pickle.loads(pickle.dumps(env[node.id]))) except: raise ExpressionError( "symbol {0} can't be included in an expression because it refers to an unserializable object in the global scope" .format(repr(node.id))) else: raise ExpressionError( "symbol {0} is not a function parameter and not in the global scope" .format(repr(node.id))) else: return Name(node.id) elif isinstance(node, getattr(ast, "NameConstant", tuple)): # node will never be tuple if node.value is None: return Const(None) elif node.value is True: return Const(True) elif node.value is False: return Const(False) else: raise AssertionError(node) elif relations and isinstance(node, ast.Compare) and len( node.ops) == 1: if isinstance(node.ops[0], ast.Eq): cmp, swap = "==", None elif isinstance(node.ops[0], ast.NotEq): cmp, swap = "!=", None elif isinstance(node.ops[0], ast.Lt): cmp, swap = "<", False elif isinstance(node.ops[0], ast.LtE): cmp, swap = "<=", False elif isinstance(node.ops[0], ast.Gt): cmp, swap = "<", True elif isinstance(node.ops[0], ast.GtE): cmp, swap = "<=", True elif isinstance(node.ops[0], ast.In): cmp, swap = "in", False elif isinstance(node.ops[0], ast.NotIn): cmp, swap = "not in", False else: raise ExpressionError( "only comparision relations supported: '==', '!=', '<', '<=', '>', '>=', 'in', and 'not in': {0}" .format(meta.dump_python_source(node).strip())) left = recurse(node.left) right = recurse(node.comparators[0]) if swap is True: left, right = right, left elif swap is None: left, right = sorted([left, right]) if (cmp == "in" or cmp == "not in") and not (isinstance( right, Const) and isinstance(right.value, set)): raise ExpressionError( "comparisons 'in' and 'not in' can only be used with a set: {0}" .format(meta.dump_python_source(node).strip())) return Relation(cmp, left, right) elif relations and isinstance(node, ast.Compare) and len(node.ops) > 1: return recurse(ast.BoolOp(ast.And(), [ ast.Compare( node.left if i == 0 else node.comparators[i - 1], [node.ops[i]], [node.comparators[i]]) for i in range(len(node.ops)) ]), relations=True) elif isinstance(node, ast.Compare): raise ExpressionError( "comparison operators are only allowed at the top of an expression: {0}" .format(meta.dump_python_source(node).strip())) elif relations and isinstance(node, ast.UnaryOp) and isinstance( node.op, ast.Not): content = recurse(node.operand, relations=True) if isinstance(content, Name): return Predicate(content.value, positive=False) else: return Logical.negate(content).simplify() elif relations and isinstance(node, ast.BoolOp) and isinstance( node.op, ast.And): return functools.reduce(LogicalAnd.combine, [ Logical.normalform(recurse(x, relations=True)) for x in node.values ]).simplify() elif relations and isinstance(node, ast.BoolOp) and isinstance( node.op, ast.Or): return functools.reduce(LogicalOr.combine, [ Logical.normalform(recurse(x, relations=True)) for x in node.values ]).simplify() elif isinstance(node, ast.BoolOp): raise ExpressionError( "logical operators are only allowed at the top of an expression: {0}" .format(meta.dump_python_source(node).strip())) elif isinstance(node, ast.UnaryOp) and isinstance( node.op, ast.USub): content = recurse(node.operand) if isinstance(content, Const): return Const(-content.value) else: return PlusMinus.negate(content).simplify() elif isinstance(node, ast.UnaryOp) and isinstance( node.op, ast.UAdd): return recurse(node.operand) elif isinstance(node, ast.UnaryOp) and isinstance( node.op, ast.Invert): content = recurse(node.operand) if isinstance(content, Const): return Const(~content.value) else: return BitAnd.negate(content).simplify() elif isinstance(node, ast.UnaryOp): raise ExpressionError( "only unary operators supported: 'not', '-', '+', and '~': {0}" .format(meta.dump_python_source(node).strip())) elif isinstance(node, ast.BinOp): if isinstance(node.op, ast.Add): fcn = "+" elif isinstance(node.op, ast.Sub): fcn = "-" elif isinstance(node.op, ast.Mult): fcn = "*" elif isinstance(node.op, ast.Div): fcn = "/" elif isinstance(node.op, ast.FloorDiv): op, fcn = "//", "floor_divide" elif isinstance(node.op, ast.Mod): op, fcn = "%", "mod" elif isinstance(node.op, ast.Pow): op, fcn = "**", "pow" elif isinstance(node.op, ast.BitOr): fcn = "|" elif isinstance(node.op, ast.BitAnd): fcn = "&" elif isinstance(node.op, ast.BitXor): op, fcn = "^", "xor" else: raise ExpressionError( "only binary operators supported: '+', '-', '*', '/', '//', '%', '**', '|', '&', and '^': {0}" .format(meta.dump_python_source(node).strip())) left = recurse(node.left) right = recurse(node.right) if isinstance(left, Const) and isinstance(right, Const): return Const(calculate[fcn](left.value, right.value)) if fcn == "+": return PlusMinus.combine(left, right).simplify() elif fcn == "-": return PlusMinus.combine( left, PlusMinus.negate(right)).simplify() elif fcn == "*": return PlusMinus.distribute(left, right).simplify() elif fcn == "/": # PlusMinus is implemented as a RingAlgebra, but it's really a Field right = PlusMinus.normalform(right) if len(right.pos) == len(right.neg) == 0: negation = TimesDiv(TimesDiv.negateval(right.const), (), ()) elif right.const == PlusMinus.identity and len( right.pos) == 1 and len(right.neg) == 0: negation = TimesDiv.negate(right.pos[0]) elif right.const == PlusMinus.identity and len( right.pos) == 0 and len(right.neg) == 1: negation = TimesDiv.negate(right.neg[0]) negation.const = PlusMinus.negateval(negation.const) else: negation = TimesDiv.negate( right) # additive terms in the denominator return PlusMinus.distribute(left, negation).simplify() elif fcn == "pow" and isinstance( right, Const) and right.value == round( right.value) and 1 <= right.value < 5: n = int(right.value) left = PlusMinus.normalform(left) if left.const == PlusMinus.identity and len( left.pos) == 1 and len(left.neg) == 0: return PlusMinus( PlusMinus.identity, (TimesDiv(left.pos[0].const**n, tuple(sorted(left.pos[0].pos * n)), tuple(sorted(left.pos[0].neg * n))), ), ()).simplify() elif left.const == PlusMinus.identity and len( left.pos) == 0 and len( left.neg) == 1 and n % 2 == 0: return PlusMinus( PlusMinus.identity, (TimesDiv(left.neg[0].const**n, tuple(sorted(left.neg[0].pos * n)), tuple(sorted(left.neg[0].neg * n))), ), ()).simplify() elif left.const == PlusMinus.identity and len( left.pos) == 0 and len(left.neg) == 1: return PlusMinus(PlusMinus.identity, (), (TimesDiv( left.neg[0].const**n, tuple(sorted(left.neg[0].pos * n)), tuple(sorted(left.neg[0].neg * n))), )).simplify() else: return PlusMinus( PlusMinus.identity, (TimesDiv(TimesDiv.identity, tuple((left, ) * n), ()), ), ()).simplify() elif fcn == "pow" and isinstance( right, Const) and right.value == round( right.value) and -5 < right.value <= -1: n = -int(right.value) left = PlusMinus.normalform(left) if left.const == PlusMinus.identity and len( left.pos) == 1 and len(left.neg) == 0: return PlusMinus( PlusMinus.identity, (TimesDiv(left.pos[0].const**(-n), tuple(sorted(left.pos[0].neg * n)), tuple(sorted(left.pos[0].pos * n))), ), ()).simplify() elif left.const == PlusMinus.identity and len( left.pos) == 0 and len( left.neg) == 1 and n % 2 == 0: return PlusMinus( PlusMinus.identity, (TimesDiv(left.neg[0].const**(-n), tuple(sorted(left.neg[0].neg * n)), tuple(sorted(left.neg[0].pos * n))), ), ()).simplify() elif left.const == PlusMinus.identity and len( left.pos) == 0 and len(left.neg) == 1: return PlusMinus(PlusMinus.identity, (), (TimesDiv( left.neg[0].const**(-n), tuple(sorted(left.neg[0].neg * n)), tuple(sorted(left.neg[0].pos * n))), )).simplify() else: return PlusMinus(PlusMinus.identity, (TimesDiv(TimesDiv.identity, (), tuple((left, ) * n)), ), ()).simplify() elif fcn == "&": raise NotImplementedError elif fcn == "|": raise NotImplementedError else: return BinOp(fcn, left, right, op) elif isinstance(node, ast.Call): fcn = Expr.recognized.get(resolve(node.func), None) if fcn is None and node.func.id in Expr.recognized.values(): fcn = node.func.id elif fcn is not None: for n, x in Expr.recognized.items(): if fcn is n: fcn = x break if fcn is None: raise ExpressionError( "unhandled function in expression: {0}".format( meta.dump_python_source(node).strip())) return Call(fcn, *(recurse(x) for x in node.args)) else: ExpressionError("unhandled syntax in expression: {0}".format( meta.dump_python_source(node).strip())) if returnlabel: return recurse(pyast, relations=True), label else: return recurse(pyast, relations=True)
def recurse(node, relations=False): if isinstance(node, ast.Num): return Const(node.n) elif isinstance(node, ast.Str): return Const(node.s) elif isinstance(node, ast.Dict) and len(node.keys) == 0: return Const(set()) elif isinstance(node, ast.Set): content = [recurse(x) for x in node.elts] if all(isinstance(x, Const) for x in content): return Const(set(x.value for x in content)) else: raise ExpressionError( "sets in expressions may not contain variable contents: {0}" .format(meta.dump_python_source(node).strip())) elif isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load): if node.id == "None": return Const(None) elif node.id == "True": return Const(True) elif node.id == "False": return Const(False) elif node.id in _defs: return _defs[node.id] elif params is not None and node.id not in params: if node.id in env: try: return Const( pickle.loads(pickle.dumps(env[node.id]))) except: raise ExpressionError( "symbol {0} can't be included in an expression because it refers to an unserializable object in the global scope" .format(repr(node.id))) else: raise ExpressionError( "symbol {0} is not a function parameter and not in the global scope" .format(repr(node.id))) else: return Name(node.id) elif isinstance(node, getattr(ast, "NameConstant", tuple)): # node will never be tuple if node.value is None: return Const(None) elif node.value is True: return Const(True) elif node.value is False: return Const(False) else: raise AssertionError(node) elif relations and isinstance(node, ast.Compare) and len( node.ops) == 1: if isinstance(node.ops[0], ast.Eq): cmp, swap = "==", None elif isinstance(node.ops[0], ast.NotEq): cmp, swap = "!=", None elif isinstance(node.ops[0], ast.Lt): cmp, swap = "<", False elif isinstance(node.ops[0], ast.LtE): cmp, swap = "<=", False elif isinstance(node.ops[0], ast.Gt): cmp, swap = "<", True elif isinstance(node.ops[0], ast.GtE): cmp, swap = "<=", True elif isinstance(node.ops[0], ast.In): cmp, swap = "in", False elif isinstance(node.ops[0], ast.NotIn): cmp, swap = "not in", False else: raise ExpressionError( "only comparision relations supported: '==', '!=', '<', '<=', '>', '>=', 'in', and 'not in': {0}" .format(meta.dump_python_source(node).strip())) left = recurse(node.left) right = recurse(node.comparators[0]) if swap is True: left, right = right, left elif swap is None: left, right = sorted([left, right]) if (cmp == "in" or cmp == "not in") and not (isinstance( right, Const) and isinstance(right.value, set)): raise ExpressionError( "comparisons 'in' and 'not in' can only be used with a set: {0}" .format(meta.dump_python_source(node).strip())) return Relation(cmp, left, right) elif relations and isinstance(node, ast.Compare) and len(node.ops) > 1: return recurse(ast.BoolOp(ast.And(), [ ast.Compare( node.left if i == 0 else node.comparators[i - 1], [node.ops[i]], [node.comparators[i]]) for i in range(len(node.ops)) ]), relations=True) elif isinstance(node, ast.Compare): raise ExpressionError( "comparison operators are only allowed at the top of an expression: {0}" .format(meta.dump_python_source(node).strip())) elif relations and isinstance(node, ast.UnaryOp) and isinstance( node.op, ast.Not): content = recurse(node.operand, relations=True) if isinstance(content, Name): return Predicate(content.value, positive=False) else: return Logical.negate(content).simplify() elif relations and isinstance(node, ast.BoolOp) and isinstance( node.op, ast.And): return functools.reduce(LogicalAnd.combine, [ Logical.normalform(recurse(x, relations=True)) for x in node.values ]).simplify() elif relations and isinstance(node, ast.BoolOp) and isinstance( node.op, ast.Or): return functools.reduce(LogicalOr.combine, [ Logical.normalform(recurse(x, relations=True)) for x in node.values ]).simplify() elif isinstance(node, ast.BoolOp): raise ExpressionError( "logical operators are only allowed at the top of an expression: {0}" .format(meta.dump_python_source(node).strip())) elif isinstance(node, ast.UnaryOp) and isinstance( node.op, ast.USub): content = recurse(node.operand) if isinstance(content, Const): return Const(-content.value) else: return PlusMinus.negate(content).simplify() elif isinstance(node, ast.UnaryOp) and isinstance( node.op, ast.UAdd): return recurse(node.operand) elif isinstance(node, ast.UnaryOp) and isinstance( node.op, ast.Invert): content = recurse(node.operand) if isinstance(content, Const): return Const(~content.value) else: return BitAnd.negate(content).simplify() elif isinstance(node, ast.UnaryOp): raise ExpressionError( "only unary operators supported: 'not', '-', '+', and '~': {0}" .format(meta.dump_python_source(node).strip())) elif isinstance(node, ast.BinOp): if isinstance(node.op, ast.Add): fcn = "+" elif isinstance(node.op, ast.Sub): fcn = "-" elif isinstance(node.op, ast.Mult): fcn = "*" elif isinstance(node.op, ast.Div): fcn = "/" elif isinstance(node.op, ast.FloorDiv): op, fcn = "//", "floor_divide" elif isinstance(node.op, ast.Mod): op, fcn = "%", "mod" elif isinstance(node.op, ast.Pow): op, fcn = "**", "pow" elif isinstance(node.op, ast.BitOr): fcn = "|" elif isinstance(node.op, ast.BitAnd): fcn = "&" elif isinstance(node.op, ast.BitXor): op, fcn = "^", "xor" else: raise ExpressionError( "only binary operators supported: '+', '-', '*', '/', '//', '%', '**', '|', '&', and '^': {0}" .format(meta.dump_python_source(node).strip())) left = recurse(node.left) right = recurse(node.right) if isinstance(left, Const) and isinstance(right, Const): return Const(calculate[fcn](left.value, right.value)) if fcn == "+": return PlusMinus.combine(left, right).simplify() elif fcn == "-": return PlusMinus.combine( left, PlusMinus.negate(right)).simplify() elif fcn == "*": return PlusMinus.distribute(left, right).simplify() elif fcn == "/": # PlusMinus is implemented as a RingAlgebra, but it's really a Field right = PlusMinus.normalform(right) if len(right.pos) == len(right.neg) == 0: negation = TimesDiv(TimesDiv.negateval(right.const), (), ()) elif right.const == PlusMinus.identity and len( right.pos) == 1 and len(right.neg) == 0: negation = TimesDiv.negate(right.pos[0]) elif right.const == PlusMinus.identity and len( right.pos) == 0 and len(right.neg) == 1: negation = TimesDiv.negate(right.neg[0]) negation.const = PlusMinus.negateval(negation.const) else: negation = TimesDiv.negate( right) # additive terms in the denominator return PlusMinus.distribute(left, negation).simplify() elif fcn == "pow" and isinstance( right, Const) and right.value == round( right.value) and 1 <= right.value < 5: n = int(right.value) left = PlusMinus.normalform(left) if left.const == PlusMinus.identity and len( left.pos) == 1 and len(left.neg) == 0: return PlusMinus( PlusMinus.identity, (TimesDiv(left.pos[0].const**n, tuple(sorted(left.pos[0].pos * n)), tuple(sorted(left.pos[0].neg * n))), ), ()).simplify() elif left.const == PlusMinus.identity and len( left.pos) == 0 and len( left.neg) == 1 and n % 2 == 0: return PlusMinus( PlusMinus.identity, (TimesDiv(left.neg[0].const**n, tuple(sorted(left.neg[0].pos * n)), tuple(sorted(left.neg[0].neg * n))), ), ()).simplify() elif left.const == PlusMinus.identity and len( left.pos) == 0 and len(left.neg) == 1: return PlusMinus(PlusMinus.identity, (), (TimesDiv( left.neg[0].const**n, tuple(sorted(left.neg[0].pos * n)), tuple(sorted(left.neg[0].neg * n))), )).simplify() else: return PlusMinus( PlusMinus.identity, (TimesDiv(TimesDiv.identity, tuple((left, ) * n), ()), ), ()).simplify() elif fcn == "pow" and isinstance( right, Const) and right.value == round( right.value) and -5 < right.value <= -1: n = -int(right.value) left = PlusMinus.normalform(left) if left.const == PlusMinus.identity and len( left.pos) == 1 and len(left.neg) == 0: return PlusMinus( PlusMinus.identity, (TimesDiv(left.pos[0].const**(-n), tuple(sorted(left.pos[0].neg * n)), tuple(sorted(left.pos[0].pos * n))), ), ()).simplify() elif left.const == PlusMinus.identity and len( left.pos) == 0 and len( left.neg) == 1 and n % 2 == 0: return PlusMinus( PlusMinus.identity, (TimesDiv(left.neg[0].const**(-n), tuple(sorted(left.neg[0].neg * n)), tuple(sorted(left.neg[0].pos * n))), ), ()).simplify() elif left.const == PlusMinus.identity and len( left.pos) == 0 and len(left.neg) == 1: return PlusMinus(PlusMinus.identity, (), (TimesDiv( left.neg[0].const**(-n), tuple(sorted(left.neg[0].neg * n)), tuple(sorted(left.neg[0].pos * n))), )).simplify() else: return PlusMinus(PlusMinus.identity, (TimesDiv(TimesDiv.identity, (), tuple((left, ) * n)), ), ()).simplify() elif fcn == "&": raise NotImplementedError elif fcn == "|": raise NotImplementedError else: return BinOp(fcn, left, right, op) elif isinstance(node, ast.Call): fcn = Expr.recognized.get(resolve(node.func), None) if fcn is None and node.func.id in Expr.recognized.values(): fcn = node.func.id elif fcn is not None: for n, x in Expr.recognized.items(): if fcn is n: fcn = x break if fcn is None: raise ExpressionError( "unhandled function in expression: {0}".format( meta.dump_python_source(node).strip())) return Call(fcn, *(recurse(x) for x in node.args)) else: ExpressionError("unhandled syntax in expression: {0}".format( meta.dump_python_source(node).strip()))
import ast import meta import astpp code = open('sample_code.py', 'r').read() tree = ast.parse(code) #Get source source = meta.dump_python_source(tree) #Get formatted AST repr formatted_ast = astpp.dump(tree, include_attributes=True) class FuncLister(ast.NodeVisitor): def visit_FunctionDef(self, node): print node.name self.generic_visit(node) print 'Functions:' FuncLister().visit(tree)
def __repr__(self): return "#%s%s"%(hash(self),dump_python_source(self.ast))
def string(expr): """Stringify an ast""" import meta return meta.dump_python_source(expr).strip()
from ast import * from astpp import dump from meta import dump_python_source class RewriteOps(NodeTransformer): _names = {Add: '__add__', Sub: '__sub__', Mult: '__mul__', Div: '__truediv__', FloorDiv: '__floordiv__', Mod: '__mod__', Pow: '__pow__'} def visit_BinOp(self, node): name = self._names.get(node.op.__class__, None) if name is not None: return copy_location(Call(func=Attribute(value=node.left, attr=name, ctx=Load()),args=[node.right], keywords=[], starargs=[], kwargs=[]),node) else: self.generic_visit(node) source = '"Hello " + "World!"' print(source,'\n') node = parse(source) print(dump(node),'\n') node = RewriteOps().visit(node) print(dump(node)) source2 = dump_python_source(node) print(source2) print(eval(source)) print(eval(source2))
def statement(node): if isinstance(node, list): out = [] for x in node: t = ("\n" + meta.dump_python_source(x).strip()).replace("\n", "\n ") out.append(makeyield(len(text))) text.append(t) out.append(statement(x)) node = out elif isinstance(node, ast.Module): node.body = statement(node.body) elif isinstance(node, (ast.ClassDef, ast.Exec, ast.FunctionDef, ast.Import, ast.Return)): raise SyntaxError("{0} statements are not allowed".format(node.__class__.__text__)) elif isinstance(node, ast.Expr): node.value = expression(node.value) elif isinstance(node, ast.Assert): node.test = expression(node.test) elif isinstance(node, ast.Assign): node.targets = [expression(x) for x in node.targets] node.value = expression(node.value) elif isinstance(node, ast.AugAssign): node.target = expression(node.target) node.value = expression(node.value) elif isinstance(node, ast.Delete): node.targets = [expression(x) for x in node.targets] elif isinstance(node, ast.ExceptHandler): node.body = statement(node.body) elif isinstance(node, ast.For): node.iter = expression(node.iter) node.body = statement(node.body) node.orelse = statement(node.orelse) elif isinstance(node, ast.If): node.test = expression(node.test) node.body = statement(node.body) node.orelse = statement(node.orelse) elif isinstance(node, ast.Print): node.values = expression(node.values) elif isinstance(node, ast.TryExcept): node.body = statement(node.body) node.orelse = statement(node.orelse) elif isinstance(node, ast.TryFinally): node.body = statement(node.body) node.finalbody = statement(node.finalbody) elif isinstance(node, ast.Try): node.body = statement(node.body) node.orelse = statement(node.orelse) node.finalbody = statement(node.finalbody) elif isinstance(node, ast.While): node.test = expression(node.test) node.body = statement(node.body) node.orelse = statement(node.orelse) elif isinstance(node, ast.With): node.body = statement(node.body) if hasattr(node, "items"): for x in node.items: x.context_expr = expression(x.context_expr) else: node.context_expr = expression(node.context_expr) return node
def ast_to_source_meta(node): import meta return meta.dump_python_source(node).strip()