def rewrite_tg(env, tg_name, code):
"""Re-write a transform generating function pipe specification by extracting the transform generating part,
and replacing it with the generated transform. so:
tgen(a,b,c).foo.bar
becomes:
tg = tgen(a,b,c)
tg.foo.bar
"""
visitor = ReplaceTG(env, tg_name)
assert visitor.tg_name
try:
tree = visitor.visit(ast.parse(code))
except SyntaxError as e:
raise SyntaxError(str(e)+"\nIn code: \n"+code)
if visitor.loc:
loc = ' #' + visitor.loc
else:
loc = file_loc() # The AST visitor didn't match a call node
if visitor.trans_gen:
tg = meta.dump_python_source(visitor.trans_gen).strip()
else:
tg = None
return meta.dump_python_source(tree).strip(), tg, loc
def _debug(cls, method=None):
import meta
def recurse(x, readers):
if isinstance(x, ast.Name) and x.id in readers:
return ast.Name(repr(readers[x.id]), ast.Load())
elif isinstance(x, ast.AST):
for field in x._fields:
setattr(x, field, recurse(getattr(x, field), readers))
return x
elif isinstance(x, list):
return [recurse(y, readers) for y in x]
else:
return x
if method is not None:
source, readers = cls._source[method]
recurse(source, readers)
source.body[0].name = "{0}.{1}".format(cls.__name__, method)
return meta.dump_python_source(source)
else:
out = "class {0}(Cursor):".format(cls.__name__)
for method, (source, readers) in cls._source.items():
recurse(source, readers)
source.body[0].name = method
if method != "__init__":
out += "\n @property"
out += meta.dump_python_source(source).replace(
"\n", "\n ").rstrip() + "\n"
return out
def transform_function(func_tree):
assert all(isinstance(t, If) for t in func_tree.body), \
'Patterns function should only have if statements'
# Adjust arglist and decorators
func_tree.args.args.append(A('value'))
func_tree.decorator_list = []
# Transform tests to pattern matching
for test in func_tree.body:
cond = test.test
if isinstance(cond, (Num, Str, List, Tuple, Dict)) and not has_vars(cond):
test.test = make_eq(N('value'), cond)
elif isinstance(cond, (Num, Str, Name, Compare, List, Tuple, Dict, BinOp)):
tests, assigns = destruct_to_tests_and_assigns(N('value'), cond)
test.test = BoolOp(op=And(), values=tests) if len(tests) > 1 else \
tests[0] if tests else V(1)
test.body = assigns + test.body
else:
raise TypeError("Don't know how to match %s" % meta.dump_python_source(cond).strip())
func_tree.body = lmap(wrap_tail_expr, func_tree.body)
func_tree.body.append(make_raise(N('Mismatch')))
# Set raise Mismatch lineno just after function end
func_tree.body[-1].lineno = last_lineno(func_tree) + 1
def destruct_to_tests_and_assigns(topic, pattern):
if isinstance(pattern, (Num, Str)):
return [make_eq(topic, pattern)], []
elif isinstance(pattern, Name):
return [], [make_assign(pattern.id, topic)]
elif isinstance(pattern, Compare) and len(pattern.ops) == 1 and isinstance(pattern.ops[0], Is):
return [make_call('isinstance', topic, pattern.comparators[0])], \
[make_assign(pattern.left.id, topic)]
elif isinstance(pattern, (List, Tuple, Dict)):
elts = getattr(pattern, 'elts', []) or getattr(pattern, 'values', [])
coll_tests = [
make_call('isinstance', topic, N(pattern.__class__.__name__.lower())),
make_eq(make_call('len', topic), len(elts))
]
tests, assigns = subscript_tests_and_assigns(topic, pattern)
return coll_tests + tests, assigns
elif isinstance(pattern, BinOp) and isinstance(pattern.op, Add) \
and isinstance(pattern.left, (List, Tuple)) and isinstance(pattern.right, Name):
coll_tests = [
make_call('isinstance', topic, N(pattern.left.__class__.__name__.lower())),
make_op(GtE, make_call('len', topic), len(pattern.left.elts)),
]
coll_assigns = [
make_assign(pattern.right.id,
Subscript(ctx=Load(),
value=topic,
slice=Slice(lower=V(len(pattern.left.elts)), upper=None, step=None)))
]
tests, assigns = subscript_tests_and_assigns(topic, pattern.left)
return coll_tests + tests, assigns + coll_assigns
else:
raise TypeError("Don't know how to match %s" % meta.dump_python_source(pattern).strip())
def transform_function(func_tree):
assert all(isinstance(t, If) for t in func_tree.body), \
'Patterns function should only have if statements'
# Adjust arglist and decorators
func_tree.args.args.append(Name(ctx=Param(), id='value'))
func_tree.decorator_list = []
# Transform tests to pattern matching
for test in func_tree.body:
cond = test.test
if isinstance(cond, (Num, Str, List, Tuple)) and not has_vars(cond):
test.test = make_eq(N('value'), cond)
if isinstance(cond, (Num, Str, Name, Compare, List, Tuple, Dict, BinOp)):
tests, assigns = destruct_to_tests_and_assigns(N('value'), cond)
test.test = BoolOp(op=And(), values=tests) if tests else V(1)
test.body = assigns + test.body
else:
raise TypeError("Don't know how to match %s" % meta.dump_python_source(cond).strip())
func_tree.body = map(wrap_tail_expr, func_tree.body)
func_tree.body.append(Raise(type=N('Mismatch'), inst=None, tback=None))
def resolve(node):
if isinstance(node, ast.Attribute):
return getattr(resolve(node.value), node.attr, None)
elif isinstance(node, ast.Name):
return env.get(node.id, None)
else:
raise ExpressionError(
"functions must be named, not constructed: {0}".format(
meta.dump_python_source(node).strip()))
def parse_ast(cls, t):
# This is done very naively
result = ''
for __x in t:
frame = inspect.currentframe()
if isinstance(__x, ast.Expr):
result += cls.parse_node(__x, frame=frame)
else:
exec(meta.dump_python_source(__x))
return result
def parse_ast(cls, t):
# This is done very naively
result = ''
for __x in t:
frame = inspect.currentframe()
if isinstance(__x, ast.Expr):
result += cls.parse_node(__x, frame=frame)
else:
exec(meta.dump_python_source(__x))
return result
def decompile(code, version="2.7"):
import meta
from uncompyle2 import uncompyle
try:
source = StringIO.StringIO()
uncompyle(version, code, source)
source = source.getvalue()
except:
try:
code_obj = meta.decompile(code)
source = meta.dump_python_source(code_obj)
except:
return None
return source.lstrip(' \n')
def decompile(code, version="2.7"):
import meta
from uncompyle2 import uncompyle
try:
source = StringIO.StringIO()
uncompyle(version, code, source)
source = source.getvalue()
except:
try:
code_obj = meta.decompile(code)
source = meta.dump_python_source(code_obj)
except:
return None
return source.lstrip(' \n')
def toSource(x):
return meta.dump_python_source(x).strip()
def translate_toplevel(self, commands):
main = filter(lambda x: isinstance(x, ast.FunctionDef) and x.name == 'main', self.template.body)[0]
main.body.extend(flatten(self.translate_commands(commands)))
substassign = filter(lambda x: isinstance(x, ast.Assign) and x.targets[0].id == 'SUBSTS', self.template.body)[0]
substassign.value.args = [ast.List([ast.Str(s) for s in self.macro_handler.substs], ast.Load())]
make_arg_parser = filter(lambda x: isinstance(x, ast.FunctionDef) and x.name == 'make_arg_parser', self.template.body)[0]
make_arg_parser.body[-1:-1] = self.make_argparse_arguments()
p = Parser(sys.stdin.read())
p.changequote('[',']')
macro_handler = MacroHandler()
macro_handler.add_macros(MACROS, p)
stream = StringIO()
# Parse m4
p.parse(stream=stream)
shell = stream.getvalue()
if len(sys.argv) > 1:
sys.stdout.write(shell)
sys.exit(0)
# now translate shell to Python
template_file = os.path.join(os.path.dirname(__file__), 'template.py')
template = ast.parse(open(template_file, 'r').read())
translator = ShellTranslator(macro_handler, template)
translator.translate(shell, toplevel=True)
sys.stdout.write(meta.dump_python_source(template))
def pyc_source(pyc_contents):
code_section = pyc_contents[8:]
code = marshal.load(code_section)
return meta.dump_python_source(meta.decompile(code))
def pyc_source(pyc_contents):
code_section = pyc_contents[8:]
code = marshal.load(code_section)
return meta.dump_python_source(meta.decompile(code))
def __repr__(self): return self.complexity() + dump_python_source(self.ast)
def parse(expression, defs=None, returnlabel=False):
"""
Convert a lambda or string ``expression`` into a :py:class:`Expr <histbook.expr.Expr>`.
Parameters
----------
expression : lambda or string
expression involving constants (literals and ``pi``, ``e``, ``inf``, and ``nan``), field variables, and functions in ``Expr.recognized``; if a lambda, the fields are given by function parameters, if a string, any unrecognized names will be identified as fields
defs : ``None`` or dict
if not ``None``, provides names to recognize in ``expression`` (to avoid repetitive code)
returnlabel : bool
if ``True``, return value is a 2-tuple: :py:class:`Expr <histbook.expr.Expr>` and string representing the expression; otherwise *(default)*, just the :py:class:`Expr <histbook.expr.Expr>`
"""
_defs = {
"pi": Const(math.pi),
"e": Const(math.e),
"inf": Const(float("inf")),
"nan": Const(float("nan"))
}
if defs is not None:
for n, x in defs.items():
if isinstance(x, Expr):
_defs[n] = x
elif sys.version_info[0] < 3 and isinstance(x, (str, unicode)):
_defs[n] = Expr.parse(x)
elif sys.version_info[0] >= 3 and isinstance(x, str):
_defs[n] = Expr.parse(x)
else:
try:
_defs[n] = Const(pickle.loads(pickle.dumps(x)))
except:
raise ExpressionError(
"object can't be included in an expression as symbol {0} because it refers to an unserializable object"
.format(repr(n)))
pyast = None
label = None
params = None
if isinstance(expression,
types.FunctionType): # more specific than callable(...)
fcn = meta.decompiler.decompile_func(expression)
if isinstance(fcn, ast.FunctionDef) and len(
fcn.body) == 1 and isinstance(fcn.body[0], ast.Return):
pyast = fcn.body[0].value
label = expression.__name__
elif isinstance(fcn, ast.Lambda):
pyast = fcn.body.value
label = meta.dump_python_source(pyast).strip()
params = expression.__code__.co_varnames[:expression.__code__.
co_argcount]
elif (sys.version_info[0] < 3 and isinstance(
expression, basestring)) or (sys.version_info[0] >= 3
and isinstance(expression, str)):
mod = ast.parse(expression)
if len(mod.body) == 1 and isinstance(mod.body[0], ast.Expr):
pyast = mod.body[0].value
label = expression
if pyast is None:
raise TypeError(
"expression must be a one-line string, one-line function, or lambda expression, not {0}"
.format(repr(expression)))
calculate = {
"+": lambda x, y: x + y,
"-": lambda x, y: x - y,
"*": lambda x, y: x * y,
"/": lambda x, y: float(x) / float(y),
"//": lambda x, y: int(x // y),
"%": lambda x, y: x % y,
"**": lambda x, y: x**y,
"|": lambda x, y: numpy.uint64(x) | numpy.uint64(y),
"&": lambda x, y: numpy.uint64(x) & numpy.uint64(y),
"^": lambda x, y: numpy.uint64(x) ^ numpy.uint64(y)
}
env = dict(globals())
def resolve(node):
if isinstance(node, ast.Attribute):
return getattr(resolve(node.value), node.attr, None)
elif isinstance(node, ast.Name):
return env.get(node.id, None)
else:
raise ExpressionError(
"functions must be named, not constructed: {0}".format(
meta.dump_python_source(node).strip()))
def recurse(node, relations=False):
if isinstance(node, ast.Num):
return Const(node.n)
elif isinstance(node, ast.Str):
return Const(node.s)
elif isinstance(node, ast.Dict) and len(node.keys) == 0:
return Const(set())
elif isinstance(node, ast.Set):
content = [recurse(x) for x in node.elts]
if all(isinstance(x, Const) for x in content):
return Const(set(x.value for x in content))
else:
raise ExpressionError(
"sets in expressions may not contain variable contents: {0}"
.format(meta.dump_python_source(node).strip()))
elif isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load):
if node.id == "None":
return Const(None)
elif node.id == "True":
return Const(True)
elif node.id == "False":
return Const(False)
elif node.id in _defs:
return _defs[node.id]
elif params is not None and node.id not in params:
if node.id in env:
try:
return Const(
pickle.loads(pickle.dumps(env[node.id])))
except:
raise ExpressionError(
"symbol {0} can't be included in an expression because it refers to an unserializable object in the global scope"
.format(repr(node.id)))
else:
raise ExpressionError(
"symbol {0} is not a function parameter and not in the global scope"
.format(repr(node.id)))
else:
return Name(node.id)
elif isinstance(node, getattr(ast, "NameConstant",
tuple)): # node will never be tuple
if node.value is None:
return Const(None)
elif node.value is True:
return Const(True)
elif node.value is False:
return Const(False)
else:
raise AssertionError(node)
elif relations and isinstance(node, ast.Compare) and len(
node.ops) == 1:
if isinstance(node.ops[0], ast.Eq): cmp, swap = "==", None
elif isinstance(node.ops[0], ast.NotEq): cmp, swap = "!=", None
elif isinstance(node.ops[0], ast.Lt): cmp, swap = "<", False
elif isinstance(node.ops[0], ast.LtE): cmp, swap = "<=", False
elif isinstance(node.ops[0], ast.Gt): cmp, swap = "<", True
elif isinstance(node.ops[0], ast.GtE): cmp, swap = "<=", True
elif isinstance(node.ops[0], ast.In): cmp, swap = "in", False
elif isinstance(node.ops[0], ast.NotIn):
cmp, swap = "not in", False
else:
raise ExpressionError(
"only comparision relations supported: '==', '!=', '<', '<=', '>', '>=', 'in', and 'not in': {0}"
.format(meta.dump_python_source(node).strip()))
left = recurse(node.left)
right = recurse(node.comparators[0])
if swap is True:
left, right = right, left
elif swap is None:
left, right = sorted([left, right])
if (cmp == "in" or cmp == "not in") and not (isinstance(
right, Const) and isinstance(right.value, set)):
raise ExpressionError(
"comparisons 'in' and 'not in' can only be used with a set: {0}"
.format(meta.dump_python_source(node).strip()))
return Relation(cmp, left, right)
elif relations and isinstance(node,
ast.Compare) and len(node.ops) > 1:
return recurse(ast.BoolOp(ast.And(), [
ast.Compare(
node.left if i == 0 else node.comparators[i - 1],
[node.ops[i]], [node.comparators[i]])
for i in range(len(node.ops))
]),
relations=True)
elif isinstance(node, ast.Compare):
raise ExpressionError(
"comparison operators are only allowed at the top of an expression: {0}"
.format(meta.dump_python_source(node).strip()))
elif relations and isinstance(node, ast.UnaryOp) and isinstance(
node.op, ast.Not):
content = recurse(node.operand, relations=True)
if isinstance(content, Name):
return Predicate(content.value, positive=False)
else:
return Logical.negate(content).simplify()
elif relations and isinstance(node, ast.BoolOp) and isinstance(
node.op, ast.And):
return functools.reduce(LogicalAnd.combine, [
Logical.normalform(recurse(x, relations=True))
for x in node.values
]).simplify()
elif relations and isinstance(node, ast.BoolOp) and isinstance(
node.op, ast.Or):
return functools.reduce(LogicalOr.combine, [
Logical.normalform(recurse(x, relations=True))
for x in node.values
]).simplify()
elif isinstance(node, ast.BoolOp):
raise ExpressionError(
"logical operators are only allowed at the top of an expression: {0}"
.format(meta.dump_python_source(node).strip()))
elif isinstance(node, ast.UnaryOp) and isinstance(
node.op, ast.USub):
content = recurse(node.operand)
if isinstance(content, Const):
return Const(-content.value)
else:
return PlusMinus.negate(content).simplify()
elif isinstance(node, ast.UnaryOp) and isinstance(
node.op, ast.UAdd):
return recurse(node.operand)
elif isinstance(node, ast.UnaryOp) and isinstance(
node.op, ast.Invert):
content = recurse(node.operand)
if isinstance(content, Const):
return Const(~content.value)
else:
return BitAnd.negate(content).simplify()
elif isinstance(node, ast.UnaryOp):
raise ExpressionError(
"only unary operators supported: 'not', '-', '+', and '~': {0}"
.format(meta.dump_python_source(node).strip()))
elif isinstance(node, ast.BinOp):
if isinstance(node.op, ast.Add): fcn = "+"
elif isinstance(node.op, ast.Sub): fcn = "-"
elif isinstance(node.op, ast.Mult): fcn = "*"
elif isinstance(node.op, ast.Div): fcn = "/"
elif isinstance(node.op, ast.FloorDiv):
op, fcn = "//", "floor_divide"
elif isinstance(node.op, ast.Mod):
op, fcn = "%", "mod"
elif isinstance(node.op, ast.Pow):
op, fcn = "**", "pow"
elif isinstance(node.op, ast.BitOr):
fcn = "|"
elif isinstance(node.op, ast.BitAnd):
fcn = "&"
elif isinstance(node.op, ast.BitXor):
op, fcn = "^", "xor"
else:
raise ExpressionError(
"only binary operators supported: '+', '-', '*', '/', '//', '%', '**', '|', '&', and '^': {0}"
.format(meta.dump_python_source(node).strip()))
left = recurse(node.left)
right = recurse(node.right)
if isinstance(left, Const) and isinstance(right, Const):
return Const(calculate[fcn](left.value, right.value))
if fcn == "+":
return PlusMinus.combine(left, right).simplify()
elif fcn == "-":
return PlusMinus.combine(
left, PlusMinus.negate(right)).simplify()
elif fcn == "*":
return PlusMinus.distribute(left, right).simplify()
elif fcn == "/":
# PlusMinus is implemented as a RingAlgebra, but it's really a Field
right = PlusMinus.normalform(right)
if len(right.pos) == len(right.neg) == 0:
negation = TimesDiv(TimesDiv.negateval(right.const),
(), ())
elif right.const == PlusMinus.identity and len(
right.pos) == 1 and len(right.neg) == 0:
negation = TimesDiv.negate(right.pos[0])
elif right.const == PlusMinus.identity and len(
right.pos) == 0 and len(right.neg) == 1:
negation = TimesDiv.negate(right.neg[0])
negation.const = PlusMinus.negateval(negation.const)
else:
negation = TimesDiv.negate(
right) # additive terms in the denominator
return PlusMinus.distribute(left, negation).simplify()
elif fcn == "pow" and isinstance(
right, Const) and right.value == round(
right.value) and 1 <= right.value < 5:
n = int(right.value)
left = PlusMinus.normalform(left)
if left.const == PlusMinus.identity and len(
left.pos) == 1 and len(left.neg) == 0:
return PlusMinus(
PlusMinus.identity,
(TimesDiv(left.pos[0].const**n,
tuple(sorted(left.pos[0].pos * n)),
tuple(sorted(left.pos[0].neg * n))), ),
()).simplify()
elif left.const == PlusMinus.identity and len(
left.pos) == 0 and len(
left.neg) == 1 and n % 2 == 0:
return PlusMinus(
PlusMinus.identity,
(TimesDiv(left.neg[0].const**n,
tuple(sorted(left.neg[0].pos * n)),
tuple(sorted(left.neg[0].neg * n))), ),
()).simplify()
elif left.const == PlusMinus.identity and len(
left.pos) == 0 and len(left.neg) == 1:
return PlusMinus(PlusMinus.identity, (), (TimesDiv(
left.neg[0].const**n,
tuple(sorted(left.neg[0].pos * n)),
tuple(sorted(left.neg[0].neg * n))), )).simplify()
else:
return PlusMinus(
PlusMinus.identity,
(TimesDiv(TimesDiv.identity, tuple((left, ) * n),
()), ), ()).simplify()
elif fcn == "pow" and isinstance(
right, Const) and right.value == round(
right.value) and -5 < right.value <= -1:
n = -int(right.value)
left = PlusMinus.normalform(left)
if left.const == PlusMinus.identity and len(
left.pos) == 1 and len(left.neg) == 0:
return PlusMinus(
PlusMinus.identity,
(TimesDiv(left.pos[0].const**(-n),
tuple(sorted(left.pos[0].neg * n)),
tuple(sorted(left.pos[0].pos * n))), ),
()).simplify()
elif left.const == PlusMinus.identity and len(
left.pos) == 0 and len(
left.neg) == 1 and n % 2 == 0:
return PlusMinus(
PlusMinus.identity,
(TimesDiv(left.neg[0].const**(-n),
tuple(sorted(left.neg[0].neg * n)),
tuple(sorted(left.neg[0].pos * n))), ),
()).simplify()
elif left.const == PlusMinus.identity and len(
left.pos) == 0 and len(left.neg) == 1:
return PlusMinus(PlusMinus.identity, (), (TimesDiv(
left.neg[0].const**(-n),
tuple(sorted(left.neg[0].neg * n)),
tuple(sorted(left.neg[0].pos * n))), )).simplify()
else:
return PlusMinus(PlusMinus.identity,
(TimesDiv(TimesDiv.identity,
(), tuple((left, ) * n)), ),
()).simplify()
elif fcn == "&":
raise NotImplementedError
elif fcn == "|":
raise NotImplementedError
else:
return BinOp(fcn, left, right, op)
elif isinstance(node, ast.Call):
fcn = Expr.recognized.get(resolve(node.func), None)
if fcn is None and node.func.id in Expr.recognized.values():
fcn = node.func.id
elif fcn is not None:
for n, x in Expr.recognized.items():
if fcn is n:
fcn = x
break
if fcn is None:
raise ExpressionError(
"unhandled function in expression: {0}".format(
meta.dump_python_source(node).strip()))
return Call(fcn, *(recurse(x) for x in node.args))
else:
ExpressionError("unhandled syntax in expression: {0}".format(
meta.dump_python_source(node).strip()))
if returnlabel:
return recurse(pyast, relations=True), label
else:
return recurse(pyast, relations=True)
def recurse(node, relations=False):
if isinstance(node, ast.Num):
return Const(node.n)
elif isinstance(node, ast.Str):
return Const(node.s)
elif isinstance(node, ast.Dict) and len(node.keys) == 0:
return Const(set())
elif isinstance(node, ast.Set):
content = [recurse(x) for x in node.elts]
if all(isinstance(x, Const) for x in content):
return Const(set(x.value for x in content))
else:
raise ExpressionError(
"sets in expressions may not contain variable contents: {0}"
.format(meta.dump_python_source(node).strip()))
elif isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load):
if node.id == "None":
return Const(None)
elif node.id == "True":
return Const(True)
elif node.id == "False":
return Const(False)
elif node.id in _defs:
return _defs[node.id]
elif params is not None and node.id not in params:
if node.id in env:
try:
return Const(
pickle.loads(pickle.dumps(env[node.id])))
except:
raise ExpressionError(
"symbol {0} can't be included in an expression because it refers to an unserializable object in the global scope"
.format(repr(node.id)))
else:
raise ExpressionError(
"symbol {0} is not a function parameter and not in the global scope"
.format(repr(node.id)))
else:
return Name(node.id)
elif isinstance(node, getattr(ast, "NameConstant",
tuple)): # node will never be tuple
if node.value is None:
return Const(None)
elif node.value is True:
return Const(True)
elif node.value is False:
return Const(False)
else:
raise AssertionError(node)
elif relations and isinstance(node, ast.Compare) and len(
node.ops) == 1:
if isinstance(node.ops[0], ast.Eq): cmp, swap = "==", None
elif isinstance(node.ops[0], ast.NotEq): cmp, swap = "!=", None
elif isinstance(node.ops[0], ast.Lt): cmp, swap = "<", False
elif isinstance(node.ops[0], ast.LtE): cmp, swap = "<=", False
elif isinstance(node.ops[0], ast.Gt): cmp, swap = "<", True
elif isinstance(node.ops[0], ast.GtE): cmp, swap = "<=", True
elif isinstance(node.ops[0], ast.In): cmp, swap = "in", False
elif isinstance(node.ops[0], ast.NotIn):
cmp, swap = "not in", False
else:
raise ExpressionError(
"only comparision relations supported: '==', '!=', '<', '<=', '>', '>=', 'in', and 'not in': {0}"
.format(meta.dump_python_source(node).strip()))
left = recurse(node.left)
right = recurse(node.comparators[0])
if swap is True:
left, right = right, left
elif swap is None:
left, right = sorted([left, right])
if (cmp == "in" or cmp == "not in") and not (isinstance(
right, Const) and isinstance(right.value, set)):
raise ExpressionError(
"comparisons 'in' and 'not in' can only be used with a set: {0}"
.format(meta.dump_python_source(node).strip()))
return Relation(cmp, left, right)
elif relations and isinstance(node,
ast.Compare) and len(node.ops) > 1:
return recurse(ast.BoolOp(ast.And(), [
ast.Compare(
node.left if i == 0 else node.comparators[i - 1],
[node.ops[i]], [node.comparators[i]])
for i in range(len(node.ops))
]),
relations=True)
elif isinstance(node, ast.Compare):
raise ExpressionError(
"comparison operators are only allowed at the top of an expression: {0}"
.format(meta.dump_python_source(node).strip()))
elif relations and isinstance(node, ast.UnaryOp) and isinstance(
node.op, ast.Not):
content = recurse(node.operand, relations=True)
if isinstance(content, Name):
return Predicate(content.value, positive=False)
else:
return Logical.negate(content).simplify()
elif relations and isinstance(node, ast.BoolOp) and isinstance(
node.op, ast.And):
return functools.reduce(LogicalAnd.combine, [
Logical.normalform(recurse(x, relations=True))
for x in node.values
]).simplify()
elif relations and isinstance(node, ast.BoolOp) and isinstance(
node.op, ast.Or):
return functools.reduce(LogicalOr.combine, [
Logical.normalform(recurse(x, relations=True))
for x in node.values
]).simplify()
elif isinstance(node, ast.BoolOp):
raise ExpressionError(
"logical operators are only allowed at the top of an expression: {0}"
.format(meta.dump_python_source(node).strip()))
elif isinstance(node, ast.UnaryOp) and isinstance(
node.op, ast.USub):
content = recurse(node.operand)
if isinstance(content, Const):
return Const(-content.value)
else:
return PlusMinus.negate(content).simplify()
elif isinstance(node, ast.UnaryOp) and isinstance(
node.op, ast.UAdd):
return recurse(node.operand)
elif isinstance(node, ast.UnaryOp) and isinstance(
node.op, ast.Invert):
content = recurse(node.operand)
if isinstance(content, Const):
return Const(~content.value)
else:
return BitAnd.negate(content).simplify()
elif isinstance(node, ast.UnaryOp):
raise ExpressionError(
"only unary operators supported: 'not', '-', '+', and '~': {0}"
.format(meta.dump_python_source(node).strip()))
elif isinstance(node, ast.BinOp):
if isinstance(node.op, ast.Add): fcn = "+"
elif isinstance(node.op, ast.Sub): fcn = "-"
elif isinstance(node.op, ast.Mult): fcn = "*"
elif isinstance(node.op, ast.Div): fcn = "/"
elif isinstance(node.op, ast.FloorDiv):
op, fcn = "//", "floor_divide"
elif isinstance(node.op, ast.Mod):
op, fcn = "%", "mod"
elif isinstance(node.op, ast.Pow):
op, fcn = "**", "pow"
elif isinstance(node.op, ast.BitOr):
fcn = "|"
elif isinstance(node.op, ast.BitAnd):
fcn = "&"
elif isinstance(node.op, ast.BitXor):
op, fcn = "^", "xor"
else:
raise ExpressionError(
"only binary operators supported: '+', '-', '*', '/', '//', '%', '**', '|', '&', and '^': {0}"
.format(meta.dump_python_source(node).strip()))
left = recurse(node.left)
right = recurse(node.right)
if isinstance(left, Const) and isinstance(right, Const):
return Const(calculate[fcn](left.value, right.value))
if fcn == "+":
return PlusMinus.combine(left, right).simplify()
elif fcn == "-":
return PlusMinus.combine(
left, PlusMinus.negate(right)).simplify()
elif fcn == "*":
return PlusMinus.distribute(left, right).simplify()
elif fcn == "/":
# PlusMinus is implemented as a RingAlgebra, but it's really a Field
right = PlusMinus.normalform(right)
if len(right.pos) == len(right.neg) == 0:
negation = TimesDiv(TimesDiv.negateval(right.const),
(), ())
elif right.const == PlusMinus.identity and len(
right.pos) == 1 and len(right.neg) == 0:
negation = TimesDiv.negate(right.pos[0])
elif right.const == PlusMinus.identity and len(
right.pos) == 0 and len(right.neg) == 1:
negation = TimesDiv.negate(right.neg[0])
negation.const = PlusMinus.negateval(negation.const)
else:
negation = TimesDiv.negate(
right) # additive terms in the denominator
return PlusMinus.distribute(left, negation).simplify()
elif fcn == "pow" and isinstance(
right, Const) and right.value == round(
right.value) and 1 <= right.value < 5:
n = int(right.value)
left = PlusMinus.normalform(left)
if left.const == PlusMinus.identity and len(
left.pos) == 1 and len(left.neg) == 0:
return PlusMinus(
PlusMinus.identity,
(TimesDiv(left.pos[0].const**n,
tuple(sorted(left.pos[0].pos * n)),
tuple(sorted(left.pos[0].neg * n))), ),
()).simplify()
elif left.const == PlusMinus.identity and len(
left.pos) == 0 and len(
left.neg) == 1 and n % 2 == 0:
return PlusMinus(
PlusMinus.identity,
(TimesDiv(left.neg[0].const**n,
tuple(sorted(left.neg[0].pos * n)),
tuple(sorted(left.neg[0].neg * n))), ),
()).simplify()
elif left.const == PlusMinus.identity and len(
left.pos) == 0 and len(left.neg) == 1:
return PlusMinus(PlusMinus.identity, (), (TimesDiv(
left.neg[0].const**n,
tuple(sorted(left.neg[0].pos * n)),
tuple(sorted(left.neg[0].neg * n))), )).simplify()
else:
return PlusMinus(
PlusMinus.identity,
(TimesDiv(TimesDiv.identity, tuple((left, ) * n),
()), ), ()).simplify()
elif fcn == "pow" and isinstance(
right, Const) and right.value == round(
right.value) and -5 < right.value <= -1:
n = -int(right.value)
left = PlusMinus.normalform(left)
if left.const == PlusMinus.identity and len(
left.pos) == 1 and len(left.neg) == 0:
return PlusMinus(
PlusMinus.identity,
(TimesDiv(left.pos[0].const**(-n),
tuple(sorted(left.pos[0].neg * n)),
tuple(sorted(left.pos[0].pos * n))), ),
()).simplify()
elif left.const == PlusMinus.identity and len(
left.pos) == 0 and len(
left.neg) == 1 and n % 2 == 0:
return PlusMinus(
PlusMinus.identity,
(TimesDiv(left.neg[0].const**(-n),
tuple(sorted(left.neg[0].neg * n)),
tuple(sorted(left.neg[0].pos * n))), ),
()).simplify()
elif left.const == PlusMinus.identity and len(
left.pos) == 0 and len(left.neg) == 1:
return PlusMinus(PlusMinus.identity, (), (TimesDiv(
left.neg[0].const**(-n),
tuple(sorted(left.neg[0].neg * n)),
tuple(sorted(left.neg[0].pos * n))), )).simplify()
else:
return PlusMinus(PlusMinus.identity,
(TimesDiv(TimesDiv.identity,
(), tuple((left, ) * n)), ),
()).simplify()
elif fcn == "&":
raise NotImplementedError
elif fcn == "|":
raise NotImplementedError
else:
return BinOp(fcn, left, right, op)
elif isinstance(node, ast.Call):
fcn = Expr.recognized.get(resolve(node.func), None)
if fcn is None and node.func.id in Expr.recognized.values():
fcn = node.func.id
elif fcn is not None:
for n, x in Expr.recognized.items():
if fcn is n:
fcn = x
break
if fcn is None:
raise ExpressionError(
"unhandled function in expression: {0}".format(
meta.dump_python_source(node).strip()))
return Call(fcn, *(recurse(x) for x in node.args))
else:
ExpressionError("unhandled syntax in expression: {0}".format(
meta.dump_python_source(node).strip()))
import ast
import meta
import astpp
code = open('sample_code.py', 'r').read()
tree = ast.parse(code)
#Get source
source = meta.dump_python_source(tree)
#Get formatted AST repr
formatted_ast = astpp.dump(tree, include_attributes=True)
class FuncLister(ast.NodeVisitor):
def visit_FunctionDef(self, node):
print node.name
self.generic_visit(node)
print 'Functions:'
FuncLister().visit(tree)
def toSource(x): return meta.dump_python_source(x).strip()
def __repr__(self): return "#%s%s"%(hash(self),dump_python_source(self.ast))
def string(expr):
"""Stringify an ast"""
import meta
return meta.dump_python_source(expr).strip()
from ast import *
from astpp import dump
from meta import dump_python_source
class RewriteOps(NodeTransformer):
_names = {Add: '__add__', Sub: '__sub__', Mult: '__mul__', Div: '__truediv__', FloorDiv: '__floordiv__', Mod: '__mod__', Pow: '__pow__'}
def visit_BinOp(self, node):
name = self._names.get(node.op.__class__, None)
if name is not None:
return copy_location(Call(func=Attribute(value=node.left, attr=name, ctx=Load()),args=[node.right], keywords=[], starargs=[], kwargs=[]),node)
else:
self.generic_visit(node)
source = '"Hello " + "World!"'
print(source,'\n')
node = parse(source)
print(dump(node),'\n')
node = RewriteOps().visit(node)
print(dump(node))
source2 = dump_python_source(node)
print(source2)
print(eval(source))
print(eval(source2))
def statement(node):
if isinstance(node, list):
out = []
for x in node:
t = ("\n" + meta.dump_python_source(x).strip()).replace("\n", "\n ")
out.append(makeyield(len(text)))
text.append(t)
out.append(statement(x))
node = out
elif isinstance(node, ast.Module):
node.body = statement(node.body)
elif isinstance(node, (ast.ClassDef, ast.Exec, ast.FunctionDef, ast.Import, ast.Return)):
raise SyntaxError("{0} statements are not allowed".format(node.__class__.__text__))
elif isinstance(node, ast.Expr):
node.value = expression(node.value)
elif isinstance(node, ast.Assert):
node.test = expression(node.test)
elif isinstance(node, ast.Assign):
node.targets = [expression(x) for x in node.targets]
node.value = expression(node.value)
elif isinstance(node, ast.AugAssign):
node.target = expression(node.target)
node.value = expression(node.value)
elif isinstance(node, ast.Delete):
node.targets = [expression(x) for x in node.targets]
elif isinstance(node, ast.ExceptHandler):
node.body = statement(node.body)
elif isinstance(node, ast.For):
node.iter = expression(node.iter)
node.body = statement(node.body)
node.orelse = statement(node.orelse)
elif isinstance(node, ast.If):
node.test = expression(node.test)
node.body = statement(node.body)
node.orelse = statement(node.orelse)
elif isinstance(node, ast.Print):
node.values = expression(node.values)
elif isinstance(node, ast.TryExcept):
node.body = statement(node.body)
node.orelse = statement(node.orelse)
elif isinstance(node, ast.TryFinally):
node.body = statement(node.body)
node.finalbody = statement(node.finalbody)
elif isinstance(node, ast.Try):
node.body = statement(node.body)
node.orelse = statement(node.orelse)
node.finalbody = statement(node.finalbody)
elif isinstance(node, ast.While):
node.test = expression(node.test)
node.body = statement(node.body)
node.orelse = statement(node.orelse)
elif isinstance(node, ast.With):
node.body = statement(node.body)
if hasattr(node, "items"):
for x in node.items:
x.context_expr = expression(x.context_expr)
else:
node.context_expr = expression(node.context_expr)
return node
def ast_to_source_meta(node):
import meta
return meta.dump_python_source(node).strip()
