def visit_compare(self, compare_node: ast.Compare): # Visit downstream symbols new_left = self.visit(compare_node.left) new_comparators = [ self.visit(node) for node in compare_node.comparators ] # Simple comparison node if len(new_comparators) == 1: return ast.Compare(left=new_left, comparators=[new_comparators[0]], ops=compare_node.ops) # Compound comparison node # Convert to multiple Compare operations new_nodes, left = [], new_left for right, op in zip(new_comparators, compare_node.ops): new_node = ast.Compare(left=left, comparators=[right], ops=[op]) new_nodes.append(new_node) left = right assert len(new_nodes) >= 2 bin_op = ast.BinOp(left=new_nodes[0], right=new_nodes[1], op=ast.BitAnd()) for new_right in new_nodes[2:]: bin_op = ast.BinOp(left=bin_op, right=new_right, op=ast.BitAnd()) return bin_op
def mutate(cls, node): if node not in config.visited_nodes: if node.__class__ is ast.BitAnd: config.mutated = True original_node = deepcopy(node) parent = config.parent_dict[node] del config.parent_dict[node] node = ast.BitOr() config.parent_dict[node] = parent config.node_pairs[node] = original_node config.current_mutated_node = node elif node.__class__ is ast.BitOr: config.mutated = True original_node = deepcopy(node) parent = config.parent_dict[node] del config.parent_dict[node] node = ast.BitAnd() config.parent_dict[node] = parent config.node_pairs[node] = original_node config.current_mutated_node = node elif node.__class__ is ast.BitXor: config.mutated = True original_node = deepcopy(node) parent = config.parent_dict[node] del config.parent_dict[node] node = ast.BitAnd() config.parent_dict[node] = parent config.node_pairs[node] = original_node config.current_mutated_node = node elif node.__class__ is ast.LShift: config.mutated = True original_node = deepcopy(node) parent = config.parent_dict[node] del config.parent_dict[node] node = ast.RShift() config.parent_dict[node] = parent config.node_pairs[node] = original_node config.current_mutated_node = node elif node.__class__ is ast.RShift: config.mutated = True original_node = deepcopy(node) parent = config.parent_dict[node] del config.parent_dict[node] node = ast.LShift() config.parent_dict[node] = parent config.node_pairs[node] = original_node config.current_mutated_node = node return node
def visit_Compare(self, ast_node_Compare: ast.Compare) -> ast.Constant: """ Compare nodes occur for all sequences of comparison (gt, lt, etc.) operators. :param ast_node_Compare: a object comparison of two or more values. :return: evaluated value. """ comparators = ast_node_Compare.comparators operators = ast_node_Compare.ops # Exception case: (A cmp ?) if operator.le(len(comparators), 0): raise ValueError('There must be at least two operands for compare') # Common case: (A cmp B) if operator.eq(len(comparators), 1): ast_node_BinOp = ast.BinOp(ast_node_Compare.left, next(iter(operators), None), next(iter(comparators), None)) return self.visit(ast_node_BinOp) else: # Recursive case: (A cmp B cmp C cmp ... etc.) ast_values_BoolOp = [] left_comparator = ast_node_Compare.left for ast_type_operator, right_comparator in zip( operators, comparators): new_ast_node_Compare = self.visit( ast.Compare(left_comparator, [ast_type_operator], [right_comparator])) left_comparator = right_comparator ast_values_BoolOp.append(new_ast_node_Compare) return self.visit(ast.BoolOp(ast.BitAnd(), ast_values_BoolOp))
def __init__(self, base_node): BaseMutator.__init__(self, base_node) self.original_bin_op = base_node.op if type(base_node.op) in [ ast.LShift, ast.RShift, ast.BitOr, ast.BitXor, ast.BitAnd, ast.FloorDiv ]: if type(base_node.op) is ast.LShift: self.mutations.append({"op": ast.RShift()}) if type(base_node.op) is ast.RShift: self.mutations.append({"op": ast.LShift()}) if type(base_node.op) is ast.BitOr: self.mutations.append({"op": ast.BitAnd()}) if type(base_node.op) is ast.BitXor: self.mutations.append({"op": ast.FloorDiv()}) if type(base_node.op) is ast.BitAnd: self.mutations.append({"op": ast.BitOr()}) if type(base_node.op) is ast.FloorDiv: self.mutations.append({"op": ast.BitXor()})
def visit_Call(self, node): 'Replace custom function with bitwise operators' self.generic_visit(node) if isinstance(node.func, ast.Name): if len(node.args) == 2: if node.func.id == "mand": op = ast.BitAnd() elif node.func.id == "mxor": op = ast.BitXor() elif node.func.id == "mor": op = ast.BitOr() elif node.func.id == "mlshift": op = ast.LShift() elif node.func.id == "mrshift": op = ast.RShift() else: return node return ast.BinOp(node.args[0], op, node.args[1]) elif len(node.args) == 1 and node.func.id == "mnot": arg = node.args[0] self.generic_visit(node) return ast.UnaryOp(ast.Invert(), arg) return self.generic_visit(node)
def visit_Operator(self, node: Operator, *args, **kwargs) -> C.operator: if node == Operator.Add: return C.Add() elif node == Operator.Sub: return C.Sub() elif node == Operator.Mult: return C.Mult() elif node == Operator.MatMult: return C.MatMult() elif node == Operator.Div: return C.Div() elif node == Operator.Mod: return C.Mod() elif node == Operator.Pow: return C.Pow() elif node == Operator.LShift: return C.LShift() elif node == Operator.RShift: return C.RShift() elif node == Operator.BitOr: return C.BitOr() elif node == Operator.BitXor: return C.BitXor() elif node == Operator.BitAnd: return C.BitAnd() elif node == Operator.FloorDiv: return C.FloorDiv() else: raise Exception(f'unknown Operator {node!r}')
def binop_action(s, loc, tokens): node = tokens[0] for op_char, right in tokens[1:]: if op_char == '+': op = ast.Add() elif op_char == '-': op = ast.Sub() elif op_char == '*': op = ast.Mult() elif op_char == '/': op = ast.Div() elif op_char == '%': op = ast.Mod() elif op_char == '<<': op = ast.LShift() elif op_char == '>>': op = ast.RShift() elif op_char == '&': op = ast.BitAnd() elif op_char == '^': op = ast.BitXor() else: # op_char == '|': op = ast.BitOr() node = ast.BinOp(left=node, op=op, right=right, lineno=1, col_offset=0) return node
def bit_and(self): left = self.bit_shift() while self.match_(TokenType.AMPERSAND): right = self.bit_shift() left = ast.BinOp(left, ast.BitAnd(), right, **self.get_loc(left, right)) return left
def get_bin_op(self, s): """Get the BinOp class for s.""" op = None if s == '+': op = ast.Add() elif s == '-': op = ast.Sub() elif s == '*': op = ast.Mult() elif s == '/': op = ast.Div() elif s == '%': op = ast.Mod() elif s == '<<': op = ast.LShift() elif s == '>>': op = ast.RShift() elif s == '&': op = ast.BitAnd() elif s == '^': op = ast.BitXor() elif s == '|': op = ast.BitOr() return op
def visit_BinOp(self, node): # operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift | RShift | BitOr | BitXor | BitAnd | FloorDiv print(" in MyTransformer.visit_BinOp()") print(" curr op =", node.op) bin_negate = node.op # use pseudorandomness to determine whether to negate or just mix up rand_num = random.randint(1, 10) # negate if rand_num >= 4: print(" negating...") if isinstance(node.op, ast.Add): bin_negate = ast.Sub() elif isinstance(node.op, ast.Sub): bin_negate = ast.Add() elif isinstance(node.op, ast.Mult): bin_negate = ast.Div() elif isinstance(node.op, ast.Div): bin_negate = ast.FloorDiv() elif isinstance(node.op, ast.FloorDiv): bin_negate = ast.Div() elif isinstance(node.op, ast.LShift): bin_negate = ast.RShift() elif isinstance(node.op, ast.RShift): bin_negate = ast.LShift() elif isinstance(node.op, ast.BitOr): bin_negate = ast.BitAnd() elif isinstance(node.op, ast.BitAnd): bin_negate = ast.BitXor() elif isinstance(node.op, ast.BitXor): bin_negate = ast.BitOr() elif isinstance(node.op, ast.Pow): bin_negate = ast.Mult() elif isinstance(node.op, ast.Mod): bin_negate = ast.Div() elif isinstance(node.op, ast.MatMult): bin_negate = ast.Mult() else: print(" did not find negation for", node.op) # mix up else: print(" mixing up...") if isinstance(node.op, ast.Add): bin_negate = ast.Mult() elif isinstance(node.op, ast.Sub): bin_negate = ast.Div() elif isinstance(node.op, ast.Mult): bin_negate = ast.Pow() elif isinstance(node.op, ast.Div): bin_negate = ast.FloorDiv() elif isinstance(node.op, ast.FloorDiv): bin_negate = ast.Div() elif isinstance(node.op, ast.BitOr): bin_negate = ast.BitXor() elif isinstance(node.op, ast.BitAnd): bin_negate = ast.BitOr() elif isinstance(node.op, ast.BitXor): bin_negate = ast.BitOr() elif isinstance(node.op, ast.Pow): bin_negate = ast.Mult() elif isinstance(node.op, ast.Mod): bin_negate = ast.FloorDiv() else: print(" did not find negation for", node.op) print(" bin_negate =", bin_negate) # create negated node | BinOp(expr left, operator op, expr right) new_node = node new_node.op = bin_negate ast.copy_location(new_node, node) ast.fix_missing_locations(new_node) return new_node
def to_node(self): node = self.left.to_node() if len(self.operator) == 0: return node else: for right_node in self.right: node = ast.BinOp(node, ast.BitAnd(), right_node.to_node()) return node
def test_differentops(self): 'Test with other types of operators' tests = [("(3 & 5 & 6)", ast.BoolOp(ast.BitAnd(), [ast.Num(3), ast.Num(5), ast.Num(6)])), ("(1 ^ 2 ^ 3) - 4", ast.BinOp(ast.BoolOp(ast.BitXor(), [ast.Num(1), ast.Num(2), ast.Num(3)]), ast.Add(), ast.BinOp(ast.Num(-1), ast.Mult(), ast.Num(4)))), ("((1 + 2 + 3) & (4 + 5))", ast.BinOp(ast.BoolOp(ast.Add(), [ast.Num(1), ast.Num(2), ast.Num(3)]), ast.BitAnd(), ast.BinOp(ast.Num(4), ast.Add(), ast.Num(5)))), ("(1 & 2 & 3) - (4 & 5)", ast.BinOp(ast.BoolOp(ast.BitAnd(), [ast.Num(1), ast.Num(2), ast.Num(3)]), ast.Add(), ast.BinOp(ast.Num(-1), ast.Mult(), ast.BinOp(ast.Num(4), ast.BitAnd(), ast.Num(5))))), ("(1 & 2 & 3) << (4 & 5)", ast.BinOp(ast.BoolOp(ast.BitAnd(), [ast.Num(1), ast.Num(2), ast.Num(3)]), ast.LShift(), ast.BinOp(ast.Num(4), ast.BitAnd(), ast.Num(5))))] for teststring, ref_ast in tests: test_ast = ast.parse(teststring, mode="eval").body test_ast = pre_processing.all_preprocessings(test_ast) test_ast = asttools.LevelOperators().visit(test_ast) self.assertTrue(asttools.Comparator().visit(test_ast, ref_ast))
def visit_BoolOp(self, node): # first transform children of the node self.generic_visit(node) # using a dict doesn't seem to work if isinstance(node.op, ast.And): new_op = ast.BitAnd() else: assert isinstance(node.op, ast.Or) new_op = ast.BitOr() values = node.values right = values.pop() while len(values): left = values.pop() right = ast.copy_location(ast.BinOp(left, new_op, right), node) return right
def visit_boolop(self, node): # Converts 'and' and 'or' into '&' and '|' which NumExpr supports # BoolOp objects have a list of values but need to be converted into a tree of BinOps if isinstance(node.op, ast.And): new_op = ast.BitAnd() elif isinstance(node.op, ast.Or): new_op = ast.BitOr() else: raise NotImplementedError(type(node.op)) values = node.values left = self.visit(values[0]) i = 1 while i < len(values): right = self.visit(values[i]) left = ast.BinOp(left=left, right=right, op=new_op) i += 1 return left
def BinOp(draw, expression) -> ast.BinOp: op = draw( sampled_from([ ast.Add(), ast.Sub(), ast.Mult(), ast.Div(), ast.FloorDiv(), ast.Mod(), ast.Pow(), ast.LShift(), ast.RShift(), ast.BitOr(), ast.BitXor(), ast.BitOr(), ast.BitAnd(), ast.MatMult() ])) le = draw(lists(expression, min_size=2, max_size=2)) return ast.BinOp(le[0], op, le[1])
def AugAssign(draw): op = draw( sampled_from([ ast.Add(), ast.Sub(), ast.Mult(), ast.Div(), ast.FloorDiv(), ast.Mod(), ast.Pow(), ast.LShift(), ast.RShift(), ast.BitOr(), ast.BitXor(), ast.BitOr(), ast.BitAnd(), ast.MatMult() ])) return ast.AugAssign(target=draw(Name(ast.Store)), op=op, value=draw(expression()))
def __init__(self, str, lineno=0): self.value = str self.lineno = lineno op_ast_map = { '+': ast.Add(), '-': ast.Sub(), '*': ast.Mult(), '/': ast.Div(), '%': ast.Mod(), '**': ast.Pow(), '<<': ast.LShift(), '>>': ast.RShift(), '|': ast.BitOr(), '^^': ast.BitXor(), '&&': ast.BitAnd(), '//': ast.FloorDiv(), '==': ast.Eq(), '!=': ast.NotEq(), '<': ast.Lt(), '<=': ast.LtE(), '>': ast.Gt(), '>=': ast.GtE(), 'is': ast.Is(), 'is_not': ast.IsNot(), 'in': ast.In(), 'not_in': ast.NotIn(), 'and': ast.And(), 'or': ast.Or() }
def mutate_BitXor(self, node): return ast.BitAnd()
def and_test(self, p): return ast.BinOp(left=p.not_test0, op=ast.BitAnd(), right=p.not_test1)
def translateExpr(n): if isinstance(n, Token): if n.type.name == 'INTEGER': if n.arg > 2**63: return ast.NameConstant('__BIGINT__') return ast.Num(n.arg) if n.type.name == 'FLOAT': return ast.Num(n.arg) if n.type.name == 'IDENT': return ast.NameConstant(mungeName(n.arg)) if n.type.name == 'FALSE': return ast.NameConstant('False') if n.type.name == 'TRUE': return ast.NameConstant('True') if n.type.name == 'BITSTRING': if n.arg.find('x') >= 0: return ast.Call(ast.NameConstant('__BITSTRING_MATCH__'), [ast.Str(n.arg)], []) return ast.NameConstant('0b' + n.arg) raise Exception('q %s' % n) if isinstance(n, Type): return ast.Num(800) if n.type == 'e-land': return ast.BoolOp( ast.And(), [translateExpr(n.children[0]), translateExpr(n.children[1])]) elif n.type == 'e-lor': return ast.BoolOp( ast.Or(), [translateExpr(n.children[0]), translateExpr(n.children[1])]) elif n.type == 'e-eq': return ast.Compare(translateExpr(n.children[0]), [ast.Eq()], [translateExpr(n.children[1])]) elif n.type == 'e-ne': return ast.Compare(translateExpr(n.children[0]), [ast.NotEq()], [translateExpr(n.children[1])]) elif n.type == 'e-call': return ast.Call(translateExpr(n.children[0]), [translateExpr(x) for x in n.children[1]], []) elif n.type == 'e-range': return ast.Subscript(translateExpr(n.children[0]), translateRange(n.children[1]), None) elif n.type == 'e-not': return ast.UnaryOp(ast.Not(), translateExpr(n.children[0])) elif n.type == 'e-negate': return ast.UnaryOp(ast.USub(), translateExpr(n.children[0])) elif n.type == 'e-add': return ast.BinOp(translateExpr(n.children[0]), ast.Add(), translateExpr(n.children[1])) elif n.type == 'e-sub': return ast.BinOp(translateExpr(n.children[0]), ast.Sub(), translateExpr(n.children[1])) elif n.type == 'e-mul': return ast.BinOp(translateExpr(n.children[0]), ast.Mult(), translateExpr(n.children[1])) elif n.type == 'e-div': return ast.BinOp(translateExpr(n.children[0]), ast.FloorDiv(), translateExpr(n.children[1])) elif n.type == 'e-fdiv': return ast.BinOp(translateExpr(n.children[0]), ast.Div(), translateExpr(n.children[1])) elif n.type == 'e-rem': return ast.BinOp(translateExpr(n.children[0]), ast.Mod(), translateExpr(n.children[1])) elif n.type == 'e-lt': return ast.Compare(translateExpr(n.children[0]), [ast.Lt()], [translateExpr(n.children[1])]) elif n.type == 'e-le': return ast.Compare(translateExpr(n.children[0]), [ast.LtE()], [translateExpr(n.children[1])]) elif n.type == 'e-gt': return ast.Compare(translateExpr(n.children[0]), [ast.Gt()], [translateExpr(n.children[1])]) elif n.type == 'e-ge': return ast.Compare(translateExpr(n.children[0]), [ast.GtE()], [translateExpr(n.children[1])]) elif n.type == 'e-lsh': return ast.BinOp(translateExpr(n.children[0]), ast.LShift(), translateExpr(n.children[1])) elif n.type == 'e-rsh': return ast.BinOp(translateExpr(n.children[0]), ast.RShift(), translateExpr(n.children[1])) elif n.type == 'e-eor': return ast.BinOp(translateExpr(n.children[0]), ast.BitXor(), translateExpr(n.children[1])) elif n.type == 'e-or': return ast.BinOp(translateExpr(n.children[0]), ast.BitOr(), translateExpr(n.children[1])) elif n.type == 'e-and': return ast.BinOp(translateExpr(n.children[0]), ast.BitAnd(), translateExpr(n.children[1])) elif n.type == 'e-ternary': return ast.IfExp(translateExpr(n.children[0]), translateExpr(n.children[1]), translateExpr(n.children[2])) elif n.type == 'e-subrange': return ast.Num(113) elif n.type == 'e-lnegate': return ast.UnaryOp(ast.Invert(), translateExpr(n.children[0])) elif n.type == 'e-subscript': return ast.Attribute(translateExpr(n.children[0]), n.children[1].arg, None) elif n.type == 'e-indexempty': return ast.Subscript(translateExpr(n.children[0]), ast.Slice(None, None, None), None) elif n.type == 'e-implementation-defined': return ast.Call(ast.NameConstant('IMPLEMENTATION_DEFINED'), [], []) elif n.type == 'e-exp': return ast.BinOp(translateExpr(n.children[0]), ast.Pow(), translateExpr(n.children[1])) elif n.type == 'e-mod': return ast.BinOp(translateExpr(n.children[0]), ast.Mod(), translateExpr(n.children[1])) elif n.type == 'e-unknown': return ast.Call(ast.NameConstant('UNKNOWN'), [], []) elif n.type == 'e-index': return ast.Subscript(translateExpr(n.children[0]), ast.Slice(None, None, None), None) elif n.type == 'e-tuple': return ast.Tuple([translateExpr(x) for x in n.children[0]], None) elif n.type == 'e-set-in': return ast.Num(114) elif n.type == 'e-concat': return ast.Call( ast.NameConstant('_CONCAT_'), [translateExpr(n.children[0]), translateExpr(n.children[1])], []) elif n.type == 'tuple-nomatch': return ast.NameConstant('_') elif n.type == 'range': return ast.Num(122) #return translateRange(n) else: raise Exception('unknown node type: %s' % (n, )) return ast.Num(42)
def and_expr(self, p): return ast.BinOp(left=p.arith_expr0, op=ast.BitAnd(), right=p.arith_expr1)
def visit_And(self, node): self.generic_visit(node) return ast.BitAnd()
def compile_subtract(p): if len(p) == 2: return ast.UnaryOp(ast.USub(), build_ast(p[1])) elif len(p) == 3: return ast.BinOp(build_ast(p[1]), ast.Sub(), build_ast(p[2])) else: return ast.BinOp(compile_subtract(p[:-1]), ast.Sub(), build_ast(p[-1])) CompileFuncs = { Symbol.new("+"): compile_add, Symbol.new("-"): compile_subtract, Symbol.new("*"): compile_multiply, Symbol.new("/"): compile_divide, Symbol.new("//"): compile_floordivide, Symbol.new("%"): lambda p: ast.BinOp(build_ast(p[1]), ast.Mod(), build_ast(p[2])), Symbol.new("&"): lambda p: ast.BinOp(build_ast(p[1]), ast.BitAnd(), build_ast(p[2])), Symbol.new("**"): lambda p: ast.BinOp(build_ast(p[1]), ast.Pow(), build_ast(p[2])), Symbol.new(">>"): lambda p: ast.BinOp(build_ast(p[1]), ast.RShift(), build_ast(p[2])), Symbol.new("<<"): lambda p: ast.BinOp(build_ast(p[1]), ast.LShift(), build_ast(p[2])), Symbol.new("^"): lambda p: ast.BinOp(build_ast(p[1]), ast.BitXor(), build_ast(p[2])), Symbol.new("<"): lambda p: ast.Compare(build_ast(p[1]), [ast.Lt() for x in p[1::2]], [build_ast(x) for x in p[2::2]]), Symbol.new(">"): lambda p: ast.Compare(build_ast(p[1]), [ast.Gt() for x in p[1::2]], [build_ast(x) for x in p[2::2]]), Symbol.new("<="): lambda p: ast.Compare(build_ast(p[1]), [ast.LtE() for x in p[1::2]], [build_ast(x) for x in p[2::2]]), Symbol.new(">="): lambda p: ast.Compare(build_ast(p[1]), [ast.GtE() for x in p[1::2]], [build_ast(x) for x in p[2::2]]), Symbol.new("=="): compile_equals, Symbol.new("!="): lambda p: ast.Compare(build_ast(p[1]), [ast.NotEq() for x in p[1::2]], [build_ast(x) for x in p[2::2]]), Symbol.new("is"): lambda p: ast.Compare(build_ast(p[1]), [ast.Is() for x in p[1::2]], [build_ast(x) for x in p[2::2]]), Symbol.new("is-not"): lambda p: ast.Compare(build_ast(p[1]), [ast.IsNot() for x in p[1::2]], [build_ast(x) for x in p[2::2]]), Symbol.new("define"): compile_define, Symbol.new("dict-set"): lambda p: ast.Assign([ast.Subscript(build_ast(p[1]), ast.Index(build_ast(p[2])), ast.Store())], build_ast(p[3])), #Symbol.new("caadr"): lambda p: compiler.ast.Subscript(compiler.ast.Subscript(build_ast(p[1]), 0, compiler.ast.Const(1)), 0, compiler.ast.Const(0)),
def test_empty_init(self): # Jython 2.5.0 did not allow empty constructors for many ast node types # but CPython ast nodes do allow this. For the moment, I don't see a # reason to allow construction of the super types (like ast.AST and # ast.stmt) as well as the op types that are implemented as enums in # Jython (like boolop), but I've left them in but commented out for # now. We may need them in the future since CPython allows this, but # it may fall under implementation detail. #ast.AST() ast.Add() ast.And() ast.Assert() ast.Assign() ast.Attribute() ast.AugAssign() ast.AugLoad() ast.AugStore() ast.BinOp() ast.BitAnd() ast.BitOr() ast.BitXor() ast.BoolOp() ast.Break() ast.Call() ast.ClassDef() ast.Compare() ast.Continue() ast.Del() ast.Delete() ast.Dict() ast.Div() ast.Ellipsis() ast.Eq() ast.Exec() ast.Expr() ast.Expression() ast.ExtSlice() ast.FloorDiv() ast.For() ast.FunctionDef() ast.GeneratorExp() ast.Global() ast.Gt() ast.GtE() ast.If() ast.IfExp() ast.Import() ast.ImportFrom() ast.In() ast.Index() ast.Interactive() ast.Invert() ast.Is() ast.IsNot() ast.LShift() ast.Lambda() ast.List() ast.ListComp() ast.Load() ast.Lt() ast.LtE() ast.Mod() ast.Module() ast.Mult() ast.Name() ast.Not() ast.NotEq() ast.NotIn() ast.Num() ast.Or() ast.Param() ast.Pass() ast.Pow() ast.Print() ast.RShift() ast.Raise() ast.Repr() ast.Return() ast.Slice() ast.Store() ast.Str() ast.Sub() ast.Subscript() ast.Suite() ast.TryExcept() ast.TryFinally() ast.Tuple() ast.UAdd() ast.USub() ast.UnaryOp() ast.While() ast.With() ast.Yield() ast.alias() ast.arguments() #ast.boolop() #ast.cmpop() ast.comprehension() #ast.excepthandler() #ast.expr() #ast.expr_context() ast.keyword()
'POW_ASN', 'FDIV_ASN', 'NAME', 'NUMBER', ) Add = ast.Add() Sub = ast.Sub() Mult = ast.Mult() Div = ast.Div() Mod = ast.Mod() Pow = ast.Pow() LShift = ast.LShift() RShift = ast.RShift() BitOr = ast.BitOr() BitAnd = ast.BitAnd() BitXor = ast.BitXor() FloorDiv = ast.FloorDiv() op_dict = { "+": Add, "-": Sub, "*": Mult, "/": Div, "//": FloorDiv, "%": Mod, "**": Pow, "<<": LShift, ">>": RShift, "|": BitOr, "&": BitAnd,
class LtnsCompiler: def _temp_func_name(self): if not hasattr(self, '_temp'): self._temp = 0 self._temp += 1 return f'_temp_func_{self._temp}' def _temp_var_name(self): if not hasattr(self, '_temp'): self._temp = 0 self._temp += 1 return f'_temp_var_{self._temp}' def _compile_branch(self, branch): result = Result() for x in branch[:-1]: res = self.compile(x) result.stmts += res.stmts result.stmts += [res.expr_statement] res = self.compile(branch[-1]) result.stmts += res.stmts result.expr = res.expr return result def _compile_args(self, args, kwargs): for i, arg in enumerate(args): if arg == '&': vararg = ast.arg(str(args[i + 1], None), None) posarg = args[:i] else: vararg = None posarg = args return ast.arguments( args=[ast.arg(str(x), None) for x in posarg], vararg=vararg, kwonlyargs=[ast.arg(str(x), None) for x in kwargs], kwarg=None, defaults=[], kw_defaults=list(kwargs.values()), ) def compile(self, node): return _model_compiler[type(node)](self, node) @model(LtnsElement) def compile_element(self, element): if element.name in _special_form_compiler: return _special_form_compiler[element.name](self, *element.childs, **element.attributes) func = ast.parse(element.name, mode='eval').body # TODO: handle exception of parsing result = Result() args = [] keywords = [] for child in element.childs: res = self.compile(child) args.append(res.expr) result.stmts += res.stmts for key, value in element.attributes.items(): res = self.compile(value) keywords.append(ast.keyword(arg=str(key), value=res.expr)) result.stmts += res.stmts expr = ast.Call(func=func, args=args, keywords=keywords) result.expr = expr return result @model(LtnsSymbol) def compile_symbol(self, symbol): expr = ast.Name(id=str(symbol), ctx=ast.Load()) return Result(expr=expr) @model(LtnsString) def compile_string(self, string): return Result(expr=ast.Str(str(string))) @model(LtnsKeyword) def compile_keyword(self, keyword): return Result(expr=ast.Call( func=ast.Name(id='LtnsKeyword', ctx=ast.Load()), args=[ast.Str(str(keyword))], )) @model(LtnsInteger) def compile_integer(self, integer): return Result(expr=ast.Num(int(integer))) @model(LtnsFloat) def compile_float_number(self, float_number): return Result(expr=ast.Num(float(float_number))) @model(LtnsComplex) def compile_complex_number(self, complex_number): return Result(expr=ast.Num(complex(complex_number))) @model(LtnsList) def compile_list(self, ltns_list): result = Result() elts = [] for e in ltns_list: res = self.compile(e) elts.append(res.expr) result.stmts += res.stmts expr = ast.List(elts=elts, ctx=ast.Load()) result.expr = expr return result @special('do') def compile_do(self, *body): return self._compile_branch(body) name_op = { 'add*': ast.Add(), 'sub*': ast.Sub(), 'mul*': ast.Mult(), 'div*': ast.Div(), 'mod': ast.Mod(), 'pow': ast.Pow(), 'lshift': ast.LShift(), 'rshift': ast.RShift(), 'bitor': ast.BitOr(), 'bitxor': ast.BitXor(), 'bitand': ast.BitAnd(), } def compile_bin_op(self, a, b, name): result = Result() left = self.compile(a) result.stmts += left.stmts left = left.expr right = self.compile(b) result.stmts += right.stmts right = right.expr expr = ast.BinOp(op=self.name_op[name], left=left, right=right) result.expr = expr return result for name in name_op: _special_form_compiler[name] = partial(compile_bin_op, name=name) @special('if') def compile_if(self, test, then, orelse=None): result = Result() pred = self.compile(test) result.stmts += pred.stmts then_body = self.compile(then) else_body = None if orelse is not None: else_body = self.compile(orelse) if then_body.stmts or else_body.stmts: temp_func_name = self._temp_func_name() temp_var_name = self._temp_var_name() body = then_body.stmts body.append( ast.Assign( targets=[ast.Name(id=temp_var_name, ctx=ast.Store())], value=then_body.expr, )) orelse = else_body.stmts orelse.append( ast.Assign( targets=[ast.Name(id=temp_var_name, ctx=ast.Store())], value=else_body.expr, )) result.stmts.append( ast.FunctionDef( name=temp_func_name, args=ast.arguments( args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[], ), body=[ ast.If(test=pred.expr, body=body, orelse=orelse), ast.Return( value=ast.Name(id=temp_var_name, ctx=ast.Load())), ], decorator_list=[], returns=None, )) result.expr = ast.Call( func=ast.Name(id=temp_func_name, ctx=ast.Load()), args=[], keywords=[], ) return result else: result.expr = ast.IfExp( test=pred.expr, body=then_body.expr, orelse=else_body.expr, ) return result @special('fn*') def compile_fn(self, args, *body, **kwargs): result = Result() body = self._compile_branch(body) args = self._compile_args(args, kwargs) if body.stmts: fdef = ast.FunctionDef() fdef.name = self._temp_func_name() fdef.args = args fdef.body = body.stmts + [ast.Return(body.expr)] fdef.decorator_list = [] fdef.returns = None result.stmts += [fdef] result.expr = ast.Name(id=fdef.name, ctx=ast.Load()) else: result.expr = ast.Lambda(args=args, body=body.expr) return result @special('def') def compile_def(self, *name_value, **def_dict): if len(name_value) % 2 != 0: raise ValueError("length of argument list should be even") result = Result() for name, value in def_dict.items(): name = LtnsSymbol(name) name_value = name_value + (name, value) for i, name in enumerate(name_value[::2]): name = ast.Name(id=str(name), ctx=ast.Store()) value = self.compile(name_value[i * 2 + 1]) result.stmts += [ast.Assign( targets=[name], value=value.expr, )] result.expr = ast.Name(id='None', ctx=ast.Load()) return result
def and_expr_rewrite(head, tail): if tail: for op, each in tail: head = ast.BinOp(head, ast.BitAnd(), each, **loc @ op) return head
def bitand(self, other: Union[T, Expr[T]]) -> BinOp[T]: return BinOp(self, ast.BitAnd(), other)
def as_ast(dct): """See https://docs.python.org/2/library/ast.html""" if dct['ast_type'] == "Module": return ast.Module(dct["body"]) elif dct['ast_type'] == "Interactive": return ast.Interactive(dct["body"]) elif dct['ast_type'] == "Expression": return ast.Expression(dct["body"]) elif dct['ast_type'] == "Suite": return ast.Suite(dct["body"]) elif dct['ast_type'] == "FunctionDef": return ast.FunctionDef(dct["name"], dct["args"], dct["body"], dct["decorator_list"]) elif dct['ast_type'] == "ClassDef": return ast.ClassDef(dct["name"], dct["bases"], dct["body"], dct["decorator_list"]) elif dct['ast_type'] == "Return": return ast.Return(dct["value"]) elif dct['ast_type'] == "Delete": return ast.Delete(dct["targets"]) elif dct['ast_type'] == "Assign": return ast.Assign(dct["targets"], dct["value"]) elif dct['ast_type'] == "AugAssign": return ast.AugAssign(dct["target"], dct["op"], dct["value"]) elif dct['ast_type'] == "Print": return ast.Print(dct["dest"], dct["values"], dct["nl"]) elif dct['ast_type'] == "For": return ast.For(dct["target"], dct["iter"], dct["body"], dct["orelse"]) elif dct['ast_type'] == "While": return ast.While(dct["test"], dct["body"], dct["orelse"]) elif dct['ast_type'] == "If": return ast.If(dct["test"], dct["body"], dct["orelse"]) elif dct['ast_type'] == "With": return ast.With(dct["context_expr"], dct["optional_vars"], dct["body"]) elif dct['ast_type'] == "Raise": return ast.Raise(dct["type"], dct["inst"], dct["tback"]) elif dct['ast_type'] == "TryExcept": return ast.TryExcept(dct["body"], dct["handlers"], dct["orelse"]) elif dct['ast_type'] == "TryFinally": return ast.TryFinally(dct["body"], dct["finalbody"]) elif dct['ast_type'] == "Assert": return ast.Assert(dct["test"], dct["msg"]) elif dct['ast_type'] == "Import": return ast.Import(dct["names"]) elif dct['ast_type'] == "ImportFrom": return ast.ImportFrom(dct["module"], dct["names"], dct["level"]) elif dct['ast_type'] == "Exec": return ast.Exec(dct["body"], dct["globals"], dct["locals"]) elif dct['ast_type'] == "Global": return ast.Global(dct["names"]) elif dct['ast_type'] == "Expr": return ast.Expr(dct["value"]) elif dct['ast_type'] == "Pass": return ast.Pass() elif dct['ast_type'] == "Break": return ast.Break() elif dct['ast_type'] == "Continue": return ast.Continue() elif dct['ast_type'] == "BoolOp": return ast.BoolOp(dct["op"], dct["values"]) elif dct['ast_type'] == "BinOp": return ast.BinOp(dct["left"], dct["op"], dct["right"]) elif dct['ast_type'] == "UnaryOp": return ast.UnaryOp(dct["op"], dct["operand"]) elif dct['ast_type'] == "Lambda": return ast.Lambda(dct["args"], dct["body"]) elif dct['ast_type'] == "IfExp": return ast.IfExp(dct["test"], dct["body"], dct["orelse"]) elif dct['ast_type'] == "Dict": return ast.Dict(dct["keys"], dct["values"]) elif dct['ast_type'] == "Set": return ast.Set(dct["elts"]) elif dct['ast_type'] == "ListComp": return ast.ListComp(dct["elt"], dct["generators"]) elif dct['ast_type'] == "SetComp": return ast.SetComp(dct["elt"], dct["generators"]) elif dct['ast_type'] == "DictComp": return ast.DictComp(dct["key"], dct["value"], dct["generators"]) elif dct['ast_type'] == "GeneratorExp": return ast.GeneratorExp(dct["elt"], dct["generators"]) elif dct['ast_type'] == "Yield": return ast.Yield(dct["value"]) elif dct['ast_type'] == "Compare": return ast.Compare(dct["left"], dct["ops"], dct["comparators"]) elif dct['ast_type'] == "Call": return ast.Call(dct["func"], dct["args"], dct["keywords"], dct["starargs"], dct["kwargs"]) elif dct['ast_type'] == "Repr": return ast.Repr(dct["value"]) elif dct['ast_type'] == "Num": return ast.Num(dct["n"]) elif dct['ast_type'] == "Str": # Converting to ASCII return ast.Str(dct["s"].encode('ascii', 'ignore')) elif dct['ast_type'] == "Attribute": return ast.Attribute(dct["value"], dct["attr"], dct["ctx"]) elif dct['ast_type'] == "Subscript": return ast.Subscript(dct["value"], dct["slice"], dct["ctx"]) elif dct['ast_type'] == "Name": return ast.Name(dct["id"], dct["ctx"]) elif dct['ast_type'] == "List": return ast.List(dct["elts"], dct["ctx"]) elif dct['ast_type'] == "Tuple": return ast.Tuple(dct["elts"], dct["ctx"]) elif dct['ast_type'] == "Load": return ast.Load() elif dct['ast_type'] == "Store": return ast.Store() elif dct['ast_type'] == "Del": return ast.Del() elif dct['ast_type'] == "AugLoad": return ast.AugLoad() elif dct['ast_type'] == "AugStore": return ast.AugStore() elif dct['ast_type'] == "Param": return ast.Param() elif dct['ast_type'] == "Ellipsis": return ast.Ellipsis() elif dct['ast_type'] == "Slice": return ast.Slice(dct["lower"], dct["upper"], dct["step"]) elif dct['ast_type'] == "ExtSlice": return ast.ExtSlice(dct["dims"]) elif dct['ast_type'] == "Index": return ast.Index(dct["value"]) elif dct['ast_type'] == "And": return ast.And() elif dct['ast_type'] == "Or": return ast.Or() elif dct['ast_type'] == "Add": return ast.Add() elif dct['ast_type'] == "Sub": return ast.Sub() elif dct['ast_type'] == "Mult": return ast.Mult() elif dct['ast_type'] == "Div": return ast.Div() elif dct['ast_type'] == "Mod": return ast.Mod() elif dct['ast_type'] == "Pow": return ast.Pow() elif dct['ast_type'] == "LShift": return ast.LShift() elif dct['ast_type'] == "RShift": return ast.RShift() elif dct['ast_type'] == "BitOr": return ast.BitOr() elif dct['ast_type'] == "BitXor": return ast.BitXor() elif dct['ast_type'] == "BitAnd": return ast.BitAnd() elif dct['ast_type'] == "FloorDiv": return ast.FloorDiv() elif dct['ast_type'] == "Invert": return ast.Invert() elif dct['ast_type'] == "Not": return ast.Not() elif dct['ast_type'] == "UAdd": return ast.UAdd() elif dct['ast_type'] == "USub": return ast.USub() elif dct['ast_type'] == "Eq": return ast.Eq() elif dct['ast_type'] == "NotEq": return ast.NotEq() elif dct['ast_type'] == "Lt": return ast.Lt() elif dct['ast_type'] == "LtE": return ast.LtE() elif dct['ast_type'] == "Gt": return ast.Gt() elif dct['ast_type'] == "GtE": return ast.GtE() elif dct['ast_type'] == "Is": return ast.Is() elif dct['ast_type'] == "IsNot": return ast.IsNot() elif dct['ast_type'] == "In": return ast.In() elif dct['ast_type'] == "NotIn": return ast.NotIn() elif dct['ast_type'] == "comprehension": return ast.comprehension(dct["target"], dct["iter"], dct["ifs"]) elif dct['ast_type'] == "ExceptHandler": return ast.ExceptHandler(dct["type"], dct["name"], dct["body"]) elif dct['ast_type'] == "arguments": return ast.arguments(dct["args"], dct["vararg"], dct["kwarg"], dct["defaults"]) elif dct['ast_type'] == "keyword": return ast.keyword(dct["arg"], dct["value"]) elif dct['ast_type'] == "alias": return ast.alias(dct["name"], dct["asname"]) else: return dct
class helper(object): def __init__(self, node): self.node = node def __ast__(self): return self.node @property def body(self): return E(self.node.body) @body.setter def body(self, e): self.node.body = e def call(self, args=None): self.node = ast.Call(func=self.node, args=args if args else [], keywords=[], starargs=None, kwargs=None) return self def args(self, args): self.node.args = args return self def subscript(self, index=None): self.node = ast.Subscript(value=self.node, slice=ast.Index(index)) return self def attr(self, attribute): assert (isinstance(attribute, str)) self.node = ast.Attribute(value=self.node, attr=attribute) return self def assign(self, value): self.node = ast.Assign(targets=[self.node], value=E(value)) return self def init(self, value): self.node.init = value return self sub_assign = _aug_assign(ast.Sub()) add_assign = _aug_assign(ast.Add()) xor_assign = _aug_assign(ast.BitXor()) or_assign = _aug_assign(ast.BitOr()) mult = _bin_op(ast.Mult()) add = _bin_op(ast.Add()) bit_and = _bin_op(ast.BitAnd()) bit_or = _bin_op(ast.BitOr()) bit_xor = _bin_op(ast.BitXor()) xor = bit_xor NotEq = _compare2(ast.NotEq()) Eq = _compare2(ast.Eq()) Gt = _compare2(ast.Gt()) Lt = _compare2(ast.Lt()) def ast(self): return self.node