def visit_compare(self, compare_node: ast.Compare):
# Visit downstream symbols
new_left = self.visit(compare_node.left)
new_comparators = [
self.visit(node) for node in compare_node.comparators
]
# Simple comparison node
if len(new_comparators) == 1:
return ast.Compare(left=new_left,
comparators=[new_comparators[0]],
ops=compare_node.ops)
# Compound comparison node
# Convert to multiple Compare operations
new_nodes, left = [], new_left
for right, op in zip(new_comparators, compare_node.ops):
new_node = ast.Compare(left=left, comparators=[right], ops=[op])
new_nodes.append(new_node)
left = right
assert len(new_nodes) >= 2
bin_op = ast.BinOp(left=new_nodes[0],
right=new_nodes[1],
op=ast.BitAnd())
for new_right in new_nodes[2:]:
bin_op = ast.BinOp(left=bin_op, right=new_right, op=ast.BitAnd())
return bin_op
def mutate(cls, node):
if node not in config.visited_nodes:
if node.__class__ is ast.BitAnd:
config.mutated = True
original_node = deepcopy(node)
parent = config.parent_dict[node]
del config.parent_dict[node]
node = ast.BitOr()
config.parent_dict[node] = parent
config.node_pairs[node] = original_node
config.current_mutated_node = node
elif node.__class__ is ast.BitOr:
config.mutated = True
original_node = deepcopy(node)
parent = config.parent_dict[node]
del config.parent_dict[node]
node = ast.BitAnd()
config.parent_dict[node] = parent
config.node_pairs[node] = original_node
config.current_mutated_node = node
elif node.__class__ is ast.BitXor:
config.mutated = True
original_node = deepcopy(node)
parent = config.parent_dict[node]
del config.parent_dict[node]
node = ast.BitAnd()
config.parent_dict[node] = parent
config.node_pairs[node] = original_node
config.current_mutated_node = node
elif node.__class__ is ast.LShift:
config.mutated = True
original_node = deepcopy(node)
parent = config.parent_dict[node]
del config.parent_dict[node]
node = ast.RShift()
config.parent_dict[node] = parent
config.node_pairs[node] = original_node
config.current_mutated_node = node
elif node.__class__ is ast.RShift:
config.mutated = True
original_node = deepcopy(node)
parent = config.parent_dict[node]
del config.parent_dict[node]
node = ast.LShift()
config.parent_dict[node] = parent
config.node_pairs[node] = original_node
config.current_mutated_node = node
return node
def visit_Compare(self, ast_node_Compare: ast.Compare) -> ast.Constant:
"""
Compare nodes occur for all sequences of comparison (gt, lt, etc.) operators.
:param ast_node_Compare: a object comparison of two or more values.
:return: evaluated value.
"""
comparators = ast_node_Compare.comparators
operators = ast_node_Compare.ops
# Exception case: (A cmp ?)
if operator.le(len(comparators), 0):
raise ValueError('There must be at least two operands for compare')
# Common case: (A cmp B)
if operator.eq(len(comparators), 1):
ast_node_BinOp = ast.BinOp(ast_node_Compare.left,
next(iter(operators), None),
next(iter(comparators), None))
return self.visit(ast_node_BinOp)
else:
# Recursive case: (A cmp B cmp C cmp ... etc.)
ast_values_BoolOp = []
left_comparator = ast_node_Compare.left
for ast_type_operator, right_comparator in zip(
operators, comparators):
new_ast_node_Compare = self.visit(
ast.Compare(left_comparator, [ast_type_operator],
[right_comparator]))
left_comparator = right_comparator
ast_values_BoolOp.append(new_ast_node_Compare)
return self.visit(ast.BoolOp(ast.BitAnd(), ast_values_BoolOp))
def __init__(self, base_node):
BaseMutator.__init__(self, base_node)
self.original_bin_op = base_node.op
if type(base_node.op) in [
ast.LShift, ast.RShift, ast.BitOr, ast.BitXor, ast.BitAnd,
ast.FloorDiv
]:
if type(base_node.op) is ast.LShift:
self.mutations.append({"op": ast.RShift()})
if type(base_node.op) is ast.RShift:
self.mutations.append({"op": ast.LShift()})
if type(base_node.op) is ast.BitOr:
self.mutations.append({"op": ast.BitAnd()})
if type(base_node.op) is ast.BitXor:
self.mutations.append({"op": ast.FloorDiv()})
if type(base_node.op) is ast.BitAnd:
self.mutations.append({"op": ast.BitOr()})
if type(base_node.op) is ast.FloorDiv:
self.mutations.append({"op": ast.BitXor()})
def visit_Call(self, node):
'Replace custom function with bitwise operators'
self.generic_visit(node)
if isinstance(node.func, ast.Name):
if len(node.args) == 2:
if node.func.id == "mand":
op = ast.BitAnd()
elif node.func.id == "mxor":
op = ast.BitXor()
elif node.func.id == "mor":
op = ast.BitOr()
elif node.func.id == "mlshift":
op = ast.LShift()
elif node.func.id == "mrshift":
op = ast.RShift()
else:
return node
return ast.BinOp(node.args[0], op, node.args[1])
elif len(node.args) == 1 and node.func.id == "mnot":
arg = node.args[0]
self.generic_visit(node)
return ast.UnaryOp(ast.Invert(), arg)
return self.generic_visit(node)
def visit_Operator(self, node: Operator, *args, **kwargs) -> C.operator:
if node == Operator.Add:
return C.Add()
elif node == Operator.Sub:
return C.Sub()
elif node == Operator.Mult:
return C.Mult()
elif node == Operator.MatMult:
return C.MatMult()
elif node == Operator.Div:
return C.Div()
elif node == Operator.Mod:
return C.Mod()
elif node == Operator.Pow:
return C.Pow()
elif node == Operator.LShift:
return C.LShift()
elif node == Operator.RShift:
return C.RShift()
elif node == Operator.BitOr:
return C.BitOr()
elif node == Operator.BitXor:
return C.BitXor()
elif node == Operator.BitAnd:
return C.BitAnd()
elif node == Operator.FloorDiv:
return C.FloorDiv()
else:
raise Exception(f'unknown Operator {node!r}')
def binop_action(s, loc, tokens):
node = tokens[0]
for op_char, right in tokens[1:]:
if op_char == '+':
op = ast.Add()
elif op_char == '-':
op = ast.Sub()
elif op_char == '*':
op = ast.Mult()
elif op_char == '/':
op = ast.Div()
elif op_char == '%':
op = ast.Mod()
elif op_char == '<<':
op = ast.LShift()
elif op_char == '>>':
op = ast.RShift()
elif op_char == '&':
op = ast.BitAnd()
elif op_char == '^':
op = ast.BitXor()
else: # op_char == '|':
op = ast.BitOr()
node = ast.BinOp(left=node, op=op, right=right, lineno=1, col_offset=0)
return node
def bit_and(self):
left = self.bit_shift()
while self.match_(TokenType.AMPERSAND):
right = self.bit_shift()
left = ast.BinOp(left, ast.BitAnd(), right,
**self.get_loc(left, right))
return left
def get_bin_op(self, s):
"""Get the BinOp class for s."""
op = None
if s == '+':
op = ast.Add()
elif s == '-':
op = ast.Sub()
elif s == '*':
op = ast.Mult()
elif s == '/':
op = ast.Div()
elif s == '%':
op = ast.Mod()
elif s == '<<':
op = ast.LShift()
elif s == '>>':
op = ast.RShift()
elif s == '&':
op = ast.BitAnd()
elif s == '^':
op = ast.BitXor()
elif s == '|':
op = ast.BitOr()
return op
def visit_BinOp(self, node):
# operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift | RShift | BitOr | BitXor | BitAnd | FloorDiv
print(" in MyTransformer.visit_BinOp()")
print(" curr op =", node.op)
bin_negate = node.op
# use pseudorandomness to determine whether to negate or just mix up
rand_num = random.randint(1, 10)
# negate
if rand_num >= 4:
print(" negating...")
if isinstance(node.op, ast.Add): bin_negate = ast.Sub()
elif isinstance(node.op, ast.Sub): bin_negate = ast.Add()
elif isinstance(node.op, ast.Mult): bin_negate = ast.Div()
elif isinstance(node.op, ast.Div): bin_negate = ast.FloorDiv()
elif isinstance(node.op, ast.FloorDiv): bin_negate = ast.Div()
elif isinstance(node.op, ast.LShift): bin_negate = ast.RShift()
elif isinstance(node.op, ast.RShift): bin_negate = ast.LShift()
elif isinstance(node.op, ast.BitOr): bin_negate = ast.BitAnd()
elif isinstance(node.op, ast.BitAnd): bin_negate = ast.BitXor()
elif isinstance(node.op, ast.BitXor): bin_negate = ast.BitOr()
elif isinstance(node.op, ast.Pow): bin_negate = ast.Mult()
elif isinstance(node.op, ast.Mod): bin_negate = ast.Div()
elif isinstance(node.op, ast.MatMult): bin_negate = ast.Mult()
else: print(" did not find negation for", node.op)
# mix up
else:
print(" mixing up...")
if isinstance(node.op, ast.Add): bin_negate = ast.Mult()
elif isinstance(node.op, ast.Sub): bin_negate = ast.Div()
elif isinstance(node.op, ast.Mult): bin_negate = ast.Pow()
elif isinstance(node.op, ast.Div): bin_negate = ast.FloorDiv()
elif isinstance(node.op, ast.FloorDiv): bin_negate = ast.Div()
elif isinstance(node.op, ast.BitOr): bin_negate = ast.BitXor()
elif isinstance(node.op, ast.BitAnd): bin_negate = ast.BitOr()
elif isinstance(node.op, ast.BitXor): bin_negate = ast.BitOr()
elif isinstance(node.op, ast.Pow): bin_negate = ast.Mult()
elif isinstance(node.op, ast.Mod): bin_negate = ast.FloorDiv()
else: print(" did not find negation for", node.op)
print(" bin_negate =", bin_negate)
# create negated node | BinOp(expr left, operator op, expr right)
new_node = node
new_node.op = bin_negate
ast.copy_location(new_node, node)
ast.fix_missing_locations(new_node)
return new_node
def to_node(self):
node = self.left.to_node()
if len(self.operator) == 0:
return node
else:
for right_node in self.right:
node = ast.BinOp(node, ast.BitAnd(), right_node.to_node())
return node
def test_differentops(self):
'Test with other types of operators'
tests = [("(3 & 5 & 6)",
ast.BoolOp(ast.BitAnd(),
[ast.Num(3), ast.Num(5), ast.Num(6)])),
("(1 ^ 2 ^ 3) - 4",
ast.BinOp(ast.BoolOp(ast.BitXor(),
[ast.Num(1), ast.Num(2), ast.Num(3)]),
ast.Add(),
ast.BinOp(ast.Num(-1), ast.Mult(), ast.Num(4)))),
("((1 + 2 + 3) & (4 + 5))",
ast.BinOp(ast.BoolOp(ast.Add(),
[ast.Num(1), ast.Num(2), ast.Num(3)]),
ast.BitAnd(),
ast.BinOp(ast.Num(4), ast.Add(), ast.Num(5)))),
("(1 & 2 & 3) - (4 & 5)",
ast.BinOp(ast.BoolOp(ast.BitAnd(),
[ast.Num(1), ast.Num(2), ast.Num(3)]),
ast.Add(),
ast.BinOp(ast.Num(-1), ast.Mult(),
ast.BinOp(ast.Num(4), ast.BitAnd(),
ast.Num(5))))),
("(1 & 2 & 3) << (4 & 5)",
ast.BinOp(ast.BoolOp(ast.BitAnd(),
[ast.Num(1), ast.Num(2), ast.Num(3)]),
ast.LShift(),
ast.BinOp(ast.Num(4), ast.BitAnd(), ast.Num(5))))]
for teststring, ref_ast in tests:
test_ast = ast.parse(teststring, mode="eval").body
test_ast = pre_processing.all_preprocessings(test_ast)
test_ast = asttools.LevelOperators().visit(test_ast)
self.assertTrue(asttools.Comparator().visit(test_ast, ref_ast))
def visit_BoolOp(self, node):
# first transform children of the node
self.generic_visit(node)
# using a dict doesn't seem to work
if isinstance(node.op, ast.And):
new_op = ast.BitAnd()
else:
assert isinstance(node.op, ast.Or)
new_op = ast.BitOr()
values = node.values
right = values.pop()
while len(values):
left = values.pop()
right = ast.copy_location(ast.BinOp(left, new_op, right), node)
return right
def visit_boolop(self, node):
# Converts 'and' and 'or' into '&' and '|' which NumExpr supports
# BoolOp objects have a list of values but need to be converted into a tree of BinOps
if isinstance(node.op, ast.And):
new_op = ast.BitAnd()
elif isinstance(node.op, ast.Or):
new_op = ast.BitOr()
else:
raise NotImplementedError(type(node.op))
values = node.values
left = self.visit(values[0])
i = 1
while i < len(values):
right = self.visit(values[i])
left = ast.BinOp(left=left, right=right, op=new_op)
i += 1
return left
def BinOp(draw, expression) -> ast.BinOp:
op = draw(
sampled_from([
ast.Add(),
ast.Sub(),
ast.Mult(),
ast.Div(),
ast.FloorDiv(),
ast.Mod(),
ast.Pow(),
ast.LShift(),
ast.RShift(),
ast.BitOr(),
ast.BitXor(),
ast.BitOr(),
ast.BitAnd(),
ast.MatMult()
]))
le = draw(lists(expression, min_size=2, max_size=2))
return ast.BinOp(le[0], op, le[1])
def AugAssign(draw):
op = draw(
sampled_from([
ast.Add(),
ast.Sub(),
ast.Mult(),
ast.Div(),
ast.FloorDiv(),
ast.Mod(),
ast.Pow(),
ast.LShift(),
ast.RShift(),
ast.BitOr(),
ast.BitXor(),
ast.BitOr(),
ast.BitAnd(),
ast.MatMult()
]))
return ast.AugAssign(target=draw(Name(ast.Store)),
op=op,
value=draw(expression()))
def __init__(self, str, lineno=0):
self.value = str
self.lineno = lineno
op_ast_map = {
'+': ast.Add(),
'-': ast.Sub(),
'*': ast.Mult(),
'/': ast.Div(),
'%': ast.Mod(),
'**': ast.Pow(),
'<<': ast.LShift(),
'>>': ast.RShift(),
'|': ast.BitOr(),
'^^': ast.BitXor(),
'&&': ast.BitAnd(),
'//': ast.FloorDiv(),
'==': ast.Eq(),
'!=': ast.NotEq(),
'<': ast.Lt(),
'<=': ast.LtE(),
'>': ast.Gt(),
'>=': ast.GtE(),
'is': ast.Is(),
'is_not': ast.IsNot(),
'in': ast.In(),
'not_in': ast.NotIn(),
'and': ast.And(),
'or': ast.Or()
}
def mutate_BitXor(self, node):
return ast.BitAnd()
def and_test(self, p):
return ast.BinOp(left=p.not_test0, op=ast.BitAnd(), right=p.not_test1)
def translateExpr(n):
if isinstance(n, Token):
if n.type.name == 'INTEGER':
if n.arg > 2**63:
return ast.NameConstant('__BIGINT__')
return ast.Num(n.arg)
if n.type.name == 'FLOAT':
return ast.Num(n.arg)
if n.type.name == 'IDENT':
return ast.NameConstant(mungeName(n.arg))
if n.type.name == 'FALSE':
return ast.NameConstant('False')
if n.type.name == 'TRUE':
return ast.NameConstant('True')
if n.type.name == 'BITSTRING':
if n.arg.find('x') >= 0:
return ast.Call(ast.NameConstant('__BITSTRING_MATCH__'),
[ast.Str(n.arg)], [])
return ast.NameConstant('0b' + n.arg)
raise Exception('q %s' % n)
if isinstance(n, Type):
return ast.Num(800)
if n.type == 'e-land':
return ast.BoolOp(
ast.And(),
[translateExpr(n.children[0]),
translateExpr(n.children[1])])
elif n.type == 'e-lor':
return ast.BoolOp(
ast.Or(),
[translateExpr(n.children[0]),
translateExpr(n.children[1])])
elif n.type == 'e-eq':
return ast.Compare(translateExpr(n.children[0]), [ast.Eq()],
[translateExpr(n.children[1])])
elif n.type == 'e-ne':
return ast.Compare(translateExpr(n.children[0]), [ast.NotEq()],
[translateExpr(n.children[1])])
elif n.type == 'e-call':
return ast.Call(translateExpr(n.children[0]),
[translateExpr(x) for x in n.children[1]], [])
elif n.type == 'e-range':
return ast.Subscript(translateExpr(n.children[0]),
translateRange(n.children[1]), None)
elif n.type == 'e-not':
return ast.UnaryOp(ast.Not(), translateExpr(n.children[0]))
elif n.type == 'e-negate':
return ast.UnaryOp(ast.USub(), translateExpr(n.children[0]))
elif n.type == 'e-add':
return ast.BinOp(translateExpr(n.children[0]), ast.Add(),
translateExpr(n.children[1]))
elif n.type == 'e-sub':
return ast.BinOp(translateExpr(n.children[0]), ast.Sub(),
translateExpr(n.children[1]))
elif n.type == 'e-mul':
return ast.BinOp(translateExpr(n.children[0]), ast.Mult(),
translateExpr(n.children[1]))
elif n.type == 'e-div':
return ast.BinOp(translateExpr(n.children[0]), ast.FloorDiv(),
translateExpr(n.children[1]))
elif n.type == 'e-fdiv':
return ast.BinOp(translateExpr(n.children[0]), ast.Div(),
translateExpr(n.children[1]))
elif n.type == 'e-rem':
return ast.BinOp(translateExpr(n.children[0]), ast.Mod(),
translateExpr(n.children[1]))
elif n.type == 'e-lt':
return ast.Compare(translateExpr(n.children[0]), [ast.Lt()],
[translateExpr(n.children[1])])
elif n.type == 'e-le':
return ast.Compare(translateExpr(n.children[0]), [ast.LtE()],
[translateExpr(n.children[1])])
elif n.type == 'e-gt':
return ast.Compare(translateExpr(n.children[0]), [ast.Gt()],
[translateExpr(n.children[1])])
elif n.type == 'e-ge':
return ast.Compare(translateExpr(n.children[0]), [ast.GtE()],
[translateExpr(n.children[1])])
elif n.type == 'e-lsh':
return ast.BinOp(translateExpr(n.children[0]), ast.LShift(),
translateExpr(n.children[1]))
elif n.type == 'e-rsh':
return ast.BinOp(translateExpr(n.children[0]), ast.RShift(),
translateExpr(n.children[1]))
elif n.type == 'e-eor':
return ast.BinOp(translateExpr(n.children[0]), ast.BitXor(),
translateExpr(n.children[1]))
elif n.type == 'e-or':
return ast.BinOp(translateExpr(n.children[0]), ast.BitOr(),
translateExpr(n.children[1]))
elif n.type == 'e-and':
return ast.BinOp(translateExpr(n.children[0]), ast.BitAnd(),
translateExpr(n.children[1]))
elif n.type == 'e-ternary':
return ast.IfExp(translateExpr(n.children[0]),
translateExpr(n.children[1]),
translateExpr(n.children[2]))
elif n.type == 'e-subrange':
return ast.Num(113)
elif n.type == 'e-lnegate':
return ast.UnaryOp(ast.Invert(), translateExpr(n.children[0]))
elif n.type == 'e-subscript':
return ast.Attribute(translateExpr(n.children[0]), n.children[1].arg,
None)
elif n.type == 'e-indexempty':
return ast.Subscript(translateExpr(n.children[0]),
ast.Slice(None, None, None), None)
elif n.type == 'e-implementation-defined':
return ast.Call(ast.NameConstant('IMPLEMENTATION_DEFINED'), [], [])
elif n.type == 'e-exp':
return ast.BinOp(translateExpr(n.children[0]), ast.Pow(),
translateExpr(n.children[1]))
elif n.type == 'e-mod':
return ast.BinOp(translateExpr(n.children[0]), ast.Mod(),
translateExpr(n.children[1]))
elif n.type == 'e-unknown':
return ast.Call(ast.NameConstant('UNKNOWN'), [], [])
elif n.type == 'e-index':
return ast.Subscript(translateExpr(n.children[0]),
ast.Slice(None, None, None), None)
elif n.type == 'e-tuple':
return ast.Tuple([translateExpr(x) for x in n.children[0]], None)
elif n.type == 'e-set-in':
return ast.Num(114)
elif n.type == 'e-concat':
return ast.Call(
ast.NameConstant('_CONCAT_'),
[translateExpr(n.children[0]),
translateExpr(n.children[1])], [])
elif n.type == 'tuple-nomatch':
return ast.NameConstant('_')
elif n.type == 'range':
return ast.Num(122) #return translateRange(n)
else:
raise Exception('unknown node type: %s' % (n, ))
return ast.Num(42)
def and_expr(self, p):
return ast.BinOp(left=p.arith_expr0,
op=ast.BitAnd(),
right=p.arith_expr1)
def visit_And(self, node):
self.generic_visit(node)
return ast.BitAnd()
def compile_subtract(p):
if len(p) == 2:
return ast.UnaryOp(ast.USub(), build_ast(p[1]))
elif len(p) == 3:
return ast.BinOp(build_ast(p[1]), ast.Sub(), build_ast(p[2]))
else:
return ast.BinOp(compile_subtract(p[:-1]), ast.Sub(), build_ast(p[-1]))
CompileFuncs = {
Symbol.new("+"): compile_add,
Symbol.new("-"): compile_subtract,
Symbol.new("*"): compile_multiply,
Symbol.new("/"): compile_divide,
Symbol.new("//"): compile_floordivide,
Symbol.new("%"): lambda p: ast.BinOp(build_ast(p[1]), ast.Mod(), build_ast(p[2])),
Symbol.new("&"): lambda p: ast.BinOp(build_ast(p[1]), ast.BitAnd(), build_ast(p[2])),
Symbol.new("**"): lambda p: ast.BinOp(build_ast(p[1]), ast.Pow(), build_ast(p[2])),
Symbol.new(">>"): lambda p: ast.BinOp(build_ast(p[1]), ast.RShift(), build_ast(p[2])),
Symbol.new("<<"): lambda p: ast.BinOp(build_ast(p[1]), ast.LShift(), build_ast(p[2])),
Symbol.new("^"): lambda p: ast.BinOp(build_ast(p[1]), ast.BitXor(), build_ast(p[2])),
Symbol.new("<"): lambda p: ast.Compare(build_ast(p[1]), [ast.Lt() for x in p[1::2]], [build_ast(x) for x in p[2::2]]),
Symbol.new(">"): lambda p: ast.Compare(build_ast(p[1]), [ast.Gt() for x in p[1::2]], [build_ast(x) for x in p[2::2]]),
Symbol.new("<="): lambda p: ast.Compare(build_ast(p[1]), [ast.LtE() for x in p[1::2]], [build_ast(x) for x in p[2::2]]),
Symbol.new(">="): lambda p: ast.Compare(build_ast(p[1]), [ast.GtE() for x in p[1::2]], [build_ast(x) for x in p[2::2]]),
Symbol.new("=="): compile_equals,
Symbol.new("!="): lambda p: ast.Compare(build_ast(p[1]), [ast.NotEq() for x in p[1::2]], [build_ast(x) for x in p[2::2]]),
Symbol.new("is"): lambda p: ast.Compare(build_ast(p[1]), [ast.Is() for x in p[1::2]], [build_ast(x) for x in p[2::2]]),
Symbol.new("is-not"): lambda p: ast.Compare(build_ast(p[1]), [ast.IsNot() for x in p[1::2]], [build_ast(x) for x in p[2::2]]),
Symbol.new("define"): compile_define,
Symbol.new("dict-set"): lambda p: ast.Assign([ast.Subscript(build_ast(p[1]), ast.Index(build_ast(p[2])), ast.Store())], build_ast(p[3])),
#Symbol.new("caadr"): lambda p: compiler.ast.Subscript(compiler.ast.Subscript(build_ast(p[1]), 0, compiler.ast.Const(1)), 0, compiler.ast.Const(0)),
def test_empty_init(self):
# Jython 2.5.0 did not allow empty constructors for many ast node types
# but CPython ast nodes do allow this. For the moment, I don't see a
# reason to allow construction of the super types (like ast.AST and
# ast.stmt) as well as the op types that are implemented as enums in
# Jython (like boolop), but I've left them in but commented out for
# now. We may need them in the future since CPython allows this, but
# it may fall under implementation detail.
#ast.AST()
ast.Add()
ast.And()
ast.Assert()
ast.Assign()
ast.Attribute()
ast.AugAssign()
ast.AugLoad()
ast.AugStore()
ast.BinOp()
ast.BitAnd()
ast.BitOr()
ast.BitXor()
ast.BoolOp()
ast.Break()
ast.Call()
ast.ClassDef()
ast.Compare()
ast.Continue()
ast.Del()
ast.Delete()
ast.Dict()
ast.Div()
ast.Ellipsis()
ast.Eq()
ast.Exec()
ast.Expr()
ast.Expression()
ast.ExtSlice()
ast.FloorDiv()
ast.For()
ast.FunctionDef()
ast.GeneratorExp()
ast.Global()
ast.Gt()
ast.GtE()
ast.If()
ast.IfExp()
ast.Import()
ast.ImportFrom()
ast.In()
ast.Index()
ast.Interactive()
ast.Invert()
ast.Is()
ast.IsNot()
ast.LShift()
ast.Lambda()
ast.List()
ast.ListComp()
ast.Load()
ast.Lt()
ast.LtE()
ast.Mod()
ast.Module()
ast.Mult()
ast.Name()
ast.Not()
ast.NotEq()
ast.NotIn()
ast.Num()
ast.Or()
ast.Param()
ast.Pass()
ast.Pow()
ast.Print()
ast.RShift()
ast.Raise()
ast.Repr()
ast.Return()
ast.Slice()
ast.Store()
ast.Str()
ast.Sub()
ast.Subscript()
ast.Suite()
ast.TryExcept()
ast.TryFinally()
ast.Tuple()
ast.UAdd()
ast.USub()
ast.UnaryOp()
ast.While()
ast.With()
ast.Yield()
ast.alias()
ast.arguments()
#ast.boolop()
#ast.cmpop()
ast.comprehension()
#ast.excepthandler()
#ast.expr()
#ast.expr_context()
ast.keyword()
'POW_ASN',
'FDIV_ASN',
'NAME',
'NUMBER',
)
Add = ast.Add()
Sub = ast.Sub()
Mult = ast.Mult()
Div = ast.Div()
Mod = ast.Mod()
Pow = ast.Pow()
LShift = ast.LShift()
RShift = ast.RShift()
BitOr = ast.BitOr()
BitAnd = ast.BitAnd()
BitXor = ast.BitXor()
FloorDiv = ast.FloorDiv()
op_dict = {
"+": Add,
"-": Sub,
"*": Mult,
"/": Div,
"//": FloorDiv,
"%": Mod,
"**": Pow,
"<<": LShift,
">>": RShift,
"|": BitOr,
"&": BitAnd,
class LtnsCompiler:
def _temp_func_name(self):
if not hasattr(self, '_temp'):
self._temp = 0
self._temp += 1
return f'_temp_func_{self._temp}'
def _temp_var_name(self):
if not hasattr(self, '_temp'):
self._temp = 0
self._temp += 1
return f'_temp_var_{self._temp}'
def _compile_branch(self, branch):
result = Result()
for x in branch[:-1]:
res = self.compile(x)
result.stmts += res.stmts
result.stmts += [res.expr_statement]
res = self.compile(branch[-1])
result.stmts += res.stmts
result.expr = res.expr
return result
def _compile_args(self, args, kwargs):
for i, arg in enumerate(args):
if arg == '&':
vararg = ast.arg(str(args[i + 1], None), None)
posarg = args[:i]
else:
vararg = None
posarg = args
return ast.arguments(
args=[ast.arg(str(x), None) for x in posarg],
vararg=vararg,
kwonlyargs=[ast.arg(str(x), None) for x in kwargs],
kwarg=None,
defaults=[],
kw_defaults=list(kwargs.values()),
)
def compile(self, node):
return _model_compiler[type(node)](self, node)
@model(LtnsElement)
def compile_element(self, element):
if element.name in _special_form_compiler:
return _special_form_compiler[element.name](self, *element.childs,
**element.attributes)
func = ast.parse(element.name,
mode='eval').body # TODO: handle exception of parsing
result = Result()
args = []
keywords = []
for child in element.childs:
res = self.compile(child)
args.append(res.expr)
result.stmts += res.stmts
for key, value in element.attributes.items():
res = self.compile(value)
keywords.append(ast.keyword(arg=str(key), value=res.expr))
result.stmts += res.stmts
expr = ast.Call(func=func, args=args, keywords=keywords)
result.expr = expr
return result
@model(LtnsSymbol)
def compile_symbol(self, symbol):
expr = ast.Name(id=str(symbol), ctx=ast.Load())
return Result(expr=expr)
@model(LtnsString)
def compile_string(self, string):
return Result(expr=ast.Str(str(string)))
@model(LtnsKeyword)
def compile_keyword(self, keyword):
return Result(expr=ast.Call(
func=ast.Name(id='LtnsKeyword', ctx=ast.Load()),
args=[ast.Str(str(keyword))],
))
@model(LtnsInteger)
def compile_integer(self, integer):
return Result(expr=ast.Num(int(integer)))
@model(LtnsFloat)
def compile_float_number(self, float_number):
return Result(expr=ast.Num(float(float_number)))
@model(LtnsComplex)
def compile_complex_number(self, complex_number):
return Result(expr=ast.Num(complex(complex_number)))
@model(LtnsList)
def compile_list(self, ltns_list):
result = Result()
elts = []
for e in ltns_list:
res = self.compile(e)
elts.append(res.expr)
result.stmts += res.stmts
expr = ast.List(elts=elts, ctx=ast.Load())
result.expr = expr
return result
@special('do')
def compile_do(self, *body):
return self._compile_branch(body)
name_op = {
'add*': ast.Add(),
'sub*': ast.Sub(),
'mul*': ast.Mult(),
'div*': ast.Div(),
'mod': ast.Mod(),
'pow': ast.Pow(),
'lshift': ast.LShift(),
'rshift': ast.RShift(),
'bitor': ast.BitOr(),
'bitxor': ast.BitXor(),
'bitand': ast.BitAnd(),
}
def compile_bin_op(self, a, b, name):
result = Result()
left = self.compile(a)
result.stmts += left.stmts
left = left.expr
right = self.compile(b)
result.stmts += right.stmts
right = right.expr
expr = ast.BinOp(op=self.name_op[name], left=left, right=right)
result.expr = expr
return result
for name in name_op:
_special_form_compiler[name] = partial(compile_bin_op, name=name)
@special('if')
def compile_if(self, test, then, orelse=None):
result = Result()
pred = self.compile(test)
result.stmts += pred.stmts
then_body = self.compile(then)
else_body = None
if orelse is not None:
else_body = self.compile(orelse)
if then_body.stmts or else_body.stmts:
temp_func_name = self._temp_func_name()
temp_var_name = self._temp_var_name()
body = then_body.stmts
body.append(
ast.Assign(
targets=[ast.Name(id=temp_var_name, ctx=ast.Store())],
value=then_body.expr,
))
orelse = else_body.stmts
orelse.append(
ast.Assign(
targets=[ast.Name(id=temp_var_name, ctx=ast.Store())],
value=else_body.expr,
))
result.stmts.append(
ast.FunctionDef(
name=temp_func_name,
args=ast.arguments(
args=[],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[],
),
body=[
ast.If(test=pred.expr, body=body, orelse=orelse),
ast.Return(
value=ast.Name(id=temp_var_name, ctx=ast.Load())),
],
decorator_list=[],
returns=None,
))
result.expr = ast.Call(
func=ast.Name(id=temp_func_name, ctx=ast.Load()),
args=[],
keywords=[],
)
return result
else:
result.expr = ast.IfExp(
test=pred.expr,
body=then_body.expr,
orelse=else_body.expr,
)
return result
@special('fn*')
def compile_fn(self, args, *body, **kwargs):
result = Result()
body = self._compile_branch(body)
args = self._compile_args(args, kwargs)
if body.stmts:
fdef = ast.FunctionDef()
fdef.name = self._temp_func_name()
fdef.args = args
fdef.body = body.stmts + [ast.Return(body.expr)]
fdef.decorator_list = []
fdef.returns = None
result.stmts += [fdef]
result.expr = ast.Name(id=fdef.name, ctx=ast.Load())
else:
result.expr = ast.Lambda(args=args, body=body.expr)
return result
@special('def')
def compile_def(self, *name_value, **def_dict):
if len(name_value) % 2 != 0:
raise ValueError("length of argument list should be even")
result = Result()
for name, value in def_dict.items():
name = LtnsSymbol(name)
name_value = name_value + (name, value)
for i, name in enumerate(name_value[::2]):
name = ast.Name(id=str(name), ctx=ast.Store())
value = self.compile(name_value[i * 2 + 1])
result.stmts += [ast.Assign(
targets=[name],
value=value.expr,
)]
result.expr = ast.Name(id='None', ctx=ast.Load())
return result
def and_expr_rewrite(head, tail):
if tail:
for op, each in tail:
head = ast.BinOp(head, ast.BitAnd(), each, **loc @ op)
return head
def bitand(self, other: Union[T, Expr[T]]) -> BinOp[T]:
return BinOp(self, ast.BitAnd(), other)
def as_ast(dct):
"""See https://docs.python.org/2/library/ast.html"""
if dct['ast_type'] == "Module":
return ast.Module(dct["body"])
elif dct['ast_type'] == "Interactive":
return ast.Interactive(dct["body"])
elif dct['ast_type'] == "Expression":
return ast.Expression(dct["body"])
elif dct['ast_type'] == "Suite":
return ast.Suite(dct["body"])
elif dct['ast_type'] == "FunctionDef":
return ast.FunctionDef(dct["name"], dct["args"], dct["body"],
dct["decorator_list"])
elif dct['ast_type'] == "ClassDef":
return ast.ClassDef(dct["name"], dct["bases"], dct["body"],
dct["decorator_list"])
elif dct['ast_type'] == "Return":
return ast.Return(dct["value"])
elif dct['ast_type'] == "Delete":
return ast.Delete(dct["targets"])
elif dct['ast_type'] == "Assign":
return ast.Assign(dct["targets"], dct["value"])
elif dct['ast_type'] == "AugAssign":
return ast.AugAssign(dct["target"], dct["op"], dct["value"])
elif dct['ast_type'] == "Print":
return ast.Print(dct["dest"], dct["values"], dct["nl"])
elif dct['ast_type'] == "For":
return ast.For(dct["target"], dct["iter"], dct["body"], dct["orelse"])
elif dct['ast_type'] == "While":
return ast.While(dct["test"], dct["body"], dct["orelse"])
elif dct['ast_type'] == "If":
return ast.If(dct["test"], dct["body"], dct["orelse"])
elif dct['ast_type'] == "With":
return ast.With(dct["context_expr"], dct["optional_vars"], dct["body"])
elif dct['ast_type'] == "Raise":
return ast.Raise(dct["type"], dct["inst"], dct["tback"])
elif dct['ast_type'] == "TryExcept":
return ast.TryExcept(dct["body"], dct["handlers"], dct["orelse"])
elif dct['ast_type'] == "TryFinally":
return ast.TryFinally(dct["body"], dct["finalbody"])
elif dct['ast_type'] == "Assert":
return ast.Assert(dct["test"], dct["msg"])
elif dct['ast_type'] == "Import":
return ast.Import(dct["names"])
elif dct['ast_type'] == "ImportFrom":
return ast.ImportFrom(dct["module"], dct["names"], dct["level"])
elif dct['ast_type'] == "Exec":
return ast.Exec(dct["body"], dct["globals"], dct["locals"])
elif dct['ast_type'] == "Global":
return ast.Global(dct["names"])
elif dct['ast_type'] == "Expr":
return ast.Expr(dct["value"])
elif dct['ast_type'] == "Pass":
return ast.Pass()
elif dct['ast_type'] == "Break":
return ast.Break()
elif dct['ast_type'] == "Continue":
return ast.Continue()
elif dct['ast_type'] == "BoolOp":
return ast.BoolOp(dct["op"], dct["values"])
elif dct['ast_type'] == "BinOp":
return ast.BinOp(dct["left"], dct["op"], dct["right"])
elif dct['ast_type'] == "UnaryOp":
return ast.UnaryOp(dct["op"], dct["operand"])
elif dct['ast_type'] == "Lambda":
return ast.Lambda(dct["args"], dct["body"])
elif dct['ast_type'] == "IfExp":
return ast.IfExp(dct["test"], dct["body"], dct["orelse"])
elif dct['ast_type'] == "Dict":
return ast.Dict(dct["keys"], dct["values"])
elif dct['ast_type'] == "Set":
return ast.Set(dct["elts"])
elif dct['ast_type'] == "ListComp":
return ast.ListComp(dct["elt"], dct["generators"])
elif dct['ast_type'] == "SetComp":
return ast.SetComp(dct["elt"], dct["generators"])
elif dct['ast_type'] == "DictComp":
return ast.DictComp(dct["key"], dct["value"], dct["generators"])
elif dct['ast_type'] == "GeneratorExp":
return ast.GeneratorExp(dct["elt"], dct["generators"])
elif dct['ast_type'] == "Yield":
return ast.Yield(dct["value"])
elif dct['ast_type'] == "Compare":
return ast.Compare(dct["left"], dct["ops"], dct["comparators"])
elif dct['ast_type'] == "Call":
return ast.Call(dct["func"], dct["args"], dct["keywords"],
dct["starargs"], dct["kwargs"])
elif dct['ast_type'] == "Repr":
return ast.Repr(dct["value"])
elif dct['ast_type'] == "Num":
return ast.Num(dct["n"])
elif dct['ast_type'] == "Str":
# Converting to ASCII
return ast.Str(dct["s"].encode('ascii', 'ignore'))
elif dct['ast_type'] == "Attribute":
return ast.Attribute(dct["value"], dct["attr"], dct["ctx"])
elif dct['ast_type'] == "Subscript":
return ast.Subscript(dct["value"], dct["slice"], dct["ctx"])
elif dct['ast_type'] == "Name":
return ast.Name(dct["id"], dct["ctx"])
elif dct['ast_type'] == "List":
return ast.List(dct["elts"], dct["ctx"])
elif dct['ast_type'] == "Tuple":
return ast.Tuple(dct["elts"], dct["ctx"])
elif dct['ast_type'] == "Load":
return ast.Load()
elif dct['ast_type'] == "Store":
return ast.Store()
elif dct['ast_type'] == "Del":
return ast.Del()
elif dct['ast_type'] == "AugLoad":
return ast.AugLoad()
elif dct['ast_type'] == "AugStore":
return ast.AugStore()
elif dct['ast_type'] == "Param":
return ast.Param()
elif dct['ast_type'] == "Ellipsis":
return ast.Ellipsis()
elif dct['ast_type'] == "Slice":
return ast.Slice(dct["lower"], dct["upper"], dct["step"])
elif dct['ast_type'] == "ExtSlice":
return ast.ExtSlice(dct["dims"])
elif dct['ast_type'] == "Index":
return ast.Index(dct["value"])
elif dct['ast_type'] == "And":
return ast.And()
elif dct['ast_type'] == "Or":
return ast.Or()
elif dct['ast_type'] == "Add":
return ast.Add()
elif dct['ast_type'] == "Sub":
return ast.Sub()
elif dct['ast_type'] == "Mult":
return ast.Mult()
elif dct['ast_type'] == "Div":
return ast.Div()
elif dct['ast_type'] == "Mod":
return ast.Mod()
elif dct['ast_type'] == "Pow":
return ast.Pow()
elif dct['ast_type'] == "LShift":
return ast.LShift()
elif dct['ast_type'] == "RShift":
return ast.RShift()
elif dct['ast_type'] == "BitOr":
return ast.BitOr()
elif dct['ast_type'] == "BitXor":
return ast.BitXor()
elif dct['ast_type'] == "BitAnd":
return ast.BitAnd()
elif dct['ast_type'] == "FloorDiv":
return ast.FloorDiv()
elif dct['ast_type'] == "Invert":
return ast.Invert()
elif dct['ast_type'] == "Not":
return ast.Not()
elif dct['ast_type'] == "UAdd":
return ast.UAdd()
elif dct['ast_type'] == "USub":
return ast.USub()
elif dct['ast_type'] == "Eq":
return ast.Eq()
elif dct['ast_type'] == "NotEq":
return ast.NotEq()
elif dct['ast_type'] == "Lt":
return ast.Lt()
elif dct['ast_type'] == "LtE":
return ast.LtE()
elif dct['ast_type'] == "Gt":
return ast.Gt()
elif dct['ast_type'] == "GtE":
return ast.GtE()
elif dct['ast_type'] == "Is":
return ast.Is()
elif dct['ast_type'] == "IsNot":
return ast.IsNot()
elif dct['ast_type'] == "In":
return ast.In()
elif dct['ast_type'] == "NotIn":
return ast.NotIn()
elif dct['ast_type'] == "comprehension":
return ast.comprehension(dct["target"], dct["iter"], dct["ifs"])
elif dct['ast_type'] == "ExceptHandler":
return ast.ExceptHandler(dct["type"], dct["name"], dct["body"])
elif dct['ast_type'] == "arguments":
return ast.arguments(dct["args"], dct["vararg"], dct["kwarg"],
dct["defaults"])
elif dct['ast_type'] == "keyword":
return ast.keyword(dct["arg"], dct["value"])
elif dct['ast_type'] == "alias":
return ast.alias(dct["name"], dct["asname"])
else:
return dct
class helper(object):
def __init__(self, node):
self.node = node
def __ast__(self):
return self.node
@property
def body(self):
return E(self.node.body)
@body.setter
def body(self, e):
self.node.body = e
def call(self, args=None):
self.node = ast.Call(func=self.node,
args=args if args else [],
keywords=[],
starargs=None,
kwargs=None)
return self
def args(self, args):
self.node.args = args
return self
def subscript(self, index=None):
self.node = ast.Subscript(value=self.node, slice=ast.Index(index))
return self
def attr(self, attribute):
assert (isinstance(attribute, str))
self.node = ast.Attribute(value=self.node, attr=attribute)
return self
def assign(self, value):
self.node = ast.Assign(targets=[self.node], value=E(value))
return self
def init(self, value):
self.node.init = value
return self
sub_assign = _aug_assign(ast.Sub())
add_assign = _aug_assign(ast.Add())
xor_assign = _aug_assign(ast.BitXor())
or_assign = _aug_assign(ast.BitOr())
mult = _bin_op(ast.Mult())
add = _bin_op(ast.Add())
bit_and = _bin_op(ast.BitAnd())
bit_or = _bin_op(ast.BitOr())
bit_xor = _bin_op(ast.BitXor())
xor = bit_xor
NotEq = _compare2(ast.NotEq())
Eq = _compare2(ast.Eq())
Gt = _compare2(ast.Gt())
Lt = _compare2(ast.Lt())
def ast(self):
return self.node
