def make_Iterator(self, gen):
if gen.ifs:
ldFilter = ast.Lambda(
ast.arguments([ast.Name(gen.target.id, ast.Param())], None,
None, []), ast.BoolOp(ast.And(), gen.ifs))
ifilterName = ast.Attribute(value=ast.Name(id='itertools',
ctx=ast.Load()),
attr='ifilter',
ctx=ast.Load())
return ast.Call(ifilterName, [ldFilter, gen.iter], [], None, None)
else:
return gen.iter
def gen_attr_check(self, node, attr_name):
"""Check if 'attr_name' is allowed on the object in node.
It generates (_getattr_(node, attr_name) and node).
"""
call_getattr = ast.Call(
func=ast.Name('__rl_getattr__', ast.Load()),
args=[node, ast.Str(attr_name)],
keywords=[])
return ast.BoolOp(op=ast.And(), values=[call_getattr, node])
def visit_Compare(self, node):
left = self.visit(node.left)
expr = None
for op, right in zip(node.ops, node.comparators):
right = self.visit(right)
compare = CmpOp(op=op, left=left, right=right)
if expr is None:
expr = compare
else:
expr = BinOp(op=ast.And(), left=expr, right=compare)
left = right
return expr
def visit_Where_of_Where(self, parent, filter):
'''
seq.Where(x: f(x)).Where(x: g(x))
=> Where(Where(seq, x: f(x)), y: g(y))
is turned into
seq.Where(x: f(x) and g(y))
=> Where(seq, x: f(x) and g(y))
'''
func_f = parent.filter
func_g = filter
arg = arg_name()
return self.visit(Where(parent.source, lambda_build(arg, ast.BoolOp(ast.And(), [lambda_call(arg, func_f), lambda_call(arg, func_g)]))))
def mutate(self, node, idx):
"""Replace AND with OR."""
# replace all occurences of And()
# A and B and C -> A or B or C
node.op = ast.Or()
# or replace an operator somewhere in the middle
# of the expression
if idx and len(node.values) > 2:
left = node.values[:idx]
if len(left) > 1:
left = [ast.BoolOp(op=ast.And(), values=left)]
right = node.values[idx:]
if len(right) > 1:
right = [ast.BoolOp(op=ast.And(), values=right)]
node.values = []
node.values.extend(left)
node.values.extend(right)
return node
def getMemWriteAddrs(self, cond, outname, exp, addrs):
assert exp.nodetype == ast.Node.IF or exp.nodetype == ast.Node.WRITEMEM or exp.nodetype == ast.Node.MEMVAR, (
exp, exp.nodetype)
if exp.nodetype == ast.Node.IF:
c1 = ast.And(cond, exp.cond)
c2 = ast.And(cond, ast.Not(exp.cond))
self.getMemWriteAddrs(c1, outname, exp.exptrue, addrs)
self.getMemWriteAddrs(c2, outname, exp.expfalse, addrs)
elif exp.nodetype == ast.Node.WRITEMEM:
if exp.mem.isMemVar():
new_addr = (cond, exp.mem, exp.addr)
addrs.append(new_addr)
else:
mem = getBaseMemory(exp)
new_addr = (cond, mem, exp.addr)
addrs.append(new_addr)
self.getMemWriteAddrs(cond, outname, exp.mem, addrs)
elif exp.nodetype == ast.Node.MEMVAR:
pass
else:
assert False
def test_BoolOp(self):
and_op = ast.BoolOp(ast.And(), [ast.Num(2), ast.Num(3)])
self.verify(and_op, '2 and 3')
or_op = ast.BoolOp(ast.Or(), [ast.Num(2), ast.Num(3)])
self.verify(or_op, '2 or 3')
many_args = ast.BoolOp(ast.And(), [ast.Num(1), ast.Num(2), ast.Num(3)])
self.verify(many_args, '1 and 2 and 3')
no_precedence = ast.BoolOp(
ast.Or(),
[ast.BoolOp(ast.And(), [ast.Num(2), ast.Num(3)]),
ast.Num(1)])
self.verify(no_precedence, '2 and 3 or 1')
no_precedence2 = ast.BoolOp(
ast.Or(),
[ast.Num(2),
ast.BoolOp(ast.And(), [ast.Num(3), ast.Num(1)])])
self.verify(no_precedence2, '2 or 3 and 1')
precedence = ast.BoolOp(
ast.And(),
[ast.Num(1),
ast.BoolOp(ast.Or(), [ast.Num(2), ast.Num(3)])])
self.verify(precedence, '1 and (2 or 3)')
def visit_ListComp(self, node):
"""
Rewrite list comprehensions to the equivalent for loops.
AST syntax:
ListComp(expr elt, comprehension* generators)
comprehension = (expr target, expr iter, expr* ifs)
'ifs' represent a chain of ANDs
"""
assert len(node.generators) > 0
# Create innermost body, i.e. list.append(expr)
# TODO: size hint for PyList_New
list_create = ast.List(elts=[], ctx=ast.Load())
list_create.type = typesystem.object_ # typesystem.list_()
list_create = nodes.CloneableNode(list_create)
list_value = nodes.CloneNode(list_create)
list_append = ast.Attribute(list_value, "append", ast.Load())
append_call = ast.Call(func=list_append,
args=[node.elt],
keywords=[],
starargs=None,
kwargs=None)
# Build up the loops from inwards to outwards
body = append_call
for comprehension in reversed(node.generators):
# Hanlde the 'if' clause
ifs = comprehension.ifs
if len(ifs) > 1:
make_boolop = lambda op1_op2: ast.BoolOp(op=ast.And(),
values=op1_op2)
if_test = reduce(make_boolop, ifs)
elif len(ifs) == 1:
if_test, = ifs
else:
if_test = None
if if_test is not None:
body = ast.If(test=if_test, body=[body], orelse=[])
# Wrap list.append() call or inner loops
body = ast.For(target=comprehension.target,
iter=comprehension.iter,
body=[body],
orelse=[])
expr = nodes.ExpressionNode(stmts=[list_create, body], expr=list_value)
return self.visit(expr)
def invert(self, node: Type[ast.AST]) -> Type[ast.AST]:
if type(node) == ast.Compare:
if len(node.ops) == 1:
return ast.Compare(
left=node.left,
ops=[self.invertComparators[type(node.ops[0])]()],
comparators=node.comparators)
else:
tmpNode = ast.BoolOp(op=ast.And(),
values=[
ast.Compare(
left=node.left,
ops=[node.ops[0]],
comparators=[node.comparators[0]])
])
for i in range(0, len(node.ops) - 1):
tmpNode.values.append(
ast.Compare(left=node.comparators[i],
ops=[node.ops[i + 1]],
comparators=[node.comparators[i + 1]]))
return self.invert(tmpNode)
elif isinstance(node, ast.BinOp) and type(
node.op) in self.invertComparators:
return ast.BinOp(node.left,
self.invertComparators[type(node.op)](),
node.right)
elif type(node) == ast.NameConstant and type(node.value) == bool:
return ast.NameConstant(value=not node.value)
elif type(node) == ast.BoolOp:
return ast.BoolOp(values=[self.invert(x) for x in node.values],
op={
ast.And: ast.Or(),
ast.Or: ast.And()
}.get(type(node.op)))
elif type(node) == ast.UnaryOp:
return self.UnaryopInvert(node)
else:
return ast.UnaryOp(op=ast.Not(), operand=node)
def visit_Compare(self, node):
self.generic_visit(node)
if len(node.ops) == 1:
return node
else:
list_compare = []
left = node.left
for op, comparator in zip(node.ops, node.comparators):
compare = ast.Compare(left=left,
ops=[op],
comparators=[comparator])
list_compare.append(compare)
left = comparator
return ast.BoolOp(op=ast.And(), values=list_compare)
def p_and_test(p):
'''and_test : not_test
| and_test TAG_AND not_test'''
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BoolOp(op=ast.And(),
values=[p[1], p[3]],
lineno=p[1].lineno,
col_offset=p[1].col_offset)
return
def merge_exitcases(exit1, exit2):
"""
Merge the exitcases of two Links.
Args:
exit1: The exitcase of a Link object.
exit2: Another exitcase to merge with exit1.
Returns:
The merged exitcases.
"""
if exit1:
if exit2:
return ast.BoolOp(ast.And(), values=[exit1, exit2])
return exit1
return exit2
def visit_ListComp(self, node):
"""Generate a curried lambda function
[x + y for x, y in [[1, 4], [2, 5], [3, 6]]]
becomes
[[1, 4], [2, 5], [3, 6]]].map(([x, y]) => x + y)
"""
try:
generator, = node.generators
except ValueError:
raise NotImplementedError(
'Only single loop comprehensions are allowed'
)
names = find_names(generator.target)
argslist = [ast.arg(arg=name.id, annotation=None) for name in names]
if len(names) <= 1:
signature = ast.arguments(
args=argslist,
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[],
)
else:
signature = ast.List(elts=argslist, ctx=ast.Load())
array = generator.iter
lam_sig = functools.partial(ast.Lambda, args=signature)
filters = generator.ifs
if filters:
filt = ast.BoolOp(op=ast.And(), values=filters)
# array.filter
method = ast.Attribute(value=array, attr='filter', ctx=ast.Load())
# array.filter(func)
array = ast.Call(
func=method, args=[lam_sig(body=filt)], keywords=[]
)
method = ast.Attribute(value=array, attr='map', ctx=ast.Load())
mapped = ast.Call(
func=method, args=[lam_sig(body=node.elt)], keywords=[]
)
result = self.visit(mapped)
return result
class BinaryInfixOperand(object):
n_terms = 2
assoc = 'LEFT'
keyword_aliases = _kw(
(['and', '&&'], ast.And()),
(['or', '||'], ast.Or()),
(['<', 'lt'], ast.Lt()),
(['==', 'eq'], ast.Eq()),
(['<=', 'le'], ast.LtE()),
(['!=', 'ne'], ast.NotEq()),
(['>=', 'ge'], ast.GtE()),
(['>', 'gt'], ast.Gt()),
)
def __init__(self, tokens):
tokens = tokens[0]
if len(tokens) % 2 == 1:
self.op_token = tokens[1]
self.comparators = tokens[::2]
else:
err = "Invalid number of infix expressions: {}"
err = err.format(len(tokens))
raise ParseException(err)
assert self.op_token in self.keyword_aliases
# Check for too many literals and not enough keywords
op = self.keyword_aliases[self.op_token]
if isinstance(op, ast.boolop):
if any(isinstance(c, Literal) for c in self.comparators):
raise ValueError("Cannot use literals as truth")
else:
if all(isinstance(c, Literal) for c in self.comparators):
raise ValueError("Cannot compare literals.")
def ast(self):
op = self.keyword_aliases[self.op_token]
if isinstance(op, ast.boolop):
# and and or use one type of AST node
value = ast.BoolOp(op=op,
values=[e.ast() for e in self.comparators])
else:
# remaining operators use another
value = ast.Compare(
left=self.comparators[0].ast(),
ops=[op],
comparators=[e.ast() for e in self.comparators[1:]])
return value
def translate_pat_Tuple(self, ctx, pat, scrutinee_trans):
scrutinee_trans_copy = astx.copy_node(scrutinee_trans)
elts = pat.elts
idx = self.idx
conditions = []
binding_translations = _util.odict()
for n, (elt, ty) in enumerate(zip(elts, idx.itervalues())):
elt_scrutinee_trans = astx.make_Subscript_Num_Index(
scrutinee_trans_copy, n)
elt_condition, elt_binding_translations = ctx.translate_pat(
elt, elt_scrutinee_trans)
conditions.append(elt_condition)
binding_translations.update(elt_binding_translations)
condition = ast.BoolOp(op=ast.And(), values=conditions)
return (condition, binding_translations)
def visit_Compare(self, node):
ops = node.ops
comps = node.comparators
if len(comps) == 1:
binop = ast.BinOp(op=ops[0], left=node.left, right=comps[0])
return self.visit(binop)
left = node.left
values = []
for op, comp in zip(ops, comps):
new_node = ast.Compare(comparators=[comp], left=left, ops=[op])
left = comp
values.append(new_node)
return self.visit(ast.BoolOp(op=ast.And(), values=values))
def visit_Compare(self, node):
if len(node.ops) > 1:
compares = []
compares.append(
ast.Compare(left=node.left,
ops=node.ops[0:1],
comparators=node.comparators[0:1]))
l = node.comparators[0]
for op, right in zip(node.ops[1:], node.comparators[1:]):
compares.append(
ast.Compare(left=l, ops=[op], comparators=[right]))
l = right
andexpr = ast.BoolOp(op=ast.And(), values=compares)
return ast.copy_location(andexpr, node)
else:
return node
def visit_Compare(self, node):
self.generic_visit(node)
ret = None
comparators = [node.left] + node.comparators
for i in range(len(node.comparators)):
new_cmp = ast.Compare(left=comparators[i], ops=[node.ops[i]], comparators=[comparators[i + 1]])
ast.copy_location(new_cmp, node)
if ret is None:
ret = new_cmp
else:
ret = ast.BoolOp(op=ast.And(), values=[ret, new_cmp])
ret = self.visit_BoolOp(ret)
ast.copy_location(ret, node)
self.generic_visit(ret)
return ret
def p_logic_and_expr(t):
'''logic_and_expr : logic_and_expr AND logic_expr
| logic_expr'''
if len(t) == 4:
if isinstance(t[1], ast.BoolOp) and isinstance(t[1].op, ast.And):
t[0] = t[1]
t[0].values.append(t[3])
else:
and_ast = ast.And()
and_ast.lineno = t.lineno(2)
and_ast.col_offset = -1 # XXX
t[0] = ast.BoolOp(and_ast, [t[1], t[3]])
t[0].lineno = t.lineno(2)
t[0].col_offset = -1 # XXX
else:
t[0] = t[1]
def _parse_logical_and_expression(self, env):
""" <logical_and_expression>
Parses the <logical_and_expression> language structure.
logical_and_expression -> logical_and_expression '&&' inclusive_or_expression
| inclusive_or_expression
"""
inclor = self._parse_inclusive_or_expression(env)
while self._look.tag == lexer.Tag.AND:
tok = self._look
self._match(lexer.Tag.AND)
inclor = ast.And(tok, inclor,
self._parse_logical_and_expression(env))
return inclor
def convert_tree_to_ssa(tree: ast.AST, defn_env: dict, phi_name: str = "phi"):
# tree = MoveReturn().visit(tree)
# tree.body.append(
# ast.Return(ast.Name("__magma_ssa_return_value", ast.Load())))
ssa_visitor = SSAVisitor(phi_name)
tree = ssa_visitor.visit(tree)
return_transformer = TransformReturn()
tree = return_transformer.visit(tree)
num_return_values = len(ssa_visitor.return_values)
for i in reversed(range(num_return_values)):
conds = ssa_visitor.return_values[i]
name = f"__magma_ssa_return_value_{i}"
if i == num_return_values - 1 or not conds:
if isinstance(tree.returns, ast.Tuple):
tree.body.append(ast.Assign(
[ast.Tuple([ast.Name(f"O{i}", ast.Store())
for i in range(len(tree.returns.elts))], ast.Store())],
ast.Name(name, ast.Load())
))
else:
tree.body.append(ast.Assign([ast.Name("O", ast.Load)],
ast.Name(name, ast.Load())))
else:
cond = conds[-1]
for c in conds[:-1]:
c = ast.BinOp(cond, ast.And(), c)
if isinstance(tree.returns, ast.Tuple):
for i in range(len(tree.returns.elts)):
tree.body.append(ast.Assign(
[ast.Name(f"O{i}", ast.Store())],
ast.Call(ast.Name(phi_name, ast.Load()), [
ast.List([
ast.Name(f"O{i}", ast.Load()),
ast.Subscript(ast.Name(name, ast.Load()),
ast.Index(ast.Num(i)),
ast.Load())
], ast.Load()),
cond], []))
)
else:
tree.body.append(ast.Assign(
[ast.Name("O", ast.Store())],
ast.Call(ast.Name(phi_name, ast.Load()), [
ast.List([ast.Name("O", ast.Load()), ast.Name(name, ast.Load())],
ast.Load()), cond], []))
)
return tree, ssa_visitor.args
def visit_Compare(self, node):
if len(node.ops) > 1:
return self.visit(ast.copy_location(
ast.BoolOp(
op=ast.And(),
values=[
ast.Compare(left=left, ops=[op], comparators=[right])
for left, op, right in zip([node.left] + node.comparators, node.ops, node.comparators)
]
), node
))
else:
func = ast.Attribute(value=self.visit(node.left),
attr=compare_op_map[type(node.ops[0])],
ctx=ast.Load())
call = ast.Call(func=func, args=[self.visit(node.comparators[0])], keywords=[])
return ast.copy_location(call, node)
def translate_pat_Dict(self, ctx, pat, scrutinee_trans):
scrutinee_trans_copy = astx.copy_node(scrutinee_trans)
keys, values = pat.keys, pat.values
idx = self.idx
conditions = []
binding_translations = _util.odict()
for key, value in zip(keys, values):
label = key.label
n = _util.odict_idx_of(idx, label)
elt_scrutinee_trans = astx.make_Subscript_Num_Index(
scrutinee_trans_copy, n)
elt_condition, elt_binding_translations = ctx.translate_pat(
value, elt_scrutinee_trans)
conditions.append(elt_condition)
binding_translations.update(elt_binding_translations)
condition = ast.BoolOp(op=ast.And(), values=conditions)
return (condition, binding_translations)
def mutate(cls, node):
if node not in config.visited_nodes:
if node.__class__ is ast.BoolOp:
if node.op.__class__ is ast.And:
config.mutated = True
original_node = deepcopy(node)
node.op = ast.Or()
config.node_pairs[node] = original_node
config.current_mutated_node = node
elif node.op.__class__ is ast.Or:
config.mutated = True
original_node = deepcopy(node)
node.op = ast.And()
config.node_pairs[node] = original_node
config.current_mutated_node = node
return node
def normalize_compare(node):
"""Rewrites a compare expression to a `and` expression
1 < 2 < 3 > 0
1 < 2 and 2 < 3 and 3 > 0"""
and_values = []
left = node.left
for (op, val) in zip(node.ops, node.comparators):
comp = ast.Compare(ops=[op],
left=left,
comparators=[val],
lineno=node.lineno,
col_offset=node.col_offset)
and_values.append(comp)
left = val
return ast.BoolOp(op=ast.And(),
values=and_values,
lineno=node.lineno,
col_offset=node.col_offset)
class Infix(Symbol):
rightAssoc = False
_operator_map = {
"in": ast.In(),
">": ast.Gt(),
"<": ast.Lt(),
"=": ast.Eq()
}
_logical_map = {"and": ast.And(), "or": ast.Or()}
def led(self, left):
self.first = left
rbp = self.lbp - int(self.rightAssoc)
self.second = self.parser.expression(rbp)
return self
def __repr__(self):
return "<'%s'>(%s, %s)" % (self.value, self.first, self.second)
def ast(self, context=None):
lhs = self.first.ast(context)
if self.second:
rhs = self.second.ast(context)
if self.value in self._logical_map:
return ast.BoolOp(op=self._logical_map[self.value],
values=[lhs, rhs])
else:
path = list(context.stack)
path.append(self.first.value)
return ast.Call(
func=ast.Name(id="filter_field", ctx=ast.Load()),
args=[
ast.Name(id="obj", ctx=ast.Load()),
ast.List(elts=[ast.Str(s=i) for i in path],
ctx=ast.Load()),
ast.Str(s=self.value),
rhs,
],
keywords=[],
)
return ast.Name(id="not", ctx=ast.Load())
def create_match_check(self, pattern: Pattern, data):
"""Given an ADT match pattern and a (Python) expression pointing to
an ADT value, this generates a Python expression that checks if the
ADT value matches the given pattern (returning True or False)."""
# wildcard or var match everything
if isinstance(pattern, (relay.PatternWildcard, relay.PatternVar)):
return NameConstant(True)
conds = []
if isinstance(pattern, relay.PatternConstructor):
# constructor patterns check whether the constructors match
# and also the matches of any nested patterns
# equiv: (arg.tag == patern_constructor.tag)
conds.append(
ast.Compare(
ast.Attribute(data, "tag", Load()),
[ast.Eq()],
[ast.Num(pattern.constructor.tag)],
)
)
assert isinstance(pattern, (relay.PatternConstructor, relay.PatternTuple))
# now check for any nested patterns
for i in range(len(pattern.patterns)):
nested_pat = pattern.patterns[i]
# can safely skip var or wildcard patterns: they will
# never cause a check to fail
if not isinstance(nested_pat, relay.PatternConstructor):
continue
# index into the value corresponding to the subpattern
field_index = ast.Subscript(
ast.Attribute(data, "fields", Load()), ast.Index(Num(i)), Load()
)
conds.append(self.create_match_check(nested_pat, field_index))
# if we do not need to check nested pattern, just return the single check
if len(conds) == 1:
return conds[0]
# otherwise AND together any nested checks
return ast.BoolOp(ast.And(), conds)
def merge_exitcases(
exit1: Union[Compare, ast.BoolOp, ast.expr, None],
exit2: Union[Compare, ast.BoolOp, ast.expr, None],
) -> Union[Compare, ast.BoolOp, ast.expr, None]:
"""
Merge the exitcases of two Links.
Args:
exit1: The exitcase of a Link object.
exit2: Another exitcase to merge with exit1.
Returns:
The merged exitcases.
"""
if exit1:
if exit2:
return ast.BoolOp(ast.And(), values=[exit1, exit2])
return exit1
return exit2
def visit_Compare(self, node, **kwargs):
ops = node.ops
comps = node.comparators
# base case: we have something like a CMP b
if len(comps) == 1:
op = self.translate_In(ops[0])
binop = ast.BinOp(op=op, left=node.left, right=comps[0])
return self.visit(binop)
# recursive case: we have a chained comparison, a CMP b CMP c, etc.
left = node.left
values = []
for op, comp in zip(ops, comps):
new_node = self.visit(ast.Compare(comparators=[comp], left=left,
ops=[self.translate_In(op)]))
left = comp
values.append(new_node)
return self.visit(ast.BoolOp(op=ast.And(), values=values))
def translate_pat_Call_constructor(self, ctx, pat, scrutinee_trans):
lbl = pat.func.id
tag_loc = ast.Subscript(value=scrutinee_trans,
slice=ast.Index(value=ast.Num(n=0)))
lbl_condition = ast.Compare(left=tag_loc,
ops=[ast.Eq()],
comparators=[ast.Str(s=lbl)])
arg = pat.args[0]
arg_scrutinee = ast.Subscript(value=scrutinee_trans,
slice=ast.Index(value=ast.Num(n=1)))
arg_condition, binding_translations = ctx.translate_pat(
arg, arg_scrutinee)
condition = ast.BoolOp(op=ast.And(),
values=[lbl_condition, arg_condition])
return condition, binding_translations
