def _build_cond_stmt(self, step_node, compare_node):
if not isinstance(step_node, (gast.Constant, gast.UnaryOp)):
raise NotImplementedError(
"Dynamic-to-Static only supports the step value is a constant or negative constant in 'for-range' statements, "
"such as '2', '-3'. But received: '{}'. Please fix code to be compatible with Dynamic-to-Static."
.format(ast_to_source_code(step_node).strip()))
if isinstance(step_node, gast.UnaryOp) or step_node.value < 0:
# eg:
# range(max, min, -2)
# ->
# i > min
return gast.Compare(left=gast.Name(
id=self.iter_var_name
if self.is_for_range_iter() else self.iter_idx_name,
ctx=gast.Load(),
annotation=None,
type_comment=None),
ops=[gast.Gt()],
comparators=[compare_node])
else:
# eg:
# range(min, max, 2)
# ->
# i < max
return gast.Compare(left=gast.Name(
id=self.iter_var_name
if self.is_for_range_iter() else self.iter_idx_name,
ctx=gast.Load(),
annotation=None,
type_comment=None),
ops=[gast.Lt()],
comparators=[compare_node])
def make_control_flow_handlers(self, cont_n, status_n, expected_return,
has_cont, has_break):
'''
Create the statements in charge of gathering control flow information
for the static_if result, and executes the expected control flow
instruction
'''
if expected_return:
assign = cont_ass = [ast.Assign(
[ast.Tuple(expected_return, ast.Store())],
ast.Name(cont_n, ast.Load(), None, None), None)]
else:
assign = cont_ass = []
if has_cont:
cmpr = ast.Compare(ast.Name(status_n, ast.Load(), None, None),
[ast.Eq()], [ast.Constant(LOOP_CONT, None)])
cont_ass = [ast.If(cmpr,
deepcopy(assign) + [ast.Continue()],
cont_ass)]
if has_break:
cmpr = ast.Compare(ast.Name(status_n, ast.Load(), None, None),
[ast.Eq()], [ast.Constant(LOOP_BREAK, None)])
cont_ass = [ast.If(cmpr,
deepcopy(assign) + [ast.Break()],
cont_ass)]
return cont_ass
def visit_Assign(self, node):
if len(node.targets) > 1:
raise NotImplementedError("cannot process multiple assignment")
if not isinstance(node.targets[0], gast.Name):
raise NotImplementedError("cannot process indexed assignment")
# $lhs = $lhs.update_($rhs, matchbox.EXECUTION_MASK) if (lhs in vars()
# or lhs in globals()) and isinstance($lhs, (matchbox.MaskedBatch,
# matchbox.TENSOR_TYPE)) else $rhs
node.value = gast.IfExp(
gast.BoolOp(
gast.And(),
[
gast.BoolOp(gast.Or(), [
gast.Compare(gast.Str(
node.targets[0].id), [gast.In()], [
gast.Call(gast.Name('vars', gast.Load, None),
[], [])
]),
gast.Compare(gast.Str(
node.targets[0].id), [gast.In()], [
gast.Call(
gast.Name('globals', gast.Load, None), [],
[])
])
]),
# gast.Compare(
# gast.Attribute(
# gast.Name('matchbox', gast.Load(), None),
# gast.Name('EXECUTION_MASK', gast.Load(), None),
# gast.Load()),
# [gast.IsNot()],
# [gast.NameConstant(None)]),
gast.Call(gast.Name('isinstance', gast.Load(), None), [
node.targets[0],
gast.Tuple([
gast.Attribute(
gast.Name('matchbox', gast.Load(), None),
gast.Name('MaskedBatch', gast.Load(), None),
gast.Load()),
gast.Attribute(
gast.Name('matchbox', gast.Load(), None),
gast.Name('TENSOR_TYPE', gast.Load(), None),
gast.Load())
], gast.Load())
], [])
]),
gast.Call(
gast.Attribute(
gast.Name(node.targets[0].id, gast.Load(), None),
gast.Name('_update', gast.Load(), None), gast.Load()), [
node.value,
gast.Attribute(
gast.Name('matchbox', gast.Load(), None),
gast.Name('EXECUTION_MASK', gast.Load(), None),
gast.Load())
], []),
node.value)
return node
def inlineFixedSizeArrayCompare(self, node):
if len(node.comparators) != 1:
return node
node_right = node.comparators[0]
alike = ast.Num, ast.List, ast.Tuple
if isinstance(node.left, alike) and isinstance(node_right, alike):
return node
lbase, lsize = self.fixedSizeArray(node.left)
rbase, rsize = self.fixedSizeArray(node_right)
if not lbase or not rbase:
return node
if rsize != 1 and lsize != 1 and rsize != lsize:
raise PythranSyntaxError("Invalid numpy broadcasting", node)
self.update = True
operands = [
ast.Compare(self.make_array_index(lbase, lsize,
i), [type(node.ops[0])()],
[self.make_array_index(rbase, rsize, i)])
for i in range(max(lsize, rsize))
]
res = ast.Call(path_to_attr(('numpy', 'array')),
[ast.Tuple(operands, ast.Load())], [])
self.aliases[res.func] = {path_to_node(('numpy', 'array'))}
return res
def visit_For(self, node):
""" Handle iterate variable in for loops.
>>> import gast as ast
>>> from pythran import passmanager, backend
>>> node = ast.parse('''
... def foo():
... a = b = c = 2
... for i in builtins.range(1):
... a -= 1
... b += 1
... return''')
>>> pm = passmanager.PassManager("test")
>>> res = pm.gather(RangeValues, node)
>>> res['a']
Interval(low=-inf, high=2)
>>> res['b']
Interval(low=2, high=inf)
>>> res['c']
Interval(low=2, high=2)
>>> node = ast.parse('''
... def foo():
... for i in (1, 2, 4):
... a = i
... return''')
>>> pm = passmanager.PassManager("test")
>>> res = pm.gather(RangeValues, node)
>>> res['a']
Interval(low=1, high=4)
"""
assert isinstance(node.target, ast.Name), "For apply on variables."
self.visit(node.iter)
init_state = self.result.copy()
bound_range(self.result, self.aliases, ast.Compare(node.target,
[ast.In()],
[node.iter]))
with predecessor_pruner(node, self.cfg):
# visit body
self.cfg_visit(node.body[0])
if self.no_backward:
return self.visit_loop_successor(node)
else:
self.no_backward += 1
prev_state = self.result
self.result = prev_state.copy()
self.cfg_visit(node.body[0])
self.widen(self.result, prev_state)
self.cfg_visit(node.body[0])
self.unionify(init_state)
self.no_backward -= 1
return self.visit_loop_successor(node)
def visit_Compare(self, node):
node = self.generic_visit(node)
if len(node.ops) > 1:
# in case we have more than one compare operator
# we generate an auxiliary function
# that lazily evaluates the needed parameters
imported_ids = self.passmanager.gather(ImportedIds, node, self.ctx)
imported_ids = sorted(imported_ids)
binded_args = [ast.Name(i, ast.Load(), None) for i in imported_ids]
# name of the new function
forged_name = "{0}_compare{1}".format(self.prefix,
len(self.compare_functions))
# call site
call = ast.Call(ast.Name(forged_name, ast.Load(), None),
binded_args, [])
# new function
arg_names = [ast.Name(i, ast.Param(), None) for i in imported_ids]
args = ast.arguments(arg_names, None, [], [], None, [])
body = [] # iteratively fill the body (yeah, feel your body!)
if is_trivially_copied(node.left):
prev_holder = node.left
else:
body.append(
ast.Assign([ast.Name('$0', ast.Store(), None)], node.left))
prev_holder = ast.Name('$0', ast.Load(), None)
for i, exp in enumerate(node.comparators):
if is_trivially_copied(exp):
holder = exp
else:
body.append(
ast.Assign(
[ast.Name('${}'.format(i + 1), ast.Store(), None)],
exp))
holder = ast.Name('${}'.format(i + 1), ast.Load(), None)
cond = ast.Compare(prev_holder, [node.ops[i]], [holder])
body.append(
ast.If(
cond, [ast.Pass()],
[ast.Return(path_to_attr(('__builtin__', 'False')))]))
prev_holder = holder
body.append(ast.Return(path_to_attr(('__builtin__', 'True'))))
forged_fdef = ast.FunctionDef(forged_name, args, body, [], None)
self.compare_functions.append(forged_fdef)
return call
else:
return node
def _build_cond_stmt(self, step_node, compare_node):
return gast.Compare(
left=gast.BinOp(
left=gast.Name(
id=self.iter_var_name
if self.is_for_range_iter() else self.iter_idx_name,
ctx=gast.Load(),
annotation=None,
type_comment=None),
op=gast.Add(),
right=step_node),
ops=[gast.LtE()],
comparators=[compare_node])
def get_for_args_stmts(self, iter_name, args_list):
'''
Returns 3 gast stmt nodes for argument.
1. Initailize of iterate variable
2. Condition for the loop
3. Statement for changing of iterate variable during the loop
NOTE(TODO): Python allows to access iteration variable after loop, such
as "for i in range(10)" will create i = 9 after the loop. But using
current conversion will make i = 10. We should find a way to change it
'''
len_range_args = len(args_list)
assert len_range_args >= 1 and len_range_args <= 3, "range() function takes 1 to 3 arguments"
if len_range_args == 1:
init_stmt = get_constant_variable_node(iter_name, 0)
else:
init_stmt = gast.Assign(
targets=[
gast.Name(
id=iter_name,
ctx=gast.Store(),
annotation=None,
type_comment=None)
],
value=args_list[0])
range_max_node = args_list[0] if len_range_args == 1 else args_list[1]
step_node = args_list[2] if len_range_args == 3 else gast.Constant(
value=1, kind=None)
cond_stmt = gast.Compare(
left=gast.BinOp(
left=gast.Name(
id=iter_name,
ctx=gast.Load(),
annotation=None,
type_comment=None),
op=gast.Add(),
right=step_node),
ops=[gast.LtE()],
comparators=[range_max_node])
change_stmt = gast.AugAssign(
target=gast.Name(
id=iter_name,
ctx=gast.Store(),
annotation=None,
type_comment=None),
op=gast.Add(),
value=step_node)
return init_stmt, cond_stmt, change_stmt
def visit_Compare(self, node):
self.generic_visit(node)
all_comparators = [node.left] + node.comparators
if len(all_comparators) == 2:
# Replace `a < b` with `not a >= b`
inverted_op = OP_INVERSIONS[type(node.ops[0])]
return gast.UnaryOp(op=gast.Not(),
operand=gast.Compare(
left=node.left,
ops=[inverted_op()],
comparators=node.comparators))
else:
# Replace `a < b < c` with `not (a >= b or b >= c)`
or_clauses = []
for left, op, right in zip(all_comparators[:-1], node.ops,
all_comparators[1:]):
inverted_op = OP_INVERSIONS[type(op)]
or_clauses.append(
gast.Compare(left=left,
ops=[inverted_op()],
comparators=[right]))
return gast.UnaryOp(op=gast.Not(),
operand=gast.BoolOp(op=gast.Or(),
values=or_clauses))
def visit_Compare(self, node):
""" Boolean are possible index.
>>> import gast as ast
>>> from pythran import passmanager, backend
>>> node = ast.parse('''
... def foo():
... a = 2 or 3
... b = 4 or 5
... c = a < b
... d = b < 3
... e = b == 4''')
>>> pm = passmanager.PassManager("test")
>>> res = pm.gather(RangeValues, node)
>>> res['c']
Interval(low=1, high=1)
>>> res['d']
Interval(low=0, high=0)
>>> res['e']
Interval(low=0, high=1)
"""
if any(
isinstance(op, (ast.In, ast.NotIn, ast.Is, ast.IsNot))
for op in node.ops):
self.generic_visit(node)
return self.add(node, Interval(0, 1))
curr = self.visit(node.left)
res = []
for op, comparator in zip(node.ops, node.comparators):
comparator = self.visit(comparator)
fake = ast.Compare(ast.Name('x', ast.Load(), None), [op],
[ast.Name('y', ast.Load(), None)])
fake = ast.Expression(fake)
ast.fix_missing_locations(fake)
expr = compile(ast.gast_to_ast(fake), '<range_values>', 'eval')
res.append(eval(expr, {'x': curr, 'y': comparator}))
if all(res):
return self.add(node, Interval(1, 1))
elif any(r.low == r.high == 0 for r in res):
return self.add(node, Interval(0, 0))
else:
return self.add(node, Interval(0, 1))
def visit_For(self, node):
""" Handle iterate variable in for loops.
>>> import gast as ast
>>> from pythran import passmanager, backend
>>> node = ast.parse('''
... def foo():
... a = b = c = 2
... for i in builtins.range(1):
... a -= 1
... b += 1''')
>>> pm = passmanager.PassManager("test")
>>> res = pm.gather(RangeValuesSimple, node)
>>> res['a']
Interval(low=-inf, high=2)
>>> res['b']
Interval(low=2, high=inf)
>>> res['c']
Interval(low=2, high=2)
>>> node = ast.parse('''
... def foo():
... for i in (1, 2, 4):
... a = i''')
>>> pm = passmanager.PassManager("test")
>>> res = pm.gather(RangeValuesSimple, node)
>>> res['a']
Interval(low=1, high=4)
"""
assert isinstance(node.target, ast.Name), "For apply on variables."
self.visit(node.iter)
if isinstance(node.iter, ast.Call):
for alias in self.aliases[node.iter.func]:
if isinstance(alias, Intrinsic):
self.add(
node.target.id,
alias.return_range_content(
[self.visit(n) for n in node.iter.args]))
self.visit_loop(node, ast.Compare(node.target, [ast.In()],
[node.iter]))
def get_for_args_stmts(self, iter_name, args_list):
'''
Returns 3 gast stmt nodes for argument.
1. Initailize of iterate variable
2. Condition for the loop
3. Statement for changing of iterate variable during the loop
'''
len_range_args = len(args_list)
assert len_range_args >= 1 and len_range_args <= 3, "range() function takes 1 to 3 arguments"
if len_range_args == 1:
init_stmt = get_constant_variable_node(iter_name, 0)
else:
init_stmt = gast.Assign(targets=[
gast.Name(id=iter_name,
ctx=gast.Store(),
annotation=None,
type_comment=None)
],
value=args_list[0])
range_max_node = args_list[0] if len_range_args == 1 else args_list[1]
step_node = args_list[2] if len_range_args == 3 else gast.Constant(
value=1, kind=None)
old_cond_stmt = gast.Compare(left=gast.BinOp(left=gast.Name(
id=iter_name, ctx=gast.Load(), annotation=None, type_comment=None),
op=gast.Add(),
right=step_node),
ops=[gast.LtE()],
comparators=[range_max_node])
cond_stmt = gast.BoolOp(op=gast.And(),
values=[old_cond_stmt, self.condition_node])
change_stmt = gast.AugAssign(target=gast.Name(id=iter_name,
ctx=gast.Store(),
annotation=None,
type_comment=None),
op=gast.Add(),
value=step_node)
return init_stmt, cond_stmt, change_stmt
def visit_BinOp(self, node):
node = self.generic_visit(node)
if not isinstance(node.op, ast.Mod):
return node
right_range = self.range_values[node.right]
left_range = self.range_values[node.left]
if right_range.low < 0 or isinf(right_range.high):
return node
if left_range.low < -right_range.low:
return node
if left_range.high > right_range.high * 2:
return node
cleft0, cleft1 = deepcopy(node.left), deepcopy(node.left)
cright = deepcopy(node.right)
self.update = True
return ast.IfExp(ast.Compare(node.left, [ast.Lt()], [node.right]),
cleft0, ast.BinOp(cleft1, ast.Sub(), cright))
def build(left, right):
return gast.Compare(left=left, ops=[op], comparators=[right])
def visit_BinOp(self, node):
if not isinstance(node.op, ast.Mod):
return self.generic_visit(node)
# check that right is a name defined once outside of loop
# TODO: handle expression instead of names
if not isinstance(node.right, ast.Name):
return self.generic_visit(node)
right_def = self.single_def(node.right)
if not right_def:
return self.generic_visit(node)
if self.range_values[node.right.id].low < 0:
return self.generic_visit(node)
# same for lhs
if not isinstance(node.left, ast.Name):
return self.generic_visit(node)
head = self.single_def(node.left)
if not head:
return self.generic_visit(node)
# check lhs is the actual index of a loop
head = head['name']
loop = self.ancestors[head][-1]
if not isinstance(loop, ast.For):
return self.generic_visit(node)
if not isinstance(loop.iter, ast.Call):
return self.generic_visit(node)
# make sure rhs is defined out of the loop
if loop in self.ancestors[right_def['name']]:
return self.generic_visit(node)
# gather range informations
range_ = None
for alias in self.aliases[loop.iter.func]:
if alias is MODULES['__builtin__']['range']:
range_ = alias
elif alias is MODULES['__builtin__']['xrange']:
range_ = alias
else:
break
if range_ is None:
return self.generic_visit(node)
# everything is setup for the transformation!
new_id = node.left.id + '_m'
i = 0
while new_id in self.identifiers:
new_id = '{}_m{}'.format(node.left.id, i)
i += 1
rargs = range_.args.args
lower = rargs[0] if len(rargs) > 1 else ast.Num(0)
header = ast.Assign([ast.Name(new_id, ast.Store(), None)],
ast.BinOp(
ast.BinOp(deepcopy(lower), ast.Sub(),
ast.Num(1)), ast.Mod(),
deepcopy(node.right)))
incr = ast.BinOp(ast.Name(new_id, ast.Load(), None), ast.Add(),
ast.Num(1))
step = ast.Assign([ast.Name(new_id, ast.Store(), None)],
ast.IfExp(
ast.Compare(incr, [ast.Eq()],
[deepcopy(node.right)]), ast.Num(0),
deepcopy(incr)))
self.loops_mod.setdefault(loop, []).append((header, step))
self.update = True
return ast.Name(new_id, ast.Load(), None)
def visit_If(self, node):
if node.test not in self.static_expressions:
return self.generic_visit(node)
imported_ids = self.gather(ImportedIds, node)
assigned_ids_left = self.escaping_ids(node, node.body)
assigned_ids_right = self.escaping_ids(node, node.orelse)
assigned_ids_both = assigned_ids_left.union(assigned_ids_right)
imported_ids.update(i for i in assigned_ids_left
if i not in assigned_ids_right)
imported_ids.update(i for i in assigned_ids_right
if i not in assigned_ids_left)
imported_ids = sorted(imported_ids)
assigned_ids = sorted(assigned_ids_both)
fbody = self.make_fake(node.body)
true_has_return = self.gather(HasReturn, fbody)
true_has_break = self.gather(HasBreak, fbody)
true_has_cont = self.gather(HasContinue, fbody)
felse = self.make_fake(node.orelse)
false_has_return = self.gather(HasReturn, felse)
false_has_break = self.gather(HasBreak, felse)
false_has_cont = self.gather(HasContinue, felse)
has_return = true_has_return or false_has_return
has_break = true_has_break or false_has_break
has_cont = true_has_cont or false_has_cont
self.generic_visit(node)
func_true = outline(self.true_name(), imported_ids, assigned_ids,
node.body, has_return, has_break, has_cont)
func_false = outline(self.false_name(), imported_ids, assigned_ids,
node.orelse, has_return, has_break, has_cont)
self.new_functions.extend((func_true, func_false))
actual_call = self.make_dispatcher(node.test,
func_true, func_false, imported_ids)
# variable modified within the static_if
expected_return = [ast.Name(ii, ast.Store(), None, None)
for ii in assigned_ids]
self.update = True
# name for various variables resulting from the static_if
n = len(self.new_functions)
status_n = "$status{}".format(n)
return_n = "$return{}".format(n)
cont_n = "$cont{}".format(n)
if has_return:
cfg = self.cfgs[-1]
always_return = all(isinstance(x, (ast.Return, ast.Yield))
for x in cfg[node])
always_return &= true_has_return and false_has_return
fast_return = [ast.Name(status_n, ast.Store(), None, None),
ast.Name(return_n, ast.Store(), None, None),
ast.Name(cont_n, ast.Store(), None, None)]
if always_return:
return [ast.Assign([ast.Tuple(fast_return, ast.Store())],
actual_call, None),
ast.Return(ast.Name(return_n, ast.Load(), None, None))]
else:
cont_ass = self.make_control_flow_handlers(cont_n, status_n,
expected_return,
has_cont, has_break)
cmpr = ast.Compare(ast.Name(status_n, ast.Load(), None, None),
[ast.Eq()], [ast.Constant(EARLY_RET, None)])
return [ast.Assign([ast.Tuple(fast_return, ast.Store())],
actual_call, None),
ast.If(cmpr,
[ast.Return(ast.Name(return_n, ast.Load(),
None, None))],
cont_ass)]
elif has_break or has_cont:
cont_ass = self.make_control_flow_handlers(cont_n, status_n,
expected_return,
has_cont, has_break)
fast_return = [ast.Name(status_n, ast.Store(), None, None),
ast.Name(cont_n, ast.Store(), None, None)]
return [ast.Assign([ast.Tuple(fast_return, ast.Store())],
actual_call, None)] + cont_ass
elif expected_return:
return ast.Assign([ast.Tuple(expected_return, ast.Store())],
actual_call, None)
else:
return ast.Expr(actual_call)
def generate_Compare(self): op = CompareSampler().sample()() return gast.Compare(self.generate_Name(), [op], [self.generate_Name()])
def negate(node):
if isinstance(node, ast.Name):
# Not type info, could be anything :(
raise UnsupportedExpression()
if isinstance(node, ast.UnaryOp):
# !~x <> ~x == 0 <> x == ~0 <> x == -1
if isinstance(node.op, ast.Invert):
return ast.Compare(node.operand,
[ast.Eq()],
[ast.Constant(-1, None)])
# !!x <> x
if isinstance(node.op, ast.Not):
return node.operand
# !+x <> +x == 0 <> x == 0 <> !x
if isinstance(node.op, ast.UAdd):
return node.operand
# !-x <> -x == 0 <> x == 0 <> !x
if isinstance(node.op, ast.USub):
return node.operand
if isinstance(node, ast.BoolOp):
new_values = [ast.UnaryOp(ast.Not(), v) for v in node.values]
# !(x or y) <> !x and !y
if isinstance(node.op, ast.Or):
return ast.BoolOp(ast.And(), new_values)
# !(x and y) <> !x or !y
if isinstance(node.op, ast.And):
return ast.BoolOp(ast.Or(), new_values)
if isinstance(node, ast.Compare):
cmps = [ast.Compare(x, [negate(o)], [y])
for x, o, y
in zip([node.left] + node.comparators[:-1], node.ops,
node.comparators)]
if len(cmps) == 1:
return cmps[0]
return ast.BoolOp(ast.Or(), cmps)
if isinstance(node, ast.Eq):
return ast.NotEq()
if isinstance(node, ast.NotEq):
return ast.Eq()
if isinstance(node, ast.Gt):
return ast.LtE()
if isinstance(node, ast.GtE):
return ast.Lt()
if isinstance(node, ast.Lt):
return ast.GtE()
if isinstance(node, ast.LtE):
return ast.Gt()
if isinstance(node, ast.In):
return ast.NotIn()
if isinstance(node, ast.NotIn):
return ast.In()
if isinstance(node, ast.Attribute):
if node.attr == 'False':
return ast.Constant(True, None)
if node.attr == 'True':
return ast.Constant(False, None)
raise UnsupportedExpression()
