def make_control_flow_handlers(self, cont_n, status_n, expected_return,
has_cont, has_break):
'''
Create the statements in charge of gathering control flow information
for the static_if result, and executes the expected control flow
instruction
'''
if expected_return:
assign = cont_ass = [
ast.Assign([ast.Tuple(expected_return, ast.Store())],
ast.Name(cont_n, ast.Load(), None))
]
else:
assign = cont_ass = []
if has_cont:
cmpr = ast.Compare(ast.Name(status_n, ast.Load(), None),
[ast.Eq()], [ast.Num(LOOP_CONT)])
cont_ass = [
ast.If(cmpr,
deepcopy(assign) + [ast.Continue()], cont_ass)
]
if has_break:
cmpr = ast.Compare(ast.Name(status_n, ast.Load(), None),
[ast.Eq()], [ast.Num(LOOP_BREAK)])
cont_ass = [
ast.If(cmpr,
deepcopy(assign) + [ast.Break()], cont_ass)
]
return cont_ass
def test_copy_location(self):
tree = gast.Num(n=1)
tree.lineno = 1
tree.col_offset = 2
node = gast.Num(n=2)
gast.copy_location(node, tree)
self.assertEqual(node.lineno, tree.lineno)
self.assertEqual(node.col_offset, tree.col_offset)
def visit_Compare(self, node):
node = self.generic_visit(node)
if len(node.ops) > 1:
# in case we have more than one compare operator
# we generate an auxiliary function
# that lazily evaluates the needed parameters
imported_ids = self.passmanager.gather(ImportedIds, node, self.ctx)
imported_ids = sorted(imported_ids)
binded_args = [ast.Name(i, ast.Load(), None) for i in imported_ids]
# name of the new function
forged_name = "{0}_compare{1}".format(self.prefix,
len(self.compare_functions))
# call site
call = ast.Call(ast.Name(forged_name, ast.Load(), None),
binded_args, [])
# new function
arg_names = [ast.Name(i, ast.Param(), None) for i in imported_ids]
args = ast.arguments(arg_names, None, [], [], None, [])
body = [] # iteratively fill the body (yeah, feel your body!)
if is_trivially_copied(node.left):
prev_holder = node.left
else:
body.append(
ast.Assign([ast.Name('$0', ast.Store(), None)], node.left))
prev_holder = ast.Name('$0', ast.Load(), None)
for i, exp in enumerate(node.comparators):
if is_trivially_copied(exp):
holder = exp
else:
body.append(
ast.Assign(
[ast.Name('${}'.format(i + 1), ast.Store(), None)],
exp))
holder = ast.Name('${}'.format(i + 1), ast.Load(), None)
cond = ast.Compare(prev_holder, [node.ops[i]], [holder])
body.append(
ast.If(cond, [ast.Pass()], [ast.Return(ast.Num(0))]))
prev_holder = holder
body.append(ast.Return(ast.Num(1)))
forged_fdef = ast.FunctionDef(forged_name, args, body, [], None)
self.compare_functions.append(forged_fdef)
return call
else:
return node
def visit_BinOp(self, node):
self.generic_visit(node)
left_val = node.left
right_val = node.right
left_is_num = isinstance(left_val, gast.Num)
right_is_num = isinstance(right_val, gast.Num)
if isinstance(node.op, gast.Mult):
if left_is_num and right_is_num:
return gast.Num(left_val.n * right_val.n)
if left_is_num:
if left_val.n == 0:
return gast.Num(0)
elif left_val.n == 1:
return right_val
if right_is_num:
if right_val.n == 0:
return gast.Num(0)
elif right_val.n == 1:
return left_val
elif isinstance(node.op, gast.Add):
if left_is_num and right_is_num:
return gast.Num(left_val.n + right_val.n)
if left_is_num and left_val.n == 0:
return right_val
if right_is_num and right_val.n == 0:
return left_val
elif isinstance(node.op, gast.Sub):
if left_is_num and right_is_num:
return gast.Num(left_val.n - right_val.n)
if left_is_num and left_val.n == 0:
return gast.UnaryOp(op=gast.USub(), operand=right_val)
if right_is_num and right_val.n == 0:
return left_val
elif isinstance(node.op, gast.Div):
if left_is_num and right_is_num:
return gast.Num(left_val.n / right_val.n)
if right_is_num and right_val.n == 1:
return left_val
elif isinstance(node.op, gast.Pow):
if left_is_num and right_is_num:
return gast.Num(left_val.n**right_val.n)
if left_is_num:
if left_val.n == 0:
return gast.Num(0)
elif left_val.n == 1:
return gast.Num(1)
if right_is_num:
if right_val.n == 0:
return gast.Num(1)
elif right_val.n == 1:
return left_val
return node
def visit_Subscript(self, node):
"""
>>> import gast as ast
>>> from pythran import passmanager, backend
>>> pm = passmanager.PassManager("test")
>>> node = ast.parse("def foo(a): a[1:][3]")
>>> _, node = pm.apply(PartialConstantFolding, node)
>>> _, node = pm.apply(ConstantFolding, node)
>>> print(pm.dump(backend.Python, node))
def foo(a):
a[4]
>>> node = ast.parse("def foo(a): a[::2][3]")
>>> _, node = pm.apply(PartialConstantFolding, node)
>>> _, node = pm.apply(ConstantFolding, node)
>>> print(pm.dump(backend.Python, node))
def foo(a):
a[6]
>>> node = ast.parse("def foo(a): a[-4:][5]")
>>> _, node = pm.apply(PartialConstantFolding, node)
>>> _, node = pm.apply(ConstantFolding, node)
>>> print(pm.dump(backend.Python, node))
def foo(a):
a[1]
"""
self.generic_visit(node)
if not isinstance(node.value, ast.Subscript):
return node
if not isinstance(node.value.slice, ast.Slice):
return node
if not isinstance(node.slice, ast.Index):
return node
if not isinstance(node.slice.value, ast.Num):
return node
slice_ = node.value.slice
index = node.slice
node = node.value
node.slice = index
lower = slice_.lower or ast.Num(0)
step = slice_.step or ast.Num(1)
node.slice.value = ast.BinOp(lower, ast.Add(),
ast.BinOp(index.value, ast.Mult(), step))
self.update = True
return node
def test_ast_to_object(self):
node = gast.FunctionDef(
name='f',
args=gast.arguments(
args=[gast.Name('a', gast.Param(), None)],
vararg=None,
kwonlyargs=[],
kwarg=None,
defaults=[],
kw_defaults=[]),
body=[
gast.Return(
gast.BinOp(
op=gast.Add(),
left=gast.Name('a', gast.Load(), None),
right=gast.Num(1)))
],
decorator_list=[],
returns=None)
module, source, _ = compiler.ast_to_object(node)
expected_source = """
# coding=utf-8
def f(a):
return a + 1
"""
self.assertEqual(
textwrap.dedent(expected_source).strip(),
source.strip())
self.assertEqual(2, module.f(1))
with open(module.__file__, 'r') as temp_output:
self.assertEqual(
textwrap.dedent(expected_source).strip(),
temp_output.read().strip())
def attach_data(self, node):
'''Generic method called for visit_XXXX() with XXXX in
GatherOMPData.statements list
'''
if self.current:
for curr in self.current:
md = OMPDirective(curr)
metadata.add(node, md)
self.current = list()
# add a Pass to hold some directives
for field_name, field in ast.iter_fields(node):
if field_name in GatherOMPData.statement_lists:
if(field and
isinstance(field[-1], ast.Expr) and
self.isompdirective(field[-1].value)):
field.append(ast.Pass())
self.generic_visit(node)
# add an If to hold scoping OpenMP directives
directives = metadata.get(node, OMPDirective)
field_names = {n for n, _ in ast.iter_fields(node)}
has_no_scope = field_names.isdisjoint(GatherOMPData.statement_lists)
if directives and has_no_scope:
# some directives create a scope, but the holding stmt may not
# artificially create one here if needed
sdirective = ''.join(d.s for d in directives)
scoping = ('parallel', 'task', 'section')
if any(s in sdirective for s in scoping):
metadata.clear(node, OMPDirective)
node = ast.If(ast.Num(1), [node], [])
for directive in directives:
metadata.add(node, directive)
return node
def visit_For(self, node):
target = node.target
if isinstance(target, ast.Tuple) or isinstance(target, ast.List):
renamings = OrderedDict()
self.traverse_tuples(target, (), renamings)
if renamings:
gtarget = self.get_new_id()
node.target = ast.Name(gtarget, node.target.ctx, None)
for rename, state in renamings.items():
nnode = reduce(
lambda x, y: ast.Subscript(
x,
ast.Index(ast.Num(y)),
ast.Load()),
state,
ast.Name(gtarget, ast.Load(), None))
if isinstance(rename, str):
node.body.insert(0,
ast.Assign(
[ast.Name(rename,
ast.Store(),
None)],
nnode)
)
else:
node.body.insert(0, ast.Assign([rename], nnode))
self.generic_visit(node)
return node
def visit_Subscript(self, node):
if isinstance(node.value,
(gast.Name, gast.Num)) and node.value.id == 'd':
if (not isinstance(node.slice, gast.Index)
or not isinstance(node.slice.value,
(gast.Subscript, gast.Name, gast.Str))):
# This happens when the gradient of a constant is taken
if self.replace_grad == Replace.TANGENT:
new_node = gast.Num(0)
else:
new_node = gast.Name(id='_', ctx=None, annotation=None)
self.remove(new_node)
elif (self.replace_grad in (Replace.FULL, Replace.TANGENT)
or isinstance(node.ctx, gast.Load)):
new_node = create.create_grad(node.slice.value, self.namer,
self.tangent)
elif isinstance(node.ctx, gast.Store):
new_node = create.create_temp_grad(node.slice.value,
self.namer, self.tangent)
else:
raise ValueError
new_node.ctx = node.ctx
if isinstance(new_node, gast.Tuple):
for elt in new_node.elts:
elt.ctx = node.ctx
node = new_node
return node
def to_ast(value):
"""
Turn a value into ast expression.
>>> a = 1
>>> print ast.dump(to_ast(a))
Num(n=1)
>>> a = [1, 2, 3]
>>> print ast.dump(to_ast(a))
List(elts=[Num(n=1), Num(n=2), Num(n=3)], ctx=Load())
"""
if isinstance(value, (type(None), bool)):
return builtin_folding(value)
if any(value is t for t in (bool, int, float)):
return builtin_folding(value)
elif isinstance(value, numpy.generic):
return to_ast(numpy.asscalar(value))
elif isinstance(value, numbers.Number):
return ast.Num(value)
elif isinstance(value, str):
return ast.Str(value)
elif isinstance(value, (list, tuple, set, dict, numpy.ndarray)):
return size_container_folding(value)
elif hasattr(value, "__module__") and value.__module__ == "__builtin__":
# TODO Can be done the same way for others modules
return builtin_folding(value)
# only meaningful for python3
elif sys.version_info.major == 3:
if isinstance(value, (filter, map, zip)):
return to_ast(list(value))
raise ToNotEval()
def visit_Assign(self, node):
self.generic_visit(node)
# if the rhs is an identifier, we don't need to duplicate it
# otherwise, better duplicate it...
no_tmp = isinstance(node.value, ast.Name)
extra_assign = [] if no_tmp else [node]
for i, t in enumerate(node.targets):
if isinstance(t, ast.Tuple) or isinstance(t, ast.List):
renamings = OrderedDict()
self.traverse_tuples(t, (), renamings)
if renamings:
gtarget = node.value.id if no_tmp else self.get_new_id()
node.targets[i] = ast.Name(gtarget,
node.targets[i].ctx,
None)
for rename, state in renamings.items():
nnode = reduce(
lambda x, y: ast.Subscript(
x,
ast.Index(ast.Num(y)),
ast.Load()),
state,
ast.Name(gtarget, ast.Load(), None))
if isinstance(rename, str):
extra_assign.append(
ast.Assign(
[ast.Name(rename, ast.Store(), None)],
nnode))
else:
extra_assign.append(ast.Assign([rename], nnode))
return extra_assign or node
def visit_Assign(self, node):
self.src = quoting.unquote(node)
self.mark(node)
self.trivializing = True
self.namer.target = node.targets[0]
if isinstance(node.targets[0], (gast.Subscript, gast.Attribute)):
node.value = self.trivialize(node.value)
node.targets[0] = self.visit(node.targets[0])
elif isinstance(node.targets[0], gast.Tuple):
node.value = self.visit(node.value)
name = self.namer.name(node.targets[0])
target = gast.Name(id=name, ctx=gast.Store(), annotation=None)
for i, elt in enumerate(node.targets[0].elts):
stmt = gast.Assign(targets=[elt],
value=gast.Subscript(
value=gast.Name(id=name,
ctx=gast.Load(),
annotation=None),
slice=gast.Index(value=gast.Num(n=i)),
ctx=gast.Load()))
self.mark(stmt)
self.append(stmt)
node.targets[0] = target
elif not isinstance(node.targets[0], gast.Name):
raise ValueError
node = self.generic_visit(node)
self.namer.target = None
self.trivializing = False
return node
def test_fix_missing_locations(self):
node = gast.Num(n=6)
tree = gast.UnaryOp(gast.USub(), node)
tree.lineno = 1
tree.col_offset = 2
gast.fix_missing_locations(tree)
self.assertEqual(node.lineno, tree.lineno)
self.assertEqual(node.col_offset, tree.col_offset)
def visit_UnaryOp(self, node):
node = self.generic_visit(node)
if isinstance(node.op, gast.USub) and isinstance(node.operand, gast.Num):
value = node.operand.n
replacement = gast.Num(n=-value)
return gast.copy_location(replacement, node)
else:
return node
def visit_range(self, node):
range_value = self.range_values[node]
if isinf(range_value.high):
return self.generic_visit(node)
elif range_value.low == range_value.high:
self.update = True
return ast.Num(range_value.low)
else:
return self.generic_visit(node)
def visit_Name(self, node):
if node.id in self.renamings:
nnode = reduce(
lambda x, y: ast.Subscript(x, ast.Index(ast.Num(y)), ast.Load(
)), self.renamings[node.id],
ast.Name(self.tuple_id, ast.Load(), None))
nnode.ctx = node.ctx
return nnode
return node
def visit_container(self, node):
adjoint = []
for i, elt in enumerate(node.elts):
adjoint.append(
template.replace('d[x] = d[t[i]]',
namer=self.namer,
t=self.target,
i=gast.Num(n=i),
x=elt))
return node, adjoint
def visit_For(self, node):
if isinstance(node.iter, gast.List):
return self.make_unrolled_loop(node, node.iter.elts)
elif isinstance(node.iter, gast.Call) and node.iter.func.id == 'range':
range_args = [arg.n for arg in node.iter.args]
# manually invoke the range to generate the list of values to use.
value_node_list = [gast.Num(i) for i in range(*range_args)]
return self.make_unrolled_loop(node, value_node_list)
else:
return node
def visit_For(self, node):
if isinstance(node.iter, (ast.List, ast.Tuple)):
range_params = self.isrange(node.iter.elts)
if range_params:
node.iter = ast.Call(ast.Attribute(
ast.Name('__builtin__', ast.Load(), None),
'xrange',
node.iter.ctx),
[ast.Num(param) for param in range_params],
[])
self.update = True
return self.generic_visit(node)
def expand_pow(self, node, n):
if n == 0:
return ast.Num(1)
elif n == 1:
return node
else:
node_square = self.replace(node)
node_pow = self.expand_pow(node_square, n >> 1)
if n & 1:
return ast.BinOp(node_pow, ast.Mult(), copy.deepcopy(node))
else:
return node_pow
def to_ast(value):
"""
Turn a value into ast expression.
>>> a = 1
>>> print ast.dump(to_ast(a))
Num(n=1)
>>> a = [1, 2, 3]
>>> print ast.dump(to_ast(a))
List(elts=[Num(n=1), Num(n=2), Num(n=3)], ctx=Load())
"""
numpy_type = (numpy.float64, numpy.float32, numpy.float16, numpy.complex_,
numpy.complex64, numpy.complex128, numpy.float_, numpy.uint8,
numpy.uint16, numpy.uint32, numpy.uint64, numpy.int8,
numpy.int16, numpy.int32, numpy.int64, numpy.intp,
numpy.intc, numpy.int_, numpy.bool_)
itertools_t = [
getattr(itertools, fun) for fun in dir(itertools)
if isinstance(getattr(itertools, fun), type)
]
unfolded_type = (types.BuiltinFunctionType, types.BuiltinMethodType,
numpy.ufunc, type(list.append), BaseException,
types.GeneratorType) + tuple(itertools_t)
if sys.version_info.major == 2:
unfolded_type += (types.FunctionType, types.FileType, types.TypeType,
types.XRangeType)
else:
unfolded_type += type, range, type(numpy.array2string)
if isinstance(value, (type(None), bool)):
return ast.Attribute(ast.Name('__builtin__', ast.Load(), None),
str(value), ast.Load())
elif isinstance(value, numpy_type):
return to_ast(numpy.asscalar(value))
elif isinstance(value, (int, long, float, complex)):
return ast.Num(value)
elif isinstance(value, str):
return ast.Str(value)
elif isinstance(value, (list, tuple, set, dict, numpy.ndarray)):
return size_container_folding(value)
elif hasattr(value, "__module__") and value.__module__ == "__builtin__":
# TODO Can be done the same way for others modules
return builtin_folding(value)
elif isinstance(value, unfolded_type):
raise ToNotEval()
elif value in numpy_type:
raise ToNotEval()
# only meaningful for python3
elif isinstance(value, (filter, map, zip)):
return to_ast(list(value))
else:
raise ConversionError()
def test_NodeTransformer(self):
node = gast.Num(n=6)
tree = gast.UnaryOp(gast.USub(), node)
class Trans(gast.NodeTransformer):
def visit_Num(self, node):
node.n *= 2
return node
tree = Trans().visit(tree)
self.assertEqual(node.n, 12)
def visit_If(self, node):
node = self.generic_visit(node)
node = self.add_mask(node, node.test)
nodes = [node]
if len(node.orelse) > 0:
test_inverse = gast.Call(
gast.Attribute(node.test, gast.Name('eq', gast.Load(), None),
gast.Load()), [gast.Num(0)], [])
else_node = gast.If(any_active(test_inverse), node.orelse, [])
node.orelse = []
self.add_mask(else_node, test_inverse)
nodes.append(else_node)
node.test = any_active(node.test)
return nodes
def test_code_block(self):
def template(block): # pylint:disable=unused-argument
def test_fn(a): # pylint:disable=unused-variable
block # pylint:disable=pointless-statement
return a
node = templates.replace(
template,
block=[
gast.Assign([gast.Name('a', gast.Store(), None)],
gast.BinOp(gast.Name('a', gast.Load(), None),
gast.Add(), gast.Num(1))),
] * 2)[0]
result = compiler.ast_to_object(node)
self.assertEquals(3, result.test_fn(1))
def test_NodeVisitor(self):
node = gast.Num(n=6)
tree = gast.UnaryOp(gast.USub(), node)
class Vis(gast.NodeTransformer):
def __init__(self):
self.state = []
def visit_Num(self, node):
self.state.append(node.n)
vis = Vis()
vis.visit(tree)
self.assertEqual(vis.state, [6])
def test_replace_code_block(self):
template = """
def test_fn(a):
block
return a
"""
node = templates.replace(
template,
block=[
gast.Assign([gast.Name('a', None, None)],
gast.BinOp(gast.Name('a', None, None), gast.Add(),
gast.Num(1))),
] * 2)[0]
result, _ = compiler.ast_to_object(node)
self.assertEquals(3, result.test_fn(1))
def create_grad_list(self, node):
assert isinstance(node, (gast.List, gast.Tuple)), 'Must be list or tuple'
list_of_nodes = node.elts
elts = []
for _node in list_of_nodes:
if isinstance(_node, (gast.Name, gast.Subscript)):
grad_node = create.create_grad(_node, self.namer, tangent=True)
grad_node.ctx = node.ctx
elts.append(grad_node)
elif isinstance(_node, gast.Num):
elts.append(gast.Num(0))
elif isinstance(_node, (gast.List, gast.Tuple)):
elts.append(self.create_grad_list(_node.elts))
else:
raise ValueError('Cannot handle node type %s' % type(_node))
return node.__class__(elts=elts, ctx=node.ctx)
def ast(self):
# The caller must adjust the context appropriately.
if self.has_subscript():
return gast.Subscript(self.parent.ast(),
gast.Index(self.qn[-1].ast()), None)
if self.has_attr():
return gast.Attribute(self.parent.ast(), self.qn[-1], None)
base = self.qn[0]
if isinstance(base, str):
return gast.Name(base, None, None)
elif isinstance(base, StringLiteral):
return gast.Str(base.value)
elif isinstance(base, NumberLiteral):
return gast.Num(base.value)
else:
assert False, ('the constructor should prevent types other than '
'str, StringLiteral and NumberLiteral')
def test_ast_to_source(self):
node = gast.If(
test=gast.Num(1),
body=[
gast.Assign(targets=[gast.Name('a', gast.Store(), None)],
value=gast.Name('b', gast.Load(), None))
],
orelse=[
gast.Assign(targets=[gast.Name('a', gast.Store(), None)],
value=gast.Str('c'))
])
source = compiler.ast_to_source(node, indentation=' ')
self.assertEqual(
textwrap.dedent("""
if 1:
a = b
else:
a = 'c'
""").strip(), source.strip())
def outline(name, formal_parameters, out_parameters, stmts, has_return,
has_break, has_cont):
args = ast.arguments(
[ast.Name(fp, ast.Param(), None) for fp in formal_parameters], None,
[], [], None, [])
if isinstance(stmts, ast.expr):
assert not out_parameters, "no out parameters with expr"
fdef = ast.FunctionDef(name, args, [ast.Return(stmts)], [], None)
else:
fdef = ast.FunctionDef(name, args, stmts, [], None)
# this is part of a huge trick that plays with delayed type inference
# it basically computes the return type based on out parameters, and
# the return statement is unconditionally added so if we have other
# returns, there will be a computation of the output type based on the
# __combined of the regular return types and this one The original
# returns have been patched above to have a different type that
# cunningly combines with this output tuple
#
# This is the only trick I found to let pythran compute both the output
# variable type and the early return type. But hey, a dirty one :-/
stmts.append(
ast.Return(
ast.Tuple(
[ast.Name(fp, ast.Load(), None) for fp in out_parameters],
ast.Load())))
if has_return:
pr = PatchReturn(stmts[-1], has_break or has_cont)
pr.visit(fdef)
if has_break or has_cont:
if not has_return:
stmts[-1].value = ast.Tuple(
[ast.Num(LOOP_NONE), stmts[-1].value], ast.Load())
pbc = PatchBreakContinue(stmts[-1])
pbc.visit(fdef)
return fdef