def visit_Call(self, node):
self.generic_visit(node)
if (isinstance(node.func, ast.Name)
and node.func.id == former_name):
node.func.id = new_name
node.args = ([
ast.Name(iin, ast.Load(), None) for iin in sorted(ii)
] + node.args)
return node
def _process(self, node):
qn = anno.getanno(node, anno.Basic.QN)
if qn in self.name_map:
new_node = gast.Name(str(self.name_map[qn]), node.ctx, None)
# All annotations get carried over.
for k in anno.keys(node):
anno.copyanno(node, new_node, k)
return new_node
return self.generic_visit(node)
def add_stararg(self, a):
self._consume_args()
self._argspec.append(
gast.Call(gast.Name('tuple',
ctx=gast.Load(),
annotation=None,
type_comment=None),
args=[a],
keywords=()))
def _create_continuation_init(self):
def template(var_name): # pylint:disable=unused-argument
var_name = False
assign, = templates.replace(template,
var_name=gast.Name(
self.continuation_uses[-1][1], None,
None))
return assign
def visit_Name(self, node):
new_node = gast.Name(
self._visit(node.id),
self._visit(node.ctx),
None,
None,
)
ast.copy_location(new_node, node)
return new_node
def __init__(self, **kwargs):
self.argument_effects = kwargs.get('argument_effects',
(UpdateEffect(),) * DefaultArgNum)
self.global_effects = kwargs.get('global_effects', False)
self.return_alias = kwargs.get('return_alias',
lambda x: {UnboundValue})
self.args = ast.arguments(
[ast.Name(n, ast.Param(), None, None)
for n in kwargs.get('args', [])],
[], None,
[ast.Name(n, ast.Param(), None, None)
for n in kwargs.get('kwonlyargs', [])],
[], None,
[to_ast(d) for d in kwargs.get('defaults', [])])
self.return_range = kwargs.get("return_range",
lambda call: UNKNOWN_RANGE)
self.return_range_content = kwargs.get("return_range_content",
lambda c: UNKNOWN_RANGE)
def attr_to_func(self, node, mod=None):
if mod is None:
mod = methods[node.attr][0]
# Submodules import full module
self.to_import.add(mangle(mod[0]))
func = reduce(lambda v, o: ast.Attribute(v, o, ast.Load()),
mod[1:] + (node.attr, ),
ast.Name(mangle(mod[0]), ast.Load(), None, None))
return func
def _build_var_slice_node(self):
return gast.Subscript(
value=self.iter_node,
slice=gast.Index(value=gast.Name(
id=self.iter_idx_name,
ctx=gast.Load(),
annotation=None,
type_comment=None)),
ctx=gast.Load())
def visit_Name(self, node):
if node.id in self.renamings:
nnode = reduce(
lambda x, y: ast.Subscript(x, ast.Index(ast.Num(y)), ast.Load(
)), self.renamings[node.id],
ast.Name(self.tuple_id, ast.Load(), None))
nnode.ctx = node.ctx
return nnode
return node
def _create_break_trigger(self):
def template(var_name): # pylint:disable=unused-argument
var_name = True
block = templates.replace(template,
var_name=gast.Name(self.break_uses[-1][1],
None, None))
block.append(gast.Continue())
return block
def make_dispatcher(static_expr, func_true, func_false, imported_ids):
dispatcher_args = [
static_expr,
ast.Name(func_true.name, ast.Load(), None, None),
ast.Name(func_false.name, ast.Load(), None, None)
]
dispatcher = ast.Call(
ast.Attribute(
ast.Attribute(ast.Name("__builtin__", ast.Load(), None, None),
"pythran", ast.Load()), "static_if", ast.Load()),
dispatcher_args, [])
actual_call = ast.Call(
dispatcher,
[ast.Name(ii, ast.Load(), None, None) for ii in imported_ids], [])
return actual_call
def visit_ExceptHandler(self, node):
if node.name:
new_node = gast.ExceptHandler(
self._visit(node.type), gast.Name(node.name,
gast.Store(), None),
self._visit(node.body))
return ast.copy_location(new_node, node)
else:
return self.generic_visit(node)
def make_Iterator(self, gen):
if gen.ifs:
ldFilter = ast.Lambda(
ast.arguments(
[ast.Name(gen.target.id, ast.Param(), None, None)], [],
None, [], [], None, []),
ast.BoolOp(ast.And(), gen.ifs)
if len(gen.ifs) > 1 else gen.ifs[0])
self.use_itertools |= MODULE == 'itertools'
ifilterName = ast.Attribute(value=ast.Name(id=ASMODULE,
ctx=ast.Load(),
annotation=None,
type_comment=None),
attr=IFILTER,
ctx=ast.Load())
return ast.Call(ifilterName, [ldFilter, gen.iter], [])
else:
return gen.iter
def makeattr(*args):
self.use_itertools |= MODULE == 'itertools'
return ast.Call(
ast.Attribute(value=ast.Name(id=ASMODULE,
ctx=ast.Load(),
annotation=None,
type_comment=None),
attr=IMAP,
ctx=ast.Load()), list(args), [])
def sub():
return ast.Call(func=ast.Attribute(value=ast.Attribute(value=ast.Name(
'builtins', ast.Load(), None, None),
attr="pythran",
ctx=ast.Load()),
attr="len_set",
ctx=ast.Load()),
args=[Placeholder(0)],
keywords=[])
def _process_body_item(self, node):
if isinstance(node, gast.Assign) and (node.value.id == 'y'):
if_node = gast.If(
gast.Name('x',
ctx=gast.Load(),
annotation=None,
type_comment=None), [node], [])
return if_node, if_node.body
return node, None
def sub():
return ast.Call(func=ast.Attribute(value=ast.Name(id='builtins',
ctx=ast.Load(),
annotation=None,
type_comment=None),
attr="tuple",
ctx=ast.Load()),
args=[Placeholder(0)],
keywords=[])
def sub():
return ast.Call(func=ast.Attribute(value=ast.Name(id=mangle('numpy'),
ctx=ast.Load(),
annotation=None,
type_comment=None),
attr="cbrt",
ctx=ast.Load()),
args=[Placeholder(0)],
keywords=[])
def visit_While(self, node):
self.generic_visit(node)
# Scrape out the data flow analysis
body_scope = anno.getanno(node, 'body_scope')
parent_scope_values = anno.getanno(node, 'parent_scope_values')
body_closure = tuple(body_scope.modified - body_scope.created)
def template(
state_args, # pylint:disable=unused-argument
state_locals,
state_results, # pylint:disable=unused-argument
test_name,
test, # pylint:disable=unused-argument
body_name,
body,
state_init):
def test_name(state_args): # pylint:disable=function-redefined,unused-argument
return test
def body_name(state_args): # pylint:disable=function-redefined,unused-argument
body # pylint:disable=pointless-statement
return state_locals
state_results = tf.while_loop(test_name, body_name, [state_init]) # pylint:disable=undefined-variable
test_name = self.namer.new_symbol('loop_test', body_scope.used)
body_name = self.namer.new_symbol('loop_body', body_scope.used)
node = templates.replace(
template,
state_args=self._tuple_or_item(
gast.Name(n, gast.Param(), None) for n in body_closure),
state_locals=self._ast_tuple_or_item(
(gast.Name(n, gast.Load(), None) for n in body_closure),
gast.Load()),
state_results=self._ast_tuple_or_item(
(gast.Name(n, gast.Store(), None) for n in body_closure),
gast.Store()),
test_name=gast.Name(test_name, gast.Load(), None),
test=node.test,
body_name=gast.Name(body_name, gast.Load(), None),
body=node.body,
state_init=[parent_scope_values.getval(n) for n in body_closure])
return node
def test_load_ast(self):
node = gast.FunctionDef(
name='f',
args=gast.arguments(
args=[
gast.Name(
'a', ctx=gast.Param(), annotation=None, type_comment=None)
],
posonlyargs=[],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[]),
body=[
gast.Return(
gast.BinOp(
op=gast.Add(),
left=gast.Name(
'a',
ctx=gast.Load(),
annotation=None,
type_comment=None),
right=gast.Constant(1, kind=None)))
],
decorator_list=[],
returns=None,
type_comment=None)
module, source, _ = loader.load_ast(node)
expected_source = """
# coding=utf-8
def f(a):
return (a + 1)
"""
self.assertEqual(
textwrap.dedent(expected_source).strip(),
source.strip())
self.assertEqual(2, module.f(1))
with open(module.__file__, 'r') as temp_output:
self.assertEqual(
textwrap.dedent(expected_source).strip(),
temp_output.read().strip())
def function_to_graph(f,
program_ctx,
arg_values,
arg_types,
owner_type=None):
"""Specialization of `entity_to_graph` for callable functions."""
node, source = parser.parse_entity(f)
node = node.body[0]
# TODO(mdan): Can we convert everything and scoop the lambda afterwards?
if f.__name__ == '<lambda>':
nodes = ast_util.find_matching_lambda_definitions(node, f)
if len(nodes) != 1:
raise ValueError(
'Unable to identify source code of lambda function {}. It was'
' defined on this line: {}, which contains multiple lambdas with'
' identical argument names. To avoid ambiguity, define each lambda'
' in a separate expression.'.format(f, source))
node, = nodes
# TODO(znado): Place inside standard_analysis.
origin_info.resolve(node, source, f)
namespace = inspect_utils.getnamespace(f)
_add_self_references(namespace, program_ctx.autograph_module)
namer = program_ctx.new_namer(namespace)
entity_info = transformer.EntityInfo(
source_code=source,
source_file='<fragment>',
namespace=namespace,
arg_values=arg_values,
arg_types=arg_types,
owner_type=owner_type)
context = converter.EntityContext(namer, entity_info, program_ctx)
node = node_to_graph(node, context)
if isinstance(node, gast.Lambda):
new_name = namer.new_symbol('tf__lambda', ())
node = gast.Assign(
targets=[gast.Name(new_name, gast.Store(), None)], value=node)
else:
# TODO(mdan): This somewhat duplicates the renaming logic in call_trees.py
new_name, did_rename = namer.compiled_function_name(f.__name__, f,
owner_type)
if did_rename:
node.name = new_name
else:
new_name = f.__name__
assert node.name == new_name
program_ctx.update_name_map(namer)
# TODO(mdan): Use this at compilation.
return [node], new_name, namespace
def function_to_graph(f, program_ctx, arg_values, arg_types, do_rename=True):
"""Specialization of `entity_to_graph` for callable functions."""
future_features = inspect_utils.getfutureimports(f)
node, source = parser.parse_entity(f, future_features=future_features)
logging.log(3, 'Source code of %s:\n\n%s\n', f, source)
# Parsed AST should contain future imports and one function def node.
# In general, the output of inspect.getsource is inexact for lambdas because
# it uses regex matching to adjust the exact location around the line number
# that CPython records. Then, the entire containing line is returned, which
# we may have trouble disambiguating. For example:
# x, y = lambda: 1, lambda: 2
if f.__name__ == '<lambda>':
nodes = ast_util.find_matching_definitions(node, f)
if len(nodes) != 1:
raise ValueError(
'Unable to identify source code of lambda function {}. It was'
' defined on this line: {}, which must contain a single lambda with'
' matching signature. To avoid ambiguity, define each lambda'
' in a separate expression.'.format(f, source))
node, = nodes
# TODO(znado): Place inside standard_analysis.
origin_info.resolve(node, source, f)
namespace = inspect_utils.getnamespace(f)
_add_self_references(namespace, program_ctx.autograph_module)
namer = naming.Namer(namespace)
entity_info = transformer.EntityInfo(source_code=source,
source_file='<fragment>',
future_features=future_features,
namespace=namespace,
arg_values=arg_values,
arg_types=arg_types)
context = converter.EntityContext(namer, entity_info, program_ctx)
try:
node = node_to_graph(node, context)
except (ValueError, AttributeError, KeyError, NotImplementedError) as e:
logging.error(1, 'Error converting %s', f, exc_info=True)
raise errors.InternalError('conversion', e)
# TODO(mdan): Catch and rethrow syntax errors.
if isinstance(node, gast.Lambda):
new_name = namer.new_symbol('tf__lambda', ())
node = gast.Assign(targets=[gast.Name(new_name, gast.Store(), None)],
value=node)
elif do_rename:
new_name = namer.function_name(f.__name__)
node.name = new_name
else:
new_name = f.__name__
assert node.name == new_name
return (node, ), new_name, entity_info
def handle_keywords(self, func, node, offset=0):
'''
Gather keywords to positional argument information
Assumes the named parameter exist, raises a KeyError otherwise
'''
func_argument_names = {}
for i, arg in enumerate(func.args.args[offset:]):
assert isinstance(arg, ast.Name)
func_argument_names[arg.id] = i
func_argument_kwonly_names = {}
for i, arg in enumerate(func.args.kwonlyargs):
assert isinstance(arg, ast.Name)
func_argument_kwonly_names[arg.id] = i
nargs = len(func.args.args) - offset
defaults = func.args.defaults
keywords = {
func_argument_names[kw.arg]: kw.value
for kw in node.keywords if kw.arg not in func_argument_kwonly_names
}
keywords_only = []
nb_kw = len(node.keywords)
for i, kw in enumerate(list(reversed(node.keywords))):
if kw.arg in func_argument_kwonly_names:
keywords_only.append(
(func_argument_kwonly_names[kw.arg], kw.value))
node.keywords.pop(nb_kw - i - 1)
keywords_only = [v for _, v in sorted(keywords_only)]
extra_keyword_offset = max(keywords.keys()) if keywords else 0
node.args.extend([None] * (1 + extra_keyword_offset - len(node.args)))
replacements = {}
for index, arg in enumerate(node.args):
if arg is None:
if index in keywords:
replacements[index] = deepcopy(keywords[index])
else: # must be a default value
replacements[index] = deepcopy(defaults[index - nargs])
if not keywords_only:
return replacements
node.args.append(
ast.Call(
ast.Attribute(
ast.Attribute(
ast.Name("__builtin__", ast.Load(), None, None),
"pythran", ast.Load()), "kwonly", ast.Load()), [], []))
node.args.extend(keywords_only)
return replacements
def parse_cond_args(parent_ids_dict,
var_ids_dict,
modified_ids_dict=None,
ctx=gast.Load):
"""
Find out the ast.Name.id list of input by analyzing node's AST information.
"""
# 1. filter the var fit the ctx
arg_name_ids = [
var_id for var_id, var_ctx in six.iteritems(var_ids_dict)
if isinstance(var_ctx[0], ctx)
]
# 2. args should contain modified var ids in if-body or else-body
# case:
#
# ```
# if b < 1:
# z = y
# else:
# z = x
# ```
#
# In the above case, `z` should be in the args of cond()
if modified_ids_dict:
arg_name_ids = set(arg_name_ids) | set(modified_ids_dict)
# 3. args should not contain the vars not in parent ids
# case :
#
# ```
# x = 1
# if x > y:
# z = [v for v in range(i)]
# ```
#
# In the above case, `v` should not be in the args of cond()
arg_name_ids = list(set(arg_name_ids) & set(parent_ids_dict))
arg_name_ids.sort()
args = [
gast.Name(id=name_id,
ctx=gast.Load(),
annotation=None,
type_comment=None) for name_id in arg_name_ids
]
arguments = gast.arguments(args=args,
posonlyargs=[],
vararg=None,
kwonlyargs=[],
kw_defaults=None,
kwarg=None,
defaults=[])
return arguments
def _build_compare_node(self):
if self.is_for_range_iter():
compare_node = self.iter_args[
0] if self.args_length == 1 else self.iter_args[1]
else:
compare_node = gast.Name(id=self.iter_var_len_name,
ctx=gast.Load(),
annotation=None,
type_comment=None)
return compare_node
def visit_Name(self, node):
new_node = gast.Name(
self._visit(node.id),
self._visit(node.ctx),
None,
None,
)
gast.copy_location(new_node, node)
new_node.end_lineno = new_node.end_col_offset = None
return new_node
def builtin_folding(value):
""" Convert builtin function to ast expression. """
if isinstance(value, (type(None), bool)):
name = str(value)
elif value.__name__ in ("bool", "float", "int", "long"):
name = value.__name__ + "_"
else:
name = value.__name__
return ast.Attribute(ast.Name('__builtin__', ast.Load(), None), name,
ast.Load())
def visit_Compare(self, node):
""" Boolean are possible index.
>>> import gast as ast
>>> from pythran import passmanager, backend
>>> node = ast.parse('''
... def foo():
... a = 2 or 3
... b = 4 or 5
... c = a < b
... d = b < 3
... e = b == 4''')
>>> pm = passmanager.PassManager("test")
>>> res = pm.gather(RangeValues, node)
>>> res['c']
Interval(low=1, high=1)
>>> res['d']
Interval(low=0, high=0)
>>> res['e']
Interval(low=0, high=1)
"""
if any(
isinstance(op, (ast.In, ast.NotIn, ast.Is, ast.IsNot))
for op in node.ops):
self.generic_visit(node)
return self.add(node, Interval(0, 1))
curr = self.visit(node.left)
res = []
for op, comparator in zip(node.ops, node.comparators):
comparator = self.visit(comparator)
fake = ast.Compare(ast.Name('x', ast.Load(), None), [op],
[ast.Name('y', ast.Load(), None)])
fake = ast.Expression(fake)
ast.fix_missing_locations(fake)
expr = compile(ast.gast_to_ast(fake), '<range_values>', 'eval')
res.append(eval(expr, {'x': curr, 'y': comparator}))
if all(res):
return self.add(node, Interval(1, 1))
elif any(r.low == r.high == 0 for r in res):
return self.add(node, Interval(0, 0))
else:
return self.add(node, Interval(0, 1))
def sub():
return ast.Call(
func=ast.Attribute(
ast.Attribute(ast.Name('__builtin__', ast.Load(), None, None),
'str', ast.Load()), 'join', ast.Load()),
args=[
ast.Constant(Placeholder(1), None),
ast.Tuple([Placeholder(0), Placeholder(2)], ast.Load())
],
keywords=[])
def outline(name, formal_parameters, out_parameters, stmts, has_return,
has_break, has_cont):
args = ast.arguments(
[ast.Name(fp, ast.Param(), None, None) for fp in formal_parameters],
[], None, [], [], None, [])
if isinstance(stmts, ast.expr):
assert not out_parameters, "no out parameters with expr"
fdef = ast.FunctionDef(name, args, [ast.Return(stmts)], [], None, None)
else:
fdef = ast.FunctionDef(name, args, stmts, [], None, None)
# this is part of a huge trick that plays with delayed type inference
# it basically computes the return type based on out parameters, and
# the return statement is unconditionally added so if we have other
# returns, there will be a computation of the output type based on the
# __combined of the regular return types and this one The original
# returns have been patched above to have a different type that
# cunningly combines with this output tuple
#
# This is the only trick I found to let pythran compute both the output
# variable type and the early return type. But hey, a dirty one :-/
stmts.append(
ast.Return(
ast.Tuple([
ast.Name(fp, ast.Load(), None, None)
for fp in out_parameters
], ast.Load())))
if has_return:
pr = PatchReturn(stmts[-1], has_break or has_cont)
pr.visit(fdef)
if has_break or has_cont:
if not has_return:
stmts[-1].value = ast.Tuple(
[ast.Constant(LOOP_NONE, None), stmts[-1].value],
ast.Load())
pbc = PatchBreakContinue(stmts[-1])
pbc.visit(fdef)
return fdef
