def visit_Call(self, node): self.generic_visit(node) if (isinstance(node.func, ast.Name) and node.func.id == former_name): node.func.id = new_name node.args = ([ ast.Name(iin, ast.Load(), None) for iin in sorted(ii) ] + node.args) return node
def _process(self, node): qn = anno.getanno(node, anno.Basic.QN) if qn in self.name_map: new_node = gast.Name(str(self.name_map[qn]), node.ctx, None) # All annotations get carried over. for k in anno.keys(node): anno.copyanno(node, new_node, k) return new_node return self.generic_visit(node)
def add_stararg(self, a): self._consume_args() self._argspec.append( gast.Call(gast.Name('tuple', ctx=gast.Load(), annotation=None, type_comment=None), args=[a], keywords=()))
def _create_continuation_init(self): def template(var_name): # pylint:disable=unused-argument var_name = False assign, = templates.replace(template, var_name=gast.Name( self.continuation_uses[-1][1], None, None)) return assign
def visit_Name(self, node): new_node = gast.Name( self._visit(node.id), self._visit(node.ctx), None, None, ) ast.copy_location(new_node, node) return new_node
def __init__(self, **kwargs): self.argument_effects = kwargs.get('argument_effects', (UpdateEffect(),) * DefaultArgNum) self.global_effects = kwargs.get('global_effects', False) self.return_alias = kwargs.get('return_alias', lambda x: {UnboundValue}) self.args = ast.arguments( [ast.Name(n, ast.Param(), None, None) for n in kwargs.get('args', [])], [], None, [ast.Name(n, ast.Param(), None, None) for n in kwargs.get('kwonlyargs', [])], [], None, [to_ast(d) for d in kwargs.get('defaults', [])]) self.return_range = kwargs.get("return_range", lambda call: UNKNOWN_RANGE) self.return_range_content = kwargs.get("return_range_content", lambda c: UNKNOWN_RANGE)
def attr_to_func(self, node, mod=None): if mod is None: mod = methods[node.attr][0] # Submodules import full module self.to_import.add(mangle(mod[0])) func = reduce(lambda v, o: ast.Attribute(v, o, ast.Load()), mod[1:] + (node.attr, ), ast.Name(mangle(mod[0]), ast.Load(), None, None)) return func
def _build_var_slice_node(self): return gast.Subscript( value=self.iter_node, slice=gast.Index(value=gast.Name( id=self.iter_idx_name, ctx=gast.Load(), annotation=None, type_comment=None)), ctx=gast.Load())
def visit_Name(self, node): if node.id in self.renamings: nnode = reduce( lambda x, y: ast.Subscript(x, ast.Index(ast.Num(y)), ast.Load( )), self.renamings[node.id], ast.Name(self.tuple_id, ast.Load(), None)) nnode.ctx = node.ctx return nnode return node
def _create_break_trigger(self): def template(var_name): # pylint:disable=unused-argument var_name = True block = templates.replace(template, var_name=gast.Name(self.break_uses[-1][1], None, None)) block.append(gast.Continue()) return block
def make_dispatcher(static_expr, func_true, func_false, imported_ids): dispatcher_args = [ static_expr, ast.Name(func_true.name, ast.Load(), None, None), ast.Name(func_false.name, ast.Load(), None, None) ] dispatcher = ast.Call( ast.Attribute( ast.Attribute(ast.Name("__builtin__", ast.Load(), None, None), "pythran", ast.Load()), "static_if", ast.Load()), dispatcher_args, []) actual_call = ast.Call( dispatcher, [ast.Name(ii, ast.Load(), None, None) for ii in imported_ids], []) return actual_call
def visit_ExceptHandler(self, node): if node.name: new_node = gast.ExceptHandler( self._visit(node.type), gast.Name(node.name, gast.Store(), None), self._visit(node.body)) return ast.copy_location(new_node, node) else: return self.generic_visit(node)
def make_Iterator(self, gen): if gen.ifs: ldFilter = ast.Lambda( ast.arguments( [ast.Name(gen.target.id, ast.Param(), None, None)], [], None, [], [], None, []), ast.BoolOp(ast.And(), gen.ifs) if len(gen.ifs) > 1 else gen.ifs[0]) self.use_itertools |= MODULE == 'itertools' ifilterName = ast.Attribute(value=ast.Name(id=ASMODULE, ctx=ast.Load(), annotation=None, type_comment=None), attr=IFILTER, ctx=ast.Load()) return ast.Call(ifilterName, [ldFilter, gen.iter], []) else: return gen.iter
def makeattr(*args): self.use_itertools |= MODULE == 'itertools' return ast.Call( ast.Attribute(value=ast.Name(id=ASMODULE, ctx=ast.Load(), annotation=None, type_comment=None), attr=IMAP, ctx=ast.Load()), list(args), [])
def sub(): return ast.Call(func=ast.Attribute(value=ast.Attribute(value=ast.Name( 'builtins', ast.Load(), None, None), attr="pythran", ctx=ast.Load()), attr="len_set", ctx=ast.Load()), args=[Placeholder(0)], keywords=[])
def _process_body_item(self, node): if isinstance(node, gast.Assign) and (node.value.id == 'y'): if_node = gast.If( gast.Name('x', ctx=gast.Load(), annotation=None, type_comment=None), [node], []) return if_node, if_node.body return node, None
def sub(): return ast.Call(func=ast.Attribute(value=ast.Name(id='builtins', ctx=ast.Load(), annotation=None, type_comment=None), attr="tuple", ctx=ast.Load()), args=[Placeholder(0)], keywords=[])
def sub(): return ast.Call(func=ast.Attribute(value=ast.Name(id=mangle('numpy'), ctx=ast.Load(), annotation=None, type_comment=None), attr="cbrt", ctx=ast.Load()), args=[Placeholder(0)], keywords=[])
def visit_While(self, node): self.generic_visit(node) # Scrape out the data flow analysis body_scope = anno.getanno(node, 'body_scope') parent_scope_values = anno.getanno(node, 'parent_scope_values') body_closure = tuple(body_scope.modified - body_scope.created) def template( state_args, # pylint:disable=unused-argument state_locals, state_results, # pylint:disable=unused-argument test_name, test, # pylint:disable=unused-argument body_name, body, state_init): def test_name(state_args): # pylint:disable=function-redefined,unused-argument return test def body_name(state_args): # pylint:disable=function-redefined,unused-argument body # pylint:disable=pointless-statement return state_locals state_results = tf.while_loop(test_name, body_name, [state_init]) # pylint:disable=undefined-variable test_name = self.namer.new_symbol('loop_test', body_scope.used) body_name = self.namer.new_symbol('loop_body', body_scope.used) node = templates.replace( template, state_args=self._tuple_or_item( gast.Name(n, gast.Param(), None) for n in body_closure), state_locals=self._ast_tuple_or_item( (gast.Name(n, gast.Load(), None) for n in body_closure), gast.Load()), state_results=self._ast_tuple_or_item( (gast.Name(n, gast.Store(), None) for n in body_closure), gast.Store()), test_name=gast.Name(test_name, gast.Load(), None), test=node.test, body_name=gast.Name(body_name, gast.Load(), None), body=node.body, state_init=[parent_scope_values.getval(n) for n in body_closure]) return node
def test_load_ast(self): node = gast.FunctionDef( name='f', args=gast.arguments( args=[ gast.Name( 'a', ctx=gast.Param(), annotation=None, type_comment=None) ], posonlyargs=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[ gast.Return( gast.BinOp( op=gast.Add(), left=gast.Name( 'a', ctx=gast.Load(), annotation=None, type_comment=None), right=gast.Constant(1, kind=None))) ], decorator_list=[], returns=None, type_comment=None) module, source, _ = loader.load_ast(node) expected_source = """ # coding=utf-8 def f(a): return (a + 1) """ self.assertEqual( textwrap.dedent(expected_source).strip(), source.strip()) self.assertEqual(2, module.f(1)) with open(module.__file__, 'r') as temp_output: self.assertEqual( textwrap.dedent(expected_source).strip(), temp_output.read().strip())
def function_to_graph(f, program_ctx, arg_values, arg_types, owner_type=None): """Specialization of `entity_to_graph` for callable functions.""" node, source = parser.parse_entity(f) node = node.body[0] # TODO(mdan): Can we convert everything and scoop the lambda afterwards? if f.__name__ == '<lambda>': nodes = ast_util.find_matching_lambda_definitions(node, f) if len(nodes) != 1: raise ValueError( 'Unable to identify source code of lambda function {}. It was' ' defined on this line: {}, which contains multiple lambdas with' ' identical argument names. To avoid ambiguity, define each lambda' ' in a separate expression.'.format(f, source)) node, = nodes # TODO(znado): Place inside standard_analysis. origin_info.resolve(node, source, f) namespace = inspect_utils.getnamespace(f) _add_self_references(namespace, program_ctx.autograph_module) namer = program_ctx.new_namer(namespace) entity_info = transformer.EntityInfo( source_code=source, source_file='<fragment>', namespace=namespace, arg_values=arg_values, arg_types=arg_types, owner_type=owner_type) context = converter.EntityContext(namer, entity_info, program_ctx) node = node_to_graph(node, context) if isinstance(node, gast.Lambda): new_name = namer.new_symbol('tf__lambda', ()) node = gast.Assign( targets=[gast.Name(new_name, gast.Store(), None)], value=node) else: # TODO(mdan): This somewhat duplicates the renaming logic in call_trees.py new_name, did_rename = namer.compiled_function_name(f.__name__, f, owner_type) if did_rename: node.name = new_name else: new_name = f.__name__ assert node.name == new_name program_ctx.update_name_map(namer) # TODO(mdan): Use this at compilation. return [node], new_name, namespace
def function_to_graph(f, program_ctx, arg_values, arg_types, do_rename=True): """Specialization of `entity_to_graph` for callable functions.""" future_features = inspect_utils.getfutureimports(f) node, source = parser.parse_entity(f, future_features=future_features) logging.log(3, 'Source code of %s:\n\n%s\n', f, source) # Parsed AST should contain future imports and one function def node. # In general, the output of inspect.getsource is inexact for lambdas because # it uses regex matching to adjust the exact location around the line number # that CPython records. Then, the entire containing line is returned, which # we may have trouble disambiguating. For example: # x, y = lambda: 1, lambda: 2 if f.__name__ == '<lambda>': nodes = ast_util.find_matching_definitions(node, f) if len(nodes) != 1: raise ValueError( 'Unable to identify source code of lambda function {}. It was' ' defined on this line: {}, which must contain a single lambda with' ' matching signature. To avoid ambiguity, define each lambda' ' in a separate expression.'.format(f, source)) node, = nodes # TODO(znado): Place inside standard_analysis. origin_info.resolve(node, source, f) namespace = inspect_utils.getnamespace(f) _add_self_references(namespace, program_ctx.autograph_module) namer = naming.Namer(namespace) entity_info = transformer.EntityInfo(source_code=source, source_file='<fragment>', future_features=future_features, namespace=namespace, arg_values=arg_values, arg_types=arg_types) context = converter.EntityContext(namer, entity_info, program_ctx) try: node = node_to_graph(node, context) except (ValueError, AttributeError, KeyError, NotImplementedError) as e: logging.error(1, 'Error converting %s', f, exc_info=True) raise errors.InternalError('conversion', e) # TODO(mdan): Catch and rethrow syntax errors. if isinstance(node, gast.Lambda): new_name = namer.new_symbol('tf__lambda', ()) node = gast.Assign(targets=[gast.Name(new_name, gast.Store(), None)], value=node) elif do_rename: new_name = namer.function_name(f.__name__) node.name = new_name else: new_name = f.__name__ assert node.name == new_name return (node, ), new_name, entity_info
def handle_keywords(self, func, node, offset=0): ''' Gather keywords to positional argument information Assumes the named parameter exist, raises a KeyError otherwise ''' func_argument_names = {} for i, arg in enumerate(func.args.args[offset:]): assert isinstance(arg, ast.Name) func_argument_names[arg.id] = i func_argument_kwonly_names = {} for i, arg in enumerate(func.args.kwonlyargs): assert isinstance(arg, ast.Name) func_argument_kwonly_names[arg.id] = i nargs = len(func.args.args) - offset defaults = func.args.defaults keywords = { func_argument_names[kw.arg]: kw.value for kw in node.keywords if kw.arg not in func_argument_kwonly_names } keywords_only = [] nb_kw = len(node.keywords) for i, kw in enumerate(list(reversed(node.keywords))): if kw.arg in func_argument_kwonly_names: keywords_only.append( (func_argument_kwonly_names[kw.arg], kw.value)) node.keywords.pop(nb_kw - i - 1) keywords_only = [v for _, v in sorted(keywords_only)] extra_keyword_offset = max(keywords.keys()) if keywords else 0 node.args.extend([None] * (1 + extra_keyword_offset - len(node.args))) replacements = {} for index, arg in enumerate(node.args): if arg is None: if index in keywords: replacements[index] = deepcopy(keywords[index]) else: # must be a default value replacements[index] = deepcopy(defaults[index - nargs]) if not keywords_only: return replacements node.args.append( ast.Call( ast.Attribute( ast.Attribute( ast.Name("__builtin__", ast.Load(), None, None), "pythran", ast.Load()), "kwonly", ast.Load()), [], [])) node.args.extend(keywords_only) return replacements
def parse_cond_args(parent_ids_dict, var_ids_dict, modified_ids_dict=None, ctx=gast.Load): """ Find out the ast.Name.id list of input by analyzing node's AST information. """ # 1. filter the var fit the ctx arg_name_ids = [ var_id for var_id, var_ctx in six.iteritems(var_ids_dict) if isinstance(var_ctx[0], ctx) ] # 2. args should contain modified var ids in if-body or else-body # case: # # ``` # if b < 1: # z = y # else: # z = x # ``` # # In the above case, `z` should be in the args of cond() if modified_ids_dict: arg_name_ids = set(arg_name_ids) | set(modified_ids_dict) # 3. args should not contain the vars not in parent ids # case : # # ``` # x = 1 # if x > y: # z = [v for v in range(i)] # ``` # # In the above case, `v` should not be in the args of cond() arg_name_ids = list(set(arg_name_ids) & set(parent_ids_dict)) arg_name_ids.sort() args = [ gast.Name(id=name_id, ctx=gast.Load(), annotation=None, type_comment=None) for name_id in arg_name_ids ] arguments = gast.arguments(args=args, posonlyargs=[], vararg=None, kwonlyargs=[], kw_defaults=None, kwarg=None, defaults=[]) return arguments
def _build_compare_node(self): if self.is_for_range_iter(): compare_node = self.iter_args[ 0] if self.args_length == 1 else self.iter_args[1] else: compare_node = gast.Name(id=self.iter_var_len_name, ctx=gast.Load(), annotation=None, type_comment=None) return compare_node
def visit_Name(self, node): new_node = gast.Name( self._visit(node.id), self._visit(node.ctx), None, None, ) gast.copy_location(new_node, node) new_node.end_lineno = new_node.end_col_offset = None return new_node
def builtin_folding(value): """ Convert builtin function to ast expression. """ if isinstance(value, (type(None), bool)): name = str(value) elif value.__name__ in ("bool", "float", "int", "long"): name = value.__name__ + "_" else: name = value.__name__ return ast.Attribute(ast.Name('__builtin__', ast.Load(), None), name, ast.Load())
def visit_Compare(self, node): """ Boolean are possible index. >>> import gast as ast >>> from pythran import passmanager, backend >>> node = ast.parse(''' ... def foo(): ... a = 2 or 3 ... b = 4 or 5 ... c = a < b ... d = b < 3 ... e = b == 4''') >>> pm = passmanager.PassManager("test") >>> res = pm.gather(RangeValues, node) >>> res['c'] Interval(low=1, high=1) >>> res['d'] Interval(low=0, high=0) >>> res['e'] Interval(low=0, high=1) """ if any( isinstance(op, (ast.In, ast.NotIn, ast.Is, ast.IsNot)) for op in node.ops): self.generic_visit(node) return self.add(node, Interval(0, 1)) curr = self.visit(node.left) res = [] for op, comparator in zip(node.ops, node.comparators): comparator = self.visit(comparator) fake = ast.Compare(ast.Name('x', ast.Load(), None), [op], [ast.Name('y', ast.Load(), None)]) fake = ast.Expression(fake) ast.fix_missing_locations(fake) expr = compile(ast.gast_to_ast(fake), '<range_values>', 'eval') res.append(eval(expr, {'x': curr, 'y': comparator})) if all(res): return self.add(node, Interval(1, 1)) elif any(r.low == r.high == 0 for r in res): return self.add(node, Interval(0, 0)) else: return self.add(node, Interval(0, 1))
def sub(): return ast.Call( func=ast.Attribute( ast.Attribute(ast.Name('__builtin__', ast.Load(), None, None), 'str', ast.Load()), 'join', ast.Load()), args=[ ast.Constant(Placeholder(1), None), ast.Tuple([Placeholder(0), Placeholder(2)], ast.Load()) ], keywords=[])
def outline(name, formal_parameters, out_parameters, stmts, has_return, has_break, has_cont): args = ast.arguments( [ast.Name(fp, ast.Param(), None, None) for fp in formal_parameters], [], None, [], [], None, []) if isinstance(stmts, ast.expr): assert not out_parameters, "no out parameters with expr" fdef = ast.FunctionDef(name, args, [ast.Return(stmts)], [], None, None) else: fdef = ast.FunctionDef(name, args, stmts, [], None, None) # this is part of a huge trick that plays with delayed type inference # it basically computes the return type based on out parameters, and # the return statement is unconditionally added so if we have other # returns, there will be a computation of the output type based on the # __combined of the regular return types and this one The original # returns have been patched above to have a different type that # cunningly combines with this output tuple # # This is the only trick I found to let pythran compute both the output # variable type and the early return type. But hey, a dirty one :-/ stmts.append( ast.Return( ast.Tuple([ ast.Name(fp, ast.Load(), None, None) for fp in out_parameters ], ast.Load()))) if has_return: pr = PatchReturn(stmts[-1], has_break or has_cont) pr.visit(fdef) if has_break or has_cont: if not has_return: stmts[-1].value = ast.Tuple( [ast.Constant(LOOP_NONE, None), stmts[-1].value], ast.Load()) pbc = PatchBreakContinue(stmts[-1]) pbc.visit(fdef) return fdef