def visit_Lambda(self, node): if MODULES['functools'] not in self.global_declarations.values(): import_ = ast.Import([ast.alias('functools', mangle('functools'))]) self.imports.append(import_) functools_module = MODULES['functools'] self.global_declarations[mangle('functools')] = functools_module self.generic_visit(node) forged_name = "{0}_lambda{1}".format(self.prefix, len(self.lambda_functions)) ii = self.passmanager.gather(ImportedIds, node, self.ctx) ii.difference_update(self.lambda_functions) # remove current lambdas binded_args = [ast.Name(iin, ast.Load(), None) for iin in sorted(ii)] node.args.args = ( [ast.Name(iin, ast.Param(), None) for iin in sorted(ii)] + node.args.args) forged_fdef = ast.FunctionDef(forged_name, copy(node.args), [ast.Return(node.body)], [], None) self.lambda_functions.append(forged_fdef) self.global_declarations[forged_name] = forged_fdef proxy_call = ast.Name(forged_name, ast.Load(), None) if binded_args: return ast.Call( ast.Attribute(ast.Name(mangle('functools'), ast.Load(), None), "partial", ast.Load()), [proxy_call] + binded_args, []) else: return proxy_call
def visit_FunctionDef(self, node): self.update = True if MODULES['functools'] not in self.global_declarations.values(): import_ = ast.Import([ast.alias('functools', mangle('functools'))]) self.ctx.module.body.insert(0, import_) functools_module = MODULES['functools'] self.global_declarations[mangle('functools')] = functools_module self.ctx.module.body.append(node) former_name = node.name seed = 0 new_name = "pythran_{}{}" while new_name.format(former_name, seed) in self.identifiers: seed += 1 new_name = new_name.format(former_name, seed) self.identifiers.add(new_name) ii = self.gather(ImportedIds, node) binded_args = [ ast.Name(iin, ast.Load(), None, None) for iin in sorted(ii) ] node.args.args = ( [ast.Name(iin, ast.Param(), None, None) for iin in sorted(ii)] + node.args.args) metadata.add(node, metadata.Local()) class Renamer(ast.NodeTransformer): def visit_Call(self, node): self.generic_visit(node) if (isinstance(node.func, ast.Name) and node.func.id == former_name): node.func.id = new_name node.args = ([ ast.Name(iin, ast.Load(), None, None) for iin in sorted(ii) ] + node.args) return node Renamer().visit(node) node.name = new_name self.global_declarations[node.name] = node proxy_call = ast.Name(new_name, ast.Load(), None, None) new_node = ast.Assign([ast.Name(former_name, ast.Store(), None, None)], ast.Call( ast.Attribute( ast.Name(mangle('functools'), ast.Load(), None, None), "partial", ast.Load()), [proxy_call] + binded_args, [], )) self.generic_visit(node) return new_node
def visit_FunctionDef(self, node): self.update = True if MODULES['functools'] not in self.global_declarations.values(): import_ = ast.Import([ast.alias('functools', mangle('functools'))]) self.ctx.module.body.insert(0, import_) functools_module = MODULES['functools'] self.global_declarations[mangle('functools')] = functools_module self.ctx.module.body.append(node) former_name = node.name seed = 0 new_name = "pythran_{}{}" while new_name.format(former_name, seed) in self.identifiers: seed += 1 new_name = new_name.format(former_name, seed) self.identifiers.add(new_name) ii = self.passmanager.gather(ImportedIds, node, self.ctx) binded_args = [ast.Name(iin, ast.Load(), None) for iin in sorted(ii)] node.args.args = ([ast.Name(iin, ast.Param(), None) for iin in sorted(ii)] + node.args.args) class Renamer(ast.NodeTransformer): def visit_Call(self, node): self.generic_visit(node) if (isinstance(node.func, ast.Name) and node.func.id == former_name): node.func.id = new_name node.args = ( [ast.Name(iin, ast.Load(), None) for iin in sorted(ii)] + node.args ) return node Renamer().visit(node) node.name = new_name self.global_declarations[node.name] = node proxy_call = ast.Name(new_name, ast.Load(), None) new_node = ast.Assign( [ast.Name(former_name, ast.Store(), None)], ast.Call( ast.Attribute( ast.Name(mangle('functools'), ast.Load(), None), "partial", ast.Load() ), [proxy_call] + binded_args, [], ) ) self.generic_visit(node) return new_node
def attr_to_func(self, node): mod = methods[node.attr][0] # Submodules import full module self.to_import.add(mangle(mod[0])) func = reduce(lambda v, o: ast.Attribute(v, o, ast.Load()), mod[1:] + (node.attr, ), ast.Name(mangle(mod[0]), ast.Load(), None, None)) return func
def visit_Attribute(self, node): node = self.generic_visit(node) # method name -> not a getattr if node.attr in methods: # Make sure parent is'nt a call, it's already handled in visit_Call for parent in reversed(self.ancestors.get(node, ())): if isinstance(parent, ast.Attribute): continue if isinstance(parent, ast.Call): return node break # we have a bound method which is not a call obj = self.baseobj(node) if obj is not None: self.update = True mod = methods[node.attr][0] self.to_import.add(mangle(mod[0])) func = self.attr_to_func(node) z = ast.Call( ast.Attribute( ast.Name(mangle('functools'), ast.Load(), None, None), "partial", ast.Load()), [func, obj], []) return z else: return node # imported module -> not a getattr elif (isinstance(node.value, ast.Name) and node.value.id in self.imports): module_id = self.imports[node.value.id] if node.attr not in MODULES[self.renamer(module_id, MODULES)[1]]: msg = ("`" + node.attr + "' is not a member of " + demangle(module_id) + " or Pythran does not support it") raise PythranSyntaxError(msg, node) node.value.id = module_id # patch module aliasing self.update = True return node # not listed as attributed -> not a getattr elif node.attr not in attributes: return node # A getattr ! else: self.update = True call = ast.Call( ast.Attribute(ast.Name('builtins', ast.Load(), None, None), 'getattr', ast.Load()), [node.value, ast.Constant(node.attr, None)], []) if isinstance(node.ctx, ast.Store): # the only situation where this arises is for real/imag of # a ndarray. As a call is not valid for a store, add a slice # to ends up with a valid lhs assert node.attr in ('real', 'imag'), "only store to imag/real" return ast.Subscript(call, ast.Slice(None, None, None), node.ctx) else: return call
class AbsSqrPattern(Pattern): # __builtin__.abs(X) ** 2 => __builtin__.pythran.abssqr(X) pattern = ast.Call(func=ast.Attribute(value=ast.Name(id=mangle('numpy'), ctx=ast.Load(), annotation=None, type_comment=None), attr="square", ctx=ast.Load()), args=[ ast.Call(func=ast.Attribute(value=ast.Name( id='__builtin__', ctx=ast.Load(), annotation=None, type_comment=None), attr="abs", ctx=ast.Load()), args=[Placeholder(0)], keywords=[]) ], keywords=[]) @staticmethod def sub(): return ast.Call(func=ast.Attribute(value=ast.Attribute(value=ast.Name( id='__builtin__', ctx=ast.Load(), annotation=None, type_comment=None), attr="pythran", ctx=ast.Load()), attr="abssqr", ctx=ast.Load()), args=[Placeholder(0)], keywords=[])
def prepare(self, node): assert isinstance(node, ast.Module) self.env = { 'builtins': __import__('builtins'), } for module_name in MODULES: # __dispatch__ is the only fake top-level module if module_name != '__dispatch__': alias_module_name = mangle(module_name) try: self.env[alias_module_name] = __import__(module_name) except ImportError: pass # we need to parse the whole code to be able to apply user-defined pure # function but import are resolved before so we remove them to avoid # ImportError (for operator_ for example) dummy_module = ast.Module([s for s in node.body if not isinstance(s, ast.Import)], []) eval(compile(ast.gast_to_ast(dummy_module), '<constant_folding>', 'exec'), self.env) super(ConstantFolding, self).prepare(node)
def prepare(self, node): assert isinstance(node, ast.Module) self.env = { 'builtins': __import__('builtins'), } for module_name in MODULES: # __dispatch__ is the only fake top-level module if module_name != '__dispatch__': import_name = module_name alias_module_name = mangle(module_name) self.env[alias_module_name] = __import__(import_name) # handle functions conflicting with c++ keywords for fun in MODULES[module_name]: if fun in ("__theitemgetter__", "pythran"): # these ones do not exist in Python continue # we need to parse the whole code to be able to apply user-defined pure # function but import are resolved before so we remove them to avoid # ImportError (for operator_ for example) dummy_module = ast.Module([s for s in node.body if not isinstance(s, ast.Import)], []) eval(compile(ast.gast_to_ast(dummy_module), '<constant_folding>', 'exec'), self.env) super(ConstantFolding, self).prepare(node)
def sub(): return ast.Call( func=ast.Attribute(value=ast.Name(id=mangle('numpy'), ctx=ast.Load(), annotation=None, type_comment=None), attr="cbrt", ctx=ast.Load()), args=[Placeholder(0)], keywords=[])
def visit_Module(self, node): """Add itertools import for imap, izip or ifilter iterator.""" self.generic_visit(node) import_alias = ast.alias(name='itertools', asname=mangle('itertools')) if self.use_itertools: importIt = ast.Import(names=[import_alias]) node.body.insert(0, importIt) return node
def visit_Module(self, node): self.need_import = False self.generic_visit(node) if self.need_import: import_alias = ast.alias(name='numpy', asname=mangle('numpy')) importIt = ast.Import(names=[import_alias]) node.body.insert(0, importIt) return node
def visit_Module(self, node): self.use_itertools = False self.generic_visit(node) if self.use_itertools: import_alias = ast.alias(name='itertools', asname=mangle('itertools')) importIt = ast.Import(names=[import_alias]) node.body.insert(0, importIt) return node
class AbsSqrPatternNumpy(AbsSqrPattern): # numpy.abs(X) ** 2 => __builtin__.pythran.abssqr(X) pattern = ast.Call(func=ast.Attribute(value=ast.Name(id=mangle('numpy'), ctx=ast.Load(), annotation=None, type_comment=None), attr="square", ctx=ast.Load()), args=[ast.Call(func=ast.Attribute( value=ast.Name(id=mangle('numpy'), ctx=ast.Load(), annotation=None, type_comment=None), attr="abs", ctx=ast.Load()), args=[Placeholder(0)], keywords=[])], keywords=[])
def visit_Lambda(self, node): op = issimpleoperator(node) if op is not None: if mangle('operator') not in self.global_declarations: import_ = ast.Import( [ast.alias('operator', mangle('operator'))]) self.imports.append(import_) operator_module = MODULES['operator'] self.global_declarations[mangle('operator')] = operator_module return ast.Attribute( ast.Name(mangle('operator'), ast.Load(), None, None), op, ast.Load()) self.generic_visit(node) forged_name = "{0}_lambda{1}".format(self.prefix, len(self.lambda_functions)) ii = self.gather(ImportedIds, node) ii.difference_update(self.lambda_functions) # remove current lambdas binded_args = [ ast.Name(iin, ast.Load(), None, None) for iin in sorted(ii) ] node.args.args = ( [ast.Name(iin, ast.Param(), None, None) for iin in sorted(ii)] + node.args.args) for patternname, pattern in self.patterns.items(): if issamelambda(pattern, node): proxy_call = ast.Name(patternname, ast.Load(), None, None) break else: duc = ExtendedDefUseChains() nodepattern = deepcopy(node) duc.visit(ast.Module([ast.Expr(nodepattern)], [])) self.patterns[forged_name] = nodepattern, duc forged_fdef = ast.FunctionDef(forged_name, copy(node.args), [ast.Return(node.body)], [], None, None) metadata.add(forged_fdef, metadata.Local()) self.lambda_functions.append(forged_fdef) self.global_declarations[forged_name] = forged_fdef proxy_call = ast.Name(forged_name, ast.Load(), None, None) if binded_args: if MODULES['functools'] not in self.global_declarations.values(): import_ = ast.Import( [ast.alias('functools', mangle('functools'))]) self.imports.append(import_) functools_module = MODULES['functools'] self.global_declarations[mangle( 'functools')] = functools_module return ast.Call( ast.Attribute( ast.Name(mangle('functools'), ast.Load(), None, None), "partial", ast.Load()), [proxy_call] + binded_args, []) else: return proxy_call
def visit_Lambda(self, node): if MODULES['functools'] not in self.global_declarations.values(): import_ = ast.Import([ast.alias('functools', mangle('functools'))]) self.imports.append(import_) functools_module = MODULES['functools'] self.global_declarations[mangle('functools')] = functools_module self.generic_visit(node) forged_name = "{0}_lambda{1}".format( self.prefix, len(self.lambda_functions)) ii = self.passmanager.gather(ImportedIds, node, self.ctx) ii.difference_update(self.lambda_functions) # remove current lambdas binded_args = [ast.Name(iin, ast.Load(), None) for iin in sorted(ii)] node.args.args = ([ast.Name(iin, ast.Param(), None) for iin in sorted(ii)] + node.args.args) forged_fdef = ast.FunctionDef( forged_name, copy(node.args), [ast.Return(node.body)], [], None) self.lambda_functions.append(forged_fdef) self.global_declarations[forged_name] = forged_fdef proxy_call = ast.Name(forged_name, ast.Load(), None) if binded_args: return ast.Call( ast.Attribute( ast.Name(mangle('functools'), ast.Load(), None), "partial", ast.Load() ), [proxy_call] + binded_args, []) else: return proxy_call
class PowFuncPattern(Pattern): # builtins.pow(X, Y) => X ** Y pattern = ast.Call(func=ast.Attribute( value=ast.Name(id=mangle('builtins'), ctx=ast.Load(), annotation=None, type_comment=None), attr='pow', ctx=ast.Load()), args=[Placeholder(0), Placeholder(1)], keywords=[]) @staticmethod def sub(): return ast.BinOp(Placeholder(0), ast.Pow(), Placeholder(1))
def keyword_based_disambiguification(self, node): assert isinstance(node.func, ast.Attribute) if getattr(node.func.value, 'id', None) != mangle('__dispatch__'): return if not node.keywords: return if node.func.attr not in duplicated_methods: return node_keywords = {kw.arg for kw in node.keywords} for disamb_path, disamb_node in duplicated_methods[node.func.attr]: disamb_args = {arg.id for arg in disamb_node.args.args} if all(kw in disamb_args for kw in node_keywords): node.func = self.attr_to_func(node.func, disamb_path) return
class CbrtPattern(Pattern): # X ** .33333 => numpy.cbrt(X) pattern = ast.BinOp(Placeholder(0), ast.Pow(), ast.Constant(1./3., None)) @staticmethod def sub(): return ast.Call( func=ast.Attribute(value=ast.Name(id=mangle('numpy'), ctx=ast.Load(), annotation=None, type_comment=None), attr="cbrt", ctx=ast.Load()), args=[Placeholder(0)], keywords=[]) extra_imports = [ast.Import([ast.alias('numpy', mangle('numpy'))])]
def visit_Module(self, node): """ Visit the whole module and add all import at the top level. >> import numpy.linalg Becomes >> import numpy """ node.body = [k for k in (self.visit(n) for n in node.body) if k] imports = [ast.Import([ast.alias(i, mangle(i))]) for i in self.imports] node.body = imports + node.body ast.fix_missing_locations(node) return node
def visit_ListComp(self, node): if node in self.optimizable_comprehension: self.update = True self.generic_visit(node) iterList = [] varList = [] for gen in node.generators: iterList.append(self.make_Iterator(gen)) varList.append(ast.Name(gen.target.id, ast.Param(), None)) # If dim = 1, product is useless if len(iterList) == 1: iterAST = iterList[0] varAST = ast.arguments([varList[0]], None, [], [], None, []) else: self.use_itertools = True prodName = ast.Attribute(value=ast.Name(id=mangle('itertools'), ctx=ast.Load(), annotation=None), attr='product', ctx=ast.Load()) varid = varList[0].id # retarget this id, it's free renamings = {v.id: (i, ) for i, v in enumerate(varList)} node.elt = ConvertToTuple(varid, renamings).visit(node.elt) iterAST = ast.Call(prodName, iterList, []) varAST = ast.arguments([ast.Name(varid, ast.Param(), None)], None, [], [], None, []) mapName = ast.Attribute(value=ast.Name(id='__builtin__', ctx=ast.Load(), annotation=None), attr='map', ctx=ast.Load()) ldBodymap = node.elt ldmap = ast.Lambda(varAST, ldBodymap) return ast.Call(mapName, [ldmap, iterAST], []) else: return self.generic_visit(node)
def path_to_attr(path): """ Transform path to ast.Attribute. >>> import gast as ast >>> path = ('__builtin__', 'my', 'constant') >>> value = path_to_attr(path) >>> ref = ast.Attribute( ... value=ast.Attribute(value=ast.Name(id="__builtin__", ... ctx=ast.Load(), ... annotation=None), ... attr="my", ctx=ast.Load()), ... attr="constant", ctx=ast.Load()) >>> ast.dump(ref) == ast.dump(value) True """ return reduce(lambda hpath, last: ast.Attribute(hpath, last, ast.Load()), path[1:], ast.Name(mangle(path[0]), ast.Load(), None))
def visit_ListComp(self, node): if node in self.optimizable_comprehension: self.update = True self.generic_visit(node) iterList = [] varList = [] for gen in node.generators: iterList.append(self.make_Iterator(gen)) varList.append(ast.Name(gen.target.id, ast.Param(), None)) # If dim = 1, product is useless if len(iterList) == 1: iterAST = iterList[0] varAST = ast.arguments([varList[0]], None, [], [], None, []) else: self.use_itertools = True prodName = ast.Attribute( value=ast.Name(id=mangle('itertools'), ctx=ast.Load(), annotation=None), attr='product', ctx=ast.Load()) iterAST = ast.Call(prodName, iterList, []) varAST = ast.arguments([ast.Tuple(varList, ast.Store())], None, [], [], None, []) mapName = ast.Attribute( value=ast.Name(id='__builtin__', ctx=ast.Load(), annotation=None), attr='map', ctx=ast.Load()) ldBodymap = node.elt ldmap = ast.Lambda(varAST, ldBodymap) return ast.Call(mapName, [ldmap, iterAST], []) else: return self.generic_visit(node)
def prepare(self, node, ctx): assert isinstance(node, ast.Module) self.env = { '__builtin__': __import__('__builtin__'), } for module_name in MODULES: # __dispatch__ is the only fake top-level module if module_name != '__dispatch__': import_name = module_name # handle module name conflicting with c++ keywords if (module_name.endswith("_") and module_name[:-1] in cxx_keywords): import_name = module_name[:-1] alias_module_name = mangle(module_name) self.env[alias_module_name] = __import__(import_name) # handle functions conflicting with c++ keywords for fun in MODULES[module_name]: if fun in ("__theitemgetter__", "pythran"): # these ones do not exist in Python continue # Set attributs pointing to another for C++ keyword # case of __builtin__.int_ that point on __builtin__.int if not hasattr(self.env[alias_module_name], fun): setattr( self.env[alias_module_name], fun, getattr(self.env[alias_module_name], fun.strip("_"))) # we need to parse the whole code to be able to apply user-defined pure # function but import are resolved before so we remove them to avoid # ImportError (for operator_ for example) dummy_module = ast.Module( [s for s in node.body if not isinstance(s, ast.Import)]) eval( compile(ast.gast_to_ast(dummy_module), '<constant_folding>', 'exec'), self.env) super(ConstantFolding, self).prepare(node, ctx)
def visitComp(self, node, make_attr): if node in self.optimizable_comprehension: self.update = True self.generic_visit(node) iters = [self.make_Iterator(gen) for gen in node.generators] variables = [ ast.Name(gen.target.id, ast.Param(), None, None) for gen in node.generators ] # If dim = 1, product is useless if len(iters) == 1: iterAST = iters[0] varAST = ast.arguments([variables[0]], [], None, [], [], None, []) else: self.use_itertools = True prodName = ast.Attribute(value=ast.Name(id=mangle('itertools'), ctx=ast.Load(), annotation=None, type_comment=None), attr='product', ctx=ast.Load()) varid = variables[0].id # retarget this id, it's free renamings = {v.id: (i, ) for i, v in enumerate(variables)} node.elt = ConvertToTuple(varid, renamings).visit(node.elt) iterAST = ast.Call(prodName, iters, []) varAST = ast.arguments( [ast.Name(varid, ast.Param(), None, None)], [], None, [], [], None, []) ldBodymap = node.elt ldmap = ast.Lambda(varAST, ldBodymap) return make_attr(ldmap, iterAST) else: return self.generic_visit(node)
def prepare(self, node): assert isinstance(node, ast.Module) self.env = { '__builtin__': __import__('__builtin__'), } for module_name in MODULES: # __dispatch__ is the only fake top-level module if module_name != '__dispatch__': import_name = module_name # handle module name conflicting with c++ keywords if(module_name.endswith("_") and module_name[:-1] in cxx_keywords): import_name = module_name[:-1] alias_module_name = mangle(module_name) self.env[alias_module_name] = __import__(import_name) # handle functions conflicting with c++ keywords for fun in MODULES[module_name]: if fun in ("__theitemgetter__", "pythran"): # these ones do not exist in Python continue # Set attributs pointing to another for C++ keyword # case of __builtin__.int_ that point on __builtin__.int if not hasattr(self.env[alias_module_name], fun): setattr(self.env[alias_module_name], fun, getattr(self.env[alias_module_name], fun.strip("_"))) # we need to parse the whole code to be able to apply user-defined pure # function but import are resolved before so we remove them to avoid # ImportError (for operator_ for example) dummy_module = ast.Module([s for s in node.body if not isinstance(s, ast.Import)]) eval(compile(ast.gast_to_ast(dummy_module), '<constant_folding>', 'exec'), self.env) super(ConstantFolding, self).prepare(node)
def visit_GeneratorExp(self, node): if node in self.optimizable_comprehension: self.update = True self.generic_visit(node) iters = [self.make_Iterator(gen) for gen in node.generators] variables = [ast.Name(gen.target.id, ast.Param(), None) for gen in node.generators] # If dim = 1, product is useless if len(iters) == 1: iterAST = iters[0] varAST = ast.arguments([variables[0]], None, [], [], None, []) else: prodName = ast.Attribute( value=ast.Name(id=mangle('itertools'), ctx=ast.Load(), annotation=None), attr='product', ctx=ast.Load()) iterAST = ast.Call(prodName, iters, []) varAST = ast.arguments([ast.Tuple(variables, ast.Store())], None, [], [], None, []) imapName = ast.Attribute( value=ast.Name(id=ASMODULE, ctx=ast.Load(), annotation=None), attr=IMAP, ctx=ast.Load()) ldBodyimap = node.elt ldimap = ast.Lambda(varAST, ldBodyimap) return ast.Call(imapName, [ldimap, iterAST], []) else: return self.generic_visit(node)
def visitComp(self, node, make_attr): if node in self.optimizable_comprehension: self.update = True self.generic_visit(node) iters = [self.make_Iterator(gen) for gen in node.generators] variables = [ast.Name(gen.target.id, ast.Param(), None) for gen in node.generators] # If dim = 1, product is useless if len(iters) == 1: iterAST = iters[0] varAST = ast.arguments([variables[0]], None, [], [], None, []) else: self.use_itertools = True prodName = ast.Attribute( value=ast.Name(id=mangle('itertools'), ctx=ast.Load(), annotation=None), attr='product', ctx=ast.Load()) varid = variables[0].id # retarget this id, it's free renamings = {v.id: (i,) for i, v in enumerate(variables)} node.elt = ConvertToTuple(varid, renamings).visit(node.elt) iterAST = ast.Call(prodName, iters, []) varAST = ast.arguments([ast.Name(varid, ast.Param(), None)], None, [], [], None, []) ldBodymap = node.elt ldmap = ast.Lambda(varAST, ldBodymap) return make_attr(ldmap, iterAST) else: return self.generic_visit(node)
def replace(self, value): self.update = self.need_import = True module_name = ast.Name(mangle('numpy'), ast.Load(), None, None) return ast.Call(ast.Attribute(module_name, 'square', ast.Load()), [value], [])
def visit_Call(self, node): """ Transform call site to have normal function call. Examples -------- For methods: >> a = [1, 2, 3] >> a.append(1) Becomes >> __list__.append(a, 1) For functions: >> __builtin__.dict.fromkeys([1, 2, 3]) Becomes >> __builtin__.__dict__.fromkeys([1, 2, 3]) """ node = self.generic_visit(node) # Only attributes function can be Pythonic and should be normalized if isinstance(node.func, ast.Attribute): if node.func.attr in methods: # Get object targeted by methods obj = lhs = node.func.value # Get the most left identifier to check if it is not an # imported module while isinstance(obj, ast.Attribute): obj = obj.value is_not_module = (not isinstance(obj, ast.Name) or obj.id not in self.imports) if is_not_module: self.update = True # As it was a methods call, push targeted object as first # arguments and add correct module prefix node.args.insert(0, lhs) mod = methods[node.func.attr][0] # Submodules import full module self.to_import.add(mangle(mod[0])) node.func = reduce( lambda v, o: ast.Attribute(v, o, ast.Load()), mod[1:] + (node.func.attr, ), ast.Name(mangle(mod[0]), ast.Load(), None)) # else methods have been called using function syntax if node.func.attr in methods or node.func.attr in functions: # Now, methods and function have both function syntax def rec(path, cur_module): """ Recursively rename path content looking in matching module. Prefers __module__ to module if it exists. This recursion is done as modules are visited top->bottom while attributes have to be visited bottom->top. """ err = "Function path is chained attributes and name" assert isinstance(path, (ast.Name, ast.Attribute)), err if isinstance(path, ast.Attribute): new_node, cur_module = rec(path.value, cur_module) new_id, mname = self.renamer(path.attr, cur_module) return (ast.Attribute(new_node, new_id, ast.Load()), cur_module[mname]) else: new_id, mname = self.renamer(path.id, cur_module) if mname not in cur_module: raise PythranSyntaxError( "Unbound identifier '{}'".format(mname), node) return (ast.Name(new_id, ast.Load(), None), cur_module[mname]) # Rename module path to avoid naming issue. node.func.value, _ = rec(node.func.value, MODULES) self.update = True return node
class Square(Transformation): """ Replaces **2 by a call to numpy.square. >>> import gast as ast >>> from pythran import passmanager, backend >>> node = ast.parse('a**2') >>> pm = passmanager.PassManager("test") >>> _, node = pm.apply(Square, node) >>> print(pm.dump(backend.Python, node)) import numpy as __pythran_import_numpy __pythran_import_numpy.square(a) >>> node = ast.parse('__pythran_import_numpy.power(a,2)') >>> pm = passmanager.PassManager("test") >>> _, node = pm.apply(Square, node) >>> print(pm.dump(backend.Python, node)) import numpy as __pythran_import_numpy __pythran_import_numpy.square(a) """ POW_PATTERN = ast.BinOp(AST_any(), ast.Pow(), ast.Constant(2, None)) POWER_PATTERN = ast.Call( ast.Attribute(ast.Name(mangle('numpy'), ast.Load(), None, None), 'power', ast.Load()), [AST_any(), ast.Constant(2, None)], []) def __init__(self): Transformation.__init__(self) def replace(self, value): self.update = self.need_import = True module_name = ast.Name(mangle('numpy'), ast.Load(), None, None) return ast.Call(ast.Attribute(module_name, 'square', ast.Load()), [value], []) def visit_Module(self, node): self.need_import = False self.generic_visit(node) if self.need_import: import_alias = ast.alias(name='numpy', asname=mangle('numpy')) importIt = ast.Import(names=[import_alias]) node.body.insert(0, importIt) return node def expand_pow(self, node, n): if n == 0: return ast.Constant(1, None) elif n == 1: return node else: node_square = self.replace(node) node_pow = self.expand_pow(node_square, n >> 1) if n & 1: return ast.BinOp(node_pow, ast.Mult(), copy.deepcopy(node)) else: return node_pow def visit_BinOp(self, node): self.generic_visit(node) if ASTMatcher(Square.POW_PATTERN).match(node): return self.replace(node.left) elif isinstance(node.op, ast.Pow) and isnum(node.right): n = node.right.value if int(n) == n and n > 0: return self.expand_pow(node.left, n) else: return node else: return node def visit_Call(self, node): self.generic_visit(node) if ASTMatcher(Square.POWER_PATTERN).match(node): return self.replace(node.args[0]) else: return node
from pythran.passmanager import Transformation from pythran.transformations.normalize_tuples import ConvertToTuple from pythran.conversion import mangle import gast as ast import sys if sys.version_info.major == 2: MODULE = 'itertools' IMAP = 'imap' IFILTER = 'ifilter' else: MODULE = '__builtin__' IMAP = 'map' IFILTER = 'filter' ASMODULE = mangle(MODULE) class ComprehensionPatterns(Transformation): if sys.version_info.major == 3: ''' Transforms list comprehension into intrinsics. >>> import gast as ast >>> from pythran import passmanager, backend >>> node = ast.parse("def foo(y) : return (x for x in y)") >>> pm = passmanager.PassManager("test") >>> _, node = pm.apply(ComprehensionPatterns, node) >>> print(pm.dump(backend.Python, node)) def foo(y): return __builtin__.map((lambda x: x), y) '''
def visit_Call(self, node): """ Transform call site to have normal function call. Examples -------- For methods: >> a = [1, 2, 3] >> a.append(1) Becomes >> __list__.append(a, 1) For functions: >> builtins.dict.fromkeys([1, 2, 3]) Becomes >> builtins.__dict__.fromkeys([1, 2, 3]) """ node = self.generic_visit(node) # Only attributes function can be Pythonic and should be normalized if isinstance(node.func, ast.Attribute): if node.func.attr in methods: # Check object targeted by methods if self.baseobj(node.func) is not None: self.update = True # As it was a methods call, push targeted object as first # arguments and add correct module prefix node.args.insert(0, node.func.value) mod = methods[node.func.attr][0] # Submodules import full module self.to_import.add(mangle(mod[0])) node.func = self.attr_to_func(node.func) # else methods have been called using function syntax if node.func.attr in methods or node.func.attr in functions: # Now, methods and function have both function syntax def rec(path, cur_module): """ Recursively rename path content looking in matching module. Prefers __module__ to module if it exists. This recursion is done as modules are visited top->bottom while attributes have to be visited bottom->top. """ err = "Function path is chained attributes and name" assert isinstance(path, (ast.Name, ast.Attribute)), err if isinstance(path, ast.Attribute): new_node, cur_module = rec(path.value, cur_module) new_id, mname = self.renamer(path.attr, cur_module) return (ast.Attribute(new_node, new_id, ast.Load()), cur_module[mname]) else: new_id, mname = self.renamer(path.id, cur_module) return (ast.Name(new_id, ast.Load(), None, None), cur_module[mname]) # Rename module path to avoid naming issue. node.func.value, _ = rec(node.func.value, MODULES) self.update = True self.keyword_based_disambiguification(node) return node
from pythran.passmanager import Transformation from pythran.transformations import NormalizeTuples from pythran.conversion import mangle import gast as ast import sys if sys.version_info.major == 2: MODULE = 'itertools' IMAP = 'imap' IFILTER = 'ifilter' else: MODULE = '__builtin__' IMAP = 'map' IFILTER = 'filter' ASMODULE = mangle(MODULE) class GenExpToImap(Transformation): ''' Transforms generator expressions into iterators. >>> import gast as ast >>> from pythran import passmanager, backend >>> node = ast.parse("(x*x for x in range(10))") >>> pm = passmanager.PassManager("test") >>> _, node = pm.apply(GenExpToImap, node) >>> print pm.dump(backend.Python, node) import itertools as __pythran_import_itertools __pythran_import_itertools.imap((lambda x: (x * x)), range(10)) '''
ctx=ast.Load()), args=[Placeholder(0)], keywords=[]) ], keywords=[]), lambda: ast.Call(func=ast.Attribute(value=ast.Attribute(value=ast.Name( id='__builtin__', ctx=ast.Load(), annotation=None), attr="pythran", ctx=ast.Load()), attr="len_set", ctx=ast.Load()), args=[Placeholder(0)], keywords=[])), # __builtin__.abs(X ** 2) => __builtin__.pythran.abssqr(X) (ast.Call(func=ast.Attribute(value=ast.Name(id=mangle('numpy'), ctx=ast.Load(), annotation=None), attr="square", ctx=ast.Load()), args=[ ast.Call(func=ast.Attribute(value=ast.Name(id='__builtin__', ctx=ast.Load(), annotation=None), attr="abs", ctx=ast.Load()), args=[Placeholder(0)], keywords=[]) ], keywords=[]), lambda: ast.Call(func=ast.Attribute(value=ast.Attribute(value=ast.Name(
def visit_Call(self, node): """ Transform call site to have normal function call. Examples -------- For methods: >> a = [1, 2, 3] >> a.append(1) Becomes >> __list__.append(a, 1) For functions: >> __builtin__.dict.fromkeys([1, 2, 3]) Becomes >> __builtin__.__dict__.fromkeys([1, 2, 3]) """ node = self.generic_visit(node) # Only attributes function can be Pythonic and should be normalized if isinstance(node.func, ast.Attribute): if node.func.attr in methods: # Get object targeted by methods obj = lhs = node.func.value # Get the most left identifier to check if it is not an # imported module while isinstance(obj, ast.Attribute): obj = obj.value is_not_module = (not isinstance(obj, ast.Name) or obj.id not in self.imports) if is_not_module: self.update = True # As it was a methods call, push targeted object as first # arguments and add correct module prefix node.args.insert(0, lhs) mod = methods[node.func.attr][0] # Submodules import full module self.to_import.add(mangle(mod[0])) node.func = reduce( lambda v, o: ast.Attribute(v, o, ast.Load()), mod[1:] + (node.func.attr,), ast.Name(mangle(mod[0]), ast.Load(), None) ) # else methods have been called using function syntax if node.func.attr in methods or node.func.attr in functions: # Now, methods and function have both function syntax def rec(path, cur_module): """ Recursively rename path content looking in matching module. Prefers __module__ to module if it exists. This recursion is done as modules are visited top->bottom while attributes have to be visited bottom->top. """ err = "Function path is chained attributes and name" assert isinstance(path, (ast.Name, ast.Attribute)), err if isinstance(path, ast.Attribute): new_node, cur_module = rec(path.value, cur_module) new_id, mname = self.renamer(path.attr, cur_module) return (ast.Attribute(new_node, new_id, ast.Load()), cur_module[mname]) else: new_id, mname = self.renamer(path.id, cur_module) if mname not in cur_module: raise PythranSyntaxError( "Unbound identifier '{}'".format(mname), node) return (ast.Name(new_id, ast.Load(), None), cur_module[mname]) # Rename module path to avoid naming issue. node.func.value, _ = rec(node.func.value, MODULES) self.update = True return node
def replace(self, value): self.update = self.need_import = True module_name = ast.Name(mangle('numpy'), ast.Load(), None) return ast.Call(ast.Attribute(module_name, 'square', ast.Load()), [value], [])
def visit_Module(self, node): """Add itertools import for imap, izip or ifilter iterator.""" self.generic_visit(node) import_alias = ast.alias(name='itertools', asname=mangle('itertools')) importIt = ast.Import(names=[import_alias]) return ast.Module(body=([importIt] + node.body))