def visit_Lambda(self, node):
if MODULES['functools'] not in self.global_declarations.values():
import_ = ast.Import([ast.alias('functools', mangle('functools'))])
self.imports.append(import_)
functools_module = MODULES['functools']
self.global_declarations[mangle('functools')] = functools_module
self.generic_visit(node)
forged_name = "{0}_lambda{1}".format(self.prefix,
len(self.lambda_functions))
ii = self.passmanager.gather(ImportedIds, node, self.ctx)
ii.difference_update(self.lambda_functions) # remove current lambdas
binded_args = [ast.Name(iin, ast.Load(), None) for iin in sorted(ii)]
node.args.args = (
[ast.Name(iin, ast.Param(), None)
for iin in sorted(ii)] + node.args.args)
forged_fdef = ast.FunctionDef(forged_name, copy(node.args),
[ast.Return(node.body)], [], None)
self.lambda_functions.append(forged_fdef)
self.global_declarations[forged_name] = forged_fdef
proxy_call = ast.Name(forged_name, ast.Load(), None)
if binded_args:
return ast.Call(
ast.Attribute(ast.Name(mangle('functools'), ast.Load(), None),
"partial", ast.Load()),
[proxy_call] + binded_args, [])
else:
return proxy_call
def visit_FunctionDef(self, node):
self.update = True
if MODULES['functools'] not in self.global_declarations.values():
import_ = ast.Import([ast.alias('functools', mangle('functools'))])
self.ctx.module.body.insert(0, import_)
functools_module = MODULES['functools']
self.global_declarations[mangle('functools')] = functools_module
self.ctx.module.body.append(node)
former_name = node.name
seed = 0
new_name = "pythran_{}{}"
while new_name.format(former_name, seed) in self.identifiers:
seed += 1
new_name = new_name.format(former_name, seed)
self.identifiers.add(new_name)
ii = self.gather(ImportedIds, node)
binded_args = [
ast.Name(iin, ast.Load(), None, None) for iin in sorted(ii)
]
node.args.args = (
[ast.Name(iin, ast.Param(), None, None)
for iin in sorted(ii)] + node.args.args)
metadata.add(node, metadata.Local())
class Renamer(ast.NodeTransformer):
def visit_Call(self, node):
self.generic_visit(node)
if (isinstance(node.func, ast.Name)
and node.func.id == former_name):
node.func.id = new_name
node.args = ([
ast.Name(iin, ast.Load(), None, None)
for iin in sorted(ii)
] + node.args)
return node
Renamer().visit(node)
node.name = new_name
self.global_declarations[node.name] = node
proxy_call = ast.Name(new_name, ast.Load(), None, None)
new_node = ast.Assign([ast.Name(former_name, ast.Store(), None, None)],
ast.Call(
ast.Attribute(
ast.Name(mangle('functools'), ast.Load(),
None, None), "partial",
ast.Load()),
[proxy_call] + binded_args,
[],
))
self.generic_visit(node)
return new_node
def visit_FunctionDef(self, node):
self.update = True
if MODULES['functools'] not in self.global_declarations.values():
import_ = ast.Import([ast.alias('functools', mangle('functools'))])
self.ctx.module.body.insert(0, import_)
functools_module = MODULES['functools']
self.global_declarations[mangle('functools')] = functools_module
self.ctx.module.body.append(node)
former_name = node.name
seed = 0
new_name = "pythran_{}{}"
while new_name.format(former_name, seed) in self.identifiers:
seed += 1
new_name = new_name.format(former_name, seed)
self.identifiers.add(new_name)
ii = self.passmanager.gather(ImportedIds, node, self.ctx)
binded_args = [ast.Name(iin, ast.Load(), None) for iin in sorted(ii)]
node.args.args = ([ast.Name(iin, ast.Param(), None)
for iin in sorted(ii)] +
node.args.args)
class Renamer(ast.NodeTransformer):
def visit_Call(self, node):
self.generic_visit(node)
if (isinstance(node.func, ast.Name) and
node.func.id == former_name):
node.func.id = new_name
node.args = (
[ast.Name(iin, ast.Load(), None)
for iin in sorted(ii)] +
node.args
)
return node
Renamer().visit(node)
node.name = new_name
self.global_declarations[node.name] = node
proxy_call = ast.Name(new_name, ast.Load(), None)
new_node = ast.Assign(
[ast.Name(former_name, ast.Store(), None)],
ast.Call(
ast.Attribute(
ast.Name(mangle('functools'), ast.Load(), None),
"partial",
ast.Load()
),
[proxy_call] + binded_args,
[],
)
)
self.generic_visit(node)
return new_node
def attr_to_func(self, node):
mod = methods[node.attr][0]
# Submodules import full module
self.to_import.add(mangle(mod[0]))
func = reduce(lambda v, o: ast.Attribute(v, o, ast.Load()),
mod[1:] + (node.attr, ),
ast.Name(mangle(mod[0]), ast.Load(), None, None))
return func
def visit_Attribute(self, node):
node = self.generic_visit(node)
# method name -> not a getattr
if node.attr in methods:
# Make sure parent is'nt a call, it's already handled in visit_Call
for parent in reversed(self.ancestors.get(node, ())):
if isinstance(parent, ast.Attribute):
continue
if isinstance(parent, ast.Call):
return node
break
# we have a bound method which is not a call
obj = self.baseobj(node)
if obj is not None:
self.update = True
mod = methods[node.attr][0]
self.to_import.add(mangle(mod[0]))
func = self.attr_to_func(node)
z = ast.Call(
ast.Attribute(
ast.Name(mangle('functools'), ast.Load(), None, None),
"partial", ast.Load()), [func, obj], [])
return z
else:
return node
# imported module -> not a getattr
elif (isinstance(node.value, ast.Name)
and node.value.id in self.imports):
module_id = self.imports[node.value.id]
if node.attr not in MODULES[self.renamer(module_id, MODULES)[1]]:
msg = ("`" + node.attr + "' is not a member of " +
demangle(module_id) + " or Pythran does not support it")
raise PythranSyntaxError(msg, node)
node.value.id = module_id # patch module aliasing
self.update = True
return node
# not listed as attributed -> not a getattr
elif node.attr not in attributes:
return node
# A getattr !
else:
self.update = True
call = ast.Call(
ast.Attribute(ast.Name('builtins', ast.Load(), None, None),
'getattr', ast.Load()),
[node.value, ast.Constant(node.attr, None)], [])
if isinstance(node.ctx, ast.Store):
# the only situation where this arises is for real/imag of
# a ndarray. As a call is not valid for a store, add a slice
# to ends up with a valid lhs
assert node.attr in ('real', 'imag'), "only store to imag/real"
return ast.Subscript(call, ast.Slice(None, None, None),
node.ctx)
else:
return call
class AbsSqrPattern(Pattern):
# __builtin__.abs(X) ** 2 => __builtin__.pythran.abssqr(X)
pattern = ast.Call(func=ast.Attribute(value=ast.Name(id=mangle('numpy'),
ctx=ast.Load(),
annotation=None,
type_comment=None),
attr="square",
ctx=ast.Load()),
args=[
ast.Call(func=ast.Attribute(value=ast.Name(
id='__builtin__',
ctx=ast.Load(),
annotation=None,
type_comment=None),
attr="abs",
ctx=ast.Load()),
args=[Placeholder(0)],
keywords=[])
],
keywords=[])
@staticmethod
def sub():
return ast.Call(func=ast.Attribute(value=ast.Attribute(value=ast.Name(
id='__builtin__',
ctx=ast.Load(),
annotation=None,
type_comment=None),
attr="pythran",
ctx=ast.Load()),
attr="abssqr",
ctx=ast.Load()),
args=[Placeholder(0)],
keywords=[])
def prepare(self, node):
assert isinstance(node, ast.Module)
self.env = {
'builtins': __import__('builtins'),
}
for module_name in MODULES:
# __dispatch__ is the only fake top-level module
if module_name != '__dispatch__':
alias_module_name = mangle(module_name)
try:
self.env[alias_module_name] = __import__(module_name)
except ImportError:
pass
# we need to parse the whole code to be able to apply user-defined pure
# function but import are resolved before so we remove them to avoid
# ImportError (for operator_ for example)
dummy_module = ast.Module([s for s in node.body
if not isinstance(s, ast.Import)],
[])
eval(compile(ast.gast_to_ast(dummy_module),
'<constant_folding>', 'exec'),
self.env)
super(ConstantFolding, self).prepare(node)
def prepare(self, node):
assert isinstance(node, ast.Module)
self.env = {
'builtins': __import__('builtins'),
}
for module_name in MODULES:
# __dispatch__ is the only fake top-level module
if module_name != '__dispatch__':
import_name = module_name
alias_module_name = mangle(module_name)
self.env[alias_module_name] = __import__(import_name)
# handle functions conflicting with c++ keywords
for fun in MODULES[module_name]:
if fun in ("__theitemgetter__", "pythran"):
# these ones do not exist in Python
continue
# we need to parse the whole code to be able to apply user-defined pure
# function but import are resolved before so we remove them to avoid
# ImportError (for operator_ for example)
dummy_module = ast.Module([s for s in node.body
if not isinstance(s, ast.Import)],
[])
eval(compile(ast.gast_to_ast(dummy_module),
'<constant_folding>', 'exec'),
self.env)
super(ConstantFolding, self).prepare(node)
def sub():
return ast.Call(
func=ast.Attribute(value=ast.Name(id=mangle('numpy'),
ctx=ast.Load(),
annotation=None,
type_comment=None),
attr="cbrt", ctx=ast.Load()),
args=[Placeholder(0)], keywords=[])
def visit_Module(self, node):
"""Add itertools import for imap, izip or ifilter iterator."""
self.generic_visit(node)
import_alias = ast.alias(name='itertools', asname=mangle('itertools'))
if self.use_itertools:
importIt = ast.Import(names=[import_alias])
node.body.insert(0, importIt)
return node
def visit_Module(self, node):
self.need_import = False
self.generic_visit(node)
if self.need_import:
import_alias = ast.alias(name='numpy', asname=mangle('numpy'))
importIt = ast.Import(names=[import_alias])
node.body.insert(0, importIt)
return node
def visit_Module(self, node):
self.need_import = False
self.generic_visit(node)
if self.need_import:
import_alias = ast.alias(name='numpy', asname=mangle('numpy'))
importIt = ast.Import(names=[import_alias])
node.body.insert(0, importIt)
return node
def visit_Module(self, node):
self.use_itertools = False
self.generic_visit(node)
if self.use_itertools:
import_alias = ast.alias(name='itertools',
asname=mangle('itertools'))
importIt = ast.Import(names=[import_alias])
node.body.insert(0, importIt)
return node
def visit_Module(self, node):
self.use_itertools = False
self.generic_visit(node)
if self.use_itertools:
import_alias = ast.alias(name='itertools',
asname=mangle('itertools'))
importIt = ast.Import(names=[import_alias])
node.body.insert(0, importIt)
return node
class AbsSqrPatternNumpy(AbsSqrPattern):
# numpy.abs(X) ** 2 => __builtin__.pythran.abssqr(X)
pattern = ast.Call(func=ast.Attribute(value=ast.Name(id=mangle('numpy'),
ctx=ast.Load(),
annotation=None,
type_comment=None),
attr="square", ctx=ast.Load()),
args=[ast.Call(func=ast.Attribute(
value=ast.Name(id=mangle('numpy'),
ctx=ast.Load(),
annotation=None,
type_comment=None),
attr="abs",
ctx=ast.Load()),
args=[Placeholder(0)],
keywords=[])],
keywords=[])
def visit_Lambda(self, node):
op = issimpleoperator(node)
if op is not None:
if mangle('operator') not in self.global_declarations:
import_ = ast.Import(
[ast.alias('operator', mangle('operator'))])
self.imports.append(import_)
operator_module = MODULES['operator']
self.global_declarations[mangle('operator')] = operator_module
return ast.Attribute(
ast.Name(mangle('operator'), ast.Load(), None, None), op,
ast.Load())
self.generic_visit(node)
forged_name = "{0}_lambda{1}".format(self.prefix,
len(self.lambda_functions))
ii = self.gather(ImportedIds, node)
ii.difference_update(self.lambda_functions) # remove current lambdas
binded_args = [
ast.Name(iin, ast.Load(), None, None) for iin in sorted(ii)
]
node.args.args = (
[ast.Name(iin, ast.Param(), None, None)
for iin in sorted(ii)] + node.args.args)
for patternname, pattern in self.patterns.items():
if issamelambda(pattern, node):
proxy_call = ast.Name(patternname, ast.Load(), None, None)
break
else:
duc = ExtendedDefUseChains()
nodepattern = deepcopy(node)
duc.visit(ast.Module([ast.Expr(nodepattern)], []))
self.patterns[forged_name] = nodepattern, duc
forged_fdef = ast.FunctionDef(forged_name, copy(node.args),
[ast.Return(node.body)], [], None,
None)
metadata.add(forged_fdef, metadata.Local())
self.lambda_functions.append(forged_fdef)
self.global_declarations[forged_name] = forged_fdef
proxy_call = ast.Name(forged_name, ast.Load(), None, None)
if binded_args:
if MODULES['functools'] not in self.global_declarations.values():
import_ = ast.Import(
[ast.alias('functools', mangle('functools'))])
self.imports.append(import_)
functools_module = MODULES['functools']
self.global_declarations[mangle(
'functools')] = functools_module
return ast.Call(
ast.Attribute(
ast.Name(mangle('functools'), ast.Load(), None, None),
"partial", ast.Load()), [proxy_call] + binded_args, [])
else:
return proxy_call
def visit_Lambda(self, node):
if MODULES['functools'] not in self.global_declarations.values():
import_ = ast.Import([ast.alias('functools', mangle('functools'))])
self.imports.append(import_)
functools_module = MODULES['functools']
self.global_declarations[mangle('functools')] = functools_module
self.generic_visit(node)
forged_name = "{0}_lambda{1}".format(
self.prefix,
len(self.lambda_functions))
ii = self.passmanager.gather(ImportedIds, node, self.ctx)
ii.difference_update(self.lambda_functions) # remove current lambdas
binded_args = [ast.Name(iin, ast.Load(), None) for iin in sorted(ii)]
node.args.args = ([ast.Name(iin, ast.Param(), None)
for iin in sorted(ii)] +
node.args.args)
forged_fdef = ast.FunctionDef(
forged_name,
copy(node.args),
[ast.Return(node.body)],
[], None)
self.lambda_functions.append(forged_fdef)
self.global_declarations[forged_name] = forged_fdef
proxy_call = ast.Name(forged_name, ast.Load(), None)
if binded_args:
return ast.Call(
ast.Attribute(
ast.Name(mangle('functools'), ast.Load(), None),
"partial",
ast.Load()
),
[proxy_call] + binded_args,
[])
else:
return proxy_call
class PowFuncPattern(Pattern):
# builtins.pow(X, Y) => X ** Y
pattern = ast.Call(func=ast.Attribute(
value=ast.Name(id=mangle('builtins'), ctx=ast.Load(),
annotation=None,
type_comment=None),
attr='pow', ctx=ast.Load()),
args=[Placeholder(0), Placeholder(1)],
keywords=[])
@staticmethod
def sub():
return ast.BinOp(Placeholder(0), ast.Pow(), Placeholder(1))
def keyword_based_disambiguification(self, node):
assert isinstance(node.func, ast.Attribute)
if getattr(node.func.value, 'id', None) != mangle('__dispatch__'):
return
if not node.keywords:
return
if node.func.attr not in duplicated_methods:
return
node_keywords = {kw.arg for kw in node.keywords}
for disamb_path, disamb_node in duplicated_methods[node.func.attr]:
disamb_args = {arg.id for arg in disamb_node.args.args}
if all(kw in disamb_args for kw in node_keywords):
node.func = self.attr_to_func(node.func, disamb_path)
return
class CbrtPattern(Pattern):
# X ** .33333 => numpy.cbrt(X)
pattern = ast.BinOp(Placeholder(0), ast.Pow(), ast.Constant(1./3., None))
@staticmethod
def sub():
return ast.Call(
func=ast.Attribute(value=ast.Name(id=mangle('numpy'),
ctx=ast.Load(),
annotation=None,
type_comment=None),
attr="cbrt", ctx=ast.Load()),
args=[Placeholder(0)], keywords=[])
extra_imports = [ast.Import([ast.alias('numpy', mangle('numpy'))])]
def visit_Module(self, node):
"""
Visit the whole module and add all import at the top level.
>> import numpy.linalg
Becomes
>> import numpy
"""
node.body = [k for k in (self.visit(n) for n in node.body) if k]
imports = [ast.Import([ast.alias(i, mangle(i))]) for i in self.imports]
node.body = imports + node.body
ast.fix_missing_locations(node)
return node
def visit_Module(self, node):
"""
Visit the whole module and add all import at the top level.
>> import numpy.linalg
Becomes
>> import numpy
"""
node.body = [k for k in (self.visit(n) for n in node.body) if k]
imports = [ast.Import([ast.alias(i, mangle(i))]) for i in self.imports]
node.body = imports + node.body
ast.fix_missing_locations(node)
return node
def visit_ListComp(self, node):
if node in self.optimizable_comprehension:
self.update = True
self.generic_visit(node)
iterList = []
varList = []
for gen in node.generators:
iterList.append(self.make_Iterator(gen))
varList.append(ast.Name(gen.target.id, ast.Param(), None))
# If dim = 1, product is useless
if len(iterList) == 1:
iterAST = iterList[0]
varAST = ast.arguments([varList[0]], None, [], [], None, [])
else:
self.use_itertools = True
prodName = ast.Attribute(value=ast.Name(id=mangle('itertools'),
ctx=ast.Load(),
annotation=None),
attr='product',
ctx=ast.Load())
varid = varList[0].id # retarget this id, it's free
renamings = {v.id: (i, ) for i, v in enumerate(varList)}
node.elt = ConvertToTuple(varid, renamings).visit(node.elt)
iterAST = ast.Call(prodName, iterList, [])
varAST = ast.arguments([ast.Name(varid, ast.Param(), None)],
None, [], [], None, [])
mapName = ast.Attribute(value=ast.Name(id='__builtin__',
ctx=ast.Load(),
annotation=None),
attr='map',
ctx=ast.Load())
ldBodymap = node.elt
ldmap = ast.Lambda(varAST, ldBodymap)
return ast.Call(mapName, [ldmap, iterAST], [])
else:
return self.generic_visit(node)
def path_to_attr(path):
"""
Transform path to ast.Attribute.
>>> import gast as ast
>>> path = ('__builtin__', 'my', 'constant')
>>> value = path_to_attr(path)
>>> ref = ast.Attribute(
... value=ast.Attribute(value=ast.Name(id="__builtin__",
... ctx=ast.Load(),
... annotation=None),
... attr="my", ctx=ast.Load()),
... attr="constant", ctx=ast.Load())
>>> ast.dump(ref) == ast.dump(value)
True
"""
return reduce(lambda hpath, last: ast.Attribute(hpath, last, ast.Load()),
path[1:], ast.Name(mangle(path[0]), ast.Load(), None))
def path_to_attr(path):
"""
Transform path to ast.Attribute.
>>> import gast as ast
>>> path = ('__builtin__', 'my', 'constant')
>>> value = path_to_attr(path)
>>> ref = ast.Attribute(
... value=ast.Attribute(value=ast.Name(id="__builtin__",
... ctx=ast.Load(),
... annotation=None),
... attr="my", ctx=ast.Load()),
... attr="constant", ctx=ast.Load())
>>> ast.dump(ref) == ast.dump(value)
True
"""
return reduce(lambda hpath, last: ast.Attribute(hpath, last, ast.Load()),
path[1:], ast.Name(mangle(path[0]), ast.Load(), None))
def visit_ListComp(self, node):
if node in self.optimizable_comprehension:
self.update = True
self.generic_visit(node)
iterList = []
varList = []
for gen in node.generators:
iterList.append(self.make_Iterator(gen))
varList.append(ast.Name(gen.target.id, ast.Param(), None))
# If dim = 1, product is useless
if len(iterList) == 1:
iterAST = iterList[0]
varAST = ast.arguments([varList[0]], None, [], [], None, [])
else:
self.use_itertools = True
prodName = ast.Attribute(
value=ast.Name(id=mangle('itertools'),
ctx=ast.Load(),
annotation=None),
attr='product', ctx=ast.Load())
iterAST = ast.Call(prodName, iterList, [])
varAST = ast.arguments([ast.Tuple(varList, ast.Store())],
None, [], [], None, [])
mapName = ast.Attribute(
value=ast.Name(id='__builtin__',
ctx=ast.Load(),
annotation=None),
attr='map', ctx=ast.Load())
ldBodymap = node.elt
ldmap = ast.Lambda(varAST, ldBodymap)
return ast.Call(mapName, [ldmap, iterAST], [])
else:
return self.generic_visit(node)
def prepare(self, node, ctx):
assert isinstance(node, ast.Module)
self.env = {
'__builtin__': __import__('__builtin__'),
}
for module_name in MODULES:
# __dispatch__ is the only fake top-level module
if module_name != '__dispatch__':
import_name = module_name
# handle module name conflicting with c++ keywords
if (module_name.endswith("_")
and module_name[:-1] in cxx_keywords):
import_name = module_name[:-1]
alias_module_name = mangle(module_name)
self.env[alias_module_name] = __import__(import_name)
# handle functions conflicting with c++ keywords
for fun in MODULES[module_name]:
if fun in ("__theitemgetter__", "pythran"):
# these ones do not exist in Python
continue
# Set attributs pointing to another for C++ keyword
# case of __builtin__.int_ that point on __builtin__.int
if not hasattr(self.env[alias_module_name], fun):
setattr(
self.env[alias_module_name], fun,
getattr(self.env[alias_module_name],
fun.strip("_")))
# we need to parse the whole code to be able to apply user-defined pure
# function but import are resolved before so we remove them to avoid
# ImportError (for operator_ for example)
dummy_module = ast.Module(
[s for s in node.body if not isinstance(s, ast.Import)])
eval(
compile(ast.gast_to_ast(dummy_module), '<constant_folding>',
'exec'), self.env)
super(ConstantFolding, self).prepare(node, ctx)
def visitComp(self, node, make_attr):
if node in self.optimizable_comprehension:
self.update = True
self.generic_visit(node)
iters = [self.make_Iterator(gen) for gen in node.generators]
variables = [
ast.Name(gen.target.id, ast.Param(), None, None)
for gen in node.generators
]
# If dim = 1, product is useless
if len(iters) == 1:
iterAST = iters[0]
varAST = ast.arguments([variables[0]], [], None, [], [], None,
[])
else:
self.use_itertools = True
prodName = ast.Attribute(value=ast.Name(id=mangle('itertools'),
ctx=ast.Load(),
annotation=None,
type_comment=None),
attr='product',
ctx=ast.Load())
varid = variables[0].id # retarget this id, it's free
renamings = {v.id: (i, ) for i, v in enumerate(variables)}
node.elt = ConvertToTuple(varid, renamings).visit(node.elt)
iterAST = ast.Call(prodName, iters, [])
varAST = ast.arguments(
[ast.Name(varid, ast.Param(), None, None)], [], None, [],
[], None, [])
ldBodymap = node.elt
ldmap = ast.Lambda(varAST, ldBodymap)
return make_attr(ldmap, iterAST)
else:
return self.generic_visit(node)
def prepare(self, node):
assert isinstance(node, ast.Module)
self.env = {
'__builtin__': __import__('__builtin__'),
}
for module_name in MODULES:
# __dispatch__ is the only fake top-level module
if module_name != '__dispatch__':
import_name = module_name
# handle module name conflicting with c++ keywords
if(module_name.endswith("_") and
module_name[:-1] in cxx_keywords):
import_name = module_name[:-1]
alias_module_name = mangle(module_name)
self.env[alias_module_name] = __import__(import_name)
# handle functions conflicting with c++ keywords
for fun in MODULES[module_name]:
if fun in ("__theitemgetter__", "pythran"):
# these ones do not exist in Python
continue
# Set attributs pointing to another for C++ keyword
# case of __builtin__.int_ that point on __builtin__.int
if not hasattr(self.env[alias_module_name], fun):
setattr(self.env[alias_module_name], fun,
getattr(self.env[alias_module_name],
fun.strip("_")))
# we need to parse the whole code to be able to apply user-defined pure
# function but import are resolved before so we remove them to avoid
# ImportError (for operator_ for example)
dummy_module = ast.Module([s for s in node.body
if not isinstance(s, ast.Import)])
eval(compile(ast.gast_to_ast(dummy_module),
'<constant_folding>', 'exec'),
self.env)
super(ConstantFolding, self).prepare(node)
def visit_GeneratorExp(self, node):
if node in self.optimizable_comprehension:
self.update = True
self.generic_visit(node)
iters = [self.make_Iterator(gen) for gen in node.generators]
variables = [ast.Name(gen.target.id, ast.Param(), None)
for gen in node.generators]
# If dim = 1, product is useless
if len(iters) == 1:
iterAST = iters[0]
varAST = ast.arguments([variables[0]], None, [], [], None, [])
else:
prodName = ast.Attribute(
value=ast.Name(id=mangle('itertools'),
ctx=ast.Load(),
annotation=None),
attr='product', ctx=ast.Load())
iterAST = ast.Call(prodName, iters, [])
varAST = ast.arguments([ast.Tuple(variables, ast.Store())],
None, [], [], None, [])
imapName = ast.Attribute(
value=ast.Name(id=ASMODULE,
ctx=ast.Load(),
annotation=None),
attr=IMAP, ctx=ast.Load())
ldBodyimap = node.elt
ldimap = ast.Lambda(varAST, ldBodyimap)
return ast.Call(imapName, [ldimap, iterAST], [])
else:
return self.generic_visit(node)
def visitComp(self, node, make_attr):
if node in self.optimizable_comprehension:
self.update = True
self.generic_visit(node)
iters = [self.make_Iterator(gen) for gen in node.generators]
variables = [ast.Name(gen.target.id, ast.Param(), None)
for gen in node.generators]
# If dim = 1, product is useless
if len(iters) == 1:
iterAST = iters[0]
varAST = ast.arguments([variables[0]], None, [], [], None, [])
else:
self.use_itertools = True
prodName = ast.Attribute(
value=ast.Name(id=mangle('itertools'),
ctx=ast.Load(),
annotation=None),
attr='product', ctx=ast.Load())
varid = variables[0].id # retarget this id, it's free
renamings = {v.id: (i,) for i, v in enumerate(variables)}
node.elt = ConvertToTuple(varid, renamings).visit(node.elt)
iterAST = ast.Call(prodName, iters, [])
varAST = ast.arguments([ast.Name(varid, ast.Param(), None)],
None, [], [], None, [])
ldBodymap = node.elt
ldmap = ast.Lambda(varAST, ldBodymap)
return make_attr(ldmap, iterAST)
else:
return self.generic_visit(node)
def replace(self, value):
self.update = self.need_import = True
module_name = ast.Name(mangle('numpy'), ast.Load(), None, None)
return ast.Call(ast.Attribute(module_name, 'square', ast.Load()),
[value], [])
def visit_Call(self, node):
"""
Transform call site to have normal function call.
Examples
--------
For methods:
>> a = [1, 2, 3]
>> a.append(1)
Becomes
>> __list__.append(a, 1)
For functions:
>> __builtin__.dict.fromkeys([1, 2, 3])
Becomes
>> __builtin__.__dict__.fromkeys([1, 2, 3])
"""
node = self.generic_visit(node)
# Only attributes function can be Pythonic and should be normalized
if isinstance(node.func, ast.Attribute):
if node.func.attr in methods:
# Get object targeted by methods
obj = lhs = node.func.value
# Get the most left identifier to check if it is not an
# imported module
while isinstance(obj, ast.Attribute):
obj = obj.value
is_not_module = (not isinstance(obj, ast.Name)
or obj.id not in self.imports)
if is_not_module:
self.update = True
# As it was a methods call, push targeted object as first
# arguments and add correct module prefix
node.args.insert(0, lhs)
mod = methods[node.func.attr][0]
# Submodules import full module
self.to_import.add(mangle(mod[0]))
node.func = reduce(
lambda v, o: ast.Attribute(v, o, ast.Load()),
mod[1:] + (node.func.attr, ),
ast.Name(mangle(mod[0]), ast.Load(), None))
# else methods have been called using function syntax
if node.func.attr in methods or node.func.attr in functions:
# Now, methods and function have both function syntax
def rec(path, cur_module):
"""
Recursively rename path content looking in matching module.
Prefers __module__ to module if it exists.
This recursion is done as modules are visited top->bottom
while attributes have to be visited bottom->top.
"""
err = "Function path is chained attributes and name"
assert isinstance(path, (ast.Name, ast.Attribute)), err
if isinstance(path, ast.Attribute):
new_node, cur_module = rec(path.value, cur_module)
new_id, mname = self.renamer(path.attr, cur_module)
return (ast.Attribute(new_node, new_id,
ast.Load()), cur_module[mname])
else:
new_id, mname = self.renamer(path.id, cur_module)
if mname not in cur_module:
raise PythranSyntaxError(
"Unbound identifier '{}'".format(mname), node)
return (ast.Name(new_id, ast.Load(),
None), cur_module[mname])
# Rename module path to avoid naming issue.
node.func.value, _ = rec(node.func.value, MODULES)
self.update = True
return node
class Square(Transformation):
"""
Replaces **2 by a call to numpy.square.
>>> import gast as ast
>>> from pythran import passmanager, backend
>>> node = ast.parse('a**2')
>>> pm = passmanager.PassManager("test")
>>> _, node = pm.apply(Square, node)
>>> print(pm.dump(backend.Python, node))
import numpy as __pythran_import_numpy
__pythran_import_numpy.square(a)
>>> node = ast.parse('__pythran_import_numpy.power(a,2)')
>>> pm = passmanager.PassManager("test")
>>> _, node = pm.apply(Square, node)
>>> print(pm.dump(backend.Python, node))
import numpy as __pythran_import_numpy
__pythran_import_numpy.square(a)
"""
POW_PATTERN = ast.BinOp(AST_any(), ast.Pow(), ast.Constant(2, None))
POWER_PATTERN = ast.Call(
ast.Attribute(ast.Name(mangle('numpy'), ast.Load(), None, None),
'power',
ast.Load()),
[AST_any(), ast.Constant(2, None)],
[])
def __init__(self):
Transformation.__init__(self)
def replace(self, value):
self.update = self.need_import = True
module_name = ast.Name(mangle('numpy'), ast.Load(), None, None)
return ast.Call(ast.Attribute(module_name, 'square', ast.Load()),
[value], [])
def visit_Module(self, node):
self.need_import = False
self.generic_visit(node)
if self.need_import:
import_alias = ast.alias(name='numpy', asname=mangle('numpy'))
importIt = ast.Import(names=[import_alias])
node.body.insert(0, importIt)
return node
def expand_pow(self, node, n):
if n == 0:
return ast.Constant(1, None)
elif n == 1:
return node
else:
node_square = self.replace(node)
node_pow = self.expand_pow(node_square, n >> 1)
if n & 1:
return ast.BinOp(node_pow, ast.Mult(), copy.deepcopy(node))
else:
return node_pow
def visit_BinOp(self, node):
self.generic_visit(node)
if ASTMatcher(Square.POW_PATTERN).match(node):
return self.replace(node.left)
elif isinstance(node.op, ast.Pow) and isnum(node.right):
n = node.right.value
if int(n) == n and n > 0:
return self.expand_pow(node.left, n)
else:
return node
else:
return node
def visit_Call(self, node):
self.generic_visit(node)
if ASTMatcher(Square.POWER_PATTERN).match(node):
return self.replace(node.args[0])
else:
return node
from pythran.passmanager import Transformation
from pythran.transformations.normalize_tuples import ConvertToTuple
from pythran.conversion import mangle
import gast as ast
import sys
if sys.version_info.major == 2:
MODULE = 'itertools'
IMAP = 'imap'
IFILTER = 'ifilter'
else:
MODULE = '__builtin__'
IMAP = 'map'
IFILTER = 'filter'
ASMODULE = mangle(MODULE)
class ComprehensionPatterns(Transformation):
if sys.version_info.major == 3:
'''
Transforms list comprehension into intrinsics.
>>> import gast as ast
>>> from pythran import passmanager, backend
>>> node = ast.parse("def foo(y) : return (x for x in y)")
>>> pm = passmanager.PassManager("test")
>>> _, node = pm.apply(ComprehensionPatterns, node)
>>> print(pm.dump(backend.Python, node))
def foo(y):
return __builtin__.map((lambda x: x), y)
'''
def visit_Call(self, node):
"""
Transform call site to have normal function call.
Examples
--------
For methods:
>> a = [1, 2, 3]
>> a.append(1)
Becomes
>> __list__.append(a, 1)
For functions:
>> builtins.dict.fromkeys([1, 2, 3])
Becomes
>> builtins.__dict__.fromkeys([1, 2, 3])
"""
node = self.generic_visit(node)
# Only attributes function can be Pythonic and should be normalized
if isinstance(node.func, ast.Attribute):
if node.func.attr in methods:
# Check object targeted by methods
if self.baseobj(node.func) is not None:
self.update = True
# As it was a methods call, push targeted object as first
# arguments and add correct module prefix
node.args.insert(0, node.func.value)
mod = methods[node.func.attr][0]
# Submodules import full module
self.to_import.add(mangle(mod[0]))
node.func = self.attr_to_func(node.func)
# else methods have been called using function syntax
if node.func.attr in methods or node.func.attr in functions:
# Now, methods and function have both function syntax
def rec(path, cur_module):
"""
Recursively rename path content looking in matching module.
Prefers __module__ to module if it exists.
This recursion is done as modules are visited top->bottom
while attributes have to be visited bottom->top.
"""
err = "Function path is chained attributes and name"
assert isinstance(path, (ast.Name, ast.Attribute)), err
if isinstance(path, ast.Attribute):
new_node, cur_module = rec(path.value, cur_module)
new_id, mname = self.renamer(path.attr, cur_module)
return (ast.Attribute(new_node, new_id,
ast.Load()), cur_module[mname])
else:
new_id, mname = self.renamer(path.id, cur_module)
return (ast.Name(new_id, ast.Load(), None,
None), cur_module[mname])
# Rename module path to avoid naming issue.
node.func.value, _ = rec(node.func.value, MODULES)
self.update = True
self.keyword_based_disambiguification(node)
return node
from pythran.passmanager import Transformation
from pythran.transformations import NormalizeTuples
from pythran.conversion import mangle
import gast as ast
import sys
if sys.version_info.major == 2:
MODULE = 'itertools'
IMAP = 'imap'
IFILTER = 'ifilter'
else:
MODULE = '__builtin__'
IMAP = 'map'
IFILTER = 'filter'
ASMODULE = mangle(MODULE)
class GenExpToImap(Transformation):
'''
Transforms generator expressions into iterators.
>>> import gast as ast
>>> from pythran import passmanager, backend
>>> node = ast.parse("(x*x for x in range(10))")
>>> pm = passmanager.PassManager("test")
>>> _, node = pm.apply(GenExpToImap, node)
>>> print pm.dump(backend.Python, node)
import itertools as __pythran_import_itertools
__pythran_import_itertools.imap((lambda x: (x * x)), range(10))
'''
ctx=ast.Load()),
args=[Placeholder(0)],
keywords=[])
],
keywords=[]),
lambda: ast.Call(func=ast.Attribute(value=ast.Attribute(value=ast.Name(
id='__builtin__', ctx=ast.Load(), annotation=None),
attr="pythran",
ctx=ast.Load()),
attr="len_set",
ctx=ast.Load()),
args=[Placeholder(0)],
keywords=[])),
# __builtin__.abs(X ** 2) => __builtin__.pythran.abssqr(X)
(ast.Call(func=ast.Attribute(value=ast.Name(id=mangle('numpy'),
ctx=ast.Load(),
annotation=None),
attr="square",
ctx=ast.Load()),
args=[
ast.Call(func=ast.Attribute(value=ast.Name(id='__builtin__',
ctx=ast.Load(),
annotation=None),
attr="abs",
ctx=ast.Load()),
args=[Placeholder(0)],
keywords=[])
],
keywords=[]),
lambda: ast.Call(func=ast.Attribute(value=ast.Attribute(value=ast.Name(
def visit_Call(self, node):
"""
Transform call site to have normal function call.
Examples
--------
For methods:
>> a = [1, 2, 3]
>> a.append(1)
Becomes
>> __list__.append(a, 1)
For functions:
>> __builtin__.dict.fromkeys([1, 2, 3])
Becomes
>> __builtin__.__dict__.fromkeys([1, 2, 3])
"""
node = self.generic_visit(node)
# Only attributes function can be Pythonic and should be normalized
if isinstance(node.func, ast.Attribute):
if node.func.attr in methods:
# Get object targeted by methods
obj = lhs = node.func.value
# Get the most left identifier to check if it is not an
# imported module
while isinstance(obj, ast.Attribute):
obj = obj.value
is_not_module = (not isinstance(obj, ast.Name) or
obj.id not in self.imports)
if is_not_module:
self.update = True
# As it was a methods call, push targeted object as first
# arguments and add correct module prefix
node.args.insert(0, lhs)
mod = methods[node.func.attr][0]
# Submodules import full module
self.to_import.add(mangle(mod[0]))
node.func = reduce(
lambda v, o: ast.Attribute(v, o, ast.Load()),
mod[1:] + (node.func.attr,),
ast.Name(mangle(mod[0]), ast.Load(), None)
)
# else methods have been called using function syntax
if node.func.attr in methods or node.func.attr in functions:
# Now, methods and function have both function syntax
def rec(path, cur_module):
"""
Recursively rename path content looking in matching module.
Prefers __module__ to module if it exists.
This recursion is done as modules are visited top->bottom
while attributes have to be visited bottom->top.
"""
err = "Function path is chained attributes and name"
assert isinstance(path, (ast.Name, ast.Attribute)), err
if isinstance(path, ast.Attribute):
new_node, cur_module = rec(path.value, cur_module)
new_id, mname = self.renamer(path.attr, cur_module)
return (ast.Attribute(new_node, new_id, ast.Load()),
cur_module[mname])
else:
new_id, mname = self.renamer(path.id, cur_module)
if mname not in cur_module:
raise PythranSyntaxError(
"Unbound identifier '{}'".format(mname), node)
return (ast.Name(new_id, ast.Load(), None),
cur_module[mname])
# Rename module path to avoid naming issue.
node.func.value, _ = rec(node.func.value, MODULES)
self.update = True
return node
def replace(self, value):
self.update = self.need_import = True
module_name = ast.Name(mangle('numpy'), ast.Load(), None)
return ast.Call(ast.Attribute(module_name, 'square', ast.Load()),
[value], [])
def visit_Module(self, node):
"""Add itertools import for imap, izip or ifilter iterator."""
self.generic_visit(node)
import_alias = ast.alias(name='itertools', asname=mangle('itertools'))
importIt = ast.Import(names=[import_alias])
return ast.Module(body=([importIt] + node.body))
