def visit_If(self, node):
test = self.visit(node.test)
body = [self.visit(n) for n in node.body]
orelse = [self.visit(n) for n in node.orelse]
if isinstance(node.test, ast.Num) and node.test.n == 1:
stmt = Block(body)
else:
stmt = If(test, Block(body), Block(orelse) if orelse else None)
return self.process_locals(node,
self.process_omp_attachements(node, stmt))
def visit_ExceptHandler(self, node):
name = self.visit(node.name) if node.name else None
body = [self.visit(m) for m in node.body]
if not isinstance(node.type, ast.Tuple):
return [
ExceptHandler(node.type and node.type.attr, Block(body), name)
]
else:
elts = [p.attr for p in node.type.elts]
return [ExceptHandler(o, Block(body), name) for o in elts]
def visit_If(self, node):
test = self.visit(node.test)
body = [self.visit(n) for n in node.body]
orelse = [self.visit(n) for n in node.orelse]
# compound statement required for some OpenMP Directives
if isnum(node.test) and node.test.value == 1:
stmt = Block(body)
else:
stmt = If(test, Block(body), Block(orelse) if orelse else None)
return self.process_locals(node,
self.process_omp_attachements(node, stmt))
def visit_Return(self, node):
if self.yields:
return Block([
Statement("{0} = -1".format(Cxx.generator_state_holder)),
Statement("goto {0}".format(Cxx.final_statement))
])
else:
value = self.visit(node.value)
if metadata.get(node, metadata.StaticReturn):
stmt = Block([
Assign("static auto tmp_global", value),
ReturnStatement("tmp_global")
])
else:
stmt = ReturnStatement(value)
return self.process_omp_attachements(node, stmt)
def visit_Yield(self, node):
num, label = self.yields[node]
return "".join(n for n in Block([
Assign(CxxGenerator.StateHolder, num),
ReturnStatement("{0} = {1}".format(CxxGenerator.StateValue,
self.visit(node.value))),
Statement("{0}:".format(label))
]).generate())
def visit_Return(self, node):
if self.yields:
return Block([
Statement("{0} = -1".format(Cxx.generator_state_holder)),
Statement("goto {0}".format(Cxx.final_statement))
])
else:
stmt = ReturnStatement(self.visit(node.value))
return self.process_omp_attachements(node, stmt)
def visit_While(self, node):
"""
Create While node for Cxx generation.
It is a cxx_loop to handle else clause.
"""
test = self.visit(node.test)
body = [self.visit(n) for n in node.body]
stmt = While(test, Block(body))
return self.process_omp_attachements(node, stmt)
def gen_for(self, node, target, local_iter, local_iter_decl, loop_body):
"""
Create For representation on iterator for Cxx generation.
Examples
--------
>> "omp parallel for"
>> for i in xrange(10):
>> ... do things ...
Becomes
>> "omp parallel for shared(__iterX)"
>> for(decltype(__iterX)::iterator __targetX = __iterX.begin();
__targetX < __iterX.end(); ++__targetX)
>> typename decltype(__targetX)::reference i = *__targetX;
>> ... do things ...
It the case of not local variable, typing for `i` disappear and typing
is removed for iterator in case of yields statement in function.
"""
# Choose target variable for iterator (which is iterator type)
local_target = "__target{0}".format(len(self.break_handlers))
local_target_decl = NamedType(
"typename decltype({0})::iterator".format(local_iter))
# For yield function, all variables are globals.
if self.yields:
self.extra_declarations.append((
local_target,
local_target_decl,
))
local_target_decl = ""
# If variable is local to the for body it's a ref to the iterator value
# type
if node.target.id in self.scope[node] and not self.yields:
self.ldecls = {d for d in self.ldecls if d.id != node.target.id}
local_type = "typename decltype({})::reference ".format(
local_target)
else:
local_type = ""
# Assign iterable value
loop_body_prelude = Statement("{} {}= *{}".format(
local_type, target, local_target))
# Create the loop
loop = For(
"{0} {1} = {2}.begin()".format(local_target_decl, local_target,
local_iter),
"{0} < {1}.end()".format(local_target,
local_iter), "++{0}".format(local_target),
Block([loop_body_prelude, loop_body]))
return [self.process_omp_attachements(node, loop)]
def visit_Return(self, node):
if self.yields:
return Block([
Statement("{0} = -1".format(Cxx.generator_state_holder)),
Statement("goto {0}".format(Cxx.final_statement))
])
else:
value = self.visit(node.value)
if metadata.get(node, metadata.StaticReturn):
# don't rely on auto because we want to make sure there's no
# conversion each time we return
# this happens for variant because the variant param
# order may differ from the init order (because of the way we
# do type inference
rtype = "typename {}::type::result_type".format(self.fname)
stmt = Block([
Assign("static %s tmp_global" % rtype, value),
ReturnStatement("tmp_global")
])
else:
stmt = ReturnStatement(value)
return self.process_omp_attachements(node, stmt)
def visit_FunctionDef(self, node):
self.fname = node.name
tmp = self.prepare_functiondef_context(node)
operator_body, formal_types, formal_args = tmp
tmp = self.prepare_types(node)
dflt_argv, dflt_argt, result_type, callable_type, pure_type = tmp
# a function has a call operator to be called
# and a default constructor to create instances
fscope = "type{0}::".format(
"<{0}>".format(", ".join(formal_types)) if formal_types else "")
ffscope = "{0}::{1}".format(node.name, fscope)
operator_declaration = [
templatize(
make_const_function_declaration(
self, node, "typename {0}result_type".format(fscope),
"operator()", formal_types, formal_args, dflt_argv),
formal_types, dflt_argt),
EmptyStatement()
]
operator_signature = make_const_function_declaration(
self, node, "typename {0}result_type".format(ffscope),
"{0}::operator()".format(node.name), formal_types, formal_args)
ctx = CachedTypeVisitor(self.lctx)
operator_local_declarations = ([
Statement("{0} {1}".format(
self.types[self.local_names[k]].generate(ctx), k))
for k in self.ldecls
])
dependent_typedefs = ctx.typedefs()
operator_definition = FunctionBody(
templatize(operator_signature, formal_types),
Block(dependent_typedefs + operator_local_declarations +
operator_body))
ctx = CachedTypeVisitor()
extra_typedefs = ([
Typedef(Value(t.generate(ctx), t.name))
for t in self.types[node][1]
] + [Typedef(Value(result_type.generate(ctx), "result_type"))])
extra_typedefs = ctx.typedefs() + extra_typedefs
return_declaration = [
templatize(Struct("type", extra_typedefs), formal_types, dflt_argt)
]
topstruct = Struct(node.name, [callable_type, pure_type] +
return_declaration + operator_declaration)
return [topstruct], [operator_definition]
def gen_for(self, node, target, local_iter, local_iter_decl, loop_body):
"""
Create For representation on iterator for Cxx generation.
Examples
--------
>> "omp parallel for"
>> for i in range(10):
>> ... do things ...
Becomes
>> "omp parallel for shared(__iterX)"
>> for(decltype(__iterX)::iterator __targetX = __iterX.begin();
__targetX < __iterX.end(); ++__targetX)
>> auto&& i = *__targetX;
>> ... do things ...
It the case of not local variable, typing for `i` disappear and typing
is removed for iterator in case of yields statement in function.
"""
# Choose target variable for iterator (which is iterator type)
local_target = "__target{0}".format(id(node))
local_target_decl = self.types.builder.IteratorOfType(local_iter_decl)
islocal = (node.target.id not in self.openmp_deps and
node.target.id in self.scope[node] and
not hasattr(self, 'yields'))
# If variable is local to the for body it's a ref to the iterator value
# type
if islocal:
local_type = "auto&&"
self.ldecls.remove(node.target.id)
else:
local_type = ""
# Assign iterable value
loop_body_prelude = Statement("{} {}= *{}".format(local_type,
target,
local_target))
# Create the loop
assign = self.make_assign(local_target_decl, local_target, local_iter)
loop = For("{}.begin()".format(assign),
"{0} < {1}.end()".format(local_target, local_iter),
"++{0}".format(local_target),
Block([loop_body_prelude, loop_body]))
return [self.process_omp_attachements(node, loop)]
def process_locals(self, node, node_visited, *skipped):
"""
Declare variable local to node and insert declaration before.
Not possible for function yielding values.
"""
local_vars = self.scope[node].difference(skipped)
if not local_vars:
return node_visited # no processing
locals_visited = []
for varname in local_vars:
vartype = self.typeof(varname)
decl = Statement("{} {}".format(vartype, varname))
locals_visited.append(decl)
self.ldecls.difference_update(local_vars)
return Block(locals_visited + [node_visited])
def loop_visitor(self, node):
"""
New decorate function.
It push the breaking flag, run the visitor and add "else" statements.
"""
if not node.orelse:
with pushpop(self.break_handlers, None):
res = visit(self, node)
return res
break_handler = "__no_breaking{0}".format(id(node))
with pushpop(self.break_handlers, break_handler):
res = visit(self, node)
# handle the body of the for loop
orelse = [self.visit(stmt) for stmt in node.orelse]
orelse_label = Label(break_handler)
return Block([res] + orelse + [orelse_label])
def loop_visitor(self, node):
"""
New decorate function.
It push the breaking flag, run the visitor and add "else" statements.
"""
if node.orelse:
break_handler = "__no_breaking{0}".format(len(self.break_handlers))
else:
break_handler = None
self.break_handlers.append(break_handler)
res = fun(self, node)
self.break_handlers.pop()
# handle the body of the for loop
if break_handler:
orelse = map(self.visit, node.orelse)
orelse_label = Statement("{0}:".format(break_handler))
return Block([res] + orelse + [orelse_label])
else:
return res
def visit_For(self, node):
"""
Create For representation for Cxx generation.
Examples
--------
>> for i in xrange(10):
>> ... work ...
Becomes
>> typename returnable<decltype(__builtin__.xrange(10))>::type __iterX
= __builtin__.xrange(10);
>> ... possible container size reservation ...
>> for (auto&& i: __iterX)
>> ... the work ...
This function also handle assignment for local variables.
We can notice that three kind of loop are possible:
- Normal for loop on iterator
- Autofor loop.
- Normal for loop using integer variable iteration
Kind of loop used depend on OpenMP, yield use and variable scope.
"""
if not isinstance(node.target, ast.Name):
raise PythranSyntaxError(
"Using something other than an identifier as loop target",
node.target)
target = self.visit(node.target)
# Handle the body of the for loop
loop_body = Block([self.visit(stmt) for stmt in node.body])
# Declare local variables at the top of the loop body
loop_body = self.process_locals(node, loop_body, node.target.id)
iterable = self.visit(node.iter)
if self.can_use_c_for(node):
header, loop = self.gen_c_for(node, target, loop_body)
else:
if self.can_use_autofor(node):
header = []
self.ldecls.remove(node.target.id)
autofor = AutoFor(target, iterable, loop_body)
loop = [self.process_omp_attachements(node, autofor)]
else:
# Iterator declaration
local_iter = "__iter{0}".format(id(node))
local_iter_decl = self.types.builder.Assignable(
self.types[node.iter])
self.handle_omp_for(node, local_iter)
# Assign iterable
# For C loop, it avoids issues
# if the upper bound is assigned in the loop
asgnt = self.make_assign(local_iter_decl, local_iter, iterable)
header = [Statement(asgnt)]
loop = self.gen_for(node, target, local_iter, local_iter_decl,
loop_body)
# For xxxComprehension, it is replaced by a for loop. In this case,
# pre-allocate size of container.
for comp in metadata.get(node, metadata.Comprehension):
header.append(
Statement("pythonic::utils::reserve({0},{1})".format(
comp.target, iterable)))
return Block(header + loop)
def visit_For(self, node):
"""
Create For representation for Cxx generation.
Examples
--------
>> for i in xrange(10):
>> ... work ...
Becomes
>> typename returnable<decltype(__builtin__.xrange(10))>::type __iterX
= __builtin__.xrange(10);
>> ... possible container size reservation ...
>> for (typename decltype(__iterX)::iterator::reference i: __iterX)
>> ... the work ...
This function also handle assignment for local variables.
We can notice that three kind of loop are possible:
- Normal for loop on iterator
- Autofor loop.
- Normal for loop using integer variable iteration
Kind of loop used depend on OpenMP, yield use and variable scope.
"""
if not isinstance(node.target, ast.Name):
raise PythranSyntaxError(
"Using something other than an identifier as loop target",
node.target)
target = self.visit(node.target)
# Handle the body of the for loop
loop_body = Block(map(self.visit, node.body))
# Declare local variables at the top of the loop body
loop_body = self.process_locals(node, loop_body, node.target.id)
iterable = self.visit(node.iter)
if self.can_use_c_for(node):
header, loop = self.gen_c_for(node, target, loop_body)
else:
# Iterator declaration
local_iter = "__iter{0}".format(len(self.break_handlers))
local_iter_decl = Assignable(DeclType(iterable))
self.handle_omp_for(node, local_iter)
# For yield function, iterable is globals.
if self.yields:
self.extra_declarations.append((
local_iter,
local_iter_decl,
))
local_iter_decl = ""
# Assign iterable
# For C loop, it avoid issue if upper bound is reassign in the loop
header = [
Statement("{0} {1} = {2}".format(local_iter_decl, local_iter,
iterable))
]
if self.can_use_autofor(node):
self.ldecls = {
d
for d in self.ldecls if d.id != node.target.id
}
autofor = AutoFor(target, local_iter, loop_body)
loop = [self.process_omp_attachements(node, autofor)]
else:
loop = self.gen_for(node, target, local_iter, local_iter_decl,
loop_body)
# For xxxComprehension, it is replaced by a for loop. In this case,
# pre-allocate size of container.
for comp in metadata.get(node, metadata.Comprehension):
header.append(
Statement("pythonic::utils::reserve({0},{1})".format(
comp.target, iterable)))
return Block(header + loop)
def visit_Return(self, node):
return Block([
Statement("{0} = -1".format(CxxGenerator.StateHolder)),
Statement("goto {0}".format(CxxGenerator.FinalStatement))
])
def visit_FunctionDef(self, node):
tmp = self.prepare_functiondef_context(node)
operator_body, formal_types, formal_args = tmp
tmp = self.prepare_types(node)
dflt_argv, dflt_argt, result_type, callable_type, pure_type = tmp
# a generator has a call operator that returns the iterator
next_name = "__generator__{0}".format(node.name)
instanciated_next_name = "{0}{1}".format(
next_name,
"<{0}>".format(", ".join(formal_types)) if formal_types else "")
operator_body.append(
Statement("{}: return result_type()".format(
CxxGenerator.FinalStatement)))
next_declaration = [
FunctionDeclaration(Value("result_type", "next"), []),
EmptyStatement()
] # empty statement to force a comma ...
# the constructors
next_constructors = [
FunctionBody(FunctionDeclaration(Value("", next_name), []),
Line(': pythonic::yielder() {}'))
]
if formal_types:
# If all parameters have a default value, we don't need default
# constructor
if dflt_argv and all(dflt_argv):
next_constructors = list()
next_constructors.append(
FunctionBody(
make_function_declaration(self, node, "", next_name,
formal_types, formal_args,
dflt_argv),
Line(": {0} {{ }}".format(", ".join(
["pythonic::yielder()"] +
["{0}({0})".format(arg) for arg in formal_args])))))
next_iterator = [
FunctionBody(FunctionDeclaration(Value("void", "operator++"), []),
Block([Statement("next()")])),
FunctionBody(
FunctionDeclaration(
Value(
"typename {0}::result_type".format(
instanciated_next_name), "operator*"),
[], "const"),
Block([ReturnStatement(CxxGenerator.StateValue)])),
FunctionBody(
FunctionDeclaration(
Value(
"pythonic::types::generator_iterator<{0}>".format(
next_name), "begin"), []),
Block([
Statement("next()"),
ReturnStatement("pythonic::types::generator_iterator<{0}>"
"(*this)".format(next_name))
])),
FunctionBody(
FunctionDeclaration(
Value(
"pythonic::types::generator_iterator<{0}>".format(
next_name), "end"), []),
Block([
ReturnStatement(
"pythonic::types::generator_iterator<{0}>()".format(
next_name))
]))
]
next_signature = templatize(
FunctionDeclaration(
Value(
"typename {0}::result_type".format(instanciated_next_name),
"{0}::next".format(instanciated_next_name)), []),
formal_types)
next_body = operator_body
# the dispatch table at the entry point
next_body.insert(
0,
Statement("switch({0}) {{ {1} }}".format(
CxxGenerator.StateHolder,
" ".join("case {0}: goto {1};".format(num, where)
for (num, where) in sorted(self.yields.values(),
key=lambda x: x[0])))))
ctx = CachedTypeVisitor(self.lctx)
next_members = ([
Statement("{0} {1}".format(ft, fa))
for (ft, fa) in zip(formal_types, formal_args)
] + [
Statement("{0} {1}".format(
self.types[self.local_names[k]].generate(ctx), k))
for k in self.ldecls
] + [
Statement("{0} {1}".format(v, k))
for k, v in self.extra_declarations
] + [
Statement("typename {0}::result_type {1}".format(
instanciated_next_name, CxxGenerator.StateValue))
])
extern_typedefs = [
Typedef(Value(t.generate(ctx), t.name))
for t in self.types[node][1]
]
iterator_typedef = [
Typedef(
Value(
"pythonic::types::generator_iterator<{0}>".format(
"{0}<{1}>".format(next_name, ", ".join(formal_types)
) if formal_types else next_name),
"iterator")),
Typedef(Value(result_type.generate(ctx), "value_type"))
]
result_typedef = [
Typedef(Value(result_type.generate(ctx), "result_type"))
]
extra_typedefs = (ctx.typedefs() + extern_typedefs + iterator_typedef +
result_typedef)
next_struct = templatize(
Struct(
next_name, extra_typedefs + next_members + next_constructors +
next_iterator + next_declaration, "pythonic::yielder"),
formal_types)
next_definition = FunctionBody(next_signature, Block(next_body))
operator_declaration = [
templatize(
make_const_function_declaration(self, node,
instanciated_next_name,
"operator()", formal_types,
formal_args, dflt_argv),
formal_types, dflt_argt),
EmptyStatement()
]
operator_signature = make_const_function_declaration(
self, node, instanciated_next_name,
"{0}::operator()".format(node.name), formal_types, formal_args)
operator_definition = FunctionBody(
templatize(operator_signature, formal_types),
Block([
ReturnStatement("{0}({1})".format(instanciated_next_name,
", ".join(formal_args)))
]))
topstruct_type = templatize(Struct("type", extra_typedefs),
formal_types)
topstruct = Struct(node.name,
[topstruct_type, callable_type, pure_type] +
operator_declaration)
return [next_struct, topstruct], [next_definition, operator_definition]
def generate_cxx(module_name, code, specs=None, optimizations=None):
'''python + pythran spec -> c++ code
returns a PythonModule object
'''
if sys.version_info[0] == 3:
raise ValueError(
"Pythran does not fully support Python3, "
"it can only be used to compile C++ code "
"generated with the -E flag with a Python2 version of Pythran. "
"Sorry about this :-/")
pm = PassManager(module_name)
# front end
ir, renamings, docstrings = frontend.parse(pm, code)
# middle-end
optimizations = (optimizations
or cfg.get('pythran', 'optimizations').split())
optimizations = [_parse_optimization(opt) for opt in optimizations]
refine(pm, ir, optimizations)
# back-end
content = pm.dump(Cxx, ir)
# instanciate the meta program
if specs is None:
class Generable(object):
def __init__(self, content):
self.content = content
def __str__(self):
return str(self.content)
generate = __str__
mod = Generable(content)
else:
# uniform typing
for fname, signatures in specs.items():
if not isinstance(signatures, tuple):
specs[fname] = (signatures, )
# verify the pythran export are compatible with the code
specs = expand_specs(specs)
check_specs(ir, specs, renamings)
specs_to_docstrings(specs, docstrings)
metainfo = {
'hash': hashlib.sha256(code).hexdigest(),
'version': __version__,
'date': datetime.now()
}
mod = PythonModule(module_name, docstrings, metainfo)
mod.add_to_preamble(Define("BOOST_SIMD_NO_STRICT_ALIASING", "1"))
mod.add_to_includes(
Include("pythonic/core.hpp"),
Include("pythonic/python/core.hpp"),
# FIXME: only include these when needed
Include("pythonic/types/bool.hpp"),
Include("pythonic/types/int.hpp"),
Line("#ifdef _OPENMP\n#include <omp.h>\n#endif"))
mod.add_to_includes(
*[Include(inc) for inc in _extract_specs_dependencies(specs)])
mod.add_to_includes(*content.body)
for function_name, signatures in specs.iteritems():
internal_func_name = renamings.get(function_name, function_name)
# global variables are functions with no signatures :-)
if not signatures:
mod.add_global_var(
function_name, "{}()()".format(
pythran_ward +
'{0}::{1}'.format(module_name, internal_func_name)))
for sigid, signature in enumerate(signatures):
numbered_function_name = "{0}{1}".format(
internal_func_name, sigid)
arguments_types = [pytype_to_ctype(t) for t in signature]
has_arguments = HasArgument(internal_func_name).visit(ir)
arguments = [
"a{0}".format(i) for i in xrange(len(arguments_types))
]
name_fmt = pythran_ward + "{0}::{1}::type{2}"
args_list = ", ".join(arguments_types)
specialized_fname = name_fmt.format(
module_name, internal_func_name,
"<{0}>".format(args_list) if has_arguments else "")
result_type = "typename %s::result_type" % specialized_fname
mod.add_function(
FunctionBody(
FunctionDeclaration(
Value(result_type, numbered_function_name), [
Value(t, a)
for t, a in zip(arguments_types, arguments)
]),
Block([
Statement("return {0}()({1})".format(
pythran_ward + '{0}::{1}'.format(
module_name, internal_func_name),
', '.join(arguments)))
])), function_name, arguments_types)
return mod
def visit_FunctionDef(self, node):
class CachedTypeVisitor:
class CachedType:
def __init__(self, s):
self.s = s
def generate(self, _):
return self.s
def __init__(self, other=None):
if other:
self.cache = other.cache.copy()
self.rcache = other.rcache.copy()
self.mapping = other.mapping.copy()
else:
self.cache = dict()
self.rcache = dict()
self.mapping = dict()
def __call__(self, node):
if node not in self.mapping:
t = node.generate(self)
if t in self.rcache:
self.mapping[node] = self.mapping[self.rcache[t]]
self.cache[node] = self.cache[self.rcache[t]]
else:
self.rcache[t] = node
self.mapping[node] = len(self.mapping)
self.cache[node] = t
return CachedTypeVisitor.CachedType("__type{0}".format(
self.mapping[node]))
def typedefs(self):
l = sorted(self.mapping.items(), key=lambda x: x[1])
L = list()
visited = set() # the same value must not be typedefed twice
for k, v in l:
if v not in visited:
typename = "__type" + str(v)
L.append(Typedef(Value(self.cache[k], typename)))
visited.add(v)
return L
self.fname = node.name
# prepare context and visit function body
fargs = node.args.args
formal_args = [arg.id for arg in fargs]
formal_types = ["argument_type" + str(i) for i in range(len(fargs))]
self.ldecls = set(self.passmanager.gather(LocalNodeDeclarations, node))
self.local_names = {sym.id: sym for sym in self.ldecls}
self.local_names.update({arg.id: arg for arg in fargs})
self.extra_declarations = []
self.lctx = CachedTypeVisitor()
# choose one node among all the ones with the same name for each name
self.ldecls = set({n.id: n for n in self.ldecls}.values())
# 0 is used as initial_state, thus the +1
self.yields = {
k: (1 + v, "yield_point{0}".format(1 + v))
for (v, k) in enumerate(self.passmanager.gather(YieldPoints, node))
}
# gather body dump
operator_body = [self.visit(stmt) for stmt in node.body]
# compute arg dump
default_arg_values = ([None] *
(len(node.args.args) - len(node.args.defaults)) +
[self.visit(n) for n in node.args.defaults])
default_arg_types = ([None] *
(len(node.args.args) - len(node.args.defaults)) +
[self.types[n] for n in node.args.defaults])
# compute type dump
result_type = self.types[node][0]
callable_type = Typedef(Value("void", "callable"))
pure_type = (Typedef(Value("void", "pure"))
if node in self.pure_expressions else EmptyStatement())
def make_function_declaration(rtype,
name,
ftypes,
fargs,
defaults=None,
attributes=None):
if defaults is None:
defaults = [None] * len(ftypes)
if attributes is None:
attributes = []
arguments = list()
for i, (t, a, d) in enumerate(zip(ftypes, fargs, defaults)):
if self.yields:
rvalue_ref = ""
elif self.argument_effects[node][i]:
rvalue_ref = "&&"
else:
rvalue_ref = " const &"
argument = Value(
t + rvalue_ref,
"{0}{1}".format(a, "= {0}".format(d) if d else ""))
arguments.append(argument)
return FunctionDeclaration(Value(rtype, name), arguments,
*attributes)
def make_const_function_declaration(rtype,
name,
ftypes,
fargs,
defaults=None):
return make_function_declaration(rtype, name, ftypes, fargs,
defaults, ["const"])
if self.yields: # generator case
# a generator has a call operator that returns the iterator
next_name = "__generator__{0}".format(node.name)
instanciated_next_name = "{0}{1}".format(
next_name, "<{0}>".format(", ".join(formal_types))
if formal_types else "")
operator_body.append(
Statement("{0}: return result_type();".format(
Cxx.final_statement)))
next_declaration = [
FunctionDeclaration(Value("result_type", "next"), []),
EmptyStatement()
] # empty statement to force a comma ...
# the constructors
next_constructors = [
FunctionBody(FunctionDeclaration(Value("", next_name), []),
Line(': pythonic::yielder() {}'))
]
if formal_types:
# If all parameters have a default value, we don't need default
# constructor
if default_arg_values and all(default_arg_values):
next_constructors = list()
next_constructors.append(
FunctionBody(
make_function_declaration("", next_name, formal_types,
formal_args,
default_arg_values),
Line(": {0} {{ }}".format(", ".join(
["pythonic::yielder()"] +
["{0}({0})".format(arg)
for arg in formal_args])))))
next_iterator = [
FunctionBody(
FunctionDeclaration(Value("void", "operator++"), []),
Block([Statement("next()")])),
FunctionBody(
FunctionDeclaration(
Value(
"typename {0}::result_type".format(
instanciated_next_name), "operator*"), [],
"const"),
Block([ReturnStatement(Cxx.generator_state_value)])),
FunctionBody(
FunctionDeclaration(
Value(
"pythonic::types::generator_iterator<{0}>".format(
next_name), "begin"), []),
Block([
Statement("next()"),
ReturnStatement(
"pythonic::types::generator_iterator<{0}>"
"(*this)".format(next_name))
])),
FunctionBody(
FunctionDeclaration(
Value(
"pythonic::types::generator_iterator<{0}>".format(
next_name), "end"), []),
Block([
ReturnStatement(
"pythonic::types::generator_iterator<{0}>()".
format(next_name))
]))
]
next_signature = templatize(
FunctionDeclaration(
Value(
"typename {0}::result_type".format(
instanciated_next_name),
"{0}::next".format(instanciated_next_name)), []),
formal_types)
next_body = operator_body
# the dispatch table at the entry point
next_body.insert(
0,
Statement("switch({0}) {{ {1} }}".format(
Cxx.generator_state_holder,
" ".join("case {0}: goto {1};".format(num, where)
for (num, where) in sorted(self.yields.values(),
key=lambda x: x[0])))))
ctx = CachedTypeVisitor(self.lctx)
next_members = ([
Statement("{0} {1}".format(ft, fa))
for (ft, fa) in zip(formal_types, formal_args)
] + [
Statement("{0} {1}".format(self.types[k].generate(ctx), k.id))
for k in self.ldecls
] + [
Statement("{0} {1}".format(v, k))
for k, v in self.extra_declarations
] + [
Statement("typename {0}::result_type {1}".format(
instanciated_next_name, Cxx.generator_state_value))
])
extern_typedefs = [
Typedef(Value(t.generate(ctx), t.name))
for t in self.types[node][1] if not t.isweak()
]
iterator_typedef = [
Typedef(
Value(
"pythonic::types::generator_iterator<{0}>".format(
"{0}<{1}>".format(next_name, ", ".join(
formal_types)) if formal_types else next_name),
"iterator")),
Typedef(Value(result_type.generate(ctx), "value_type"))
]
result_typedef = [
Typedef(Value(result_type.generate(ctx), "result_type"))
]
extra_typedefs = (ctx.typedefs() + extern_typedefs +
iterator_typedef + result_typedef)
next_struct = templatize(
Struct(
next_name, extra_typedefs + next_members +
next_constructors + next_iterator + next_declaration,
"pythonic::yielder"), formal_types)
next_definition = FunctionBody(next_signature, Block(next_body))
operator_declaration = [
templatize(
make_const_function_declaration(instanciated_next_name,
"operator()", formal_types,
formal_args,
default_arg_values),
formal_types, default_arg_types),
EmptyStatement()
]
operator_signature = make_const_function_declaration(
instanciated_next_name, "{0}::operator()".format(node.name),
formal_types, formal_args)
operator_definition = FunctionBody(
templatize(operator_signature, formal_types),
Block([
ReturnStatement("{0}({1})".format(instanciated_next_name,
", ".join(formal_args)))
]))
topstruct_type = templatize(Struct("type", extra_typedefs),
formal_types)
topstruct = Struct(node.name,
[topstruct_type, callable_type, pure_type] +
operator_declaration)
self.declarations.append(next_struct)
self.definitions.append(next_definition)
else: # regular function case
# a function has a call operator to be called
# and a default constructor to create instances
fscope = "type{0}::".format("<{0}>".format(", ".join(formal_types))
if formal_types else "")
ffscope = "{0}::{1}".format(node.name, fscope)
operator_declaration = [
templatize(
make_const_function_declaration(
"typename {0}result_type".format(fscope), "operator()",
formal_types, formal_args, default_arg_values),
formal_types, default_arg_types),
EmptyStatement()
]
operator_signature = make_const_function_declaration(
"typename {0}result_type".format(ffscope),
"{0}::operator()".format(node.name), formal_types, formal_args)
ctx = CachedTypeVisitor(self.lctx)
operator_local_declarations = ([
Statement("{0} {1}".format(self.types[k].generate(ctx), k.id))
for k in self.ldecls
] + [
Statement("{0} {1}".format(v, k))
for k, v in self.extra_declarations
])
dependent_typedefs = ctx.typedefs()
operator_definition = FunctionBody(
templatize(operator_signature, formal_types),
Block(dependent_typedefs + operator_local_declarations +
operator_body))
ctx = CachedTypeVisitor()
extra_typedefs = ([
Typedef(Value(t.generate(ctx), t.name))
for t in self.types[node][1] if not t.isweak()
] + [Typedef(Value(result_type.generate(ctx), "result_type"))])
extra_typedefs = ctx.typedefs() + extra_typedefs
return_declaration = [
templatize(Struct("type", extra_typedefs), formal_types,
default_arg_types)
]
topstruct = Struct(node.name, [callable_type, pure_type] +
return_declaration + operator_declaration)
self.declarations.append(topstruct)
self.definitions.append(operator_definition)
return EmptyStatement()
def visit_Try(self, node):
body = [self.visit(n) for n in node.body]
except_ = list()
for n in node.handlers:
except_.extend(self.visit(n))
return TryExcept(Block(body), except_)
def generate_cxx(module_name, code, specs=None, optimizations=None):
'''python + pythran spec -> c++ code
returns a PythonModule object
'''
pm = PassManager(module_name)
# front end
ir, renamings, docstrings = frontend.parse(pm, code)
# middle-end
optimizations = (optimizations or
cfg.get('pythran', 'optimizations').split())
optimizations = [_parse_optimization(opt) for opt in optimizations]
refine(pm, ir, optimizations)
# type check
types = tog.typecheck(ir)
# back-end
content = pm.dump(Cxx, ir)
# instantiate the meta program
if specs is None:
class Generable(object):
def __init__(self, content):
self.content = content
def __str__(self):
return str(self.content)
generate = __str__
mod = Generable(content)
else:
# uniform typing
for fname, signatures in list(specs.items()):
if not isinstance(signatures, tuple):
specs[fname] = (signatures,)
# verify the pythran export are compatible with the code
specs = expand_specs(specs)
check_specs(ir, specs, renamings, types)
specs_to_docstrings(specs, docstrings)
if isinstance(code, bytes):
code_bytes = code
else:
code_bytes = code.encode('ascii', 'ignore')
metainfo = {'hash': hashlib.sha256(code_bytes).hexdigest(),
'version': __version__,
'date': datetime.now()}
mod = PythonModule(module_name, docstrings, metainfo)
mod.add_to_preamble(Define("BOOST_SIMD_NO_STRICT_ALIASING", "1"))
mod.add_to_includes(
Include("pythonic/core.hpp"),
Include("pythonic/python/core.hpp"),
# FIXME: only include these when needed
Include("pythonic/types/bool.hpp"),
Include("pythonic/types/int.hpp"),
Line("#ifdef _OPENMP\n#include <omp.h>\n#endif")
)
mod.add_to_includes(*[Include(inc) for inc in
_extract_specs_dependencies(specs)])
mod.add_to_includes(*content.body)
mod.add_to_includes(
Include("pythonic/python/exception_handler.hpp"),
)
for function_name, signatures in specs.items():
internal_func_name = renamings.get(function_name,
function_name)
# global variables are functions with no signatures :-)
if not signatures:
mod.add_global_var(function_name,
"{}()()".format(
pythran_ward + '{0}::{1}'.format(
module_name, internal_func_name)))
for sigid, signature in enumerate(signatures):
numbered_function_name = "{0}{1}".format(internal_func_name,
sigid)
arguments_types = [pytype_to_ctype(t) for t in signature]
has_arguments = HasArgument(internal_func_name).visit(ir)
arguments = ["a{0}".format(i)
for i in range(len(arguments_types))]
name_fmt = pythran_ward + "{0}::{1}::type{2}"
args_list = ", ".join(arguments_types)
specialized_fname = name_fmt.format(module_name,
internal_func_name,
"<{0}>".format(args_list)
if has_arguments else "")
result_type = "typename %s::result_type" % specialized_fname
mod.add_function(
FunctionBody(
FunctionDeclaration(
Value(
result_type,
numbered_function_name),
[Value(t, a)
for t, a in zip(arguments_types, arguments)]),
Block([Statement("""
PyThreadState *_save = PyEval_SaveThread();
try {{
auto res = {0}()({1});
PyEval_RestoreThread(_save);
return res;
}}
catch(...) {{
PyEval_RestoreThread(_save);
throw;
}}
""".format(
pythran_ward + '{0}::{1}'.format(
module_name, internal_func_name),
', '.join(arguments)))])
),
function_name,
arguments_types
)
return mod
def generate_cxx(module_name, code, specs=None, optimizations=None,
module_dir=None):
'''python + pythran spec -> c++ code
returns a PythonModule object and an error checker
the error checker can be used to print more detailed info on the origin of
a compile error (e.g. due to bad typing)
'''
pm, ir, renamings, docstrings = front_middle_end(module_name, code,
optimizations, module_dir)
# back-end
content = pm.dump(Cxx, ir)
# instantiate the meta program
if specs is None:
class Generable(object):
def __init__(self, content):
self.content = content
def __str__(self):
return str(self.content)
generate = __str__
mod = Generable(content)
def error_checker():
tog.typecheck(ir)
else:
# uniform typing
if isinstance(specs, dict):
specs = Spec(specs, {})
def error_checker():
types = tog.typecheck(ir)
check_specs(ir, specs, renamings, types)
specs.to_docstrings(docstrings)
if isinstance(code, bytes):
code_bytes = code
else:
code_bytes = code.encode('ascii', 'ignore')
metainfo = {'hash': hashlib.sha256(code_bytes).hexdigest(),
'version': __version__,
'date': datetime.now()}
mod = PythonModule(module_name, docstrings, metainfo)
mod.add_to_preamble(Define("BOOST_SIMD_NO_STRICT_ALIASING", "1"))
mod.add_to_includes(
Include("pythonic/core.hpp"),
Include("pythonic/python/core.hpp"),
# FIXME: only include these when needed
Include("pythonic/types/bool.hpp"),
Include("pythonic/types/int.hpp"),
Line("#ifdef _OPENMP\n#include <omp.h>\n#endif")
)
mod.add_to_includes(*[Include(inc) for inc in
_extract_specs_dependencies(specs)])
mod.add_to_includes(*content.body)
mod.add_to_includes(
Include("pythonic/python/exception_handler.hpp"),
)
def warded(module_name, internal_name):
return pythran_ward + '{0}::{1}'.format(module_name, internal_name)
for function_name, signatures in specs.functions.items():
internal_func_name = renamings.get(function_name,
function_name)
# global variables are functions with no signatures :-)
if not signatures:
mod.add_global_var(function_name,
"{}()()".format(warded(module_name,
internal_func_name)))
for sigid, signature in enumerate(signatures):
numbered_function_name = "{0}{1}".format(internal_func_name,
sigid)
arguments_types = [pytype_to_ctype(t) for t in signature]
arguments_names = HasArgument(internal_func_name).visit(ir)
arguments = [n for n, _ in
zip(arguments_names, arguments_types)]
name_fmt = pythran_ward + "{0}::{1}::type{2}"
args_list = ", ".join(arguments_types)
specialized_fname = name_fmt.format(module_name,
internal_func_name,
"<{0}>".format(args_list)
if arguments_names else "")
result_type = "typename %s::result_type" % specialized_fname
mod.add_pyfunction(
FunctionBody(
FunctionDeclaration(
Value(
result_type,
numbered_function_name),
[Value(t + '&&', a)
for t, a in zip(arguments_types, arguments)]),
Block([Statement("""
PyThreadState *_save = PyEval_SaveThread();
try {{
auto res = {0}()({1});
PyEval_RestoreThread(_save);
return res;
}}
catch(...) {{
PyEval_RestoreThread(_save);
throw;
}}
""".format(warded(module_name,
internal_func_name),
', '.join(arguments)))])
),
function_name,
arguments_types,
signature
)
for function_name, signature in specs.capsules.items():
internal_func_name = renamings.get(function_name,
function_name)
arguments_types = [pytype_to_ctype(t) for t in signature]
arguments_names = HasArgument(internal_func_name).visit(ir)
arguments = [n for n, _ in
zip(arguments_names, arguments_types)]
name_fmt = pythran_ward + "{0}::{1}::type{2}"
args_list = ", ".join(arguments_types)
specialized_fname = name_fmt.format(module_name,
internal_func_name,
"<{0}>".format(args_list)
if arguments_names else "")
result_type = "typename %s::result_type" % specialized_fname
docstring = spec_to_string(function_name, signature)
mod.add_capsule(
FunctionBody(
FunctionDeclaration(
Value(result_type, function_name),
[Value(t, a)
for t, a in zip(arguments_types, arguments)]),
Block([ReturnStatement("{0}()({1})".format(
warded(module_name, internal_func_name),
', '.join(arguments)))])
),
function_name,
docstring
)
return mod, error_checker
def generate_cxx(module_name, code, specs=None, optimizations=None):
'''python + pythran spec -> c++ code
returns a BoostPythonModule object
'''
pm = PassManager(module_name)
# front end
ir, renamings = frontend.parse(pm, code)
# middle-end
optimizations = (optimizations or
cfg.get('pythran', 'optimizations').split())
optimizations = map(_parse_optimization, optimizations)
refine(pm, ir, optimizations)
# back-end
content = pm.dump(Cxx, ir)
# instanciate the meta program
if specs is None:
class Generable:
def __init__(self, content):
self.content = content
def __str__(self):
return str(self.content)
generate = __str__
mod = Generable(content)
else:
# uniform typing
for fname, signatures in specs.items():
if not isinstance(signatures, tuple):
specs[fname] = (signatures,)
# verify the pythran export are compatible with the code
specs = expand_specs(specs)
check_specs(ir, specs, renamings)
mod = BoostPythonModule(module_name)
mod.use_private_namespace = False
# very low value for max_arity leads to various bugs
min_val = 2
specs_max = [max(map(len, s)) for s in specs.itervalues()]
max_arity = max([min_val] + specs_max)
mod.add_to_preamble([Define("BOOST_PYTHON_MAX_ARITY", max_arity)])
mod.add_to_preamble([Define("BOOST_SIMD_NO_STRICT_ALIASING", "1")])
mod.add_to_preamble([Include("pythonic/core.hpp")])
mod.add_to_preamble([Include("pythonic/python/core.hpp")])
mod.add_to_preamble([Line("#ifdef _OPENMP\n#include <omp.h>\n#endif")])
mod.add_to_preamble(map(Include, _extract_specs_dependencies(specs)))
mod.add_to_preamble(content.body)
mod.add_to_init([
Line('#ifdef PYTHONIC_TYPES_NDARRAY_HPP\nimport_array()\n#endif')])
# topologically sorted exceptions based on the inheritance hierarchy.
# needed because otherwise boost python register_exception handlers
# do not catch exception type in the right way
# (first valid exception is selected)
# Inheritance has to be taken into account in the registration order.
exceptions = nx.DiGraph()
for function_name, v in functions.iteritems():
for mname, symbol in v:
if isinstance(symbol, ConstExceptionIntr):
exceptions.add_node(
getattr(sys.modules[".".join(mname)], function_name))
# add edges based on class relationships
for n in exceptions:
if n.__base__ in exceptions:
exceptions.add_edge(n.__base__, n)
sorted_exceptions = nx.topological_sort(exceptions)
mod.add_to_init([
# register exception only if they can be raise from C++ world to
# Python world. Preprocessors variables are set only if deps
# analysis detect that this exception can be raised
Line('#ifdef PYTHONIC_BUILTIN_%s_HPP\n'
'boost::python::register_exception_translator<'
'pythonic::types::%s>(&pythonic::translate_%s);\n'
'#endif' % (n.__name__.upper(), n.__name__, n.__name__)
) for n in sorted_exceptions])
mod.add_to_init([
# make sure we get no nested parallelism that wreaks havoc in perf
Line('#ifdef _OPENMP\n'
'omp_set_max_active_levels(1);\n'
'#endif')])
for function_name, signatures in specs.iteritems():
internal_func_name = renamings.get(function_name,
function_name)
for sigid, signature in enumerate(signatures):
numbered_function_name = "{0}{1}".format(internal_func_name,
sigid)
arguments_types = [pytype_to_ctype(t) for t in signature]
has_arguments = HasArgument(internal_func_name).visit(ir)
arguments = ["a{0}".format(i)
for i in xrange(len(arguments_types))]
name_fmt = pythran_ward + "{0}::{1}::type{2}"
args_list = ", ".join(arguments_types)
specialized_fname = name_fmt.format(module_name,
internal_func_name,
"<{0}>".format(args_list)
if has_arguments else "")
result_type = ("typename std::remove_cv<"
"typename std::remove_reference"
"<typename {0}::result_type>::type"
">::type").format(specialized_fname)
mod.add_to_init(
[Statement("pythonic::python_to_pythran<{0}>()".format(t))
for t in _extract_all_constructed_types(signature)])
mod.add_to_init([Statement(
"pythonic::pythran_to_python<{0}>()".format(result_type))])
mod.add_function(
FunctionBody(
FunctionDeclaration(
Value(
result_type,
numbered_function_name),
[Value(t, a)
for t, a in zip(arguments_types, arguments)]),
Block([Statement("return {0}()({1})".format(
pythran_ward + '{0}::{1}'.format(
module_name, internal_func_name),
', '.join(arguments)))])
),
function_name
)
# call __init__() to execute top-level statements
init_call = '::'.join([pythran_ward + module_name, '__init__()()'])
mod.add_to_init([Statement(init_call)])
return mod