def check_ast(code, ref, optimizations):
"""
Check if a final node is the same as expected.
Parameters
----------
code : str
code we want to check after refine and optimizations
ref : str
The expected dump for the AST
optimizations : [optimization]
list of optimisation to apply
Raises
------
is_same : AssertionError
Raise if the result is not the one expected.
"""
pm = PassManager("testing")
ir, _, _, _ = frontend.parse(pm, code)
optimizations = map(_parse_optimization, optimizations)
refine(pm, ir, optimizations)
content = pm.dump(Python, ir)
if content != ref:
raise AssertionError(
"AST is not the one expected. Reference was %s,"
"but received %s" % (repr(ref), repr(content)))
def check_ast(code, ref, optimizations):
"""
Check if a final node is the same as expected.
Parameters
----------
code : str
code we want to check after refine and optimizations
ref : str
The expected dump for the AST
optimizations : [optimization]
list of optimisation to apply
Raises
------
is_same : AssertionError
Raise if the result is not the one expected.
"""
pm = PassManager("testing")
ir, _, _ = frontend.parse(pm, code)
optimizations = [_parse_optimization(opt) for opt in optimizations]
refine(pm, ir, optimizations)
content = pm.dump(Python, ir)
if content != ref:
raise AssertionError(
"AST is not the one expected. Reference was %s,"
"but received %s" % (repr(ref), repr(content)))
def front_middle_end(module_name, code, optimizations=None, module_dir=None):
"""Front-end and middle-end compilation steps"""
pm = PassManager(module_name, module_dir)
# front end
ir, renamings, docstrings = frontend.parse(pm, code)
# middle-end
if optimizations is None:
optimizations = cfg.get('pythran', 'optimizations').split()
optimizations = [_parse_optimization(opt) for opt in optimizations]
refine(pm, ir, optimizations)
return pm, ir, renamings, docstrings
def front_middle_end(module_name, code, optimizations=None, module_dir=None):
"""Front-end and middle-end compilation steps"""
pm = PassManager(module_name, module_dir)
# front end
ir, renamings, docstrings = frontend.parse(pm, code)
# middle-end
if optimizations is None:
optimizations = cfg.get('pythran', 'optimizations').split()
optimizations = [_parse_optimization(opt) for opt in optimizations]
refine(pm, ir, optimizations)
return pm, ir, renamings, docstrings
def front_middle_end(module_name, code, optimizations=None, module_dir=None,
entry_points=None):
"""Front-end and middle-end compilation steps"""
pm = PassManager(module_name, module_dir)
# front end
ir, docstrings = frontend.parse(pm, code)
if entry_points is not None:
ir = mark_unexported_functions(ir, entry_points)
# middle-end
if optimizations is None:
optimizations = cfg.get('pythran', 'optimizations').split()
optimizations = [_parse_optimization(opt) for opt in optimizations]
refine(pm, ir, optimizations)
return pm, ir, docstrings
def generate_cxx(module_name, code, specs=None, optimizations=None):
'''python + pythran spec -> c++ code
returns a PythonModule object
'''
if sys.version_info[0] == 3:
raise ValueError(
"Pythran does not fully support Python3, "
"it can only be used to compile C++ code "
"generated with the -E flag with a Python2 version of Pythran. "
"Sorry about this :-/")
pm = PassManager(module_name)
# front end
ir, renamings, docstrings = frontend.parse(pm, code)
# middle-end
optimizations = (optimizations
or cfg.get('pythran', 'optimizations').split())
optimizations = [_parse_optimization(opt) for opt in optimizations]
refine(pm, ir, optimizations)
# back-end
content = pm.dump(Cxx, ir)
# instanciate the meta program
if specs is None:
class Generable(object):
def __init__(self, content):
self.content = content
def __str__(self):
return str(self.content)
generate = __str__
mod = Generable(content)
else:
# uniform typing
for fname, signatures in specs.items():
if not isinstance(signatures, tuple):
specs[fname] = (signatures, )
# verify the pythran export are compatible with the code
specs = expand_specs(specs)
check_specs(ir, specs, renamings)
specs_to_docstrings(specs, docstrings)
metainfo = {
'hash': hashlib.sha256(code).hexdigest(),
'version': __version__,
'date': datetime.now()
}
mod = PythonModule(module_name, docstrings, metainfo)
mod.add_to_preamble(Define("BOOST_SIMD_NO_STRICT_ALIASING", "1"))
mod.add_to_includes(
Include("pythonic/core.hpp"),
Include("pythonic/python/core.hpp"),
# FIXME: only include these when needed
Include("pythonic/types/bool.hpp"),
Include("pythonic/types/int.hpp"),
Line("#ifdef _OPENMP\n#include <omp.h>\n#endif"))
mod.add_to_includes(
*[Include(inc) for inc in _extract_specs_dependencies(specs)])
mod.add_to_includes(*content.body)
for function_name, signatures in specs.iteritems():
internal_func_name = renamings.get(function_name, function_name)
# global variables are functions with no signatures :-)
if not signatures:
mod.add_global_var(
function_name, "{}()()".format(
pythran_ward +
'{0}::{1}'.format(module_name, internal_func_name)))
for sigid, signature in enumerate(signatures):
numbered_function_name = "{0}{1}".format(
internal_func_name, sigid)
arguments_types = [pytype_to_ctype(t) for t in signature]
has_arguments = HasArgument(internal_func_name).visit(ir)
arguments = [
"a{0}".format(i) for i in xrange(len(arguments_types))
]
name_fmt = pythran_ward + "{0}::{1}::type{2}"
args_list = ", ".join(arguments_types)
specialized_fname = name_fmt.format(
module_name, internal_func_name,
"<{0}>".format(args_list) if has_arguments else "")
result_type = "typename %s::result_type" % specialized_fname
mod.add_function(
FunctionBody(
FunctionDeclaration(
Value(result_type, numbered_function_name), [
Value(t, a)
for t, a in zip(arguments_types, arguments)
]),
Block([
Statement("return {0}()({1})".format(
pythran_ward + '{0}::{1}'.format(
module_name, internal_func_name),
', '.join(arguments)))
])), function_name, arguments_types)
return mod
def generate_cxx(module_name, code, specs=None, optimizations=None):
'''python + pythran spec -> c++ code
returns a BoostPythonModule object
'''
pm = PassManager(module_name)
# front end
ir, renamings = frontend.parse(pm, code)
# middle-end
optimizations = (optimizations or
cfg.get('pythran', 'optimizations').split())
optimizations = map(_parse_optimization, optimizations)
refine(pm, ir, optimizations)
# back-end
content = pm.dump(Cxx, ir)
# instanciate the meta program
if specs is None:
class Generable:
def __init__(self, content):
self.content = content
def __str__(self):
return str(self.content)
generate = __str__
mod = Generable(content)
else:
# uniform typing
for fname, signatures in specs.items():
if not isinstance(signatures, tuple):
specs[fname] = (signatures,)
# verify the pythran export are compatible with the code
specs = expand_specs(specs)
check_specs(ir, specs, renamings)
mod = BoostPythonModule(module_name)
mod.use_private_namespace = False
# very low value for max_arity leads to various bugs
min_val = 2
specs_max = [max(map(len, s)) for s in specs.itervalues()]
max_arity = max([min_val] + specs_max)
mod.add_to_preamble([Define("BOOST_PYTHON_MAX_ARITY", max_arity)])
mod.add_to_preamble([Define("BOOST_SIMD_NO_STRICT_ALIASING", "1")])
mod.add_to_preamble([Include("pythonic/core.hpp")])
mod.add_to_preamble([Include("pythonic/python/core.hpp")])
mod.add_to_preamble([Line("#ifdef _OPENMP\n#include <omp.h>\n#endif")])
mod.add_to_preamble(map(Include, _extract_specs_dependencies(specs)))
mod.add_to_preamble(content.body)
mod.add_to_init([
Line('#ifdef PYTHONIC_TYPES_NDARRAY_HPP\nimport_array()\n#endif')])
# topologically sorted exceptions based on the inheritance hierarchy.
# needed because otherwise boost python register_exception handlers
# do not catch exception type in the right way
# (first valid exception is selected)
# Inheritance has to be taken into account in the registration order.
exceptions = nx.DiGraph()
for function_name, v in functions.iteritems():
for mname, symbol in v:
if isinstance(symbol, ConstExceptionIntr):
exceptions.add_node(
getattr(sys.modules[".".join(mname)], function_name))
# add edges based on class relationships
for n in exceptions:
if n.__base__ in exceptions:
exceptions.add_edge(n.__base__, n)
sorted_exceptions = nx.topological_sort(exceptions)
mod.add_to_init([
# register exception only if they can be raise from C++ world to
# Python world. Preprocessors variables are set only if deps
# analysis detect that this exception can be raised
Line('#ifdef PYTHONIC_BUILTIN_%s_HPP\n'
'boost::python::register_exception_translator<'
'pythonic::types::%s>(&pythonic::translate_%s);\n'
'#endif' % (n.__name__.upper(), n.__name__, n.__name__)
) for n in sorted_exceptions])
mod.add_to_init([
# make sure we get no nested parallelism that wreaks havoc in perf
Line('#ifdef _OPENMP\n'
'omp_set_max_active_levels(1);\n'
'#endif')])
for function_name, signatures in specs.iteritems():
internal_func_name = renamings.get(function_name,
function_name)
for sigid, signature in enumerate(signatures):
numbered_function_name = "{0}{1}".format(internal_func_name,
sigid)
arguments_types = [pytype_to_ctype(t) for t in signature]
has_arguments = HasArgument(internal_func_name).visit(ir)
arguments = ["a{0}".format(i)
for i in xrange(len(arguments_types))]
name_fmt = pythran_ward + "{0}::{1}::type{2}"
args_list = ", ".join(arguments_types)
specialized_fname = name_fmt.format(module_name,
internal_func_name,
"<{0}>".format(args_list)
if has_arguments else "")
result_type = ("typename std::remove_cv<"
"typename std::remove_reference"
"<typename {0}::result_type>::type"
">::type").format(specialized_fname)
mod.add_to_init(
[Statement("pythonic::python_to_pythran<{0}>()".format(t))
for t in _extract_all_constructed_types(signature)])
mod.add_to_init([Statement(
"pythonic::pythran_to_python<{0}>()".format(result_type))])
mod.add_function(
FunctionBody(
FunctionDeclaration(
Value(
result_type,
numbered_function_name),
[Value(t, a)
for t, a in zip(arguments_types, arguments)]),
Block([Statement("return {0}()({1})".format(
pythran_ward + '{0}::{1}'.format(
module_name, internal_func_name),
', '.join(arguments)))])
),
function_name
)
# call __init__() to execute top-level statements
init_call = '::'.join([pythran_ward + module_name, '__init__()()'])
mod.add_to_init([Statement(init_call)])
return mod
def generate_cxx(module_name, code, specs=None, optimizations=None):
'''python + pythran spec -> c++ code
returns a PythonModule object
'''
if sys.version_info[0] == 3:
raise ValueError(
"Pythran does not fully support Python3, "
"it can only be used to compile C++ code "
"generated with the -E flag with a Python2 version of Pythran. "
"Sorry about this :-/")
pm = PassManager(module_name)
# front end
ir, renamings, docstrings = frontend.parse(pm, code)
# middle-end
optimizations = (optimizations or
cfg.get('pythran', 'optimizations').split())
optimizations = [_parse_optimization(opt) for opt in optimizations]
refine(pm, ir, optimizations)
# back-end
content = pm.dump(Cxx, ir)
# instanciate the meta program
if specs is None:
class Generable(object):
def __init__(self, content):
self.content = content
def __str__(self):
return str(self.content)
generate = __str__
mod = Generable(content)
else:
# uniform typing
for fname, signatures in specs.items():
if not isinstance(signatures, tuple):
specs[fname] = (signatures,)
# verify the pythran export are compatible with the code
specs = expand_specs(specs)
check_specs(ir, specs, renamings)
specs_to_docstrings(specs, docstrings)
metainfo = {'hash': hashlib.sha256(code).hexdigest(),
'version': __version__,
'date': datetime.now()}
mod = PythonModule(module_name, docstrings, metainfo)
mod.add_to_preamble(Define("BOOST_SIMD_NO_STRICT_ALIASING", "1"))
mod.add_to_includes(Include("pythonic/core.hpp"),
Include("pythonic/python/core.hpp"),
# FIXME: only include these when needed
Include("pythonic/types/bool.hpp"),
Include("pythonic/types/int.hpp"),
Line("#ifdef _OPENMP\n#include <omp.h>\n#endif")
)
mod.add_to_includes(*[Include(inc) for inc in
_extract_specs_dependencies(specs)])
mod.add_to_includes(*content.body)
for function_name, signatures in specs.iteritems():
internal_func_name = renamings.get(function_name,
function_name)
# global variables are functions with no signatures :-)
if not signatures:
mod.add_global_var(function_name,
"{}()()".format(
pythran_ward + '{0}::{1}'.format(
module_name, internal_func_name)))
for sigid, signature in enumerate(signatures):
numbered_function_name = "{0}{1}".format(internal_func_name,
sigid)
arguments_types = [pytype_to_ctype(t) for t in signature]
has_arguments = HasArgument(internal_func_name).visit(ir)
arguments = ["a{0}".format(i)
for i in xrange(len(arguments_types))]
name_fmt = pythran_ward + "{0}::{1}::type{2}"
args_list = ", ".join(arguments_types)
specialized_fname = name_fmt.format(module_name,
internal_func_name,
"<{0}>".format(args_list)
if has_arguments else "")
result_type = "typename %s::result_type" % specialized_fname
mod.add_function(
FunctionBody(
FunctionDeclaration(
Value(
result_type,
numbered_function_name),
[Value(t, a)
for t, a in zip(arguments_types, arguments)]),
Block([Statement("return {0}()({1})".format(
pythran_ward + '{0}::{1}'.format(
module_name, internal_func_name),
', '.join(arguments)))])
),
function_name,
arguments_types
)
return mod
def generate_cxx(module_name, code, specs=None, optimizations=None):
'''python + pythran spec -> c++ code
returns a PythonModule object
'''
pm = PassManager(module_name)
# front end
ir, renamings, docstrings = frontend.parse(pm, code)
# middle-end
optimizations = (optimizations or
cfg.get('pythran', 'optimizations').split())
optimizations = [_parse_optimization(opt) for opt in optimizations]
refine(pm, ir, optimizations)
# type check
types = tog.typecheck(ir)
# back-end
content = pm.dump(Cxx, ir)
# instantiate the meta program
if specs is None:
class Generable(object):
def __init__(self, content):
self.content = content
def __str__(self):
return str(self.content)
generate = __str__
mod = Generable(content)
else:
# uniform typing
for fname, signatures in list(specs.items()):
if not isinstance(signatures, tuple):
specs[fname] = (signatures,)
# verify the pythran export are compatible with the code
specs = expand_specs(specs)
check_specs(ir, specs, renamings, types)
specs_to_docstrings(specs, docstrings)
if isinstance(code, bytes):
code_bytes = code
else:
code_bytes = code.encode('ascii', 'ignore')
metainfo = {'hash': hashlib.sha256(code_bytes).hexdigest(),
'version': __version__,
'date': datetime.now()}
mod = PythonModule(module_name, docstrings, metainfo)
mod.add_to_preamble(Define("BOOST_SIMD_NO_STRICT_ALIASING", "1"))
mod.add_to_includes(
Include("pythonic/core.hpp"),
Include("pythonic/python/core.hpp"),
# FIXME: only include these when needed
Include("pythonic/types/bool.hpp"),
Include("pythonic/types/int.hpp"),
Line("#ifdef _OPENMP\n#include <omp.h>\n#endif")
)
mod.add_to_includes(*[Include(inc) for inc in
_extract_specs_dependencies(specs)])
mod.add_to_includes(*content.body)
mod.add_to_includes(
Include("pythonic/python/exception_handler.hpp"),
)
for function_name, signatures in specs.items():
internal_func_name = renamings.get(function_name,
function_name)
# global variables are functions with no signatures :-)
if not signatures:
mod.add_global_var(function_name,
"{}()()".format(
pythran_ward + '{0}::{1}'.format(
module_name, internal_func_name)))
for sigid, signature in enumerate(signatures):
numbered_function_name = "{0}{1}".format(internal_func_name,
sigid)
arguments_types = [pytype_to_ctype(t) for t in signature]
has_arguments = HasArgument(internal_func_name).visit(ir)
arguments = ["a{0}".format(i)
for i in range(len(arguments_types))]
name_fmt = pythran_ward + "{0}::{1}::type{2}"
args_list = ", ".join(arguments_types)
specialized_fname = name_fmt.format(module_name,
internal_func_name,
"<{0}>".format(args_list)
if has_arguments else "")
result_type = "typename %s::result_type" % specialized_fname
mod.add_function(
FunctionBody(
FunctionDeclaration(
Value(
result_type,
numbered_function_name),
[Value(t, a)
for t, a in zip(arguments_types, arguments)]),
Block([Statement("""
PyThreadState *_save = PyEval_SaveThread();
try {{
auto res = {0}()({1});
PyEval_RestoreThread(_save);
return res;
}}
catch(...) {{
PyEval_RestoreThread(_save);
throw;
}}
""".format(
pythran_ward + '{0}::{1}'.format(
module_name, internal_func_name),
', '.join(arguments)))])
),
function_name,
arguments_types
)
return mod
def generate_cxx(module_name, code, specs=None, optimizations=None):
'''python + pythran spec -> c++ code
returns a BoostPythonModule object
'''
pm = PassManager(module_name)
# front end
ir, renamings = frontend.parse(pm, code)
# middle-end
optimizations = (optimizations or
cfg.get('pythran', 'optimizations').split())
optimizations = map(_parse_optimization, optimizations)
refine(pm, ir, optimizations)
# back-end
content = pm.dump(Cxx, ir)
# instanciate the meta program
if specs is None:
class Generable:
def __init__(self, content):
self.content = content
def __str__(self):
return str(self.content)
generate = __str__
mod = Generable(content)
else:
# uniform typing
for fname, signatures in specs.items():
if not isinstance(signatures, tuple):
specs[fname] = (signatures,)
# verify the pythran export are compatible with the code
specs = expand_specs(specs)
check_specs(ir, specs, renamings)
mod = BoostPythonModule(module_name)
mod.use_private_namespace = False
# very low value for max_arity leads to various bugs
min_val = 2
specs_max = [max(map(len, s)) for s in specs.itervalues()]
max_arity = max([min_val] + specs_max)
mod.add_to_preamble([Define("BOOST_PYTHON_MAX_ARITY", max_arity)])
mod.add_to_preamble([Define("BOOST_SIMD_NO_STRICT_ALIASING", "1")])
mod.add_to_preamble([Include("pythonic/core.hpp")])
mod.add_to_preamble([Include("pythonic/python/core.hpp")])
mod.add_to_preamble([Line("#ifdef _OPENMP\n#include <omp.h>\n#endif")])
mod.add_to_preamble(map(Include, _extract_specs_dependencies(specs)))
mod.add_to_preamble(content.body)
mod.add_to_init([
Line('#ifdef PYTHONIC_TYPES_NDARRAY_HPP\nimport_array()\n#endif')])
# topologically sorted exceptions based on the inheritance hierarchy.
# needed because otherwise boost python register_exception handlers
# do not catch exception type in the right way
# (first valid exception is selected)
# Inheritance has to be taken into account in the registration order.
exceptions = nx.DiGraph()
for function_name, v in functions.iteritems():
for mname, symbol in v:
if isinstance(symbol, ConstExceptionIntr):
exceptions.add_node(
getattr(sys.modules[".".join(mname)], function_name))
# add edges based on class relationships
for n in exceptions:
if n.__base__ in exceptions:
exceptions.add_edge(n.__base__, n)
sorted_exceptions = nx.topological_sort(exceptions)
mod.add_to_init([
# register exception only if they can be raise from C++ world to
# Python world. Preprocessors variables are set only if deps
# analysis detect that this exception can be raised
Line('#ifdef PYTHONIC_BUILTIN_%s_HPP\n'
'boost::python::register_exception_translator<'
'pythonic::types::%s>(&pythonic::translate_%s);\n'
'#endif' % (n.__name__.upper(), n.__name__, n.__name__)
) for n in sorted_exceptions])
mod.add_to_init([
# make sure we get no nested parallelism that wreaks havoc in perf
Line('#ifdef _OPENMP\n'
'omp_set_max_active_levels(1);\n'
'#endif')])
for function_name, signatures in specs.iteritems():
internal_func_name = renamings.get(function_name,
function_name)
for sigid, signature in enumerate(signatures):
numbered_function_name = "{0}{1}".format(internal_func_name,
sigid)
arguments_types = [pytype_to_ctype(t) for t in signature]
has_arguments = HasArgument(internal_func_name).visit(ir)
arguments = ["a{0}".format(i)
for i in xrange(len(arguments_types))]
name_fmt = pythran_ward + "{0}::{1}::type{2}"
args_list = ", ".join(arguments_types)
specialized_fname = name_fmt.format(module_name,
internal_func_name,
"<{0}>".format(args_list)
if has_arguments else "")
result_type = ("typename std::remove_cv<"
"typename std::remove_reference"
"<typename {0}::result_type>::type"
">::type").format(specialized_fname)
mod.add_to_init(
[Statement("pythonic::python_to_pythran<{0}>()".format(t))
for t in _extract_all_constructed_types(signature)])
mod.add_to_init([Statement(
"pythonic::pythran_to_python<{0}>()".format(result_type))])
mod.add_function(
FunctionBody(
FunctionDeclaration(
Value(
result_type,
numbered_function_name),
[Value(t, a)
for t, a in zip(arguments_types, arguments)]),
Block([Statement("return {0}()({1})".format(
pythran_ward + '{0}::{1}'.format(
module_name, internal_func_name),
', '.join(arguments)))])
),
function_name
)
# call __init__() to execute top-level statements
init_call = '::'.join([pythran_ward + module_name, '__init__()()'])
mod.add_to_init([Statement(init_call)])
return mod