Esempio n. 1
0
    def check_ast(code, ref, optimizations):
        """
        Check if a final node is the same as expected.

        Parameters
        ----------
        code : str
            code we want to check after refine and optimizations
        ref : str
            The expected dump for the AST
        optimizations : [optimization]
            list of optimisation to apply

        Raises
        ------
        is_same : AssertionError
            Raise if the result is not the one expected.
        """
        pm = PassManager("testing")

        ir, _, _, _ = frontend.parse(pm, code)

        optimizations = map(_parse_optimization, optimizations)
        refine(pm, ir, optimizations)

        content = pm.dump(Python, ir)

        if content != ref:
            raise AssertionError(
                "AST is not the one expected. Reference was %s,"
                "but received %s" % (repr(ref), repr(content)))
Esempio n. 2
0
    def check_ast(code, ref, optimizations):
        """
        Check if a final node is the same as expected.

        Parameters
        ----------
        code : str
            code we want to check after refine and optimizations
        ref : str
            The expected dump for the AST
        optimizations : [optimization]
            list of optimisation to apply

        Raises
        ------
        is_same : AssertionError
            Raise if the result is not the one expected.
        """
        pm = PassManager("testing")

        ir, _, _ = frontend.parse(pm, code)

        optimizations = [_parse_optimization(opt) for opt in optimizations]
        refine(pm, ir, optimizations)

        content = pm.dump(Python, ir)

        if content != ref:
            raise AssertionError(
                "AST is not the one expected. Reference was %s,"
                "but received %s" % (repr(ref), repr(content)))
Esempio n. 3
0
def front_middle_end(module_name, code, optimizations=None, module_dir=None):
    """Front-end and middle-end compilation steps"""
    pm = PassManager(module_name, module_dir)

    # front end
    ir, renamings, docstrings = frontend.parse(pm, code)

    # middle-end
    if optimizations is None:
        optimizations = cfg.get('pythran', 'optimizations').split()
    optimizations = [_parse_optimization(opt) for opt in optimizations]
    refine(pm, ir, optimizations)

    return pm, ir, renamings, docstrings
Esempio n. 4
0
def front_middle_end(module_name, code, optimizations=None, module_dir=None):
    """Front-end and middle-end compilation steps"""
    pm = PassManager(module_name, module_dir)

    # front end
    ir, renamings, docstrings = frontend.parse(pm, code)

    # middle-end
    if optimizations is None:
        optimizations = cfg.get('pythran', 'optimizations').split()
    optimizations = [_parse_optimization(opt) for opt in optimizations]
    refine(pm, ir, optimizations)

    return pm, ir, renamings, docstrings
Esempio n. 5
0
def front_middle_end(module_name, code, optimizations=None, module_dir=None,
                     entry_points=None):
    """Front-end and middle-end compilation steps"""

    pm = PassManager(module_name, module_dir)

    # front end
    ir, docstrings = frontend.parse(pm, code)

    if entry_points is not None:
        ir = mark_unexported_functions(ir, entry_points)

    # middle-end
    if optimizations is None:
        optimizations = cfg.get('pythran', 'optimizations').split()
    optimizations = [_parse_optimization(opt) for opt in optimizations]
    refine(pm, ir, optimizations)

    return pm, ir, docstrings
Esempio n. 6
0
def generate_cxx(module_name, code, specs=None, optimizations=None):
    '''python + pythran spec -> c++ code
    returns a PythonModule object

    '''
    if sys.version_info[0] == 3:
        raise ValueError(
            "Pythran does not fully support Python3, "
            "it can only be used to compile C++ code "
            "generated with the -E flag with a Python2 version of Pythran. "
            "Sorry about this :-/")

    pm = PassManager(module_name)

    # front end
    ir, renamings, docstrings = frontend.parse(pm, code)

    # middle-end
    optimizations = (optimizations
                     or cfg.get('pythran', 'optimizations').split())
    optimizations = [_parse_optimization(opt) for opt in optimizations]
    refine(pm, ir, optimizations)

    # back-end
    content = pm.dump(Cxx, ir)

    # instanciate the meta program
    if specs is None:

        class Generable(object):
            def __init__(self, content):
                self.content = content

            def __str__(self):
                return str(self.content)

            generate = __str__

        mod = Generable(content)
    else:

        # uniform typing
        for fname, signatures in specs.items():
            if not isinstance(signatures, tuple):
                specs[fname] = (signatures, )

        # verify the pythran export are compatible with the code
        specs = expand_specs(specs)
        check_specs(ir, specs, renamings)
        specs_to_docstrings(specs, docstrings)

        metainfo = {
            'hash': hashlib.sha256(code).hexdigest(),
            'version': __version__,
            'date': datetime.now()
        }

        mod = PythonModule(module_name, docstrings, metainfo)
        mod.add_to_preamble(Define("BOOST_SIMD_NO_STRICT_ALIASING", "1"))
        mod.add_to_includes(
            Include("pythonic/core.hpp"),
            Include("pythonic/python/core.hpp"),
            # FIXME: only include these when needed
            Include("pythonic/types/bool.hpp"),
            Include("pythonic/types/int.hpp"),
            Line("#ifdef _OPENMP\n#include <omp.h>\n#endif"))
        mod.add_to_includes(
            *[Include(inc) for inc in _extract_specs_dependencies(specs)])
        mod.add_to_includes(*content.body)

        for function_name, signatures in specs.iteritems():
            internal_func_name = renamings.get(function_name, function_name)
            # global variables are functions with no signatures :-)
            if not signatures:
                mod.add_global_var(
                    function_name, "{}()()".format(
                        pythran_ward +
                        '{0}::{1}'.format(module_name, internal_func_name)))

            for sigid, signature in enumerate(signatures):
                numbered_function_name = "{0}{1}".format(
                    internal_func_name, sigid)
                arguments_types = [pytype_to_ctype(t) for t in signature]
                has_arguments = HasArgument(internal_func_name).visit(ir)
                arguments = [
                    "a{0}".format(i) for i in xrange(len(arguments_types))
                ]
                name_fmt = pythran_ward + "{0}::{1}::type{2}"
                args_list = ", ".join(arguments_types)
                specialized_fname = name_fmt.format(
                    module_name, internal_func_name,
                    "<{0}>".format(args_list) if has_arguments else "")
                result_type = "typename %s::result_type" % specialized_fname
                mod.add_function(
                    FunctionBody(
                        FunctionDeclaration(
                            Value(result_type, numbered_function_name), [
                                Value(t, a)
                                for t, a in zip(arguments_types, arguments)
                            ]),
                        Block([
                            Statement("return {0}()({1})".format(
                                pythran_ward + '{0}::{1}'.format(
                                    module_name, internal_func_name),
                                ', '.join(arguments)))
                        ])), function_name, arguments_types)
    return mod
Esempio n. 7
0
def generate_cxx(module_name, code, specs=None, optimizations=None):
    '''python + pythran spec -> c++ code
    returns a BoostPythonModule object

    '''
    pm = PassManager(module_name)

    # front end
    ir, renamings = frontend.parse(pm, code)

    # middle-end
    optimizations = (optimizations or
                     cfg.get('pythran', 'optimizations').split())
    optimizations = map(_parse_optimization, optimizations)
    refine(pm, ir, optimizations)

    # back-end
    content = pm.dump(Cxx, ir)

    # instanciate the meta program
    if specs is None:

        class Generable:
            def __init__(self, content):
                self.content = content

            def __str__(self):
                return str(self.content)

            generate = __str__

        mod = Generable(content)
    else:

        # uniform typing
        for fname, signatures in specs.items():
            if not isinstance(signatures, tuple):
                specs[fname] = (signatures,)

        # verify the pythran export are compatible with the code
        specs = expand_specs(specs)
        check_specs(ir, specs, renamings)

        mod = BoostPythonModule(module_name)
        mod.use_private_namespace = False
        # very low value for max_arity leads to various bugs
        min_val = 2
        specs_max = [max(map(len, s)) for s in specs.itervalues()]
        max_arity = max([min_val] + specs_max)
        mod.add_to_preamble([Define("BOOST_PYTHON_MAX_ARITY", max_arity)])
        mod.add_to_preamble([Define("BOOST_SIMD_NO_STRICT_ALIASING", "1")])
        mod.add_to_preamble([Include("pythonic/core.hpp")])
        mod.add_to_preamble([Include("pythonic/python/core.hpp")])
        mod.add_to_preamble([Line("#ifdef _OPENMP\n#include <omp.h>\n#endif")])
        mod.add_to_preamble(map(Include, _extract_specs_dependencies(specs)))
        mod.add_to_preamble(content.body)
        mod.add_to_init([
            Line('#ifdef PYTHONIC_TYPES_NDARRAY_HPP\nimport_array()\n#endif')])

        # topologically sorted exceptions based on the inheritance hierarchy.
        # needed because otherwise boost python register_exception handlers
        # do not catch exception type in the right way
        # (first valid exception is selected)
        # Inheritance has to be taken into account in the registration order.
        exceptions = nx.DiGraph()
        for function_name, v in functions.iteritems():
            for mname, symbol in v:
                if isinstance(symbol, ConstExceptionIntr):
                    exceptions.add_node(
                        getattr(sys.modules[".".join(mname)], function_name))

        # add edges based on class relationships
        for n in exceptions:
            if n.__base__ in exceptions:
                exceptions.add_edge(n.__base__, n)

        sorted_exceptions = nx.topological_sort(exceptions)
        mod.add_to_init([
            # register exception only if they can be raise from C++ world to
            # Python world. Preprocessors variables are set only if deps
            # analysis detect that this exception can be raised
            Line('#ifdef PYTHONIC_BUILTIN_%s_HPP\n'
                 'boost::python::register_exception_translator<'
                 'pythonic::types::%s>(&pythonic::translate_%s);\n'
                 '#endif' % (n.__name__.upper(), n.__name__, n.__name__)
                 ) for n in sorted_exceptions])

        mod.add_to_init([
            # make sure we get no nested parallelism that wreaks havoc in perf
            Line('#ifdef _OPENMP\n'
                 'omp_set_max_active_levels(1);\n'
                 '#endif')])

        for function_name, signatures in specs.iteritems():
            internal_func_name = renamings.get(function_name,
                                               function_name)
            for sigid, signature in enumerate(signatures):
                numbered_function_name = "{0}{1}".format(internal_func_name,
                                                         sigid)
                arguments_types = [pytype_to_ctype(t) for t in signature]
                has_arguments = HasArgument(internal_func_name).visit(ir)
                arguments = ["a{0}".format(i)
                             for i in xrange(len(arguments_types))]
                name_fmt = pythran_ward + "{0}::{1}::type{2}"
                args_list = ", ".join(arguments_types)
                specialized_fname = name_fmt.format(module_name,
                                                    internal_func_name,
                                                    "<{0}>".format(args_list)
                                                    if has_arguments else "")
                result_type = ("typename std::remove_cv<"
                               "typename std::remove_reference"
                               "<typename {0}::result_type>::type"
                               ">::type").format(specialized_fname)
                mod.add_to_init(
                    [Statement("pythonic::python_to_pythran<{0}>()".format(t))
                     for t in _extract_all_constructed_types(signature)])
                mod.add_to_init([Statement(
                    "pythonic::pythran_to_python<{0}>()".format(result_type))])
                mod.add_function(
                    FunctionBody(
                        FunctionDeclaration(
                            Value(
                                result_type,
                                numbered_function_name),
                            [Value(t, a)
                             for t, a in zip(arguments_types, arguments)]),
                        Block([Statement("return {0}()({1})".format(
                            pythran_ward + '{0}::{1}'.format(
                                module_name, internal_func_name),
                            ', '.join(arguments)))])
                    ),
                    function_name
                )
        # call __init__() to execute top-level statements
        init_call = '::'.join([pythran_ward + module_name, '__init__()()'])
        mod.add_to_init([Statement(init_call)])
    return mod
Esempio n. 8
0
def generate_cxx(module_name, code, specs=None, optimizations=None):
    '''python + pythran spec -> c++ code
    returns a PythonModule object

    '''
    if sys.version_info[0] == 3:
        raise ValueError(
            "Pythran does not fully support Python3, "
            "it can only be used to compile C++ code "
            "generated with the -E flag with a Python2 version of Pythran. "
            "Sorry about this :-/")

    pm = PassManager(module_name)

    # front end
    ir, renamings, docstrings = frontend.parse(pm, code)

    # middle-end
    optimizations = (optimizations or
                     cfg.get('pythran', 'optimizations').split())
    optimizations = [_parse_optimization(opt) for opt in optimizations]
    refine(pm, ir, optimizations)

    # back-end
    content = pm.dump(Cxx, ir)

    # instanciate the meta program
    if specs is None:

        class Generable(object):
            def __init__(self, content):
                self.content = content

            def __str__(self):
                return str(self.content)

            generate = __str__

        mod = Generable(content)
    else:

        # uniform typing
        for fname, signatures in specs.items():
            if not isinstance(signatures, tuple):
                specs[fname] = (signatures,)

        # verify the pythran export are compatible with the code
        specs = expand_specs(specs)
        check_specs(ir, specs, renamings)
        specs_to_docstrings(specs, docstrings)

        metainfo = {'hash': hashlib.sha256(code).hexdigest(),
                    'version': __version__,
                    'date': datetime.now()}

        mod = PythonModule(module_name, docstrings, metainfo)
        mod.add_to_preamble(Define("BOOST_SIMD_NO_STRICT_ALIASING", "1"))
        mod.add_to_includes(Include("pythonic/core.hpp"),
                            Include("pythonic/python/core.hpp"),
                            # FIXME: only include these when needed
                            Include("pythonic/types/bool.hpp"),
                            Include("pythonic/types/int.hpp"),
                            Line("#ifdef _OPENMP\n#include <omp.h>\n#endif")
                            )
        mod.add_to_includes(*[Include(inc) for inc in
                              _extract_specs_dependencies(specs)])
        mod.add_to_includes(*content.body)

        for function_name, signatures in specs.iteritems():
            internal_func_name = renamings.get(function_name,
                                               function_name)
            # global variables are functions with no signatures :-)
            if not signatures:
                mod.add_global_var(function_name,
                                   "{}()()".format(
                                       pythran_ward + '{0}::{1}'.format(
                                           module_name, internal_func_name)))

            for sigid, signature in enumerate(signatures):
                numbered_function_name = "{0}{1}".format(internal_func_name,
                                                         sigid)
                arguments_types = [pytype_to_ctype(t) for t in signature]
                has_arguments = HasArgument(internal_func_name).visit(ir)
                arguments = ["a{0}".format(i)
                             for i in xrange(len(arguments_types))]
                name_fmt = pythran_ward + "{0}::{1}::type{2}"
                args_list = ", ".join(arguments_types)
                specialized_fname = name_fmt.format(module_name,
                                                    internal_func_name,
                                                    "<{0}>".format(args_list)
                                                    if has_arguments else "")
                result_type = "typename %s::result_type" % specialized_fname
                mod.add_function(
                    FunctionBody(
                        FunctionDeclaration(
                            Value(
                                result_type,
                                numbered_function_name),
                            [Value(t, a)
                             for t, a in zip(arguments_types, arguments)]),
                        Block([Statement("return {0}()({1})".format(
                            pythran_ward + '{0}::{1}'.format(
                                module_name, internal_func_name),
                            ', '.join(arguments)))])
                    ),
                    function_name,
                    arguments_types
                )
    return mod
Esempio n. 9
0
def generate_cxx(module_name, code, specs=None, optimizations=None):
    '''python + pythran spec -> c++ code
    returns a PythonModule object

    '''

    pm = PassManager(module_name)

    # front end
    ir, renamings, docstrings = frontend.parse(pm, code)

    # middle-end
    optimizations = (optimizations or
                     cfg.get('pythran', 'optimizations').split())
    optimizations = [_parse_optimization(opt) for opt in optimizations]
    refine(pm, ir, optimizations)

    # type check
    types = tog.typecheck(ir)

    # back-end
    content = pm.dump(Cxx, ir)

    # instantiate the meta program
    if specs is None:

        class Generable(object):
            def __init__(self, content):
                self.content = content

            def __str__(self):
                return str(self.content)

            generate = __str__

        mod = Generable(content)
    else:

        # uniform typing
        for fname, signatures in list(specs.items()):
            if not isinstance(signatures, tuple):
                specs[fname] = (signatures,)

        # verify the pythran export are compatible with the code
        specs = expand_specs(specs)
        check_specs(ir, specs, renamings, types)
        specs_to_docstrings(specs, docstrings)

        if isinstance(code, bytes):
            code_bytes = code
        else:
            code_bytes = code.encode('ascii', 'ignore')
        metainfo = {'hash': hashlib.sha256(code_bytes).hexdigest(),
                    'version': __version__,
                    'date': datetime.now()}

        mod = PythonModule(module_name, docstrings, metainfo)
        mod.add_to_preamble(Define("BOOST_SIMD_NO_STRICT_ALIASING", "1"))
        mod.add_to_includes(
            Include("pythonic/core.hpp"),
            Include("pythonic/python/core.hpp"),
            # FIXME: only include these when needed
            Include("pythonic/types/bool.hpp"),
            Include("pythonic/types/int.hpp"),
            Line("#ifdef _OPENMP\n#include <omp.h>\n#endif")
        )
        mod.add_to_includes(*[Include(inc) for inc in
                              _extract_specs_dependencies(specs)])
        mod.add_to_includes(*content.body)
        mod.add_to_includes(
            Include("pythonic/python/exception_handler.hpp"),
        )

        for function_name, signatures in specs.items():
            internal_func_name = renamings.get(function_name,
                                               function_name)
            # global variables are functions with no signatures :-)
            if not signatures:
                mod.add_global_var(function_name,
                                   "{}()()".format(
                                       pythran_ward + '{0}::{1}'.format(
                                           module_name, internal_func_name)))

            for sigid, signature in enumerate(signatures):
                numbered_function_name = "{0}{1}".format(internal_func_name,
                                                         sigid)
                arguments_types = [pytype_to_ctype(t) for t in signature]
                has_arguments = HasArgument(internal_func_name).visit(ir)
                arguments = ["a{0}".format(i)
                             for i in range(len(arguments_types))]
                name_fmt = pythran_ward + "{0}::{1}::type{2}"
                args_list = ", ".join(arguments_types)
                specialized_fname = name_fmt.format(module_name,
                                                    internal_func_name,
                                                    "<{0}>".format(args_list)
                                                    if has_arguments else "")
                result_type = "typename %s::result_type" % specialized_fname
                mod.add_function(
                    FunctionBody(
                        FunctionDeclaration(
                            Value(
                                result_type,
                                numbered_function_name),
                            [Value(t, a)
                             for t, a in zip(arguments_types, arguments)]),
                        Block([Statement("""
                            PyThreadState *_save = PyEval_SaveThread();
                            try {{
                                auto res = {0}()({1});
                                PyEval_RestoreThread(_save);
                                return res;
                            }}
                            catch(...) {{
                                PyEval_RestoreThread(_save);
                                throw;
                            }}
                            """.format(
                            pythran_ward + '{0}::{1}'.format(
                                module_name, internal_func_name),
                            ', '.join(arguments)))])
                    ),
                    function_name,
                    arguments_types
                )
    return mod
Esempio n. 10
0
def generate_cxx(module_name, code, specs=None, optimizations=None):
    '''python + pythran spec -> c++ code
    returns a BoostPythonModule object

    '''
    pm = PassManager(module_name)

    # front end
    ir, renamings = frontend.parse(pm, code)

    # middle-end
    optimizations = (optimizations or
                     cfg.get('pythran', 'optimizations').split())
    optimizations = map(_parse_optimization, optimizations)
    refine(pm, ir, optimizations)

    # back-end
    content = pm.dump(Cxx, ir)

    # instanciate the meta program
    if specs is None:

        class Generable:
            def __init__(self, content):
                self.content = content

            def __str__(self):
                return str(self.content)

            generate = __str__

        mod = Generable(content)
    else:

        # uniform typing
        for fname, signatures in specs.items():
            if not isinstance(signatures, tuple):
                specs[fname] = (signatures,)

        # verify the pythran export are compatible with the code
        specs = expand_specs(specs)
        check_specs(ir, specs, renamings)

        mod = BoostPythonModule(module_name)
        mod.use_private_namespace = False
        # very low value for max_arity leads to various bugs
        min_val = 2
        specs_max = [max(map(len, s)) for s in specs.itervalues()]
        max_arity = max([min_val] + specs_max)
        mod.add_to_preamble([Define("BOOST_PYTHON_MAX_ARITY", max_arity)])
        mod.add_to_preamble([Define("BOOST_SIMD_NO_STRICT_ALIASING", "1")])
        mod.add_to_preamble([Include("pythonic/core.hpp")])
        mod.add_to_preamble([Include("pythonic/python/core.hpp")])
        mod.add_to_preamble([Line("#ifdef _OPENMP\n#include <omp.h>\n#endif")])
        mod.add_to_preamble(map(Include, _extract_specs_dependencies(specs)))
        mod.add_to_preamble(content.body)
        mod.add_to_init([
            Line('#ifdef PYTHONIC_TYPES_NDARRAY_HPP\nimport_array()\n#endif')])

        # topologically sorted exceptions based on the inheritance hierarchy.
        # needed because otherwise boost python register_exception handlers
        # do not catch exception type in the right way
        # (first valid exception is selected)
        # Inheritance has to be taken into account in the registration order.
        exceptions = nx.DiGraph()
        for function_name, v in functions.iteritems():
            for mname, symbol in v:
                if isinstance(symbol, ConstExceptionIntr):
                    exceptions.add_node(
                        getattr(sys.modules[".".join(mname)], function_name))

        # add edges based on class relationships
        for n in exceptions:
            if n.__base__ in exceptions:
                exceptions.add_edge(n.__base__, n)

        sorted_exceptions = nx.topological_sort(exceptions)
        mod.add_to_init([
            # register exception only if they can be raise from C++ world to
            # Python world. Preprocessors variables are set only if deps
            # analysis detect that this exception can be raised
            Line('#ifdef PYTHONIC_BUILTIN_%s_HPP\n'
                 'boost::python::register_exception_translator<'
                 'pythonic::types::%s>(&pythonic::translate_%s);\n'
                 '#endif' % (n.__name__.upper(), n.__name__, n.__name__)
                 ) for n in sorted_exceptions])

        mod.add_to_init([
            # make sure we get no nested parallelism that wreaks havoc in perf
            Line('#ifdef _OPENMP\n'
                 'omp_set_max_active_levels(1);\n'
                 '#endif')])

        for function_name, signatures in specs.iteritems():
            internal_func_name = renamings.get(function_name,
                                               function_name)
            for sigid, signature in enumerate(signatures):
                numbered_function_name = "{0}{1}".format(internal_func_name,
                                                         sigid)
                arguments_types = [pytype_to_ctype(t) for t in signature]
                has_arguments = HasArgument(internal_func_name).visit(ir)
                arguments = ["a{0}".format(i)
                             for i in xrange(len(arguments_types))]
                name_fmt = pythran_ward + "{0}::{1}::type{2}"
                args_list = ", ".join(arguments_types)
                specialized_fname = name_fmt.format(module_name,
                                                    internal_func_name,
                                                    "<{0}>".format(args_list)
                                                    if has_arguments else "")
                result_type = ("typename std::remove_cv<"
                               "typename std::remove_reference"
                               "<typename {0}::result_type>::type"
                               ">::type").format(specialized_fname)
                mod.add_to_init(
                    [Statement("pythonic::python_to_pythran<{0}>()".format(t))
                     for t in _extract_all_constructed_types(signature)])
                mod.add_to_init([Statement(
                    "pythonic::pythran_to_python<{0}>()".format(result_type))])
                mod.add_function(
                    FunctionBody(
                        FunctionDeclaration(
                            Value(
                                result_type,
                                numbered_function_name),
                            [Value(t, a)
                             for t, a in zip(arguments_types, arguments)]),
                        Block([Statement("return {0}()({1})".format(
                            pythran_ward + '{0}::{1}'.format(
                                module_name, internal_func_name),
                            ', '.join(arguments)))])
                    ),
                    function_name
                )
        # call __init__() to execute top-level statements
        init_call = '::'.join([pythran_ward + module_name, '__init__()()'])
        mod.add_to_init([Statement(init_call)])
    return mod