Пример #1
0
    def get_module_group_prefix(self, module_name: str) -> str:
        """Get the group prefix for a module (relative to the current group).

        The prefix should be prepended to the object name whenever
        accessing an object from this module.

        If the module lives is in the current compilation group, there is
        no prefix.  But if it lives in a different group (and hence a separate
        extension module), we need to access objects from it indirectly via an
        export table.

        For example, for code in group `a` to call a function `bar` in group `b`,
        it would need to do `exports_b.CPyDef_bar(...)`, while code that is
        also in group `b` can simply do `CPyDef_bar(...)`.

        Thus the prefix for a module in group `b` is 'exports_b.' if the current
        group is *not* b and just '' if it is.
        """
        groups = self.context.group_map
        target_group_name = groups.get(module_name)
        if target_group_name and target_group_name != self.context.group_name:
            self.context.group_deps.add(target_group_name)
            return 'exports_{}.'.format(exported_name(target_group_name))
        else:
            return ''
Пример #2
0
def generate_c_extension_shim(full_module_name: str, module_name: str,
                              dir_name: str, group_name: str) -> str:
    """Create a C extension shim with a passthrough PyInit function.

    Arguments:
        full_module_name: the dotted full module name
        module_name: the final component of the module name
        dir_name: the directory to place source code
        group_name: the name of the group
    """
    cname = '%s.c' % full_module_name.replace('.', os.sep)
    cpath = os.path.join(dir_name, cname)

    # We load the C extension shim template from a file.
    # (So that the file could be reused as a bazel template also.)
    with open(os.path.join(include_dir(), 'module_shim.tmpl')) as f:
        shim_template = f.read()

    write_file(
        cpath,
        shim_template.format(modname=module_name,
                             libname=shared_lib_name(group_name),
                             full_modname=exported_name(full_module_name)))

    return cpath
Пример #3
0
def build_c_extension_shim(full_module_name: str,
                           shared_lib_path: str,
                           dirname: str,
                           is_package: bool = False) -> str:
    module_parts = full_module_name.split('.')
    module_name = module_parts[-1]
    if is_package:
        module_parts.append('__init__')
    shared_lib = os.path.basename(shared_lib_path)
    assert shared_lib.startswith('lib') and shared_lib.endswith('.so')
    libname = shared_lib[3:-3]
    cname = '%s.c' % full_module_name.replace('.', '___')  # XXX
    cpath = os.path.join(dirname, cname)
    if '.' in full_module_name:
        packages = 'packages=[{}],'.format(
            repr('.'.join(full_module_name.split('.')[:-1])))
    else:
        packages = ''
    if len(module_parts) > 1:
        relative_lib_path = os.path.join(*(['..'] * (len(module_parts) - 1)))
    else:
        relative_lib_path = '.'
    with open(cpath, 'w') as f:
        f.write(
            shim_template.format(modname=module_name,
                                 full_modname=exported_name(full_module_name)))
    setup_path = make_setup_py(
        full_module_name,
        packages,
        cname,
        dirname,
        libraries=[libname],
        library_dirs=[os.path.abspath(os.path.dirname(shared_lib_path))],
        runtime_library_dirs=[relative_lib_path])
    return run_setup_py_build(setup_path, module_name, dirname)
Пример #4
0
def group_name(modules: List[str]) -> str:
    """Produce a probably unique name for a group from a list of module names."""
    if len(modules) == 1:
        return exported_name(modules[0])

    h = hashlib.sha1()
    h.update(','.join(modules).encode())
    return h.hexdigest()[:20]
Пример #5
0
def generate_c_extension_shim(full_module_name: str, module_name: str,
                              dir_name: str, group_name: str) -> str:
    """Create a C extension shim with a passthrough PyInit function.

    Arguments:
        full_module_name: the dotted full module name
        module_name: the final component of the module name
        dir_name: the directory to place source code
        group_name: the name of the group
    """
    cname = '%s.c' % exported_name(full_module_name)
    cpath = os.path.join(dir_name, cname)

    write_file(
        cpath,
        shim_template.format(modname=module_name,
                             libname=shared_lib_name(group_name),
                             full_modname=exported_name(full_module_name)))

    return cpath
Пример #6
0
def generate_c_extension_shim(full_module_name: str, module_name: str,
                              dirname: str) -> str:
    """Create a C extension shim with a passthrough PyInit function."""
    cname = '%s.c' % full_module_name.replace('.', '___')  # XXX
    cpath = os.path.join(dirname, cname)

    with open(cpath, 'w') as f:
        f.write(
            shim_template.format(modname=module_name,
                                 full_modname=exported_name(full_module_name)))

    return cpath
Пример #7
0
    def generate_shared_lib_init(self, emitter: Emitter) -> None:
        """Generate the init function for a shared library.

        A shared library contains all of the actual code for a set of modules.

        The init function is responsible for creating Capsules that wrap
        pointers to the initialization function of all the real init functions
        for modules in this shared library.
        """
        emitter.emit_line()
        emitter.emit_lines(
            'PyMODINIT_FUNC PyInit_{}(void)'.format(self.shared_lib_name),
            '{',
            ('static PyModuleDef def = {{ PyModuleDef_HEAD_INIT, "{}", NULL, -1, NULL, NULL }};'
             .format(self.shared_lib_name)),
            'int res;',
            'PyObject *capsule;',
            'PyObject *module = PyModule_Create(&def);',
            'if (!module) {',
            'goto fail;',
            '}',
            '',
        )

        for mod, _ in self.modules:
            name = exported_name(mod)
            emitter.emit_lines(
                'extern PyObject *CPyInit_{}(void);'.format(name),
                'capsule = PyCapsule_New((void *)CPyInit_{}, "{}.{}", NULL);'.
                format(name, self.shared_lib_name, name),
                'if (!capsule) {',
                'goto fail;',
                '}',
                'res = PyObject_SetAttrString(module, "{}", capsule);'.format(
                    name),
                'Py_DECREF(capsule);',
                'if (res < 0) {',
                'goto fail;',
                '}',
                '',
            )

        emitter.emit_lines(
            'return module;',
            'fail:',
            'Py_XDECREF(module);'
            'return NULL;',
            '}',
        )
Пример #8
0
def generate_c_extension_shim(
        full_module_name: str, module_name: str, dirname: str, libname: str) -> str:
    """Create a C extension shim with a passthrough PyInit function.

    Arguments:
      * full_module_name: the dotted full module name
      * module_name: the final component of the module name
      * dirname: the directory to place source code
      * libname: the name of the module where the code actually lives
    """
    cname = '%s.c' % full_module_name.replace('.', '___')  # XXX
    cpath = os.path.join(dirname, cname)

    write_file(
        cpath,
        shim_template.format(modname=module_name,
                             libname=libname,
                             full_modname=exported_name(full_module_name)))

    return cpath
Пример #9
0
 def test_exported_name(self) -> None:
     assert exported_name('foo') == 'foo'
     assert exported_name('foo.bar') == 'foo___bar'
Пример #10
0
    def generate_module_def(self, emitter: Emitter, module_name: str,
                            module: ModuleIR) -> None:
        """Emit the PyModuleDef struct for a module and the module init function."""
        # Emit module methods
        module_prefix = emitter.names.private_name(module_name)
        emitter.emit_line(
            'static PyMethodDef {}module_methods[] = {{'.format(module_prefix))
        for fn in module.functions:
            if fn.class_name is not None or fn.name == TOP_LEVEL_NAME:
                continue
            emitter.emit_line((
                '{{"{name}", (PyCFunction){prefix}{cname}, METH_VARARGS | METH_KEYWORDS, '
                'NULL /* docstring */}},').format(name=fn.name,
                                                  cname=fn.cname(
                                                      emitter.names),
                                                  prefix=PREFIX))
        emitter.emit_line('{NULL, NULL, 0, NULL}')
        emitter.emit_line('};')
        emitter.emit_line()

        # Emit module definition struct
        emitter.emit_lines(
            'static struct PyModuleDef {}module = {{'.format(module_prefix),
            'PyModuleDef_HEAD_INIT,', '"{}",'.format(module_name),
            'NULL, /* docstring */',
            '-1,       /* size of per-interpreter state of the module,',
            '             or -1 if the module keeps state in global variables. */',
            '{}module_methods'.format(module_prefix), '};')
        emitter.emit_line()
        # Emit module init function. If we are compiling just one module, this
        # will be the C API init function. If we are compiling 2+ modules, we
        # generate a shared library for the modules and shims that call into
        # the shared library, and in this case we use an internal module
        # initialized function that will be called by the shim.
        if not self.use_shared_lib:
            declaration = 'PyMODINIT_FUNC PyInit_{}(void)'.format(module_name)
        else:
            declaration = 'PyObject *CPyInit_{}(void)'.format(
                exported_name(module_name))
        emitter.emit_lines(declaration, '{')
        # Store the module reference in a static and return it when necessary.
        # This is separate from the *global* reference to the module that will
        # be populated when it is imported by a compiled module. We want that
        # reference to only be populated when the module has been successfully
        # imported, whereas this we want to have to stop a circular import.
        module_static = self.module_internal_static_name(module_name, emitter)

        emitter.emit_lines('if ({}) {{'.format(module_static),
                           'Py_INCREF({});'.format(module_static),
                           'return {};'.format(module_static), '}')

        emitter.emit_lines(
            '{} = PyModule_Create(&{}module);'.format(module_static,
                                                      module_prefix),
            'if (unlikely({} == NULL))'.format(module_static),
            '    return NULL;')
        emitter.emit_line(
            'PyObject *modname = PyObject_GetAttrString((PyObject *){}, "__name__");'
            .format(module_static))

        module_globals = emitter.static_name('globals', module_name)
        emitter.emit_lines(
            '{} = PyModule_GetDict({});'.format(module_globals, module_static),
            'if (unlikely({} == NULL))'.format(module_globals),
            '    return NULL;')

        # HACK: Manually instantiate generated classes here
        for cl in module.classes:
            if cl.is_generated:
                type_struct = emitter.type_struct_name(cl)
                emitter.emit_lines(
                    '{t} = (PyTypeObject *)CPyType_FromTemplate({t}_template, NULL, modname);'
                    .format(t=type_struct))
                emitter.emit_lines('if (unlikely(!{}))'.format(type_struct),
                                   '    return NULL;')

        emitter.emit_lines('if (CPyGlobalsInit() < 0)', '    return NULL;')

        self.generate_top_level_call(module, emitter)

        emitter.emit_lines('Py_DECREF(modname);')

        emitter.emit_line('return {};'.format(module_static))
        emitter.emit_line('}')
Пример #11
0
    def generate_shared_lib_init(self, emitter: Emitter) -> None:
        """Generate the init function for a shared library.

        A shared library contains all of the actual code for a
        compilation group.

        The init function is responsible for creating Capsules that
        wrap pointers to the initialization function of all the real
        init functions for modules in this shared library as well as
        the export table containing all of the exported functions and
        values from all the modules.

        These capsules are stored in attributes of the shared library.
        """
        assert self.group_name is not None

        emitter.emit_line()
        emitter.emit_lines(
            'PyMODINIT_FUNC PyInit_{}(void)'.format(
                shared_lib_name(self.group_name).split('.')[-1]),
            '{',
            ('static PyModuleDef def = {{ PyModuleDef_HEAD_INIT, "{}", NULL, -1, NULL, NULL }};'
             .format(shared_lib_name(self.group_name))),
            'int res;',
            'PyObject *capsule;',
            'PyObject *tmp;',
            'static PyObject *module;',
            'if (module) {',
            'Py_INCREF(module);',
            'return module;',
            '}',
            'module = PyModule_Create(&def);',
            'if (!module) {',
            'goto fail;',
            '}',
            '',
        )

        emitter.emit_lines(
            'capsule = PyCapsule_New(&exports, "{}.exports", NULL);'.format(
                shared_lib_name(self.group_name)),
            'if (!capsule) {',
            'goto fail;',
            '}',
            'res = PyObject_SetAttrString(module, "exports", capsule);',
            'Py_DECREF(capsule);',
            'if (res < 0) {',
            'goto fail;',
            '}',
            '',
        )

        for mod, _ in self.modules:
            name = exported_name(mod)
            emitter.emit_lines(
                'extern PyObject *CPyInit_{}(void);'.format(name),
                'capsule = PyCapsule_New((void *)CPyInit_{}, "{}.init_{}", NULL);'
                .format(name, shared_lib_name(self.group_name), name),
                'if (!capsule) {',
                'goto fail;',
                '}',
                'res = PyObject_SetAttrString(module, "init_{}", capsule);'.
                format(name),
                'Py_DECREF(capsule);',
                'if (res < 0) {',
                'goto fail;',
                '}',
                '',
            )

        for group in sorted(self.context.group_deps):
            egroup = exported_name(group)
            emitter.emit_lines(
                'tmp = PyImport_ImportModule("{}"); if (!tmp) goto fail; Py_DECREF(tmp);'
                .format(shared_lib_name(group)),
                'struct export_table_{} *pexports_{} = PyCapsule_Import("{}.exports", 0);'
                .format(egroup, egroup, shared_lib_name(group)),
                'if (!pexports_{}) {{'.format(egroup),
                'goto fail;',
                '}',
                'memcpy(&exports_{group}, pexports_{group}, sizeof(exports_{group}));'
                .format(group=egroup),
                '',
            )

        emitter.emit_lines(
            'return module;',
            'fail:',
            'Py_XDECREF(module);',
            'return NULL;',
            '}',
        )
Пример #12
0
    def generate_c_for_modules(self) -> List[Tuple[str, str]]:
        file_contents = []
        multi_file = self.use_shared_lib and self.multi_file

        base_emitter = Emitter(self.context)
        # Optionally just include the runtime library c files to
        # reduce the number of compiler invocations needed
        if self.compiler_options.include_runtime_files:
            base_emitter.emit_line('#include "CPy.c"')
            base_emitter.emit_line('#include "getargs.c"')
        base_emitter.emit_line('#include "__native{}.h"'.format(
            self.short_group_suffix))
        base_emitter.emit_line('#include "__native_internal{}.h"'.format(
            self.short_group_suffix))
        emitter = base_emitter

        for (_, literal), identifier in self.literals.items():
            if isinstance(literal, int):
                symbol = emitter.static_name(identifier, None)
                self.declare_global('CPyTagged ', symbol)
            else:
                self.declare_static_pyobject(identifier, emitter)

        for module_name, module in self.modules:
            if multi_file:
                emitter = Emitter(self.context)
                emitter.emit_line('#include "__native{}.h"'.format(
                    self.short_group_suffix))
                emitter.emit_line('#include "__native_internal{}.h"'.format(
                    self.short_group_suffix))

            self.declare_module(module_name, emitter)
            self.declare_internal_globals(module_name, emitter)
            self.declare_imports(module.imports, emitter)

            for cl in module.classes:
                if cl.is_ext_class:
                    generate_class(cl, module_name, emitter)

            # Generate Python extension module definitions and module initialization functions.
            self.generate_module_def(emitter, module_name, module)

            for fn in module.functions:
                emitter.emit_line()
                generate_native_function(fn, emitter,
                                         self.source_paths[module_name],
                                         module_name)
                if fn.name != TOP_LEVEL_NAME:
                    emitter.emit_line()
                    generate_wrapper_function(fn, emitter,
                                              self.source_paths[module_name],
                                              module_name)

            if multi_file:
                name = ('__native_{}.c'.format(
                    emitter.names.private_name(module_name)))
                file_contents.append((name, ''.join(emitter.fragments)))

        # The external header file contains type declarations while
        # the internal contains declarations of functions and objects
        # (which are shared between shared libraries via dynamic
        # exports tables and not accessed directly.)
        ext_declarations = Emitter(self.context)
        ext_declarations.emit_line('#ifndef MYPYC_NATIVE{}_H'.format(
            self.group_suffix))
        ext_declarations.emit_line('#define MYPYC_NATIVE{}_H'.format(
            self.group_suffix))
        ext_declarations.emit_line('#include <Python.h>')
        ext_declarations.emit_line('#include <CPy.h>')

        declarations = Emitter(self.context)
        declarations.emit_line('#ifndef MYPYC_NATIVE_INTERNAL{}_H'.format(
            self.group_suffix))
        declarations.emit_line('#define MYPYC_NATIVE_INTERNAL{}_H'.format(
            self.group_suffix))
        declarations.emit_line('#include <Python.h>')
        declarations.emit_line('#include <CPy.h>')
        declarations.emit_line('#include "__native{}.h"'.format(
            self.short_group_suffix))
        declarations.emit_line()
        declarations.emit_line('int CPyGlobalsInit(void);')
        declarations.emit_line()

        for module_name, module in self.modules:
            self.declare_finals(module_name, module.final_names, declarations)
            for cl in module.classes:
                generate_class_type_decl(cl, emitter, ext_declarations,
                                         declarations)
            for fn in module.functions:
                generate_function_declaration(fn, declarations)

        for lib in sorted(self.context.group_deps):
            elib = exported_name(lib)
            short_lib = exported_name(lib.split('.')[-1])
            declarations.emit_lines(
                '#include <{}>'.format(
                    os.path.join(group_dir(lib),
                                 "__native_{}.h".format(short_lib))),
                'struct export_table_{} exports_{};'.format(elib, elib))

        sorted_decls = self.toposort_declarations()

        emitter = base_emitter
        self.generate_globals_init(emitter)

        emitter.emit_line()

        for declaration in sorted_decls:
            decls = ext_declarations if declaration.is_type else declarations
            if not declaration.is_type:
                decls.emit_lines('extern {}'.format(declaration.decl[0]),
                                 *declaration.decl[1:])
                # If there is a definition, emit it. Otherwise repeat the declaration
                # (without an extern).
                if declaration.defn:
                    emitter.emit_lines(*declaration.defn)
                else:
                    emitter.emit_lines(*declaration.decl)
            else:
                decls.emit_lines(*declaration.decl)

        if self.group_name:
            self.generate_export_table(ext_declarations, emitter)

            self.generate_shared_lib_init(emitter)

        ext_declarations.emit_line('#endif')
        declarations.emit_line('#endif')

        output_dir = group_dir(self.group_name) if self.group_name else ''
        return file_contents + [
            (os.path.join(output_dir, '__native{}.c'.format(
                self.short_group_suffix)), ''.join(emitter.fragments)),
            (os.path.join(
                output_dir, '__native_internal{}.h'.format(
                    self.short_group_suffix)), ''.join(
                        declarations.fragments)),
            (os.path.join(output_dir, '__native{}.h'.format(
                self.short_group_suffix)), ''.join(
                    ext_declarations.fragments)),
        ]
Пример #13
0
 def short_group_suffix(self) -> str:
     return '_' + exported_name(
         self.group_name.split('.')[-1]) if self.group_name else ''
Пример #14
0
 def group_suffix(self) -> str:
     return '_' + exported_name(self.group_name) if self.group_name else ''
Пример #15
0
 def struct_name(self, names: NameGenerator) -> str:
     return '{}Object'.format(exported_name(self.fullname))
Пример #16
0
    def generate_c_for_modules(self) -> List[Tuple[str, str]]:
        file_contents = []
        multi_file = self.use_shared_lib and self.multi_file

        # Collect all literal refs in IR.
        for _, module in self.modules:
            for fn in module.functions:
                collect_literals(fn, self.context.literals)

        base_emitter = Emitter(self.context)
        # Optionally just include the runtime library c files to
        # reduce the number of compiler invocations needed
        if self.compiler_options.include_runtime_files:
            for name in RUNTIME_C_FILES:
                base_emitter.emit_line(f'#include "{name}"')
        base_emitter.emit_line(
            f'#include "__native{self.short_group_suffix}.h"')
        base_emitter.emit_line(
            f'#include "__native_internal{self.short_group_suffix}.h"')
        emitter = base_emitter

        self.generate_literal_tables()

        for module_name, module in self.modules:
            if multi_file:
                emitter = Emitter(self.context)
                emitter.emit_line(
                    f'#include "__native{self.short_group_suffix}.h"')
                emitter.emit_line(
                    f'#include "__native_internal{self.short_group_suffix}.h"')

            self.declare_module(module_name, emitter)
            self.declare_internal_globals(module_name, emitter)
            self.declare_imports(module.imports, emitter)

            for cl in module.classes:
                if cl.is_ext_class:
                    generate_class(cl, module_name, emitter)

            # Generate Python extension module definitions and module initialization functions.
            self.generate_module_def(emitter, module_name, module)

            for fn in module.functions:
                emitter.emit_line()
                generate_native_function(fn, emitter,
                                         self.source_paths[module_name],
                                         module_name)
                if fn.name != TOP_LEVEL_NAME:
                    emitter.emit_line()
                    if is_fastcall_supported(fn, emitter.capi_version):
                        generate_wrapper_function(
                            fn, emitter, self.source_paths[module_name],
                            module_name)
                    else:
                        generate_legacy_wrapper_function(
                            fn, emitter, self.source_paths[module_name],
                            module_name)
            if multi_file:
                name = (
                    f'__native_{emitter.names.private_name(module_name)}.c')
                file_contents.append((name, ''.join(emitter.fragments)))

        # The external header file contains type declarations while
        # the internal contains declarations of functions and objects
        # (which are shared between shared libraries via dynamic
        # exports tables and not accessed directly.)
        ext_declarations = Emitter(self.context)
        ext_declarations.emit_line(
            f'#ifndef MYPYC_NATIVE{self.group_suffix}_H')
        ext_declarations.emit_line(
            f'#define MYPYC_NATIVE{self.group_suffix}_H')
        ext_declarations.emit_line('#include <Python.h>')
        ext_declarations.emit_line('#include <CPy.h>')

        declarations = Emitter(self.context)
        declarations.emit_line(
            f'#ifndef MYPYC_NATIVE_INTERNAL{self.group_suffix}_H')
        declarations.emit_line(
            f'#define MYPYC_NATIVE_INTERNAL{self.group_suffix}_H')
        declarations.emit_line('#include <Python.h>')
        declarations.emit_line('#include <CPy.h>')
        declarations.emit_line(
            f'#include "__native{self.short_group_suffix}.h"')
        declarations.emit_line()
        declarations.emit_line('int CPyGlobalsInit(void);')
        declarations.emit_line()

        for module_name, module in self.modules:
            self.declare_finals(module_name, module.final_names, declarations)
            for cl in module.classes:
                generate_class_type_decl(cl, emitter, ext_declarations,
                                         declarations)
            for fn in module.functions:
                generate_function_declaration(fn, declarations)

        for lib in sorted(self.context.group_deps):
            elib = exported_name(lib)
            short_lib = exported_name(lib.split('.')[-1])
            declarations.emit_lines(
                '#include <{}>'.format(
                    os.path.join(group_dir(lib), f"__native_{short_lib}.h")),
                f'struct export_table_{elib} exports_{elib};')

        sorted_decls = self.toposort_declarations()

        emitter = base_emitter
        self.generate_globals_init(emitter)

        emitter.emit_line()

        for declaration in sorted_decls:
            decls = ext_declarations if declaration.is_type else declarations
            if not declaration.is_type:
                decls.emit_lines(f'extern {declaration.decl[0]}',
                                 *declaration.decl[1:])
                # If there is a definition, emit it. Otherwise repeat the declaration
                # (without an extern).
                if declaration.defn:
                    emitter.emit_lines(*declaration.defn)
                else:
                    emitter.emit_lines(*declaration.decl)
            else:
                decls.emit_lines(*declaration.decl)

        if self.group_name:
            self.generate_export_table(ext_declarations, emitter)

            self.generate_shared_lib_init(emitter)

        ext_declarations.emit_line('#endif')
        declarations.emit_line('#endif')

        output_dir = group_dir(self.group_name) if self.group_name else ''
        return file_contents + [
            (os.path.join(output_dir, f'__native{self.short_group_suffix}.c'),
             ''.join(emitter.fragments)),
            (os.path.join(output_dir,
                          f'__native_internal{self.short_group_suffix}.h'),
             ''.join(declarations.fragments)),
            (os.path.join(output_dir, f'__native{self.short_group_suffix}.h'),
             ''.join(ext_declarations.fragments)),
        ]