def generate_c_extension_shim(full_module_name: str, module_name: str, dir_name: str, group_name: str) -> str: """Create a C extension shim with a passthrough PyInit function. Arguments: full_module_name: the dotted full module name module_name: the final component of the module name dir_name: the directory to place source code group_name: the name of the group """ cname = '%s.c' % full_module_name.replace('.', os.sep) cpath = os.path.join(dir_name, cname) # We load the C extension shim template from a file. # (So that the file could be reused as a bazel template also.) with open(os.path.join(include_dir(), 'module_shim.tmpl')) as f: shim_template = f.read() write_file( cpath, shim_template.format(modname=module_name, libname=shared_lib_name(group_name), full_modname=exported_name(full_module_name))) return cpath
def build_using_shared_lib( sources: List[BuildSource], group_name: str, cfiles: List[str], deps: List[str], build_dir: str, extra_compile_args: List[str], ) -> List[Extension]: """Produce the list of extension modules when a shared library is needed. This creates one shared library extension module that all of the others import and then one shim extension module for each module in the build, that simply calls an initialization function in the shared library. The shared library (which lib_name is the name of) is a python extension module that exports the real initialization functions in Capsules stored in module attributes. """ extensions = [ Extension( shared_lib_name(group_name), sources=cfiles, include_dirs=[include_dir()], depends=deps, extra_compile_args=extra_compile_args, ) ] for source in sources: module_name = source.module.split('.')[-1] shim_file = generate_c_extension_shim(source.module, module_name, build_dir, group_name) # We include the __init__ in the "module name" we stick in the Extension, # since this seems to be needed for it to end up in the right place. full_module_name = source.module assert source.path if os.path.split(source.path)[1] == '__init__.py': full_module_name += '.__init__' extensions.append( Extension( full_module_name, sources=[shim_file], extra_compile_args=extra_compile_args, )) return extensions
def generate_c_extension_shim(full_module_name: str, module_name: str, dir_name: str, group_name: str) -> str: """Create a C extension shim with a passthrough PyInit function. Arguments: full_module_name: the dotted full module name module_name: the final component of the module name dir_name: the directory to place source code group_name: the name of the group """ cname = '%s.c' % exported_name(full_module_name) cpath = os.path.join(dir_name, cname) write_file( cpath, shim_template.format(modname=module_name, libname=shared_lib_name(group_name), full_modname=exported_name(full_module_name))) return cpath
def generate_shared_lib_init(self, emitter: Emitter) -> None: """Generate the init function for a shared library. A shared library contains all of the actual code for a compilation group. The init function is responsible for creating Capsules that wrap pointers to the initialization function of all the real init functions for modules in this shared library as well as the export table containing all of the exported functions and values from all the modules. These capsules are stored in attributes of the shared library. """ assert self.group_name is not None emitter.emit_line() emitter.emit_lines( 'PyMODINIT_FUNC PyInit_{}(void)'.format( shared_lib_name(self.group_name).split('.')[-1]), '{', ('static PyModuleDef def = {{ PyModuleDef_HEAD_INIT, "{}", NULL, -1, NULL, NULL }};' .format(shared_lib_name(self.group_name))), 'int res;', 'PyObject *capsule;', 'PyObject *tmp;', 'static PyObject *module;', 'if (module) {', 'Py_INCREF(module);', 'return module;', '}', 'module = PyModule_Create(&def);', 'if (!module) {', 'goto fail;', '}', '', ) emitter.emit_lines( 'capsule = PyCapsule_New(&exports, "{}.exports", NULL);'.format( shared_lib_name(self.group_name)), 'if (!capsule) {', 'goto fail;', '}', 'res = PyObject_SetAttrString(module, "exports", capsule);', 'Py_DECREF(capsule);', 'if (res < 0) {', 'goto fail;', '}', '', ) for mod, _ in self.modules: name = exported_name(mod) emitter.emit_lines( 'extern PyObject *CPyInit_{}(void);'.format(name), 'capsule = PyCapsule_New((void *)CPyInit_{}, "{}.init_{}", NULL);' .format(name, shared_lib_name(self.group_name), name), 'if (!capsule) {', 'goto fail;', '}', 'res = PyObject_SetAttrString(module, "init_{}", capsule);'. format(name), 'Py_DECREF(capsule);', 'if (res < 0) {', 'goto fail;', '}', '', ) for group in sorted(self.context.group_deps): egroup = exported_name(group) emitter.emit_lines( 'tmp = PyImport_ImportModule("{}"); if (!tmp) goto fail; Py_DECREF(tmp);' .format(shared_lib_name(group)), 'struct export_table_{} *pexports_{} = PyCapsule_Import("{}.exports", 0);' .format(egroup, egroup, shared_lib_name(group)), 'if (!pexports_{}) {{'.format(egroup), 'goto fail;', '}', 'memcpy(&exports_{group}, pexports_{group}, sizeof(exports_{group}));' .format(group=egroup), '', ) emitter.emit_lines( 'return module;', 'fail:', 'Py_XDECREF(module);', 'return NULL;', '}', )