def get_module_group_prefix(self, module_name: str) -> str:
"""Get the group prefix for a module (relative to the current group).
The prefix should be prepended to the object name whenever
accessing an object from this module.
If the module lives is in the current compilation group, there is
no prefix. But if it lives in a different group (and hence a separate
extension module), we need to access objects from it indirectly via an
export table.
For example, for code in group `a` to call a function `bar` in group `b`,
it would need to do `exports_b.CPyDef_bar(...)`, while code that is
also in group `b` can simply do `CPyDef_bar(...)`.
Thus the prefix for a module in group `b` is 'exports_b.' if the current
group is *not* b and just '' if it is.
"""
groups = self.context.group_map
target_group_name = groups.get(module_name)
if target_group_name and target_group_name != self.context.group_name:
self.context.group_deps.add(target_group_name)
return 'exports_{}.'.format(exported_name(target_group_name))
else:
return ''
def generate_c_extension_shim(full_module_name: str, module_name: str,
dir_name: str, group_name: str) -> str:
"""Create a C extension shim with a passthrough PyInit function.
Arguments:
full_module_name: the dotted full module name
module_name: the final component of the module name
dir_name: the directory to place source code
group_name: the name of the group
"""
cname = '%s.c' % full_module_name.replace('.', os.sep)
cpath = os.path.join(dir_name, cname)
# We load the C extension shim template from a file.
# (So that the file could be reused as a bazel template also.)
with open(os.path.join(include_dir(), 'module_shim.tmpl')) as f:
shim_template = f.read()
write_file(
cpath,
shim_template.format(modname=module_name,
libname=shared_lib_name(group_name),
full_modname=exported_name(full_module_name)))
return cpath
def build_c_extension_shim(full_module_name: str,
shared_lib_path: str,
dirname: str,
is_package: bool = False) -> str:
module_parts = full_module_name.split('.')
module_name = module_parts[-1]
if is_package:
module_parts.append('__init__')
shared_lib = os.path.basename(shared_lib_path)
assert shared_lib.startswith('lib') and shared_lib.endswith('.so')
libname = shared_lib[3:-3]
cname = '%s.c' % full_module_name.replace('.', '___') # XXX
cpath = os.path.join(dirname, cname)
if '.' in full_module_name:
packages = 'packages=[{}],'.format(
repr('.'.join(full_module_name.split('.')[:-1])))
else:
packages = ''
if len(module_parts) > 1:
relative_lib_path = os.path.join(*(['..'] * (len(module_parts) - 1)))
else:
relative_lib_path = '.'
with open(cpath, 'w') as f:
f.write(
shim_template.format(modname=module_name,
full_modname=exported_name(full_module_name)))
setup_path = make_setup_py(
full_module_name,
packages,
cname,
dirname,
libraries=[libname],
library_dirs=[os.path.abspath(os.path.dirname(shared_lib_path))],
runtime_library_dirs=[relative_lib_path])
return run_setup_py_build(setup_path, module_name, dirname)
def group_name(modules: List[str]) -> str:
"""Produce a probably unique name for a group from a list of module names."""
if len(modules) == 1:
return exported_name(modules[0])
h = hashlib.sha1()
h.update(','.join(modules).encode())
return h.hexdigest()[:20]
def generate_c_extension_shim(full_module_name: str, module_name: str,
dir_name: str, group_name: str) -> str:
"""Create a C extension shim with a passthrough PyInit function.
Arguments:
full_module_name: the dotted full module name
module_name: the final component of the module name
dir_name: the directory to place source code
group_name: the name of the group
"""
cname = '%s.c' % exported_name(full_module_name)
cpath = os.path.join(dir_name, cname)
write_file(
cpath,
shim_template.format(modname=module_name,
libname=shared_lib_name(group_name),
full_modname=exported_name(full_module_name)))
return cpath
def generate_c_extension_shim(full_module_name: str, module_name: str,
dirname: str) -> str:
"""Create a C extension shim with a passthrough PyInit function."""
cname = '%s.c' % full_module_name.replace('.', '___') # XXX
cpath = os.path.join(dirname, cname)
with open(cpath, 'w') as f:
f.write(
shim_template.format(modname=module_name,
full_modname=exported_name(full_module_name)))
return cpath
def generate_shared_lib_init(self, emitter: Emitter) -> None:
"""Generate the init function for a shared library.
A shared library contains all of the actual code for a set of modules.
The init function is responsible for creating Capsules that wrap
pointers to the initialization function of all the real init functions
for modules in this shared library.
"""
emitter.emit_line()
emitter.emit_lines(
'PyMODINIT_FUNC PyInit_{}(void)'.format(self.shared_lib_name),
'{',
('static PyModuleDef def = {{ PyModuleDef_HEAD_INIT, "{}", NULL, -1, NULL, NULL }};'
.format(self.shared_lib_name)),
'int res;',
'PyObject *capsule;',
'PyObject *module = PyModule_Create(&def);',
'if (!module) {',
'goto fail;',
'}',
'',
)
for mod, _ in self.modules:
name = exported_name(mod)
emitter.emit_lines(
'extern PyObject *CPyInit_{}(void);'.format(name),
'capsule = PyCapsule_New((void *)CPyInit_{}, "{}.{}", NULL);'.
format(name, self.shared_lib_name, name),
'if (!capsule) {',
'goto fail;',
'}',
'res = PyObject_SetAttrString(module, "{}", capsule);'.format(
name),
'Py_DECREF(capsule);',
'if (res < 0) {',
'goto fail;',
'}',
'',
)
emitter.emit_lines(
'return module;',
'fail:',
'Py_XDECREF(module);'
'return NULL;',
'}',
)
def generate_c_extension_shim(
full_module_name: str, module_name: str, dirname: str, libname: str) -> str:
"""Create a C extension shim with a passthrough PyInit function.
Arguments:
* full_module_name: the dotted full module name
* module_name: the final component of the module name
* dirname: the directory to place source code
* libname: the name of the module where the code actually lives
"""
cname = '%s.c' % full_module_name.replace('.', '___') # XXX
cpath = os.path.join(dirname, cname)
write_file(
cpath,
shim_template.format(modname=module_name,
libname=libname,
full_modname=exported_name(full_module_name)))
return cpath
def test_exported_name(self) -> None:
assert exported_name('foo') == 'foo'
assert exported_name('foo.bar') == 'foo___bar'
def generate_module_def(self, emitter: Emitter, module_name: str,
module: ModuleIR) -> None:
"""Emit the PyModuleDef struct for a module and the module init function."""
# Emit module methods
module_prefix = emitter.names.private_name(module_name)
emitter.emit_line(
'static PyMethodDef {}module_methods[] = {{'.format(module_prefix))
for fn in module.functions:
if fn.class_name is not None or fn.name == TOP_LEVEL_NAME:
continue
emitter.emit_line((
'{{"{name}", (PyCFunction){prefix}{cname}, METH_VARARGS | METH_KEYWORDS, '
'NULL /* docstring */}},').format(name=fn.name,
cname=fn.cname(
emitter.names),
prefix=PREFIX))
emitter.emit_line('{NULL, NULL, 0, NULL}')
emitter.emit_line('};')
emitter.emit_line()
# Emit module definition struct
emitter.emit_lines(
'static struct PyModuleDef {}module = {{'.format(module_prefix),
'PyModuleDef_HEAD_INIT,', '"{}",'.format(module_name),
'NULL, /* docstring */',
'-1, /* size of per-interpreter state of the module,',
' or -1 if the module keeps state in global variables. */',
'{}module_methods'.format(module_prefix), '};')
emitter.emit_line()
# Emit module init function. If we are compiling just one module, this
# will be the C API init function. If we are compiling 2+ modules, we
# generate a shared library for the modules and shims that call into
# the shared library, and in this case we use an internal module
# initialized function that will be called by the shim.
if not self.use_shared_lib:
declaration = 'PyMODINIT_FUNC PyInit_{}(void)'.format(module_name)
else:
declaration = 'PyObject *CPyInit_{}(void)'.format(
exported_name(module_name))
emitter.emit_lines(declaration, '{')
# Store the module reference in a static and return it when necessary.
# This is separate from the *global* reference to the module that will
# be populated when it is imported by a compiled module. We want that
# reference to only be populated when the module has been successfully
# imported, whereas this we want to have to stop a circular import.
module_static = self.module_internal_static_name(module_name, emitter)
emitter.emit_lines('if ({}) {{'.format(module_static),
'Py_INCREF({});'.format(module_static),
'return {};'.format(module_static), '}')
emitter.emit_lines(
'{} = PyModule_Create(&{}module);'.format(module_static,
module_prefix),
'if (unlikely({} == NULL))'.format(module_static),
' return NULL;')
emitter.emit_line(
'PyObject *modname = PyObject_GetAttrString((PyObject *){}, "__name__");'
.format(module_static))
module_globals = emitter.static_name('globals', module_name)
emitter.emit_lines(
'{} = PyModule_GetDict({});'.format(module_globals, module_static),
'if (unlikely({} == NULL))'.format(module_globals),
' return NULL;')
# HACK: Manually instantiate generated classes here
for cl in module.classes:
if cl.is_generated:
type_struct = emitter.type_struct_name(cl)
emitter.emit_lines(
'{t} = (PyTypeObject *)CPyType_FromTemplate({t}_template, NULL, modname);'
.format(t=type_struct))
emitter.emit_lines('if (unlikely(!{}))'.format(type_struct),
' return NULL;')
emitter.emit_lines('if (CPyGlobalsInit() < 0)', ' return NULL;')
self.generate_top_level_call(module, emitter)
emitter.emit_lines('Py_DECREF(modname);')
emitter.emit_line('return {};'.format(module_static))
emitter.emit_line('}')
def generate_shared_lib_init(self, emitter: Emitter) -> None:
"""Generate the init function for a shared library.
A shared library contains all of the actual code for a
compilation group.
The init function is responsible for creating Capsules that
wrap pointers to the initialization function of all the real
init functions for modules in this shared library as well as
the export table containing all of the exported functions and
values from all the modules.
These capsules are stored in attributes of the shared library.
"""
assert self.group_name is not None
emitter.emit_line()
emitter.emit_lines(
'PyMODINIT_FUNC PyInit_{}(void)'.format(
shared_lib_name(self.group_name).split('.')[-1]),
'{',
('static PyModuleDef def = {{ PyModuleDef_HEAD_INIT, "{}", NULL, -1, NULL, NULL }};'
.format(shared_lib_name(self.group_name))),
'int res;',
'PyObject *capsule;',
'PyObject *tmp;',
'static PyObject *module;',
'if (module) {',
'Py_INCREF(module);',
'return module;',
'}',
'module = PyModule_Create(&def);',
'if (!module) {',
'goto fail;',
'}',
'',
)
emitter.emit_lines(
'capsule = PyCapsule_New(&exports, "{}.exports", NULL);'.format(
shared_lib_name(self.group_name)),
'if (!capsule) {',
'goto fail;',
'}',
'res = PyObject_SetAttrString(module, "exports", capsule);',
'Py_DECREF(capsule);',
'if (res < 0) {',
'goto fail;',
'}',
'',
)
for mod, _ in self.modules:
name = exported_name(mod)
emitter.emit_lines(
'extern PyObject *CPyInit_{}(void);'.format(name),
'capsule = PyCapsule_New((void *)CPyInit_{}, "{}.init_{}", NULL);'
.format(name, shared_lib_name(self.group_name), name),
'if (!capsule) {',
'goto fail;',
'}',
'res = PyObject_SetAttrString(module, "init_{}", capsule);'.
format(name),
'Py_DECREF(capsule);',
'if (res < 0) {',
'goto fail;',
'}',
'',
)
for group in sorted(self.context.group_deps):
egroup = exported_name(group)
emitter.emit_lines(
'tmp = PyImport_ImportModule("{}"); if (!tmp) goto fail; Py_DECREF(tmp);'
.format(shared_lib_name(group)),
'struct export_table_{} *pexports_{} = PyCapsule_Import("{}.exports", 0);'
.format(egroup, egroup, shared_lib_name(group)),
'if (!pexports_{}) {{'.format(egroup),
'goto fail;',
'}',
'memcpy(&exports_{group}, pexports_{group}, sizeof(exports_{group}));'
.format(group=egroup),
'',
)
emitter.emit_lines(
'return module;',
'fail:',
'Py_XDECREF(module);',
'return NULL;',
'}',
)
def generate_c_for_modules(self) -> List[Tuple[str, str]]:
file_contents = []
multi_file = self.use_shared_lib and self.multi_file
base_emitter = Emitter(self.context)
# Optionally just include the runtime library c files to
# reduce the number of compiler invocations needed
if self.compiler_options.include_runtime_files:
base_emitter.emit_line('#include "CPy.c"')
base_emitter.emit_line('#include "getargs.c"')
base_emitter.emit_line('#include "__native{}.h"'.format(
self.short_group_suffix))
base_emitter.emit_line('#include "__native_internal{}.h"'.format(
self.short_group_suffix))
emitter = base_emitter
for (_, literal), identifier in self.literals.items():
if isinstance(literal, int):
symbol = emitter.static_name(identifier, None)
self.declare_global('CPyTagged ', symbol)
else:
self.declare_static_pyobject(identifier, emitter)
for module_name, module in self.modules:
if multi_file:
emitter = Emitter(self.context)
emitter.emit_line('#include "__native{}.h"'.format(
self.short_group_suffix))
emitter.emit_line('#include "__native_internal{}.h"'.format(
self.short_group_suffix))
self.declare_module(module_name, emitter)
self.declare_internal_globals(module_name, emitter)
self.declare_imports(module.imports, emitter)
for cl in module.classes:
if cl.is_ext_class:
generate_class(cl, module_name, emitter)
# Generate Python extension module definitions and module initialization functions.
self.generate_module_def(emitter, module_name, module)
for fn in module.functions:
emitter.emit_line()
generate_native_function(fn, emitter,
self.source_paths[module_name],
module_name)
if fn.name != TOP_LEVEL_NAME:
emitter.emit_line()
generate_wrapper_function(fn, emitter,
self.source_paths[module_name],
module_name)
if multi_file:
name = ('__native_{}.c'.format(
emitter.names.private_name(module_name)))
file_contents.append((name, ''.join(emitter.fragments)))
# The external header file contains type declarations while
# the internal contains declarations of functions and objects
# (which are shared between shared libraries via dynamic
# exports tables and not accessed directly.)
ext_declarations = Emitter(self.context)
ext_declarations.emit_line('#ifndef MYPYC_NATIVE{}_H'.format(
self.group_suffix))
ext_declarations.emit_line('#define MYPYC_NATIVE{}_H'.format(
self.group_suffix))
ext_declarations.emit_line('#include <Python.h>')
ext_declarations.emit_line('#include <CPy.h>')
declarations = Emitter(self.context)
declarations.emit_line('#ifndef MYPYC_NATIVE_INTERNAL{}_H'.format(
self.group_suffix))
declarations.emit_line('#define MYPYC_NATIVE_INTERNAL{}_H'.format(
self.group_suffix))
declarations.emit_line('#include <Python.h>')
declarations.emit_line('#include <CPy.h>')
declarations.emit_line('#include "__native{}.h"'.format(
self.short_group_suffix))
declarations.emit_line()
declarations.emit_line('int CPyGlobalsInit(void);')
declarations.emit_line()
for module_name, module in self.modules:
self.declare_finals(module_name, module.final_names, declarations)
for cl in module.classes:
generate_class_type_decl(cl, emitter, ext_declarations,
declarations)
for fn in module.functions:
generate_function_declaration(fn, declarations)
for lib in sorted(self.context.group_deps):
elib = exported_name(lib)
short_lib = exported_name(lib.split('.')[-1])
declarations.emit_lines(
'#include <{}>'.format(
os.path.join(group_dir(lib),
"__native_{}.h".format(short_lib))),
'struct export_table_{} exports_{};'.format(elib, elib))
sorted_decls = self.toposort_declarations()
emitter = base_emitter
self.generate_globals_init(emitter)
emitter.emit_line()
for declaration in sorted_decls:
decls = ext_declarations if declaration.is_type else declarations
if not declaration.is_type:
decls.emit_lines('extern {}'.format(declaration.decl[0]),
*declaration.decl[1:])
# If there is a definition, emit it. Otherwise repeat the declaration
# (without an extern).
if declaration.defn:
emitter.emit_lines(*declaration.defn)
else:
emitter.emit_lines(*declaration.decl)
else:
decls.emit_lines(*declaration.decl)
if self.group_name:
self.generate_export_table(ext_declarations, emitter)
self.generate_shared_lib_init(emitter)
ext_declarations.emit_line('#endif')
declarations.emit_line('#endif')
output_dir = group_dir(self.group_name) if self.group_name else ''
return file_contents + [
(os.path.join(output_dir, '__native{}.c'.format(
self.short_group_suffix)), ''.join(emitter.fragments)),
(os.path.join(
output_dir, '__native_internal{}.h'.format(
self.short_group_suffix)), ''.join(
declarations.fragments)),
(os.path.join(output_dir, '__native{}.h'.format(
self.short_group_suffix)), ''.join(
ext_declarations.fragments)),
]
def short_group_suffix(self) -> str:
return '_' + exported_name(
self.group_name.split('.')[-1]) if self.group_name else ''
def group_suffix(self) -> str:
return '_' + exported_name(self.group_name) if self.group_name else ''
def struct_name(self, names: NameGenerator) -> str:
return '{}Object'.format(exported_name(self.fullname))
def generate_c_for_modules(self) -> List[Tuple[str, str]]:
file_contents = []
multi_file = self.use_shared_lib and self.multi_file
# Collect all literal refs in IR.
for _, module in self.modules:
for fn in module.functions:
collect_literals(fn, self.context.literals)
base_emitter = Emitter(self.context)
# Optionally just include the runtime library c files to
# reduce the number of compiler invocations needed
if self.compiler_options.include_runtime_files:
for name in RUNTIME_C_FILES:
base_emitter.emit_line(f'#include "{name}"')
base_emitter.emit_line(
f'#include "__native{self.short_group_suffix}.h"')
base_emitter.emit_line(
f'#include "__native_internal{self.short_group_suffix}.h"')
emitter = base_emitter
self.generate_literal_tables()
for module_name, module in self.modules:
if multi_file:
emitter = Emitter(self.context)
emitter.emit_line(
f'#include "__native{self.short_group_suffix}.h"')
emitter.emit_line(
f'#include "__native_internal{self.short_group_suffix}.h"')
self.declare_module(module_name, emitter)
self.declare_internal_globals(module_name, emitter)
self.declare_imports(module.imports, emitter)
for cl in module.classes:
if cl.is_ext_class:
generate_class(cl, module_name, emitter)
# Generate Python extension module definitions and module initialization functions.
self.generate_module_def(emitter, module_name, module)
for fn in module.functions:
emitter.emit_line()
generate_native_function(fn, emitter,
self.source_paths[module_name],
module_name)
if fn.name != TOP_LEVEL_NAME:
emitter.emit_line()
if is_fastcall_supported(fn, emitter.capi_version):
generate_wrapper_function(
fn, emitter, self.source_paths[module_name],
module_name)
else:
generate_legacy_wrapper_function(
fn, emitter, self.source_paths[module_name],
module_name)
if multi_file:
name = (
f'__native_{emitter.names.private_name(module_name)}.c')
file_contents.append((name, ''.join(emitter.fragments)))
# The external header file contains type declarations while
# the internal contains declarations of functions and objects
# (which are shared between shared libraries via dynamic
# exports tables and not accessed directly.)
ext_declarations = Emitter(self.context)
ext_declarations.emit_line(
f'#ifndef MYPYC_NATIVE{self.group_suffix}_H')
ext_declarations.emit_line(
f'#define MYPYC_NATIVE{self.group_suffix}_H')
ext_declarations.emit_line('#include <Python.h>')
ext_declarations.emit_line('#include <CPy.h>')
declarations = Emitter(self.context)
declarations.emit_line(
f'#ifndef MYPYC_NATIVE_INTERNAL{self.group_suffix}_H')
declarations.emit_line(
f'#define MYPYC_NATIVE_INTERNAL{self.group_suffix}_H')
declarations.emit_line('#include <Python.h>')
declarations.emit_line('#include <CPy.h>')
declarations.emit_line(
f'#include "__native{self.short_group_suffix}.h"')
declarations.emit_line()
declarations.emit_line('int CPyGlobalsInit(void);')
declarations.emit_line()
for module_name, module in self.modules:
self.declare_finals(module_name, module.final_names, declarations)
for cl in module.classes:
generate_class_type_decl(cl, emitter, ext_declarations,
declarations)
for fn in module.functions:
generate_function_declaration(fn, declarations)
for lib in sorted(self.context.group_deps):
elib = exported_name(lib)
short_lib = exported_name(lib.split('.')[-1])
declarations.emit_lines(
'#include <{}>'.format(
os.path.join(group_dir(lib), f"__native_{short_lib}.h")),
f'struct export_table_{elib} exports_{elib};')
sorted_decls = self.toposort_declarations()
emitter = base_emitter
self.generate_globals_init(emitter)
emitter.emit_line()
for declaration in sorted_decls:
decls = ext_declarations if declaration.is_type else declarations
if not declaration.is_type:
decls.emit_lines(f'extern {declaration.decl[0]}',
*declaration.decl[1:])
# If there is a definition, emit it. Otherwise repeat the declaration
# (without an extern).
if declaration.defn:
emitter.emit_lines(*declaration.defn)
else:
emitter.emit_lines(*declaration.decl)
else:
decls.emit_lines(*declaration.decl)
if self.group_name:
self.generate_export_table(ext_declarations, emitter)
self.generate_shared_lib_init(emitter)
ext_declarations.emit_line('#endif')
declarations.emit_line('#endif')
output_dir = group_dir(self.group_name) if self.group_name else ''
return file_contents + [
(os.path.join(output_dir, f'__native{self.short_group_suffix}.c'),
''.join(emitter.fragments)),
(os.path.join(output_dir,
f'__native_internal{self.short_group_suffix}.h'),
''.join(declarations.fragments)),
(os.path.join(output_dir, f'__native{self.short_group_suffix}.h'),
''.join(ext_declarations.fragments)),
]
