def generate_literals(self, emitter: Emitter) -> None:
emitter.emit_lines('static int CPyLiteralsInit(void)', '{',
'static int is_initialized = 0;',
'if (is_initialized) return 0;', '')
for (_, literal), identifier in self.literals.items():
symbol = emitter.static_name(identifier, None)
if isinstance(literal, int):
emitter.emit_line(
'{} = PyLong_FromString(\"{}\", NULL, 10);'.format(
symbol, str(literal)))
elif isinstance(literal, float):
emitter.emit_line('{} = PyFloat_FromDouble({});'.format(
symbol, str(literal)))
elif isinstance(literal, str):
emitter.emit_line(
'{} = PyUnicode_FromStringAndSize({}, {});'.format(
symbol, *encode_as_c_string(literal)))
elif isinstance(literal, bytes):
emitter.emit_line(
'{} = PyBytes_FromStringAndSize({}, {});'.format(
symbol, *encode_bytes_as_c_string(literal)))
else:
assert False, (
'Literals must be integers, floating point numbers, or strings,',
'but the provided literal is of type {}'.format(
type(literal)))
emitter.emit_lines('if ({} == NULL)'.format(symbol),
' return -1;')
emitter.emit_lines(
'is_initialized = 1;',
'return 0;',
'}',
)
def generate_wrapper_function(fn: FuncIR,
emitter: Emitter,
source_path: str,
module_name: str) -> None:
"""Generates a CPython-compatible wrapper function for a native function.
In particular, this handles unboxing the arguments, calling the native function, and
then boxing the return value.
"""
emitter.emit_line('{} {{'.format(wrapper_function_header(fn, emitter.names)))
# If we hit an error while processing arguments, then we emit a
# traceback frame to make it possible to debug where it happened.
# Unlike traceback frames added for exceptions seen in IR, we do this
# even if there is no `traceback_name`. This is because the error will
# have originated here and so we need it in the traceback.
globals_static = emitter.static_name('globals', module_name)
traceback_code = 'CPy_AddTraceback("%s", "%s", %d, %s);' % (
source_path.replace("\\", "\\\\"),
fn.traceback_name or fn.name,
fn.line,
globals_static)
# If fn is a method, then the first argument is a self param
real_args = list(fn.args)
if fn.class_name and not fn.decl.kind == FUNC_STATICMETHOD:
arg = real_args.pop(0)
emitter.emit_line('PyObject *obj_{} = self;'.format(arg.name))
# Need to order args as: required, optional, kwonly optional, kwonly required
# This is because CPyArg_ParseTupleAndKeywords format string requires
# them grouped in that way.
groups = [[arg for arg in real_args if arg.kind == k] for k in range(ARG_NAMED_OPT + 1)]
reordered_args = groups[ARG_POS] + groups[ARG_OPT] + groups[ARG_NAMED_OPT] + groups[ARG_NAMED]
arg_names = ''.join('"{}", '.format(arg.name) for arg in reordered_args)
emitter.emit_line('static char *kwlist[] = {{{}0}};'.format(arg_names))
for arg in real_args:
emitter.emit_line('PyObject *obj_{}{};'.format(
arg.name, ' = NULL' if arg.optional else ''))
cleanups = ['CPy_DECREF(obj_{});'.format(arg.name)
for arg in groups[ARG_STAR] + groups[ARG_STAR2]]
arg_ptrs = [] # type: List[str]
if groups[ARG_STAR] or groups[ARG_STAR2]:
arg_ptrs += ['&obj_{}'.format(groups[ARG_STAR][0].name) if groups[ARG_STAR] else 'NULL']
arg_ptrs += ['&obj_{}'.format(groups[ARG_STAR2][0].name) if groups[ARG_STAR2] else 'NULL']
arg_ptrs += ['&obj_{}'.format(arg.name) for arg in reordered_args]
emitter.emit_lines(
'if (!CPyArg_ParseTupleAndKeywords(args, kw, "{}", kwlist{})) {{'.format(
make_format_string(fn.name, groups), ''.join(', ' + n for n in arg_ptrs)),
'return NULL;',
'}')
generate_wrapper_core(fn, emitter, groups[ARG_OPT] + groups[ARG_NAMED_OPT],
cleanups=cleanups,
traceback_code=traceback_code)
emitter.emit_line('}')
def declare_finals(
self, module: str, final_names: Iterable[Tuple[str, RType]], emitter: Emitter) -> None:
for name, typ in final_names:
static_name = emitter.static_name(name, module)
emitter.context.declarations[static_name] = HeaderDeclaration(
'{}{};'.format(emitter.ctype_spaced(typ), static_name),
needs_export=True)
def declare_finals(self, module: str, final_names: Iterable[Tuple[str,
RType]],
emitter: Emitter) -> None:
for name, typ in final_names:
static_name = emitter.static_name(name, module)
emitter.emit_line('extern {}{};'.format(emitter.ctype_spaced(typ),
static_name))
def generate_globals_init(self, emitter: Emitter) -> None:
emitter.emit_lines(
'',
'int CPyGlobalsInit(void)',
'{',
'static int is_initialized = 0;',
'if (is_initialized) return 0;',
''
)
emitter.emit_line('CPy_Init();')
for symbol, fixup in self.simple_inits:
emitter.emit_line('{} = {};'.format(symbol, fixup))
for (_, literal), identifier in self.literals.items():
symbol = emitter.static_name(identifier, None)
if isinstance(literal, int):
actual_symbol = symbol
symbol = INT_PREFIX + symbol
emitter.emit_line(
'PyObject * {} = PyLong_FromString(\"{}\", NULL, 10);'.format(
symbol, str(literal))
)
elif isinstance(literal, float):
emitter.emit_line(
'{} = PyFloat_FromDouble({});'.format(symbol, str(literal))
)
elif isinstance(literal, complex):
emitter.emit_line(
'{} = PyComplex_FromDoubles({}, {});'.format(
symbol, str(literal.real), str(literal.imag))
)
elif isinstance(literal, str):
emitter.emit_line(
'{} = PyUnicode_FromStringAndSize({}, {});'.format(
symbol, *encode_as_c_string(literal))
)
elif isinstance(literal, bytes):
emitter.emit_line(
'{} = PyBytes_FromStringAndSize({}, {});'.format(
symbol, *encode_bytes_as_c_string(literal))
)
else:
assert False, ('Literals must be integers, floating point numbers, or strings,',
'but the provided literal is of type {}'.format(type(literal)))
emitter.emit_lines('if (unlikely({} == NULL))'.format(symbol),
' return -1;')
# Ints have an unboxed representation.
if isinstance(literal, int):
emitter.emit_line(
'{} = CPyTagged_FromObject({});'.format(actual_symbol, symbol)
)
emitter.emit_lines(
'is_initialized = 1;',
'return 0;',
'}',
)
def declare_module(self, module_name: str, emitter: Emitter) -> None:
# We declare two globals for each module:
# one used internally in the implementation of module init to cache results
# and prevent infinite recursion in import cycles, and one used
# by other modules to refer to it.
internal_static_name = self.module_internal_static_name(module_name, emitter)
self.declare_global('CPyModule *', internal_static_name, initializer='NULL')
static_name = emitter.static_name('module', module_name)
self.declare_global('CPyModule *', static_name)
self.simple_inits.append((static_name, 'Py_None'))
def final_definition(
self, module: str, name: str, typ: RType, emitter: Emitter) -> str:
static_name = emitter.static_name(name, module)
# Here we rely on the fact that undefined value and error value are always the same
if isinstance(typ, RTuple):
# We need to inline because initializer must be static
undefined = '{{ {} }}'.format(''.join(emitter.tuple_undefined_value_helper(typ)))
else:
undefined = emitter.c_undefined_value(typ)
return '{}{} = {};'.format(emitter.ctype_spaced(typ), static_name, undefined)
def define_finals(self, final_names: Iterable[Tuple[str, RType]], emitter: Emitter) -> None:
for name, typ in final_names:
static_name = emitter.static_name(name, 'final')
# Here we rely on the fact that undefined value and error value are always the same
if isinstance(typ, RTuple):
# We need to inline because initializer must be static
undefined = '{{ {} }}'.format(''.join(emitter.tuple_undefined_value_helper(typ)))
else:
undefined = emitter.c_undefined_value(typ)
emitter.emit_line('{}{} = {};'.format(emitter.ctype_spaced(typ), static_name,
undefined))
def declare_static_pyobject(self, identifier: str,
emitter: Emitter) -> None:
symbol = emitter.static_name(identifier, None)
self.declare_global('PyObject *', symbol)
def module_static_name(self, module_name: str, emitter: Emitter) -> str:
return emitter.static_name('module', module_name)
def declare_internal_globals(self, module_name: str,
emitter: Emitter) -> None:
static_name = emitter.static_name('globals', module_name)
self.declare_global('PyObject *', static_name)
def generate_module_def(self, emitter: Emitter, module_name: str,
module: ModuleIR) -> None:
"""Emit the PyModuleDef struct for a module and the module init function."""
# Emit module methods
module_prefix = emitter.names.private_name(module_name)
emitter.emit_line(
'static PyMethodDef {}module_methods[] = {{'.format(module_prefix))
for fn in module.functions:
if fn.class_name is not None or fn.name == TOP_LEVEL_NAME:
continue
emitter.emit_line((
'{{"{name}", (PyCFunction){prefix}{cname}, METH_VARARGS | METH_KEYWORDS, '
'NULL /* docstring */}},').format(name=fn.name,
cname=fn.cname(
emitter.names),
prefix=PREFIX))
emitter.emit_line('{NULL, NULL, 0, NULL}')
emitter.emit_line('};')
emitter.emit_line()
# Emit module definition struct
emitter.emit_lines(
'static struct PyModuleDef {}module = {{'.format(module_prefix),
'PyModuleDef_HEAD_INIT,', '"{}",'.format(module_name),
'NULL, /* docstring */',
'-1, /* size of per-interpreter state of the module,',
' or -1 if the module keeps state in global variables. */',
'{}module_methods'.format(module_prefix), '};')
emitter.emit_line()
# Emit module init function. If we are compiling just one module, this
# will be the C API init function. If we are compiling 2+ modules, we
# generate a shared library for the modules and shims that call into
# the shared library, and in this case we use an internal module
# initialized function that will be called by the shim.
if len(self.modules) == 1:
declaration = 'PyMODINIT_FUNC PyInit_{}(void)'
else:
declaration = 'PyObject *CPyInit_{}(void)'
emitter.emit_lines(declaration.format(module_name), '{')
module_static = self.module_static_name(module_name, emitter)
emitter.emit_lines('if ({} != NULL) {{'.format(module_static),
'Py_INCREF({});'.format(module_static),
'return {};'.format(module_static), '}')
emitter.emit_lines(
'{} = PyModule_Create(&{}module);'.format(module_static,
module_prefix),
'if ({} == NULL)'.format(module_static), ' return NULL;')
module_globals = emitter.static_name('globals', module_name)
emitter.emit_lines(
'{} = PyModule_GetDict({});'.format(module_globals, module_static),
'if ({} == NULL)'.format(module_globals), ' return NULL;')
self.generate_imports_init_section(module.imports, emitter)
self.generate_from_imports_init_section(
module_static,
module.imports,
module.from_imports,
emitter,
)
for cl in module.classes:
type_struct = emitter.type_struct_name(cl)
# FIXME: This ought to be driven by emitclass, maybe?
if cl.traits:
real_base = cl.real_base()
bases = ([real_base] if real_base else []) + cl.traits
emitter.emit_lines(
'{}.tp_bases = PyTuple_Pack({}, {});'.format(
type_struct, len(bases),
', '.join('&{}'.format(emitter.type_struct_name(b))
for b in bases)))
emitter.emit_lines(
'if (PyType_Ready(&{}) < 0)'.format(type_struct),
' return NULL;')
for (_, literal), identifier in module.literals.items():
symbol = emitter.static_name(identifier, None)
if isinstance(literal, int):
emitter.emit_lines(
'{} = PyLong_FromString(\"{}\", NULL, 10);'.format(
symbol, str(literal)))
elif isinstance(literal, float):
emitter.emit_lines('{} = PyFloat_FromDouble({});'.format(
symbol, str(literal)))
elif isinstance(literal, str):
emitter.emit_lines(
'{} = PyUnicode_FromStringAndSize({}, {});'.format(
symbol, *encode_as_c_string(literal)),
'if ({} == NULL)'.format(symbol),
' return NULL;',
)
elif isinstance(literal, bytes):
emitter.emit_lines(
'{} = PyBytes_FromStringAndSize({}, {});'.format(
symbol, *encode_bytes_as_c_string(literal)),
'if ({} == NULL)'.format(symbol),
' return NULL;',
)
else:
assert False, (
'Literals must be integers, floating point numbers, or strings,',
'but the provided literal is of type {}'.format(
type(literal)))
for cl in module.classes:
name = cl.name
type_struct = emitter.type_struct_name(cl)
emitter.emit_lines(
'Py_INCREF(&{});'.format(type_struct),
'PyModule_AddObject({}, "{}", (PyObject *)&{});'.format(
module_static, name, type_struct))
self.generate_top_level_call(module, emitter)
emitter.emit_line('return {};'.format(module_static))
emitter.emit_line('}')
def generate_module_def(self, emitter: Emitter, module_name: str, module: ModuleIR) -> None:
"""Emit the PyModuleDef struct for a module and the module init function."""
# Emit module methods
module_prefix = emitter.names.private_name(module_name)
emitter.emit_line('static PyMethodDef {}module_methods[] = {{'.format(module_prefix))
for fn in module.functions:
if fn.class_name is not None or fn.name == TOP_LEVEL_NAME:
continue
emitter.emit_line(
('{{"{name}", (PyCFunction){prefix}{cname}, METH_VARARGS | METH_KEYWORDS, '
'NULL /* docstring */}},').format(
name=fn.name,
cname=fn.cname(emitter.names),
prefix=PREFIX))
emitter.emit_line('{NULL, NULL, 0, NULL}')
emitter.emit_line('};')
emitter.emit_line()
# Emit module definition struct
emitter.emit_lines('static struct PyModuleDef {}module = {{'.format(module_prefix),
'PyModuleDef_HEAD_INIT,',
'"{}",'.format(module_name),
'NULL, /* docstring */',
'-1, /* size of per-interpreter state of the module,',
' or -1 if the module keeps state in global variables. */',
'{}module_methods'.format(module_prefix),
'};')
emitter.emit_line()
# Emit module init function. If we are compiling just one module, this
# will be the C API init function. If we are compiling 2+ modules, we
# generate a shared library for the modules and shims that call into
# the shared library, and in this case we use an internal module
# initialized function that will be called by the shim.
if not self.use_shared_lib:
declaration = 'PyMODINIT_FUNC PyInit_{}(void)'.format(module_name)
else:
declaration = 'PyObject *CPyInit_{}(void)'.format(exported_name(module_name))
emitter.emit_lines(declaration,
'{')
# Store the module reference in a static and return it when necessary.
# This is separate from the *global* reference to the module that will
# be populated when it is imported by a compiled module. We want that
# reference to only be populated when the module has been succesfully
# imported, whereas this we want to have to stop a circular import.
module_static = self.module_internal_static_name(module_name, emitter)
emitter.emit_lines('if ({}) {{'.format(module_static),
'Py_INCREF({});'.format(module_static),
'return {};'.format(module_static),
'}')
emitter.emit_lines('{} = PyModule_Create(&{}module);'.format(module_static, module_prefix),
'if (unlikely({} == NULL))'.format(module_static),
' return NULL;')
emitter.emit_line(
'PyObject *modname = PyObject_GetAttrString((PyObject *){}, "__name__");'.format(
module_static))
module_globals = emitter.static_name('globals', module_name)
emitter.emit_lines('{} = PyModule_GetDict({});'.format(module_globals, module_static),
'if (unlikely({} == NULL))'.format(module_globals),
' return NULL;')
# HACK: Manually instantiate generated classes here
for cl in module.classes:
if cl.is_generated:
type_struct = emitter.type_struct_name(cl)
emitter.emit_lines(
'{t} = (PyTypeObject *)CPyType_FromTemplate({t}_template, NULL, modname);'.
format(t=type_struct))
emitter.emit_lines('if (unlikely(!{}))'.format(type_struct),
' return NULL;')
emitter.emit_lines('if (CPyGlobalsInit() < 0)',
' return NULL;')
self.generate_top_level_call(module, emitter)
emitter.emit_lines('Py_DECREF(modname);')
emitter.emit_line('return {};'.format(module_static))
emitter.emit_line('}')
def module_internal_static_name(self, module_name: str,
emitter: Emitter) -> str:
return emitter.static_name(module_name + '_internal',
None,
prefix=MODULE_PREFIX)
def generate_c_for_modules(self) -> str:
emitter = Emitter(self.context)
module_irs = [module_ir for _, module_ir in self.modules]
for module_name, module in self.modules:
self.declare_module(module_name, emitter)
self.declare_internal_globals(module_name, emitter)
self.declare_imports(module.imports, emitter)
for (_, literal), identifier in self.literals.items():
if isinstance(literal, int):
symbol = emitter.static_name(identifier, None)
self.declare_global('CPyTagged ', symbol)
else:
self.declare_static_pyobject(identifier, emitter)
for module_name, module in self.modules:
# Finals must be last (types can depend on declared above)
self.declare_finals(module.final_names, emitter)
for module in module_irs:
for fn in module.functions:
generate_function_declaration(fn, emitter)
self.generate_literals(emitter)
classes = []
for module_name, module in self.modules:
classes.extend([(module_name, cl) for cl in module.classes])
# We must topo sort so that base classes are generated first.
classes = sort_classes(classes)
for module_name, cl in classes:
generate_class_type_decl(cl, emitter)
for module_name, cl in classes:
generate_class(cl, module_name, emitter)
emitter.emit_line()
# Generate Python extension module definitions and module initialization functions.
for module_name, module in self.modules:
self.generate_module_def(emitter, module_name, module)
for module_name, module in self.modules:
for fn in module.functions:
emitter.emit_line()
generate_native_function(fn, emitter,
self.source_paths[module_name],
module_name)
if fn.name != TOP_LEVEL_NAME:
emitter.emit_line()
generate_wrapper_function(fn, emitter)
declarations = Emitter(self.context)
declarations.emit_line('#include <Python.h>')
declarations.emit_line('#include <CPy.h>')
declarations.emit_line()
for declaration in self.toposort_declarations():
declarations.emit_lines(*declaration.body)
for static_def in self.context.statics.values():
declarations.emit_line(static_def)
return ''.join(declarations.fragments + emitter.fragments)
