def generate_constructor_for_class(cl: ClassIR, fn: FuncDecl,
init_fn: Optional[FuncIR], setup_name: str,
vtable_name: str, emitter: Emitter) -> None:
"""Generate a native function that allocates and initializes an instance of a class."""
emitter.emit_line('{}'.format(native_function_header(fn, emitter)))
emitter.emit_line('{')
emitter.emit_line('PyObject *self = {}({});'.format(
setup_name, emitter.type_struct_name(cl)))
emitter.emit_line('if (self == NULL)')
emitter.emit_line(' return NULL;')
args = ', '.join(['self'] + [REG_PREFIX + arg.name for arg in fn.sig.args])
if init_fn is not None:
emitter.emit_line('char res = {}{}{}({});'.format(
emitter.get_group_prefix(init_fn.decl), NATIVE_PREFIX,
init_fn.cname(emitter.names), args))
emitter.emit_line('if (res == 2) {')
emitter.emit_line('Py_DECREF(self);')
emitter.emit_line('return NULL;')
emitter.emit_line('}')
# If there is a nontrivial ctor that we didn't define, invoke it via tp_init
elif len(fn.sig.args) > 1:
emitter.emit_line('int res = {}->tp_init({});'.format(
emitter.type_struct_name(cl), args))
emitter.emit_line('if (res < 0) {')
emitter.emit_line('Py_DECREF(self);')
emitter.emit_line('return NULL;')
emitter.emit_line('}')
emitter.emit_line('return self;')
emitter.emit_line('}')
def generate_vtable(entries: VTableEntries,
vtable_name: str,
emitter: Emitter,
subtables: List[Tuple[ClassIR, str, str]],
shadow: bool) -> None:
emitter.emit_line('CPyVTableItem {}_scratch[] = {{'.format(vtable_name))
if subtables:
emitter.emit_line('/* Array of trait vtables */')
for trait, table, offset_table in subtables:
emitter.emit_line(
'(CPyVTableItem){}, (CPyVTableItem){}, (CPyVTableItem){},'.format(
emitter.type_struct_name(trait), table, offset_table))
emitter.emit_line('/* Start of real vtable */')
for entry in entries:
method = entry.shadow_method if shadow and entry.shadow_method else entry.method
emitter.emit_line('(CPyVTableItem){}{}{},'.format(
emitter.get_group_prefix(entry.method.decl),
NATIVE_PREFIX,
method.cname(emitter.names)))
# msvc doesn't allow empty arrays; maybe allowing them at all is an extension?
if not entries:
emitter.emit_line('NULL')
emitter.emit_line('};')
emitter.emit_line('memcpy({name}, {name}_scratch, sizeof({name}));'.format(name=vtable_name))
def generate_class_type_decl(cl: ClassIR, c_emitter: Emitter,
external_emitter: Emitter,
emitter: Emitter) -> None:
context = c_emitter.context
name = emitter.type_struct_name(cl)
context.declarations[name] = HeaderDeclaration(
'PyTypeObject *{};'.format(emitter.type_struct_name(cl)),
needs_export=True)
# If this is a non-extension class, all we want is the type object decl.
if not cl.is_ext_class:
return
generate_object_struct(cl, external_emitter)
generate_full = not cl.is_trait and not cl.builtin_base
if generate_full:
context.declarations[emitter.native_function_name(cl.ctor)] = HeaderDeclaration(
'{};'.format(native_function_header(cl.ctor, emitter)),
needs_export=True,
)
def generate_bin_op_both_wrappers(cl: ClassIR,
fn: FuncIR,
fn_rev: FuncIR,
emitter: Emitter,
gen: 'WrapperGenerator') -> None:
# There's both a forward and a reverse operator method. First
# check if we should try calling the forward one. If the
# argument type check fails, fall back to the reverse method.
#
# Similar to above, we can't perfectly match Python semantics.
# In regular Python code you'd return NotImplemented if the
# operand has the wrong type, but in compiled code we'll never
# get to execute the type check.
emitter.emit_line('if (PyObject_IsInstance(obj_left, (PyObject *){})) {{'.format(
emitter.type_struct_name(cl)))
gen.emit_arg_processing(error=GotoHandler('typefail'), raise_exception=False)
gen.emit_call(not_implemented_handler='goto typefail;')
gen.emit_error_handling()
emitter.emit_line('}')
emitter.emit_label('typefail')
emitter.emit_line('if (PyObject_IsInstance(obj_right, (PyObject *){})) {{'.format(
emitter.type_struct_name(cl)))
gen.set_target(fn_rev)
gen.arg_names = ['right', 'left']
gen.emit_arg_processing(error=GotoHandler('typefail2'), raise_exception=False)
gen.emit_call()
gen.emit_error_handling()
emitter.emit_line('} else {')
emitter.emit_line('_Py_IDENTIFIER({});'.format(fn_rev.name))
emitter.emit_line(
'return CPy_CallReverseOpMethod(obj_left, obj_right, "{}", &PyId_{});'.format(
op_methods_to_symbols[fn.name],
fn_rev.name))
emitter.emit_line('}')
emitter.emit_label('typefail2')
emitter.emit_line('Py_INCREF(Py_NotImplemented);')
emitter.emit_line('return Py_NotImplemented;')
gen.finish()
def generate_setup_for_class(cl: ClassIR,
func_name: str,
defaults_fn: Optional[FuncIR],
vtable_name: str,
shadow_vtable_name: Optional[str],
emitter: Emitter) -> None:
"""Generate a native function that allocates an instance of a class."""
emitter.emit_line('static PyObject *')
emitter.emit_line('{}(PyTypeObject *type)'.format(func_name))
emitter.emit_line('{')
emitter.emit_line('{} *self;'.format(cl.struct_name(emitter.names)))
emitter.emit_line('self = ({struct} *)type->tp_alloc(type, 0);'.format(
struct=cl.struct_name(emitter.names)))
emitter.emit_line('if (self == NULL)')
emitter.emit_line(' return NULL;')
if shadow_vtable_name:
emitter.emit_line('if (type != {}) {{'.format(emitter.type_struct_name(cl)))
emitter.emit_line('self->vtable = {};'.format(shadow_vtable_name))
emitter.emit_line('} else {')
emitter.emit_line('self->vtable = {};'.format(vtable_name))
emitter.emit_line('}')
else:
emitter.emit_line('self->vtable = {};'.format(vtable_name))
if cl.has_method('__call__') and emitter.use_vectorcall():
name = cl.method_decl('__call__').cname(emitter.names)
emitter.emit_line('self->vectorcall = {}{};'.format(PREFIX, name))
for base in reversed(cl.base_mro):
for attr, rtype in base.attributes.items():
emitter.emit_line('self->{} = {};'.format(
emitter.attr(attr), emitter.c_undefined_value(rtype)))
# Initialize attributes to default values, if necessary
if defaults_fn is not None:
emitter.emit_lines(
'if ({}{}((PyObject *)self) == 0) {{'.format(
NATIVE_PREFIX, defaults_fn.cname(emitter.names)),
'Py_DECREF(self);',
'return NULL;',
'}')
emitter.emit_line('return (PyObject *)self;')
emitter.emit_line('}')
def generate_new_for_trait(cl: ClassIR, func_name: str,
emitter: Emitter) -> None:
emitter.emit_line('static PyObject *')
emitter.emit_line(
'{}(PyTypeObject *type, PyObject *args, PyObject *kwds)'.format(
func_name))
emitter.emit_line('{')
emitter.emit_line('if (type != {}) {{'.format(
emitter.type_struct_name(cl)))
emitter.emit_line(
'PyErr_SetString(PyExc_TypeError, '
'"interpreted classes cannot inherit from compiled traits");')
emitter.emit_line('} else {')
emitter.emit_line(
'PyErr_SetString(PyExc_TypeError, "traits may not be directly created");'
)
emitter.emit_line('}')
emitter.emit_line('return NULL;')
emitter.emit_line('}')
def generate_new_for_class(cl: ClassIR, func_name: str, vtable_name: str,
setup_name: str, emitter: Emitter) -> None:
emitter.emit_line('static PyObject *')
emitter.emit_line(
'{}(PyTypeObject *type, PyObject *args, PyObject *kwds)'.format(
func_name))
emitter.emit_line('{')
# TODO: Check and unbox arguments
if not cl.allow_interpreted_subclasses:
emitter.emit_line('if (type != {}) {{'.format(
emitter.type_struct_name(cl)))
emitter.emit_line(
'PyErr_SetString(PyExc_TypeError, "interpreted classes cannot inherit from compiled");'
)
emitter.emit_line('return NULL;')
emitter.emit_line('}')
emitter.emit_line('return {}(type);'.format(setup_name))
emitter.emit_line('}')
def generate_module_def(self, emitter: Emitter, module_name: str,
module: ModuleIR) -> None:
"""Emit the PyModuleDef struct for a module and the module init function."""
# Emit module methods
module_prefix = emitter.names.private_name(module_name)
emitter.emit_line(
'static PyMethodDef {}module_methods[] = {{'.format(module_prefix))
for fn in module.functions:
if fn.class_name is not None or fn.name == TOP_LEVEL_NAME:
continue
emitter.emit_line((
'{{"{name}", (PyCFunction){prefix}{cname}, METH_VARARGS | METH_KEYWORDS, '
'NULL /* docstring */}},').format(name=fn.name,
cname=fn.cname(
emitter.names),
prefix=PREFIX))
emitter.emit_line('{NULL, NULL, 0, NULL}')
emitter.emit_line('};')
emitter.emit_line()
# Emit module definition struct
emitter.emit_lines(
'static struct PyModuleDef {}module = {{'.format(module_prefix),
'PyModuleDef_HEAD_INIT,', '"{}",'.format(module_name),
'NULL, /* docstring */',
'-1, /* size of per-interpreter state of the module,',
' or -1 if the module keeps state in global variables. */',
'{}module_methods'.format(module_prefix), '};')
emitter.emit_line()
# Emit module init function. If we are compiling just one module, this
# will be the C API init function. If we are compiling 2+ modules, we
# generate a shared library for the modules and shims that call into
# the shared library, and in this case we use an internal module
# initialized function that will be called by the shim.
if not self.use_shared_lib:
declaration = 'PyMODINIT_FUNC PyInit_{}(void)'.format(module_name)
else:
declaration = 'PyObject *CPyInit_{}(void)'.format(
exported_name(module_name))
emitter.emit_lines(declaration, '{')
# Store the module reference in a static and return it when necessary.
# This is separate from the *global* reference to the module that will
# be populated when it is imported by a compiled module. We want that
# reference to only be populated when the module has been successfully
# imported, whereas this we want to have to stop a circular import.
module_static = self.module_internal_static_name(module_name, emitter)
emitter.emit_lines('if ({}) {{'.format(module_static),
'Py_INCREF({});'.format(module_static),
'return {};'.format(module_static), '}')
emitter.emit_lines(
'{} = PyModule_Create(&{}module);'.format(module_static,
module_prefix),
'if (unlikely({} == NULL))'.format(module_static),
' return NULL;')
emitter.emit_line(
'PyObject *modname = PyObject_GetAttrString((PyObject *){}, "__name__");'
.format(module_static))
module_globals = emitter.static_name('globals', module_name)
emitter.emit_lines(
'{} = PyModule_GetDict({});'.format(module_globals, module_static),
'if (unlikely({} == NULL))'.format(module_globals),
' return NULL;')
# HACK: Manually instantiate generated classes here
for cl in module.classes:
if cl.is_generated:
type_struct = emitter.type_struct_name(cl)
emitter.emit_lines(
'{t} = (PyTypeObject *)CPyType_FromTemplate({t}_template, NULL, modname);'
.format(t=type_struct))
emitter.emit_lines('if (unlikely(!{}))'.format(type_struct),
' return NULL;')
emitter.emit_lines('if (CPyGlobalsInit() < 0)', ' return NULL;')
self.generate_top_level_call(module, emitter)
emitter.emit_lines('Py_DECREF(modname);')
emitter.emit_line('return {};'.format(module_static))
emitter.emit_line('}')
def generate_class(cl: ClassIR, module: str, emitter: Emitter) -> None:
"""Generate C code for a class.
This is the main entry point to the module.
"""
name = cl.name
name_prefix = cl.name_prefix(emitter.names)
setup_name = '{}_setup'.format(name_prefix)
new_name = '{}_new'.format(name_prefix)
members_name = '{}_members'.format(name_prefix)
getseters_name = '{}_getseters'.format(name_prefix)
vtable_name = '{}_vtable'.format(name_prefix)
traverse_name = '{}_traverse'.format(name_prefix)
clear_name = '{}_clear'.format(name_prefix)
dealloc_name = '{}_dealloc'.format(name_prefix)
methods_name = '{}_methods'.format(name_prefix)
vtable_setup_name = '{}_trait_vtable_setup'.format(name_prefix)
fields: Dict[str, str] = OrderedDict()
fields['tp_name'] = '"{}"'.format(name)
generate_full = not cl.is_trait and not cl.builtin_base
needs_getseters = cl.needs_getseters or not cl.is_generated
if not cl.builtin_base:
fields['tp_new'] = new_name
if generate_full:
fields['tp_dealloc'] = '(destructor){}_dealloc'.format(name_prefix)
fields['tp_traverse'] = '(traverseproc){}_traverse'.format(name_prefix)
fields['tp_clear'] = '(inquiry){}_clear'.format(name_prefix)
if needs_getseters:
fields['tp_getset'] = getseters_name
fields['tp_methods'] = methods_name
def emit_line() -> None:
emitter.emit_line()
emit_line()
# If the class has a method to initialize default attribute
# values, we need to call it during initialization.
defaults_fn = cl.get_method('__mypyc_defaults_setup')
# If there is a __init__ method, we'll use it in the native constructor.
init_fn = cl.get_method('__init__')
# Fill out slots in the type object from dunder methods.
fields.update(generate_slots(cl, SLOT_DEFS, emitter))
# Fill out dunder methods that live in tables hanging off the side.
for table_name, type, slot_defs in SIDE_TABLES:
slots = generate_slots(cl, slot_defs, emitter)
if slots:
table_struct_name = generate_side_table_for_class(
cl, table_name, type, slots, emitter)
fields['tp_{}'.format(table_name)] = '&{}'.format(
table_struct_name)
richcompare_name = generate_richcompare_wrapper(cl, emitter)
if richcompare_name:
fields['tp_richcompare'] = richcompare_name
# If the class inherits from python, make space for a __dict__
struct_name = cl.struct_name(emitter.names)
if cl.builtin_base:
base_size = 'sizeof({})'.format(cl.builtin_base)
elif cl.is_trait:
base_size = 'sizeof(PyObject)'
else:
base_size = 'sizeof({})'.format(struct_name)
# Since our types aren't allocated using type() we need to
# populate these fields ourselves if we want them to have correct
# values. PyType_Ready will inherit the offsets from tp_base but
# that isn't what we want.
# XXX: there is no reason for the __weakref__ stuff to be mixed up with __dict__
if cl.has_dict:
# __dict__ lives right after the struct and __weakref__ lives right after that
# TODO: They should get members in the struct instead of doing this nonsense.
weak_offset = '{} + sizeof(PyObject *)'.format(base_size)
emitter.emit_lines(
'PyMemberDef {}[] = {{'.format(members_name),
'{{"__dict__", T_OBJECT_EX, {}, 0, NULL}},'.format(base_size),
'{{"__weakref__", T_OBJECT_EX, {}, 0, NULL}},'.format(weak_offset),
'{0}',
'};',
)
fields['tp_members'] = members_name
fields['tp_basicsize'] = '{} + 2*sizeof(PyObject *)'.format(base_size)
fields['tp_dictoffset'] = base_size
fields['tp_weaklistoffset'] = weak_offset
else:
fields['tp_basicsize'] = base_size
if generate_full:
# Declare setup method that allocates and initializes an object. type is the
# type of the class being initialized, which could be another class if there
# is an interpreted subclass.
emitter.emit_line(
'static PyObject *{}(PyTypeObject *type);'.format(setup_name))
assert cl.ctor is not None
emitter.emit_line(native_function_header(cl.ctor, emitter) + ';')
emit_line()
generate_new_for_class(cl, new_name, vtable_name, setup_name, emitter)
emit_line()
generate_traverse_for_class(cl, traverse_name, emitter)
emit_line()
generate_clear_for_class(cl, clear_name, emitter)
emit_line()
generate_dealloc_for_class(cl, dealloc_name, clear_name, emitter)
emit_line()
if cl.allow_interpreted_subclasses:
shadow_vtable_name: Optional[str] = generate_vtables(
cl,
vtable_setup_name + "_shadow",
vtable_name + "_shadow",
emitter,
shadow=True)
emit_line()
else:
shadow_vtable_name = None
vtable_name = generate_vtables(cl,
vtable_setup_name,
vtable_name,
emitter,
shadow=False)
emit_line()
if needs_getseters:
generate_getseter_declarations(cl, emitter)
emit_line()
generate_getseters_table(cl, getseters_name, emitter)
emit_line()
if cl.is_trait:
generate_new_for_trait(cl, new_name, emitter)
generate_methods_table(cl, methods_name, emitter)
emit_line()
flags = [
'Py_TPFLAGS_DEFAULT', 'Py_TPFLAGS_HEAPTYPE', 'Py_TPFLAGS_BASETYPE'
]
if generate_full:
flags.append('Py_TPFLAGS_HAVE_GC')
if cl.has_method('__call__') and emitter.use_vectorcall():
fields['tp_vectorcall_offset'] = 'offsetof({}, vectorcall)'.format(
cl.struct_name(emitter.names))
flags.append('_Py_TPFLAGS_HAVE_VECTORCALL')
fields['tp_flags'] = ' | '.join(flags)
emitter.emit_line("static PyTypeObject {}_template_ = {{".format(
emitter.type_struct_name(cl)))
emitter.emit_line("PyVarObject_HEAD_INIT(NULL, 0)")
for field, value in fields.items():
emitter.emit_line(".{} = {},".format(field, value))
emitter.emit_line("};")
emitter.emit_line(
"static PyTypeObject *{t}_template = &{t}_template_;".format(
t=emitter.type_struct_name(cl)))
emitter.emit_line()
if generate_full:
generate_setup_for_class(cl, setup_name, defaults_fn, vtable_name,
shadow_vtable_name, emitter)
emitter.emit_line()
generate_constructor_for_class(cl, cl.ctor, init_fn, setup_name,
vtable_name, emitter)
emitter.emit_line()
if needs_getseters:
generate_getseters(cl, emitter)
def generate_module_def(self, emitter: Emitter, module_name: str,
module: ModuleIR) -> None:
"""Emit the PyModuleDef struct for a module and the module init function."""
# Emit module methods
module_prefix = emitter.names.private_name(module_name)
emitter.emit_line(
f'static PyMethodDef {module_prefix}module_methods[] = {{')
for fn in module.functions:
if fn.class_name is not None or fn.name == TOP_LEVEL_NAME:
continue
name = short_id_from_name(fn.name, fn.decl.shortname, fn.line)
if is_fastcall_supported(fn, emitter.capi_version):
flag = 'METH_FASTCALL'
else:
flag = 'METH_VARARGS'
emitter.emit_line((
'{{"{name}", (PyCFunction){prefix}{cname}, {flag} | METH_KEYWORDS, '
'NULL /* docstring */}},').format(name=name,
cname=fn.cname(
emitter.names),
prefix=PREFIX,
flag=flag))
emitter.emit_line('{NULL, NULL, 0, NULL}')
emitter.emit_line('};')
emitter.emit_line()
# Emit module definition struct
emitter.emit_lines(
f'static struct PyModuleDef {module_prefix}module = {{',
'PyModuleDef_HEAD_INIT,', f'"{module_name}",',
'NULL, /* docstring */',
'-1, /* size of per-interpreter state of the module,',
' or -1 if the module keeps state in global variables. */',
f'{module_prefix}module_methods', '};')
emitter.emit_line()
# Emit module init function. If we are compiling just one module, this
# will be the C API init function. If we are compiling 2+ modules, we
# generate a shared library for the modules and shims that call into
# the shared library, and in this case we use an internal module
# initialized function that will be called by the shim.
if not self.use_shared_lib:
declaration = f'PyMODINIT_FUNC PyInit_{module_name}(void)'
else:
declaration = f'PyObject *CPyInit_{exported_name(module_name)}(void)'
emitter.emit_lines(declaration, '{')
emitter.emit_line('PyObject* modname = NULL;')
# Store the module reference in a static and return it when necessary.
# This is separate from the *global* reference to the module that will
# be populated when it is imported by a compiled module. We want that
# reference to only be populated when the module has been successfully
# imported, whereas this we want to have to stop a circular import.
module_static = self.module_internal_static_name(module_name, emitter)
emitter.emit_lines(f'if ({module_static}) {{',
f'Py_INCREF({module_static});',
f'return {module_static};', '}')
emitter.emit_lines(
f'{module_static} = PyModule_Create(&{module_prefix}module);',
f'if (unlikely({module_static} == NULL))', ' goto fail;')
emitter.emit_line(
'modname = PyObject_GetAttrString((PyObject *){}, "__name__");'.
format(module_static))
module_globals = emitter.static_name('globals', module_name)
emitter.emit_lines(
f'{module_globals} = PyModule_GetDict({module_static});',
f'if (unlikely({module_globals} == NULL))', ' goto fail;')
# HACK: Manually instantiate generated classes here
type_structs: List[str] = []
for cl in module.classes:
type_struct = emitter.type_struct_name(cl)
type_structs.append(type_struct)
if cl.is_generated:
emitter.emit_lines(
'{t} = (PyTypeObject *)CPyType_FromTemplate('
'(PyObject *){t}_template, NULL, modname);'.format(
t=type_struct))
emitter.emit_lines(f'if (unlikely(!{type_struct}))',
' goto fail;')
emitter.emit_lines('if (CPyGlobalsInit() < 0)', ' goto fail;')
self.generate_top_level_call(module, emitter)
emitter.emit_lines('Py_DECREF(modname);')
emitter.emit_line(f'return {module_static};')
emitter.emit_lines('fail:', f'Py_CLEAR({module_static});',
'Py_CLEAR(modname);')
for name, typ in module.final_names:
static_name = emitter.static_name(name, module_name)
emitter.emit_dec_ref(static_name, typ, is_xdec=True)
undef = emitter.c_undefined_value(typ)
emitter.emit_line(f'{static_name} = {undef};')
# the type objects returned from CPyType_FromTemplate are all new references
# so we have to decref them
for t in type_structs:
emitter.emit_line(f'Py_CLEAR({t});')
emitter.emit_line('return NULL;')
emitter.emit_line('}')
