def toposort_declarations(self) -> List[HeaderDeclaration]:
"""Topologically sort the declaration dict by dependencies.
Declarations can require other declarations to come prior in C (such as declaring structs).
In order to guarantee that the C output will compile the declarations will thus need to
be properly ordered. This simple DFS guarantees that we have a proper ordering.
This runs in O(V + E).
"""
result = []
marked_declarations = OrderedDict() # type: Dict[str, MarkedDeclaration]
for k, v in self.context.declarations.items():
marked_declarations[k] = MarkedDeclaration(v, False)
def _toposort_visit(name: str) -> None:
decl = marked_declarations[name]
if decl.mark:
return
for child in decl.declaration.dependencies:
_toposort_visit(child)
result.append(decl.declaration)
decl.mark = True
for name, marked_declaration in marked_declarations.items():
_toposort_visit(name)
return result
def generate_class(cl: ClassIR, module: str, emitter: Emitter) -> None:
"""Generate C code for a class.
This is the main entry point to the module.
"""
name = cl.name
name_prefix = cl.name_prefix(emitter.names)
setup_name = '{}_setup'.format(name_prefix)
new_name = '{}_new'.format(name_prefix)
members_name = '{}_members'.format(name_prefix)
getseters_name = '{}_getseters'.format(name_prefix)
vtable_name = '{}_vtable'.format(name_prefix)
traverse_name = '{}_traverse'.format(name_prefix)
clear_name = '{}_clear'.format(name_prefix)
dealloc_name = '{}_dealloc'.format(name_prefix)
methods_name = '{}_methods'.format(name_prefix)
vtable_setup_name = '{}_trait_vtable_setup'.format(name_prefix)
fields = OrderedDict() # type: Dict[str, str]
fields['tp_name'] = '"{}"'.format(name)
generate_full = not cl.is_trait and not cl.builtin_base
needs_getseters = not cl.is_generated
if not cl.builtin_base:
fields['tp_new'] = new_name
if generate_full:
fields['tp_dealloc'] = '(destructor){}_dealloc'.format(name_prefix)
fields['tp_traverse'] = '(traverseproc){}_traverse'.format(name_prefix)
fields['tp_clear'] = '(inquiry){}_clear'.format(name_prefix)
if needs_getseters:
fields['tp_getset'] = getseters_name
fields['tp_methods'] = methods_name
def emit_line() -> None:
emitter.emit_line()
emit_line()
# If the class has a method to initialize default attribute
# values, we need to call it during initialization.
defaults_fn = cl.get_method('__mypyc_defaults_setup')
# If there is a __init__ method, we'll use it in the native constructor.
init_fn = cl.get_method('__init__')
# Fill out slots in the type object from dunder methods.
fields.update(generate_slots(cl, SLOT_DEFS, emitter))
# Fill out dunder methods that live in tables hanging off the side.
for table_name, type, slot_defs in SIDE_TABLES:
slots = generate_slots(cl, slot_defs, emitter)
if slots:
table_struct_name = generate_side_table_for_class(
cl, table_name, type, slots, emitter)
fields['tp_{}'.format(table_name)] = '&{}'.format(
table_struct_name)
richcompare_name = generate_richcompare_wrapper(cl, emitter)
if richcompare_name:
fields['tp_richcompare'] = richcompare_name
# If the class inherits from python, make space for a __dict__
struct_name = cl.struct_name(emitter.names)
if cl.builtin_base:
base_size = 'sizeof({})'.format(cl.builtin_base)
elif cl.is_trait:
base_size = 'sizeof(PyObject)'
else:
base_size = 'sizeof({})'.format(struct_name)
# Since our types aren't allocated using type() we need to
# populate these fields ourselves if we want them to have correct
# values. PyType_Ready will inherit the offsets from tp_base but
# that isn't what we want.
# XXX: there is no reason for the __weakref__ stuff to be mixed up with __dict__
if cl.has_dict:
# __dict__ lives right after the struct and __weakref__ lives right after that
# TODO: They should get members in the struct instead of doing this nonsense.
weak_offset = '{} + sizeof(PyObject *)'.format(base_size)
emitter.emit_lines(
'PyMemberDef {}[] = {{'.format(members_name),
'{{"__dict__", T_OBJECT_EX, {}, 0, NULL}},'.format(base_size),
'{{"__weakref__", T_OBJECT_EX, {}, 0, NULL}},'.format(weak_offset),
'{0}',
'};',
)
fields['tp_members'] = members_name
fields['tp_basicsize'] = '{} + 2*sizeof(PyObject *)'.format(base_size)
fields['tp_dictoffset'] = base_size
fields['tp_weaklistoffset'] = weak_offset
else:
fields['tp_basicsize'] = base_size
if generate_full:
# Declare setup method that allocates and initializes an object. type is the
# type of the class being initialized, which could be another class if there
# is an interpreted subclass.
emitter.emit_line(
'static PyObject *{}(PyTypeObject *type);'.format(setup_name))
assert cl.ctor is not None
emitter.emit_line(native_function_header(cl.ctor, emitter) + ';')
emit_line()
generate_new_for_class(cl, new_name, vtable_name, setup_name, emitter)
emit_line()
generate_traverse_for_class(cl, traverse_name, emitter)
emit_line()
generate_clear_for_class(cl, clear_name, emitter)
emit_line()
generate_dealloc_for_class(cl, dealloc_name, clear_name, emitter)
emit_line()
if cl.allow_interpreted_subclasses:
shadow_vtable_name = generate_vtables(
cl,
vtable_setup_name + "_shadow",
vtable_name + "_shadow",
emitter,
shadow=True) # type: Optional[str]
emit_line()
else:
shadow_vtable_name = None
vtable_name = generate_vtables(cl,
vtable_setup_name,
vtable_name,
emitter,
shadow=False)
emit_line()
if needs_getseters:
generate_getseter_declarations(cl, emitter)
emit_line()
generate_getseters_table(cl, getseters_name, emitter)
emit_line()
if cl.is_trait:
generate_new_for_trait(cl, new_name, emitter)
generate_methods_table(cl, methods_name, emitter)
emit_line()
flags = [
'Py_TPFLAGS_DEFAULT', 'Py_TPFLAGS_HEAPTYPE', 'Py_TPFLAGS_BASETYPE'
]
if generate_full:
flags.append('Py_TPFLAGS_HAVE_GC')
if cl.has_method('__call__') and USE_VECTORCALL:
fields['tp_vectorcall_offset'] = 'offsetof({}, vectorcall)'.format(
cl.struct_name(emitter.names))
flags.append('_Py_TPFLAGS_HAVE_VECTORCALL')
fields['tp_flags'] = ' | '.join(flags)
emitter.emit_line("static PyTypeObject {}_template_ = {{".format(
emitter.type_struct_name(cl)))
emitter.emit_line("PyVarObject_HEAD_INIT(NULL, 0)")
for field, value in fields.items():
emitter.emit_line(".{} = {},".format(field, value))
emitter.emit_line("};")
emitter.emit_line(
"static PyTypeObject *{t}_template = &{t}_template_;".format(
t=emitter.type_struct_name(cl)))
emitter.emit_line()
if generate_full:
generate_setup_for_class(cl, setup_name, defaults_fn, vtable_name,
shadow_vtable_name, emitter)
emitter.emit_line()
generate_constructor_for_class(cl, cl.ctor, init_fn, setup_name,
vtable_name, emitter)
emitter.emit_line()
if needs_getseters:
generate_getseters(cl, emitter)
class ClassIR:
"""Intermediate representation of a class.
This also describes the runtime structure of native instances.
"""
def __init__(self, name: str, module_name: str, is_trait: bool = False,
is_generated: bool = False, is_abstract: bool = False,
is_ext_class: bool = True) -> None:
self.name = name
self.module_name = module_name
self.is_trait = is_trait
self.is_generated = is_generated
self.is_abstract = is_abstract
self.is_ext_class = is_ext_class
# An augmented class has additional methods separate from what mypyc generates.
# Right now the only one is dataclasses.
self.is_augmented = False
# Does this inherit from a Python class?
self.inherits_python = False
# Do instances of this class have __dict__?
self.has_dict = False
# Do we allow interpreted subclasses? Derived from a mypyc_attr.
self.allow_interpreted_subclasses = False
# If this a subclass of some built-in python class, the name
# of the object for that class. We currently only support this
# in a few ad-hoc cases.
self.builtin_base = None # type: Optional[str]
# Default empty constructor
self.ctor = FuncDecl(name, None, module_name, FuncSignature([], RInstance(self)))
self.attributes = OrderedDict() # type: OrderedDict[str, RType]
# We populate method_types with the signatures of every method before
# we generate methods, and we rely on this information being present.
self.method_decls = OrderedDict() # type: OrderedDict[str, FuncDecl]
# Map of methods that are actually present in an extension class
self.methods = OrderedDict() # type: OrderedDict[str, FuncIR]
# Glue methods for boxing/unboxing when a class changes the type
# while overriding a method. Maps from (parent class overrided, method)
# to IR of glue method.
self.glue_methods = OrderedDict() # type: Dict[Tuple[ClassIR, str], FuncIR]
# Properties are accessed like attributes, but have behavior like method calls.
# They don't belong in the methods dictionary, since we don't want to expose them to
# Python's method API. But we want to put them into our own vtable as methods, so that
# they are properly handled and overridden. The property dictionary values are a tuple
# containing a property getter and an optional property setter.
self.properties = OrderedDict() # type: OrderedDict[str, Tuple[FuncIR, Optional[FuncIR]]]
# We generate these in prepare_class_def so that we have access to them when generating
# other methods and properties that rely on these types.
self.property_types = OrderedDict() # type: OrderedDict[str, RType]
self.vtable = None # type: Optional[Dict[str, int]]
self.vtable_entries = [] # type: VTableEntries
self.trait_vtables = OrderedDict() # type: OrderedDict[ClassIR, VTableEntries]
# N.B: base might not actually quite be the direct base.
# It is the nearest concrete base, but we allow a trait in between.
self.base = None # type: Optional[ClassIR]
self.traits = [] # type: List[ClassIR]
# Supply a working mro for most generated classes. Real classes will need to
# fix it up.
self.mro = [self] # type: List[ClassIR]
# base_mro is the chain of concrete (non-trait) ancestors
self.base_mro = [self] # type: List[ClassIR]
# Direct subclasses of this class (use subclasses() to also incude non-direct ones)
# None if separate compilation prevents this from working
self.children = [] # type: Optional[List[ClassIR]]
@property
def fullname(self) -> str:
return "{}.{}".format(self.module_name, self.name)
def real_base(self) -> Optional['ClassIR']:
"""Return the actual concrete base class, if there is one."""
if len(self.mro) > 1 and not self.mro[1].is_trait:
return self.mro[1]
return None
def vtable_entry(self, name: str) -> int:
assert self.vtable is not None, "vtable not computed yet"
assert name in self.vtable, '%r has no attribute %r' % (self.name, name)
return self.vtable[name]
def attr_details(self, name: str) -> Tuple[RType, 'ClassIR']:
for ir in self.mro:
if name in ir.attributes:
return ir.attributes[name], ir
if name in ir.property_types:
return ir.property_types[name], ir
raise KeyError('%r has no attribute %r' % (self.name, name))
def attr_type(self, name: str) -> RType:
return self.attr_details(name)[0]
def method_decl(self, name: str) -> FuncDecl:
for ir in self.mro:
if name in ir.method_decls:
return ir.method_decls[name]
raise KeyError('%r has no attribute %r' % (self.name, name))
def method_sig(self, name: str) -> FuncSignature:
return self.method_decl(name).sig
def has_method(self, name: str) -> bool:
try:
self.method_decl(name)
except KeyError:
return False
return True
def is_method_final(self, name: str) -> bool:
subs = self.subclasses()
if subs is None:
# TODO: Look at the final attribute!
return False
if self.has_method(name):
method_decl = self.method_decl(name)
for subc in subs:
if subc.method_decl(name) != method_decl:
return False
return True
else:
return not any(subc.has_method(name) for subc in subs)
def has_attr(self, name: str) -> bool:
try:
self.attr_type(name)
except KeyError:
return False
return True
def name_prefix(self, names: NameGenerator) -> str:
return names.private_name(self.module_name, self.name)
def struct_name(self, names: NameGenerator) -> str:
return '{}Object'.format(exported_name(self.fullname))
def get_method_and_class(self, name: str) -> Optional[Tuple[FuncIR, 'ClassIR']]:
for ir in self.mro:
if name in ir.methods:
return ir.methods[name], ir
return None
def get_method(self, name: str) -> Optional[FuncIR]:
res = self.get_method_and_class(name)
return res[0] if res else None
def subclasses(self) -> Optional[Set['ClassIR']]:
"""Return all subclassses of this class, both direct and indirect.
Return None if it is impossible to identify all subclasses, for example
because we are performing separate compilation.
"""
if self.children is None or self.allow_interpreted_subclasses:
return None
result = set(self.children)
for child in self.children:
if child.children:
child_subs = child.subclasses()
if child_subs is None:
return None
result.update(child_subs)
return result
def concrete_subclasses(self) -> Optional[List['ClassIR']]:
"""Return all concrete (i.e. non-trait and non-abstract) subclasses.
Include both direct and indirect subclasses. Place classes with no children first.
"""
subs = self.subclasses()
if subs is None:
return None
concrete = {c for c in subs if not (c.is_trait or c.is_abstract)}
# We place classes with no children first because they are more likely
# to appear in various isinstance() checks. We then sort leafs by name
# to get stable order.
return sorted(concrete, key=lambda c: (len(c.children or []), c.name))
def serialize(self) -> JsonDict:
return {
'name': self.name,
'module_name': self.module_name,
'is_trait': self.is_trait,
'is_ext_class': self.is_ext_class,
'is_abstract': self.is_abstract,
'is_generated': self.is_generated,
'is_augmented': self.is_augmented,
'inherits_python': self.inherits_python,
'has_dict': self.has_dict,
'allow_interpreted_subclasses': self.allow_interpreted_subclasses,
'builtin_base': self.builtin_base,
'ctor': self.ctor.serialize(),
# We serialize dicts as lists to ensure order is preserved
'attributes': [(k, t.serialize()) for k, t in self.attributes.items()],
# We try to serialize a name reference, but if the decl isn't in methods
# then we can't be sure that will work so we serialize the whole decl.
'method_decls': [(k, d.fullname if k in self.methods else d.serialize())
for k, d in self.method_decls.items()],
# We serialize method fullnames out and put methods in a separate dict
'methods': [(k, m.fullname) for k, m in self.methods.items()],
'glue_methods': [
((cir.fullname, k), m.fullname)
for (cir, k), m in self.glue_methods.items()
],
# We serialize properties and property_types separately out of an
# abundance of caution about preserving dict ordering...
'property_types': [(k, t.serialize()) for k, t in self.property_types.items()],
'properties': list(self.properties),
'vtable': self.vtable,
'vtable_entries': serialize_vtable(self.vtable_entries),
'trait_vtables': [
(cir.fullname, serialize_vtable(v)) for cir, v in self.trait_vtables.items()
],
# References to class IRs are all just names
'base': self.base.fullname if self.base else None,
'traits': [cir.fullname for cir in self.traits],
'mro': [cir.fullname for cir in self.mro],
'base_mro': [cir.fullname for cir in self.base_mro],
'children': [
cir.fullname for cir in self.children
] if self.children is not None else None,
}
@classmethod
def deserialize(cls, data: JsonDict, ctx: 'DeserMaps') -> 'ClassIR':
fullname = data['module_name'] + '.' + data['name']
assert fullname in ctx.classes, "Class %s not in deser class map" % fullname
ir = ctx.classes[fullname]
ir.is_trait = data['is_trait']
ir.is_generated = data['is_generated']
ir.is_abstract = data['is_abstract']
ir.is_ext_class = data['is_ext_class']
ir.is_augmented = data['is_augmented']
ir.inherits_python = data['inherits_python']
ir.has_dict = data['has_dict']
ir.allow_interpreted_subclasses = data['allow_interpreted_subclasses']
ir.builtin_base = data['builtin_base']
ir.ctor = FuncDecl.deserialize(data['ctor'], ctx)
ir.attributes = OrderedDict(
(k, deserialize_type(t, ctx)) for k, t in data['attributes']
)
ir.method_decls = OrderedDict((k, ctx.functions[v].decl
if isinstance(v, str) else FuncDecl.deserialize(v, ctx))
for k, v in data['method_decls'])
ir.methods = OrderedDict((k, ctx.functions[v]) for k, v in data['methods'])
ir.glue_methods = OrderedDict(
((ctx.classes[c], k), ctx.functions[v]) for (c, k), v in data['glue_methods']
)
ir.property_types = OrderedDict(
(k, deserialize_type(t, ctx)) for k, t in data['property_types']
)
ir.properties = OrderedDict(
(k, (ir.methods[k], ir.methods.get(PROPSET_PREFIX + k))) for k in data['properties']
)
ir.vtable = data['vtable']
ir.vtable_entries = deserialize_vtable(data['vtable_entries'], ctx)
ir.trait_vtables = OrderedDict(
(ctx.classes[k], deserialize_vtable(v, ctx)) for k, v in data['trait_vtables']
)
base = data['base']
ir.base = ctx.classes[base] if base else None
ir.traits = [ctx.classes[s] for s in data['traits']]
ir.mro = [ctx.classes[s] for s in data['mro']]
ir.base_mro = [ctx.classes[s] for s in data['base_mro']]
ir.children = data['children'] and [ctx.classes[s] for s in data['children']]
return ir