def add_send_to_generator_class(builder: IRBuilder, fn_info: FuncInfo,
fn_decl: FuncDecl, sig: FuncSignature) -> None:
"""Generates the 'send' method for a generator class."""
# FIXME: this is basically the same as add_next...
builder.enter(fn_info)
self_reg = builder.read(
add_self_to_env(builder.environment, fn_info.generator_class.ir))
arg = builder.environment.add_local_reg(Var('arg'), object_rprimitive,
True)
none_reg = builder.none_object()
# Call the helper function with error flags set to Py_None, and return that result.
result = builder.add(
Call(fn_decl,
[self_reg, none_reg, none_reg, none_reg,
builder.read(arg)], fn_info.fitem.line))
builder.add(Return(result))
blocks, env, _, fn_info = builder.leave()
sig = FuncSignature((
RuntimeArg(SELF_NAME, object_rprimitive),
RuntimeArg('arg', object_rprimitive),
), sig.ret_type)
next_fn_decl = FuncDecl('send', fn_info.generator_class.ir.name,
builder.module_name, sig)
next_fn_ir = FuncIR(next_fn_decl, blocks, env)
fn_info.generator_class.ir.methods['send'] = next_fn_ir
builder.functions.append(next_fn_ir)
def test_register(self) -> None:
op = LoadInt(5)
self.block.ops.append(op)
fn = FuncIR(
FuncDecl('myfunc', None, 'mod',
FuncSignature([self.arg], list_rprimitive)), [self.reg],
[self.block])
value_names = generate_names_for_ir(fn.arg_regs, fn.blocks)
emitter = Emitter(EmitterContext(NameGenerator([['mod']])),
value_names)
generate_native_function(fn,
emitter,
'prog.py',
'prog',
optimize_int=False)
result = emitter.fragments
assert_string_arrays_equal([
'PyObject *CPyDef_myfunc(CPyTagged cpy_r_arg) {\n',
' CPyTagged cpy_r_i0;\n',
'CPyL0: ;\n',
' cpy_r_i0 = 10;\n',
'}\n',
],
result,
msg='Generated code invalid')
def transform_mypy_file(builder: IRBuilder, mypyfile: MypyFile) -> None:
if mypyfile.fullname in ('typing', 'abc'):
# These module are special; their contents are currently all
# built-in primitives.
return
builder.set_module(mypyfile.fullname, mypyfile.path)
classes = [node for node in mypyfile.defs if isinstance(node, ClassDef)]
# Collect all classes.
for cls in classes:
ir = builder.mapper.type_to_ir[cls.info]
builder.classes.append(ir)
builder.enter('<top level>')
# Make sure we have a builtins import
builder.gen_import('builtins', -1)
# Generate ops.
for node in mypyfile.defs:
builder.accept(node)
builder.maybe_add_implicit_return()
# Generate special function representing module top level.
blocks, env, ret_type, _ = builder.leave()
sig = FuncSignature([], none_rprimitive)
func_ir = FuncIR(FuncDecl(TOP_LEVEL_NAME, None, builder.module_name, sig),
blocks,
env,
traceback_name="<module>")
builder.functions.append(func_ir)
def test_call_two_args(self) -> None:
decl = FuncDecl('myfn', None, 'mod',
FuncSignature([RuntimeArg('m', int_rprimitive),
RuntimeArg('n', int_rprimitive)],
int_rprimitive))
self.assert_emit(Call(decl, [self.m, self.k], 55),
"cpy_r_r0 = CPyDef_myfn(cpy_r_m, cpy_r_k);")
def prepare_func_def(module_name: str, class_name: Optional[str],
fdef: FuncDef, mapper: Mapper) -> FuncDecl:
kind = FUNC_STATICMETHOD if fdef.is_static else (
FUNC_CLASSMETHOD if fdef.is_class else FUNC_NORMAL)
decl = FuncDecl(fdef.name, class_name, module_name, mapper.fdef_to_sig(fdef), kind)
mapper.func_to_decl[fdef] = decl
return decl
def gen_func_ir(builder: IRBuilder,
blocks: List[BasicBlock],
sig: FuncSignature,
env: Environment,
fn_info: FuncInfo,
cdef: Optional[ClassDef]) -> Tuple[FuncIR, Optional[Value]]:
"""Generate the FuncIR for a function.
This takes the basic blocks, environment, and function info of a
particular function and returns the IR. If the function is nested,
also returns the register containing the instance of the
corresponding callable class.
"""
func_reg = None # type: Optional[Value]
if fn_info.is_nested or fn_info.in_non_ext:
func_ir = add_call_to_callable_class(builder, blocks, sig, env, fn_info)
add_get_to_callable_class(builder, fn_info)
func_reg = instantiate_callable_class(builder, fn_info)
else:
assert isinstance(fn_info.fitem, FuncDef)
func_decl = builder.mapper.func_to_decl[fn_info.fitem]
if fn_info.is_decorated:
class_name = None if cdef is None else cdef.name
func_decl = FuncDecl(fn_info.name, class_name, builder.module_name, sig,
func_decl.kind,
func_decl.is_prop_getter, func_decl.is_prop_setter)
func_ir = FuncIR(func_decl, blocks, env, fn_info.fitem.line,
traceback_name=fn_info.fitem.name)
else:
func_ir = FuncIR(func_decl, blocks, env,
fn_info.fitem.line, traceback_name=fn_info.fitem.name)
return (func_ir, func_reg)
def gen_glue_property(builder: IRBuilder,
sig: FuncSignature,
target: FuncIR,
cls: ClassIR,
base: ClassIR,
line: int,
do_pygetattr: bool) -> FuncIR:
"""Generate glue methods for properties that mediate between different subclass types.
Similarly to methods, properties of derived types can be covariantly subtyped. Thus,
properties also require glue. However, this only requires the return type to change.
Further, instead of a method call, an attribute get is performed.
If do_pygetattr is True, then get the attribute using the Python C
API instead of a native call.
"""
builder.enter()
rt_arg = RuntimeArg(SELF_NAME, RInstance(cls))
arg = builder.read(add_self_to_env(builder.environment, cls), line)
builder.ret_types[-1] = sig.ret_type
if do_pygetattr:
retval = builder.py_get_attr(arg, target.name, line)
else:
retval = builder.add(GetAttr(arg, target.name, line))
retbox = builder.coerce(retval, sig.ret_type, line)
builder.add(Return(retbox))
blocks, env, return_type, _ = builder.leave()
return FuncIR(
FuncDecl(target.name + '__' + base.name + '_glue',
cls.name, builder.module_name, FuncSignature([rt_arg], return_type)),
blocks, env)
def deserialize(cls, data: JsonDict, ctx: 'DeserMaps') -> 'ClassIR':
fullname = data['module_name'] + '.' + data['name']
assert fullname in ctx.classes, "Class %s not in deser class map" % fullname
ir = ctx.classes[fullname]
ir.is_trait = data['is_trait']
ir.is_generated = data['is_generated']
ir.is_abstract = data['is_abstract']
ir.is_ext_class = data['is_ext_class']
ir.is_augmented = data['is_augmented']
ir.inherits_python = data['inherits_python']
ir.has_dict = data['has_dict']
ir.allow_interpreted_subclasses = data['allow_interpreted_subclasses']
ir.needs_getseters = data['needs_getseters']
ir.builtin_base = data['builtin_base']
ir.ctor = FuncDecl.deserialize(data['ctor'], ctx)
ir.attributes = OrderedDict(
(k, deserialize_type(t, ctx)) for k, t in data['attributes']
)
ir.method_decls = OrderedDict((k, ctx.functions[v].decl
if isinstance(v, str) else FuncDecl.deserialize(v, ctx))
for k, v in data['method_decls'])
ir.methods = OrderedDict((k, ctx.functions[v]) for k, v in data['methods'])
ir.glue_methods = OrderedDict(
((ctx.classes[c], k), ctx.functions[v]) for (c, k), v in data['glue_methods']
)
ir.property_types = OrderedDict(
(k, deserialize_type(t, ctx)) for k, t in data['property_types']
)
ir.properties = OrderedDict(
(k, (ir.methods[k], ir.methods.get(PROPSET_PREFIX + k))) for k in data['properties']
)
ir.vtable = data['vtable']
ir.vtable_entries = deserialize_vtable(data['vtable_entries'], ctx)
ir.trait_vtables = OrderedDict(
(ctx.classes[k], deserialize_vtable(v, ctx)) for k, v in data['trait_vtables']
)
base = data['base']
ir.base = ctx.classes[base] if base else None
ir.traits = [ctx.classes[s] for s in data['traits']]
ir.mro = [ctx.classes[s] for s in data['mro']]
ir.base_mro = [ctx.classes[s] for s in data['base_mro']]
ir.children = data['children'] and [ctx.classes[s] for s in data['children']]
return ir
def gen_glue_method(builder: IRBuilder, sig: FuncSignature, target: FuncIR,
cls: ClassIR, base: ClassIR, line: int,
do_pycall: bool,
) -> FuncIR:
"""Generate glue methods that mediate between different method types in subclasses.
For example, if we have:
class A:
def f(builder: IRBuilder, x: int) -> object: ...
then it is totally permissible to have a subclass
class B(A):
def f(builder: IRBuilder, x: object) -> int: ...
since '(object) -> int' is a subtype of '(int) -> object' by the usual
contra/co-variant function subtyping rules.
The trickiness here is that int and object have different
runtime representations in mypyc, so A.f and B.f have
different signatures at the native C level. To deal with this,
we need to generate glue methods that mediate between the
different versions by coercing the arguments and return
values.
If do_pycall is True, then make the call using the C API
instead of a native call.
"""
builder.enter()
builder.ret_types[-1] = sig.ret_type
rt_args = list(sig.args)
if target.decl.kind == FUNC_NORMAL:
rt_args[0] = RuntimeArg(sig.args[0].name, RInstance(cls))
# The environment operates on Vars, so we make some up
fake_vars = [(Var(arg.name), arg.type) for arg in rt_args]
args = [builder.read(builder.add_local_reg(var, type, is_arg=True), line)
for var, type in fake_vars]
arg_names = [arg.name if arg.kind in (ARG_NAMED, ARG_NAMED_OPT) else None
for arg in rt_args]
arg_kinds = [concrete_arg_kind(arg.kind) for arg in rt_args]
if do_pycall:
retval = builder.builder.py_method_call(
args[0], target.name, args[1:], line, arg_kinds[1:], arg_names[1:])
else:
retval = builder.builder.call(target.decl, args, arg_kinds, arg_names, line)
retval = builder.coerce(retval, sig.ret_type, line)
builder.add(Return(retval))
arg_regs, _, blocks, ret_type, _ = builder.leave()
return FuncIR(
FuncDecl(target.name + '__' + base.name + '_glue',
cls.name, builder.module_name,
FuncSignature(rt_args, ret_type),
target.decl.kind),
arg_regs, blocks)
def func_decl(self, name: str) -> FuncDecl:
return FuncDecl(name=name,
class_name=None,
module_name="module",
sig=FuncSignature(
args=[],
ret_type=none_rprimitive,
))
def leave_method(self) -> None:
"""Finish the generation of IR for a method."""
arg_regs, args, blocks, ret_type, fn_info = self.leave()
sig = FuncSignature(args, ret_type)
name = self.function_name_stack.pop()
class_ir = self.class_ir_stack.pop()
decl = FuncDecl(name, class_ir.name, self.module_name, sig)
ir = FuncIR(decl, arg_regs, blocks)
class_ir.methods[name] = ir
class_ir.method_decls[name] = ir.decl
self.functions.append(ir)
def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> 'ModuleIR':
return ModuleIR(
data['fullname'],
data['imports'],
[
ctx.functions[FuncDecl.get_name_from_json(f['decl'])]
for f in data['functions']
],
[ClassIR.deserialize(c, ctx) for c in data['classes']],
[(k, deserialize_type(t, ctx)) for k, t in data['final_names']],
)
def func_decl(self,
name: str,
ret_type: Optional[RType] = None) -> FuncDecl:
if ret_type is None:
ret_type = none_rprimitive
return FuncDecl(
name=name,
class_name=None,
module_name="module",
sig=FuncSignature(
args=[],
ret_type=ret_type,
),
)
def test_simple(self) -> None:
self.block.ops.append(Return(self.reg))
fn = FuncIR(FuncDecl('myfunc', None, 'mod', FuncSignature([self.arg], int_rprimitive)),
[self.block], self.env)
emitter = Emitter(EmitterContext(NameGenerator([['mod']])))
generate_native_function(fn, emitter, 'prog.py', 'prog', optimize_int=False)
result = emitter.fragments
assert_string_arrays_equal(
[
'CPyTagged CPyDef_myfunc(CPyTagged cpy_r_arg) {\n',
'CPyL0: ;\n',
' return cpy_r_arg;\n',
'}\n',
],
result, msg='Generated code invalid')
def add_throw_to_generator_class(builder: IRBuilder, fn_info: FuncInfo,
fn_decl: FuncDecl,
sig: FuncSignature) -> None:
"""Generates the 'throw' method for a generator class."""
builder.enter(fn_info)
self_reg = builder.read(
add_self_to_env(builder.environment, fn_info.generator_class.ir))
# Add the type, value, and traceback variables to the environment.
typ = builder.environment.add_local_reg(Var('type'), object_rprimitive,
True)
val = builder.environment.add_local_reg(Var('value'), object_rprimitive,
True)
tb = builder.environment.add_local_reg(Var('traceback'), object_rprimitive,
True)
# Because the value and traceback arguments are optional and hence
# can be NULL if not passed in, we have to assign them Py_None if
# they are not passed in.
none_reg = builder.none_object()
builder.assign_if_null(val, lambda: none_reg, builder.fn_info.fitem.line)
builder.assign_if_null(tb, lambda: none_reg, builder.fn_info.fitem.line)
# Call the helper function using the arguments passed in, and return that result.
result = builder.add(
Call(fn_decl, [
self_reg,
builder.read(typ),
builder.read(val),
builder.read(tb), none_reg
], fn_info.fitem.line))
builder.add(Return(result))
blocks, env, _, fn_info = builder.leave()
# Create the FuncSignature for the throw function. Note that the
# value and traceback fields are optional, and are assigned to if
# they are not passed in inside the body of the throw function.
sig = FuncSignature((RuntimeArg(
SELF_NAME, object_rprimitive), RuntimeArg('type', object_rprimitive),
RuntimeArg('value', object_rprimitive, ARG_OPT),
RuntimeArg('traceback', object_rprimitive, ARG_OPT)),
sig.ret_type)
throw_fn_decl = FuncDecl('throw', fn_info.generator_class.ir.name,
builder.module_name, sig)
throw_fn_ir = FuncIR(throw_fn_decl, blocks, env)
fn_info.generator_class.ir.methods['throw'] = throw_fn_ir
builder.functions.append(throw_fn_ir)
def add_iter_to_generator_class(builder: IRBuilder, fn_info: FuncInfo) -> None:
"""Generates the '__iter__' method for a generator class."""
builder.enter(fn_info)
self_target = add_self_to_env(builder.environment,
fn_info.generator_class.ir)
builder.add(Return(builder.read(self_target, fn_info.fitem.line)))
blocks, env, _, fn_info = builder.leave()
# Next, add the actual function as a method of the generator class.
sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive), ),
object_rprimitive)
iter_fn_decl = FuncDecl('__iter__', fn_info.generator_class.ir.name,
builder.module_name, sig)
iter_fn_ir = FuncIR(iter_fn_decl, blocks, env)
fn_info.generator_class.ir.methods['__iter__'] = iter_fn_ir
builder.functions.append(iter_fn_ir)
def gen_dispatch_func_ir(
builder: IRBuilder,
fitem: FuncDef,
main_func_name: str,
dispatch_name: str,
sig: FuncSignature,
) -> FuncIR:
"""Create a dispatch function (a function that checks the first argument type and dispatches
to the correct implementation)
"""
builder.enter()
generate_singledispatch_dispatch_function(builder, main_func_name, fitem)
args, _, blocks, _, fn_info = builder.leave()
func_decl = FuncDecl(dispatch_name, None, builder.module_name, sig)
dispatch_func_ir = FuncIR(func_decl, args, blocks)
return dispatch_func_ir
def add_call_to_callable_class(builder: IRBuilder,
blocks: List[BasicBlock],
sig: FuncSignature,
env: Environment,
fn_info: FuncInfo) -> FuncIR:
"""Generates a '__call__' method for a callable class representing a nested function.
This takes the blocks, signature, and environment associated with a function definition and
uses those to build the '__call__' method of a given callable class, used to represent that
function. Note that a 'self' parameter is added to its list of arguments, as the nested
function becomes a class method.
"""
sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive),) + sig.args, sig.ret_type)
call_fn_decl = FuncDecl('__call__', fn_info.callable_class.ir.name, builder.module_name, sig)
call_fn_ir = FuncIR(call_fn_decl, blocks, env,
fn_info.fitem.line, traceback_name=fn_info.fitem.name)
fn_info.callable_class.ir.methods['__call__'] = call_fn_ir
return call_fn_ir
def gen_glue_ne_method(builder: IRBuilder, cls: ClassIR, line: int) -> FuncIR:
"""Generate a "__ne__" method from a "__eq__" method. """
builder.enter()
rt_args = (RuntimeArg("self", RInstance(cls)), RuntimeArg("rhs", object_rprimitive))
# The environment operates on Vars, so we make some up
fake_vars = [(Var(arg.name), arg.type) for arg in rt_args]
args = [
builder.read(
builder.environment.add_local_reg(
var, type, is_arg=True
),
line
)
for var, type in fake_vars
] # type: List[Value]
builder.ret_types[-1] = object_rprimitive
# If __eq__ returns NotImplemented, then __ne__ should also
not_implemented_block, regular_block = BasicBlock(), BasicBlock()
eqval = builder.add(MethodCall(args[0], '__eq__', [args[1]], line))
not_implemented = builder.add(LoadAddress(not_implemented_op.type,
not_implemented_op.src, line))
builder.add(Branch(
builder.translate_is_op(eqval, not_implemented, 'is', line),
not_implemented_block,
regular_block,
Branch.BOOL_EXPR))
builder.activate_block(regular_block)
retval = builder.coerce(
builder.unary_op(eqval, 'not', line), object_rprimitive, line
)
builder.add(Return(retval))
builder.activate_block(not_implemented_block)
builder.add(Return(not_implemented))
blocks, env, ret_type, _ = builder.leave()
return FuncIR(
FuncDecl('__ne__', cls.name, builder.module_name,
FuncSignature(rt_args, ret_type)),
blocks, env)
def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value:
# OK AND NOW THE FUN PART
base_exprs = cdef.base_type_exprs + cdef.removed_base_type_exprs
if base_exprs:
bases = [builder.accept(x) for x in base_exprs]
tp_bases = builder.new_tuple(bases, cdef.line)
else:
tp_bases = builder.add(
LoadErrorValue(object_rprimitive, is_borrowed=True))
modname = builder.load_static_unicode(builder.module_name)
template = builder.add(
LoadStatic(object_rprimitive, cdef.name + "_template",
builder.module_name, NAMESPACE_TYPE))
# Create the class
tp = builder.call_c(pytype_from_template_op, [template, tp_bases, modname],
cdef.line)
# Immediately fix up the trait vtables, before doing anything with the class.
ir = builder.mapper.type_to_ir[cdef.info]
if not ir.is_trait and not ir.builtin_base:
builder.add(
Call(
FuncDecl(cdef.name + '_trait_vtable_setup',
None, builder.module_name,
FuncSignature([], bool_rprimitive)), [], -1))
# Populate a '__mypyc_attrs__' field containing the list of attrs
builder.call_c(py_setattr_op, [
tp,
builder.load_static_unicode('__mypyc_attrs__'),
create_mypyc_attrs_tuple(builder, builder.mapper.type_to_ir[cdef.info],
cdef.line)
], cdef.line)
# Save the class
builder.add(InitStatic(tp, cdef.name, builder.module_name, NAMESPACE_TYPE))
# Add it to the dict
builder.call_c(dict_set_item_op, [
builder.load_globals_dict(),
builder.load_static_unicode(cdef.name),
tp,
], cdef.line)
return tp
def add_helper_to_generator_class(builder: IRBuilder,
arg_regs: List[Register],
blocks: List[BasicBlock],
sig: FuncSignature,
fn_info: FuncInfo) -> FuncDecl:
"""Generates a helper method for a generator class, called by '__next__' and 'throw'."""
sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive),
RuntimeArg('type', object_rprimitive),
RuntimeArg('value', object_rprimitive),
RuntimeArg('traceback', object_rprimitive),
RuntimeArg('arg', object_rprimitive)
), sig.ret_type)
helper_fn_decl = FuncDecl('__mypyc_generator_helper__', fn_info.generator_class.ir.name,
builder.module_name, sig)
helper_fn_ir = FuncIR(helper_fn_decl, arg_regs, blocks,
fn_info.fitem.line, traceback_name=fn_info.fitem.name)
fn_info.generator_class.ir.methods['__mypyc_generator_helper__'] = helper_fn_ir
builder.functions.append(helper_fn_ir)
return helper_fn_decl
def add_call_to_callable_class(builder: IRBuilder,
args: List[Register],
blocks: List[BasicBlock],
sig: FuncSignature,
fn_info: FuncInfo) -> FuncIR:
"""Generate a '__call__' method for a callable class representing a nested function.
This takes the blocks and signature associated with a function
definition and uses those to build the '__call__' method of a
given callable class, used to represent that function.
"""
# Since we create a method, we also add a 'self' parameter.
sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive),) + sig.args, sig.ret_type)
call_fn_decl = FuncDecl('__call__', fn_info.callable_class.ir.name, builder.module_name, sig)
call_fn_ir = FuncIR(call_fn_decl, args, blocks,
fn_info.fitem.line, traceback_name=fn_info.fitem.name)
fn_info.callable_class.ir.methods['__call__'] = call_fn_ir
fn_info.callable_class.ir.method_decls['__call__'] = call_fn_decl
return call_fn_ir
def test_register(self) -> None:
self.env.temp_index = 0
op = LoadInt(5)
self.block.ops.append(op)
self.env.add_op(op)
fn = FuncIR(FuncDecl('myfunc', None, 'mod', FuncSignature([self.arg], list_rprimitive)),
[self.block], self.env)
emitter = Emitter(EmitterContext(NameGenerator([['mod']])))
generate_native_function(fn, emitter, 'prog.py', 'prog', False)
result = emitter.fragments
assert_string_arrays_equal(
[
'PyObject *CPyDef_myfunc(CPyTagged cpy_r_arg) {\n',
' CPyTagged cpy_r_i0;\n',
'CPyL0: ;\n',
' cpy_r_i0 = 10;\n',
'}\n',
],
result, msg='Generated code invalid')
def add_get_to_callable_class(builder: IRBuilder, fn_info: FuncInfo) -> None:
"""Generate the '__get__' method for a callable class."""
line = fn_info.fitem.line
builder.enter(fn_info)
vself = builder.read(
builder.environment.add_local_reg(Var(SELF_NAME), object_rprimitive,
True))
instance = builder.environment.add_local_reg(Var('instance'),
object_rprimitive, True)
builder.environment.add_local_reg(Var('owner'), object_rprimitive, True)
# If accessed through the class, just return the callable
# object. If accessed through an object, create a new bound
# instance method object.
instance_block, class_block = BasicBlock(), BasicBlock()
comparison = builder.translate_is_op(builder.read(instance),
builder.none_object(), 'is', line)
builder.add_bool_branch(comparison, class_block, instance_block)
builder.activate_block(class_block)
builder.add(Return(vself))
builder.activate_block(instance_block)
builder.add(
Return(
builder.call_c(method_new_op,
[vself, builder.read(instance)], line)))
blocks, env, _, fn_info = builder.leave()
sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive),
RuntimeArg('instance', object_rprimitive),
RuntimeArg('owner', object_rprimitive)),
object_rprimitive)
get_fn_decl = FuncDecl('__get__', fn_info.callable_class.ir.name,
builder.module_name, sig)
get_fn_ir = FuncIR(get_fn_decl, blocks, env)
fn_info.callable_class.ir.methods['__get__'] = get_fn_ir
builder.functions.append(get_fn_ir)
def add_close_to_generator_class(builder: IRBuilder,
fn_info: FuncInfo) -> None:
"""Generates the '__close__' method for a generator class."""
# TODO: Currently this method just triggers a runtime error,
# we should fill this out eventually.
builder.enter(fn_info)
add_self_to_env(builder.environment, fn_info.generator_class.ir)
builder.add(
RaiseStandardError(RaiseStandardError.RUNTIME_ERROR,
'close method on generator classes uimplemented',
fn_info.fitem.line))
builder.add(Unreachable())
blocks, env, _, fn_info = builder.leave()
# Next, add the actual function as a method of the generator class.
sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive), ),
object_rprimitive)
close_fn_decl = FuncDecl('close', fn_info.generator_class.ir.name,
builder.module_name, sig)
close_fn_ir = FuncIR(close_fn_decl, blocks, env)
fn_info.generator_class.ir.methods['close'] = close_fn_ir
builder.functions.append(close_fn_ir)
def __init__(self,
name: str,
module_name: str,
is_trait: bool = False,
is_generated: bool = False,
is_abstract: bool = False,
is_ext_class: bool = True) -> None:
self.name = name
self.module_name = module_name
self.is_trait = is_trait
self.is_generated = is_generated
self.is_abstract = is_abstract
self.is_ext_class = is_ext_class
# An augmented class has additional methods separate from what mypyc generates.
# Right now the only one is dataclasses.
self.is_augmented = False
# Does this inherit from a Python class?
self.inherits_python = False
# Do instances of this class have __dict__?
self.has_dict = False
# Do we allow interpreted subclasses? Derived from a mypyc_attr.
self.allow_interpreted_subclasses = False
# Does this class need getseters to be generated for its attributes? (getseters are also
# added if is_generated is False)
self.needs_getseters = False
# Is this class declared as serializable (supports copy.copy
# and pickle) using @mypyc_attr(serializable=True)?
#
# Additionally, any class with this attribute False but with
# an __init__ that can be called without any arguments is
# *implicitly serializable*. In this case __init__ will be
# called during deserialization without arguments. If this is
# True, we match Python semantics and __init__ won't be called
# during deserialization.
#
# This impacts also all subclasses. Use is_serializable() to
# also consider base classes.
self._serializable = False
# If this a subclass of some built-in python class, the name
# of the object for that class. We currently only support this
# in a few ad-hoc cases.
self.builtin_base: Optional[str] = None
# Default empty constructor
self.ctor = FuncDecl(name, None, module_name,
FuncSignature([], RInstance(self)))
self.attributes: OrderedDict[str, RType] = OrderedDict()
# Deletable attributes
self.deletable: List[str] = []
# We populate method_types with the signatures of every method before
# we generate methods, and we rely on this information being present.
self.method_decls: OrderedDict[str, FuncDecl] = OrderedDict()
# Map of methods that are actually present in an extension class
self.methods: OrderedDict[str, FuncIR] = OrderedDict()
# Glue methods for boxing/unboxing when a class changes the type
# while overriding a method. Maps from (parent class overridden, method)
# to IR of glue method.
self.glue_methods: Dict[Tuple[ClassIR, str], FuncIR] = OrderedDict()
# Properties are accessed like attributes, but have behavior like method calls.
# They don't belong in the methods dictionary, since we don't want to expose them to
# Python's method API. But we want to put them into our own vtable as methods, so that
# they are properly handled and overridden. The property dictionary values are a tuple
# containing a property getter and an optional property setter.
self.properties: OrderedDict[str,
Tuple[FuncIR,
Optional[FuncIR]]] = OrderedDict()
# We generate these in prepare_class_def so that we have access to them when generating
# other methods and properties that rely on these types.
self.property_types: OrderedDict[str, RType] = OrderedDict()
self.vtable: Optional[Dict[str, int]] = None
self.vtable_entries: VTableEntries = []
self.trait_vtables: OrderedDict[ClassIR, VTableEntries] = OrderedDict()
# N.B: base might not actually quite be the direct base.
# It is the nearest concrete base, but we allow a trait in between.
self.base: Optional[ClassIR] = None
self.traits: List[ClassIR] = []
# Supply a working mro for most generated classes. Real classes will need to
# fix it up.
self.mro: List[ClassIR] = [self]
# base_mro is the chain of concrete (non-trait) ancestors
self.base_mro: List[ClassIR] = [self]
# Direct subclasses of this class (use subclasses() to also include non-direct ones)
# None if separate compilation prevents this from working
self.children: Optional[List[ClassIR]] = []
# Instance attributes that are initialized in the class body.
self.attrs_with_defaults: Set[str] = set()
# Attributes that are always initialized in __init__ or class body
# (inferred in mypyc.analysis.attrdefined using interprocedural analysis)
self._always_initialized_attrs: Set[str] = set()
# Attributes that are sometimes initialized in __init__
self._sometimes_initialized_attrs: Set[str] = set()
# If True, __init__ can make 'self' visible to unanalyzed/arbitrary code
self.init_self_leak = False
def prepare_class_def(path: str, module_name: str, cdef: ClassDef,
errors: Errors, mapper: Mapper) -> None:
ir = mapper.type_to_ir[cdef.info]
info = cdef.info
attrs = get_mypyc_attrs(cdef)
if attrs.get("allow_interpreted_subclasses") is True:
ir.allow_interpreted_subclasses = True
if attrs.get("serializable") is True:
# Supports copy.copy and pickle (including subclasses)
ir._serializable = True
# We sort the table for determinism here on Python 3.5
for name, node in sorted(info.names.items()):
# Currently all plugin generated methods are dummies and not included.
if node.plugin_generated:
continue
if isinstance(node.node, Var):
assert node.node.type, "Class member %s missing type" % name
if not node.node.is_classvar and name not in ('__slots__', '__deletable__'):
ir.attributes[name] = mapper.type_to_rtype(node.node.type)
elif isinstance(node.node, (FuncDef, Decorator)):
prepare_method_def(ir, module_name, cdef, mapper, node.node)
elif isinstance(node.node, OverloadedFuncDef):
# Handle case for property with both a getter and a setter
if node.node.is_property:
if is_valid_multipart_property_def(node.node):
for item in node.node.items:
prepare_method_def(ir, module_name, cdef, mapper, item)
else:
errors.error("Unsupported property decorator semantics", path, cdef.line)
# Handle case for regular function overload
else:
assert node.node.impl
prepare_method_def(ir, module_name, cdef, mapper, node.node.impl)
# Check for subclassing from builtin types
for cls in info.mro:
# Special case exceptions and dicts
# XXX: How do we handle *other* things??
if cls.fullname == 'builtins.BaseException':
ir.builtin_base = 'PyBaseExceptionObject'
elif cls.fullname == 'builtins.dict':
ir.builtin_base = 'PyDictObject'
elif cls.fullname.startswith('builtins.'):
if not can_subclass_builtin(cls.fullname):
# Note that if we try to subclass a C extension class that
# isn't in builtins, bad things will happen and we won't
# catch it here! But this should catch a lot of the most
# common pitfalls.
errors.error("Inheriting from most builtin types is unimplemented",
path, cdef.line)
if ir.builtin_base:
ir.attributes.clear()
# Set up a constructor decl
init_node = cdef.info['__init__'].node
if not ir.is_trait and not ir.builtin_base and isinstance(init_node, FuncDef):
init_sig = mapper.fdef_to_sig(init_node)
defining_ir = mapper.type_to_ir.get(init_node.info)
# If there is a nontrivial __init__ that wasn't defined in an
# extension class, we need to make the constructor take *args,
# **kwargs so it can call tp_init.
if ((defining_ir is None or not defining_ir.is_ext_class
or cdef.info['__init__'].plugin_generated)
and init_node.info.fullname != 'builtins.object'):
init_sig = FuncSignature(
[init_sig.args[0],
RuntimeArg("args", tuple_rprimitive, ARG_STAR),
RuntimeArg("kwargs", dict_rprimitive, ARG_STAR2)],
init_sig.ret_type)
ctor_sig = FuncSignature(init_sig.args[1:], RInstance(ir))
ir.ctor = FuncDecl(cdef.name, None, module_name, ctor_sig)
mapper.func_to_decl[cdef.info] = ir.ctor
# Set up the parent class
bases = [mapper.type_to_ir[base.type] for base in info.bases
if base.type in mapper.type_to_ir]
if not all(c.is_trait for c in bases[1:]):
errors.error("Non-trait bases must appear first in parent list", path, cdef.line)
ir.traits = [c for c in bases if c.is_trait]
mro = []
base_mro = []
for cls in info.mro:
if cls not in mapper.type_to_ir:
if cls.fullname != 'builtins.object':
ir.inherits_python = True
continue
base_ir = mapper.type_to_ir[cls]
if not base_ir.is_trait:
base_mro.append(base_ir)
mro.append(base_ir)
if cls.defn.removed_base_type_exprs or not base_ir.is_ext_class:
ir.inherits_python = True
base_idx = 1 if not ir.is_trait else 0
if len(base_mro) > base_idx:
ir.base = base_mro[base_idx]
ir.mro = mro
ir.base_mro = base_mro
for base in bases:
if base.children is not None:
base.children.append(ir)
if is_dataclass(cdef):
ir.is_augmented = True
def native_function_name(self, fn: FuncDecl) -> str:
return '{}{}'.format(NATIVE_PREFIX, fn.cname(self.names))
def __init__(self, name: str, module_name: str, is_trait: bool = False,
is_generated: bool = False, is_abstract: bool = False,
is_ext_class: bool = True) -> None:
self.name = name
self.module_name = module_name
self.is_trait = is_trait
self.is_generated = is_generated
self.is_abstract = is_abstract
self.is_ext_class = is_ext_class
# An augmented class has additional methods separate from what mypyc generates.
# Right now the only one is dataclasses.
self.is_augmented = False
# Does this inherit from a Python class?
self.inherits_python = False
# Do instances of this class have __dict__?
self.has_dict = False
# Do we allow interpreted subclasses? Derived from a mypyc_attr.
self.allow_interpreted_subclasses = False
# If this a subclass of some built-in python class, the name
# of the object for that class. We currently only support this
# in a few ad-hoc cases.
self.builtin_base = None # type: Optional[str]
# Default empty constructor
self.ctor = FuncDecl(name, None, module_name, FuncSignature([], RInstance(self)))
self.attributes = OrderedDict() # type: OrderedDict[str, RType]
# We populate method_types with the signatures of every method before
# we generate methods, and we rely on this information being present.
self.method_decls = OrderedDict() # type: OrderedDict[str, FuncDecl]
# Map of methods that are actually present in an extension class
self.methods = OrderedDict() # type: OrderedDict[str, FuncIR]
# Glue methods for boxing/unboxing when a class changes the type
# while overriding a method. Maps from (parent class overrided, method)
# to IR of glue method.
self.glue_methods = OrderedDict() # type: Dict[Tuple[ClassIR, str], FuncIR]
# Properties are accessed like attributes, but have behavior like method calls.
# They don't belong in the methods dictionary, since we don't want to expose them to
# Python's method API. But we want to put them into our own vtable as methods, so that
# they are properly handled and overridden. The property dictionary values are a tuple
# containing a property getter and an optional property setter.
self.properties = OrderedDict() # type: OrderedDict[str, Tuple[FuncIR, Optional[FuncIR]]]
# We generate these in prepare_class_def so that we have access to them when generating
# other methods and properties that rely on these types.
self.property_types = OrderedDict() # type: OrderedDict[str, RType]
self.vtable = None # type: Optional[Dict[str, int]]
self.vtable_entries = [] # type: VTableEntries
self.trait_vtables = OrderedDict() # type: OrderedDict[ClassIR, VTableEntries]
# N.B: base might not actually quite be the direct base.
# It is the nearest concrete base, but we allow a trait in between.
self.base = None # type: Optional[ClassIR]
self.traits = [] # type: List[ClassIR]
# Supply a working mro for most generated classes. Real classes will need to
# fix it up.
self.mro = [self] # type: List[ClassIR]
# base_mro is the chain of concrete (non-trait) ancestors
self.base_mro = [self] # type: List[ClassIR]
# Direct subclasses of this class (use subclasses() to also incude non-direct ones)
# None if separate compilation prevents this from working
self.children = [] # type: Optional[List[ClassIR]]
class ClassIR:
"""Intermediate representation of a class.
This also describes the runtime structure of native instances.
"""
def __init__(self,
name: str,
module_name: str,
is_trait: bool = False,
is_generated: bool = False,
is_abstract: bool = False,
is_ext_class: bool = True) -> None:
self.name = name
self.module_name = module_name
self.is_trait = is_trait
self.is_generated = is_generated
self.is_abstract = is_abstract
self.is_ext_class = is_ext_class
# An augmented class has additional methods separate from what mypyc generates.
# Right now the only one is dataclasses.
self.is_augmented = False
# Does this inherit from a Python class?
self.inherits_python = False
# Do instances of this class have __dict__?
self.has_dict = False
# Do we allow interpreted subclasses? Derived from a mypyc_attr.
self.allow_interpreted_subclasses = False
# Does this class need getseters to be generated for its attributes? (getseters are also
# added if is_generated is False)
self.needs_getseters = False
# Is this class declared as serializable (supports copy.copy
# and pickle) using @mypyc_attr(serializable=True)?
#
# Additionally, any class with this attribute False but with
# an __init__ that can be called without any arguments is
# *implicitly serializable*. In this case __init__ will be
# called during deserialization without arguments. If this is
# True, we match Python semantics and __init__ won't be called
# during deserialization.
#
# This impacts also all subclasses. Use is_serializable() to
# also consider base classes.
self._serializable = False
# If this a subclass of some built-in python class, the name
# of the object for that class. We currently only support this
# in a few ad-hoc cases.
self.builtin_base: Optional[str] = None
# Default empty constructor
self.ctor = FuncDecl(name, None, module_name,
FuncSignature([], RInstance(self)))
self.attributes: OrderedDict[str, RType] = OrderedDict()
# Deletable attributes
self.deletable: List[str] = []
# We populate method_types with the signatures of every method before
# we generate methods, and we rely on this information being present.
self.method_decls: OrderedDict[str, FuncDecl] = OrderedDict()
# Map of methods that are actually present in an extension class
self.methods: OrderedDict[str, FuncIR] = OrderedDict()
# Glue methods for boxing/unboxing when a class changes the type
# while overriding a method. Maps from (parent class overridden, method)
# to IR of glue method.
self.glue_methods: Dict[Tuple[ClassIR, str], FuncIR] = OrderedDict()
# Properties are accessed like attributes, but have behavior like method calls.
# They don't belong in the methods dictionary, since we don't want to expose them to
# Python's method API. But we want to put them into our own vtable as methods, so that
# they are properly handled and overridden. The property dictionary values are a tuple
# containing a property getter and an optional property setter.
self.properties: OrderedDict[str,
Tuple[FuncIR,
Optional[FuncIR]]] = OrderedDict()
# We generate these in prepare_class_def so that we have access to them when generating
# other methods and properties that rely on these types.
self.property_types: OrderedDict[str, RType] = OrderedDict()
self.vtable: Optional[Dict[str, int]] = None
self.vtable_entries: VTableEntries = []
self.trait_vtables: OrderedDict[ClassIR, VTableEntries] = OrderedDict()
# N.B: base might not actually quite be the direct base.
# It is the nearest concrete base, but we allow a trait in between.
self.base: Optional[ClassIR] = None
self.traits: List[ClassIR] = []
# Supply a working mro for most generated classes. Real classes will need to
# fix it up.
self.mro: List[ClassIR] = [self]
# base_mro is the chain of concrete (non-trait) ancestors
self.base_mro: List[ClassIR] = [self]
# Direct subclasses of this class (use subclasses() to also include non-direct ones)
# None if separate compilation prevents this from working
self.children: Optional[List[ClassIR]] = []
# Instance attributes that are initialized in the class body.
self.attrs_with_defaults: Set[str] = set()
# Attributes that are always initialized in __init__ or class body
# (inferred in mypyc.analysis.attrdefined using interprocedural analysis)
self._always_initialized_attrs: Set[str] = set()
# Attributes that are sometimes initialized in __init__
self._sometimes_initialized_attrs: Set[str] = set()
# If True, __init__ can make 'self' visible to unanalyzed/arbitrary code
self.init_self_leak = False
def __repr__(self) -> str:
return (
"ClassIR("
"name={self.name}, module_name={self.module_name}, "
"is_trait={self.is_trait}, is_generated={self.is_generated}, "
"is_abstract={self.is_abstract}, is_ext_class={self.is_ext_class}"
")".format(self=self))
@property
def fullname(self) -> str:
return f"{self.module_name}.{self.name}"
def real_base(self) -> Optional['ClassIR']:
"""Return the actual concrete base class, if there is one."""
if len(self.mro) > 1 and not self.mro[1].is_trait:
return self.mro[1]
return None
def vtable_entry(self, name: str) -> int:
assert self.vtable is not None, "vtable not computed yet"
assert name in self.vtable, f'{self.name!r} has no attribute {name!r}'
return self.vtable[name]
def attr_details(self, name: str) -> Tuple[RType, 'ClassIR']:
for ir in self.mro:
if name in ir.attributes:
return ir.attributes[name], ir
if name in ir.property_types:
return ir.property_types[name], ir
raise KeyError(f'{self.name!r} has no attribute {name!r}')
def attr_type(self, name: str) -> RType:
return self.attr_details(name)[0]
def method_decl(self, name: str) -> FuncDecl:
for ir in self.mro:
if name in ir.method_decls:
return ir.method_decls[name]
raise KeyError(f'{self.name!r} has no attribute {name!r}')
def method_sig(self, name: str) -> FuncSignature:
return self.method_decl(name).sig
def has_method(self, name: str) -> bool:
try:
self.method_decl(name)
except KeyError:
return False
return True
def is_method_final(self, name: str) -> bool:
subs = self.subclasses()
if subs is None:
# TODO: Look at the final attribute!
return False
if self.has_method(name):
method_decl = self.method_decl(name)
for subc in subs:
if subc.method_decl(name) != method_decl:
return False
return True
else:
return not any(subc.has_method(name) for subc in subs)
def has_attr(self, name: str) -> bool:
try:
self.attr_type(name)
except KeyError:
return False
return True
def is_deletable(self, name: str) -> bool:
for ir in self.mro:
if name in ir.deletable:
return True
return False
def is_always_defined(self, name: str) -> bool:
if self.is_deletable(name):
return False
return name in self._always_initialized_attrs
def name_prefix(self, names: NameGenerator) -> str:
return names.private_name(self.module_name, self.name)
def struct_name(self, names: NameGenerator) -> str:
return f'{exported_name(self.fullname)}Object'
def get_method_and_class(self,
name: str) -> Optional[Tuple[FuncIR, 'ClassIR']]:
for ir in self.mro:
if name in ir.methods:
return ir.methods[name], ir
return None
def get_method(self, name: str) -> Optional[FuncIR]:
res = self.get_method_and_class(name)
return res[0] if res else None
def subclasses(self) -> Optional[Set['ClassIR']]:
"""Return all subclasses of this class, both direct and indirect.
Return None if it is impossible to identify all subclasses, for example
because we are performing separate compilation.
"""
if self.children is None or self.allow_interpreted_subclasses:
return None
result = set(self.children)
for child in self.children:
if child.children:
child_subs = child.subclasses()
if child_subs is None:
return None
result.update(child_subs)
return result
def concrete_subclasses(self) -> Optional[List['ClassIR']]:
"""Return all concrete (i.e. non-trait and non-abstract) subclasses.
Include both direct and indirect subclasses. Place classes with no children first.
"""
subs = self.subclasses()
if subs is None:
return None
concrete = {c for c in subs if not (c.is_trait or c.is_abstract)}
# We place classes with no children first because they are more likely
# to appear in various isinstance() checks. We then sort leaves by name
# to get stable order.
return sorted(concrete, key=lambda c: (len(c.children or []), c.name))
def is_serializable(self) -> bool:
return any(ci._serializable for ci in self.mro)
def serialize(self) -> JsonDict:
return {
'name':
self.name,
'module_name':
self.module_name,
'is_trait':
self.is_trait,
'is_ext_class':
self.is_ext_class,
'is_abstract':
self.is_abstract,
'is_generated':
self.is_generated,
'is_augmented':
self.is_augmented,
'inherits_python':
self.inherits_python,
'has_dict':
self.has_dict,
'allow_interpreted_subclasses':
self.allow_interpreted_subclasses,
'needs_getseters':
self.needs_getseters,
'_serializable':
self._serializable,
'builtin_base':
self.builtin_base,
'ctor':
self.ctor.serialize(),
# We serialize dicts as lists to ensure order is preserved
'attributes':
[(k, t.serialize()) for k, t in self.attributes.items()],
# We try to serialize a name reference, but if the decl isn't in methods
# then we can't be sure that will work so we serialize the whole decl.
'method_decls': [(k, d.id if k in self.methods else d.serialize())
for k, d in self.method_decls.items()],
# We serialize method fullnames out and put methods in a separate dict
'methods': [(k, m.id) for k, m in self.methods.items()],
'glue_methods': [((cir.fullname, k), m.id)
for (cir, k), m in self.glue_methods.items()],
# We serialize properties and property_types separately out of an
# abundance of caution about preserving dict ordering...
'property_types':
[(k, t.serialize()) for k, t in self.property_types.items()],
'properties':
list(self.properties),
'vtable':
self.vtable,
'vtable_entries':
serialize_vtable(self.vtable_entries),
'trait_vtables': [(cir.fullname, serialize_vtable(v))
for cir, v in self.trait_vtables.items()],
# References to class IRs are all just names
'base':
self.base.fullname if self.base else None,
'traits': [cir.fullname for cir in self.traits],
'mro': [cir.fullname for cir in self.mro],
'base_mro': [cir.fullname for cir in self.base_mro],
'children':
[cir.fullname
for cir in self.children] if self.children is not None else None,
'deletable':
self.deletable,
'attrs_with_defaults':
sorted(self.attrs_with_defaults),
'_always_initialized_attrs':
sorted(self._always_initialized_attrs),
'_sometimes_initialized_attrs':
sorted(self._sometimes_initialized_attrs),
'init_self_leak':
self.init_self_leak,
}
@classmethod
def deserialize(cls, data: JsonDict, ctx: 'DeserMaps') -> 'ClassIR':
fullname = data['module_name'] + '.' + data['name']
assert fullname in ctx.classes, "Class %s not in deser class map" % fullname
ir = ctx.classes[fullname]
ir.is_trait = data['is_trait']
ir.is_generated = data['is_generated']
ir.is_abstract = data['is_abstract']
ir.is_ext_class = data['is_ext_class']
ir.is_augmented = data['is_augmented']
ir.inherits_python = data['inherits_python']
ir.has_dict = data['has_dict']
ir.allow_interpreted_subclasses = data['allow_interpreted_subclasses']
ir.needs_getseters = data['needs_getseters']
ir._serializable = data['_serializable']
ir.builtin_base = data['builtin_base']
ir.ctor = FuncDecl.deserialize(data['ctor'], ctx)
ir.attributes = OrderedDict(
(k, deserialize_type(t, ctx)) for k, t in data['attributes'])
ir.method_decls = OrderedDict(
(k, ctx.functions[v].decl if isinstance(v, str) else FuncDecl.
deserialize(v, ctx)) for k, v in data['method_decls'])
ir.methods = OrderedDict(
(k, ctx.functions[v]) for k, v in data['methods'])
ir.glue_methods = OrderedDict(((ctx.classes[c], k), ctx.functions[v])
for (c, k), v in data['glue_methods'])
ir.property_types = OrderedDict(
(k, deserialize_type(t, ctx)) for k, t in data['property_types'])
ir.properties = OrderedDict(
(k, (ir.methods[k], ir.methods.get(PROPSET_PREFIX + k)))
for k in data['properties'])
ir.vtable = data['vtable']
ir.vtable_entries = deserialize_vtable(data['vtable_entries'], ctx)
ir.trait_vtables = OrderedDict(
(ctx.classes[k], deserialize_vtable(v, ctx))
for k, v in data['trait_vtables'])
base = data['base']
ir.base = ctx.classes[base] if base else None
ir.traits = [ctx.classes[s] for s in data['traits']]
ir.mro = [ctx.classes[s] for s in data['mro']]
ir.base_mro = [ctx.classes[s] for s in data['base_mro']]
ir.children = data['children'] and [
ctx.classes[s] for s in data['children']
]
ir.deletable = data['deletable']
ir.attrs_with_defaults = set(data['attrs_with_defaults'])
ir._always_initialized_attrs = set(data['_always_initialized_attrs'])
ir._sometimes_initialized_attrs = set(
data['_sometimes_initialized_attrs'])
ir.init_self_leak = data['init_self_leak']
return ir
