def transform_mypy_file(builder: IRBuilder, mypyfile: MypyFile) -> None:
if mypyfile.fullname in ('typing', 'abc'):
# These module are special; their contents are currently all
# built-in primitives.
return
builder.set_module(mypyfile.fullname, mypyfile.path)
classes = [node for node in mypyfile.defs if isinstance(node, ClassDef)]
# Collect all classes.
for cls in classes:
ir = builder.mapper.type_to_ir[cls.info]
builder.classes.append(ir)
builder.enter('<top level>')
# Make sure we have a builtins import
builder.gen_import('builtins', -1)
# Generate ops.
for node in mypyfile.defs:
builder.accept(node)
builder.maybe_add_implicit_return()
# Generate special function representing module top level.
blocks, env, ret_type, _ = builder.leave()
sig = FuncSignature([], none_rprimitive)
func_ir = FuncIR(FuncDecl(TOP_LEVEL_NAME, None, builder.module_name, sig),
blocks,
env,
traceback_name="<module>")
builder.functions.append(func_ir)
def gen_glue_property(builder: IRBuilder,
sig: FuncSignature,
target: FuncIR,
cls: ClassIR,
base: ClassIR,
line: int,
do_pygetattr: bool) -> FuncIR:
"""Generate glue methods for properties that mediate between different subclass types.
Similarly to methods, properties of derived types can be covariantly subtyped. Thus,
properties also require glue. However, this only requires the return type to change.
Further, instead of a method call, an attribute get is performed.
If do_pygetattr is True, then get the attribute using the Python C
API instead of a native call.
"""
builder.enter()
rt_arg = RuntimeArg(SELF_NAME, RInstance(cls))
arg = builder.read(add_self_to_env(builder.environment, cls), line)
builder.ret_types[-1] = sig.ret_type
if do_pygetattr:
retval = builder.py_get_attr(arg, target.name, line)
else:
retval = builder.add(GetAttr(arg, target.name, line))
retbox = builder.coerce(retval, sig.ret_type, line)
builder.add(Return(retbox))
blocks, env, return_type, _ = builder.leave()
return FuncIR(
FuncDecl(target.name + '__' + base.name + '_glue',
cls.name, builder.module_name, FuncSignature([rt_arg], return_type)),
blocks, env)
def add_send_to_generator_class(builder: IRBuilder, fn_info: FuncInfo,
fn_decl: FuncDecl, sig: FuncSignature) -> None:
"""Generates the 'send' method for a generator class."""
# FIXME: this is basically the same as add_next...
builder.enter(fn_info)
self_reg = builder.read(
add_self_to_env(builder.environment, fn_info.generator_class.ir))
arg = builder.environment.add_local_reg(Var('arg'), object_rprimitive,
True)
none_reg = builder.none_object()
# Call the helper function with error flags set to Py_None, and return that result.
result = builder.add(
Call(fn_decl,
[self_reg, none_reg, none_reg, none_reg,
builder.read(arg)], fn_info.fitem.line))
builder.add(Return(result))
blocks, env, _, fn_info = builder.leave()
sig = FuncSignature((
RuntimeArg(SELF_NAME, object_rprimitive),
RuntimeArg('arg', object_rprimitive),
), sig.ret_type)
next_fn_decl = FuncDecl('send', fn_info.generator_class.ir.name,
builder.module_name, sig)
next_fn_ir = FuncIR(next_fn_decl, blocks, env)
fn_info.generator_class.ir.methods['send'] = next_fn_ir
builder.functions.append(next_fn_ir)
def gen_dispatch_func_ir(
builder: IRBuilder,
fitem: FuncDef,
main_func_name: str,
dispatch_name: str,
sig: FuncSignature,
) -> Tuple[FuncIR, Value]:
"""Create a dispatch function (a function that checks the first argument type and dispatches
to the correct implementation)
"""
builder.enter(FuncInfo(fitem, dispatch_name))
setup_callable_class(builder)
builder.fn_info.callable_class.ir.attributes['registry'] = dict_rprimitive
builder.fn_info.callable_class.ir.attributes[
'dispatch_cache'] = dict_rprimitive
builder.fn_info.callable_class.ir.has_dict = True
builder.fn_info.callable_class.ir.needs_getseters = True
generate_singledispatch_callable_class_ctor(builder)
generate_singledispatch_dispatch_function(builder, main_func_name, fitem)
args, _, blocks, _, fn_info = builder.leave()
dispatch_callable_class = add_call_to_callable_class(
builder, args, blocks, sig, fn_info)
builder.functions.append(dispatch_callable_class)
add_get_to_callable_class(builder, fn_info)
add_register_method_to_callable_class(builder, fn_info)
func_reg = instantiate_callable_class(builder, fn_info)
dispatch_func_ir = generate_dispatch_glue_native_function(
builder, fitem, dispatch_callable_class.decl, dispatch_name)
return dispatch_func_ir, func_reg
def gen_glue_method(builder: IRBuilder, sig: FuncSignature, target: FuncIR,
cls: ClassIR, base: ClassIR, line: int,
do_pycall: bool,
) -> FuncIR:
"""Generate glue methods that mediate between different method types in subclasses.
For example, if we have:
class A:
def f(builder: IRBuilder, x: int) -> object: ...
then it is totally permissible to have a subclass
class B(A):
def f(builder: IRBuilder, x: object) -> int: ...
since '(object) -> int' is a subtype of '(int) -> object' by the usual
contra/co-variant function subtyping rules.
The trickiness here is that int and object have different
runtime representations in mypyc, so A.f and B.f have
different signatures at the native C level. To deal with this,
we need to generate glue methods that mediate between the
different versions by coercing the arguments and return
values.
If do_pycall is True, then make the call using the C API
instead of a native call.
"""
builder.enter()
builder.ret_types[-1] = sig.ret_type
rt_args = list(sig.args)
if target.decl.kind == FUNC_NORMAL:
rt_args[0] = RuntimeArg(sig.args[0].name, RInstance(cls))
# The environment operates on Vars, so we make some up
fake_vars = [(Var(arg.name), arg.type) for arg in rt_args]
args = [builder.read(builder.add_local_reg(var, type, is_arg=True), line)
for var, type in fake_vars]
arg_names = [arg.name if arg.kind in (ARG_NAMED, ARG_NAMED_OPT) else None
for arg in rt_args]
arg_kinds = [concrete_arg_kind(arg.kind) for arg in rt_args]
if do_pycall:
retval = builder.builder.py_method_call(
args[0], target.name, args[1:], line, arg_kinds[1:], arg_names[1:])
else:
retval = builder.builder.call(target.decl, args, arg_kinds, arg_names, line)
retval = builder.coerce(retval, sig.ret_type, line)
builder.add(Return(retval))
arg_regs, _, blocks, ret_type, _ = builder.leave()
return FuncIR(
FuncDecl(target.name + '__' + base.name + '_glue',
cls.name, builder.module_name,
FuncSignature(rt_args, ret_type),
target.decl.kind),
arg_regs, blocks)
def add_throw_to_generator_class(builder: IRBuilder, fn_info: FuncInfo,
fn_decl: FuncDecl,
sig: FuncSignature) -> None:
"""Generates the 'throw' method for a generator class."""
builder.enter(fn_info)
self_reg = builder.read(
add_self_to_env(builder.environment, fn_info.generator_class.ir))
# Add the type, value, and traceback variables to the environment.
typ = builder.environment.add_local_reg(Var('type'), object_rprimitive,
True)
val = builder.environment.add_local_reg(Var('value'), object_rprimitive,
True)
tb = builder.environment.add_local_reg(Var('traceback'), object_rprimitive,
True)
# Because the value and traceback arguments are optional and hence
# can be NULL if not passed in, we have to assign them Py_None if
# they are not passed in.
none_reg = builder.none_object()
builder.assign_if_null(val, lambda: none_reg, builder.fn_info.fitem.line)
builder.assign_if_null(tb, lambda: none_reg, builder.fn_info.fitem.line)
# Call the helper function using the arguments passed in, and return that result.
result = builder.add(
Call(fn_decl, [
self_reg,
builder.read(typ),
builder.read(val),
builder.read(tb), none_reg
], fn_info.fitem.line))
builder.add(Return(result))
blocks, env, _, fn_info = builder.leave()
# Create the FuncSignature for the throw function. Note that the
# value and traceback fields are optional, and are assigned to if
# they are not passed in inside the body of the throw function.
sig = FuncSignature((RuntimeArg(
SELF_NAME, object_rprimitive), RuntimeArg('type', object_rprimitive),
RuntimeArg('value', object_rprimitive, ARG_OPT),
RuntimeArg('traceback', object_rprimitive, ARG_OPT)),
sig.ret_type)
throw_fn_decl = FuncDecl('throw', fn_info.generator_class.ir.name,
builder.module_name, sig)
throw_fn_ir = FuncIR(throw_fn_decl, blocks, env)
fn_info.generator_class.ir.methods['throw'] = throw_fn_ir
builder.functions.append(throw_fn_ir)
def add_iter_to_generator_class(builder: IRBuilder, fn_info: FuncInfo) -> None:
"""Generates the '__iter__' method for a generator class."""
builder.enter(fn_info)
self_target = add_self_to_env(builder.environment,
fn_info.generator_class.ir)
builder.add(Return(builder.read(self_target, fn_info.fitem.line)))
blocks, env, _, fn_info = builder.leave()
# Next, add the actual function as a method of the generator class.
sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive), ),
object_rprimitive)
iter_fn_decl = FuncDecl('__iter__', fn_info.generator_class.ir.name,
builder.module_name, sig)
iter_fn_ir = FuncIR(iter_fn_decl, blocks, env)
fn_info.generator_class.ir.methods['__iter__'] = iter_fn_ir
builder.functions.append(iter_fn_ir)
def gen_dispatch_func_ir(
builder: IRBuilder,
fitem: FuncDef,
main_func_name: str,
dispatch_name: str,
sig: FuncSignature,
) -> FuncIR:
"""Create a dispatch function (a function that checks the first argument type and dispatches
to the correct implementation)
"""
builder.enter()
generate_singledispatch_dispatch_function(builder, main_func_name, fitem)
args, _, blocks, _, fn_info = builder.leave()
func_decl = FuncDecl(dispatch_name, None, builder.module_name, sig)
dispatch_func_ir = FuncIR(func_decl, args, blocks)
return dispatch_func_ir
def gen_glue_ne_method(builder: IRBuilder, cls: ClassIR, line: int) -> FuncIR:
"""Generate a "__ne__" method from a "__eq__" method. """
builder.enter()
rt_args = (RuntimeArg("self", RInstance(cls)), RuntimeArg("rhs", object_rprimitive))
# The environment operates on Vars, so we make some up
fake_vars = [(Var(arg.name), arg.type) for arg in rt_args]
args = [
builder.read(
builder.environment.add_local_reg(
var, type, is_arg=True
),
line
)
for var, type in fake_vars
] # type: List[Value]
builder.ret_types[-1] = object_rprimitive
# If __eq__ returns NotImplemented, then __ne__ should also
not_implemented_block, regular_block = BasicBlock(), BasicBlock()
eqval = builder.add(MethodCall(args[0], '__eq__', [args[1]], line))
not_implemented = builder.add(LoadAddress(not_implemented_op.type,
not_implemented_op.src, line))
builder.add(Branch(
builder.translate_is_op(eqval, not_implemented, 'is', line),
not_implemented_block,
regular_block,
Branch.BOOL_EXPR))
builder.activate_block(regular_block)
retval = builder.coerce(
builder.unary_op(eqval, 'not', line), object_rprimitive, line
)
builder.add(Return(retval))
builder.activate_block(not_implemented_block)
builder.add(Return(not_implemented))
blocks, env, ret_type, _ = builder.leave()
return FuncIR(
FuncDecl('__ne__', cls.name, builder.module_name,
FuncSignature(rt_args, ret_type)),
blocks, env)
def generate_dispatch_glue_native_function(
builder: IRBuilder,
fitem: FuncDef,
callable_class_decl: FuncDecl,
dispatch_name: str,
) -> FuncIR:
line = fitem.line
builder.enter()
# We store the callable class in the globals dict for this function
callable_class = builder.load_global_str(dispatch_name, line)
decl = builder.mapper.func_to_decl[fitem]
arg_info = get_args(builder, decl.sig.args, line)
args = [callable_class] + arg_info.args
arg_kinds = [ArgKind.ARG_POS] + arg_info.arg_kinds
arg_names = arg_info.arg_names
arg_names.insert(0, 'self')
ret_val = builder.builder.call(callable_class_decl, args, arg_kinds,
arg_names, line)
builder.add(Return(ret_val))
arg_regs, _, blocks, _, fn_info = builder.leave()
return FuncIR(decl, arg_regs, blocks)
def add_get_to_callable_class(builder: IRBuilder, fn_info: FuncInfo) -> None:
"""Generate the '__get__' method for a callable class."""
line = fn_info.fitem.line
builder.enter(fn_info)
vself = builder.read(
builder.environment.add_local_reg(Var(SELF_NAME), object_rprimitive,
True))
instance = builder.environment.add_local_reg(Var('instance'),
object_rprimitive, True)
builder.environment.add_local_reg(Var('owner'), object_rprimitive, True)
# If accessed through the class, just return the callable
# object. If accessed through an object, create a new bound
# instance method object.
instance_block, class_block = BasicBlock(), BasicBlock()
comparison = builder.translate_is_op(builder.read(instance),
builder.none_object(), 'is', line)
builder.add_bool_branch(comparison, class_block, instance_block)
builder.activate_block(class_block)
builder.add(Return(vself))
builder.activate_block(instance_block)
builder.add(
Return(
builder.call_c(method_new_op,
[vself, builder.read(instance)], line)))
blocks, env, _, fn_info = builder.leave()
sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive),
RuntimeArg('instance', object_rprimitive),
RuntimeArg('owner', object_rprimitive)),
object_rprimitive)
get_fn_decl = FuncDecl('__get__', fn_info.callable_class.ir.name,
builder.module_name, sig)
get_fn_ir = FuncIR(get_fn_decl, blocks, env)
fn_info.callable_class.ir.methods['__get__'] = get_fn_ir
builder.functions.append(get_fn_ir)
def add_close_to_generator_class(builder: IRBuilder,
fn_info: FuncInfo) -> None:
"""Generates the '__close__' method for a generator class."""
# TODO: Currently this method just triggers a runtime error,
# we should fill this out eventually.
builder.enter(fn_info)
add_self_to_env(builder.environment, fn_info.generator_class.ir)
builder.add(
RaiseStandardError(RaiseStandardError.RUNTIME_ERROR,
'close method on generator classes uimplemented',
fn_info.fitem.line))
builder.add(Unreachable())
blocks, env, _, fn_info = builder.leave()
# Next, add the actual function as a method of the generator class.
sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive), ),
object_rprimitive)
close_fn_decl = FuncDecl('close', fn_info.generator_class.ir.name,
builder.module_name, sig)
close_fn_ir = FuncIR(close_fn_decl, blocks, env)
fn_info.generator_class.ir.methods['close'] = close_fn_ir
builder.functions.append(close_fn_ir)
def gen_func_item(builder: IRBuilder,
fitem: FuncItem,
name: str,
sig: FuncSignature,
cdef: Optional[ClassDef] = None,
) -> Tuple[FuncIR, Optional[Value]]:
"""Generate and return the FuncIR for a given FuncDef.
If the given FuncItem is a nested function, then we generate a
callable class representing the function and use that instead of
the actual function. if the given FuncItem contains a nested
function, then we generate an environment class so that inner
nested functions can access the environment of the given FuncDef.
Consider the following nested function:
def a() -> None:
def b() -> None:
def c() -> None:
return None
return None
return None
The classes generated would look something like the following.
has pointer to +-------+
+--------------------------> | a_env |
| +-------+
| ^
| | has pointer to
+-------+ associated with +-------+
| b_obj | -------------------> | b_env |
+-------+ +-------+
^
|
+-------+ has pointer to |
| c_obj | --------------------------+
+-------+
"""
# TODO: do something about abstract methods.
func_reg = None # type: Optional[Value]
# We treat lambdas as always being nested because we always generate
# a class for lambdas, no matter where they are. (It would probably also
# work to special case toplevel lambdas and generate a non-class function.)
is_nested = fitem in builder.nested_fitems or isinstance(fitem, LambdaExpr)
contains_nested = fitem in builder.encapsulating_funcs.keys()
is_decorated = fitem in builder.fdefs_to_decorators
in_non_ext = False
class_name = None
if cdef:
ir = builder.mapper.type_to_ir[cdef.info]
in_non_ext = not ir.is_ext_class
class_name = cdef.name
builder.enter(FuncInfo(fitem, name, class_name, gen_func_ns(builder),
is_nested, contains_nested, is_decorated, in_non_ext))
# Functions that contain nested functions need an environment class to store variables that
# are free in their nested functions. Generator functions need an environment class to
# store a variable denoting the next instruction to be executed when the __next__ function
# is called, along with all the variables inside the function itself.
if builder.fn_info.contains_nested or builder.fn_info.is_generator:
setup_env_class(builder)
if builder.fn_info.is_nested or builder.fn_info.in_non_ext:
setup_callable_class(builder)
if builder.fn_info.is_generator:
# Do a first-pass and generate a function that just returns a generator object.
gen_generator_func(builder)
blocks, env, ret_type, fn_info = builder.leave()
func_ir, func_reg = gen_func_ir(builder, blocks, sig, env, fn_info, cdef)
# Re-enter the FuncItem and visit the body of the function this time.
builder.enter(fn_info)
setup_env_for_generator_class(builder)
load_outer_envs(builder, builder.fn_info.generator_class)
if builder.fn_info.is_nested and isinstance(fitem, FuncDef):
setup_func_for_recursive_call(builder, fitem, builder.fn_info.generator_class)
create_switch_for_generator_class(builder)
add_raise_exception_blocks_to_generator_class(builder, fitem.line)
else:
load_env_registers(builder)
gen_arg_defaults(builder)
if builder.fn_info.contains_nested and not builder.fn_info.is_generator:
finalize_env_class(builder)
builder.ret_types[-1] = sig.ret_type
# Add all variables and functions that are declared/defined within this
# function and are referenced in functions nested within this one to this
# function's environment class so the nested functions can reference
# them even if they are declared after the nested function's definition.
# Note that this is done before visiting the body of this function.
env_for_func = builder.fn_info # type: Union[FuncInfo, ImplicitClass]
if builder.fn_info.is_generator:
env_for_func = builder.fn_info.generator_class
elif builder.fn_info.is_nested or builder.fn_info.in_non_ext:
env_for_func = builder.fn_info.callable_class
if builder.fn_info.fitem in builder.free_variables:
# Sort the variables to keep things deterministic
for var in sorted(builder.free_variables[builder.fn_info.fitem],
key=lambda x: x.name):
if isinstance(var, Var):
rtype = builder.type_to_rtype(var.type)
builder.add_var_to_env_class(var, rtype, env_for_func, reassign=False)
if builder.fn_info.fitem in builder.encapsulating_funcs:
for nested_fn in builder.encapsulating_funcs[builder.fn_info.fitem]:
if isinstance(nested_fn, FuncDef):
# The return type is 'object' instead of an RInstance of the
# callable class because differently defined functions with
# the same name and signature across conditional blocks
# will generate different callable classes, so the callable
# class that gets instantiated must be generic.
builder.add_var_to_env_class(
nested_fn, object_rprimitive, env_for_func, reassign=False
)
builder.accept(fitem.body)
builder.maybe_add_implicit_return()
if builder.fn_info.is_generator:
populate_switch_for_generator_class(builder)
blocks, env, ret_type, fn_info = builder.leave()
if fn_info.is_generator:
add_methods_to_generator_class(builder, fn_info, sig, env, blocks, fitem.is_coroutine)
else:
func_ir, func_reg = gen_func_ir(builder, blocks, sig, env, fn_info, cdef)
calculate_arg_defaults(builder, fn_info, env, func_reg)
return (func_ir, func_reg)
def generate_attr_defaults(builder: IRBuilder, cdef: ClassDef) -> None:
"""Generate an initialization method for default attr values (from class vars)."""
cls = builder.mapper.type_to_ir[cdef.info]
if cls.builtin_base:
return
# Pull out all assignments in classes in the mro so we can initialize them
# TODO: Support nested statements
default_assignments = []
for info in reversed(cdef.info.mro):
if info not in builder.mapper.type_to_ir:
continue
for stmt in info.defn.defs.body:
if (isinstance(stmt, AssignmentStmt)
and isinstance(stmt.lvalues[0], NameExpr)
and not is_class_var(stmt.lvalues[0])
and not isinstance(stmt.rvalue, TempNode)):
if stmt.lvalues[0].name == '__slots__':
continue
# Skip type annotated assignments in dataclasses
if is_dataclass(cdef) and stmt.type:
continue
default_assignments.append(stmt)
if not default_assignments:
return
builder.enter()
builder.ret_types[-1] = bool_rprimitive
rt_args = (RuntimeArg(SELF_NAME, RInstance(cls)), )
self_var = builder.read(add_self_to_env(builder.environment, cls), -1)
for stmt in default_assignments:
lvalue = stmt.lvalues[0]
assert isinstance(lvalue, NameExpr)
if not stmt.is_final_def and not is_constant(stmt.rvalue):
builder.warning('Unsupported default attribute value',
stmt.rvalue.line)
# If the attribute is initialized to None and type isn't optional,
# don't initialize it to anything.
attr_type = cls.attr_type(lvalue.name)
if isinstance(stmt.rvalue,
RefExpr) and stmt.rvalue.fullname == 'builtins.None':
if (not is_optional_type(attr_type)
and not is_object_rprimitive(attr_type)
and not is_none_rprimitive(attr_type)):
continue
val = builder.coerce(builder.accept(stmt.rvalue), attr_type, stmt.line)
builder.add(SetAttr(self_var, lvalue.name, val, -1))
builder.add(Return(builder.true()))
blocks, env, ret_type, _ = builder.leave()
ir = FuncIR(
FuncDecl('__mypyc_defaults_setup', cls.name, builder.module_name,
FuncSignature(rt_args, ret_type)), blocks, env)
builder.functions.append(ir)
cls.methods[ir.name] = ir
def gen_glue_method(
builder: IRBuilder,
sig: FuncSignature,
target: FuncIR,
cls: ClassIR,
base: ClassIR,
line: int,
do_pycall: bool,
) -> FuncIR:
"""Generate glue methods that mediate between different method types in subclasses.
For example, if we have:
class A:
def f(builder: IRBuilder, x: int) -> object: ...
then it is totally permissible to have a subclass
class B(A):
def f(builder: IRBuilder, x: object) -> int: ...
since '(object) -> int' is a subtype of '(int) -> object' by the usual
contra/co-variant function subtyping rules.
The trickiness here is that int and object have different
runtime representations in mypyc, so A.f and B.f have
different signatures at the native C level. To deal with this,
we need to generate glue methods that mediate between the
different versions by coercing the arguments and return
values.
If do_pycall is True, then make the call using the C API
instead of a native call.
"""
builder.enter()
builder.ret_types[-1] = sig.ret_type
rt_args = list(sig.args)
if target.decl.kind == FUNC_NORMAL:
rt_args[0] = RuntimeArg(sig.args[0].name, RInstance(cls))
arg_info = get_args(builder, rt_args, line)
args, arg_kinds, arg_names = arg_info.args, arg_info.arg_kinds, arg_info.arg_names
# We can do a passthrough *args/**kwargs with a native call, but if the
# args need to get distributed out to arguments, we just let python handle it
if (any(kind.is_star() for kind in arg_kinds)
and any(not arg.kind.is_star() for arg in target.decl.sig.args)):
do_pycall = True
if do_pycall:
if target.decl.kind == FUNC_STATICMETHOD:
# FIXME: this won't work if we can do interpreted subclasses
first = builder.builder.get_native_type(cls)
st = 0
else:
first = args[0]
st = 1
retval = builder.builder.py_method_call(first, target.name, args[st:],
line, arg_kinds[st:],
arg_names[st:])
else:
retval = builder.builder.call(target.decl, args, arg_kinds, arg_names,
line)
retval = builder.coerce(retval, sig.ret_type, line)
builder.add(Return(retval))
arg_regs, _, blocks, ret_type, _ = builder.leave()
return FuncIR(
FuncDecl(target.name + '__' + base.name + '_glue', cls.name,
builder.module_name, FuncSignature(rt_args, ret_type),
target.decl.kind), arg_regs, blocks)
