def generate_wrapper_core(fn: FuncIR, emitter: Emitter,
optional_args: Optional[List[RuntimeArg]] = None,
arg_names: Optional[List[str]] = None,
cleanups: Optional[List[str]] = None,
traceback_code: Optional[str] = None) -> None:
"""Generates the core part of a wrapper function for a native function.
This expects each argument as a PyObject * named obj_{arg} as a precondition.
It converts the PyObject *s to the necessary types, checking and unboxing if necessary,
makes the call, then boxes the result if necessary and returns it.
"""
optional_args = optional_args or []
cleanups = cleanups or []
use_goto = bool(cleanups or traceback_code)
error_code = 'return NULL;' if not use_goto else 'goto fail;'
arg_names = arg_names or [arg.name for arg in fn.args]
for arg_name, arg in zip(arg_names, fn.args):
# Suppress the argument check for *args/**kwargs, since we know it must be right.
typ = arg.type if arg.kind not in (ARG_STAR, ARG_STAR2) else object_rprimitive
generate_arg_check(arg_name, typ, emitter, error_code, arg in optional_args)
native_args = ', '.join('arg_{}'.format(arg) for arg in arg_names)
if fn.ret_type.is_unboxed or use_goto:
# TODO: The Py_RETURN macros return the correct PyObject * with reference count handling.
# Are they relevant?
emitter.emit_line('{}retval = {}{}({});'.format(emitter.ctype_spaced(fn.ret_type),
NATIVE_PREFIX,
fn.cname(emitter.names),
native_args))
emitter.emit_lines(*cleanups)
if fn.ret_type.is_unboxed:
emitter.emit_error_check('retval', fn.ret_type, 'return NULL;')
emitter.emit_box('retval', 'retbox', fn.ret_type, declare_dest=True)
emitter.emit_line('return {};'.format('retbox' if fn.ret_type.is_unboxed else 'retval'))
else:
emitter.emit_line('return {}{}({});'.format(NATIVE_PREFIX,
fn.cname(emitter.names),
native_args))
# TODO: Tracebacks?
if use_goto:
emitter.emit_label('fail')
emitter.emit_lines(*cleanups)
if traceback_code:
emitter.emit_lines(traceback_code)
emitter.emit_lines('return NULL;')
def gen_glue_method(builder: IRBuilder, sig: FuncSignature, target: FuncIR,
cls: ClassIR, base: ClassIR, line: int,
do_pycall: bool,
) -> FuncIR:
"""Generate glue methods that mediate between different method types in subclasses.
For example, if we have:
class A:
def f(builder: IRBuilder, x: int) -> object: ...
then it is totally permissible to have a subclass
class B(A):
def f(builder: IRBuilder, x: object) -> int: ...
since '(object) -> int' is a subtype of '(int) -> object' by the usual
contra/co-variant function subtyping rules.
The trickiness here is that int and object have different
runtime representations in mypyc, so A.f and B.f have
different signatures at the native C level. To deal with this,
we need to generate glue methods that mediate between the
different versions by coercing the arguments and return
values.
If do_pycall is True, then make the call using the C API
instead of a native call.
"""
builder.enter()
builder.ret_types[-1] = sig.ret_type
rt_args = list(sig.args)
if target.decl.kind == FUNC_NORMAL:
rt_args[0] = RuntimeArg(sig.args[0].name, RInstance(cls))
# The environment operates on Vars, so we make some up
fake_vars = [(Var(arg.name), arg.type) for arg in rt_args]
args = [builder.read(builder.add_local_reg(var, type, is_arg=True), line)
for var, type in fake_vars]
arg_names = [arg.name if arg.kind in (ARG_NAMED, ARG_NAMED_OPT) else None
for arg in rt_args]
arg_kinds = [concrete_arg_kind(arg.kind) for arg in rt_args]
if do_pycall:
retval = builder.builder.py_method_call(
args[0], target.name, args[1:], line, arg_kinds[1:], arg_names[1:])
else:
retval = builder.builder.call(target.decl, args, arg_kinds, arg_names, line)
retval = builder.coerce(retval, sig.ret_type, line)
builder.add(Return(retval))
arg_regs, _, blocks, ret_type, _ = builder.leave()
return FuncIR(
FuncDecl(target.name + '__' + base.name + '_glue',
cls.name, builder.module_name,
FuncSignature(rt_args, ret_type),
target.decl.kind),
arg_regs, blocks)
def test_block_not_terminated_empty_block(self) -> None:
block = self.basic_block([])
fn = FuncIR(
decl=self.func_decl(name="func_1"),
arg_regs=[],
blocks=[block],
)
assert_has_error(fn, FnError(source=block,
desc="Block not terminated"))
def test_valid_goto(self) -> None:
block_1 = self.basic_block([Return(value=NONE_VALUE)])
block_2 = self.basic_block([Goto(label=block_1)])
fn = FuncIR(
decl=self.func_decl(name="func_1"),
arg_regs=[],
blocks=[block_1, block_2],
)
assert_no_errors(fn)
def generate_readonly_getter(cl: ClassIR,
attr: str,
rtype: RType,
func_ir: FuncIR,
emitter: Emitter) -> None:
emitter.emit_line('static PyObject *')
emitter.emit_line('{}({} *self, void *closure)'.format(getter_name(cl, attr, emitter.names),
cl.struct_name(emitter.names)))
emitter.emit_line('{')
if rtype.is_unboxed:
emitter.emit_line('{}retval = {}{}((PyObject *) self);'.format(
emitter.ctype_spaced(rtype), NATIVE_PREFIX, func_ir.cname(emitter.names)))
emitter.emit_box('retval', 'retbox', rtype, declare_dest=True)
emitter.emit_line('return retbox;')
else:
emitter.emit_line('return {}{}((PyObject *) self);'.format(NATIVE_PREFIX,
func_ir.cname(emitter.names)))
emitter.emit_line('}')
def generate_function_declaration(fn: FuncIR, emitter: Emitter) -> None:
emitter.context.declarations[emitter.native_function_name(
fn.decl)] = HeaderDeclaration('{};'.format(
native_function_header(fn.decl, emitter)),
needs_export=True)
if fn.name != TOP_LEVEL_NAME:
emitter.context.declarations[
PREFIX + fn.cname(emitter.names)] = HeaderDeclaration('{};'.format(
wrapper_function_header(fn, emitter.names)))
def wrapper_function_header(fn: FuncIR, names: NameGenerator) -> str:
"""Return header of a vectorcall wrapper function.
See comment above for a summary of the arguments.
"""
return (
'PyObject *{prefix}{name}('
'PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames)'
).format(prefix=PREFIX, name=fn.cname(names))
def test_valid_fn(self) -> None:
assert_no_errors(
FuncIR(
decl=self.func_decl(name="func_1"),
arg_regs=[],
blocks=[self.basic_block(ops=[
Return(value=NONE_VALUE),
])],
))
def insert_uninit_checks(ir: FuncIR) -> None:
# Remove dead blocks from the CFG, which helps avoid spurious
# checks due to unused error handling blocks.
cleanup_cfg(ir.blocks)
cfg = get_cfg(ir.blocks)
must_defined = analyze_must_defined_regs(
ir.blocks, cfg, set(ir.arg_regs), all_values(ir.arg_regs, ir.blocks))
ir.blocks = split_blocks_at_uninits(ir.blocks, must_defined.before)
def insert_uninit_checks(ir: FuncIR) -> None:
# Remove dead blocks from the CFG, which helps avoid spurious
# checks due to unused error handling blocks.
cleanup_cfg(ir.blocks)
cfg = get_cfg(ir.blocks)
args = set(reg for reg in ir.env.regs() if ir.env.indexes[reg] < len(ir.args))
must_defined = analyze_must_defined_regs(ir.blocks, cfg, args, ir.env.regs())
ir.blocks = split_blocks_at_uninits(ir.env, ir.blocks, must_defined.before)
def gen_func_ir(
builder: IRBuilder,
args: List[Register],
blocks: List[BasicBlock],
sig: FuncSignature,
fn_info: FuncInfo,
cdef: Optional[ClassDef],
is_singledispatch_main_func: bool = False
) -> Tuple[FuncIR, Optional[Value]]:
"""Generate the FuncIR for a function.
This takes the basic blocks and function info of a particular
function and returns the IR. If the function is nested,
also returns the register containing the instance of the
corresponding callable class.
"""
func_reg: Optional[Value] = None
if fn_info.is_nested or fn_info.in_non_ext:
func_ir = add_call_to_callable_class(builder, args, blocks, sig,
fn_info)
add_get_to_callable_class(builder, fn_info)
func_reg = instantiate_callable_class(builder, fn_info)
else:
assert isinstance(fn_info.fitem, FuncDef)
func_decl = builder.mapper.func_to_decl[fn_info.fitem]
if fn_info.is_decorated or is_singledispatch_main_func:
class_name = None if cdef is None else cdef.name
func_decl = FuncDecl(fn_info.name, class_name, builder.module_name,
sig, func_decl.kind, func_decl.is_prop_getter,
func_decl.is_prop_setter)
func_ir = FuncIR(func_decl,
args,
blocks,
fn_info.fitem.line,
traceback_name=fn_info.fitem.name)
else:
func_ir = FuncIR(func_decl,
args,
blocks,
fn_info.fitem.line,
traceback_name=fn_info.fitem.name)
return (func_ir, func_reg)
def set_target(self, fn: FuncIR) -> None:
"""Set the wrapped function.
It's fine to modify the attributes initialized here later to customize
the wrapper function.
"""
self.target_name = fn.name
self.target_cname = fn.cname(self.emitter.names)
self.arg_names = [arg.name for arg in fn.args]
self.args = fn.args[:]
self.ret_type = fn.ret_type
def generate_property_setter(cl: ClassIR, attr: str, arg_type: RType,
func_ir: FuncIR, emitter: Emitter) -> None:
emitter.emit_line('static int')
emitter.emit_line('{}({} *self, PyObject *value, void *closure)'.format(
setter_name(cl, attr, emitter.names), cl.struct_name(emitter.names)))
emitter.emit_line('{')
if arg_type.is_unboxed:
emitter.emit_unbox('value',
'tmp',
arg_type,
error=ReturnHandler('-1'),
declare_dest=True)
emitter.emit_line('{}{}((PyObject *) self, tmp);'.format(
NATIVE_PREFIX, func_ir.cname(emitter.names)))
else:
emitter.emit_line('{}{}((PyObject *) self, value);'.format(
NATIVE_PREFIX, func_ir.cname(emitter.names)))
emitter.emit_line('return 0;')
emitter.emit_line('}')
def leave_method(self) -> None:
"""Finish the generation of IR for a method."""
arg_regs, args, blocks, ret_type, fn_info = self.leave()
sig = FuncSignature(args, ret_type)
name = self.function_name_stack.pop()
class_ir = self.class_ir_stack.pop()
decl = FuncDecl(name, class_ir.name, self.module_name, sig)
ir = FuncIR(decl, arg_regs, blocks)
class_ir.methods[name] = ir
class_ir.method_decls[name] = ir.decl
self.functions.append(ir)
def test_invalid_return_type(self) -> None:
ret = Return(value=Integer(value=5, rtype=int32_rprimitive))
fn = FuncIR(
decl=self.func_decl(name="func_1", ret_type=int64_rprimitive),
arg_regs=[],
blocks=[self.basic_block([ret])],
)
assert_has_error(
fn,
FnError(source=ret,
desc="Cannot coerce source type int32 to dest type int64"),
)
def test_duplicate_op(self) -> None:
arg_reg = Register(type=int32_rprimitive, name="r1")
assign = Assign(dest=arg_reg,
src=Integer(value=5, rtype=int32_rprimitive))
block = self.basic_block([assign, assign, Return(value=NONE_VALUE)])
fn = FuncIR(
decl=self.func_decl(name="func_1"),
arg_regs=[],
blocks=[block],
)
assert_has_error(
fn, FnError(source=assign, desc="Func has a duplicate op"))
def insert_exception_handling(ir: FuncIR) -> None:
# Generate error block if any ops may raise an exception. If an op
# fails without its own error handler, we'll branch to this
# block. The block just returns an error value.
error_label = None
for block in ir.blocks:
can_raise = any(op.can_raise() for op in block.ops)
if can_raise:
error_label = add_handler_block(ir)
break
if error_label:
ir.blocks = split_blocks_at_errors(ir.blocks, error_label,
ir.traceback_name)
def generate_get_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str:
"""Generates a wrapper for native __get__ methods."""
name = '{}{}{}'.format(DUNDER_PREFIX, fn.name, cl.name_prefix(emitter.names))
emitter.emit_line(
'static PyObject *{name}(PyObject *self, PyObject *instance, PyObject *owner) {{'.
format(name=name))
emitter.emit_line('instance = instance ? instance : Py_None;')
emitter.emit_line('return {}{}(self, instance, owner);'.format(
NATIVE_PREFIX,
fn.cname(emitter.names)))
emitter.emit_line('}')
return name
def test_invalid_goto(self) -> None:
block_1 = self.basic_block([Return(value=NONE_VALUE)])
goto = Goto(label=block_1)
block_2 = self.basic_block([goto])
fn = FuncIR(
decl=self.func_decl(name="func_1"),
arg_regs=[],
# block_1 omitted
blocks=[block_2],
)
assert_has_error(
fn, FnError(source=goto,
desc="Invalid control operation target: 1"))
def generate_set_del_item_wrapper_inner(fn: FuncIR, emitter: Emitter,
args: Sequence[RuntimeArg]) -> None:
for arg in args:
generate_arg_check(arg.name, arg.type, emitter, 'goto fail;', False)
native_args = ', '.join('arg_{}'.format(arg.name) for arg in args)
emitter.emit_line('{}val = {}{}({});'.format(
emitter.ctype_spaced(fn.ret_type), NATIVE_PREFIX,
fn.cname(emitter.names), native_args))
emitter.emit_error_check('val', fn.ret_type, 'goto fail;')
emitter.emit_dec_ref('val', fn.ret_type)
emitter.emit_line('return 0;')
emitter.emit_label('fail')
emitter.emit_line('return -1;')
emitter.emit_line('}')
def test_invalid_register_source(self) -> None:
ret = Return(value=Register(
type=none_rprimitive,
name="r1",
))
block = self.basic_block([ret])
fn = FuncIR(
decl=self.func_decl(name="func_1"),
arg_regs=[],
blocks=[block],
)
assert_has_error(
fn, FnError(source=ret,
desc="Invalid op reference to register r1"))
def test_simple(self) -> None:
self.block.ops.append(Return(self.reg))
fn = FuncIR(FuncDecl('myfunc', None, 'mod', FuncSignature([self.arg], int_rprimitive)),
[self.block], self.env)
emitter = Emitter(EmitterContext(NameGenerator([['mod']])))
generate_native_function(fn, emitter, 'prog.py', 'prog', optimize_int=False)
result = emitter.fragments
assert_string_arrays_equal(
[
'CPyTagged CPyDef_myfunc(CPyTagged cpy_r_arg) {\n',
'CPyL0: ;\n',
' return cpy_r_arg;\n',
'}\n',
],
result, msg='Generated code invalid')
def test_invalid_assign(self) -> None:
arg_reg = Register(type=int64_rprimitive, name="r1")
assign = Assign(dest=arg_reg,
src=Integer(value=5, rtype=int32_rprimitive))
ret = Return(value=NONE_VALUE)
fn = FuncIR(
decl=self.func_decl(name="func_1"),
arg_regs=[arg_reg],
blocks=[self.basic_block([assign, ret])],
)
assert_has_error(
fn,
FnError(source=assign,
desc="Cannot coerce source type int32 to dest type int64"),
)
def add_throw_to_generator_class(builder: IRBuilder, fn_info: FuncInfo,
fn_decl: FuncDecl,
sig: FuncSignature) -> None:
"""Generates the 'throw' method for a generator class."""
builder.enter(fn_info)
self_reg = builder.read(
add_self_to_env(builder.environment, fn_info.generator_class.ir))
# Add the type, value, and traceback variables to the environment.
typ = builder.environment.add_local_reg(Var('type'), object_rprimitive,
True)
val = builder.environment.add_local_reg(Var('value'), object_rprimitive,
True)
tb = builder.environment.add_local_reg(Var('traceback'), object_rprimitive,
True)
# Because the value and traceback arguments are optional and hence
# can be NULL if not passed in, we have to assign them Py_None if
# they are not passed in.
none_reg = builder.none_object()
builder.assign_if_null(val, lambda: none_reg, builder.fn_info.fitem.line)
builder.assign_if_null(tb, lambda: none_reg, builder.fn_info.fitem.line)
# Call the helper function using the arguments passed in, and return that result.
result = builder.add(
Call(fn_decl, [
self_reg,
builder.read(typ),
builder.read(val),
builder.read(tb), none_reg
], fn_info.fitem.line))
builder.add(Return(result))
blocks, env, _, fn_info = builder.leave()
# Create the FuncSignature for the throw function. Note that the
# value and traceback fields are optional, and are assigned to if
# they are not passed in inside the body of the throw function.
sig = FuncSignature((RuntimeArg(
SELF_NAME, object_rprimitive), RuntimeArg('type', object_rprimitive),
RuntimeArg('value', object_rprimitive, ARG_OPT),
RuntimeArg('traceback', object_rprimitive, ARG_OPT)),
sig.ret_type)
throw_fn_decl = FuncDecl('throw', fn_info.generator_class.ir.name,
builder.module_name, sig)
throw_fn_ir = FuncIR(throw_fn_decl, blocks, env)
fn_info.generator_class.ir.methods['throw'] = throw_fn_ir
builder.functions.append(throw_fn_ir)
def add_iter_to_generator_class(builder: IRBuilder, fn_info: FuncInfo) -> None:
"""Generates the '__iter__' method for a generator class."""
builder.enter(fn_info)
self_target = add_self_to_env(builder.environment,
fn_info.generator_class.ir)
builder.add(Return(builder.read(self_target, fn_info.fitem.line)))
blocks, env, _, fn_info = builder.leave()
# Next, add the actual function as a method of the generator class.
sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive), ),
object_rprimitive)
iter_fn_decl = FuncDecl('__iter__', fn_info.generator_class.ir.name,
builder.module_name, sig)
iter_fn_ir = FuncIR(iter_fn_decl, blocks, env)
fn_info.generator_class.ir.methods['__iter__'] = iter_fn_ir
builder.functions.append(iter_fn_ir)
def test_invalid_op_source(self) -> None:
ret = Return(value=LoadLiteral(
value="foo",
rtype=str_rprimitive,
))
block = self.basic_block([ret])
fn = FuncIR(
decl=self.func_decl(name="func_1"),
arg_regs=[],
blocks=[block],
)
assert_has_error(
fn,
FnError(source=ret,
desc="Invalid op reference to op of type LoadLiteral"),
)
def gen_dispatch_func_ir(
builder: IRBuilder,
fitem: FuncDef,
main_func_name: str,
dispatch_name: str,
sig: FuncSignature,
) -> FuncIR:
"""Create a dispatch function (a function that checks the first argument type and dispatches
to the correct implementation)
"""
builder.enter()
generate_singledispatch_dispatch_function(builder, main_func_name, fitem)
args, _, blocks, _, fn_info = builder.leave()
func_decl = FuncDecl(dispatch_name, None, builder.module_name, sig)
dispatch_func_ir = FuncIR(func_decl, args, blocks)
return dispatch_func_ir
def add_call_to_callable_class(builder: IRBuilder,
blocks: List[BasicBlock],
sig: FuncSignature,
env: Environment,
fn_info: FuncInfo) -> FuncIR:
"""Generates a '__call__' method for a callable class representing a nested function.
This takes the blocks, signature, and environment associated with a function definition and
uses those to build the '__call__' method of a given callable class, used to represent that
function. Note that a 'self' parameter is added to its list of arguments, as the nested
function becomes a class method.
"""
sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive),) + sig.args, sig.ret_type)
call_fn_decl = FuncDecl('__call__', fn_info.callable_class.ir.name, builder.module_name, sig)
call_fn_ir = FuncIR(call_fn_decl, blocks, env,
fn_info.fitem.line, traceback_name=fn_info.fitem.name)
fn_info.callable_class.ir.methods['__call__'] = call_fn_ir
return call_fn_ir
def generate_bool_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str:
"""Generates a wrapper for native __bool__ methods."""
name = '{}{}{}'.format(DUNDER_PREFIX, fn.name, cl.name_prefix(emitter.names))
emitter.emit_line('static int {name}(PyObject *self) {{'.format(
name=name
))
emitter.emit_line('{}val = {}{}(self);'.format(emitter.ctype_spaced(fn.ret_type),
NATIVE_PREFIX,
fn.cname(emitter.names)))
emitter.emit_error_check('val', fn.ret_type, 'return -1;')
# This wouldn't be that hard to fix but it seems unimportant and
# getting error handling and unboxing right would be fiddly. (And
# way easier to do in IR!)
assert is_bool_rprimitive(fn.ret_type), "Only bool return supported for __bool__"
emitter.emit_line('return val;')
emitter.emit_line('}')
return name
def test_pprint(self) -> None:
block_1 = self.basic_block([Return(value=NONE_VALUE)])
goto = Goto(label=block_1)
block_2 = self.basic_block([goto])
fn = FuncIR(
decl=self.func_decl(name="func_1"),
arg_regs=[],
# block_1 omitted
blocks=[block_2],
)
errors = [(goto, "Invalid control operation target: 1")]
formatted = format_func(fn, errors)
assert formatted == [
"def func_1():",
"L0:",
" goto L1",
" ERR: Invalid control operation target: 1",
]
