def generate_wrapper_core(fn: FuncIR, emitter: Emitter, optional_args: Optional[List[RuntimeArg]] = None, arg_names: Optional[List[str]] = None, cleanups: Optional[List[str]] = None, traceback_code: Optional[str] = None) -> None: """Generates the core part of a wrapper function for a native function. This expects each argument as a PyObject * named obj_{arg} as a precondition. It converts the PyObject *s to the necessary types, checking and unboxing if necessary, makes the call, then boxes the result if necessary and returns it. """ optional_args = optional_args or [] cleanups = cleanups or [] use_goto = bool(cleanups or traceback_code) error_code = 'return NULL;' if not use_goto else 'goto fail;' arg_names = arg_names or [arg.name for arg in fn.args] for arg_name, arg in zip(arg_names, fn.args): # Suppress the argument check for *args/**kwargs, since we know it must be right. typ = arg.type if arg.kind not in (ARG_STAR, ARG_STAR2) else object_rprimitive generate_arg_check(arg_name, typ, emitter, error_code, arg in optional_args) native_args = ', '.join('arg_{}'.format(arg) for arg in arg_names) if fn.ret_type.is_unboxed or use_goto: # TODO: The Py_RETURN macros return the correct PyObject * with reference count handling. # Are they relevant? emitter.emit_line('{}retval = {}{}({});'.format(emitter.ctype_spaced(fn.ret_type), NATIVE_PREFIX, fn.cname(emitter.names), native_args)) emitter.emit_lines(*cleanups) if fn.ret_type.is_unboxed: emitter.emit_error_check('retval', fn.ret_type, 'return NULL;') emitter.emit_box('retval', 'retbox', fn.ret_type, declare_dest=True) emitter.emit_line('return {};'.format('retbox' if fn.ret_type.is_unboxed else 'retval')) else: emitter.emit_line('return {}{}({});'.format(NATIVE_PREFIX, fn.cname(emitter.names), native_args)) # TODO: Tracebacks? if use_goto: emitter.emit_label('fail') emitter.emit_lines(*cleanups) if traceback_code: emitter.emit_lines(traceback_code) emitter.emit_lines('return NULL;')
def gen_glue_method(builder: IRBuilder, sig: FuncSignature, target: FuncIR, cls: ClassIR, base: ClassIR, line: int, do_pycall: bool, ) -> FuncIR: """Generate glue methods that mediate between different method types in subclasses. For example, if we have: class A: def f(builder: IRBuilder, x: int) -> object: ... then it is totally permissible to have a subclass class B(A): def f(builder: IRBuilder, x: object) -> int: ... since '(object) -> int' is a subtype of '(int) -> object' by the usual contra/co-variant function subtyping rules. The trickiness here is that int and object have different runtime representations in mypyc, so A.f and B.f have different signatures at the native C level. To deal with this, we need to generate glue methods that mediate between the different versions by coercing the arguments and return values. If do_pycall is True, then make the call using the C API instead of a native call. """ builder.enter() builder.ret_types[-1] = sig.ret_type rt_args = list(sig.args) if target.decl.kind == FUNC_NORMAL: rt_args[0] = RuntimeArg(sig.args[0].name, RInstance(cls)) # The environment operates on Vars, so we make some up fake_vars = [(Var(arg.name), arg.type) for arg in rt_args] args = [builder.read(builder.add_local_reg(var, type, is_arg=True), line) for var, type in fake_vars] arg_names = [arg.name if arg.kind in (ARG_NAMED, ARG_NAMED_OPT) else None for arg in rt_args] arg_kinds = [concrete_arg_kind(arg.kind) for arg in rt_args] if do_pycall: retval = builder.builder.py_method_call( args[0], target.name, args[1:], line, arg_kinds[1:], arg_names[1:]) else: retval = builder.builder.call(target.decl, args, arg_kinds, arg_names, line) retval = builder.coerce(retval, sig.ret_type, line) builder.add(Return(retval)) arg_regs, _, blocks, ret_type, _ = builder.leave() return FuncIR( FuncDecl(target.name + '__' + base.name + '_glue', cls.name, builder.module_name, FuncSignature(rt_args, ret_type), target.decl.kind), arg_regs, blocks)
def test_block_not_terminated_empty_block(self) -> None: block = self.basic_block([]) fn = FuncIR( decl=self.func_decl(name="func_1"), arg_regs=[], blocks=[block], ) assert_has_error(fn, FnError(source=block, desc="Block not terminated"))
def test_valid_goto(self) -> None: block_1 = self.basic_block([Return(value=NONE_VALUE)]) block_2 = self.basic_block([Goto(label=block_1)]) fn = FuncIR( decl=self.func_decl(name="func_1"), arg_regs=[], blocks=[block_1, block_2], ) assert_no_errors(fn)
def generate_readonly_getter(cl: ClassIR, attr: str, rtype: RType, func_ir: FuncIR, emitter: Emitter) -> None: emitter.emit_line('static PyObject *') emitter.emit_line('{}({} *self, void *closure)'.format(getter_name(cl, attr, emitter.names), cl.struct_name(emitter.names))) emitter.emit_line('{') if rtype.is_unboxed: emitter.emit_line('{}retval = {}{}((PyObject *) self);'.format( emitter.ctype_spaced(rtype), NATIVE_PREFIX, func_ir.cname(emitter.names))) emitter.emit_box('retval', 'retbox', rtype, declare_dest=True) emitter.emit_line('return retbox;') else: emitter.emit_line('return {}{}((PyObject *) self);'.format(NATIVE_PREFIX, func_ir.cname(emitter.names))) emitter.emit_line('}')
def generate_function_declaration(fn: FuncIR, emitter: Emitter) -> None: emitter.context.declarations[emitter.native_function_name( fn.decl)] = HeaderDeclaration('{};'.format( native_function_header(fn.decl, emitter)), needs_export=True) if fn.name != TOP_LEVEL_NAME: emitter.context.declarations[ PREFIX + fn.cname(emitter.names)] = HeaderDeclaration('{};'.format( wrapper_function_header(fn, emitter.names)))
def wrapper_function_header(fn: FuncIR, names: NameGenerator) -> str: """Return header of a vectorcall wrapper function. See comment above for a summary of the arguments. """ return ( 'PyObject *{prefix}{name}(' 'PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames)' ).format(prefix=PREFIX, name=fn.cname(names))
def test_valid_fn(self) -> None: assert_no_errors( FuncIR( decl=self.func_decl(name="func_1"), arg_regs=[], blocks=[self.basic_block(ops=[ Return(value=NONE_VALUE), ])], ))
def insert_uninit_checks(ir: FuncIR) -> None: # Remove dead blocks from the CFG, which helps avoid spurious # checks due to unused error handling blocks. cleanup_cfg(ir.blocks) cfg = get_cfg(ir.blocks) must_defined = analyze_must_defined_regs( ir.blocks, cfg, set(ir.arg_regs), all_values(ir.arg_regs, ir.blocks)) ir.blocks = split_blocks_at_uninits(ir.blocks, must_defined.before)
def insert_uninit_checks(ir: FuncIR) -> None: # Remove dead blocks from the CFG, which helps avoid spurious # checks due to unused error handling blocks. cleanup_cfg(ir.blocks) cfg = get_cfg(ir.blocks) args = set(reg for reg in ir.env.regs() if ir.env.indexes[reg] < len(ir.args)) must_defined = analyze_must_defined_regs(ir.blocks, cfg, args, ir.env.regs()) ir.blocks = split_blocks_at_uninits(ir.env, ir.blocks, must_defined.before)
def gen_func_ir( builder: IRBuilder, args: List[Register], blocks: List[BasicBlock], sig: FuncSignature, fn_info: FuncInfo, cdef: Optional[ClassDef], is_singledispatch_main_func: bool = False ) -> Tuple[FuncIR, Optional[Value]]: """Generate the FuncIR for a function. This takes the basic blocks and function info of a particular function and returns the IR. If the function is nested, also returns the register containing the instance of the corresponding callable class. """ func_reg: Optional[Value] = None if fn_info.is_nested or fn_info.in_non_ext: func_ir = add_call_to_callable_class(builder, args, blocks, sig, fn_info) add_get_to_callable_class(builder, fn_info) func_reg = instantiate_callable_class(builder, fn_info) else: assert isinstance(fn_info.fitem, FuncDef) func_decl = builder.mapper.func_to_decl[fn_info.fitem] if fn_info.is_decorated or is_singledispatch_main_func: class_name = None if cdef is None else cdef.name func_decl = FuncDecl(fn_info.name, class_name, builder.module_name, sig, func_decl.kind, func_decl.is_prop_getter, func_decl.is_prop_setter) func_ir = FuncIR(func_decl, args, blocks, fn_info.fitem.line, traceback_name=fn_info.fitem.name) else: func_ir = FuncIR(func_decl, args, blocks, fn_info.fitem.line, traceback_name=fn_info.fitem.name) return (func_ir, func_reg)
def set_target(self, fn: FuncIR) -> None: """Set the wrapped function. It's fine to modify the attributes initialized here later to customize the wrapper function. """ self.target_name = fn.name self.target_cname = fn.cname(self.emitter.names) self.arg_names = [arg.name for arg in fn.args] self.args = fn.args[:] self.ret_type = fn.ret_type
def generate_property_setter(cl: ClassIR, attr: str, arg_type: RType, func_ir: FuncIR, emitter: Emitter) -> None: emitter.emit_line('static int') emitter.emit_line('{}({} *self, PyObject *value, void *closure)'.format( setter_name(cl, attr, emitter.names), cl.struct_name(emitter.names))) emitter.emit_line('{') if arg_type.is_unboxed: emitter.emit_unbox('value', 'tmp', arg_type, error=ReturnHandler('-1'), declare_dest=True) emitter.emit_line('{}{}((PyObject *) self, tmp);'.format( NATIVE_PREFIX, func_ir.cname(emitter.names))) else: emitter.emit_line('{}{}((PyObject *) self, value);'.format( NATIVE_PREFIX, func_ir.cname(emitter.names))) emitter.emit_line('return 0;') emitter.emit_line('}')
def leave_method(self) -> None: """Finish the generation of IR for a method.""" arg_regs, args, blocks, ret_type, fn_info = self.leave() sig = FuncSignature(args, ret_type) name = self.function_name_stack.pop() class_ir = self.class_ir_stack.pop() decl = FuncDecl(name, class_ir.name, self.module_name, sig) ir = FuncIR(decl, arg_regs, blocks) class_ir.methods[name] = ir class_ir.method_decls[name] = ir.decl self.functions.append(ir)
def test_invalid_return_type(self) -> None: ret = Return(value=Integer(value=5, rtype=int32_rprimitive)) fn = FuncIR( decl=self.func_decl(name="func_1", ret_type=int64_rprimitive), arg_regs=[], blocks=[self.basic_block([ret])], ) assert_has_error( fn, FnError(source=ret, desc="Cannot coerce source type int32 to dest type int64"), )
def test_duplicate_op(self) -> None: arg_reg = Register(type=int32_rprimitive, name="r1") assign = Assign(dest=arg_reg, src=Integer(value=5, rtype=int32_rprimitive)) block = self.basic_block([assign, assign, Return(value=NONE_VALUE)]) fn = FuncIR( decl=self.func_decl(name="func_1"), arg_regs=[], blocks=[block], ) assert_has_error( fn, FnError(source=assign, desc="Func has a duplicate op"))
def insert_exception_handling(ir: FuncIR) -> None: # Generate error block if any ops may raise an exception. If an op # fails without its own error handler, we'll branch to this # block. The block just returns an error value. error_label = None for block in ir.blocks: can_raise = any(op.can_raise() for op in block.ops) if can_raise: error_label = add_handler_block(ir) break if error_label: ir.blocks = split_blocks_at_errors(ir.blocks, error_label, ir.traceback_name)
def generate_get_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: """Generates a wrapper for native __get__ methods.""" name = '{}{}{}'.format(DUNDER_PREFIX, fn.name, cl.name_prefix(emitter.names)) emitter.emit_line( 'static PyObject *{name}(PyObject *self, PyObject *instance, PyObject *owner) {{'. format(name=name)) emitter.emit_line('instance = instance ? instance : Py_None;') emitter.emit_line('return {}{}(self, instance, owner);'.format( NATIVE_PREFIX, fn.cname(emitter.names))) emitter.emit_line('}') return name
def test_invalid_goto(self) -> None: block_1 = self.basic_block([Return(value=NONE_VALUE)]) goto = Goto(label=block_1) block_2 = self.basic_block([goto]) fn = FuncIR( decl=self.func_decl(name="func_1"), arg_regs=[], # block_1 omitted blocks=[block_2], ) assert_has_error( fn, FnError(source=goto, desc="Invalid control operation target: 1"))
def generate_set_del_item_wrapper_inner(fn: FuncIR, emitter: Emitter, args: Sequence[RuntimeArg]) -> None: for arg in args: generate_arg_check(arg.name, arg.type, emitter, 'goto fail;', False) native_args = ', '.join('arg_{}'.format(arg.name) for arg in args) emitter.emit_line('{}val = {}{}({});'.format( emitter.ctype_spaced(fn.ret_type), NATIVE_PREFIX, fn.cname(emitter.names), native_args)) emitter.emit_error_check('val', fn.ret_type, 'goto fail;') emitter.emit_dec_ref('val', fn.ret_type) emitter.emit_line('return 0;') emitter.emit_label('fail') emitter.emit_line('return -1;') emitter.emit_line('}')
def test_invalid_register_source(self) -> None: ret = Return(value=Register( type=none_rprimitive, name="r1", )) block = self.basic_block([ret]) fn = FuncIR( decl=self.func_decl(name="func_1"), arg_regs=[], blocks=[block], ) assert_has_error( fn, FnError(source=ret, desc="Invalid op reference to register r1"))
def test_simple(self) -> None: self.block.ops.append(Return(self.reg)) fn = FuncIR(FuncDecl('myfunc', None, 'mod', FuncSignature([self.arg], int_rprimitive)), [self.block], self.env) emitter = Emitter(EmitterContext(NameGenerator([['mod']]))) generate_native_function(fn, emitter, 'prog.py', 'prog', optimize_int=False) result = emitter.fragments assert_string_arrays_equal( [ 'CPyTagged CPyDef_myfunc(CPyTagged cpy_r_arg) {\n', 'CPyL0: ;\n', ' return cpy_r_arg;\n', '}\n', ], result, msg='Generated code invalid')
def test_invalid_assign(self) -> None: arg_reg = Register(type=int64_rprimitive, name="r1") assign = Assign(dest=arg_reg, src=Integer(value=5, rtype=int32_rprimitive)) ret = Return(value=NONE_VALUE) fn = FuncIR( decl=self.func_decl(name="func_1"), arg_regs=[arg_reg], blocks=[self.basic_block([assign, ret])], ) assert_has_error( fn, FnError(source=assign, desc="Cannot coerce source type int32 to dest type int64"), )
def add_throw_to_generator_class(builder: IRBuilder, fn_info: FuncInfo, fn_decl: FuncDecl, sig: FuncSignature) -> None: """Generates the 'throw' method for a generator class.""" builder.enter(fn_info) self_reg = builder.read( add_self_to_env(builder.environment, fn_info.generator_class.ir)) # Add the type, value, and traceback variables to the environment. typ = builder.environment.add_local_reg(Var('type'), object_rprimitive, True) val = builder.environment.add_local_reg(Var('value'), object_rprimitive, True) tb = builder.environment.add_local_reg(Var('traceback'), object_rprimitive, True) # Because the value and traceback arguments are optional and hence # can be NULL if not passed in, we have to assign them Py_None if # they are not passed in. none_reg = builder.none_object() builder.assign_if_null(val, lambda: none_reg, builder.fn_info.fitem.line) builder.assign_if_null(tb, lambda: none_reg, builder.fn_info.fitem.line) # Call the helper function using the arguments passed in, and return that result. result = builder.add( Call(fn_decl, [ self_reg, builder.read(typ), builder.read(val), builder.read(tb), none_reg ], fn_info.fitem.line)) builder.add(Return(result)) blocks, env, _, fn_info = builder.leave() # Create the FuncSignature for the throw function. Note that the # value and traceback fields are optional, and are assigned to if # they are not passed in inside the body of the throw function. sig = FuncSignature((RuntimeArg( SELF_NAME, object_rprimitive), RuntimeArg('type', object_rprimitive), RuntimeArg('value', object_rprimitive, ARG_OPT), RuntimeArg('traceback', object_rprimitive, ARG_OPT)), sig.ret_type) throw_fn_decl = FuncDecl('throw', fn_info.generator_class.ir.name, builder.module_name, sig) throw_fn_ir = FuncIR(throw_fn_decl, blocks, env) fn_info.generator_class.ir.methods['throw'] = throw_fn_ir builder.functions.append(throw_fn_ir)
def add_iter_to_generator_class(builder: IRBuilder, fn_info: FuncInfo) -> None: """Generates the '__iter__' method for a generator class.""" builder.enter(fn_info) self_target = add_self_to_env(builder.environment, fn_info.generator_class.ir) builder.add(Return(builder.read(self_target, fn_info.fitem.line))) blocks, env, _, fn_info = builder.leave() # Next, add the actual function as a method of the generator class. sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive), ), object_rprimitive) iter_fn_decl = FuncDecl('__iter__', fn_info.generator_class.ir.name, builder.module_name, sig) iter_fn_ir = FuncIR(iter_fn_decl, blocks, env) fn_info.generator_class.ir.methods['__iter__'] = iter_fn_ir builder.functions.append(iter_fn_ir)
def test_invalid_op_source(self) -> None: ret = Return(value=LoadLiteral( value="foo", rtype=str_rprimitive, )) block = self.basic_block([ret]) fn = FuncIR( decl=self.func_decl(name="func_1"), arg_regs=[], blocks=[block], ) assert_has_error( fn, FnError(source=ret, desc="Invalid op reference to op of type LoadLiteral"), )
def gen_dispatch_func_ir( builder: IRBuilder, fitem: FuncDef, main_func_name: str, dispatch_name: str, sig: FuncSignature, ) -> FuncIR: """Create a dispatch function (a function that checks the first argument type and dispatches to the correct implementation) """ builder.enter() generate_singledispatch_dispatch_function(builder, main_func_name, fitem) args, _, blocks, _, fn_info = builder.leave() func_decl = FuncDecl(dispatch_name, None, builder.module_name, sig) dispatch_func_ir = FuncIR(func_decl, args, blocks) return dispatch_func_ir
def add_call_to_callable_class(builder: IRBuilder, blocks: List[BasicBlock], sig: FuncSignature, env: Environment, fn_info: FuncInfo) -> FuncIR: """Generates a '__call__' method for a callable class representing a nested function. This takes the blocks, signature, and environment associated with a function definition and uses those to build the '__call__' method of a given callable class, used to represent that function. Note that a 'self' parameter is added to its list of arguments, as the nested function becomes a class method. """ sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive),) + sig.args, sig.ret_type) call_fn_decl = FuncDecl('__call__', fn_info.callable_class.ir.name, builder.module_name, sig) call_fn_ir = FuncIR(call_fn_decl, blocks, env, fn_info.fitem.line, traceback_name=fn_info.fitem.name) fn_info.callable_class.ir.methods['__call__'] = call_fn_ir return call_fn_ir
def generate_bool_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: """Generates a wrapper for native __bool__ methods.""" name = '{}{}{}'.format(DUNDER_PREFIX, fn.name, cl.name_prefix(emitter.names)) emitter.emit_line('static int {name}(PyObject *self) {{'.format( name=name )) emitter.emit_line('{}val = {}{}(self);'.format(emitter.ctype_spaced(fn.ret_type), NATIVE_PREFIX, fn.cname(emitter.names))) emitter.emit_error_check('val', fn.ret_type, 'return -1;') # This wouldn't be that hard to fix but it seems unimportant and # getting error handling and unboxing right would be fiddly. (And # way easier to do in IR!) assert is_bool_rprimitive(fn.ret_type), "Only bool return supported for __bool__" emitter.emit_line('return val;') emitter.emit_line('}') return name
def test_pprint(self) -> None: block_1 = self.basic_block([Return(value=NONE_VALUE)]) goto = Goto(label=block_1) block_2 = self.basic_block([goto]) fn = FuncIR( decl=self.func_decl(name="func_1"), arg_regs=[], # block_1 omitted blocks=[block_2], ) errors = [(goto, "Invalid control operation target: 1")] formatted = format_func(fn, errors) assert formatted == [ "def func_1():", "L0:", " goto L1", " ERR: Invalid control operation target: 1", ]