def transfer_to_runnerfunc(self, args, vararg, kwonlyargs, kwarg): ## PyObject **tmp = calloc(sizeof(PyObject*), <len(args)> + 2) #0: set to the function object #1: set to the generator object itself #2: used as a slot to communicate a yielded value #3+: args in canonical order argsname = self.v.scope.ctx.reserve_name('gen_argslist', self.v.tu) decl = c.Decl(argsname, c.PtrDecl(PyObjectLL.typedecl()), init=c.ID('NULL')) self.v.scope.ctx.add_variable(decl, False) self.v.ctx.add(c.Assignment('=', c.ID(argsname), c.FuncCall(c.ID('calloc'), c.ExprList(c.Constant('integer', self.N_EXTRA_PARAMS + len(self.stub_arg_insts)), c.FuncCall(c.ID('sizeof'), c.ExprList(PyObjectLL.typedecl())))))) self.fail_if_null(argsname) self.v.ctx.add(c.Assignment('=', c.ArrayRef(c.ID(argsname), c.Constant('integer', self.SELF_INDEX)), c.ID('self'))) self.v.ctx.add(c.Assignment('=', c.ArrayRef(c.ID(argsname), c.Constant('integer', self.GENERATOR_INDEX)), c.ID('NULL'))) self.v.ctx.add(c.Assignment('=', c.ArrayRef(c.ID(argsname), c.Constant('integer', self.RETURN_INDEX)), c.ID('NULL'))) for i, arg_inst in enumerate(self.stub_arg_insts, self.ARGS_INDEX): self.v.ctx.add(c.Assignment('=', c.ArrayRef(c.ID(argsname), c.Constant('integer', i)), c.ID(arg_inst.name))) self.v.ctx.add(c.FuncCall(c.ID('Py_XINCREF'), c.ExprList(c.ID(arg_inst.name)))) self.v.ctx.add(c.Assignment('=', c.ID('__return_value__'), c.FuncCall(c.ID('MpGenerator_New'), c.ExprList( c.FuncCall(c.ID('strdup'), c.ExprList(c.Constant('string', PyStringLL.name_to_c_string(self.hlnode.owner.name)))), c.ID(self.c_runner_func.decl.name), c.ID(argsname), c.Constant('integer', self.STACKSIZE) #FIXME: try to discover and set a good size for the stack )))) self.fail_if_null('__return_value__') self.v.ctx.add(c.Assignment('=', c.ArrayRef(c.ID(argsname), c.Constant('integer', self.GENERATOR_INDEX)), c.ID('__return_value__')))
def runner_intro(self): '''Set the generator context on the TLS so that we can get to it from generators we call into.''' self.v.ctx.add(c.Comment('cast args to PyObject**')) self.args_name = self.v.scope.ctx.reserve_name('__args__', self.v.tu) self.v.scope.ctx.add_variable(c.Decl(self.args_name, c.PtrDecl(c.PtrDecl(c.TypeDecl(self.args_name, c.IdentifierType('PyObject')))), init=c.ID('NULL')), False) self.v.ctx.add(c.Assignment('=', c.ID(self.args_name), c.Cast(c.PtrDecl(PyObjectLL.typedecl()), c.ID('gen_args')))) self.fail_if_null(self.args_name) #NOTE: we should _not_ incref our __self__ or __gen__ since this function _is_ us -- if we keep this ref and don't # fully drain, then we will never decref ourself and clean up the other references we're holding onto self.v.ctx.add(c.Comment('get function instance')) self.self_inst = PyObjectLL(None, self.v) self.self_inst.declare_tmp(name='__self__') self.v.ctx.add(c.Assignment('=', c.ID(self.self_inst.name), c.ArrayRef(c.ID(self.args_name), c.Constant('integer', self.SELF_INDEX)))) self.fail_if_null(self.self_inst.name) self.v.ctx.add(c.Comment('get generator instance')) self.gen_inst = PyObjectLL(None, self.v) self.gen_inst.declare_tmp(name='__gen__') self.v.ctx.add(c.Assignment('=', c.ID(self.gen_inst.name), c.ArrayRef(c.ID(self.args_name), c.Constant('integer', self.GENERATOR_INDEX)))) self.fail_if_null(self.gen_inst.name) super().runner_intro() self.v.ctx.add(c.Comment('mark us as in the generator')) tmp = CIntegerLL(None, self.v) tmp.declare_tmp() self.v.ctx.add(c.Assignment('=', c.ID(tmp.name), c.FuncCall(c.ID('MpGenerator_EnterContext'), c.ExprList(c.ID(self.gen_inst.name))))) self.fail_if_nonzero(tmp.name) tmp.decref()
def runner_load_args(self, args, vararg, kwonlyargs, kwarg): # put all args into the new MpLocals array, no further decl required for locals args = self._buildargs(args, vararg, kwonlyargs, kwarg) for i, arg in enumerate(args): self.v.ctx.add(c.Comment("set arg '{}'".format(str(arg.arg)))) inst = PyObjectLL(arg.arg.hl, self.v) inst.name = str(arg.arg) self.set_attr_string(str(arg.arg), inst)
def get_item(self, offset:CIntegerLL, out_inst=None): if not isinstance(offset, CIntegerLL): offset = offset.as_ssize() if not out_inst: out_inst = PyObjectLL(None, self.v) out_inst.declare_tmp(name="_item") self.v.ctx.add(c.Assignment('=', c.ID(out_inst.name), c.FuncCall(c.ID('PyTuple_GetItem'), c.ExprList(c.ID(self.name), c.ID(offset.name))))) self.fail_if_null(out_inst.name) out_inst.incref() return out_inst
def runner_load_args(self, args, vararg, kwonlyargs, kwarg): arg_list = self._buildargs(args, vararg, kwonlyargs, kwarg) #FIXME: do not re-declare here... maybe share this with pyfunction? for offset, arg in enumerate(arg_list, self.ARGS_INDEX): #NOTE: incref happened in stub to preserve the value for us inst = PyObjectLL(arg.arg.hl, self.v) inst.declare() self.v.ctx.add(c.Assignment('=', c.ID(inst.name), c.ArrayRef(c.ID(self.args_name), c.Constant('integer', offset)))) self.locals_map[str(arg.arg)] = str(arg.arg) self.args_pos_map.append(str(arg.arg))
def create_builder_funcdef(self): self.v.ctx.add(c.Comment('Declare Class creation pycfunction "{}"'.format(self.hlnode.owner.name))) # create the function pyobject itself builder_func = PyObjectLL(self.hlnode, self.v) builder_func.declare_tmp(name=self.hlnode.owner.name + "_builder_pycfunc") c_name = c.Constant('string', PyStringLL.name_to_c_string(self.hlnode.owner.name)) self.v.ctx.add(c.Assignment('=', c.ID(builder_func.name), c.FuncCall(c.ID('MpFunction_New'), c.ExprList( c_name, c.ID(self.c_builder_func.decl.name), c.ID('NULL'))))) self.fail_if_null(builder_func.name) return builder_func
def create_runnerfunc(self, args, vararg, kwonlyargs, kwarg): body = c.Compound() base_decl = c.Decl('__self__', c.PtrDecl(c.TypeDecl('__self__', c.IdentifierType('PyObject')))) arg_decls = [] for arg in args: ll_inst = self.v.create_ll_instance(arg.arg.hl) ll_inst.name = body.reserve_name(str(arg.arg), self.v.tu) arg_decls.append(c.Decl(ll_inst.name, c.PtrDecl(c.TypeDecl(ll_inst.name, c.IdentifierType('PyObject'))))) if vararg: ll_inst = self.v.create_ll_instance(vararg.hl) ll_inst.name = body.reserve_name(str(vararg), self.v.tu) arg_decls.append(c.Decl(ll_inst.name, c.PtrDecl(c.TypeDecl(ll_inst.name, c.IdentifierType('PyObject'))))) kw_decls = [] for arg in kwonlyargs: ll_inst = self.v.create_ll_instance(arg.arg.hl) ll_inst.name = body.reserve_name(str(arg.arg), self.v.tu) kw_decls.append(c.Decl(ll_inst.name, c.PtrDecl(c.TypeDecl(ll_inst.name, c.IdentifierType('PyObject'))))) if kwarg: ll_inst = self.v.create_ll_instance(kwarg.hl) ll_inst.name = body.reserve_name(str(kwarg), self.v.tu) kw_decls.append(c.Decl(ll_inst.name, c.PtrDecl(c.TypeDecl(ll_inst.name, c.IdentifierType('PyObject'))))) param_list = c.ParamList(base_decl, *(arg_decls + kw_decls)) self._create_runner_common(param_list, PyObjectLL.typedecl(), body)
def set_item_string(self, name:str, var:PyObjectLL): tmp = CIntegerLL(None, self.v) tmp.declare_tmp(name='_set_str_rv') var = var.as_pyobject() #var.incref() self.v.ctx.add(c.Assignment('=', c.ID(tmp.name), c.FuncCall(c.ID('PyDict_SetItemString'), c.ExprList( c.ID(self.name), c.Constant('string', name), c.ID(var.name))))) self.fail_if_nonzero(tmp.name) tmp.decref()
def stub_intro(self): self.stub_self_inst = PyObjectLL(None, self.v) self.stub_self_inst.name = self.v.scope.ctx.reserve_name('self', self.v.tu) self.stub_args_tuple = PyTupleLL(None, self.v) self.stub_args_tuple.name = self.v.scope.ctx.reserve_name('args', self.v.tu) self.stub_kwargs_dict = PyDictLL(None, self.v) self.stub_kwargs_dict.name = self.v.scope.ctx.reserve_name('kwargs', self.v.tu) #FIXME: make this an instance and and inst self.v.scope.ctx.add_variable(c.Decl('__return_value__', PyObjectLL.typedecl('__return_value__'), init=c.ID('NULL')), False)
def declare_function_object(self, docstring): # create the function definition structure c_name = c.Constant('string', PyStringLL.name_to_c_string(self.hlnode.owner.name)) c_docstring = c.Constant('string', PyStringLL.python_to_c_string(docstring)) if docstring else c.ID('NULL') # create the function pyobject itself self.c_obj = PyObjectLL(self.hlnode, self.v) self.c_obj.declare(is_global=True, quals=['static'], name=self.hlnode.owner.global_c_name + "_pycfunc") self.c_obj.xdecref() self.v.ctx.add(c.Assignment('=', c.ID(self.c_obj.name), c.FuncCall(c.ID('MpFunction_New'), c.ExprList( c_name, c.ID(self.c_pystub_func.decl.name), c_docstring)))) self.fail_if_null(self.c_obj.name) # Note: this incref is for the global reference and needs to get cleaned up too # -- the inst we return gets owned by the local scope self.c_obj.incref() return self.c_obj
def intro(self, docstring, module_name): self.v.ctx.add_variable(c.Decl('__return_value__', PyObjectLL.typedecl('__return_value__')), False) # set the docstring ds = PyStringLL(None, self.v) ds.declare_tmp() if docstring: ds.new(docstring) else: ds.assign_none() self.c_namespace_dict.set_item_string('__doc__', ds) ds.decref() # set the module name ds = PyStringLL(None, self.v) ds.declare_tmp() if module_name: ds.new(module_name) self.c_namespace_dict.set_item_string('__module__', ds) ds.decref()
def declare(self): # create the namespace dict self.ll_dict = PyDictLL(self.hlnode, self.v) self.ll_dict.declare(is_global=True, quals=["static"], name=self.hlnode.name + "_dict") # create the module creation function self.c_builder_func = c.FuncDef( c.Decl( self.c_builder_name, c.FuncDecl(c.ParamList(), c.PtrDecl(c.TypeDecl(self.c_builder_name, c.IdentifierType("PyObject")))), quals=["static"], ), c.Compound(), ) self.v.tu.add_fwddecl(self.c_builder_func.decl) self.v.tu.add(self.c_builder_func) # declare the module self.ll_mod = PyObjectLL(self.hlnode, self.v) self.ll_mod.declare(is_global=True, quals=["static"])
def declare_pyclass(self): self.c_obj = PyObjectLL(self.hlnode, self.v) self.c_obj.declare(is_global=True, quals=['static'], name=self.hlnode.owner.global_c_name) return self.c_obj
class PyGeneratorLL(PyFunctionLL): SELF_INDEX = 0 GENERATOR_INDEX = 1 RETURN_INDEX = 2 SEND_INDEX = 3 ARGS_INDEX = 4 N_EXTRA_PARAMS = 4 STACKSIZE = 1024 * 1024 * 1 def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.args_name = None self.self_inst = None self.gen_inst = None def create_runnerfunc(self, args, vararg, kwonlyargs, kwarg): body = c.Compound() body.reserve_name('gen_args', self.v.tu) # create ll insts and declare all args here to match normal functions decl order for arg in args: ll_inst = self.v.create_ll_instance(arg.arg.hl) ll_inst.declare() if vararg: ll_inst = self.v.create_ll_instance(vararg.hl) ll_inst.declare() for arg in kwonlyargs: ll_inst = self.v.create_ll_instance(arg.arg.hl) ll_inst.declare() if kwarg: ll_inst = self.v.create_ll_instance(kwarg.hl) ll_inst.declare() param_list = c.ParamList(c.Decl('gen_args', c.PtrDecl(c.TypeDecl('gen_args', c.IdentifierType('void'))))) return_ty = c.TypeDecl(None, c.IdentifierType('void')) self._create_runner_common(param_list, return_ty, body) def transfer_to_runnerfunc(self, args, vararg, kwonlyargs, kwarg): ## PyObject **tmp = calloc(sizeof(PyObject*), <len(args)> + 2) #0: set to the function object #1: set to the generator object itself #2: used as a slot to communicate a yielded value #3+: args in canonical order argsname = self.v.scope.ctx.reserve_name('gen_argslist', self.v.tu) decl = c.Decl(argsname, c.PtrDecl(PyObjectLL.typedecl()), init=c.ID('NULL')) self.v.scope.ctx.add_variable(decl, False) self.v.ctx.add(c.Assignment('=', c.ID(argsname), c.FuncCall(c.ID('calloc'), c.ExprList(c.Constant('integer', self.N_EXTRA_PARAMS + len(self.stub_arg_insts)), c.FuncCall(c.ID('sizeof'), c.ExprList(PyObjectLL.typedecl())))))) self.fail_if_null(argsname) self.v.ctx.add(c.Assignment('=', c.ArrayRef(c.ID(argsname), c.Constant('integer', self.SELF_INDEX)), c.ID('self'))) self.v.ctx.add(c.Assignment('=', c.ArrayRef(c.ID(argsname), c.Constant('integer', self.GENERATOR_INDEX)), c.ID('NULL'))) self.v.ctx.add(c.Assignment('=', c.ArrayRef(c.ID(argsname), c.Constant('integer', self.RETURN_INDEX)), c.ID('NULL'))) for i, arg_inst in enumerate(self.stub_arg_insts, self.ARGS_INDEX): self.v.ctx.add(c.Assignment('=', c.ArrayRef(c.ID(argsname), c.Constant('integer', i)), c.ID(arg_inst.name))) self.v.ctx.add(c.FuncCall(c.ID('Py_XINCREF'), c.ExprList(c.ID(arg_inst.name)))) self.v.ctx.add(c.Assignment('=', c.ID('__return_value__'), c.FuncCall(c.ID('MpGenerator_New'), c.ExprList( c.FuncCall(c.ID('strdup'), c.ExprList(c.Constant('string', PyStringLL.name_to_c_string(self.hlnode.owner.name)))), c.ID(self.c_runner_func.decl.name), c.ID(argsname), c.Constant('integer', self.STACKSIZE) #FIXME: try to discover and set a good size for the stack )))) self.fail_if_null('__return_value__') self.v.ctx.add(c.Assignment('=', c.ArrayRef(c.ID(argsname), c.Constant('integer', self.GENERATOR_INDEX)), c.ID('__return_value__'))) def runner_load_args(self, args, vararg, kwonlyargs, kwarg): arg_list = self._buildargs(args, vararg, kwonlyargs, kwarg) #FIXME: do not re-declare here... maybe share this with pyfunction? for offset, arg in enumerate(arg_list, self.ARGS_INDEX): #NOTE: incref happened in stub to preserve the value for us inst = PyObjectLL(arg.arg.hl, self.v) inst.declare() self.v.ctx.add(c.Assignment('=', c.ID(inst.name), c.ArrayRef(c.ID(self.args_name), c.Constant('integer', offset)))) self.locals_map[str(arg.arg)] = str(arg.arg) self.args_pos_map.append(str(arg.arg)) def runner_intro(self): '''Set the generator context on the TLS so that we can get to it from generators we call into.''' self.v.ctx.add(c.Comment('cast args to PyObject**')) self.args_name = self.v.scope.ctx.reserve_name('__args__', self.v.tu) self.v.scope.ctx.add_variable(c.Decl(self.args_name, c.PtrDecl(c.PtrDecl(c.TypeDecl(self.args_name, c.IdentifierType('PyObject')))), init=c.ID('NULL')), False) self.v.ctx.add(c.Assignment('=', c.ID(self.args_name), c.Cast(c.PtrDecl(PyObjectLL.typedecl()), c.ID('gen_args')))) self.fail_if_null(self.args_name) #NOTE: we should _not_ incref our __self__ or __gen__ since this function _is_ us -- if we keep this ref and don't # fully drain, then we will never decref ourself and clean up the other references we're holding onto self.v.ctx.add(c.Comment('get function instance')) self.self_inst = PyObjectLL(None, self.v) self.self_inst.declare_tmp(name='__self__') self.v.ctx.add(c.Assignment('=', c.ID(self.self_inst.name), c.ArrayRef(c.ID(self.args_name), c.Constant('integer', self.SELF_INDEX)))) self.fail_if_null(self.self_inst.name) self.v.ctx.add(c.Comment('get generator instance')) self.gen_inst = PyObjectLL(None, self.v) self.gen_inst.declare_tmp(name='__gen__') self.v.ctx.add(c.Assignment('=', c.ID(self.gen_inst.name), c.ArrayRef(c.ID(self.args_name), c.Constant('integer', self.GENERATOR_INDEX)))) self.fail_if_null(self.gen_inst.name) super().runner_intro() self.v.ctx.add(c.Comment('mark us as in the generator')) tmp = CIntegerLL(None, self.v) tmp.declare_tmp() self.v.ctx.add(c.Assignment('=', c.ID(tmp.name), c.FuncCall(c.ID('MpGenerator_EnterContext'), c.ExprList(c.ID(self.gen_inst.name))))) self.fail_if_nonzero(tmp.name) tmp.decref() def _runner_cleanup(self): '''Make sure to leave the __gen__ context before we decref it.''' self.v.ctx.add(c.FuncCall(c.ID('MpGenerator_LeaveContext'), c.ExprList(c.ID(self.gen_inst.name)))) super()._runner_cleanup() def _runner_leave(self): '''In order to leave a coroutine, we set the return context to NULL and transfer back. The generator will raise a StopError in the owning context for us.''' self.v.ctx.add(c.Assignment('=', c.ArrayRef(c.ID(self.args_name), c.Constant('integer', self.RETURN_INDEX)), c.ID('NULL'))) self.v.ctx.add(c.FuncCall(c.ID('MpGenerator_Yield'), c.ExprList(c.ID(self.gen_inst.name)))) def do_yield(self, rv_inst): # assign to our yielded slot slot if not rv_inst: rv_inst = self.v.none # set yielded slot: tp_iter is responsible for incref, the caller of PyIter_Next is responsible for decref ret_ref = c.ArrayRef(c.ID(self.args_name), c.Constant('integer', self.RETURN_INDEX)) self.v.ctx.add(c.Assignment('=', ret_ref, c.ID(rv_inst.name))) with self.v.scope.ll.maybe_recursive_call(): # transfer control back to originator self.v.ctx.add(c.FuncCall(c.ID('MpGenerator_LeaveContext'), c.ExprList(c.ID(self.gen_inst.name)))) self.v.ctx.add(c.FuncCall(c.ID('MpGenerator_Yield'), c.ExprList(c.ID(self.gen_inst.name)))) self.v.ctx.add(c.FuncCall(c.ID('MpGenerator_EnterContext'), c.ExprList(c.ID(self.gen_inst.name)))) # check for dealloc and jump to cleanup if_exhausted = self.v.ctx.add(c.If( c.ArrayRef(c.ID(self.args_name), c.Constant('integer', self.SEND_INDEX)), c.Compound(), None)) with self.v.new_context(if_exhausted.iftrue): self.v.ctx.add(c.Goto('end'))
class PyModuleLL(PyObjectLL): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # the ll instance representing the dict of global variables self.ll_dict = None # the ll name and instance representing the function that builds the module self.c_builder_name = self.v.tu.reserve_global_name(self.hlnode.name + "_builder") self.c_builder_func = None def declare(self): # create the namespace dict self.ll_dict = PyDictLL(self.hlnode, self.v) self.ll_dict.declare(is_global=True, quals=["static"], name=self.hlnode.name + "_dict") # create the module creation function self.c_builder_func = c.FuncDef( c.Decl( self.c_builder_name, c.FuncDecl(c.ParamList(), c.PtrDecl(c.TypeDecl(self.c_builder_name, c.IdentifierType("PyObject")))), quals=["static"], ), c.Compound(), ) self.v.tu.add_fwddecl(self.c_builder_func.decl) self.v.tu.add(self.c_builder_func) # declare the module self.ll_mod = PyObjectLL(self.hlnode, self.v) self.ll_mod.declare(is_global=True, quals=["static"]) def return_existing(self): self.v.ctx.add(c.If(c.ID(self.ll_mod.name), c.Compound(c.Return(c.ID(self.ll_mod.name))), None)) def new(self): self.name = self.ll_mod.name n = "__main__" if self.hlnode.is_main else self.hlnode.python_name self.v.ctx.add(c.Comment('Create module "{}" with __name__ "{}"'.format(self.hlnode.python_name, n))) self.v.ctx.add( c.Assignment( "=", c.ID(self.ll_mod.name), c.FuncCall(c.ID("PyModule_New"), c.ExprList(c.Constant("string", n))) ) ) self.fail_if_null(self.ll_mod.name) # get the modules dict mods = PyDictLL(None, self.v) mods.declare_tmp(name="_modules") self.v.ctx.add(c.Comment("Insert into sys.modules")) self.v.ctx.add(c.Assignment("=", c.ID(mods.name), c.FuncCall(c.ID("PyImport_GetModuleDict"), c.ExprList()))) self.fail_if_null(self.ll_mod.name) mods.incref() # add ourself to the modules dict mods.set_item_string(n, self) # clear the ref so we don't free it later mods.clear() # grab the module dict self.v.ctx.add( c.Assignment( "=", c.ID(self.ll_dict.name), c.FuncCall(c.ID("PyModule_GetDict"), c.ExprList(c.ID(self.ll_mod.name))) ) ) self.fail_if_null(self.ll_dict.name) # set the builtins on the module self.set_attr_string("__builtins__", self.v.builtins) # set builtin properties self.set_initial_string_attribute("__name__", n) # self.ll_module.set_initial_string_attribute(self.context, '__name__', self.hl_module.owner.python_name) def set_initial_string_attribute(self, name: str, s: str): if s is not None: ps = PyStringLL(None, self.v) ps.declare_tmp() ps.new(s) else: ps = PyObjectLL(None, self.v) ps.declare_tmp() ps.assign_none() self.set_attr_string(name, ps) ps.decref() def intro(self): self.v.scope.ctx.add_variable( c.Decl( "__return_value__", c.PtrDecl(c.TypeDecl("__return_value__", c.IdentifierType("PyObject"))), init=c.ID("NULL"), ), False, ) def outro(self): self.v.ctx.add(c.Assignment("=", c.ID("__return_value__"), c.ID(self.ll_mod.name))) self.v.ctx.add(c.Label("end")) for name in reversed(self.v.scope.ctx.cleanup): self.v.ctx.add(c.FuncCall(c.ID("Py_XDECREF"), c.ExprList(c.ID(name)))) self.v.ctx.add(c.Return(c.ID("__return_value__"))) @contextmanager def maybe_recursive_call(self): yield def del_attr_string(self, name: str): self.ll_dict.del_item_string(name) def set_attr_string(self, name: str, val: LLType): self.ll_dict.set_item_string(name, val) # FIXME: do we really need both dict and attr? don't these go to the same place? # super().set_attr_string(name, val) def get_attr_string(self, attrname: str, out: LLType): if str(attrname) in PY_BUILTINS: mode = "likely" else: mode = "unlikely" # access globals first, fall back to builtins -- remember to ref the global if we get it, since dict get item borrows # out.xdecref() self.ll_dict.get_item_string_nofail(attrname, out) frombuiltins = self.v.ctx.add( c.If(c.FuncCall(c.ID(mode), c.ExprList(c.UnaryOp("!", c.ID(out.name)))), c.Compound(), None) ) with self.v.new_context(frombuiltins.iftrue): self.v.builtins.get_attr_string_with_exception( attrname, out, "PyExc_NameError", "name '{}' is not defined".format(attrname) )
class PyClassLL(PyObjectLL): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # the low-level func responsible for building the class self.c_builder_func = None # the pyobject that our builder will return into self.c_obj = None # passed into the class creation function and used by us to set attrs in the local namespace self.c_namespace_dict = None def create_builderfunc(self): param_list = c.ParamList( c.Decl('self', c.PtrDecl(c.TypeDecl('self', c.IdentifierType('PyObject')))), c.Decl('args', c.PtrDecl(c.TypeDecl('args', c.IdentifierType('PyObject')))), c.Decl('kwargs', c.PtrDecl(c.TypeDecl('kwargs', c.IdentifierType('PyObject'))))) name = self.v.tu.reserve_global_name(self.hlnode.owner.global_c_name + '_builder') self.c_builder_func = c.FuncDef( c.Decl(name, c.FuncDecl(param_list, c.PtrDecl(c.TypeDecl(name, c.IdentifierType('PyObject')))), quals=['static']), c.Compound() ) self.v.tu.add_fwddecl(self.c_builder_func.decl) self.v.tu.add(self.c_builder_func) def create_builder_funcdef(self): self.v.ctx.add(c.Comment('Declare Class creation pycfunction "{}"'.format(self.hlnode.owner.name))) # create the function pyobject itself builder_func = PyObjectLL(self.hlnode, self.v) builder_func.declare_tmp(name=self.hlnode.owner.name + "_builder_pycfunc") c_name = c.Constant('string', PyStringLL.name_to_c_string(self.hlnode.owner.name)) self.v.ctx.add(c.Assignment('=', c.ID(builder_func.name), c.FuncCall(c.ID('MpFunction_New'), c.ExprList( c_name, c.ID(self.c_builder_func.decl.name), c.ID('NULL'))))) self.fail_if_null(builder_func.name) return builder_func def declare_pyclass(self): self.c_obj = PyObjectLL(self.hlnode, self.v) self.c_obj.declare(is_global=True, quals=['static'], name=self.hlnode.owner.global_c_name) return self.c_obj def set_namespace(self, ns_dict): self.c_namespace_dict = ns_dict def intro(self, docstring, module_name): self.v.ctx.add_variable(c.Decl('__return_value__', PyObjectLL.typedecl('__return_value__')), False) # set the docstring ds = PyStringLL(None, self.v) ds.declare_tmp() if docstring: ds.new(docstring) else: ds.assign_none() self.c_namespace_dict.set_item_string('__doc__', ds) ds.decref() # set the module name ds = PyStringLL(None, self.v) ds.declare_tmp() if module_name: ds.new(module_name) self.c_namespace_dict.set_item_string('__module__', ds) ds.decref() def outro(self): self.v.none.incref() self.v.ctx.add(c.Assignment('=', c.ID('__return_value__'), c.ID(self.v.none.name))) self.v.ctx.add(c.Label('end')) for name in reversed(self.v.ctx.cleanup): self.v.ctx.add(c.FuncCall(c.ID('Py_XDECREF'), c.ExprList(c.ID(name)))) self.v.ctx.add(c.Return(c.ID('__return_value__'))) @contextmanager def maybe_recursive_call(self): yield def del_attr_string(self, attrname): return self.c_namespace_dict.del_item_string(attrname) def set_attr_string(self, attrname, attrval): return self.c_namespace_dict.set_item_string(attrname, attrval) def get_attr_string(self, attrname, out): self.c_namespace_dict.get_item_string(attrname, out, 'PyExc_NameError', "name '{}' is not defined".format(attrname))
def get_item_string_nofail(self, name:str, out:PyObjectLL): self.v.ctx.add(c.Assignment('=', c.ID(out.name), c.FuncCall(c.ID('PyDict_GetItemString'), c.ExprList( c.ID(self.name), c.Constant('string', name))))) out.xincref() return out
class PyFunctionLL(PyObjectLL): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # map python names to c level names for locals self.locals_map = {} # {str: str} self.args_pos_map = [] # map position to c level names # the ll name and instance representing the c function that does arg decoding for python style calls self.c_pystub_func = None # the ll name and instance representing the c function that runs the python function self.c_runner_func = None # the ll name and instance representing the py callable function object self.c_obj = None # args to the stub functions self.stub_self_inst = None self.stub_args_tuple = None self.stub_kwargs_dict = None self.stub_arg_insts = [] def prepare(self): pass def attach_defaults(self, default_insts, kwdefault_insts): if default_insts: tmp = PyTupleLL(None, self.v) tmp.declare_tmp(name=self.hlnode.owner.name + "_defaults") tmp.pack(*default_insts) self.c_obj.set_attr_string('__defaults__', tmp) tmp.decref() if kwdefault_insts: tmp = PyDictLL(None, self.v) tmp.declare_tmp(name=self.hlnode.owner.name + "_kwdefaults") tmp.new() for name, inst in kwdefault_insts: if inst is None: self.v.none.incref() tmp.set_item_string(name, self.v.none) else: tmp.set_item_string(name, inst) inst.decref() self.c_obj.set_attr_string('__kwdefaults__', tmp) tmp.decref() def attach_annotations(self, ret, args, vararg_name, vararg, kwonlyargs, kwarg_name, kwarg): if not (ret or args or vararg or kwonlyargs or kwarg): return self.v.ctx.add(c.Comment("build annotations dict")) tmp = PyDictLL(None, self.v) tmp.declare_tmp(name=self.hlnode.owner.name + '_annotations') tmp.new() if ret: tmp.set_item_string('return', ret) if vararg: tmp.set_item_string(vararg_name, vararg) if kwarg: tmp.set_item_string(kwarg_name, kwarg) for name, ann in args: if ann: tmp.set_item_string(str(name), ann) for name, ann in kwonlyargs: if ann: tmp.set_item_string(str(name), ann) self.c_obj.set_attr_string('__annotations__', tmp) tmp.decref() ### MpFunction def declare_function_object(self, docstring): # create the function definition structure c_name = c.Constant('string', PyStringLL.name_to_c_string(self.hlnode.owner.name)) c_docstring = c.Constant('string', PyStringLL.python_to_c_string(docstring)) if docstring else c.ID('NULL') # create the function pyobject itself self.c_obj = PyObjectLL(self.hlnode, self.v) self.c_obj.declare(is_global=True, quals=['static'], name=self.hlnode.owner.global_c_name + "_pycfunc") self.c_obj.xdecref() self.v.ctx.add(c.Assignment('=', c.ID(self.c_obj.name), c.FuncCall(c.ID('MpFunction_New'), c.ExprList( c_name, c.ID(self.c_pystub_func.decl.name), c_docstring)))) self.fail_if_null(self.c_obj.name) # Note: this incref is for the global reference and needs to get cleaned up too # -- the inst we return gets owned by the local scope self.c_obj.incref() return self.c_obj ### Stub Func def create_pystubfunc(self): # NOTE: always use kwargs calling convention, because we don't know how external code will call us param_list = c.ParamList( c.Decl('self', c.PtrDecl(c.TypeDecl('args', c.IdentifierType('PyObject')))), c.Decl('args', c.PtrDecl(c.TypeDecl('args', c.IdentifierType('PyObject')))), c.Decl('kwargs', c.PtrDecl(c.TypeDecl('kwargs', c.IdentifierType('PyObject'))))) # create the c function that will correspond to the py function name = self.v.tu.reserve_global_name(self.hlnode.owner.global_c_name + '_pystub') self.c_pystub_func = c.FuncDef( c.Decl(name, c.FuncDecl(param_list, c.PtrDecl(c.TypeDecl(name, c.IdentifierType('PyObject')))), quals=['static']), c.Compound() ) self.v.tu.add_fwddecl(self.c_pystub_func.decl) self.v.tu.add(c.WhiteSpace('\n')) self.v.tu.add(self.c_pystub_func) def stub_intro(self): self.stub_self_inst = PyObjectLL(None, self.v) self.stub_self_inst.name = self.v.scope.ctx.reserve_name('self', self.v.tu) self.stub_args_tuple = PyTupleLL(None, self.v) self.stub_args_tuple.name = self.v.scope.ctx.reserve_name('args', self.v.tu) self.stub_kwargs_dict = PyDictLL(None, self.v) self.stub_kwargs_dict.name = self.v.scope.ctx.reserve_name('kwargs', self.v.tu) #FIXME: make this an instance and and inst self.v.scope.ctx.add_variable(c.Decl('__return_value__', PyObjectLL.typedecl('__return_value__'), init=c.ID('NULL')), False) def stub_load_args(self, args, defaults, vararg, kwonlyargs, kw_defaults, kwarg): self.stub_arg_insts = [] # get args ref args_tuple = self.stub_args_tuple kwargs_dict = self.stub_kwargs_dict # get copy of kwargs -- clear items out of it as we load them, or skip entirely if kwarg: if_have_kwargs = self.v.ctx.add(c.If(c.ID(kwargs_dict.name), c.Compound(), c.Compound())) with self.v.new_context(if_have_kwargs.iftrue): kwargs_inst = kwargs_dict.copy() with self.v.new_context(if_have_kwargs.iffalse): kwargs_dict.new() kwargs_inst = kwargs_dict else: kwargs_inst = None # load positional and normal keyword args if args: c_args_size = CIntegerLL(None, self.v) c_args_size.declare_tmp(name='_args_size') args_tuple.get_size_unchecked(c_args_size) arg_insts = [None] * len(args) for i, arg in enumerate(args): # Note: different scope than the actual args are declared in.. need to stub them out here #TODO: make this type pull from the arg.arg.hl.get_type() through lookup... maybe create dup_ll_type or something arg_insts[i] = PyObjectLL(arg.arg.hl, self.v) arg_insts[i].declare() # query if in positional args self.v.ctx.add(c.Comment("Grab arg {}".format(str(arg.arg)))) query_inst = self.v.ctx.add(c.If(c.BinaryOp('>', c.ID(c_args_size.name), c.Constant('integer', i)), c.Compound(), c.Compound())) ## get the positional arg on the true side with self.v.new_context(query_inst.iftrue): args_tuple.get_unchecked(i, arg_insts[i]) ## get the keyword arg on the false side with self.v.new_context(query_inst.iffalse): have_kwarg = self.v.ctx.add(c.If(c.ID('kwargs'), c.Compound(), None)) ### if we took kwargs, then get it directly with self.v.new_context(have_kwarg.iftrue): kwargs_dict.get_item_string_nofail(str(arg.arg), arg_insts[i]) ### if no kwargs passed or the item was not in the kwargs, load the default from defaults query_default_inst = self.v.ctx.add(c.If(c.UnaryOp('!', c.ID(arg_insts[i].name)), c.Compound(), c.Compound())) with self.v.new_context(query_default_inst.iftrue): kwstartoffset = len(args) - len(defaults) if i >= kwstartoffset: # try loading from defaults default_offset = i - kwstartoffset tmp = PyTupleLL(None, self.v) tmp.declare_tmp() self.c_obj.get_attr_string('__defaults__', tmp) #self.v.ctx.add(c.Assignment('=', c.ID(tmp.name), # c.FuncCall(c.ID('PyObject_GetAttrString'), c.ExprList(c.ID(self.c_obj.name), c.Constant('string', '__defaults__'))))) # tmp.get_unchecked(default_offset, arg_insts[i]) #self.v.ctx.add(c.Assignment('=', c.ID(arg_insts[i].name), # c.FuncCall(c.ID('PyTuple_GetItem'), c.ExprList(c.ID(tmp.name), c.Constant('integer', default_offset))))) arg_insts[i].incref() tmp.decref() #self.v.ctx.add(c.FuncCall(c.ID('Py_INCREF'), c.ID(arg_insts[i].name))) else: # emit an error for an unpassed arg with self.v.new_context(query_default_inst.iftrue): self.fail('PyExc_TypeError', 'Missing arg {}'.format(str(arg))) ### if we did get the item out of the kwargs, delete it from the inst copy so it's not duped in the args we pass with self.v.new_context(query_default_inst.iffalse): if kwargs_inst: kwargs_inst.del_item_string(str(arg.arg)) self.stub_arg_insts.extend(arg_insts) # add unused args to varargs and pass if in taken args or error if not if vararg: self.v.ctx.add(c.Comment('load varargs')) vararg_inst = args_tuple.get_slice(len(args), args_tuple.get_length()) self.stub_arg_insts.append(vararg_inst) else: len_inst = args_tuple.get_length() ifstmt = self.v.ctx.add(c.If(c.BinaryOp('>', c.ID(len_inst.name), c.Constant('integer', len(args))), c.Compound(), None)) with self.v.new_context(ifstmt.iftrue): self.fail_formatted('PyExc_TypeError', "{}() takes exactly {} positional arguments (%d given)".format(self.hlnode.owner.name, len(args)), len_inst) # load all keyword only args if kwonlyargs: kwarg_insts = [None] * len(kwonlyargs) for i, arg in enumerate(kwonlyargs): kwarg_insts[i] = PyObjectLL(arg.arg.hl, self.v) kwarg_insts[i].declare() # ensure we have kwargs at all have_kwarg = self.v.ctx.add(c.If(c.ID('kwargs'), c.Compound(), c.Compound())) ## in have_kwarg.iftrue, load all kwargs from the kwargs dict with self.v.new_context(have_kwarg.iftrue): for i, arg in enumerate(kwonlyargs): #FIXME: we can make this significantly more efficient with a bit of work kwargs_dict.get_item_string_nofail(str(arg.arg), kwarg_insts[i]) need_default = self.v.ctx.add(c.If(c.UnaryOp('!', c.ID(kwarg_insts[i].name)), c.Compound(), None)) ### not found in kwdict, means we need to load from default with self.v.new_context(need_default.iftrue): kwdefaults0 = PyDictLL(None, self.v) kwdefaults0.declare_tmp(name='_kwdefaults') self.c_obj.get_attr_string('__kwdefaults__', kwdefaults0) kwdefaults0.get_item_string(str(arg.arg), kwarg_insts[i]) kwdefaults0.decref() ### found in kwdict, means we need to delete from kwdict to avoid passing duplicate arg in kwargs if kwargs_inst: need_default.iffalse = c.Compound() with self.v.new_context(need_default.iffalse): kwargs_inst.del_item_string(str(arg.arg)) ## if have_kwarg.iffalse, need to load from the kwdefaults dict #TODO: this is identical to the failure case from above with self.v.new_context(have_kwarg.iffalse): kwdefaults1 = PyDictLL(None, self.v) kwdefaults1.declare_tmp(name='_kwdefaults') self.c_obj.get_attr_string('__kwdefaults__', kwdefaults1) for i, arg in enumerate(kwonlyargs): #have_kwarg.iffalse.add(c.Assignment('=', c.ID(kwdefaults1.name), # c.FuncCall(c.ID('PyObject_GetAttrString'), c.ExprList(c.ID(self.c_obj.name), c.Constant('string', '__kwdefaults__'))))) kwdefaults1.get_item_string(str(arg.arg), kwarg_insts[i]) #have_kwarg.iffalse.add(c.Assignment('=', c.ID(kwarg_insts[i].name), # c.FuncCall(c.ID('PyDict_GetItemString'), c.ExprList(c.ID(kwdefaults1.name), c.Constant('string', str(arg.arg)))))) #self.fail_if_null(kwarg_insts[i].name) kwdefaults1.decref() self.stub_arg_insts.extend(kwarg_insts) # pass remainder of kwargs dict in as the kwarg slot if kwarg: self.stub_arg_insts.append(kwargs_inst) ''' # if we have items left in the kwargs dict: # - if we don't take a kwargs slot, then raise a TypeError # - if the args would have overrided an argument we took, then raise a TypeError #import pdb; pdb.set_trace() if kwargs_dict: ifkwargs = self.v.context.add(c.If(c.ID(kwargs_dict.name), c.Compound(), None)) with self.v.new_context(ifkwargs.iftrue): tmp = kwargs_dict.mapping_size(self.v.context) have_extra = self.v.context.add(c.If(c.BinaryOp('<', c.Constant('integer', 0), c.ID(tmp.name)), c.Compound(), None)) with self.v.new_context(have_extra.iftrue): if not kwarg: #tmp = kwargs_dict.mapping_keys(self.v.context) #key = CIntegerLL(None, self.v) #key.declare(self.v.scope.context, name='_key') #key.set_constant(self.v.context, 0) #first_extra_inst = tmp.sequence_get_item(self.v.context, key) #first_extra_inst = first_extra_inst.str(self.v.context) #first_extra_inst = first_extra_inst.as_c_string(self.v.context) self.fail('PyExc_TypeError', self.hlnode.owner.name + "() got an unexpected keyword argument '%s'") #out: foo() got an unexpected keyword argument 'e' #out: foo() got multiple values for keyword argument 'a' ''' def _buildargs(self, args, vararg, kwonlyargs, kwarg): out = [] if args: out.extend(args) if vararg: out.extend([vararg]) if kwonlyargs: out.extend(kwonlyargs) if kwarg: out.extend([kwarg]) return out def transfer_to_runnerfunc(self, args, vararg, kwonlyargs, kwarg): args = [c.ID(inst.name) for inst in self.stub_arg_insts] self.v.ctx.add(c.Assignment('=', c.ID('__return_value__'), c.FuncCall(c.ID(self.c_runner_func.decl.name), c.ExprList(c.ID('self'), *args)))) def stub_outro(self): self.v.ctx.add(c.Label('end')) for name in reversed(self.v.ctx.cleanup): self.v.ctx.add(c.FuncCall(c.ID('Py_XDECREF'), c.ExprList(c.ID(name)))) self.v.ctx.add(c.Return(c.ID('__return_value__'))) # Runner def create_runnerfunc(self, args, vararg, kwonlyargs, kwarg): body = c.Compound() base_decl = c.Decl('__self__', c.PtrDecl(c.TypeDecl('__self__', c.IdentifierType('PyObject')))) arg_decls = [] for arg in args: ll_inst = self.v.create_ll_instance(arg.arg.hl) ll_inst.name = body.reserve_name(str(arg.arg), self.v.tu) arg_decls.append(c.Decl(ll_inst.name, c.PtrDecl(c.TypeDecl(ll_inst.name, c.IdentifierType('PyObject'))))) if vararg: ll_inst = self.v.create_ll_instance(vararg.hl) ll_inst.name = body.reserve_name(str(vararg), self.v.tu) arg_decls.append(c.Decl(ll_inst.name, c.PtrDecl(c.TypeDecl(ll_inst.name, c.IdentifierType('PyObject'))))) kw_decls = [] for arg in kwonlyargs: ll_inst = self.v.create_ll_instance(arg.arg.hl) ll_inst.name = body.reserve_name(str(arg.arg), self.v.tu) kw_decls.append(c.Decl(ll_inst.name, c.PtrDecl(c.TypeDecl(ll_inst.name, c.IdentifierType('PyObject'))))) if kwarg: ll_inst = self.v.create_ll_instance(kwarg.hl) ll_inst.name = body.reserve_name(str(kwarg), self.v.tu) kw_decls.append(c.Decl(ll_inst.name, c.PtrDecl(c.TypeDecl(ll_inst.name, c.IdentifierType('PyObject'))))) param_list = c.ParamList(base_decl, *(arg_decls + kw_decls)) self._create_runner_common(param_list, PyObjectLL.typedecl(), body) def _create_runner_common(self, param_list, return_ty, body): name = self.v.tu.reserve_global_name(self.hlnode.owner.global_c_name + '_runner') self.c_runner_func = c.FuncDef( c.Decl(name, c.FuncDecl(param_list, return_ty), quals=['static']), body ) self.v.tu.add_fwddecl(self.c_runner_func.decl) self.v.tu.add(self.c_runner_func) def runner_load_args(self, args, vararg, kwonlyargs, kwarg): # load args from parameter list into the locals for arg in args: arg_inst = arg.arg.hl.ll self.locals_map[str(arg.arg)] = arg_inst.name self.args_pos_map.append(str(arg.arg)) if vararg: arg_inst = vararg.hl.ll self.locals_map[str(vararg)] = arg_inst.name self.args_pos_map.append(str(vararg)) for arg in kwonlyargs: arg_inst = arg.arg.hl.ll self.locals_map[str(arg.arg)] = arg_inst.name self.args_pos_map.append(str(arg.arg)) if kwarg: arg_inst = kwarg.hl.ll self.locals_map[str(kwarg)] = arg_inst.name self.args_pos_map.append(str(kwarg)) def runner_load_locals(self): for name, sym in self.hlnode.symbols.items(): #if name not in self.locals_map and isinstance(sym, Name): if name not in self.locals_map and sym.parent.ll is self: arg_inst = self.v.create_ll_instance(sym) arg_inst.declare() self.locals_map[name] = arg_inst.name def get_self_accessor(self): '''For use by "super" so we can find ourself automaticlaly when called without args''' return c.ID(self.args_pos_map[0]) def runner_intro(self): self.v.ctx.add_variable(c.Decl('__return_value__', PyObjectLL.typedecl('__return_value__'), init=c.ID('NULL')), False) def runner_outro(self): '''Toplevel interface to runner cleanup and exit, called by high-level users. Internal overriders of function should not override this method, but the lower-level methods instead.''' self._runner_end() self._runner_cleanup() self._runner_leave() def _runner_end(self): self.v.none.incref() self.v.ctx.add(c.Assignment('=', c.ID('__return_value__'), c.ID(self.v.none.name))) self.v.ctx.add(c.Label('end')) def _runner_cleanup(self): for name in reversed(self.v.ctx.cleanup): self.v.ctx.add(c.FuncCall(c.ID('Py_XDECREF'), c.ExprList(c.ID(name)))) def _runner_leave(self): self.v.ctx.add(c.Return(c.ID('__return_value__'))) def del_attr_string(self, attrname): #NOTE: we don't own the ref on our args if attrname not in self.args_pos_map: self.v.ctx.add(c.FuncCall(c.ID('Py_CLEAR'), c.ExprList(c.ID(self.locals_map[attrname])))) else: self.v.ctx.add(c.Assignment('=', c.ID(self.locals_map[attrname]), c.ID('NULL'))) def set_attr_string(self, attrname, val): self.v.ctx.add(c.FuncCall(c.ID('Py_XDECREF'), c.ExprList(c.ID(self.locals_map[attrname])))) val = val.as_pyobject() val.incref() self.v.ctx.add(c.Assignment('=', c.ID(self.locals_map[attrname]), c.ID(val.name))) def get_attr_string(self, attrname, outvar): self.v.ctx.add(c.Assignment('=', c.ID(outvar.name), c.ID(self.locals_map[attrname]))) self.except_if_null(outvar.name, 'PyExc_UnboundLocalError', "local variable '{}' referenced before assignment".format(attrname)) outvar.incref() @contextmanager def maybe_recursive_call(self): yield
def get_item_string(self, name:str, out:PyObjectLL, error_type='PyExc_KeyError', error_str=None): self.v.ctx.add(c.Assignment('=', c.ID(out.name), c.FuncCall(c.ID('PyDict_GetItemString'), c.ExprList( c.ID(self.name), c.Constant('string', name))))) self.except_if_null(out.name, error_type, error_str) out.incref() return out
def runner_intro(self): self.v.ctx.add_variable(c.Decl('__return_value__', PyObjectLL.typedecl('__return_value__'), init=c.ID('NULL')), False)
def _new_from_long(self, c_ast): PyObjectLL.new(self) self.v.ctx.add(c.Assignment('=', c.ID(self.name), c.FuncCall(c.ID('PyBool_FromLong'), c.ExprList(c_ast)))) self.fail_if_null(self.name)
def declare(self, *, quals=[], name=None): super().declare(quals=quals, name=name) self.v.scope.ctx.add_variable(c.Decl(self.name, PyObjectLL.typedecl(self.name), quals=quals, init=c.ID('NULL')), True)