def kill_assertion_link(graph, link): block = link.prevblock exits = list(block.exits) if len(exits) <= 1: return False remove_condition = len(exits) == 2 if block.exitswitch == c_last_exception: if link is exits[0]: return False # cannot remove the non-exceptional path else: if block.exitswitch.concretetype is not lltype.Bool: # a switch remove_condition = False else: # common case: if <cond>: raise AssertionError # turn it into a debug_assert operation assert remove_condition newops = LowLevelOpList() if link.exitcase: v = newops.genop('bool_not', [block.exitswitch], resulttype=lltype.Bool) else: v = block.exitswitch msg = "assertion failed in %s" % (graph.name, ) c_msg = inputconst(lltype.Void, msg) newops.genop('debug_assert', [v, c_msg]) block.operations.extend(newops) exits.remove(link) if remove_condition: # condition no longer necessary block.exitswitch = None exits[0].exitcase = None exits[0].llexitcase = None block.recloseblock(*exits) return True
def kill_assertion_link(graph, link): block = link.prevblock exits = list(block.exits) if len(exits) <= 1: return False remove_condition = len(exits) == 2 if block.exitswitch == c_last_exception: if link is exits[0]: return False # cannot remove the non-exceptional path else: if block.exitswitch.concretetype is not lltype.Bool: # a switch remove_condition = False else: # common case: if <cond>: raise AssertionError # turn it into a debug_assert operation assert remove_condition newops = LowLevelOpList() if link.exitcase: v = newops.genop('bool_not', [block.exitswitch], resulttype = lltype.Bool) else: v = block.exitswitch msg = "assertion failed in %s" % (graph.name,) c_msg = inputconst(lltype.Void, msg) newops.genop('debug_assert', [v, c_msg]) block.operations.extend(newops) exits.remove(link) if remove_condition: # condition no longer necessary block.exitswitch = None exits[0].exitcase = None exits[0].llexitcase = None block.recloseblock(*exits) return True
def finish(self): # compute the final masterarray by copying over the masterarray1, # which is a list of dicts of attributes if SAVE_STATISTICS: import cPickle cPickle.dump(self.stats, open('stackless-stats.pickle', 'wb')) # fun fun fun patching the call_function_retval_xyz() functions! for RESTYPE, typename in frame.STORAGE_TYPES_AND_FIELDS: rettype_index = STORAGE_TYPES.index(RESTYPE) cache = self.signaturecodes[rettype_index] if not cache: continue # not used anyway, don't produce a broken empty switch func = getattr(code, 'call_function_retval_' + typename) desc = self.translator.annotator.bookkeeper.getdesc(func) graph = desc.getuniquegraph() [v_fnaddr, v_signature_index] = graph.getargs() block = model.Block([v_fnaddr, v_signature_index]) block.exitswitch = v_signature_index block.isstartblock = True graph.startblock = block switchlinks = [] for ARGTYPES, signature_index in cache.items(): # XXX because of type erasure, the following cast is # kind of invalid, but we hope that nobody will notice FUNCTYPE = lltype.Ptr(lltype.FuncType(ARGTYPES, RESTYPE)) v_fnaddr1 = varoftype(v_fnaddr.concretetype) callblock = model.Block([v_fnaddr1]) llops = LowLevelOpList() args_v = [model.Constant(TYPE._defl(), concretetype=TYPE) for TYPE in ARGTYPES] v_res = llops.genop('adr_call', [v_fnaddr1] + args_v, resulttype = RESTYPE) callblock.operations[:] = llops callblock.closeblock(model.Link([v_res], graph.returnblock)) link = model.Link([v_fnaddr], callblock) link.exitcase = signature_index link.llexitcase = signature_index switchlinks.append(link) block.closeblock(*switchlinks) model.checkgraph(graph) self.is_finished = True masterarray = lltype.malloc(frame.FRAME_INFO_ARRAY, len(self.masterarray1), immortal=True) for dst, src in zip(masterarray, self.masterarray1): dst.fnaddr, dst.info = src # horrors in the same spirit as in rpython.memory.gctransform # (shorter, though) ll_global_state = self.ll_global_state.value ll_global_state.inst_masterarray = masterarray return [masterarray]
def test_nclc_nongc_not_passed_on(): # +--- inputargs: pointer_to_gc # | v0 <- op_getsubstruct pointer_to_gc 'b' # +--- exitargs: pointer_to_gc (i.e. the pointer to non-gc doesn't leave the block) llops = LowLevelOpList() ptr_a = varoftype(lltype.Ptr(GcA)) v_res = llops.genop("getsubstruct", [ptr_a, model.Constant('b', lltype.Void)], resulttype=lltype.Ptr(NonGcB)) block = model.Block([ptr_a]) block.operations.extend(llops) block.closeblock(model.Link([ptr_a], None)) assert not needs_conservative_livevar_calculation(block)
def transform_graph(self, graph): if graph in self.minimal_transform: if self.minimalgctransformer: self.minimalgctransformer.transform_graph(graph) del self.minimal_transform[graph] return if graph in self.seen_graphs: return self.seen_graphs[graph] = True self.links_to_split = {} # link -> vars to pop_alive across the link # for sanity, we need an empty block at the start of the graph inserted_empty_startblock = False if not starts_with_empty_block(graph): insert_empty_startblock(self.translator.annotator, graph) inserted_empty_startblock = True is_borrowed = self.compute_borrowed_vars(graph) for block in graph.iterblocks(): self.transform_block(block, is_borrowed) for link, livecounts in self.links_to_split.iteritems(): llops = LowLevelOpList() for var, livecount in livecounts.iteritems(): for i in range(livecount): self.pop_alive(var, llops) for i in range(-livecount): self.push_alive(var, llops) if llops: if link.prevblock.exitswitch is None: link.prevblock.operations.extend(llops) else: insert_empty_block(self.translator.annotator, link, llops) # remove the empty block at the start of the graph, which should # still be empty (but let's check) if starts_with_empty_block(graph) and inserted_empty_startblock: old_startblock = graph.startblock graph.startblock.isstartblock = False graph.startblock = graph.startblock.exits[0].target graph.startblock.isstartblock = True checkgraph(graph) self.links_to_split = None v = Variable('vanishing_exc_value') v.concretetype = self.get_lltype_of_exception_value() llops = LowLevelOpList() self.pop_alive(v, llops) graph.exc_cleanup = (v, list(llops)) return is_borrowed # xxx for tests only
def test_nclc_should_be_true(): # this is testing a block like: # +--- inputargs: pointer_to_gc # | v0 <- op_getsubstruct pointer_to_gc 'b' # +--- exitargs: v0 (i.e. pointer to non-gc) llops = LowLevelOpList() ptr_a = varoftype(lltype.Ptr(GcA)) v_res = llops.genop("getsubstruct", [ptr_a, model.Constant('b', lltype.Void)], resulttype=lltype.Ptr(NonGcB)) block = model.Block([ptr_a]) block.operations.extend(llops) block.closeblock(model.Link([v_res], None)) assert needs_conservative_livevar_calculation(block)
def test_sbwk_should_insert_keepalives(): # this is testing something like: # v0 <- op_producing_non_gc # v1 <- op_using_v0 <- split here llops = LowLevelOpList() ptr_a = varoftype(lltype.Ptr(GcA)) v_res = llops.genop("getfield", [ptr_a, model.Constant('b', lltype.Void)], resulttype=lltype.Ptr(NonGcB)) llops.genop("direct_call", [model.Constant(None, lltype.Void), v_res], resulttype=lltype.Void) block = model.Block([ptr_a]) block.operations.extend(llops) block.closeblock(model.Link([], None)) link = split_block_with_keepalive(block, 1) assert 'keepalive' in [op.opname for op in link.target.operations]
def test_nclc_ignore_functype(): # +--- inputargs: pointer_to_gc # | v0 <- op_getfield pointer_to_gc 'c' # +--- exitargs: v0 (i.e. a pointer to function) # pointers to functions are 'not gc' but functions are also # immortal so you don't need to muck around inserting keepalives # so *they* don't die! llops = LowLevelOpList() ptr_a = varoftype(lltype.Ptr(GcA)) v_res = llops.genop("getfield", [ptr_a, model.Constant('c', lltype.Void)], resulttype=GcA.c) block = model.Block([ptr_a]) block.operations.extend(llops) block.closeblock(model.Link([v_res], None)) assert not needs_conservative_livevar_calculation(block)
def _make_resume_return_block(self, erased_types, erased_retval_type, except_links, sorted_vars): inputargs = [varoftype(lltype.Signed)] + [varoftype(t) for t in erased_types] return_block = model.Block(inputargs) return_block.exitswitch = model.c_last_exception llops = LowLevelOpList() getretval = self.fetch_retvals[erased_retval_type] v_retval = llops.genop("direct_call", [getretval], resulttype=erased_retval_type) switch_block = model.Block([copyvar(v) for v in inputargs]) switch_block.exitswitch = switch_block.inputargs[0] retlink = model.Link(inputargs, switch_block, None) if erased_retval_type != lltype.Void: retlink.args.append(v_retval) switch_block.inputargs.append(copyvar(v_retval)) links = [retlink] for except_link in except_links: cast_block, cast_args = self._make_cast_block( erased_types, [v.concretetype for v in sorted_vars]) varmap = dict([(v, cast_args[sorted_vars.index(v)]) for v in sorted_vars]) link = model.Link(inputargs[1:], cast_block, except_link.exitcase) link.llexitcase = except_link.llexitcase for attr in "last_exception", "last_exc_value": old = getattr(except_link, attr) new = copyvar(old) setattr(link, attr, new) link.args.append(new) newnew = copyvar(new) cast_block.inputargs.append(newnew) varmap[old] = newnew links.append(link) link = copy_link_with_varmap(except_link, varmap) link.exitcase = link.llexitcase = None link.last_exception = link.last_exc_value = None cast_block.closeblock(link) return_block.operations = llops return_block.closeblock(*links) return return_block, switch_block
def _generate_save_block(self, varsforcall, v_unwind_exception, saver): conc_types = tuple([v.concretetype for v in varsforcall]) if conc_types in self.curr_graph_save_blocks: return self.curr_graph_save_blocks[conc_types] rtyper = self.translator.rtyper edata = rtyper.getexceptiondata() etype = edata.lltype_of_exception_type evalue = edata.lltype_of_exception_value inputargs = [copyvar(v) for v in varsforcall] v_unwind_exception = copyvar(v_unwind_exception) save_state_block = model.Block(inputargs + [v_unwind_exception]) saveops = LowLevelOpList() v_exc = gen_cast(saveops, self.unwind_exception_type, v_unwind_exception) realvarsforcall = [v_exc] for v in inputargs: realvarsforcall.append(gen_cast(saveops, storage_type(v.concretetype), v)) saveops.genop('direct_call', [model.Constant(saver, lltype.typeOf(saver))] + realvarsforcall, resulttype=lltype.Void) save_state_block.operations = saveops type_repr = rclass.get_type_repr(rtyper) c_unwindexception = model.Constant( type_repr.convert_const(code.UnwindException), etype) if not hasattr(self.curr_graph.exceptblock.inputargs[0], 'concretetype'): self.curr_graph.exceptblock.inputargs[0].concretetype = etype if not hasattr(self.curr_graph.exceptblock.inputargs[1], 'concretetype'): self.curr_graph.exceptblock.inputargs[1].concretetype = evalue save_state_block.closeblock(model.Link( [c_unwindexception, v_unwind_exception], self.curr_graph.exceptblock)) self.translator.rtyper._convert_link( save_state_block, save_state_block.exits[0]) if SAVE_STATISTICS: self.stats.saveops += len(save_state_block.operations) self.curr_graph_save_blocks[conc_types] = save_state_block return save_state_block
def transform_block(self, block, is_borrowed): llops = LowLevelOpList() #self.curr_block = block self.livevars = [ var for var in block.inputargs if var_needsgc(var) and not is_borrowed(var) ] allvars = [var for var in block.getvariables() if var_needsgc(var)] self.var_last_needed_in = dict.fromkeys(allvars, 0) for i, op in enumerate(block.operations): for var in op.args: if not var_needsgc(var): continue self.var_last_needed_in[var] = i for link in block.exits: for var in link.args: if not var_needsgc(var): continue self.var_last_needed_in[var] = len(block.operations) + 1 for i, op in enumerate(block.operations): hop = GcHighLevelOp(self, op, i, llops) hop.dispatch() if len(block.exits) != 0: # i.e not the return block assert block.exitswitch is not c_last_exception deadinallexits = set(self.livevars) for link in block.exits: deadinallexits.difference_update(set(link.args)) for var in deadinallexits: self.pop_alive(var, llops) for link in block.exits: livecounts = dict.fromkeys( set(self.livevars) - deadinallexits, 1) for v, v2 in zip(link.args, link.target.inputargs): if is_borrowed(v2): continue if v in livecounts: livecounts[v] -= 1 elif var_needsgc(v): # 'v' is typically a Constant here, but it can be # a borrowed variable going into a non-borrowed one livecounts[v] = -1 self.links_to_split[link] = livecounts block.operations[:] = llops self.livevars = None self.var_last_needed_in = None
def write_barrier_check(spaceop, needs_write_barrier=True): t = TranslationContext() t.buildannotator().build_types(lambda x: x, [SomeInteger()]) t.buildrtyper().specialize() transformer = WriteBarrierTransformer(t) llops = LowLevelOpList() hop = GcHighLevelOp(transformer, spaceop, 0, llops) hop.dispatch() found = False print spaceop, '======>' for op in llops: print '\t', op if op.opname == 'direct_call': found = True assert found == needs_write_barrier
def saving_function_for_type(self, FRAME_TYPE): v_exception = varoftype(self.transformer.unwind_exception_type) v_restart = varoftype(lltype.Signed) save_block = model.Block([v_exception, v_restart]) llops = LowLevelOpList() flags = {'flavor': 'gc'} if self.stackless_gc: flags['nocollect'] = True v_state = llops.genop('malloc', [model.Constant(FRAME_TYPE, lltype.Void), model.Constant(flags, lltype.Void)], resulttype=lltype.Ptr(FRAME_TYPE)) for fieldname in FRAME_TYPE._names[1:]: # skip the 'header' field v_arg = varoftype(FRAME_TYPE._flds[fieldname]) save_block.inputargs.append(v_arg) llops.genop('setfield', [v_state, model.Constant(fieldname, lltype.Void), v_arg], resulttype=lltype.Void) v_header = gen_cast(llops, lltype.Ptr(STATE_HEADER), v_state) llops.genop('direct_call', [self.transformer.add_frame_state_ptr, v_exception, v_header], resulttype=lltype.Void) llops.genop("setfield", [v_header, self.transformer.c_f_restart_name, v_restart], resulttype=lltype.Void) save_state_graph = model.FunctionGraph('save_' + FRAME_TYPE._name, save_block, varoftype(lltype.Void)) save_block.operations = llops save_block.closeblock(model.Link([v_header], save_state_graph.returnblock)) FUNC_TYPE = lltype.FuncType([v.concretetype for v in save_block.inputargs], lltype.Void) return lltype.functionptr(FUNC_TYPE, save_state_graph.name, graph=save_state_graph)
def _make_resume_retrieval_block(self, FRAME_TYPE, erased_types): retrieve_block = model.Block([varoftype(lltype.Signed)]) retrieve_block.exitswitch = retrieve_block.inputargs[0] llops = LowLevelOpList() llops.genop("setfield", [self.ll_global_state, self.c_restart_substate_name, self.c_minus_one]) v_state_hdr = llops.genop("getfield", [self.ll_global_state, self.c_inst_top_name], resulttype=lltype.Ptr(STATE_HEADER)) v_state = gen_cast(llops, lltype.Ptr(FRAME_TYPE), v_state_hdr) llops.genop("setfield", [self.ll_global_state, self.c_inst_top_name, self.c_null_state]) output_args = [retrieve_block.inputargs[0]] assert len(FRAME_TYPE._names[1:]) == len(erased_types) for fieldname, TYPE in zip(FRAME_TYPE._names[1:], erased_types): assert FRAME_TYPE._flds[fieldname] == TYPE output_args.append(llops.genop("getfield", [v_state, model.Constant(fieldname, lltype.Void)], resulttype=TYPE)) retrieve_block.operations = llops return retrieve_block, output_args
def new_wrapper(func, translator, newname=None): # The basic idea is to produce a flow graph from scratch, using the # help of the rtyper for the conversion of the arguments after they # have been decoded. bk = translator.annotator.bookkeeper graph = bk.getdesc(func).getuniquegraph() f = getfunctionptr(graph) FUNCTYPE = typeOf(f).TO newops = LowLevelOpList(translator.rtyper) varguments = [] for var in graph.startblock.inputargs: v = Variable(var) v.concretetype = PyObjPtr varguments.append(v) wrapper_inputargs = varguments[:] # use the rtyper to produce the conversions inputargs = f._obj.graph.getargs() for i in range(len(varguments)): if FUNCTYPE.ARGS[i] != PyObjPtr: rtyper = translator.rtyper r_arg = rtyper.bindingrepr(inputargs[i]) # give the rtyper a chance to know which function we are wrapping rtyper.set_wrapper_context(func) varguments[i] = newops.convertvar(varguments[i], r_from = pyobj_repr, r_to = r_arg) rtyper.set_wrapper_context(None) vlist = [inputconst(typeOf(f), f)] + varguments vresult = newops.genop('direct_call', vlist, resulttype=FUNCTYPE.RESULT) if FUNCTYPE.RESULT != PyObjPtr: # convert "result" back to a PyObject rtyper = translator.rtyper assert rtyper is not None, ( "needs the rtyper to perform function result conversions") r_result = rtyper.bindingrepr(f._obj.graph.getreturnvar()) vresult = newops.convertvar(vresult, r_from = r_result, r_to = pyobj_repr) # "return result" block = Block(wrapper_inputargs) wgraph = FunctionGraph('pyfn_' + (newname or func.func_name), block) translator.update_call_graph(wgraph, graph, object()) translator.graphs.append(wgraph) block.operations[:] = newops block.closeblock(Link([vresult], wgraph.returnblock)) wgraph.getreturnvar().concretetype = PyObjPtr checkgraph(wgraph) # the above convertvar()s may have created and annotated new helpers # that need to be specialized now translator.rtyper.specialize_more_blocks() return functionptr(FuncType([PyObjPtr] * len(wrapper_inputargs), PyObjPtr), wgraph.name, graph = wgraph, exception_policy = "CPython")
def build_pytypeobject(r_inst): rtyper = r_inst.rtyper cache = rtyper.classdef_to_pytypeobject try: return cache[r_inst.classdef] except KeyError: for parentdef in r_inst.classdef.getmro(): cpytype = parentdef._cpy_exported_type_ if cpytype is not None: break else: # for classes that cannot be exported at all return lltype.nullptr(lltype.PyObject) from pypy.rpython.lltypesystem.rclass import CPYOBJECTPTR from pypy.rpython.rtyper import LowLevelOpList typetype = lltype.pyobjectptr(type) # XXX default tp_new should go away # make the graph of tp_new manually v1 = Variable('tp'); v1.concretetype = lltype.Ptr(PY_TYPE_OBJECT) v2 = Variable('args'); v2.concretetype = PyObjPtr v3 = Variable('kwds'); v3.concretetype = PyObjPtr block = Block([v1, v2, v3]) llops = LowLevelOpList(None) v4 = r_inst.new_instance(llops, v_cpytype = v1) v5 = llops.genop('cast_pointer', [v4], resulttype = PyObjPtr) block.operations = list(llops) tp_new_graph = FunctionGraph('ll_tp_new', block) block.closeblock(Link([v5], tp_new_graph.returnblock)) tp_new_graph.getreturnvar().concretetype = v5.concretetype # build the PyTypeObject structure pytypeobj = lltype.malloc(PY_TYPE_OBJECT, flavor='cpy', extra_args=(typetype,)) name = cpytype.name T = lltype.FixedSizeArray(lltype.Char, len(name)+1) p = lltype.malloc(T, immortal=True) for i in range(len(name)): p[i] = name[i] p[len(name)] = '\x00' pytypeobj.c_tp_name = lltype.direct_arrayitems(p) pytypeobj.c_tp_basicsize = llmemory.sizeof(r_inst.lowleveltype.TO) if cpytype.subclassable and False: # XXX deallocation of subclass object segfaults! pytypeobj.c_tp_flags = CDefinedIntSymbolic('''(Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES | Py_TPFLAGS_BASETYPE)''') else: pytypeobj.c_tp_flags = CDefinedIntSymbolic('''(Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES)''') pytypeobj.c_tp_new = rtyper.type_system.getcallable(tp_new_graph) pytypeobj.c_tp_dealloc = rtyper.annotate_helper_fn(ll_tp_dealloc, [PyObjPtr]) pytypeobj.c_tp_as_number = lltype.malloc(PyNumberMethods, immortal=True) pytypeobj.c_tp_as_sequence = lltype.malloc(PySequenceMethods, immortal=True) pytypeobj.c_tp_as_mapping = lltype.malloc(PyMappingMethods, immortal=True) result = lltype.cast_pointer(PyObjPtr, pytypeobj) # the llsetup function that will store the 'objects' into the # type's tp_dict Py_TPFLAGS_HEAPTYPE = CDefinedIntSymbolic('Py_TPFLAGS_HEAPTYPE') if cpytype.objects: objects = [(lltype.pyobjectptr(name), value) for name, value in cpytype.objects.items() if name != '__new__'] if '__new__' in cpytype.objects: new = cpytype.objects['__new__']._obj.value objects.append((lltype.pyobjectptr('__new__'), lltype.pyobjectptr(staticmethod(new)))) def ll_type_setup(p): tp = lltype.cast_pointer(lltype.Ptr(PY_TYPE_OBJECT), p) old_flags = tp.c_tp_flags tp.c_tp_flags |= Py_TPFLAGS_HEAPTYPE for name, value in objects: llop.setattr(PyObjPtr, tp, name, value) tp.c_tp_flags = old_flags result._obj.setup_fnptr = rtyper.annotate_helper_fn(ll_type_setup, [PyObjPtr]) cache[r_inst.classdef] = result return result