def test_secondary_backendopt(self): # checks an issue with a newly added graph that calls an # already-exception-transformed graph. This can occur e.g. # from a late-seen destructor added by the GC transformer # which ends up calling existing code. def common(n): if n > 5: raise ValueError def main(n): common(n) def later(n): try: common(n) return 0 except ValueError: return 1 t = TranslationContext() t.buildannotator().build_types(main, [int]) t.buildrtyper().specialize() exctransformer = t.getexceptiontransformer() exctransformer.create_exception_handling(graphof(t, common)) from rpython.annotator import model as annmodel from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator annhelper = MixLevelHelperAnnotator(t.rtyper) later_graph = annhelper.getgraph(later, [annmodel.SomeInteger()], annmodel.SomeInteger()) annhelper.finish() annhelper.backend_optimize() # ^^^ as the inliner can't handle exception-transformed graphs, # this should *not* inline common() into later(). if option.view: later_graph.show() common_graph = graphof(t, common) found = False for block in later_graph.iterblocks(): for op in block.operations: if (op.opname == 'direct_call' and op.args[0].value._obj.graph is common_graph): found = True assert found, "cannot find the call (buggily inlined?)" from rpython.rtyper.llinterp import LLInterpreter llinterp = LLInterpreter(t.rtyper) res = llinterp.eval_graph(later_graph, [10]) assert res == 1
def test_secondary_backendopt(self): # checks an issue with a newly added graph that calls an # already-exception-transformed graph. This can occur e.g. # from a late-seen destructor added by the GC transformer # which ends up calling existing code. def common(n): if n > 5: raise ValueError def main(n): common(n) def later(n): try: common(n) return 0 except ValueError: return 1 t = TranslationContext() t.buildannotator().build_types(main, [int]) t.buildrtyper().specialize() exctransformer = t.getexceptiontransformer() exctransformer.create_exception_handling(graphof(t, common)) from rpython.annotator import model as annmodel from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator annhelper = MixLevelHelperAnnotator(t.rtyper) later_graph = annhelper.getgraph(later, [annmodel.SomeInteger()], annmodel.SomeInteger()) annhelper.finish() annhelper.backend_optimize() # ^^^ as the inliner can't handle exception-transformed graphs, # this should *not* inline common() into later(). if option.view: later_graph.show() common_graph = graphof(t, common) found = False for block in later_graph.iterblocks(): for op in block.operations: if op.opname == "direct_call" and op.args[0].value._obj.graph is common_graph: found = True assert found, "cannot find the call (buggily inlined?)" from rpython.rtyper.llinterp import LLInterpreter llinterp = LLInterpreter(t.rtyper) res = llinterp.eval_graph(later_graph, [10]) assert res == 1
class BaseGCTransformer(object): finished_helpers = False curr_block = None def __init__(self, translator, inline=False): self.translator = translator self.seen_graphs = set() self.prepared = False self.minimal_transform = set() if translator: self.mixlevelannotator = MixLevelHelperAnnotator(translator.rtyper) else: self.mixlevelannotator = None self.inline = inline if translator and inline: self.lltype_to_classdef = translator.rtyper.lltype_to_classdef_mapping() self.graphs_to_inline = {} self.graph_dependencies = {} self.ll_finalizers_ptrs = [] if self.MinimalGCTransformer: self.minimalgctransformer = self.MinimalGCTransformer(self) else: self.minimalgctransformer = None def get_lltype_of_exception_value(self): exceptiondata = self.translator.rtyper.exceptiondata return exceptiondata.lltype_of_exception_value def need_minimal_transform(self, graph): self.seen_graphs.add(graph) self.minimal_transform.add(graph) def inline_helpers(self, graphs): from rpython.translator.backendopt.inline import iter_callsites raise_analyzer = RaiseAnalyzer(self.translator) for graph in graphs: to_enum = [] for called, block, i in iter_callsites(graph, None): if called in self.graphs_to_inline: to_enum.append(called) must_constfold = False for inline_graph in to_enum: try: inline.inline_function(self.translator, inline_graph, graph, self.lltype_to_classdef, raise_analyzer, cleanup=False) must_constfold = True except inline.CannotInline as e: print 'CANNOT INLINE:', e print '\t%s into %s' % (inline_graph, graph) cleanup_graph(graph) if must_constfold: constant_fold_graph(graph) def compute_borrowed_vars(self, graph): # the input args are borrowed, and stay borrowed for as long as they # are not merged with other values. var_families = DataFlowFamilyBuilder(graph).get_variable_families() borrowed_reps = {} for v in graph.getargs(): borrowed_reps[var_families.find_rep(v)] = True # no support for returning borrowed values so far retvar = graph.getreturnvar() def is_borrowed(v1): return (var_families.find_rep(v1) in borrowed_reps and v1 is not retvar) return is_borrowed def transform_block(self, block, is_borrowed): llops = LowLevelOpList() self.curr_block = block self.livevars = [var for var in block.inputargs if var_needsgc(var) and not is_borrowed(var)] allvars = [var for var in block.getvariables() if var_needsgc(var)] self.var_last_needed_in = dict.fromkeys(allvars, 0) for i, op in enumerate(block.operations): for var in op.args: if not var_needsgc(var): continue self.var_last_needed_in[var] = i for link in block.exits: for var in link.args: if not var_needsgc(var): continue self.var_last_needed_in[var] = len(block.operations) + 1 for i, op in enumerate(block.operations): hop = GcHighLevelOp(self, op, i, llops) hop.dispatch() if len(block.exits) != 0: # i.e not the return block assert not block.canraise deadinallexits = set(self.livevars) for link in block.exits: deadinallexits.difference_update(set(link.args)) for var in deadinallexits: self.pop_alive(var, llops) for link in block.exits: livecounts = dict.fromkeys(set(self.livevars) - deadinallexits, 1) for v, v2 in zip(link.args, link.target.inputargs): if is_borrowed(v2): continue if v in livecounts: livecounts[v] -= 1 elif var_needsgc(v): # 'v' is typically a Constant here, but it can be # a borrowed variable going into a non-borrowed one livecounts[v] = -1 self.links_to_split[link] = livecounts block.operations[:] = llops self.livevars = None self.var_last_needed_in = None self.curr_block = None def transform_graph(self, graph): if graph in self.minimal_transform: if self.minimalgctransformer: self.minimalgctransformer.transform_graph(graph) self.minimal_transform.remove(graph) return if graph in self.seen_graphs: return self.seen_graphs.add(graph) self.links_to_split = {} # link -> vars to pop_alive across the link # for sanity, we need an empty block at the start of the graph inserted_empty_startblock = False if not starts_with_empty_block(graph): insert_empty_startblock(graph) inserted_empty_startblock = True is_borrowed = self.compute_borrowed_vars(graph) for block in graph.iterblocks(): self.transform_block(block, is_borrowed) for link, livecounts in self.links_to_split.iteritems(): llops = LowLevelOpList() for var, livecount in livecounts.iteritems(): for i in range(livecount): self.pop_alive(var, llops) for i in range(-livecount): self.push_alive(var, llops) if llops: if link.prevblock.exitswitch is None: link.prevblock.operations.extend(llops) else: insert_empty_block(link, llops) # remove the empty block at the start of the graph, which should # still be empty (but let's check) if starts_with_empty_block(graph) and inserted_empty_startblock: old_startblock = graph.startblock graph.startblock = graph.startblock.exits[0].target checkgraph(graph) self.links_to_split = None v = Variable('vanishing_exc_value') v.concretetype = self.get_lltype_of_exception_value() llops = LowLevelOpList() self.pop_alive(v, llops) graph.exc_cleanup = (v, list(llops)) return is_borrowed # xxx for tests only def annotate_helper(self, ll_helper, ll_args, ll_result, inline=False): assert not self.finished_helpers args_s = map(lltype_to_annotation, ll_args) s_result = lltype_to_annotation(ll_result) graph = self.mixlevelannotator.getgraph(ll_helper, args_s, s_result) # the produced graphs does not need to be fully transformed self.need_minimal_transform(graph) if inline: self.graphs_to_inline[graph] = True FUNCTYPE = lltype.FuncType(ll_args, ll_result) return self.mixlevelannotator.graph2delayed(graph, FUNCTYPE=FUNCTYPE) def inittime_helper(self, ll_helper, ll_args, ll_result, inline=True): ptr = self.annotate_helper(ll_helper, ll_args, ll_result, inline=inline) return Constant(ptr, lltype.typeOf(ptr)) def annotate_finalizer(self, ll_finalizer, ll_args, ll_result): fptr = self.annotate_helper(ll_finalizer, ll_args, ll_result) self.ll_finalizers_ptrs.append(fptr) return fptr def finish_helpers(self, backendopt=True): if self.translator is not None: self.mixlevelannotator.finish_annotate() if self.translator is not None: self.mixlevelannotator.finish_rtype() if backendopt: self.mixlevelannotator.backend_optimize() self.finished_helpers = True # Make sure that the database also sees all finalizers now. # It is likely that the finalizers need special support there newgcdependencies = self.ll_finalizers_ptrs return newgcdependencies def get_finish_helpers(self): return self.finish_helpers def finish_tables(self): pass def get_finish_tables(self): return self.finish_tables def finish(self, backendopt=True): self.finish_helpers(backendopt=backendopt) self.finish_tables() def transform_generic_set(self, hop): opname = hop.spaceop.opname v_new = hop.spaceop.args[-1] v_old = hop.genop('g' + opname[1:], hop.inputargs()[:-1], resulttype=v_new.concretetype) self.push_alive(v_new, hop.llops) hop.rename('bare_' + opname) self.pop_alive(v_old, hop.llops) def push_alive(self, var, llops): pass def pop_alive(self, var, llops): pass def var_needs_set_transform(self, var): return False def default(self, hop): hop.llops.append(hop.spaceop) def gct_setfield(self, hop): if self.var_needs_set_transform(hop.spaceop.args[-1]): self.transform_generic_set(hop) else: hop.rename('bare_' + hop.spaceop.opname) gct_setarrayitem = gct_setfield gct_setinteriorfield = gct_setfield gct_raw_store = gct_setfield gct_getfield = default def gct_zero_gc_pointers_inside(self, hop): pass def gct_gc_writebarrier_before_copy(self, hop): # We take the conservative default and return False here, meaning # that rgc.ll_arraycopy() will do the copy by hand (i.e. with a # 'for' loop). Subclasses that have their own logic, or that don't # need any kind of write barrier, may return True. op = hop.spaceop hop.genop("same_as", [rmodel.inputconst(lltype.Bool, False)], resultvar=op.result) def gct_gc_pin(self, hop): op = hop.spaceop hop.genop("same_as", [rmodel.inputconst(lltype.Bool, False)], resultvar=op.result) def gct_gc_unpin(self, hop): pass def gct_gc__is_pinned(self, hop): op = hop.spaceop hop.genop("same_as", [rmodel.inputconst(lltype.Bool, False)], resultvar=op.result) def gct_gc_identityhash(self, hop): # must be implemented in the various GCs raise NotImplementedError def gct_gc_id(self, hop): # this assumes a non-moving GC. Moving GCs need to override this hop.rename('cast_ptr_to_int') def gct_gc_heap_stats(self, hop): from rpython.memory.gc.base import ARRAY_TYPEID_MAP return hop.cast_result(rmodel.inputconst(lltype.Ptr(ARRAY_TYPEID_MAP), lltype.nullptr(ARRAY_TYPEID_MAP))) def get_prebuilt_hash(self, obj): return None
class BaseGCTransformer(object): finished_helpers = False curr_block = None def __init__(self, translator, inline=False): self.translator = translator self.seen_graphs = set() self.prepared = False self.minimal_transform = set() if translator: self.mixlevelannotator = MixLevelHelperAnnotator(translator.rtyper) else: self.mixlevelannotator = None self.inline = inline if translator and inline: self.lltype_to_classdef = translator.rtyper.lltype_to_classdef_mapping() self.raise_analyzer = RaiseAnalyzer(translator) self.graphs_to_inline = {} self.graph_dependencies = {} self.ll_finalizers_ptrs = [] if self.MinimalGCTransformer: self.minimalgctransformer = self.MinimalGCTransformer(self) else: self.minimalgctransformer = None def get_lltype_of_exception_value(self): exceptiondata = self.translator.rtyper.exceptiondata return exceptiondata.lltype_of_exception_value def need_minimal_transform(self, graph): self.seen_graphs.add(graph) self.minimal_transform.add(graph) def inline_helpers_into(self, graph): from rpython.translator.backendopt.inline import iter_callsites to_enum = [] for called, block, i in iter_callsites(graph, None): if called in self.graphs_to_inline: to_enum.append(called) any_inlining = False for inline_graph in to_enum: try: inline.inline_function(self.translator, inline_graph, graph, self.lltype_to_classdef, self.raise_analyzer, cleanup=False) any_inlining = True except inline.CannotInline as e: print 'CANNOT INLINE:', e print '\t%s into %s' % (inline_graph, graph) raise # for now, make it a fatal error cleanup_graph(graph) if any_inlining: constant_fold_graph(graph) return any_inlining def inline_helpers_and_postprocess(self, graphs): for graph in graphs: any_inlining = self.inline and self.inline_helpers_into(graph) self.postprocess_graph(graph, any_inlining) def postprocess_graph(self, graph, any_inlining): pass def compute_borrowed_vars(self, graph): # the input args are borrowed, and stay borrowed for as long as they # are not merged with other values. var_families = DataFlowFamilyBuilder(graph).get_variable_families() borrowed_reps = {} for v in graph.getargs(): borrowed_reps[var_families.find_rep(v)] = True # no support for returning borrowed values so far retvar = graph.getreturnvar() def is_borrowed(v1): return (var_families.find_rep(v1) in borrowed_reps and v1 is not retvar) return is_borrowed def transform_block(self, block, is_borrowed): llops = LowLevelOpList() self.curr_block = block self.livevars = [var for var in block.inputargs if var_needsgc(var) and not is_borrowed(var)] allvars = [var for var in block.getvariables() if var_needsgc(var)] self.var_last_needed_in = dict.fromkeys(allvars, 0) for i, op in enumerate(block.operations): for var in op.args: if not var_needsgc(var): continue self.var_last_needed_in[var] = i for link in block.exits: for var in link.args: if not var_needsgc(var): continue self.var_last_needed_in[var] = len(block.operations) + 1 for i, op in enumerate(block.operations): hop = GcHighLevelOp(self, op, i, llops) hop.dispatch() if len(block.exits) != 0: # i.e not the return block assert not block.canraise deadinallexits = set(self.livevars) for link in block.exits: deadinallexits.difference_update(set(link.args)) for var in deadinallexits: self.pop_alive(var, llops) for link in block.exits: livecounts = dict.fromkeys(set(self.livevars) - deadinallexits, 1) for v, v2 in zip(link.args, link.target.inputargs): if is_borrowed(v2): continue if v in livecounts: livecounts[v] -= 1 elif var_needsgc(v): # 'v' is typically a Constant here, but it can be # a borrowed variable going into a non-borrowed one livecounts[v] = -1 self.links_to_split[link] = livecounts block.operations[:] = llops self.livevars = None self.var_last_needed_in = None self.curr_block = None def start_transforming_graph(self, graph): pass # for asmgcc.py def transform_graph(self, graph): if graph in self.minimal_transform: if self.minimalgctransformer: self.minimalgctransformer.transform_graph(graph) self.minimal_transform.remove(graph) return if graph in self.seen_graphs: return self.seen_graphs.add(graph) self.start_transforming_graph(graph) self.links_to_split = {} # link -> vars to pop_alive across the link # for sanity, we need an empty block at the start of the graph inserted_empty_startblock = False if not starts_with_empty_block(graph): insert_empty_startblock(graph) inserted_empty_startblock = True is_borrowed = self.compute_borrowed_vars(graph) for block in graph.iterblocks(): self.transform_block(block, is_borrowed) for link, livecounts in self.links_to_split.iteritems(): llops = LowLevelOpList() for var, livecount in livecounts.iteritems(): for i in range(livecount): self.pop_alive(var, llops) for i in range(-livecount): self.push_alive(var, llops) if llops: if link.prevblock.exitswitch is None: link.prevblock.operations.extend(llops) else: insert_empty_block(link, llops) # remove the empty block at the start of the graph, which should # still be empty (but let's check) if starts_with_empty_block(graph) and inserted_empty_startblock: old_startblock = graph.startblock graph.startblock = graph.startblock.exits[0].target checkgraph(graph) self.links_to_split = None v = Variable('vanishing_exc_value') v.concretetype = self.get_lltype_of_exception_value() llops = LowLevelOpList() self.pop_alive(v, llops) graph.exc_cleanup = (v, list(llops)) return is_borrowed # xxx for tests only def annotate_helper(self, ll_helper, ll_args, ll_result, inline=False): assert not self.finished_helpers args_s = map(lltype_to_annotation, ll_args) s_result = lltype_to_annotation(ll_result) graph = self.mixlevelannotator.getgraph(ll_helper, args_s, s_result) # the produced graphs does not need to be fully transformed self.need_minimal_transform(graph) if inline: self.graphs_to_inline[graph] = True FUNCTYPE = lltype.FuncType(ll_args, ll_result) return self.mixlevelannotator.graph2delayed(graph, FUNCTYPE=FUNCTYPE) def inittime_helper(self, ll_helper, ll_args, ll_result, inline=True): ptr = self.annotate_helper(ll_helper, ll_args, ll_result, inline=inline) return Constant(ptr, lltype.typeOf(ptr)) def annotate_finalizer(self, ll_finalizer, ll_args, ll_result): fptr = self.annotate_helper(ll_finalizer, ll_args, ll_result) self.ll_finalizers_ptrs.append(fptr) return fptr def finish_helpers(self, backendopt=True): if self.translator is not None: self.mixlevelannotator.finish_annotate() if self.translator is not None: self.mixlevelannotator.finish_rtype() if backendopt: self.mixlevelannotator.backend_optimize() self.finished_helpers = True # Make sure that the database also sees all finalizers now. # It is likely that the finalizers need special support there newgcdependencies = self.ll_finalizers_ptrs return newgcdependencies def get_finish_helpers(self): return self.finish_helpers def finish_tables(self): pass def get_finish_tables(self): return self.finish_tables def finish(self, backendopt=True): self.finish_helpers(backendopt=backendopt) self.finish_tables() def transform_generic_set(self, hop): opname = hop.spaceop.opname v_new = hop.spaceop.args[-1] v_old = hop.genop('g' + opname[1:], hop.inputargs()[:-1], resulttype=v_new.concretetype) self.push_alive(v_new, hop.llops) hop.rename('bare_' + opname) self.pop_alive(v_old, hop.llops) def push_alive(self, var, llops): pass def pop_alive(self, var, llops): pass def var_needs_set_transform(self, var): return False def default(self, hop): hop.llops.append(hop.spaceop) def gct_setfield(self, hop): if self.var_needs_set_transform(hop.spaceop.args[-1]): self.transform_generic_set(hop) else: hop.rename('bare_' + hop.spaceop.opname) gct_setarrayitem = gct_setfield gct_setinteriorfield = gct_setfield gct_raw_store = gct_setfield gct_getfield = default def gct_zero_gc_pointers_inside(self, hop): pass def gct_gc_writebarrier_before_copy(self, hop): # We take the conservative default and return False here, meaning # that rgc.ll_arraycopy() will do the copy by hand (i.e. with a # 'for' loop). Subclasses that have their own logic, or that don't # need any kind of write barrier, may return True. op = hop.spaceop hop.genop("same_as", [rmodel.inputconst(lltype.Bool, False)], resultvar=op.result) def gct_gc_pin(self, hop): op = hop.spaceop hop.genop("same_as", [rmodel.inputconst(lltype.Bool, False)], resultvar=op.result) def gct_gc_unpin(self, hop): pass def gct_gc__is_pinned(self, hop): op = hop.spaceop hop.genop("same_as", [rmodel.inputconst(lltype.Bool, False)], resultvar=op.result) def gct_gc_identityhash(self, hop): # must be implemented in the various GCs raise NotImplementedError def gct_gc_id(self, hop): # this assumes a non-moving GC. Moving GCs need to override this hop.rename('cast_ptr_to_int') def gct_gc_heap_stats(self, hop): from rpython.memory.gc.base import ARRAY_TYPEID_MAP return hop.cast_result(rmodel.inputconst(lltype.Ptr(ARRAY_TYPEID_MAP), lltype.nullptr(ARRAY_TYPEID_MAP)))