def checkgraphs(self, blocks): seen = {} for block in blocks: graph = self.annotated[block] if graph not in seen: checkgraph(graph) seen[graph] = True
def split_graph_and_record_jitdriver(self, graph, block, pos): op = block.operations[pos] jd = JitDriverStaticData() jd._jit_merge_point_in = graph args = op.args[2:] s_binding = self.translator.annotator.binding jd._portal_args_s = [s_binding(v) for v in args] graph = copygraph(graph) [jmpp] = find_jit_merge_points([graph]) graph.startblock = support.split_before_jit_merge_point(*jmpp) # a crash in the following checkgraph() means that you forgot # to list some variable in greens=[] or reds=[] in JitDriver, # or that a jit_merge_point() takes a constant as an argument. checkgraph(graph) for v in graph.getargs(): assert isinstance(v, Variable) assert len(dict.fromkeys(graph.getargs())) == len(graph.getargs()) self.translator.graphs.append(graph) jd.portal_graph = graph # it's a bit unbelievable to have a portal without func assert hasattr(graph, "func") graph.func._dont_inline_ = True graph.func._jit_unroll_safe_ = True jd.jitdriver = block.operations[pos].args[1].value jd.portal_runner_ptr = "<not set so far>" jd.result_type = history.getkind(jd.portal_graph.getreturnvar() .concretetype)[0] self.jitdrivers_sd.append(jd)
def test_wrong_startblock_incref(): class B(object): pass def g(b): while True: b.x -= 10 if b.x < 0: return b.x def f(n): b = B() b.x = n return g(b) # XXX obscure: remove the first empty block in the graph of 'g' t = TranslationContext() graph = t.buildflowgraph(g) assert graph.startblock.operations == [] graph.startblock = graph.startblock.exits[0].target from pypy.objspace.flow.model import checkgraph checkgraph(graph) t._prebuilt_graphs[g] = graph fn = compile_func(f, [int], t) res = fn(112) assert res == -8
def split_graph_and_record_jitdriver(self, graph, block, pos): op = block.operations[pos] jd = JitDriverStaticData() jd._jit_merge_point_pos = (graph, op) args = op.args[2:] s_binding = self.translator.annotator.binding jd._portal_args_s = [s_binding(v) for v in args] graph = copygraph(graph) graph.startblock.isstartblock = False [jmpp] = find_jit_merge_points([graph]) graph.startblock = support.split_before_jit_merge_point(*jmpp) graph.startblock.isstartblock = True # a crash in the following checkgraph() means that you forgot # to list some variable in greens=[] or reds=[] in JitDriver. checkgraph(graph) for v in graph.getargs(): assert isinstance(v, Variable) assert len(dict.fromkeys(graph.getargs())) == len(graph.getargs()) self.translator.graphs.append(graph) jd.portal_graph = graph # it's a bit unbelievable to have a portal without func assert hasattr(graph, "func") graph.func._dont_inline_ = True graph.func._jit_unroll_safe_ = True jd.jitdriver = block.operations[pos].args[1].value jd.portal_runner_ptr = "<not set so far>" jd.result_type = history.getkind(jd.portal_graph.getreturnvar() .concretetype)[0] self.jitdrivers_sd.append(jd)
def builder(translator, func): # build a hacked graph that doesn't take a *arg any more, but # individual extra arguments graph = translator.buildflowgraph(func) argnames, vararg, kwarg = graph.signature assert vararg, "graph should have a *arg at this point" assert not kwarg, "where does this **arg come from??" argscopy = [Variable(v) for v in graph.getargs()] starargs = [ Variable('stararg%d' % i) for i in range(nb_extra_args) ] newstartblock = Block(argscopy[:-1] + starargs) newtup = SpaceOperation('newtuple', starargs, argscopy[-1]) newstartblock.operations.append(newtup) newstartblock.closeblock(Link(argscopy, graph.startblock)) graph.startblock.isstartblock = False graph.startblock = newstartblock newstartblock.isstartblock = True argnames = argnames + ['.star%d' % i for i in range(nb_extra_args)] graph.signature = Signature(argnames) # note that we can mostly ignore defaults: if nb_extra_args > 0, # then defaults aren't applied. if nb_extra_args == 0, then this # just removes the *arg and the defaults keep their meaning. if nb_extra_args > 0: graph.defaults = None # shouldn't be used in this case checkgraph(graph) return graph
def test_split_block_exceptions(): for i in range(2): def raises(x): if x == 1: raise ValueError elif x == 2: raise KeyError return x def catches(x): try: y = x + 1 raises(y) except ValueError: return 0 except KeyError: return 1 return x graph, t = translate(catches, [int]) split_block(t.annotator, graph.startblock, i) checkgraph(graph) interp = LLInterpreter(t.rtyper) result = interp.eval_graph(graph, [0]) assert result == 0 result = interp.eval_graph(graph, [1]) assert result == 1 result = interp.eval_graph(graph, [2]) assert result == 2
def builder(translator, func): # build a hacked graph that doesn't take a *arg any more, but # individual extra arguments graph = translator.buildflowgraph(func) argnames, vararg, kwarg = graph.signature assert vararg, "graph should have a *arg at this point" assert not kwarg, "where does this **arg come from??" argscopy = [Variable(v) for v in graph.getargs()] starargs = [Variable('stararg%d'%i) for i in range(nb_extra_args)] newstartblock = Block(argscopy[:-1] + starargs) newtup = SpaceOperation('newtuple', starargs, argscopy[-1]) newstartblock.operations.append(newtup) newstartblock.closeblock(Link(argscopy, graph.startblock)) graph.startblock.isstartblock = False graph.startblock = newstartblock newstartblock.isstartblock = True argnames += tuple(['.star%d' % i for i in range(nb_extra_args)]) graph.signature = argnames, None, None # note that we can mostly ignore defaults: if nb_extra_args > 0, # then defaults aren't applied. if nb_extra_args == 0, then this # just removes the *arg and the defaults keep their meaning. if nb_extra_args > 0: graph.defaults = None # shouldn't be used in this case checkgraph(graph) return graph
def check_graph(graph, args, expected_result, t): if conftest.option.view: t.view() checkgraph(graph) interp = LLInterpreter(t.rtyper) res = interp.eval_graph(graph, args) assert res == expected_result
def remove_asserts(translator, graphs): rtyper = translator.rtyper clsdef = translator.annotator.bookkeeper.getuniqueclassdef(AssertionError) r_AssertionError = rclass.getclassrepr(rtyper, clsdef) ll_AssertionError = r_AssertionError.convert_const(AssertionError) total_count = [0, 0] for graph in graphs: count = 0 morework = True while morework: morework = False eliminate_empty_blocks(graph) join_blocks(graph) for link in graph.iterlinks(): if (link.target is graph.exceptblock and isinstance(link.args[0], Constant) and link.args[0].value == ll_AssertionError): if kill_assertion_link(graph, link): count += 1 morework = True break else: total_count[0] += 1 if translator.config.translation.verbose: log.removeassert("cannot remove an assert from %s" % (graph.name,)) if count: # now melt away the (hopefully) dead operation that compute # the condition total_count[1] += count if translator.config.translation.verbose: log.removeassert("removed %d asserts in %s" % (count, graph.name)) checkgraph(graph) #transform_dead_op_vars(graph, translator) log.removeassert("Could not remove %d asserts, but removed %d asserts." % tuple(total_count))
def find_portal(self): graphs = self.translator.graphs self.jit_merge_point_pos = find_jit_merge_point(graphs) graph, block, pos = self.jit_merge_point_pos op = block.operations[pos] args = op.args[2:] s_binding = self.translator.annotator.binding self.portal_args_s = [s_binding(v) for v in args] graph = copygraph(graph) graph.startblock.isstartblock = False graph.startblock = support.split_before_jit_merge_point( *find_jit_merge_point([graph])) graph.startblock.isstartblock = True # a crash in the following checkgraph() means that you forgot # to list some variable in greens=[] or reds=[] in JitDriver. checkgraph(graph) for v in graph.getargs(): assert isinstance(v, Variable) assert len(dict.fromkeys(graph.getargs())) == len(graph.getargs()) self.translator.graphs.append(graph) self.portal_graph = graph # it's a bit unbelievable to have a portal without func assert hasattr(graph, "func") graph.func._dont_inline_ = True graph.func._jit_unroll_safe_ = True self.jitdriver = block.operations[pos].args[1].value
def show_incremental_progress(gv_func): from pypy import conftest graph = _getgraph(gv_func) fixduplicatevars(graph) flowmodel.checkgraph(graph) if conftest.option.view: eliminate_empty_blocks(graph) graph.show()
def simplify_graph(graph, passes=True): # can take a list of passes to apply, True meaning all """inplace-apply all the existing optimisations to the graph.""" if passes is True: passes = all_passes checkgraph(graph) for pass_ in passes: pass_(graph) checkgraph(graph)
def finish(self): # compute the final masterarray by copying over the masterarray1, # which is a list of dicts of attributes if SAVE_STATISTICS: import cPickle cPickle.dump(self.stats, open('stackless-stats.pickle', 'wb')) # fun fun fun patching the call_function_retval_xyz() functions! for RESTYPE, typename in frame.STORAGE_TYPES_AND_FIELDS: rettype_index = STORAGE_TYPES.index(RESTYPE) cache = self.signaturecodes[rettype_index] if not cache: continue # not used anyway, don't produce a broken empty switch func = getattr(code, 'call_function_retval_' + typename) desc = self.translator.annotator.bookkeeper.getdesc(func) graph = desc.getuniquegraph() [v_fnaddr, v_signature_index] = graph.getargs() block = model.Block([v_fnaddr, v_signature_index]) block.exitswitch = v_signature_index block.isstartblock = True graph.startblock = block switchlinks = [] for ARGTYPES, signature_index in cache.items(): # XXX because of type erasure, the following cast is # kind of invalid, but we hope that nobody will notice FUNCTYPE = lltype.Ptr(lltype.FuncType(ARGTYPES, RESTYPE)) v_fnaddr1 = varoftype(v_fnaddr.concretetype) callblock = model.Block([v_fnaddr1]) llops = LowLevelOpList() args_v = [model.Constant(TYPE._defl(), concretetype=TYPE) for TYPE in ARGTYPES] v_res = llops.genop('adr_call', [v_fnaddr1] + args_v, resulttype = RESTYPE) callblock.operations[:] = llops callblock.closeblock(model.Link([v_res], graph.returnblock)) link = model.Link([v_fnaddr], callblock) link.exitcase = signature_index link.llexitcase = signature_index switchlinks.append(link) block.closeblock(*switchlinks) model.checkgraph(graph) self.is_finished = True masterarray = lltype.malloc(frame.FRAME_INFO_ARRAY, len(self.masterarray1), immortal=True) for dst, src in zip(masterarray, self.masterarray1): dst.fnaddr, dst.info = src # horrors in the same spirit as in rpython.memory.gctransform # (shorter, though) ll_global_state = self.ll_global_state.value ll_global_state.inst_masterarray = masterarray return [masterarray]
def merge_if_blocks_once(graph): """Convert consecutive blocks that all compare a variable (of Primitive type) with a constant into one block with multiple exits. The backends can in turn output this block as a switch statement. """ candidates = [block for block in graph.iterblocks() if is_chain_block(block, first=True)] entrymap = mkentrymap(graph) for firstblock in candidates: chain = [] checkvars = [] varmap = {} # {var in a block in the chain: var in the first block} for var in firstblock.exits[0].args: varmap[var] = var for var in firstblock.exits[1].args: varmap[var] = var def add_to_varmap(var, newvar): if isinstance(var, Variable): varmap[newvar] = varmap[var] else: varmap[newvar] = var current = firstblock while 1: # check whether the chain can be extended with the block that follows the # False link checkvar = [var for var in current.operations[-1].args if isinstance(var, Variable)][0] case = [var for var in current.operations[-1].args if isinstance(var, Constant)][0] chain.append((current, case)) checkvars.append(checkvar) falseexit = current.exits[0] assert not falseexit.exitcase trueexit = current.exits[1] targetblock = falseexit.target if len(entrymap[targetblock]) != 1: break if checkvar not in falseexit.args: break newcheckvar = targetblock.inputargs[falseexit.args.index(checkvar)] if not is_chain_block(targetblock): break if newcheckvar not in targetblock.operations[0].args: break for i, var in enumerate(trueexit.args): add_to_varmap(var, trueexit.target.inputargs[i]) for i, var in enumerate(falseexit.args): add_to_varmap(var, falseexit.target.inputargs[i]) current = targetblock if len(chain) > 1: break else: return False merge_chain(chain, checkvars[0], varmap, graph) checkgraph(graph) return True
def virtualize_mallocs(translator, graphs, verbose=False): newgraphs = graphs[:] mallocv = MallocVirtualizer(newgraphs, translator.rtyper, verbose) while mallocv.remove_mallocs_once(): pass for graph in newgraphs: checkgraph(graph) join_blocks(graph) assert newgraphs[:len(graphs)] == graphs del newgraphs[:len(graphs)] translator.graphs.extend(newgraphs)
def _buildgraph(graph): assert graph.startblock.operations[0].opname == 'debug_assert' del graph.startblock.operations[0] # rgenop makes graphs that use the same variable in several blocks, fixduplicatevars(graph) # fix this now flowmodel.checkgraph(graph) eliminate_empty_blocks(graph) # we cannot call join_blocks(graph) here! It has a subtle problem: # it copies operations between blocks without renaming op.result. # See test_promotion.test_many_promotions for a failure. graph.rgenop = True return graph
def test_split_blocks_simple(): for i in range(4): def f(x, y): z = x + y w = x * y return z + w graph, t = translate(f, [int, int]) split_block(t.annotator, graph.startblock, i) checkgraph(graph) interp = LLInterpreter(t.rtyper) result = interp.eval_graph(graph, [1, 2]) assert result == 5
def transform_graph(self, graph): if graph in self.minimal_transform: if self.minimalgctransformer: self.minimalgctransformer.transform_graph(graph) del self.minimal_transform[graph] return if graph in self.seen_graphs: return self.seen_graphs[graph] = True self.links_to_split = {} # link -> vars to pop_alive across the link # for sanity, we need an empty block at the start of the graph inserted_empty_startblock = False if not starts_with_empty_block(graph): insert_empty_startblock(self.translator.annotator, graph) inserted_empty_startblock = True is_borrowed = self.compute_borrowed_vars(graph) for block in graph.iterblocks(): self.transform_block(block, is_borrowed) for link, livecounts in self.links_to_split.iteritems(): llops = LowLevelOpList() for var, livecount in livecounts.iteritems(): for i in range(livecount): self.pop_alive(var, llops) for i in range(-livecount): self.push_alive(var, llops) if llops: if link.prevblock.exitswitch is None: link.prevblock.operations.extend(llops) else: insert_empty_block(self.translator.annotator, link, llops) # remove the empty block at the start of the graph, which should # still be empty (but let's check) if starts_with_empty_block(graph) and inserted_empty_startblock: old_startblock = graph.startblock graph.startblock.isstartblock = False graph.startblock = graph.startblock.exits[0].target graph.startblock.isstartblock = True checkgraph(graph) self.links_to_split = None v = Variable('vanishing_exc_value') v.concretetype = self.get_lltype_of_exception_value() llops = LowLevelOpList() self.pop_alive(v, llops) graph.exc_cleanup = (v, list(llops)) return is_borrowed # xxx for tests only
def normalize_calltable_row_annotation(annotator, graphs): if len(graphs) <= 1: return False # nothing to do graph_bindings = {} for graph in graphs: graph_bindings[graph] = [annotator.binding(v) for v in graph.getargs()] iterbindings = graph_bindings.itervalues() nbargs = len(iterbindings.next()) for binding in iterbindings: assert len(binding) == nbargs generalizedargs = [] for i in range(nbargs): args_s = [] for graph, bindings in graph_bindings.items(): args_s.append(bindings[i]) s_value = annmodel.unionof(*args_s) generalizedargs.append(s_value) result_s = [annotator.binding(graph.getreturnvar()) for graph in graph_bindings] generalizedresult = annmodel.unionof(*result_s) conversion = False for graph in graphs: bindings = graph_bindings[graph] need_conversion = (generalizedargs != bindings) if need_conversion: conversion = True oldblock = graph.startblock inlist = [] for j, s_value in enumerate(generalizedargs): v = Variable(graph.getargs()[j]) annotator.setbinding(v, s_value) inlist.append(v) newblock = Block(inlist) # prepare the output args of newblock and link outlist = inlist[:] newblock.closeblock(Link(outlist, oldblock)) oldblock.isstartblock = False newblock.isstartblock = True graph.startblock = newblock # finished checkgraph(graph) annotator.annotated[newblock] = annotator.annotated[oldblock] # convert the return value too if annotator.binding(graph.getreturnvar()) != generalizedresult: conversion = True annotator.setbinding(graph.getreturnvar(), generalizedresult) return conversion
def build_graph_types(self, flowgraph, inputcells, complete_now=True): checkgraph(flowgraph) nbarg = len(flowgraph.getargs()) if len(inputcells) != nbarg: raise TypeError("%s expects %d args, got %d" % (flowgraph, nbarg, len(inputcells))) # register the entry point self.addpendinggraph(flowgraph, inputcells) # recursively proceed until no more pending block is left if complete_now: self.complete() return self.binding(flowgraph.getreturnvar(), None)
def build_graph_types(self, flowgraph, inputcells, complete_now=True): checkgraph(flowgraph) nbarg = len(flowgraph.getargs()) if len(inputcells) != nbarg: raise TypeError("%s expects %d args, got %d" %( flowgraph, nbarg, len(inputcells))) # register the entry point self.addpendinggraph(flowgraph, inputcells) # recursively proceed until no more pending block is left if complete_now: self.complete() return self.binding(flowgraph.getreturnvar(), None)
def check_malloc_removed(cls, graph): remover = cls.MallocRemover() checkgraph(graph) count1 = count2 = 0 for node in graph.iterblocks(): for op in node.operations: if op.opname == cls.MallocRemover.MALLOC_OP: S = op.args[0].value if not remover.union_wrapper(S): # union wrappers are fine count1 += 1 if op.opname in ('direct_call', 'indirect_call'): count2 += 1 assert count1 == 0 # number of mallocs left assert count2 == 0 # number of calls left
def test_split_blocks_conditional(): for i in range(3): def f(x, y): if x + 12: return y + 1 else: return y + 2 graph, t = translate(f, [int, int]) split_block(t.annotator, graph.startblock, i) checkgraph(graph) interp = LLInterpreter(t.rtyper) result = interp.eval_graph(graph, [-12, 2]) assert result == 4 result = interp.eval_graph(graph, [0, 2]) assert result == 3
def check(self, f, argtypes, no_getfields=0): t = self.translate(f, argtypes) getfields = 0 graph = graphof(t, f) checkgraph(graph) storesink_graph(graph) checkgraph(graph) if option.view: t.view() for block in graph.iterblocks(): for op in block.operations: if op.opname == 'getfield': getfields += 1 if no_getfields != getfields: py.test.fail("Expected %d, got %d getfields" % (no_getfields, getfields))
def remove_tail_calls_to_self(translator, graph): entrymap = mkentrymap(graph) changed = False for link in entrymap[graph.returnblock]: block = link.prevblock if (len(block.exits) == 1 and len(block.operations) > 0 and block.operations[-1].opname == 'direct_call' and block.operations[-1].result == link.args[0]): call = get_graph(block.operations[-1].args[0], translator) print "getgraph", graph if graph is graph: _remove_tail_call(translator, graph, block) changed = True if changed: from pypy.translator import simplify checkgraph(graph) simplify.remove_identical_vars(graph) simplify.eliminate_empty_blocks(graph) simplify.join_blocks(graph)
def sanity_check(t): # look for missing '.concretetype' for graph in t.graphs: checkgraph(graph) for node in graph.iterblocks(): for v in node.inputargs: assert hasattr(v, 'concretetype') for op in node.operations: for v in op.args: assert hasattr(v, 'concretetype') assert hasattr(op.result, 'concretetype') for node in graph.iterlinks(): if node.exitcase is not None: assert hasattr(node, 'llexitcase') for v in node.args: assert hasattr(v, 'concretetype') if isinstance(node.last_exception, (Variable, Constant)): assert hasattr(node.last_exception, 'concretetype') if isinstance(node.last_exc_value, (Variable, Constant)): assert hasattr(node.last_exc_value, 'concretetype')
def test_ignore_breaking_transformations(): def f(): pass f._annspecialcase_ = "override:ignore" def g(i): if i == 1: return "ab" else: try: return f() except: return "hello!" t, typer, graph = gengraph(g, [int]) from pypy.translator import simplify from pypy.translator.backendopt import removenoops from pypy.objspace.flow.model import checkgraph removenoops.remove_same_as(graph) simplify.eliminate_empty_blocks(graph) #should not crash: checkgraph(graph)
def remove_asserts(translator, graphs): rtyper = translator.rtyper clsdef = translator.annotator.bookkeeper.getuniqueclassdef(AssertionError) r_AssertionError = rclass.getclassrepr(rtyper, clsdef) ll_AssertionError = r_AssertionError.convert_const(AssertionError) total_count = [0, 0] for graph in graphs: count = 0 morework = True while morework: morework = False eliminate_empty_blocks(graph) join_blocks(graph) for link in graph.iterlinks(): if (link.target is graph.exceptblock and isinstance(link.args[0], Constant) and link.args[0].value == ll_AssertionError): if kill_assertion_link(graph, link): count += 1 morework = True break else: total_count[0] += 1 if translator.config.translation.verbose: log.removeassert( "cannot remove an assert from %s" % (graph.name, )) if count: # now melt away the (hopefully) dead operation that compute # the condition total_count[1] += count if translator.config.translation.verbose: log.removeassert("removed %d asserts in %s" % (count, graph.name)) checkgraph(graph) #transform_dead_op_vars(graph, translator) log.removeassert("Could not remove %d asserts, but removed %d asserts." % tuple(total_count))
def backend_optimizations(translator, graphs=None, secondary=False, **kwds): # sensible keywords are # raisingop2direct_call, inline_threshold, mallocs # merge_if_blocks, constfold, heap2stack # clever_malloc_removal, remove_asserts config = translator.config.translation.backendopt.copy(as_default=True) config.set(**kwds) if graphs is None: graphs = translator.graphs for graph in graphs: assert not hasattr(graph, '_seen_by_the_backend') if config.print_statistics: print "before optimizations:" print_statistics(translator.graphs[0], translator, "per-graph.txt") if config.raisingop2direct_call: raisingop2direct_call(translator, graphs) if translator.rtyper.type_system.name == 'ootypesystem': check_virtual_methods() if config.remove_asserts: constfold(config, graphs) remove_asserts(translator, graphs) if config.really_remove_asserts: for graph in graphs: removenoops.remove_debug_assert(graph) # the dead operations will be killed by the remove_obvious_noops below # remove obvious no-ops def remove_obvious_noops(): for graph in graphs: removenoops.remove_same_as(graph) simplify.eliminate_empty_blocks(graph) simplify.transform_dead_op_vars(graph, translator) removenoops.remove_duplicate_casts(graph, translator) if config.print_statistics: print "after no-op removal:" print_statistics(translator.graphs[0], translator) remove_obvious_noops() if config.inline or config.mallocs: heuristic = get_function(config.inline_heuristic) if config.inline: threshold = config.inline_threshold else: threshold = 0 inline_malloc_removal_phase(config, translator, graphs, threshold, inline_heuristic=heuristic) constfold(config, graphs) if config.clever_malloc_removal: threshold = config.clever_malloc_removal_threshold heuristic = get_function(config.clever_malloc_removal_heuristic) log.inlineandremove("phase with threshold factor: %s" % threshold) log.inlineandremove("heuristic: %s.%s" % (heuristic.__module__, heuristic.__name__)) count = mallocprediction.clever_inlining_and_malloc_removal( translator, graphs, threshold=threshold, heuristic=heuristic) log.inlineandremove("removed %d simple mallocs in total" % count) constfold(config, graphs) if config.print_statistics: print "after clever inlining and malloc removal" print_statistics(translator.graphs[0], translator) if config.storesink: for graph in graphs: storesink_graph(graph) if config.profile_based_inline and not secondary: threshold = config.profile_based_inline_threshold heuristic = get_function(config.profile_based_inline_heuristic) inline.instrument_inline_candidates(graphs, threshold) counters = translator.driver_instrument_result( config.profile_based_inline) n = len(counters) def call_count_pred(label): if label >= n: return False return counters[label] > 250 # xxx introduce an option for this inline_malloc_removal_phase(config, translator, graphs, threshold, inline_heuristic=heuristic, call_count_pred=call_count_pred) constfold(config, graphs) if config.merge_if_blocks: log.mergeifblocks("starting to merge if blocks") for graph in graphs: merge_if_blocks(graph, translator.config.translation.verbose) if config.print_statistics: print "after if-to-switch:" print_statistics(translator.graphs[0], translator) remove_obvious_noops() for graph in graphs: checkgraph(graph)
def backend_optimizations(translator, graphs=None, secondary=False, **kwds): # sensible keywords are # raisingop2direct_call, inline_threshold, mallocs # merge_if_blocks, constfold, heap2stack # clever_malloc_removal, remove_asserts config = translator.config.translation.backendopt.copy(as_default=True) config.set(**kwds) if graphs is None: graphs = translator.graphs for graph in graphs: assert not hasattr(graph, '_seen_by_the_backend') if config.print_statistics: print "before optimizations:" print_statistics(translator.graphs[0], translator, "per-graph.txt") if config.raisingop2direct_call: raisingop2direct_call(translator, graphs) if translator.rtyper.type_system.name == 'ootypesystem': check_virtual_methods() if config.remove_asserts: constfold(config, graphs) remove_asserts(translator, graphs) if config.really_remove_asserts: for graph in graphs: removenoops.remove_debug_assert(graph) # the dead operations will be killed by the remove_obvious_noops below # remove obvious no-ops def remove_obvious_noops(): for graph in graphs: removenoops.remove_same_as(graph) simplify.eliminate_empty_blocks(graph) simplify.transform_dead_op_vars(graph, translator) removenoops.remove_duplicate_casts(graph, translator) if config.print_statistics: print "after no-op removal:" print_statistics(translator.graphs[0], translator) remove_obvious_noops() if config.inline or config.mallocs: heuristic = get_function(config.inline_heuristic) if config.inline: threshold = config.inline_threshold else: threshold = 0 inline_malloc_removal_phase(config, translator, graphs, threshold, inline_heuristic=heuristic) constfold(config, graphs) if config.clever_malloc_removal: threshold = config.clever_malloc_removal_threshold heuristic = get_function(config.clever_malloc_removal_heuristic) log.inlineandremove("phase with threshold factor: %s" % threshold) log.inlineandremove("heuristic: %s.%s" % (heuristic.__module__, heuristic.__name__)) count = mallocprediction.clever_inlining_and_malloc_removal( translator, graphs, threshold = threshold, heuristic=heuristic) log.inlineandremove("removed %d simple mallocs in total" % count) constfold(config, graphs) if config.print_statistics: print "after clever inlining and malloc removal" print_statistics(translator.graphs[0], translator) if config.profile_based_inline and not secondary: threshold = config.profile_based_inline_threshold heuristic = get_function(config.profile_based_inline_heuristic) inline.instrument_inline_candidates(graphs, threshold) counters = translator.driver_instrument_result( config.profile_based_inline) n = len(counters) def call_count_pred(label): if label >= n: return False return counters[label] > 250 # xxx introduce an option for this inline_malloc_removal_phase(config, translator, graphs, threshold, inline_heuristic=heuristic, call_count_pred=call_count_pred) constfold(config, graphs) if config.merge_if_blocks: log.mergeifblocks("starting to merge if blocks") for graph in graphs: merge_if_blocks(graph, translator.config.translation.verbose) if config.print_statistics: print "after if-to-switch:" print_statistics(translator.graphs[0], translator) remove_obvious_noops() for graph in graphs: checkgraph(graph)
def merge_if_blocks_once(graph): """Convert consecutive blocks that all compare a variable (of Primitive type) with a constant into one block with multiple exits. The backends can in turn output this block as a switch statement. """ candidates = [ block for block in graph.iterblocks() if is_chain_block(block, first=True) ] entrymap = mkentrymap(graph) for firstblock in candidates: chain = [] checkvars = [] varmap = {} # {var in a block in the chain: var in the first block} for var in firstblock.exits[0].args: varmap[var] = var for var in firstblock.exits[1].args: varmap[var] = var def add_to_varmap(var, newvar): if isinstance(var, Variable): varmap[newvar] = varmap[var] else: varmap[newvar] = var current = firstblock while 1: # check whether the chain can be extended with the block that follows the # False link checkvar = [ var for var in current.operations[-1].args if isinstance(var, Variable) ][0] resvar = current.operations[-1].result case = [ var for var in current.operations[-1].args if isinstance(var, Constant) ][0] checkvars.append(checkvar) falseexit = current.exits[0] assert not falseexit.exitcase trueexit = current.exits[1] targetblock = falseexit.target # if the result of the check is also passed through the link, we # cannot construct the chain if resvar in falseexit.args or resvar in trueexit.args: break chain.append((current, case)) if len(entrymap[targetblock]) != 1: break if checkvar not in falseexit.args: break newcheckvar = targetblock.inputargs[falseexit.args.index(checkvar)] if not is_chain_block(targetblock): break if newcheckvar not in targetblock.operations[0].args: break for i, var in enumerate(trueexit.args): add_to_varmap(var, trueexit.target.inputargs[i]) for i, var in enumerate(falseexit.args): add_to_varmap(var, falseexit.target.inputargs[i]) current = targetblock if len(chain) > 1: break else: return False merge_chain(chain, checkvars[0], varmap, graph) checkgraph(graph) return True
def cleanup_graph(graph): checkgraph(graph) eliminate_empty_blocks(graph) join_blocks(graph) remove_identical_vars(graph) checkgraph(graph)
def new_wrapper(func, translator, newname=None): # The basic idea is to produce a flow graph from scratch, using the # help of the rtyper for the conversion of the arguments after they # have been decoded. bk = translator.annotator.bookkeeper graph = bk.getdesc(func).getuniquegraph() f = getfunctionptr(graph) FUNCTYPE = typeOf(f).TO newops = LowLevelOpList(translator.rtyper) varguments = [] for var in graph.startblock.inputargs: v = Variable(var) v.concretetype = PyObjPtr varguments.append(v) wrapper_inputargs = varguments[:] # use the rtyper to produce the conversions inputargs = f._obj.graph.getargs() for i in range(len(varguments)): if FUNCTYPE.ARGS[i] != PyObjPtr: rtyper = translator.rtyper r_arg = rtyper.bindingrepr(inputargs[i]) # give the rtyper a chance to know which function we are wrapping rtyper.set_wrapper_context(func) varguments[i] = newops.convertvar(varguments[i], r_from = pyobj_repr, r_to = r_arg) rtyper.set_wrapper_context(None) vlist = [inputconst(typeOf(f), f)] + varguments vresult = newops.genop('direct_call', vlist, resulttype=FUNCTYPE.RESULT) if FUNCTYPE.RESULT != PyObjPtr: # convert "result" back to a PyObject rtyper = translator.rtyper assert rtyper is not None, ( "needs the rtyper to perform function result conversions") r_result = rtyper.bindingrepr(f._obj.graph.getreturnvar()) vresult = newops.convertvar(vresult, r_from = r_result, r_to = pyobj_repr) # "return result" block = Block(wrapper_inputargs) wgraph = FunctionGraph('pyfn_' + (newname or func.func_name), block) translator.update_call_graph(wgraph, graph, object()) translator.graphs.append(wgraph) block.operations[:] = newops block.closeblock(Link([vresult], wgraph.returnblock)) wgraph.getreturnvar().concretetype = PyObjPtr checkgraph(wgraph) # the above convertvar()s may have created and annotated new helpers # that need to be specialized now translator.rtyper.specialize_more_blocks() return functionptr(FuncType([PyObjPtr] * len(wrapper_inputargs), PyObjPtr), wgraph.name, graph = wgraph, exception_policy = "CPython")
def sanity_check(t): # look for missing '.concretetype' for graph in t.graphs: checkgraph(graph) traverse(no_missing_concretetype, graph)
def checkgraphs(self): for graph in self.graphs: checkgraph(graph)
def normalize_calltable_row_signature(annotator, shape, row): graphs = row.values() assert graphs, "no graph??" sig0 = graphs[0].signature defaults0 = graphs[0].defaults for graph in graphs[1:]: if graph.signature != sig0: break if graph.defaults != defaults0: break else: return False # nothing to do, all signatures already match shape_cnt, shape_keys, shape_star, shape_stst = shape assert not shape_star, "XXX not implemented" assert not shape_stst, "XXX not implemented" # for the first 'shape_cnt' arguments we need to generalize to # a common type call_nbargs = shape_cnt + len(shape_keys) did_something = False NODEFAULT = object() for graph in graphs: argnames, varargname, kwargname = graph.signature assert not varargname, "XXX not implemented" assert not kwargname, "XXX not implemented" # ? inputargs_s = [annotator.binding(v) for v in graph.getargs()] argorder = range(shape_cnt) for key in shape_keys: i = list(argnames).index(key) assert i not in argorder argorder.append(i) need_reordering = (argorder != range(call_nbargs)) if need_reordering or len(graph.getargs()) != call_nbargs: oldblock = graph.startblock inlist = [] defaults = graph.defaults or () num_nondefaults = len(inputargs_s) - len(defaults) defaults = [NODEFAULT] * num_nondefaults + list(defaults) newdefaults = [] for j in argorder: v = Variable(graph.getargs()[j]) annotator.setbinding(v, inputargs_s[j]) inlist.append(v) newdefaults.append(defaults[j]) newblock = Block(inlist) # prepare the output args of newblock: # 1. collect the positional arguments outlist = inlist[:shape_cnt] # 2. add defaults and keywords for j in range(shape_cnt, len(inputargs_s)): try: i = argorder.index(j) v = inlist[i] except ValueError: default = defaults[j] if default is NODEFAULT: raise TyperError( "call pattern has %d positional arguments, " "but %r takes at least %d arguments" % ( shape_cnt, graph.name, num_nondefaults)) v = Constant(default) outlist.append(v) newblock.closeblock(Link(outlist, oldblock)) graph.startblock = newblock for i in range(len(newdefaults)-1,-1,-1): if newdefaults[i] is NODEFAULT: newdefaults = newdefaults[i:] break graph.defaults = tuple(newdefaults) graph.signature = Signature([argnames[j] for j in argorder], None, None) # finished checkgraph(graph) annotator.annotated[newblock] = annotator.annotated[oldblock] did_something = True return did_something
def normalize_calltable_row_signature(annotator, shape, row): graphs = row.values() assert graphs, "no graph??" sig0 = graphs[0].signature defaults0 = graphs[0].defaults for graph in graphs[1:]: if graph.signature != sig0: break if graph.defaults != defaults0: break else: return False # nothing to do, all signatures already match shape_cnt, shape_keys, shape_star, shape_stst = shape assert not shape_star, "XXX not implemented" assert not shape_stst, "XXX not implemented" # for the first 'shape_cnt' arguments we need to generalize to # a common type call_nbargs = shape_cnt + len(shape_keys) did_something = False NODEFAULT = object() for graph in graphs: argnames, varargname, kwargname = graph.signature assert not varargname, "XXX not implemented" assert not kwargname, "XXX not implemented" # ? inputargs_s = [annotator.binding(v) for v in graph.getargs()] argorder = range(shape_cnt) for key in shape_keys: i = list(argnames).index(key) assert i not in argorder argorder.append(i) need_reordering = (argorder != range(call_nbargs)) if need_reordering or len(graph.getargs()) != call_nbargs: oldblock = graph.startblock inlist = [] defaults = graph.defaults or () num_nondefaults = len(inputargs_s) - len(defaults) defaults = [NODEFAULT] * num_nondefaults + list(defaults) newdefaults = [] for j in argorder: v = Variable(graph.getargs()[j]) annotator.setbinding(v, inputargs_s[j]) inlist.append(v) newdefaults.append(defaults[j]) newblock = Block(inlist) # prepare the output args of newblock: # 1. collect the positional arguments outlist = inlist[:shape_cnt] # 2. add defaults and keywords for j in range(shape_cnt, len(inputargs_s)): try: i = argorder.index(j) v = inlist[i] except ValueError: default = defaults[j] if default is NODEFAULT: raise TyperError( "call pattern has %d positional arguments, " "but %r takes at least %d arguments" % ( shape_cnt, graph.name, num_nondefaults)) v = Constant(default) outlist.append(v) newblock.closeblock(Link(outlist, oldblock)) oldblock.isstartblock = False newblock.isstartblock = True graph.startblock = newblock for i in range(len(newdefaults)-1,-1,-1): if newdefaults[i] is NODEFAULT: newdefaults = newdefaults[i:] break graph.defaults = tuple(newdefaults) graph.signature = Signature([argnames[j] for j in argorder], None, None) # finished checkgraph(graph) annotator.annotated[newblock] = annotator.annotated[oldblock] did_something = True return did_something
_, origblock, origindex = self.jit_merge_point_pos op = origblock.operations[origindex] assert op.opname == 'jit_marker' assert op.args[0].value == 'jit_merge_point' greens_v, reds_v = decode_hp_hint_args(op) vlist = [Constant(self.portal_runner_ptr, self.PTR_PORTAL_FUNCTYPE)] vlist += greens_v vlist += reds_v v_result = Variable() v_result.concretetype = PORTALFUNC.RESULT newop = SpaceOperation('direct_call', vlist, v_result) del origblock.operations[origindex:] origblock.operations.append(newop) origblock.exitswitch = None origblock.recloseblock(Link([v_result], origportalgraph.returnblock)) checkgraph(origportalgraph) def add_finish(self): def finish(): if self.metainterp_sd.profiler.initialized: self.metainterp_sd.profiler.finish() self.metainterp_sd.cpu.finish_once() if self.cpu.translate_support_code: call_final_function(self.translator, finish, annhelper = self.annhelper) def rewrite_set_param(self): closures = {} graphs = self.translator.graphs _, PTR_SET_PARAM_FUNCTYPE = self.cpu.ts.get_FuncType([lltype.Signed],
def tweak_generator_body_graph(Entry, graph): # First, always run simplify_graph in order to reduce the number of # variables passed around simplify_graph(graph) # assert graph.startblock.operations[0].opname == 'generator_mark' graph.startblock.operations.pop(0) # insert_empty_startblock(None, graph) _insert_reads(graph.startblock, Entry.varnames) Entry.block = graph.startblock # mappings = [Entry] # stopblock = Block([]) v0 = Variable() v1 = Variable() stopblock.operations = [ SpaceOperation('simple_call', [Constant(StopIteration)], v0), SpaceOperation('type', [v0], v1), ] stopblock.closeblock(Link([v1, v0], graph.exceptblock)) # for block in list(graph.iterblocks()): for exit in block.exits: if exit.target is graph.returnblock: exit.args = [] exit.target = stopblock assert block is not stopblock for index in range(len(block.operations) - 1, -1, -1): op = block.operations[index] if op.opname == 'yield': [v_yielded_value] = op.args del block.operations[index] newlink = split_block(None, block, index) newblock = newlink.target # class Resume(AbstractPosition): _immutable_ = True block = newblock Resume.__name__ = 'Resume%d' % len(mappings) mappings.append(Resume) varnames = get_variable_names(newlink.args) # _insert_reads(newblock, varnames) # v_resume = Variable('resume') block.operations.append( SpaceOperation('simple_call', [Constant(Resume)], v_resume)) for i, name in enumerate(varnames): block.operations.append( SpaceOperation( 'setattr', [v_resume, Constant(name), newlink.args[i]], Variable())) v_pair = Variable('pair') block.operations.append( SpaceOperation('newtuple', [v_resume, v_yielded_value], v_pair)) newlink.args = [v_pair] newlink.target = graph.returnblock # regular_entry_block = Block([Variable('entry')]) block = regular_entry_block for Resume in mappings: v_check = Variable() block.operations.append( SpaceOperation( 'simple_call', [Constant(isinstance), block.inputargs[0], Constant(Resume)], v_check)) block.exitswitch = v_check link1 = Link([block.inputargs[0]], Resume.block) link1.exitcase = True nextblock = Block([Variable('entry')]) link2 = Link([block.inputargs[0]], nextblock) link2.exitcase = False block.closeblock(link1, link2) block = nextblock block.closeblock( Link([ Constant(AssertionError), Constant(AssertionError("bad generator class")) ], graph.exceptblock)) graph.startblock = regular_entry_block graph.signature = Signature(['entry']) graph.defaults = () checkgraph(graph) eliminate_empty_blocks(graph)