def test_data_flow_families(): def snippet_fn(xx, yy): while yy > 0: if 0 < xx: yy = yy - xx else: yy = yy + xx return yy t = TranslationContext() graph = t.buildflowgraph(snippet_fn) operations = [] for block in graph.iterblocks(): operations += block.operations variable_families = DataFlowFamilyBuilder(graph).get_variable_families() # we expect to find xx only once: v_xx = variable_families.find_rep(graph.getargs()[0]) found = 0 for op in operations: if op.opname in ('add', 'sub', 'lt'): assert variable_families.find_rep(op.args[1]) == v_xx found += 1 assert found == 3
def test_pseudohighlevelcallable(): t = TranslationContext() t.buildannotator() rtyper = t.buildrtyper() rtyper.specialize() a = MixLevelHelperAnnotator(rtyper) class A: value = 5 def double(self): return self.value * 2 def fn1(a): a2 = A() a2.value = a.double() return a2 s_A, r_A = a.s_r_instanceof(A) fn1ptr = a.delayedfunction(fn1, [s_A], s_A) pseudo = PseudoHighLevelCallable(fn1ptr, [s_A], s_A) def fn2(n): a = A() a.value = n a2 = pseudo(a) return a2.value graph = a.getgraph(fn2, [annmodel.SomeInteger()], annmodel.SomeInteger()) a.finish() llinterp = LLInterpreter(rtyper) res = llinterp.eval_graph(graph, [21]) assert res == 42
def compile(f, gc, **kwds): from rpython.annotator.listdef import s_list_of_strings from rpython.translator.translator import TranslationContext from rpython.jit.metainterp.warmspot import apply_jit from rpython.translator.c import genc # t = TranslationContext() t.config.translation.gc = gc if gc != 'boehm': t.config.translation.gcremovetypeptr = True for name, value in kwds.items(): setattr(t.config.translation, name, value) ann = t.buildannotator() ann.build_types(f, [s_list_of_strings], main_entry_point=True) t.buildrtyper().specialize() if kwds['jit']: patch = get_functions_to_patch() old_value = {} try: for (obj, attr), value in patch.items(): old_value[obj, attr] = getattr(obj, attr) setattr(obj, attr, value) # apply_jit(t) # finally: for (obj, attr), oldvalue in old_value.items(): setattr(obj, attr, oldvalue) cbuilder = genc.CStandaloneBuilder(t, f, t.config) cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES) cbuilder.compile() return cbuilder
def make_deallocator(TYPE, attr="static_deallocation_funcptr_for_type", cls=RefcountingGCTransformer): if TYPE._is_varsize(): def f(): return lltype.malloc(TYPE, 1) else: def f(): return lltype.malloc(TYPE) t = TranslationContext() t.buildannotator().build_types(f, []) t.buildrtyper().specialize() transformer = cls(t) fptr = getattr(transformer, attr)(TYPE) transformer.transform_graph(graphof(t, f)) transformer.finish(backendopt=False) if option.view: t.view() if fptr: return fptr._obj.graph, t else: return None, t
def check(self, fn, signature, args, expected_result, must_be_removed=True, inline=None): remover = self.MallocRemover() t = TranslationContext() t.buildannotator().build_types(fn, signature) t.buildrtyper().specialize() graph = graphof(t, fn) if inline is not None: from rpython.translator.backendopt.inline import auto_inline_graphs auto_inline_graphs(t, t.graphs, inline) if option.view: t.view() # to detect broken intermediate graphs, # we do the loop ourselves instead of calling remove_simple_mallocs() while True: progress = remover.remove_mallocs_once(graph) simplify.transform_dead_op_vars_in_blocks(list(graph.iterblocks()), [graph]) if progress and option.view: t.view() if expected_result is not Ellipsis: interp = LLInterpreter(t.rtyper) res = interp.eval_graph(graph, args) assert res == expected_result if not progress: break if must_be_removed: self.check_malloc_removed(graph) return graph
def test_remove_same_as_nonconst(): from rpython.rlib.nonconst import NonConstant from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.lltypesystem import lltype def f(): if NonConstant(False): x = llop.same_as(lltype.Signed, 666) return 42 t = TranslationContext() t.buildannotator().build_types(f, []) t.buildrtyper().specialize() f_graph = graphof(t, f) #simple_inline_function(t, nothing, f_graph) # here, the graph looks like v21=same_as(True); exitswitch: v21 remove_same_as(f_graph) t.checkgraphs() # only one path should be left for block in f_graph.iterblocks(): assert len(block.exits) <= 1 for block in t.annotator.annotated: assert None not in block.operations interp = LLInterpreter(t.rtyper) result = interp.eval_graph(f_graph, []) assert result == 42
def __init__(self, translator=None, policy=None, bookkeeper=None, keepgoing=False): import rpython.rtyper.extfuncregistry # has side effects if translator is None: # interface for tests from rpython.translator.translator import TranslationContext translator = TranslationContext() translator.annotator = self self.translator = translator self.pendingblocks = ShuffleDict() # map {block: graph-containing-it} self.annotated = {} # set of blocks already seen self.added_blocks = None # see processblock() below self.links_followed = {} # set of links that have ever been followed self.notify = {} # {block: {positions-to-reflow-from-when-done}} self.fixed_graphs = {} # set of graphs not to annotate again self.blocked_blocks = {} # set of {blocked_block: (graph, index)} # --- the following information is recorded for debugging --- self.blocked_graphs = {} # set of graphs that have blocked blocks # --- end of debugging information --- self.frozen = False if policy is None: from rpython.annotator.policy import AnnotatorPolicy self.policy = AnnotatorPolicy() else: self.policy = policy if bookkeeper is None: bookkeeper = Bookkeeper(self) self.bookkeeper = bookkeeper self.keepgoing = keepgoing self.failed_blocks = set() self.errors = []
def test_replace_exitswitch_by_constant_bug(): class X: pass def constant9(): x = X() x.n = 3 x.n = 9 return x.n def fn(): n = constant9() if n == 1: return 5 elif n == 2: return 6 elif n == 3: return 8 elif n == 4: return -123 elif n == 5: return 12973 else: return n t = TranslationContext() a = t.buildannotator() a.build_types(fn, []) rtyper = t.buildrtyper() rtyper.specialize() graph = t.graphs[0] remove_same_as(graph) merge_if_blocks_once(graph) from rpython.translator.backendopt import malloc, inline inline.auto_inlining(t, 20) malloc.remove_mallocs(t, t.graphs) from rpython.translator import simplify simplify.join_blocks(graph)
def rtype(func, inputtypes, specialize=True, gcname='ref', backendopt=False, **extraconfigopts): from rpython.translator.translator import TranslationContext t = TranslationContext() # XXX XXX XXX mess t.config.translation.gc = gcname t.config.translation.gcremovetypeptr = True t.config.set(**extraconfigopts) ann = t.buildannotator() ann.build_types(func, inputtypes) rtyper = t.buildrtyper() rtyper.backend = llinterp_backend if specialize: rtyper.specialize() if backendopt: from rpython.translator.backendopt.all import backend_optimizations backend_optimizations(t) if option.view: t.viewcg() return t
def annotate_at(f, policy=None): t = TranslationContext() t.config.translation.check_str_without_nul = True a = t.buildannotator(policy=policy) a.annotate_helper(f, [model.s_ImpossibleValue] * f.__code__.co_argcount, policy=policy) return a
def test_merge_several(): def merge(n, m): r = -1 if n == 0: if m == 0: r = 0 elif m == 1: r = 1 else: r = 2 elif n == 1: r = 4 else: r = 6 return r t = TranslationContext() a = t.buildannotator() a.build_types(merge, [int, int]) rtyper = t.buildrtyper() rtyper.specialize() graph = tgraphof(t, merge) remove_same_as(graph) merge_if_blocks(graph) assert len(graph.startblock.exits) == 3 assert len(list(graph.iterblocks())) == 3 interp = LLInterpreter(rtyper) for m in range(3): res = interp.eval_graph(graph, [0, m]) assert res == m res = interp.eval_graph(graph, [1, 0]) assert res == 4 res = interp.eval_graph(graph, [2, 0]) assert res == 6
def analyze(self, func, sig): t = TranslationContext() t.buildannotator().build_types(func, sig) t.buildrtyper().specialize() fgraph = graphof(t, func) return VirtualizableAnalyzer(t).analyze( fgraph.startblock.operations[0])
def translate(func, argtypes, backendopt=False): t = TranslationContext() t.buildannotator().build_types(func, argtypes) t.buildrtyper(type_system='ootype').specialize() if backendopt: backend_optimizations(t, merge_if_blocks=True) return t
def makegraph(func, argtypes): t = TranslationContext() t.buildannotator().build_types(func, [int]) t.buildrtyper().specialize() bk = t.annotator.bookkeeper graph = bk.getdesc(func).getuniquegraph() return t, graph
def rtype(self, fn, argtypes, resulttype, checkfunction=None): t = TranslationContext() a = t.buildannotator() a.build_types(prefn, [int]) typer = t.buildrtyper() typer.specialize() #t.view() s_result = a.typeannotation(resulttype) from rpython.rtyper import annlowlevel # annotate, normalize and rtype fn after the fact annhelper = annlowlevel.MixLevelHelperAnnotator(typer) graph = annhelper.getgraph( fn, [a.typeannotation(argtype) for argtype in argtypes], s_result) annhelper.finish() t.checkgraphs() if checkfunction is not None: checkfunction(t) # sanity check prefn llinterp = LLInterpreter(typer) res = llinterp.eval_graph(graphof(t, prefn), [1]) assert res == 100 res = llinterp.eval_graph(graphof(t, prefn), [2]) assert res == 201 return t
def test_del_basic(self): py.test.skip("xxx fix or kill") S = lltype.GcStruct('S', ('x', lltype.Signed), rtti=True) TRASH = lltype.GcStruct('TRASH', ('x', lltype.Signed)) GLOBAL = lltype.Struct('GLOBAL', ('x', lltype.Signed)) glob = lltype.malloc(GLOBAL, immortal=True) def destructor(s): glob.x = s.x + 1 def type_info_S(s): return lltype.getRuntimeTypeInfo(S) def g(n): s = lltype.malloc(S) s.x = n # now 's' should go away def entrypoint(n): g(n) # llop.gc__collect(lltype.Void) return glob.x t = TranslationContext() t.buildannotator().build_types(entrypoint, [int]) rtyper = t.buildrtyper() destrptr = rtyper.annotate_helper_fn(destructor, [lltype.Ptr(S)]) rtyper.attachRuntimeTypeInfoFunc(S, type_info_S, destrptr=destrptr) rtyper.specialize() fn = self.compile_func(entrypoint, None, t) res = fn(123) assert res == 124
def test_remove_same_as(): def nothing(x): return x def f(): nothing(False) if nothing(True): return 42 else: return 666 t = TranslationContext() t.buildannotator().build_types(f, []) t.buildrtyper().specialize() # now we make the 'if True' appear f_graph = graphof(t, f) simple_inline_function(t, nothing, f_graph) # here, the graph looks like v21=same_as(True); exitswitch: v21 remove_same_as(f_graph) t.checkgraphs() # only one path should be left for block in f_graph.iterblocks(): assert len(block.exits) <= 1 interp = LLInterpreter(t.rtyper) result = interp.eval_graph(f_graph, []) assert result == 42
def _build_gen(func, annotation, graph=None, backendopt=True, exctrans=False, annotatorpolicy=None, nowrap=False): try: func = func.im_func except AttributeError: pass t = TranslationContext() if graph is not None: graph.func = func ann = t.buildannotator(policy=annotatorpolicy) inputcells = [ann.typeannotation(a) for a in annotation] ann.build_graph_types(graph, inputcells) t.graphs.insert(0, graph) else: ann = t.buildannotator(policy=annotatorpolicy) ann.build_types(func, annotation) if getoption('view'): t.view() t.buildrtyper(type_system="ootype").specialize() if backendopt: check_virtual_methods(ootype.ROOT) backend_optimizations(t) main_graph = t.graphs[0] if getoption('view'): t.view() return _build_gen_from_graph(main_graph, t, exctrans, nowrap)
def test_type_erase(self): class A(object): pass class B(object): pass def f(): d = {} d[A()] = B() d2 = {} d2[B()] = A() return d, d2 t = TranslationContext() s = t.buildannotator().build_types(f, []) rtyper = t.buildrtyper() rtyper.specialize() s_AB_dic = s.items[0] s_BA_dic = s.items[1] r_AB_dic = rtyper.getrepr(s_AB_dic) r_BA_dic = rtyper.getrepr(s_AB_dic) assert r_AB_dic.lowleveltype == r_BA_dic.lowleveltype
def test_lookup_graphs_abstract(): from rpython.translator.translator import TranslationContext, graphof class A: pass class B(A): def foo(self): pass class C(A): def foo(self): pass def fn(flag): obj = flag and B() or C() obj.foo() return obj t = TranslationContext() t.buildannotator().build_types(fn, [int]) t.buildrtyper(type_system='ootype').specialize() graph = graphof(t, fn) TYPE_A = graph.getreturnvar().concretetype TYPE_B = TYPE_A._subclasses[0] TYPE_C = TYPE_A._subclasses[1] assert len(TYPE_A._lookup_graphs('ofoo')) == 2 assert len(TYPE_B._lookup_graphs('ofoo')) == 1 assert len(TYPE_C._lookup_graphs('ofoo')) == 1
def _get_TranslationContext(self): t = TranslationContext() t.config.translation.gc = DEFL_GC # 'hybrid' or 'minimark' t.config.translation.gcrootfinder = 'shadowstack' t.config.translation.list_comprehension_operations = True t.config.translation.gcremovetypeptr = True return t
def translate(self, func, sig): t = TranslationContext() t.buildannotator().build_types(func, sig) t.buildrtyper().specialize() if option.view: t.view() return t, self.Analyzer(t)
def test_dont_remove_with__del__(self): import os delcalls = [0] class A(object): nextid = 0 def __init__(self): self.id = self.nextid self.nextid += 1 def __del__(self): delcalls[0] += 1 #os.write(1, "__del__\n") def f(x=int): a = A() i = 0 while i < x: a = A() os.write(1, str(delcalls[0]) + "\n") i += 1 return 1 t = TranslationContext() t.buildannotator().build_types(f, [int]) t.buildrtyper().specialize() graph = graphof(t, f) backend_optimizations(t) op = graph.startblock.exits[0].target.exits[1].target.operations[0] assert op.opname == "malloc"
def gengraph(func, argtypes=[], viewbefore='auto', policy=None, backendopt=False, config=None, **extraconfigopts): t = TranslationContext(config=config) t.config.set(**extraconfigopts) a = t.buildannotator(policy=policy) a.build_types(func, argtypes, main_entry_point=True) a.validate() if viewbefore == 'auto': viewbefore = getattr(option, 'view', False) if viewbefore: a.simplify() t.view() global typer # we need it for find_exception typer = t.buildrtyper() typer.backend = llinterp_backend typer.specialize() #t.view() t.checkgraphs() if backendopt: from rpython.translator.backendopt.all import backend_optimizations backend_optimizations(t) t.checkgraphs() if viewbefore: t.view() desc = t.annotator.bookkeeper.getdesc(func) graph = desc.specialize(argtypes) return t, typer, graph
def translates(self, func=None, argtypes=None, seeobj_w=[], **kwds): config = make_config(None, **kwds) if func is not None: if argtypes is None: nb_args = func.func_code.co_argcount argtypes = [W_Root] * nb_args # t = TranslationContext(config=config) self.t = t # for debugging ann = t.buildannotator() def _do_startup(): self.threadlocals.enter_thread(self) W_SliceObject(w_some_obj(), w_some_obj(), w_some_obj()) ann.build_types(_do_startup, [], complete_now=False) if func is not None: ann.build_types(func, argtypes, complete_now=False) if seeobj_w: def seeme(n): return seeobj_w[n] ann.build_types(seeme, [int], complete_now=False) # # annotate all _seen_extras, knowing that annotating some may # grow the list done = 0 while done < len(self._seen_extras): #print self._seen_extras ann.build_types(self._seen_extras[done], [], complete_now=False) ann.complete_pending_blocks() done += 1 ann.complete() assert done == len(self._seen_extras) #t.viewcg() t.buildrtyper().specialize() t.checkgraphs()
def test_funny_links(): from rpython.flowspace.model import Block, FunctionGraph, \ Variable, Constant, Link from rpython.flowspace.operation import op for i in range(2): v_i = Variable("i") block = Block([v_i]) g = FunctionGraph("is_one", block) op1 = op.eq(v_i, Constant(1)) block.operations.append(op1) block.exitswitch = op1.result tlink = Link([Constant(1)], g.returnblock, True) flink = Link([Constant(0)], g.returnblock, False) links = [tlink, flink] if i: links.reverse() block.closeblock(*links) t = TranslationContext() a = t.buildannotator() a.build_graph_types(g, [annmodel.SomeInteger()]) rtyper = t.buildrtyper() rtyper.specialize() interp = LLInterpreter(rtyper) assert interp.eval_graph(g, [1]) == 1 assert interp.eval_graph(g, [0]) == 0
def annotate(func, values, inline=None, backendoptimize=True, translationoptions={}): # build the normal ll graphs for ll_function t = TranslationContext() for key, value in translationoptions.items(): setattr(t.config.translation, key, value) annpolicy = AnnotatorPolicy() a = t.buildannotator(policy=annpolicy) argtypes = getargtypes(a, values) a.build_types(func, argtypes, main_entry_point=True) rtyper = t.buildrtyper() rtyper.specialize() #if inline: # auto_inlining(t, threshold=inline) if backendoptimize: from rpython.translator.backendopt.all import backend_optimizations backend_optimizations(t, inline_threshold=inline or 0, remove_asserts=True, really_remove_asserts=True) return rtyper
def setup(self, entry_point, inputtypes, policy=None, extra={}, empty_translator=None): standalone = inputtypes is None self.standalone = standalone if standalone: # the 'argv' parameter inputtypes = [s_list_of_strings] self.inputtypes = inputtypes if policy is None: policy = annpolicy.AnnotatorPolicy() self.policy = policy self.extra = extra if empty_translator: translator = empty_translator else: translator = TranslationContext(config=self.config) self.entry_point = entry_point self.translator = translator self.libdef = None self.secondary_entrypoints = [] if self.config.translation.secondaryentrypoints: for key in self.config.translation.secondaryentrypoints.split(","): try: points = secondary_entrypoints[key] except KeyError: raise KeyError("Entrypoint %r not found (not in %r)" % (key, secondary_entrypoints.keys())) self.secondary_entrypoints.extend(points) self.translator.driver_instrument_result = self.instrument_result
def test_type_erase_var_size(self): class A(object): pass class B(object): pass def f(): la = [A()] lb = [B()] la.append(None) lb.append(None) return la, lb t = TranslationContext() s = t.buildannotator().build_types(f, []) rtyper = t.buildrtyper() rtyper.specialize() s_A_list = s.items[0] s_B_list = s.items[1] r_A_list = rtyper.getrepr(s_A_list) assert isinstance(r_A_list, self.rlist.ListRepr) r_B_list = rtyper.getrepr(s_B_list) assert isinstance(r_B_list, self.rlist.ListRepr) assert r_A_list.lowleveltype == r_B_list.lowleveltype
def test_switch_on_symbolic(): symb1 = CDefinedIntSymbolic("1", 1) symb2 = CDefinedIntSymbolic("2", 2) symb3 = CDefinedIntSymbolic("3", 3) def fn(x): res = 0 if x == symb1: res += x + 1 elif x == symb2: res += x + 2 elif x == symb3: res += x + 3 res += 1 return res t = TranslationContext() a = t.buildannotator() a.build_types(fn, [int]) rtyper = t.buildrtyper() rtyper.specialize() graph = t.graphs[0] remove_same_as(graph) res = merge_if_blocks_once(graph) assert not res checkgraph(graph)