def test_llexternal_with_callback(self): from rpython.rtyper.lltypesystem.rffi import llexternal from rpython.rtyper.lltypesystem import lltype class Abc: pass abc = Abc() FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) z = llexternal('z', [lltype.Ptr(FUNC)], lltype.Signed) def g(n): abc.foobar = n return n + 1 def f(x): return z(g) t, wa = self.translate(f, [int]) fgraph = graphof(t, f) backend_optimizations(t) assert fgraph.startblock.operations[0].opname == 'direct_call' result = wa.analyze(fgraph.startblock.operations[0]) assert len(result) == 1 (struct, T, name), = result assert struct == "struct" assert name.endswith("foobar")
def translate(func, argtypes, backendopt=False): t = TranslationContext() t.buildannotator().build_types(func, argtypes) t.buildrtyper(type_system='ootype').specialize() if backendopt: backend_optimizations(t, merge_if_blocks=True) return t
def test_llexternal(self): from rpython.rtyper.lltypesystem.rffi import llexternal from rpython.rtyper.lltypesystem import lltype z = llexternal('z', [lltype.Signed], lltype.Signed) def f(x): return z(x) t, ra = self.translate(f, [int]) fgraph = graphof(t, f) backend_optimizations(t) assert fgraph.startblock.operations[0].opname == 'direct_call' result = ra.can_raise(fgraph.startblock.operations[0]) assert not result z = lltype.functionptr(lltype.FuncType([lltype.Signed], lltype.Signed), 'foobar') def g(x): return z(x) t, ra = self.translate(g, [int]) ggraph = graphof(t, g) assert ggraph.startblock.operations[0].opname == 'direct_call' result = ra.can_raise(ggraph.startblock.operations[0]) assert result
def rtype(func, inputtypes, specialize=True, gcname='ref', backendopt=False, **extraconfigopts): from rpython.translator.translator import TranslationContext t = TranslationContext() # XXX XXX XXX mess t.config.translation.gc = gcname t.config.translation.gcremovetypeptr = True t.config.set(**extraconfigopts) ann = t.buildannotator() ann.build_types(func, inputtypes) rtyper = t.buildrtyper() rtyper.backend = llinterp_backend if specialize: rtyper.specialize() if backendopt: from rpython.translator.backendopt.all import backend_optimizations backend_optimizations(t) if option.view: t.viewcg() return t
def _build_gen(func, annotation, graph=None, backendopt=True, exctrans=False, annotatorpolicy=None, nowrap=False): try: func = func.im_func except AttributeError: pass t = TranslationContext() if graph is not None: graph.func = func ann = t.buildannotator(policy=annotatorpolicy) inputcells = [ann.typeannotation(a) for a in annotation] ann.build_graph_types(graph, inputcells) t.graphs.insert(0, graph) else: ann = t.buildannotator(policy=annotatorpolicy) ann.build_types(func, annotation) if getoption('view'): t.view() t.buildrtyper(type_system="ootype").specialize() if backendopt: check_virtual_methods(ootype.ROOT) backend_optimizations(t) main_graph = t.graphs[0] if getoption('view'): t.view() return _build_gen_from_graph(main_graph, t, exctrans, nowrap)
def test_optimize_method(): def fn(n): if n > 0: x = B(n) else: x = C(n) return x.meth(100) interp, graph = get_interpreter(fn, [-1000], taggedpointers=True) t = interp.typer.annotator.translator t.config.translation.backendopt.constfold = True backend_optimizations(t) if option.view: t.view() LLFrame = interp.frame_class class MyFrame(LLFrame): def op_indirect_call(self, f, *args): raise AssertionError("this call should be optimized away") interp.frame_class = MyFrame res = interp.eval_graph(graph, [-1000]) assert res == -897
def generate_source_for_function(func, annotation, backendopt=False): """ Given a Python function and some hints about its argument types, generates JVM sources that call it and print the result. Returns the JvmGeneratedSource object. """ if hasattr(func, "im_func"): func = func.im_func t = TranslationContext() ann = t.buildannotator() ann.build_types(func, annotation) t.buildrtyper(type_system="ootype").specialize() if backendopt: check_virtual_methods(ootype.ROOT) backend_optimizations(t) main_graph = t.graphs[0] if getoption("view"): t.view() if getoption("wd"): tmpdir = py.path.local(".") else: tmpdir = udir jvm = GenJvm(tmpdir, t, EntryPoint(main_graph, True, True)) return jvm.generate_source()
def test_dont_remove_with__del__(self): import os delcalls = [0] class A(object): nextid = 0 def __init__(self): self.id = self.nextid self.nextid += 1 def __del__(self): delcalls[0] += 1 #os.write(1, "__del__\n") def f(x=int): a = A() i = 0 while i < x: a = A() os.write(1, str(delcalls[0]) + "\n") i += 1 return 1 t = TranslationContext() t.buildannotator().build_types(f, [int]) t.buildrtyper().specialize() graph = graphof(t, f) backend_optimizations(t) op = graph.startblock.exits[0].target.exits[1].target.operations[0] assert op.opname == "malloc"
def annotate(func, values, inline=None, backendoptimize=True, translationoptions={}): # build the normal ll graphs for ll_function t = TranslationContext() for key, value in translationoptions.items(): setattr(t.config.translation, key, value) annpolicy = AnnotatorPolicy() a = t.buildannotator(policy=annpolicy) argtypes = getargtypes(a, values) a.build_types(func, argtypes, main_entry_point=True) rtyper = t.buildrtyper() rtyper.specialize() #if inline: # auto_inlining(t, threshold=inline) if backendoptimize: from rpython.translator.backendopt.all import backend_optimizations backend_optimizations(t, inline_threshold=inline or 0, remove_asserts=True, really_remove_asserts=True) return rtyper
def gengraph(func, argtypes=[], viewbefore='auto', policy=None, backendopt=False, config=None, **extraconfigopts): t = TranslationContext(config=config) t.config.set(**extraconfigopts) a = t.buildannotator(policy=policy) a.build_types(func, argtypes, main_entry_point=True) a.validate() if viewbefore == 'auto': viewbefore = getattr(option, 'view', False) if viewbefore: a.simplify() t.view() global typer # we need it for find_exception typer = t.buildrtyper() typer.backend = llinterp_backend typer.specialize() #t.view() t.checkgraphs() if backendopt: from rpython.translator.backendopt.all import backend_optimizations backend_optimizations(t) t.checkgraphs() if viewbefore: t.view() desc = t.annotator.bookkeeper.getdesc(func) graph = desc.specialize(argtypes) return t, typer, graph
def backend_optimize(self, **flags): # only optimize the newly created graphs from rpython.translator.backendopt.all import backend_optimizations translator = self.rtyper.annotator.translator newgraphs = list(self.newgraphs) backend_optimizations(translator, newgraphs, secondary=True, inline_graph_from_anywhere=True, **flags) self.newgraphs.clear()
def prejit_optimizations(self, policy, graphs): from rpython.translator.backendopt.all import backend_optimizations backend_optimizations(self.translator, graphs=graphs, merge_if_blocks=True, constfold=True, remove_asserts=True, really_remove_asserts=True, replace_we_are_jitted=False)
def prejit_optimizations(self, policy, graphs): from rpython.translator.backendopt.all import backend_optimizations backend_optimizations(self.translator, graphs=graphs, merge_if_blocks=True, constfold=True, raisingop2direct_call=False, remove_asserts=True, really_remove_asserts=True)
def translate(func, argtypes, backend_optimize=True): t = TranslationContext() t.buildannotator().build_types(func, argtypes) t.buildrtyper().specialize() if backend_optimize: backend_optimizations(t) if option.view: t.view() return graphof(t, func), t
def get_graph(fn, signature, all_opts=True): t = TranslationContext() t.buildannotator().build_types(fn, signature) t.buildrtyper().specialize() if all_opts: backend_optimizations(t, inline_threshold=INLINE_THRESHOLD_FOR_TEST, constfold=False) graph = graphof(t, fn) if option.view: t.view() return graph, t
def translateopt(self, func, sig, **optflags): t = TranslationContext() opts = {'translation.list_comprehension_operations': True} t.config.set(**opts) t.buildannotator().build_types(func, sig) t.buildrtyper().specialize() if option.view: t.view() backend_optimizations(t, **optflags) if option.view: t.view() return t
def test_llexternal(self): from rpython.rtyper.lltypesystem.rffi import llexternal z = llexternal('z', [lltype.Signed], lltype.Signed) def f(x): return z(x) t, wa = self.translate(f, [int]) fgraph = graphof(t, f) backend_optimizations(t) assert fgraph.startblock.operations[0].opname == 'direct_call' result = wa.analyze(fgraph.startblock.operations[0]) assert not result
def getgraph(f, argtypes): from rpython.translator.translator import TranslationContext, graphof from rpython.translator.backendopt.all import backend_optimizations t = TranslationContext() a = t.buildannotator() typer = t.buildrtyper() a.build_types(f, argtypes) typer.specialize() backend_optimizations(t) graph = graphof(t, f) if option.view: graph.show() return graph
def specialize(self, func, argtypes): from rpython.rtyper.llinterp import LLInterpreter t = TranslationContext(list_comprehension_operations=True) t.buildannotator().build_types(func, argtypes) if option.view: t.view() t.buildrtyper().specialize() backend_optimizations(t) if option.view: t.view() graph = graphof(t, func) interp = LLInterpreter(t.rtyper) return interp, graph
def analyze(self, func, sig, func_to_analyze=None, backendopt=False): if func_to_analyze is None: func_to_analyze = func t = TranslationContext() t.buildannotator().build_types(func, sig) t.buildrtyper().specialize() if backendopt: backend_optimizations(t) if option.view: t.view() a = FinalizerAnalyzer(t) fgraph = graphof(t, func_to_analyze) result = a.analyze_light_finalizer(fgraph) return result
def test_simple(): t = TranslationContext() a = t.buildannotator() a.build_types(g, [int]) a.simplify() t.buildrtyper().specialize() backend_optimizations(t) t.checkgraphs() n = insert_ll_stackcheck(t) t.checkgraphs() assert n == 1 if option.view: t.view() check(graphof(t, f), "f")
def test_count_vars_big(): from rpython.translator.goal.targetrpystonex import make_target_definition from rpython.translator.backendopt.all import backend_optimizations entrypoint, _, _ = make_target_definition(10) t = rtype(entrypoint, [int]) backend_optimizations(t) # does not crash rel = relevant_gcvars(t) print rel print sum(rel) / float(len(rel)), max(rel), min(rel) rel = relevant_gcvars(t, filter_for_nongcptr) print rel print sum(rel) / float(len(rel)), max(rel), min(rel)
def test_simple(): t = TranslationContext() a = t.buildannotator() a.build_types(g, [int]) a.simplify() t.buildrtyper().specialize() backend_optimizations(t) t.checkgraphs() n = insert_ll_stackcheck(t) t.checkgraphs() assert n == 1 if option.view: t.view() check(graphof(t, f), 'f')
def test_newlist_negativ(): def f(n): l = [0] * n return len(l) rtyper = support.annotate(f, [-1]) jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) cw = CodeWriter(FakeCPU(rtyper), [jitdriver_sd]) graphs = cw.find_all_graphs(FakePolicy()) backend_optimizations(rtyper.annotator.translator, graphs=graphs) cw.make_jitcodes(verbose=True) s = jitdriver_sd.mainjitcode.dump() assert 'int_force_ge_zero' in s assert 'new_array' in s
def test_two_constants(): def fn(): r = range(10, 37, 4) r.reverse() return r[0] t = TranslationContext() a = t.buildannotator() a.build_types(fn, []) rtyper = t.buildrtyper() rtyper.specialize() backend_optimizations(t, merge_if_blocks=True) graph = tgraphof(t, fn) blocknum = len(list(graph.iterblocks())) merge_if_blocks(graph) assert blocknum == len(list(graph.iterblocks()))
def transform_func(self, fn, inputtypes, backendopt=False): t = TranslationContext() t.buildannotator().build_types(fn, inputtypes) t.buildrtyper().specialize() if option.view: t.view() if backendopt: backend_optimizations(t) g = graphof(t, fn) etrafo = exceptiontransform.ExceptionTransformer(t) etrafo.create_exception_handling(g) join_blocks(g) if option.view: t.view() return t, g
def test_join_blocks_cleans_links(): from rpython.rtyper.lltypesystem import lltype from rpython.flowspace.model import Constant from rpython.translator.backendopt.removenoops import remove_same_as def f(x): return bool(x + 2) def g(x): if f(x): return 1 else: return 2 graph, t = translate(g, [int], backend_optimize=False) fgraph = graphof(t, f) fgraph.startblock.exits[0].args = [Constant(True, lltype.Bool)] # does not crash: previously join_blocks would barf on this remove_same_as(graph) backend_optimizations(t)
def test_list_operations(): class A(object): pass def f(): l = [A(), A()] l.append(A()) l[1] = l[0] return len(l) t = rtype(f, []) backend_optimizations(t, clever_malloc_removal=False, storesink=True) etrafo = ExceptionTransformer(t) graph = etrafo.transform_completely() collect_analyzer = CollectAnalyzer(t) clean_setarrayitems = find_clean_setarrayitems(collect_analyzer, t.graphs[0]) assert len(clean_setarrayitems) == 1
def test_idempotent(self): def s(x): res = 0 i = 1 while i <= x: res += i i += 1 return res def g(x): return s(100) + s(1) + x def idempotent(n1, n2): c = [i for i in range(n2)] return 33 + big() + g(10) t = self.translateopt(idempotent, [int, int], raisingop2direct_call=True, constfold=False) #backend_optimizations(t, raisingop2direct_call=True, # inline_threshold=0, constfold=False) digest1 = md5digest(t) digest2 = md5digest(t) def compare(digest1, digest2): diffs = [] assert digest1.keys() == digest2.keys() for name in digest1: if digest1[name] != digest2[name]: diffs.append(name) assert not diffs compare(digest1, digest2) #XXX Inlining and constfold are currently non-idempotent. # Maybe they just renames variables but the graph changes in some way. backend_optimizations(t, raisingop2direct_call=True, inline_threshold=0, constfold=False) digest3 = md5digest(t) compare(digest1, digest3)
def rtype(func, inputtypes, specialize=True, gcname='ref', backendopt=False, **extraconfigopts): from rpython.translator.translator import TranslationContext t = TranslationContext() # XXX XXX XXX mess t.config.translation.gc = gcname t.config.translation.gcremovetypeptr = True t.config.set(**extraconfigopts) ann = t.buildannotator() ann.build_types(func, inputtypes) if specialize: t.buildrtyper().specialize() if backendopt: from rpython.translator.backendopt.all import backend_optimizations backend_optimizations(t) if option.view: t.viewcg() return t
def test_gctransformed(): t = TranslationContext() a = t.buildannotator() a.build_types(g, [int]) a.simplify() t.buildrtyper().specialize() backend_optimizations(t) t.checkgraphs() n = insert_ll_stackcheck(t) t.checkgraphs() assert n == 1 exctransf = t.getexceptiontransformer() f_graph = graphof(t, f) exctransf.create_exception_handling(f_graph) if option.view: f_graph.show() check(f_graph, 'f') class GCTransform(shadowstack.ShadowStackFrameworkGCTransformer): from rpython.memory.gc.generation import GenerationGC as \ GCClass GC_PARAMS = {} gctransf = GCTransform(t) gctransf.transform_graph(f_graph) if option.view: f_graph.show() relevant = check(f_graph, 'f') for p in relevant: in_between = False reload = 0 for spaceop in p: if spaceop.opname == 'direct_call': target = direct_target(spaceop) if target == 'f': in_between = False elif target == 'stack_check___': in_between = True if in_between and spaceop.opname == 'gc_reload_possibly_moved': reload += 1 assert reload == 0
def test_gctransformed(): t = TranslationContext() a = t.buildannotator() a.build_types(g, [int]) a.simplify() t.buildrtyper().specialize() backend_optimizations(t) t.checkgraphs() n = insert_ll_stackcheck(t) t.checkgraphs() assert n == 1 exctransf = t.getexceptiontransformer() f_graph = graphof(t, f) exctransf.create_exception_handling(f_graph) if option.view: f_graph.show() check(f_graph, "f") class GCTransform(shadowstack.ShadowStackFrameworkGCTransformer): from rpython.memory.gc.generation import GenerationGC as GCClass GC_PARAMS = {} gctransf = GCTransform(t) gctransf.transform_graph(f_graph) if option.view: f_graph.show() relevant = check(f_graph, "f") for p in relevant: in_between = False reload = 0 for spaceop in p: if spaceop.opname == "direct_call": target = direct_target(spaceop) if target == "f": in_between = False elif target == "stack_check___": in_between = True if in_between and spaceop.opname == "gc_reload_possibly_moved": reload += 1 assert reload == 0
def test_implicit_cast(self): z = llexternal('z', [USHORT, ULONG, USHORT, DOUBLE], USHORT, sandboxsafe=True) # to allow the wrapper to be inlined def f(x, y, xx, yy): return z(x, y, xx, yy) a = RPythonAnnotator() r = a.build_types(f, [int, int, int, int]) rtyper = RPythonTyper(a) rtyper.specialize() a.translator.rtyper = rtyper backend_optimizations(a.translator) if option.view: a.translator.view() graph = graphof(a.translator, f) s = summary(graph) # there should be not too many operations here by now expected = {'force_cast': 3, 'cast_int_to_float': 1, 'direct_call': 1} for k, v in expected.items(): assert s[k] == v
def generate_source_for_function(func, annotation, backendopt=False): """ Given a Python function and some hints about its argument types, generates JVM sources that call it and print the result. Returns the JvmGeneratedSource object. """ if hasattr(func, 'im_func'): func = func.im_func t = TranslationContext() ann = t.buildannotator() ann.build_types(func, annotation) t.buildrtyper(type_system="ootype").specialize() if backendopt: check_virtual_methods(ootype.ROOT) backend_optimizations(t) main_graph = t.graphs[0] if getoption('view'): t.view() if getoption('wd'): tmpdir = py.path.local('.') else: tmpdir = udir jvm = GenJvm(tmpdir, t, EntryPoint(main_graph, True, True)) return jvm.generate_source()
def test_same_cases(): def fn(x): if x == 42: r = 1 elif x == 42: r = 2 else: r = 3 return r t = TranslationContext() a = t.buildannotator() a.build_types(fn, [int]) rtyper = t.buildrtyper() rtyper.specialize() backend_optimizations(t, merge_if_blocks=True) graph = tgraphof(t, fn) assert len(graph.startblock.exits) == 2 interp = LLInterpreter(rtyper) for i in [42, 43]: expected = fn(i) actual = interp.eval_graph(graph, [i]) assert actual == expected