def hannotate(func, argtypes, policy=P_DEFAULT, annotator=False, inline=None, backendoptimize=False): # build the normal ll graphs for ll_function t = TranslationContext() a = t.buildannotator() a.build_types(func, argtypes) rtyper = t.buildrtyper() rtyper.specialize() if inline: auto_inlining(t, threshold=inline) if backendoptimize: from pypy.translator.backendopt.all import backend_optimizations backend_optimizations(t) graph1 = graphof(t, func) # build hint annotator types hannotator = HintAnnotator(base_translator=t, policy=policy) hs = hannotator.build_types(graph1, [SomeLLAbstractConstant(v.concretetype, {OriginFlags(): True}) for v in graph1.getargs()]) hannotator.simplify() t = hannotator.translator if conftest.option.view: t.view() if annotator: return hs, hannotator else: return hs
def check(self, fn, signature, args, expected_result, expected_mallocs=0, expected_calls=0): t = TranslationContext() self.translator = t t.buildannotator().build_types(fn, signature) t.buildrtyper(type_system=self.type_system).specialize() graph = graphof(t, fn) if option.view: t.view() self.original_graph_count = len(t.graphs) # to detect missing keepalives and broken intermediate graphs, # we do the loop ourselves instead of calling remove_simple_mallocs() maxiter = 100 mallocv = MallocVirtualizer(t.graphs, t.rtyper, verbose=True) while True: progress = mallocv.remove_mallocs_once() #simplify.transform_dead_op_vars_in_blocks(list(graph.iterblocks())) if progress and option.view: t.view() t.checkgraphs() if expected_result is not DONT_CHECK_RESULT: interp = LLInterpreter(t.rtyper) if not isinstance(expected_result, CHECK_RAISES): res = interp.eval_graph(graph, args) assert res == expected_result else: excinfo = py.test.raises(LLException, interp.eval_graph, graph, args) assert expected_result.excname in str(excinfo.value) if not progress: break maxiter -= 1 assert maxiter > 0, "infinite loop?" self.check_malloc_removed(graph, expected_mallocs, expected_calls) return graph
def check(self, fn, signature, args, expected_result, must_be_removed=True, inline=None): remover = self.MallocRemover() t = TranslationContext() t.buildannotator().build_types(fn, signature) t.buildrtyper(type_system=self.type_system).specialize() graph = graphof(t, fn) if inline is not None: from pypy.translator.backendopt.inline import auto_inline_graphs auto_inline_graphs(t, t.graphs, inline) if option.view: t.view() # to detect missing keepalives and broken intermediate graphs, # we do the loop ourselves instead of calling remove_simple_mallocs() while True: progress = remover.remove_mallocs_once(graph) simplify.transform_dead_op_vars_in_blocks(list(graph.iterblocks())) if progress and option.view: t.view() if expected_result is not Ellipsis: interp = LLInterpreter(t.rtyper) res = interp.eval_graph(graph, args) assert res == expected_result if not progress: break if must_be_removed: self.check_malloc_removed(graph) return graph
def _build_gen(func, annotation, graph=None, backendopt=True, exctrans=False, annotatorpolicy=None, nowrap=False): try: func = func.im_func except AttributeError: pass t = TranslationContext() if graph is not None: graph.func = func ann = t.buildannotator(policy=annotatorpolicy) inputcells = [ann.typeannotation(a) for a in annotation] ann.build_graph_types(graph, inputcells) t.graphs.insert(0, graph) else: ann = t.buildannotator(policy=annotatorpolicy) ann.build_types(func, annotation) if getoption('view'): t.view() t.buildrtyper(type_system="ootype").specialize() if backendopt: check_virtual_methods(ootype.ROOT) backend_optimizations(t) main_graph = t.graphs[0] if getoption('view'): t.view() return _build_gen_from_graph(main_graph, t, exctrans, nowrap)
def __init__(self, function, annotations, stackless=False, view=False, html=None, is_interactive=False, root = None, run_browser = True, policy = None): if not use_browsertest and not _CLI_is_on_path(): py.test.skip('Javascript CLI (js) not found') self.html = html self.is_interactive = is_interactive t = TranslationContext() if policy is None: from pypy.annotation.policy import AnnotatorPolicy policy = AnnotatorPolicy() policy.allow_someobjects = False ann = t.buildannotator(policy=policy) ann.build_types(function, annotations) if view or option.view: t.view() t.buildrtyper(type_system="ootype").specialize() if view or option.view: t.view() #self.js = JS(t, [function, callback_function], stackless) self.js = JS(t, function, stackless) self.js.write_source() if root is None and use_tg: from pypy.translator.js.demo.jsdemo.controllers import Root self.root = Root else: self.root = root self.run_browser = run_browser self.function_calls = []
def check(self, fn, signature, args, expected_result, expected_mallocs=0, expected_calls=0): t = TranslationContext() self.translator = t t.buildannotator().build_types(fn, signature) t.buildrtyper(type_system=self.type_system).specialize() graph = graphof(t, fn) if option.view: t.view() self.original_graph_count = len(t.graphs) # to detect broken intermediate graphs, # we do the loop ourselves instead of calling remove_simple_mallocs() maxiter = 100 mallocv = MallocVirtualizer(t.graphs, t.rtyper, verbose=True) while True: progress = mallocv.remove_mallocs_once() #simplify.transform_dead_op_vars_in_blocks(list(graph.iterblocks())) if progress and option.view: t.view() t.checkgraphs() if expected_result is not DONT_CHECK_RESULT: interp = LLInterpreter(t.rtyper) if not isinstance(expected_result, CHECK_RAISES): res = interp.eval_graph(graph, args) assert res == expected_result else: excinfo = py.test.raises(LLException, interp.eval_graph, graph, args) assert expected_result.excname in str(excinfo.value) if not progress: break maxiter -= 1 assert maxiter > 0, "infinite loop?" self.check_malloc_removed(graph, expected_mallocs, expected_calls) return graph
def _build_gen(func, annotation, graph=None, backendopt=True): try: func = func.im_func except AttributeError: pass t = TranslationContext() if graph is not None: graph.func = func ann = t.buildannotator() inputcells = [ann.typeannotation(a) for a in annotation] ann.build_graph_types(graph, inputcells) t.graphs.insert(0, graph) else: ann = t.buildannotator() ann.build_types(func, annotation) if getoption('view'): t.view() t.buildrtyper(type_system="ootype").specialize() if backendopt: check_virtual_methods(ootype.ROOT) backend_optimizations(t) main_graph = t.graphs[0] if getoption('view'): t.view() if getoption('wd'): tmpdir = py.path.local('.') else: tmpdir = udir return GenCli(tmpdir, t, TestEntryPoint(main_graph, True))
def hannotate(self, func, argtypes, policy=P_DEFAULT, annotator=False, inline=None, backendoptimize=False): # build the normal ll graphs for ll_function t = TranslationContext() a = t.buildannotator() a.build_types(func, argtypes) rtyper = t.buildrtyper(type_system = self.type_system) rtyper.specialize() if inline: auto_inlining(t, threshold=inline) if backendoptimize: from pypy.translator.backendopt.all import backend_optimizations backend_optimizations(t) graph1 = graphof(t, func) # build hint annotator types policy = self.fixpolicy(policy) hannotator = HintAnnotator(base_translator=t, policy=policy) hs = hannotator.build_types(graph1, [SomeLLAbstractConstant(v.concretetype, {OriginFlags(): True}) for v in graph1.getargs()]) hannotator.simplify() t = hannotator.translator if conftest.option.view: t.view() if annotator: return hs, hannotator else: return hs
def translate(self, func, sig): t = TranslationContext() t.buildannotator().build_types(func, sig) t.buildrtyper(type_system=self.type_system).specialize() if option.view: t.view() return t, RaiseAnalyzer(t)
def rcompile(rgenop, entrypoint, argtypes, random_seed=0, type_system='lltype'): from pypy.translator.translator import TranslationContext from pypy.annotation.policy import AnnotatorPolicy from pypy import conftest t = TranslationContext() policy = AnnotatorPolicy() policy.allow_someobjects = False t.buildannotator(policy=policy).build_types(entrypoint, argtypes) t.buildrtyper(type_system=type_system).specialize() # note that backend optimizations will constant-fold simple operations, # which is required by some backends that don't accept calls like # genop1("add", constant, constant). from pypy.translator.backendopt.all import backend_optimizations backend_optimizations(t) if conftest.option.view: t.view() entrygraph = t._graphof(entrypoint) return compile_graph(rgenop, entrygraph, random_seed=random_seed)
def compile_func(fn, inputtypes, t=None, gcpolicy="ref"): from pypy.config.pypyoption import get_pypy_config config = get_pypy_config(translating=True) config.translation.gc = gcpolicy config.translation.countmallocs = True if t is None: t = TranslationContext(config=config) if inputtypes is not None: t.buildannotator().build_types(fn, inputtypes) t.buildrtyper().specialize() builder = genc.CExtModuleBuilder(t, fn, config=config) builder.generate_source() builder.compile() if conftest.option.view: t.view() compiled_fn = builder.get_entry_point() malloc_counters = builder.get_malloc_counters() def checking_fn(*args, **kwds): try: return compiled_fn(*args, **kwds) finally: mallocs, frees = malloc_counters() assert mallocs == frees return checking_fn
def test_annotate_byval(self): t = TranslationContext() a = t.buildannotator() s = a.build_types(test_testfunc_byval, []) if conftest.option.view: t.view() assert s.knowntype == int
def gengraph(func, argtypes=[], viewbefore='auto', policy=None, type_system="lltype", backendopt=False, config=None, **extraconfigopts): t = TranslationContext(config=config) t.config.set(**extraconfigopts) a = t.buildannotator(policy=policy) timelog("annotating", a.build_types, func, argtypes, main_entry_point=True) if viewbefore == 'auto': viewbefore = getattr(conftest.option, 'view', False) if viewbefore: a.simplify() t.view() global typer # we need it for find_exception typer = t.buildrtyper(type_system=type_system) timelog("rtyper-specializing", typer.specialize) #t.view() timelog("checking graphs", t.checkgraphs) if backendopt: from pypy.translator.backendopt.all import backend_optimizations backend_optimizations(t) timelog("checking graphs", t.checkgraphs) if viewbefore: t.view() desc = t.annotator.bookkeeper.getdesc(func) graph = desc.specialize(argtypes) return t, typer, graph
def check(self, fn, signature, args, expected_result, must_be_removed=True): remover = self.MallocRemover() t = TranslationContext() t.buildannotator().build_types(fn, signature) t.buildrtyper(type_system=self.type_system).specialize() graph = graphof(t, fn) if option.view: t.view() # to detect missing keepalives and broken intermediate graphs, # we do the loop ourselves instead of calling remove_simple_mallocs() while True: progress = remover.remove_mallocs_once(graph) simplify.transform_dead_op_vars_in_blocks(list(graph.iterblocks())) if progress and option.view: t.view() if expected_result is not Ellipsis: interp = LLInterpreter(t.rtyper) res = interp.eval_graph(graph, args) assert res == expected_result if not progress: break if must_be_removed: self.check_malloc_removed(graph) return graph
def make_deallocator(TYPE, attr="static_deallocation_funcptr_for_type", cls=RefcountingGCTransformer): if TYPE._is_varsize(): def f(): return lltype.malloc(TYPE, 1) else: def f(): return lltype.malloc(TYPE) t = TranslationContext() t.buildannotator().build_types(f, []) t.buildrtyper().specialize() transformer = cls(t) fptr = getattr(transformer, attr)(TYPE) transformer.transform_graph(graphof(t, f)) transformer.finish(backendopt=False) if conftest.option.view: t.view() if fptr: return fptr._obj.graph, t else: return None, t
def test_annotate_array_access_float(self): t = TranslationContext() a = t.buildannotator() s = a.build_types(access_array, [float]) assert s.knowntype == float if conftest.option.view: t.view()
def get_graph(fn, signature): t = TranslationContext() t.buildannotator().build_types(fn, signature) t.buildrtyper().specialize() graph = graphof(t, fn) if conftest.option.view: t.view() return graph, t
def rtype(func, inputtypes, specialize=True): t = TranslationContext() t.buildannotator().build_types(func, inputtypes) if specialize: t.buildrtyper().specialize() if conftest.option.view: t.view() return t
def rtype(fn, signature): t = TranslationContext() t.buildannotator().build_types(fn, signature) t.buildrtyper().specialize() graph = graphof(t, fn) if option.view: t.view() return t, graph
def translate(func, argtypes, backend_optimize=True): t = TranslationContext() t.buildannotator().build_types(func, argtypes) t.buildrtyper().specialize() if backend_optimize: backend_optimizations(t) if conftest.option.view: t.view() return graphof(t, func), t
def build_generator(func, argtypes=[]): context = TranslationContext() context.buildannotator().build_types(func, argtypes) context.buildrtyper(type_system="ootype").specialize() if conftest.option.view: context.view() gen = GenCL(context, func) return gen
def specialize_view(self, f, args=[], opt=False): t = TranslationContext() a = t.buildannotator() a = a.build_types(f, args) r = t.buildrtyper() r.specialize() if opt: from pypy.translator.backendopt.all import backend_optimizations backend_optimizations(t) t.view()
def gengraph(f, args=[], viewBefore=False, viewAfter=False, mangle=True): t = TranslationContext() t.config.translation.ootype.mangle = mangle t.buildannotator().build_types(f, args) if viewBefore or conftest.option.view: t.view() t.buildrtyper(type_system="ootype").specialize() if viewAfter or conftest.option.view: t.view() return graphof(t, f)
def translateopt(self, func, sig, **optflags): t = TranslationContext() t.buildannotator().build_types(func, sig) t.buildrtyper(type_system=self.type_system).specialize() if conftest.option.view: t.view() backend_optimizations(t, **optflags) if conftest.option.view: t.view() return t
def _makefunc_str_int(cls, func): def main(argv): arg0 = argv[1] arg1 = int(argv[2]) try: res = func(arg0, arg1) except MemoryError: print 'Result: MemoryError' else: print 'Result: "%s"' % (res,) return 0 from pypy.config.pypyoption import get_pypy_config config = get_pypy_config(translating=True) config.translation.gc = cls.gcpolicy config.translation.gcrootfinder = "asmgcc" if sys.platform == 'win32': config.translation.cc = 'mingw32' t = TranslationContext(config=config) a = t.buildannotator() a.build_types(main, [s_list_of_strings]) t.buildrtyper().specialize() t.checkgraphs() cbuilder = CStandaloneBuilder(t, main, config=config) c_source_filename = cbuilder.generate_source( defines = cbuilder.DEBUG_DEFINES) cls._patch_makefile(cbuilder.targetdir) if conftest.option.view: t.view() exe_name = cbuilder.compile() def run(arg0, arg1): lines = [] print >> sys.stderr, 'RUN: starting', exe_name if sys.platform == 'win32': redirect = ' 2> NUL' else: redirect = '' g = os.popen('"%s" %s %d%s' % (exe_name, arg0, arg1, redirect), 'r') for line in g: print >> sys.stderr, 'RUN:', line.rstrip() lines.append(line) g.close() if not lines: py.test.fail("no output from subprocess") if not lines[-1].startswith('Result:'): py.test.fail("unexpected output from subprocess") result = lines[-1][len('Result:'):].strip() if result == 'MemoryError': raise MemoryError("subprocess got an RPython MemoryError") if result.startswith('"') and result.endswith('"'): return result[1:-1] else: return int(result) return run
def rtype(func, inputtypes, specialize=True, gcname='ref'): from pypy.translator.translator import TranslationContext t = TranslationContext() # XXX XXX XXX mess t.config.translation.gc = gcname t.buildannotator().build_types(func, inputtypes) if specialize: t.buildrtyper().specialize() if conftest.option.view: t.view() return t
def test_annotate_prebuilt(self): p = c_char_p("hello") def func(): return p.value t = TranslationContext() a = t.buildannotator() s = a.build_types(func, []) assert s.knowntype == str if conftest.option.view: t.view()
def build_adi(function, types): t = TranslationContext() t.buildannotator().build_types(function, types) t.buildrtyper().specialize() if conftest.option.view: t.view() adi = AbstractDataFlowInterpreter(t) graph = graphof(t, function) adi.schedule_function(graph) adi.complete() return t, adi, graph
def get_graph(fn, signature, all_opts=True): t = TranslationContext() t.buildannotator().build_types(fn, signature) t.buildrtyper().specialize() if all_opts: backend_optimizations(t, inline_threshold=INLINE_THRESHOLD_FOR_TEST, constfold=False, raisingop2direct_call=False) graph = graphof(t, fn) if conftest.option.view: t.view() return graph, t
def test_annotate_primitive_ptritem(self): def func(x): cs = pointer(c_short(x)) return cs[0] t = TranslationContext() a = t.buildannotator() s = a.build_types(func, [int]) assert s.knowntype == int if conftest.option.view: t.view()
def test_annotate_array_access_bytype(self): def access_array_bytype(dummy): my_array = numpy.array([1], 'd') return my_array[0] t = TranslationContext() a = t.buildannotator() s = a.build_types(access_array_bytype, [int]) assert s.knowntype == float if conftest.option.view: t.view()
def test_annotate_array_access_bytype(self): def access_array_bytype(dummy): my_array = numpy.array([1],'f') return my_array[0] t = TranslationContext() a = t.buildannotator() s = a.build_types(access_array_bytype, [int]) assert s.knowntype == float if conftest.option.view: t.view()
def test_annotate_pointer_fn(self): def func(): p = pointer(c_int(123)) return p.contents.value t = TranslationContext() a = t.buildannotator() s = a.build_types(func, []) if conftest.option.view: t.view() assert s.knowntype == int
def test_annotate_primitive_arrayitem(self): CSA = c_short * 1 def func(x): csa = CSA(x) return csa[0] t = TranslationContext() a = t.buildannotator() s = a.build_types(func, [int]) assert s.knowntype == int if conftest.option.view: t.view()
def test_annotate_set_primitive_value(self): def func(x): cs = c_short() cs.value = x return cs.value t = TranslationContext() a = t.buildannotator() s = a.build_types(func, [int]) assert s.knowntype == int if conftest.option.view: t.view()
def transform_func(fn, inputtypes): t = TranslationContext() t.buildannotator().build_types(fn, inputtypes) t.buildrtyper().specialize() if conftest.option.view: t.view() g = graphof(t, fn) etrafo = exceptiontransform.ExceptionTransformer(t) etrafo.create_exception_handling(g) join_blocks(g) if conftest.option.view: t.view() return t, g
def specialize(self, func, argtypes): from pypy.rpython.llinterp import LLInterpreter t = TranslationContext(list_comprehension_operations=True) t.buildannotator().build_types(func, argtypes) if conftest.option.view: t.view() t.buildrtyper(self.typesystem).specialize() backend_optimizations(t) if conftest.option.view: t.view() graph = graphof(t, func) interp = LLInterpreter(t.rtyper) return interp, graph
def getcompiled(self, func): def main(argv): try: res = func() except MemoryError: print 'Result: MemoryError' else: if isinstance(res, int): print 'Result:', res else: print 'Result: "%s"' % (res,) return 0 from pypy.config.pypyoption import get_pypy_config config = get_pypy_config(translating=True) config.translation.gc = self.gcpolicy config.translation.gcrootfinder = "asmgcc" t = TranslationContext(config=config) self.t = t a = t.buildannotator() a.build_types(main, [s_list_of_strings]) t.buildrtyper().specialize() t.checkgraphs() cbuilder = CStandaloneBuilder(t, main, config=config) c_source_filename = cbuilder.generate_source( defines = cbuilder.DEBUG_DEFINES) self.patch_makefile(cbuilder.targetdir) if conftest.option.view: t.view() exe_name = cbuilder.compile() def run(): lines = [] print >> sys.stderr, 'RUN: starting', exe_name g = os.popen("'%s'" % (exe_name,), 'r') for line in g: print >> sys.stderr, 'RUN:', line.rstrip() lines.append(line) g.close() if not lines: py.test.fail("no output from subprocess") if not lines[-1].startswith('Result:'): py.test.fail("unexpected output from subprocess") result = lines[-1][len('Result:'):].strip() if result == 'MemoryError': raise MemoryError("subprocess got an RPython MemoryError") if result.startswith('"') and result.endswith('"'): return result[1:-1] else: return int(result) return run
def compile(self, entry_point, debug=True): t = TranslationContext(self.config) t.buildannotator().build_types(entry_point, [s_list_of_strings]) t.buildrtyper().specialize() cbuilder = CStandaloneBuilder(t, entry_point, t.config) if debug: cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES) else: cbuilder.generate_source() cbuilder.compile() if option.view: t.view() return t, cbuilder
def test_is_exception_instance(): def f(): return NameError() t = TranslationContext() t.buildannotator().build_types(f, []) if conftest.option.view: t.view() rtyper = t.buildrtyper(type_system="ootype") rtyper.specialize() graph = graphof(t, f) INST = graph.getreturnvar().concretetype assert rtyper.exceptiondata.is_exception_instance(INST)
def test_simple(): t = TranslationContext() a = t.buildannotator() a.build_types(g, [int]) a.simplify() t.buildrtyper().specialize() backend_optimizations(t) t.checkgraphs() n = insert_ll_stackcheck(t) t.checkgraphs() assert n == 1 if conftest.option.view: t.view() check(graphof(t, f), 'f')
def analyze(self, func, sig, func_to_analyze=None, backendopt=False): if func_to_analyze is None: func_to_analyze = func t = TranslationContext() t.buildannotator().build_types(func, sig) t.buildrtyper(type_system=self.type_system).specialize() if backendopt: backend_optimizations(t) if option.view: t.view() a = FinalizerAnalyzer(t) fgraph = graphof(t, func_to_analyze) result = a.analyze_light_finalizer(fgraph) return result
def transform_func(self, fn, inputtypes, backendopt=False): t = TranslationContext() t.buildannotator().build_types(fn, inputtypes) t.buildrtyper(type_system=self.type_system).specialize() if conftest.option.view: t.view() if backendopt: backend_optimizations(t) g = graphof(t, fn) etrafo = exceptiontransform.ExceptionTransformer(t) etrafo.create_exception_handling(g) join_blocks(g) if conftest.option.view: t.view() return t, g
def test_simple_loop(): def snippet_fn(x, y): while y > 0: y -= x return y t = TranslationContext() graph = t.buildflowgraph(snippet_fn) if option.view: t.view() loops = find_inner_loops(graph) assert len(loops) == 1 loop = loops[0] assert loop.headblock.operations[0].opname == 'gt' assert len(loop.links) == 2 assert loop.links[0] in loop.headblock.exits assert loop.links[1] in loop.links[0].target.exits assert loop.links[1].target is loop.headblock
def check_malloc_to_coalloc(function, types, args, expected_result, must_remove=-1): t = TranslationContext() t.buildannotator().build_types(function, types) t.buildrtyper().specialize() interp = LLInterpreter(t.rtyper) graph = graphof(t, function) res = interp.eval_graph(graph, args) assert res == expected_result num = malloc_to_coalloc(t) if must_remove == -1: for block in graph.iterblocks(): for op in block.operations: assert op.opname != "malloc" else: assert num == must_remove if conftest.option.view: t.view() res = interp.eval_graph(graph, args) assert res == expected_result return t