def compile(self, entry_point, debug=True, shared=False, stackcheck=False, entrypoints=None): t = TranslationContext(self.config) ann = t.buildannotator() ann.build_types(entry_point, [s_list_of_strings]) if entrypoints is not None: anns = {} for func, annotation in secondary_entrypoints['test']: anns[func] = annotation for item in entrypoints: ann.build_types(item, anns[item]) t.buildrtyper().specialize() if stackcheck: from rpython.translator.transform import insert_ll_stackcheck insert_ll_stackcheck(t) t.config.translation.shared = shared if entrypoints is not None: kwds = {'secondary_entrypoints': [(i, None) for i in entrypoints]} else: kwds = {} cbuilder = CStandaloneBuilder(t, entry_point, t.config, **kwds) if debug: cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES) else: cbuilder.generate_source() cbuilder.compile() if option is not None and option.view: t.view() return t, cbuilder
def translate(self, func, sig): t = TranslationContext() t.buildannotator().build_types(func, sig) t.buildrtyper().specialize() if option.view: t.view() return t, self.Analyzer(t)
def check(self, fn, signature, args, expected_result, must_be_removed=True, inline=None): remover = self.MallocRemover() t = TranslationContext() t.buildannotator().build_types(fn, signature) t.buildrtyper().specialize() graph = graphof(t, fn) if inline is not None: from rpython.translator.backendopt.inline import auto_inline_graphs auto_inline_graphs(t, t.graphs, inline) if option.view: t.view() # to detect broken intermediate graphs, # we do the loop ourselves instead of calling remove_simple_mallocs() while True: progress = remover.remove_mallocs_once(graph) simplify.transform_dead_op_vars_in_blocks(list(graph.iterblocks()), [graph]) if progress and option.view: t.view() if expected_result is not Ellipsis: interp = LLInterpreter(t.rtyper) res = interp.eval_graph(graph, args) assert res == expected_result if not progress: break if must_be_removed: self.check_malloc_removed(graph) return graph
def _build_gen(func, annotation, graph=None, backendopt=True, exctrans=False, annotatorpolicy=None, nowrap=False): try: func = func.im_func except AttributeError: pass t = TranslationContext() if graph is not None: graph.func = func ann = t.buildannotator(policy=annotatorpolicy) inputcells = [ann.typeannotation(a) for a in annotation] ann.build_graph_types(graph, inputcells) t.graphs.insert(0, graph) else: ann = t.buildannotator(policy=annotatorpolicy) ann.build_types(func, annotation) if getoption('view'): t.view() t.buildrtyper(type_system="ootype").specialize() if backendopt: check_virtual_methods(ootype.ROOT) backend_optimizations(t) main_graph = t.graphs[0] if getoption('view'): t.view() return _build_gen_from_graph(main_graph, t, exctrans, nowrap)
def generate_source_for_function(func, annotation, backendopt=False): """ Given a Python function and some hints about its argument types, generates JVM sources that call it and print the result. Returns the JvmGeneratedSource object. """ if hasattr(func, "im_func"): func = func.im_func t = TranslationContext() ann = t.buildannotator() ann.build_types(func, annotation) t.buildrtyper(type_system="ootype").specialize() if backendopt: check_virtual_methods(ootype.ROOT) backend_optimizations(t) main_graph = t.graphs[0] if getoption("view"): t.view() if getoption("wd"): tmpdir = py.path.local(".") else: tmpdir = udir jvm = GenJvm(tmpdir, t, EntryPoint(main_graph, True, True)) return jvm.generate_source()
def gengraph(func, argtypes=[], viewbefore='auto', policy=None, backendopt=False, config=None, **extraconfigopts): t = TranslationContext(config=config) t.config.set(**extraconfigopts) a = t.buildannotator(policy=policy) a.build_types(func, argtypes, main_entry_point=True) a.validate() if viewbefore == 'auto': viewbefore = getattr(option, 'view', False) if viewbefore: a.simplify() t.view() global typer # we need it for find_exception typer = t.buildrtyper() typer.backend = llinterp_backend typer.specialize() #t.view() t.checkgraphs() if backendopt: from rpython.translator.backendopt.all import backend_optimizations backend_optimizations(t) t.checkgraphs() if viewbefore: t.view() desc = t.annotator.bookkeeper.getdesc(func) graph = desc.specialize(argtypes) return t, typer, graph
def make_deallocator(TYPE, attr="static_deallocation_funcptr_for_type", cls=RefcountingGCTransformer): if TYPE._is_varsize(): def f(): return lltype.malloc(TYPE, 1) else: def f(): return lltype.malloc(TYPE) t = TranslationContext() t.buildannotator().build_types(f, []) t.buildrtyper().specialize() transformer = cls(t) fptr = getattr(transformer, attr)(TYPE) transformer.transform_graph(graphof(t, f)) transformer.finish(backendopt=False) if option.view: t.view() if fptr: return fptr._obj.graph, t else: return None, t
def check(self, fn, signature, args, expected_result, expected_mallocs=0, expected_calls=0): t = TranslationContext() self.translator = t t.buildannotator().build_types(fn, signature) t.buildrtyper().specialize() graph = graphof(t, fn) if option.view: t.view() self.original_graph_count = len(t.graphs) # to detect broken intermediate graphs, # we do the loop ourselves instead of calling remove_simple_mallocs() maxiter = 100 mallocv = MallocVirtualizer(t.graphs, t.rtyper, verbose=True) while True: progress = mallocv.remove_mallocs_once() if progress and option.view: t.view() t.checkgraphs() if expected_result is not DONT_CHECK_RESULT: interp = LLInterpreter(t.rtyper) if not isinstance(expected_result, CHECK_RAISES): res = interp.eval_graph(graph, args) assert res == expected_result else: excinfo = py.test.raises(LLException, interp.eval_graph, graph, args) assert expected_result.excname in str(excinfo.value) if not progress: break maxiter -= 1 assert maxiter > 0, "infinite loop?" self.check_malloc_removed(graph, expected_mallocs, expected_calls) return graph
def rtype(func, inputtypes, specialize=True): t = TranslationContext() t.buildannotator().build_types(func, inputtypes) if specialize: t.buildrtyper().specialize() if option.view: t.view() return t
def rtype(fn, signature): t = TranslationContext() t.buildannotator().build_types(fn, signature) t.buildrtyper().specialize() graph = graphof(t, fn) if option.view: t.view() return t, graph
def translate(func, argtypes, backend_optimize=True): t = TranslationContext() t.buildannotator().build_types(func, argtypes) t.buildrtyper().specialize() if backend_optimize: backend_optimizations(t) if option.view: t.view() return graphof(t, func), t
def gengraph(f, args=[], viewBefore=False, viewAfter=False, mangle=True): t = TranslationContext() t.config.translation.ootype.mangle = mangle t.buildannotator().build_types(f, args) if viewBefore or option.view: t.view() t.buildrtyper(type_system="ootype").specialize() if viewAfter or option.view: t.view() return graphof(t, f)
def rtype(func, inputtypes, specialize=True): t = TranslationContext() t.buildannotator().build_types(func, inputtypes) rtyper = t.buildrtyper() rtyper.backend = llinterp_backend if specialize: rtyper.specialize() if option.view: t.view() return t
def get_graph(fn, signature, all_opts=True): t = TranslationContext() t.buildannotator().build_types(fn, signature) t.buildrtyper().specialize() if all_opts: backend_optimizations(t, inline_threshold=INLINE_THRESHOLD_FOR_TEST, constfold=False) graph = graphof(t, fn) if option.view: t.view() return graph, t
def translateopt(self, func, sig, **optflags): t = TranslationContext() opts = {'translation.list_comprehension_operations': True} t.config.set(**opts) t.buildannotator().build_types(func, sig) t.buildrtyper().specialize() if option.view: t.view() backend_optimizations(t, **optflags) if option.view: t.view() return t
def specialize(self, func, argtypes): from rpython.rtyper.llinterp import LLInterpreter t = TranslationContext(list_comprehension_operations=True) t.buildannotator().build_types(func, argtypes) if option.view: t.view() t.buildrtyper().specialize() backend_optimizations(t) if option.view: t.view() graph = graphof(t, func) interp = LLInterpreter(t.rtyper) return interp, graph
def test_simple(): t = TranslationContext() a = t.buildannotator() a.build_types(g, [int]) a.simplify() t.buildrtyper().specialize() backend_optimizations(t) t.checkgraphs() n = insert_ll_stackcheck(t) t.checkgraphs() assert n == 1 if option.view: t.view() check(graphof(t, f), 'f')
def analyze(self, func, sig, func_to_analyze=None, backendopt=False): if func_to_analyze is None: func_to_analyze = func t = TranslationContext() t.buildannotator().build_types(func, sig) t.buildrtyper().specialize() if backendopt: backend_optimizations(t) if option.view: t.view() a = FinalizerAnalyzer(t) fgraph = graphof(t, func_to_analyze) result = a.analyze_light_finalizer(fgraph) return result
def test_is_exception_instance(): def f(): return NameError() t = TranslationContext() t.buildannotator().build_types(f, []) if option.view: t.view() rtyper = t.buildrtyper(type_system="ootype") rtyper.specialize() graph = graphof(t, f) INST = graph.getreturnvar().concretetype assert rtyper.exceptiondata.is_exception_instance(INST)
def test_simple(): t = TranslationContext() a = t.buildannotator() a.build_types(g, [int]) a.simplify() t.buildrtyper().specialize() backend_optimizations(t) t.checkgraphs() n = insert_ll_stackcheck(t) t.checkgraphs() assert n == 1 if option.view: t.view() check(graphof(t, f), "f")
def transform_func(self, fn, inputtypes, backendopt=False): t = TranslationContext() t.buildannotator().build_types(fn, inputtypes) t.buildrtyper().specialize() if option.view: t.view() if backendopt: backend_optimizations(t) g = graphof(t, fn) etrafo = exceptiontransform.ExceptionTransformer(t) etrafo.create_exception_handling(g) join_blocks(g) if option.view: t.view() return t, g
def test_simple_loop(): def snippet_fn(x, y): while y > 0: y -= x return y t = TranslationContext() graph = t.buildflowgraph(snippet_fn) if option.view: t.view() loops = find_inner_loops(graph) assert len(loops) == 1 loop = loops[0] assert loop.headblock.operations[0].opname == 'gt' assert len(loop.links) == 2 assert loop.links[0] in loop.headblock.exits assert loop.links[1] in loop.links[0].target.exits assert loop.links[1].target is loop.headblock
def test_two_loops(): def snippet_fn(x, y): while y > 0: y -= x while y < 0: y += x return y t = TranslationContext() graph = t.buildflowgraph(snippet_fn) if option.view: t.view() loops = find_inner_loops(graph) assert len(loops) == 2 assert loops[0].headblock is not loops[1].headblock for loop in loops: assert loop.headblock.operations[0].opname in ('gt', 'lt') assert len(loop.links) == 2 assert loop.links[0] in loop.headblock.exits assert loop.links[1] in loop.links[0].target.exits assert loop.links[1].target is loop.headblock
def test_nested_loops(): def snippet_fn(x, z): y = 0 while y <= 10: while z < y: z += y y += 1 return z t = TranslationContext() graph = t.buildflowgraph(snippet_fn) if option.view: t.view() loops = find_inner_loops(graph) assert len(loops) == 1 loop = loops[0] assert loop.headblock.operations[0].opname == 'lt' assert len(loop.links) == 2 assert loop.links[0] in loop.headblock.exits assert loop.links[1] in loop.links[0].target.exits assert loop.links[1].target is loop.headblock
def generate_source_for_function(func, annotation, backendopt=False): """ Given a Python function and some hints about its argument types, generates JVM sources that call it and print the result. Returns the JvmGeneratedSource object. """ if hasattr(func, 'im_func'): func = func.im_func t = TranslationContext() ann = t.buildannotator() ann.build_types(func, annotation) t.buildrtyper(type_system="ootype").specialize() if backendopt: check_virtual_methods(ootype.ROOT) backend_optimizations(t) main_graph = t.graphs[0] if getoption('view'): t.view() if getoption('wd'): tmpdir = py.path.local('.') else: tmpdir = udir jvm = GenJvm(tmpdir, t, EntryPoint(main_graph, True, True)) return jvm.generate_source()
def _makefunc_str_int(cls, func): def main(argv): arg0 = argv[1] arg1 = int(argv[2]) try: res = func(arg0, arg1) except MemoryError: print 'Result: MemoryError' else: print 'Result: "%s"' % (res,) return 0 config = cls.make_config() t = TranslationContext(config=config) a = t.buildannotator() sec_ep = getattr(cls, 'secondary_entrypoints', []) for f, inputtypes in sec_ep: a.build_types(f, inputtypes, False) a.build_types(main, [s_list_of_strings]) t.buildrtyper().specialize() t.checkgraphs() cbuilder = CStandaloneBuilder(t, main, config=config, secondary_entrypoints=sec_ep) c_source_filename = cbuilder.generate_source( defines = cbuilder.DEBUG_DEFINES) cls._patch_makefile(cbuilder.targetdir) if option.view: t.view() exe_name = cbuilder.compile() def run(arg0, arg1, runner=None): if runner is not None: py.test.skip("unsupported test: runner=%r" % (runner,)) lines = [] print >> sys.stderr, 'RUN: starting', exe_name, arg0, arg1 if sys.platform == 'win32': redirect = ' 2> NUL' else: redirect = '' if config.translation.shared and os.name == 'posix': library_path = exe_name.dirpath() if sys.platform == 'darwin': env = 'DYLD_LIBRARY_PATH="%s" ' % library_path else: env = 'LD_LIBRARY_PATH="%s" ' % library_path else: env = '' cwd = os.getcwd() try: os.chdir(str(exe_name.dirpath())) g = os.popen( '%s"%s" %s %d%s' % (env, exe_name, arg0, arg1, redirect), 'r') finally: os.chdir(cwd) for line in g: print >> sys.stderr, 'RUN:', line.rstrip() lines.append(line) g.close() if not lines: py.test.fail("no output from subprocess") if not lines[-1].startswith('Result:'): py.test.fail("unexpected output from subprocess") result = lines[-1][len('Result:'):].strip() if result == 'MemoryError': raise MemoryError("subprocess got an RPython MemoryError") if result.startswith('"') and result.endswith('"'): return result[1:-1] else: return int(result) return run
class Translation(object): def __init__(self, entry_point, argtypes=None, **kwds): self.driver = driver.TranslationDriver(overrides=DEFAULTS) self.config = self.driver.config self.entry_point = export_symbol(entry_point) self.context = TranslationContext(config=self.config) policy = kwds.pop('policy', None) self.update_options(kwds) self.ensure_setup(argtypes, policy) # for t.view() to work just after construction graph = self.context.buildflowgraph(entry_point) self.context._prebuilt_graphs[entry_point] = graph def view(self): self.context.view() def viewcg(self): self.context.viewcg() def ensure_setup(self, argtypes=None, policy=None): standalone = argtypes is None if standalone: assert argtypes is None else: if argtypes is None: argtypes = [] self.driver.setup(self.entry_point, argtypes, policy, empty_translator=self.context) self.ann_argtypes = argtypes self.ann_policy = policy def update_options(self, kwds): gc = kwds.pop('gc', None) if gc: self.config.translation.gc = gc self.config.translation.set(**kwds) def ensure_opt(self, name, value=None, fallback=None): if value is not None: self.update_options({name: value}) return value val = getattr(self.config.translation, name, None) if fallback is not None and val is None: self.update_options({name: fallback}) return fallback if val is not None: return val raise Exception( "the %r option should have been specified at this point" % name) def ensure_type_system(self, type_system=None): if self.config.translation.backend is not None: return self.ensure_opt('type_system') return self.ensure_opt('type_system', type_system, 'lltype') def ensure_backend(self, backend=None): backend = self.ensure_opt('backend', backend) self.ensure_type_system() return backend # disable some goals (steps) def disable(self, to_disable): self.driver.disable(to_disable) def set_backend_extra_options(self, **extra_options): for name in extra_options: backend, option = name.split('_', 1) self.ensure_backend(backend) self.driver.set_backend_extra_options(extra_options) # backend independent def annotate(self, **kwds): self.update_options(kwds) return self.driver.annotate() # type system dependent def rtype(self, **kwds): self.update_options(kwds) ts = self.ensure_type_system() return getattr(self.driver, 'rtype_' + ts)() def backendopt(self, **kwds): self.update_options(kwds) ts = self.ensure_type_system('lltype') return getattr(self.driver, 'backendopt_' + ts)() # backend depedent def source(self, **kwds): self.update_options(kwds) backend = self.ensure_backend() getattr(self.driver, 'source_' + backend)() def source_c(self, **kwds): self.update_options(kwds) self.ensure_backend('c') self.driver.source_c() def source_cl(self, **kwds): self.update_options(kwds) self.ensure_backend('cl') self.driver.source_cl() def compile(self, **kwds): self.update_options(kwds) backend = self.ensure_backend() getattr(self.driver, 'compile_' + backend)() return self.driver.c_entryp def compile_c(self, **kwds): self.update_options(kwds) self.ensure_backend('c') self.driver.compile_c() return self.driver.c_entryp
def _makefunc_str_int(cls, func): def main(argv): arg0 = argv[1] arg1 = int(argv[2]) try: res = func(arg0, arg1) except MemoryError: print 'Result: MemoryError' else: print 'Result: "%s"' % (res,) return 0 config = cls.make_config() t = TranslationContext(config=config) a = t.buildannotator() sec_ep = getattr(cls, 'secondary_entrypoints', []) for f, inputtypes in sec_ep: a.build_types(f, inputtypes, False) a.build_types(main, [s_list_of_strings]) t.buildrtyper().specialize() t.checkgraphs() cbuilder = CStandaloneBuilder(t, main, config=config, secondary_entrypoints=sec_ep) c_source_filename = cbuilder.generate_source( defines = cbuilder.DEBUG_DEFINES) cls._patch_makefile(cbuilder.targetdir) if option.view: t.view() exe_name = cbuilder.compile() def run(arg0, arg1): lines = [] print >> sys.stderr, 'RUN: starting', exe_name, arg0, arg1 if sys.platform == 'win32': redirect = ' 2> NUL' else: redirect = '' if config.translation.shared and os.name == 'posix': library_path = exe_name.dirpath() if sys.platform == 'darwin': env = 'DYLD_LIBRARY_PATH="%s" ' % library_path else: env = 'LD_LIBRARY_PATH="%s" ' % library_path else: env = '' cwd = os.getcwd() try: os.chdir(str(exe_name.dirpath())) g = os.popen( '%s"%s" %s %d%s' % (env, exe_name, arg0, arg1, redirect), 'r') finally: os.chdir(cwd) for line in g: print >> sys.stderr, 'RUN:', line.rstrip() lines.append(line) g.close() if not lines: py.test.fail("no output from subprocess") if not lines[-1].startswith('Result:'): py.test.fail("unexpected output from subprocess") result = lines[-1][len('Result:'):].strip() if result == 'MemoryError': raise MemoryError("subprocess got an RPython MemoryError") if result.startswith('"') and result.endswith('"'): return result[1:-1] else: return int(result) return run