def test_no_collect_detection(): from rpython.rlib import rgc from rpython.translator.c.genc import CStandaloneBuilder class A(object): def __init__(self, x): self.x = x @rgc.no_collect def g(): return A(1).x assert g._dont_inline_ assert g._gc_no_collect_ def entrypoint(argv): return g() + 2 t = rtype(entrypoint, [s_list_of_strings]) t.config.translation.gc = "minimark" cbuild = CStandaloneBuilder(t, entrypoint, t.config, gcpolicy=FrameworkGcPolicy2) with py.test.raises(Exception) as f: cbuild.build_database() expected = "'no_collect' function can trigger collection: <function g at " assert str(f.value).startswith(expected)
def task_database_c(self): """ Create a database for further backend generation """ translator = self.translator if translator.annotator is not None: translator.frozen = True standalone = self.standalone if standalone: from rpython.translator.c.genc import CStandaloneBuilder cbuilder = CStandaloneBuilder( self.translator, self.entry_point, config=self.config, secondary_entrypoints=self.secondary_entrypoints + annotated_jit_entrypoints) else: from rpython.translator.c.dlltool import CLibraryBuilder functions = [ (self.entry_point, None) ] + self.secondary_entrypoints + annotated_jit_entrypoints cbuilder = CLibraryBuilder(self.translator, self.entry_point, functions=functions, name='libtesting', config=self.config) if not standalone: # xxx more messy cbuilder.modulename = self.extmod_name database = cbuilder.build_database() self.log.info("database for generating C source was created") self.cbuilder = cbuilder self.database = database
def test_remove_duplicate_write_barrier(): from rpython.translator.c.genc import CStandaloneBuilder from rpython.flowspace.model import summary class A(object): pass glob_a_1 = A() glob_a_2 = A() def f(a, cond): a.x = a a.z = a if cond: a.y = a def g(): f(glob_a_1, 5) f(glob_a_2, 0) t = rtype(g, []) t.config.translation.gc = "minimark" cbuild = CStandaloneBuilder(t, g, t.config, gcpolicy=FrameworkGcPolicy2) db = cbuild.generate_graphs_for_llinterp() ff = graphof(t, f) #ff.show() assert summary(ff)['direct_call'] == 1 # only one remember_young_pointer
def test_no_collect_detection(): from rpython.rlib import rgc from rpython.translator.c.genc import CStandaloneBuilder class A(object): def __init__(self, x): self.x = x @rgc.no_collect def g(): return A(1).x assert g._dont_inline_ assert g._gc_no_collect_ def entrypoint(argv): return g() + 2 t = rtype(entrypoint, [s_list_of_strings]) t.config.translation.gc = "minimark" cbuild = CStandaloneBuilder(t, entrypoint, t.config, gcpolicy=FrameworkGcPolicy2) cbuild.make_entrypoint_wrapper = False with py.test.raises(Exception) as f: cbuild.build_database() expected = "'no_collect' function can trigger collection: <function g at " assert str(f.value).startswith(expected)
def test_custom_trace_function_no_collect(): from rpython.rlib import rgc from rpython.translator.c.genc import CStandaloneBuilder S = lltype.GcStruct("MyStructure") class Glob: pass glob = Glob() def trace_func(gc, obj, callback, arg): glob.foo = (gc, obj) lambda_trace_func = lambda: trace_func def entrypoint(argv): lltype.malloc(S) rgc.register_custom_trace_hook(S, lambda_trace_func) return 0 t = rtype(entrypoint, [s_list_of_strings]) t.config.translation.gc = "minimark" cbuild = CStandaloneBuilder(t, entrypoint, t.config, gcpolicy=FrameworkGcPolicy2) with py.test.raises(Exception) as f: cbuild.build_database() assert 'can cause the GC to be called' in str(f.value) assert 'trace_func' in str(f.value) assert 'MyStructure' in str(f.value)
def test_framework_simple(): def g(x): return x + 1 class A(object): pass def entrypoint(argv): a = A() a.b = g(1) return str(a.b) from rpython.rtyper.llinterp import LLInterpreter from rpython.translator.c.genc import CStandaloneBuilder t = rtype(entrypoint, [s_list_of_strings]) t.config.translation.gc = "minimark" cbuild = CStandaloneBuilder(t, entrypoint, t.config, gcpolicy=FrameworkGcPolicy2) db = cbuild.generate_graphs_for_llinterp() entrypointptr = cbuild.getentrypointptr() entrygraph = entrypointptr._obj.graph r_list_of_strings = t.rtyper.getrepr(s_list_of_strings) ll_argv = r_list_of_strings.convert_const([]) llinterp = LLInterpreter(t.rtyper) # FIIIIISH setupgraph = db.gctransformer.frameworkgc_setup_ptr.value._obj.graph llinterp.eval_graph(setupgraph, []) res = llinterp.eval_graph(entrygraph, [ll_argv]) assert ''.join(res.chars) == "2"
def test_custom_trace_function_no_collect(): from rpython.rlib import rgc from rpython.translator.c.genc import CStandaloneBuilder S = lltype.GcStruct("MyStructure") class Glob: pass glob = Glob() def trace_func(gc, obj, callback, arg): glob.foo = (gc, obj) lambda_trace_func = lambda: trace_func def entrypoint(argv): lltype.malloc(S) rgc.register_custom_trace_hook(S, lambda_trace_func) return 0 t = rtype(entrypoint, [s_list_of_strings]) t.config.translation.gc = "minimark" cbuild = CStandaloneBuilder(t, entrypoint, t.config, gcpolicy=FrameworkGcPolicy2) cbuild.make_entrypoint_wrapper = False with py.test.raises(Exception) as f: cbuild.build_database() assert 'can cause the GC to be called' in str(f.value) assert 'trace_func' in str(f.value) assert 'MyStructure' in str(f.value)
def test_remove_duplicate_write_barrier(): from rpython.translator.c.genc import CStandaloneBuilder from rpython.flowspace.model import summary class A(object): pass glob_a_1 = A() glob_a_2 = A() def f(a, cond): a.x = a a.z = a if cond: a.y = a def g(): f(glob_a_1, 5) f(glob_a_2, 0) t = rtype(g, []) t.config.translation.gc = "minimark" cbuild = CStandaloneBuilder(t, g, t.config, gcpolicy=FrameworkGcPolicy2) cbuild.make_entrypoint_wrapper = False db = cbuild.build_database() ff = graphof(t, f) #ff.show() assert summary(ff)['direct_call'] == 1 # only one remember_young_pointer
def test_framework_simple(): def g(x): return x + 1 class A(object): pass def entrypoint(argv): a = A() a.b = g(1) return str(a.b) from rpython.rtyper.llinterp import LLInterpreter from rpython.translator.c.genc import CStandaloneBuilder t = rtype(entrypoint, [s_list_of_strings]) t.config.translation.gc = "minimark" cbuild = CStandaloneBuilder(t, entrypoint, t.config, gcpolicy=FrameworkGcPolicy2) cbuild.make_entrypoint_wrapper = False db = cbuild.build_database() entrypointptr = cbuild.getentrypointptr() entrygraph = entrypointptr._obj.graph r_list_of_strings = t.rtyper.getrepr(s_list_of_strings) ll_argv = r_list_of_strings.convert_const([]) llinterp = LLInterpreter(t.rtyper) # FIIIIISH setupgraph = db.gctransformer.frameworkgc_setup_ptr.value._obj.graph llinterp.eval_graph(setupgraph, []) res = llinterp.eval_graph(entrygraph, [ll_argv]) assert ''.join(res.chars) == "2"
def task_database_c(self): """ Create a database for further backend generation """ translator = self.translator if translator.annotator is not None: translator.frozen = True standalone = self.standalone if standalone: from rpython.translator.c.genc import CStandaloneBuilder cbuilder = CStandaloneBuilder(self.translator, self.entry_point, config=self.config, secondary_entrypoints=self.secondary_entrypoints) else: from rpython.translator.c.dlltool import CLibraryBuilder functions = [(self.entry_point, None)] + self.secondary_entrypoints cbuilder = CLibraryBuilder(self.translator, self.entry_point, functions=functions, name='libtesting', config=self.config) if not standalone: # xxx more messy cbuilder.modulename = self.extmod_name database = cbuilder.build_database() self.log.info("database for generating C source was created") self.cbuilder = cbuilder self.database = database
def run_rtyper(fn): t = rtype(fn, [s_list_of_strings]) t.config.translation.gc = "minimark" cbuild = CStandaloneBuilder(t, fn, t.config, gcpolicy=FrameworkGcPolicy2) cbuild.make_entrypoint_wrapper = False cbuild.build_database() return True
def _compile_and_run(self, t, entry_point, entry_point_graph, args): from rpython.translator.c.genc import CStandaloneBuilder as CBuilder # XXX patch exceptions cbuilder = CBuilder(t, entry_point, config=t.config) cbuilder.generate_source() self._check_cbuilder(cbuilder) exe_name = cbuilder.compile() debug_print('---------- Test starting ----------') stdout = cbuilder.cmdexec(" ".join([str(arg) for arg in args])) res = int(stdout) debug_print('---------- Test done (%d) ----------' % (res, )) return res
def _compile_and_run(self, t, entry_point, entry_point_graph, args): from rpython.translator.c.genc import CStandaloneBuilder as CBuilder # XXX patch exceptions cbuilder = CBuilder(t, entry_point, config=t.config) cbuilder.generate_source() self._check_cbuilder(cbuilder) exe_name = cbuilder.compile() debug_print('---------- Test starting ----------') stdout = cbuilder.cmdexec(" ".join([str(arg) for arg in args])) res = int(stdout) debug_print('---------- Test done (%d) ----------' % (res,)) return res
def compile(self, entry_point): t = TranslationContext(self.config) t.config.translation.gc = "semispace" t.config.translation.gcrootfinder = self.gcrootfinder t.config.translation.thread = True t.buildannotator().build_types(entry_point, [s_list_of_strings]) t.buildrtyper().specialize() # cbuilder = CStandaloneBuilder(t, entry_point, t.config) cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES) cbuilder.compile() # return t, cbuilder
def compile(self, entry_point, debug=True, shared=False, stackcheck=False, entrypoints=None): t = TranslationContext(self.config) ann = t.buildannotator() ann.build_types(entry_point, [s_list_of_strings]) if entrypoints is not None: anns = {} for func, annotation in secondary_entrypoints['test']: anns[func] = annotation for item in entrypoints: ann.build_types(item, anns[item]) t.buildrtyper().specialize() if stackcheck: from rpython.translator.transform import insert_ll_stackcheck insert_ll_stackcheck(t) t.config.translation.shared = shared if entrypoints is not None: kwds = {'secondary_entrypoints': [(i, None) for i in entrypoints]} else: kwds = {} cbuilder = CStandaloneBuilder(t, entry_point, t.config, **kwds) if debug: cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES) else: cbuilder.generate_source() cbuilder.compile() if option is not None and option.view: t.view() return t, cbuilder
def llinterpreter_for_transformed_graph(self, f, args_s): from rpython.rtyper.llinterp import LLInterpreter from rpython.translator.c.genc import CStandaloneBuilder t = rtype(f, args_s) # XXX we shouldn't need an actual gcpolicy here. cbuild = CStandaloneBuilder(t, f, t.config, gcpolicy=self.gcpolicy) cbuild.generate_graphs_for_llinterp() graph = cbuild.getentrypointptr()._obj.graph # arguments cannot be GC objects because nobody would put a # proper header on them for v in graph.getargs(): if isinstance(v.concretetype, lltype.Ptr): assert v.concretetype.TO._gckind != 'gc', "fix the test!" llinterp = LLInterpreter(t.rtyper) if option.view: t.view() return llinterp, graph
def test_no_collect(): from rpython.rlib import rgc from rpython.translator.c.genc import CStandaloneBuilder @rgc.no_collect def g(): return 1 assert g._dont_inline_ assert g._gc_no_collect_ def entrypoint(argv): return g() + 2 t = rtype(entrypoint, [s_list_of_strings]) t.config.translation.gc = "minimark" cbuild = CStandaloneBuilder(t, entrypoint, t.config, gcpolicy=FrameworkGcPolicy2) db = cbuild.generate_graphs_for_llinterp()
def test_counters(self): from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.lltypesystem.lloperation import llop def entry_point(argv): llop.instrument_count(lltype.Void, 'test', 2) llop.instrument_count(lltype.Void, 'test', 1) llop.instrument_count(lltype.Void, 'test', 1) llop.instrument_count(lltype.Void, 'test', 2) llop.instrument_count(lltype.Void, 'test', 1) return 0 t = TranslationContext(self.config) t.config.translation.instrument = True t.buildannotator().build_types(entry_point, [s_list_of_strings]) t.buildrtyper().specialize() cbuilder = CStandaloneBuilder(t, entry_point, config=t.config) # xxx cbuilder.generate_source() cbuilder.compile() counters_fname = udir.join("_counters_") os.environ['PYPY_INSTRUMENT_COUNTERS'] = str(counters_fname) try: data = cbuilder.cmdexec() finally: del os.environ['PYPY_INSTRUMENT_COUNTERS'] f = counters_fname.open('rb') counters_data = f.read() f.close() import struct counters = struct.unpack("LLL", counters_data) assert counters == (0,3,2)
def test_separate_files(self): # One file in translator/c/src fname = py.path.local(cdir).join('src', 'll_strtod.c') # One file in (another) subdir of the temp directory dirname = udir.join("test_dir").ensure(dir=1) fname2 = dirname.join("test_genc.c") fname2.write(""" void f() { LL_strtod_formatd(12.3, 'f', 5); }""") files = [fname, fname2] def entry_point(argv): return 0 t = TranslationContext(self.config) t.buildannotator().build_types(entry_point, [s_list_of_strings]) t.buildrtyper().specialize() cbuilder = CStandaloneBuilder(t, entry_point, t.config) cbuilder.eci = cbuilder.eci.merge( ExternalCompilationInfo(separate_module_files=files)) cbuilder.generate_source() makefile = udir.join(cbuilder.modulename, 'Makefile').read() # generated files are compiled in the same directory assert " ../test_dir/test_genc.c" in makefile assert " ../test_dir/test_genc.o" in makefile # but files from pypy source dir must be copied assert "translator/c/src" not in makefile assert " ll_strtod.c" in makefile assert " ll_strtod.o" in makefile
def translates(self, func=None, argtypes=None, seeobj_w=[], extra_func=None, c_compile=False, **kwds): config = make_config(None, **kwds) if func is not None: if argtypes is None: nb_args = func.func_code.co_argcount argtypes = [W_Root] * nb_args # t = TranslationContext(config=config) self.t = t # for debugging ann = t.buildannotator() def entry_point(argv): self.threadlocals.enter_thread(self) W_SliceObject(w_some_obj(), w_some_obj(), w_some_obj()) if extra_func: extra_func(self) return 0 ann.build_types(entry_point, [s_list_of_strings], complete_now=False) if func is not None: ann.build_types(func, argtypes, complete_now=False) if seeobj_w: def seeme(n): return seeobj_w[n] ann.build_types(seeme, [int], complete_now=False) # # annotate all _seen_extras, knowing that annotating some may # grow the list done = 0 while done < len(self._seen_extras): #print self._seen_extras ann.build_types(self._seen_extras[done], [], complete_now=False) ann.complete_pending_blocks() done += 1 ann.complete() assert done == len(self._seen_extras) #t.viewcg() t.buildrtyper().specialize() t.checkgraphs() if c_compile: cbuilder = CStandaloneBuilder(t, entry_point, t.config) cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES) cbuilder.compile() return t, cbuilder
def test_build(): def entry_point(argv): parser = interp_pyexpat.XML_ParserCreate("test") interp_pyexpat.XML_ParserFree(parser) res = interp_pyexpat.XML_ErrorString(3) os.write(1, rffi.charp2str(res)) return 0 t = TranslationContext() t.buildannotator().build_types(entry_point, [s_list_of_strings]) t.buildrtyper().specialize() builder = CStandaloneBuilder(t, entry_point, t.config) builder.generate_source() builder.compile() data = builder.cmdexec() assert data == pyexpat.ErrorString(3)
def compile(self, entry_point, no__thread=True): t = TranslationContext(self.config) t.config.translation.gc = "semispace" t.config.translation.gcrootfinder = self.gcrootfinder t.config.translation.thread = True t.config.translation.no__thread = no__thread t.buildannotator().build_types(entry_point, [s_list_of_strings]) t.buildrtyper().specialize() # cbuilder = CStandaloneBuilder(t, entry_point, t.config) cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES) cbuilder.compile() # return t, cbuilder
def test_counters(self): from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.lltypesystem.lloperation import llop def entry_point(argv): llop.instrument_count(lltype.Void, 'test', 2) llop.instrument_count(lltype.Void, 'test', 1) llop.instrument_count(lltype.Void, 'test', 1) llop.instrument_count(lltype.Void, 'test', 2) llop.instrument_count(lltype.Void, 'test', 1) return 0 t = TranslationContext(self.config) t.config.translation.instrument = True t.buildannotator().build_types(entry_point, [s_list_of_strings]) t.buildrtyper().specialize() cbuilder = CStandaloneBuilder(t, entry_point, config=t.config) # xxx cbuilder.generate_source() cbuilder.compile() counters_fname = udir.join("_counters_") os.environ['PYPY_INSTRUMENT_COUNTERS'] = str(counters_fname) try: data = cbuilder.cmdexec() finally: del os.environ['PYPY_INSTRUMENT_COUNTERS'] f = counters_fname.open('rb') counters_data = f.read() f.close() import struct counters = struct.unpack("LLL", counters_data) assert counters == (0, 3, 2)
def setup_class(cls): cls.marker = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw', zero=True) funcs0 = [] funcs2 = [] cleanups = [] name_to_func = {} mixlevelstuff = [] for fullname in dir(cls): if not fullname.startswith('define'): continue definefunc = getattr(cls, fullname) _, name = fullname.split('_', 1) func_fixup = definefunc.im_func(cls) cleanup = None if isinstance(func_fixup, tuple): func, cleanup, fixup = func_fixup mixlevelstuff.append(fixup) else: func = func_fixup func.func_name = "f_%s" % name if cleanup: cleanup.func_name = "clean_%s" % name nargs = len(inspect.getargspec(func)[0]) name_to_func[name] = len(funcs0) if nargs == 2: funcs2.append(func) funcs0.append(None) elif nargs == 0: funcs0.append(func) funcs2.append(None) else: raise NotImplementedError( "defined test functions should have 0/2 arguments") # used to let test cleanup static root pointing to runtime # allocated stuff cleanups.append(cleanup) def entrypoint(args): num = args[0] func = funcs0[num] if func: res = func() else: func = funcs2[num] res = func(args[1], args[2]) cleanup = cleanups[num] if cleanup: cleanup() return res from rpython.translator.c.genc import CStandaloneBuilder s_args = SomePtr(lltype.Ptr(ARGS)) t = rtype(entrypoint, [s_args], gcname=cls.gcname, taggedpointers=cls.taggedpointers) for fixup in mixlevelstuff: if fixup: fixup(t) cbuild = CStandaloneBuilder(t, entrypoint, config=t.config, gcpolicy=cls.gcpolicy) db = cbuild.generate_graphs_for_llinterp() entrypointptr = cbuild.getentrypointptr() entrygraph = entrypointptr._obj.graph if option.view: t.viewcg() cls.name_to_func = name_to_func cls.entrygraph = entrygraph cls.rtyper = t.rtyper cls.db = db
def _makefunc_str_int(cls, func): def main(argv): arg0 = argv[1] arg1 = int(argv[2]) try: res = func(arg0, arg1) except MemoryError: print 'Result: MemoryError' else: print 'Result: "%s"' % (res,) return 0 config = cls.make_config() t = TranslationContext(config=config) a = t.buildannotator() sec_ep = getattr(cls, 'secondary_entrypoints', []) for f, inputtypes in sec_ep: a.build_types(f, inputtypes, False) a.build_types(main, [s_list_of_strings]) t.buildrtyper().specialize() t.checkgraphs() cbuilder = CStandaloneBuilder(t, main, config=config, secondary_entrypoints=sec_ep) c_source_filename = cbuilder.generate_source( defines = cbuilder.DEBUG_DEFINES) cls._patch_makefile(cbuilder.targetdir) if option.view: t.view() exe_name = cbuilder.compile() def run(arg0, arg1, runner=None): if runner is not None: py.test.skip("unsupported test: runner=%r" % (runner,)) lines = [] print >> sys.stderr, 'RUN: starting', exe_name, arg0, arg1 if sys.platform == 'win32': redirect = ' 2> NUL' else: redirect = '' if config.translation.shared and os.name == 'posix': library_path = exe_name.dirpath() if sys.platform == 'darwin': env = 'DYLD_LIBRARY_PATH="%s" ' % library_path else: env = 'LD_LIBRARY_PATH="%s" ' % library_path else: env = '' cwd = os.getcwd() try: os.chdir(str(exe_name.dirpath())) g = os.popen( '%s"%s" %s %d%s' % (env, exe_name, arg0, arg1, redirect), 'r') finally: os.chdir(cwd) for line in g: print >> sys.stderr, 'RUN:', line.rstrip() lines.append(line) g.close() if not lines: py.test.fail("no output from subprocess") if not lines[-1].startswith('Result:'): py.test.fail("unexpected output from subprocess") result = lines[-1][len('Result:'):].strip() if result == 'MemoryError': raise MemoryError("subprocess got an RPython MemoryError") if result.startswith('"') and result.endswith('"'): return result[1:-1] else: return int(result) return run
def _makefunc_str_int(cls, func): def main(argv): arg0 = argv[1] arg1 = int(argv[2]) try: res = func(arg0, arg1) except MemoryError: print 'Result: MemoryError' else: print 'Result: "%s"' % (res,) return 0 config = cls.make_config() t = TranslationContext(config=config) a = t.buildannotator() sec_ep = getattr(cls, 'secondary_entrypoints', []) for f, inputtypes in sec_ep: a.build_types(f, inputtypes, False) a.build_types(main, [s_list_of_strings]) t.buildrtyper().specialize() t.checkgraphs() cbuilder = CStandaloneBuilder(t, main, config=config, secondary_entrypoints=sec_ep) c_source_filename = cbuilder.generate_source( defines = cbuilder.DEBUG_DEFINES) cls._patch_makefile(cbuilder.targetdir) if option.view: t.view() exe_name = cbuilder.compile() def run(arg0, arg1): lines = [] print >> sys.stderr, 'RUN: starting', exe_name, arg0, arg1 if sys.platform == 'win32': redirect = ' 2> NUL' else: redirect = '' if config.translation.shared and os.name == 'posix': library_path = exe_name.dirpath() if sys.platform == 'darwin': env = 'DYLD_LIBRARY_PATH="%s" ' % library_path else: env = 'LD_LIBRARY_PATH="%s" ' % library_path else: env = '' cwd = os.getcwd() try: os.chdir(str(exe_name.dirpath())) g = os.popen( '%s"%s" %s %d%s' % (env, exe_name, arg0, arg1, redirect), 'r') finally: os.chdir(cwd) for line in g: print >> sys.stderr, 'RUN:', line.rstrip() lines.append(line) g.close() if not lines: py.test.fail("no output from subprocess") if not lines[-1].startswith('Result:'): py.test.fail("unexpected output from subprocess") result = lines[-1][len('Result:'):].strip() if result == 'MemoryError': raise MemoryError("subprocess got an RPython MemoryError") if result.startswith('"') and result.endswith('"'): return result[1:-1] else: return int(result) return run