def test_annotator_folding(): from rpython.translator.interactive import Translation gcoption = ChoiceOption('name', 'GC name', ['ref', 'framework'], 'ref') gcgroup = OptionDescription('gc', '', [gcoption]) descr = OptionDescription('pypy', '', [gcgroup]) config = Config(descr) def f(x): if config.gc.name == 'ref': return x + 1 else: return 'foo' t = Translation(f, [int]) t.rtype() block = t.context.graphs[0].startblock assert len(block.exits[0].target.operations) == 0 assert len(block.operations) == 1 assert len(block.exits) == 1 assert block.operations[0].opname == 'int_add' assert config._freeze_() # does not raise, since it does not change the attribute config.gc.name = "ref" py.test.raises(TypeError, 'config.gc.name = "framework"')
def _makefunc_str_int(cls, f): def main(argv): arg0 = argv[1] arg1 = int(argv[2]) try: res = f(arg0, arg1) except MemoryError: print "MEMORY-ERROR" else: print res return 0 t = Translation(main, gc=cls.gcpolicy, taggedpointers=cls.taggedpointers, gcremovetypeptr=cls.removetypeptr) t.disable(['backendopt']) t.set_backend_extra_options(c_debug_defines=True) t.rtype() if option.view: t.viewcg() exename = t.compile() def run(s, i): data = py.process.cmdexec("%s %s %d" % (exename, s, i)) data = data.strip() if data == 'MEMORY-ERROR': raise MemoryError return data return run
def test_annotator_folding(): from rpython.translator.interactive import Translation gcoption = ChoiceOption('name', 'GC name', ['ref', 'framework'], 'ref') gcgroup = OptionDescription('gc', '', [gcoption]) descr = OptionDescription('pypy', '', [gcgroup]) config = Config(descr) def f(x): if config.gc.name == 'ref': return x + 1 else: return 'foo' t = Translation(f, [int]) t.rtype() block = t.context.graphs[0].startblock assert len(block.exits[0].target.operations) == 0 assert len(block.operations) == 1 assert len(block.exits) == 1 assert block.operations[0].opname == 'int_add' assert config._freeze_() # does not raise, since it does not change the attribute config.gc.name = "ref" py.test.raises(TypeError, 'config.gc.name = "framework"')
def test_simple_rtype(): def f(x, y): return x + y t = Translation(f, [int, int]) t.annotate() t.rtype() assert 'rtype_lltype' in t.driver.done
def build_adi(function, types): t = Translation(function, types) t.rtype() if option.view: t.view() adi = AbstractDataFlowInterpreter(t.context) graph = graphof(t.context, function) adi.schedule_function(graph) adi.complete() return t.context, adi, graph
def build_adi(function, types): t = Translation(function, types) t.rtype() if option.view: t.view() adi = AbstractDataFlowInterpreter(t.context) graph = graphof(t.context, function) adi.schedule_function(graph) adi.complete() return t.context, adi, graph
def test_simple_rtype(): def f(x,y): return x+y t = Translation(f, [int, int]) t.annotate() t.rtype() assert 'rtype_lltype' in t.driver.done
def test_jitted(): from rpython.jit.backend import detect_cpu if not detect_cpu.autodetect().startswith('x86'): py.test.skip("HAS_CODEMAP is only in the x86 jit backend for now") class MyCode: pass def get_name(mycode): raise NotImplementedError rvmprof.register_code_object_class(MyCode, get_name) jitdriver = jit.JitDriver( greens=['code'], reds='auto', is_recursive=True, get_unique_id=lambda code: rvmprof.get_unique_id(code)) @rvmprof.vmprof_execute_code("mycode", lambda code, level, total_i: code) def mainloop(code, level, total_i): i = 20 while i > 0: jitdriver.jit_merge_point(code=code) i -= 1 if level > 0: mainloop(code, level - 1, total_i + i) if level == 0 and total_i == 0: p, length = traceback.traceback(20) traceback.walk_traceback(MyCode, my_callback, 42, p, length) lltype.free(p, flavor='raw') def my_callback(code, loc, arg): print code, loc, arg return 0 def f(argv): jit.set_param(jitdriver, "inlining", 0) code1 = MyCode() rvmprof.register_code(code1, "foo") mainloop(code1, 2, 0) return 0 t = Translation(f, None, gc="boehm") t.rtype() t.driver.pyjitpl_lltype() t.compile_c() stdout = t.driver.cbuilder.cmdexec('') r = re.compile("[<]MyCode object at 0x([0-9a-f]+)[>] (\d) 42\n") got = r.findall(stdout) addr = got[0][0] assert got == [(addr, '1'), (addr, '1'), (addr, '0')]
def test_enforced_args(): from rpython.annotator.model import s_None from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator from rpython.translator.interactive import Translation def f1(): str2charp("hello") def f2(): str2charp("world") t = Translation(f1, []) t.rtype() mixann = MixLevelHelperAnnotator(t.context.rtyper) mixann.getgraph(f2, [], s_None) mixann.finish()
def test_enforced_args(): from rpython.annotator.model import s_None from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator from rpython.translator.interactive import Translation def f1(): str2charp("hello") def f2(): str2charp("world") t = Translation(f1, []) t.rtype() mixann = MixLevelHelperAnnotator(t.context.rtyper) mixann.getgraph(f2, [], s_None) mixann.finish()
def test_jitted(): from rpython.jit.backend import detect_cpu if not detect_cpu.autodetect().startswith("x86"): py.test.skip("HAS_CODEMAP is only in the x86 jit backend for now") class MyCode: pass def get_name(mycode): raise NotImplementedError rvmprof.register_code_object_class(MyCode, get_name) jitdriver = jit.JitDriver( greens=["code"], reds="auto", is_recursive=True, get_unique_id=lambda code: rvmprof.get_unique_id(code) ) @rvmprof.vmprof_execute_code("mycode", lambda code, level, total_i: code) def mainloop(code, level, total_i): i = 20 while i > 0: jitdriver.jit_merge_point(code=code) i -= 1 if level > 0: mainloop(code, level - 1, total_i + i) if level == 0 and total_i == 0: p, length = traceback.traceback(20) traceback.walk_traceback(MyCode, my_callback, 42, p, length) lltype.free(p, flavor="raw") def my_callback(code, loc, arg): print code, loc, arg return 0 def f(argv): jit.set_param(jitdriver, "inlining", 0) code1 = MyCode() rvmprof.register_code(code1, "foo") mainloop(code1, 2, 0) return 0 t = Translation(f, None, gc="boehm") t.rtype() t.driver.pyjitpl_lltype() t.compile_c() stdout = t.driver.cbuilder.cmdexec("") r = re.compile("[<]MyCode object at 0x([0-9a-f]+)[>] (\d) 42\n") got = r.findall(stdout) addr = got[0][0] assert got == [(addr, "1"), (addr, "1"), (addr, "0")]
def translate_to_graph(function, arguments): """ Translate a function to basic blocks and visualize these blocks (see requirements.txt for necessary pip packages); use this mechanism in conjunction with interpret_from_graph :param function: the RPython function to interpret :param arguments: a list of the arguments passed to 'function' :return: the translator RTyper and the basic block graph for the function passed """ assert callable(function) assert isinstance(arguments, list) translation = Translation(function, arguments) translation.annotate() translation.rtype() translation.backendopt() translation.view() return translation.driver.translator.rtyper, translation.driver.translator.graphs[0]
def test_computed_int_symbolic(): too_early = True def compute_fn(): assert not too_early return 7 k = ComputedIntSymbolic(compute_fn) def f(ignored): return k*6 t = Translation(f) t.rtype() if option.view: t.view() too_early = False fn = compile(f, [int]) res = fn(0) assert res == 42
def test_parser(): def f(x): if x: s = "a(X, Y, Z)." else: s = "f(a, X, _, _, X, f(X, 2.455))." term = parsing.parse_file(s) assert isinstance(term, parsing.Nonterminal) return term.symbol assert f(True) == "file" assert f(True) == "file" t = Translation(f) t.annotate([bool]) t.rtype() t.backendopt() func = t.compile_c() assert func(True) == "file" assert func(False) == "file"
def test_inhibit_tail_call(): def foobar_fn(n): return 42 foobar_fn._dont_inline_ = True def main(n): return foobar_fn(n) # t = Translation(main, [int], backend="c") t.rtype() t.context._graphof(foobar_fn).inhibit_tail_call = True t.source_c() lines = t.driver.cbuilder.c_source_filename.join('..', 'rpython_translator_c_test_test_genc.c').readlines() for i, line in enumerate(lines): if '= pypy_g_foobar_fn' in line: break else: assert 0, "the call was not found in the C source" assert 'PYPY_INHIBIT_TAIL_CALL();' in lines[i+1]
def test_record_known_result(self): from rpython.translator.interactive import Translation @elidable def f(x): return x + 1 @elidable def g(x): return x - 1 def call_f(x): y = f(x) record_known_result(x, g, y) return y call_f(10) # doesn't crash t = Translation(call_f, [int]) t.rtype() # does not crash
def test_computed_int_symbolic(): too_early = True def compute_fn(): assert not too_early return 7 k = ComputedIntSymbolic(compute_fn) def f(ignored): return k * 6 t = Translation(f) t.rtype() if option.view: t.view() too_early = False fn = compile(f, [int]) res = fn(0) assert res == 42
def test_engine(): e = get_engine(""" g(a, a). g(a, b). g(b, c). f(X, Z) :- g(X, Y), g(Y, Z). """) t1 = parse_query_term("f(a, c).") t2 = parse_query_term("f(X, c).") def run(): e.run(t1) e.run(t2) v0 = e.heap.getvar(0) if isinstance(v0, Atom): return v0.name() return "no!" assert run() == "a" t = Translation(run) t.annotate() t.rtype() func = t.compile_c() assert func() == "a"
def compile(self, entry_point, backendopt=True, withsmallfuncsets=None, shared=False, thread=False): t = Translation(entry_point, None, gc="boehm") self.t = t t.set_backend_extra_options(c_debug_defines=True) t.config.translation.reverse_debugger = True t.config.translation.lldebug0 = True t.config.translation.shared = shared t.config.translation.thread = thread if withsmallfuncsets is not None: t.config.translation.withsmallfuncsets = withsmallfuncsets if not backendopt: t.disable(["backendopt_lltype"]) t.annotate() t.rtype() if t.backendopt: t.backendopt() self.exename = t.compile_c() self.rdbname = os.path.join(os.path.dirname(str(self.exename)), 'log.rdb')
def compile(fn, argtypes, view=False, gcpolicy="none", backendopt=True, annotatorpolicy=None, thread=False, **kwds): argtypes_unroll = unrolling_iterable(enumerate(argtypes)) for argtype in argtypes: if argtype not in [ int, float, str, bool, r_ulonglong, r_longlong, r_uint ]: raise Exception("Unsupported argtype, %r" % (argtype, )) def entry_point(argv): args = () for i, argtype in argtypes_unroll: a = argv[i + 1] if argtype is int: args += (int(a), ) elif argtype is r_uint: args += (r_uint(int(a)), ) elif argtype is r_longlong: args += (parse_longlong(a), ) elif argtype is r_ulonglong: args += (parse_ulonglong(a), ) elif argtype is bool: if a == 'True': args += (True, ) else: assert a == 'False' args += (False, ) elif argtype is float: if a == 'inf': args += (INFINITY, ) elif a == '-inf': args += (-INFINITY, ) elif a == 'nan': args += (NAN, ) else: args += (float(a), ) else: if a.startswith('/'): # escaped string if len(a) == 1: a = '' else: l = a[1:].split(',') a = ''.join([chr(int(x)) for x in l]) args += (a, ) res = fn(*args) print "THE RESULT IS:", llrepr_out(res), ";" return 0 t = Translation(entry_point, None, gc=gcpolicy, backend="c", policy=annotatorpolicy, thread=thread, **kwds) if not backendopt: t.disable(["backendopt_lltype"]) t.driver.config.translation.countmallocs = True t.annotate() try: if py.test.config.option.view: t.view() except AttributeError: pass t.rtype() if backendopt: t.backendopt() try: if py.test.config.option.view: t.view() except AttributeError: pass t.compile_c() ll_res = graphof(t.context, fn).getreturnvar().concretetype def output(stdout): for line in stdout.splitlines(False): if len(repr(line)) == len(line) + 2: # no escaped char print line else: print 'REPR:', repr(line) def f(*args, **kwds): expected_extra_mallocs = kwds.pop('expected_extra_mallocs', 0) expected_exception_name = kwds.pop('expected_exception_name', None) assert not kwds assert len(args) == len(argtypes) for arg, argtype in zip(args, argtypes): assert isinstance(arg, argtype) stdout = t.driver.cbuilder.cmdexec( " ".join([llrepr_in(arg) for arg in args]), expect_crash=(expected_exception_name is not None)) # if expected_exception_name is not None: stdout, stderr = stdout print '--- stdout ---' output(stdout) print '--- stderr ---' output(stderr) print '--------------' stderr, prevline, lastline, empty = stderr.rsplit('\n', 3) assert empty == '' expected = 'Fatal RPython error: ' + expected_exception_name assert lastline == expected or prevline == expected return None output(stdout) stdout, lastline, empty = stdout.rsplit('\n', 2) assert empty == '' assert lastline.startswith('MALLOC COUNTERS: ') mallocs, frees = map(int, lastline.split()[2:]) assert stdout.endswith(' ;') pos = stdout.rindex('THE RESULT IS: ') res = stdout[pos + len('THE RESULT IS: '):-2] # if isinstance(expected_extra_mallocs, int): assert mallocs - frees == expected_extra_mallocs else: assert mallocs - frees in expected_extra_mallocs # if ll_res in [ lltype.Signed, lltype.Unsigned, lltype.SignedLongLong, lltype.UnsignedLongLong ]: return int(res) elif ll_res == lltype.Bool: return bool(int(res)) elif ll_res == lltype.Char: assert len(res) == 1 return res elif ll_res == lltype.Float: return float(res) elif ll_res == lltype.Ptr(STR): return res elif ll_res == lltype.Void: return None raise NotImplementedError("parsing %s" % (ll_res, )) class CompilationResult(object): def __repr__(self): return 'CompilationResult(%s)' % (fn.__name__, ) def __call__(self, *args, **kwds): return f(*args, **kwds) cr = CompilationResult() cr.t = t cr.builder = t.driver.cbuilder return cr
def compile(fn, argtypes, view=False, gcpolicy="none", backendopt=True, annotatorpolicy=None, thread=False, **kwds): argtypes_unroll = unrolling_iterable(enumerate(argtypes)) for argtype in argtypes: if argtype not in [int, float, str, bool, r_ulonglong, r_longlong, r_uint]: raise Exception("Unsupported argtype, %r" % (argtype,)) def entry_point(argv): args = () for i, argtype in argtypes_unroll: a = argv[i + 1] if argtype is int: args += (int(a),) elif argtype is r_uint: args += (r_uint(int(a)),) elif argtype is r_longlong: args += (parse_longlong(a),) elif argtype is r_ulonglong: args += (parse_ulonglong(a),) elif argtype is bool: if a == 'True': args += (True,) else: assert a == 'False' args += (False,) elif argtype is float: if a == 'inf': args += (INFINITY,) elif a == '-inf': args += (-INFINITY,) elif a == 'nan': args += (NAN,) else: args += (float(a),) else: if a.startswith('/'): # escaped string if len(a) == 1: a = '' else: l = a[1:].split(',') a = ''.join([chr(int(x)) for x in l]) args += (a,) res = fn(*args) print "THE RESULT IS:", llrepr_out(res), ";" return 0 t = Translation(entry_point, None, gc=gcpolicy, backend="c", policy=annotatorpolicy, thread=thread, **kwds) if not backendopt: t.disable(["backendopt_lltype"]) t.driver.config.translation.countmallocs = True t.annotate() try: if py.test.config.option.view: t.view() except AttributeError: pass t.rtype() if backendopt: t.backendopt() try: if py.test.config.option.view: t.view() except AttributeError: pass t.compile_c() ll_res = graphof(t.context, fn).getreturnvar().concretetype def output(stdout): for line in stdout.splitlines(False): if len(repr(line)) == len(line) + 2: # no escaped char print line else: print 'REPR:', repr(line) def f(*args, **kwds): expected_extra_mallocs = kwds.pop('expected_extra_mallocs', 0) expected_exception_name = kwds.pop('expected_exception_name', None) assert not kwds assert len(args) == len(argtypes) for arg, argtype in zip(args, argtypes): assert isinstance(arg, argtype) stdout = t.driver.cbuilder.cmdexec( " ".join([llrepr_in(arg) for arg in args]), expect_crash=(expected_exception_name is not None)) # if expected_exception_name is not None: stdout, stderr = stdout print '--- stdout ---' output(stdout) print '--- stderr ---' output(stderr) print '--------------' stderr, prevline, lastline, empty = stderr.rsplit('\n', 3) assert empty == '' expected = 'Fatal RPython error: ' + expected_exception_name assert lastline == expected or prevline == expected return None output(stdout) stdout, lastline, empty = stdout.rsplit('\n', 2) assert empty == '' assert lastline.startswith('MALLOC COUNTERS: ') mallocs, frees = map(int, lastline.split()[2:]) assert stdout.endswith(' ;') pos = stdout.rindex('THE RESULT IS: ') res = stdout[pos + len('THE RESULT IS: '):-2] # if isinstance(expected_extra_mallocs, int): assert mallocs - frees == expected_extra_mallocs else: assert mallocs - frees in expected_extra_mallocs # if ll_res in [lltype.Signed, lltype.Unsigned, lltype.SignedLongLong, lltype.UnsignedLongLong]: return int(res) elif ll_res == lltype.Bool: return bool(int(res)) elif ll_res == lltype.Char: assert len(res) == 1 return res elif ll_res == lltype.Float: return float(res) elif ll_res == lltype.Ptr(STR): return res elif ll_res == lltype.Void: return None raise NotImplementedError("parsing %s" % (ll_res,)) class CompilationResult(object): def __repr__(self): return 'CompilationResult(%s)' % (fn.__name__,) def __call__(self, *args, **kwds): return f(*args, **kwds) cr = CompilationResult() cr.t = t cr.builder = t.driver.cbuilder return cr