def compile(fn, argtypes, view=False, gcpolicy="ref", backendopt=True, annotatorpolicy=None): if argtypes is not None and "__pypy__" in sys.builtin_module_names: py.test.skip("requires building cpython extension modules") t = Translation(fn, argtypes, gc=gcpolicy, backend="c", policy=annotatorpolicy) if not backendopt: t.disable(["backendopt_lltype"]) t.annotate() # XXX fish t.driver.config.translation.countmallocs = True compiled_fn = t.compile_c() try: if py.test.config.option.view: t.view() except AttributeError: pass malloc_counters = t.driver.cbuilder.get_malloc_counters() def checking_fn(*args, **kwds): if 'expected_extra_mallocs' in kwds: expected_extra_mallocs = kwds.pop('expected_extra_mallocs') else: expected_extra_mallocs = 0 res = compiled_fn(*args, **kwds) mallocs, frees = malloc_counters() if isinstance(expected_extra_mallocs, int): assert mallocs - frees == expected_extra_mallocs else: assert mallocs - frees in expected_extra_mallocs return res return checking_fn
def compile(fn, argtypes, view=False, gcpolicy="ref", backendopt=True, annotatorpolicy=None): t = Translation(fn, argtypes, gc=gcpolicy, backend="c", policy=annotatorpolicy) if not backendopt: t.disable(["backendopt_lltype"]) t.annotate() # XXX fish t.driver.config.translation.countmallocs = True compiled_fn = t.compile_c() if getattr(py.test.config.option, 'view', False): t.view() malloc_counters = t.driver.cbuilder.get_malloc_counters() def checking_fn(*args, **kwds): if 'expected_extra_mallocs' in kwds: expected_extra_mallocs = kwds.pop('expected_extra_mallocs') else: expected_extra_mallocs = 0 res = compiled_fn(*args, **kwds) mallocs, frees = malloc_counters() if isinstance(expected_extra_mallocs, int): assert mallocs - frees == expected_extra_mallocs else: assert mallocs - frees in expected_extra_mallocs return res return checking_fn
def test_translate_compiled_parser(): r0 = Rule("expression", [["additive", "EOF"]]) r1 = Rule("additive", [["multitive", "+", "additive"], ["multitive"]]) r2 = Rule("multitive", [["primary", "*", "multitive"], ["primary"]]) r3 = Rule("primary", [["(", "additive", ")"], ["decimal"]]) r4 = Rule("decimal", [[symb] for symb in "0123456789"]) p = PackratParser([r0, r1, r2, r3, r4], "expression") compiler = ParserCompiler(p) kls = compiler.compile() p = kls() tree = p.parse([ Token(c, i, SourcePos(i, 0, i)) for i, c in enumerate(list("2*(3+4)") + ["EOF"]) ]) data = [ Token(c, i, SourcePos(i, 0, i)) for i, c in enumerate(list("2*(3+4)") + ["EOF"]) ] print tree p = kls() def parse(choose): tree = p.parse(data) return tree.symbol + " " + "-%-".join( [c.symbol for c in tree.children]) t = Translation(parse) t.annotate([bool]) t.backendopt() t.rtype() func = t.compile_c() res1 = parse(True) res2 = func(True) assert res1 == res2
def test_translate_ast_visitor(): from pypy.rlib.parsing.ebnfparse import parse_ebnf, make_parse_function regexs, rules, ToAST = parse_ebnf(""" DECIMAL: "0|[1-9][0-9]*"; IGNORE: " "; additive: multitive ["+!"] additive | <multitive>; multitive: primary ["*!"] multitive | <primary>; #nonsense! primary: "(" <additive> ")" | <DECIMAL>; """) parse = make_parse_function(regexs, rules) def f(): tree = parse("(0 +! 10) *! (999 +! 10) +! 1") tree = ToAST().visit_additive(tree) assert len(tree) == 1 tree = tree[0] return tree.symbol + " " + "-&-".join( [c.symbol for c in tree.children]) res1 = f() t = Translation(f) t.annotate() t.rtype() t.backendopt() func = t.compile_c() res2 = func() assert res1 == res2
def test_translate_pypackrat_regex(): from pypy.rlib.parsing.pypackrat import PackratParser class parser(PackratParser): """ num: `([1-9][0-9]*)|0`; """ print parser._code def parse(s): p = parser(s) return p.num() res = parse("1234") assert res == '1234' t = Translation(parse) t.annotate([str]) t.rtype() t.backendopt() if option.view: t.view() func = t.compile_c() res = func("12345") assert res == '12345' res = func("0") assert res == '0'
def test_translate_compiled_parser(): r0 = Rule("expression", [["additive", "EOF"]]) r1 = Rule("additive", [["multitive", "+", "additive"], ["multitive"]]) r2 = Rule("multitive", [["primary", "*", "multitive"], ["primary"]]) r3 = Rule("primary", [["(", "additive", ")"], ["decimal"]]) r4 = Rule("decimal", [[symb] for symb in "0123456789"]) p = PackratParser([r0, r1, r2, r3, r4], "expression") compiler = ParserCompiler(p) kls = compiler.compile() p = kls() tree = p.parse([Token(c, i, SourcePos(i, 0, i)) for i, c in enumerate(list("2*(3+4)") + ["EOF"])]) data = [Token(c, i, SourcePos(i, 0, i)) for i, c in enumerate(list("2*(3+4)") + ["EOF"])] print tree p = kls() def parse(choose): tree = p.parse(data) return tree.symbol + " " + "-%-".join([c.symbol for c in tree.children]) t = Translation(parse) t.annotate([bool]) t.backendopt() t.rtype() func = t.compile_c() res1 = parse(True) res2 = func(True) assert res1 == res2
def test_translate_simple(self): digits = RangeExpression("0", "9") lower = RangeExpression("a", "z") upper = RangeExpression("A", "Z") keywords = StringExpression("if") | StringExpression("else") | StringExpression("def") | StringExpression("class") underscore = StringExpression("_") atoms = lower + (upper | lower | digits | underscore).kleene() vars = underscore | (upper + (upper | lower | underscore | digits).kleene()) integers = StringExpression("0") | (RangeExpression("1", "9") + digits.kleene()) white = StringExpression(" ") l1 = self.get_lexer([keywords, atoms, vars, integers, white], ["KEYWORD", "ATOM", "VAR", "INT", "WHITE"]) l2 = self.get_lexer([keywords, atoms, vars, integers, white], ["KEYWORD", "ATOM", "VAR", "INT", "WHITE"], ["WHITE"]) def lex(s, ignore=False): if ignore: tokens = l2.tokenize(s) else: tokens = l1.tokenize(s) return "-%-".join([t.name for t in tokens]) res = lex("if A a 12341 0 else").split("-%-") assert res == ("KEYWORD WHITE VAR WHITE ATOM WHITE INT WHITE " "INT WHITE KEYWORD").split() res = lex("if A a 12341 0 else", True).split("-%-") assert res == "KEYWORD VAR ATOM INT INT KEYWORD".split() t = Translation(lex) t.annotate([str, bool]) t.rtype() func = t.compile_c() res = lex("if A a 12341 0 else", False).split("-%-") assert res == ("KEYWORD WHITE VAR WHITE ATOM WHITE INT WHITE " "INT WHITE KEYWORD").split() res = lex("if A a 12341 0 else", True).split("-%-") assert res == "KEYWORD VAR ATOM INT INT KEYWORD".split()
def test_translate(): from pypy.translator.interactive import Translation def f(x, y): rnd = Random(x) rnd.init_by_array([x, y]) rnd.jumpahead(y) return rnd.genrand32(), rnd.random() t = Translation(f) fc = t.compile_c([int, int]) assert fc(1, 2) == f(1, 2)
def test_translate(): from pypy.translator.interactive import Translation def f(x, y): rnd = Random(x) rnd.init_by_array([x, y]) rnd.jumpahead(intmask(y)) return rnd.genrand32(), rnd.random() t = Translation(f) fc = t.compile_c([r_uint, r_uint]) assert fc(r_uint(1), r_uint(2)) == f(r_uint(1), r_uint(2))
def test_tagged_boehm(): t = Translation(entry_point, standalone=True, gc='boehm') try: exename = t.compile_c() finally: if conftest.option.view: t.view() g = os.popen(exename, 'r') data = g.read() g.close() assert data.rstrip().endswith('ALL OK')
def test_tagged_boehm(): t = Translation(entry_point, standalone=True, gc='boehm', taggedpointers=True) try: exename = str(t.compile_c()) finally: if conftest.option.view: t.view() g = os.popen(exename, 'r') data = g.read() g.close() assert data.rstrip().endswith('ALL OK')
def test_name(): def f(): return 3 f.c_name = 'pypy_xyz_f' t = Translation(f, [], backend="c") t.annotate() compiled_fn = t.compile_c() if py.test.config.option.view: t.view() assert 'pypy_xyz_f' in t.driver.cbuilder.c_source_filename.read()
def compile_rex(rex): try: from pypy.translator.interactive import Translation except ImportError: py.test.skip("pypy not found") fda = rex.make_automaton().make_deterministic() fda.optimize() fn = fda.make_code() t = Translation(fn) t.backendopt([str], backend="c") if py.test.config.option.view: t.view() return t.compile_c()
def test_entrypoints(): def f(): return 3 key = "test_entrypoints42" @entrypoint(key, [int], "foobar") def g(x): return x + 42 t = Translation(f, [], backend="c", secondaryentrypoints="test_entrypoints42") t.annotate() compiled_fn = t.compile_c() if py.test.config.option.view: t.view() assert 'foobar' in t.driver.cbuilder.c_source_filename.read()
def test_exportstruct(): from pypy.rlib.exports import export_struct def f(): return 42 FOO = Struct("FOO", ("field1", Signed)) foo = malloc(FOO, flavor="raw") foo.field1 = 43 export_struct("BarStruct", foo._obj) t = Translation(f, [], backend="c") t.annotate() compiled_fn = t.compile_c() if py.test.config.option.view: t.view() assert ' BarStruct ' in t.driver.cbuilder.c_source_filename.read() free(foo, flavor="raw")
def test_simple_compile_c(): def f(x, y): return x + y t = Translation(f, [int, int]) t.source(backend='c') t_f = t.compile() res = t_f(2, 3) assert res == 5 t = Translation(f, [int, int]) t_f = t.compile_c() res = t_f(2, 3) assert res == 5
def test_simple_compile_c(): def f(x,y): return x+y t = Translation(f, [int, int]) t.source(backend='c') t_f = t.compile() res = t_f(2,3) assert res == 5 t = Translation(f, [int, int]) t_f = t.compile_c() res = t_f(2,3) assert res == 5
def test_computed_int_symbolic(): too_early = True def compute_fn(): assert not too_early return 7 k = ComputedIntSymbolic(compute_fn) def f(): return k*6 t = Translation(f) t.rtype() if conftest.option.view: t.view() too_early = False fn = t.compile_c() res = fn() assert res == 42
def test_parser(): def f(x): if x: s = "a(X, Y, Z)." else: s = "f(a, X, _, _, X, f(X, 2.455))." term = parsing.parse_file(s) assert isinstance(term, parsing.Nonterminal) return term.symbol assert f(True) == "file" assert f(True) == "file" t = Translation(f) t.annotate([bool]) t.rtype() t.backendopt() func = t.compile_c() assert func(True) == "file" assert func(False) == "file"
def test_compile_recognizer(): try: from pypy.translator.interactive import Translation except ImportError: py.test.skip("pypy not found on path") a = DFA() s0 = a.add_state("start") s1 = a.add_state() s2 = a.add_state(final=True) a[s0, "a"] = s0 a[s0, "c"] = s1 a[s0, "b"] = s2 a[s1, "b"] = s2 recognize = a.make_code() t = Translation(recognize) t.backendopt([str], backend="c") cfn = t.compile_c() assert cfn("aaaaaaaaaab") assert cfn("b") assert cfn("aaaacb")
def test_translate_pypackrat(): from pypy.rlib.parsing.pypackrat import PackratParser class parser(PackratParser): """ expr: additive; additive: a = additive '-' b = multitive return {'(%s - %s)' % (a, b)} | multitive; multitive: a = multitive '*' b = simple return {'(%s * %s)' % (a, b)} | simple; simple: ('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9'); """ print parser._code def parse(s): p = parser(s) return p.expr() res = parse("5-5-5") assert res == '((5 - 5) - 5)' t = Translation(parse) t.annotate([str]) t.rtype() t.backendopt() if option.view: t.view() func = t.compile_c() res = func("5-5-5") assert res == '((5 - 5) - 5)'
def test_translate_simple(self): digits = RangeExpression("0", "9") lower = RangeExpression("a", "z") upper = RangeExpression("A", "Z") keywords = StringExpression("if") | StringExpression( "else") | StringExpression("def") | StringExpression("class") underscore = StringExpression("_") atoms = lower + (upper | lower | digits | underscore).kleene() vars = underscore | (upper + (upper | lower | underscore | digits).kleene()) integers = StringExpression("0") | (RangeExpression("1", "9") + digits.kleene()) white = StringExpression(" ") l1 = self.get_lexer([keywords, atoms, vars, integers, white], ["KEYWORD", "ATOM", "VAR", "INT", "WHITE"]) l2 = self.get_lexer([keywords, atoms, vars, integers, white], ["KEYWORD", "ATOM", "VAR", "INT", "WHITE"], ["WHITE"]) def lex(s, ignore=False): if ignore: tokens = l2.tokenize(s) else: tokens = l1.tokenize(s) return "-%-".join([t.name for t in tokens]) res = lex("if A a 12341 0 else").split("-%-") assert res == ("KEYWORD WHITE VAR WHITE ATOM WHITE INT WHITE " "INT WHITE KEYWORD").split() res = lex("if A a 12341 0 else", True).split("-%-") assert res == "KEYWORD VAR ATOM INT INT KEYWORD".split() t = Translation(lex) t.annotate([str, bool]) t.rtype() func = t.compile_c() res = lex("if A a 12341 0 else", False).split("-%-") assert res == ("KEYWORD WHITE VAR WHITE ATOM WHITE INT WHITE " "INT WHITE KEYWORD").split() res = lex("if A a 12341 0 else", True).split("-%-") assert res == "KEYWORD VAR ATOM INT INT KEYWORD".split()
def test_engine(): e = get_engine(""" g(a, a). g(a, b). g(b, c). f(X, Z) :- g(X, Y), g(Y, Z). """) t1 = parse_query_term("f(a, c).") t2 = parse_query_term("f(X, c).") def run(): e.run(t1) e.run(t2) v0 = e.heap.getvar(0) if isinstance(v0, Atom): return v0.name return "no!" assert run() == "a" t = Translation(run) t.annotate() t.rtype() func = t.compile_c() assert func() == "a"
def test_translate_ast_visitor(): from pypy.rlib.parsing.ebnfparse import parse_ebnf, make_parse_function regexs, rules, ToAST = parse_ebnf(""" DECIMAL: "0|[1-9][0-9]*"; IGNORE: " "; additive: multitive ["+!"] additive | <multitive>; multitive: primary ["*!"] multitive | <primary>; #nonsense! primary: "(" <additive> ")" | <DECIMAL>; """) parse = make_parse_function(regexs, rules) def f(): tree = parse("(0 +! 10) *! (999 +! 10) +! 1") tree = ToAST().visit_additive(tree) assert len(tree) == 1 tree = tree[0] return tree.symbol + " " + "-&-".join([c.symbol for c in tree.children]) res1 = f() t = Translation(f) t.annotate() t.rtype() t.backendopt() func = t.compile_c() res2 = func() assert res1 == res2
def compile(fn, argtypes, view=False, gcpolicy="ref", backendopt=True, annotatorpolicy=None): t = Translation(fn, argtypes, gc=gcpolicy, backend="c", policy=annotatorpolicy) if not backendopt: t.disable(["backendopt_lltype"]) t.annotate() # XXX fish t.driver.config.translation.countmallocs = True compiled_fn = t.compile_c() if conftest.option.view: t.view() # XXX fish fish fish some more module = t.driver.cbuilder.c_ext_module def checking_fn(*args, **kwds): if 'expected_extra_mallocs' in kwds: expected_extra_mallocs = kwds.pop('expected_extra_mallocs') else: expected_extra_mallocs = 0 res = compiled_fn(*args, **kwds) mallocs, frees = module.malloc_counters() assert mallocs - frees == expected_extra_mallocs return res return checking_fn
return True if '--subprocess' in sys.argv: os.putenv('SDL_WINDOWID', sys.argv[-1]) def pypy_entry_point(): def jitpolicy(*args): from pypy.jit.metainterp.policy import JitPolicy return JitPolicy() brain = Brain() brain.loop() if '--pypy' in sys.argv: from pypy.translator.interactive import Translation t = Translation(pypy_entry_point) ## NotImplementedError: --gcrootfinder=asmgcc requires standalone ## #t.config.translation.suggest(jit=True, jit_debug='steps', jit_backend='x86', gc='boehm') t.annotate() t.rtype() f = t.compile_c() f() else: pypy_entry_point() else: a = App() gtk.main() print '-------------------exit toplevel-----------------'
def getcompiled(f, args): t = Translation(f) fn = t.compile_c(args) if conftest.option.view: t.view() return fn, t
if __name__ == '__main__': print 'Loading...' from pypy.translator.interactive import Translation t = Translation(demo) print 'Annotating...' t.annotate([]) t.viewcg() print 'Specializing...' t.rtype() # enable this to see (some) lower-level Cish operations print 'Compiling...' f = t.compile_c() print 'Running...' T = time.time() for i in range(10): f() t1 = time.time() - T print "that took", t1 T = time.time() for i in range(10): demo() t2 = time.time() - T print "compared to", t2 print "a speed-up of", t2/t1