def test_translate_compiled_parser():
r0 = Rule("expression", [["additive", "EOF"]])
r1 = Rule("additive", [["multitive", "+", "additive"], ["multitive"]])
r2 = Rule("multitive", [["primary", "*", "multitive"], ["primary"]])
r3 = Rule("primary", [["(", "additive", ")"], ["decimal"]])
r4 = Rule("decimal", [[symb] for symb in "0123456789"])
p = PackratParser([r0, r1, r2, r3, r4], "expression")
compiler = ParserCompiler(p)
kls = compiler.compile()
p = kls()
tree = p.parse([
Token(c, i, SourcePos(i, 0, i))
for i, c in enumerate(list("2*(3+4)") + ["EOF"])
])
data = [
Token(c, i, SourcePos(i, 0, i))
for i, c in enumerate(list("2*(3+4)") + ["EOF"])
]
print tree
p = kls()
def parse(choose):
tree = p.parse(data)
return tree.symbol + " " + "-%-".join(
[c.symbol for c in tree.children])
t = Translation(parse)
t.annotate([bool])
t.backendopt()
t.rtype()
func = t.compile_c()
res1 = parse(True)
res2 = func(True)
assert res1 == res2
def test_translate_pypackrat_regex():
from pypy.rlib.parsing.pypackrat import PackratParser
class parser(PackratParser):
"""
num:
`([1-9][0-9]*)|0`;
"""
print parser._code
def parse(s):
p = parser(s)
return p.num()
res = parse("1234")
assert res == '1234'
t = Translation(parse)
t.annotate([str])
t.rtype()
t.backendopt()
if option.view:
t.view()
func = t.compile_c()
res = func("12345")
assert res == '12345'
res = func("0")
assert res == '0'
def test_annotator_folding():
from pypy.translator.interactive import Translation
gcoption = ChoiceOption('name', 'GC name', ['ref', 'framework'], 'ref')
gcgroup = OptionDescription('gc', '', [gcoption])
descr = OptionDescription('pypy', '', [gcgroup])
config = Config(descr)
def f(x):
if config.gc.name == 'ref':
return x + 1
else:
return 'foo'
t = Translation(f)
t.rtype([int])
block = t.context.graphs[0].startblock
assert len(block.exits[0].target.operations) == 0
assert len(block.operations) == 1
assert len(block.exits) == 1
assert block.operations[0].opname == 'int_add'
assert config._freeze_()
# does not raise, since it does not change the attribute
config.gc.name = "ref"
py.test.raises(TypeError, 'config.gc.name = "framework"')
def _makefunc_str_int(cls, f):
def main(argv):
arg0 = argv[1]
arg1 = int(argv[2])
try:
res = f(arg0, arg1)
except MemoryError:
print "MEMORY-ERROR"
else:
print res
return 0
t = Translation(main, standalone=True, gc=cls.gcpolicy,
policy=annpolicy.StrictAnnotatorPolicy(),
taggedpointers=cls.taggedpointers,
gcremovetypeptr=cls.removetypeptr)
t.disable(['backendopt'])
t.set_backend_extra_options(c_debug_defines=True)
t.rtype()
if conftest.option.view:
t.viewcg()
exename = t.compile()
def run(s, i):
data = py.process.cmdexec("%s %s %d" % (exename, s, i))
data = data.strip()
if data == 'MEMORY-ERROR':
raise MemoryError
return data
return run
def _makefunc_str_int(cls, f):
def main(argv):
arg0 = argv[1]
arg1 = int(argv[2])
try:
res = f(arg0, arg1)
except MemoryError:
print "MEMORY-ERROR"
else:
print res
return 0
t = Translation(main, standalone=True, gc=cls.gcpolicy,
policy=annpolicy.StrictAnnotatorPolicy(),
taggedpointers=cls.taggedpointers,
gcremovetypeptr=cls.removetypeptr)
t.disable(['backendopt'])
t.set_backend_extra_options(c_debug_defines=True)
t.rtype()
if conftest.option.view:
t.viewcg()
exename = t.compile()
def run(s, i):
data = py.process.cmdexec("%s %s %d" % (exename, s, i))
data = data.strip()
if data == 'MEMORY-ERROR':
raise MemoryError
return data
return run
def test_translate_compiled_parser():
r0 = Rule("expression", [["additive", "EOF"]])
r1 = Rule("additive", [["multitive", "+", "additive"], ["multitive"]])
r2 = Rule("multitive", [["primary", "*", "multitive"], ["primary"]])
r3 = Rule("primary", [["(", "additive", ")"], ["decimal"]])
r4 = Rule("decimal", [[symb] for symb in "0123456789"])
p = PackratParser([r0, r1, r2, r3, r4], "expression")
compiler = ParserCompiler(p)
kls = compiler.compile()
p = kls()
tree = p.parse([Token(c, i, SourcePos(i, 0, i))
for i, c in enumerate(list("2*(3+4)") + ["EOF"])])
data = [Token(c, i, SourcePos(i, 0, i))
for i, c in enumerate(list("2*(3+4)") + ["EOF"])]
print tree
p = kls()
def parse(choose):
tree = p.parse(data)
return tree.symbol + " " + "-%-".join([c.symbol for c in tree.children])
t = Translation(parse)
t.annotate([bool])
t.backendopt()
t.rtype()
func = t.compile_c()
res1 = parse(True)
res2 = func(True)
assert res1 == res2
def test_translate_simple(self):
digits = RangeExpression("0", "9")
lower = RangeExpression("a", "z")
upper = RangeExpression("A", "Z")
keywords = StringExpression("if") | StringExpression("else") | StringExpression("def") | StringExpression("class")
underscore = StringExpression("_")
atoms = lower + (upper | lower | digits | underscore).kleene()
vars = underscore | (upper + (upper | lower | underscore | digits).kleene())
integers = StringExpression("0") | (RangeExpression("1", "9") + digits.kleene())
white = StringExpression(" ")
l1 = self.get_lexer([keywords, atoms, vars, integers, white], ["KEYWORD", "ATOM", "VAR", "INT", "WHITE"])
l2 = self.get_lexer([keywords, atoms, vars, integers, white], ["KEYWORD", "ATOM", "VAR", "INT", "WHITE"], ["WHITE"])
def lex(s, ignore=False):
if ignore:
tokens = l2.tokenize(s)
else:
tokens = l1.tokenize(s)
return "-%-".join([t.name for t in tokens])
res = lex("if A a 12341 0 else").split("-%-")
assert res == ("KEYWORD WHITE VAR WHITE ATOM WHITE INT WHITE "
"INT WHITE KEYWORD").split()
res = lex("if A a 12341 0 else", True).split("-%-")
assert res == "KEYWORD VAR ATOM INT INT KEYWORD".split()
t = Translation(lex)
t.annotate([str, bool])
t.rtype()
func = t.compile_c()
res = lex("if A a 12341 0 else", False).split("-%-")
assert res == ("KEYWORD WHITE VAR WHITE ATOM WHITE INT WHITE "
"INT WHITE KEYWORD").split()
res = lex("if A a 12341 0 else", True).split("-%-")
assert res == "KEYWORD VAR ATOM INT INT KEYWORD".split()
def test_translate_ast_visitor():
from pypy.rlib.parsing.ebnfparse import parse_ebnf, make_parse_function
regexs, rules, ToAST = parse_ebnf("""
DECIMAL: "0|[1-9][0-9]*";
IGNORE: " ";
additive: multitive ["+!"] additive | <multitive>;
multitive: primary ["*!"] multitive | <primary>; #nonsense!
primary: "(" <additive> ")" | <DECIMAL>;
""")
parse = make_parse_function(regexs, rules)
def f():
tree = parse("(0 +! 10) *! (999 +! 10) +! 1")
tree = ToAST().visit_additive(tree)
assert len(tree) == 1
tree = tree[0]
return tree.symbol + " " + "-&-".join(
[c.symbol for c in tree.children])
res1 = f()
t = Translation(f)
t.annotate()
t.rtype()
t.backendopt()
func = t.compile_c()
res2 = func()
assert res1 == res2
def test_annotator_folding():
from pypy.translator.interactive import Translation
gcoption = ChoiceOption('name', 'GC name', ['ref', 'framework'], 'ref')
gcgroup = OptionDescription('gc', '', [gcoption])
descr = OptionDescription('pypy', '', [gcgroup])
config = Config(descr)
def f(x):
if config.gc.name == 'ref':
return x + 1
else:
return 'foo'
t = Translation(f)
t.rtype([int])
block = t.context.graphs[0].startblock
assert len(block.exits[0].target.operations) == 0
assert len(block.operations) == 1
assert len(block.exits) == 1
assert block.operations[0].opname == 'int_add'
assert config._freeze_()
# does not raise, since it does not change the attribute
config.gc.name = "ref"
py.test.raises(TypeError, 'config.gc.name = "framework"')
def build_adi(function, types):
t = Translation(function)
t.rtype(types)
if conftest.option.view:
t.view()
adi = AbstractDataFlowInterpreter(t.context)
graph = graphof(t.context, function)
adi.schedule_function(graph)
adi.complete()
return t.context, adi, graph
def build_adi(function, types):
t = Translation(function)
t.rtype(types)
if conftest.option.view:
t.view()
adi = AbstractDataFlowInterpreter(t.context)
graph = graphof(t.context, function)
adi.schedule_function(graph)
adi.complete()
return t.context, adi, graph
def test_enforced_args():
from pypy.annotation.model import s_None
from pypy.rpython.annlowlevel import MixLevelHelperAnnotator
from pypy.translator.interactive import Translation
def f1():
str2charp("hello")
def f2():
str2charp("world")
t = Translation(f1, [])
t.rtype()
mixann = MixLevelHelperAnnotator(t.context.rtyper)
mixann.getgraph(f2, [], s_None)
mixann.finish()
def test_simple_rtype():
def f(x, y):
return x + y
t = Translation(f, [int, int])
s = t.annotate()
t.rtype()
assert "rtype_lltype" in t.driver.done
t = Translation(f)
s = t.annotate([int, int])
t.rtype()
assert "rtype_lltype" in t.driver.done
def test_simple_rtype():
def f(x, y):
return x + y
t = Translation(f, [int, int])
s = t.annotate()
t.rtype()
assert 'rtype_lltype' in t.driver.done
t = Translation(f)
s = t.annotate([int, int])
t.rtype()
assert 'rtype_lltype' in t.driver.done
def test_enforced_args():
from pypy.annotation.model import s_None
from pypy.rpython.annlowlevel import MixLevelHelperAnnotator
from pypy.translator.interactive import Translation
def f1():
str2charp("hello")
def f2():
str2charp("world")
t = Translation(f1, [])
t.rtype()
mixann = MixLevelHelperAnnotator(t.context.rtyper)
mixann.getgraph(f2, [], s_None)
mixann.finish()
def test_computed_int_symbolic():
too_early = True
def compute_fn():
assert not too_early
return 7
k = ComputedIntSymbolic(compute_fn)
def f():
return k*6
t = Translation(f)
t.rtype()
if conftest.option.view:
t.view()
too_early = False
fn = t.compile_c()
res = fn()
assert res == 42
def test_computed_int_symbolic():
too_early = True
def compute_fn():
assert not too_early
return 7
k = ComputedIntSymbolic(compute_fn)
def f():
return k*6
t = Translation(f)
t.rtype()
if conftest.option.view:
t.view()
too_early = False
fn = t.compile_c()
res = fn()
assert res == 42
def test_parser():
def f(x):
if x:
s = "a(X, Y, Z)."
else:
s = "f(a, X, _, _, X, f(X, 2.455))."
term = parsing.parse_file(s)
assert isinstance(term, parsing.Nonterminal)
return term.symbol
assert f(True) == "file"
assert f(True) == "file"
t = Translation(f)
t.annotate([bool])
t.rtype()
t.backendopt()
func = t.compile_c()
assert func(True) == "file"
assert func(False) == "file"
def test_parser():
def f(x):
if x:
s = "a(X, Y, Z)."
else:
s = "f(a, X, _, _, X, f(X, 2.455))."
term = parsing.parse_file(s)
assert isinstance(term, parsing.Nonterminal)
return term.symbol
assert f(True) == "file"
assert f(True) == "file"
t = Translation(f)
t.annotate([bool])
t.rtype()
t.backendopt()
func = t.compile_c()
assert func(True) == "file"
assert func(False) == "file"
def test_inhibit_tail_call():
from pypy.rpython.lltypesystem import lltype
def foobar_fn(n):
return 42
foobar_fn._dont_inline_ = True
def main(n):
return foobar_fn(n)
#
t = Translation(main, [int], backend="c")
t.rtype()
t.context._graphof(foobar_fn).inhibit_tail_call = True
t.source_c()
lines = t.driver.cbuilder.c_source_filename.readlines()
for i, line in enumerate(lines):
if '= pypy_g_foobar_fn' in line:
break
else:
assert 0, "the call was not found in the C source"
assert 'PYPY_INHIBIT_TAIL_CALL();' in lines[i+1]
def test_translate_pypackrat():
from pypy.rlib.parsing.pypackrat import PackratParser
class parser(PackratParser):
"""
expr:
additive;
additive:
a = additive
'-'
b = multitive
return {'(%s - %s)' % (a, b)}
| multitive;
multitive:
a = multitive
'*'
b = simple
return {'(%s * %s)' % (a, b)}
| simple;
simple:
('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9');
"""
print parser._code
def parse(s):
p = parser(s)
return p.expr()
res = parse("5-5-5")
assert res == '((5 - 5) - 5)'
t = Translation(parse)
t.annotate([str])
t.rtype()
t.backendopt()
if option.view:
t.view()
func = t.compile_c()
res = func("5-5-5")
assert res == '((5 - 5) - 5)'
def test_translate_simple(self):
digits = RangeExpression("0", "9")
lower = RangeExpression("a", "z")
upper = RangeExpression("A", "Z")
keywords = StringExpression("if") | StringExpression(
"else") | StringExpression("def") | StringExpression("class")
underscore = StringExpression("_")
atoms = lower + (upper | lower | digits | underscore).kleene()
vars = underscore | (upper +
(upper | lower | underscore | digits).kleene())
integers = StringExpression("0") | (RangeExpression("1", "9") +
digits.kleene())
white = StringExpression(" ")
l1 = self.get_lexer([keywords, atoms, vars, integers, white],
["KEYWORD", "ATOM", "VAR", "INT", "WHITE"])
l2 = self.get_lexer([keywords, atoms, vars, integers, white],
["KEYWORD", "ATOM", "VAR", "INT", "WHITE"],
["WHITE"])
def lex(s, ignore=False):
if ignore:
tokens = l2.tokenize(s)
else:
tokens = l1.tokenize(s)
return "-%-".join([t.name for t in tokens])
res = lex("if A a 12341 0 else").split("-%-")
assert res == ("KEYWORD WHITE VAR WHITE ATOM WHITE INT WHITE "
"INT WHITE KEYWORD").split()
res = lex("if A a 12341 0 else", True).split("-%-")
assert res == "KEYWORD VAR ATOM INT INT KEYWORD".split()
t = Translation(lex)
t.annotate([str, bool])
t.rtype()
func = t.compile_c()
res = lex("if A a 12341 0 else", False).split("-%-")
assert res == ("KEYWORD WHITE VAR WHITE ATOM WHITE INT WHITE "
"INT WHITE KEYWORD").split()
res = lex("if A a 12341 0 else", True).split("-%-")
assert res == "KEYWORD VAR ATOM INT INT KEYWORD".split()
def test_simple_rtype_with_type_system():
def f(x,y):
return x+y
t = Translation(f, [int, int])
s = t.annotate()
t.rtype(type_system='lltype')
assert 'rtype_lltype' in t.driver.done
t = Translation(f, [int, int])
s = t.annotate()
t.rtype(type_system='ootype')
assert 'rtype_ootype' in t.driver.done
t = Translation(f, type_system='ootype')
s = t.annotate([int, int])
t.rtype()
assert 'rtype_ootype' in t.driver.done
t = Translation(f)
s = t.annotate([int, int])
t.rtype(backend='cli')
assert 'rtype_ootype' in t.driver.done
t = Translation(f, backend='cli', type_system='ootype')
s = t.annotate([int, int])
t.rtype()
assert 'rtype_ootype' in t.driver.done
t = Translation(f, type_system='lltype')
s = t.annotate([int, int])
py.test.raises(Exception, "t.rtype(backend='cli')")
t = Translation(f, backend='cli')
s = t.annotate([int, int])
py.test.raises(Exception, "t.rtype(type_system='lltype')")
def test_engine():
e = get_engine("""
g(a, a).
g(a, b).
g(b, c).
f(X, Z) :- g(X, Y), g(Y, Z).
""")
t1 = parse_query_term("f(a, c).")
t2 = parse_query_term("f(X, c).")
def run():
e.run(t1)
e.run(t2)
v0 = e.heap.getvar(0)
if isinstance(v0, Atom):
return v0.name
return "no!"
assert run() == "a"
t = Translation(run)
t.annotate()
t.rtype()
func = t.compile_c()
assert func() == "a"
def test_inhibit_tail_call():
from pypy.rpython.lltypesystem import lltype
def foobar_fn(n):
return 42
foobar_fn._dont_inline_ = True
def main(n):
return foobar_fn(n)
#
t = Translation(main, [int], backend="c")
t.rtype()
t.context._graphof(foobar_fn).inhibit_tail_call = True
t.source_c()
lines = t.driver.cbuilder.c_source_filename.readlines()
for i, line in enumerate(lines):
if '= pypy_g_foobar_fn' in line:
break
else:
assert 0, "the call was not found in the C source"
assert 'PYPY_INHIBIT_TAIL_CALL();' in lines[i + 1]
def test_simple_rtype_with_type_system():
def f(x, y):
return x + y
t = Translation(f, [int, int])
s = t.annotate()
t.rtype(type_system='lltype')
assert 'rtype_lltype' in t.driver.done
t = Translation(f, [int, int])
s = t.annotate()
t.rtype(type_system='ootype')
assert 'rtype_ootype' in t.driver.done
t = Translation(f, type_system='ootype')
s = t.annotate([int, int])
t.rtype()
assert 'rtype_ootype' in t.driver.done
t = Translation(f)
s = t.annotate([int, int])
t.rtype(backend='cli')
assert 'rtype_ootype' in t.driver.done
t = Translation(f, backend='cli', type_system='ootype')
s = t.annotate([int, int])
t.rtype()
assert 'rtype_ootype' in t.driver.done
t = Translation(f, type_system='lltype')
s = t.annotate([int, int])
py.test.raises(Exception, "t.rtype(backend='cli')")
t = Translation(f, backend='cli')
s = t.annotate([int, int])
py.test.raises(Exception, "t.rtype(type_system='lltype')")
def test_translate_pypackrat_regex():
from pypy.rlib.parsing.pypackrat import PackratParser
class parser(PackratParser):
"""
num:
`([1-9][0-9]*)|0`;
"""
print parser._code
def parse(s):
p = parser(s)
return p.num()
res = parse("1234")
assert res == '1234'
t = Translation(parse)
t.annotate([str])
t.rtype()
t.backendopt()
if option.view:
t.view()
func = t.compile_c()
res = func("12345")
assert res == '12345'
res = func("0")
assert res == '0'
def test_translate_ast_visitor():
from pypy.rlib.parsing.ebnfparse import parse_ebnf, make_parse_function
regexs, rules, ToAST = parse_ebnf("""
DECIMAL: "0|[1-9][0-9]*";
IGNORE: " ";
additive: multitive ["+!"] additive | <multitive>;
multitive: primary ["*!"] multitive | <primary>; #nonsense!
primary: "(" <additive> ")" | <DECIMAL>;
""")
parse = make_parse_function(regexs, rules)
def f():
tree = parse("(0 +! 10) *! (999 +! 10) +! 1")
tree = ToAST().visit_additive(tree)
assert len(tree) == 1
tree = tree[0]
return tree.symbol + " " + "-&-".join([c.symbol for c in tree.children])
res1 = f()
t = Translation(f)
t.annotate()
t.rtype()
t.backendopt()
func = t.compile_c()
res2 = func()
assert res1 == res2
def test_translate_pypackrat():
from pypy.rlib.parsing.pypackrat import PackratParser
class parser(PackratParser):
"""
expr:
additive;
additive:
a = additive
'-'
b = multitive
return {'(%s - %s)' % (a, b)}
| multitive;
multitive:
a = multitive
'*'
b = simple
return {'(%s * %s)' % (a, b)}
| simple;
simple:
('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9');
"""
print parser._code
def parse(s):
p = parser(s)
return p.expr()
res = parse("5-5-5")
assert res == '((5 - 5) - 5)'
t = Translation(parse)
t.annotate([str])
t.rtype()
t.backendopt()
if option.view:
t.view()
func = t.compile_c()
res = func("5-5-5")
assert res == '((5 - 5) - 5)'
def test_engine():
e = get_engine("""
g(a, a).
g(a, b).
g(b, c).
f(X, Z) :- g(X, Y), g(Y, Z).
""")
t1 = parse_query_term("f(a, c).")
t2 = parse_query_term("f(X, c).")
def run():
e.run(t1)
e.run(t2)
v0 = e.heap.getvar(0)
if isinstance(v0, Atom):
return v0.name
return "no!"
assert run() == "a"
t = Translation(run)
t.annotate()
t.rtype()
func = t.compile_c()
assert func() == "a"
return True
if '--subprocess' in sys.argv:
os.putenv('SDL_WINDOWID', sys.argv[-1])
def pypy_entry_point():
def jitpolicy(*args):
from pypy.jit.metainterp.policy import JitPolicy
return JitPolicy()
brain = Brain()
brain.loop()
if '--pypy' in sys.argv:
from pypy.translator.interactive import Translation
t = Translation(pypy_entry_point)
## NotImplementedError: --gcrootfinder=asmgcc requires standalone ##
#t.config.translation.suggest(jit=True, jit_debug='steps', jit_backend='x86', gc='boehm')
t.annotate()
t.rtype()
f = t.compile_c()
f()
else:
pypy_entry_point()
else:
a = App()
gtk.main()
print '-------------------exit toplevel-----------------'
# test it
n.test(pat)
if __name__ == '__main__':
print 'Loading...'
from pypy.translator.interactive import Translation
t = Translation(demo)
print 'Annotating...'
t.annotate([])
t.viewcg()
print 'Specializing...'
t.rtype() # enable this to see (some) lower-level Cish operations
print 'Compiling...'
f = t.compile_c()
print 'Running...'
T = time.time()
for i in range(10):
f()
t1 = time.time() - T
print "that took", t1
T = time.time()
for i in range(10):
demo()
t2 = time.time() - T
n.train(pat, 2000)
# test it
n.test(pat)
if __name__ == '__main__':
print 'Loading...'
from pypy.translator.interactive import Translation
t = Translation(demo)
print 'Annotating...'
t.annotate([])
t.viewcg()
print 'Specializing...'
t.rtype() # enable this to see (some) lower-level Cish operations
print 'Compiling...'
f = t.compile_c()
print 'Running...'
T = time.time()
for i in range(10):
f()
t1 = time.time() - T
print "that took", t1
T = time.time()
for i in range(10):
demo()
t2 = time.time() - T
func = getattr(self, func)
func( eval(arg) )
return True
if '--subprocess' in sys.argv:
os.putenv('SDL_WINDOWID', sys.argv[-1])
def pypy_entry_point():
def jitpolicy(*args):
from pypy.jit.metainterp.policy import JitPolicy
return JitPolicy()
brain = Brain()
brain.loop()
if '--pypy' in sys.argv:
from pypy.translator.interactive import Translation
t = Translation( pypy_entry_point )
## NotImplementedError: --gcrootfinder=asmgcc requires standalone ##
#t.config.translation.suggest(jit=True, jit_debug='steps', jit_backend='x86', gc='boehm')
t.annotate()
t.rtype()
f = t.compile_c()
f()
else:
pypy_entry_point()
else:
a = App()
gtk.main()
print '-------------------exit toplevel-----------------'
def make_graph(self, f, arguments):
t = Translation(f)
t.rtype(arguments)
return t.context.graphs[0]