def test_prof_inline(self):
if sys.platform == 'win32':
py.test.skip("instrumentation support is unix only for now")
def add(a, b):
return a + b - b + b - b + b - b + b - b + b - b + b - b + b
def entry_point(argv):
tot = 0
x = int(argv[1])
while x > 0:
tot = add(tot, x)
x -= 1
os.write(1, str(tot))
return 0
from pypy.translator.interactive import Translation
t = Translation(entry_point, backend='c', standalone=True)
# no counters
t.backendopt(inline_threshold=100, profile_based_inline="500")
exe = t.compile()
out = py.process.cmdexec("%s 500" % exe)
assert int(out) == 500 * 501 / 2
t = Translation(entry_point, backend='c', standalone=True)
# counters
t.backendopt(inline_threshold=all.INLINE_THRESHOLD_FOR_TEST * 0.5,
profile_based_inline="500")
exe = t.compile()
out = py.process.cmdexec("%s 500" % exe)
assert int(out) == 500 * 501 / 2
def test_profopt(self):
def add(a, b):
return a + b - b + b - b + b - b + b - b + b - b + b - b + b
def entry_point(argv):
tot = 0
x = int(argv[1])
while x > 0:
tot = add(tot, x)
x -= 1
os.write(1, str(tot))
return 0
from pypy.translator.interactive import Translation
# XXX this is mostly a "does not crash option"
t = Translation(entry_point, backend='c', standalone=True, profopt="")
# no counters
t.backendopt()
exe = t.compile()
out = py.process.cmdexec("%s 500" % exe)
assert int(out) == 500 * 501 / 2
t = Translation(entry_point,
backend='c',
standalone=True,
profopt="",
noprofopt=True)
# no counters
t.backendopt()
exe = t.compile()
out = py.process.cmdexec("%s 500" % exe)
assert int(out) == 500 * 501 / 2
def test_profopt_mac_osx_bug(self):
if sys.platform == 'win32':
py.test.skip("no profopt on win32")
def entry_point(argv):
import os
pid = os.fork()
if pid:
os.waitpid(pid, 0)
else:
os._exit(0)
return 0
from pypy.translator.interactive import Translation
# XXX this is mostly a "does not crash option"
t = Translation(entry_point, backend='c', standalone=True, profopt="")
# no counters
t.backendopt()
exe = t.compile()
#py.process.cmdexec(exe)
t = Translation(entry_point,
backend='c',
standalone=True,
profopt="",
noprofopt=True)
# no counters
t.backendopt()
exe = t.compile()
def test_simple_backendopt():
def f(x, y):
return x, y
t = Translation(f, [int, int], backend='c')
t.backendopt()
assert 'backendopt_lltype' in t.driver.done
t = Translation(f, [int, int])
t.backendopt()
assert 'backendopt_lltype' in t.driver.done
def test_simple_source_llvm():
from pypy.translator.llvm.test.runtest import llvm_test
llvm_test()
def f(x, y):
return x + y
t = Translation(f, [int, int], backend='llvm')
t.source(gc='boehm')
assert 'source_llvm' in t.driver.done
t = Translation(f, [int, int])
t.source_llvm()
assert 'source_llvm' in t.driver.done
def test_simple_rtype():
def f(x, y):
return x + y
t = Translation(f, [int, int])
s = t.annotate()
t.rtype()
assert 'rtype_lltype' in t.driver.done
t = Translation(f)
s = t.annotate([int, int])
t.rtype()
assert 'rtype_lltype' in t.driver.done
def test_simple_source():
def f(x, y):
return x, y
t = Translation(f, backend='c')
t.annotate([int, int])
t.source()
assert 'source_c' in t.driver.done
t = Translation(f, [int, int])
t.source_c()
assert 'source_c' in t.driver.done
t = Translation(f, [int, int])
py.test.raises(Exception, "t.source()")
def compile(fn,
argtypes,
view=False,
gcpolicy="ref",
backendopt=True,
annotatorpolicy=None):
t = Translation(fn,
argtypes,
gc=gcpolicy,
backend="c",
policy=annotatorpolicy)
if not backendopt:
t.disable(["backendopt_lltype"])
t.annotate()
# XXX fish
t.driver.config.translation.countmallocs = True
compiled_fn = t.compile_c()
if getattr(py.test.config.option, 'view', False):
t.view()
malloc_counters = t.driver.cbuilder.get_malloc_counters()
def checking_fn(*args, **kwds):
if 'expected_extra_mallocs' in kwds:
expected_extra_mallocs = kwds.pop('expected_extra_mallocs')
else:
expected_extra_mallocs = 0
res = compiled_fn(*args, **kwds)
mallocs, frees = malloc_counters()
if isinstance(expected_extra_mallocs, int):
assert mallocs - frees == expected_extra_mallocs
else:
assert mallocs - frees in expected_extra_mallocs
return res
return checking_fn
def test_annotator_folding():
from pypy.translator.interactive import Translation
gcoption = ChoiceOption('name', 'GC name', ['ref', 'framework'], 'ref')
gcgroup = OptionDescription('gc', '', [gcoption])
descr = OptionDescription('pypy', '', [gcgroup])
config = Config(descr)
def f(x):
if config.gc.name == 'ref':
return x + 1
else:
return 'foo'
t = Translation(f)
t.rtype([int])
block = t.context.graphs[0].startblock
assert len(block.exits[0].target.operations) == 0
assert len(block.operations) == 1
assert len(block.exits) == 1
assert block.operations[0].opname == 'int_add'
assert config._freeze_()
# does not raise, since it does not change the attribute
config.gc.name = "ref"
py.test.raises(TypeError, 'config.gc.name = "framework"')
def _makefunc_str_int(cls, f):
def main(argv):
arg0 = argv[1]
arg1 = int(argv[2])
try:
res = f(arg0, arg1)
except MemoryError:
print "MEMORY-ERROR"
else:
print res
return 0
t = Translation(main, standalone=True, gc=cls.gcpolicy,
policy=annpolicy.StrictAnnotatorPolicy(),
taggedpointers=cls.taggedpointers,
gcremovetypeptr=cls.removetypeptr)
t.disable(['backendopt'])
t.set_backend_extra_options(c_debug_defines=True)
t.rtype()
if conftest.option.view:
t.viewcg()
exename = t.compile()
def run(s, i):
data = py.process.cmdexec("%s %s %d" % (exename, s, i))
data = data.strip()
if data == 'MEMORY-ERROR':
raise MemoryError
return data
return run
def test_lib():
def entry_point(argv):
fd = os.open("/tmp/foobar", os.O_RDONLY, 0777)
assert fd == 77
res = os.read(fd, 123)
assert res == "he\x00llo"
count = os.write(fd, "world\x00!\x00")
assert count == 42
for arg in argv:
count = os.write(fd, arg)
assert count == 61
os.close(fd)
return 0
t = Translation(entry_point, backend='c', standalone=True, sandbox=True)
exe = t.compile()
proc = MySandboxedProc([exe, 'x1', 'y2'],
expected=[
("open", ("/tmp/foobar", os.O_RDONLY, 0777),
77),
("read", (77, 123), "he\x00llo"),
("write", (77, "world\x00!\x00"), 42),
("write", (77, exe), 61),
("write", (77, "x1"), 61),
("write", (77, "y2"), 61),
("close", (77, ), None),
])
proc.handle_forever()
assert proc.seen == len(proc.expected)
def test_translate_pypackrat_regex():
from pypy.rlib.parsing.pypackrat import PackratParser
class parser(PackratParser):
"""
num:
`([1-9][0-9]*)|0`;
"""
print parser._code
def parse(s):
p = parser(s)
return p.num()
res = parse("1234")
assert res == '1234'
t = Translation(parse)
t.annotate([str])
t.rtype()
t.backendopt()
if option.view:
t.view()
func = t.compile_c()
res = func("12345")
assert res == '12345'
res = func("0")
assert res == '0'
def test_translate_ast_visitor():
from pypy.rlib.parsing.ebnfparse import parse_ebnf, make_parse_function
regexs, rules, ToAST = parse_ebnf("""
DECIMAL: "0|[1-9][0-9]*";
IGNORE: " ";
additive: multitive ["+!"] additive | <multitive>;
multitive: primary ["*!"] multitive | <primary>; #nonsense!
primary: "(" <additive> ")" | <DECIMAL>;
""")
parse = make_parse_function(regexs, rules)
def f():
tree = parse("(0 +! 10) *! (999 +! 10) +! 1")
tree = ToAST().visit_additive(tree)
assert len(tree) == 1
tree = tree[0]
return tree.symbol + " " + "-&-".join(
[c.symbol for c in tree.children])
res1 = f()
t = Translation(f)
t.annotate()
t.rtype()
t.backendopt()
func = t.compile_c()
res2 = func()
assert res1 == res2
def test_simple_compile_c():
def f(x, y):
return x + y
t = Translation(f, [int, int])
t.source(backend='c')
t_f = t.compile()
res = t_f(2, 3)
assert res == 5
t = Translation(f, [int, int])
t_f = t.compile_c()
res = t_f(2, 3)
assert res == 5
def test_translate_compiled_parser():
r0 = Rule("expression", [["additive", "EOF"]])
r1 = Rule("additive", [["multitive", "+", "additive"], ["multitive"]])
r2 = Rule("multitive", [["primary", "*", "multitive"], ["primary"]])
r3 = Rule("primary", [["(", "additive", ")"], ["decimal"]])
r4 = Rule("decimal", [[symb] for symb in "0123456789"])
p = PackratParser([r0, r1, r2, r3, r4], "expression")
compiler = ParserCompiler(p)
kls = compiler.compile()
p = kls()
tree = p.parse([
Token(c, i, SourcePos(i, 0, i))
for i, c in enumerate(list("2*(3+4)") + ["EOF"])
])
data = [
Token(c, i, SourcePos(i, 0, i))
for i, c in enumerate(list("2*(3+4)") + ["EOF"])
]
print tree
p = kls()
def parse(choose):
tree = p.parse(data)
return tree.symbol + " " + "-%-".join(
[c.symbol for c in tree.children])
t = Translation(parse)
t.annotate([bool])
t.backendopt()
t.rtype()
func = t.compile_c()
res1 = parse(True)
res2 = func(True)
assert res1 == res2
def test_disable_logic():
def f(x, y):
return x + y
t = Translation(f, [int, int])
t.disable(['backendopt'])
t.source_c()
assert 'backendopt' not in t.driver.done
def run(self, entry_point, args, expected):
t = Translation(entry_point, backend='c', standalone=True,
sandbox=True)
exe = t.compile()
from pypy.translator.sandbox.sandlib import SimpleIOSandboxedProc
proc = SimpleIOSandboxedProc([exe] + args)
output, error = proc.communicate()
assert error == ''
assert output == expected
def build_adi(function, types):
t = Translation(function)
t.rtype(types)
if conftest.option.view:
t.view()
adi = AbstractDataFlowInterpreter(t.context)
graph = graphof(t.context, function)
adi.schedule_function(graph)
adi.complete()
return t.context, adi, graph
def test_translate():
from pypy.translator.interactive import Translation
def f(x, y):
rnd = Random(x)
rnd.init_by_array([x, y])
rnd.jumpahead(intmask(y))
return rnd.genrand32(), rnd.random()
t = Translation(f)
fc = t.compile_c([r_uint, r_uint])
assert fc(r_uint(1), r_uint(2)) == f(r_uint(1), r_uint(2))
def test_tagged_boehm():
t = Translation(entry_point, standalone=True, gc='boehm')
try:
exename = t.compile_c()
finally:
if conftest.option.view:
t.view()
g = os.popen(exename, 'r')
data = g.read()
g.close()
assert data.rstrip().endswith('ALL OK')
def test_simple_annotate():
def f(x, y):
return x + y
t = Translation(f, [int, int])
assert t.context is t.driver.translator
assert t.config is t.driver.config is t.context.config
s = t.annotate([int, int])
assert s.knowntype == int
t = Translation(f, [int, int])
s = t.annotate()
assert s.knowntype == int
t = Translation(f)
s = t.annotate([int, int])
assert s.knowntype == int
t = Translation(f, [int, int])
py.test.raises(Exception, "t.annotate([int, float])")
def test_name():
def f():
return 3
f.c_name = 'pypy_xyz_f'
t = Translation(f, [], backend="c")
t.annotate()
compiled_fn = t.compile_c()
if py.test.config.option.view:
t.view()
assert 'pypy_xyz_f' in t.driver.cbuilder.c_source_filename.read()
def main():
import_benchmarks()
benchmarks = []
for name, thing in globals().iteritems():
if getattr(thing, 'benchmark', False):
benchmarks.append((name, thing))
benchmarks.sort()
def entry_point(argv):
for name, func in benchmarks:
print name, ':', func()
return 0
t = Translation(entry_point, standalone=True, backend='c')
c_exe = t.compile()
t = Translation(entry_point, standalone=True, backend='cli')
cli_exe = t.compile()
t = Translation(entry_point, standalone=True, backend='jvm')
jvm_exe = t.compile()
c_res = run_benchmark(c_exe)
cli_res = run_benchmark(cli_exe)
jvm_res = run_benchmark(jvm_exe)
print 'benchmark genc gencli cli_ratio genjvm jvm_ratio'
print
for name, _ in benchmarks:
c_time = c_res[name]
cli_time = cli_res[name]
jvm_time = jvm_res[name]
if c_time == 0:
cli_ratio = '%10s' % '---'
else:
cli_ratio = '%10.2f' % (cli_time / c_time)
if c_time == 0:
jvm_ratio = '%10s' % '---'
else:
jvm_ratio = '%10.2f' % (jvm_time / c_time)
print '%-32s %10.2f %10.2f %s %10.2f %s' % (
name, c_time, cli_time, cli_ratio, jvm_time, jvm_ratio)
def compile_rex(rex):
try:
from pypy.translator.interactive import Translation
except ImportError:
py.test.skip("pypy not found")
fda = rex.make_automaton().make_deterministic()
fda.optimize()
fn = fda.make_code()
t = Translation(fn)
t.backendopt([str], backend="c")
if py.test.config.option.view:
t.view()
return t.compile_c()
def test_time():
def entry_point(argv):
t = time.time()
os.dup(int(t*1000))
return 0
t = Translation(entry_point, backend='c', standalone=True, sandbox=True)
exe = t.compile()
g, f = os.popen2(exe, "t", 0)
expect(f, g, "ll_time.ll_time_time", (), 3.141592)
expect(f, g, "ll_os.ll_os_dup", (3141,), 3)
g.close()
tail = f.read()
f.close()
assert tail == ""
def test_dup2_access():
def entry_point(argv):
os.dup2(34, 56)
y = os.access("spam", 77)
return 1 - y
t = Translation(entry_point, backend='c', standalone=True, sandbox=True)
exe = t.compile()
g, f = os.popen2(exe, "t", 0)
expect(f, g, "ll_os.ll_os_dup2", (34, 56), None)
expect(f, g, "ll_os.ll_os_access", ("spam", 77), True)
g.close()
tail = f.read()
f.close()
assert tail == ""
def test_enforced_args():
from pypy.annotation.model import s_None
from pypy.rpython.annlowlevel import MixLevelHelperAnnotator
from pypy.translator.interactive import Translation
def f1():
str2charp("hello")
def f2():
str2charp("world")
t = Translation(f1, [])
t.rtype()
mixann = MixLevelHelperAnnotator(t.context.rtyper)
mixann.getgraph(f2, [], s_None)
mixann.finish()
def test_simple_compile_c_isolate():
from pypy.tool import isolate
def f(x, y):
return x + y
t = Translation(f, [int, int])
t.set_backend_extra_options(c_isolated=True)
t_f = t.compile()
assert isinstance(t_f, isolate.IsolateInvoker)
res = t_f(2, 3)
assert res == 5
# cleanup
t_f.close_isolate()
def test_socketio():
class SocketProc(VirtualizedSocketProc, SimpleIOSandboxedProc):
def build_virtual_root(self):
pass
def entry_point(argv):
fd = os.open("tcp://google.com:80", os.O_RDONLY, 0777)
os.write(fd, 'GET /\n')
print os.read(fd, 30)
return 0
t = Translation(entry_point, backend='c', standalone=True, sandbox=True)
exe = t.compile()
proc = SocketProc([exe])
output, error = proc.communicate("")
assert output.startswith('<HTML><HEAD>')
def test_open_dup():
def entry_point(argv):
fd = os.open("/tmp/foobar", os.O_RDONLY, 0777)
assert fd == 77
fd2 = os.dup(fd)
assert fd2 == 78
return 0
t = Translation(entry_point, backend='c', standalone=True, sandbox=True)
exe = t.compile()
g, f = os.popen2(exe, "t", 0)
expect(f, g, "ll_os.ll_os_open", ("/tmp/foobar", os.O_RDONLY, 0777), 77)
expect(f, g, "ll_os.ll_os_dup", (77,), 78)
g.close()
tail = f.read()
f.close()
assert tail == ""