def rtype(self, fn, argtypes, resulttype, checkfunction=None):
t = TranslationContext()
a = t.buildannotator()
a.build_types(prefn, [int])
typer = t.buildrtyper()
typer.specialize()
#t.view()
s_result = a.typeannotation(resulttype)
from rpython.rtyper import annlowlevel
# annotate, normalize and rtype fn after the fact
annhelper = annlowlevel.MixLevelHelperAnnotator(typer)
graph = annhelper.getgraph(
fn, [a.typeannotation(argtype) for argtype in argtypes], s_result)
annhelper.finish()
t.checkgraphs()
if checkfunction is not None:
checkfunction(t)
# sanity check prefn
llinterp = LLInterpreter(typer)
res = llinterp.eval_graph(graphof(t, prefn), [1])
assert res == 100
res = llinterp.eval_graph(graphof(t, prefn), [2])
assert res == 201
return t
def test_remove_same_as_nonconst():
from rpython.rlib.nonconst import NonConstant
from rpython.rtyper.lltypesystem.lloperation import llop
from rpython.rtyper.lltypesystem import lltype
def f():
if NonConstant(False):
x = llop.same_as(lltype.Signed, 666)
return 42
t = TranslationContext()
t.buildannotator().build_types(f, [])
t.buildrtyper().specialize()
f_graph = graphof(t, f)
#simple_inline_function(t, nothing, f_graph)
# here, the graph looks like v21=same_as(True); exitswitch: v21
remove_same_as(f_graph)
t.checkgraphs()
# only one path should be left
for block in f_graph.iterblocks():
assert len(block.exits) <= 1
for block in t.annotator.annotated:
assert None not in block.operations
interp = LLInterpreter(t.rtyper)
result = interp.eval_graph(f_graph, [])
assert result == 42
def test_remove_same_as():
def nothing(x):
return x
def f():
nothing(False)
if nothing(True):
return 42
else:
return 666
t = TranslationContext()
t.buildannotator().build_types(f, [])
t.buildrtyper().specialize()
# now we make the 'if True' appear
f_graph = graphof(t, f)
simple_inline_function(t, nothing, f_graph)
# here, the graph looks like v21=same_as(True); exitswitch: v21
remove_same_as(f_graph)
t.checkgraphs()
# only one path should be left
for block in f_graph.iterblocks():
assert len(block.exits) <= 1
interp = LLInterpreter(t.rtyper)
result = interp.eval_graph(f_graph, [])
assert result == 42
def translates(self, func=None, argtypes=None, seeobj_w=[], **kwds):
config = make_config(None, **kwds)
if func is not None:
if argtypes is None:
nb_args = func.func_code.co_argcount
argtypes = [W_Root] * nb_args
#
t = TranslationContext(config=config)
self.t = t # for debugging
ann = t.buildannotator()
def _do_startup():
self.threadlocals.enter_thread(self)
W_SliceObject(w_some_obj(), w_some_obj(), w_some_obj())
ann.build_types(_do_startup, [], complete_now=False)
if func is not None:
ann.build_types(func, argtypes, complete_now=False)
if seeobj_w:
def seeme(n):
return seeobj_w[n]
ann.build_types(seeme, [int], complete_now=False)
#
# annotate all _seen_extras, knowing that annotating some may
# grow the list
done = 0
while done < len(self._seen_extras):
#print self._seen_extras
ann.build_types(self._seen_extras[done], [],
complete_now=False)
ann.complete_pending_blocks()
done += 1
ann.complete()
assert done == len(self._seen_extras)
#t.viewcg()
t.buildrtyper().specialize()
t.checkgraphs()
def test_remove_same_as_nonconst():
from rpython.rlib.nonconst import NonConstant
from rpython.rtyper.lltypesystem.lloperation import llop
from rpython.rtyper.lltypesystem import lltype
def f():
if NonConstant(False):
x = llop.same_as(lltype.Signed, 666)
return 42
t = TranslationContext()
t.buildannotator().build_types(f, [])
t.buildrtyper().specialize()
f_graph = graphof(t, f)
# simple_inline_function(t, nothing, f_graph)
# here, the graph looks like v21=same_as(True); exitswitch: v21
remove_same_as(f_graph)
t.checkgraphs()
# only one path should be left
for block in f_graph.iterblocks():
assert len(block.exits) <= 1
for block in t.annotator.annotated:
assert None not in block.operations
interp = LLInterpreter(t.rtyper)
result = interp.eval_graph(f_graph, [])
assert result == 42
def gengraph(func,
argtypes=[],
viewbefore='auto',
policy=None,
backendopt=False,
config=None,
**extraconfigopts):
t = TranslationContext(config=config)
t.config.set(**extraconfigopts)
a = t.buildannotator(policy=policy)
a.build_types(func, argtypes, main_entry_point=True)
a.validate()
if viewbefore == 'auto':
viewbefore = getattr(option, 'view', False)
if viewbefore:
a.simplify()
t.view()
global typer # we need it for find_exception
typer = t.buildrtyper()
typer.backend = llinterp_backend
typer.specialize()
#t.view()
t.checkgraphs()
if backendopt:
from rpython.translator.backendopt.all import backend_optimizations
backend_optimizations(t)
t.checkgraphs()
if viewbefore:
t.view()
desc = t.annotator.bookkeeper.getdesc(func)
graph = desc.specialize(argtypes)
return t, typer, graph
def gengraph(func, argtypes=[], viewbefore='auto', policy=None,
backendopt=False, config=None, **extraconfigopts):
t = TranslationContext(config=config)
t.config.set(**extraconfigopts)
a = t.buildannotator(policy=policy)
a.build_types(func, argtypes, main_entry_point=True)
a.validate()
if viewbefore == 'auto':
viewbefore = getattr(option, 'view', False)
if viewbefore:
a.simplify()
t.view()
global typer # we need it for find_exception
typer = t.buildrtyper()
typer.backend = llinterp_backend
typer.specialize()
#t.view()
t.checkgraphs()
if backendopt:
from rpython.translator.backendopt.all import backend_optimizations
backend_optimizations(t)
t.checkgraphs()
if viewbefore:
t.view()
desc = t.annotator.bookkeeper.getdesc(func)
graph = desc.specialize(argtypes)
return t, typer, graph
def rtype(self, fn, argtypes, resulttype, checkfunction=None):
t = TranslationContext()
a = t.buildannotator()
a.build_types(prefn, [int])
typer = t.buildrtyper()
typer.specialize()
#t.view()
s_result = a.typeannotation(resulttype)
from rpython.rtyper import annlowlevel
# annotate, normalize and rtype fn after the fact
annhelper = annlowlevel.MixLevelHelperAnnotator(typer)
graph = annhelper.getgraph(fn, [a.typeannotation(argtype) for argtype in argtypes],
s_result)
annhelper.finish()
t.checkgraphs()
if checkfunction is not None:
checkfunction(t)
# sanity check prefn
llinterp = LLInterpreter(typer)
res = llinterp.eval_graph(graphof(t, prefn), [1])
assert res == 100
res = llinterp.eval_graph(graphof(t, prefn), [2])
assert res == 201
return t
def check(self, fn, signature, args, expected_result, expected_mallocs=0, expected_calls=0):
t = TranslationContext()
self.translator = t
t.buildannotator().build_types(fn, signature)
t.buildrtyper().specialize()
graph = graphof(t, fn)
if option.view:
t.view()
self.original_graph_count = len(t.graphs)
# to detect broken intermediate graphs,
# we do the loop ourselves instead of calling remove_simple_mallocs()
maxiter = 100
mallocv = MallocVirtualizer(t.graphs, t.rtyper, verbose=True)
while True:
progress = mallocv.remove_mallocs_once()
if progress and option.view:
t.view()
t.checkgraphs()
if expected_result is not DONT_CHECK_RESULT:
interp = LLInterpreter(t.rtyper)
if not isinstance(expected_result, CHECK_RAISES):
res = interp.eval_graph(graph, args)
assert res == expected_result
else:
excinfo = py.test.raises(LLException, interp.eval_graph, graph, args)
assert expected_result.excname in str(excinfo.value)
if not progress:
break
maxiter -= 1
assert maxiter > 0, "infinite loop?"
self.check_malloc_removed(graph, expected_mallocs, expected_calls)
return graph
def test_remove_same_as():
def nothing(x):
return x
def f():
nothing(False)
if nothing(True):
return 42
else:
return 666
t = TranslationContext()
t.buildannotator().build_types(f, [])
t.buildrtyper().specialize()
# now we make the 'if True' appear
f_graph = graphof(t, f)
simple_inline_function(t, nothing, f_graph)
# here, the graph looks like v21=same_as(True); exitswitch: v21
remove_same_as(f_graph)
t.checkgraphs()
# only one path should be left
for block in f_graph.iterblocks():
assert len(block.exits) <= 1
interp = LLInterpreter(t.rtyper)
result = interp.eval_graph(f_graph, [])
assert result == 42
def translates(self, func=None, argtypes=None, seeobj_w=[], **kwds):
config = make_config(None, **kwds)
if func is not None:
if argtypes is None:
nb_args = func.func_code.co_argcount
argtypes = [W_Root] * nb_args
#
t = TranslationContext(config=config)
self.t = t # for debugging
ann = t.buildannotator()
def _do_startup():
self.threadlocals.enter_thread(self)
ann.build_types(_do_startup, [], complete_now=False)
if func is not None:
ann.build_types(func, argtypes, complete_now=False)
if seeobj_w:
def seeme(n):
return seeobj_w[n]
ann.build_types(seeme, [int], complete_now=False)
#
# annotate all _seen_extras, knowing that annotating some may
# grow the list
done = 0
while done < len(self._seen_extras):
#print self._seen_extras
ann.build_types(self._seen_extras[done], [],
complete_now=False)
ann.complete_pending_blocks()
done += 1
ann.complete()
assert done == len(self._seen_extras)
#t.viewcg()
t.buildrtyper().specialize()
t.checkgraphs()
def rtype(fn, argtypes=[]):
t = TranslationContext()
t.buildannotator().build_types(fn, argtypes)
typer = t.buildrtyper()
typer.specialize()
#t.view()
t.checkgraphs()
return t
def rtype(fn, argtypes=[]):
t = TranslationContext()
t.buildannotator().build_types(fn, argtypes)
typer = t.buildrtyper()
typer.specialize()
#t.view()
t.checkgraphs()
return t
def rtype(self, fn, argtypes, resulttype):
t = TranslationContext()
a = t.buildannotator()
s = a.build_types(fn, argtypes)
assert s == a.typeannotation(resulttype)
typer = t.buildrtyper()
typer.specialize()
#t.view()
t.checkgraphs()
return t
def rtype(self, fn, argtypes, resulttype):
t = TranslationContext()
a = t.buildannotator()
s = a.build_types(fn, argtypes)
assert s == a.typeannotation(resulttype)
typer = t.buildrtyper()
typer.specialize()
#t.view()
t.checkgraphs()
return t
def test_retval_None():
def f(x):
pass
t = TranslationContext()
t.buildannotator().build_types(f, [int])
t.buildrtyper().specialize()
#t.view()
t.checkgraphs()
graph = graphof(t, f)
assert graph.getreturnvar().concretetype == Void
assert graph.startblock.exits[0].args[0].concretetype == Void
def test_retval_None():
def f(x):
pass
t = TranslationContext()
t.buildannotator().build_types(f, [int])
t.buildrtyper().specialize()
#t.view()
t.checkgraphs()
graph = graphof(t, f)
assert graph.getreturnvar().concretetype == Void
assert graph.startblock.exits[0].args[0].concretetype == Void
def translates(self,
func=None,
argtypes=None,
seeobj_w=[],
extra_func=None,
c_compile=False,
**kwds):
config = make_config(None, **kwds)
if func is not None:
if argtypes is None:
nb_args = func.func_code.co_argcount
argtypes = [W_Root] * nb_args
#
t = TranslationContext(config=config)
self.t = t # for debugging
ann = t.buildannotator()
def entry_point(argv):
self.threadlocals.enter_thread(self)
W_SliceObject(w_some_obj(), w_some_obj(), w_some_obj())
if extra_func:
extra_func(self)
return 0
ann.build_types(entry_point, [s_list_of_strings], complete_now=False)
if func is not None:
ann.build_types(func, argtypes, complete_now=False)
if seeobj_w:
def seeme(n):
return seeobj_w[n]
ann.build_types(seeme, [int], complete_now=False)
#
# annotate all _seen_extras, knowing that annotating some may
# grow the list
done = 0
while done < len(self._seen_extras):
#print self._seen_extras
ann.build_types(self._seen_extras[done], [], complete_now=False)
ann.complete_pending_blocks()
done += 1
ann.complete()
assert done == len(self._seen_extras)
#t.viewcg()
t.buildrtyper().specialize()
t.checkgraphs()
if c_compile:
cbuilder = CStandaloneBuilder(t, entry_point, t.config)
cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES)
cbuilder.compile()
return t, cbuilder
def test_simple():
t = TranslationContext()
a = t.buildannotator()
a.build_types(g, [int])
a.simplify()
t.buildrtyper().specialize()
backend_optimizations(t)
t.checkgraphs()
n = insert_ll_stackcheck(t)
t.checkgraphs()
assert n == 1
if option.view:
t.view()
check(graphof(t, f), "f")
def test_simple():
t = TranslationContext()
a = t.buildannotator()
a.build_types(g, [int])
a.simplify()
t.buildrtyper().specialize()
backend_optimizations(t)
t.checkgraphs()
n = insert_ll_stackcheck(t)
t.checkgraphs()
assert n == 1
if option.view:
t.view()
check(graphof(t, f), 'f')
def test_remove_unaryops():
# We really want to use remove_unaryops for more complex operations, but
# it's easier to test it with operations on ints here.
def f(x):
i = llop.int_invert(lltype.Signed, x)
i = llop.int_add(lltype.Signed, x, 1)
return llop.int_neg(lltype.Signed, i)
t = TranslationContext()
t.buildannotator().build_types(f, [int])
t.buildrtyper().specialize()
f_graph = graphof(t, f)
remove_unaryops(f_graph, ["int_neg", "int_invert"])
t.checkgraphs()
interp = LLInterpreter(t.rtyper)
result = interp.eval_graph(f_graph, [-2])
assert result == -1
def test_remove_unaryops():
# We really want to use remove_unaryops for more complex operations, but
# it's easier to test it with operations on ints here.
def f(x):
i = llop.int_invert(lltype.Signed, x)
i = llop.int_add(lltype.Signed, x, 1)
return llop.int_neg(lltype.Signed, i)
t = TranslationContext()
t.buildannotator().build_types(f, [int])
t.buildrtyper().specialize()
f_graph = graphof(t, f)
remove_unaryops(f_graph, ["int_neg", "int_invert"])
t.checkgraphs()
interp = LLInterpreter(t.rtyper)
result = interp.eval_graph(f_graph, [-2])
assert result == -1
def test_gctransformed():
t = TranslationContext()
a = t.buildannotator()
a.build_types(g, [int])
a.simplify()
t.buildrtyper().specialize()
backend_optimizations(t)
t.checkgraphs()
n = insert_ll_stackcheck(t)
t.checkgraphs()
assert n == 1
exctransf = t.getexceptiontransformer()
f_graph = graphof(t, f)
exctransf.create_exception_handling(f_graph)
if option.view:
f_graph.show()
check(f_graph, 'f')
class GCTransform(shadowstack.ShadowStackFrameworkGCTransformer):
from rpython.memory.gc.generation import GenerationGC as \
GCClass
GC_PARAMS = {}
gctransf = GCTransform(t)
gctransf.transform_graph(f_graph)
if option.view:
f_graph.show()
relevant = check(f_graph, 'f')
for p in relevant:
in_between = False
reload = 0
for spaceop in p:
if spaceop.opname == 'direct_call':
target = direct_target(spaceop)
if target == 'f':
in_between = False
elif target == 'stack_check___':
in_between = True
if in_between and spaceop.opname == 'gc_reload_possibly_moved':
reload += 1
assert reload == 0
def test_gctransformed():
t = TranslationContext()
a = t.buildannotator()
a.build_types(g, [int])
a.simplify()
t.buildrtyper().specialize()
backend_optimizations(t)
t.checkgraphs()
n = insert_ll_stackcheck(t)
t.checkgraphs()
assert n == 1
exctransf = t.getexceptiontransformer()
f_graph = graphof(t, f)
exctransf.create_exception_handling(f_graph)
if option.view:
f_graph.show()
check(f_graph, "f")
class GCTransform(shadowstack.ShadowStackFrameworkGCTransformer):
from rpython.memory.gc.generation import GenerationGC as GCClass
GC_PARAMS = {}
gctransf = GCTransform(t)
gctransf.transform_graph(f_graph)
if option.view:
f_graph.show()
relevant = check(f_graph, "f")
for p in relevant:
in_between = False
reload = 0
for spaceop in p:
if spaceop.opname == "direct_call":
target = direct_target(spaceop)
if target == "f":
in_between = False
elif target == "stack_check___":
in_between = True
if in_between and spaceop.opname == "gc_reload_possibly_moved":
reload += 1
assert reload == 0
def check(self,
fn,
signature,
args,
expected_result,
expected_mallocs=0,
expected_calls=0):
t = TranslationContext()
self.translator = t
t.buildannotator().build_types(fn, signature)
t.buildrtyper().specialize()
graph = graphof(t, fn)
if option.view:
t.view()
self.original_graph_count = len(t.graphs)
# to detect broken intermediate graphs,
# we do the loop ourselves instead of calling remove_simple_mallocs()
maxiter = 100
mallocv = MallocVirtualizer(t.graphs, t.rtyper, verbose=True)
while True:
progress = mallocv.remove_mallocs_once()
if progress and option.view:
t.view()
t.checkgraphs()
if expected_result is not DONT_CHECK_RESULT:
interp = LLInterpreter(t.rtyper)
if not isinstance(expected_result, CHECK_RAISES):
res = interp.eval_graph(graph, args)
assert res == expected_result
else:
excinfo = py.test.raises(LLException, interp.eval_graph,
graph, args)
assert expected_result.excname in str(excinfo.value)
if not progress:
break
maxiter -= 1
assert maxiter > 0, "infinite loop?"
self.check_malloc_removed(graph, expected_mallocs, expected_calls)
return graph
def _makefunc_str_int(cls, func):
def main(argv):
arg0 = argv[1]
arg1 = int(argv[2])
try:
res = func(arg0, arg1)
except MemoryError:
print 'Result: MemoryError'
else:
print 'Result: "%s"' % (res,)
return 0
config = cls.make_config()
t = TranslationContext(config=config)
a = t.buildannotator()
sec_ep = getattr(cls, 'secondary_entrypoints', [])
for f, inputtypes in sec_ep:
a.build_types(f, inputtypes, False)
a.build_types(main, [s_list_of_strings])
t.buildrtyper().specialize()
t.checkgraphs()
cbuilder = CStandaloneBuilder(t, main, config=config,
secondary_entrypoints=sec_ep)
c_source_filename = cbuilder.generate_source(
defines = cbuilder.DEBUG_DEFINES)
cls._patch_makefile(cbuilder.targetdir)
if option.view:
t.view()
exe_name = cbuilder.compile()
def run(arg0, arg1, runner=None):
if runner is not None:
py.test.skip("unsupported test: runner=%r" % (runner,))
lines = []
print >> sys.stderr, 'RUN: starting', exe_name, arg0, arg1
if sys.platform == 'win32':
redirect = ' 2> NUL'
else:
redirect = ''
if config.translation.shared and os.name == 'posix':
library_path = exe_name.dirpath()
if sys.platform == 'darwin':
env = 'DYLD_LIBRARY_PATH="%s" ' % library_path
else:
env = 'LD_LIBRARY_PATH="%s" ' % library_path
else:
env = ''
cwd = os.getcwd()
try:
os.chdir(str(exe_name.dirpath()))
g = os.popen(
'%s"%s" %s %d%s' % (env, exe_name, arg0, arg1, redirect), 'r')
finally:
os.chdir(cwd)
for line in g:
print >> sys.stderr, 'RUN:', line.rstrip()
lines.append(line)
g.close()
if not lines:
py.test.fail("no output from subprocess")
if not lines[-1].startswith('Result:'):
py.test.fail("unexpected output from subprocess")
result = lines[-1][len('Result:'):].strip()
if result == 'MemoryError':
raise MemoryError("subprocess got an RPython MemoryError")
if result.startswith('"') and result.endswith('"'):
return result[1:-1]
else:
return int(result)
return run
def _test(self, func, types):
t = TranslationContext()
t.buildannotator().build_types(func, types)
t.buildrtyper().specialize()
t.checkgraphs()
def _makefunc_str_int(cls, func):
def main(argv):
arg0 = argv[1]
arg1 = int(argv[2])
try:
res = func(arg0, arg1)
except MemoryError:
print 'Result: MemoryError'
else:
print 'Result: "%s"' % (res,)
return 0
config = cls.make_config()
t = TranslationContext(config=config)
a = t.buildannotator()
sec_ep = getattr(cls, 'secondary_entrypoints', [])
for f, inputtypes in sec_ep:
a.build_types(f, inputtypes, False)
a.build_types(main, [s_list_of_strings])
t.buildrtyper().specialize()
t.checkgraphs()
cbuilder = CStandaloneBuilder(t, main, config=config,
secondary_entrypoints=sec_ep)
c_source_filename = cbuilder.generate_source(
defines = cbuilder.DEBUG_DEFINES)
cls._patch_makefile(cbuilder.targetdir)
if option.view:
t.view()
exe_name = cbuilder.compile()
def run(arg0, arg1):
lines = []
print >> sys.stderr, 'RUN: starting', exe_name, arg0, arg1
if sys.platform == 'win32':
redirect = ' 2> NUL'
else:
redirect = ''
if config.translation.shared and os.name == 'posix':
library_path = exe_name.dirpath()
if sys.platform == 'darwin':
env = 'DYLD_LIBRARY_PATH="%s" ' % library_path
else:
env = 'LD_LIBRARY_PATH="%s" ' % library_path
else:
env = ''
cwd = os.getcwd()
try:
os.chdir(str(exe_name.dirpath()))
g = os.popen(
'%s"%s" %s %d%s' % (env, exe_name, arg0, arg1, redirect), 'r')
finally:
os.chdir(cwd)
for line in g:
print >> sys.stderr, 'RUN:', line.rstrip()
lines.append(line)
g.close()
if not lines:
py.test.fail("no output from subprocess")
if not lines[-1].startswith('Result:'):
py.test.fail("unexpected output from subprocess")
result = lines[-1][len('Result:'):].strip()
if result == 'MemoryError':
raise MemoryError("subprocess got an RPython MemoryError")
if result.startswith('"') and result.endswith('"'):
return result[1:-1]
else:
return int(result)
return run
