def test_data_flow_families():
def snippet_fn(xx, yy):
while yy > 0:
if 0 < xx:
yy = yy - xx
else:
yy = yy + xx
return yy
t = TranslationContext()
graph = t.buildflowgraph(snippet_fn)
operations = []
for block in graph.iterblocks():
operations += block.operations
variable_families = DataFlowFamilyBuilder(graph).get_variable_families()
# we expect to find xx only once:
v_xx = variable_families.find_rep(graph.getargs()[0])
found = 0
for op in operations:
if op.opname in ('add', 'sub', 'lt'):
assert variable_families.find_rep(op.args[1]) == v_xx
found += 1
assert found == 3
def test_pseudohighlevelcallable():
t = TranslationContext()
t.buildannotator()
rtyper = t.buildrtyper()
rtyper.specialize()
a = MixLevelHelperAnnotator(rtyper)
class A:
value = 5
def double(self):
return self.value * 2
def fn1(a):
a2 = A()
a2.value = a.double()
return a2
s_A, r_A = a.s_r_instanceof(A)
fn1ptr = a.delayedfunction(fn1, [s_A], s_A)
pseudo = PseudoHighLevelCallable(fn1ptr, [s_A], s_A)
def fn2(n):
a = A()
a.value = n
a2 = pseudo(a)
return a2.value
graph = a.getgraph(fn2, [annmodel.SomeInteger()], annmodel.SomeInteger())
a.finish()
llinterp = LLInterpreter(rtyper)
res = llinterp.eval_graph(graph, [21])
assert res == 42
def compile(f, gc, **kwds):
from rpython.annotator.listdef import s_list_of_strings
from rpython.translator.translator import TranslationContext
from rpython.jit.metainterp.warmspot import apply_jit
from rpython.translator.c import genc
#
t = TranslationContext()
t.config.translation.gc = gc
if gc != 'boehm':
t.config.translation.gcremovetypeptr = True
for name, value in kwds.items():
setattr(t.config.translation, name, value)
ann = t.buildannotator()
ann.build_types(f, [s_list_of_strings], main_entry_point=True)
t.buildrtyper().specialize()
if kwds['jit']:
patch = get_functions_to_patch()
old_value = {}
try:
for (obj, attr), value in patch.items():
old_value[obj, attr] = getattr(obj, attr)
setattr(obj, attr, value)
#
apply_jit(t)
#
finally:
for (obj, attr), oldvalue in old_value.items():
setattr(obj, attr, oldvalue)
cbuilder = genc.CStandaloneBuilder(t, f, t.config)
cbuilder.generate_source(defines=cbuilder.DEBUG_DEFINES)
cbuilder.compile()
return cbuilder
def make_deallocator(TYPE,
attr="static_deallocation_funcptr_for_type",
cls=RefcountingGCTransformer):
if TYPE._is_varsize():
def f():
return lltype.malloc(TYPE, 1)
else:
def f():
return lltype.malloc(TYPE)
t = TranslationContext()
t.buildannotator().build_types(f, [])
t.buildrtyper().specialize()
transformer = cls(t)
fptr = getattr(transformer, attr)(TYPE)
transformer.transform_graph(graphof(t, f))
transformer.finish(backendopt=False)
if option.view:
t.view()
if fptr:
return fptr._obj.graph, t
else:
return None, t
def check(self, fn, signature, args, expected_result, must_be_removed=True,
inline=None):
remover = self.MallocRemover()
t = TranslationContext()
t.buildannotator().build_types(fn, signature)
t.buildrtyper().specialize()
graph = graphof(t, fn)
if inline is not None:
from rpython.translator.backendopt.inline import auto_inline_graphs
auto_inline_graphs(t, t.graphs, inline)
if option.view:
t.view()
# to detect broken intermediate graphs,
# we do the loop ourselves instead of calling remove_simple_mallocs()
while True:
progress = remover.remove_mallocs_once(graph)
simplify.transform_dead_op_vars_in_blocks(list(graph.iterblocks()),
[graph])
if progress and option.view:
t.view()
if expected_result is not Ellipsis:
interp = LLInterpreter(t.rtyper)
res = interp.eval_graph(graph, args)
assert res == expected_result
if not progress:
break
if must_be_removed:
self.check_malloc_removed(graph)
return graph
def test_remove_same_as_nonconst():
from rpython.rlib.nonconst import NonConstant
from rpython.rtyper.lltypesystem.lloperation import llop
from rpython.rtyper.lltypesystem import lltype
def f():
if NonConstant(False):
x = llop.same_as(lltype.Signed, 666)
return 42
t = TranslationContext()
t.buildannotator().build_types(f, [])
t.buildrtyper().specialize()
f_graph = graphof(t, f)
#simple_inline_function(t, nothing, f_graph)
# here, the graph looks like v21=same_as(True); exitswitch: v21
remove_same_as(f_graph)
t.checkgraphs()
# only one path should be left
for block in f_graph.iterblocks():
assert len(block.exits) <= 1
for block in t.annotator.annotated:
assert None not in block.operations
interp = LLInterpreter(t.rtyper)
result = interp.eval_graph(f_graph, [])
assert result == 42
def __init__(self,
translator=None,
policy=None,
bookkeeper=None,
keepgoing=False):
import rpython.rtyper.extfuncregistry # has side effects
if translator is None:
# interface for tests
from rpython.translator.translator import TranslationContext
translator = TranslationContext()
translator.annotator = self
self.translator = translator
self.pendingblocks = ShuffleDict() # map {block: graph-containing-it}
self.annotated = {} # set of blocks already seen
self.added_blocks = None # see processblock() below
self.links_followed = {} # set of links that have ever been followed
self.notify = {} # {block: {positions-to-reflow-from-when-done}}
self.fixed_graphs = {} # set of graphs not to annotate again
self.blocked_blocks = {} # set of {blocked_block: (graph, index)}
# --- the following information is recorded for debugging ---
self.blocked_graphs = {} # set of graphs that have blocked blocks
# --- end of debugging information ---
self.frozen = False
if policy is None:
from rpython.annotator.policy import AnnotatorPolicy
self.policy = AnnotatorPolicy()
else:
self.policy = policy
if bookkeeper is None:
bookkeeper = Bookkeeper(self)
self.bookkeeper = bookkeeper
self.keepgoing = keepgoing
self.failed_blocks = set()
self.errors = []
def test_replace_exitswitch_by_constant_bug():
class X:
pass
def constant9():
x = X()
x.n = 3
x.n = 9
return x.n
def fn():
n = constant9()
if n == 1: return 5
elif n == 2: return 6
elif n == 3: return 8
elif n == 4: return -123
elif n == 5: return 12973
else: return n
t = TranslationContext()
a = t.buildannotator()
a.build_types(fn, [])
rtyper = t.buildrtyper()
rtyper.specialize()
graph = t.graphs[0]
remove_same_as(graph)
merge_if_blocks_once(graph)
from rpython.translator.backendopt import malloc, inline
inline.auto_inlining(t, 20)
malloc.remove_mallocs(t, t.graphs)
from rpython.translator import simplify
simplify.join_blocks(graph)
def rtype(func,
inputtypes,
specialize=True,
gcname='ref',
backendopt=False,
**extraconfigopts):
from rpython.translator.translator import TranslationContext
t = TranslationContext()
# XXX XXX XXX mess
t.config.translation.gc = gcname
t.config.translation.gcremovetypeptr = True
t.config.set(**extraconfigopts)
ann = t.buildannotator()
ann.build_types(func, inputtypes)
rtyper = t.buildrtyper()
rtyper.backend = llinterp_backend
if specialize:
rtyper.specialize()
if backendopt:
from rpython.translator.backendopt.all import backend_optimizations
backend_optimizations(t)
if option.view:
t.viewcg()
return t
def annotate_at(f, policy=None):
t = TranslationContext()
t.config.translation.check_str_without_nul = True
a = t.buildannotator(policy=policy)
a.annotate_helper(f, [model.s_ImpossibleValue] * f.__code__.co_argcount,
policy=policy)
return a
def test_merge_several():
def merge(n, m):
r = -1
if n == 0:
if m == 0:
r = 0
elif m == 1:
r = 1
else:
r = 2
elif n == 1:
r = 4
else:
r = 6
return r
t = TranslationContext()
a = t.buildannotator()
a.build_types(merge, [int, int])
rtyper = t.buildrtyper()
rtyper.specialize()
graph = tgraphof(t, merge)
remove_same_as(graph)
merge_if_blocks(graph)
assert len(graph.startblock.exits) == 3
assert len(list(graph.iterblocks())) == 3
interp = LLInterpreter(rtyper)
for m in range(3):
res = interp.eval_graph(graph, [0, m])
assert res == m
res = interp.eval_graph(graph, [1, 0])
assert res == 4
res = interp.eval_graph(graph, [2, 0])
assert res == 6
def analyze(self, func, sig):
t = TranslationContext()
t.buildannotator().build_types(func, sig)
t.buildrtyper().specialize()
fgraph = graphof(t, func)
return VirtualizableAnalyzer(t).analyze(
fgraph.startblock.operations[0])
def translate(func, argtypes, backendopt=False):
t = TranslationContext()
t.buildannotator().build_types(func, argtypes)
t.buildrtyper(type_system='ootype').specialize()
if backendopt: backend_optimizations(t, merge_if_blocks=True)
return t
def makegraph(func, argtypes):
t = TranslationContext()
t.buildannotator().build_types(func, [int])
t.buildrtyper().specialize()
bk = t.annotator.bookkeeper
graph = bk.getdesc(func).getuniquegraph()
return t, graph
def rtype(self, fn, argtypes, resulttype, checkfunction=None):
t = TranslationContext()
a = t.buildannotator()
a.build_types(prefn, [int])
typer = t.buildrtyper()
typer.specialize()
#t.view()
s_result = a.typeannotation(resulttype)
from rpython.rtyper import annlowlevel
# annotate, normalize and rtype fn after the fact
annhelper = annlowlevel.MixLevelHelperAnnotator(typer)
graph = annhelper.getgraph(
fn, [a.typeannotation(argtype) for argtype in argtypes], s_result)
annhelper.finish()
t.checkgraphs()
if checkfunction is not None:
checkfunction(t)
# sanity check prefn
llinterp = LLInterpreter(typer)
res = llinterp.eval_graph(graphof(t, prefn), [1])
assert res == 100
res = llinterp.eval_graph(graphof(t, prefn), [2])
assert res == 201
return t
def test_del_basic(self):
py.test.skip("xxx fix or kill")
S = lltype.GcStruct('S', ('x', lltype.Signed), rtti=True)
TRASH = lltype.GcStruct('TRASH', ('x', lltype.Signed))
GLOBAL = lltype.Struct('GLOBAL', ('x', lltype.Signed))
glob = lltype.malloc(GLOBAL, immortal=True)
def destructor(s):
glob.x = s.x + 1
def type_info_S(s):
return lltype.getRuntimeTypeInfo(S)
def g(n):
s = lltype.malloc(S)
s.x = n
# now 's' should go away
def entrypoint(n):
g(n)
# llop.gc__collect(lltype.Void)
return glob.x
t = TranslationContext()
t.buildannotator().build_types(entrypoint, [int])
rtyper = t.buildrtyper()
destrptr = rtyper.annotate_helper_fn(destructor, [lltype.Ptr(S)])
rtyper.attachRuntimeTypeInfoFunc(S, type_info_S, destrptr=destrptr)
rtyper.specialize()
fn = self.compile_func(entrypoint, None, t)
res = fn(123)
assert res == 124
def test_remove_same_as():
def nothing(x):
return x
def f():
nothing(False)
if nothing(True):
return 42
else:
return 666
t = TranslationContext()
t.buildannotator().build_types(f, [])
t.buildrtyper().specialize()
# now we make the 'if True' appear
f_graph = graphof(t, f)
simple_inline_function(t, nothing, f_graph)
# here, the graph looks like v21=same_as(True); exitswitch: v21
remove_same_as(f_graph)
t.checkgraphs()
# only one path should be left
for block in f_graph.iterblocks():
assert len(block.exits) <= 1
interp = LLInterpreter(t.rtyper)
result = interp.eval_graph(f_graph, [])
assert result == 42
def _build_gen(func, annotation, graph=None, backendopt=True, exctrans=False,
annotatorpolicy=None, nowrap=False):
try:
func = func.im_func
except AttributeError:
pass
t = TranslationContext()
if graph is not None:
graph.func = func
ann = t.buildannotator(policy=annotatorpolicy)
inputcells = [ann.typeannotation(a) for a in annotation]
ann.build_graph_types(graph, inputcells)
t.graphs.insert(0, graph)
else:
ann = t.buildannotator(policy=annotatorpolicy)
ann.build_types(func, annotation)
if getoption('view'):
t.view()
t.buildrtyper(type_system="ootype").specialize()
if backendopt:
check_virtual_methods(ootype.ROOT)
backend_optimizations(t)
main_graph = t.graphs[0]
if getoption('view'):
t.view()
return _build_gen_from_graph(main_graph, t, exctrans, nowrap)
def test_type_erase(self):
class A(object):
pass
class B(object):
pass
def f():
d = {}
d[A()] = B()
d2 = {}
d2[B()] = A()
return d, d2
t = TranslationContext()
s = t.buildannotator().build_types(f, [])
rtyper = t.buildrtyper()
rtyper.specialize()
s_AB_dic = s.items[0]
s_BA_dic = s.items[1]
r_AB_dic = rtyper.getrepr(s_AB_dic)
r_BA_dic = rtyper.getrepr(s_AB_dic)
assert r_AB_dic.lowleveltype == r_BA_dic.lowleveltype
def test_lookup_graphs_abstract():
from rpython.translator.translator import TranslationContext, graphof
class A:
pass
class B(A):
def foo(self):
pass
class C(A):
def foo(self):
pass
def fn(flag):
obj = flag and B() or C()
obj.foo()
return obj
t = TranslationContext()
t.buildannotator().build_types(fn, [int])
t.buildrtyper(type_system='ootype').specialize()
graph = graphof(t, fn)
TYPE_A = graph.getreturnvar().concretetype
TYPE_B = TYPE_A._subclasses[0]
TYPE_C = TYPE_A._subclasses[1]
assert len(TYPE_A._lookup_graphs('ofoo')) == 2
assert len(TYPE_B._lookup_graphs('ofoo')) == 1
assert len(TYPE_C._lookup_graphs('ofoo')) == 1
def _get_TranslationContext(self):
t = TranslationContext()
t.config.translation.gc = DEFL_GC # 'hybrid' or 'minimark'
t.config.translation.gcrootfinder = 'shadowstack'
t.config.translation.list_comprehension_operations = True
t.config.translation.gcremovetypeptr = True
return t
def translate(self, func, sig):
t = TranslationContext()
t.buildannotator().build_types(func, sig)
t.buildrtyper().specialize()
if option.view:
t.view()
return t, self.Analyzer(t)
def test_dont_remove_with__del__(self):
import os
delcalls = [0]
class A(object):
nextid = 0
def __init__(self):
self.id = self.nextid
self.nextid += 1
def __del__(self):
delcalls[0] += 1
#os.write(1, "__del__\n")
def f(x=int):
a = A()
i = 0
while i < x:
a = A()
os.write(1, str(delcalls[0]) + "\n")
i += 1
return 1
t = TranslationContext()
t.buildannotator().build_types(f, [int])
t.buildrtyper().specialize()
graph = graphof(t, f)
backend_optimizations(t)
op = graph.startblock.exits[0].target.exits[1].target.operations[0]
assert op.opname == "malloc"
def gengraph(func,
argtypes=[],
viewbefore='auto',
policy=None,
backendopt=False,
config=None,
**extraconfigopts):
t = TranslationContext(config=config)
t.config.set(**extraconfigopts)
a = t.buildannotator(policy=policy)
a.build_types(func, argtypes, main_entry_point=True)
a.validate()
if viewbefore == 'auto':
viewbefore = getattr(option, 'view', False)
if viewbefore:
a.simplify()
t.view()
global typer # we need it for find_exception
typer = t.buildrtyper()
typer.backend = llinterp_backend
typer.specialize()
#t.view()
t.checkgraphs()
if backendopt:
from rpython.translator.backendopt.all import backend_optimizations
backend_optimizations(t)
t.checkgraphs()
if viewbefore:
t.view()
desc = t.annotator.bookkeeper.getdesc(func)
graph = desc.specialize(argtypes)
return t, typer, graph
def translates(self, func=None, argtypes=None, seeobj_w=[], **kwds):
config = make_config(None, **kwds)
if func is not None:
if argtypes is None:
nb_args = func.func_code.co_argcount
argtypes = [W_Root] * nb_args
#
t = TranslationContext(config=config)
self.t = t # for debugging
ann = t.buildannotator()
def _do_startup():
self.threadlocals.enter_thread(self)
W_SliceObject(w_some_obj(), w_some_obj(), w_some_obj())
ann.build_types(_do_startup, [], complete_now=False)
if func is not None:
ann.build_types(func, argtypes, complete_now=False)
if seeobj_w:
def seeme(n):
return seeobj_w[n]
ann.build_types(seeme, [int], complete_now=False)
#
# annotate all _seen_extras, knowing that annotating some may
# grow the list
done = 0
while done < len(self._seen_extras):
#print self._seen_extras
ann.build_types(self._seen_extras[done], [],
complete_now=False)
ann.complete_pending_blocks()
done += 1
ann.complete()
assert done == len(self._seen_extras)
#t.viewcg()
t.buildrtyper().specialize()
t.checkgraphs()
def test_funny_links():
from rpython.flowspace.model import Block, FunctionGraph, \
Variable, Constant, Link
from rpython.flowspace.operation import op
for i in range(2):
v_i = Variable("i")
block = Block([v_i])
g = FunctionGraph("is_one", block)
op1 = op.eq(v_i, Constant(1))
block.operations.append(op1)
block.exitswitch = op1.result
tlink = Link([Constant(1)], g.returnblock, True)
flink = Link([Constant(0)], g.returnblock, False)
links = [tlink, flink]
if i:
links.reverse()
block.closeblock(*links)
t = TranslationContext()
a = t.buildannotator()
a.build_graph_types(g, [annmodel.SomeInteger()])
rtyper = t.buildrtyper()
rtyper.specialize()
interp = LLInterpreter(rtyper)
assert interp.eval_graph(g, [1]) == 1
assert interp.eval_graph(g, [0]) == 0
def annotate(func,
values,
inline=None,
backendoptimize=True,
translationoptions={}):
# build the normal ll graphs for ll_function
t = TranslationContext()
for key, value in translationoptions.items():
setattr(t.config.translation, key, value)
annpolicy = AnnotatorPolicy()
a = t.buildannotator(policy=annpolicy)
argtypes = getargtypes(a, values)
a.build_types(func, argtypes, main_entry_point=True)
rtyper = t.buildrtyper()
rtyper.specialize()
#if inline:
# auto_inlining(t, threshold=inline)
if backendoptimize:
from rpython.translator.backendopt.all import backend_optimizations
backend_optimizations(t,
inline_threshold=inline or 0,
remove_asserts=True,
really_remove_asserts=True)
return rtyper
def setup(self, entry_point, inputtypes, policy=None, extra={}, empty_translator=None):
standalone = inputtypes is None
self.standalone = standalone
if standalone:
# the 'argv' parameter
inputtypes = [s_list_of_strings]
self.inputtypes = inputtypes
if policy is None:
policy = annpolicy.AnnotatorPolicy()
self.policy = policy
self.extra = extra
if empty_translator:
translator = empty_translator
else:
translator = TranslationContext(config=self.config)
self.entry_point = entry_point
self.translator = translator
self.libdef = None
self.secondary_entrypoints = []
if self.config.translation.secondaryentrypoints:
for key in self.config.translation.secondaryentrypoints.split(","):
try:
points = secondary_entrypoints[key]
except KeyError:
raise KeyError("Entrypoint %r not found (not in %r)" %
(key, secondary_entrypoints.keys()))
self.secondary_entrypoints.extend(points)
self.translator.driver_instrument_result = self.instrument_result
def test_type_erase_var_size(self):
class A(object):
pass
class B(object):
pass
def f():
la = [A()]
lb = [B()]
la.append(None)
lb.append(None)
return la, lb
t = TranslationContext()
s = t.buildannotator().build_types(f, [])
rtyper = t.buildrtyper()
rtyper.specialize()
s_A_list = s.items[0]
s_B_list = s.items[1]
r_A_list = rtyper.getrepr(s_A_list)
assert isinstance(r_A_list, self.rlist.ListRepr)
r_B_list = rtyper.getrepr(s_B_list)
assert isinstance(r_B_list, self.rlist.ListRepr)
assert r_A_list.lowleveltype == r_B_list.lowleveltype
def test_switch_on_symbolic():
symb1 = CDefinedIntSymbolic("1", 1)
symb2 = CDefinedIntSymbolic("2", 2)
symb3 = CDefinedIntSymbolic("3", 3)
def fn(x):
res = 0
if x == symb1:
res += x + 1
elif x == symb2:
res += x + 2
elif x == symb3:
res += x + 3
res += 1
return res
t = TranslationContext()
a = t.buildannotator()
a.build_types(fn, [int])
rtyper = t.buildrtyper()
rtyper.specialize()
graph = t.graphs[0]
remove_same_as(graph)
res = merge_if_blocks_once(graph)
assert not res
checkgraph(graph)
