def test_secondary_backendopt(self):
# checks an issue with a newly added graph that calls an
# already-exception-transformed graph. This can occur e.g.
# from a late-seen destructor added by the GC transformer
# which ends up calling existing code.
def common(n):
if n > 5:
raise ValueError
def main(n):
common(n)
def later(n):
try:
common(n)
return 0
except ValueError:
return 1
t = TranslationContext()
t.buildannotator().build_types(main, [int])
t.buildrtyper().specialize()
exctransformer = t.getexceptiontransformer()
exctransformer.create_exception_handling(graphof(t, common))
from rpython.annotator import model as annmodel
from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator
annhelper = MixLevelHelperAnnotator(t.rtyper)
later_graph = annhelper.getgraph(later, [annmodel.SomeInteger()],
annmodel.SomeInteger())
annhelper.finish()
annhelper.backend_optimize()
# ^^^ as the inliner can't handle exception-transformed graphs,
# this should *not* inline common() into later().
if option.view:
later_graph.show()
common_graph = graphof(t, common)
found = False
for block in later_graph.iterblocks():
for op in block.operations:
if (op.opname == 'direct_call'
and op.args[0].value._obj.graph is common_graph):
found = True
assert found, "cannot find the call (buggily inlined?)"
from rpython.rtyper.llinterp import LLInterpreter
llinterp = LLInterpreter(t.rtyper)
res = llinterp.eval_graph(later_graph, [10])
assert res == 1
def test_secondary_backendopt(self):
# checks an issue with a newly added graph that calls an
# already-exception-transformed graph. This can occur e.g.
# from a late-seen destructor added by the GC transformer
# which ends up calling existing code.
def common(n):
if n > 5:
raise ValueError
def main(n):
common(n)
def later(n):
try:
common(n)
return 0
except ValueError:
return 1
t = TranslationContext()
t.buildannotator().build_types(main, [int])
t.buildrtyper().specialize()
exctransformer = t.getexceptiontransformer()
exctransformer.create_exception_handling(graphof(t, common))
from rpython.annotator import model as annmodel
from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator
annhelper = MixLevelHelperAnnotator(t.rtyper)
later_graph = annhelper.getgraph(later, [annmodel.SomeInteger()], annmodel.SomeInteger())
annhelper.finish()
annhelper.backend_optimize()
# ^^^ as the inliner can't handle exception-transformed graphs,
# this should *not* inline common() into later().
if option.view:
later_graph.show()
common_graph = graphof(t, common)
found = False
for block in later_graph.iterblocks():
for op in block.operations:
if op.opname == "direct_call" and op.args[0].value._obj.graph is common_graph:
found = True
assert found, "cannot find the call (buggily inlined?)"
from rpython.rtyper.llinterp import LLInterpreter
llinterp = LLInterpreter(t.rtyper)
res = llinterp.eval_graph(later_graph, [10])
assert res == 1
class BaseGCTransformer(object):
finished_helpers = False
curr_block = None
def __init__(self, translator, inline=False):
self.translator = translator
self.seen_graphs = set()
self.prepared = False
self.minimal_transform = set()
if translator:
self.mixlevelannotator = MixLevelHelperAnnotator(translator.rtyper)
else:
self.mixlevelannotator = None
self.inline = inline
if translator and inline:
self.lltype_to_classdef = translator.rtyper.lltype_to_classdef_mapping()
self.graphs_to_inline = {}
self.graph_dependencies = {}
self.ll_finalizers_ptrs = []
if self.MinimalGCTransformer:
self.minimalgctransformer = self.MinimalGCTransformer(self)
else:
self.minimalgctransformer = None
def get_lltype_of_exception_value(self):
exceptiondata = self.translator.rtyper.exceptiondata
return exceptiondata.lltype_of_exception_value
def need_minimal_transform(self, graph):
self.seen_graphs.add(graph)
self.minimal_transform.add(graph)
def inline_helpers(self, graphs):
from rpython.translator.backendopt.inline import iter_callsites
raise_analyzer = RaiseAnalyzer(self.translator)
for graph in graphs:
to_enum = []
for called, block, i in iter_callsites(graph, None):
if called in self.graphs_to_inline:
to_enum.append(called)
must_constfold = False
for inline_graph in to_enum:
try:
inline.inline_function(self.translator, inline_graph, graph,
self.lltype_to_classdef,
raise_analyzer,
cleanup=False)
must_constfold = True
except inline.CannotInline as e:
print 'CANNOT INLINE:', e
print '\t%s into %s' % (inline_graph, graph)
cleanup_graph(graph)
if must_constfold:
constant_fold_graph(graph)
def compute_borrowed_vars(self, graph):
# the input args are borrowed, and stay borrowed for as long as they
# are not merged with other values.
var_families = DataFlowFamilyBuilder(graph).get_variable_families()
borrowed_reps = {}
for v in graph.getargs():
borrowed_reps[var_families.find_rep(v)] = True
# no support for returning borrowed values so far
retvar = graph.getreturnvar()
def is_borrowed(v1):
return (var_families.find_rep(v1) in borrowed_reps
and v1 is not retvar)
return is_borrowed
def transform_block(self, block, is_borrowed):
llops = LowLevelOpList()
self.curr_block = block
self.livevars = [var for var in block.inputargs
if var_needsgc(var) and not is_borrowed(var)]
allvars = [var for var in block.getvariables() if var_needsgc(var)]
self.var_last_needed_in = dict.fromkeys(allvars, 0)
for i, op in enumerate(block.operations):
for var in op.args:
if not var_needsgc(var):
continue
self.var_last_needed_in[var] = i
for link in block.exits:
for var in link.args:
if not var_needsgc(var):
continue
self.var_last_needed_in[var] = len(block.operations) + 1
for i, op in enumerate(block.operations):
hop = GcHighLevelOp(self, op, i, llops)
hop.dispatch()
if len(block.exits) != 0: # i.e not the return block
assert not block.canraise
deadinallexits = set(self.livevars)
for link in block.exits:
deadinallexits.difference_update(set(link.args))
for var in deadinallexits:
self.pop_alive(var, llops)
for link in block.exits:
livecounts = dict.fromkeys(set(self.livevars) - deadinallexits, 1)
for v, v2 in zip(link.args, link.target.inputargs):
if is_borrowed(v2):
continue
if v in livecounts:
livecounts[v] -= 1
elif var_needsgc(v):
# 'v' is typically a Constant here, but it can be
# a borrowed variable going into a non-borrowed one
livecounts[v] = -1
self.links_to_split[link] = livecounts
block.operations[:] = llops
self.livevars = None
self.var_last_needed_in = None
self.curr_block = None
def transform_graph(self, graph):
if graph in self.minimal_transform:
if self.minimalgctransformer:
self.minimalgctransformer.transform_graph(graph)
self.minimal_transform.remove(graph)
return
if graph in self.seen_graphs:
return
self.seen_graphs.add(graph)
self.links_to_split = {} # link -> vars to pop_alive across the link
# for sanity, we need an empty block at the start of the graph
inserted_empty_startblock = False
if not starts_with_empty_block(graph):
insert_empty_startblock(graph)
inserted_empty_startblock = True
is_borrowed = self.compute_borrowed_vars(graph)
for block in graph.iterblocks():
self.transform_block(block, is_borrowed)
for link, livecounts in self.links_to_split.iteritems():
llops = LowLevelOpList()
for var, livecount in livecounts.iteritems():
for i in range(livecount):
self.pop_alive(var, llops)
for i in range(-livecount):
self.push_alive(var, llops)
if llops:
if link.prevblock.exitswitch is None:
link.prevblock.operations.extend(llops)
else:
insert_empty_block(link, llops)
# remove the empty block at the start of the graph, which should
# still be empty (but let's check)
if starts_with_empty_block(graph) and inserted_empty_startblock:
old_startblock = graph.startblock
graph.startblock = graph.startblock.exits[0].target
checkgraph(graph)
self.links_to_split = None
v = Variable('vanishing_exc_value')
v.concretetype = self.get_lltype_of_exception_value()
llops = LowLevelOpList()
self.pop_alive(v, llops)
graph.exc_cleanup = (v, list(llops))
return is_borrowed # xxx for tests only
def annotate_helper(self, ll_helper, ll_args, ll_result, inline=False):
assert not self.finished_helpers
args_s = map(lltype_to_annotation, ll_args)
s_result = lltype_to_annotation(ll_result)
graph = self.mixlevelannotator.getgraph(ll_helper, args_s, s_result)
# the produced graphs does not need to be fully transformed
self.need_minimal_transform(graph)
if inline:
self.graphs_to_inline[graph] = True
FUNCTYPE = lltype.FuncType(ll_args, ll_result)
return self.mixlevelannotator.graph2delayed(graph, FUNCTYPE=FUNCTYPE)
def inittime_helper(self, ll_helper, ll_args, ll_result, inline=True):
ptr = self.annotate_helper(ll_helper, ll_args, ll_result, inline=inline)
return Constant(ptr, lltype.typeOf(ptr))
def annotate_finalizer(self, ll_finalizer, ll_args, ll_result):
fptr = self.annotate_helper(ll_finalizer, ll_args, ll_result)
self.ll_finalizers_ptrs.append(fptr)
return fptr
def finish_helpers(self, backendopt=True):
if self.translator is not None:
self.mixlevelannotator.finish_annotate()
if self.translator is not None:
self.mixlevelannotator.finish_rtype()
if backendopt:
self.mixlevelannotator.backend_optimize()
self.finished_helpers = True
# Make sure that the database also sees all finalizers now.
# It is likely that the finalizers need special support there
newgcdependencies = self.ll_finalizers_ptrs
return newgcdependencies
def get_finish_helpers(self):
return self.finish_helpers
def finish_tables(self):
pass
def get_finish_tables(self):
return self.finish_tables
def finish(self, backendopt=True):
self.finish_helpers(backendopt=backendopt)
self.finish_tables()
def transform_generic_set(self, hop):
opname = hop.spaceop.opname
v_new = hop.spaceop.args[-1]
v_old = hop.genop('g' + opname[1:],
hop.inputargs()[:-1],
resulttype=v_new.concretetype)
self.push_alive(v_new, hop.llops)
hop.rename('bare_' + opname)
self.pop_alive(v_old, hop.llops)
def push_alive(self, var, llops):
pass
def pop_alive(self, var, llops):
pass
def var_needs_set_transform(self, var):
return False
def default(self, hop):
hop.llops.append(hop.spaceop)
def gct_setfield(self, hop):
if self.var_needs_set_transform(hop.spaceop.args[-1]):
self.transform_generic_set(hop)
else:
hop.rename('bare_' + hop.spaceop.opname)
gct_setarrayitem = gct_setfield
gct_setinteriorfield = gct_setfield
gct_raw_store = gct_setfield
gct_getfield = default
def gct_zero_gc_pointers_inside(self, hop):
pass
def gct_gc_writebarrier_before_copy(self, hop):
# We take the conservative default and return False here, meaning
# that rgc.ll_arraycopy() will do the copy by hand (i.e. with a
# 'for' loop). Subclasses that have their own logic, or that don't
# need any kind of write barrier, may return True.
op = hop.spaceop
hop.genop("same_as",
[rmodel.inputconst(lltype.Bool, False)],
resultvar=op.result)
def gct_gc_pin(self, hop):
op = hop.spaceop
hop.genop("same_as",
[rmodel.inputconst(lltype.Bool, False)],
resultvar=op.result)
def gct_gc_unpin(self, hop):
pass
def gct_gc__is_pinned(self, hop):
op = hop.spaceop
hop.genop("same_as",
[rmodel.inputconst(lltype.Bool, False)],
resultvar=op.result)
def gct_gc_identityhash(self, hop):
# must be implemented in the various GCs
raise NotImplementedError
def gct_gc_id(self, hop):
# this assumes a non-moving GC. Moving GCs need to override this
hop.rename('cast_ptr_to_int')
def gct_gc_heap_stats(self, hop):
from rpython.memory.gc.base import ARRAY_TYPEID_MAP
return hop.cast_result(rmodel.inputconst(lltype.Ptr(ARRAY_TYPEID_MAP),
lltype.nullptr(ARRAY_TYPEID_MAP)))
def get_prebuilt_hash(self, obj):
return None
class BaseGCTransformer(object):
finished_helpers = False
curr_block = None
def __init__(self, translator, inline=False):
self.translator = translator
self.seen_graphs = set()
self.prepared = False
self.minimal_transform = set()
if translator:
self.mixlevelannotator = MixLevelHelperAnnotator(translator.rtyper)
else:
self.mixlevelannotator = None
self.inline = inline
if translator and inline:
self.lltype_to_classdef = translator.rtyper.lltype_to_classdef_mapping()
self.raise_analyzer = RaiseAnalyzer(translator)
self.graphs_to_inline = {}
self.graph_dependencies = {}
self.ll_finalizers_ptrs = []
if self.MinimalGCTransformer:
self.minimalgctransformer = self.MinimalGCTransformer(self)
else:
self.minimalgctransformer = None
def get_lltype_of_exception_value(self):
exceptiondata = self.translator.rtyper.exceptiondata
return exceptiondata.lltype_of_exception_value
def need_minimal_transform(self, graph):
self.seen_graphs.add(graph)
self.minimal_transform.add(graph)
def inline_helpers_into(self, graph):
from rpython.translator.backendopt.inline import iter_callsites
to_enum = []
for called, block, i in iter_callsites(graph, None):
if called in self.graphs_to_inline:
to_enum.append(called)
any_inlining = False
for inline_graph in to_enum:
try:
inline.inline_function(self.translator, inline_graph, graph,
self.lltype_to_classdef,
self.raise_analyzer,
cleanup=False)
any_inlining = True
except inline.CannotInline as e:
print 'CANNOT INLINE:', e
print '\t%s into %s' % (inline_graph, graph)
raise # for now, make it a fatal error
cleanup_graph(graph)
if any_inlining:
constant_fold_graph(graph)
return any_inlining
def inline_helpers_and_postprocess(self, graphs):
for graph in graphs:
any_inlining = self.inline and self.inline_helpers_into(graph)
self.postprocess_graph(graph, any_inlining)
def postprocess_graph(self, graph, any_inlining):
pass
def compute_borrowed_vars(self, graph):
# the input args are borrowed, and stay borrowed for as long as they
# are not merged with other values.
var_families = DataFlowFamilyBuilder(graph).get_variable_families()
borrowed_reps = {}
for v in graph.getargs():
borrowed_reps[var_families.find_rep(v)] = True
# no support for returning borrowed values so far
retvar = graph.getreturnvar()
def is_borrowed(v1):
return (var_families.find_rep(v1) in borrowed_reps
and v1 is not retvar)
return is_borrowed
def transform_block(self, block, is_borrowed):
llops = LowLevelOpList()
self.curr_block = block
self.livevars = [var for var in block.inputargs
if var_needsgc(var) and not is_borrowed(var)]
allvars = [var for var in block.getvariables() if var_needsgc(var)]
self.var_last_needed_in = dict.fromkeys(allvars, 0)
for i, op in enumerate(block.operations):
for var in op.args:
if not var_needsgc(var):
continue
self.var_last_needed_in[var] = i
for link in block.exits:
for var in link.args:
if not var_needsgc(var):
continue
self.var_last_needed_in[var] = len(block.operations) + 1
for i, op in enumerate(block.operations):
hop = GcHighLevelOp(self, op, i, llops)
hop.dispatch()
if len(block.exits) != 0: # i.e not the return block
assert not block.canraise
deadinallexits = set(self.livevars)
for link in block.exits:
deadinallexits.difference_update(set(link.args))
for var in deadinallexits:
self.pop_alive(var, llops)
for link in block.exits:
livecounts = dict.fromkeys(set(self.livevars) - deadinallexits, 1)
for v, v2 in zip(link.args, link.target.inputargs):
if is_borrowed(v2):
continue
if v in livecounts:
livecounts[v] -= 1
elif var_needsgc(v):
# 'v' is typically a Constant here, but it can be
# a borrowed variable going into a non-borrowed one
livecounts[v] = -1
self.links_to_split[link] = livecounts
block.operations[:] = llops
self.livevars = None
self.var_last_needed_in = None
self.curr_block = None
def start_transforming_graph(self, graph):
pass # for asmgcc.py
def transform_graph(self, graph):
if graph in self.minimal_transform:
if self.minimalgctransformer:
self.minimalgctransformer.transform_graph(graph)
self.minimal_transform.remove(graph)
return
if graph in self.seen_graphs:
return
self.seen_graphs.add(graph)
self.start_transforming_graph(graph)
self.links_to_split = {} # link -> vars to pop_alive across the link
# for sanity, we need an empty block at the start of the graph
inserted_empty_startblock = False
if not starts_with_empty_block(graph):
insert_empty_startblock(graph)
inserted_empty_startblock = True
is_borrowed = self.compute_borrowed_vars(graph)
for block in graph.iterblocks():
self.transform_block(block, is_borrowed)
for link, livecounts in self.links_to_split.iteritems():
llops = LowLevelOpList()
for var, livecount in livecounts.iteritems():
for i in range(livecount):
self.pop_alive(var, llops)
for i in range(-livecount):
self.push_alive(var, llops)
if llops:
if link.prevblock.exitswitch is None:
link.prevblock.operations.extend(llops)
else:
insert_empty_block(link, llops)
# remove the empty block at the start of the graph, which should
# still be empty (but let's check)
if starts_with_empty_block(graph) and inserted_empty_startblock:
old_startblock = graph.startblock
graph.startblock = graph.startblock.exits[0].target
checkgraph(graph)
self.links_to_split = None
v = Variable('vanishing_exc_value')
v.concretetype = self.get_lltype_of_exception_value()
llops = LowLevelOpList()
self.pop_alive(v, llops)
graph.exc_cleanup = (v, list(llops))
return is_borrowed # xxx for tests only
def annotate_helper(self, ll_helper, ll_args, ll_result, inline=False):
assert not self.finished_helpers
args_s = map(lltype_to_annotation, ll_args)
s_result = lltype_to_annotation(ll_result)
graph = self.mixlevelannotator.getgraph(ll_helper, args_s, s_result)
# the produced graphs does not need to be fully transformed
self.need_minimal_transform(graph)
if inline:
self.graphs_to_inline[graph] = True
FUNCTYPE = lltype.FuncType(ll_args, ll_result)
return self.mixlevelannotator.graph2delayed(graph, FUNCTYPE=FUNCTYPE)
def inittime_helper(self, ll_helper, ll_args, ll_result, inline=True):
ptr = self.annotate_helper(ll_helper, ll_args, ll_result, inline=inline)
return Constant(ptr, lltype.typeOf(ptr))
def annotate_finalizer(self, ll_finalizer, ll_args, ll_result):
fptr = self.annotate_helper(ll_finalizer, ll_args, ll_result)
self.ll_finalizers_ptrs.append(fptr)
return fptr
def finish_helpers(self, backendopt=True):
if self.translator is not None:
self.mixlevelannotator.finish_annotate()
if self.translator is not None:
self.mixlevelannotator.finish_rtype()
if backendopt:
self.mixlevelannotator.backend_optimize()
self.finished_helpers = True
# Make sure that the database also sees all finalizers now.
# It is likely that the finalizers need special support there
newgcdependencies = self.ll_finalizers_ptrs
return newgcdependencies
def get_finish_helpers(self):
return self.finish_helpers
def finish_tables(self):
pass
def get_finish_tables(self):
return self.finish_tables
def finish(self, backendopt=True):
self.finish_helpers(backendopt=backendopt)
self.finish_tables()
def transform_generic_set(self, hop):
opname = hop.spaceop.opname
v_new = hop.spaceop.args[-1]
v_old = hop.genop('g' + opname[1:],
hop.inputargs()[:-1],
resulttype=v_new.concretetype)
self.push_alive(v_new, hop.llops)
hop.rename('bare_' + opname)
self.pop_alive(v_old, hop.llops)
def push_alive(self, var, llops):
pass
def pop_alive(self, var, llops):
pass
def var_needs_set_transform(self, var):
return False
def default(self, hop):
hop.llops.append(hop.spaceop)
def gct_setfield(self, hop):
if self.var_needs_set_transform(hop.spaceop.args[-1]):
self.transform_generic_set(hop)
else:
hop.rename('bare_' + hop.spaceop.opname)
gct_setarrayitem = gct_setfield
gct_setinteriorfield = gct_setfield
gct_raw_store = gct_setfield
gct_getfield = default
def gct_zero_gc_pointers_inside(self, hop):
pass
def gct_gc_writebarrier_before_copy(self, hop):
# We take the conservative default and return False here, meaning
# that rgc.ll_arraycopy() will do the copy by hand (i.e. with a
# 'for' loop). Subclasses that have their own logic, or that don't
# need any kind of write barrier, may return True.
op = hop.spaceop
hop.genop("same_as",
[rmodel.inputconst(lltype.Bool, False)],
resultvar=op.result)
def gct_gc_pin(self, hop):
op = hop.spaceop
hop.genop("same_as",
[rmodel.inputconst(lltype.Bool, False)],
resultvar=op.result)
def gct_gc_unpin(self, hop):
pass
def gct_gc__is_pinned(self, hop):
op = hop.spaceop
hop.genop("same_as",
[rmodel.inputconst(lltype.Bool, False)],
resultvar=op.result)
def gct_gc_identityhash(self, hop):
# must be implemented in the various GCs
raise NotImplementedError
def gct_gc_id(self, hop):
# this assumes a non-moving GC. Moving GCs need to override this
hop.rename('cast_ptr_to_int')
def gct_gc_heap_stats(self, hop):
from rpython.memory.gc.base import ARRAY_TYPEID_MAP
return hop.cast_result(rmodel.inputconst(lltype.Ptr(ARRAY_TYPEID_MAP),
lltype.nullptr(ARRAY_TYPEID_MAP)))
