def __init__(self,
translator=None,
policy=None,
bookkeeper=None,
keepgoing=False):
import rpython.rtyper.extfuncregistry # has side effects
if translator is None:
# interface for tests
from rpython.translator.translator import TranslationContext
translator = TranslationContext()
translator.annotator = self
self.translator = translator
self.pendingblocks = ShuffleDict() # map {block: graph-containing-it}
self.annotated = {} # set of blocks already seen
self.added_blocks = None # see processblock() below
self.links_followed = {} # set of links that have ever been followed
self.notify = {} # {block: {positions-to-reflow-from-when-done}}
self.fixed_graphs = {} # set of graphs not to annotate again
self.blocked_blocks = {} # set of {blocked_block: (graph, index)}
# --- the following information is recorded for debugging ---
self.blocked_graphs = {} # set of graphs that have blocked blocks
# --- end of debugging information ---
self.frozen = False
if policy is None:
from rpython.annotator.policy import AnnotatorPolicy
self.policy = AnnotatorPolicy()
else:
self.policy = policy
if bookkeeper is None:
bookkeeper = Bookkeeper(self)
self.bookkeeper = bookkeeper
self.keepgoing = keepgoing
self.failed_blocks = set()
self.errors = []
def build_types(self,
function,
input_arg_types,
complete_now=True,
main_entry_point=False):
"""Recursively build annotations about the specific entry point."""
assert isinstance(function, types.FunctionType), "fix that!"
from rpython.annotator.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
# make input arguments and set their type
args_s = [self.typeannotation(t) for t in input_arg_types]
# XXX hack
annmodel.TLS.check_str_without_nul = (
self.translator.config.translation.check_str_without_nul)
with self.using_policy(policy):
flowgraph, inputs_s = self.get_call_parameters(function, args_s)
if main_entry_point:
self.translator.entry_point_graph = flowgraph
return self.build_graph_types(flowgraph,
inputs_s,
complete_now=complete_now)
def default_specialize(pol, funcdesc, args_s):
name = funcdesc.name
if name.startswith('ll_') or name.startswith('_ll_'): # xxx can we do better?
return super(MixLevelAnnotatorPolicy, pol).default_specialize(
funcdesc, args_s)
else:
return AnnotatorPolicy.default_specialize(funcdesc, args_s)
def test_basic(self):
"""
A ExtFuncEntry provides an annotation for a function, no need to flow
its graph.
"""
def b(x):
"NOT_RPYTHON"
return eval("x+40")
class BTestFuncEntry(ExtFuncEntry):
_about_ = b
name = 'b'
signature_args = [annmodel.SomeInteger()]
signature_result = annmodel.SomeInteger()
def f():
return b(2)
policy = AnnotatorPolicy()
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeInteger)
res = interpret(f, [])
assert res == 42
def annotate(func,
values,
inline=None,
backendoptimize=True,
translationoptions={}):
# build the normal ll graphs for ll_function
t = TranslationContext()
for key, value in translationoptions.items():
setattr(t.config.translation, key, value)
annpolicy = AnnotatorPolicy()
a = t.buildannotator(policy=annpolicy)
argtypes = getargtypes(a, values)
a.build_types(func, argtypes, main_entry_point=True)
rtyper = t.buildrtyper()
rtyper.specialize()
#if inline:
# auto_inlining(t, threshold=inline)
if backendoptimize:
from rpython.translator.backendopt.all import backend_optimizations
backend_optimizations(t,
inline_threshold=inline or 0,
remove_asserts=True,
really_remove_asserts=True)
return rtyper
def test_list_of_str0_unchecked(self):
str0 = SomeString(no_nul=True)
def os_execve(l):
pass
register_external(os_execve, [[str0]], None)
def f(l):
return os_execve(l)
policy = AnnotatorPolicy()
a = RPythonAnnotator(policy=policy)
assert a.translator.config.translation.check_str_without_nul == False
a.build_types(f, [[str]]) # Does not raise
# Now enable the str0 check, and try again with a similar function
a.translator.config.translation.check_str_without_nul=True
def g(l):
return os_execve(l)
with py.test.raises(AnnotatorError):
# fails with TooLateForChange
a.build_types(g, [[str]])
a.build_types(g, [[str0]]) # Does not raise
def default_specialize(self, funcdesc, args_s):
name = funcdesc.name
if name.startswith('ll_') or name.startswith('_ll_'): # xxx can we do better?
return super(MixLevelAnnotatorPolicy, self).default_specialize(
funcdesc, args_s)
else:
return AnnotatorPolicy.default_specialize(funcdesc, args_s)
def __init__(self, translator=None, policy=None, bookkeeper=None):
import rpython.rtyper.extfuncregistry # has side effects
if translator is None:
# interface for tests
from rpython.translator.translator import TranslationContext
translator = TranslationContext()
translator.annotator = self
self.translator = translator
self.pendingblocks = {} # map {block: graph-containing-it}
self.annotated = {} # set of blocks already seen
self.added_blocks = None # see processblock() below
self.links_followed = {} # set of links that have ever been followed
self.notify = {} # {block: {positions-to-reflow-from-when-done}}
self.fixed_graphs = {} # set of graphs not to annotate again
self.blocked_blocks = {} # set of {blocked_block: (graph, index)}
# --- the following information is recorded for debugging ---
self.blocked_graphs = {} # set of graphs that have blocked blocks
# --- end of debugging information ---
self.frozen = False
if policy is None:
from rpython.annotator.policy import AnnotatorPolicy
self.policy = AnnotatorPolicy()
else:
self.policy = policy
if bookkeeper is None:
bookkeeper = Bookkeeper(self)
self.bookkeeper = bookkeeper
def annotate_helper(self, function, args_s, policy=None):
if policy is None:
from rpython.annotator.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
# XXX hack
annmodel.TLS.check_str_without_nul = (
self.translator.config.translation.check_str_without_nul)
graph, inputcells = self.get_call_parameters(function, args_s, policy)
self.build_graph_types(graph, inputcells, complete_now=False)
self.complete_helpers(policy)
return graph
def test_register_external_signature(self):
"""
Test the standard interface for external functions.
"""
def dd():
pass
register_external(dd, [int], int)
def f():
return dd(3)
policy = AnnotatorPolicy()
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeInteger)
def test_register_external_specialcase(self):
"""
When args=None, the external function accepts any arguments unmodified.
"""
def function_withspecialcase(arg):
return repr(arg)
register_external(function_withspecialcase, args=None, result=str)
def f():
x = function_withspecialcase
return x(33) + x("aaa") + x([]) + "\n"
policy = AnnotatorPolicy()
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeString)
def test_list_of_str0(self):
str0 = annmodel.SomeString(no_nul=True)
def os_execve(l):
pass
register_external(os_execve, [[str0]], None)
def f(l):
return os_execve(l)
policy = AnnotatorPolicy()
a = RPythonAnnotator(policy=policy)
a.build_types(f, [[str]]) # Does not raise
assert a.translator.config.translation.check_str_without_nul == False
# Now enable the str0 check, and try again with a similar function
a.translator.config.translation.check_str_without_nul=True
def g(l):
return os_execve(l)
py.test.raises(Exception, a.build_types, g, [[str]])
a.build_types(g, [[str0]]) # Does not raise
def test_str0(self):
str0 = SomeString(no_nul=True)
def os_open(s):
pass
register_external(os_open, [str0], None)
def f(s):
return os_open(s)
policy = AnnotatorPolicy()
a = RPythonAnnotator(policy=policy)
a.build_types(f, [str]) # Does not raise
assert a.translator.config.translation.check_str_without_nul == False
# Now enable the str0 check, and try again with a similar function
a.translator.config.translation.check_str_without_nul=True
def g(s):
return os_open(s)
with py.test.raises(SignatureError):
a.build_types(g, [str])
a.build_types(g, [str0]) # Does not raise
def get_interpreter(func,
values,
view='auto',
viewbefore='auto',
policy=None,
type_system="lltype",
backendopt=False,
config=None,
**extraconfigopts):
extra_key = [(key, value) for key, value in extraconfigopts.iteritems()]
extra_key.sort()
extra_key = tuple(extra_key)
key = ((func, ) + tuple([typeOf(x)
for x in values]) + (backendopt, extra_key))
try:
(t, interp, graph) = _tcache[key]
except KeyError:
def annotation(x):
T = typeOf(x)
return lltype_to_annotation(T)
if policy is None:
from rpython.annotator.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
t, typer, graph = gengraph(func, [annotation(x) for x in values],
viewbefore,
policy,
backendopt=backendopt,
config=config,
**extraconfigopts)
interp = LLInterpreter(typer)
_tcache[key] = (t, interp, graph)
# keep the cache small
_lastinterpreted.append(key)
if len(_lastinterpreted) >= 4:
del _tcache[_lastinterpreted.pop(0)]
if view == 'auto':
view = getattr(option, 'view', False)
if view:
t.view()
return interp, graph
def test_basic(self):
"""
A ExtFuncEntry provides an annotation for a function, no need to flow
its graph.
"""
def b(x):
"NOT_RPYTHON"
return eval("x+40")
register_external(b, [int], result=int)
def f():
return b(2)
policy = AnnotatorPolicy()
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, SomeInteger)
res = interpret(f, [])
assert res == 42
def test_register_external_return_goes_back(self):
"""
Check whether it works to pass the same list from one external
fun to another
[bookkeeper and list joining issues]
"""
def function_with_list():
pass
register_external(function_with_list, [[int]], int)
def function_returning_list():
pass
register_external(function_returning_list, [], [int])
def f():
return function_with_list(function_returning_list())
policy = AnnotatorPolicy()
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, SomeInteger)
def test_register_external_tuple_args(self):
"""
Verify the annotation of a registered external function which takes a
tuple argument.
"""
def function_with_tuple_arg():
"""
Dummy function which is declared via register_external to take a
tuple as an argument so that register_external's behavior for
tuple-taking functions can be verified.
"""
register_external(function_with_tuple_arg, [(int,)], int)
def f():
return function_with_tuple_arg((1,))
policy = AnnotatorPolicy()
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
# Not a very good assertion, but at least it means _something_ happened.
assert isinstance(s, SomeInteger)
class RPythonAnnotator(object):
"""Block annotator for RPython.
See description in doc/translation.txt."""
def __init__(self, translator=None, policy=None, bookkeeper=None,
keepgoing=False):
import rpython.rtyper.extfuncregistry # has side effects
if translator is None:
# interface for tests
from rpython.translator.translator import TranslationContext
translator = TranslationContext()
translator.annotator = self
self.translator = translator
self.pendingblocks = {} # map {block: graph-containing-it}
self.annotated = {} # set of blocks already seen
self.added_blocks = None # see processblock() below
self.links_followed = {} # set of links that have ever been followed
self.notify = {} # {block: {positions-to-reflow-from-when-done}}
self.fixed_graphs = {} # set of graphs not to annotate again
self.blocked_blocks = {} # set of {blocked_block: (graph, index)}
# --- the following information is recorded for debugging ---
self.blocked_graphs = {} # set of graphs that have blocked blocks
# --- end of debugging information ---
self.frozen = False
if policy is None:
from rpython.annotator.policy import AnnotatorPolicy
self.policy = AnnotatorPolicy()
else:
self.policy = policy
if bookkeeper is None:
bookkeeper = Bookkeeper(self)
self.bookkeeper = bookkeeper
self.keepgoing = keepgoing
self.failed_blocks = set()
self.errors = []
def __getstate__(self):
attrs = """translator pendingblocks annotated links_followed
notify bookkeeper frozen policy added_blocks""".split()
ret = self.__dict__.copy()
for key, value in ret.items():
if key not in attrs:
assert type(value) is dict, (
"%r is not dict. please update %s.__getstate__" %
(key, self.__class__.__name__))
ret[key] = {}
return ret
#___ convenience high-level interface __________________
def build_types(self, function, input_arg_types, complete_now=True,
main_entry_point=False):
"""Recursively build annotations about the specific entry point."""
assert isinstance(function, types.FunctionType), "fix that!"
from rpython.annotator.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
# make input arguments and set their type
args_s = [self.typeannotation(t) for t in input_arg_types]
# XXX hack
annmodel.TLS.check_str_without_nul = (
self.translator.config.translation.check_str_without_nul)
with self.using_policy(policy):
flowgraph, inputs_s = self.get_call_parameters(function, args_s)
if main_entry_point:
self.translator.entry_point_graph = flowgraph
return self.build_graph_types(flowgraph, inputs_s, complete_now=complete_now)
def get_call_parameters(self, function, args_s):
with self.bookkeeper.at_position(None):
desc = self.bookkeeper.getdesc(function)
return desc.get_call_parameters(args_s)
def annotate_helper(self, function, args_s, policy=None):
if policy is None:
from rpython.annotator.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
# XXX hack
annmodel.TLS.check_str_without_nul = (
self.translator.config.translation.check_str_without_nul)
with self.using_policy(policy):
graph, inputcells = self.get_call_parameters(function, args_s)
self.build_graph_types(graph, inputcells, complete_now=False)
self.complete_helpers()
return graph
def complete_helpers(self):
saved = self.added_blocks
self.added_blocks = {}
try:
self.complete()
# invoke annotation simplifications for the new blocks
self.simplify(block_subset=self.added_blocks)
finally:
self.added_blocks = saved
@contextmanager
def using_policy(self, policy):
"""A context manager that temporarily replaces the annotator policy"""
old_policy = self.policy
self.policy = policy
yield
self.policy = old_policy
def build_graph_types(self, flowgraph, inputcells, complete_now=True):
checkgraph(flowgraph)
nbarg = len(flowgraph.getargs())
assert len(inputcells) == nbarg # wrong number of args
# register the entry point
self.addpendinggraph(flowgraph, inputcells)
# recursively proceed until no more pending block is left
if complete_now:
self.complete()
return self.annotation(flowgraph.getreturnvar())
def gettype(self, variable):
"""Return the known type of a control flow graph variable,
defaulting to 'object'."""
if isinstance(variable, Constant):
return type(variable.value)
elif isinstance(variable, Variable):
s_variable = variable.annotation
if s_variable:
return s_variable.knowntype
else:
return object
else:
raise TypeError("Variable or Constant instance expected, "
"got %r" % (variable,))
def getuserclassdefinitions(self):
"""Return a list of ClassDefs."""
return self.bookkeeper.classdefs
#___ medium-level interface ____________________________
def addpendinggraph(self, flowgraph, inputcells):
self.addpendingblock(flowgraph, flowgraph.startblock, inputcells)
def addpendingblock(self, graph, block, cells):
"""Register an entry point into block with the given input cells."""
if graph in self.fixed_graphs:
# special case for annotating/rtyping in several phases: calling
# a graph that has already been rtyped. Safety-check the new
# annotations that are passed in, and don't annotate the old
# graph -- it's already low-level operations!
for a, s_newarg in zip(block.inputargs, cells):
s_oldarg = a.annotation
# XXX: Should use s_oldarg.contains(s_newarg) but that breaks
# PyPy translation
if annmodel.unionof(s_oldarg, s_newarg) != s_oldarg:
raise annmodel.AnnotatorError(
"Late-stage annotation is not allowed to modify the "
"existing annotation for variable %s: %s" %
(a, s_oldarg))
else:
assert not self.frozen
if block not in self.annotated:
self.bindinputargs(graph, block, cells)
else:
self.mergeinputargs(graph, block, cells)
if not self.annotated[block]:
self.pendingblocks[block] = graph
def complete_pending_blocks(self):
while self.pendingblocks:
block, graph = self.pendingblocks.popitem()
self.processblock(graph, block)
def complete(self):
"""Process pending blocks until none is left."""
while True:
self.complete_pending_blocks()
self.policy.no_more_blocks_to_annotate(self)
if not self.pendingblocks:
break # finished
# make sure that the return variables of all graphs is annotated
if self.added_blocks is not None:
newgraphs = [self.annotated[block] for block in self.added_blocks]
newgraphs = dict.fromkeys(newgraphs)
got_blocked_blocks = False in newgraphs
else:
newgraphs = self.translator.graphs #all of them
got_blocked_blocks = False in self.annotated.values()
if self.failed_blocks:
text = ('Annotation failed, %s errors were recorded:' %
len(self.errors))
text += '\n-----'.join(str(e) for e in self.errors)
raise annmodel.AnnotatorError(text)
if got_blocked_blocks:
for graph in self.blocked_graphs.values():
self.blocked_graphs[graph] = True
blocked_blocks = [block for block, done in self.annotated.items()
if done is False]
assert len(blocked_blocks) == len(self.blocked_blocks)
text = format_blocked_annotation_error(self, self.blocked_blocks)
#raise SystemExit()
raise annmodel.AnnotatorError(text)
for graph in newgraphs:
v = graph.getreturnvar()
if v.annotation is None:
self.setbinding(v, s_ImpossibleValue)
v = graph.exceptblock.inputargs[1]
if v.annotation is not None and v.annotation.can_be_none():
raise annmodel.AnnotatorError(
"%r is found by annotation to possibly raise None, "
"but the None was not suppressed by the flow space" %
(graph,))
def validate(self):
"""Check that the annotation results are valid"""
self.bookkeeper.check_no_flags_on_instances()
def annotation(self, arg):
"Gives the SomeValue corresponding to the given Variable or Constant."
if isinstance(arg, Variable):
return arg.annotation
elif isinstance(arg, Constant):
return self.bookkeeper.immutablevalue(arg.value)
else:
raise TypeError('Variable or Constant expected, got %r' % (arg,))
def binding(self, arg):
"Gives the SomeValue corresponding to the given Variable or Constant."
s_arg = self.annotation(arg)
if s_arg is None:
raise KeyError
return s_arg
def typeannotation(self, t):
return signature.annotation(t, self.bookkeeper)
def setbinding(self, arg, s_value):
s_old = arg.annotation
if s_old is not None:
if not s_value.contains(s_old):
log.WARNING("%s does not contain %s" % (s_value, s_old))
log.WARNING("%s" % annmodel.union(s_value, s_old))
assert False
arg.annotation = s_value
def warning(self, msg, pos=None):
if pos is None:
try:
pos = self.bookkeeper.position_key
except AttributeError:
pos = '?'
if pos != '?':
pos = self.whereami(pos)
log.WARNING("%s/ %s" % (pos, msg))
#___ interface for annotator.bookkeeper _______
def recursivecall(self, graph, whence, inputcells):
if isinstance(whence, tuple):
parent_graph, parent_block, parent_index = whence
tag = parent_block, parent_index
self.translator.update_call_graph(parent_graph, graph, tag)
# self.notify[graph.returnblock] is a dictionary of call
# points to this func which triggers a reflow whenever the
# return block of this graph has been analysed.
callpositions = self.notify.setdefault(graph.returnblock, {})
if whence is not None:
if callable(whence):
def callback():
whence(self, graph)
else:
callback = whence
callpositions[callback] = True
# generalize the function's input arguments
self.addpendingblock(graph, graph.startblock, inputcells)
# get the (current) return value
v = graph.getreturnvar()
try:
return self.binding(v)
except KeyError:
# the function didn't reach any return statement so far.
# (some functions actually never do, they always raise exceptions)
return s_ImpossibleValue
def reflowfromposition(self, position_key):
graph, block, index = position_key
self.reflowpendingblock(graph, block)
def call_sites(self):
newblocks = self.added_blocks
if newblocks is None:
newblocks = self.annotated # all of them
for block in newblocks:
for op in block.operations:
if op.opname in ('simple_call', 'call_args'):
yield op
# some blocks are partially annotated
if op.result.annotation is None:
break # ignore the unannotated part
#___ simplification (should be moved elsewhere?) _______
def simplify(self, block_subset=None, extra_passes=None):
# Generic simplifications
transform.transform_graph(self, block_subset=block_subset,
extra_passes=extra_passes)
if block_subset is None:
graphs = self.translator.graphs
else:
graphs = {}
for block in block_subset:
graph = self.annotated.get(block)
if graph:
graphs[graph] = True
for graph in graphs:
simplify.eliminate_empty_blocks(graph)
self.bookkeeper.compute_at_fixpoint()
if block_subset is None:
perform_normalizations(self)
#___ flowing annotations in blocks _____________________
def processblock(self, graph, block):
# Important: this is not called recursively.
# self.flowin() can only issue calls to self.addpendingblock().
# The analysis of a block can be in three states:
# * block not in self.annotated:
# never seen the block.
# * self.annotated[block] == False:
# the input variables of the block have bindings but we
# still have to consider all the operations in the block.
# * self.annotated[block] == graph-containing-block:
# analysis done (at least until we find we must generalize the
# input variables).
#print '* processblock', block, cells
self.annotated[block] = graph
if block in self.failed_blocks:
return
if block in self.blocked_blocks:
del self.blocked_blocks[block]
try:
self.flowin(graph, block)
except BlockedInference as e:
self.annotated[block] = False # failed, hopefully temporarily
self.blocked_blocks[block] = (graph, e.opindex)
except Exception as e:
# hack for debug tools only
if not hasattr(e, '__annotator_block'):
setattr(e, '__annotator_block', block)
raise
# The dict 'added_blocks' is used by rpython.annlowlevel to
# detect which are the new blocks that annotating an additional
# small helper creates.
if self.added_blocks is not None:
self.added_blocks[block] = True
def reflowpendingblock(self, graph, block):
assert not self.frozen
assert graph not in self.fixed_graphs
self.pendingblocks[block] = graph
assert block in self.annotated
self.annotated[block] = False # must re-flow
self.blocked_blocks[block] = (graph, None)
def bindinputargs(self, graph, block, inputcells):
# Create the initial bindings for the input args of a block.
assert len(block.inputargs) == len(inputcells)
for a, cell in zip(block.inputargs, inputcells):
self.setbinding(a, cell)
self.annotated[block] = False # must flowin.
self.blocked_blocks[block] = (graph, None)
def mergeinputargs(self, graph, block, inputcells):
# Merge the new 'cells' with each of the block's existing input
# variables.
oldcells = [self.binding(a) for a in block.inputargs]
try:
unions = [annmodel.unionof(c1,c2) for c1, c2 in zip(oldcells,inputcells)]
except annmodel.UnionError as e:
# Add source code to the UnionError
e.source = '\n'.join(source_lines(graph, block, None, long=True))
if self.keepgoing:
self.errors.append(e)
self.failed_blocks.add(block)
return
raise
# if the merged cells changed, we must redo the analysis
if unions != oldcells:
self.bindinputargs(graph, block, unions)
def apply_renaming(self, s_out, renaming):
if hasattr(s_out, 'is_type_of'):
renamed_is_type_of = []
for v in s_out.is_type_of:
renamed_is_type_of += renaming[v]
assert s_out.knowntype is type
newcell = typeof(renamed_is_type_of)
if s_out.is_constant():
newcell.const = s_out.const
s_out = newcell
if hasattr(s_out, 'knowntypedata'):
renamed_knowntypedata = {}
for value, constraints in s_out.knowntypedata.items():
renamed_knowntypedata[value] = {}
for v, s in constraints.items():
new_vs = renaming.get(v, [])
for new_v in new_vs:
renamed_knowntypedata[value][new_v] = s
assert isinstance(s_out, annmodel.SomeBool)
newcell = annmodel.SomeBool()
if s_out.is_constant():
newcell.const = s_out.const
s_out = newcell
s_out.set_knowntypedata(renamed_knowntypedata)
return s_out
def whereami(self, position_key):
graph, block, i = position_key
blk = ""
if block:
at = block.at()
if at:
blk = " block"+at
opid=""
if i is not None:
opid = " op=%d" % i
return repr(graph) + blk + opid
def flowin(self, graph, block):
try:
i = 0
while i < len(block.operations):
op = block.operations[i]
with self.bookkeeper.at_position((graph, block, i)):
new_ops = op.transform(self)
if new_ops is not None:
block.operations[i:i+1] = new_ops
if not new_ops:
continue
new_ops[-1].result = op.result
op = new_ops[0]
self.consider_op(op)
i += 1
except BlockedInference as e:
if e.op is block.raising_op:
# this is the case where the last operation of the block will
# always raise an exception which is immediately caught by
# an exception handler. We then only follow the exceptional
# branches.
exits = [link for link in block.exits
if link.exitcase is not None]
elif e.op.opname in ('simple_call', 'call_args', 'next'):
# XXX warning, keep the name of the call operations in sync
# with the flow object space. These are the operations for
# which it is fine to always raise an exception. We then
# swallow the BlockedInference and that's it.
# About 'next': see test_annotate_iter_empty_container().
return
else:
# other cases are problematic (but will hopefully be solved
# later by reflowing). Throw the BlockedInference up to
# processblock().
e.opindex = i
raise
except annmodel.HarmlesslyBlocked:
return
except annmodel.AnnotatorError as e: # note that UnionError is a subclass
e.source = gather_error(self, graph, block, i)
if self.keepgoing:
self.errors.append(e)
self.failed_blocks.add(block)
return
raise
else:
# dead code removal: don't follow all exits if the exitswitch
# is known
exits = block.exits
if isinstance(block.exitswitch, Variable):
s_exitswitch = self.binding(block.exitswitch)
if s_exitswitch.is_constant():
exits = [link for link in exits
if link.exitcase == s_exitswitch.const]
if block.canraise:
op = block.raising_op
s_exception = self.get_exception(op)
for link in exits:
case = link.exitcase
if case is None:
self.follow_link(graph, link, {})
continue
if s_exception == s_ImpossibleValue:
break
s_case = SomeInstance(self.bookkeeper.getuniqueclassdef(case))
s_matching_exc = intersection(s_exception, s_case)
if s_matching_exc != s_ImpossibleValue:
self.follow_raise_link(graph, link, s_matching_exc)
s_exception = difference(s_exception, s_case)
else:
if isinstance(block.exitswitch, Variable):
knowntypedata = getattr(
block.exitswitch.annotation, "knowntypedata", {})
else:
knowntypedata = {}
for link in exits:
constraints = knowntypedata.get(link.exitcase, {})
self.follow_link(graph, link, constraints)
if block in self.notify:
# reflow from certain positions when this block is done
for callback in self.notify[block]:
if isinstance(callback, tuple):
self.reflowfromposition(callback) # callback is a position
else:
callback()
def follow_link(self, graph, link, constraints):
assert not (isinstance(link.exitcase, (types.ClassType, type)) and
issubclass(link.exitcase, BaseException))
ignore_link = False
inputs_s = []
renaming = defaultdict(list)
for v_out, v_input in zip(link.args, link.target.inputargs):
renaming[v_out].append(v_input)
for v_out in link.args:
s_out = self.annotation(v_out)
if v_out in constraints:
s_constraint = constraints[v_out]
s_out = pair(s_out, s_constraint).improve()
# ignore links that try to pass impossible values
if s_out == s_ImpossibleValue:
ignore_link = True
s_out = self.apply_renaming(s_out, renaming)
inputs_s.append(s_out)
if ignore_link:
return
self.links_followed[link] = True
self.addpendingblock(graph, link.target, inputs_s)
def follow_raise_link(self, graph, link, s_last_exc_value):
v_last_exc_type = link.last_exception
v_last_exc_value = link.last_exc_value
assert (isinstance(link.exitcase, (types.ClassType, type)) and
issubclass(link.exitcase, BaseException))
assert v_last_exc_type and v_last_exc_value
if isinstance(v_last_exc_value, Variable):
self.setbinding(v_last_exc_value, s_last_exc_value)
if isinstance(v_last_exc_type, Variable):
self.setbinding(v_last_exc_type, typeof([v_last_exc_value]))
inputs_s = []
renaming = defaultdict(list)
for v_out, v_input in zip(link.args, link.target.inputargs):
renaming[v_out].append(v_input)
for v_out, v_input in zip(link.args, link.target.inputargs):
if v_out == v_last_exc_type:
s_out = typeof(renaming[v_last_exc_value])
if isinstance(v_last_exc_type, Constant):
s_out.const = v_last_exc_type.value
elif v_last_exc_type.annotation.is_constant():
s_out.const = v_last_exc_type.annotation.const
inputs_s.append(s_out)
else:
s_out = self.annotation(v_out)
s_out = self.apply_renaming(s_out, renaming)
inputs_s.append(s_out)
self.links_followed[link] = True
self.addpendingblock(graph, link.target, inputs_s)
#___ creating the annotations based on operations ______
def consider_op(self, op):
# let's be careful about avoiding propagated SomeImpossibleValues
# to enter an op; the latter can result in violations of the
# more general results invariant: e.g. if SomeImpossibleValue enters is_
# is_(SomeImpossibleValue, None) -> SomeBool
# is_(SomeInstance(not None), None) -> SomeBool(const=False) ...
# boom -- in the assert of setbinding()
for arg in op.args:
if isinstance(self.annotation(arg), annmodel.SomeImpossibleValue):
raise BlockedInference(self, op, -1)
resultcell = op.consider(self)
if resultcell is None:
resultcell = s_ImpossibleValue
elif resultcell == s_ImpossibleValue:
raise BlockedInference(self, op, -1) # the operation cannot succeed
assert isinstance(resultcell, annmodel.SomeObject)
assert isinstance(op.result, Variable)
self.setbinding(op.result, resultcell) # bind resultcell to op.result
def get_exception(self, operation):
"""
Return the annotation for all exceptions that `operation` may raise.
"""
can_only_throw = operation.get_can_only_throw(self)
if can_only_throw is None:
return SomeInstance(self.bookkeeper.getuniqueclassdef(Exception))
else:
return self.bookkeeper.new_exception(can_only_throw)
class RPythonAnnotator(object):
"""Block annotator for RPython.
See description in doc/translation.txt."""
def __init__(self, translator=None, policy=None, bookkeeper=None):
import rpython.rtyper.extfuncregistry # has side effects
if translator is None:
# interface for tests
from rpython.translator.translator import TranslationContext
translator = TranslationContext()
translator.annotator = self
self.translator = translator
self.pendingblocks = {} # map {block: graph-containing-it}
self.annotated = {} # set of blocks already seen
self.added_blocks = None # see processblock() below
self.links_followed = {} # set of links that have ever been followed
self.notify = {} # {block: {positions-to-reflow-from-when-done}}
self.fixed_graphs = {} # set of graphs not to annotate again
self.blocked_blocks = {} # set of {blocked_block: (graph, index)}
# --- the following information is recorded for debugging ---
self.blocked_graphs = {} # set of graphs that have blocked blocks
# --- end of debugging information ---
self.frozen = False
if policy is None:
from rpython.annotator.policy import AnnotatorPolicy
self.policy = AnnotatorPolicy()
else:
self.policy = policy
if bookkeeper is None:
bookkeeper = Bookkeeper(self)
self.bookkeeper = bookkeeper
def __getstate__(self):
attrs = """translator pendingblocks annotated links_followed
notify bookkeeper frozen policy added_blocks""".split()
ret = self.__dict__.copy()
for key, value in ret.items():
if key not in attrs:
assert type(value) is dict, (
"%r is not dict. please update %s.__getstate__" %
(key, self.__class__.__name__))
ret[key] = {}
return ret
#___ convenience high-level interface __________________
def build_types(self, function, input_arg_types, complete_now=True,
main_entry_point=False):
"""Recursively build annotations about the specific entry point."""
assert isinstance(function, types.FunctionType), "fix that!"
from rpython.annotator.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
# make input arguments and set their type
args_s = [self.typeannotation(t) for t in input_arg_types]
# XXX hack
annmodel.TLS.check_str_without_nul = (
self.translator.config.translation.check_str_without_nul)
flowgraph, inputs_s = self.get_call_parameters(function, args_s, policy)
if main_entry_point:
self.translator.entry_point_graph = flowgraph
return self.build_graph_types(flowgraph, inputs_s, complete_now=complete_now)
def get_call_parameters(self, function, args_s, policy):
desc = self.bookkeeper.getdesc(function)
prevpolicy = self.policy
self.policy = policy
self.bookkeeper.enter(None)
try:
return desc.get_call_parameters(args_s)
finally:
self.bookkeeper.leave()
self.policy = prevpolicy
def annotate_helper(self, function, args_s, policy=None):
if policy is None:
from rpython.annotator.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
# XXX hack
annmodel.TLS.check_str_without_nul = (
self.translator.config.translation.check_str_without_nul)
graph, inputcells = self.get_call_parameters(function, args_s, policy)
self.build_graph_types(graph, inputcells, complete_now=False)
self.complete_helpers(policy)
return graph
def complete_helpers(self, policy):
saved = self.policy, self.added_blocks
self.policy = policy
try:
self.added_blocks = {}
self.complete()
# invoke annotation simplifications for the new blocks
self.simplify(block_subset=self.added_blocks)
finally:
self.policy, self.added_blocks = saved
def build_graph_types(self, flowgraph, inputcells, complete_now=True):
checkgraph(flowgraph)
nbarg = len(flowgraph.getargs())
assert len(inputcells) == nbarg # wrong number of args
# register the entry point
self.addpendinggraph(flowgraph, inputcells)
# recursively proceed until no more pending block is left
if complete_now:
self.complete()
return self.annotation(flowgraph.getreturnvar())
def gettype(self, variable):
"""Return the known type of a control flow graph variable,
defaulting to 'object'."""
if isinstance(variable, Constant):
return type(variable.value)
elif isinstance(variable, Variable):
s_variable = variable.annotation
if s_variable:
return s_variable.knowntype
else:
return object
else:
raise TypeError("Variable or Constant instance expected, "
"got %r" % (variable,))
def getuserclassdefinitions(self):
"""Return a list of ClassDefs."""
return self.bookkeeper.classdefs
#___ medium-level interface ____________________________
def addpendinggraph(self, flowgraph, inputcells):
self.addpendingblock(flowgraph, flowgraph.startblock, inputcells)
def addpendingblock(self, graph, block, cells):
"""Register an entry point into block with the given input cells."""
if graph in self.fixed_graphs:
# special case for annotating/rtyping in several phases: calling
# a graph that has already been rtyped. Safety-check the new
# annotations that are passed in, and don't annotate the old
# graph -- it's already low-level operations!
for a, s_newarg in zip(block.inputargs, cells):
s_oldarg = self.binding(a)
assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg
else:
assert not self.frozen
if block not in self.annotated:
self.bindinputargs(graph, block, cells)
else:
self.mergeinputargs(graph, block, cells)
if not self.annotated[block]:
self.pendingblocks[block] = graph
def complete_pending_blocks(self):
while self.pendingblocks:
block, graph = self.pendingblocks.popitem()
self.processblock(graph, block)
def complete(self):
"""Process pending blocks until none is left."""
while True:
self.complete_pending_blocks()
self.policy.no_more_blocks_to_annotate(self)
if not self.pendingblocks:
break # finished
# make sure that the return variables of all graphs is annotated
if self.added_blocks is not None:
newgraphs = [self.annotated[block] for block in self.added_blocks]
newgraphs = dict.fromkeys(newgraphs)
got_blocked_blocks = False in newgraphs
else:
newgraphs = self.translator.graphs #all of them
got_blocked_blocks = False in self.annotated.values()
if got_blocked_blocks:
for graph in self.blocked_graphs.values():
self.blocked_graphs[graph] = True
blocked_blocks = [block for block, done in self.annotated.items()
if done is False]
assert len(blocked_blocks) == len(self.blocked_blocks)
text = format_blocked_annotation_error(self, self.blocked_blocks)
#raise SystemExit()
raise annmodel.AnnotatorError(text)
for graph in newgraphs:
v = graph.getreturnvar()
if v.annotation is None:
self.setbinding(v, s_ImpossibleValue)
def validate(self):
"""Check that the annotation results are valid"""
self.bookkeeper.check_no_flags_on_instances()
def annotation(self, arg):
"Gives the SomeValue corresponding to the given Variable or Constant."
if isinstance(arg, Variable):
return arg.annotation
elif isinstance(arg, Constant):
return self.bookkeeper.immutablevalue(arg.value)
else:
raise TypeError('Variable or Constant expected, got %r' % (arg,))
def binding(self, arg):
"Gives the SomeValue corresponding to the given Variable or Constant."
s_arg = self.annotation(arg)
if s_arg is None:
raise KeyError
return s_arg
def typeannotation(self, t):
return signature.annotation(t, self.bookkeeper)
def setbinding(self, arg, s_value):
s_old = arg.annotation
if s_old is not None:
assert s_value.contains(s_old)
arg.annotation = s_value
def warning(self, msg, pos=None):
if pos is None:
try:
pos = self.bookkeeper.position_key
except AttributeError:
pos = '?'
if pos != '?':
pos = self.whereami(pos)
log.WARNING("%s/ %s" % (pos, msg))
#___ interface for annotator.bookkeeper _______
def recursivecall(self, graph, whence, inputcells):
if isinstance(whence, tuple):
parent_graph, parent_block, parent_index = whence
tag = parent_block, parent_index
self.translator.update_call_graph(parent_graph, graph, tag)
# self.notify[graph.returnblock] is a dictionary of call
# points to this func which triggers a reflow whenever the
# return block of this graph has been analysed.
callpositions = self.notify.setdefault(graph.returnblock, {})
if whence is not None:
if callable(whence):
def callback():
whence(self, graph)
else:
callback = whence
callpositions[callback] = True
# generalize the function's input arguments
self.addpendingblock(graph, graph.startblock, inputcells)
# get the (current) return value
v = graph.getreturnvar()
try:
return self.binding(v)
except KeyError:
# the function didn't reach any return statement so far.
# (some functions actually never do, they always raise exceptions)
return s_ImpossibleValue
def reflowfromposition(self, position_key):
graph, block, index = position_key
self.reflowpendingblock(graph, block)
def call_sites(self):
newblocks = self.added_blocks
if newblocks is None:
newblocks = self.annotated # all of them
for block in newblocks:
for op in block.operations:
if op.opname in ('simple_call', 'call_args'):
yield op
# some blocks are partially annotated
if op.result.annotation is None:
break # ignore the unannotated part
#___ simplification (should be moved elsewhere?) _______
def simplify(self, block_subset=None, extra_passes=None):
# Generic simplifications
transform.transform_graph(self, block_subset=block_subset,
extra_passes=extra_passes)
if block_subset is None:
graphs = self.translator.graphs
else:
graphs = {}
for block in block_subset:
graph = self.annotated.get(block)
if graph:
graphs[graph] = True
for graph in graphs:
simplify.eliminate_empty_blocks(graph)
self.bookkeeper.compute_at_fixpoint()
if block_subset is None:
perform_normalizations(self)
#___ flowing annotations in blocks _____________________
def processblock(self, graph, block):
# Important: this is not called recursively.
# self.flowin() can only issue calls to self.addpendingblock().
# The analysis of a block can be in three states:
# * block not in self.annotated:
# never seen the block.
# * self.annotated[block] == False:
# the input variables of the block have bindings but we
# still have to consider all the operations in the block.
# * self.annotated[block] == graph-containing-block:
# analysis done (at least until we find we must generalize the
# input variables).
#print '* processblock', block, cells
self.annotated[block] = graph
if block in self.blocked_blocks:
del self.blocked_blocks[block]
try:
self.flowin(graph, block)
except BlockedInference, e:
self.annotated[block] = False # failed, hopefully temporarily
self.blocked_blocks[block] = (graph, e.opindex)
except Exception, e:
# hack for debug tools only
if not hasattr(e, '__annotator_block'):
setattr(e, '__annotator_block', block)
raise
class RPythonAnnotator(object):
"""Block annotator for RPython.
See description in doc/translation.txt."""
def __init__(self, translator=None, policy=None, bookkeeper=None):
import rpython.rtyper.extfuncregistry # has side effects
if translator is None:
# interface for tests
from rpython.translator.translator import TranslationContext
translator = TranslationContext()
translator.annotator = self
self.translator = translator
self.pendingblocks = {} # map {block: graph-containing-it}
self.annotated = {} # set of blocks already seen
self.added_blocks = None # see processblock() below
self.links_followed = {} # set of links that have ever been followed
self.notify = {} # {block: {positions-to-reflow-from-when-done}}
self.fixed_graphs = {} # set of graphs not to annotate again
self.blocked_blocks = {} # set of {blocked_block: (graph, index)}
# --- the following information is recorded for debugging ---
self.blocked_graphs = {} # set of graphs that have blocked blocks
# --- end of debugging information ---
self.frozen = False
if policy is None:
from rpython.annotator.policy import AnnotatorPolicy
self.policy = AnnotatorPolicy()
else:
self.policy = policy
if bookkeeper is None:
bookkeeper = Bookkeeper(self)
self.bookkeeper = bookkeeper
def __getstate__(self):
attrs = """translator pendingblocks annotated links_followed
notify bookkeeper frozen policy added_blocks""".split()
ret = self.__dict__.copy()
for key, value in ret.items():
if key not in attrs:
assert type(value) is dict, (
"%r is not dict. please update %s.__getstate__" %
(key, self.__class__.__name__))
ret[key] = {}
return ret
#___ convenience high-level interface __________________
def build_types(self,
function,
input_arg_types,
complete_now=True,
main_entry_point=False):
"""Recursively build annotations about the specific entry point."""
assert isinstance(function, types.FunctionType), "fix that!"
from rpython.annotator.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
# make input arguments and set their type
args_s = [self.typeannotation(t) for t in input_arg_types]
# XXX hack
annmodel.TLS.check_str_without_nul = (
self.translator.config.translation.check_str_without_nul)
flowgraph, inputcells = self.get_call_parameters(
function, args_s, policy)
if not isinstance(flowgraph, FunctionGraph):
assert isinstance(flowgraph, annmodel.SomeObject)
return flowgraph
if main_entry_point:
self.translator.entry_point_graph = flowgraph
return self.build_graph_types(flowgraph,
inputcells,
complete_now=complete_now)
def get_call_parameters(self, function, args_s, policy):
desc = self.bookkeeper.getdesc(function)
args = simple_args(args_s)
result = []
def schedule(graph, inputcells):
result.append((graph, inputcells))
return annmodel.s_ImpossibleValue
prevpolicy = self.policy
self.policy = policy
self.bookkeeper.enter(None)
try:
desc.pycall(schedule, args, annmodel.s_ImpossibleValue)
finally:
self.bookkeeper.leave()
self.policy = prevpolicy
[(graph, inputcells)] = result
return graph, inputcells
def annotate_helper(self, function, args_s, policy=None):
if policy is None:
from rpython.annotator.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
# XXX hack
annmodel.TLS.check_str_without_nul = (
self.translator.config.translation.check_str_without_nul)
graph, inputcells = self.get_call_parameters(function, args_s, policy)
self.build_graph_types(graph, inputcells, complete_now=False)
self.complete_helpers(policy)
return graph
def complete_helpers(self, policy):
saved = self.policy, self.added_blocks
self.policy = policy
try:
self.added_blocks = {}
self.complete()
# invoke annotation simplifications for the new blocks
self.simplify(block_subset=self.added_blocks)
finally:
self.policy, self.added_blocks = saved
def build_graph_types(self, flowgraph, inputcells, complete_now=True):
checkgraph(flowgraph)
nbarg = len(flowgraph.getargs())
assert len(inputcells) == nbarg # wrong number of args
# register the entry point
self.addpendinggraph(flowgraph, inputcells)
# recursively proceed until no more pending block is left
if complete_now:
self.complete()
return self.annotation(flowgraph.getreturnvar())
def gettype(self, variable):
"""Return the known type of a control flow graph variable,
defaulting to 'object'."""
if isinstance(variable, Constant):
return type(variable.value)
elif isinstance(variable, Variable):
s_variable = variable.annotation
if s_variable:
return s_variable.knowntype
else:
return object
else:
raise TypeError("Variable or Constant instance expected, "
"got %r" % (variable, ))
def getuserclassdefinitions(self):
"""Return a list of ClassDefs."""
return self.bookkeeper.classdefs
#___ medium-level interface ____________________________
def addpendinggraph(self, flowgraph, inputcells):
self.addpendingblock(flowgraph, flowgraph.startblock, inputcells)
def addpendingblock(self, graph, block, cells):
"""Register an entry point into block with the given input cells."""
if graph in self.fixed_graphs:
# special case for annotating/rtyping in several phases: calling
# a graph that has already been rtyped. Safety-check the new
# annotations that are passed in, and don't annotate the old
# graph -- it's already low-level operations!
for a, s_newarg in zip(block.inputargs, cells):
s_oldarg = self.binding(a)
assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg
else:
assert not self.frozen
if block not in self.annotated:
self.bindinputargs(graph, block, cells)
else:
self.mergeinputargs(graph, block, cells)
if not self.annotated[block]:
self.pendingblocks[block] = graph
def complete_pending_blocks(self):
while self.pendingblocks:
block, graph = self.pendingblocks.popitem()
self.processblock(graph, block)
def complete(self):
"""Process pending blocks until none is left."""
while True:
self.complete_pending_blocks()
self.policy.no_more_blocks_to_annotate(self)
if not self.pendingblocks:
break # finished
# make sure that the return variables of all graphs is annotated
if self.added_blocks is not None:
newgraphs = [self.annotated[block] for block in self.added_blocks]
newgraphs = dict.fromkeys(newgraphs)
got_blocked_blocks = False in newgraphs
else:
newgraphs = self.translator.graphs #all of them
got_blocked_blocks = False in self.annotated.values()
if got_blocked_blocks:
for graph in self.blocked_graphs.values():
self.blocked_graphs[graph] = True
blocked_blocks = [
block for block, done in self.annotated.items()
if done is False
]
assert len(blocked_blocks) == len(self.blocked_blocks)
text = format_blocked_annotation_error(self, self.blocked_blocks)
#raise SystemExit()
raise annmodel.AnnotatorError(text)
for graph in newgraphs:
v = graph.getreturnvar()
if v.annotation is None:
self.setbinding(v, annmodel.s_ImpossibleValue)
# policy-dependent computation
self.bookkeeper.compute_at_fixpoint()
def validate(self):
"""Check that the annotation results are valid"""
self.bookkeeper.check_no_flags_on_instances()
def annotation(self, arg):
"Gives the SomeValue corresponding to the given Variable or Constant."
if isinstance(arg, Variable):
return arg.annotation
elif isinstance(arg, Constant):
return self.bookkeeper.immutablevalue(arg.value)
else:
raise TypeError('Variable or Constant expected, got %r' % (arg, ))
def binding(self, arg):
"Gives the SomeValue corresponding to the given Variable or Constant."
s_arg = self.annotation(arg)
if s_arg is None:
raise KeyError
return s_arg
def typeannotation(self, t):
return signature.annotation(t, self.bookkeeper)
def setbinding(self, arg, s_value):
s_old = arg.annotation
if s_old is not None:
assert s_value.contains(s_old)
arg.annotation = s_value
def warning(self, msg, pos=None):
if pos is None:
try:
pos = self.bookkeeper.position_key
except AttributeError:
pos = '?'
if pos != '?':
pos = self.whereami(pos)
log.WARNING("%s/ %s" % (pos, msg))
#___ interface for annotator.bookkeeper _______
def recursivecall(self, graph, whence, inputcells):
if isinstance(whence, tuple):
parent_graph, parent_block, parent_index = whence
tag = parent_block, parent_index
self.translator.update_call_graph(parent_graph, graph, tag)
# self.notify[graph.returnblock] is a dictionary of call
# points to this func which triggers a reflow whenever the
# return block of this graph has been analysed.
callpositions = self.notify.setdefault(graph.returnblock, {})
if whence is not None:
if callable(whence):
def callback():
whence(self, graph)
else:
callback = whence
callpositions[callback] = True
# generalize the function's input arguments
self.addpendingblock(graph, graph.startblock, inputcells)
# get the (current) return value
v = graph.getreturnvar()
try:
return self.binding(v)
except KeyError:
# the function didn't reach any return statement so far.
# (some functions actually never do, they always raise exceptions)
return annmodel.s_ImpossibleValue
def reflowfromposition(self, position_key):
graph, block, index = position_key
self.reflowpendingblock(graph, block)
#___ simplification (should be moved elsewhere?) _______
def simplify(self, block_subset=None, extra_passes=None):
# Generic simplifications
transform.transform_graph(self,
block_subset=block_subset,
extra_passes=extra_passes)
if block_subset is None:
graphs = self.translator.graphs
else:
graphs = {}
for block in block_subset:
graph = self.annotated.get(block)
if graph:
graphs[graph] = True
for graph in graphs:
simplify.eliminate_empty_blocks(graph)
#___ flowing annotations in blocks _____________________
def processblock(self, graph, block):
# Important: this is not called recursively.
# self.flowin() can only issue calls to self.addpendingblock().
# The analysis of a block can be in three states:
# * block not in self.annotated:
# never seen the block.
# * self.annotated[block] == False:
# the input variables of the block have bindings but we
# still have to consider all the operations in the block.
# * self.annotated[block] == graph-containing-block:
# analysis done (at least until we find we must generalize the
# input variables).
#print '* processblock', block, cells
self.annotated[block] = graph
if block in self.blocked_blocks:
del self.blocked_blocks[block]
try:
self.flowin(graph, block)
except BlockedInference, e:
self.annotated[block] = False # failed, hopefully temporarily
self.blocked_blocks[block] = (graph, e.opindex)
except Exception, e:
# hack for debug tools only
if not hasattr(e, '__annotator_block'):
setattr(e, '__annotator_block', block)
raise
class RPythonAnnotator(object):
"""Block annotator for RPython.
See description in doc/translation.txt."""
def __init__(self,
translator=None,
policy=None,
bookkeeper=None,
keepgoing=False):
import rpython.rtyper.extfuncregistry # has side effects
if translator is None:
# interface for tests
from rpython.translator.translator import TranslationContext
translator = TranslationContext()
translator.annotator = self
self.translator = translator
self.pendingblocks = ShuffleDict() # map {block: graph-containing-it}
self.annotated = {} # set of blocks already seen
self.added_blocks = None # see processblock() below
self.links_followed = {} # set of links that have ever been followed
self.notify = {} # {block: {positions-to-reflow-from-when-done}}
self.fixed_graphs = {} # set of graphs not to annotate again
self.blocked_blocks = {} # set of {blocked_block: (graph, index)}
# --- the following information is recorded for debugging ---
self.blocked_graphs = {} # set of graphs that have blocked blocks
# --- end of debugging information ---
self.frozen = False
if policy is None:
from rpython.annotator.policy import AnnotatorPolicy
self.policy = AnnotatorPolicy()
else:
self.policy = policy
if bookkeeper is None:
bookkeeper = Bookkeeper(self)
self.bookkeeper = bookkeeper
self.keepgoing = keepgoing
self.failed_blocks = set()
self.errors = []
def __getstate__(self):
attrs = """translator pendingblocks annotated links_followed
notify bookkeeper frozen policy added_blocks""".split()
ret = self.__dict__.copy()
for key, value in ret.items():
if key not in attrs:
assert type(value) is dict, (
"%r is not dict. please update %s.__getstate__" %
(key, self.__class__.__name__))
ret[key] = {}
return ret
#___ convenience high-level interface __________________
def build_types(self,
function,
input_arg_types,
complete_now=True,
main_entry_point=False):
"""Recursively build annotations about the specific entry point."""
assert isinstance(function, types.FunctionType), "fix that!"
from rpython.annotator.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
# make input arguments and set their type
args_s = [self.typeannotation(t) for t in input_arg_types]
# XXX hack
annmodel.TLS.check_str_without_nul = (
self.translator.config.translation.check_str_without_nul)
with self.using_policy(policy):
flowgraph, inputs_s = self.get_call_parameters(function, args_s)
if main_entry_point:
self.translator.entry_point_graph = flowgraph
return self.build_graph_types(flowgraph,
inputs_s,
complete_now=complete_now)
def get_call_parameters(self, function, args_s):
with self.bookkeeper.at_position(None):
desc = self.bookkeeper.getdesc(function)
return desc.get_call_parameters(args_s)
def annotate_helper(self, function, args_s, policy=None):
if policy is None:
from rpython.annotator.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
# XXX hack
annmodel.TLS.check_str_without_nul = (
self.translator.config.translation.check_str_without_nul)
with self.using_policy(policy):
graph, inputcells = self.get_call_parameters(function, args_s)
self.build_graph_types(graph, inputcells, complete_now=False)
self.complete_helpers()
return graph
def complete_helpers(self):
saved = self.added_blocks
self.added_blocks = {}
try:
self.complete()
# invoke annotation simplifications for the new blocks
self.simplify(block_subset=self.added_blocks)
finally:
self.added_blocks = saved
@contextmanager
def using_policy(self, policy):
"""A context manager that temporarily replaces the annotator policy"""
old_policy = self.policy
self.policy = policy
yield
self.policy = old_policy
def build_graph_types(self, flowgraph, inputcells, complete_now=True):
checkgraph(flowgraph)
nbarg = len(flowgraph.getargs())
assert len(inputcells) == nbarg # wrong number of args
# register the entry point
self.addpendinggraph(flowgraph, inputcells)
# recursively proceed until no more pending block is left
if complete_now:
self.complete()
return self.annotation(flowgraph.getreturnvar())
def gettype(self, variable):
"""Return the known type of a control flow graph variable,
defaulting to 'object'."""
if isinstance(variable, Constant):
return type(variable.value)
elif isinstance(variable, Variable):
s_variable = variable.annotation
if s_variable:
return s_variable.knowntype
else:
return object
else:
raise TypeError("Variable or Constant instance expected, "
"got %r" % (variable, ))
def getuserclassdefinitions(self):
"""Return a list of ClassDefs."""
return self.bookkeeper.classdefs
#___ medium-level interface ____________________________
def addpendinggraph(self, flowgraph, inputcells):
self.addpendingblock(flowgraph, flowgraph.startblock, inputcells)
def addpendingblock(self, graph, block, cells):
"""Register an entry point into block with the given input cells."""
if graph in self.fixed_graphs:
# special case for annotating/rtyping in several phases: calling
# a graph that has already been rtyped. Safety-check the new
# annotations that are passed in, and don't annotate the old
# graph -- it's already low-level operations!
for a, s_newarg in zip(block.inputargs, cells):
s_oldarg = a.annotation
# XXX: Should use s_oldarg.contains(s_newarg) but that breaks
# PyPy translation
if annmodel.unionof(s_oldarg, s_newarg) != s_oldarg:
raise annmodel.AnnotatorError(
"Late-stage annotation is not allowed to modify the "
"existing annotation for variable %s: %s" %
(a, s_oldarg))
else:
assert not self.frozen
if block not in self.annotated:
self.bindinputargs(graph, block, cells)
else:
self.mergeinputargs(graph, block, cells)
if not self.annotated[block]:
self.pendingblocks[block] = graph
def complete_pending_blocks(self):
while self.pendingblocks:
block, graph = self.pendingblocks.popitem()
self.processblock(graph, block)
def complete(self):
"""Process pending blocks until none is left."""
while True:
self.complete_pending_blocks()
self.policy.no_more_blocks_to_annotate(self)
if not self.pendingblocks:
break # finished
# make sure that the return variables of all graphs is annotated
if self.added_blocks is not None:
newgraphs = [self.annotated[block] for block in self.added_blocks]
newgraphs = dict.fromkeys(newgraphs)
got_blocked_blocks = False in newgraphs
else:
newgraphs = self.translator.graphs #all of them
got_blocked_blocks = False in self.annotated.values()
if self.failed_blocks:
text = ('Annotation failed, %s errors were recorded:' %
len(self.errors))
text += '\n-----'.join(str(e) for e in self.errors)
raise annmodel.AnnotatorError(text)
if got_blocked_blocks:
for graph in self.blocked_graphs.values():
self.blocked_graphs[graph] = True
blocked_blocks = [
block for block, done in self.annotated.items()
if done is False
]
assert len(blocked_blocks) == len(self.blocked_blocks)
text = format_blocked_annotation_error(self, self.blocked_blocks)
#raise SystemExit()
raise annmodel.AnnotatorError(text)
for graph in newgraphs:
v = graph.getreturnvar()
if v.annotation is None:
self.setbinding(v, s_ImpossibleValue)
v = graph.exceptblock.inputargs[1]
if v.annotation is not None and v.annotation.can_be_none():
raise annmodel.AnnotatorError(
"%r is found by annotation to possibly raise None, "
"but the None was not suppressed by the flow space" %
(graph, ))
def validate(self):
"""Check that the annotation results are valid"""
self.bookkeeper.check_no_flags_on_instances()
def annotation(self, arg):
"Gives the SomeValue corresponding to the given Variable or Constant."
if isinstance(arg, Variable):
return arg.annotation
elif isinstance(arg, Constant):
return self.bookkeeper.immutablevalue(arg.value)
else:
raise TypeError('Variable or Constant expected, got %r' % (arg, ))
def binding(self, arg):
"Gives the SomeValue corresponding to the given Variable or Constant."
s_arg = self.annotation(arg)
if s_arg is None:
raise KeyError
return s_arg
def typeannotation(self, t):
return signature.annotation(t, self.bookkeeper)
def setbinding(self, arg, s_value):
s_old = arg.annotation
if s_old is not None:
if not s_value.contains(s_old):
log.WARNING("%s does not contain %s" % (s_value, s_old))
log.WARNING("%s" % annmodel.union(s_value, s_old))
assert False
arg.annotation = s_value
def warning(self, msg, pos=None):
if pos is None:
try:
pos = self.bookkeeper.position_key
except AttributeError:
pos = '?'
if pos != '?':
pos = self.whereami(pos)
log.WARNING("%s/ %s" % (pos, msg))
#___ interface for annotator.bookkeeper _______
def recursivecall(self, graph, whence, inputcells):
if isinstance(whence, tuple):
parent_graph, parent_block, parent_index = whence
tag = parent_block, parent_index
self.translator.update_call_graph(parent_graph, graph, tag)
# self.notify[graph.returnblock] is a dictionary of call
# points to this func which triggers a reflow whenever the
# return block of this graph has been analysed.
callpositions = self.notify.setdefault(graph.returnblock, {})
if whence is not None:
if callable(whence):
def callback():
whence(self, graph)
else:
callback = whence
callpositions[callback] = True
# generalize the function's input arguments
self.addpendingblock(graph, graph.startblock, inputcells)
# get the (current) return value
v = graph.getreturnvar()
try:
return self.binding(v)
except KeyError:
# the function didn't reach any return statement so far.
# (some functions actually never do, they always raise exceptions)
return s_ImpossibleValue
def reflowfromposition(self, position_key):
graph, block, index = position_key
self.reflowpendingblock(graph, block)
def call_sites(self):
newblocks = self.added_blocks
if newblocks is None:
newblocks = self.annotated # all of them
for block in newblocks:
for op in block.operations:
if op.opname in ('simple_call', 'call_args'):
yield op
# some blocks are partially annotated
if op.result.annotation is None:
break # ignore the unannotated part
#___ simplification (should be moved elsewhere?) _______
def simplify(self, block_subset=None, extra_passes=None):
# Generic simplifications
transform.transform_graph(self,
block_subset=block_subset,
extra_passes=extra_passes)
if block_subset is None:
graphs = self.translator.graphs
else:
graphs = {}
for block in block_subset:
graph = self.annotated.get(block)
if graph:
graphs[graph] = True
for graph in graphs:
simplify.eliminate_empty_blocks(graph)
self.bookkeeper.compute_at_fixpoint()
if block_subset is None:
perform_normalizations(self)
#___ flowing annotations in blocks _____________________
def processblock(self, graph, block):
# Important: this is not called recursively.
# self.flowin() can only issue calls to self.addpendingblock().
# The analysis of a block can be in three states:
# * block not in self.annotated:
# never seen the block.
# * self.annotated[block] == False:
# the input variables of the block have bindings but we
# still have to consider all the operations in the block.
# * self.annotated[block] == graph-containing-block:
# analysis done (at least until we find we must generalize the
# input variables).
#print '* processblock', block, cells
self.annotated[block] = graph
if block in self.failed_blocks:
return
if block in self.blocked_blocks:
del self.blocked_blocks[block]
try:
self.flowin(graph, block)
except BlockedInference as e:
self.annotated[block] = False # failed, hopefully temporarily
self.blocked_blocks[block] = (graph, e.opindex)
except Exception as e:
# hack for debug tools only
if not hasattr(e, '__annotator_block'):
setattr(e, '__annotator_block', block)
raise
# The dict 'added_blocks' is used by rpython.annlowlevel to
# detect which are the new blocks that annotating an additional
# small helper creates.
if self.added_blocks is not None:
self.added_blocks[block] = True
def reflowpendingblock(self, graph, block):
assert not self.frozen
assert graph not in self.fixed_graphs
self.pendingblocks[block] = graph
assert block in self.annotated
self.annotated[block] = False # must re-flow
self.blocked_blocks[block] = (graph, None)
def bindinputargs(self, graph, block, inputcells):
# Create the initial bindings for the input args of a block.
assert len(block.inputargs) == len(inputcells)
for a, cell in zip(block.inputargs, inputcells):
self.setbinding(a, cell)
self.annotated[block] = False # must flowin.
self.blocked_blocks[block] = (graph, None)
def mergeinputargs(self, graph, block, inputcells):
# Merge the new 'cells' with each of the block's existing input
# variables.
oldcells = [self.binding(a) for a in block.inputargs]
try:
unions = [
annmodel.unionof(c1, c2)
for c1, c2 in zip(oldcells, inputcells)
]
except annmodel.UnionError as e:
# Add source code to the UnionError
e.source = '\n'.join(source_lines(graph, block, None, long=True))
if self.keepgoing:
self.errors.append(e)
self.failed_blocks.add(block)
return
raise
# if the merged cells changed, we must redo the analysis
if unions != oldcells:
self.bindinputargs(graph, block, unions)
def apply_renaming(self, s_out, renaming):
if hasattr(s_out, 'is_type_of'):
renamed_is_type_of = []
for v in s_out.is_type_of:
renamed_is_type_of += renaming[v]
assert s_out.knowntype is type
newcell = typeof(renamed_is_type_of)
if s_out.is_constant():
newcell.const = s_out.const
s_out = newcell
if hasattr(s_out, 'knowntypedata'):
renamed_knowntypedata = {}
for value, constraints in s_out.knowntypedata.items():
renamed_knowntypedata[value] = {}
for v, s in constraints.items():
new_vs = renaming.get(v, [])
for new_v in new_vs:
renamed_knowntypedata[value][new_v] = s
assert isinstance(s_out, annmodel.SomeBool)
newcell = annmodel.SomeBool()
if s_out.is_constant():
newcell.const = s_out.const
s_out = newcell
s_out.set_knowntypedata(renamed_knowntypedata)
return s_out
def whereami(self, position_key):
graph, block, i = position_key
blk = ""
if block:
at = block.at()
if at:
blk = " block" + at
opid = ""
if i is not None:
opid = " op=%d" % i
return repr(graph) + blk + opid
def flowin(self, graph, block):
try:
i = 0
while i < len(block.operations):
op = block.operations[i]
with self.bookkeeper.at_position((graph, block, i)):
new_ops = op.transform(self)
if new_ops is not None:
block.operations[i:i + 1] = new_ops
if not new_ops:
continue
new_ops[-1].result = op.result
op = new_ops[0]
self.consider_op(op)
i += 1
except BlockedInference as e:
if e.op is block.raising_op:
# this is the case where the last operation of the block will
# always raise an exception which is immediately caught by
# an exception handler. We then only follow the exceptional
# branches.
exits = [
link for link in block.exits if link.exitcase is not None
]
elif e.op.opname in ('simple_call', 'call_args', 'next'):
# XXX warning, keep the name of the call operations in sync
# with the flow object space. These are the operations for
# which it is fine to always raise an exception. We then
# swallow the BlockedInference and that's it.
# About 'next': see test_annotate_iter_empty_container().
return
else:
# other cases are problematic (but will hopefully be solved
# later by reflowing). Throw the BlockedInference up to
# processblock().
e.opindex = i
raise
except annmodel.HarmlesslyBlocked:
return
except annmodel.AnnotatorError as e: # note that UnionError is a subclass
e.source = gather_error(self, graph, block, i)
if self.keepgoing:
self.errors.append(e)
self.failed_blocks.add(block)
return
raise
else:
# dead code removal: don't follow all exits if the exitswitch
# is known
exits = block.exits
if isinstance(block.exitswitch, Variable):
s_exitswitch = self.binding(block.exitswitch)
if s_exitswitch.is_constant():
exits = [
link for link in exits
if link.exitcase == s_exitswitch.const
]
if block.canraise:
op = block.raising_op
s_exception = self.get_exception(op)
for link in exits:
case = link.exitcase
if case is None:
self.follow_link(graph, link, {})
continue
if s_exception == s_ImpossibleValue:
break
s_case = SomeInstance(self.bookkeeper.getuniqueclassdef(case))
s_matching_exc = intersection(s_exception, s_case)
if s_matching_exc != s_ImpossibleValue:
self.follow_raise_link(graph, link, s_matching_exc)
s_exception = difference(s_exception, s_case)
else: