def __init__(self, function, annotations, stackless=False, view=False, html=None, is_interactive=False, root = None, run_browser = True, policy = None):
if not use_browsertest and not _CLI_is_on_path():
py.test.skip('Javascript CLI (js) not found')
self.html = html
self.is_interactive = is_interactive
t = TranslationContext()
if policy is None:
from pypy.annotation.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
policy.allow_someobjects = False
ann = t.buildannotator(policy=policy)
ann.build_types(function, annotations)
if view or option.view:
t.view()
t.buildrtyper(type_system="ootype").specialize()
if view or option.view:
t.view()
#self.js = JS(t, [function, callback_function], stackless)
self.js = JS(t, function, stackless)
self.js.write_source()
if root is None and use_tg:
from pypy.translator.js.demo.jsdemo.controllers import Root
self.root = Root
else:
self.root = root
self.run_browser = run_browser
self.function_calls = []
def get_interpreter(func, values, view='auto', viewbefore='auto', policy=None,
someobjects=False, type_system="lltype", backendopt=False, config=None):
key = (func,) + tuple([typeOf(x) for x in values])+ (someobjects,
backendopt)
try:
(t, interp, graph) = _tcache[key]
except KeyError:
def annotation(x):
T = typeOf(x)
if T == Ptr(PyObject) and someobjects:
return object
elif T == Ptr(rstr.STR):
return str
else:
return lltype_to_annotation(T)
if policy is None and not someobjects:
from pypy.annotation.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
policy.allow_someobjects = False
t, typer, graph = gengraph(func, [annotation(x) for x in values],
viewbefore, policy, type_system=type_system,
backendopt=backendopt, config=config)
interp = LLInterpreter(typer)
_tcache[key] = (t, interp, graph)
# keep the cache small
_lastinterpreted.append(key)
if len(_lastinterpreted) >= 4:
del _tcache[_lastinterpreted.pop(0)]
if view == 'auto':
view = conftest.option.view
if view:
t.view()
return interp, graph
def annotate(func,
values,
inline=None,
backendoptimize=True,
type_system="lltype"):
# build the normal ll graphs for ll_function
t = TranslationContext()
annpolicy = AnnotatorPolicy()
annpolicy.allow_someobjects = False
a = t.buildannotator(policy=annpolicy)
argtypes = getargtypes(a, values)
a.build_types(func, argtypes)
rtyper = t.buildrtyper(type_system=type_system)
rtyper.specialize()
if inline:
auto_inlining(t, threshold=inline)
if backendoptimize:
from pypy.translator.backendopt.all import backend_optimizations
backend_optimizations(t,
inline_threshold=inline or 0,
remove_asserts=True,
really_remove_asserts=True)
#if conftest.option.view:
# t.view()
return rtyper
def test_register_external_tuple_args(self):
"""
Verify the annotation of a registered external function which takes a
tuple argument.
"""
def function_with_tuple_arg():
"""
Dummy function which is declared via register_external to take a
tuple as an argument so that register_external's behavior for
tuple-taking functions can be verified.
"""
register_external(function_with_tuple_arg, [(int,)], int)
def f():
return function_with_tuple_arg((1,))
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
# Not a very good assertion, but at least it means _something_ happened.
assert isinstance(s, annmodel.SomeInteger)
def test_callback(self):
"""
Verify annotation when a callback function is in the arguments list.
"""
def d(y):
return eval("y()")
class DTestFuncEntry(ExtFuncEntry):
_about_ = d
name = "d"
signature_args = [annmodel.SomeGenericCallable(args=[], result=annmodel.SomeFloat())]
signature_result = annmodel.SomeFloat()
def callback():
return 2.5
def f():
return d(callback)
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeFloat)
assert a.translator._graphof(callback)
def test_register_external_return_goes_back(self):
"""
Check whether it works to pass the same list from one external
fun to another
[bookkeeper and list joining issues]
"""
def function_with_list():
pass
register_external(function_with_list, [[int]], int)
def function_returning_list():
pass
register_external(function_returning_list, [], [int])
def f():
return function_with_list(function_returning_list())
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeInteger)
def test_basic(self):
"""
A ExtFuncEntry provides an annotation for a function, no need to flow
its graph.
"""
def b(x):
"NOT_RPYTHON"
return eval("x+40")
class BTestFuncEntry(ExtFuncEntry):
_about_ = b
name = "b"
signature_args = [annmodel.SomeInteger()]
signature_result = annmodel.SomeInteger()
def f():
return b(2)
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeInteger)
res = interpret(f, [])
assert res == 42
def test_basic(self):
"""
A ExtFuncEntry provides an annotation for a function, no need to flow
its graph.
"""
def b(x):
"NOT_RPYTHON"
return eval("x+40")
class BTestFuncEntry(ExtFuncEntry):
_about_ = b
name = 'b'
signature_args = [annmodel.SomeInteger()]
signature_result = annmodel.SomeInteger()
def f():
return b(2)
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeInteger)
res = interpret(f, [])
assert res == 42
def rcompile(rgenop,
entrypoint,
argtypes,
random_seed=0,
type_system='lltype'):
from pypy.translator.translator import TranslationContext
from pypy.annotation.policy import AnnotatorPolicy
from pypy import conftest
t = TranslationContext()
policy = AnnotatorPolicy()
policy.allow_someobjects = False
t.buildannotator(policy=policy).build_types(entrypoint, argtypes)
t.buildrtyper(type_system=type_system).specialize()
# note that backend optimizations will constant-fold simple operations,
# which is required by some backends that don't accept calls like
# genop1("add", constant, constant).
from pypy.translator.backendopt.all import backend_optimizations
backend_optimizations(t)
if conftest.option.view:
t.view()
entrygraph = t._graphof(entrypoint)
return compile_graph(rgenop, entrygraph, random_seed=random_seed)
def test_callback(self):
"""
Verify annotation when a callback function is in the arguments list.
"""
def d(y):
return eval("y()")
class DTestFuncEntry(ExtFuncEntry):
_about_ = d
name = 'd'
signature_args = [
annmodel.SomeGenericCallable(args=[],
result=annmodel.SomeFloat())
]
signature_result = annmodel.SomeFloat()
def callback():
return 2.5
def f():
return d(callback)
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeFloat)
assert a.translator._graphof(callback)
def annotate(func,
values,
inline=None,
backendoptimize=True,
type_system="lltype",
translationoptions={}):
# build the normal ll graphs for ll_function
t = TranslationContext()
for key, value in translationoptions.items():
setattr(t.config.translation, key, value)
annpolicy = AnnotatorPolicy()
annpolicy.allow_someobjects = False
a = t.buildannotator(policy=annpolicy)
argtypes = getargtypes(a, values)
a.build_types(func, argtypes, main_entry_point=True)
rtyper = t.buildrtyper(type_system=type_system)
rtyper.specialize()
#if inline:
# auto_inlining(t, threshold=inline)
if backendoptimize:
from pypy.translator.backendopt.all import backend_optimizations
backend_optimizations(t,
inline_threshold=inline or 0,
remove_asserts=True,
really_remove_asserts=True)
return rtyper
def test_register_external_signature():
def f():
return dd(3)
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeInteger)
def test_annotation_b():
def f():
return b(1)
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeInteger)
def test_register_external_signature():
def f():
return dd(3)
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeInteger)
def test_annotation_b():
def f():
return b(1)
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeInteger)
def test_register_external_specialcase():
def f():
x = function_withspecialcase
return x(33) + x("aaa") + x([]) + "\n"
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeString)
def test_register_external_specialcase():
def f():
x = function_withspecialcase
return x(33) + x("aaa") + x([]) + "\n"
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeString)
def get_interpreter(
func,
values,
view="auto",
viewbefore="auto",
policy=None,
someobjects=False,
type_system="lltype",
backendopt=False,
config=None,
**extraconfigopts
):
extra_key = [(key, value) for key, value in extraconfigopts.iteritems()]
extra_key.sort()
extra_key = tuple(extra_key)
key = (func,) + tuple([typeOf(x) for x in values]) + (someobjects, backendopt, extra_key)
try:
(t, interp, graph) = _tcache[key]
except KeyError:
def annotation(x):
T = typeOf(x)
if T == Ptr(PyObject) and someobjects:
return object
else:
return lltype_to_annotation(T)
if policy is None and not someobjects:
from pypy.annotation.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
policy.allow_someobjects = False
t, typer, graph = gengraph(
func,
[annotation(x) for x in values],
viewbefore,
policy,
type_system=type_system,
backendopt=backendopt,
config=config,
**extraconfigopts
)
interp = LLInterpreter(typer)
_tcache[key] = (t, interp, graph)
# keep the cache small
_lastinterpreted.append(key)
if len(_lastinterpreted) >= 4:
del _tcache[_lastinterpreted.pop(0)]
if view == "auto":
view = getattr(conftest.option, "view", False)
if view:
t.view()
return interp, graph
def test_callback():
def callback():
return 2.5
def f():
return d(callback)
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeFloat)
assert a.translator._graphof(callback)
def test_callback():
def callback():
return 2.5
def f():
return d(callback)
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeFloat)
assert a.translator._graphof(callback)
def get_interpreter(func,
values,
view='auto',
viewbefore='auto',
policy=None,
someobjects=False,
type_system="lltype",
backendopt=False,
config=None,
malloc_check=True,
**extraconfigopts):
extra_key = [(key, value) for key, value in extraconfigopts.iteritems()]
extra_key.sort()
extra_key = tuple(extra_key)
key = ((func, ) + tuple([typeOf(x) for x in values]) +
(someobjects, backendopt, extra_key))
try:
(t, interp, graph) = _tcache[key]
except KeyError:
def annotation(x):
T = typeOf(x)
if T == Ptr(PyObject) and someobjects:
return object
else:
return lltype_to_annotation(T)
if policy is None and not someobjects:
from pypy.annotation.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
policy.allow_someobjects = False
t, typer, graph = gengraph(func, [annotation(x) for x in values],
viewbefore,
policy,
type_system=type_system,
backendopt=backendopt,
config=config,
**extraconfigopts)
interp = LLInterpreter(typer, malloc_check=malloc_check)
_tcache[key] = (t, interp, graph)
# keep the cache small
_lastinterpreted.append(key)
if len(_lastinterpreted) >= 4:
del _tcache[_lastinterpreted.pop(0)]
if view == 'auto':
view = getattr(conftest.option, 'view', False)
if view:
t.view()
return interp, graph
def test_register_external_return_goes_back():
"""
Check whether it works to pass the same list from one external
fun to another
[bookkeeper and list joining issues]
"""
def f():
return function_with_list(function_returning_list())
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeInteger)
def test_register_external_tuple_args():
"""
Verify the annotation of a registered external function which takes a tuple
argument.
"""
def f():
return function_with_tuple_arg((1,))
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
# Not a very good assertion, but at least it means _something_ happened.
assert isinstance(s, annmodel.SomeInteger)
def test_register_external_tuple_args():
"""
Verify the annotation of a registered external function which takes a tuple
argument.
"""
def f():
return function_with_tuple_arg((1, ))
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
# Not a very good assertion, but at least it means _something_ happened.
assert isinstance(s, annmodel.SomeInteger)
def default_specialize(pol, funcdesc, args_s):
name = funcdesc.name
if name.startswith('ll_') or name.startswith('_ll_'): # xxx can we do better?
return super(MixLevelAnnotatorPolicy, pol).default_specialize(
funcdesc, args_s)
else:
return AnnotatorPolicy.default_specialize(funcdesc, args_s)
def default_specialize(pol, funcdesc, args_s):
name = funcdesc.name
if name.startswith('ll_') or name.startswith('_ll_'): # xxx can we do better?
return super(MixLevelAnnotatorPolicy, pol).default_specialize(
funcdesc, args_s)
else:
return AnnotatorPolicy.default_specialize(funcdesc, args_s)
def test_register_external_signature(self):
"""
Test the standard interface for external functions.
"""
def dd():
pass
register_external(dd, [int], int)
def f():
return dd(3)
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeInteger)
def test_register_external_signature(self):
"""
Test the standard interface for external functions.
"""
def dd():
pass
register_external(dd, [int], int)
def f():
return dd(3)
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeInteger)
def test_register_external_specialcase(self):
"""
When args=None, the external function accepts any arguments unmodified.
"""
def function_withspecialcase(arg):
return repr(arg)
register_external(function_withspecialcase, args=None, result=str)
def f():
x = function_withspecialcase
return x(33) + x("aaa") + x([]) + "\n"
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeString)
def __init__(self, translator=None, policy=None, bookkeeper=None):
import pypy.rpython.ootypesystem.ooregistry # has side effects
import pypy.rpython.ootypesystem.bltregistry # has side effects
import pypy.rpython.extfuncregistry # has side effects
import pypy.rlib.nonconst # has side effects
if translator is None:
# interface for tests
from pypy.translator.translator import TranslationContext
translator = TranslationContext()
translator.annotator = self
self.translator = translator
self.pendingblocks = {} # map {block: graph-containing-it}
self.bindings = {} # map Variables to SomeValues
self.annotated = {} # set of blocks already seen
self.added_blocks = None # see processblock() below
self.links_followed = {} # set of links that have ever been followed
self.notify = {} # {block: {positions-to-reflow-from-when-done}}
self.fixed_graphs = {} # set of graphs not to annotate again
self.blocked_blocks = {} # set of {blocked_block: graph}
# --- the following information is recorded for debugging only ---
# --- and only if annotation.model.DEBUG is kept to True
self.why_not_annotated = {
} # {block: (exc_type, exc_value, traceback)}
# records the location of BlockedInference
# exceptions that blocked some blocks.
self.blocked_graphs = {} # set of graphs that have blocked blocks
self.bindingshistory = {} # map Variables to lists of SomeValues
self.binding_caused_by = {} # map Variables to position_keys
# records the caller position that caused bindings of inputargs
# to be updated
self.binding_cause_history = {} # map Variables to lists of positions
# history of binding_caused_by, kept in sync with
# bindingshistory
self.reflowcounter = {}
self.return_bindings = {} # map return Variables to their graphs
# --- end of debugging information ---
self.frozen = False
if policy is None:
from pypy.annotation.policy import AnnotatorPolicy
self.policy = AnnotatorPolicy()
else:
self.policy = policy
if bookkeeper is None:
bookkeeper = Bookkeeper(self)
self.bookkeeper = bookkeeper
def test_prebuilt_lock():
import thread
import pypy.module.thread.rpython.exttable # for declare()/declaretype()
lock0 = thread.allocate_lock()
lock1 = thread.allocate_lock()
lock1.acquire()
def fn(i):
lock = [lock0, lock1][i]
ok = lock.acquire(False)
if ok: lock.release()
return ok
policy = AnnotatorPolicy()
policy.allow_someobjects = False
res = interpret(fn, [0], policy=policy)
assert res is True
res = interpret(fn, [1], policy=policy)
assert res is False
def test_lock_or_None():
import thread
import pypy.module.thread.rpython.exttable # for declare()/declaretype()
def makelock(i):
if i > 0:
return thread.allocate_lock()
else:
return None
def fn(i):
lock = makelock(i)
return lock is not None and lock.acquire(False)
policy = AnnotatorPolicy()
policy.allow_someobjects = False
res = interpret(fn, [0], policy=policy)
assert res is False
res = interpret(fn, [1], policy=policy)
assert res is True
def annotate_helper(self, function, args_s, policy=None):
if policy is None:
from pypy.annotation.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
graph, inputcells = self.get_call_parameters(function, args_s, policy)
self.build_graph_types(graph, inputcells, complete_now=False)
self.complete_helpers(policy)
return graph
def test_register_external_specialcase(self):
"""
When args=None, the external function accepts any arguments unmodified.
"""
def function_withspecialcase(arg):
return repr(arg)
register_external(function_withspecialcase, args=None, result=str)
def f():
x = function_withspecialcase
return x(33) + x("aaa") + x([]) + "\n"
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
assert isinstance(s, annmodel.SomeString)
def annotate(func, values, inline=None, backendoptimize=True,
type_system="lltype"):
# build the normal ll graphs for ll_function
t = TranslationContext()
annpolicy = AnnotatorPolicy()
annpolicy.allow_someobjects = False
a = t.buildannotator(policy=annpolicy)
argtypes = getargtypes(a, values)
a.build_types(func, argtypes, main_entry_point=True)
rtyper = t.buildrtyper(type_system = type_system)
rtyper.specialize()
#if inline:
# auto_inlining(t, threshold=inline)
if backendoptimize:
from pypy.translator.backendopt.all import backend_optimizations
backend_optimizations(t, inline_threshold=inline or 0,
remove_asserts=True, really_remove_asserts=True)
return rtyper
def rcompile(rgenop, entrypoint, argtypes, random_seed=0, type_system='lltype'):
from pypy.translator.translator import TranslationContext
from pypy.annotation.policy import AnnotatorPolicy
from pypy import conftest
t = TranslationContext()
policy = AnnotatorPolicy()
policy.allow_someobjects = False
t.buildannotator(policy=policy).build_types(entrypoint, argtypes)
t.buildrtyper(type_system = type_system).specialize()
# note that backend optimizations will constant-fold simple operations,
# which is required by some backends that don't accept calls like
# genop1("add", constant, constant).
from pypy.translator.backendopt.all import backend_optimizations
backend_optimizations(t)
if conftest.option.view:
t.view()
entrygraph = t._graphof(entrypoint)
return compile_graph(rgenop, entrygraph, random_seed=random_seed)
def __init__(self,
function,
annotations,
stackless=False,
view=False,
html=None,
is_interactive=False,
root=None,
run_browser=True,
policy=None):
if not use_browsertest and not _CLI_is_on_path():
py.test.skip('Javascript CLI (js) not found')
self.html = html
self.is_interactive = is_interactive
t = TranslationContext()
if policy is None:
from pypy.annotation.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
policy.allow_someobjects = False
ann = t.buildannotator(policy=policy)
ann.build_types(function, annotations)
if view or option.view:
t.view()
t.buildrtyper(type_system="ootype").specialize()
if view or option.view:
t.view()
#self.js = JS(t, [function, callback_function], stackless)
self.js = JS(t, function, stackless)
self.js.write_source()
if root is None and use_tg:
from pypy.translator.js.demo.jsdemo.controllers import Root
self.root = Root
else:
self.root = root
self.run_browser = run_browser
self.function_calls = []
def test_list_of_str0(self):
str0 = annmodel.SomeString(no_nul=True)
def os_execve(l):
pass
register_external(os_execve, [[str0]], None)
def f(l):
return os_execve(l)
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
a.build_types(f, [[str]]) # Does not raise
assert a.translator.config.translation.check_str_without_nul == False
# Now enable the str0 check, and try again with a similar function
a.translator.config.translation.check_str_without_nul = True
def g(l):
return os_execve(l)
raises(Exception, a.build_types, g, [[str]])
a.build_types(g, [[str0]]) # Does not raise
def test_register_external_tuple_args(self):
"""
Verify the annotation of a registered external function which takes a
tuple argument.
"""
def function_with_tuple_arg():
"""
Dummy function which is declared via register_external to take a
tuple as an argument so that register_external's behavior for
tuple-taking functions can be verified.
"""
register_external(function_with_tuple_arg, [(int, )], int)
def f():
return function_with_tuple_arg((1, ))
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
s = a.build_types(f, [])
# Not a very good assertion, but at least it means _something_ happened.
assert isinstance(s, annmodel.SomeInteger)
def test_list_of_str0(self):
str0 = annmodel.SomeString(no_nul=True)
def os_execve(l):
pass
register_external(os_execve, [[str0]], None)
def f(l):
return os_execve(l)
policy = AnnotatorPolicy()
policy.allow_someobjects = False
a = RPythonAnnotator(policy=policy)
a.build_types(f, [[str]]) # Does not raise
assert a.translator.config.translation.check_str_without_nul == False
# Now enable the str0 check, and try again with a similar function
a.translator.config.translation.check_str_without_nul = True
def g(l):
return os_execve(l)
raises(Exception, a.build_types, g, [[str]])
a.build_types(g, [[str0]]) # Does not raise
def __init__(self, translator=None, policy=None, bookkeeper=None):
import pypy.rpython.ootypesystem.ooregistry # has side effects
import pypy.rpython.extfuncregistry # has side effects
import pypy.rlib.nonconst # has side effects
if translator is None:
# interface for tests
from pypy.translator.translator import TranslationContext
translator = TranslationContext()
translator.annotator = self
self.translator = translator
self.pendingblocks = {} # map {block: graph-containing-it}
self.bindings = {} # map Variables to SomeValues
self.annotated = {} # set of blocks already seen
self.added_blocks = None # see processblock() below
self.links_followed = {} # set of links that have ever been followed
self.notify = {} # {block: {positions-to-reflow-from-when-done}}
self.fixed_graphs = {} # set of graphs not to annotate again
self.blocked_blocks = {} # set of {blocked_block: graph}
# --- the following information is recorded for debugging only ---
# --- and only if annotation.model.DEBUG is kept to True
self.why_not_annotated = {
} # {block: (exc_type, exc_value, traceback)}
# records the location of BlockedInference
# exceptions that blocked some blocks.
self.blocked_graphs = {} # set of graphs that have blocked blocks
self.bindingshistory = {} # map Variables to lists of SomeValues
self.binding_caused_by = {} # map Variables to position_keys
# records the caller position that caused bindings of inputargs
# to be updated
self.binding_cause_history = {} # map Variables to lists of positions
# history of binding_caused_by, kept in sync with
# bindingshistory
self.reflowcounter = {}
self.return_bindings = {} # map return Variables to their graphs
# --- end of debugging information ---
self.frozen = False
if policy is None:
from pypy.annotation.policy import AnnotatorPolicy
self.policy = AnnotatorPolicy()
else:
self.policy = policy
if bookkeeper is None:
bookkeeper = Bookkeeper(self)
self.bookkeeper = bookkeeper
def build_types(self, function, input_arg_types, complete_now=True,
main_entry_point=False):
"""Recursively build annotations about the specific entry point."""
assert isinstance(function, types.FunctionType), "fix that!"
from pypy.annotation.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
# make input arguments and set their type
args_s = [self.typeannotation(t) for t in input_arg_types]
# XXX hack
annmodel.TLS.check_str_without_nul = (
self.translator.config.translation.check_str_without_nul)
flowgraph, inputcells = self.get_call_parameters(function, args_s, policy)
if not isinstance(flowgraph, FunctionGraph):
assert isinstance(flowgraph, annmodel.SomeObject)
return flowgraph
if main_entry_point:
self.translator.entry_point_graph = flowgraph
return self.build_graph_types(flowgraph, inputcells, complete_now=complete_now)
def annotate_helper_method(self, _class, attr, args_s, policy=None):
""" Warning! this method is meant to be used between
annotation and rtyping
"""
if policy is None:
from pypy.annotation.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
assert attr != '__class__'
classdef = self.bookkeeper.getuniqueclassdef(_class)
attrdef = classdef.find_attribute(attr)
s_result = attrdef.getvalue()
classdef.add_source_for_attribute(attr, classdef.classdesc)
self.bookkeeper
assert isinstance(s_result, annmodel.SomePBC)
olddesc = s_result.descriptions.iterkeys().next()
desc = olddesc.bind_self(classdef)
args = self.bookkeeper.build_args("simple_call", args_s[:])
desc.consider_call_site(self.bookkeeper, desc.getcallfamily(), [desc],
args, annmodel.s_ImpossibleValue)
result = []
def schedule(graph, inputcells):
result.append((graph, inputcells))
return annmodel.s_ImpossibleValue
prevpolicy = self.policy
self.policy = policy
self.bookkeeper.enter(None)
try:
desc.pycall(schedule, args, annmodel.s_ImpossibleValue)
finally:
self.bookkeeper.leave()
self.policy = prevpolicy
[(graph, inputcells)] = result
self.build_graph_types(graph, inputcells, complete_now=False)
self.complete_helpers(policy)
return graph
res = self.do(func)
assert res == 101
def test_recursive_structure(self):
P1 = RPointer(None)
S1 = RStruct('S1', [('next', P1)])
P1.setpointertype(S1)
def func():
s1 = S1.allocate()
s2 = S1.allocate()
s2.ref_next().set_contents(s1)
return s2.ref_next().get_contents().sameaddr(s1)
res = self.do(func)
assert res == True
POLICY = AnnotatorPolicy()
POLICY.allow_someobjects = False
class TestLLInterpreted(TestBasic):
def do(self, func):
return interpret(func, [], policy=POLICY, backendopt=True)
def test_simple_struct(self):
S0 = RStruct('S0', [('x', rc_int)])
def func():
s = S0.allocate()
s.ref_x().set_value(12)
return s.ref_x().get_value()
interp, graph = get_interpreter(func, [], policy=POLICY,
class RPythonAnnotator(object):
"""Block annotator for RPython.
See description in doc/translation.txt."""
def __init__(self, translator=None, policy=None, bookkeeper=None):
import pypy.rpython.ootypesystem.ooregistry # has side effects
import pypy.rpython.ootypesystem.bltregistry # has side effects
import pypy.rpython.extfuncregistry # has side effects
import pypy.rlib.nonconst # has side effects
if translator is None:
# interface for tests
from pypy.translator.translator import TranslationContext
translator = TranslationContext()
translator.annotator = self
self.translator = translator
self.pendingblocks = {} # map {block: graph-containing-it}
self.bindings = {} # map Variables to SomeValues
self.annotated = {} # set of blocks already seen
self.added_blocks = None # see processblock() below
self.links_followed = {} # set of links that have ever been followed
self.notify = {} # {block: {positions-to-reflow-from-when-done}}
self.fixed_graphs = {} # set of graphs not to annotate again
self.blocked_blocks = {} # set of {blocked_block: graph}
# --- the following information is recorded for debugging only ---
# --- and only if annotation.model.DEBUG is kept to True
self.why_not_annotated = {
} # {block: (exc_type, exc_value, traceback)}
# records the location of BlockedInference
# exceptions that blocked some blocks.
self.blocked_graphs = {} # set of graphs that have blocked blocks
self.bindingshistory = {} # map Variables to lists of SomeValues
self.binding_caused_by = {} # map Variables to position_keys
# records the caller position that caused bindings of inputargs
# to be updated
self.binding_cause_history = {} # map Variables to lists of positions
# history of binding_caused_by, kept in sync with
# bindingshistory
self.reflowcounter = {}
self.return_bindings = {} # map return Variables to their graphs
# --- end of debugging information ---
self.frozen = False
if policy is None:
from pypy.annotation.policy import AnnotatorPolicy
self.policy = AnnotatorPolicy()
else:
self.policy = policy
if bookkeeper is None:
bookkeeper = Bookkeeper(self)
self.bookkeeper = bookkeeper
def __getstate__(self):
attrs = """translator pendingblocks bindings annotated links_followed
notify bookkeeper frozen policy added_blocks""".split()
ret = self.__dict__.copy()
for key, value in ret.items():
if key not in attrs:
assert type(value) is dict, (
"%r is not dict. please update %s.__getstate__" %
(key, self.__class__.__name__))
ret[key] = {}
return ret
def _register_returnvar(self, flowgraph):
if annmodel.DEBUG:
self.return_bindings[flowgraph.getreturnvar()] = flowgraph
#___ convenience high-level interface __________________
def build_types(self, function, input_arg_types, complete_now=True):
"""Recursively build annotations about the specific entry point."""
assert isinstance(function, FunctionType), "fix that!"
# make input arguments and set their type
inputcells = [self.typeannotation(t) for t in input_arg_types]
desc = self.bookkeeper.getdesc(function)
desc.getcallfamily() # record this implicit call (hint for back-ends)
flowgraph = desc.specialize(inputcells)
if not isinstance(flowgraph, FunctionGraph):
assert isinstance(flowgraph, annmodel.SomeObject)
return flowgraph
return self.build_graph_types(flowgraph,
inputcells,
complete_now=complete_now)
def get_call_parameters(self, function, args_s, policy):
desc = self.bookkeeper.getdesc(function)
args = self.bookkeeper.build_args("simple_call", args_s[:])
result = []
def schedule(graph, inputcells):
result.append((graph, inputcells))
return annmodel.s_ImpossibleValue
prevpolicy = self.policy
self.policy = policy
self.bookkeeper.enter(None)
try:
desc.pycall(schedule, args, annmodel.s_ImpossibleValue)
finally:
self.bookkeeper.leave()
self.policy = prevpolicy
[(graph, inputcells)] = result
return graph, inputcells
def annotate_helper(self, function, args_s, policy=None):
if policy is None:
from pypy.annotation.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
graph, inputcells = self.get_call_parameters(function, args_s, policy)
self.build_graph_types(graph, inputcells, complete_now=False)
self.complete_helpers(policy)
return graph
def annotate_helper_method(self, _class, attr, args_s, policy=None):
""" Warning! this method is meant to be used between
annotation and rtyping
"""
if policy is None:
from pypy.annotation.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
assert attr != '__class__'
classdef = self.bookkeeper.getuniqueclassdef(_class)
attrdef = classdef.find_attribute(attr)
s_result = attrdef.getvalue()
classdef.add_source_for_attribute(attr, classdef.classdesc)
self.bookkeeper
assert isinstance(s_result, annmodel.SomePBC)
olddesc = s_result.descriptions.iterkeys().next()
desc = olddesc.bind_self(classdef)
args = self.bookkeeper.build_args("simple_call", args_s[:])
desc.consider_call_site(self.bookkeeper, desc.getcallfamily(), [desc],
args, annmodel.s_ImpossibleValue)
result = []
def schedule(graph, inputcells):
result.append((graph, inputcells))
return annmodel.s_ImpossibleValue
prevpolicy = self.policy
self.policy = policy
self.bookkeeper.enter(None)
try:
desc.pycall(schedule, args, annmodel.s_ImpossibleValue)
finally:
self.bookkeeper.leave()
self.policy = prevpolicy
[(graph, inputcells)] = result
self.build_graph_types(graph, inputcells, complete_now=False)
self.complete_helpers(policy)
return graph
def complete_helpers(self, policy):
saved = self.policy, self.added_blocks
self.policy = policy
try:
self.added_blocks = {}
self.complete()
# invoke annotation simplifications for the new blocks
self.simplify(block_subset=self.added_blocks)
finally:
self.policy, self.added_blocks = saved
def build_graph_types(self, flowgraph, inputcells, complete_now=True):
checkgraph(flowgraph)
nbarg = len(flowgraph.getargs())
if len(inputcells) != nbarg:
raise TypeError("%s expects %d args, got %d" %
(flowgraph, nbarg, len(inputcells)))
# register the entry point
self.addpendinggraph(flowgraph, inputcells)
# recursively proceed until no more pending block is left
if complete_now:
self.complete()
return self.binding(flowgraph.getreturnvar(), None)
def gettype(self, variable):
"""Return the known type of a control flow graph variable,
defaulting to 'object'."""
if isinstance(variable, Constant):
return type(variable.value)
elif isinstance(variable, Variable):
cell = self.bindings.get(variable)
if cell:
return cell.knowntype
else:
return object
else:
raise TypeError, ("Variable or Constant instance expected, "
"got %r" % (variable, ))
def getuserclassdefinitions(self):
"""Return a list of ClassDefs."""
return self.bookkeeper.classdefs
#___ medium-level interface ____________________________
def addpendinggraph(self, flowgraph, inputcells):
self._register_returnvar(flowgraph)
self.addpendingblock(flowgraph, flowgraph.startblock, inputcells)
def addpendingblock(self, graph, block, cells, called_from_graph=None):
"""Register an entry point into block with the given input cells."""
if graph in self.fixed_graphs:
# special case for annotating/rtyping in several phases: calling
# a graph that has already been rtyped. Safety-check the new
# annotations that are passed in, and don't annotate the old
# graph -- it's already low-level operations!
for a, s_newarg in zip(graph.getargs(), cells):
s_oldarg = self.binding(a)
assert s_oldarg.contains(s_newarg)
else:
assert not self.frozen
for a in cells:
assert isinstance(a, annmodel.SomeObject)
if block not in self.annotated:
self.bindinputargs(graph, block, cells, called_from_graph)
else:
self.mergeinputargs(graph, block, cells, called_from_graph)
if not self.annotated[block]:
self.pendingblocks[block] = graph
def complete(self):
"""Process pending blocks until none is left."""
while True:
while self.pendingblocks:
block, graph = self.pendingblocks.popitem()
if annmodel.DEBUG:
self.flowin_block = block # we need to keep track of block
self.processblock(graph, block)
self.policy.no_more_blocks_to_annotate(self)
if not self.pendingblocks:
break # finished
# make sure that the return variables of all graphs is annotated
if self.added_blocks is not None:
newgraphs = [self.annotated[block] for block in self.added_blocks]
newgraphs = dict.fromkeys(newgraphs)
got_blocked_blocks = False in newgraphs
else:
newgraphs = self.translator.graphs #all of them
got_blocked_blocks = False in self.annotated.values()
if got_blocked_blocks:
for graph in self.blocked_graphs.values():
self.blocked_graphs[graph] = True
blocked_blocks = [
block for block, done in self.annotated.items()
if done is False
]
assert len(blocked_blocks) == len(self.blocked_blocks)
text = format_blocked_annotation_error(self, self.blocked_blocks)
#raise SystemExit()
raise AnnotatorError(text)
for graph in newgraphs:
v = graph.getreturnvar()
if v not in self.bindings:
self.setbinding(v, annmodel.s_ImpossibleValue)
# policy-dependent computation
self.bookkeeper.compute_at_fixpoint()
def binding(self, arg, default=FAIL):
"Gives the SomeValue corresponding to the given Variable or Constant."
if isinstance(arg, Variable):
try:
return self.bindings[arg]
except KeyError:
if default is not FAIL:
return default
else:
raise
elif isinstance(arg, Constant):
#if arg.value is undefined_value: # undefined local variables
# return annmodel.s_ImpossibleValue
return self.bookkeeper.immutableconstant(arg)
else:
raise TypeError, 'Variable or Constant expected, got %r' % (arg, )
def typeannotation(self, t):
return signature.annotation(t, self.bookkeeper)
def ondegenerated(self, what, s_value, where=None, called_from_graph=None):
if self.policy.allow_someobjects:
return
# is the function itself tagged with allow_someobjects?
position_key = where or getattr(self.bookkeeper, 'position_key', None)
if position_key is not None:
graph, block, i = position_key
try:
if graph.func.allow_someobjects:
return
except AttributeError:
pass
graph = position_key[0]
msgstr = format_someobject_error(self, position_key, what, s_value,
called_from_graph,
self.bindings.get(what, "(none)"))
raise AnnotatorError(msgstr)
def setbinding(self, arg, s_value, called_from_graph=None, where=None):
if arg in self.bindings:
assert s_value.contains(self.bindings[arg])
# for debugging purposes, record the history of bindings that
# have been given to this variable
if annmodel.DEBUG:
history = self.bindingshistory.setdefault(arg, [])
history.append(self.bindings[arg])
cause_history = self.binding_cause_history.setdefault(arg, [])
cause_history.append(self.binding_caused_by[arg])
degenerated = annmodel.isdegenerated(s_value)
if degenerated:
self.ondegenerated(arg,
s_value,
where=where,
called_from_graph=called_from_graph)
self.bindings[arg] = s_value
if annmodel.DEBUG:
if arg in self.return_bindings:
log.event("%s -> %s" % (self.whereami(
(self.return_bindings[arg], None, None)), s_value))
if arg in self.return_bindings and degenerated:
self.warning("result degenerated to SomeObject",
(self.return_bindings[arg], None, None))
self.binding_caused_by[arg] = called_from_graph
def transfer_binding(self, v_target, v_source):
assert v_source in self.bindings
self.bindings[v_target] = self.bindings[v_source]
if annmodel.DEBUG:
self.binding_caused_by[v_target] = None
def warning(self, msg, pos=None):
if pos is None:
try:
pos = self.bookkeeper.position_key
except AttributeError:
pos = '?'
if pos != '?':
pos = self.whereami(pos)
log.WARNING("%s/ %s" % (pos, msg))
#___ interface for annotator.bookkeeper _______
def recursivecall(
self, graph, whence, inputcells
): # whence = position_key|callback taking the annotator, graph
if isinstance(whence, tuple):
parent_graph, parent_block, parent_index = position_key = whence
tag = parent_block, parent_index
self.translator.update_call_graph(parent_graph, graph, tag)
else:
position_key = None
self._register_returnvar(graph)
# self.notify[graph.returnblock] is a dictionary of call
# points to this func which triggers a reflow whenever the
# return block of this graph has been analysed.
callpositions = self.notify.setdefault(graph.returnblock, {})
if whence is not None:
if callable(whence):
def callback():
whence(self, graph)
else:
callback = whence
callpositions[callback] = True
# generalize the function's input arguments
self.addpendingblock(graph, graph.startblock, inputcells, position_key)
# get the (current) return value
v = graph.getreturnvar()
try:
return self.bindings[v]
except KeyError:
# the function didn't reach any return statement so far.
# (some functions actually never do, they always raise exceptions)
return annmodel.s_ImpossibleValue
def reflowfromposition(self, position_key):
graph, block, index = position_key
self.reflowpendingblock(graph, block)
#___ simplification (should be moved elsewhere?) _______
# it should be!
# now simplify_calls is moved to transform.py.
# i kept reverse_binding here for future(?) purposes though. --sanxiyn
def reverse_binding(self, known_variables, cell):
"""This is a hack."""
# In simplify_calls, when we are trying to create the new
# SpaceOperation, all we have are SomeValues. But SpaceOperations take
# Variables, not SomeValues. Trouble is, we don't always have a
# Variable that just happens to be bound to the given SomeValue.
# A typical example would be if the tuple of arguments was created
# from another basic block or even another function. Well I guess
# there is no clean solution, short of making the transformations
# more syntactic (e.g. replacing a specific sequence of SpaceOperations
# with another one). This is a real hack because we have to use
# the identity of 'cell'.
if cell.is_constant():
return Constant(cell.const)
else:
for v in known_variables:
if self.bindings[v] is cell:
return v
else:
raise CannotSimplify
def simplify(self, block_subset=None, extra_passes=None):
# Generic simplifications
from pypy.translator import transform
transform.transform_graph(self,
block_subset=block_subset,
extra_passes=extra_passes)
from pypy.translator import simplify
if block_subset is None:
graphs = self.translator.graphs
else:
graphs = {}
for block in block_subset:
graph = self.annotated.get(block)
if graph:
graphs[graph] = True
for graph in graphs:
simplify.eliminate_empty_blocks(graph)
#___ flowing annotations in blocks _____________________
def processblock(self, graph, block):
# Important: this is not called recursively.
# self.flowin() can only issue calls to self.addpendingblock().
# The analysis of a block can be in three states:
# * block not in self.annotated:
# never seen the block.
# * self.annotated[block] == False:
# the input variables of the block are in self.bindings but we
# still have to consider all the operations in the block.
# * self.annotated[block] == graph-containing-block:
# analysis done (at least until we find we must generalize the
# input variables).
#print '* processblock', block, cells
if annmodel.DEBUG:
self.reflowcounter.setdefault(block, 0)
self.reflowcounter[block] += 1
self.annotated[block] = graph
if block in self.blocked_blocks:
del self.blocked_blocks[block]
try:
self.flowin(graph, block)
except BlockedInference, e:
self.annotated[block] = False # failed, hopefully temporarily
self.blocked_blocks[block] = graph
except Exception, e:
# hack for debug tools only
if not hasattr(e, '__annotator_block'):
setattr(e, '__annotator_block', block)
raise
class RPythonAnnotator(object):
"""Block annotator for RPython.
See description in doc/translation.txt."""
def __init__(self, translator=None, policy=None, bookkeeper=None):
import pypy.rpython.ootypesystem.ooregistry # has side effects
import pypy.rpython.extfuncregistry # has side effects
import pypy.rlib.nonconst # has side effects
if translator is None:
# interface for tests
from pypy.translator.translator import TranslationContext
translator = TranslationContext()
translator.annotator = self
self.translator = translator
self.pendingblocks = {} # map {block: graph-containing-it}
self.bindings = {} # map Variables to SomeValues
self.annotated = {} # set of blocks already seen
self.added_blocks = None # see processblock() below
self.links_followed = {} # set of links that have ever been followed
self.notify = {} # {block: {positions-to-reflow-from-when-done}}
self.fixed_graphs = {} # set of graphs not to annotate again
self.blocked_blocks = {} # set of {blocked_block: graph}
# --- the following information is recorded for debugging only ---
# --- and only if annotation.model.DEBUG is kept to True
self.why_not_annotated = {} # {block: (exc_type, exc_value, traceback)}
# records the location of BlockedInference
# exceptions that blocked some blocks.
self.blocked_graphs = {} # set of graphs that have blocked blocks
self.bindingshistory = {}# map Variables to lists of SomeValues
self.binding_caused_by = {} # map Variables to position_keys
# records the caller position that caused bindings of inputargs
# to be updated
self.binding_cause_history = {} # map Variables to lists of positions
# history of binding_caused_by, kept in sync with
# bindingshistory
self.reflowcounter = {}
self.return_bindings = {} # map return Variables to their graphs
# --- end of debugging information ---
self.frozen = False
if policy is None:
from pypy.annotation.policy import AnnotatorPolicy
self.policy = AnnotatorPolicy()
else:
self.policy = policy
if bookkeeper is None:
bookkeeper = Bookkeeper(self)
self.bookkeeper = bookkeeper
def __getstate__(self):
attrs = """translator pendingblocks bindings annotated links_followed
notify bookkeeper frozen policy added_blocks""".split()
ret = self.__dict__.copy()
for key, value in ret.items():
if key not in attrs:
assert type(value) is dict, (
"%r is not dict. please update %s.__getstate__" %
(key, self.__class__.__name__))
ret[key] = {}
return ret
def _register_returnvar(self, flowgraph):
if annmodel.DEBUG:
self.return_bindings[flowgraph.getreturnvar()] = flowgraph
#___ convenience high-level interface __________________
def build_types(self, function, input_arg_types, complete_now=True,
main_entry_point=False):
"""Recursively build annotations about the specific entry point."""
assert isinstance(function, types.FunctionType), "fix that!"
from pypy.annotation.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
# make input arguments and set their type
args_s = [self.typeannotation(t) for t in input_arg_types]
flowgraph, inputcells = self.get_call_parameters(function, args_s, policy)
if not isinstance(flowgraph, FunctionGraph):
assert isinstance(flowgraph, annmodel.SomeObject)
return flowgraph
if main_entry_point:
self.translator.entry_point_graph = flowgraph
return self.build_graph_types(flowgraph, inputcells, complete_now=complete_now)
def get_call_parameters(self, function, args_s, policy):
desc = self.bookkeeper.getdesc(function)
args = self.bookkeeper.build_args("simple_call", args_s[:])
result = []
def schedule(graph, inputcells):
result.append((graph, inputcells))
return annmodel.s_ImpossibleValue
prevpolicy = self.policy
self.policy = policy
self.bookkeeper.enter(None)
try:
desc.pycall(schedule, args, annmodel.s_ImpossibleValue)
finally:
self.bookkeeper.leave()
self.policy = prevpolicy
[(graph, inputcells)] = result
return graph, inputcells
def annotate_helper(self, function, args_s, policy=None):
if policy is None:
from pypy.annotation.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
graph, inputcells = self.get_call_parameters(function, args_s, policy)
self.build_graph_types(graph, inputcells, complete_now=False)
self.complete_helpers(policy)
return graph
def annotate_helper_method(self, _class, attr, args_s, policy=None):
""" Warning! this method is meant to be used between
annotation and rtyping
"""
if policy is None:
from pypy.annotation.policy import AnnotatorPolicy
policy = AnnotatorPolicy()
assert attr != '__class__'
classdef = self.bookkeeper.getuniqueclassdef(_class)
attrdef = classdef.find_attribute(attr)
s_result = attrdef.getvalue()
classdef.add_source_for_attribute(attr, classdef.classdesc)
self.bookkeeper
assert isinstance(s_result, annmodel.SomePBC)
olddesc = s_result.any_description()
desc = olddesc.bind_self(classdef)
args = self.bookkeeper.build_args("simple_call", args_s[:])
desc.consider_call_site(self.bookkeeper, desc.getcallfamily(), [desc],
args, annmodel.s_ImpossibleValue)
result = []
def schedule(graph, inputcells):
result.append((graph, inputcells))
return annmodel.s_ImpossibleValue
prevpolicy = self.policy
self.policy = policy
self.bookkeeper.enter(None)
try:
desc.pycall(schedule, args, annmodel.s_ImpossibleValue)
finally:
self.bookkeeper.leave()
self.policy = prevpolicy
[(graph, inputcells)] = result
self.build_graph_types(graph, inputcells, complete_now=False)
self.complete_helpers(policy)
return graph
def complete_helpers(self, policy):
saved = self.policy, self.added_blocks
self.policy = policy
try:
self.added_blocks = {}
self.complete()
# invoke annotation simplifications for the new blocks
self.simplify(block_subset=self.added_blocks)
finally:
self.policy, self.added_blocks = saved
def build_graph_types(self, flowgraph, inputcells, complete_now=True):
checkgraph(flowgraph)
nbarg = len(flowgraph.getargs())
if len(inputcells) != nbarg:
raise TypeError("%s expects %d args, got %d" %(
flowgraph, nbarg, len(inputcells)))
# register the entry point
self.addpendinggraph(flowgraph, inputcells)
# recursively proceed until no more pending block is left
if complete_now:
self.complete()
return self.binding(flowgraph.getreturnvar(), None)
def gettype(self, variable):
"""Return the known type of a control flow graph variable,
defaulting to 'object'."""
if isinstance(variable, Constant):
return type(variable.value)
elif isinstance(variable, Variable):
cell = self.bindings.get(variable)
if cell:
return cell.knowntype
else:
return object
else:
raise TypeError, ("Variable or Constant instance expected, "
"got %r" % (variable,))
def getuserclassdefinitions(self):
"""Return a list of ClassDefs."""
return self.bookkeeper.classdefs
#___ medium-level interface ____________________________
def addpendinggraph(self, flowgraph, inputcells):
self._register_returnvar(flowgraph)
self.addpendingblock(flowgraph, flowgraph.startblock, inputcells)
def addpendingblock(self, graph, block, cells, called_from_graph=None):
"""Register an entry point into block with the given input cells."""
if graph in self.fixed_graphs:
# special case for annotating/rtyping in several phases: calling
# a graph that has already been rtyped. Safety-check the new
# annotations that are passed in, and don't annotate the old
# graph -- it's already low-level operations!
for a, s_newarg in zip(graph.getargs(), cells):
s_oldarg = self.binding(a)
assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg
else:
assert not self.frozen
for a in cells:
assert isinstance(a, annmodel.SomeObject)
if block not in self.annotated:
self.bindinputargs(graph, block, cells, called_from_graph)
else:
self.mergeinputargs(graph, block, cells, called_from_graph)
if not self.annotated[block]:
self.pendingblocks[block] = graph
def complete(self):
"""Process pending blocks until none is left."""
while True:
while self.pendingblocks:
block, graph = self.pendingblocks.popitem()
if annmodel.DEBUG:
self.flowin_block = block # we need to keep track of block
self.processblock(graph, block)
self.policy.no_more_blocks_to_annotate(self)
if not self.pendingblocks:
break # finished
# make sure that the return variables of all graphs is annotated
if self.added_blocks is not None:
newgraphs = [self.annotated[block] for block in self.added_blocks]
newgraphs = dict.fromkeys(newgraphs)
got_blocked_blocks = False in newgraphs
else:
newgraphs = self.translator.graphs #all of them
got_blocked_blocks = False in self.annotated.values()
if got_blocked_blocks:
for graph in self.blocked_graphs.values():
self.blocked_graphs[graph] = True
blocked_blocks = [block for block, done in self.annotated.items()
if done is False]
assert len(blocked_blocks) == len(self.blocked_blocks)
text = format_blocked_annotation_error(self, self.blocked_blocks)
#raise SystemExit()
raise AnnotatorError(text)
for graph in newgraphs:
v = graph.getreturnvar()
if v not in self.bindings:
self.setbinding(v, annmodel.s_ImpossibleValue)
# policy-dependent computation
self.bookkeeper.compute_at_fixpoint()
def binding(self, arg, default=FAIL):
"Gives the SomeValue corresponding to the given Variable or Constant."
if isinstance(arg, Variable):
try:
return self.bindings[arg]
except KeyError:
if default is not FAIL:
return default
else:
raise
elif isinstance(arg, Constant):
#if arg.value is undefined_value: # undefined local variables
# return annmodel.s_ImpossibleValue
return self.bookkeeper.immutableconstant(arg)
else:
raise TypeError, 'Variable or Constant expected, got %r' % (arg,)
def typeannotation(self, t):
return signature.annotation(t, self.bookkeeper)
def ondegenerated(self, what, s_value, where=None, called_from_graph=None):
if self.policy.allow_someobjects:
return
# is the function itself tagged with allow_someobjects?
position_key = where or getattr(self.bookkeeper, 'position_key', None)
if position_key is not None:
graph, block, i = position_key
try:
if graph.func.allow_someobjects:
return
except AttributeError:
pass
msgstr = format_someobject_error(self, position_key, what, s_value,
called_from_graph,
self.bindings.get(what, "(none)"))
raise AnnotatorError(msgstr)
def setbinding(self, arg, s_value, called_from_graph=None, where=None):
if arg in self.bindings:
assert s_value.contains(self.bindings[arg])
# for debugging purposes, record the history of bindings that
# have been given to this variable
if annmodel.DEBUG:
history = self.bindingshistory.setdefault(arg, [])
history.append(self.bindings[arg])
cause_history = self.binding_cause_history.setdefault(arg, [])
cause_history.append(self.binding_caused_by[arg])
degenerated = annmodel.isdegenerated(s_value)
if degenerated:
self.ondegenerated(arg, s_value, where=where,
called_from_graph=called_from_graph)
self.bindings[arg] = s_value
if annmodel.DEBUG:
if arg in self.return_bindings:
log.event("%s -> %s" %
(self.whereami((self.return_bindings[arg], None, None)),
s_value))
if arg in self.return_bindings and degenerated:
self.warning("result degenerated to SomeObject",
(self.return_bindings[arg],None, None))
self.binding_caused_by[arg] = called_from_graph
def transfer_binding(self, v_target, v_source):
assert v_source in self.bindings
self.bindings[v_target] = self.bindings[v_source]
if annmodel.DEBUG:
self.binding_caused_by[v_target] = None
def warning(self, msg, pos=None):
if pos is None:
try:
pos = self.bookkeeper.position_key
except AttributeError:
pos = '?'
if pos != '?':
pos = self.whereami(pos)
log.WARNING("%s/ %s" % (pos, msg))
#___ interface for annotator.bookkeeper _______
def recursivecall(self, graph, whence, inputcells): # whence = position_key|callback taking the annotator, graph
if isinstance(whence, tuple):
parent_graph, parent_block, parent_index = position_key = whence
tag = parent_block, parent_index
self.translator.update_call_graph(parent_graph, graph, tag)
else:
position_key = None
self._register_returnvar(graph)
# self.notify[graph.returnblock] is a dictionary of call
# points to this func which triggers a reflow whenever the
# return block of this graph has been analysed.
callpositions = self.notify.setdefault(graph.returnblock, {})
if whence is not None:
if callable(whence):
def callback():
whence(self, graph)
else:
callback = whence
callpositions[callback] = True
# generalize the function's input arguments
self.addpendingblock(graph, graph.startblock, inputcells,
position_key)
# get the (current) return value
v = graph.getreturnvar()
try:
return self.bindings[v]
except KeyError:
# the function didn't reach any return statement so far.
# (some functions actually never do, they always raise exceptions)
return annmodel.s_ImpossibleValue
def reflowfromposition(self, position_key):
graph, block, index = position_key
self.reflowpendingblock(graph, block)
#___ simplification (should be moved elsewhere?) _______
def simplify(self, block_subset=None, extra_passes=None):
# Generic simplifications
transform.transform_graph(self, block_subset=block_subset,
extra_passes=extra_passes)
if block_subset is None:
graphs = self.translator.graphs
else:
graphs = {}
for block in block_subset:
graph = self.annotated.get(block)
if graph:
graphs[graph] = True
for graph in graphs:
simplify.eliminate_empty_blocks(graph)
#___ flowing annotations in blocks _____________________
def processblock(self, graph, block):
# Important: this is not called recursively.
# self.flowin() can only issue calls to self.addpendingblock().
# The analysis of a block can be in three states:
# * block not in self.annotated:
# never seen the block.
# * self.annotated[block] == False:
# the input variables of the block are in self.bindings but we
# still have to consider all the operations in the block.
# * self.annotated[block] == graph-containing-block:
# analysis done (at least until we find we must generalize the
# input variables).
#print '* processblock', block, cells
if annmodel.DEBUG:
self.reflowcounter.setdefault(block, 0)
self.reflowcounter[block] += 1
self.annotated[block] = graph
if block in self.blocked_blocks:
del self.blocked_blocks[block]
try:
self.flowin(graph, block)
except BlockedInference, e:
self.annotated[block] = False # failed, hopefully temporarily
self.blocked_blocks[block] = graph
except Exception, e:
# hack for debug tools only
if not hasattr(e, '__annotator_block'):
setattr(e, '__annotator_block', block)
raise
def test_compile_timer():
policy = AnnotatorPolicy()
policy.allow_someobjects = False
f_compiled = compile(timer_user, [], annotatorpolicy=policy)
f_compiled(expected_extra_mallocs=2)
def test_compile_timer():
policy = AnnotatorPolicy()
policy.allow_someobjects = False
f_compiled = compile(timer_user, [], annotatorpolicy=policy)
f_compiled(expected_extra_mallocs=2)
