def __init__(self, annotator):
self.annotator = annotator
self.lowlevel_ann_policy = LowLevelAnnotatorPolicy(self)
self.type_system = LowLevelTypeSystem()
self.reprs = {}
self._reprs_must_call_setup = []
self._seen_reprs_must_call_setup = {}
self._dict_traits = {}
self.rootclass_repr = RootClassRepr(self)
self.rootclass_repr.setup()
self.instance_reprs = {}
self.type_for_typeptr = {}
self.pbc_reprs = {}
self.classes_with_wrapper = {}
self.wrapper_context = None # or add an extra arg to convertvar?
self.classdef_to_pytypeobject = {}
self.concrete_calltables = {}
self.cache_dummy_values = {}
self.lltype2vtable = {}
self.typererrors = []
self.typererror_count = 0
# make the primitive_to_repr constant mapping
self.primitive_to_repr = {}
self.isinstance_helpers = {}
self.exceptiondata = ExceptionData(self)
self.custom_trace_funcs = []
try:
self.seed = int(os.getenv('RTYPERSEED'))
s = 'Using %d as seed for block shuffling' % self.seed
self.log.info(s)
except:
self.seed = 0
self.order = None
def __init__(self, annotator, backend=genc_backend):
self.annotator = annotator
self.backend = backend
self.lowlevel_ann_policy = LowLevelAnnotatorPolicy(self)
self.reprs = {}
self._reprs_must_call_setup = []
self._seen_reprs_must_call_setup = {}
self._dict_traits = {}
self.rootclass_repr = RootClassRepr(self)
self.rootclass_repr.setup()
self.instance_reprs = {}
self.type_for_typeptr = {}
self.pbc_reprs = {}
self.classes_with_wrapper = {}
self.wrapper_context = None # or add an extra arg to convertvar?
self.classdef_to_pytypeobject = {}
self.concrete_calltables = {}
self.cache_dummy_values = {}
self.lltype2vtable = {}
# make the primitive_to_repr constant mapping
self.primitive_to_repr = {}
self.isinstance_helpers = {}
self.exceptiondata = ExceptionData(self)
self.custom_trace_funcs = []
try:
self.seed = int(os.getenv('RTYPERSEED'))
s = 'Using %d as seed for block shuffling' % self.seed
self.log.info(s)
except:
self.seed = 0
def __init__(self, annotator):
self.annotator = annotator
self.lowlevel_ann_policy = LowLevelAnnotatorPolicy(self)
self.type_system = LowLevelTypeSystem.instance
self.reprs = {}
self._reprs_must_call_setup = []
self._seen_reprs_must_call_setup = {}
self._dict_traits = {}
self.class_reprs = {}
self.instance_reprs = {}
self.type_for_typeptr = {}
self.pbc_reprs = {}
self.classes_with_wrapper = {}
self.wrapper_context = None # or add an extra arg to convertvar?
self.classdef_to_pytypeobject = {}
self.concrete_calltables = {}
self.class_pbc_attributes = {}
self.cache_dummy_values = {}
self.lltype2vtable = {}
self.typererrors = []
self.typererror_count = 0
# make the primitive_to_repr constant mapping
self.primitive_to_repr = {}
self.exceptiondata = ExceptionData(self)
try:
self.seed = int(os.getenv('RTYPERSEED'))
s = 'Using %d as seed for block shuffling' % self.seed
self.log.info(s)
except:
self.seed = 0
self.order = None
class RPythonTyper(object):
from rpython.rtyper.rmodel import log
def __init__(self, annotator):
self.annotator = annotator
self.lowlevel_ann_policy = LowLevelAnnotatorPolicy(self)
self.type_system = LowLevelTypeSystem()
self.reprs = {}
self._reprs_must_call_setup = []
self._seen_reprs_must_call_setup = {}
self._dict_traits = {}
self.rootclass_repr = RootClassRepr(self)
self.rootclass_repr.setup()
self.instance_reprs = {}
self.type_for_typeptr = {}
self.pbc_reprs = {}
self.classes_with_wrapper = {}
self.wrapper_context = None # or add an extra arg to convertvar?
self.classdef_to_pytypeobject = {}
self.concrete_calltables = {}
self.cache_dummy_values = {}
self.lltype2vtable = {}
self.typererrors = []
self.typererror_count = 0
# make the primitive_to_repr constant mapping
self.primitive_to_repr = {}
self.isinstance_helpers = {}
self.exceptiondata = ExceptionData(self)
self.custom_trace_funcs = []
try:
self.seed = int(os.getenv('RTYPERSEED'))
s = 'Using %d as seed for block shuffling' % self.seed
self.log.info(s)
except:
self.seed = 0
self.order = None
def getconfig(self):
return self.annotator.translator.config
def getprimitiverepr(self, lltype):
try:
return self.primitive_to_repr[lltype]
except KeyError:
pass
if isinstance(lltype, Primitive):
repr = self.primitive_to_repr[lltype] = self.getrepr(
lltype_to_annotation(lltype))
return repr
raise TyperError('There is no primitive repr for %r' % (lltype, ))
def add_wrapper(self, clsdef):
# record that this class has a wrapper, and what the __init__ is
cls = clsdef.classdesc.pyobj
init = getattr(cls.__init__, 'im_func', None)
self.classes_with_wrapper[cls] = init
def set_wrapper_context(self, obj):
# not nice, but we sometimes need to know which function we are wrapping
self.wrapper_context = obj
def add_pendingsetup(self, repr):
assert isinstance(repr, Repr)
if repr in self._seen_reprs_must_call_setup:
#warning("ignoring already seen repr for setup: %r" %(repr,))
return
self._reprs_must_call_setup.append(repr)
self._seen_reprs_must_call_setup[repr] = True
def lltype_to_classdef_mapping(self):
result = {}
for (classdef, _), repr in self.instance_reprs.iteritems():
result[repr.lowleveltype] = classdef
return result
def get_type_for_typeptr(self, typeptr):
search = typeptr._obj
try:
return self.type_for_typeptr[search]
except KeyError:
# rehash the dictionary, and perform a linear scan
# for the case of ll2ctypes typeptr
found = None
type_for_typeptr = {}
for key, value in self.type_for_typeptr.items():
type_for_typeptr[key] = value
if key == search:
found = value
self.type_for_typeptr = type_for_typeptr
if found is None:
raise KeyError(search)
return found
def set_type_for_typeptr(self, typeptr, TYPE):
self.type_for_typeptr[typeptr._obj] = TYPE
self.lltype2vtable[TYPE] = typeptr
def get_real_typeptr_for_typeptr(self, typeptr):
# perform a linear scan for the case of ll2ctypes typeptr
search = typeptr._obj
for key, value in self.type_for_typeptr.items():
if key == search:
return key._as_ptr()
raise KeyError(search)
def getrepr(self, s_obj):
# s_objs are not hashable... try hard to find a unique key anyway
key = s_obj.rtyper_makekey()
assert key[0] is s_obj.__class__
try:
result = self.reprs[key]
except KeyError:
self.reprs[key] = None
result = s_obj.rtyper_makerepr(self)
assert not isinstance(result.lowleveltype, ContainerType), (
"missing a Ptr in the type specification "
"of %s:\n%r" % (s_obj, result.lowleveltype))
self.reprs[key] = result
self.add_pendingsetup(result)
assert result is not None # recursive getrepr()!
return result
def annotation(self, var):
s_obj = self.annotator.annotation(var)
return s_obj
def binding(self, var):
s_obj = self.annotator.binding(var)
return s_obj
def bindingrepr(self, var):
return self.getrepr(self.binding(var))
def specialize(self, dont_simplify_again=False):
"""Main entry point: specialize all annotated blocks of the program."""
# specialize depends on annotator simplifications
assert dont_simplify_again in (False, True) # safety check
if not dont_simplify_again:
self.annotator.simplify()
# first make sure that all functions called in a group have exactly
# the same signature, by hacking their flow graphs if needed
perform_normalizations(self.annotator)
self.exceptiondata.finish(self)
# new blocks can be created as a result of specialize_block(), so
# we need to be careful about the loop here.
self.already_seen = {}
self.specialize_more_blocks()
if self.exceptiondata is not None:
self.exceptiondata.make_helpers(self)
self.specialize_more_blocks() # for the helpers just made
def getannmixlevel(self):
if self.annmixlevel is not None:
return self.annmixlevel
from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator
self.annmixlevel = MixLevelHelperAnnotator(self)
return self.annmixlevel
def specialize_more_blocks(self):
if self.already_seen:
newtext = ' more'
else:
newtext = ''
blockcount = 0
self.annmixlevel = None
while True:
# look for blocks not specialized yet
pending = [
block for block in self.annotator.annotated
if block not in self.already_seen
]
if not pending:
break
# shuffle blocks a bit
if self.seed:
import random
r = random.Random(self.seed)
r.shuffle(pending)
if self.order:
tracking = self.order(self.annotator, pending)
else:
tracking = lambda block: None
previous_percentage = 0
# specialize all blocks in the 'pending' list
for block in pending:
tracking(block)
blockcount += 1
self.specialize_block(block)
self.already_seen[block] = True
# progress bar
n = len(self.already_seen)
if n % 100 == 0:
total = len(self.annotator.annotated)
percentage = 100 * n // total
if percentage >= previous_percentage + 5:
previous_percentage = percentage
if self.typererror_count:
error_report = " but %d errors" % self.typererror_count
else:
error_report = ''
self.log.event(
'specializing: %d / %d blocks (%d%%)%s' %
(n, total, percentage, error_report))
# make sure all reprs so far have had their setup() called
self.call_all_setups()
if self.typererrors:
self.dump_typererrors(to_log=True)
raise TyperError("there were %d error" % len(self.typererrors))
self.log.event('-=- specialized %d%s blocks -=-' %
(blockcount, newtext))
annmixlevel = self.annmixlevel
del self.annmixlevel
if annmixlevel is not None:
annmixlevel.finish()
def dump_typererrors(self, num=None, minimize=True, to_log=False):
c = 0
bc = 0
for err in self.typererrors[:num]:
c += 1
if minimize and isinstance(err, BrokenReprTyperError):
bc += 1
continue
graph, block, position = err.where
errmsg = ("TyperError-%d: %s\n" % (c, graph) + str(err) + "\n")
if to_log:
self.log.ERROR(errmsg)
else:
print errmsg
if bc:
minmsg = "(minimized %d errors away for this dump)" % (bc, )
if to_log:
self.log.ERROR(minmsg)
else:
print minmsg
def call_all_setups(self):
# make sure all reprs so far have had their setup() called
must_setup_more = []
delayed = []
while self._reprs_must_call_setup:
r = self._reprs_must_call_setup.pop()
if r.is_setup_delayed():
delayed.append(r)
else:
r.setup()
must_setup_more.append(r)
for r in must_setup_more:
r.setup_final()
self._reprs_must_call_setup.extend(delayed)
def setconcretetype(self, v):
assert isinstance(v, Variable)
v.concretetype = self.bindingrepr(v).lowleveltype
def setup_block_entry(self, block):
if block.operations == () and len(block.inputargs) == 2:
# special case for exception blocks: force them to return an
# exception type and value in a standardized format
v1, v2 = block.inputargs
v1.concretetype = self.exceptiondata.lltype_of_exception_type
v2.concretetype = self.exceptiondata.lltype_of_exception_value
return [
self.exceptiondata.r_exception_type,
self.exceptiondata.r_exception_value
]
else:
# normal path
result = []
for a in block.inputargs:
r = self.bindingrepr(a)
a.concretetype = r.lowleveltype
result.append(r)
return result
def make_new_lloplist(self, block):
return LowLevelOpList(self, block)
def specialize_block(self, block):
graph = self.annotator.annotated[block]
if graph not in self.annotator.fixed_graphs:
self.annotator.fixed_graphs[graph] = True
# make sure that the return variables of all graphs
# are concretetype'd
self.setconcretetype(graph.getreturnvar())
# give the best possible types to the input args
try:
self.setup_block_entry(block)
except TyperError, e:
self.gottypererror(e, block, "block-entry", None)
return # cannot continue this block
# specialize all the operations, as far as possible
if block.operations == (): # return or except block
return
newops = self.make_new_lloplist(block)
varmapping = {}
for v in block.getvariables():
varmapping[v] = v # records existing Variables
for hop in self.highlevelops(block, newops):
try:
hop.setup() # this is called from here to catch TyperErrors...
self.translate_hl_to_ll(hop, varmapping)
except TyperError, e:
self.gottypererror(e, block, hop.spaceop, newops)
return # cannot continue this block: no op.result.concretetype
class RPythonTyper(object):
from rpython.rtyper.rmodel import log
def __init__(self, annotator, backend=genc_backend):
self.annotator = annotator
self.backend = backend
self.lowlevel_ann_policy = LowLevelAnnotatorPolicy(self)
self.reprs = {}
self._reprs_must_call_setup = []
self._seen_reprs_must_call_setup = {}
self._dict_traits = {}
self.rootclass_repr = RootClassRepr(self)
self.rootclass_repr.setup()
self.instance_reprs = {}
self.type_for_typeptr = {}
self.pbc_reprs = {}
self.classes_with_wrapper = {}
self.wrapper_context = None # or add an extra arg to convertvar?
self.classdef_to_pytypeobject = {}
self.concrete_calltables = {}
self.cache_dummy_values = {}
self.lltype2vtable = {}
# make the primitive_to_repr constant mapping
self.primitive_to_repr = {}
self.isinstance_helpers = {}
self.exceptiondata = ExceptionData(self)
self.custom_trace_funcs = []
try:
self.seed = int(os.getenv('RTYPERSEED'))
s = 'Using %d as seed for block shuffling' % self.seed
self.log.info(s)
except:
self.seed = 0
def getconfig(self):
return self.annotator.translator.config
def getprimitiverepr(self, lltype):
try:
return self.primitive_to_repr[lltype]
except KeyError:
pass
if isinstance(lltype, Primitive):
repr = self.primitive_to_repr[lltype] = self.getrepr(lltype_to_annotation(lltype))
return repr
raise TyperError('There is no primitive repr for %r' % (lltype,))
def add_wrapper(self, clsdef):
# record that this class has a wrapper, and what the __init__ is
cls = clsdef.classdesc.pyobj
init = getattr(cls.__init__, 'im_func', None)
self.classes_with_wrapper[cls] = init
def set_wrapper_context(self, obj):
# not nice, but we sometimes need to know which function we are wrapping
self.wrapper_context = obj
def add_pendingsetup(self, repr):
assert isinstance(repr, Repr)
if repr in self._seen_reprs_must_call_setup:
#warning("ignoring already seen repr for setup: %r" %(repr,))
return
self._reprs_must_call_setup.append(repr)
self._seen_reprs_must_call_setup[repr] = True
def lltype_to_classdef_mapping(self):
result = {}
for (classdef, _), repr in self.instance_reprs.iteritems():
result[repr.lowleveltype] = classdef
return result
def get_type_for_typeptr(self, typeptr):
search = typeptr._obj
try:
return self.type_for_typeptr[search]
except KeyError:
# rehash the dictionary, and perform a linear scan
# for the case of ll2ctypes typeptr
found = None
type_for_typeptr = {}
for key, value in self.type_for_typeptr.items():
type_for_typeptr[key] = value
if key == search:
found = value
self.type_for_typeptr = type_for_typeptr
if found is None:
raise KeyError(search)
return found
def set_type_for_typeptr(self, typeptr, TYPE):
self.type_for_typeptr[typeptr._obj] = TYPE
self.lltype2vtable[TYPE] = typeptr
def get_real_typeptr_for_typeptr(self, typeptr):
# perform a linear scan for the case of ll2ctypes typeptr
search = typeptr._obj
for key, value in self.type_for_typeptr.items():
if key == search:
return key._as_ptr()
raise KeyError(search)
def getrepr(self, s_obj):
# s_objs are not hashable... try hard to find a unique key anyway
key = s_obj.rtyper_makekey()
assert key[0] is s_obj.__class__
try:
result = self.reprs[key]
except KeyError:
self.reprs[key] = None
result = s_obj.rtyper_makerepr(self)
assert not isinstance(result.lowleveltype, ContainerType), (
"missing a Ptr in the type specification "
"of %s:\n%r" % (s_obj, result.lowleveltype))
self.reprs[key] = result
self.add_pendingsetup(result)
assert result is not None # recursive getrepr()!
return result
def annotation(self, var):
s_obj = self.annotator.annotation(var)
return s_obj
def binding(self, var):
s_obj = self.annotator.binding(var)
return s_obj
def bindingrepr(self, var):
return self.getrepr(self.binding(var))
def specialize(self, dont_simplify_again=False):
"""Main entry point: specialize all annotated blocks of the program."""
# specialize depends on annotator simplifications
if not dont_simplify_again:
self.annotator.simplify()
self.exceptiondata.finish(self)
# new blocks can be created as a result of specialize_block(), so
# we need to be careful about the loop here.
self.already_seen = {}
self.specialize_more_blocks()
self.exceptiondata.make_helpers(self)
self.specialize_more_blocks() # for the helpers just made
def getannmixlevel(self):
if self.annmixlevel is not None:
return self.annmixlevel
from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator
self.annmixlevel = MixLevelHelperAnnotator(self)
return self.annmixlevel
def specialize_more_blocks(self):
if self.already_seen:
newtext = ' more'
else:
newtext = ''
blockcount = 0
self.annmixlevel = None
while True:
# make sure all reprs so far have had their setup() called
self.call_all_setups()
# look for blocks not specialized yet
pending = [block for block in self.annotator.annotated
if block not in self.already_seen]
if not pending:
break
# shuffle blocks a bit
if self.seed:
import random
r = random.Random(self.seed)
r.shuffle(pending)
previous_percentage = 0
# specialize all blocks in the 'pending' list
for block in pending:
blockcount += 1
self.specialize_block(block)
self.already_seen[block] = True
# progress bar
n = len(self.already_seen)
if n % 100 == 0:
total = len(self.annotator.annotated)
percentage = 100 * n // total
if percentage >= previous_percentage + 5:
previous_percentage = percentage
self.log.event('specializing: %d / %d blocks (%d%%)' %
(n, total, percentage))
self.log.event('-=- specialized %d%s blocks -=-' % (
blockcount, newtext))
annmixlevel = self.annmixlevel
del self.annmixlevel
if annmixlevel is not None:
annmixlevel.finish()
def call_all_setups(self):
# make sure all reprs so far have had their setup() called
must_setup_more = []
delayed = []
while self._reprs_must_call_setup:
r = self._reprs_must_call_setup.pop()
if r.is_setup_delayed():
delayed.append(r)
else:
r.setup()
must_setup_more.append(r)
for r in must_setup_more:
r.setup_final()
self._reprs_must_call_setup.extend(delayed)
def setconcretetype(self, v):
assert isinstance(v, Variable)
v.concretetype = self.bindingrepr(v).lowleveltype
def setup_block_entry(self, block):
if block.operations == () and len(block.inputargs) == 2:
# special case for exception blocks: force them to return an
# exception type and value in a standardized format
v1, v2 = block.inputargs
v1.concretetype = self.exceptiondata.lltype_of_exception_type
v2.concretetype = self.exceptiondata.lltype_of_exception_value
return [self.exceptiondata.r_exception_type,
self.exceptiondata.r_exception_value]
else:
# normal path
result = []
for a in block.inputargs:
r = self.bindingrepr(a)
a.concretetype = r.lowleveltype
result.append(r)
return result
def make_new_lloplist(self, block):
return LowLevelOpList(self, block)
def specialize_block(self, block):
graph = self.annotator.annotated[block]
if graph not in self.annotator.fixed_graphs:
self.annotator.fixed_graphs[graph] = True
# make sure that the return variables of all graphs
# are concretetype'd
self.setconcretetype(graph.getreturnvar())
# give the best possible types to the input args
try:
self.setup_block_entry(block)
except TyperError as e:
self.gottypererror(e, block, "block-entry")
raise
# specialize all the operations, as far as possible
if block.operations == (): # return or except block
return
newops = self.make_new_lloplist(block)
varmapping = {}
for v in block.getvariables():
varmapping[v] = v # records existing Variables
for hop in self.highlevelops(block, newops):
try:
hop.setup() # this is called from here to catch TyperErrors...
self.translate_hl_to_ll(hop, varmapping)
except TyperError as e:
self.gottypererror(e, block, hop.spaceop)
raise
block.operations[:] = newops
block.renamevariables(varmapping)
extrablock = None
pos = newops.llop_raising_exceptions
if (pos is not None and pos != len(newops) - 1):
# this is for the case where the llop that raises the exceptions
# is not the last one in the list.
assert block.canraise
noexclink = block.exits[0]
assert noexclink.exitcase is None
if pos == "removed":
# the exception cannot actually occur at all.
# This is set by calling exception_cannot_occur().
# We just remove all exception links.
block.exitswitch = None
block.exits = block.exits[:1]
else:
# We have to split the block in two, with the exception-catching
# exitswitch after the llop at 'pos', and the extra operations
# in the new part of the block, corresponding to the
# no-exception case. See for example test_rlist.test_indexerror
# or test_rpbc.test_multiple_ll_one_hl_op.
assert 0 <= pos < len(newops) - 1
extraops = block.operations[pos+1:]
del block.operations[pos+1:]
extrablock = insert_empty_block(noexclink, newops=extraops)
if extrablock is None:
self.insert_link_conversions(block)
else:
# skip the extrablock as a link target, its link doesn't need conversions
# by construction, OTOH some of involved vars have no annotation
# so proceeding with it would kill information
self.insert_link_conversions(block, skip=1)
# consider it as a link source instead
self.insert_link_conversions(extrablock)
def _convert_link(self, block, link):
if link.exitcase is not None and link.exitcase != 'default':
if isinstance(block.exitswitch, Variable):
r_case = self.bindingrepr(block.exitswitch)
else:
assert block.canraise
r_case = rclass.get_type_repr(self)
link.llexitcase = r_case.convert_const(link.exitcase)
else:
link.llexitcase = None
a = link.last_exception
if isinstance(a, Variable):
a.concretetype = self.exceptiondata.lltype_of_exception_type
elif isinstance(a, Constant):
link.last_exception = inputconst(
self.exceptiondata.r_exception_type, a.value)
a = link.last_exc_value
if isinstance(a, Variable):
a.concretetype = self.exceptiondata.lltype_of_exception_value
elif isinstance(a, Constant):
link.last_exc_value = inputconst(
self.exceptiondata.r_exception_value, a.value)
def insert_link_conversions(self, block, skip=0):
# insert the needed conversions on the links
can_insert_here = block.exitswitch is None and len(block.exits) == 1
for link in block.exits[skip:]:
self._convert_link(block, link)
inputargs_reprs = self.setup_block_entry(link.target)
newops = self.make_new_lloplist(block)
newlinkargs = {}
for i in range(len(link.args)):
a1 = link.args[i]
r_a2 = inputargs_reprs[i]
if isinstance(a1, Constant):
link.args[i] = inputconst(r_a2, a1.value)
continue # the Constant was typed, done
if a1 is link.last_exception:
r_a1 = self.exceptiondata.r_exception_type
elif a1 is link.last_exc_value:
r_a1 = self.exceptiondata.r_exception_value
else:
r_a1 = self.bindingrepr(a1)
if r_a1 == r_a2:
continue # no conversion needed
try:
new_a1 = newops.convertvar(a1, r_a1, r_a2)
except TyperError as e:
self.gottypererror(e, block, link)
raise
if new_a1 != a1:
newlinkargs[i] = new_a1
if newops:
if can_insert_here:
block.operations.extend(newops)
else:
# cannot insert conversion operations around a single
# link, unless it is the only exit of this block.
# create a new block along the link...
newblock = insert_empty_block(link,
# ...and store the conversions there.
newops=newops)
link = newblock.exits[0]
for i, new_a1 in newlinkargs.items():
link.args[i] = new_a1
def highlevelops(self, block, llops):
# enumerate the HighLevelOps in a block.
if block.operations:
for op in block.operations[:-1]:
yield HighLevelOp(self, op, [], llops)
# look for exception links for the last operation
if block.canraise:
exclinks = block.exits[1:]
else:
exclinks = []
yield HighLevelOp(self, block.operations[-1], exclinks, llops)
def translate_hl_to_ll(self, hop, varmapping):
#self.log.translating(hop.spaceop.opname, hop.args_s)
resultvar = hop.dispatch()
if hop.exceptionlinks and hop.llops.llop_raising_exceptions is None:
raise TyperError("the graph catches %s, but the rtyper did not "
"take exceptions into account "
"(exception_is_here() not called)" % (
[link.exitcase.__name__ for link in hop.exceptionlinks],))
if resultvar is None:
# no return value
self.translate_no_return_value(hop)
else:
assert isinstance(resultvar, (Variable, Constant))
op = hop.spaceop
# for simplicity of the translate_meth, resultvar is usually not
# op.result here. We have to replace resultvar with op.result
# in all generated operations.
if hop.s_result.is_constant():
if isinstance(resultvar, Constant) and \
isinstance(hop.r_result.lowleveltype, Primitive) and \
hop.r_result.lowleveltype is not Void:
# assert that they are equal, or both are 'nan'
assert resultvar.value == hop.s_result.const or (
math.isnan(resultvar.value) and
math.isnan(hop.s_result.const))
resulttype = resultvar.concretetype
op.result.concretetype = hop.r_result.lowleveltype
if op.result.concretetype != resulttype:
raise TyperError("inconsistent type for the result of '%s':\n"
"annotator says %s,\n"
"whose repr is %r\n"
"but rtype_%s returned %r" % (
op.opname, hop.s_result,
hop.r_result, op.opname, resulttype))
# figure out if the resultvar is a completely fresh Variable or not
if (isinstance(resultvar, Variable) and
resultvar.annotation is None and
resultvar not in varmapping):
# fresh Variable: rename it to the previously existing op.result
varmapping[resultvar] = op.result
elif resultvar is op.result:
# special case: we got the previous op.result Variable again
assert varmapping[resultvar] is resultvar
else:
# renaming unsafe. Insert a 'same_as' operation...
hop.llops.append(SpaceOperation('same_as', [resultvar],
op.result))
def translate_no_return_value(self, hop):
op = hop.spaceop
if hop.s_result != annmodel.s_ImpossibleValue:
raise TyperError("the annotator doesn't agree that '%s' "
"has no return value" % op.opname)
op.result.concretetype = Void
def gottypererror(self, exc, block, position):
"""Record information about the location of a TyperError"""
graph = self.annotator.annotated.get(block)
exc.where = (graph, block, position)
# __________ regular operations __________
def _registeroperations(cls, unary_ops, binary_ops):
d = {}
# All unary operations
for opname in unary_ops:
fnname = 'translate_op_' + opname
exec py.code.compile("""
def translate_op_%s(self, hop):
r_arg1 = hop.args_r[0]
return r_arg1.rtype_%s(hop)
""" % (opname, opname)) in globals(), d
setattr(cls, fnname, d[fnname])
# All binary operations
for opname in binary_ops:
fnname = 'translate_op_' + opname
exec py.code.compile("""
def translate_op_%s(self, hop):
r_arg1 = hop.args_r[0]
r_arg2 = hop.args_r[1]
return pair(r_arg1, r_arg2).rtype_%s(hop)
""" % (opname, opname)) in globals(), d
setattr(cls, fnname, d[fnname])
_registeroperations = classmethod(_registeroperations)
# this one is not in BINARY_OPERATIONS
def translate_op_contains(self, hop):
r_arg1 = hop.args_r[0]
r_arg2 = hop.args_r[1]
return pair(r_arg1, r_arg2).rtype_contains(hop)
# __________ irregular operations __________
def translate_op_newlist(self, hop):
return rlist.rtype_newlist(hop)
def translate_op_newdict(self, hop):
return rdict.rtype_newdict(hop)
def translate_op_alloc_and_set(self, hop):
return rlist.rtype_alloc_and_set(hop)
def translate_op_extend_with_str_slice(self, hop):
r_arg1 = hop.args_r[0]
r_arg2 = hop.args_r[3]
return pair(r_arg1, r_arg2).rtype_extend_with_str_slice(hop)
def translate_op_extend_with_char_count(self, hop):
r_arg1 = hop.args_r[0]
r_arg2 = hop.args_r[1]
return pair(r_arg1, r_arg2).rtype_extend_with_char_count(hop)
def translate_op_newtuple(self, hop):
from rpython.rtyper.rtuple import rtype_newtuple
return rtype_newtuple(hop)
def translate_op_instantiate1(self, hop):
if not isinstance(hop.s_result, annmodel.SomeInstance):
raise TyperError("instantiate1 got s_result=%r" % (hop.s_result,))
classdef = hop.s_result.classdef
return rclass.rtype_new_instance(self, classdef, hop.llops)
def default_translate_operation(self, hop):
raise TyperError("unimplemented operation: '%s'" % hop.spaceop.opname)
# __________ utilities __________
def needs_wrapper(self, cls):
return cls in self.classes_with_wrapper
def get_wrapping_hint(self, clsdef):
cls = clsdef.classdesc.pyobj
return self.classes_with_wrapper[cls], self.wrapper_context
def getcallable(self, graph):
def getconcretetype(v):
return self.bindingrepr(v).lowleveltype
if self.annotator.translator.config.translation.sandbox:
try:
name = graph.func._sandbox_external_name
except AttributeError:
pass
else:
args_s = [v.annotation for v in graph.getargs()]
s_result = graph.getreturnvar().annotation
sandboxed = make_sandbox_trampoline(name, args_s, s_result)
return self.getannmixlevel().delayedfunction(
sandboxed, args_s, s_result)
return getfunctionptr(graph, getconcretetype)
def annotate_helper(self, ll_function, argtypes):
"""Annotate the given low-level helper function and return its graph
"""
args_s = []
for s in argtypes:
# assume 's' is a low-level type, unless it is already an annotation
if not isinstance(s, annmodel.SomeObject):
s = lltype_to_annotation(s)
args_s.append(s)
# hack for bound methods
if hasattr(ll_function, 'im_func'):
bk = self.annotator.bookkeeper
args_s.insert(0, bk.immutablevalue(ll_function.im_self))
ll_function = ll_function.im_func
helper_graph = annotate_lowlevel_helper(self.annotator,
ll_function, args_s,
policy=self.lowlevel_ann_policy)
return helper_graph
def annotate_helper_fn(self, ll_function, argtypes):
"""Annotate the given low-level helper function
and return it as a function pointer
"""
graph = self.annotate_helper(ll_function, argtypes)
return self.getcallable(graph)
class RPythonTyper(object):
from rpython.rtyper.rmodel import log
def __init__(self, annotator, backend=genc_backend):
self.annotator = annotator
self.backend = backend
self.lowlevel_ann_policy = LowLevelAnnotatorPolicy(self)
self.reprs = {}
self._reprs_must_call_setup = []
self._seen_reprs_must_call_setup = {}
self._dict_traits = {}
self.rootclass_repr = RootClassRepr(self)
self.rootclass_repr.setup()
self.instance_reprs = {}
self.type_for_typeptr = {}
self.pbc_reprs = {}
self.classes_with_wrapper = {}
self.wrapper_context = None # or add an extra arg to convertvar?
self.classdef_to_pytypeobject = {}
self.concrete_calltables = {}
self.cache_dummy_values = {}
self.lltype2vtable = {}
# make the primitive_to_repr constant mapping
self.primitive_to_repr = {}
self.isinstance_helpers = {}
self.exceptiondata = ExceptionData(self)
self.custom_trace_funcs = []
try:
self.seed = int(os.getenv('RTYPERSEED'))
s = 'Using %d as seed for block shuffling' % self.seed
self.log.info(s)
except:
self.seed = 0
self.order = None
def getconfig(self):
return self.annotator.translator.config
def getprimitiverepr(self, lltype):
try:
return self.primitive_to_repr[lltype]
except KeyError:
pass
if isinstance(lltype, Primitive):
repr = self.primitive_to_repr[lltype] = self.getrepr(lltype_to_annotation(lltype))
return repr
raise TyperError('There is no primitive repr for %r' % (lltype,))
def add_wrapper(self, clsdef):
# record that this class has a wrapper, and what the __init__ is
cls = clsdef.classdesc.pyobj
init = getattr(cls.__init__, 'im_func', None)
self.classes_with_wrapper[cls] = init
def set_wrapper_context(self, obj):
# not nice, but we sometimes need to know which function we are wrapping
self.wrapper_context = obj
def add_pendingsetup(self, repr):
assert isinstance(repr, Repr)
if repr in self._seen_reprs_must_call_setup:
#warning("ignoring already seen repr for setup: %r" %(repr,))
return
self._reprs_must_call_setup.append(repr)
self._seen_reprs_must_call_setup[repr] = True
def lltype_to_classdef_mapping(self):
result = {}
for (classdef, _), repr in self.instance_reprs.iteritems():
result[repr.lowleveltype] = classdef
return result
def get_type_for_typeptr(self, typeptr):
search = typeptr._obj
try:
return self.type_for_typeptr[search]
except KeyError:
# rehash the dictionary, and perform a linear scan
# for the case of ll2ctypes typeptr
found = None
type_for_typeptr = {}
for key, value in self.type_for_typeptr.items():
type_for_typeptr[key] = value
if key == search:
found = value
self.type_for_typeptr = type_for_typeptr
if found is None:
raise KeyError(search)
return found
def set_type_for_typeptr(self, typeptr, TYPE):
self.type_for_typeptr[typeptr._obj] = TYPE
self.lltype2vtable[TYPE] = typeptr
def get_real_typeptr_for_typeptr(self, typeptr):
# perform a linear scan for the case of ll2ctypes typeptr
search = typeptr._obj
for key, value in self.type_for_typeptr.items():
if key == search:
return key._as_ptr()
raise KeyError(search)
def getrepr(self, s_obj):
# s_objs are not hashable... try hard to find a unique key anyway
key = s_obj.rtyper_makekey()
assert key[0] is s_obj.__class__
try:
result = self.reprs[key]
except KeyError:
self.reprs[key] = None
result = s_obj.rtyper_makerepr(self)
assert not isinstance(result.lowleveltype, ContainerType), (
"missing a Ptr in the type specification "
"of %s:\n%r" % (s_obj, result.lowleveltype))
self.reprs[key] = result
self.add_pendingsetup(result)
assert result is not None # recursive getrepr()!
return result
def annotation(self, var):
s_obj = self.annotator.annotation(var)
return s_obj
def binding(self, var):
s_obj = self.annotator.binding(var)
return s_obj
def bindingrepr(self, var):
return self.getrepr(self.binding(var))
def specialize(self, dont_simplify_again=False):
"""Main entry point: specialize all annotated blocks of the program."""
# specialize depends on annotator simplifications
if not dont_simplify_again:
self.annotator.simplify()
self.exceptiondata.finish(self)
# new blocks can be created as a result of specialize_block(), so
# we need to be careful about the loop here.
self.already_seen = {}
self.specialize_more_blocks()
self.exceptiondata.make_helpers(self)
self.specialize_more_blocks() # for the helpers just made
def getannmixlevel(self):
if self.annmixlevel is not None:
return self.annmixlevel
from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator
self.annmixlevel = MixLevelHelperAnnotator(self)
return self.annmixlevel
def specialize_more_blocks(self):
if self.already_seen:
newtext = ' more'
else:
newtext = ''
blockcount = 0
self.annmixlevel = None
while True:
# look for blocks not specialized yet
pending = [block for block in self.annotator.annotated
if block not in self.already_seen]
if not pending:
break
# shuffle blocks a bit
if self.seed:
import random
r = random.Random(self.seed)
r.shuffle(pending)
if self.order:
tracking = self.order(self.annotator, pending)
else:
tracking = lambda block: None
previous_percentage = 0
# specialize all blocks in the 'pending' list
for block in pending:
tracking(block)
blockcount += 1
self.specialize_block(block)
self.already_seen[block] = True
# progress bar
n = len(self.already_seen)
if n % 100 == 0:
total = len(self.annotator.annotated)
percentage = 100 * n // total
if percentage >= previous_percentage + 5:
previous_percentage = percentage
self.log.event('specializing: %d / %d blocks (%d%%)' %
(n, total, percentage))
# make sure all reprs so far have had their setup() called
self.call_all_setups()
self.log.event('-=- specialized %d%s blocks -=-' % (
blockcount, newtext))
annmixlevel = self.annmixlevel
del self.annmixlevel
if annmixlevel is not None:
annmixlevel.finish()
def call_all_setups(self):
# make sure all reprs so far have had their setup() called
must_setup_more = []
delayed = []
while self._reprs_must_call_setup:
r = self._reprs_must_call_setup.pop()
if r.is_setup_delayed():
delayed.append(r)
else:
r.setup()
must_setup_more.append(r)
for r in must_setup_more:
r.setup_final()
self._reprs_must_call_setup.extend(delayed)
def setconcretetype(self, v):
assert isinstance(v, Variable)
v.concretetype = self.bindingrepr(v).lowleveltype
def setup_block_entry(self, block):
if block.operations == () and len(block.inputargs) == 2:
# special case for exception blocks: force them to return an
# exception type and value in a standardized format
v1, v2 = block.inputargs
v1.concretetype = self.exceptiondata.lltype_of_exception_type
v2.concretetype = self.exceptiondata.lltype_of_exception_value
return [self.exceptiondata.r_exception_type,
self.exceptiondata.r_exception_value]
else:
# normal path
result = []
for a in block.inputargs:
r = self.bindingrepr(a)
a.concretetype = r.lowleveltype
result.append(r)
return result
def make_new_lloplist(self, block):
return LowLevelOpList(self, block)
def specialize_block(self, block):
graph = self.annotator.annotated[block]
if graph not in self.annotator.fixed_graphs:
self.annotator.fixed_graphs[graph] = True
# make sure that the return variables of all graphs
# are concretetype'd
self.setconcretetype(graph.getreturnvar())
# give the best possible types to the input args
try:
self.setup_block_entry(block)
except TyperError as e:
self.gottypererror(e, block, "block-entry")
raise
# specialize all the operations, as far as possible
if block.operations == (): # return or except block
return
newops = self.make_new_lloplist(block)
varmapping = {}
for v in block.getvariables():
varmapping[v] = v # records existing Variables
for hop in self.highlevelops(block, newops):
try:
hop.setup() # this is called from here to catch TyperErrors...
self.translate_hl_to_ll(hop, varmapping)
except TyperError as e:
self.gottypererror(e, block, hop.spaceop)
raise
block.operations[:] = newops
block.renamevariables(varmapping)
extrablock = None
pos = newops.llop_raising_exceptions
if (pos is not None and pos != len(newops) - 1):
# this is for the case where the llop that raises the exceptions
# is not the last one in the list.
assert block.canraise
noexclink = block.exits[0]
assert noexclink.exitcase is None
if pos == "removed":
# the exception cannot actually occur at all.
# This is set by calling exception_cannot_occur().
# We just remove all exception links.
block.exitswitch = None
block.exits = block.exits[:1]
else:
# We have to split the block in two, with the exception-catching
# exitswitch after the llop at 'pos', and the extra operations
# in the new part of the block, corresponding to the
# no-exception case. See for example test_rlist.test_indexerror
# or test_rpbc.test_multiple_ll_one_hl_op.
assert 0 <= pos < len(newops) - 1
extraops = block.operations[pos+1:]
del block.operations[pos+1:]
extrablock = insert_empty_block(noexclink, newops=extraops)
if extrablock is None:
self.insert_link_conversions(block)
else:
# skip the extrablock as a link target, its link doesn't need conversions
# by construction, OTOH some of involved vars have no annotation
# so proceeding with it would kill information
self.insert_link_conversions(block, skip=1)
# consider it as a link source instead
self.insert_link_conversions(extrablock)
def _convert_link(self, block, link):
if link.exitcase is not None and link.exitcase != 'default':
if isinstance(block.exitswitch, Variable):
r_case = self.bindingrepr(block.exitswitch)
else:
assert block.canraise
r_case = rclass.get_type_repr(self)
link.llexitcase = r_case.convert_const(link.exitcase)
else:
link.llexitcase = None
a = link.last_exception
if isinstance(a, Variable):
a.concretetype = self.exceptiondata.lltype_of_exception_type
elif isinstance(a, Constant):
link.last_exception = inputconst(
self.exceptiondata.r_exception_type, a.value)
a = link.last_exc_value
if isinstance(a, Variable):
a.concretetype = self.exceptiondata.lltype_of_exception_value
elif isinstance(a, Constant):
link.last_exc_value = inputconst(
self.exceptiondata.r_exception_value, a.value)
def insert_link_conversions(self, block, skip=0):
# insert the needed conversions on the links
can_insert_here = block.exitswitch is None and len(block.exits) == 1
for link in block.exits[skip:]:
self._convert_link(block, link)
inputargs_reprs = self.setup_block_entry(link.target)
newops = self.make_new_lloplist(block)
newlinkargs = {}
for i in range(len(link.args)):
a1 = link.args[i]
r_a2 = inputargs_reprs[i]
if isinstance(a1, Constant):
link.args[i] = inputconst(r_a2, a1.value)
continue # the Constant was typed, done
if a1 is link.last_exception:
r_a1 = self.exceptiondata.r_exception_type
elif a1 is link.last_exc_value:
r_a1 = self.exceptiondata.r_exception_value
else:
r_a1 = self.bindingrepr(a1)
if r_a1 == r_a2:
continue # no conversion needed
try:
new_a1 = newops.convertvar(a1, r_a1, r_a2)
except TyperError as e:
self.gottypererror(e, block, link)
raise
if new_a1 != a1:
newlinkargs[i] = new_a1
if newops:
if can_insert_here:
block.operations.extend(newops)
else:
# cannot insert conversion operations around a single
# link, unless it is the only exit of this block.
# create a new block along the link...
newblock = insert_empty_block(link,
# ...and store the conversions there.
newops=newops)
link = newblock.exits[0]
for i, new_a1 in newlinkargs.items():
link.args[i] = new_a1
def highlevelops(self, block, llops):
# enumerate the HighLevelOps in a block.
if block.operations:
for op in block.operations[:-1]:
yield HighLevelOp(self, op, [], llops)
# look for exception links for the last operation
if block.canraise:
exclinks = block.exits[1:]
else:
exclinks = []
yield HighLevelOp(self, block.operations[-1], exclinks, llops)
def translate_hl_to_ll(self, hop, varmapping):
#self.log.translating(hop.spaceop.opname, hop.args_s)
resultvar = hop.dispatch()
if hop.exceptionlinks and hop.llops.llop_raising_exceptions is None:
raise TyperError("the graph catches %s, but the rtyper did not "
"take exceptions into account "
"(exception_is_here() not called)" % (
[link.exitcase.__name__ for link in hop.exceptionlinks],))
if resultvar is None:
# no return value
self.translate_no_return_value(hop)
else:
assert isinstance(resultvar, (Variable, Constant))
op = hop.spaceop
# for simplicity of the translate_meth, resultvar is usually not
# op.result here. We have to replace resultvar with op.result
# in all generated operations.
if hop.s_result.is_constant():
if isinstance(resultvar, Constant) and \
isinstance(hop.r_result.lowleveltype, Primitive) and \
hop.r_result.lowleveltype is not Void:
# assert that they are equal, or both are 'nan'
assert resultvar.value == hop.s_result.const or (
math.isnan(resultvar.value) and
math.isnan(hop.s_result.const))
resulttype = resultvar.concretetype
op.result.concretetype = hop.r_result.lowleveltype
if op.result.concretetype != resulttype:
raise TyperError("inconsistent type for the result of '%s':\n"
"annotator says %s,\n"
"whose repr is %r\n"
"but rtype_%s returned %r" % (
op.opname, hop.s_result,
hop.r_result, op.opname, resulttype))
# figure out if the resultvar is a completely fresh Variable or not
if (isinstance(resultvar, Variable) and
resultvar.annotation is None and
resultvar not in varmapping):
# fresh Variable: rename it to the previously existing op.result
varmapping[resultvar] = op.result
elif resultvar is op.result:
# special case: we got the previous op.result Variable again
assert varmapping[resultvar] is resultvar
else:
# renaming unsafe. Insert a 'same_as' operation...
hop.llops.append(SpaceOperation('same_as', [resultvar],
op.result))
def translate_no_return_value(self, hop):
op = hop.spaceop
if hop.s_result != annmodel.s_ImpossibleValue:
raise TyperError("the annotator doesn't agree that '%s' "
"has no return value" % op.opname)
op.result.concretetype = Void
def gottypererror(self, exc, block, position):
"""Record information about the location of a TyperError"""
graph = self.annotator.annotated.get(block)
exc.where = (graph, block, position)
# __________ regular operations __________
def _registeroperations(cls, unary_ops, binary_ops):
d = {}
# All unary operations
for opname in unary_ops:
fnname = 'translate_op_' + opname
exec py.code.compile("""
def translate_op_%s(self, hop):
r_arg1 = hop.args_r[0]
return r_arg1.rtype_%s(hop)
""" % (opname, opname)) in globals(), d
setattr(cls, fnname, d[fnname])
# All binary operations
for opname in binary_ops:
fnname = 'translate_op_' + opname
exec py.code.compile("""
def translate_op_%s(self, hop):
r_arg1 = hop.args_r[0]
r_arg2 = hop.args_r[1]
return pair(r_arg1, r_arg2).rtype_%s(hop)
""" % (opname, opname)) in globals(), d
setattr(cls, fnname, d[fnname])
_registeroperations = classmethod(_registeroperations)
# this one is not in BINARY_OPERATIONS
def translate_op_contains(self, hop):
r_arg1 = hop.args_r[0]
r_arg2 = hop.args_r[1]
return pair(r_arg1, r_arg2).rtype_contains(hop)
# __________ irregular operations __________
def translate_op_newlist(self, hop):
return rlist.rtype_newlist(hop)
def translate_op_newdict(self, hop):
return rdict.rtype_newdict(hop)
def translate_op_alloc_and_set(self, hop):
return rlist.rtype_alloc_and_set(hop)
def translate_op_extend_with_str_slice(self, hop):
r_arg1 = hop.args_r[0]
r_arg2 = hop.args_r[3]
return pair(r_arg1, r_arg2).rtype_extend_with_str_slice(hop)
def translate_op_extend_with_char_count(self, hop):
r_arg1 = hop.args_r[0]
r_arg2 = hop.args_r[1]
return pair(r_arg1, r_arg2).rtype_extend_with_char_count(hop)
def translate_op_newtuple(self, hop):
from rpython.rtyper.rtuple import rtype_newtuple
return rtype_newtuple(hop)
def translate_op_instantiate1(self, hop):
if not isinstance(hop.s_result, annmodel.SomeInstance):
raise TyperError("instantiate1 got s_result=%r" % (hop.s_result,))
classdef = hop.s_result.classdef
return rclass.rtype_new_instance(self, classdef, hop.llops)
def default_translate_operation(self, hop):
raise TyperError("unimplemented operation: '%s'" % hop.spaceop.opname)
# __________ utilities __________
def needs_wrapper(self, cls):
return cls in self.classes_with_wrapper
def get_wrapping_hint(self, clsdef):
cls = clsdef.classdesc.pyobj
return self.classes_with_wrapper[cls], self.wrapper_context
def getcallable(self, graph):
def getconcretetype(v):
return self.bindingrepr(v).lowleveltype
if self.annotator.translator.config.translation.sandbox:
try:
name = graph.func._sandbox_external_name
except AttributeError:
pass
else:
args_s = [v.annotation for v in graph.getargs()]
s_result = graph.getreturnvar().annotation
sandboxed = make_sandbox_trampoline(name, args_s, s_result)
return self.getannmixlevel().delayedfunction(
sandboxed, args_s, s_result)
return getfunctionptr(graph, getconcretetype)
def annotate_helper(self, ll_function, argtypes):
"""Annotate the given low-level helper function and return its graph
"""
args_s = []
for s in argtypes:
# assume 's' is a low-level type, unless it is already an annotation
if not isinstance(s, annmodel.SomeObject):
s = lltype_to_annotation(s)
args_s.append(s)
# hack for bound methods
if hasattr(ll_function, 'im_func'):
bk = self.annotator.bookkeeper
args_s.insert(0, bk.immutablevalue(ll_function.im_self))
ll_function = ll_function.im_func
helper_graph = annotate_lowlevel_helper(self.annotator,
ll_function, args_s,
policy=self.lowlevel_ann_policy)
return helper_graph
def annotate_helper_fn(self, ll_function, argtypes):
"""Annotate the given low-level helper function
and return it as a function pointer
"""
graph = self.annotate_helper(ll_function, argtypes)
return self.getcallable(graph)
class RPythonTyper(object):
from rpython.rtyper.rmodel import log
def __init__(self, annotator):
self.annotator = annotator
self.lowlevel_ann_policy = LowLevelAnnotatorPolicy(self)
self.type_system = LowLevelTypeSystem()
self.reprs = {}
self._reprs_must_call_setup = []
self._seen_reprs_must_call_setup = {}
self._dict_traits = {}
self.rootclass_repr = RootClassRepr(self)
self.rootclass_repr.setup()
self.instance_reprs = {}
self.type_for_typeptr = {}
self.pbc_reprs = {}
self.classes_with_wrapper = {}
self.wrapper_context = None # or add an extra arg to convertvar?
self.classdef_to_pytypeobject = {}
self.concrete_calltables = {}
self.cache_dummy_values = {}
self.lltype2vtable = {}
self.typererrors = []
self.typererror_count = 0
# make the primitive_to_repr constant mapping
self.primitive_to_repr = {}
self.isinstance_helpers = {}
self.exceptiondata = ExceptionData(self)
self.custom_trace_funcs = []
try:
self.seed = int(os.getenv('RTYPERSEED'))
s = 'Using %d as seed for block shuffling' % self.seed
self.log.info(s)
except:
self.seed = 0
self.order = None
def getconfig(self):
return self.annotator.translator.config
def getprimitiverepr(self, lltype):
try:
return self.primitive_to_repr[lltype]
except KeyError:
pass
if isinstance(lltype, Primitive):
repr = self.primitive_to_repr[lltype] = self.getrepr(lltype_to_annotation(lltype))
return repr
raise TyperError('There is no primitive repr for %r' % (lltype,))
def add_wrapper(self, clsdef):
# record that this class has a wrapper, and what the __init__ is
cls = clsdef.classdesc.pyobj
init = getattr(cls.__init__, 'im_func', None)
self.classes_with_wrapper[cls] = init
def set_wrapper_context(self, obj):
# not nice, but we sometimes need to know which function we are wrapping
self.wrapper_context = obj
def add_pendingsetup(self, repr):
assert isinstance(repr, Repr)
if repr in self._seen_reprs_must_call_setup:
#warning("ignoring already seen repr for setup: %r" %(repr,))
return
self._reprs_must_call_setup.append(repr)
self._seen_reprs_must_call_setup[repr] = True
def lltype_to_classdef_mapping(self):
result = {}
for (classdef, _), repr in self.instance_reprs.iteritems():
result[repr.lowleveltype] = classdef
return result
def get_type_for_typeptr(self, typeptr):
search = typeptr._obj
try:
return self.type_for_typeptr[search]
except KeyError:
# rehash the dictionary, and perform a linear scan
# for the case of ll2ctypes typeptr
found = None
type_for_typeptr = {}
for key, value in self.type_for_typeptr.items():
type_for_typeptr[key] = value
if key == search:
found = value
self.type_for_typeptr = type_for_typeptr
if found is None:
raise KeyError(search)
return found
def set_type_for_typeptr(self, typeptr, TYPE):
self.type_for_typeptr[typeptr._obj] = TYPE
self.lltype2vtable[TYPE] = typeptr
def get_real_typeptr_for_typeptr(self, typeptr):
# perform a linear scan for the case of ll2ctypes typeptr
search = typeptr._obj
for key, value in self.type_for_typeptr.items():
if key == search:
return key._as_ptr()
raise KeyError(search)
def getrepr(self, s_obj):
# s_objs are not hashable... try hard to find a unique key anyway
key = s_obj.rtyper_makekey()
assert key[0] is s_obj.__class__
try:
result = self.reprs[key]
except KeyError:
self.reprs[key] = None
result = s_obj.rtyper_makerepr(self)
assert not isinstance(result.lowleveltype, ContainerType), (
"missing a Ptr in the type specification "
"of %s:\n%r" % (s_obj, result.lowleveltype))
self.reprs[key] = result
self.add_pendingsetup(result)
assert result is not None # recursive getrepr()!
return result
def annotation(self, var):
s_obj = self.annotator.annotation(var)
return s_obj
def binding(self, var):
s_obj = self.annotator.binding(var)
return s_obj
def bindingrepr(self, var):
return self.getrepr(self.binding(var))
def specialize(self, dont_simplify_again=False):
"""Main entry point: specialize all annotated blocks of the program."""
# specialize depends on annotator simplifications
assert dont_simplify_again in (False, True) # safety check
if not dont_simplify_again:
self.annotator.simplify()
# first make sure that all functions called in a group have exactly
# the same signature, by hacking their flow graphs if needed
perform_normalizations(self.annotator)
self.exceptiondata.finish(self)
# new blocks can be created as a result of specialize_block(), so
# we need to be careful about the loop here.
self.already_seen = {}
self.specialize_more_blocks()
if self.exceptiondata is not None:
self.exceptiondata.make_helpers(self)
self.specialize_more_blocks() # for the helpers just made
def getannmixlevel(self):
if self.annmixlevel is not None:
return self.annmixlevel
from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator
self.annmixlevel = MixLevelHelperAnnotator(self)
return self.annmixlevel
def specialize_more_blocks(self):
if self.already_seen:
newtext = ' more'
else:
newtext = ''
blockcount = 0
self.annmixlevel = None
while True:
# look for blocks not specialized yet
pending = [block for block in self.annotator.annotated
if block not in self.already_seen]
if not pending:
break
# shuffle blocks a bit
if self.seed:
import random
r = random.Random(self.seed)
r.shuffle(pending)
if self.order:
tracking = self.order(self.annotator, pending)
else:
tracking = lambda block: None
previous_percentage = 0
# specialize all blocks in the 'pending' list
for block in pending:
tracking(block)
blockcount += 1
self.specialize_block(block)
self.already_seen[block] = True
# progress bar
n = len(self.already_seen)
if n % 100 == 0:
total = len(self.annotator.annotated)
percentage = 100 * n // total
if percentage >= previous_percentage + 5:
previous_percentage = percentage
if self.typererror_count:
error_report = " but %d errors" % self.typererror_count
else:
error_report = ''
self.log.event('specializing: %d / %d blocks (%d%%)%s' %
(n, total, percentage, error_report))
# make sure all reprs so far have had their setup() called
self.call_all_setups()
if self.typererrors:
self.dump_typererrors(to_log=True)
raise TyperError("there were %d error" % len(self.typererrors))
self.log.event('-=- specialized %d%s blocks -=-' % (
blockcount, newtext))
annmixlevel = self.annmixlevel
del self.annmixlevel
if annmixlevel is not None:
annmixlevel.finish()
def dump_typererrors(self, num=None, minimize=True, to_log=False):
c = 0
bc = 0
for err in self.typererrors[:num]:
c += 1
if minimize and isinstance(err, BrokenReprTyperError):
bc += 1
continue
graph, block, position = err.where
errmsg = ("TyperError-%d: %s\n" % (c, graph) +
str(err) +
"\n")
if to_log:
self.log.ERROR(errmsg)
else:
print errmsg
if bc:
minmsg = "(minimized %d errors away for this dump)" % (bc,)
if to_log:
self.log.ERROR(minmsg)
else:
print minmsg
def call_all_setups(self):
# make sure all reprs so far have had their setup() called
must_setup_more = []
delayed = []
while self._reprs_must_call_setup:
r = self._reprs_must_call_setup.pop()
if r.is_setup_delayed():
delayed.append(r)
else:
r.setup()
must_setup_more.append(r)
for r in must_setup_more:
r.setup_final()
self._reprs_must_call_setup.extend(delayed)
def setconcretetype(self, v):
assert isinstance(v, Variable)
v.concretetype = self.bindingrepr(v).lowleveltype
def setup_block_entry(self, block):
if block.operations == () and len(block.inputargs) == 2:
# special case for exception blocks: force them to return an
# exception type and value in a standardized format
v1, v2 = block.inputargs
v1.concretetype = self.exceptiondata.lltype_of_exception_type
v2.concretetype = self.exceptiondata.lltype_of_exception_value
return [self.exceptiondata.r_exception_type,
self.exceptiondata.r_exception_value]
else:
# normal path
result = []
for a in block.inputargs:
r = self.bindingrepr(a)
a.concretetype = r.lowleveltype
result.append(r)
return result
def make_new_lloplist(self, block):
return LowLevelOpList(self, block)
def specialize_block(self, block):
graph = self.annotator.annotated[block]
if graph not in self.annotator.fixed_graphs:
self.annotator.fixed_graphs[graph] = True
# make sure that the return variables of all graphs
# are concretetype'd
self.setconcretetype(graph.getreturnvar())
# give the best possible types to the input args
try:
self.setup_block_entry(block)
except TyperError, e:
self.gottypererror(e, block, "block-entry", None)
return # cannot continue this block
# specialize all the operations, as far as possible
if block.operations == (): # return or except block
return
newops = self.make_new_lloplist(block)
varmapping = {}
for v in block.getvariables():
varmapping[v] = v # records existing Variables
for hop in self.highlevelops(block, newops):
try:
hop.setup() # this is called from here to catch TyperErrors...
self.translate_hl_to_ll(hop, varmapping)
except TyperError, e:
self.gottypererror(e, block, hop.spaceop, newops)
return # cannot continue this block: no op.result.concretetype