def test_setfield_forced_virtual(self):
loop = """
[p1, p2]
i1 = getfield_gc_i(p1, descr=valuedescr)
setfield_gc(p2, i1, descr=valuedescr)
p3 = new_with_vtable(descr=nodesize)
jump(p2, p3)
"""
es, loop, preamble = self.optimize(loop)
sb = ShortPreambleBuilder(loop.inputargs, es.short_boxes,
es.short_inputargs,
es.exported_infos)
short_boxes = [box for box in es.short_boxes
if not isinstance(box.short_op, ShortInputArg)]
op = short_boxes[0].short_op.res
pop = sb.use_box(op, short_boxes[0].preamble_op, FakeOptimizer())
sb.add_preamble_op(PreambleOp(op, pop, False))
exp_short = """
[p0, p1]
guard_nonnull(p0) []
guard_subclass(p0, ConstClass(node_vtable)) []
i1 = getfield_gc_i(p0, descr=valuedescr)
jump(i1)
"""
self.compare_short(sb.build_short_preamble(), exp_short)
def import_state(self, targetargs, exported_state):
# the mapping between input args (from old label) and what we need
# to actually emit. Update the info
assert (len(exported_state.next_iteration_args) ==
len(targetargs))
self._check_no_forwarding([targetargs])
exported_state._check_no_forwarding()
for i, target in enumerate(exported_state.next_iteration_args):
source = targetargs[i]
assert source is not target
source.set_forwarded(target)
info = exported_state.exported_infos.get(target, None)
if info is not None:
self.optimizer.setinfo_from_preamble(source, info,
exported_state.exported_infos)
# import the optimizer state, starting from boxes that can be produced
# by short preamble
label_args = exported_state.virtual_state.make_inputargs(
targetargs, self.optimizer)
self.short_preamble_producer = ShortPreambleBuilder(
label_args, exported_state.short_boxes,
exported_state.short_inputargs, exported_state.exported_infos,
self.optimizer)
for produced_op in exported_state.short_boxes:
produced_op.produce_op(self, exported_state.exported_infos)
return label_args
def test_short_boxes_heapcache(self):
loop = """
[p0, i1]
i0 = getfield_gc_i(p0, descr=valuedescr)
jump(p0, i1)
"""
es, loop, preamble = self.optimize(loop)
op = preamble.operations[0]
short_boxes = [box for box in es.short_boxes
if not isinstance(box.short_op, ShortInputArg)]
assert len(short_boxes) == 1
assert short_boxes[0].short_op.res is op
sb = ShortPreambleBuilder(loop.inputargs,
es.short_boxes, es.short_inputargs,
es.exported_infos, FakeOptimizer())
op = preamble.operations[0]
short_op = sb.use_box(op, short_boxes[0].preamble_op, FakeOptimizer())
sb.add_preamble_op(PreambleOp(op, short_op, False))
exp_short = """
[p0, i1]
guard_nonnull(p0) []
guard_subclass(p0, ConstClass(node_vtable)) []
i0 = getfield_gc_i(p0, descr=valuedescr)
jump(i0)
"""
self.compare_short(sb.build_short_preamble(), exp_short)
def import_state(self, targetop, exported_state):
# the mapping between input args (from old label) and what we need
# to actually emit. Update the info
assert (len(exported_state.next_iteration_args) ==
len(targetop.getarglist()))
for i, target in enumerate(exported_state.next_iteration_args):
source = targetop.getarg(i)
assert source is not target
source.set_forwarded(target)
info = exported_state.exported_infos.get(target, None)
if info is not None:
self.optimizer.setinfo_from_preamble(source, info,
exported_state.exported_infos)
# import the optimizer state, starting from boxes that can be produced
# by short preamble
label_args = exported_state.virtual_state.make_inputargs(
targetop.getarglist(), self.optimizer)
self.short_preamble_producer = ShortPreambleBuilder(
label_args, exported_state.short_boxes,
exported_state.short_inputargs, exported_state.exported_infos,
self.optimizer)
for produced_op in exported_state.short_boxes:
produced_op.produce_op(self, exported_state.exported_infos)
return label_args
def test_setfield_forced_virtual(self):
loop = """
[p1, p2]
i1 = getfield_gc_i(p1, descr=valuedescr)
setfield_gc(p2, i1, descr=valuedescr)
p3 = new_with_vtable(descr=nodesize)
jump(p2, p3)
"""
es, loop, preamble = self.optimize(loop)
sb = ShortPreambleBuilder(loop.inputargs, es.short_boxes,
es.short_inputargs, es.exported_infos)
short_boxes = [
box for box in es.short_boxes
if not isinstance(box.short_op, ShortInputArg)
]
op = short_boxes[0].short_op.res
pop = sb.use_box(op, short_boxes[0].preamble_op, FakeOptimizer())
sb.add_preamble_op(PreambleOp(op, pop, False))
exp_short = """
[p0, p1]
guard_nonnull(p0) []
guard_subclass(p0, ConstClass(node_vtable)) []
i1 = getfield_gc_i(p0, descr=valuedescr)
jump(i1)
"""
self.compare_short(sb.build_short_preamble(), exp_short)
def test_short_boxes_heapcache(self):
loop = """
[p0, i1]
i0 = getfield_gc_i(p0, descr=valuedescr)
jump(p0, i1)
"""
es, loop, preamble = self.optimize(loop)
op = preamble.operations[0]
short_boxes = [
box for box in es.short_boxes
if not isinstance(box.short_op, ShortInputArg)
]
assert len(short_boxes) == 1
assert short_boxes[0].short_op.res is op
sb = ShortPreambleBuilder(loop.inputargs, es.short_boxes,
es.short_inputargs, es.exported_infos,
FakeOptimizer())
op = preamble.operations[0]
short_op = sb.use_box(op, short_boxes[0].preamble_op, FakeOptimizer())
sb.add_preamble_op(PreambleOp(op, short_op, False))
exp_short = """
[p0, i1]
guard_nonnull(p0) []
guard_subclass(p0, ConstClass(node_vtable)) []
i0 = getfield_gc_i(p0, descr=valuedescr)
jump(i0)
"""
self.compare_short(sb.build_short_preamble(), exp_short)
def test_simple(self):
loop = """
[i0]
i1 = int_add(i0, 1)
guard_value(i1, 1) []
jump(i1)
"""
es, loop, preamble = self.optimize(loop)
vs = es.virtual_state
assert isinstance(vs.state[0], NotVirtualStateInfo)
# the virtual state is constant, so we don't need to have it in
# inputargs
assert vs.make_inputargs([1], FakeOptimizer()) == []
assert vs.state[0].level == LEVEL_CONSTANT
# we have exported values for i1, which happens to be an inputarg
sb = ShortPreambleBuilder([], es.short_boxes, es.short_inputargs,
es.exported_infos)
sp = sb.build_short_preamble()
exp = """
[]
jump()
"""
self.compare_short(sp, exp)
def test_simple(self):
loop = """
[i0]
i1 = int_add(i0, 1)
guard_value(i1, 1) []
jump(i1)
"""
es, loop, preamble = self.optimize(loop)
vs = es.virtual_state
assert isinstance(vs.state[0], NotVirtualStateInfo)
# the virtual state is constant, so we don't need to have it in
# inputargs
assert vs.make_inputargs([1], FakeOptimizer()) == []
assert vs.state[0].level == LEVEL_CONSTANT
# we have exported values for i1, which happens to be an inputarg
sb = ShortPreambleBuilder([], es.short_boxes, es.short_inputargs,
es.exported_infos)
sp = sb.build_short_preamble()
exp = """
[]
jump()
"""
self.compare_short(sp, exp)
def finalize_short_preamble(self, label_op, virtual_state):
sb = self.short_preamble_producer
self.optimizer._clean_optimization_info(sb.short_inputargs)
short_preamble = sb.build_short_preamble()
jitcelltoken = label_op.getdescr()
assert isinstance(jitcelltoken, JitCellToken)
if jitcelltoken.target_tokens is None:
jitcelltoken.target_tokens = []
target_token = TargetToken(jitcelltoken,
original_jitcell_token=jitcelltoken)
target_token.original_jitcell_token = jitcelltoken
target_token.virtual_state = virtual_state
target_token.short_preamble = short_preamble
jitcelltoken.target_tokens.append(target_token)
self.short_preamble_producer = ExtendedShortPreambleBuilder(
target_token, sb)
label_op.initarglist(label_op.getarglist() + sb.used_boxes)
return target_token
def finalize_short_preamble(self, label_op, virtual_state):
sb = self.short_preamble_producer
self.optimizer._clean_optimization_info(sb.short_inputargs)
short_preamble = sb.build_short_preamble()
jitcelltoken = label_op.getdescr()
assert isinstance(jitcelltoken, JitCellToken)
if jitcelltoken.target_tokens is None:
jitcelltoken.target_tokens = []
target_token = TargetToken(jitcelltoken,
original_jitcell_token=jitcelltoken)
target_token.original_jitcell_token = jitcelltoken
target_token.virtual_state = virtual_state
target_token.short_preamble = short_preamble
jitcelltoken.target_tokens.append(target_token)
self.short_preamble_producer = ExtendedShortPreambleBuilder(
target_token, sb)
label_op.initarglist(label_op.getarglist() + sb.used_boxes)
return target_token
class UnrollOptimizer(Optimization):
"""Unroll the loop into two iterations. The first one will
become the preamble or entry bridge (don't think there is a
distinction anymore)"""
short_preamble_producer = None
def __init__(self, metainterp_sd, jitdriver_sd, optimizations):
self.optimizer = UnrollableOptimizer(metainterp_sd, jitdriver_sd,
optimizations)
self.optimizer.optunroll = self
def get_virtual_state(self, args):
modifier = VirtualStateConstructor(self.optimizer)
return modifier.get_virtual_state(args)
def _check_no_forwarding(self, lsts, check_newops=True):
check_no_forwarding(lsts)
if check_newops:
assert not self.optimizer._newoperations
def optimize_preamble(self, trace, runtime_boxes, call_pure_results, memo):
info, newops = self.optimizer.propagate_all_forward(
trace.get_iter(), call_pure_results, flush=False)
exported_state = self.export_state(info.jump_op.getarglist(),
info.inputargs, memo)
exported_state.quasi_immutable_deps = info.quasi_immutable_deps
# we need to absolutely make sure that we've cleaned up all
# the optimization info
self.optimizer._clean_optimization_info(self.optimizer._newoperations)
return exported_state, self.optimizer._newoperations
def optimize_peeled_loop(self, trace, celltoken, state, runtime_boxes,
call_pure_results, inline_short_preamble=True):
trace = trace.get_iter()
try:
label_args = self.import_state(trace.inputargs, state)
except VirtualStatesCantMatch:
raise InvalidLoop("Cannot import state, virtual states don't match")
self.potential_extra_ops = {}
self.optimizer.init_inparg_dict_from(label_args)
try:
info, _ = self.optimizer.propagate_all_forward(
trace, call_pure_results, flush=False)
except SpeculativeError:
raise InvalidLoop("Speculative heap access would be ill-typed")
end_jump = info.jump_op
label_op = ResOperation(rop.LABEL, label_args,
descr=celltoken)
for a in end_jump.getarglist():
self.optimizer.force_box_for_end_of_preamble(
self.optimizer.get_box_replacement(a))
current_vs = self.get_virtual_state(end_jump.getarglist())
# pick the vs we want to jump to
assert isinstance(celltoken, JitCellToken)
target_virtual_state = self.pick_virtual_state(current_vs,
state.virtual_state,
celltoken.target_tokens)
# force the boxes for virtual state to match
try:
args = target_virtual_state.make_inputargs(
[self.get_box_replacement(x) for x in end_jump.getarglist()],
self.optimizer, force_boxes=True)
for arg in args:
if arg is not None:
self.optimizer.force_box(arg)
except VirtualStatesCantMatch:
raise InvalidLoop("Virtual states did not match "
"after picking the virtual state, when forcing"
" boxes")
extra_same_as = self.short_preamble_producer.extra_same_as[:]
target_token = self.finalize_short_preamble(label_op,
state.virtual_state)
label_op.setdescr(target_token)
if not inline_short_preamble:
self.jump_to_preamble(celltoken, end_jump, info)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
try:
new_virtual_state = self.jump_to_existing_trace(
end_jump, label_op, runtime_boxes, force_boxes=False)
except InvalidLoop:
# inlining short preamble failed, jump to preamble
self.jump_to_preamble(celltoken, end_jump, info)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
if new_virtual_state is not None:
# Attempt to force virtual boxes in order to avoid jumping
# to the preamble.
try:
new_virtual_state = self.jump_to_existing_trace(
end_jump, label_op, runtime_boxes, force_boxes=True)
except InvalidLoop:
pass
if new_virtual_state is not None:
self.jump_to_preamble(celltoken, end_jump, info)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
self.disable_retracing_if_max_retrace_guards(
self.optimizer._newoperations, target_token)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
def disable_retracing_if_max_retrace_guards(self, ops, target_token):
maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards
count = 0
for op in ops:
if op.is_guard():
count += 1
if count > maxguards:
assert isinstance(target_token, TargetToken)
target_token.targeting_jitcell_token.retraced_count = sys.maxint
def pick_virtual_state(self, my_vs, label_vs, target_tokens):
if target_tokens is None:
return label_vs # for tests
for token in target_tokens:
if token.virtual_state is None:
continue
if token.virtual_state.generalization_of(my_vs, self.optimizer):
return token.virtual_state
return label_vs
def optimize_bridge(self, trace, runtime_boxes, call_pure_results,
inline_short_preamble, box_names_memo, resumestorage):
from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge
frontend_inputargs = trace.inputargs
trace = trace.get_iter()
self._check_no_forwarding([trace.inputargs])
if resumestorage:
deserialize_optimizer_knowledge(self.optimizer,
resumestorage, frontend_inputargs,
trace.inputargs)
info, ops = self.optimizer.propagate_all_forward(trace,
call_pure_results, False)
jump_op = info.jump_op
cell_token = jump_op.getdescr()
assert isinstance(cell_token, JitCellToken)
if not inline_short_preamble or len(cell_token.target_tokens) == 1:
return self.jump_to_preamble(cell_token, jump_op, info)
# force all the information that does not go to the short
# preamble at all
self.optimizer.flush()
for a in jump_op.getarglist():
self.optimizer.force_box_for_end_of_preamble(a)
try:
vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
force_boxes=False)
except InvalidLoop:
return self.jump_to_preamble(cell_token, jump_op, info)
if vs is None:
return info, self.optimizer._newoperations[:]
warmrunnerdescr = self.optimizer.metainterp_sd.warmrunnerdesc
limit = warmrunnerdescr.memory_manager.retrace_limit
if cell_token.retraced_count < limit:
cell_token.retraced_count += 1
debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit))
else:
# Try forcing boxes to avoid jumping to the preamble
try:
vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
force_boxes=True)
except InvalidLoop:
pass
if vs is None:
return info, self.optimizer._newoperations[:]
debug_print("Retrace count reached, jumping to preamble")
return self.jump_to_preamble(cell_token, jump_op, info)
exported_state = self.export_state(info.jump_op.getarglist(),
info.inputargs, box_names_memo)
exported_state.quasi_immutable_deps = self.optimizer.quasi_immutable_deps
self.optimizer._clean_optimization_info(self.optimizer._newoperations)
return exported_state, self.optimizer._newoperations
def finalize_short_preamble(self, label_op, virtual_state):
sb = self.short_preamble_producer
self.optimizer._clean_optimization_info(sb.short_inputargs)
short_preamble = sb.build_short_preamble()
jitcelltoken = label_op.getdescr()
assert isinstance(jitcelltoken, JitCellToken)
if jitcelltoken.target_tokens is None:
jitcelltoken.target_tokens = []
target_token = TargetToken(jitcelltoken,
original_jitcell_token=jitcelltoken)
target_token.original_jitcell_token = jitcelltoken
target_token.virtual_state = virtual_state
target_token.short_preamble = short_preamble
jitcelltoken.target_tokens.append(target_token)
self.short_preamble_producer = ExtendedShortPreambleBuilder(
target_token, sb)
label_op.initarglist(label_op.getarglist() + sb.used_boxes)
return target_token
def jump_to_preamble(self, cell_token, jump_op, info):
assert cell_token.target_tokens[0].virtual_state is None
jump_op = jump_op.copy_and_change(rop.JUMP,
descr=cell_token.target_tokens[0])
self.optimizer.send_extra_operation(jump_op)
return info, self.optimizer._newoperations[:]
def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes, force_boxes=False):
jitcelltoken = jump_op.getdescr()
assert isinstance(jitcelltoken, JitCellToken)
virtual_state = self.get_virtual_state(jump_op.getarglist())
args = [self.get_box_replacement(op) for op in jump_op.getarglist()]
for target_token in jitcelltoken.target_tokens:
target_virtual_state = target_token.virtual_state
if target_virtual_state is None:
continue
try:
extra_guards = target_virtual_state.generate_guards(
virtual_state, args, runtime_boxes, self.optimizer,
force_boxes=force_boxes)
patchguardop = self.optimizer.patchguardop
for guard in extra_guards.extra_guards:
if isinstance(guard, GuardResOp):
guard.rd_resume_position = patchguardop.rd_resume_position
guard.setdescr(compile.ResumeAtPositionDescr())
self.send_extra_operation(guard)
except VirtualStatesCantMatch:
continue
# When force_boxes == True, creating the virtual args can fail when
# components of the virtual state alias. If this occurs, we must
# recompute the virtual state as boxes will have been forced.
try:
args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
args, self.optimizer, force_boxes=force_boxes)
except VirtualStatesCantMatch:
assert force_boxes
virtual_state = self.get_virtual_state(args)
continue
short_preamble = target_token.short_preamble
try:
extra = self.inline_short_preamble(args + virtuals, args,
short_preamble, self.optimizer.patchguardop,
target_token, label_op)
except KeyError:
# SHOULD NOT OCCUR BUT DOES: WHY?? issue #2185
self.optimizer.metainterp_sd.logger_ops.log_short_preamble([],
short_preamble, {})
raise
self.send_extra_operation(jump_op.copy_and_change(rop.JUMP,
args=args + extra,
descr=target_token))
return None # explicit because the return can be non-None
return virtual_state
def _map_args(self, mapping, arglist):
result = []
for box in arglist:
if not isinstance(box, Const):
box = mapping[box]
result.append(box)
return result
def inline_short_preamble(self, jump_args, args_no_virtuals, short,
patchguardop, target_token, label_op):
short_inputargs = short[0].getarglist()
short_jump_args = short[-1].getarglist()
sb = self.short_preamble_producer
if sb is not None:
assert isinstance(sb, ExtendedShortPreambleBuilder)
if sb.target_token is target_token:
# this means we're inlining the short preamble that's being
# built. Make sure we modify the correct things in-place
self.short_preamble_producer.setup(short_jump_args,
short, label_op.getarglist())
# after this call, THE REST OF THIS FUNCTION WILL MODIFY ALL
# THE LISTS PROVIDED, POTENTIALLY
# We need to make a list of fresh new operations corresponding
# to the short preamble operations. We could temporarily forward
# the short operations to the fresh ones, but there are obscure
# issues: send_extra_operation() below might occasionally invoke
# use_box(), which assumes the short operations are not forwarded.
# So we avoid such temporary forwarding and just use a dict here.
assert len(short_inputargs) == len(jump_args)
mapping = {}
for i in range(len(jump_args)):
mapping[short_inputargs[i]] = jump_args[i]
# a fix-point loop, runs only once in almost all cases
i = 1
while 1:
self._check_no_forwarding([short_inputargs, short], False)
while i < len(short) - 1:
sop = short[i]
arglist = self._map_args(mapping, sop.getarglist())
if sop.is_guard():
op = sop.copy_and_change(sop.getopnum(), arglist,
descr=compile.ResumeAtPositionDescr())
assert isinstance(op, GuardResOp)
op.rd_resume_position = patchguardop.rd_resume_position
else:
op = sop.copy_and_change(sop.getopnum(), arglist)
mapping[sop] = op
i += 1
self.optimizer.send_extra_operation(op)
# force all of them except the virtuals
for arg in (args_no_virtuals +
self._map_args(mapping, short_jump_args)):
self.optimizer.force_box(self.get_box_replacement(arg))
self.optimizer.flush()
# done unless "short" has grown again
if i == len(short) - 1:
break
return [self.get_box_replacement(box)
for box in self._map_args(mapping, short_jump_args)]
def _expand_info(self, arg, infos):
if isinstance(arg, AbstractResOp) and rop.is_same_as(arg.opnum):
info = self.optimizer.getinfo(arg.getarg(0))
else:
info = self.optimizer.getinfo(arg)
if arg in infos:
return
if info:
infos[arg] = info
if info.is_virtual():
self._expand_infos_from_virtual(info, infos)
def _expand_infos_from_virtual(self, info, infos):
items = info.all_items()
for item in items:
if item is None:
continue
self._expand_info(item, infos)
def export_state(self, original_label_args, renamed_inputargs, memo):
end_args = [self.optimizer.force_box_for_end_of_preamble(a)
for a in original_label_args]
self.optimizer.flush()
virtual_state = self.get_virtual_state(end_args)
end_args = [self.get_box_replacement(arg) for arg in end_args]
infos = {}
for arg in end_args:
self._expand_info(arg, infos)
label_args, virtuals = virtual_state.make_inputargs_and_virtuals(
end_args, self.optimizer)
for arg in label_args:
self._expand_info(arg, infos)
sb = ShortBoxes()
short_boxes = sb.create_short_boxes(self.optimizer, renamed_inputargs,
label_args + virtuals)
short_inputargs = sb.create_short_inputargs(label_args + virtuals)
for produced_op in short_boxes:
op = produced_op.short_op.res
if not isinstance(op, Const):
self._expand_info(op, infos)
return ExportedState(label_args, end_args, virtual_state, infos,
short_boxes, renamed_inputargs,
short_inputargs, memo)
def import_state(self, targetargs, exported_state):
# the mapping between input args (from old label) and what we need
# to actually emit. Update the info
assert (len(exported_state.next_iteration_args) ==
len(targetargs))
self._check_no_forwarding([targetargs])
exported_state._check_no_forwarding()
for i, target in enumerate(exported_state.next_iteration_args):
source = targetargs[i]
assert source is not target
source.set_forwarded(target)
info = exported_state.exported_infos.get(target, None)
if info is not None:
self.optimizer.setinfo_from_preamble(source, info,
exported_state.exported_infos)
# import the optimizer state, starting from boxes that can be produced
# by short preamble
label_args = exported_state.virtual_state.make_inputargs(
targetargs, self.optimizer)
self.short_preamble_producer = ShortPreambleBuilder(
label_args, exported_state.short_boxes,
exported_state.short_inputargs, exported_state.exported_infos,
self.optimizer)
for produced_op in exported_state.short_boxes:
produced_op.produce_op(self, exported_state.exported_infos)
return label_args
class UnrollOptimizer(Optimization):
"""Unroll the loop into two iterations. The first one will
become the preamble or entry bridge (don't think there is a
distinction anymore)"""
short_preamble_producer = None
def __init__(self, metainterp_sd, jitdriver_sd, optimizations):
self.optimizer = UnrollableOptimizer(metainterp_sd, jitdriver_sd,
optimizations)
self.optimizer.optunroll = self
def get_virtual_state(self, args):
modifier = VirtualStateConstructor(self.optimizer)
return modifier.get_virtual_state(args)
def _check_no_forwarding(self, lsts, check_newops=True):
for lst in lsts:
for op in lst:
assert op.get_forwarded() is None
if check_newops:
assert not self.optimizer._newoperations
def optimize_preamble(self, start_label, end_label, ops, call_pure_results,
memo):
self._check_no_forwarding([[start_label, end_label], ops])
info, newops = self.optimizer.propagate_all_forward(
start_label.getarglist()[:], ops, call_pure_results, True,
flush=False)
exported_state = self.export_state(start_label, end_label.getarglist(),
info.inputargs, memo)
exported_state.quasi_immutable_deps = info.quasi_immutable_deps
# we need to absolutely make sure that we've cleaned up all
# the optimization info
self.optimizer._clean_optimization_info(self.optimizer._newoperations)
return exported_state, self.optimizer._newoperations
def optimize_peeled_loop(self, start_label, end_jump, ops, state,
call_pure_results, inline_short_preamble=True):
self._check_no_forwarding([[start_label, end_jump], ops])
try:
label_args = self.import_state(start_label, state)
except VirtualStatesCantMatch:
raise InvalidLoop("Cannot import state, virtual states don't match")
self.potential_extra_ops = {}
self.optimizer.init_inparg_dict_from(label_args)
try:
info, _ = self.optimizer.propagate_all_forward(
start_label.getarglist()[:], ops, call_pure_results, False,
flush=False)
except SpeculativeError:
raise InvalidLoop("Speculative heap access would be ill-typed")
label_op = ResOperation(rop.LABEL, label_args, start_label.getdescr())
for a in end_jump.getarglist():
self.optimizer.force_box_for_end_of_preamble(
self.optimizer.get_box_replacement(a))
current_vs = self.get_virtual_state(end_jump.getarglist())
# pick the vs we want to jump to
celltoken = start_label.getdescr()
assert isinstance(celltoken, JitCellToken)
target_virtual_state = self.pick_virtual_state(current_vs,
state.virtual_state,
celltoken.target_tokens)
# force the boxes for virtual state to match
try:
args = target_virtual_state.make_inputargs(
[self.get_box_replacement(x) for x in end_jump.getarglist()],
self.optimizer, force_boxes=True)
for arg in args:
self.optimizer.force_box(arg)
except VirtualStatesCantMatch:
raise InvalidLoop("Virtual states did not match "
"after picking the virtual state, when forcing"
" boxes")
extra_same_as = self.short_preamble_producer.extra_same_as[:]
target_token = self.finalize_short_preamble(label_op,
state.virtual_state)
label_op.setdescr(target_token)
if not inline_short_preamble:
self.jump_to_preamble(celltoken, end_jump, info)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
try:
new_virtual_state = self.jump_to_existing_trace(end_jump, label_op)
except InvalidLoop:
# inlining short preamble failed, jump to preamble
self.jump_to_preamble(celltoken, end_jump, info)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
if new_virtual_state is not None:
self.jump_to_preamble(celltoken, end_jump, info)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
self.disable_retracing_if_max_retrace_guards(
self.optimizer._newoperations, target_token)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
def disable_retracing_if_max_retrace_guards(self, ops, target_token):
maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards
count = 0
for op in ops:
if op.is_guard():
count += 1
if count > maxguards:
assert isinstance(target_token, TargetToken)
target_token.targeting_jitcell_token.retraced_count = sys.maxint
def pick_virtual_state(self, my_vs, label_vs, target_tokens):
if target_tokens is None:
return label_vs # for tests
for token in target_tokens:
if token.virtual_state is None:
continue
if token.virtual_state.generalization_of(my_vs, self.optimizer):
return token.virtual_state
return label_vs
def optimize_bridge(self, start_label, operations, call_pure_results,
inline_short_preamble, box_names_memo):
self._check_no_forwarding([start_label.getarglist(),
operations])
info, ops = self.optimizer.propagate_all_forward(
start_label.getarglist()[:], operations[:-1],
call_pure_results, True)
jump_op = operations[-1]
cell_token = jump_op.getdescr()
assert isinstance(cell_token, JitCellToken)
if not inline_short_preamble or len(cell_token.target_tokens) == 1:
return self.jump_to_preamble(cell_token, jump_op, info)
# force all the information that does not go to the short
# preamble at all
self.optimizer.flush()
for a in jump_op.getarglist():
self.optimizer.force_box_for_end_of_preamble(a)
try:
vs = self.jump_to_existing_trace(jump_op, None)
except InvalidLoop:
return self.jump_to_preamble(cell_token, jump_op, info)
if vs is None:
return info, self.optimizer._newoperations[:]
warmrunnerdescr = self.optimizer.metainterp_sd.warmrunnerdesc
limit = warmrunnerdescr.memory_manager.retrace_limit
if cell_token.retraced_count < limit:
cell_token.retraced_count += 1
debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit))
else:
debug_print("Retrace count reached, jumping to preamble")
return self.jump_to_preamble(cell_token, jump_op, info)
exported_state = self.export_state(start_label,
operations[-1].getarglist(),
info.inputargs, box_names_memo)
exported_state.quasi_immutable_deps = self.optimizer.quasi_immutable_deps
self.optimizer._clean_optimization_info(self.optimizer._newoperations)
return exported_state, self.optimizer._newoperations
def finalize_short_preamble(self, label_op, virtual_state):
sb = self.short_preamble_producer
self.optimizer._clean_optimization_info(sb.short_inputargs)
short_preamble = sb.build_short_preamble()
jitcelltoken = label_op.getdescr()
assert isinstance(jitcelltoken, JitCellToken)
if jitcelltoken.target_tokens is None:
jitcelltoken.target_tokens = []
target_token = TargetToken(jitcelltoken,
original_jitcell_token=jitcelltoken)
target_token.original_jitcell_token = jitcelltoken
target_token.virtual_state = virtual_state
target_token.short_preamble = short_preamble
jitcelltoken.target_tokens.append(target_token)
self.short_preamble_producer = ExtendedShortPreambleBuilder(
target_token, sb)
label_op.initarglist(label_op.getarglist() + sb.used_boxes)
return target_token
def jump_to_preamble(self, cell_token, jump_op, info):
assert cell_token.target_tokens[0].virtual_state is None
jump_op = jump_op.copy_and_change(rop.JUMP,
descr=cell_token.target_tokens[0])
self.optimizer.send_extra_operation(jump_op)
return info, self.optimizer._newoperations[:]
def jump_to_existing_trace(self, jump_op, label_op):
jitcelltoken = jump_op.getdescr()
assert isinstance(jitcelltoken, JitCellToken)
virtual_state = self.get_virtual_state(jump_op.getarglist())
args = [self.get_box_replacement(op) for op in jump_op.getarglist()]
for target_token in jitcelltoken.target_tokens:
target_virtual_state = target_token.virtual_state
if target_virtual_state is None:
continue
try:
extra_guards = target_virtual_state.generate_guards(
virtual_state, args, jump_op.getarglist(), self.optimizer)
patchguardop = self.optimizer.patchguardop
for guard in extra_guards.extra_guards:
if isinstance(guard, GuardResOp):
guard.rd_snapshot = patchguardop.rd_snapshot
guard.rd_frame_info_list = patchguardop.rd_frame_info_list
guard.setdescr(compile.ResumeAtPositionDescr())
self.send_extra_operation(guard)
except VirtualStatesCantMatch:
continue
args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
args, self.optimizer)
short_preamble = target_token.short_preamble
try:
extra = self.inline_short_preamble(args + virtuals, args,
short_preamble, self.optimizer.patchguardop,
target_token, label_op)
except KeyError:
# SHOULD NOT OCCUR BUT DOES: WHY?? issue #2185
self.optimizer.metainterp_sd.logger_ops.log_short_preamble([],
short_preamble, {})
raise
self.send_extra_operation(jump_op.copy_and_change(rop.JUMP,
args=args + extra,
descr=target_token))
return None # explicit because the return can be non-None
return virtual_state
def _map_args(self, mapping, arglist):
result = []
for box in arglist:
if not isinstance(box, Const):
box = mapping[box]
result.append(box)
return result
def inline_short_preamble(self, jump_args, args_no_virtuals, short,
patchguardop, target_token, label_op):
short_inputargs = short[0].getarglist()
short_jump_args = short[-1].getarglist()
sb = self.short_preamble_producer
if sb is not None:
assert isinstance(sb, ExtendedShortPreambleBuilder)
if sb.target_token is target_token:
# this means we're inlining the short preamble that's being
# built. Make sure we modify the correct things in-place
self.short_preamble_producer.setup(short_jump_args,
short, label_op.getarglist())
# after this call, THE REST OF THIS FUNCTION WILL MODIFY ALL
# THE LISTS PROVIDED, POTENTIALLY
# We need to make a list of fresh new operations corresponding
# to the short preamble operations. We could temporarily forward
# the short operations to the fresh ones, but there are obscure
# issues: send_extra_operation() below might occasionally invoke
# use_box(), which assumes the short operations are not forwarded.
# So we avoid such temporary forwarding and just use a dict here.
assert len(short_inputargs) == len(jump_args)
mapping = {}
for i in range(len(jump_args)):
mapping[short_inputargs[i]] = jump_args[i]
# a fix-point loop, runs only once in almost all cases
i = 1
while 1:
self._check_no_forwarding([short_inputargs, short], False)
while i < len(short) - 1:
sop = short[i]
arglist = self._map_args(mapping, sop.getarglist())
if sop.is_guard():
op = sop.copy_and_change(sop.getopnum(), arglist,
descr=compile.ResumeAtPositionDescr())
assert isinstance(op, GuardResOp)
op.rd_snapshot = patchguardop.rd_snapshot
op.rd_frame_info_list = patchguardop.rd_frame_info_list
else:
op = sop.copy_and_change(sop.getopnum(), arglist)
mapping[sop] = op
i += 1
self.optimizer.send_extra_operation(op)
# force all of them except the virtuals
for arg in args_no_virtuals + short_jump_args:
self.optimizer.force_box(self.get_box_replacement(arg))
self.optimizer.flush()
# done unless "short" has grown again
if i == len(short) - 1:
break
return [self.get_box_replacement(box)
for box in self._map_args(mapping, short_jump_args)]
def _expand_info(self, arg, infos):
if isinstance(arg, AbstractResOp) and arg.is_same_as():
info = self.optimizer.getinfo(arg.getarg(0))
else:
info = self.optimizer.getinfo(arg)
if arg in infos:
return
if info:
infos[arg] = info
if info.is_virtual():
self._expand_infos_from_virtual(info, infos)
def _expand_infos_from_virtual(self, info, infos):
items = info.all_items()
for item in items:
if item is None:
continue
self._expand_info(item, infos)
def export_state(self, start_label, original_label_args, renamed_inputargs,
memo):
end_args = [self.optimizer.force_box_for_end_of_preamble(a)
for a in original_label_args]
self.optimizer.flush()
virtual_state = self.get_virtual_state(end_args)
end_args = [self.get_box_replacement(arg) for arg in end_args]
infos = {}
for arg in end_args:
self._expand_info(arg, infos)
label_args, virtuals = virtual_state.make_inputargs_and_virtuals(
end_args, self.optimizer)
for arg in label_args:
self._expand_info(arg, infos)
sb = ShortBoxes()
short_boxes = sb.create_short_boxes(self.optimizer, renamed_inputargs,
label_args + virtuals)
short_inputargs = sb.create_short_inputargs(label_args + virtuals)
for produced_op in short_boxes:
op = produced_op.short_op.res
if not isinstance(op, Const):
self._expand_info(op, infos)
self.optimizer._clean_optimization_info(end_args)
self.optimizer._clean_optimization_info(start_label.getarglist())
return ExportedState(label_args, end_args, virtual_state, infos,
short_boxes, renamed_inputargs,
short_inputargs, memo)
def import_state(self, targetop, exported_state):
# the mapping between input args (from old label) and what we need
# to actually emit. Update the info
assert (len(exported_state.next_iteration_args) ==
len(targetop.getarglist()))
for i, target in enumerate(exported_state.next_iteration_args):
source = targetop.getarg(i)
assert source is not target
source.set_forwarded(target)
info = exported_state.exported_infos.get(target, None)
if info is not None:
self.optimizer.setinfo_from_preamble(source, info,
exported_state.exported_infos)
# import the optimizer state, starting from boxes that can be produced
# by short preamble
label_args = exported_state.virtual_state.make_inputargs(
targetop.getarglist(), self.optimizer)
self.short_preamble_producer = ShortPreambleBuilder(
label_args, exported_state.short_boxes,
exported_state.short_inputargs, exported_state.exported_infos,
self.optimizer)
for produced_op in exported_state.short_boxes:
produced_op.produce_op(self, exported_state.exported_infos)
return label_args
class OptUnroll(Optimization):
"""Unroll the loop into two iterations. The first one will
become the preamble or entry bridge (don't think there is a
distinction anymore)"""
short_preamble_producer = None
def get_virtual_state(self, args):
modifier = VirtualStateConstructor(self.optimizer)
return modifier.get_virtual_state(args)
def _check_no_forwarding(self, lsts, check_newops=True):
for lst in lsts:
for op in lst:
assert op.get_forwarded() is None
if check_newops:
assert not self.optimizer._newoperations
def disable_retracing_if_max_retrace_guards(self, ops, target_token):
maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards
count = 0
for op in ops:
if op.is_guard():
count += 1
if count > maxguards:
assert isinstance(target_token, TargetToken)
target_token.targeting_jitcell_token.retraced_count = sys.maxint
def pick_virtual_state(self, my_vs, label_vs, target_tokens):
if target_tokens is None:
return label_vs # for tests
for token in target_tokens:
if token.virtual_state is None:
continue
if token.virtual_state.generalization_of(my_vs, self.optimizer):
return token.virtual_state
return label_vs
def finalize_short_preamble(self, label_op, virtual_state):
sb = self.short_preamble_producer
self.optimizer._clean_optimization_info(sb.short_inputargs)
short_preamble = sb.build_short_preamble()
jitcelltoken = label_op.getdescr()
assert isinstance(jitcelltoken, JitCellToken)
if jitcelltoken.target_tokens is None:
jitcelltoken.target_tokens = []
target_token = TargetToken(jitcelltoken,
original_jitcell_token=jitcelltoken)
target_token.original_jitcell_token = jitcelltoken
target_token.virtual_state = virtual_state
target_token.short_preamble = short_preamble
jitcelltoken.target_tokens.append(target_token)
self.short_preamble_producer = ExtendedShortPreambleBuilder(
target_token, sb)
label_op.initarglist(label_op.getarglist() + sb.used_boxes)
return target_token
def jump_to_existing_trace(self,
jump_op,
label_op,
runtime_boxes,
force_boxes=False):
# there is a big conceptual problem here: it's not safe at all to catch
# InvalidLoop in the callers of _jump_to_existing_trace and then
# continue trying to jump to some other label, because inlining the
# short preamble could have worked partly, leaving some unwanted new
# ops at the end of the trace. Here's at least a stopgap to stop
# terrible things from happening: we *must not* move any of those bogus
# guards earlier into the trace. see
# test_unroll_shortpreamble_mutates_bug in test_loop, and issue #3598
# leaving the bogus operations at the end of the trace is not great,
# but should be safe: at worst, they just always do a bit of stuff and
# then fail
with self.optimizer.cant_replace_guards():
return self._jump_to_existing_trace(jump_op, label_op,
runtime_boxes, force_boxes)
def _jump_to_existing_trace(self,
jump_op,
label_op,
runtime_boxes,
force_boxes=False):
jitcelltoken = jump_op.getdescr()
assert isinstance(jitcelltoken, JitCellToken)
virtual_state = self.get_virtual_state(jump_op.getarglist())
args = [get_box_replacement(op) for op in jump_op.getarglist()]
for target_token in jitcelltoken.target_tokens:
target_virtual_state = target_token.virtual_state
if target_virtual_state is None:
continue
try:
extra_guards = target_virtual_state.generate_guards(
virtual_state,
args,
runtime_boxes,
self.optimizer,
force_boxes=force_boxes)
patchguardop = self.optimizer.patchguardop
for guard in extra_guards.extra_guards:
if isinstance(guard, GuardResOp):
guard.rd_resume_position = patchguardop.rd_resume_position
guard.setdescr(compile.ResumeAtPositionDescr())
self.optimizer.send_extra_operation(guard)
except VirtualStatesCantMatch:
continue
# When force_boxes == True, creating the virtual args can fail when
# components of the virtual state alias. If this occurs, we must
# recompute the virtual state as boxes will have been forced.
try:
args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
args, self.optimizer, force_boxes=force_boxes)
except VirtualStatesCantMatch:
assert force_boxes
virtual_state = self.get_virtual_state(args)
continue
short_preamble = target_token.short_preamble
extra = self.inline_short_preamble(args + virtuals, args,
short_preamble,
self.optimizer.patchguardop,
target_token, label_op)
self.optimizer.send_extra_operation(
jump_op.copy_and_change(rop.JUMP,
args=args + extra,
descr=target_token))
return None # explicit because the return can be non-None
return virtual_state
def _map_args(self, mapping, arglist):
result = []
for box in arglist:
if not isinstance(box, Const):
box = mapping[box]
result.append(box)
return result
def inline_short_preamble(self, jump_args, args_no_virtuals, short,
patchguardop, target_token, label_op):
short_inputargs = short[0].getarglist()
short_jump_args = short[-1].getarglist()
sb = self.short_preamble_producer
if sb is not None:
assert isinstance(sb, ExtendedShortPreambleBuilder)
if sb.target_token is target_token:
# this means we're inlining the short preamble that's being
# built. Make sure we modify the correct things in-place
self.short_preamble_producer.setup(short_jump_args, short,
label_op.getarglist())
# after this call, THE REST OF THIS FUNCTION WILL MODIFY ALL
# THE LISTS PROVIDED, POTENTIALLY
# We need to make a list of fresh new operations corresponding
# to the short preamble operations. We could temporarily forward
# the short operations to the fresh ones, but there are obscure
# issues: send_extra_operation() below might occasionally invoke
# use_box(), which assumes the short operations are not forwarded.
# So we avoid such temporary forwarding and just use a dict here.
assert len(short_inputargs) == len(jump_args)
mapping = {}
for i in range(len(jump_args)):
mapping[short_inputargs[i]] = jump_args[i]
# a fix-point loop, runs only once in almost all cases
i = 1
while 1:
self._check_no_forwarding([short_inputargs, short], False)
while i < len(short) - 1:
sop = short[i]
arglist = self._map_args(mapping, sop.getarglist())
if sop.is_guard():
op = sop.copy_and_change(
sop.getopnum(),
arglist,
descr=compile.ResumeAtPositionDescr())
assert isinstance(op, GuardResOp)
op.rd_resume_position = patchguardop.rd_resume_position
else:
op = sop.copy_and_change(sop.getopnum(), arglist)
mapping[sop] = op
i += 1
self.optimizer.send_extra_operation(op)
# force all of them except the virtuals
for arg in (args_no_virtuals +
self._map_args(mapping, short_jump_args)):
self.optimizer.force_box(get_box_replacement(arg))
self.optimizer.flush()
# done unless "short" has grown again
if i == len(short) - 1:
break
return [
get_box_replacement(box)
for box in self._map_args(mapping, short_jump_args)
]
def _expand_info(self, arg, infos):
arg1 = self.optimizer.as_operation(arg)
if arg1 is not None and rop.is_same_as(arg1.opnum):
info = self.optimizer.getinfo(arg1.getarg(0))
else:
info = self.optimizer.getinfo(arg)
if arg in infos:
return
if info:
infos[arg] = info
if info.is_virtual():
self._expand_infos_from_virtual(info, infos)
def _expand_infos_from_virtual(self, info, infos):
items = info.all_items()
for item in items:
if item is None:
continue
self._expand_info(item, infos)
def export_state(self, original_label_args, renamed_inputargs,
runtime_boxes, memo):
end_args = [
self.optimizer.force_box_for_end_of_preamble(a)
for a in original_label_args
]
self.optimizer.flush()
virtual_state = self.get_virtual_state(end_args)
end_args = [get_box_replacement(arg) for arg in end_args]
infos = {}
for arg in end_args:
self._expand_info(arg, infos)
label_args, virtuals = virtual_state.make_inputargs_and_virtuals(
end_args, self.optimizer)
for arg in label_args:
self._expand_info(arg, infos)
sb = ShortBoxes()
short_boxes = sb.create_short_boxes(self.optimizer, renamed_inputargs,
label_args + virtuals)
short_inputargs = sb.create_short_inputargs(label_args + virtuals)
for produced_op in short_boxes:
op = produced_op.short_op.res
if not isinstance(op, Const):
self._expand_info(op, infos)
self.optimizer._clean_optimization_info(end_args)
return ExportedState(label_args, end_args, virtual_state, infos,
short_boxes, renamed_inputargs, short_inputargs,
runtime_boxes, memo)
def import_state(self, targetargs, exported_state):
# the mapping between input args (from old label) and what we need
# to actually emit. Update the info
assert len(exported_state.next_iteration_args) == len(targetargs)
for i, target in enumerate(exported_state.next_iteration_args):
source = targetargs[i]
assert source is not target
source.set_forwarded(target)
info = exported_state.exported_infos.get(target, None)
if info is not None:
self.optimizer.setinfo_from_preamble(
source, info, exported_state.exported_infos)
# import the optimizer state, starting from boxes that can be produced
# by short preamble
label_args = exported_state.virtual_state.make_inputargs(
targetargs, self.optimizer)
self.short_preamble_producer = ShortPreambleBuilder(
label_args, exported_state.short_boxes,
exported_state.short_inputargs, exported_state.exported_infos,
self.optimizer)
for produced_op in exported_state.short_boxes:
produced_op.produce_op(self, exported_state.exported_infos)
return label_args