def parse_loop(self, ops, add_label=True):
loop = self.parse(ops)
loop.operations = filter(
lambda op: op.getopnum() != rop.DEBUG_MERGE_POINT, loop.operations)
token = JitCellToken()
if add_label:
label = ResOperation(rop.LABEL,
loop.inputargs,
descr=TargetToken(token))
else:
label = loop.operations[0]
label.setdescr(TargetToken(token))
jump = loop.operations[-1]
loop = VectorLoop(label, loop.operations[0:-1], jump)
loop.jump.setdescr(token)
class Optimizer(object):
metainterp_sd = FakeMetaInterpStaticData(self.cpu)
jitdriver_sd = FakeJitDriverStaticData()
opt = Optimizer()
opt.jitdriver_sd.vec = True
for op in loop.operations:
if op.is_guard() and not op.getdescr():
descr = invent_fail_descr_for_op(op.getopnum(), opt)
op.setdescr(descr)
return loop
def exc_handling(guard_op):
# operations need to start with correct GUARD_EXCEPTION
if guard_op._exc_box is None:
op = ResOperation(rop.GUARD_NO_EXCEPTION, [])
else:
op = ResOperation(rop.GUARD_EXCEPTION, [guard_op._exc_box])
op.setdescr(self.builder.getfaildescr())
op.setfailargs([])
return op
def exc_handling(guard_op):
# operations need to start with correct GUARD_EXCEPTION
if guard_op._exc_box is None:
op = ResOperation(rop.GUARD_NO_EXCEPTION, [])
else:
op = ResOperation(rop.GUARD_EXCEPTION, [guard_op._exc_box])
op.setdescr(self.builder.getfaildescr())
op.setfailargs([])
return op
def produce_into(self, builder, r):
fail_subset = builder.subset_of_intvars(r)
original_intvars = builder.intvars[:]
builder.fakemetainterp.ovf_flag = False
super(AbstractOvfOperation, self).produce_into(builder, r)
if builder.fakemetainterp.ovf_flag: # overflow detected
op = ResOperation(rop.GUARD_OVERFLOW, [])
# the overflowed result should not be used any more, but can
# be used on the failure path: recompute fail_subset including
# the result, and then remove it from builder.intvars.
fail_subset = builder.subset_of_intvars(r)
builder.intvars[:] = original_intvars
else:
op = ResOperation(rop.GUARD_NO_OVERFLOW, [])
op.setdescr(builder.getfaildescr())
op.setfailargs(fail_subset)
builder.loop.operations.append(op)
def produce_into(self, builder, r):
fail_subset = builder.subset_of_intvars(r)
original_intvars = builder.intvars[:]
builder.fakemetainterp.ovf_flag = False
super(AbstractOvfOperation, self).produce_into(builder, r)
if builder.fakemetainterp.ovf_flag: # overflow detected
op = ResOperation(rop.GUARD_OVERFLOW, [])
# the overflowed result should not be used any more, but can
# be used on the failure path: recompute fail_subset including
# the result, and then remove it from builder.intvars.
fail_subset = builder.subset_of_intvars(r)
builder.intvars[:] = original_intvars
else:
op = ResOperation(rop.GUARD_NO_OVERFLOW, [])
op.setdescr(builder.getfaildescr())
op.setfailargs(fail_subset)
builder.loop.operations.append(op)
def next(self):
opnum = self._next()
argnum = oparity[opnum]
if argnum == -1:
argnum = self._next()
if not (0 <= oparity[opnum] <= 3):
args = []
for i in range(argnum):
args.append(self._untag(self._next()))
res = ResOperation(opnum, args)
else:
cls = opclasses[opnum]
res = cls()
argnum = oparity[opnum]
if argnum == 0:
pass
elif argnum == 1:
res.setarg(0, self._untag(self._next()))
elif argnum == 2:
res.setarg(0, self._untag(self._next()))
res.setarg(1, self._untag(self._next()))
else:
assert argnum == 3
res.setarg(0, self._untag(self._next()))
res.setarg(1, self._untag(self._next()))
res.setarg(2, self._untag(self._next()))
descr_index = -1
if opwithdescr[opnum]:
descr_index = self._next()
if descr_index == 0 or rop.is_guard(opnum):
descr = None
else:
if descr_index < self.all_descr_len + 1:
descr = self.metainterp_sd.all_descrs[descr_index - 1]
else:
descr = self.trace._descrs[descr_index -
self.all_descr_len - 1]
res.setdescr(descr)
if rop.is_guard(opnum): # all guards have descrs
assert isinstance(res, GuardResOp)
res.rd_resume_position = descr_index
if res.type != 'v':
self._cache[self._index] = res
self._index += 1
self._count += 1
return res
def parse_loop(self, ops, add_label=True):
loop = self.parse(ops, postprocess=self.postprocess)
loop.operations = filter(lambda op: op.getopnum() != rop.DEBUG_MERGE_POINT, loop.operations)
token = JitCellToken()
if add_label:
label = ResOperation(rop.LABEL, loop.inputargs, descr=TargetToken(token))
else:
label = loop.operations[0]
label.setdescr(TargetToken(token))
jump = loop.operations[-1]
loop = VectorLoop(label, loop.operations[0:-1], jump)
loop.jump.setdescr(token)
class Optimizer(object):
metainterp_sd = FakeMetaInterpStaticData(self.cpu)
jitdriver_sd = FakeJitDriverStaticData()
opt = Optimizer()
opt.jitdriver_sd.vec = True
for op in loop.operations:
if op.is_guard() and not op.getdescr():
descr = invent_fail_descr_for_op(op.getopnum(), opt)
op.setdescr(descr)
return loop
def compile_simple_loop(metainterp, greenkey, start, inputargs, ops, jumpargs,
enable_opts):
from rpython.jit.metainterp.optimizeopt import optimize_trace
jitdriver_sd = metainterp.jitdriver_sd
metainterp_sd = metainterp.staticdata
jitcell_token = make_jitcell_token(jitdriver_sd)
label = ResOperation(rop.LABEL, inputargs[:], descr=jitcell_token)
jump_op = ResOperation(rop.JUMP, jumpargs[:], descr=jitcell_token)
call_pure_results = metainterp.call_pure_results
data = SimpleCompileData(label, ops + [jump_op],
call_pure_results=call_pure_results,
enable_opts=enable_opts)
try:
loop_info, ops = optimize_trace(metainterp_sd, jitdriver_sd,
data, metainterp.box_names_memo)
except InvalidLoop:
return None
loop = create_empty_loop(metainterp)
loop.original_jitcell_token = jitcell_token
loop.inputargs = loop_info.inputargs
if loop_info.quasi_immutable_deps:
loop.quasi_immutable_deps = loop_info.quasi_immutable_deps
jump_op = ops[-1]
target_token = TargetToken(jitcell_token)
target_token.original_jitcell_token = jitcell_token
label = ResOperation(rop.LABEL, loop_info.inputargs[:], descr=target_token)
jump_op.setdescr(target_token)
loop.operations = [label] + ops
if not we_are_translated():
loop.check_consistency()
jitcell_token.target_tokens = [target_token]
send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop",
inputargs, metainterp.box_names_memo)
record_loop_or_bridge(metainterp_sd, loop)
return target_token
def propagate_all_forward(self, starting_state, export_state=True):
self.optimizer.exporting_state = export_state
loop = self.optimizer.loop
self.optimizer.clear_newoperations()
start_label = loop.operations[0]
if start_label.getopnum() == rop.LABEL:
loop.operations = loop.operations[1:]
# We need to emit the label op before import_state() as emitting it
# will clear heap caches
self.optimizer.send_extra_operation(start_label)
else:
start_label = None
patchguardop = None
if len(loop.operations) > 1:
patchguardop = loop.operations[-2]
if patchguardop.getopnum() != rop.GUARD_FUTURE_CONDITION:
patchguardop = None
jumpop = loop.operations[-1]
if jumpop.getopnum() == rop.JUMP or jumpop.getopnum() == rop.LABEL:
loop.operations = loop.operations[:-1]
else:
jumpop = None
self.import_state(start_label, starting_state)
self.optimizer.propagate_all_forward(clear=False)
if not jumpop:
return
cell_token = jumpop.getdescr()
assert isinstance(cell_token, JitCellToken)
stop_label = ResOperation(rop.LABEL, jumpop.getarglist(), None, TargetToken(cell_token))
if jumpop.getopnum() == rop.JUMP:
if self.jump_to_already_compiled_trace(jumpop, patchguardop):
# Found a compiled trace to jump to
if self.short:
# Construct our short preamble
assert start_label
self.close_bridge(start_label)
return
if start_label and self.jump_to_start_label(start_label, stop_label):
# Initial label matches, jump to it
jumpop = ResOperation(rop.JUMP, stop_label.getarglist(), None,
descr=start_label.getdescr())
if self.short:
# Construct our short preamble
self.close_loop(start_label, jumpop, patchguardop)
else:
self.optimizer.send_extra_operation(jumpop)
return
if cell_token.target_tokens:
limit = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.retrace_limit
if cell_token.retraced_count < limit:
cell_token.retraced_count += 1
debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit))
else:
debug_print("Retrace count reached, jumping to preamble")
assert cell_token.target_tokens[0].virtual_state is None
jumpop = jumpop.clone()
jumpop.setdescr(cell_token.target_tokens[0])
self.optimizer.send_extra_operation(jumpop)
return
# Found nothing to jump to, emit a label instead
if self.short:
# Construct our short preamble
assert start_label
self.close_bridge(start_label)
self.optimizer.flush()
if export_state:
KillHugeIntBounds(self.optimizer).apply()
loop.operations = self.optimizer.get_newoperations()
if export_state:
jd_sd = self.optimizer.jitdriver_sd
try:
threshold = jd_sd.warmstate.disable_unrolling_threshold
except AttributeError: # tests only
threshold = sys.maxint
if len(loop.operations) > threshold:
if loop.operations[0].getopnum() == rop.LABEL:
# abandoning unrolling, too long
new_descr = stop_label.getdescr()
if loop.operations[0].getopnum() == rop.LABEL:
new_descr = loop.operations[0].getdescr()
stop_label = stop_label.copy_and_change(rop.JUMP,
descr=new_descr)
self.optimizer.send_extra_operation(stop_label)
loop.operations = self.optimizer.get_newoperations()
return None
final_state = self.export_state(stop_label)
else:
final_state = None
loop.operations.append(stop_label)
return final_state
def optimize_peeled_loop(self, trace, celltoken, state, runtime_boxes,
call_pure_results, inline_short_preamble=True):
trace = trace.get_iter()
try:
label_args = self.import_state(trace.inputargs, state)
except VirtualStatesCantMatch:
raise InvalidLoop("Cannot import state, virtual states don't match")
self.potential_extra_ops = {}
self.optimizer.init_inparg_dict_from(label_args)
try:
info, _ = self.optimizer.propagate_all_forward(
trace, call_pure_results, flush=False)
except SpeculativeError:
raise InvalidLoop("Speculative heap access would be ill-typed")
end_jump = info.jump_op
label_op = ResOperation(rop.LABEL, label_args,
descr=celltoken)
for a in end_jump.getarglist():
self.optimizer.force_box_for_end_of_preamble(
self.optimizer.get_box_replacement(a))
current_vs = self.get_virtual_state(end_jump.getarglist())
# pick the vs we want to jump to
assert isinstance(celltoken, JitCellToken)
target_virtual_state = self.pick_virtual_state(current_vs,
state.virtual_state,
celltoken.target_tokens)
# force the boxes for virtual state to match
try:
args = target_virtual_state.make_inputargs(
[self.get_box_replacement(x) for x in end_jump.getarglist()],
self.optimizer, force_boxes=True)
for arg in args:
if arg is not None:
self.optimizer.force_box(arg)
except VirtualStatesCantMatch:
raise InvalidLoop("Virtual states did not match "
"after picking the virtual state, when forcing"
" boxes")
extra_same_as = self.short_preamble_producer.extra_same_as[:]
target_token = self.finalize_short_preamble(label_op,
state.virtual_state)
label_op.setdescr(target_token)
if not inline_short_preamble:
self.jump_to_preamble(celltoken, end_jump, info)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
try:
new_virtual_state = self.jump_to_existing_trace(
end_jump, label_op, runtime_boxes, force_boxes=False)
except InvalidLoop:
# inlining short preamble failed, jump to preamble
self.jump_to_preamble(celltoken, end_jump, info)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
if new_virtual_state is not None:
# Attempt to force virtual boxes in order to avoid jumping
# to the preamble.
try:
new_virtual_state = self.jump_to_existing_trace(
end_jump, label_op, runtime_boxes, force_boxes=True)
except InvalidLoop:
pass
if new_virtual_state is not None:
self.jump_to_preamble(celltoken, end_jump, info)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
self.disable_retracing_if_max_retrace_guards(
self.optimizer._newoperations, target_token)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
def optimize_peeled_loop(self, start_label, end_jump, ops, state,
call_pure_results, inline_short_preamble=True):
self._check_no_forwarding([[start_label, end_jump], ops])
try:
label_args = self.import_state(start_label, state)
except VirtualStatesCantMatch:
raise InvalidLoop("Cannot import state, virtual states don't match")
self.potential_extra_ops = {}
self.optimizer.init_inparg_dict_from(label_args)
try:
info, _ = self.optimizer.propagate_all_forward(
start_label.getarglist()[:], ops, call_pure_results, False,
flush=False)
except SpeculativeError:
raise InvalidLoop("Speculative heap access would be ill-typed")
label_op = ResOperation(rop.LABEL, label_args, start_label.getdescr())
for a in end_jump.getarglist():
self.optimizer.force_box_for_end_of_preamble(
self.optimizer.get_box_replacement(a))
current_vs = self.get_virtual_state(end_jump.getarglist())
# pick the vs we want to jump to
celltoken = start_label.getdescr()
assert isinstance(celltoken, JitCellToken)
target_virtual_state = self.pick_virtual_state(current_vs,
state.virtual_state,
celltoken.target_tokens)
# force the boxes for virtual state to match
try:
args = target_virtual_state.make_inputargs(
[self.get_box_replacement(x) for x in end_jump.getarglist()],
self.optimizer, force_boxes=True)
for arg in args:
self.optimizer.force_box(arg)
except VirtualStatesCantMatch:
raise InvalidLoop("Virtual states did not match "
"after picking the virtual state, when forcing"
" boxes")
extra_same_as = self.short_preamble_producer.extra_same_as[:]
target_token = self.finalize_short_preamble(label_op,
state.virtual_state)
label_op.setdescr(target_token)
if not inline_short_preamble:
self.jump_to_preamble(celltoken, end_jump, info)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
try:
new_virtual_state = self.jump_to_existing_trace(end_jump, label_op)
except InvalidLoop:
# inlining short preamble failed, jump to preamble
self.jump_to_preamble(celltoken, end_jump, info)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
if new_virtual_state is not None:
self.jump_to_preamble(celltoken, end_jump, info)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
self.disable_retracing_if_max_retrace_guards(
self.optimizer._newoperations, target_token)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
def propagate_all_forward(self, starting_state, export_state=True):
self.optimizer.exporting_state = export_state
loop = self.optimizer.loop
self.optimizer.clear_newoperations()
start_label = loop.operations[0]
if start_label.getopnum() == rop.LABEL:
loop.operations = loop.operations[1:]
# We need to emit the label op before import_state() as emitting it
# will clear heap caches
self.optimizer.send_extra_operation(start_label)
else:
start_label = None
patchguardop = None
if len(loop.operations) > 1:
patchguardop = loop.operations[-2]
if patchguardop.getopnum() != rop.GUARD_FUTURE_CONDITION:
patchguardop = None
jumpop = loop.operations[-1]
if jumpop.getopnum() == rop.JUMP or jumpop.getopnum() == rop.LABEL:
loop.operations = loop.operations[:-1]
else:
jumpop = None
self.import_state(start_label, starting_state)
self.optimizer.propagate_all_forward(clear=False)
if not jumpop:
return
cell_token = jumpop.getdescr()
assert isinstance(cell_token, JitCellToken)
stop_label = ResOperation(rop.LABEL, jumpop.getarglist(), None, TargetToken(cell_token))
if jumpop.getopnum() == rop.JUMP:
if self.jump_to_already_compiled_trace(jumpop, patchguardop):
# Found a compiled trace to jump to
if self.short:
# Construct our short preamble
assert start_label
self.close_bridge(start_label)
return
if start_label and self.jump_to_start_label(start_label, stop_label):
# Initial label matches, jump to it
jumpop = ResOperation(rop.JUMP, stop_label.getarglist(), None,
descr=start_label.getdescr())
if self.short:
# Construct our short preamble
self.close_loop(start_label, jumpop, patchguardop)
else:
self.optimizer.send_extra_operation(jumpop)
return
if cell_token.target_tokens:
limit = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.retrace_limit
if cell_token.retraced_count < limit:
cell_token.retraced_count += 1
debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit))
else:
debug_print("Retrace count reached, jumping to preamble")
assert cell_token.target_tokens[0].virtual_state is None
jumpop = jumpop.clone()
jumpop.setdescr(cell_token.target_tokens[0])
self.optimizer.send_extra_operation(jumpop)
return
# Found nothing to jump to, emit a label instead
if self.short:
# Construct our short preamble
assert start_label
self.close_bridge(start_label)
self.optimizer.flush()
if export_state:
KillHugeIntBounds(self.optimizer).apply()
loop.operations = self.optimizer.get_newoperations()
if export_state:
final_state = self.export_state(stop_label)
else:
final_state = None
loop.operations.append(stop_label)
return final_state