def compile_simple_loop(metainterp, greenkey, trace, runtime_args, enable_opts,
cut_at):
from rpython.jit.metainterp.optimizeopt import optimize_trace
jitdriver_sd = metainterp.jitdriver_sd
metainterp_sd = metainterp.staticdata
jitcell_token = make_jitcell_token(jitdriver_sd)
call_pure_results = metainterp.call_pure_results
data = SimpleCompileData(trace, call_pure_results=call_pure_results,
enable_opts=enable_opts)
try:
loop_info, ops = optimize_trace(metainterp_sd, jitdriver_sd,
data, metainterp.box_names_memo)
except InvalidLoop:
trace.cut_at(cut_at)
return None
loop = create_empty_loop(metainterp)
loop.original_jitcell_token = jitcell_token
loop.inputargs = loop_info.inputargs
if loop_info.quasi_immutable_deps:
loop.quasi_immutable_deps = loop_info.quasi_immutable_deps
jump_op = ops[-1]
target_token = TargetToken(jitcell_token)
target_token.original_jitcell_token = jitcell_token
label = ResOperation(rop.LABEL, loop_info.inputargs[:], descr=target_token)
jump_op.setdescr(target_token)
loop.operations = [label] + ops
if not we_are_translated():
loop.check_consistency()
jitcell_token.target_tokens = [target_token]
send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop",
runtime_args, metainterp.box_names_memo)
record_loop_or_bridge(metainterp_sd, loop)
return target_token
def compile_trace(metainterp, resumekey):
"""Try to compile a new bridge leading from the beginning of the history
to some existing place.
"""
from rpython.jit.metainterp.optimizeopt import optimize_trace
# The history contains new operations to attach as the code for the
# failure of 'resumekey.guard_op'.
#
# Attempt to use optimize_bridge(). This may return None in case
# it does not work -- i.e. none of the existing old_loop_tokens match.
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
if isinstance(resumekey, ResumeAtPositionDescr):
inline_short_preamble = False
else:
inline_short_preamble = True
inputargs = metainterp.history.inputargs[:]
operations = metainterp.history.operations
label = ResOperation(rop.LABEL, inputargs)
jitdriver_sd = metainterp.jitdriver_sd
enable_opts = jitdriver_sd.warmstate.enable_opts
call_pure_results = metainterp.call_pure_results
if operations[-1].getopnum() == rop.JUMP:
data = BridgeCompileData(label, operations[:],
call_pure_results=call_pure_results,
enable_opts=enable_opts,
inline_short_preamble=inline_short_preamble)
else:
data = SimpleCompileData(label, operations[:],
call_pure_results=call_pure_results,
enable_opts=enable_opts)
try:
info, newops = optimize_trace(metainterp_sd, jitdriver_sd,
data, metainterp.box_names_memo)
except InvalidLoop:
#pdb.post_mortem(sys.exc_info()[2])
debug_print("compile_new_bridge: got an InvalidLoop")
# XXX I am fairly convinced that optimize_bridge cannot actually raise
# InvalidLoop
debug_print('InvalidLoop in compile_new_bridge')
return None
new_trace = create_empty_loop(metainterp)
new_trace.operations = newops
if info.quasi_immutable_deps:
new_trace.quasi_immutable_deps = info.quasi_immutable_deps
if info.final():
new_trace.inputargs = info.inputargs
target_token = new_trace.operations[-1].getdescr()
resumekey.compile_and_attach(metainterp, new_trace, inputargs)
record_loop_or_bridge(metainterp_sd, new_trace)
return target_token
new_trace.inputargs = info.renamed_inputargs
metainterp.retrace_needed(new_trace, info)
return None
def _do_optimize_loop(self,
loop,
call_pure_results,
start_state=None,
export_state=False):
from rpython.jit.metainterp.optimizeopt import optimize_trace
from rpython.jit.metainterp.optimizeopt.util import args_dict
self.loop = loop
loop.call_pure_results = args_dict()
if call_pure_results is not None:
for k, v in call_pure_results.items():
loop.call_pure_results[list(k)] = v
metainterp_sd = FakeMetaInterpStaticData(self.cpu)
if hasattr(self, 'vrefinfo'):
metainterp_sd.virtualref_info = self.vrefinfo
if hasattr(self, 'callinfocollection'):
metainterp_sd.callinfocollection = self.callinfocollection
#
return optimize_trace(metainterp_sd,
None,
loop,
self.enable_opts,
start_state=start_state,
export_state=export_state)
def _do_optimize_loop(self, loop, call_pure_results):
from rpython.jit.metainterp.optimizeopt import optimize_trace
from rpython.jit.metainterp.optimizeopt.util import args_dict
self.loop = loop
loop.call_pure_results = args_dict()
if call_pure_results is not None:
for k, v in call_pure_results.items():
loop.call_pure_results[list(k)] = v
metainterp_sd = FakeMetaInterpStaticData(self.cpu)
if hasattr(self, 'vrefinfo'):
metainterp_sd.virtualref_info = self.vrefinfo
if hasattr(self, 'callinfocollection'):
metainterp_sd.callinfocollection = self.callinfocollection
#
optimize_trace(metainterp_sd, loop, self.enable_opts)
def compile_trace(metainterp, resumekey, runtime_boxes):
"""Try to compile a new bridge leading from the beginning of the history
to some existging place.
"""
from rpython.jit.metainterp.optimizeopt import optimize_trace
# The history contains new operations to attach as the code for the
# failure of 'resumekey.guard_op'.
#
# Attempt to use optimize_bridge(). This may return None in case
# it does not work -- i.e. none of the existing old_loop_tokens match.
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
if isinstance(resumekey, ResumeAtPositionDescr):
inline_short_preamble = False
else:
inline_short_preamble = True
inputargs = metainterp.history.inputargs[:]
trace = metainterp.history.trace
jitdriver_sd = metainterp.jitdriver_sd
enable_opts = jitdriver_sd.warmstate.enable_opts
call_pure_results = metainterp.call_pure_results
if metainterp.history.ends_with_jump:
data = BridgeCompileData(trace, runtime_boxes,
call_pure_results=call_pure_results,
enable_opts=enable_opts,
inline_short_preamble=inline_short_preamble)
else:
data = SimpleCompileData(trace,
call_pure_results=call_pure_results,
enable_opts=enable_opts)
try:
info, newops = optimize_trace(metainterp_sd, jitdriver_sd,
data, metainterp.box_names_memo)
except InvalidLoop:
#pdb.post_mortem(sys.exc_info()[2])
debug_print("compile_new_bridge: got an InvalidLoop")
# XXX I am fairly convinced that optimize_bridge cannot actually raise
# InvalidLoop
debug_print('InvalidLoop in compile_new_bridge')
return None
new_trace = create_empty_loop(metainterp)
new_trace.operations = newops
if info.quasi_immutable_deps:
new_trace.quasi_immutable_deps = info.quasi_immutable_deps
if info.final():
new_trace.inputargs = info.inputargs
target_token = new_trace.operations[-1].getdescr()
resumekey.compile_and_attach(metainterp, new_trace, inputargs)
record_loop_or_bridge(metainterp_sd, new_trace)
return target_token
new_trace.inputargs = info.renamed_inputargs
metainterp.retrace_needed(new_trace, info)
return None
def _do_optimize_bridge(self, bridge, call_pure_results):
from rpython.jit.metainterp.optimizeopt import optimize_trace
from rpython.jit.metainterp.optimizeopt.util import args_dict
self.bridge = bridge
bridge.call_pure_results = args_dict()
if call_pure_results is not None:
for k, v in call_pure_results.items():
bridge.call_pure_results[list(k)] = v
metainterp_sd = FakeMetaInterpStaticData(self.cpu)
if hasattr(self, "vrefinfo"):
metainterp_sd.virtualref_info = self.vrefinfo
if hasattr(self, "callinfocollection"):
metainterp_sd.callinfocollection = self.callinfocollection
#
bridge.resume_at_jump_descr = FakeDescrWithSnapshot()
optimize_trace(metainterp_sd, bridge, self.enable_opts)
def _do_optimize_bridge(self, bridge, call_pure_results):
from rpython.jit.metainterp.optimizeopt import optimize_trace
from rpython.jit.metainterp.optimizeopt.util import args_dict
self.bridge = bridge
bridge.call_pure_results = args_dict()
if call_pure_results is not None:
for k, v in call_pure_results.items():
bridge.call_pure_results[list(k)] = v
metainterp_sd = FakeMetaInterpStaticData(self.cpu)
if hasattr(self, 'vrefinfo'):
metainterp_sd.virtualref_info = self.vrefinfo
if hasattr(self, 'callinfocollection'):
metainterp_sd.callinfocollection = self.callinfocollection
#
bridge.resume_at_jump_descr = FakeDescrWithSnapshot()
optimize_trace(metainterp_sd, bridge, self.enable_opts)
def compile_trace(metainterp, resumekey, resume_at_jump_descr=None):
"""Try to compile a new bridge leading from the beginning of the history
to some existing place.
"""
from rpython.jit.metainterp.optimizeopt import optimize_trace
# The history contains new operations to attach as the code for the
# failure of 'resumekey.guard_op'.
#
# Attempt to use optimize_bridge(). This may return None in case
# it does not work -- i.e. none of the existing old_loop_tokens match.
new_trace = create_empty_loop(metainterp)
new_trace.inputargs = metainterp.history.inputargs[:]
# clone ops, as optimize_bridge can mutate the ops
new_trace.operations = [op.clone() for op in metainterp.history.operations]
new_trace.resume_at_jump_descr = resume_at_jump_descr
metainterp_sd = metainterp.staticdata
state = metainterp.jitdriver_sd.warmstate
if isinstance(resumekey, ResumeAtPositionDescr):
inline_short_preamble = False
else:
inline_short_preamble = True
try:
optimize_trace(metainterp_sd, new_trace, state.enable_opts, inline_short_preamble)
except InvalidLoop:
debug_print("compile_new_bridge: got an InvalidLoop")
# XXX I am fairly convinced that optimize_bridge cannot actually raise
# InvalidLoop
debug_print('InvalidLoop in compile_new_bridge')
return None
if new_trace.operations[-1].getopnum() != rop.LABEL:
# We managed to create a bridge. Dispatch to resumekey to
# know exactly what we must do (ResumeGuardDescr/ResumeFromInterpDescr)
target_token = new_trace.operations[-1].getdescr()
resumekey.compile_and_attach(metainterp, new_trace)
record_loop_or_bridge(metainterp_sd, new_trace)
return target_token
else:
metainterp.retrace_needed(new_trace)
return None
def _do_optimize_loop(self, compile_data):
from rpython.jit.metainterp.optimizeopt import optimize_trace
metainterp_sd = FakeMetaInterpStaticData(self.cpu)
if hasattr(self, 'vrefinfo'):
metainterp_sd.virtualref_info = self.vrefinfo
if hasattr(self, 'callinfocollection'):
metainterp_sd.callinfocollection = self.callinfocollection
#
compile_data.enable_opts = self.enable_opts
state = optimize_trace(metainterp_sd, None, compile_data)
return state
def _do_optimize_loop(self, compile_data):
from rpython.jit.metainterp.optimizeopt import optimize_trace
metainterp_sd = FakeMetaInterpStaticData(self.cpu)
if hasattr(self, 'vrefinfo'):
metainterp_sd.virtualref_info = self.vrefinfo
if hasattr(self, 'callinfocollection'):
metainterp_sd.callinfocollection = self.callinfocollection
#
compile_data.enable_opts = self.enable_opts
state = optimize_trace(metainterp_sd, None, compile_data)
return state
def _do_optimize_loop(self, compile_data):
from rpython.jit.metainterp.optimizeopt import optimize_trace
metainterp_sd = FakeMetaInterpStaticData(self.cpu)
if hasattr(self, 'vrefinfo'):
metainterp_sd.virtualref_info = self.vrefinfo
if hasattr(self, 'callinfocollection'):
metainterp_sd.callinfocollection = self.callinfocollection
compute_bitstrings(self.cpu.fetch_all_descrs())
#
compile_data.enable_opts = self.enable_opts
state = optimize_trace(metainterp_sd, None, compile_data)
state[0]._check_no_forwarding()
return state
def _do_optimize_loop(self, compile_data):
from rpython.jit.metainterp.optimizeopt import optimize_trace
metainterp_sd = FakeMetaInterpStaticData(self.cpu)
if hasattr(self, 'vrefinfo'):
metainterp_sd.virtualref_info = self.vrefinfo
if hasattr(self, 'callinfocollection'):
metainterp_sd.callinfocollection = self.callinfocollection
compute_bitstrings(self.cpu.fetch_all_descrs())
#
compile_data.enable_opts = self.enable_opts
state = optimize_trace(metainterp_sd, None, compile_data)
state[0]._check_no_forwarding()
return state
def compile_trace(metainterp, resumekey):
"""Try to compile a new bridge leading from the beginning of the history
to some existing place.
"""
from rpython.jit.metainterp.optimizeopt import optimize_trace
# The history contains new operations to attach as the code for the
# failure of 'resumekey.guard_op'.
#
# Attempt to use optimize_bridge(). This may return None in case
# it does not work -- i.e. none of the existing old_loop_tokens match.
new_trace = create_empty_loop(metainterp)
new_trace.inputargs = metainterp.history.inputargs[:]
new_trace.operations = metainterp.history.operations[:]
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
state = jitdriver_sd.warmstate
if isinstance(resumekey, ResumeAtPositionDescr):
inline_short_preamble = False
else:
inline_short_preamble = True
try:
state = optimize_trace(metainterp_sd,
jitdriver_sd,
new_trace,
state.enable_opts,
inline_short_preamble,
export_state=True)
except InvalidLoop:
debug_print("compile_new_bridge: got an InvalidLoop")
# XXX I am fairly convinced that optimize_bridge cannot actually raise
# InvalidLoop
debug_print('InvalidLoop in compile_new_bridge')
return None
if new_trace.operations[-1].getopnum() != rop.LABEL:
# We managed to create a bridge. Dispatch to resumekey to
# know exactly what we must do (ResumeGuardDescr/ResumeFromInterpDescr)
target_token = new_trace.operations[-1].getdescr()
resumekey.compile_and_attach(metainterp, new_trace)
record_loop_or_bridge(metainterp_sd, new_trace)
return target_token
else:
metainterp.retrace_needed(new_trace, state)
return None
def compile_retrace(metainterp, greenkey, start,
inputargs, jumpargs,
partial_trace, resumekey, start_state):
"""Try to compile a new procedure by closing the current history back
to the first operation.
"""
from rpython.jit.metainterp.optimizeopt import optimize_trace
from rpython.jit.metainterp.optimizeopt.optimizer import BasicLoopInfo
history = metainterp.history
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
loop_jitcell_token = metainterp.get_procedure_token(greenkey)
assert loop_jitcell_token
end_label = ResOperation(rop.LABEL, inputargs[:],
descr=loop_jitcell_token)
jump_op = ResOperation(rop.JUMP, jumpargs[:], descr=loop_jitcell_token)
enable_opts = jitdriver_sd.warmstate.enable_opts
ops = history.operations[start:]
call_pure_results = metainterp.call_pure_results
loop_data = UnrolledLoopData(end_label, jump_op, ops, start_state,
call_pure_results=call_pure_results,
enable_opts=enable_opts)
try:
loop_info, loop_ops = optimize_trace(metainterp_sd, jitdriver_sd,
loop_data,
metainterp.box_names_memo)
except InvalidLoop:
# Fall back on jumping directly to preamble
jump_op = ResOperation(rop.JUMP, inputargs[:], descr=loop_jitcell_token)
loop_data = UnrolledLoopData(end_label, jump_op, [jump_op], start_state,
call_pure_results=call_pure_results,
enable_opts=enable_opts,
inline_short_preamble=False)
try:
loop_info, loop_ops = optimize_trace(metainterp_sd, jitdriver_sd,
loop_data,
metainterp.box_names_memo)
except InvalidLoop:
return None
label_token = loop_info.label_op.getdescr()
assert isinstance(label_token, TargetToken)
if label_token.short_preamble:
metainterp_sd.logger_ops.log_short_preamble([],
label_token.short_preamble, metainterp.box_names_memo)
loop = partial_trace
loop.original_jitcell_token = loop_jitcell_token
loop.operations = (loop.operations + loop_info.extra_same_as +
[loop_info.label_op]
+ loop_ops)
quasi_immutable_deps = {}
if loop_info.quasi_immutable_deps:
quasi_immutable_deps.update(loop_info.quasi_immutable_deps)
if start_state.quasi_immutable_deps:
quasi_immutable_deps.update(start_state.quasi_immutable_deps)
if quasi_immutable_deps:
loop.quasi_immutable_deps = quasi_immutable_deps
target_token = loop.operations[-1].getdescr()
resumekey.compile_and_attach(metainterp, loop, inputargs)
record_loop_or_bridge(metainterp_sd, loop)
return target_token
def compile_loop(metainterp, greenkey, start, inputargs, jumpargs,
full_preamble_needed=True, try_disabling_unroll=False):
"""Try to compile a new procedure by closing the current history back
to the first operation.
"""
from rpython.jit.metainterp.optimizeopt import optimize_trace
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
history = metainterp.history
enable_opts = jitdriver_sd.warmstate.enable_opts
if try_disabling_unroll:
if 'unroll' not in enable_opts:
return None
enable_opts = enable_opts.copy()
del enable_opts['unroll']
ops = history.operations[start:]
if 'unroll' not in enable_opts or not metainterp.cpu.supports_guard_gc_type:
return compile_simple_loop(metainterp, greenkey, start, inputargs, ops,
jumpargs, enable_opts)
jitcell_token = make_jitcell_token(jitdriver_sd)
label = ResOperation(rop.LABEL, inputargs,
descr=TargetToken(jitcell_token))
end_label = ResOperation(rop.LABEL, jumpargs, descr=jitcell_token)
call_pure_results = metainterp.call_pure_results
preamble_data = LoopCompileData(label, end_label, ops,
call_pure_results=call_pure_results,
enable_opts=enable_opts)
try:
start_state, preamble_ops = optimize_trace(metainterp_sd, jitdriver_sd,
preamble_data,
metainterp.box_names_memo)
except InvalidLoop:
return None
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
end_label = ResOperation(rop.LABEL, inputargs,
descr=jitcell_token)
jump_op = ResOperation(rop.JUMP, jumpargs, descr=jitcell_token)
start_descr = TargetToken(jitcell_token,
original_jitcell_token=jitcell_token)
jitcell_token.target_tokens = [start_descr]
loop_data = UnrolledLoopData(end_label, jump_op, ops, start_state,
call_pure_results=call_pure_results,
enable_opts=enable_opts)
try:
loop_info, loop_ops = optimize_trace(metainterp_sd, jitdriver_sd,
loop_data,
metainterp.box_names_memo)
except InvalidLoop:
return None
loop = create_empty_loop(metainterp)
loop.original_jitcell_token = jitcell_token
loop.inputargs = start_state.renamed_inputargs
quasi_immutable_deps = {}
if start_state.quasi_immutable_deps:
quasi_immutable_deps.update(start_state.quasi_immutable_deps)
if loop_info.quasi_immutable_deps:
quasi_immutable_deps.update(loop_info.quasi_immutable_deps)
if quasi_immutable_deps:
loop.quasi_immutable_deps = quasi_immutable_deps
start_label = ResOperation(rop.LABEL, start_state.renamed_inputargs,
descr=start_descr)
label_token = loop_info.label_op.getdescr()
assert isinstance(label_token, TargetToken)
if label_token.short_preamble:
metainterp_sd.logger_ops.log_short_preamble([],
label_token.short_preamble, metainterp.box_names_memo)
loop.operations = ([start_label] + preamble_ops + loop_info.extra_same_as +
[loop_info.label_op] + loop_ops)
if not we_are_translated():
loop.check_consistency()
send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop",
inputargs, metainterp.box_names_memo)
record_loop_or_bridge(metainterp_sd, loop)
return start_descr
def compile_retrace(metainterp, greenkey, start,
inputargs, jumpargs,
partial_trace, resumekey, start_state):
"""Try to compile a new procedure by closing the current history back
to the first operation.
"""
from rpython.jit.metainterp.optimizeopt import optimize_trace
trace = metainterp.history.trace.cut_trace_from(start, inputargs)
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
history = metainterp.history
loop_jitcell_token = metainterp.get_procedure_token(greenkey)
assert loop_jitcell_token
cut = history.get_trace_position()
history.record(rop.JUMP, jumpargs[:], None, descr=loop_jitcell_token)
enable_opts = jitdriver_sd.warmstate.enable_opts
call_pure_results = metainterp.call_pure_results
loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state,
call_pure_results=call_pure_results,
enable_opts=enable_opts)
try:
loop_info, loop_ops = optimize_trace(metainterp_sd, jitdriver_sd,
loop_data,
metainterp.box_names_memo)
except InvalidLoop:
# Fall back on jumping directly to preamble
history.cut(cut)
history.record(rop.JUMP, jumpargs[:], None, descr=loop_jitcell_token)
loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state,
call_pure_results=call_pure_results,
enable_opts=enable_opts,
inline_short_preamble=False)
try:
loop_info, loop_ops = optimize_trace(metainterp_sd, jitdriver_sd,
loop_data,
metainterp.box_names_memo)
except InvalidLoop:
history.cut(cut)
return None
label_op = loop_info.label_op
if label_op is None:
assert False, "unreachable code" # hint for some strange tests
label_token = label_op.getdescr()
assert isinstance(label_token, TargetToken)
if label_token.short_preamble:
metainterp_sd.logger_ops.log_short_preamble([],
label_token.short_preamble, metainterp.box_names_memo)
loop = partial_trace
loop.original_jitcell_token = loop_jitcell_token
loop.operations = (loop.operations + loop_info.extra_same_as +
[loop_info.label_op]
+ loop_ops)
quasi_immutable_deps = {}
if loop_info.quasi_immutable_deps:
quasi_immutable_deps.update(loop_info.quasi_immutable_deps)
if start_state.quasi_immutable_deps:
quasi_immutable_deps.update(start_state.quasi_immutable_deps)
if quasi_immutable_deps:
loop.quasi_immutable_deps = quasi_immutable_deps
target_token = loop.operations[-1].getdescr()
resumekey.compile_and_attach(metainterp, loop, inputargs)
record_loop_or_bridge(metainterp_sd, loop)
return target_token
def compile_loop(metainterp, greenkey, start, inputargs, jumpargs,
full_preamble_needed=True, try_disabling_unroll=False):
"""Try to compile a new procedure by closing the current history back
to the first operation.
"""
from rpython.jit.metainterp.optimizeopt import optimize_trace
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
history = metainterp.history
trace = history.trace
warmstate = jitdriver_sd.warmstate
enable_opts = jitdriver_sd.warmstate.enable_opts
if try_disabling_unroll:
if 'unroll' not in enable_opts:
return None
enable_opts = enable_opts.copy()
del enable_opts['unroll']
jitcell_token = make_jitcell_token(jitdriver_sd)
cut_at = history.get_trace_position()
history.record(rop.JUMP, jumpargs, None, descr=jitcell_token)
if start != (0, 0, 0):
trace = trace.cut_trace_from(start, inputargs)
if 'unroll' not in enable_opts or not metainterp.cpu.supports_guard_gc_type:
return compile_simple_loop(metainterp, greenkey, trace, jumpargs,
enable_opts, cut_at)
call_pure_results = metainterp.call_pure_results
preamble_data = LoopCompileData(trace, jumpargs,
call_pure_results=call_pure_results,
enable_opts=enable_opts)
try:
start_state, preamble_ops = optimize_trace(metainterp_sd, jitdriver_sd,
preamble_data,
metainterp.box_names_memo)
except InvalidLoop:
history.cut(cut_at)
return None
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
start_descr = TargetToken(jitcell_token,
original_jitcell_token=jitcell_token)
jitcell_token.target_tokens = [start_descr]
loop_data = UnrolledLoopData(trace, jitcell_token, start_state,
call_pure_results=call_pure_results,
enable_opts=enable_opts)
try:
loop_info, loop_ops = optimize_trace(metainterp_sd, jitdriver_sd,
loop_data,
metainterp.box_names_memo)
except InvalidLoop:
history.cut(cut_at)
return None
if ((warmstate.vec and jitdriver_sd.vec) or warmstate.vec_all):
from rpython.jit.metainterp.optimizeopt.vector import optimize_vector
loop_info, loop_ops = optimize_vector(trace, metainterp_sd,
jitdriver_sd, warmstate,
loop_info, loop_ops,
jitcell_token)
#
loop = create_empty_loop(metainterp)
loop.original_jitcell_token = jitcell_token
loop.inputargs = start_state.renamed_inputargs
quasi_immutable_deps = {}
if start_state.quasi_immutable_deps:
quasi_immutable_deps.update(start_state.quasi_immutable_deps)
if loop_info.quasi_immutable_deps:
quasi_immutable_deps.update(loop_info.quasi_immutable_deps)
if quasi_immutable_deps:
loop.quasi_immutable_deps = quasi_immutable_deps
start_label = ResOperation(rop.LABEL, start_state.renamed_inputargs,
descr=start_descr)
label_token = loop_info.label_op.getdescr()
assert isinstance(label_token, TargetToken)
if label_token.short_preamble:
metainterp_sd.logger_ops.log_short_preamble([],
label_token.short_preamble, metainterp.box_names_memo)
loop.operations = ([start_label] + preamble_ops + loop_info.extra_same_as +
[loop_info.label_op] + loop_ops)
if not we_are_translated():
loop.check_consistency()
send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop",
inputargs, metainterp.box_names_memo)
record_loop_or_bridge(metainterp_sd, loop)
loop_info.post_loop_compilation(loop, jitdriver_sd, metainterp, jitcell_token)
return start_descr
def compile_loop(metainterp,
greenkey,
start,
inputargs,
jumpargs,
full_preamble_needed=True,
try_disabling_unroll=False):
"""Try to compile a new procedure by closing the current history back
to the first operation.
"""
from rpython.jit.metainterp.optimizeopt import optimize_trace
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
history = metainterp.history
enable_opts = jitdriver_sd.warmstate.enable_opts
if try_disabling_unroll:
if 'unroll' not in enable_opts:
return None
enable_opts = enable_opts.copy()
del enable_opts['unroll']
jitcell_token = make_jitcell_token(jitdriver_sd)
part = create_empty_loop(metainterp)
part.inputargs = inputargs[:]
h_ops = history.operations
label = ResOperation(rop.LABEL,
inputargs,
None,
descr=TargetToken(jitcell_token))
end_label = ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)
part.operations = [label] + h_ops[start:] + [end_label]
try:
start_state = optimize_trace(metainterp_sd,
jitdriver_sd,
part,
enable_opts,
export_state=True)
except InvalidLoop:
return None
target_token = part.operations[0].getdescr()
assert isinstance(target_token, TargetToken)
all_target_tokens = [target_token]
loop = create_empty_loop(metainterp)
loop.inputargs = part.inputargs
loop.operations = part.operations
loop.quasi_immutable_deps = {}
if part.quasi_immutable_deps:
loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
if part.operations[-1].getopnum() == rop.LABEL:
inliner = Inliner(inputargs, jumpargs)
part.quasi_immutable_deps = None
part.operations = [part.operations[-1]] + \
[inliner.inline_op(h_ops[i]) for i in range(start, len(h_ops))] + \
[ResOperation(rop.JUMP, [inliner.inline_arg(a) for a in jumpargs],
None, descr=jitcell_token)]
target_token = part.operations[0].getdescr()
assert isinstance(target_token, TargetToken)
all_target_tokens.append(target_token)
inputargs = jumpargs
jumpargs = part.operations[-1].getarglist()
try:
optimize_trace(metainterp_sd,
jitdriver_sd,
part,
enable_opts,
start_state=start_state,
export_state=False)
except InvalidLoop:
return None
loop.operations = loop.operations[:-1] + part.operations
if part.quasi_immutable_deps:
loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
assert part.operations[-1].getopnum() != rop.LABEL
if not loop.quasi_immutable_deps:
loop.quasi_immutable_deps = None
for box in loop.inputargs:
assert isinstance(box, Box)
loop.original_jitcell_token = jitcell_token
for label in all_target_tokens:
assert isinstance(label, TargetToken)
if label.virtual_state and label.short_preamble:
metainterp_sd.logger_ops.log_short_preamble([],
label.short_preamble)
jitcell_token.target_tokens = all_target_tokens
propagate_original_jitcell_token(loop)
send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
record_loop_or_bridge(metainterp_sd, loop)
return all_target_tokens[0]
def compile_retrace(metainterp, greenkey, start, inputargs, jumpargs,
partial_trace, resumekey, start_state):
"""Try to compile a new procedure by closing the current history back
to the first operation.
"""
from rpython.jit.metainterp.optimizeopt import optimize_trace
history = metainterp.history
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
loop_jitcell_token = metainterp.get_procedure_token(greenkey)
assert loop_jitcell_token
assert partial_trace.operations[-1].getopnum() == rop.LABEL
part = create_empty_loop(metainterp)
part.inputargs = inputargs[:]
h_ops = history.operations
part.operations = [partial_trace.operations[-1]] + \
h_ops[start:] + \
[ResOperation(rop.JUMP, jumpargs, None, descr=loop_jitcell_token)]
label = part.operations[0]
orignial_label = label.clone()
assert label.getopnum() == rop.LABEL
try:
optimize_trace(metainterp_sd,
jitdriver_sd,
part,
jitdriver_sd.warmstate.enable_opts,
start_state=start_state,
export_state=False)
except InvalidLoop:
# Fall back on jumping to preamble
target_token = label.getdescr()
assert isinstance(target_token, TargetToken)
part.operations = [orignial_label] + \
[ResOperation(rop.JUMP, inputargs[:],
None, descr=loop_jitcell_token)]
try:
optimize_trace(metainterp_sd,
jitdriver_sd,
part,
jitdriver_sd.warmstate.enable_opts,
inline_short_preamble=False,
start_state=start_state,
export_state=False)
except InvalidLoop:
return None
assert part.operations[-1].getopnum() != rop.LABEL
target_token = label.getdescr()
assert isinstance(target_token, TargetToken)
assert loop_jitcell_token.target_tokens
loop_jitcell_token.target_tokens.append(target_token)
if target_token.short_preamble:
metainterp_sd.logger_ops.log_short_preamble(
[], target_token.short_preamble)
loop = partial_trace
loop.operations = loop.operations[:-1] + part.operations
quasi_immutable_deps = {}
if loop.quasi_immutable_deps:
quasi_immutable_deps.update(loop.quasi_immutable_deps)
if part.quasi_immutable_deps:
quasi_immutable_deps.update(part.quasi_immutable_deps)
if quasi_immutable_deps:
loop.quasi_immutable_deps = quasi_immutable_deps
for box in loop.inputargs:
assert isinstance(box, Box)
target_token = loop.operations[-1].getdescr()
resumekey.compile_and_attach(metainterp, loop)
target_token = label.getdescr()
assert isinstance(target_token, TargetToken)
record_loop_or_bridge(metainterp_sd, loop)
return target_token
def compile_loop(metainterp, greenkey, start,
inputargs, jumpargs,
full_preamble_needed=True,
try_disabling_unroll=False):
"""Try to compile a new procedure by closing the current history back
to the first operation.
"""
from rpython.jit.metainterp.optimizeopt import optimize_trace
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
history = metainterp.history
enable_opts = jitdriver_sd.warmstate.enable_opts
if try_disabling_unroll:
if 'unroll' not in enable_opts:
return None
enable_opts = enable_opts.copy()
del enable_opts['unroll']
jitcell_token = make_jitcell_token(jitdriver_sd)
part = create_empty_loop(metainterp)
part.inputargs = inputargs[:]
h_ops = history.operations
label = ResOperation(rop.LABEL, inputargs, None,
descr=TargetToken(jitcell_token))
end_label = ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)
part.operations = [label] + h_ops[start:] + [end_label]
try:
start_state = optimize_trace(metainterp_sd, jitdriver_sd, part,
enable_opts, export_state=True)
except InvalidLoop:
return None
target_token = part.operations[0].getdescr()
assert isinstance(target_token, TargetToken)
all_target_tokens = [target_token]
loop = create_empty_loop(metainterp)
loop.inputargs = part.inputargs
loop.operations = part.operations
loop.quasi_immutable_deps = {}
if part.quasi_immutable_deps:
loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
if part.operations[-1].getopnum() == rop.LABEL:
inliner = Inliner(inputargs, jumpargs)
part.quasi_immutable_deps = None
part.operations = [part.operations[-1]] + \
[inliner.inline_op(h_ops[i]) for i in range(start, len(h_ops))] + \
[ResOperation(rop.JUMP, [inliner.inline_arg(a) for a in jumpargs],
None, descr=jitcell_token)]
target_token = part.operations[0].getdescr()
assert isinstance(target_token, TargetToken)
all_target_tokens.append(target_token)
inputargs = jumpargs
jumpargs = part.operations[-1].getarglist()
try:
optimize_trace(metainterp_sd, jitdriver_sd, part, enable_opts,
start_state=start_state, export_state=False)
except InvalidLoop:
return None
loop.operations = loop.operations[:-1] + part.operations
if part.quasi_immutable_deps:
loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
assert part.operations[-1].getopnum() != rop.LABEL
if not loop.quasi_immutable_deps:
loop.quasi_immutable_deps = None
for box in loop.inputargs:
assert isinstance(box, Box)
loop.original_jitcell_token = jitcell_token
for label in all_target_tokens:
assert isinstance(label, TargetToken)
if label.virtual_state and label.short_preamble:
metainterp_sd.logger_ops.log_short_preamble([], label.short_preamble)
jitcell_token.target_tokens = all_target_tokens
propagate_original_jitcell_token(loop)
send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
record_loop_or_bridge(metainterp_sd, loop)
return all_target_tokens[0]
def compile_retrace(metainterp, greenkey, start,
inputargs, jumpargs,
partial_trace, resumekey, start_state):
"""Try to compile a new procedure by closing the current history back
to the first operation.
"""
from rpython.jit.metainterp.optimizeopt import optimize_trace
history = metainterp.history
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
loop_jitcell_token = metainterp.get_procedure_token(greenkey)
assert loop_jitcell_token
assert partial_trace.operations[-1].getopnum() == rop.LABEL
part = create_empty_loop(metainterp)
part.inputargs = inputargs[:]
h_ops = history.operations
part.operations = [partial_trace.operations[-1]] + \
h_ops[start:] + \
[ResOperation(rop.JUMP, jumpargs, None, descr=loop_jitcell_token)]
label = part.operations[0]
orignial_label = label.clone()
assert label.getopnum() == rop.LABEL
try:
optimize_trace(metainterp_sd, jitdriver_sd, part,
jitdriver_sd.warmstate.enable_opts,
start_state=start_state, export_state=False)
except InvalidLoop:
# Fall back on jumping to preamble
target_token = label.getdescr()
assert isinstance(target_token, TargetToken)
part.operations = [orignial_label] + \
[ResOperation(rop.JUMP, inputargs[:],
None, descr=loop_jitcell_token)]
try:
optimize_trace(metainterp_sd, jitdriver_sd, part,
jitdriver_sd.warmstate.enable_opts,
inline_short_preamble=False, start_state=start_state,
export_state=False)
except InvalidLoop:
return None
assert part.operations[-1].getopnum() != rop.LABEL
target_token = label.getdescr()
assert isinstance(target_token, TargetToken)
assert loop_jitcell_token.target_tokens
loop_jitcell_token.target_tokens.append(target_token)
if target_token.short_preamble:
metainterp_sd.logger_ops.log_short_preamble([], target_token.short_preamble)
loop = partial_trace
loop.operations = loop.operations[:-1] + part.operations
quasi_immutable_deps = {}
if loop.quasi_immutable_deps:
quasi_immutable_deps.update(loop.quasi_immutable_deps)
if part.quasi_immutable_deps:
quasi_immutable_deps.update(part.quasi_immutable_deps)
if quasi_immutable_deps:
loop.quasi_immutable_deps = quasi_immutable_deps
for box in loop.inputargs:
assert isinstance(box, Box)
target_token = loop.operations[-1].getdescr()
resumekey.compile_and_attach(metainterp, loop)
target_token = label.getdescr()
assert isinstance(target_token, TargetToken)
record_loop_or_bridge(metainterp_sd, loop)
return target_token
def compile_loop(metainterp, greenkey, start, inputargs, jumpargs,
full_preamble_needed=True, try_disabling_unroll=False):
"""Try to compile a new procedure by closing the current history back
to the first operation.
"""
from rpython.jit.metainterp.optimizeopt import optimize_trace
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
history = metainterp.history
trace = history.trace
warmstate = jitdriver_sd.warmstate
#
metainterp_sd.jitlog.start_new_trace(metainterp_sd,
faildescr=None, entry_bridge=False)
#
enable_opts = jitdriver_sd.warmstate.enable_opts
if try_disabling_unroll:
if 'unroll' not in enable_opts:
return None
enable_opts = enable_opts.copy()
del enable_opts['unroll']
jitcell_token = make_jitcell_token(jitdriver_sd)
cut_at = history.get_trace_position()
history.record(rop.JUMP, jumpargs, None, descr=jitcell_token)
if start != (0, 0, 0):
trace = trace.cut_trace_from(start, inputargs)
if 'unroll' not in enable_opts or not metainterp.cpu.supports_guard_gc_type:
return compile_simple_loop(metainterp, greenkey, trace, jumpargs,
enable_opts, cut_at)
call_pure_results = metainterp.call_pure_results
preamble_data = LoopCompileData(trace, jumpargs,
call_pure_results=call_pure_results,
enable_opts=enable_opts)
try:
start_state, preamble_ops = optimize_trace(metainterp_sd, jitdriver_sd,
preamble_data,
metainterp.box_names_memo)
except InvalidLoop:
metainterp_sd.jitlog.trace_aborted()
history.cut(cut_at)
return None
metainterp_sd = metainterp.staticdata
jitdriver_sd = metainterp.jitdriver_sd
start_descr = TargetToken(jitcell_token,
original_jitcell_token=jitcell_token)
jitcell_token.target_tokens = [start_descr]
loop_data = UnrolledLoopData(trace, jitcell_token, start_state, jumpargs,
call_pure_results=call_pure_results,
enable_opts=enable_opts)
try:
loop_info, loop_ops = optimize_trace(metainterp_sd, jitdriver_sd,
loop_data,
metainterp.box_names_memo)
except InvalidLoop:
metainterp_sd.jitlog.trace_aborted()
history.cut(cut_at)
return None
if ((warmstate.vec and jitdriver_sd.vec) or warmstate.vec_all) and \
metainterp.cpu.vector_ext and metainterp.cpu.vector_ext.is_enabled():
from rpython.jit.metainterp.optimizeopt.vector import optimize_vector
loop_info, loop_ops = optimize_vector(trace, metainterp_sd,
jitdriver_sd, warmstate,
loop_info, loop_ops,
jitcell_token)
#
loop = create_empty_loop(metainterp)
loop.original_jitcell_token = jitcell_token
loop.inputargs = start_state.renamed_inputargs
quasi_immutable_deps = {}
if start_state.quasi_immutable_deps:
quasi_immutable_deps.update(start_state.quasi_immutable_deps)
if loop_info.quasi_immutable_deps:
quasi_immutable_deps.update(loop_info.quasi_immutable_deps)
if quasi_immutable_deps:
loop.quasi_immutable_deps = quasi_immutable_deps
start_label = ResOperation(rop.LABEL, start_state.renamed_inputargs,
descr=start_descr)
label_token = loop_info.label_op.getdescr()
assert isinstance(label_token, TargetToken)
if label_token.short_preamble:
metainterp_sd.logger_ops.log_short_preamble([],
label_token.short_preamble, metainterp.box_names_memo)
loop.operations = ([start_label] + preamble_ops + loop_info.extra_same_as +
loop_info.extra_before_label + [loop_info.label_op] + loop_ops)
if not we_are_translated():
loop.check_consistency()
send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop",
inputargs, metainterp.box_names_memo)
record_loop_or_bridge(metainterp_sd, loop)
loop_info.post_loop_compilation(loop, jitdriver_sd, metainterp, jitcell_token)
return start_descr
