def test_setfield_forced_virtual(self): loop = """ [p1, p2] i1 = getfield_gc_i(p1, descr=valuedescr) setfield_gc(p2, i1, descr=valuedescr) p3 = new_with_vtable(descr=nodesize) jump(p2, p3) """ es, loop, preamble = self.optimize(loop) sb = ShortPreambleBuilder(loop.inputargs, es.short_boxes, es.short_inputargs, es.exported_infos) short_boxes = [box for box in es.short_boxes if not isinstance(box.short_op, ShortInputArg)] op = short_boxes[0].short_op.res pop = sb.use_box(op, short_boxes[0].preamble_op, FakeOptimizer()) sb.add_preamble_op(PreambleOp(op, pop, False)) exp_short = """ [p0, p1] guard_nonnull(p0) [] guard_subclass(p0, ConstClass(node_vtable)) [] i1 = getfield_gc_i(p0, descr=valuedescr) jump(i1) """ self.compare_short(sb.build_short_preamble(), exp_short)
def import_state(self, targetargs, exported_state): # the mapping between input args (from old label) and what we need # to actually emit. Update the info assert (len(exported_state.next_iteration_args) == len(targetargs)) self._check_no_forwarding([targetargs]) exported_state._check_no_forwarding() for i, target in enumerate(exported_state.next_iteration_args): source = targetargs[i] assert source is not target source.set_forwarded(target) info = exported_state.exported_infos.get(target, None) if info is not None: self.optimizer.setinfo_from_preamble(source, info, exported_state.exported_infos) # import the optimizer state, starting from boxes that can be produced # by short preamble label_args = exported_state.virtual_state.make_inputargs( targetargs, self.optimizer) self.short_preamble_producer = ShortPreambleBuilder( label_args, exported_state.short_boxes, exported_state.short_inputargs, exported_state.exported_infos, self.optimizer) for produced_op in exported_state.short_boxes: produced_op.produce_op(self, exported_state.exported_infos) return label_args
def test_short_boxes_heapcache(self): loop = """ [p0, i1] i0 = getfield_gc_i(p0, descr=valuedescr) jump(p0, i1) """ es, loop, preamble = self.optimize(loop) op = preamble.operations[0] short_boxes = [box for box in es.short_boxes if not isinstance(box.short_op, ShortInputArg)] assert len(short_boxes) == 1 assert short_boxes[0].short_op.res is op sb = ShortPreambleBuilder(loop.inputargs, es.short_boxes, es.short_inputargs, es.exported_infos, FakeOptimizer()) op = preamble.operations[0] short_op = sb.use_box(op, short_boxes[0].preamble_op, FakeOptimizer()) sb.add_preamble_op(PreambleOp(op, short_op, False)) exp_short = """ [p0, i1] guard_nonnull(p0) [] guard_subclass(p0, ConstClass(node_vtable)) [] i0 = getfield_gc_i(p0, descr=valuedescr) jump(i0) """ self.compare_short(sb.build_short_preamble(), exp_short)
def import_state(self, targetop, exported_state): # the mapping between input args (from old label) and what we need # to actually emit. Update the info assert (len(exported_state.next_iteration_args) == len(targetop.getarglist())) for i, target in enumerate(exported_state.next_iteration_args): source = targetop.getarg(i) assert source is not target source.set_forwarded(target) info = exported_state.exported_infos.get(target, None) if info is not None: self.optimizer.setinfo_from_preamble(source, info, exported_state.exported_infos) # import the optimizer state, starting from boxes that can be produced # by short preamble label_args = exported_state.virtual_state.make_inputargs( targetop.getarglist(), self.optimizer) self.short_preamble_producer = ShortPreambleBuilder( label_args, exported_state.short_boxes, exported_state.short_inputargs, exported_state.exported_infos, self.optimizer) for produced_op in exported_state.short_boxes: produced_op.produce_op(self, exported_state.exported_infos) return label_args
def test_setfield_forced_virtual(self): loop = """ [p1, p2] i1 = getfield_gc_i(p1, descr=valuedescr) setfield_gc(p2, i1, descr=valuedescr) p3 = new_with_vtable(descr=nodesize) jump(p2, p3) """ es, loop, preamble = self.optimize(loop) sb = ShortPreambleBuilder(loop.inputargs, es.short_boxes, es.short_inputargs, es.exported_infos) short_boxes = [ box for box in es.short_boxes if not isinstance(box.short_op, ShortInputArg) ] op = short_boxes[0].short_op.res pop = sb.use_box(op, short_boxes[0].preamble_op, FakeOptimizer()) sb.add_preamble_op(PreambleOp(op, pop, False)) exp_short = """ [p0, p1] guard_nonnull(p0) [] guard_subclass(p0, ConstClass(node_vtable)) [] i1 = getfield_gc_i(p0, descr=valuedescr) jump(i1) """ self.compare_short(sb.build_short_preamble(), exp_short)
def test_short_boxes_heapcache(self): loop = """ [p0, i1] i0 = getfield_gc_i(p0, descr=valuedescr) jump(p0, i1) """ es, loop, preamble = self.optimize(loop) op = preamble.operations[0] short_boxes = [ box for box in es.short_boxes if not isinstance(box.short_op, ShortInputArg) ] assert len(short_boxes) == 1 assert short_boxes[0].short_op.res is op sb = ShortPreambleBuilder(loop.inputargs, es.short_boxes, es.short_inputargs, es.exported_infos, FakeOptimizer()) op = preamble.operations[0] short_op = sb.use_box(op, short_boxes[0].preamble_op, FakeOptimizer()) sb.add_preamble_op(PreambleOp(op, short_op, False)) exp_short = """ [p0, i1] guard_nonnull(p0) [] guard_subclass(p0, ConstClass(node_vtable)) [] i0 = getfield_gc_i(p0, descr=valuedescr) jump(i0) """ self.compare_short(sb.build_short_preamble(), exp_short)
def test_simple(self): loop = """ [i0] i1 = int_add(i0, 1) guard_value(i1, 1) [] jump(i1) """ es, loop, preamble = self.optimize(loop) vs = es.virtual_state assert isinstance(vs.state[0], NotVirtualStateInfo) # the virtual state is constant, so we don't need to have it in # inputargs assert vs.make_inputargs([1], FakeOptimizer()) == [] assert vs.state[0].level == LEVEL_CONSTANT # we have exported values for i1, which happens to be an inputarg sb = ShortPreambleBuilder([], es.short_boxes, es.short_inputargs, es.exported_infos) sp = sb.build_short_preamble() exp = """ [] jump() """ self.compare_short(sp, exp)
def finalize_short_preamble(self, label_op, virtual_state): sb = self.short_preamble_producer self.optimizer._clean_optimization_info(sb.short_inputargs) short_preamble = sb.build_short_preamble() jitcelltoken = label_op.getdescr() assert isinstance(jitcelltoken, JitCellToken) if jitcelltoken.target_tokens is None: jitcelltoken.target_tokens = [] target_token = TargetToken(jitcelltoken, original_jitcell_token=jitcelltoken) target_token.original_jitcell_token = jitcelltoken target_token.virtual_state = virtual_state target_token.short_preamble = short_preamble jitcelltoken.target_tokens.append(target_token) self.short_preamble_producer = ExtendedShortPreambleBuilder( target_token, sb) label_op.initarglist(label_op.getarglist() + sb.used_boxes) return target_token
class UnrollOptimizer(Optimization): """Unroll the loop into two iterations. The first one will become the preamble or entry bridge (don't think there is a distinction anymore)""" short_preamble_producer = None def __init__(self, metainterp_sd, jitdriver_sd, optimizations): self.optimizer = UnrollableOptimizer(metainterp_sd, jitdriver_sd, optimizations) self.optimizer.optunroll = self def get_virtual_state(self, args): modifier = VirtualStateConstructor(self.optimizer) return modifier.get_virtual_state(args) def _check_no_forwarding(self, lsts, check_newops=True): check_no_forwarding(lsts) if check_newops: assert not self.optimizer._newoperations def optimize_preamble(self, trace, runtime_boxes, call_pure_results, memo): info, newops = self.optimizer.propagate_all_forward( trace.get_iter(), call_pure_results, flush=False) exported_state = self.export_state(info.jump_op.getarglist(), info.inputargs, memo) exported_state.quasi_immutable_deps = info.quasi_immutable_deps # we need to absolutely make sure that we've cleaned up all # the optimization info self.optimizer._clean_optimization_info(self.optimizer._newoperations) return exported_state, self.optimizer._newoperations def optimize_peeled_loop(self, trace, celltoken, state, runtime_boxes, call_pure_results, inline_short_preamble=True): trace = trace.get_iter() try: label_args = self.import_state(trace.inputargs, state) except VirtualStatesCantMatch: raise InvalidLoop("Cannot import state, virtual states don't match") self.potential_extra_ops = {} self.optimizer.init_inparg_dict_from(label_args) try: info, _ = self.optimizer.propagate_all_forward( trace, call_pure_results, flush=False) except SpeculativeError: raise InvalidLoop("Speculative heap access would be ill-typed") end_jump = info.jump_op label_op = ResOperation(rop.LABEL, label_args, descr=celltoken) for a in end_jump.getarglist(): self.optimizer.force_box_for_end_of_preamble( self.optimizer.get_box_replacement(a)) current_vs = self.get_virtual_state(end_jump.getarglist()) # pick the vs we want to jump to assert isinstance(celltoken, JitCellToken) target_virtual_state = self.pick_virtual_state(current_vs, state.virtual_state, celltoken.target_tokens) # force the boxes for virtual state to match try: args = target_virtual_state.make_inputargs( [self.get_box_replacement(x) for x in end_jump.getarglist()], self.optimizer, force_boxes=True) for arg in args: if arg is not None: self.optimizer.force_box(arg) except VirtualStatesCantMatch: raise InvalidLoop("Virtual states did not match " "after picking the virtual state, when forcing" " boxes") extra_same_as = self.short_preamble_producer.extra_same_as[:] target_token = self.finalize_short_preamble(label_op, state.virtual_state) label_op.setdescr(target_token) if not inline_short_preamble: self.jump_to_preamble(celltoken, end_jump, info) return (UnrollInfo(target_token, label_op, extra_same_as, self.optimizer.quasi_immutable_deps), self.optimizer._newoperations) try: new_virtual_state = self.jump_to_existing_trace( end_jump, label_op, runtime_boxes, force_boxes=False) except InvalidLoop: # inlining short preamble failed, jump to preamble self.jump_to_preamble(celltoken, end_jump, info) return (UnrollInfo(target_token, label_op, extra_same_as, self.optimizer.quasi_immutable_deps), self.optimizer._newoperations) if new_virtual_state is not None: # Attempt to force virtual boxes in order to avoid jumping # to the preamble. try: new_virtual_state = self.jump_to_existing_trace( end_jump, label_op, runtime_boxes, force_boxes=True) except InvalidLoop: pass if new_virtual_state is not None: self.jump_to_preamble(celltoken, end_jump, info) return (UnrollInfo(target_token, label_op, extra_same_as, self.optimizer.quasi_immutable_deps), self.optimizer._newoperations) self.disable_retracing_if_max_retrace_guards( self.optimizer._newoperations, target_token) return (UnrollInfo(target_token, label_op, extra_same_as, self.optimizer.quasi_immutable_deps), self.optimizer._newoperations) def disable_retracing_if_max_retrace_guards(self, ops, target_token): maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards count = 0 for op in ops: if op.is_guard(): count += 1 if count > maxguards: assert isinstance(target_token, TargetToken) target_token.targeting_jitcell_token.retraced_count = sys.maxint def pick_virtual_state(self, my_vs, label_vs, target_tokens): if target_tokens is None: return label_vs # for tests for token in target_tokens: if token.virtual_state is None: continue if token.virtual_state.generalization_of(my_vs, self.optimizer): return token.virtual_state return label_vs def optimize_bridge(self, trace, runtime_boxes, call_pure_results, inline_short_preamble, box_names_memo, resumestorage): from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge frontend_inputargs = trace.inputargs trace = trace.get_iter() self._check_no_forwarding([trace.inputargs]) if resumestorage: deserialize_optimizer_knowledge(self.optimizer, resumestorage, frontend_inputargs, trace.inputargs) info, ops = self.optimizer.propagate_all_forward(trace, call_pure_results, False) jump_op = info.jump_op cell_token = jump_op.getdescr() assert isinstance(cell_token, JitCellToken) if not inline_short_preamble or len(cell_token.target_tokens) == 1: return self.jump_to_preamble(cell_token, jump_op, info) # force all the information that does not go to the short # preamble at all self.optimizer.flush() for a in jump_op.getarglist(): self.optimizer.force_box_for_end_of_preamble(a) try: vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes, force_boxes=False) except InvalidLoop: return self.jump_to_preamble(cell_token, jump_op, info) if vs is None: return info, self.optimizer._newoperations[:] warmrunnerdescr = self.optimizer.metainterp_sd.warmrunnerdesc limit = warmrunnerdescr.memory_manager.retrace_limit if cell_token.retraced_count < limit: cell_token.retraced_count += 1 debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit)) else: # Try forcing boxes to avoid jumping to the preamble try: vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes, force_boxes=True) except InvalidLoop: pass if vs is None: return info, self.optimizer._newoperations[:] debug_print("Retrace count reached, jumping to preamble") return self.jump_to_preamble(cell_token, jump_op, info) exported_state = self.export_state(info.jump_op.getarglist(), info.inputargs, box_names_memo) exported_state.quasi_immutable_deps = self.optimizer.quasi_immutable_deps self.optimizer._clean_optimization_info(self.optimizer._newoperations) return exported_state, self.optimizer._newoperations def finalize_short_preamble(self, label_op, virtual_state): sb = self.short_preamble_producer self.optimizer._clean_optimization_info(sb.short_inputargs) short_preamble = sb.build_short_preamble() jitcelltoken = label_op.getdescr() assert isinstance(jitcelltoken, JitCellToken) if jitcelltoken.target_tokens is None: jitcelltoken.target_tokens = [] target_token = TargetToken(jitcelltoken, original_jitcell_token=jitcelltoken) target_token.original_jitcell_token = jitcelltoken target_token.virtual_state = virtual_state target_token.short_preamble = short_preamble jitcelltoken.target_tokens.append(target_token) self.short_preamble_producer = ExtendedShortPreambleBuilder( target_token, sb) label_op.initarglist(label_op.getarglist() + sb.used_boxes) return target_token def jump_to_preamble(self, cell_token, jump_op, info): assert cell_token.target_tokens[0].virtual_state is None jump_op = jump_op.copy_and_change(rop.JUMP, descr=cell_token.target_tokens[0]) self.optimizer.send_extra_operation(jump_op) return info, self.optimizer._newoperations[:] def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes, force_boxes=False): jitcelltoken = jump_op.getdescr() assert isinstance(jitcelltoken, JitCellToken) virtual_state = self.get_virtual_state(jump_op.getarglist()) args = [self.get_box_replacement(op) for op in jump_op.getarglist()] for target_token in jitcelltoken.target_tokens: target_virtual_state = target_token.virtual_state if target_virtual_state is None: continue try: extra_guards = target_virtual_state.generate_guards( virtual_state, args, runtime_boxes, self.optimizer, force_boxes=force_boxes) patchguardop = self.optimizer.patchguardop for guard in extra_guards.extra_guards: if isinstance(guard, GuardResOp): guard.rd_resume_position = patchguardop.rd_resume_position guard.setdescr(compile.ResumeAtPositionDescr()) self.send_extra_operation(guard) except VirtualStatesCantMatch: continue # When force_boxes == True, creating the virtual args can fail when # components of the virtual state alias. If this occurs, we must # recompute the virtual state as boxes will have been forced. try: args, virtuals = target_virtual_state.make_inputargs_and_virtuals( args, self.optimizer, force_boxes=force_boxes) except VirtualStatesCantMatch: assert force_boxes virtual_state = self.get_virtual_state(args) continue short_preamble = target_token.short_preamble try: extra = self.inline_short_preamble(args + virtuals, args, short_preamble, self.optimizer.patchguardop, target_token, label_op) except KeyError: # SHOULD NOT OCCUR BUT DOES: WHY?? issue #2185 self.optimizer.metainterp_sd.logger_ops.log_short_preamble([], short_preamble, {}) raise self.send_extra_operation(jump_op.copy_and_change(rop.JUMP, args=args + extra, descr=target_token)) return None # explicit because the return can be non-None return virtual_state def _map_args(self, mapping, arglist): result = [] for box in arglist: if not isinstance(box, Const): box = mapping[box] result.append(box) return result def inline_short_preamble(self, jump_args, args_no_virtuals, short, patchguardop, target_token, label_op): short_inputargs = short[0].getarglist() short_jump_args = short[-1].getarglist() sb = self.short_preamble_producer if sb is not None: assert isinstance(sb, ExtendedShortPreambleBuilder) if sb.target_token is target_token: # this means we're inlining the short preamble that's being # built. Make sure we modify the correct things in-place self.short_preamble_producer.setup(short_jump_args, short, label_op.getarglist()) # after this call, THE REST OF THIS FUNCTION WILL MODIFY ALL # THE LISTS PROVIDED, POTENTIALLY # We need to make a list of fresh new operations corresponding # to the short preamble operations. We could temporarily forward # the short operations to the fresh ones, but there are obscure # issues: send_extra_operation() below might occasionally invoke # use_box(), which assumes the short operations are not forwarded. # So we avoid such temporary forwarding and just use a dict here. assert len(short_inputargs) == len(jump_args) mapping = {} for i in range(len(jump_args)): mapping[short_inputargs[i]] = jump_args[i] # a fix-point loop, runs only once in almost all cases i = 1 while 1: self._check_no_forwarding([short_inputargs, short], False) while i < len(short) - 1: sop = short[i] arglist = self._map_args(mapping, sop.getarglist()) if sop.is_guard(): op = sop.copy_and_change(sop.getopnum(), arglist, descr=compile.ResumeAtPositionDescr()) assert isinstance(op, GuardResOp) op.rd_resume_position = patchguardop.rd_resume_position else: op = sop.copy_and_change(sop.getopnum(), arglist) mapping[sop] = op i += 1 self.optimizer.send_extra_operation(op) # force all of them except the virtuals for arg in (args_no_virtuals + self._map_args(mapping, short_jump_args)): self.optimizer.force_box(self.get_box_replacement(arg)) self.optimizer.flush() # done unless "short" has grown again if i == len(short) - 1: break return [self.get_box_replacement(box) for box in self._map_args(mapping, short_jump_args)] def _expand_info(self, arg, infos): if isinstance(arg, AbstractResOp) and rop.is_same_as(arg.opnum): info = self.optimizer.getinfo(arg.getarg(0)) else: info = self.optimizer.getinfo(arg) if arg in infos: return if info: infos[arg] = info if info.is_virtual(): self._expand_infos_from_virtual(info, infos) def _expand_infos_from_virtual(self, info, infos): items = info.all_items() for item in items: if item is None: continue self._expand_info(item, infos) def export_state(self, original_label_args, renamed_inputargs, memo): end_args = [self.optimizer.force_box_for_end_of_preamble(a) for a in original_label_args] self.optimizer.flush() virtual_state = self.get_virtual_state(end_args) end_args = [self.get_box_replacement(arg) for arg in end_args] infos = {} for arg in end_args: self._expand_info(arg, infos) label_args, virtuals = virtual_state.make_inputargs_and_virtuals( end_args, self.optimizer) for arg in label_args: self._expand_info(arg, infos) sb = ShortBoxes() short_boxes = sb.create_short_boxes(self.optimizer, renamed_inputargs, label_args + virtuals) short_inputargs = sb.create_short_inputargs(label_args + virtuals) for produced_op in short_boxes: op = produced_op.short_op.res if not isinstance(op, Const): self._expand_info(op, infos) return ExportedState(label_args, end_args, virtual_state, infos, short_boxes, renamed_inputargs, short_inputargs, memo) def import_state(self, targetargs, exported_state): # the mapping between input args (from old label) and what we need # to actually emit. Update the info assert (len(exported_state.next_iteration_args) == len(targetargs)) self._check_no_forwarding([targetargs]) exported_state._check_no_forwarding() for i, target in enumerate(exported_state.next_iteration_args): source = targetargs[i] assert source is not target source.set_forwarded(target) info = exported_state.exported_infos.get(target, None) if info is not None: self.optimizer.setinfo_from_preamble(source, info, exported_state.exported_infos) # import the optimizer state, starting from boxes that can be produced # by short preamble label_args = exported_state.virtual_state.make_inputargs( targetargs, self.optimizer) self.short_preamble_producer = ShortPreambleBuilder( label_args, exported_state.short_boxes, exported_state.short_inputargs, exported_state.exported_infos, self.optimizer) for produced_op in exported_state.short_boxes: produced_op.produce_op(self, exported_state.exported_infos) return label_args
class UnrollOptimizer(Optimization): """Unroll the loop into two iterations. The first one will become the preamble or entry bridge (don't think there is a distinction anymore)""" short_preamble_producer = None def __init__(self, metainterp_sd, jitdriver_sd, optimizations): self.optimizer = UnrollableOptimizer(metainterp_sd, jitdriver_sd, optimizations) self.optimizer.optunroll = self def get_virtual_state(self, args): modifier = VirtualStateConstructor(self.optimizer) return modifier.get_virtual_state(args) def _check_no_forwarding(self, lsts, check_newops=True): for lst in lsts: for op in lst: assert op.get_forwarded() is None if check_newops: assert not self.optimizer._newoperations def optimize_preamble(self, start_label, end_label, ops, call_pure_results, memo): self._check_no_forwarding([[start_label, end_label], ops]) info, newops = self.optimizer.propagate_all_forward( start_label.getarglist()[:], ops, call_pure_results, True, flush=False) exported_state = self.export_state(start_label, end_label.getarglist(), info.inputargs, memo) exported_state.quasi_immutable_deps = info.quasi_immutable_deps # we need to absolutely make sure that we've cleaned up all # the optimization info self.optimizer._clean_optimization_info(self.optimizer._newoperations) return exported_state, self.optimizer._newoperations def optimize_peeled_loop(self, start_label, end_jump, ops, state, call_pure_results, inline_short_preamble=True): self._check_no_forwarding([[start_label, end_jump], ops]) try: label_args = self.import_state(start_label, state) except VirtualStatesCantMatch: raise InvalidLoop("Cannot import state, virtual states don't match") self.potential_extra_ops = {} self.optimizer.init_inparg_dict_from(label_args) try: info, _ = self.optimizer.propagate_all_forward( start_label.getarglist()[:], ops, call_pure_results, False, flush=False) except SpeculativeError: raise InvalidLoop("Speculative heap access would be ill-typed") label_op = ResOperation(rop.LABEL, label_args, start_label.getdescr()) for a in end_jump.getarglist(): self.optimizer.force_box_for_end_of_preamble( self.optimizer.get_box_replacement(a)) current_vs = self.get_virtual_state(end_jump.getarglist()) # pick the vs we want to jump to celltoken = start_label.getdescr() assert isinstance(celltoken, JitCellToken) target_virtual_state = self.pick_virtual_state(current_vs, state.virtual_state, celltoken.target_tokens) # force the boxes for virtual state to match try: args = target_virtual_state.make_inputargs( [self.get_box_replacement(x) for x in end_jump.getarglist()], self.optimizer, force_boxes=True) for arg in args: self.optimizer.force_box(arg) except VirtualStatesCantMatch: raise InvalidLoop("Virtual states did not match " "after picking the virtual state, when forcing" " boxes") extra_same_as = self.short_preamble_producer.extra_same_as[:] target_token = self.finalize_short_preamble(label_op, state.virtual_state) label_op.setdescr(target_token) if not inline_short_preamble: self.jump_to_preamble(celltoken, end_jump, info) return (UnrollInfo(target_token, label_op, extra_same_as, self.optimizer.quasi_immutable_deps), self.optimizer._newoperations) try: new_virtual_state = self.jump_to_existing_trace(end_jump, label_op) except InvalidLoop: # inlining short preamble failed, jump to preamble self.jump_to_preamble(celltoken, end_jump, info) return (UnrollInfo(target_token, label_op, extra_same_as, self.optimizer.quasi_immutable_deps), self.optimizer._newoperations) if new_virtual_state is not None: self.jump_to_preamble(celltoken, end_jump, info) return (UnrollInfo(target_token, label_op, extra_same_as, self.optimizer.quasi_immutable_deps), self.optimizer._newoperations) self.disable_retracing_if_max_retrace_guards( self.optimizer._newoperations, target_token) return (UnrollInfo(target_token, label_op, extra_same_as, self.optimizer.quasi_immutable_deps), self.optimizer._newoperations) def disable_retracing_if_max_retrace_guards(self, ops, target_token): maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards count = 0 for op in ops: if op.is_guard(): count += 1 if count > maxguards: assert isinstance(target_token, TargetToken) target_token.targeting_jitcell_token.retraced_count = sys.maxint def pick_virtual_state(self, my_vs, label_vs, target_tokens): if target_tokens is None: return label_vs # for tests for token in target_tokens: if token.virtual_state is None: continue if token.virtual_state.generalization_of(my_vs, self.optimizer): return token.virtual_state return label_vs def optimize_bridge(self, start_label, operations, call_pure_results, inline_short_preamble, box_names_memo): self._check_no_forwarding([start_label.getarglist(), operations]) info, ops = self.optimizer.propagate_all_forward( start_label.getarglist()[:], operations[:-1], call_pure_results, True) jump_op = operations[-1] cell_token = jump_op.getdescr() assert isinstance(cell_token, JitCellToken) if not inline_short_preamble or len(cell_token.target_tokens) == 1: return self.jump_to_preamble(cell_token, jump_op, info) # force all the information that does not go to the short # preamble at all self.optimizer.flush() for a in jump_op.getarglist(): self.optimizer.force_box_for_end_of_preamble(a) try: vs = self.jump_to_existing_trace(jump_op, None) except InvalidLoop: return self.jump_to_preamble(cell_token, jump_op, info) if vs is None: return info, self.optimizer._newoperations[:] warmrunnerdescr = self.optimizer.metainterp_sd.warmrunnerdesc limit = warmrunnerdescr.memory_manager.retrace_limit if cell_token.retraced_count < limit: cell_token.retraced_count += 1 debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit)) else: debug_print("Retrace count reached, jumping to preamble") return self.jump_to_preamble(cell_token, jump_op, info) exported_state = self.export_state(start_label, operations[-1].getarglist(), info.inputargs, box_names_memo) exported_state.quasi_immutable_deps = self.optimizer.quasi_immutable_deps self.optimizer._clean_optimization_info(self.optimizer._newoperations) return exported_state, self.optimizer._newoperations def finalize_short_preamble(self, label_op, virtual_state): sb = self.short_preamble_producer self.optimizer._clean_optimization_info(sb.short_inputargs) short_preamble = sb.build_short_preamble() jitcelltoken = label_op.getdescr() assert isinstance(jitcelltoken, JitCellToken) if jitcelltoken.target_tokens is None: jitcelltoken.target_tokens = [] target_token = TargetToken(jitcelltoken, original_jitcell_token=jitcelltoken) target_token.original_jitcell_token = jitcelltoken target_token.virtual_state = virtual_state target_token.short_preamble = short_preamble jitcelltoken.target_tokens.append(target_token) self.short_preamble_producer = ExtendedShortPreambleBuilder( target_token, sb) label_op.initarglist(label_op.getarglist() + sb.used_boxes) return target_token def jump_to_preamble(self, cell_token, jump_op, info): assert cell_token.target_tokens[0].virtual_state is None jump_op = jump_op.copy_and_change(rop.JUMP, descr=cell_token.target_tokens[0]) self.optimizer.send_extra_operation(jump_op) return info, self.optimizer._newoperations[:] def jump_to_existing_trace(self, jump_op, label_op): jitcelltoken = jump_op.getdescr() assert isinstance(jitcelltoken, JitCellToken) virtual_state = self.get_virtual_state(jump_op.getarglist()) args = [self.get_box_replacement(op) for op in jump_op.getarglist()] for target_token in jitcelltoken.target_tokens: target_virtual_state = target_token.virtual_state if target_virtual_state is None: continue try: extra_guards = target_virtual_state.generate_guards( virtual_state, args, jump_op.getarglist(), self.optimizer) patchguardop = self.optimizer.patchguardop for guard in extra_guards.extra_guards: if isinstance(guard, GuardResOp): guard.rd_snapshot = patchguardop.rd_snapshot guard.rd_frame_info_list = patchguardop.rd_frame_info_list guard.setdescr(compile.ResumeAtPositionDescr()) self.send_extra_operation(guard) except VirtualStatesCantMatch: continue args, virtuals = target_virtual_state.make_inputargs_and_virtuals( args, self.optimizer) short_preamble = target_token.short_preamble try: extra = self.inline_short_preamble(args + virtuals, args, short_preamble, self.optimizer.patchguardop, target_token, label_op) except KeyError: # SHOULD NOT OCCUR BUT DOES: WHY?? issue #2185 self.optimizer.metainterp_sd.logger_ops.log_short_preamble([], short_preamble, {}) raise self.send_extra_operation(jump_op.copy_and_change(rop.JUMP, args=args + extra, descr=target_token)) return None # explicit because the return can be non-None return virtual_state def _map_args(self, mapping, arglist): result = [] for box in arglist: if not isinstance(box, Const): box = mapping[box] result.append(box) return result def inline_short_preamble(self, jump_args, args_no_virtuals, short, patchguardop, target_token, label_op): short_inputargs = short[0].getarglist() short_jump_args = short[-1].getarglist() sb = self.short_preamble_producer if sb is not None: assert isinstance(sb, ExtendedShortPreambleBuilder) if sb.target_token is target_token: # this means we're inlining the short preamble that's being # built. Make sure we modify the correct things in-place self.short_preamble_producer.setup(short_jump_args, short, label_op.getarglist()) # after this call, THE REST OF THIS FUNCTION WILL MODIFY ALL # THE LISTS PROVIDED, POTENTIALLY # We need to make a list of fresh new operations corresponding # to the short preamble operations. We could temporarily forward # the short operations to the fresh ones, but there are obscure # issues: send_extra_operation() below might occasionally invoke # use_box(), which assumes the short operations are not forwarded. # So we avoid such temporary forwarding and just use a dict here. assert len(short_inputargs) == len(jump_args) mapping = {} for i in range(len(jump_args)): mapping[short_inputargs[i]] = jump_args[i] # a fix-point loop, runs only once in almost all cases i = 1 while 1: self._check_no_forwarding([short_inputargs, short], False) while i < len(short) - 1: sop = short[i] arglist = self._map_args(mapping, sop.getarglist()) if sop.is_guard(): op = sop.copy_and_change(sop.getopnum(), arglist, descr=compile.ResumeAtPositionDescr()) assert isinstance(op, GuardResOp) op.rd_snapshot = patchguardop.rd_snapshot op.rd_frame_info_list = patchguardop.rd_frame_info_list else: op = sop.copy_and_change(sop.getopnum(), arglist) mapping[sop] = op i += 1 self.optimizer.send_extra_operation(op) # force all of them except the virtuals for arg in args_no_virtuals + short_jump_args: self.optimizer.force_box(self.get_box_replacement(arg)) self.optimizer.flush() # done unless "short" has grown again if i == len(short) - 1: break return [self.get_box_replacement(box) for box in self._map_args(mapping, short_jump_args)] def _expand_info(self, arg, infos): if isinstance(arg, AbstractResOp) and arg.is_same_as(): info = self.optimizer.getinfo(arg.getarg(0)) else: info = self.optimizer.getinfo(arg) if arg in infos: return if info: infos[arg] = info if info.is_virtual(): self._expand_infos_from_virtual(info, infos) def _expand_infos_from_virtual(self, info, infos): items = info.all_items() for item in items: if item is None: continue self._expand_info(item, infos) def export_state(self, start_label, original_label_args, renamed_inputargs, memo): end_args = [self.optimizer.force_box_for_end_of_preamble(a) for a in original_label_args] self.optimizer.flush() virtual_state = self.get_virtual_state(end_args) end_args = [self.get_box_replacement(arg) for arg in end_args] infos = {} for arg in end_args: self._expand_info(arg, infos) label_args, virtuals = virtual_state.make_inputargs_and_virtuals( end_args, self.optimizer) for arg in label_args: self._expand_info(arg, infos) sb = ShortBoxes() short_boxes = sb.create_short_boxes(self.optimizer, renamed_inputargs, label_args + virtuals) short_inputargs = sb.create_short_inputargs(label_args + virtuals) for produced_op in short_boxes: op = produced_op.short_op.res if not isinstance(op, Const): self._expand_info(op, infos) self.optimizer._clean_optimization_info(end_args) self.optimizer._clean_optimization_info(start_label.getarglist()) return ExportedState(label_args, end_args, virtual_state, infos, short_boxes, renamed_inputargs, short_inputargs, memo) def import_state(self, targetop, exported_state): # the mapping between input args (from old label) and what we need # to actually emit. Update the info assert (len(exported_state.next_iteration_args) == len(targetop.getarglist())) for i, target in enumerate(exported_state.next_iteration_args): source = targetop.getarg(i) assert source is not target source.set_forwarded(target) info = exported_state.exported_infos.get(target, None) if info is not None: self.optimizer.setinfo_from_preamble(source, info, exported_state.exported_infos) # import the optimizer state, starting from boxes that can be produced # by short preamble label_args = exported_state.virtual_state.make_inputargs( targetop.getarglist(), self.optimizer) self.short_preamble_producer = ShortPreambleBuilder( label_args, exported_state.short_boxes, exported_state.short_inputargs, exported_state.exported_infos, self.optimizer) for produced_op in exported_state.short_boxes: produced_op.produce_op(self, exported_state.exported_infos) return label_args
class OptUnroll(Optimization): """Unroll the loop into two iterations. The first one will become the preamble or entry bridge (don't think there is a distinction anymore)""" short_preamble_producer = None def get_virtual_state(self, args): modifier = VirtualStateConstructor(self.optimizer) return modifier.get_virtual_state(args) def _check_no_forwarding(self, lsts, check_newops=True): for lst in lsts: for op in lst: assert op.get_forwarded() is None if check_newops: assert not self.optimizer._newoperations def disable_retracing_if_max_retrace_guards(self, ops, target_token): maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards count = 0 for op in ops: if op.is_guard(): count += 1 if count > maxguards: assert isinstance(target_token, TargetToken) target_token.targeting_jitcell_token.retraced_count = sys.maxint def pick_virtual_state(self, my_vs, label_vs, target_tokens): if target_tokens is None: return label_vs # for tests for token in target_tokens: if token.virtual_state is None: continue if token.virtual_state.generalization_of(my_vs, self.optimizer): return token.virtual_state return label_vs def finalize_short_preamble(self, label_op, virtual_state): sb = self.short_preamble_producer self.optimizer._clean_optimization_info(sb.short_inputargs) short_preamble = sb.build_short_preamble() jitcelltoken = label_op.getdescr() assert isinstance(jitcelltoken, JitCellToken) if jitcelltoken.target_tokens is None: jitcelltoken.target_tokens = [] target_token = TargetToken(jitcelltoken, original_jitcell_token=jitcelltoken) target_token.original_jitcell_token = jitcelltoken target_token.virtual_state = virtual_state target_token.short_preamble = short_preamble jitcelltoken.target_tokens.append(target_token) self.short_preamble_producer = ExtendedShortPreambleBuilder( target_token, sb) label_op.initarglist(label_op.getarglist() + sb.used_boxes) return target_token def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes, force_boxes=False): # there is a big conceptual problem here: it's not safe at all to catch # InvalidLoop in the callers of _jump_to_existing_trace and then # continue trying to jump to some other label, because inlining the # short preamble could have worked partly, leaving some unwanted new # ops at the end of the trace. Here's at least a stopgap to stop # terrible things from happening: we *must not* move any of those bogus # guards earlier into the trace. see # test_unroll_shortpreamble_mutates_bug in test_loop, and issue #3598 # leaving the bogus operations at the end of the trace is not great, # but should be safe: at worst, they just always do a bit of stuff and # then fail with self.optimizer.cant_replace_guards(): return self._jump_to_existing_trace(jump_op, label_op, runtime_boxes, force_boxes) def _jump_to_existing_trace(self, jump_op, label_op, runtime_boxes, force_boxes=False): jitcelltoken = jump_op.getdescr() assert isinstance(jitcelltoken, JitCellToken) virtual_state = self.get_virtual_state(jump_op.getarglist()) args = [get_box_replacement(op) for op in jump_op.getarglist()] for target_token in jitcelltoken.target_tokens: target_virtual_state = target_token.virtual_state if target_virtual_state is None: continue try: extra_guards = target_virtual_state.generate_guards( virtual_state, args, runtime_boxes, self.optimizer, force_boxes=force_boxes) patchguardop = self.optimizer.patchguardop for guard in extra_guards.extra_guards: if isinstance(guard, GuardResOp): guard.rd_resume_position = patchguardop.rd_resume_position guard.setdescr(compile.ResumeAtPositionDescr()) self.optimizer.send_extra_operation(guard) except VirtualStatesCantMatch: continue # When force_boxes == True, creating the virtual args can fail when # components of the virtual state alias. If this occurs, we must # recompute the virtual state as boxes will have been forced. try: args, virtuals = target_virtual_state.make_inputargs_and_virtuals( args, self.optimizer, force_boxes=force_boxes) except VirtualStatesCantMatch: assert force_boxes virtual_state = self.get_virtual_state(args) continue short_preamble = target_token.short_preamble extra = self.inline_short_preamble(args + virtuals, args, short_preamble, self.optimizer.patchguardop, target_token, label_op) self.optimizer.send_extra_operation( jump_op.copy_and_change(rop.JUMP, args=args + extra, descr=target_token)) return None # explicit because the return can be non-None return virtual_state def _map_args(self, mapping, arglist): result = [] for box in arglist: if not isinstance(box, Const): box = mapping[box] result.append(box) return result def inline_short_preamble(self, jump_args, args_no_virtuals, short, patchguardop, target_token, label_op): short_inputargs = short[0].getarglist() short_jump_args = short[-1].getarglist() sb = self.short_preamble_producer if sb is not None: assert isinstance(sb, ExtendedShortPreambleBuilder) if sb.target_token is target_token: # this means we're inlining the short preamble that's being # built. Make sure we modify the correct things in-place self.short_preamble_producer.setup(short_jump_args, short, label_op.getarglist()) # after this call, THE REST OF THIS FUNCTION WILL MODIFY ALL # THE LISTS PROVIDED, POTENTIALLY # We need to make a list of fresh new operations corresponding # to the short preamble operations. We could temporarily forward # the short operations to the fresh ones, but there are obscure # issues: send_extra_operation() below might occasionally invoke # use_box(), which assumes the short operations are not forwarded. # So we avoid such temporary forwarding and just use a dict here. assert len(short_inputargs) == len(jump_args) mapping = {} for i in range(len(jump_args)): mapping[short_inputargs[i]] = jump_args[i] # a fix-point loop, runs only once in almost all cases i = 1 while 1: self._check_no_forwarding([short_inputargs, short], False) while i < len(short) - 1: sop = short[i] arglist = self._map_args(mapping, sop.getarglist()) if sop.is_guard(): op = sop.copy_and_change( sop.getopnum(), arglist, descr=compile.ResumeAtPositionDescr()) assert isinstance(op, GuardResOp) op.rd_resume_position = patchguardop.rd_resume_position else: op = sop.copy_and_change(sop.getopnum(), arglist) mapping[sop] = op i += 1 self.optimizer.send_extra_operation(op) # force all of them except the virtuals for arg in (args_no_virtuals + self._map_args(mapping, short_jump_args)): self.optimizer.force_box(get_box_replacement(arg)) self.optimizer.flush() # done unless "short" has grown again if i == len(short) - 1: break return [ get_box_replacement(box) for box in self._map_args(mapping, short_jump_args) ] def _expand_info(self, arg, infos): arg1 = self.optimizer.as_operation(arg) if arg1 is not None and rop.is_same_as(arg1.opnum): info = self.optimizer.getinfo(arg1.getarg(0)) else: info = self.optimizer.getinfo(arg) if arg in infos: return if info: infos[arg] = info if info.is_virtual(): self._expand_infos_from_virtual(info, infos) def _expand_infos_from_virtual(self, info, infos): items = info.all_items() for item in items: if item is None: continue self._expand_info(item, infos) def export_state(self, original_label_args, renamed_inputargs, runtime_boxes, memo): end_args = [ self.optimizer.force_box_for_end_of_preamble(a) for a in original_label_args ] self.optimizer.flush() virtual_state = self.get_virtual_state(end_args) end_args = [get_box_replacement(arg) for arg in end_args] infos = {} for arg in end_args: self._expand_info(arg, infos) label_args, virtuals = virtual_state.make_inputargs_and_virtuals( end_args, self.optimizer) for arg in label_args: self._expand_info(arg, infos) sb = ShortBoxes() short_boxes = sb.create_short_boxes(self.optimizer, renamed_inputargs, label_args + virtuals) short_inputargs = sb.create_short_inputargs(label_args + virtuals) for produced_op in short_boxes: op = produced_op.short_op.res if not isinstance(op, Const): self._expand_info(op, infos) self.optimizer._clean_optimization_info(end_args) return ExportedState(label_args, end_args, virtual_state, infos, short_boxes, renamed_inputargs, short_inputargs, runtime_boxes, memo) def import_state(self, targetargs, exported_state): # the mapping between input args (from old label) and what we need # to actually emit. Update the info assert len(exported_state.next_iteration_args) == len(targetargs) for i, target in enumerate(exported_state.next_iteration_args): source = targetargs[i] assert source is not target source.set_forwarded(target) info = exported_state.exported_infos.get(target, None) if info is not None: self.optimizer.setinfo_from_preamble( source, info, exported_state.exported_infos) # import the optimizer state, starting from boxes that can be produced # by short preamble label_args = exported_state.virtual_state.make_inputargs( targetargs, self.optimizer) self.short_preamble_producer = ShortPreambleBuilder( label_args, exported_state.short_boxes, exported_state.short_inputargs, exported_state.exported_infos, self.optimizer) for produced_op in exported_state.short_boxes: produced_op.produce_op(self, exported_state.exported_infos) return label_args