def ensure_operations(self, opstrlist, trace): oparse = OpParser('', self.cpu, self.namespace, None, None, True, None) oplist = [] for op_str in opstrlist: op = oparse.parse_next_op(op_str) if not op.returns_void(): var = op_str.split('=')[0].strip() if '[' in var: var = var[:var.find('[')] elem = op_str[:len(var)] oparse._cache['lltype', elem] = op oplist.append(op) oplist_i = 0 remap = {} last_match = 0 for i, op in enumerate(trace.operations): if oplist_i >= len(oplist): break curtomatch = oplist[oplist_i] if self.match_op(curtomatch, op, remap): if not op.returns_void(): remap[curtomatch] = op oplist_i += 1 last_match = i msg = "could not find all ops in the trace sequence\n\n" if oplist_i != len(oplist): l = [str(o) for o in oplist[oplist_i:]] msg += "sequence\n " + '\n '.join(l) msg += "\n\ndoes not match\n " l = [str(o) for o in trace.operations[last_match + 1:]] msg += '\n '.join(l) assert oplist_i == len(oplist), msg
def ensure_operations(self, opstrlist, trace, inthatorder=True): oparse = OpParser('', self.cpu, self.namespace, None, None, True, None) oplist = [] for op_str in opstrlist: op = oparse.parse_next_op(op_str) if not op.returns_void(): var = op_str.split('=')[0].strip() if '[' in var: var = var[:var.find('[')] elem = op_str[:len(var)] oparse._cache['lltype', elem] = op oplist.append(op) oplist_i = 0 match = False remap = {} last_match = 0 for i, op in enumerate(trace.operations): if oplist_i >= len(oplist): break curtomatch = oplist[oplist_i] if self.match_op(curtomatch, op, remap): if not op.returns_void(): remap[curtomatch] = op oplist_i += 1 last_match = i msg = "could not find all ops in the trace sequence\n\n" if oplist_i != len(oplist): l = [str(o) for o in oplist[oplist_i:]] msg += "sequence\n " + '\n '.join(l) msg += "\n\ndoes not match\n " l = [str(o) for o in trace.operations[last_match+1:]] msg += '\n '.join(l) assert oplist_i == len(oplist), msg
def assert_contains_sequence(self, loop, instr): class Glob(object): next = None prev = None def __repr__(self): return '*' from rpython.jit.tool.oparser import OpParser, default_fail_descr parser = OpParser(instr, self.cpu, self.namespace, None, default_fail_descr, True, None) parser.vars = { arg.repr_short(arg._repr_memo) : arg for arg in loop.inputargs} operations = [] last_glob = None prev_op = None for line in instr.splitlines(): line = line.strip() if line.startswith("#") or \ line == "": continue if line.startswith("..."): last_glob = Glob() last_glob.prev = prev_op operations.append(last_glob) continue op = parser.parse_next_op(line) if last_glob is not None: last_glob.next = op last_glob = None operations.append(op) def check(op, candidate, rename): m = 0 if isinstance(candidate, Glob): if candidate.next is None: return 0 # consumes the rest if op.getopnum() != candidate.next.getopnum(): return 0 m = 1 candidate = candidate.next if op.getopnum() == candidate.getopnum(): for i,arg in enumerate(op.getarglist()): oarg = candidate.getarg(i) if arg in rename: assert rename[arg].same_box(oarg) else: rename[arg] = oarg if not op.returns_void(): rename[op] = candidate m += 1 return m return 0 j = 0 rename = {} ops = loop.finaloplist() for i, op in enumerate(ops): candidate = operations[j] j += check(op, candidate, rename) if isinstance(operations[-1], Glob): assert j == len(operations)-1, self.debug_print_operations(loop) else: assert j == len(operations), self.debug_print_operations(loop)
def parse(self, s, boxkinds=None, want_fail_descr=True, postprocess=None): AbstractValue._repr_memo.counter = 0 self.oparse = OpParser(s, self.cpu, self.namespace, boxkinds, None, False, postprocess) return self.oparse.parse()
class BaseTest(LLtypeMixin): @pytest.fixture(autouse=True) def cls_attributes(self): metainterp_sd = FakeMetaInterpStaticData(self.cpu) metainterp_sd.virtualref_info = self.vrefinfo compute_bitstrings(self.cpu.fetch_all_descrs()) self.metainterp_sd = metainterp_sd def parse(self, s, boxkinds=None, want_fail_descr=True, postprocess=None): AbstractValue._repr_memo.counter = 0 self.oparse = OpParser(s, self.cpu, self.namespace, boxkinds, None, False, postprocess) return self.oparse.parse() def add_guard_future_condition(self, res): # invent a GUARD_FUTURE_CONDITION to not have to change all tests if res.operations[-1].getopnum() == rop.JUMP: guard = ResOperation(rop.GUARD_FUTURE_CONDITION, []) res.operations.insert(-1, guard) @staticmethod def assert_equal(optimized, expected, text_right=None): assert len(optimized.inputargs) == len(expected.inputargs) remap = {} for box1, box2 in zip(optimized.inputargs, expected.inputargs): assert box1.type == box2.type remap[box2] = box1 assert equaloplists(optimized.operations, expected.operations, False, remap, text_right) def _convert_call_pure_results(self, d): if d is None: return call_pure_results = args_dict() for k, v in d.items(): call_pure_results[list(k)] = v return call_pure_results def convert_values(self, inpargs, values): if values: r = [] for arg, v in zip(inpargs, values): if arg.type == 'i': n = IntFrontendOp(0) if v is not None: n.setint(v) else: n = RefFrontendOp(0) if v is not None: n.setref_base(v) assert arg.type == 'r' r.append(n) return r return inpargs def unroll_and_optimize(self, loop, call_pure_results=None, jump_values=None): self.add_guard_future_condition(loop) jump_op = loop.operations[-1] assert jump_op.getopnum() == rop.JUMP celltoken = JitCellToken() runtime_boxes = self.pack_into_boxes(jump_op, jump_values) jump_op.setdescr(celltoken) call_pure_results = self._convert_call_pure_results(call_pure_results) t = convert_loop_to_trace(loop, self.metainterp_sd) preamble_data = compile.PreambleCompileData( t, runtime_boxes, call_pure_results, enable_opts=self.enable_opts) start_state, preamble_ops = preamble_data.optimize_trace( self.metainterp_sd, None, {}) preamble_data.forget_optimization_info() loop_data = compile.UnrolledLoopData(preamble_data.trace, celltoken, start_state, call_pure_results, enable_opts=self.enable_opts) loop_info, ops = loop_data.optimize_trace(self.metainterp_sd, None, {}) preamble = TreeLoop('preamble') preamble.inputargs = start_state.renamed_inputargs start_label = ResOperation(rop.LABEL, start_state.renamed_inputargs) preamble.operations = ([start_label] + preamble_ops + loop_info.extra_same_as + [loop_info.label_op]) loop.inputargs = loop_info.label_op.getarglist()[:] loop.operations = [loop_info.label_op] + ops return Info(preamble, loop_info.target_token.short_preamble, start_state.virtual_state) def pack_into_boxes(self, jump_op, jump_values): assert jump_op.getopnum() == rop.JUMP r = [] if jump_values is not None: assert len(jump_values) == len(jump_op.getarglist()) for i, v in enumerate(jump_values): if v is not None: r.append(InputArgRef(v)) else: r.append(None) else: for i, box in enumerate(jump_op.getarglist()): if box.type == 'r' and not box.is_constant(): # NOTE: we arbitrarily set the box contents to a NODE2 # object here. If you need something different, you # need to pass a 'jump_values' argument to e.g. # optimize_loop() r.append(InputArgRef(self.nodefulladdr)) else: r.append(None) return r
class BaseTest(object): def parse(self, s, boxkinds=None, want_fail_descr=True, postprocess=None): AbstractValue._repr_memo.counter = 0 self.oparse = OpParser(s, self.cpu, self.namespace, boxkinds, None, False, postprocess) return self.oparse.parse() def postprocess(self, op): class FakeJitCode(object): index = 0 if op.is_guard(): op.rd_snapshot = resume.TopSnapshot( resume.Snapshot(None, op.getfailargs()), [], []) op.rd_frame_info_list = resume.FrameInfo(None, FakeJitCode(), 11) def add_guard_future_condition(self, res): # invent a GUARD_FUTURE_CONDITION to not have to change all tests if res.operations[-1].getopnum() == rop.JUMP: guard = ResOperation(rop.GUARD_FUTURE_CONDITION, [], None) guard.rd_snapshot = resume.TopSnapshot(None, [], []) res.operations.insert(-1, guard) def assert_equal(self, optimized, expected, text_right=None): from rpython.jit.metainterp.optimizeopt.util import equaloplists assert len(optimized.inputargs) == len(expected.inputargs) remap = {} for box1, box2 in zip(optimized.inputargs, expected.inputargs): assert box1.type == box2.type remap[box2] = box1 assert equaloplists(optimized.operations, expected.operations, False, remap, text_right) def _do_optimize_loop(self, compile_data): from rpython.jit.metainterp.optimizeopt import optimize_trace metainterp_sd = FakeMetaInterpStaticData(self.cpu) if hasattr(self, 'vrefinfo'): metainterp_sd.virtualref_info = self.vrefinfo if hasattr(self, 'callinfocollection'): metainterp_sd.callinfocollection = self.callinfocollection # compile_data.enable_opts = self.enable_opts state = optimize_trace(metainterp_sd, None, compile_data) return state def _convert_call_pure_results(self, d): from rpython.jit.metainterp.optimizeopt.util import args_dict if d is None: return call_pure_results = args_dict() for k, v in d.items(): call_pure_results[list(k)] = v return call_pure_results def unroll_and_optimize(self, loop, call_pure_results=None): self.add_guard_future_condition(loop) jump_op = loop.operations[-1] assert jump_op.getopnum() == rop.JUMP ops = loop.operations[:-1] jump_op.setdescr(JitCellToken()) start_label = ResOperation(rop.LABEL, loop.inputargs, jump_op.getdescr()) end_label = jump_op.copy_and_change(opnum=rop.LABEL) call_pure_results = self._convert_call_pure_results(call_pure_results) preamble_data = compile.LoopCompileData(start_label, end_label, ops, call_pure_results) start_state, preamble_ops = self._do_optimize_loop(preamble_data) preamble_data.forget_optimization_info() loop_data = compile.UnrolledLoopData(start_label, jump_op, ops, start_state, call_pure_results) loop_info, ops = self._do_optimize_loop(loop_data) preamble = TreeLoop('preamble') preamble.inputargs = start_state.renamed_inputargs start_label = ResOperation(rop.LABEL, start_state.renamed_inputargs) preamble.operations = ([start_label] + preamble_ops + loop_info.extra_same_as + [loop_info.label_op]) loop.inputargs = loop_info.label_op.getarglist()[:] loop.operations = [loop_info.label_op] + ops return Info(preamble, loop_info.target_token.short_preamble, start_state.virtual_state) def set_values(self, ops, jump_values=None): jump_op = ops[-1] assert jump_op.getopnum() == rop.JUMP if jump_values is not None: for i, v in enumerate(jump_values): if v is not None: jump_op.getarg(i).setref_base(v) else: for i, box in enumerate(jump_op.getarglist()): if box.type == 'r' and not box.is_constant(): # NOTE: we arbitrarily set the box contents to a NODE2 # object here. If you need something different, you # need to pass a 'jump_values' argument to e.g. # optimize_loop() box.setref_base(self.nodefulladdr)
def parse(self, s, boxkinds=None, want_fail_descr=True, postprocess=None): self.oparse = OpParser(s, self.cpu, self.namespace, 'lltype', boxkinds, None, False, postprocess) return self.oparse.parse()
class BaseTest(object): def parse(self, s, boxkinds=None, want_fail_descr=True, postprocess=None): self.oparse = OpParser(s, self.cpu, self.namespace, 'lltype', boxkinds, None, False, postprocess) return self.oparse.parse() def postprocess(self, op): if op.is_guard(): op.rd_snapshot = resume.Snapshot(None, op.getfailargs()) op.rd_frame_info_list = resume.FrameInfo(None, "code", 11) def add_guard_future_condition(self, res): # invent a GUARD_FUTURE_CONDITION to not have to change all tests if res.operations[-1].getopnum() == rop.JUMP: guard = ResOperation(rop.GUARD_FUTURE_CONDITION, [], None) guard.rd_snapshot = resume.Snapshot(None, []) res.operations.insert(-1, guard) def assert_equal(self, optimized, expected, text_right=None): from rpython.jit.metainterp.optimizeopt.util import equaloplists assert len(optimized.inputargs) == len(expected.inputargs) remap = {} for box1, box2 in zip(optimized.inputargs, expected.inputargs): assert box1.__class__ == box2.__class__ remap[box2] = box1 assert equaloplists(optimized.operations, expected.operations, False, remap, text_right) def _do_optimize_loop(self, loop, call_pure_results, start_state=None, export_state=False): from rpython.jit.metainterp.optimizeopt import optimize_trace from rpython.jit.metainterp.optimizeopt.util import args_dict self.loop = loop loop.call_pure_results = args_dict() if call_pure_results is not None: for k, v in call_pure_results.items(): loop.call_pure_results[list(k)] = v metainterp_sd = FakeMetaInterpStaticData(self.cpu) if hasattr(self, 'vrefinfo'): metainterp_sd.virtualref_info = self.vrefinfo if hasattr(self, 'callinfocollection'): metainterp_sd.callinfocollection = self.callinfocollection # return optimize_trace(metainterp_sd, None, loop, self.enable_opts, start_state=start_state, export_state=export_state) def unroll_and_optimize(self, loop, call_pure_results=None): self.add_guard_future_condition(loop) operations = loop.operations jumpop = operations[-1] assert jumpop.getopnum() == rop.JUMP inputargs = loop.inputargs jump_args = jumpop.getarglist()[:] operations = operations[:-1] cloned_operations = [op.clone() for op in operations] preamble = TreeLoop('preamble') preamble.inputargs = inputargs token = JitCellToken() preamble.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(token))] + \ operations + \ [ResOperation(rop.LABEL, jump_args, None, descr=token)] start_state = self._do_optimize_loop(preamble, call_pure_results, export_state=True) assert preamble.operations[-1].getopnum() == rop.LABEL inliner = Inliner(inputargs, jump_args) loop.operations = [preamble.operations[-1]] + \ [inliner.inline_op(op, clone=False) for op in cloned_operations] + \ [ResOperation(rop.JUMP, [inliner.inline_arg(a) for a in jump_args], None, descr=token)] #[inliner.inline_op(jumpop)] assert loop.operations[-1].getopnum() == rop.JUMP assert loop.operations[0].getopnum() == rop.LABEL loop.inputargs = loop.operations[0].getarglist() self._do_optimize_loop(loop, call_pure_results, start_state, export_state=False) extra_same_as = [] while loop.operations[0].getopnum() != rop.LABEL: extra_same_as.append(loop.operations[0]) del loop.operations[0] # Hack to prevent random order of same_as ops extra_same_as.sort(key=lambda op: str(preamble.operations).find(str(op.getarg(0)))) for op in extra_same_as: preamble.operations.insert(-1, op) return preamble
class BaseTest(object): def parse(self, s, boxkinds=None, want_fail_descr=True, postprocess=None): AbstractValue._repr_memo.counter = 0 self.oparse = OpParser(s, self.cpu, self.namespace, boxkinds, None, False, postprocess) return self.oparse.parse() def add_guard_future_condition(self, res): # invent a GUARD_FUTURE_CONDITION to not have to change all tests if res.operations[-1].getopnum() == rop.JUMP: guard = ResOperation(rop.GUARD_FUTURE_CONDITION, []) res.operations.insert(-1, guard) @staticmethod def assert_equal(optimized, expected, text_right=None): from rpython.jit.metainterp.optimizeopt.util import equaloplists assert len(optimized.inputargs) == len(expected.inputargs) remap = {} for box1, box2 in zip(optimized.inputargs, expected.inputargs): assert box1.type == box2.type remap[box2] = box1 assert equaloplists(optimized.operations, expected.operations, False, remap, text_right) def _do_optimize_loop(self, compile_data): from rpython.jit.metainterp.optimizeopt import optimize_trace metainterp_sd = FakeMetaInterpStaticData(self.cpu) if hasattr(self, 'vrefinfo'): metainterp_sd.virtualref_info = self.vrefinfo if hasattr(self, 'callinfocollection'): metainterp_sd.callinfocollection = self.callinfocollection # compile_data.enable_opts = self.enable_opts state = optimize_trace(metainterp_sd, None, compile_data) return state def _convert_call_pure_results(self, d): from rpython.jit.metainterp.optimizeopt.util import args_dict if d is None: return call_pure_results = args_dict() for k, v in d.items(): call_pure_results[list(k)] = v return call_pure_results def convert_values(self, inpargs, values): from rpython.jit.metainterp.history import IntFrontendOp, RefFrontendOp if values: r = [] for arg, v in zip(inpargs, values): if arg.type == 'i': n = IntFrontendOp(0) if v is not None: n.setint(v) else: n = RefFrontendOp(0) if v is not None: n.setref_base(v) assert arg.type == 'r' r.append(n) return r return inpargs def unroll_and_optimize(self, loop, call_pure_results=None, jump_values=None): self.add_guard_future_condition(loop) jump_op = loop.operations[-1] assert jump_op.getopnum() == rop.JUMP celltoken = JitCellToken() runtime_boxes = self.pack_into_boxes(jump_op, jump_values) jump_op.setdescr(celltoken) #start_label = ResOperation(rop.LABEL, loop.inputargs, # descr=jump_op.getdescr()) #end_label = jump_op.copy_and_change(opnum=rop.LABEL) call_pure_results = self._convert_call_pure_results(call_pure_results) t = convert_loop_to_trace(loop, FakeMetaInterpStaticData(self.cpu)) preamble_data = compile.LoopCompileData(t, runtime_boxes, call_pure_results) start_state, preamble_ops = self._do_optimize_loop(preamble_data) preamble_data.forget_optimization_info() loop_data = compile.UnrolledLoopData(preamble_data.trace, celltoken, start_state, call_pure_results) loop_info, ops = self._do_optimize_loop(loop_data) preamble = TreeLoop('preamble') preamble.inputargs = start_state.renamed_inputargs start_label = ResOperation(rop.LABEL, start_state.renamed_inputargs) preamble.operations = ([start_label] + preamble_ops + loop_info.extra_same_as + [loop_info.label_op]) loop.inputargs = loop_info.label_op.getarglist()[:] loop.operations = [loop_info.label_op] + ops return Info(preamble, loop_info.target_token.short_preamble, start_state.virtual_state) def pack_into_boxes(self, jump_op, jump_values): assert jump_op.getopnum() == rop.JUMP r = [] if jump_values is not None: assert len(jump_values) == len(jump_op.getarglist()) for i, v in enumerate(jump_values): if v is not None: r.append(InputArgRef(v)) else: r.append(None) else: for i, box in enumerate(jump_op.getarglist()): if box.type == 'r' and not box.is_constant(): # NOTE: we arbitrarily set the box contents to a NODE2 # object here. If you need something different, you # need to pass a 'jump_values' argument to e.g. # optimize_loop() r.append(InputArgRef(self.nodefulladdr)) else: r.append(None) return r