def test_to_json_meta_bridges(): forest = TraceForest(1) forest.resops = {15: 'guard_true'} trunk = forest.add_trace('loop', 0, 0) bridge1 = forest.add_trace('bridge', 1, 0) bridge2 = forest.add_trace('bridge', 2, 0) bridge3 = forest.add_trace('bridge', 3, 0) forest.descr_nmr_to_point_in_trace[10] = PointInTrace(trunk, None) forest.descr_nmr_to_point_in_trace[11] = PointInTrace(trunk, None) forest.descr_nmr_to_point_in_trace[12] = PointInTrace(bridge1, None) forest.descr_nmr_to_point_in_trace[13] = PointInTrace(bridge2, None) # trunk.set_addr_bounds(99, 100) bridge1.set_addr_bounds(100, 101) bridge2.set_addr_bounds(200, 201) bridge3.set_addr_bounds(300, 301) # forest.stitch_bridge(10, 100) forest.stitch_bridge(11, 200) forest.stitch_bridge(12, 300) # stage = trunk.start_mark(const.MARK_TRACE_OPT) j = LogMetaSerializer().to_representation(FakeJitLog(forest)) assert len(j['bridges']) == 4 bridges = j['bridges'] assert bridges[0] == {10: 100, 11: 200} assert bridges[1] == {12: 300} assert bridges[2] == {} assert bridges[3] == {}
def test_read_jitlog_counter(): forest = TraceForest(1) ta = forest.add_trace('loop', 1, 0) ta.start_mark(const.MARK_TRACE_ASM) op = FlatOp(0, 'hello', '', '?', 0, 2) ta.add_instr(op) op2 = FlatOp(0, 'increment_debug_counter', '', '?', 0, 2) ta.add_instr(op2) tb = forest.add_trace('bridge', 22, 101) fw = FileObjWrapper(FileObj([encode_le_u64(0x0), b'l', encode_le_u64(20)])) assert marks.read_jitlog_counter(ParseContext(forest), None, fw) == False, \ "must not find trace" fw = FileObjWrapper( FileObj([ encode_le_u64(1), b'e', encode_le_u64(145), encode_le_u64(2), b'l', encode_le_u64(45), encode_le_u64(22), b'b', encode_le_u64(100), ])) # read the entry, the label, and the bridge assert marks.read_jitlog_counter(ParseContext(forest), None, fw) == True assert marks.read_jitlog_counter(ParseContext(forest), None, fw) == True assert marks.read_jitlog_counter(ParseContext(forest), None, fw) == True assert ta.counter == 145 assert ta.point_counters[1] == 45 assert tb.counter == 100
def test_v3_redirect_assembler(): # prepare a forest that already got two traces, # the first with call assembler and the target of # call assembler already included. then a read mark # redirect assembler is emulated. forest = TraceForest(3) trace = forest.add_trace('loop', 0, 0) trace.start_mark(const.MARK_TRACE_ASM) op = FlatOp(0, 'call_assembler_i', '', 'i0', 0, 15) trace.add_instr(op) # trace2 = forest.add_trace('loop', 16, 0) trace2.start_mark(const.MARK_TRACE_ASM) trace2.set_addr_bounds(42, 44) # fobj = FileObj([ const.MARK_REDIRECT_ASSEMBLER, encode_le_u64(15), encode_le_u64(17), encode_le_u64(16), ]) fw = FileObjWrapper(fobj) forest = construct_forest(fw, forest=forest) asm = forest.get_trace(16) parent = forest.get_trace(0) assert asm.get_parent() == parent assert len(parent.links) == 1
def test_to_json_meta_links(): forest = TraceForest(1) forest.resops = { 15: 'guard_true' } trunk = forest.add_trace('loop', 0, 0) bridge1 = forest.add_trace('bridge', 1, 0) bridge2 = forest.add_trace('bridge', 2, 0) bridge3 = forest.add_trace('bridge', 3, 0) op = FlatOp(0,'',[],None,0,0) op.index = 42 forest.descr_nmr_to_point_in_trace[10] = PointInTrace(trunk, op) forest.descr_nmr_to_point_in_trace[11] = PointInTrace(trunk, None) forest.descr_nmr_to_point_in_trace[12] = PointInTrace(bridge1, None) forest.descr_nmr_to_point_in_trace[13] = PointInTrace(bridge2, None) # trunk.set_addr_bounds(99,100) bridge1.set_addr_bounds(100,101) bridge2.set_addr_bounds(200,201) bridge3.set_addr_bounds(300,301) # forest.stitch_bridge(10, 100) forest.stitch_bridge(11, 200) forest.stitch_bridge(12, 300) # stage = trunk.start_mark(const.MARK_TRACE_OPT) j = LogMetaSerializer().to_representation(forest) assert len(j['links']) == 2 links = j['links'] assert links[0] == {42: 1, 0: 2} assert links[1] == {0: 3}
def test_func_filter(self): from jitlog.query import FuncFilter as func f = TraceForest(3, is_32bit=False, machine='s390x') loop = f.add_trace('loop', 0, 0, 'su') stage = loop.start_mark(c.MARK_TRACE) stage.append_op(MergePoint({ c.MP_SCOPE[0]: '_sake_in_a_glass' })) assert not func('hello')._filter(loop) assert func('sake')._filter(loop) assert not func('s_a_k_e')._filter(loop)
def test_query_loops_and_bridges(self): f = TraceForest(3, is_32bit=False, machine='s390x') # t = f.add_trace('loop', 0, 0, 'jd') t2 = f.add_trace('bridge', 1, 1, 'jd') # assert len(f.traces) == 2 assert self.q(f, 'loops') == [t] assert self.q(f, 'bridges') == [t2]
def test_merge_point_extract_source_code(): forest = TraceForest(1) trace = forest.add_trace('loop', 0, 0) trace.start_mark(const.MARK_TRACE_OPT) trace.add_instr(MergePoint({0x1: 'jitlog/test/data/code.py', 0x2: 2})) trace.add_instr(FlatOp(0, 'INT_ADD', ['i1', 'i2'], 'i3')) forest.extract_source_code_lines() assert forest.source_lines['jitlog/test/data/code.py'][2] == ( 4, 'return a + b')
def test_patch_asm_timeval(): forest = TraceForest(1) trace = Trace(forest, 'bridge', 0, 0) trace.start_mark(const.MARK_TRACE_ASM) trace.set_addr_bounds(0, 9) trace.add_instr(FlatOp(0, 'hello', ['world'], None, None)) trace.get_stage('asm').get_last_op().set_core_dump(0, 'abcdef') trace.add_instr(FlatOp(0, 'add', ['i1', 'i2'], 'i3', None)) trace.get_stage('asm').get_last_op().set_core_dump(6, 'a312') forest.patch_memory(4, '4321', 1) trace.get_core_dump(0) == "abcdefa312" trace.get_core_dump(1) == "abcd432112"
def _parse_jitlog(fileobj): """ JitLog is parsed. if an exception is raised after the header has been read, it is saved in the field 'exc' on the forest returned. This allows to parse an incomplete log file an still display everything that was readable! If a ParseException is directly raised, this is an error that cannot be recovered from! """ try: is_jit_log = fileobj.read(1) == const.MARK_JITLOG_HEADER version = ord(fileobj.read(1)) | (ord(fileobj.read(1)) << 8) is_32bit = ord(fileobj.read(1)) machine = read_string(fileobj, True) forest = TraceForest(version, is_32bit, machine) ctx = ParseContext(forest) except Exception: raise ParseException("Header malformed") # if not is_jit_log: raise ParseException("Missing header. Provided input might not be a jitlog!") if version < JITLOG_MIN_VERSION: raise ParseException("Version %d is not supported" % version) while True: marker = fileobj.read(1) if len(marker) == 0: break # end of file! if not forest.is_jitlog_marker(marker): msg = "marker unknown: 0x%x at pos 0x%x" % \ (ord(marker), fileobj.tell()) forest.exc = ParseException(msg) break trace = forest.last_trace try: read = marks.get_reader(version, marker) read(ctx, trace, fileobj) forest.time_tick() except KeyError as e: forest.exc = e break except ParseException as e: forest.exc = e break except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() tb = traceback.extract_tb(exc_traceback, limit=3) msg = "failed at 0x%x with marker %s with exc \"%s\". trace back: \"%s\"" %\ (fileobj.tell(), marker, str(e), tb) forest.exc = ParseException(msg) break return forest
def test_merge_point_duplicate_source_lines(): forest = TraceForest(1) trace = forest.add_trace('loop', 0, 0) trace.start_mark(const.MARK_TRACE_OPT) trace.add_instr(MergePoint({0x1: 'jitlog/test/data/code.py', 0x2: 5})) trace.add_instr(MergePoint({0x1: 'jitlog/test/data/code.py', 0x2: 5})) trace.add_instr(MergePoint({0x1: 'jitlog/test/data/code.py', 0x2: 5})) trace.add_instr(MergePoint({0x1: 'jitlog/test/data/code.py', 0x2: 5})) forest.extract_source_code_lines() assert forest.source_lines['jitlog/test/data/code.py'][5] == (4, 'c = a * 2') assert len(forest.source_lines['jitlog/test/data/code.py']) == 1
def test_filter(self): from jitlog.query import loops, bridges, Filter f = TraceForest(3, is_32bit=False, machine='s390x') loop = f.add_trace('loop', 0, 0, 'su') bridge = f.add_trace('bridge', 1, 1, 'shi') assert loops._filter(loop) assert not loops._filter(bridge) assert not bridges._filter(loop) assert bridges._filter(bridge) r = loops | bridges assert isinstance(r, Filter) assert r._filter(loop) assert r._filter(bridge)
def test_failing_guard(): forest = TraceForest(3) trace = forest.add_trace('loop', 0, 0) trace.start_mark(const.MARK_TRACE_ASM) op = FlatOp(0, 'gurad_true', '', 'i0', 0, 15) trace.add_instr(op) # trace2 = forest.add_trace('bridge', 16, 0) trace2.start_mark(const.MARK_TRACE_OPT) trace2.set_addr_bounds(42, 44) # forest.stitch_bridge(15, 42) assert trace2.get_failing_guard() == op
def test_point_in_trace(): forest = TraceForest(1) trace = forest.add_trace('loop', 0, 0) trace.start_mark(const.MARK_TRACE_ASM) op = FlatOp(0, 'hello', '', '?', 0, 1) trace.add_instr(op) trace.add_up_enter_count(10) point_in_trace = forest.get_point_in_trace_by_descr(1) point_in_trace.set_inc_op(FakeOp(1)) point_in_trace.add_up_enter_count(20) assert trace.counter == 10 assert trace.point_counters[1] == 20
def test_query_small_forest(self): f = TraceForest(3, is_32bit=False, machine='s390x') # t = f.add_trace('loop', 0, 0, 'jd') stage = t.start_mark(c.MARK_TRACE_OPT) stage.append_op(FlatOp(0, 'load', [], '?')) # t2 = f.add_trace('loop', 1, 1, 'jd') stage = t2.start_mark(c.MARK_TRACE_OPT) stage.append_op(FlatOp(0, 'store', [], '?')) # assert len(f.traces) == 2 assert self.q(f, '') == None assert self.q(f, 'traces(op(name="load"))') == [t]
def construct_forest(fileobj, version=1, forest=None): if forest is None: forest = TraceForest(version) ctx = ParseContext(forest) try: while True: marker = fileobj.read(1) read = marks.get_reader(version, marker) trace = forest.last_trace read(ctx, trace, fileobj) forest.time_tick() except BufferTooSmallError: pass return forest
def test_merge_point_extract_multiple_lines(): forest = TraceForest(1) trace = forest.add_trace('loop', 0, 0) trace.start_mark(const.MARK_TRACE_OPT) trace.add_instr(MergePoint({0x1: 'jitlog/test/data/code.py', 0x2: 5})) trace.add_instr(FlatOp(0, 'INT_MUL', ['i1', 'i2'], 'i3')) trace.add_instr(MergePoint({0x1: 'jitlog/test/data/code.py', 0x2: 7})) forest.extract_source_code_lines() assert forest.source_lines['jitlog/test/data/code.py'][5] == (4, 'c = a * 2') assert forest.source_lines['jitlog/test/data/code.py'][6] == (8, 'd = c * 3') assert forest.source_lines['jitlog/test/data/code.py'][7] == ( 4, 'return d + 5')
def test_to_json_meta_info(): forest = TraceForest(1, False, 'x86') forest.resops = {15: 'divide'} trace = forest.add_trace('loop', 0, 0) trace.counter = 42 stage = trace.start_mark(const.MARK_TRACE_OPT) stage.append_op(MergePoint({const.MP_SCOPE[0]: 'my_func'})) json = LogMetaSerializer().to_representation(FakeJitLog(forest)) del json['bridges'] # do not care for this test assert json == \ { 'resops': { 15: 'divide' }, 'traces': { 0: { 'scope': 'my_func', 'filename': None, 'lineno': 0, 'type': 'loop', 'counter_points': { 'enter': 42 } } }, 'word_size': 8, 'machine': 'x86' }
def test_counter_points(): forest = TraceForest(1) trace = forest.add_trace('loop', 0, 0) d = trace.get_counter_points() assert d[0] == 0 assert len(d) == 1 trace.counter = 100 d = trace.get_counter_points() assert d[0] == 100 assert len(d) == 1 pit = PointInTrace(trace, FakeOp(10)) assert not pit.add_up_enter_count(55) pit.set_inc_op(FakeOp(11)) assert pit.add_up_enter_count(55) d = trace.get_counter_points() assert d[11] == 55 assert len(d) == 2
def test_to_json_meta_info(): forest = TraceForest(1, False, 'x86') forest.resops = { 15: 'divide' } trace = forest.add_trace('loop', 0, 0, 'john') trace.counter = 42 stage = trace.start_mark(const.MARK_TRACE_OPT) stage.append_op(MergePoint({const.MP_SCOPE[0]: 'my_func' })) json = LogMetaSerializer().to_representation(forest) assert json == \ { 'resops': { 15: 'divide' }, 'traces': { 0: { 'scope': 'my_func', 'filename': None, 'lineno': 0, 'type': 'loop', 'counter_points': { 0: 42 }, 'jd_name': 'john', 'stamp': 0 } }, 'word_size': 8, 'machine': 'x86', 'links': {}, }
def test_merge_point_encode(): forest = TraceForest(1) trace = forest.add_trace('loop', 0, 0) trace.start_mark(const.MARK_TRACE_OPT) trace.add_instr(MergePoint({0x1: 'jitlog/test/data/code.py', 0x2: 5})) trace.add_instr(FlatOp(0, 'INT_MUL', ['i1', 'i2'], 'i3')) trace.add_instr(MergePoint({0x1: 'jitlog/test/data/code.py', 0x2: 7})) trace.add_instr(MergePoint({0x1: 'jitlog/test/data/code2.py', 0x2: 3})) forest.extract_source_code_lines() binary = trace.forest.encode_source_code_lines() parta = b'\x22\x19\x00\x00\x00jitlog/test/data/code2.py' \ b'\x01\x00' \ b'\x03\x00\x07\x13\x00\x00\x00self.unique = False' partb = b'\x22\x18\x00\x00\x00jitlog/test/data/code.py' \ b'\x03\x00' \ b'\x05\x00\x04\x09\x00\x00\x00c = a * 2' \ b'\x06\x00\x08\x09\x00\x00\x00d = c * 3' \ b'\x07\x00\x04\x0c\x00\x00\x00return d + 5' equals = binary == parta + partb if not equals: assert binary == partb + parta
def test_merge_point_extract_source_code(encoding, text, decoded, bom, tmpdir): forest = TraceForest(1) trace = forest.add_trace('loop', 0, 0) trace.start_mark(const.MARK_TRACE_OPT) file = tmpdir.join("file" + encoding + ".py") l = [] if bom: l.append(bom) l.append("# coding: ".encode(encoding)) l.append(encoding.encode(encoding)) l.append("\r\n".encode(encoding)) l.append("print(\"".encode(encoding)) l.append(text) l.append("\")".encode(encoding)) l.append("\r\n".encode(encoding)) file.write_binary(b''.join(l)) trace.add_instr(MergePoint({0x1: str(file), 0x2: 2})) forest.extract_source_code_lines() line = forest.source_lines[str(file)][2] if PY3: assert line == (0, "print(\"" + decoded + "\")") else: assert line == (0, "print(\"" + text + "\")")
def forest(): forest = TraceForest(1) forest.resops = DEFAULT_TEST_RESOPS return forest
def test_add_source_code_lines_to_forest(): forest = TraceForest(1) forest.add_source_code_line("x.py", 12, 12, "x = 1") forest.add_source_code_line("x.py", 13, 12, "y = 1")
def test_query_empty_forest(self): f = TraceForest(3, is_32bit=False, machine='s390x') assert self.q(f, '') == None assert self.q(f, 'traces()') == []