def main(progname, logfilename, outfilename): storage = LoopStorage(extrapath=os.path.dirname(progname)) log, loops = import_log(logfilename) parse_log_counts(extract_category(log, 'jit-backend-count'), loops) storage.loops = [loop for loop in loops if not loop.descr.startswith('bridge')] storage.loop_dict = create_loop_dict(loops) json.dump([loop.force_asm().as_json() for loop in storage.loops], open(outfilename, "w"), indent=4)
def __init__(self, rawtraces): storage = LoopStorage() traces = [ SimpleParser.parse_from_input(rawtrace) for rawtrace in rawtraces ] traces = storage.reconnect_loops(traces) self.loops = [ TraceWithIds.from_trace(trace, storage) for trace in traces ]
def main(progname, logfilename, outfilename): storage = LoopStorage(extrapath=os.path.dirname(progname)) log, loops = import_log(logfilename) parse_log_counts(extract_category(log, 'jit-backend-count'), loops) storage.loops = [ loop for loop in loops if not loop.descr.startswith('bridge') ] storage.loop_dict = create_loop_dict(loops) json.dump([loop.force_asm().as_json() for loop in storage.loops], open(outfilename, "w"), indent=4)
def run(self, topaz, tmpdir, code): tmpdir.join("t.rb").write(code) proc = subprocess.Popen( [str(topaz), str(tmpdir.join("t.rb"))], cwd=str(tmpdir), env={"PYPYLOG": "jit-log-opt:%s" % tmpdir.join("x.pypylog")} ) proc.wait() data = logparser.parse_log_file(str(tmpdir.join("x.pypylog")), verbose=False) data = logparser.extract_category(data, "jit-log-opt-") storage = LoopStorage() traces = [SimpleParser.parse_from_input(t) for t in data] traces = storage.reconnect_loops(traces) return [Trace(t) for t in traces]
def run(self, topaz, tmpdir, code): tmpdir.join("t.rb").write(code) proc = subprocess.Popen( [str(topaz), str(tmpdir.join("t.rb"))], cwd=str(tmpdir), env={"PYPYLOG": "jit-log-opt:%s" % tmpdir.join("x.pypylog")}) proc.wait() data = logparser.parse_log_file(str(tmpdir.join("x.pypylog")), verbose=False) data = logparser.extract_category(data, "jit-log-opt-") storage = LoopStorage() traces = [SimpleParser.parse_from_input(t) for t in data] traces = storage.reconnect_loops(traces) return [Trace(t) for t in traces]
def test_parsing_strliteral(): loop = parse(""" debug_merge_point(0, 0, 'StrLiteralSearch at 11/51 [17, 8, 3, 1, 1, 1, 1, 51, 0, 19, 51, 1]') """) ops = Function.from_operations(loop.operations, LoopStorage()) chunk = ops.chunks[0] assert chunk.bytecode_name.startswith('StrLiteralSearch')
def extract_traces(file, remove_debug=True, remove_main_labels=True, remove_all_labels=False): data = logparser.parse_log_file(file, verbose=False) data = logparser.extract_category(data, "jit-log-opt-") storage = LoopStorage() traces = [SimpleParser.parse_from_input(t) for t in data] main_loops = storage.reconnect_loops(traces) traces_w = [] for trace in traces: if trace in main_loops: traces_w.append(Trace(trace)) else: traces_w[len(traces_w) - 1].addbridge(trace) for trace in traces_w: trace.parse(remove_debug, remove_main_labels, remove_all_labels) return traces_w
def test_parse_nonpython(): loop = parse(""" [] debug_merge_point(0, 0, 'random') """) f = Function.from_operations(loop.operations, LoopStorage()) assert f.filename is None
def test_parse_non_code(): ops = parse(''' [] debug_merge_point(0, 0, "SomeRandomStuff") ''') res = Function.from_operations(ops.operations, LoopStorage()) assert len(res.chunks) == 1 assert 'SomeRandomStuff' in res.chunks[0].repr()
def test_parse_from_inside(): loop = parse(""" [] debug_merge_point(1, 0, 'two') debug_merge_point(2, 0, 'three') debug_merge_point(0, 0, 'one') """) f = Function.from_operations(loop.operations, LoopStorage()) assert len(f.chunks) == 2
def test_lineno(): fname = str(py.path.local(__file__).join('..', 'x.py')) ops = parse(''' [i0, i1] debug_merge_point(0, 0, "<code object f. file '%(fname)s'. line 5> #0 LOAD_FAST") debug_merge_point(0, 0, "<code object f. file '%(fname)s'. line 5> #3 LOAD_FAST") debug_merge_point(0, 0, "<code object f. file '%(fname)s'. line 5> #6 BINARY_ADD") debug_merge_point(0, 0, "<code object f. file '%(fname)s'. line 5> #7 RETURN_VALUE") ''' % locals()) res = Function.from_operations(ops.operations, LoopStorage()) assert res.chunks[1].lineno == 6
def test_embedded_lineno(): # debug_merge_point() can have a text that is either: # # * the PyPy2's <code object %s. file '%s'. line %d> #%d %s> # funcname, filename, lineno, bytecode_no, bytecode_name # # * a standard text of the form %s;%s:%d-%d-%d %s # funcname, filename, startlineno, curlineno, endlineno, anything # # * or anything else, which is not specially recognized but shouldn't crash # sourcefile = str(udir.join('test_embedded_lineno.src')) with open(sourcefile, 'w') as f: print >> f, "A#1" print >> f, "B#2" print >> f, "C#3" print >> f, "D#4" print >> f, "E#5" print >> f, "F#6" loop = parse(""" [] debug_merge_point(0, 0, 'myfunc;%(filename)s:2-2~one') debug_merge_point(0, 0, 'myfunc;%(filename)s:2-2~two') debug_merge_point(0, 0, 'myfunc;%(filename)s:2-4~') debug_merge_point(0, 0, 'myfunc;%(filename)s:2-4~four') """ % {'filename': sourcefile}) f = Function.from_operations(loop.operations, LoopStorage()) expect = [(2, 'one', True), (2, 'two', False), (4, '', True), (4, 'four', False)] assert len(f.chunks) == len(expect) code_seen = set() for chunk, (expected_lineno, expected_bytecode_name, expected_line_starts_here) in zip(f.chunks, expect): assert chunk.name == 'myfunc' assert chunk.bytecode_name == expected_bytecode_name assert chunk.filename == sourcefile assert chunk.startlineno == 2 assert chunk.bytecode_no == ~expected_lineno # half-abuse assert chunk.has_valid_code() assert chunk.lineno == expected_lineno assert chunk.line_starts_here == expected_line_starts_here code_seen.add(chunk.code) assert len(code_seen) == 1 code, = code_seen assert code.source[0] == "B#2" assert code.source[1] == "C#3" assert code.source[4] == "F#6" py.test.raises(IndexError, "code.source[5]")
def test_name_no_first(): ops = parse(''' [i0] i3 = int_add(i0, 1) debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 200> #10 ADD") debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 201> #11 SUB") i1 = int_add(i0, 1) debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 202> #11 SUB") i2 = int_add(i1, 1) ''') res = Function.from_operations(ops.operations, LoopStorage()) assert res.repr() == res.chunks[1].repr()
def test_linerange_notstarts(): if sys.version_info > (2, 6): py.test.skip("unportable test") fname = str(py.path.local(__file__).join('..', 'x.py')) ops = parse(""" [p6, p1] debug_merge_point(0, 0, '<code object h. file '%(fname)s'. line 11> #17 FOR_ITER') guard_class(p6, 144264192, descr=<Guard0x2>) p12 = getfield_gc(p6, descr=<GcPtrFieldDescr pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_w_seq 12>) """ % locals()) res = Function.from_operations(ops.operations, LoopStorage()) assert res.lineset
def test_adjust_bridges(): main = parse(''' [v0] guard_false(v0, descr=<Guard0x1a>) guard_true(v0, descr=<Guard0x5>) ''') bridge = parse(''' # bridge out of Guard 0x1a [] int_add(0, 1) ''') LoopStorage().reconnect_loops([main, bridge]) assert adjust_bridges(main, {})[1].name == 'guard_true' assert adjust_bridges(main, {'loop-1a': True})[1].name == 'int_add'
def test_name(): ops = parse(''' [i0] debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 200> #10 ADD") debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 201> #11 SUB") i1 = int_add(i0, 1) debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 202> #11 SUB") i2 = int_add(i1, 1) ''') res = Function.from_operations(ops.operations, LoopStorage()) assert res.repr() == res.chunks[0].repr() assert res.repr() == "stuff, file '/I/dont/exist.py', line 200" assert res.startlineno == 200 assert res.filename == '/I/dont/exist.py' assert res.name == 'stuff'
def test_linerange(): if sys.version_info > (2, 6): py.test.skip("unportable test") fname = str(py.path.local(__file__).join('..', 'x.py')) ops = parse(''' [i0, i1] debug_merge_point(0, 0, "<code object g. file '%(fname)s'. line 5> #9 LOAD_FAST") debug_merge_point(0, 0, "<code object g. file '%(fname)s'. line 5> #12 LOAD_CONST") debug_merge_point(0, 0, "<code object g. file '%(fname)s'. line 5> #22 LOAD_CONST") debug_merge_point(0, 0, "<code object g. file '%(fname)s'. line 5> #28 LOAD_CONST") debug_merge_point(0, 0, "<code object g. file '%(fname)s'. line 5> #6 SETUP_LOOP") ''' % locals()) res = Function.from_operations(ops.operations, LoopStorage()) assert res.linerange == (7, 9) assert res.lineset == set([7, 8, 9])
def test_inlined_call(): ops = parse(""" [] debug_merge_point(0, 0, '<code object inlined_call. file 'source.py'. line 12> #28 CALL_FUNCTION') i18 = getfield_gc(p0, descr=<BoolFieldDescr pypy.interpreter.pyframe.PyFrame.inst_is_being_profiled 89>) debug_merge_point(1, 1, '<code object inner. file 'source.py'. line 9> #0 LOAD_FAST') debug_merge_point(1, 1, '<code object inner. file 'source.py'. line 9> #3 LOAD_CONST') debug_merge_point(1, 1, '<code object inner. file 'source.py'. line 9> #7 RETURN_VALUE') debug_merge_point(0, 0, '<code object inlined_call. file 'source.py'. line 12> #31 STORE_FAST') """) res = Function.from_operations(ops.operations, LoopStorage()) assert len(res.chunks) == 3 # two chunks + inlined call assert isinstance(res.chunks[0], TraceForOpcode) assert isinstance(res.chunks[1], Function) assert isinstance(res.chunks[2], TraceForOpcode) assert res.chunks[1].path == "1" assert len(res.chunks[1].chunks) == 3
def test_split(): ops = parse(''' [i0] label() debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 200> #10 ADD") debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 200> #11 SUB") i1 = int_add(i0, 1) debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 200> #11 SUB") i2 = int_add(i1, 1) ''') res = Function.from_operations(ops.operations, LoopStorage(), loopname='<loopname>') assert len(res.chunks) == 4 assert len(res.chunks[0].operations) == 1 assert len(res.chunks[1].operations) == 1 assert len(res.chunks[2].operations) == 2 assert len(res.chunks[3].operations) == 2 assert res.chunks[3].bytecode_no == 11 assert res.chunks[0].bytecode_name == '<loopname>'
def test_reassign_loops(): main = parse(''' [v0] guard_false(v0, descr=<Guard0x18>) [] ''') main.count = 10 bridge = parse(''' # bridge out of Guard 0x18 with 13 ops [i0, i1] int_add(i0, i1) ''') bridge.count = 3 entry_bridge = parse(''' # Loop 3 : entry bridge [] ''') loops = LoopStorage().reconnect_loops([main, bridge, entry_bridge]) assert len(loops) == 2 assert len(loops[0].operations[0].bridge.operations) == 1 assert loops[0].operations[0].bridge.no == 0x18 assert loops[0].operations[0].percentage == 30
def test_load_codes(): tmppath = py.test.ensuretemp('load_codes') tmppath.join("x.py").write("def f(): pass") # one code s = LoopStorage(str(tmppath)) assert s.load_code(str(tmppath.join('x.py'))) == s.load_code('x.py')
def __init__(self, rawtraces): storage = LoopStorage() traces = [SimpleParser.parse_from_input(rawtrace) for rawtrace in rawtraces] traces = storage.reconnect_loops(traces) self.loops = [TraceWithIds.from_trace(trace, storage) for trace in traces]