def __init__(self, rawtraces): storage = LoopStorage() traces = [ SimpleParser.parse_from_input(rawtrace) for rawtrace in rawtraces ] traces = storage.reconnect_loops(traces) self.loops = [ TraceWithIds.from_trace(trace, storage) for trace in traces ]
def __init__(self, rawtraces): storage = LoopStorage() traces = [ SimpleParser.parse_from_input(rawtrace) for rawtrace in rawtraces ] traces = storage.reconnect_loops(traces) self.loops = [ LoopWithIds.from_trace(trace, storage) for trace in traces ]
def getMySources(filename): extra_path = os.path.dirname(filename) storage = LoopStorage(extra_path) log, loops = import_log(filename, ParserWithHtmlRepr) parse_log_counts(extract_category(log, "jit-backend-count"), loops) storage.loops = [loop for loop in loops if not loop.descr.startswith("bridge")] [l.force_asm() for l in storage.loops] storage.loop_dict = create_loop_dict(loops) loops = index(storage) ids = [item.descr for item in loops] mySources = [] for item in ids: source, name, up, filename, startline, callstack = loopfunc(item, storage) i = 1 mySource = [] pending = False pendingLine = 0 pendingCode = "" for sourceline in source.lines: # print str(i) + ": " + sourceline.line line = sourceline.line chunks = [] if sourceline.in_loop and line.strip != "": if sourceline.chunks: makePending = True for chunk in sourceline.chunks: if chunk.is_bytecode: ops = [] for op in chunk.operations: if op.name != "debug_merge_point": op_repr = (op.html_repr(), op.asm) ops.append(op_repr) chunks.append((chunk.html_repr(), chunk, ops)) if len(chunks) == 0: if pending: pendingCode += "\n" + line else: pendingCode = line pendingLine = i pending = True else: if pending: pending = False mySource.append((pendingLine, pendingCode, None)) mySource.append((i, line, chunks)) i += 1 if pending: pending = False mySource.append((pendingLine, pendingCode, None)) mySources.append((filename, mySource)) return mySources
def main(): filename = sys.argv[1] extra_path = os.path.dirname(filename) storage = LoopStorage(extra_path) log, loops = import_log(filename, ParserWithHtmlRepr) parse_log_counts(extract_category(log, 'jit-backend-count'), loops) storage.loops = [loop for loop in loops if not loop.descr.startswith('bridge')] storage.loop_dict = create_loop_dict(loops) print loops,log,storage server = Server(filename, storage)
def run(self, topaz, tmpdir, code): tmpdir.join("t.rb").write(code) proc = subprocess.Popen( [str(topaz), str(tmpdir.join("t.rb"))], cwd=str(tmpdir), env={"PYPYLOG": "jit-log-opt:%s" % tmpdir.join("x.pypylog")}) proc.wait() data = logparser.parse_log_file(str(tmpdir.join("x.pypylog")), verbose=False) data = logparser.extract_category(data, "jit-log-opt-") storage = LoopStorage() traces = [SimpleParser.parse_from_input(t) for t in data] traces = storage.reconnect_loops(traces) return [Trace(t) for t in traces]
def run(self, topaz, tmpdir, code): tmpdir.join("t.rb").write(code) proc = subprocess.Popen( [str(topaz), str(tmpdir.join("t.rb"))], cwd=str(tmpdir), env={"PYPYLOG": "jit-log-opt:%s" % tmpdir.join("x.pypylog")} ) proc.wait() data = logparser.parse_log_file(str(tmpdir.join("x.pypylog")), verbose=False) data = logparser.extract_category(data, "jit-log-opt-") storage = LoopStorage() traces = [SimpleParser.parse_from_input(t) for t in data] traces = storage.reconnect_loops(traces) return [Trace(t) for t in traces]
def test_parsing_strliteral(): loop = parse(""" debug_merge_point(0, 0, 'StrLiteralSearch at 11/51 [17, 8, 3, 1, 1, 1, 1, 51, 0, 19, 51, 1]') """) ops = Function.from_operations(loop.operations, LoopStorage()) chunk = ops.chunks[0] assert chunk.bytecode_name.startswith('StrLiteralSearch')
def test_parse_nonpython(): loop = parse(""" [] debug_merge_point(0, 0, 'random') """) f = Function.from_operations(loop.operations, LoopStorage()) assert f.filename is None
def test_parse_non_code(): ops = parse(''' [] debug_merge_point(0, 0, "SomeRandomStuff") ''') res = Function.from_operations(ops.operations, LoopStorage()) assert len(res.chunks) == 1 assert 'SomeRandomStuff' in res.chunks[0].repr()
def test_parse_from_inside(): loop = parse(""" [] debug_merge_point(1, 0, 'two') debug_merge_point(2, 0, 'three') debug_merge_point(0, 0, 'one') """) f = Function.from_operations(loop.operations, LoopStorage()) assert len(f.chunks) == 2
def test_parse_nonpython(): loop = parse(""" [] debug_merge_point(0, 0, 'random') debug_merge_point(0, 0, '<code object f. file 'x.py'. line 2> #15 COMPARE_OP') """) f = Function.from_operations(loop.operations, LoopStorage()) assert f.chunks[-1].filename == 'x.py' assert f.filename is None
def test_lineno(): fname = str(py.path.local(__file__).join('..', 'x.py')) ops = parse(''' [i0, i1] debug_merge_point(0, 0, "<code object f. file '%(fname)s'. line 5> #0 LOAD_FAST") debug_merge_point(0, 0, "<code object f. file '%(fname)s'. line 5> #3 LOAD_FAST") debug_merge_point(0, 0, "<code object f. file '%(fname)s'. line 5> #6 BINARY_ADD") debug_merge_point(0, 0, "<code object f. file '%(fname)s'. line 5> #7 RETURN_VALUE") ''' % locals()) res = Function.from_operations(ops.operations, LoopStorage()) assert res.chunks[1].lineno == 6
def run(self, spy, tmpdir, code): proc = subprocess.Popen( [str(spy), "-r", code.replace("\n", "\r\n"), BenchmarkImage], cwd=str(tmpdir), env={"PYPYLOG": "jit-log-opt:%s" % tmpdir.join("x.pypylog")} ) proc.wait() data = logparser.parse_log_file(str(tmpdir.join("x.pypylog")), verbose=False) data = logparser.extract_category(data, "jit-log-opt-") storage = LoopStorage() traces = [SimpleParser.parse_from_input(t) for t in data] main_loops = storage.reconnect_loops(traces) traces_w = [] for trace in traces: if trace in main_loops: traces_w.append(Trace(trace)) else: traces_w[len(traces_w) - 1].addbridge(trace) return traces_w
def test_linerange_notstarts(): if sys.version_info > (2, 6): py.test.skip("unportable test") fname = str(py.path.local(__file__).join('..', 'x.py')) ops = parse(""" [p6, p1] debug_merge_point(0, 0, '<code object h. file '%(fname)s'. line 11> #17 FOR_ITER') guard_class(p6, 144264192, descr=<Guard0x2>) p12 = getfield_gc(p6, descr=<GcPtrFieldDescr pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_w_seq 12>) """ % locals()) res = Function.from_operations(ops.operations, LoopStorage()) assert res.lineset
def test_name_no_first(): ops = parse(''' [i0] i3 = int_add(i0, 1) debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 200> #10 ADD") debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 201> #11 SUB") i1 = int_add(i0, 1) debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 202> #11 SUB") i2 = int_add(i1, 1) ''') res = Function.from_operations(ops.operations, LoopStorage()) assert res.repr() == res.chunks[1].repr()
def main(): if not '__pypy__' in sys.builtin_module_names: print "Please run it using pypy-c" sys.exit(1) # server_mode = True if '--qt' in sys.argv: server_mode = False sys.argv.remove('--qt') # if len(sys.argv) != 2 and len(sys.argv) != 3: print __doc__ sys.exit(1) filename = sys.argv[1] extra_path = os.path.dirname(filename) if len(sys.argv) != 3: port = 5000 else: port = int(sys.argv[2]) storage = LoopStorage(extra_path) log, loops = import_log(filename, ParserWithHtmlRepr) parse_log_counts(extract_category(log, 'jit-backend-count'), loops) storage.loops = [loop for loop in loops if not loop.descr.startswith('bridge')] storage.loop_dict = create_loop_dict(loops) print loops,log,storage app = OverrideFlask('_jitviewer') server = Server(filename, storage) app.debug = True app.route('/')(server.index) app.route('/loop')(server.loop) def run(): app.run(use_reloader=False, host='0.0.0.0', port=port) if server_mode: run() else: url = "http://localhost:%d/" % port run_server_and_browser(app, run, url, filename)
def test_adjust_bridges(): main = parse(''' [v0] guard_false(v0, descr=<Guard0x1a>) guard_true(v0, descr=<Guard0x5>) ''') bridge = parse(''' # bridge out of Guard 0x1a [] int_add(0, 1) ''') LoopStorage().reconnect_loops([main, bridge]) assert adjust_bridges(main, {})[1].name == 'guard_true' assert adjust_bridges(main, {'loop-1a': True})[1].name == 'int_add'
def test_linerange(): if sys.version_info > (2, 6): py.test.skip("unportable test") fname = str(py.path.local(__file__).join('..', 'x.py')) ops = parse(''' [i0, i1] debug_merge_point(0, 0, "<code object g. file '%(fname)s'. line 5> #9 LOAD_FAST") debug_merge_point(0, 0, "<code object g. file '%(fname)s'. line 5> #12 LOAD_CONST") debug_merge_point(0, 0, "<code object g. file '%(fname)s'. line 5> #22 LOAD_CONST") debug_merge_point(0, 0, "<code object g. file '%(fname)s'. line 5> #28 LOAD_CONST") debug_merge_point(0, 0, "<code object g. file '%(fname)s'. line 5> #6 SETUP_LOOP") ''' % locals()) res = Function.from_operations(ops.operations, LoopStorage()) assert res.linerange == (7, 9) assert res.lineset == set([7, 8, 9])
def test_name(): ops = parse(''' [i0] debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 200> #10 ADD") debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 201> #11 SUB") i1 = int_add(i0, 1) debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 202> #11 SUB") i2 = int_add(i1, 1) ''') res = Function.from_operations(ops.operations, LoopStorage()) assert res.repr() == res.chunks[0].repr() assert res.repr() == "stuff, file '/I/dont/exist.py', line 200" assert res.startlineno == 200 assert res.filename == '/I/dont/exist.py' assert res.name == 'stuff'
def test_inlined_call(): ops = parse(""" [] debug_merge_point(0, 0, '<code object inlined_call. file 'source.py'. line 12> #28 CALL_FUNCTION') i18 = getfield_gc(p0, descr=<BoolFieldDescr pypy.interpreter.pyframe.PyFrame.inst_is_being_profiled 89>) debug_merge_point(1, 1, '<code object inner. file 'source.py'. line 9> #0 LOAD_FAST') debug_merge_point(1, 1, '<code object inner. file 'source.py'. line 9> #3 LOAD_CONST') debug_merge_point(1, 1, '<code object inner. file 'source.py'. line 9> #7 RETURN_VALUE') debug_merge_point(0, 0, '<code object inlined_call. file 'source.py'. line 12> #31 STORE_FAST') """) res = Function.from_operations(ops.operations, LoopStorage()) assert len(res.chunks) == 3 # two chunks + inlined call assert isinstance(res.chunks[0], TraceForOpcode) assert isinstance(res.chunks[1], Function) assert isinstance(res.chunks[2], TraceForOpcode) assert res.chunks[1].path == "1" assert len(res.chunks[1].chunks) == 3
def test_split(): ops = parse(''' [i0] label() debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 200> #10 ADD") debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 200> #11 SUB") i1 = int_add(i0, 1) debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 200> #11 SUB") i2 = int_add(i1, 1) ''') res = Function.from_operations(ops.operations, LoopStorage(), loopname='<loopname>') assert len(res.chunks) == 4 assert len(res.chunks[0].operations) == 1 assert len(res.chunks[1].operations) == 1 assert len(res.chunks[2].operations) == 2 assert len(res.chunks[3].operations) == 2 assert res.chunks[3].bytecode_no == 11 assert res.chunks[0].bytecode_name == '<loopname>'
def test_reassign_loops(): main = parse(''' [v0] guard_false(v0, descr=<Guard0x18>) [] ''') main.count = 10 bridge = parse(''' # bridge out of Guard 0x18 with 13 ops [i0, i1] int_add(i0, i1) ''') bridge.count = 3 entry_bridge = parse(''' # Loop 3 : entry bridge [] ''') loops = LoopStorage().reconnect_loops([main, bridge, entry_bridge]) assert len(loops) == 2 assert len(loops[0].operations[0].bridge.operations) == 1 assert loops[0].operations[0].bridge.no == 0x18 assert loops[0].operations[0].percentage == 30
def hook3(*args): l3.append(args) pypyjit.set_compile_hook(hook) pypyjit.set_optimize_hook(hook2) pypyjit.set_abort_hook(hook3) runpy.run_path(sys.argv[1]) pypyjit.set_compile_hook(None) pypyjit.set_optimize_hook(None) pypyjit.set_abort_hook(None) for jitdriver_name, loop_type, greenkey, operations, assembler_addr, assembler_length in l: storage = LoopStorage() try: code = storage.load_code(loop.filename)[(loop.startlineno, loop.name)] if code.co_name == "<module>" and code.co_firstlineno == 1: with open(code.co_filename) as f: source = CodeRepr(f.read(), code, loop) else: source = CodeRepr(inspect.getsource(code), code, loop) except (IOError, OSError): source = CodeReprNoFile(loop) print source parse = parser.Function.from_operations(operations, storage) print "b" print parse
def test_load_codes(): tmppath = py.test.ensuretemp('load_codes') tmppath.join("x.py").write("def f(): pass") # one code s = LoopStorage(str(tmppath)) assert s.load_code(str(tmppath.join('x.py'))) == s.load_code('x.py')