def main(loopfile, options): print 'Loading file:' log = logparser.parse_log_file(loopfile) loops, summary = consider_category(log, options, "jit-log-opt-") if not options.quiet: for loop in loops: loop.show() if options.summary: print print 'Summary:' print_summary(summary) if options.diff: # non-optimized loops and summary nloops, nsummary = consider_category(log, options, "jit-log-noopt-") print print 'Summary of optimized-away operations' print diff = {} keys = set(summary.keys()).union(set(nsummary)) for key in keys: before = nsummary.get(key, 0) after = summary.get(key, 0) diff[key] = (before - after, before, after) print_diff(diff)
def main(loopfile, options): print 'Loading file:' log = logparser.parse_log_file(loopfile) loops, summary = consider_category(log, options, "jit-log-opt-") if not options.quiet: for loop in loops: loop.show() if options.summary: print print 'Summary:' print_summary(summary) if options.diff: # non-optimized loops and summary nloops, nsummary = consider_category(log, options, "jit-log-noopt-") print print 'Summary of optimized-away operations' print diff = {} keys = set(summary.keys()).union(set(nsummary)) for key in keys: before = nsummary.get(key, 0) after = summary.get(key, 0) diff[key] = (before-after, before, after) print_diff(diff)
def run(self, func_or_src, args=[], import_site=False, **jitopts): jitopts.setdefault('threshold', 200) src = py.code.Source(func_or_src) if isinstance(func_or_src, types.FunctionType): funcname = func_or_src.func_name else: funcname = 'main' # write the snippet arglist = ', '.join(map(repr, args)) with self.filepath.open("w") as f: # we don't want to see the small bridges created # by the checkinterval reaching the limit f.write("import sys\n") f.write("sys.setcheckinterval(10000000)\n") f.write(str(src) + "\n") f.write("print %s(%s)\n" % (funcname, arglist)) # # run a child pypy-c with logging enabled logfile = self.filepath.new(ext='.log') # cmdline = [sys.executable] if not import_site: cmdline.append('-S') if jitopts: jitcmdline = ['%s=%s' % (key, value) for key, value in jitopts.items()] cmdline += ['--jit', ','.join(jitcmdline)] cmdline.append(str(self.filepath)) # env = os.environ.copy() env['PYPYLOG'] = self.log_string + ':' + str(logfile) pipe = subprocess.Popen(cmdline, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = pipe.communicate() if getattr(pipe, 'returncode', 0) < 0: raise IOError("subprocess was killed by signal %d" % ( pipe.returncode,)) if stderr.startswith('SKIP:'): py.test.skip(stderr) if stderr.startswith('debug_alloc.h:'): # lldebug builds stderr = '' assert not stderr # # parse the JIT log rawlog = logparser.parse_log_file(str(logfile)) rawtraces = logparser.extract_category(rawlog, 'jit-log-opt-') log = Log(rawtraces) log.result = eval(stdout) log.logfile = str(logfile) # summaries = logparser.extract_category(rawlog, 'jit-summary') if len(summaries) > 0: log.jit_summary = parse_prof(summaries[-1]) else: log.jit_summary = None # return log
def import_log(logname, ParserCls=SimpleParser): log = parse_log_file(logname) hex_re = '0x(-?[\da-f]+)' addrs = {} for entry in extract_category(log, 'jit-backend-addr'): m = re.search('bootstrap ' + hex_re, entry) if not m: # a bridge m = re.search('has address ' + hex_re, entry) addr = int(m.group(1), 16) entry = entry.lower() m = re.search('guard ' + hex_re, entry) name = 'guard ' + m.group(1) else: name = entry[:entry.find('(') - 1].lower() addr = int(m.group(1), 16) addrs.setdefault(addr, []).append(name) from rpython.jit.backend.tool.viewcode import World world = World() for entry in extract_category(log, 'jit-backend-dump'): world.parse(entry.splitlines(True)) dumps = {} for r in world.ranges: if r.addr in addrs and addrs[r.addr]: name = addrs[r.addr].pop(0) # they should come in order data = r.data.encode('hex') # backward compatibility dumps[name] = (world.backend_name, r.addr, data) loops = [] cat = extract_category(log, 'jit-log-opt') if not cat: extract_category(log, 'jit-log-rewritten') if not cat: extract_category(log, 'jit-log-noopt') for entry in cat: parser = ParserCls(entry, None, {}, 'lltype', None, nonstrict=True) loop = parser.parse() comm = loop.comment comm = comm.lower() if comm.startswith('# bridge'): m = re.search('guard 0x(-?[\da-f]+)', comm) name = 'guard ' + m.group(1) elif "(" in comm: name = comm[2:comm.find('(')-1] else: name = " ".join(comm[2:].split(" ", 2)[:2]) if name in dumps: bname, start_ofs, dump = dumps[name] loop.force_asm = (lambda dump=dump, start_ofs=start_ofs, bname=bname, loop=loop: parser.postprocess(loop, backend_tp=bname, backend_dump=dump, dump_start=start_ofs)) loops += split_trace(loop) return log, loops
def import_log(logname, ParserCls=SimpleParser): log = parse_log_file(logname) hex_re = '0x(-?[\da-f]+)' addrs = {} for entry in extract_category(log, 'jit-backend-addr'): m = re.search('bootstrap ' + hex_re, entry) if not m: # a bridge m = re.search('has address ' + hex_re, entry) addr = int(m.group(1), 16) entry = entry.lower() m = re.search('guard ' + hex_re, entry) name = 'guard ' + m.group(1) else: name = entry[:entry.find('(') - 1].lower() addr = int(m.group(1), 16) addrs.setdefault(addr, []).append(name) from rpython.jit.backend.tool.viewcode import World world = World() for entry in extract_category(log, 'jit-backend-dump'): world.parse(entry.splitlines(True)) dumps = {} for r in world.ranges: if r.addr in addrs and addrs[r.addr]: name = addrs[r.addr].pop(0) # they should come in order data = r.data.encode('hex') # backward compatibility dumps[name] = (world.backend_name, r.addr, data) loops = [] cat = extract_category(log, 'jit-log-opt') if not cat: cat = extract_category(log, 'jit-log-rewritten') if not cat: cat = extract_category(log, 'jit-log-noopt') for entry in cat: parser = ParserCls(entry, None, {}, 'lltype', None, nonstrict=True) loop = parser.parse() comm = loop.comment comm = comm.lower() if comm.startswith('# bridge'): m = re.search('guard 0x(-?[\da-f]+)', comm) name = 'guard ' + m.group(1) elif "(" in comm: name = comm[2:comm.find('(') - 1] else: name = " ".join(comm[2:].split(" ", 2)[:2]) if name in dumps: bname, start_ofs, dump = dumps[name] loop.force_asm = ( lambda dump=dump, start_ofs=start_ofs, bname=bname, loop=loop: parser.postprocess(loop, backend_tp=bname, backend_dump=dump, dump_start=start_ofs)) loops += split_trace(loop) return log, loops
def run(self, topaz, tmpdir, code): tmpdir.join("t.rb").write(code) proc = subprocess.Popen( [str(topaz), str(tmpdir.join("t.rb"))], cwd=str(tmpdir), env={"PYPYLOG": "jit-log-opt:%s" % tmpdir.join("x.pypylog")} ) proc.wait() data = logparser.parse_log_file(str(tmpdir.join("x.pypylog")), verbose=False) data = logparser.extract_category(data, "jit-log-opt-") storage = LoopStorage() traces = [SimpleParser.parse_from_input(t) for t in data] traces = storage.reconnect_loops(traces) return [Trace(t) for t in traces]
def run(self, topaz, tmpdir, code): tmpdir.join("t.rb").write(code) proc = subprocess.Popen( [str(topaz), str(tmpdir.join("t.rb"))], cwd=str(tmpdir), env={"PYPYLOG": "jit-log-opt:%s" % tmpdir.join("x.pypylog")}) proc.wait() data = logparser.parse_log_file(str(tmpdir.join("x.pypylog")), verbose=False) data = logparser.extract_category(data, "jit-log-opt-") storage = LoopStorage() traces = [SimpleParser.parse_from_input(t) for t in data] traces = storage.reconnect_loops(traces) return [Trace(t) for t in traces]
def extract_traces(file, remove_debug=True, remove_main_labels=True, remove_all_labels=False): data = logparser.parse_log_file(file, verbose=False) data = logparser.extract_category(data, "jit-log-opt-") storage = LoopStorage() traces = [SimpleParser.parse_from_input(t) for t in data] main_loops = storage.reconnect_loops(traces) traces_w = [] for trace in traces: if trace in main_loops: traces_w.append(Trace(trace)) else: traces_w[len(traces_w) - 1].addbridge(trace) for trace in traces_w: trace.parse(remove_debug, remove_main_labels, remove_all_labels) return traces_w
def run(self, src, call, **jitopts): jitopts.setdefault('threshold', 200) # write the snippet with self.filepath.open("w") as f: # we don't want to see the small bridges created # by the checkinterval reaching the limit f.write(str(src) + "\n") # # run a child pyrolog-c with logging enabled logfile = self.filepath.new(ext='.log') # cmdline = [strexecutable] cmdline.append(str(self.filepath)) # print cmdline, logfile env = {'PYPYLOG': 'jit-log-opt,jit-summary:' + str(logfile)} #env={'PYPYLOG': ':' + str(logfile)} pipe = subprocess.Popen(cmdline, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) pipe.stdin.write(call + "\n") stdout, stderr = pipe.communicate() if stderr.startswith('SKIP:'): py.test.skip(stderr) if stderr.startswith('debug_alloc.h:'): # lldebug builds stderr = '' assert not stderr # # parse the JIT log rawlog = logparser.parse_log_file(str(logfile)) rawtraces = logparser.extract_category(rawlog, 'jit-log-opt-') log = Log(rawtraces) log.result = stdout if "ParseError" in log.result or "ERROR" in log.result: assert 0, log.result # summaries = logparser.extract_category(rawlog, 'jit-summary') if len(summaries) > 0: log.jit_summary = parse_prof(summaries[-1]) else: log.jit_summary = None # return log
def run(self, src, call, **jitopts): jitopts.setdefault('threshold', 200) # write the snippet with self.filepath.open("w") as f: # we don't want to see the small bridges created # by the checkinterval reaching the limit f.write(str(src) + "\n") # # run a child pyrolog-c with logging enabled logfile = self.filepath.new(ext='.log') # cmdline = [strexecutable] cmdline.append(str(self.filepath)) # print cmdline, logfile env={'PYPYLOG': 'jit-log-opt,jit-summary:' + str(logfile)} #env={'PYPYLOG': ':' + str(logfile)} pipe = subprocess.Popen(cmdline, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) pipe.stdin.write(call + "\n") stdout, stderr = pipe.communicate() if stderr.startswith('SKIP:'): py.test.skip(stderr) if stderr.startswith('debug_alloc.h:'): # lldebug builds stderr = '' assert not stderr # # parse the JIT log rawlog = logparser.parse_log_file(str(logfile)) rawtraces = logparser.extract_category(rawlog, 'jit-log-opt-') log = Log(rawtraces) log.result = stdout if "ParseError" in log.result or "ERROR" in log.result: assert 0, log.result # summaries = logparser.extract_category(rawlog, 'jit-summary') if len(summaries) > 0: log.jit_summary = parse_prof(summaries[-1]) else: log.jit_summary = None # return log
def main(argv): log = logparser.parse_log_file(argv[0]) parts = logparser.extract_category(log, "jit-log-opt-") for i, oplist in enumerate(parts): loop = parse(oplist, no_namespace=True, nonstrict=True) num_ops = 0 num_dmp = 0 num_guards = 0 for op in loop.operations: if op.getopnum() == rop.DEBUG_MERGE_POINT: num_dmp += 1 else: num_ops += 1 if op.is_guard(): num_guards += 1 if num_dmp == 0: print "Loop #%d, length: %d, opcodes: %d, guards: %d" % (i, num_ops, num_dmp, num_guards) else: print "Loop #%d, length: %d, opcodes: %d, guards: %d, %f" % (i, num_ops, num_dmp, num_guards, num_ops/num_dmp)
def import_log(logname, ParserCls=SimpleParser): log = parse_log_file(logname) addrs = parse_addresses(extract_category(log, 'jit-backend-addr')) from rpython.jit.backend.tool.viewcode import World world = World() for entry in extract_category(log, 'jit-backend-dump'): world.parse(entry.splitlines(True)) dumps = {} for r in world.ranges: for pos1 in range(r.addr, r.addr + len(r.data)): if pos1 in addrs and addrs[pos1]: name = addrs[pos1].pop(0) # they should come in order data = r.data.encode('hex') dumps[name] = (world.backend_name, r.addr, data) loops = [] cat = extract_category(log, 'jit-log-opt') if not cat: cat = extract_category(log, 'jit-log-rewritten') if not cat: cat = extract_category(log, 'jit-log-noopt') for entry in cat: parser = ParserCls(entry, None, {}, 'lltype', None, nonstrict=True) loop = parser.parse() comm = loop.comment comm = comm.lower() if comm.startswith('# bridge'): m = re.search('guard 0x(-?[\da-f]+)', comm) name = 'guard ' + m.group(1) elif "(" in comm: name = comm[2:comm.find('(')-1] else: name = " ".join(comm[2:].split(" ", 2)[:2]) if name in dumps: bname, start_ofs, dump = dumps[name] loop.force_asm = (lambda dump=dump, start_ofs=start_ofs, bname=bname, loop=loop: parser.postprocess(loop, backend_tp=bname, backend_dump=dump, dump_start=start_ofs)) loops += split_trace(loop) return log, loops
def run(self, spy, tmpdir, code): proc = subprocess.Popen( [str(spy), "-r", code.replace("\n", "\r\n"), BenchmarkImage], cwd=str(tmpdir), env={"PYPYLOG": "jit-log-opt:%s" % tmpdir.join("x.pypylog")} ) proc.wait() data = logparser.parse_log_file(str(tmpdir.join("x.pypylog")), verbose=False) data = logparser.extract_category(data, "jit-log-opt-") storage = LoopStorage() traces = [SimpleParser.parse_from_input(t) for t in data] main_loops = storage.reconnect_loops(traces) traces_w = [] for trace in traces: if trace in main_loops: traces_w.append(Trace(trace)) else: traces_w[len(traces_w) - 1].addbridge(trace) return traces_w
def main(argv): log = logparser.parse_log_file(argv[0]) parts = logparser.extract_category(log, "jit-log-opt-") for i, oplist in enumerate(parts): loop = parse(oplist, no_namespace=True, nonstrict=True) num_ops = 0 num_dmp = 0 num_guards = 0 for op in loop.operations: if op.getopnum() == rop.DEBUG_MERGE_POINT: num_dmp += 1 else: num_ops += 1 if op.is_guard(): num_guards += 1 if num_dmp == 0: print "Loop #%d, length: %d, opcodes: %d, guards: %d" % ( i, num_ops, num_dmp, num_guards) else: print "Loop #%d, length: %d, opcodes: %d, guards: %d, %f" % ( i, num_ops, num_dmp, num_guards, num_ops / num_dmp)
def main(loopfile, use_threshold, view=True): countname = py.path.local(loopfile + '.count') if countname.check(): #counts = [line.split(':', 1) for line in countname.readlines()] #counts = Counts([('<code' + k.strip("\n"), int(v.strip('\n').strip())) # for v, k in counts]) counts = Counts([]) l = list(sorted(counts.values())) if len(l) > 20 and use_threshold: counts.threshold = l[-20] else: counts.threshold = 0 for_print = [(v, k) for k, v in counts.iteritems()] for_print.sort() else: counts = {} log = logparser.parse_log_file(loopfile) loops = logparser.extract_category(log, "jit-log-opt-") real_loops, allloops = splitloops(loops) postprocess(real_loops, allloops, counts) if view: Page(allloops, counts).display()
mod.udir = udir sys.modules['rpython.tool.udir'] = mod rpython.tool.udir = mod if '--text' in sys.argv: sys.argv.remove('--text') showgraph = False else: showgraph = True if len(sys.argv) != 2: print >> sys.stderr, __doc__ sys.exit(2) # import cStringIO from rpython.tool import logparser log1 = logparser.parse_log_file(sys.argv[1]) text1 = logparser.extract_category(log1, catprefix='jit-backend-dump') f = cStringIO.StringIO() f.writelines(text1) f.seek(0) del log1, text1 # world = World() world.parse(f) if showgraph: world.find_cross_references() world.show(showtext=True) else: world.showtextonly() else: from rpython.tool.udir import udir
def run(self, func_or_src, args=[], import_site=False, discard_stdout_before_last_line=False, **jitopts): jitopts.setdefault('threshold', 200) src = py.code.Source(func_or_src) if isinstance(func_or_src, types.FunctionType): funcname = func_or_src.func_name else: funcname = 'main' # write the snippet arglist = ', '.join(map(repr, args)) with self.filepath.open("w") as f: # we don't want to see the small bridges created # by the checkinterval reaching the limit f.write("import sys\n") f.write("sys.setcheckinterval(10000000)\n") f.write(str(src) + "\n") f.write("print %s(%s)\n" % (funcname, arglist)) # # run a child pypy-c with logging enabled logfile = self.filepath.new(ext='.log') # cmdline = [sys.executable] if not import_site: cmdline.append('-S') if jitopts: jitcmdline = [ '%s=%s' % (key, value) for key, value in jitopts.items() ] cmdline += ['--jit', ','.join(jitcmdline)] cmdline.append(str(self.filepath)) # env = os.environ.copy() env['PYPYLOG'] = self.log_string + ':' + str(logfile) pipe = subprocess.Popen(cmdline, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = pipe.communicate() if pipe.wait() < 0: raise IOError("subprocess was killed by signal %d" % (pipe.returncode, )) if stderr.startswith('SKIP:'): py.test.skip(stderr) if stderr.startswith('debug_alloc.h:'): # lldebug builds stderr = '' assert not stderr # if discard_stdout_before_last_line: stdout = stdout.splitlines(True)[-1] # # parse the JIT log rawlog = logparser.parse_log_file(str(logfile)) rawtraces = logparser.extract_category(rawlog, 'jit-log-opt-') log = Log(rawtraces) log.result = eval(stdout) log.logfile = str(logfile) # summaries = logparser.extract_category(rawlog, 'jit-summary') if len(summaries) > 0: log.jit_summary = parse_prof(summaries[-1]) else: log.jit_summary = None # return log
import re from rpython.tool import logparser log = logparser.parse_log_file("log") log = logparser.extract_category(log, "jit-log-opt") d = {} counted = 0 not_counted = 0 result = {} for item in log: first = item.split("\n")[0] if '# Loop' in first: m = re.search("<code object (.*), file '(.*)', line (\d+)", first) if not m: continue key = (m.group(1), m.group(2), int(m.group(3))) for i, op in enumerate(item.split("\n")[2:-1]): if 'guard_' in op: m = re.search("descr=<Guard(0x[0-9a-f]+)>", op) d[int(m.group(1), 16)] = (key, i) #print op, i, int(m.group(1), 16) else: m = re.search("bridge out of Guard (0x[a-f0-9]+)", first) bridge_no = int(m.group(1), 16) if bridge_no in d: result.setdefault(d[bridge_no][0], []).append((bridge_no, d[bridge_no][1])) else: not_counted += 1 for i, op in enumerate(item.split("\n")[2:-1]): if 'guard_' in op:
return _Page(self.graph_builder) # ____________________________________________________________ if __name__ == '__main__': if '--text' in sys.argv: sys.argv.remove('--text') showgraph = False else: showgraph = True if len(sys.argv) != 2: print >> sys.stderr, __doc__ sys.exit(2) # import cStringIO from rpython.tool import logparser log1 = logparser.parse_log_file(sys.argv[1]) text1 = logparser.extract_category(log1, catprefix='jit-backend-dump') f = cStringIO.StringIO() f.writelines(text1) f.seek(0) del log1, text1 # world = World() world.parse(f) if showgraph: world.find_cross_references() world.show(showtext=True) else: world.showtextonly()