Exemple #1
0
def main(loopfile, options):
    print 'Loading file:'
    log = logparser.parse_log_file(loopfile)
    loops, summary = consider_category(log, options, "jit-log-opt-")
    if not options.quiet:
        for loop in loops:
            loop.show()

    if options.summary:
        print
        print 'Summary:'
        print_summary(summary)

    if options.diff:
        # non-optimized loops and summary
        nloops, nsummary = consider_category(log, options, "jit-log-noopt-")
        print
        print 'Summary of optimized-away operations'
        print
        diff = {}
        keys = set(summary.keys()).union(set(nsummary))
        for key in keys:
            before = nsummary.get(key, 0)
            after = summary.get(key, 0)
            diff[key] = (before-after, before, after)
        print_diff(diff)
Exemple #2
0
 def run(self, func_or_src, args=[], import_site=False, **jitopts):
     jitopts.setdefault('threshold', 200)
     src = py.code.Source(func_or_src)
     if isinstance(func_or_src, types.FunctionType):
         funcname = func_or_src.func_name
     else:
         funcname = 'main'
     # write the snippet
     arglist = ', '.join(map(repr, args))
     with self.filepath.open("w") as f:
         # we don't want to see the small bridges created
         # by the checkinterval reaching the limit
         f.write("import sys\n")
         f.write("sys.setcheckinterval(10000000)\n")
         f.write(str(src) + "\n")
         f.write("print %s(%s)\n" % (funcname, arglist))
     #
     # run a child pypy-c with logging enabled
     logfile = self.filepath.new(ext='.log')
     #
     cmdline = [sys.executable]
     if not import_site:
         cmdline.append('-S')
     if jitopts:
         jitcmdline = ['%s=%s' % (key, value)
                       for key, value in jitopts.items()]
         cmdline += ['--jit', ','.join(jitcmdline)]
     cmdline.append(str(self.filepath))
     #
     env = os.environ.copy()
     env['PYPYLOG'] = self.log_string + ':' + str(logfile)
     pipe = subprocess.Popen(cmdline,
                             env=env,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
     stdout, stderr = pipe.communicate()
     if getattr(pipe, 'returncode', 0) < 0:
         raise IOError("subprocess was killed by signal %d" % (
             pipe.returncode,))
     if stderr.startswith('SKIP:'):
         py.test.skip(stderr)
     if stderr.startswith('debug_alloc.h:'):   # lldebug builds
         stderr = ''
     assert not stderr
     #
     # parse the JIT log
     rawlog = logparser.parse_log_file(str(logfile))
     rawtraces = logparser.extract_category(rawlog, 'jit-log-opt-')
     log = Log(rawtraces)
     log.result = eval(stdout)
     log.logfile = str(logfile)
     #
     summaries  = logparser.extract_category(rawlog, 'jit-summary')
     if len(summaries) > 0:
         log.jit_summary = parse_prof(summaries[-1])
     else:
         log.jit_summary = None
     #
     return log
Exemple #3
0
def import_log(logname, ParserCls=SimpleParser):
    log = parse_log_file(logname)
    hex_re = '0x(-?[\da-f]+)'
    addrs = {}
    for entry in extract_category(log, 'jit-backend-addr'):
        m = re.search('bootstrap ' + hex_re, entry)
        if not m:
            # a bridge
            m = re.search('has address ' + hex_re, entry)
            addr = int(m.group(1), 16)
            entry = entry.lower()
            m = re.search('guard ' + hex_re, entry)
            name = 'guard ' + m.group(1)
        else:
            name = entry[:entry.find('(') - 1].lower()
            addr = int(m.group(1), 16)
        addrs.setdefault(addr, []).append(name)
    from rpython.jit.backend.tool.viewcode import World
    world = World()
    for entry in extract_category(log, 'jit-backend-dump'):
        world.parse(entry.splitlines(True))
    dumps = {}
    for r in world.ranges:
        if r.addr in addrs and addrs[r.addr]:
            name = addrs[r.addr].pop(0) # they should come in order
            data = r.data.encode('hex')       # backward compatibility
            dumps[name] = (world.backend_name, r.addr, data)
    loops = []
    cat = extract_category(log, 'jit-log-opt')
    if not cat:
        extract_category(log, 'jit-log-rewritten')
    if not cat:
        extract_category(log, 'jit-log-noopt')        
    for entry in cat:
        parser = ParserCls(entry, None, {}, 'lltype', None,
                           nonstrict=True)
        loop = parser.parse()
        comm = loop.comment
        comm = comm.lower()
        if comm.startswith('# bridge'):
            m = re.search('guard 0x(-?[\da-f]+)', comm)
            name = 'guard ' + m.group(1)
        elif "(" in comm:
            name = comm[2:comm.find('(')-1]
        else:
            name = " ".join(comm[2:].split(" ", 2)[:2])
        if name in dumps:
            bname, start_ofs, dump = dumps[name]
            loop.force_asm = (lambda dump=dump, start_ofs=start_ofs,
                              bname=bname, loop=loop:
                              parser.postprocess(loop, backend_tp=bname,
                                                 backend_dump=dump,
                                                 dump_start=start_ofs))
        loops += split_trace(loop)
    return log, loops
Exemple #4
0
    def run(self, topaz, tmpdir, code):
        tmpdir.join("t.rb").write(code)
        proc = subprocess.Popen(
            [str(topaz), str(tmpdir.join("t.rb"))],
            cwd=str(tmpdir),
            env={"PYPYLOG": "jit-log-opt:%s" % tmpdir.join("x.pypylog")}
        )
        proc.wait()
        data = logparser.parse_log_file(str(tmpdir.join("x.pypylog")), verbose=False)
        data = logparser.extract_category(data, "jit-log-opt-")

        storage = LoopStorage()
        traces = [SimpleParser.parse_from_input(t) for t in data]
        traces = storage.reconnect_loops(traces)
        return [Trace(t) for t in traces]
Exemple #5
0
def extract_traces(file, remove_debug=True, remove_main_labels=True, remove_all_labels=False):
    data = logparser.parse_log_file(file, verbose=False)
    data = logparser.extract_category(data, "jit-log-opt-")
    
    storage = LoopStorage()
    traces = [SimpleParser.parse_from_input(t) for t in data]
    main_loops = storage.reconnect_loops(traces)
    traces_w = []
    for trace in traces:
        if trace in main_loops:
            traces_w.append(Trace(trace))
        else:
            traces_w[len(traces_w) - 1].addbridge(trace)
    for trace in traces_w:
        trace.parse(remove_debug, remove_main_labels, remove_all_labels)
    return traces_w
 def run(self, src, call, **jitopts):
     jitopts.setdefault('threshold', 200)
     # write the snippet
     with self.filepath.open("w") as f:
         # we don't want to see the small bridges created
         # by the checkinterval reaching the limit
         f.write(str(src) + "\n")
     #
     # run a child pyrolog-c with logging enabled
     logfile = self.filepath.new(ext='.log')
     #
     cmdline = [strexecutable]
     cmdline.append(str(self.filepath))
     #
     print cmdline, logfile
     env={'PYPYLOG': 'jit-log-opt,jit-summary:' + str(logfile)}
     #env={'PYPYLOG': ':' + str(logfile)}
     pipe = subprocess.Popen(cmdline,
                             env=env,
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
     pipe.stdin.write(call + "\n")
     stdout, stderr = pipe.communicate()
     if stderr.startswith('SKIP:'):
         py.test.skip(stderr)
     if stderr.startswith('debug_alloc.h:'):   # lldebug builds
         stderr = ''
     assert not stderr
     #
     # parse the JIT log
     rawlog = logparser.parse_log_file(str(logfile))
     rawtraces = logparser.extract_category(rawlog, 'jit-log-opt-')
     log = Log(rawtraces)
     log.result = stdout
     if "ParseError" in log.result or "ERROR" in log.result:
         assert 0, log.result
     #
     summaries  = logparser.extract_category(rawlog, 'jit-summary')
     if len(summaries) > 0:
         log.jit_summary = parse_prof(summaries[-1])
     else:
         log.jit_summary = None
     #
     return log
Exemple #7
0
def main(argv):
    log = logparser.parse_log_file(argv[0])
    parts = logparser.extract_category(log, "jit-log-opt-")
    for i, oplist in enumerate(parts):
        loop = parse(oplist, no_namespace=True, nonstrict=True)
        num_ops = 0
        num_dmp = 0
        num_guards = 0
        for op in loop.operations:
            if op.getopnum() == rop.DEBUG_MERGE_POINT:
                num_dmp += 1
            else:
                num_ops += 1
            if op.is_guard():
                num_guards += 1
        if num_dmp == 0:
            print "Loop #%d, length: %d, opcodes: %d, guards: %d" % (i, num_ops, num_dmp, num_guards)
        else:
            print "Loop #%d, length: %d, opcodes: %d, guards: %d, %f" % (i, num_ops, num_dmp, num_guards, num_ops/num_dmp)
Exemple #8
0
def import_log(logname, ParserCls=SimpleParser):
    log = parse_log_file(logname)
    addrs = parse_addresses(extract_category(log, 'jit-backend-addr'))
    from rpython.jit.backend.tool.viewcode import World
    world = World()
    for entry in extract_category(log, 'jit-backend-dump'):
        world.parse(entry.splitlines(True))
    dumps = {}
    for r in world.ranges:
        for pos1 in range(r.addr, r.addr + len(r.data)):
            if pos1 in addrs and addrs[pos1]:
                name = addrs[pos1].pop(0) # they should come in order
                data = r.data.encode('hex')
                dumps[name] = (world.backend_name, r.addr, data)
    loops = []
    cat = extract_category(log, 'jit-log-opt')
    if not cat:
        cat = extract_category(log, 'jit-log-rewritten')
    if not cat:
        cat = extract_category(log, 'jit-log-noopt')        
    for entry in cat:
        parser = ParserCls(entry, None, {}, 'lltype', None,
                           nonstrict=True)
        loop = parser.parse()
        comm = loop.comment
        comm = comm.lower()
        if comm.startswith('# bridge'):
            m = re.search('guard 0x(-?[\da-f]+)', comm)
            name = 'guard ' + m.group(1)
        elif "(" in comm:
            name = comm[2:comm.find('(')-1]
        else:
            name = " ".join(comm[2:].split(" ", 2)[:2])
        if name in dumps:
            bname, start_ofs, dump = dumps[name]
            loop.force_asm = (lambda dump=dump, start_ofs=start_ofs,
                              bname=bname, loop=loop:
                              parser.postprocess(loop, backend_tp=bname,
                                                 backend_dump=dump,
                                                 dump_start=start_ofs))
        loops += split_trace(loop)
    return log, loops
Exemple #9
0
    def run(self, spy, tmpdir, code):
        proc = subprocess.Popen(
            [str(spy), "-r", code.replace("\n", "\r\n"), BenchmarkImage],
            cwd=str(tmpdir),
            env={"PYPYLOG": "jit-log-opt:%s" % tmpdir.join("x.pypylog")}
        )
        proc.wait()
        data = logparser.parse_log_file(str(tmpdir.join("x.pypylog")), verbose=False)
        data = logparser.extract_category(data, "jit-log-opt-")

        storage = LoopStorage()
        traces = [SimpleParser.parse_from_input(t) for t in data]
        main_loops = storage.reconnect_loops(traces)
        traces_w = []
        for trace in traces:
            if trace in main_loops:
                traces_w.append(Trace(trace))
            else:
                traces_w[len(traces_w) - 1].addbridge(trace)
        return traces_w
Exemple #10
0
def main(loopfile, use_threshold, view=True):
    countname = py.path.local(loopfile + '.count')
    if countname.check():
        #counts = [line.split(':', 1) for line in countname.readlines()]
        #counts = Counts([('<code' + k.strip("\n"), int(v.strip('\n').strip()))
        #                 for v, k in counts])
        counts = Counts([])
        l = list(sorted(counts.values()))
        if len(l) > 20 and use_threshold:
            counts.threshold = l[-20]
        else:
            counts.threshold = 0
        for_print = [(v, k) for k, v in counts.iteritems()]
        for_print.sort()
    else:
        counts = {}
    log = logparser.parse_log_file(loopfile)
    loops = logparser.extract_category(log, "jit-log-opt-")
    real_loops, allloops = splitloops(loops)
    postprocess(real_loops, allloops, counts)
    if view:
        Page(allloops, counts).display()
Exemple #11
0
            return _Page(self.graph_builder)

# ____________________________________________________________

if __name__ == '__main__':
    if '--text' in sys.argv:
        sys.argv.remove('--text')
        showgraph = False
    else:
        showgraph = True
    if len(sys.argv) != 2:
        print >> sys.stderr, __doc__
        sys.exit(2)
    #
    import cStringIO
    from rpython.tool import logparser
    log1 = logparser.parse_log_file(sys.argv[1])
    text1 = logparser.extract_category(log1, catprefix='jit-backend-dump')
    f = cStringIO.StringIO()
    f.writelines(text1)
    f.seek(0)
    del log1, text1
    #
    world = World()
    world.parse(f)
    if showgraph:
        world.find_cross_references()
        world.show(showtext=True)
    else:
        world.showtextonly()
Exemple #12
0
import re
from rpython.tool import logparser

log = logparser.parse_log_file("log")
log = logparser.extract_category(log, "jit-log-opt")
d = {}
counted = 0
not_counted = 0
result = {}
for item in log:
    first = item.split("\n")[0]
    if '# Loop' in first:
        m = re.search("<code object (.*), file '(.*)', line (\d+)", first)
        if not m:
            continue
        key = (m.group(1), m.group(2), int(m.group(3)))
        for i, op in enumerate(item.split("\n")[2:-1]):
            if 'guard_' in op:
                m = re.search("descr=<Guard(0x[0-9a-f]+)>", op)
                d[int(m.group(1), 16)] = (key, i)
                #print op, i, int(m.group(1), 16)
    else:
        m = re.search("bridge out of Guard (0x[a-f0-9]+)", first)
        bridge_no = int(m.group(1), 16)
        if bridge_no in d:
            result.setdefault(d[bridge_no][0], []).append((bridge_no, d[bridge_no][1]))
        else:
            not_counted += 1
        for i, op in enumerate(item.split("\n")[2:-1]):
            if 'guard_' in op: