def run(self, func_or_src, args=[], import_site=False, **jitopts):
jitopts.setdefault('threshold', 200)
src = py.code.Source(func_or_src)
if isinstance(func_or_src, types.FunctionType):
funcname = func_or_src.func_name
else:
funcname = 'main'
# write the snippet
arglist = ', '.join(map(repr, args))
with self.filepath.open("w") as f:
# we don't want to see the small bridges created
# by the checkinterval reaching the limit
f.write("import sys\n")
f.write("sys.setcheckinterval(10000000)\n")
f.write(str(src) + "\n")
f.write("print %s(%s)\n" % (funcname, arglist))
#
# run a child pypy-c with logging enabled
logfile = self.filepath.new(ext='.log')
#
cmdline = [sys.executable]
if not import_site:
cmdline.append('-S')
if jitopts:
jitcmdline = ['%s=%s' % (key, value)
for key, value in jitopts.items()]
cmdline += ['--jit', ','.join(jitcmdline)]
cmdline.append(str(self.filepath))
#
env = os.environ.copy()
env['PYPYLOG'] = self.log_string + ':' + str(logfile)
pipe = subprocess.Popen(cmdline,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = pipe.communicate()
if getattr(pipe, 'returncode', 0) < 0:
raise IOError("subprocess was killed by signal %d" % (
pipe.returncode,))
if stderr.startswith('SKIP:'):
py.test.skip(stderr)
if stderr.startswith('debug_alloc.h:'): # lldebug builds
stderr = ''
assert not stderr
#
# parse the JIT log
rawlog = logparser.parse_log_file(str(logfile))
rawtraces = logparser.extract_category(rawlog, 'jit-log-opt-')
log = Log(rawtraces)
log.result = eval(stdout)
#
summaries = logparser.extract_category(rawlog, 'jit-summary')
if len(summaries) > 0:
log.jit_summary = parse_prof(summaries[-1])
else:
log.jit_summary = None
#
return log
def main(loopfile, view=True):
log = logparser.parse_log_file(loopfile)
loops = logparser.extract_category(log, "jit-log-opt-")
allloops = splitloops(loops)
postprocess(allloops)
if view:
Page(allloops).display()
def test_really_run():
""" This test checks whether output of jitprof did not change.
It'll explode when someone touches jitprof.py
"""
mydriver = JitDriver(reds = ['i', 'n'], greens = [])
def f(n):
i = 0
while i < n:
mydriver.can_enter_jit(i=i, n=n)
mydriver.jit_merge_point(i=i, n=n)
i += 1
cap = py.io.StdCaptureFD()
try:
ll_meta_interp(f, [10], CPUClass=runner.LLtypeCPU, type_system='lltype',
ProfilerClass=Profiler)
finally:
out, err = cap.reset()
log = parse_log(err.splitlines(True))
err_sections = list(extract_category(log, 'jit-summary'))
[err1] = err_sections # there should be exactly one jit-summary
assert err1.count("\n") == JITPROF_LINES
info = parse_prof(err1)
# assert did not crash
# asserts below are a bit delicate, possibly they might be deleted
assert info.tracing_no == 1
assert info.backend_no == 1
assert info.ops.total == 2
assert info.recorded_ops.total == 2
assert info.recorded_ops.calls == 0
assert info.guards == 1
assert info.opt_ops == 13
assert info.opt_guards == 2
assert info.forcings == 0
def import_log(logname, ParserCls=SimpleParser):
log = parse_log_file(logname)
addrs = {}
for entry in extract_category(log, 'jit-backend-addr'):
m = re.search('bootstrap ([-\da-f]+)', entry)
if not m:
# a bridge
m = re.search('has address ([-\da-f]+)', entry)
addr = int(m.group(1), 16)
entry = entry.lower()
m = re.search('guard \d+', entry)
name = m.group(0)
else:
name = entry[:entry.find('(') - 1].lower()
addr = int(m.group(1), 16)
addrs.setdefault(addr, []).append(name)
dumps = {}
for entry in extract_category(log, 'jit-backend-dump'):
backend, _, dump, _ = entry.split("\n")
_, addr, _, data = re.split(" +", dump)
backend_name = backend.split(" ")[1]
addr = int(addr[1:], 16)
if addr in addrs and addrs[addr]:
name = addrs[addr].pop(0) # they should come in order
dumps[name] = (backend_name, addr, data)
loops = []
for entry in extract_category(log, 'jit-log-opt'):
parser = ParserCls(entry, None, {}, 'lltype', None,
nonstrict=True)
loop = parser.parse()
comm = loop.comment
comm = comm.lower()
if comm.startswith('# bridge'):
m = re.search('guard \d+', comm)
name = m.group(0)
else:
name = " ".join(comm[2:].split(" ", 2)[:2])
if name in dumps:
bname, start_ofs, dump = dumps[name]
loop.force_asm = (lambda dump=dump, start_ofs=start_ofs,
bname=bname, loop=loop:
parser.postprocess(loop, backend_tp=bname,
backend_dump=dump,
dump_start=start_ofs))
loops += split_trace(loop)
return log, loops
def import_log(logname, ParserCls=SimpleParser):
log = parse_log_file(logname)
addrs = {}
for entry in extract_category(log, 'jit-backend-addr'):
m = re.search('bootstrap ([-\da-f]+)', entry)
if not m:
# a bridge
m = re.search('has address ([-\da-f]+)', entry)
addr = int(m.group(1), 16)
entry = entry.lower()
m = re.search('guard \d+', entry)
name = m.group(0)
else:
name = entry[:entry.find('(') - 1].lower()
addr = int(m.group(1), 16)
addrs.setdefault(addr, []).append(name)
dumps = {}
for entry in extract_category(log, 'jit-backend-dump'):
backend, _, dump, _ = entry.split("\n")
_, addr, _, data = re.split(" +", dump)
backend_name = backend.split(" ")[1]
addr = int(addr[1:], 16)
if addr in addrs and addrs[addr]:
name = addrs[addr].pop(0) # they should come in order
dumps[name] = (backend_name, addr, data)
loops = []
for entry in extract_category(log, 'jit-log-opt'):
parser = ParserCls(entry, None, {}, 'lltype', None,
nonstrict=True)
loop = parser.parse()
comm = loop.comment
comm = comm.lower()
if comm.startswith('# bridge'):
m = re.search('guard \d+', comm)
name = m.group(0)
else:
name = comm[2:comm.find(':')-1]
if name in dumps:
bname, start_ofs, dump = dumps[name]
loop.force_asm = (lambda dump=dump, start_ofs=start_ofs,
bname=bname, loop=loop:
parser.postprocess(loop, backend_tp=bname,
backend_dump=dump,
dump_start=start_ofs))
loops.append(loop)
return log, loops
def getMySources(filename):
extra_path = os.path.dirname(filename)
storage = LoopStorage(extra_path)
log, loops = import_log(filename, ParserWithHtmlRepr)
parse_log_counts(extract_category(log, "jit-backend-count"), loops)
storage.loops = [loop for loop in loops if not loop.descr.startswith("bridge")]
[l.force_asm() for l in storage.loops]
storage.loop_dict = create_loop_dict(loops)
loops = index(storage)
ids = [item.descr for item in loops]
mySources = []
for item in ids:
source, name, up, filename, startline, callstack = loopfunc(item, storage)
i = 1
mySource = []
pending = False
pendingLine = 0
pendingCode = ""
for sourceline in source.lines:
# print str(i) + ": " + sourceline.line
line = sourceline.line
chunks = []
if sourceline.in_loop and line.strip != "":
if sourceline.chunks:
makePending = True
for chunk in sourceline.chunks:
if chunk.is_bytecode:
ops = []
for op in chunk.operations:
if op.name != "debug_merge_point":
op_repr = (op.html_repr(), op.asm)
ops.append(op_repr)
chunks.append((chunk.html_repr(), chunk, ops))
if len(chunks) == 0:
if pending:
pendingCode += "\n" + line
else:
pendingCode = line
pendingLine = i
pending = True
else:
if pending:
pending = False
mySource.append((pendingLine, pendingCode, None))
mySource.append((i, line, chunks))
i += 1
if pending:
pending = False
mySource.append((pendingLine, pendingCode, None))
mySources.append((filename, mySource))
return mySources
def consider_category(log, options, category):
loops = logparser.extract_category(log, category)
if options.loopnum is None:
input_loops = loops
else:
input_loops = [loops[options.loopnum]]
loops = [parse(inp, no_namespace=True, nonstrict=True)
for inp in input_loops]
summary = {}
for loop in loops:
summary = loop.summary(summary)
return loops, summary
def main():
filename = sys.argv[1]
extra_path = os.path.dirname(filename)
storage = LoopStorage(extra_path)
log, loops = import_log(filename, ParserWithHtmlRepr)
parse_log_counts(extract_category(log, 'jit-backend-count'), loops)
storage.loops = [loop for loop in loops
if not loop.descr.startswith('bridge')]
storage.loop_dict = create_loop_dict(loops)
print loops,log,storage
server = Server(filename, storage)
def consider_category(log, options, category):
loops = logparser.extract_category(log, category)
if options.loopnum is None:
input_loops = loops
else:
input_loops = [loops[options.loopnum]]
loops = [
parse(inp, no_namespace=True, nonstrict=True) for inp in input_loops
]
summary = {}
for loop in loops:
summary = loop.summary(summary)
return loops, summary
def main(argv):
log = logparser.parse_log_file(argv[0])
parts = logparser.extract_category(log, "jit-log-opt-")
for i, oplist in enumerate(parts):
loop = parse(oplist, no_namespace=True)
num_ops = 0
num_dmp = 0
num_guards = 0
for op in loop.operations:
if op.opnum == rop.DEBUG_MERGE_POINT:
num_dmp += 1
else:
num_ops += 1
if op.is_guard():
num_guards += 1
print "Loop #%d, length: %d, opcodes: %d, guards: %d, %f" % (i, num_ops, num_dmp, num_guards, num_ops/num_dmp)
def main(argv):
log = logparser.parse_log_file(argv[0])
parts = logparser.extract_category(log, "jit-log-opt-")
for i, oplist in enumerate(parts):
loop = parse(oplist, no_namespace=True)
num_ops = 0
num_dmp = 0
num_guards = 0
for op in loop.operations:
if op.opnum == rop.DEBUG_MERGE_POINT:
num_dmp += 1
else:
num_ops += 1
if op.is_guard():
num_guards += 1
print "Loop #%d, length: %d, opcodes: %d, guards: %d, %f" % (
i, num_ops, num_dmp, num_guards, num_ops / num_dmp)
def parse_loops(self, opslogfile):
from pypy.jit.metainterp.test.oparser import parse
from pypy.tool import logparser
assert opslogfile.check()
log = logparser.parse_log_file(str(opslogfile))
parts = logparser.extract_category(log, 'jit-log-opt-')
# skip entry bridges, they can contain random things
self.loops = [parse(part, no_namespace=True) for part in parts
if not from_entry_bridge(part, parts)]
self.sliced_loops = [] # contains all bytecodes of all loops
self.total_ops = 0
for loop in self.loops:
self.total_ops += len(loop.operations)
for op in loop.operations:
if op.getopname() == "debug_merge_point":
sliced_loop = BytecodeTrace()
sliced_loop.bytecode = op.args[0]._get_str().rsplit(" ", 1)[1]
self.sliced_loops.append(sliced_loop)
else:
sliced_loop.append(op)
self.check_0_op_bytecodes()
def main(loopfile, use_threshold, view=True):
countname = py.path.local(loopfile + '.count')
if countname.check():
#counts = [line.split(':', 1) for line in countname.readlines()]
#counts = Counts([('<code' + k.strip("\n"), int(v.strip('\n').strip()))
# for v, k in counts])
counts = Counts([])
l = list(sorted(counts.values()))
if len(l) > 20 and use_threshold:
counts.threshold = l[-20]
else:
counts.threshold = 0
for_print = [(v, k) for k, v in counts.iteritems()]
for_print.sort()
else:
counts = {}
log = logparser.parse_log_file(loopfile)
loops = logparser.extract_category(log, "jit-log-opt-")
real_loops, allloops = splitloops(loops)
postprocess(real_loops, allloops, counts)
if view:
Page(allloops, counts).display()
def main():
if not '__pypy__' in sys.builtin_module_names:
print "Please run it using pypy-c"
sys.exit(1)
#
server_mode = True
if '--qt' in sys.argv:
server_mode = False
sys.argv.remove('--qt')
#
if len(sys.argv) != 2 and len(sys.argv) != 3:
print __doc__
sys.exit(1)
filename = sys.argv[1]
extra_path = os.path.dirname(filename)
if len(sys.argv) != 3:
port = 5000
else:
port = int(sys.argv[2])
storage = LoopStorage(extra_path)
log, loops = import_log(filename, ParserWithHtmlRepr)
parse_log_counts(extract_category(log, 'jit-backend-count'), loops)
storage.loops = [loop for loop in loops
if not loop.descr.startswith('bridge')]
storage.loop_dict = create_loop_dict(loops)
print loops,log,storage
app = OverrideFlask('_jitviewer')
server = Server(filename, storage)
app.debug = True
app.route('/')(server.index)
app.route('/loop')(server.loop)
def run():
app.run(use_reloader=False, host='0.0.0.0', port=port)
if server_mode:
run()
else:
url = "http://localhost:%d/" % port
run_server_and_browser(app, run, url, filename)
def main(loopfile, use_threshold, view=True):
countname = py.path.local(loopfile + '.count')
if countname.check():
#counts = [line.split(':', 1) for line in countname.readlines()]
#counts = Counts([('<code' + k.strip("\n"), int(v.strip('\n').strip()))
# for v, k in counts])
counts = Counts([])
l = list(sorted(counts.values()))
if len(l) > 20 and use_threshold:
counts.threshold = l[-20]
else:
counts.threshold = 0
for_print = [(v, k) for k, v in counts.iteritems()]
for_print.sort()
else:
counts = {}
log = logparser.parse_log_file(loopfile)
loops = logparser.extract_category(log, "jit-log-opt-")
real_loops, allloops = splitloops(loops)
postprocess(real_loops, allloops, counts)
if view:
Page(allloops, counts).display()
def test_really_run():
""" This test checks whether output of jitprof did not change.
It'll explode when someone touches jitprof.py
"""
mydriver = JitDriver(reds=['i', 'n'], greens=[])
def f(n):
i = 0
while i < n:
mydriver.can_enter_jit(i=i, n=n)
mydriver.jit_merge_point(i=i, n=n)
i += 1
cap = py.io.StdCaptureFD()
try:
ll_meta_interp(f, [10],
CPUClass=runner.LLtypeCPU,
type_system='lltype',
ProfilerClass=Profiler)
finally:
out, err = cap.reset()
log = parse_log(err.splitlines(True))
err_sections = list(extract_category(log, 'jit-summary'))
[err1] = err_sections # there should be exactly one jit-summary
assert err1.count("\n") == JITPROF_LINES
info = parse_prof(err1)
# assert did not crash
# asserts below are a bit delicate, possibly they might be deleted
assert info.tracing_no == 1
assert info.backend_no == 1
assert info.ops.total == 2
assert info.recorded_ops.total == 2
assert info.recorded_ops.calls == 0
assert info.guards == 1
assert info.opt_ops == 13
assert info.opt_guards == 2
assert info.forcings == 0
def parse_loops(self, opslogfile):
from pypy.jit.metainterp.test.oparser import parse
from pypy.tool import logparser
assert opslogfile.check()
log = logparser.parse_log_file(str(opslogfile))
parts = logparser.extract_category(log, 'jit-log-opt-')
# skip entry bridges, they can contain random things
self.loops = [
parse(part, no_namespace=True) for part in parts
if not from_entry_bridge(part, parts)
]
self.sliced_loops = [] # contains all bytecodes of all loops
self.total_ops = 0
for loop in self.loops:
self.total_ops += len(loop.operations)
for op in loop.operations:
if op.getopname() == "debug_merge_point":
sliced_loop = BytecodeTrace()
sliced_loop.bytecode = op.args[0]._get_str().rsplit(
" ", 1)[1]
self.sliced_loops.append(sliced_loop)
else:
sliced_loop.append(op)
self.check_0_op_bytecodes()
def main(loopnum, loopfile):
log = logparser.parse_log_file(loopfile)
loops = logparser.extract_category(log, "jit-log-opt-")
inp = loops[loopnum]
loop = parse(inp, no_namespace=True)
loop.show()
import autopath
import sys, re
from pypy.tool import logparser
# fflush(pypy_debug_file)
if len(sys.argv) != 3:
print "Usage: %s <log file> <address>" % sys.argv[0]
log = logparser.parse_log_file(sys.argv[1])
text = logparser.extract_category(log, catprefix="jit-backend")
address = int(sys.argv[2], 16)
for l in text:
m = re.match("(Loop|Bridge)(.*?) \(.*has address (\w+) to (\w+)", l)
if m is not None:
trace = m.group(1) + m.group(2)
start = int(m.group(3), 16)
stop = int(m.group(4), 16)
if start <= address <= stop:
offset = address - start
print trace
print "at offset ", offset
break
else:
print "Not found"
exit(0)
if trace.startswith("Bridge"):
cat = "jit-log-opt-bridge"
else:
import autopath
import sys, re
from pypy.tool import logparser
# fflush(pypy_debug_file)
if len(sys.argv) != 3:
print "Usage: %s <log file> <address>" % sys.argv[0]
log = logparser.parse_log_file(sys.argv[1])
text = logparser.extract_category(log, catprefix='jit-backend')
address = int(sys.argv[2], 16)
for l in text:
m = re.match('(Loop|Bridge)(.*?) \(.*has address (\w+) to (\w+)', l)
if m is not None:
trace = m.group(1) + m.group(2)
start = int(m.group(3), 16)
stop = int(m.group(4), 16)
if start <= address <= stop:
offset = address - start
print trace
print 'at offset ', offset
break
else:
print "Not found"
exit(0)
if trace.startswith('Bridge'):
cat = 'jit-log-opt-bridge'
else:
return _Page(self.graph_builder)
# ____________________________________________________________
if __name__ == '__main__':
if '--text' in sys.argv:
sys.argv.remove('--text')
showgraph = False
else:
showgraph = True
if len(sys.argv) != 2:
print >> sys.stderr, __doc__
sys.exit(2)
#
import cStringIO
from pypy.tool import logparser
log1 = logparser.parse_log_file(sys.argv[1])
text1 = logparser.extract_category(log1, catprefix='jit-backend-dump')
f = cStringIO.StringIO()
f.writelines(text1)
f.seek(0)
del log1, text1
#
world = World()
world.parse(f)
if showgraph:
world.find_cross_references()
world.show(showtext=True)
else:
world.showtextonly()
# ____________________________________________________________
if __name__ == '__main__':
if '--text' in sys.argv:
sys.argv.remove('--text')
showgraph = False
else:
showgraph = True
if len(sys.argv) != 2:
print >> sys.stderr, __doc__
sys.exit(2)
#
import cStringIO
from pypy.tool import logparser
log1 = logparser.parse_log_file(sys.argv[1])
text1 = logparser.extract_category(log1, catprefix='jit-backend-dump')
f = cStringIO.StringIO()
f.writelines(text1)
f.seek(0)
del log1, text1
#
world = World()
world.parse(f)
if showgraph:
world.find_cross_references()
world.show(showtext=True)
else:
world.showtextonly()
def main(loopnum, loopfile):
log = logparser.parse_log_file(loopfile)
loops = logparser.extract_category(log, "jit-log-opt-")
inp = loops[loopnum]
loop = parse(inp, no_namespace=True)
loop.show()
