def main(progname, logfilename, outfilename):
storage = LoopStorage(extrapath=os.path.dirname(progname))
log, loops = import_log(logfilename)
parse_log_counts(extract_category(log, 'jit-backend-count'), loops)
storage.loops = [loop for loop in loops
if not loop.descr.startswith('bridge')]
storage.loop_dict = create_loop_dict(loops)
json.dump([loop.force_asm().as_json() for loop in storage.loops],
open(outfilename, "w"), indent=4)
def __init__(self, rawtraces):
storage = LoopStorage()
traces = [
SimpleParser.parse_from_input(rawtrace) for rawtrace in rawtraces
]
traces = storage.reconnect_loops(traces)
self.loops = [
TraceWithIds.from_trace(trace, storage) for trace in traces
]
def main(progname, logfilename, outfilename):
storage = LoopStorage(extrapath=os.path.dirname(progname))
log, loops = import_log(logfilename)
parse_log_counts(extract_category(log, 'jit-backend-count'), loops)
storage.loops = [
loop for loop in loops if not loop.descr.startswith('bridge')
]
storage.loop_dict = create_loop_dict(loops)
json.dump([loop.force_asm().as_json() for loop in storage.loops],
open(outfilename, "w"),
indent=4)
def run(self, topaz, tmpdir, code):
tmpdir.join("t.rb").write(code)
proc = subprocess.Popen(
[str(topaz), str(tmpdir.join("t.rb"))],
cwd=str(tmpdir),
env={"PYPYLOG": "jit-log-opt:%s" % tmpdir.join("x.pypylog")}
)
proc.wait()
data = logparser.parse_log_file(str(tmpdir.join("x.pypylog")), verbose=False)
data = logparser.extract_category(data, "jit-log-opt-")
storage = LoopStorage()
traces = [SimpleParser.parse_from_input(t) for t in data]
traces = storage.reconnect_loops(traces)
return [Trace(t) for t in traces]
def run(self, topaz, tmpdir, code):
tmpdir.join("t.rb").write(code)
proc = subprocess.Popen(
[str(topaz), str(tmpdir.join("t.rb"))],
cwd=str(tmpdir),
env={"PYPYLOG": "jit-log-opt:%s" % tmpdir.join("x.pypylog")})
proc.wait()
data = logparser.parse_log_file(str(tmpdir.join("x.pypylog")),
verbose=False)
data = logparser.extract_category(data, "jit-log-opt-")
storage = LoopStorage()
traces = [SimpleParser.parse_from_input(t) for t in data]
traces = storage.reconnect_loops(traces)
return [Trace(t) for t in traces]
def test_parsing_strliteral():
loop = parse("""
debug_merge_point(0, 0, 'StrLiteralSearch at 11/51 [17, 8, 3, 1, 1, 1, 1, 51, 0, 19, 51, 1]')
""")
ops = Function.from_operations(loop.operations, LoopStorage())
chunk = ops.chunks[0]
assert chunk.bytecode_name.startswith('StrLiteralSearch')
def extract_traces(file, remove_debug=True, remove_main_labels=True, remove_all_labels=False):
data = logparser.parse_log_file(file, verbose=False)
data = logparser.extract_category(data, "jit-log-opt-")
storage = LoopStorage()
traces = [SimpleParser.parse_from_input(t) for t in data]
main_loops = storage.reconnect_loops(traces)
traces_w = []
for trace in traces:
if trace in main_loops:
traces_w.append(Trace(trace))
else:
traces_w[len(traces_w) - 1].addbridge(trace)
for trace in traces_w:
trace.parse(remove_debug, remove_main_labels, remove_all_labels)
return traces_w
def test_parse_nonpython():
loop = parse("""
[]
debug_merge_point(0, 0, 'random')
""")
f = Function.from_operations(loop.operations, LoopStorage())
assert f.filename is None
def test_parse_non_code():
ops = parse('''
[]
debug_merge_point(0, 0, "SomeRandomStuff")
''')
res = Function.from_operations(ops.operations, LoopStorage())
assert len(res.chunks) == 1
assert 'SomeRandomStuff' in res.chunks[0].repr()
def extract_traces(file, remove_debug=True, remove_main_labels=True,
remove_all_labels=False):
data = logparser.parse_log_file(file, verbose=False)
data = logparser.extract_category(data, "jit-log-opt-")
storage = LoopStorage()
traces = [SimpleParser.parse_from_input(t) for t in data]
main_loops = storage.reconnect_loops(traces)
traces_w = []
for trace in traces:
if trace in main_loops:
traces_w.append(Trace(trace))
else:
traces_w[len(traces_w) - 1].addbridge(trace)
for trace in traces_w:
trace.parse(remove_debug, remove_main_labels, remove_all_labels)
return traces_w
def test_parse_from_inside():
loop = parse("""
[]
debug_merge_point(1, 0, 'two')
debug_merge_point(2, 0, 'three')
debug_merge_point(0, 0, 'one')
""")
f = Function.from_operations(loop.operations, LoopStorage())
assert len(f.chunks) == 2
def test_lineno():
fname = str(py.path.local(__file__).join('..', 'x.py'))
ops = parse('''
[i0, i1]
debug_merge_point(0, 0, "<code object f. file '%(fname)s'. line 5> #0 LOAD_FAST")
debug_merge_point(0, 0, "<code object f. file '%(fname)s'. line 5> #3 LOAD_FAST")
debug_merge_point(0, 0, "<code object f. file '%(fname)s'. line 5> #6 BINARY_ADD")
debug_merge_point(0, 0, "<code object f. file '%(fname)s'. line 5> #7 RETURN_VALUE")
''' % locals())
res = Function.from_operations(ops.operations, LoopStorage())
assert res.chunks[1].lineno == 6
def test_embedded_lineno():
# debug_merge_point() can have a text that is either:
#
# * the PyPy2's <code object %s. file '%s'. line %d> #%d %s>
# funcname, filename, lineno, bytecode_no, bytecode_name
#
# * a standard text of the form %s;%s:%d-%d-%d %s
# funcname, filename, startlineno, curlineno, endlineno, anything
#
# * or anything else, which is not specially recognized but shouldn't crash
#
sourcefile = str(udir.join('test_embedded_lineno.src'))
with open(sourcefile, 'w') as f:
print >> f, "A#1"
print >> f, "B#2"
print >> f, "C#3"
print >> f, "D#4"
print >> f, "E#5"
print >> f, "F#6"
loop = parse("""
[]
debug_merge_point(0, 0, 'myfunc;%(filename)s:2-2~one')
debug_merge_point(0, 0, 'myfunc;%(filename)s:2-2~two')
debug_merge_point(0, 0, 'myfunc;%(filename)s:2-4~')
debug_merge_point(0, 0, 'myfunc;%(filename)s:2-4~four')
""" % {'filename': sourcefile})
f = Function.from_operations(loop.operations, LoopStorage())
expect = [(2, 'one', True),
(2, 'two', False),
(4, '', True),
(4, 'four', False)]
assert len(f.chunks) == len(expect)
code_seen = set()
for chunk, (expected_lineno,
expected_bytecode_name,
expected_line_starts_here) in zip(f.chunks, expect):
assert chunk.name == 'myfunc'
assert chunk.bytecode_name == expected_bytecode_name
assert chunk.filename == sourcefile
assert chunk.startlineno == 2
assert chunk.bytecode_no == ~expected_lineno # half-abuse
assert chunk.has_valid_code()
assert chunk.lineno == expected_lineno
assert chunk.line_starts_here == expected_line_starts_here
code_seen.add(chunk.code)
assert len(code_seen) == 1
code, = code_seen
assert code.source[0] == "B#2"
assert code.source[1] == "C#3"
assert code.source[4] == "F#6"
py.test.raises(IndexError, "code.source[5]")
def test_name_no_first():
ops = parse('''
[i0]
i3 = int_add(i0, 1)
debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 200> #10 ADD")
debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 201> #11 SUB")
i1 = int_add(i0, 1)
debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 202> #11 SUB")
i2 = int_add(i1, 1)
''')
res = Function.from_operations(ops.operations, LoopStorage())
assert res.repr() == res.chunks[1].repr()
def test_linerange_notstarts():
if sys.version_info > (2, 6):
py.test.skip("unportable test")
fname = str(py.path.local(__file__).join('..', 'x.py'))
ops = parse("""
[p6, p1]
debug_merge_point(0, 0, '<code object h. file '%(fname)s'. line 11> #17 FOR_ITER')
guard_class(p6, 144264192, descr=<Guard0x2>)
p12 = getfield_gc(p6, descr=<GcPtrFieldDescr pypy.objspace.std.iterobject.W_AbstractSeqIterObject.inst_w_seq 12>)
""" % locals())
res = Function.from_operations(ops.operations, LoopStorage())
assert res.lineset
def test_adjust_bridges():
main = parse('''
[v0]
guard_false(v0, descr=<Guard0x1a>)
guard_true(v0, descr=<Guard0x5>)
''')
bridge = parse('''
# bridge out of Guard 0x1a
[]
int_add(0, 1)
''')
LoopStorage().reconnect_loops([main, bridge])
assert adjust_bridges(main, {})[1].name == 'guard_true'
assert adjust_bridges(main, {'loop-1a': True})[1].name == 'int_add'
def test_name():
ops = parse('''
[i0]
debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 200> #10 ADD")
debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 201> #11 SUB")
i1 = int_add(i0, 1)
debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 202> #11 SUB")
i2 = int_add(i1, 1)
''')
res = Function.from_operations(ops.operations, LoopStorage())
assert res.repr() == res.chunks[0].repr()
assert res.repr() == "stuff, file '/I/dont/exist.py', line 200"
assert res.startlineno == 200
assert res.filename == '/I/dont/exist.py'
assert res.name == 'stuff'
def test_linerange():
if sys.version_info > (2, 6):
py.test.skip("unportable test")
fname = str(py.path.local(__file__).join('..', 'x.py'))
ops = parse('''
[i0, i1]
debug_merge_point(0, 0, "<code object g. file '%(fname)s'. line 5> #9 LOAD_FAST")
debug_merge_point(0, 0, "<code object g. file '%(fname)s'. line 5> #12 LOAD_CONST")
debug_merge_point(0, 0, "<code object g. file '%(fname)s'. line 5> #22 LOAD_CONST")
debug_merge_point(0, 0, "<code object g. file '%(fname)s'. line 5> #28 LOAD_CONST")
debug_merge_point(0, 0, "<code object g. file '%(fname)s'. line 5> #6 SETUP_LOOP")
''' % locals())
res = Function.from_operations(ops.operations, LoopStorage())
assert res.linerange == (7, 9)
assert res.lineset == set([7, 8, 9])
def test_inlined_call():
ops = parse("""
[]
debug_merge_point(0, 0, '<code object inlined_call. file 'source.py'. line 12> #28 CALL_FUNCTION')
i18 = getfield_gc(p0, descr=<BoolFieldDescr pypy.interpreter.pyframe.PyFrame.inst_is_being_profiled 89>)
debug_merge_point(1, 1, '<code object inner. file 'source.py'. line 9> #0 LOAD_FAST')
debug_merge_point(1, 1, '<code object inner. file 'source.py'. line 9> #3 LOAD_CONST')
debug_merge_point(1, 1, '<code object inner. file 'source.py'. line 9> #7 RETURN_VALUE')
debug_merge_point(0, 0, '<code object inlined_call. file 'source.py'. line 12> #31 STORE_FAST')
""")
res = Function.from_operations(ops.operations, LoopStorage())
assert len(res.chunks) == 3 # two chunks + inlined call
assert isinstance(res.chunks[0], TraceForOpcode)
assert isinstance(res.chunks[1], Function)
assert isinstance(res.chunks[2], TraceForOpcode)
assert res.chunks[1].path == "1"
assert len(res.chunks[1].chunks) == 3
def test_split():
ops = parse('''
[i0]
label()
debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 200> #10 ADD")
debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 200> #11 SUB")
i1 = int_add(i0, 1)
debug_merge_point(0, 0, "<code object stuff. file '/I/dont/exist.py'. line 200> #11 SUB")
i2 = int_add(i1, 1)
''')
res = Function.from_operations(ops.operations, LoopStorage(), loopname='<loopname>')
assert len(res.chunks) == 4
assert len(res.chunks[0].operations) == 1
assert len(res.chunks[1].operations) == 1
assert len(res.chunks[2].operations) == 2
assert len(res.chunks[3].operations) == 2
assert res.chunks[3].bytecode_no == 11
assert res.chunks[0].bytecode_name == '<loopname>'
def test_reassign_loops():
main = parse('''
[v0]
guard_false(v0, descr=<Guard0x18>) []
''')
main.count = 10
bridge = parse('''
# bridge out of Guard 0x18 with 13 ops
[i0, i1]
int_add(i0, i1)
''')
bridge.count = 3
entry_bridge = parse('''
# Loop 3 : entry bridge
[]
''')
loops = LoopStorage().reconnect_loops([main, bridge, entry_bridge])
assert len(loops) == 2
assert len(loops[0].operations[0].bridge.operations) == 1
assert loops[0].operations[0].bridge.no == 0x18
assert loops[0].operations[0].percentage == 30
def test_load_codes():
tmppath = py.test.ensuretemp('load_codes')
tmppath.join("x.py").write("def f(): pass") # one code
s = LoopStorage(str(tmppath))
assert s.load_code(str(tmppath.join('x.py'))) == s.load_code('x.py')
def test_load_codes():
tmppath = py.test.ensuretemp('load_codes')
tmppath.join("x.py").write("def f(): pass") # one code
s = LoopStorage(str(tmppath))
assert s.load_code(str(tmppath.join('x.py'))) == s.load_code('x.py')
def __init__(self, rawtraces):
storage = LoopStorage()
traces = [SimpleParser.parse_from_input(rawtrace) for rawtrace in rawtraces]
traces = storage.reconnect_loops(traces)
self.loops = [TraceWithIds.from_trace(trace, storage) for trace in traces]
