def testSnapshots(self):
archive = self._storage.OpenArchive('snapshots', create=True)
t1 = archive.StartNewSnapshot()
archive.StoreMemMaps(memory_map.Map())
time.sleep(0.01) # Max snapshot resolution is in the order of usecs.
t2 = archive.StartNewSnapshot()
archive.StoreMemMaps(memory_map.Map())
archive.StoreNativeHeap(native_heap.NativeHeap())
self.assertIn(t1, archive.ListSnapshots())
self.assertIn(t2, archive.ListSnapshots())
self.assertTrue(archive.HasMemMaps(t1))
self.assertFalse(archive.HasNativeHeap(t1))
self.assertTrue(archive.HasMemMaps(t2))
self.assertTrue(archive.HasNativeHeap(t2))
self._storage.DeleteArchive('snapshots')
def decode(self, json_str): # pylint: disable=W0221
d = super(MmapDecoder, self).decode(json_str)
mmap = memory_map.Map()
for entry_dict in d:
entry = memory_map.MapEntry(**entry_dict)
mmap.Add(entry)
return mmap
def runTest(self):
mmap = memory_map.Map()
map_entry1 = memory_map.MapEntry(4096, 8191, 'rw--', '/foo', 0)
map_entry2 = memory_map.MapEntry(65536, 81919, 'rw--', '/bar', 4096)
# Test the de-offset logic.
self.assertEqual(map_entry1.GetRelativeOffset(4096), 0)
self.assertEqual(map_entry1.GetRelativeOffset(4100), 4)
self.assertEqual(map_entry2.GetRelativeOffset(65536), 4096)
# Test the page-resident logic.
map_entry2.resident_pages = [5] # 5 -> 101b.
self.assertTrue(map_entry2.IsPageResident(0))
self.assertFalse(map_entry2.IsPageResident(1))
self.assertTrue(map_entry2.IsPageResident(2))
# Test the lookup logic.
mmap.Add(map_entry1)
mmap.Add(map_entry2)
self.assertIsNone(mmap.Lookup(1024))
self.assertEqual(mmap.Lookup(4096), map_entry1)
self.assertEqual(mmap.Lookup(6000), map_entry1)
self.assertEqual(mmap.Lookup(8191), map_entry1)
self.assertIsNone(mmap.Lookup(8192))
self.assertIsNone(mmap.Lookup(65535))
self.assertEqual(mmap.Lookup(65536), map_entry2)
self.assertEqual(mmap.Lookup(67000), map_entry2)
self.assertEqual(mmap.Lookup(81919), map_entry2)
self.assertIsNone(mmap.Lookup(81920))
def runTest(self):
rule_tree = mmap_classifier.LoadRules(_TEST_RULES)
mmap = memory_map.Map()
for m in _TEST_MMAPS:
mmap.Add(memory_map.MapEntry(
m[0], m[1], m[2], m[3], 0, m[4], m[5], m[6], m[7]))
res = mmap_classifier.Classify(mmap, rule_tree)
def CheckResult(node, prefix):
node_name = prefix + node.name
self.assertIn(node_name, _EXPECTED_RESULTS)
subtotal = node.values[0]
values = node.values[1:]
# First check that the subtotal matches clean + dirty + shared + priv.
self.assertEqual(subtotal, values[0] + values[1] + values[2] + values[3])
# Then check that the single values match the expectations.
self.assertEqual(values, _EXPECTED_RESULTS[node_name])
for child in node.children:
CheckResult(child, node_name + '::')
CheckResult(res.total, '')
def testMmap(self):
archive = self._storage.OpenArchive('mmap', create=True)
timestamp = archive.StartNewSnapshot()
mmap = memory_map.Map()
map_entry1 = memory_map.MapEntry(4096, 8191, 'rw--', '/foo', 0)
map_entry2 = memory_map.MapEntry(65536, 81919, 'rw--', '/bar', 4096)
map_entry2.resident_pages = [5]
mmap.Add(map_entry1)
mmap.Add(map_entry2)
archive.StoreMemMaps(mmap)
mmap_deser = archive.LoadMemMaps(timestamp)
self._DeepCompare(mmap, mmap_deser)
self._storage.DeleteArchive('mmap')
def Parse(lines):
"""Parses the output of Android's am dumpheap -n.
am dumpheap dumps the oustanding malloc information (when the system property
libc.debug.malloc == 1).
The expected dumpheap output looks like this:
------------------------------------------------------------------------------
... Some irrelevant banner lines ...
z 0 sz 1000 num 3 bt 1234 5678 9abc ...
...
MAPS
9dcd0000-9dcd6000 r-xp 00000000 103:00 815 /system/lib/libnbaio.so
...
------------------------------------------------------------------------------
The lines before MAPS list the allocations grouped by {size, backtrace}. In
the example above, "1000" is the size of each alloc, "3" is their cardinality
and "1234 5678 9abc" are the first N stack frames (absolute addresses in the
process virtual address space). The lines after MAPS provide essentially the
same information of /proc/PID/smaps.
See tests/android_backend_test.py for a more complete example.
Args:
lines: array of strings containing the am dumpheap -n output.
Returns:
An instance of |native_heap.NativeHeap|.
"""
(STATE_PARSING_BACKTRACES, STATE_PARSING_MAPS, STATE_ENDED) = range(3)
BT_RE = re.compile(
r'^\w+\s+\d+\s+sz\s+(\d+)\s+num\s+(\d+)\s+bt\s+((?:[0-9a-f]+\s?)+)$')
MAP_RE = re.compile(
r'^([0-9a-f]+)-([0-9a-f]+)\s+....\s*([0-9a-f]+)\s+\w+:\w+\s+\d+\s*(.*)$')
state = STATE_PARSING_BACKTRACES
skip_first_n_lines = 5
mmap = memory_map.Map()
stack_frames = {} # absolute_address (int) -> |stacktrace.Frame|.
nativeheap = native_heap.NativeHeap()
for line in lines:
line = line.rstrip('\r\n')
if skip_first_n_lines > 0:
skip_first_n_lines -= 1
continue
if state == STATE_PARSING_BACKTRACES:
if line == 'MAPS':
state = STATE_PARSING_MAPS
continue
m = BT_RE.match(line)
if not m:
logging.warning('Skipping unrecognized dumpheap alloc: "%s"' % line)
continue
alloc_size = int(m.group(1))
alloc_count = int(m.group(2))
alloc_bt_str = m.group(3)
strace = stacktrace.Stacktrace()
# Keep only one |stacktrace.Frame| per distinct |absolute_addr|, in order
# to ease the complexity of the final de-offset pass.
for absolute_addr in alloc_bt_str.split():
absolute_addr = int(absolute_addr, 16)
stack_frame = stack_frames.get(absolute_addr)
if not stack_frame:
stack_frame = stacktrace.Frame(absolute_addr)
stack_frames[absolute_addr] = stack_frame
strace.Add(stack_frame)
nativeheap.Add(native_heap.Allocation(alloc_size, alloc_count, strace))
# The am dumpheap output contains also a list of mmaps. This information is
# used in this module for the only purpose of normalizing addresses (i.e.
# translating an absolute addr into its offset inside the mmap-ed library).
# The mmap information is not further retained. A more complete mmap dump is
# performed (and retained) using the memdump tool (see memdump_parser.py).
elif state == STATE_PARSING_MAPS:
if line == 'END':
state = STATE_ENDED
continue
m = MAP_RE.match(line)
if not m:
logging.warning('Skipping unrecognized dumpheap mmap: "%s"' % line)
continue
mmap.Add(memory_map.MapEntry(
start=int(m.group(1), 16),
end=int(m.group(2), 16),
prot_flags='----', # Not really needed for lookup
mapped_file=m.group(4),
mapped_offset=int(m.group(3), 16)))
elif state == STATE_ENDED:
pass
else:
assert(False)
# Final pass: translate all the stack frames' absolute addresses into
# relative offsets (exec_file + offset) using the memory maps just processed.
for abs_addr, stack_frame in stack_frames.iteritems():
assert(abs_addr == stack_frame.address)
map_entry = mmap.Lookup(abs_addr)
if not map_entry:
continue
stack_frame.SetExecFileInfo(map_entry.mapped_file,
map_entry.GetRelativeOffset(abs_addr))
return nativeheap
def Parse(lines):
"""Parses the output of memdump.
memdump (see chrome/src/tools/memdump) is a Linux/Android binary meant to be
executed on the target device which extracts memory map information about one
or more processes. In principle is can be seen as an alternative to cat-ing
/proc/PID/smaps, but with extra features (multiprocess accounting and resident
pages reporting).
The expected memdump output looks like this:
------------------------------------------------------------------------------
[ PID=1234]
1000-2000 r-xp 0 private_unevictable=4096 private=8192 shared_app=[] \
shared_other_unevictable=4096 shared_other=4096 "/lib/foo.so" [v///fv0D]
... other entries like the one above.
------------------------------------------------------------------------------
The output is extremely similar to /proc/PID/smaps, with the following notes:
- unevictable has pretty much the same meaning of "dirty", in VM terms.
- private and shared_other are cumulative. This means the the "clean" part
must be calculated as difference of (private - private_unevictable).
- The final field [v///fv0D] is a base64 encoded bitmap which contains the
information about which pages inside the mapping are resident (present).
See tests/android_backend_test.py for a more complete example.
Args:
lines: array of strings containing memdump output.
Returns:
An instance of |memory_map.Map|.
"""
RE = (r'^([0-9a-f]+)-([0-9a-f]+)\s+'
r'([rwxps-]{4})\s+'
r'([0-9a-f]+)\s+'
r'private_unevictable=(\d+) private=(\d+) '
r'shared_app=(.*?) '
r'shared_other_unevictable=(\d+) shared_other=(\d+) '
r'\"(.*)\" '
r'\[([a-zA-Z0-9+/=-_:]*)\]$')
map_re = re.compile(RE)
skip_first_n_lines = 1
maps = memory_map.Map()
for line in lines:
line = line.rstrip('\r\n')
if skip_first_n_lines > 0:
skip_first_n_lines -= 1
continue
m = map_re.match(line)
if not m:
logging.warning('Skipping unrecognized memdump line "%s"' % line)
continue
start = int(m.group(1), 16)
end = int(m.group(2),
16) - 1 # end addr is inclusive in memdump output.
if (start > end):
# Sadly, this actually happened. Probably a kernel bug, see b/17402069.
logging.warning('Skipping unfeasible mmap "%s"' % line)
continue
entry = memory_map.MapEntry(start=start,
end=end,
prot_flags=m.group(3),
mapped_file=m.group(10),
mapped_offset=int(m.group(4), 16))
entry.priv_dirty_bytes = int(m.group(5))
entry.priv_clean_bytes = int(m.group(6)) - entry.priv_dirty_bytes
entry.shared_dirty_bytes = int(m.group(8))
entry.shared_clean_bytes = int(m.group(9)) - entry.shared_dirty_bytes
entry.resident_pages = [ord(c) for c in base64.b64decode(m.group(11))]
maps.Add(entry)
return maps
def runTest(self):
nheap = native_heap.NativeHeap()
EXE_1_MM_BASE = 64 * PAGE_SIZE
EXE_2_MM_BASE = 65 * PAGE_SIZE
EXE_2_FILE_OFF = 8192
st1 = stacktrace.Stacktrace()
st1.Add(nheap.GetStackFrame(EXE_1_MM_BASE))
st1.Add(nheap.GetStackFrame(EXE_1_MM_BASE + 4))
st2 = stacktrace.Stacktrace()
st2.Add(nheap.GetStackFrame(EXE_1_MM_BASE))
st2.Add(nheap.GetStackFrame(EXE_2_MM_BASE + 4))
st2.Add(nheap.GetStackFrame(EXE_2_MM_BASE + PAGE_SIZE + 4))
# Check that GetStackFrames keeps one unique object instance per address.
# This is to guarantee that the symbolization logic (SymbolizeUsingSymbolDB)
# can cheaply iterate on distinct stack frames rather than re-processing
# every stack frame for each allocation (and save memory as well).
self.assertIs(st1[0], st2[0])
self.assertIsNot(st1[0], st1[1])
self.assertIsNot(st2[0], st2[1])
alloc1 = native_heap.Allocation(start=4, size=4, stack_trace=st1)
alloc2 = native_heap.Allocation(start=4090, size=8, stack_trace=st1)
alloc3 = native_heap.Allocation(start=8190,
size=10000,
stack_trace=st2)
nheap.Add(alloc1)
nheap.Add(alloc2)
nheap.Add(alloc3)
self.assertEqual(len(nheap.allocations), 3)
self.assertIn(alloc1, nheap.allocations)
self.assertIn(alloc2, nheap.allocations)
self.assertIn(alloc3, nheap.allocations)
############################################################################
# Test the relativization (absolute address -> mmap + offset) logic.
############################################################################
mmap = memory_map
mmap = memory_map.Map()
mmap.Add(
memory_map.MapEntry(EXE_1_MM_BASE, EXE_1_MM_BASE + PAGE_SIZE - 1,
'rw--', '/d/exe1', 0))
mmap.Add(
memory_map.MapEntry(EXE_2_MM_BASE, EXE_2_MM_BASE + PAGE_SIZE - 1,
'rw--', 'exe2', EXE_2_FILE_OFF))
# Entry for EXE_3 is deliberately missing to check the fallback behavior.
nheap.RelativizeStackFrames(mmap)
self.assertEqual(st1[0].exec_file_rel_path, '/d/exe1')
self.assertEqual(st1[0].exec_file_name, 'exe1')
self.assertEqual(st1[0].offset, 0)
self.assertEqual(st1[1].exec_file_rel_path, '/d/exe1')
self.assertEqual(st1[1].exec_file_name, 'exe1')
self.assertEqual(st1[1].offset, 4)
self.assertEqual(st2[0].exec_file_rel_path, '/d/exe1')
self.assertEqual(st2[0].exec_file_name, 'exe1')
self.assertEqual(st2[0].offset, 0)
self.assertEqual(st2[1].exec_file_rel_path, 'exe2')
self.assertEqual(st2[1].exec_file_name, 'exe2')
self.assertEqual(st2[1].offset, 4 + EXE_2_FILE_OFF)
self.assertIsNone(st2[2].exec_file_rel_path)
self.assertIsNone(st2[2].exec_file_name)
self.assertIsNone(st2[2].offset)
############################################################################
# Test the symbolization logic.
############################################################################
syms = symbol.Symbols()
syms.Add('/d/exe1', 0, symbol.Symbol('sym1', 'src1.c', 1)) # st1[0]
syms.Add('exe2', 4 + EXE_2_FILE_OFF, symbol.Symbol('sym3')) # st2[1]
nheap.SymbolizeUsingSymbolDB(syms)
self.assertEqual(st1[0].symbol.name, 'sym1')
self.assertEqual(st1[0].symbol.source_info[0].source_file_path,
'src1.c')
self.assertEqual(st1[0].symbol.source_info[0].line_number, 1)
# st1[1] should have no symbol info, because we didn't provide any above.
self.assertIsNone(st1[1].symbol)
# st2[0] and st1[0] were the same Frame. Expect identical symbols instances.
self.assertIs(st2[0].symbol, st1[0].symbol)
# st2[1] should have a symbols name, but no source line info.
self.assertEqual(st2[1].symbol.name, 'sym3')
self.assertEqual(len(st2[1].symbol.source_info), 0)
# st2[2] should have no sym because we didn't even provide a mmap for exe3.
self.assertIsNone(st2[2].symbol)
############################################################################
# Test the resident size calculation logic (intersects mmaps and allocs).
############################################################################
mmap.Add(
memory_map.MapEntry(0, 8191, 'rw--', '', 0, resident_pages=[1]))
mmap.Add(
memory_map.MapEntry(8192, 12287, 'rw--', '', 0,
resident_pages=[1]))
# [12k, 16k] is deliberately missing to check the fallback behavior.
mmap.Add(
memory_map.MapEntry(16384,
20479,
'rw--',
'',
0,
resident_pages=[1]))
nheap.CalculateResidentSize(mmap)
# alloc1 [4, 8] is fully resident because it lays in the first resident 4k.
self.assertEqual(alloc1.resident_size, 4)
# alloc2 [4090, 4098] should have only 6 resident bytes ([4090,4096]), but
# not the last two, which lay on the second page which is noijt resident.
self.assertEqual(alloc2.resident_size, 6)
# alloc3 [8190, 18190] is split as follows (* = resident):
# [8190, 8192]: these 2 bytes are NOT resident, they lay in the 2nd page.
# *[8192, 12288]: the 3rd page is resident and is fully covered by alloc3.
# [12288, 16384]: the 4th page is fully covered as well, but not resident.
# *[16384, 18190]: the 5th page is partially covered and resident.
self.assertEqual(alloc3.resident_size,
(12288 - 8192) + (18190 - 16384))
