Example #1
0
  def runTest(self):
    mmap = memory_map.Map()
    map_entry1 = memory_map.MapEntry(4096, 8191, 'rw--', '/foo', 0)
    map_entry2 = memory_map.MapEntry(65536, 81919, 'rw--', '/bar', 4096)

    # Test the de-offset logic.
    self.assertEqual(map_entry1.GetRelativeOffset(4096), 0)
    self.assertEqual(map_entry1.GetRelativeOffset(4100), 4)
    self.assertEqual(map_entry2.GetRelativeOffset(65536), 4096)

    # Test the page-resident logic.
    map_entry2.resident_pages = [5] # 5 -> 101b.
    self.assertTrue(map_entry2.IsPageResident(0))
    self.assertFalse(map_entry2.IsPageResident(1))
    self.assertTrue(map_entry2.IsPageResident(2))

    # Test the lookup logic.
    mmap.Add(map_entry1)
    mmap.Add(map_entry2)
    self.assertIsNone(mmap.Lookup(1024))
    self.assertEqual(mmap.Lookup(4096), map_entry1)
    self.assertEqual(mmap.Lookup(6000), map_entry1)
    self.assertEqual(mmap.Lookup(8191), map_entry1)
    self.assertIsNone(mmap.Lookup(8192))
    self.assertIsNone(mmap.Lookup(65535))
    self.assertEqual(mmap.Lookup(65536), map_entry2)
    self.assertEqual(mmap.Lookup(67000), map_entry2)
    self.assertEqual(mmap.Lookup(81919), map_entry2)
    self.assertIsNone(mmap.Lookup(81920))
Example #2
0
 def testMmap(self):
   archive = self._storage.OpenArchive('mmap', create=True)
   timestamp = archive.StartNewSnapshot()
   mmap = memory_map.Map()
   map_entry1 = memory_map.MapEntry(4096, 8191, 'rw--', '/foo', 0)
   map_entry2 = memory_map.MapEntry(65536, 81919, 'rw--', '/bar', 4096)
   map_entry2.resident_pages = [5]
   mmap.Add(map_entry1)
   mmap.Add(map_entry2)
   archive.StoreMemMaps(mmap)
   mmap_deser = archive.LoadMemMaps(timestamp)
   self._DeepCompare(mmap, mmap_deser)
   self._storage.DeleteArchive('mmap')
Example #3
0
 def decode(self, json_str):  # pylint: disable=W0221
     d = super(MmapDecoder, self).decode(json_str)
     mmap = memory_map.Map()
     for entry_dict in d:
         entry = memory_map.MapEntry(**entry_dict)
         mmap.Add(entry)
     return mmap
Example #4
0
  def runTest(self):
    rule_tree = mmap_classifier.LoadRules(_TEST_RULES)
    mmap = memory_map.Map()
    for m in _TEST_MMAPS:
      mmap.Add(memory_map.MapEntry(
          m[0], m[1], m[2], m[3], 0, m[4], m[5], m[6], m[7]))

    res = mmap_classifier.Classify(mmap, rule_tree)

    def CheckResult(node, prefix):
      node_name = prefix + node.name
      self.assertIn(node_name, _EXPECTED_RESULTS)
      subtotal = node.values[0]
      values = node.values[1:]

      # First check that the subtotal matches clean + dirty + shared + priv.
      self.assertEqual(subtotal, values[0] + values[1] + values[2] + values[3])

      # Then check that the single values match the expectations.
      self.assertEqual(values, _EXPECTED_RESULTS[node_name])

      for child in node.children:
        CheckResult(child, node_name + '::')

    CheckResult(res.total, '')
def Parse(lines):
  """Parses the output of Android's am dumpheap -n.

  am dumpheap dumps the oustanding malloc information (when the system property
  libc.debug.malloc == 1).

  The expected dumpheap output looks like this:
  ------------------------------------------------------------------------------
  ... Some irrelevant banner lines ...
  z 0  sz    1000   num    3  bt 1234 5678 9abc ...
  ...
  MAPS
  9dcd0000-9dcd6000 r-xp 00000000 103:00 815       /system/lib/libnbaio.so
  ...
  ------------------------------------------------------------------------------
  The lines before MAPS list the allocations grouped by {size, backtrace}. In
  the example above, "1000" is the size of each alloc, "3" is their cardinality
  and "1234 5678 9abc" are the first N stack frames (absolute addresses in the
  process virtual address space).  The lines after MAPS provide essentially the
  same information of /proc/PID/smaps.
  See tests/android_backend_test.py for a more complete example.

  Args:
      lines: array of strings containing the am dumpheap -n output.

  Returns:
      An instance of |native_heap.NativeHeap|.
  """
  (STATE_PARSING_BACKTRACES, STATE_PARSING_MAPS, STATE_ENDED) = range(3)
  BT_RE = re.compile(
      r'^\w+\s+\d+\s+sz\s+(\d+)\s+num\s+(\d+)\s+bt\s+((?:[0-9a-f]+\s?)+)$')
  MAP_RE = re.compile(
      r'^([0-9a-f]+)-([0-9a-f]+)\s+....\s*([0-9a-f]+)\s+\w+:\w+\s+\d+\s*(.*)$')

  state = STATE_PARSING_BACKTRACES
  skip_first_n_lines = 5
  mmap = memory_map.Map()
  stack_frames = {}  # absolute_address (int) -> |stacktrace.Frame|.
  nativeheap = native_heap.NativeHeap()

  for line in lines:
    line = line.rstrip('\r\n')
    if skip_first_n_lines > 0:
      skip_first_n_lines -= 1
      continue

    if state == STATE_PARSING_BACKTRACES:
      if line == 'MAPS':
        state = STATE_PARSING_MAPS
        continue
      m = BT_RE.match(line)
      if not m:
        logging.warning('Skipping unrecognized dumpheap alloc: "%s"' % line)
        continue
      alloc_size = int(m.group(1))
      alloc_count = int(m.group(2))
      alloc_bt_str = m.group(3)
      strace = stacktrace.Stacktrace()
      # Keep only one |stacktrace.Frame| per distinct |absolute_addr|, in order
      # to ease the complexity of the final de-offset pass.
      for absolute_addr in alloc_bt_str.split():
        absolute_addr = int(absolute_addr, 16)
        stack_frame = stack_frames.get(absolute_addr)
        if not stack_frame:
          stack_frame = stacktrace.Frame(absolute_addr)
          stack_frames[absolute_addr] = stack_frame
        strace.Add(stack_frame)
      nativeheap.Add(native_heap.Allocation(alloc_size, alloc_count, strace))

    # The am dumpheap output contains also a list of mmaps. This information is
    # used in this module for the only purpose of normalizing addresses (i.e.
    # translating an absolute addr into its offset inside the mmap-ed library).
    # The mmap information is not further retained. A more complete mmap dump is
    # performed (and retained) using the memdump tool (see memdump_parser.py).
    elif state == STATE_PARSING_MAPS:
      if line == 'END':
        state = STATE_ENDED
        continue
      m = MAP_RE.match(line)
      if not m:
        logging.warning('Skipping unrecognized dumpheap mmap: "%s"' % line)
        continue
      mmap.Add(memory_map.MapEntry(
          start=int(m.group(1), 16),
          end=int(m.group(2), 16),
          prot_flags='----', # Not really needed for lookup
          mapped_file=m.group(4),
          mapped_offset=int(m.group(3), 16)))

    elif state == STATE_ENDED:
      pass

    else:
      assert(False)

  # Final pass: translate all the stack frames' absolute addresses into
  # relative offsets (exec_file + offset) using the memory maps just processed.
  for abs_addr, stack_frame in stack_frames.iteritems():
    assert(abs_addr == stack_frame.address)
    map_entry = mmap.Lookup(abs_addr)
    if not map_entry:
      continue
    stack_frame.SetExecFileInfo(map_entry.mapped_file,
                                map_entry.GetRelativeOffset(abs_addr))

  return nativeheap
def Parse(lines):
    """Parses the output of memdump.

  memdump (see chrome/src/tools/memdump) is a Linux/Android binary meant to be
  executed on the target device which extracts memory map information about one
  or more processes. In principle is can be seen as an alternative to cat-ing
  /proc/PID/smaps, but with extra features (multiprocess accounting and resident
  pages reporting).

  The expected memdump output looks like this:
  ------------------------------------------------------------------------------
  [ PID=1234]
  1000-2000 r-xp 0 private_unevictable=4096 private=8192 shared_app=[] \
      shared_other_unevictable=4096 shared_other=4096 "/lib/foo.so" [v///fv0D]
  ... other entries like the one above.
  ------------------------------------------------------------------------------
  The output is extremely similar to /proc/PID/smaps, with the following notes:
   - unevictable has pretty much the same meaning of "dirty", in VM terms.
   - private and shared_other are cumulative. This means the the "clean" part
     must be calculated as difference of (private - private_unevictable).
   - The final field [v///fv0D] is a base64 encoded bitmap which contains the
     information about which pages inside the mapping are resident (present).
  See tests/android_backend_test.py for a more complete example.

  Args:
      lines: array of strings containing memdump output.

  Returns:
      An instance of |memory_map.Map|.
  """
    RE = (r'^([0-9a-f]+)-([0-9a-f]+)\s+'
          r'([rwxps-]{4})\s+'
          r'([0-9a-f]+)\s+'
          r'private_unevictable=(\d+) private=(\d+) '
          r'shared_app=(.*?) '
          r'shared_other_unevictable=(\d+) shared_other=(\d+) '
          r'\"(.*)\" '
          r'\[([a-zA-Z0-9+/=-_:]*)\]$')
    map_re = re.compile(RE)
    skip_first_n_lines = 1
    maps = memory_map.Map()

    for line in lines:
        line = line.rstrip('\r\n')

        if skip_first_n_lines > 0:
            skip_first_n_lines -= 1
            continue

        m = map_re.match(line)
        if not m:
            logging.warning('Skipping unrecognized memdump line "%s"' % line)
            continue

        start = int(m.group(1), 16)
        end = int(m.group(2),
                  16) - 1  # end addr is inclusive in memdump output.
        if (start > end):
            # Sadly, this actually happened. Probably a kernel bug, see b/17402069.
            logging.warning('Skipping unfeasible mmap "%s"' % line)
            continue
        entry = memory_map.MapEntry(start=start,
                                    end=end,
                                    prot_flags=m.group(3),
                                    mapped_file=m.group(10),
                                    mapped_offset=int(m.group(4), 16))
        entry.priv_dirty_bytes = int(m.group(5))
        entry.priv_clean_bytes = int(m.group(6)) - entry.priv_dirty_bytes
        entry.shared_dirty_bytes = int(m.group(8))
        entry.shared_clean_bytes = int(m.group(9)) - entry.shared_dirty_bytes
        entry.resident_pages = [ord(c) for c in base64.b64decode(m.group(11))]
        maps.Add(entry)

    return maps
Example #7
0
    def runTest(self):
        nheap = native_heap.NativeHeap()

        EXE_1_MM_BASE = 64 * PAGE_SIZE
        EXE_2_MM_BASE = 65 * PAGE_SIZE
        EXE_2_FILE_OFF = 8192
        st1 = stacktrace.Stacktrace()
        st1.Add(nheap.GetStackFrame(EXE_1_MM_BASE))
        st1.Add(nheap.GetStackFrame(EXE_1_MM_BASE + 4))

        st2 = stacktrace.Stacktrace()
        st2.Add(nheap.GetStackFrame(EXE_1_MM_BASE))
        st2.Add(nheap.GetStackFrame(EXE_2_MM_BASE + 4))
        st2.Add(nheap.GetStackFrame(EXE_2_MM_BASE + PAGE_SIZE + 4))

        # Check that GetStackFrames keeps one unique object instance per address.
        # This is to guarantee that the symbolization logic (SymbolizeUsingSymbolDB)
        # can cheaply iterate on distinct stack frames rather than re-processing
        # every stack frame for each allocation (and save memory as well).
        self.assertIs(st1[0], st2[0])
        self.assertIsNot(st1[0], st1[1])
        self.assertIsNot(st2[0], st2[1])

        alloc1 = native_heap.Allocation(start=4, size=4, stack_trace=st1)
        alloc2 = native_heap.Allocation(start=4090, size=8, stack_trace=st1)
        alloc3 = native_heap.Allocation(start=8190,
                                        size=10000,
                                        stack_trace=st2)
        nheap.Add(alloc1)
        nheap.Add(alloc2)
        nheap.Add(alloc3)

        self.assertEqual(len(nheap.allocations), 3)
        self.assertIn(alloc1, nheap.allocations)
        self.assertIn(alloc2, nheap.allocations)
        self.assertIn(alloc3, nheap.allocations)

        ############################################################################
        # Test the relativization (absolute address -> mmap + offset) logic.
        ############################################################################
        mmap = memory_map
        mmap = memory_map.Map()
        mmap.Add(
            memory_map.MapEntry(EXE_1_MM_BASE, EXE_1_MM_BASE + PAGE_SIZE - 1,
                                'rw--', '/d/exe1', 0))
        mmap.Add(
            memory_map.MapEntry(EXE_2_MM_BASE, EXE_2_MM_BASE + PAGE_SIZE - 1,
                                'rw--', 'exe2', EXE_2_FILE_OFF))
        # Entry for EXE_3 is deliberately missing to check the fallback behavior.

        nheap.RelativizeStackFrames(mmap)

        self.assertEqual(st1[0].exec_file_rel_path, '/d/exe1')
        self.assertEqual(st1[0].exec_file_name, 'exe1')
        self.assertEqual(st1[0].offset, 0)

        self.assertEqual(st1[1].exec_file_rel_path, '/d/exe1')
        self.assertEqual(st1[1].exec_file_name, 'exe1')
        self.assertEqual(st1[1].offset, 4)

        self.assertEqual(st2[0].exec_file_rel_path, '/d/exe1')
        self.assertEqual(st2[0].exec_file_name, 'exe1')
        self.assertEqual(st2[0].offset, 0)

        self.assertEqual(st2[1].exec_file_rel_path, 'exe2')
        self.assertEqual(st2[1].exec_file_name, 'exe2')
        self.assertEqual(st2[1].offset, 4 + EXE_2_FILE_OFF)

        self.assertIsNone(st2[2].exec_file_rel_path)
        self.assertIsNone(st2[2].exec_file_name)
        self.assertIsNone(st2[2].offset)

        ############################################################################
        # Test the symbolization logic.
        ############################################################################
        syms = symbol.Symbols()
        syms.Add('/d/exe1', 0, symbol.Symbol('sym1', 'src1.c', 1))  # st1[0]
        syms.Add('exe2', 4 + EXE_2_FILE_OFF, symbol.Symbol('sym3'))  # st2[1]

        nheap.SymbolizeUsingSymbolDB(syms)
        self.assertEqual(st1[0].symbol.name, 'sym1')
        self.assertEqual(st1[0].symbol.source_info[0].source_file_path,
                         'src1.c')
        self.assertEqual(st1[0].symbol.source_info[0].line_number, 1)

        # st1[1] should have no symbol info, because we didn't provide any above.
        self.assertIsNone(st1[1].symbol)

        # st2[0] and st1[0] were the same Frame. Expect identical symbols instances.
        self.assertIs(st2[0].symbol, st1[0].symbol)

        # st2[1] should have a symbols name, but no source line info.
        self.assertEqual(st2[1].symbol.name, 'sym3')
        self.assertEqual(len(st2[1].symbol.source_info), 0)

        # st2[2] should have no sym because we didn't even provide a mmap for exe3.
        self.assertIsNone(st2[2].symbol)

        ############################################################################
        # Test the resident size calculation logic (intersects mmaps and allocs).
        ############################################################################
        mmap.Add(
            memory_map.MapEntry(0, 8191, 'rw--', '', 0, resident_pages=[1]))
        mmap.Add(
            memory_map.MapEntry(8192, 12287, 'rw--', '', 0,
                                resident_pages=[1]))
        # [12k, 16k] is deliberately missing to check the fallback behavior.
        mmap.Add(
            memory_map.MapEntry(16384,
                                20479,
                                'rw--',
                                '',
                                0,
                                resident_pages=[1]))
        nheap.CalculateResidentSize(mmap)

        # alloc1 [4, 8] is fully resident because it lays in the first resident 4k.
        self.assertEqual(alloc1.resident_size, 4)

        # alloc2 [4090, 4098] should have only 6 resident bytes ([4090,4096]), but
        # not the last two, which lay on the second page which is noijt resident.
        self.assertEqual(alloc2.resident_size, 6)

        # alloc3 [8190, 18190] is split as follows (* = resident):
        #  [8190, 8192]: these 2 bytes are NOT resident, they lay in the 2nd page.
        # *[8192, 12288]: the 3rd page is resident and is fully covered by alloc3.
        #  [12288, 16384]: the 4th page is fully covered as well, but not resident.
        # *[16384, 18190]: the 5th page is partially covered and resident.
        self.assertEqual(alloc3.resident_size,
                         (12288 - 8192) + (18190 - 16384))