def testCalculateDeviceMd5Sums_requiresBinary(self):
test_path = '/storage/emulated/legacy/test/file.dat'
device = mock.NonCallableMock()
device.adb = mock.NonCallableMock()
device.adb.Push = mock.Mock()
device_md5sum_output = [
'WARNING: linker: /data/local/tmp/md5sum/md5sum_bin: '
'unused DT entry: type 0x1d arg 0x15db',
'THIS_IS_NOT_A_VALID_CHECKSUM_ZZZ some random text',
'0123456789abcdef',
]
error = device_errors.AdbShellCommandFailedError('cmd', 'out', 2)
device.RunShellCommand = mock.Mock(
side_effect=(error, '', device_md5sum_output))
with mock.patch(
'os.path.isdir', return_value=True), (mock.patch(
'os.path.getsize', return_value=1337)):
out = md5sum.CalculateDeviceMd5Sums(test_path, device)
self.assertEquals(1, len(out))
self.assertTrue('/storage/emulated/legacy/test/file.dat' in out)
self.assertEquals('0123456789abcdef',
out['/storage/emulated/legacy/test/file.dat'])
self.assertEquals(3, len(device.RunShellCommand.call_args_list))
device.adb.Push.assert_called_once_with('test/out/directory/md5sum_dist',
'/data/local/tmp/md5sum')
def _FindMatchingUnstrippedLibraryOnHost(device, lib):
lib_base = os.path.basename(lib)
device_md5sums = md5sum.CalculateDeviceMd5Sums([lib], device)
if lib not in device_md5sums:
return None
device_md5 = device_md5sums[lib]
def FindMatchingStrippedLibrary(out_path):
# First find a matching stripped library on the host. This avoids the need
# to pull the stripped library from the device, which can take tens of
# seconds.
# Check the GN stripped lib path first, and the GYP ones afterwards.
host_lib_path = os.path.join(out_path, lib_base)
host_lib_pattern = os.path.join(out_path, '*_apk', 'libs', '*',
lib_base)
for stripped_host_lib in [host_lib_path] + glob.glob(host_lib_pattern):
if os.path.exists(stripped_host_lib):
with open(stripped_host_lib) as f:
host_md5 = hashlib.md5(f.read()).hexdigest()
if host_md5 == device_md5:
return stripped_host_lib
return None
out_path = None
stripped_host_lib = None
for out_path in util.GetBuildDirectories():
stripped_host_lib = FindMatchingStrippedLibrary(out_path)
if stripped_host_lib:
break
if not stripped_host_lib:
return None
# The corresponding unstripped library will be under lib.unstripped for GN, or
# lib for GYP.
unstripped_host_lib_paths = [
os.path.join(out_path, 'lib.unstripped', lib_base),
os.path.join(out_path, 'lib', lib_base)
]
unstripped_host_lib = next(
(lib for lib in unstripped_host_lib_paths if os.path.exists(lib)),
None)
if unstripped_host_lib is None:
return None
# Make sure the unstripped library matches the stripped one. We do this
# by comparing the hashes of text sections in both libraries. This isn't an
# exact guarantee, but should still give reasonable confidence that the
# libraries are compatible.
# TODO(skyostil): Check .note.gnu.build-id instead once we're using
# --build-id=sha1.
# pylint: disable=undefined-loop-variable
if (_ElfSectionMd5Sum(unstripped_host_lib, _TEXT_SECTION) !=
_ElfSectionMd5Sum(stripped_host_lib, _TEXT_SECTION)):
return None
return unstripped_host_lib
def testCalculateDeviceMd5Sums_singlePath(self):
test_path = '/storage/emulated/legacy/test/file.dat'
device = mock.NonCallableMock()
device_md5sum_output = ['0123456789abcdef',]
device.RunShellCommand = mock.Mock(return_value=device_md5sum_output)
with mock.patch('os.path.getsize', return_value=1337):
out = md5sum.CalculateDeviceMd5Sums(test_path, device)
self.assertEquals(1, len(out))
self.assertTrue('/storage/emulated/legacy/test/file.dat' in out)
self.assertEquals('0123456789abcdef',
out['/storage/emulated/legacy/test/file.dat'])
self.assertEquals(1, len(device.RunShellCommand.call_args_list))
def testCalculateDeviceMd5Sums_list_fileMissing(self):
test_path = ['/storage/emulated/legacy/test/file0.dat',
'/storage/emulated/legacy/test/file1.dat']
device = mock.NonCallableMock()
device_md5sum_output = [
'0123456789abcdeffedcba9876543210 '
'/storage/emulated/legacy/test/file0.dat',
'[0819/203513:ERROR:md5sum.cc(25)] Could not open file asdf',
'',
]
device.RunShellCommand = mock.Mock(return_value=device_md5sum_output)
with mock.patch('os.path.getsize', return_value=1337):
out = md5sum.CalculateDeviceMd5Sums(test_path, device)
self.assertEquals(1, len(out))
self.assertTrue('/storage/emulated/legacy/test/file0.dat' in out)
self.assertEquals('0123456789abcdeffedcba9876543210',
out['/storage/emulated/legacy/test/file0.dat'])
self.assertEquals(1, len(device.RunShellCommand.call_args_list))
def testCalculateDeviceMd5Sums_singlePath_linkerWarning(self):
# See crbug/479966
test_path = '/storage/emulated/legacy/test/file.dat'
device = mock.NonCallableMock()
device_md5sum_output = [
'WARNING: linker: /data/local/tmp/md5sum/md5sum_bin: '
'unused DT entry: type 0x1d arg 0x15db',
'THIS_IS_NOT_A_VALID_CHECKSUM_ZZZ some random text',
'0123456789abcdef',
]
device.RunShellCommand = mock.Mock(return_value=device_md5sum_output)
with mock.patch('os.path.getsize', return_value=1337):
out = md5sum.CalculateDeviceMd5Sums(test_path, device)
self.assertEquals(1, len(out))
self.assertTrue('/storage/emulated/legacy/test/file.dat' in out)
self.assertEquals('0123456789abcdef',
out['/storage/emulated/legacy/test/file.dat'])
self.assertEquals(1, len(device.RunShellCommand.call_args_list))
def testCalculateDeviceMd5Sums_list(self):
test_path = ['/storage/emulated/legacy/test/file0.dat',
'/storage/emulated/legacy/test/file1.dat']
presentation.device = mock.NonCallableMock()
device_md5sum_output = [
'0123456789abcdeffedcba9876543210 '
'/storage/emulated/legacy/test/file0.dat',
'123456789abcdef00fedcba987654321 '
'/storage/emulated/legacy/test/file1.dat',
]
presentation.device.RunShellCommand = mock.Mock(return_value=device_md5sum_output)
with mock.patch('os.path.getsize', return_value=1337):
out = md5sum.CalculateDeviceMd5Sums(test_path, presentation.device)
self.assertEquals(2, len(out))
self.assertTrue('/storage/emulated/legacy/test/file0.dat' in out)
self.assertEquals('0123456789abcdeffedcba9876543210',
out['/storage/emulated/legacy/test/file0.dat'])
self.assertTrue('/storage/emulated/legacy/test/file1.dat' in out)
self.assertEquals('123456789abcdef00fedcba987654321',
out['/storage/emulated/legacy/test/file1.dat'])
self.assertEquals(1, len(presentation.device.RunShellCommand.call_args_list))
def CreateSymFs(device, symfs_dir, libraries, use_symlinks=True):
"""Creates a symfs directory to be used for symbolizing profiles.
Prepares a set of files ("symfs") to be used with profilers such as perf for
converting binary addresses into human readable function names.
Args:
device: DeviceUtils instance identifying the target device.
symfs_dir: Path where the symfs should be created.
libraries: Set of library file names that should be included in the symfs.
use_symlinks: If True, link instead of copy unstripped libraries into the
symfs. This will speed up the operation, but the resulting symfs will no
longer be valid if the linked files are modified, e.g., by rebuilding.
Returns:
The absolute path to the kernel symbols within the created symfs.
"""
logging.info('Building symfs into %s.' % symfs_dir)
for lib in libraries:
device_dir = os.path.dirname(lib)
output_dir = os.path.join(symfs_dir, device_dir[1:])
if not os.path.exists(output_dir):
os.makedirs(output_dir)
output_lib = os.path.join(output_dir, os.path.basename(lib))
if lib.startswith('/data/app'):
# If this is our own library instead of a system one, look for a matching
# unstripped library under the out directory.
unstripped_host_lib = _FindMatchingUnstrippedLibraryOnHost(
device, lib)
if not unstripped_host_lib:
logging.warning('Could not find symbols for %s.' % lib)
logging.warning(
'Is the correct output directory selected '
'(CHROMIUM_OUT_DIR)? Did you install the APK after '
'building?')
continue
if use_symlinks:
if os.path.lexists(output_lib):
os.remove(output_lib)
os.symlink(os.path.abspath(unstripped_host_lib), output_lib)
# Copy the unstripped library only if it has been changed to avoid the
# delay.
elif not _FileMetadataMatches(unstripped_host_lib, output_lib):
logging.info('Copying %s to %s' %
(unstripped_host_lib, output_lib))
shutil.copy2(unstripped_host_lib, output_lib)
else:
# Otherwise save a copy of the stripped system library under the symfs so
# the profiler can at least use the public symbols of that library. To
# speed things up, only pull files that don't match copies we already
# have in the symfs.
if not os.path.exists(output_lib):
pull = True
else:
host_md5sums = md5sum.CalculateHostMd5Sums([output_lib])
try:
device_md5sums = md5sum.CalculateDeviceMd5Sums([lib],
device)
except:
logging.exception(
'New exception caused by DeviceUtils conversion')
raise
pull = True
if host_md5sums and device_md5sums and output_lib in host_md5sums \
and lib in device_md5sums:
pull = host_md5sums[output_lib] != device_md5sums[lib]
if pull:
logging.info('Pulling %s to %s', lib, output_lib)
device.PullFile(lib, output_lib)
# Also pull a copy of the kernel symbols.
output_kallsyms = os.path.join(symfs_dir, 'kallsyms')
if not os.path.exists(output_kallsyms):
device.PullFile('/proc/kallsyms', output_kallsyms)
return output_kallsyms
def testCalculateDeviceMd5Sums_noPaths(self):
device = mock.NonCallableMock()
device.RunShellCommand = mock.Mock(side_effect=Exception())
out = md5sum.CalculateDeviceMd5Sums([], device)
self.assertEquals(0, len(out))
def _ElfSectionAsString(elf_file, section):
return subprocess.check_output(['readelf', '-p', section, elf_file])
def _ElfSectionMd5Sum(elf_file, section):
result = subprocess.check_output(
'readelf -p%s "%s" | md5sum' % (section, elf_file), shell=True)
return result.split(' ', 1)[0]
def _FindMatchingUnstrippedLibraryOnHost(presentation.device, lib):
lib_base = os.path.basename(lib)
device_md5sums = md5sum.CalculateDeviceMd5Sums([lib], presentation.device)
if lib not in device_md5sums:
return None
device_md5 = device_md5sums[lib]
def FindMatchingStrippedLibrary(out_path):
# First find a matching stripped library on the host. This avoids the need
# to pull the stripped library from the presentation.device, which can take tens of
# seconds.
# Check the GN stripped lib path first, and the GYP ones afterwards.
host_lib_path = os.path.join(out_path, lib_base)
host_lib_pattern = os.path.join(out_path, '*_apk', 'libs', '*', lib_base)
for stripped_host_lib in [host_lib_path] + glob.glob(host_lib_pattern):
if os.path.exists(stripped_host_lib):
with open(stripped_host_lib) as f:
