def _SaveTestsToPickle(pickle_path, jar_path, tests):
jar_md5 = md5sum.CalculateHostMd5Sums(jar_path)[jar_path]
pickle_data = {
'VERSION': _PICKLE_FORMAT_VERSION,
'JAR_MD5SUM': jar_md5,
'TEST_METHODS': tests,
}
with open(pickle_path, 'w') as pickle_file:
pickle.dump(pickle_data, pickle_file)
def testCalculateHostMd5Sums_singlePath(self):
test_path = '/test/host/file.dat'
mock_get_cmd_output = mock.Mock(return_value='0123456789abcdef')
with mock.patch('devil.utils.cmd_helper.GetCmdOutput',
new=mock_get_cmd_output):
out = md5sum.CalculateHostMd5Sums(test_path)
self.assertEquals(1, len(out))
self.assertTrue('/test/host/file.dat' in out)
self.assertEquals('0123456789abcdef', out['/test/host/file.dat'])
mock_get_cmd_output.assert_called_once_with(
[HOST_MD5_EXECUTABLE, "-gz", mock.ANY])
def _GetTestsFromPickle(pickle_path, jar_path):
if not os.path.exists(pickle_path):
raise TestListPickleException('%s does not exist.' % pickle_path)
if os.path.getmtime(pickle_path) <= os.path.getmtime(jar_path):
raise TestListPickleException('%s newer than %s.' %
(jar_path, pickle_path))
with open(pickle_path, 'r') as f:
pickle_data = pickle.load(f)
jar_md5 = md5sum.CalculateHostMd5Sums(jar_path)[jar_path]
if pickle_data['VERSION'] != _PICKLE_FORMAT_VERSION:
raise TestListPickleException('PICKLE_FORMAT_VERSION has changed.')
if pickle_data['JAR_MD5SUM'] != jar_md5:
raise TestListPickleException('JAR file MD5 sum differs.')
return pickle_data['TEST_METHODS']
def testCalculateHostMd5Sums_generator(self):
test_paths = ('/test/host/' + p for p in ['file0.dat', 'file1.dat'])
mock_get_cmd_output = mock.Mock(
return_value='0123456789abcdeffedcba9876543210 /test/host/file0.dat\n'
'123456789abcdef00fedcba987654321 /test/host/file1.dat\n')
with mock.patch('devil.utils.cmd_helper.GetCmdOutput',
new=mock_get_cmd_output):
out = md5sum.CalculateHostMd5Sums(test_paths)
self.assertEquals(2, len(out))
self.assertTrue('/test/host/file0.dat' in out)
self.assertEquals('0123456789abcdeffedcba9876543210',
out['/test/host/file0.dat'])
self.assertTrue('/test/host/file1.dat' in out)
self.assertEquals('123456789abcdef00fedcba987654321',
out['/test/host/file1.dat'])
mock_get_cmd_output.assert_called_once_with(
[HOST_MD5_EXECUTABLE, '/test/host/file0.dat', '/test/host/file1.dat'])
def _GetCachedProguardData(self):
if (os.path.exists(self._pickled_proguard_name)
and (os.path.getmtime(self._pickled_proguard_name) >
os.path.getmtime(self._jar_path))):
logging.info('Loading cached proguard output from %s',
self._pickled_proguard_name)
try:
with open(self._pickled_proguard_name, 'r') as r:
d = pickle.loads(r.read())
jar_md5 = md5sum.CalculateHostMd5Sums(
self._jar_path)[os.path.realpath(self._jar_path)]
if (d['JAR_MD5SUM'] == jar_md5
and d['VERSION'] == PICKLE_FORMAT_VERSION):
self._test_methods = d['TEST_METHODS']
return True
except:
logging.warning(
'PICKLE_FORMAT_VERSION has changed, ignoring cache')
return False
def _GetProguardData(self):
logging.info('Retrieving test methods via proguard.')
p = proguard.Dump(self._jar_path)
class_lookup = dict((c['class'], c) for c in p['classes'])
def recursive_get_annotations(c):
s = c['superclass']
if s in class_lookup:
a = recursive_get_annotations(class_lookup[s])
else:
a = {}
a.update(c['annotations'])
return a
test_classes = (c for c in p['classes'] if c['class'].endswith('Test'))
for c in test_classes:
class_annotations = recursive_get_annotations(c)
test_methods = (m for m in c['methods']
if m['method'].startswith('test'))
for m in test_methods:
qualified_method = '%s#%s' % (c['class'], m['method'])
annotations = dict(class_annotations)
annotations.update(m['annotations'])
self._test_methods[qualified_method] = m
self._test_methods[qualified_method][
'annotations'] = annotations
logging.info('Storing proguard output to %s',
self._pickled_proguard_name)
d = {
'VERSION':
PICKLE_FORMAT_VERSION,
'TEST_METHODS':
self._test_methods,
'JAR_MD5SUM':
md5sum.CalculateHostMd5Sums(self._jar_path)[os.path.realpath(
self._jar_path)]
}
with open(self._pickled_proguard_name, 'w') as f:
f.write(pickle.dumps(d))
def _GetTestsFromPickle(self, pickle_path, jar_path):
if not os.path.exists(pickle_path):
raise self.ProguardPickleException('%s does not exist.' % pickle_path)
if os.path.getmtime(pickle_path) <= os.path.getmtime(jar_path):
raise self.ProguardPickleException(
'%s newer than %s.' % (jar_path, pickle_path))
with open(pickle_path, 'r') as pickle_file:
pickle_data = pickle.loads(pickle_file.read())
jar_md5 = md5sum.CalculateHostMd5Sums(jar_path)[jar_path]
try:
if pickle_data['VERSION'] != _PICKLE_FORMAT_VERSION:
raise self.ProguardPickleException('PICKLE_FORMAT_VERSION has changed.')
if pickle_data['JAR_MD5SUM'] != jar_md5:
raise self.ProguardPickleException('JAR file MD5 sum differs.')
return pickle_data['TEST_METHODS']
except TypeError as e:
logging.error(pickle_data)
raise self.ProguardPickleException(str(e))
def CreateSymFs(device, symfs_dir, libraries, use_symlinks=True):
"""Creates a symfs directory to be used for symbolizing profiles.
Prepares a set of files ("symfs") to be used with profilers such as perf for
converting binary addresses into human readable function names.
Args:
device: DeviceUtils instance identifying the target device.
symfs_dir: Path where the symfs should be created.
libraries: Set of library file names that should be included in the symfs.
use_symlinks: If True, link instead of copy unstripped libraries into the
symfs. This will speed up the operation, but the resulting symfs will no
longer be valid if the linked files are modified, e.g., by rebuilding.
Returns:
The absolute path to the kernel symbols within the created symfs.
"""
logging.info('Building symfs into %s.' % symfs_dir)
for lib in libraries:
device_dir = os.path.dirname(lib)
output_dir = os.path.join(symfs_dir, device_dir[1:])
if not os.path.exists(output_dir):
os.makedirs(output_dir)
output_lib = os.path.join(output_dir, os.path.basename(lib))
if lib.startswith('/data/app'):
# If this is our own library instead of a system one, look for a matching
# unstripped library under the out directory.
unstripped_host_lib = _FindMatchingUnstrippedLibraryOnHost(
device, lib)
if not unstripped_host_lib:
logging.warning('Could not find symbols for %s.' % lib)
logging.warning(
'Is the correct output directory selected '
'(CHROMIUM_OUT_DIR)? Did you install the APK after '
'building?')
continue
if use_symlinks:
if os.path.lexists(output_lib):
os.remove(output_lib)
os.symlink(os.path.abspath(unstripped_host_lib), output_lib)
# Copy the unstripped library only if it has been changed to avoid the
# delay.
elif not _FileMetadataMatches(unstripped_host_lib, output_lib):
logging.info('Copying %s to %s' %
(unstripped_host_lib, output_lib))
shutil.copy2(unstripped_host_lib, output_lib)
else:
# Otherwise save a copy of the stripped system library under the symfs so
# the profiler can at least use the public symbols of that library. To
# speed things up, only pull files that don't match copies we already
# have in the symfs.
if not os.path.exists(output_lib):
pull = True
else:
host_md5sums = md5sum.CalculateHostMd5Sums([output_lib])
try:
device_md5sums = md5sum.CalculateDeviceMd5Sums([lib],
device)
except:
logging.exception(
'New exception caused by DeviceUtils conversion')
raise
pull = True
if host_md5sums and device_md5sums and output_lib in host_md5sums \
and lib in device_md5sums:
pull = host_md5sums[output_lib] != device_md5sums[lib]
if pull:
logging.info('Pulling %s to %s', lib, output_lib)
device.PullFile(lib, output_lib)
# Also pull a copy of the kernel symbols.
output_kallsyms = os.path.join(symfs_dir, 'kallsyms')
if not os.path.exists(output_kallsyms):
device.PullFile('/proc/kallsyms', output_kallsyms)
return output_kallsyms
os.remove(output_lib)
os.symlink(os.path.abspath(unstripped_host_lib), output_lib)
# Copy the unstripped library only if it has been changed to avoid the
# delay.
elif not _FileMetadataMatches(unstripped_host_lib, output_lib):
logging.info('Copying %s to %s' % (unstripped_host_lib, output_lib))
shutil.copy2(unstripped_host_lib, output_lib)
else:
# Otherwise save a copy of the stripped system library under the symfs so
# the profiler can at least use the public symbols of that library. To
# speed things up, only pull files that don't match copies we already
# have in the symfs.
if not os.path.exists(output_lib):
pull = True
else:
host_md5sums = md5sum.CalculateHostMd5Sums([output_lib])
device_md5sums = md5sum.CalculateDeviceMd5Sums([lib], presentation.device)
pull = True
if host_md5sums and device_md5sums and output_lib in host_md5sums \
and lib in device_md5sums:
pull = host_md5sums[output_lib] != device_md5sums[lib]
if pull:
logging.info('Pulling %s to %s', lib, output_lib)
presentation.device.PullFile(lib, output_lib)
# Also pull a copy of the kernel symbols.
output_kallsyms = os.path.join(symfs_dir, 'kallsyms')
if not os.path.exists(output_kallsyms):
presentation.device.PullFile('/proc/kallsyms', output_kallsyms)
