def _GetChangedFilesImpl(self, host_path, device_path):
real_host_path = os.path.realpath(host_path)
try:
real_device_path = self.RunShellCommand(
['realpath', device_path], single_line=True, check_return=True)
except device_errors.CommandFailedError:
real_device_path = None
if not real_device_path:
return [(host_path, device_path)]
host_hash_tuples = md5sum.CalculateHostMd5Sums([real_host_path])
device_paths_to_md5 = (
real_device_path if os.path.isfile(real_host_path)
else ('%s/%s' % (real_device_path, os.path.relpath(p, real_host_path))
for _, p in host_hash_tuples))
device_hash_tuples = md5sum.CalculateDeviceMd5Sums(
device_paths_to_md5, self)
if os.path.isfile(host_path):
if (not device_hash_tuples
or device_hash_tuples[0].hash != host_hash_tuples[0].hash):
return [(host_path, device_path)]
else:
return []
else:
device_tuple_dict = dict((d.path, d.hash) for d in device_hash_tuples)
to_push = []
for host_hash, host_abs_path in (
(h.hash, h.path) for h in host_hash_tuples):
device_abs_path = '%s/%s' % (
real_device_path, os.path.relpath(host_abs_path, real_host_path))
if (device_abs_path not in device_tuple_dict
or device_tuple_dict[device_abs_path] != host_hash):
to_push.append((host_abs_path, device_abs_path))
return to_push
def _SaveTestsToPickle(self, pickle_path, jar_path, tests):
jar_md5 = md5sum.CalculateHostMd5Sums(jar_path)[jar_path]
pickle_data = {
'VERSION': _PICKLE_FORMAT_VERSION,
'JAR_MD5SUM': jar_md5,
'TEST_METHODS': tests,
}
with open(pickle_path, 'w') as pickle_file:
pickle.dump(pickle_data, pickle_file)
def testCalculateHostMd5Sums_singlePath(self):
test_path = '/test/host/file.dat'
mock_get_cmd_output = mock.Mock(
return_value='0123456789abcdeffedcba9876543210 /test/host/file.dat')
with mock.patch('pylib.cmd_helper.GetCmdOutput', new=mock_get_cmd_output):
out = md5sum.CalculateHostMd5Sums(test_path)
self.assertEquals(1, len(out))
self.assertTrue('/test/host/file.dat' in out)
self.assertEquals('0123456789abcdeffedcba9876543210',
out['/test/host/file.dat'])
mock_get_cmd_output.assert_called_once_with(
[HOST_MD5_EXECUTABLE, '/test/host/file.dat'])
def testCalculateHostMd5Sums_generator(self):
test_paths = ('/test/host/' + p for p in ['file0.dat', 'file1.dat'])
mock_get_cmd_output = mock.Mock(
return_value='0123456789abcdeffedcba9876543210 /test/host/file0.dat\n'
'123456789abcdef00fedcba987654321 /test/host/file1.dat\n')
with mock.patch('pylib.cmd_helper.GetCmdOutput', new=mock_get_cmd_output):
out = md5sum.CalculateHostMd5Sums(test_paths)
self.assertEquals(2, len(out))
self.assertTrue('/test/host/file0.dat' in out)
self.assertEquals('0123456789abcdeffedcba9876543210',
out['/test/host/file0.dat'])
self.assertTrue('/test/host/file1.dat' in out)
self.assertEquals('123456789abcdef00fedcba987654321',
out['/test/host/file1.dat'])
mock_get_cmd_output.assert_called_once_with(
[HOST_MD5_EXECUTABLE, '/test/host/file0.dat', '/test/host/file1.dat'])
def _GetCachedProguardData(self):
if (os.path.exists(self._pickled_proguard_name)
and (os.path.getmtime(self._pickled_proguard_name) >
os.path.getmtime(self._jar_path))):
logging.info('Loading cached proguard output from %s',
self._pickled_proguard_name)
try:
with open(self._pickled_proguard_name, 'r') as r:
d = pickle.loads(r.read())
jar_md5 = md5sum.CalculateHostMd5Sums(self._jar_path)[0].hash
if (d['JAR_MD5SUM'] == jar_md5
and d['VERSION'] == PICKLE_FORMAT_VERSION):
self._test_methods = d['TEST_METHODS']
return True
except:
logging.warning(
'PICKLE_FORMAT_VERSION has changed, ignoring cache')
return False
def _GetProguardData(self):
logging.info('Retrieving test methods via proguard.')
p = proguard.Dump(self._jar_path)
class_lookup = dict((c['class'], c) for c in p['classes'])
def recursive_get_annotations(c):
s = c['superclass']
if s in class_lookup:
a = recursive_get_annotations(class_lookup[s])
else:
a = {}
a.update(c['annotations'])
return a
test_classes = (c for c in p['classes'] if c['class'].endswith('Test'))
for c in test_classes:
class_annotations = recursive_get_annotations(c)
test_methods = (m for m in c['methods']
if m['method'].startswith('test'))
for m in test_methods:
qualified_method = '%s#%s' % (c['class'], m['method'])
annotations = dict(class_annotations)
annotations.update(m['annotations'])
self._test_methods[qualified_method] = m
self._test_methods[qualified_method][
'annotations'] = annotations
logging.info('Storing proguard output to %s',
self._pickled_proguard_name)
d = {
'VERSION':
PICKLE_FORMAT_VERSION,
'TEST_METHODS':
self._test_methods,
'JAR_MD5SUM':
md5sum.CalculateHostMd5Sums(self._jar_path)[os.path.realpath(
self._jar_path)]
}
with open(self._pickled_proguard_name, 'w') as f:
f.write(pickle.dumps(d))
def _GetTestsFromPickle(self, pickle_path, jar_path):
if not os.path.exists(pickle_path):
raise self.ProguardPickleException('%s does not exist.' % pickle_path)
if os.path.getmtime(pickle_path) <= os.path.getmtime(jar_path):
raise self.ProguardPickleException(
'%s newer than %s.' % (jar_path, pickle_path))
with open(pickle_path, 'r') as pickle_file:
pickle_data = pickle.loads(pickle_file.read())
jar_md5, _ = md5sum.CalculateHostMd5Sums(jar_path)[0]
try:
if pickle_data['VERSION'] != _PICKLE_FORMAT_VERSION:
raise self.ProguardPickleException('PICKLE_FORMAT_VERSION has changed.')
if pickle_data['JAR_MD5SUM'] != jar_md5:
raise self.ProguardPickleException('JAR file MD5 sum differs.')
return pickle_data['TEST_METHODS']
except TypeError as e:
logging.error(pickle_data)
raise self.ProguardPickleException(str(e))
def CreateSymFs(device, symfs_dir, libraries, use_symlinks=True):
"""Creates a symfs directory to be used for symbolizing profiles.
Prepares a set of files ("symfs") to be used with profilers such as perf for
converting binary addresses into human readable function names.
Args:
device: DeviceUtils instance identifying the target device.
symfs_dir: Path where the symfs should be created.
libraries: Set of library file names that should be included in the symfs.
use_symlinks: If True, link instead of copy unstripped libraries into the
symfs. This will speed up the operation, but the resulting symfs will no
longer be valid if the linked files are modified, e.g., by rebuilding.
Returns:
The absolute path to the kernel symbols within the created symfs.
"""
logging.info('Building symfs into %s.' % symfs_dir)
for lib in libraries:
device_dir = os.path.dirname(lib)
output_dir = os.path.join(symfs_dir, device_dir[1:])
if not os.path.exists(output_dir):
os.makedirs(output_dir)
output_lib = os.path.join(output_dir, os.path.basename(lib))
if lib.startswith('/data/app/'):
# If this is our own library instead of a system one, look for a matching
# unstripped library under the out directory.
unstripped_host_lib = _FindMatchingUnstrippedLibraryOnHost(device, lib)
if not unstripped_host_lib:
logging.warning('Could not find symbols for %s.' % lib)
logging.warning('Is the correct output directory selected '
'(CHROMIUM_OUT_DIR)? Did you install the APK after '
'building?')
continue
if use_symlinks:
if os.path.lexists(output_lib):
os.remove(output_lib)
os.symlink(os.path.abspath(unstripped_host_lib), output_lib)
# Copy the unstripped library only if it has been changed to avoid the
# delay. Add one second to the modification time to guard against file
# systems with poor timestamp resolution.
elif not os.path.exists(output_lib) or \
(os.stat(unstripped_host_lib).st_mtime >
os.stat(output_lib).st_mtime + 1):
logging.info('Copying %s to %s' % (unstripped_host_lib, output_lib))
shutil.copy2(unstripped_host_lib, output_lib)
else:
# Otherwise save a copy of the stripped system library under the symfs so
# the profiler can at least use the public symbols of that library. To
# speed things up, only pull files that don't match copies we already
# have in the symfs.
if not os.path.exists(output_lib):
pull = True
else:
host_md5sums = md5sum.CalculateHostMd5Sums([output_lib])
device_md5sums = md5sum.CalculateDeviceMd5Sums([lib], device)
pull = (not host_md5sums or not device_md5sums
or host_md5sums[0] != device_md5sums[0])
if pull:
logging.info('Pulling %s to %s', lib, output_lib)
device.PullFile(lib, output_lib)
# Also pull a copy of the kernel symbols.
output_kallsyms = os.path.join(symfs_dir, 'kallsyms')
if not os.path.exists(output_kallsyms):
device.PullFile('/proc/kallsyms', output_kallsyms)
return output_kallsyms
