def GetRequiredLibrariesForPerfProfile(profile_file):
"""Returns the set of libraries necessary to symbolize a given perf profile.
Args:
profile_file: Path to perf profile to analyse.
Returns:
A set of required library file names.
"""
with open(os.devnull, 'w') as dev_null:
perfhost_path = support_binaries.FindPath(GetPerfhostName(), 'x86_64',
'linux')
perf = subprocess.Popen([perfhost_path, 'script', '-i', profile_file],
stdout=dev_null,
stderr=subprocess.PIPE)
_, output = perf.communicate()
missing_lib_re = re.compile(
r'^Failed to open (.*), continuing without symbols')
libs = set()
for line in output.split('\n'):
lib = missing_lib_re.match(line)
if lib:
lib = lib.group(1)
path = os.path.dirname(lib)
if (any(
path.startswith(ignored_path)
for ignored_path in _IGNORED_LIB_PATHS) or path == '/'
or not path):
continue
libs.add(lib)
return libs
def testFindPath(self):
md5sum_path = support_binaries.FindPath(
'md5sum_bin_host', 'x86_64', 'linux')
self.assertNotEquals(md5sum_path, None)
self.assertTrue(os.path.isabs(md5sum_path))
st = os.stat(md5sum_path)
self.assertTrue(st.st_mode & stat.S_IXUSR)
def FetchPath(self, dependency, platform, try_support_binaries=False):
"""Get a path to an executable for |dependency|, downloading as needed.
A path to a default executable may be returned if a platform specific
version is not specified in the config(s).
Args:
dependency: Name of the desired dependency, as given in the config(s)
used in this DependencyManager.
platform: Name of the platform the dependency will run on. Often of the
form 'os_architecture'. Must match those specified in the config(s)
used in this DependencyManager.
try_support_binaries: True if support_binaries should be queried if the
dependency_manager was not initialized with data for |dependency|.
Returns:
A path to an executable of |dependency| that will run on |platform|,
downloading from cloud storage if needed.
Raises:
NoPathFoundError: If a local copy of the executable cannot be found and
a remote path could not be downloaded from cloud_storage.
CredentialsError: If cloud_storage credentials aren't configured.
PermissionError: If cloud_storage credentials are configured, but not
with an account that has permission to download the remote file.
NotFoundError: If the remote file does not exist where expected in
cloud_storage.
ServerError: If an internal server error is hit while downloading the
remote file.
CloudStorageError: If another error occured while downloading the remote
path.
FileNotFoundError: If an attempted download was otherwise unsuccessful.
"""
dependency_info = self._GetDependencyInfo(dependency, platform)
if not dependency_info:
if not try_support_binaries:
raise exceptions.NoPathFoundError(dependency, platform)
# TODO(aiolos): Remove the support_binaries call and always raise
# NoPathFound once the binary dependencies are moved over to the new
# system.
# platform should be of the form '%s_%s' % (os_name, arch_name) when
# called from the binary_manager.
platform_parts = platform.split('_', 1)
assert len(platform_parts) == 2
platform_os, platform_arch = platform_parts
logging.info(
'Calling into support_binaries with dependency %s, platform '
'%s and arch %s. support_binaries is deprecated.' %
(dependency, platform_os, platform_arch))
return support_binaries.FindPath(dependency, platform_arch,
platform_os)
path = self._LocalPath(dependency_info)
if not path or not os.path.exists(path):
path = dependency_info.GetRemotePath()
if not path or not os.path.exists(path):
raise exceptions.NoPathFoundError(dependency, platform)
return path
def _InstallIpfw(self):
ipfw_bin = support_binaries.FindPath('ipfw', self.GetArchName(),
self.GetOSName())
ipfw_mod = support_binaries.FindPath('ipfw_mod.ko', self.GetArchName(),
self.GetOSName())
try:
changed = cloud_storage.GetIfChanged(ipfw_bin,
cloud_storage.INTERNAL_BUCKET)
changed |= cloud_storage.GetIfChanged(
ipfw_mod, cloud_storage.INTERNAL_BUCKET)
except cloud_storage.CloudStorageError, e:
logging.error(str(e))
logging.error('You may proceed by manually building and installing'
'dummynet for your kernel. See: '
'http://info.iet.unipi.it/~luigi/dummynet/')
sys.exit(1)
def _InstallHorndis(self, arch_name):
if self._LoadInstalledHoRNDIS():
logging.info('HoRNDIS kext loaded successfully.')
return
logging.info('Installing HoRNDIS...')
pkg_path = support_binaries.FindPath('HoRNDIS-rel5.pkg', arch_name, 'mac')
subprocess.check_call(
['/usr/bin/sudo', 'installer', '-pkg', pkg_path, '-target', '/'])
def _SetupPrebuiltTools(adb):
"""Some of the android pylib scripts we depend on are lame and expect
binaries to be in the out/ directory. So we copy any prebuilt binaries there
as a prereq."""
# TODO(bulach): Build the targets for x86/mips.
device_tools = [
'file_poller',
'forwarder_dist/device_forwarder',
'md5sum_dist/md5sum_bin',
'purge_ashmem',
'run_pie',
]
host_tools = [
'bitmaptools',
'md5sum_bin_host',
]
if platform.GetHostPlatform().GetOSName() == 'linux':
host_tools.append('host_forwarder')
arch_name = adb.device().GetABI()
has_device_prebuilt = (arch_name.startswith('armeabi')
or arch_name.startswith('arm64'))
if not has_device_prebuilt:
logging.warning('Unknown architecture type: %s' % arch_name)
return all(
[support_binaries.FindLocallyBuiltPath(t) for t in device_tools])
build_type = None
for t in device_tools + host_tools:
executable = os.path.basename(t)
locally_built_path = support_binaries.FindLocallyBuiltPath(t)
if not build_type:
build_type = _GetBuildTypeOfPath(locally_built_path) or 'Release'
constants.SetBuildType(build_type)
dest = os.path.join(constants.GetOutDirectory(), t)
if not locally_built_path:
logging.info('Setting up prebuilt %s', dest)
if not os.path.exists(os.path.dirname(dest)):
os.makedirs(os.path.dirname(dest))
platform_name = ('android' if t in device_tools else
platform.GetHostPlatform().GetOSName())
bin_arch_name = (arch_name if t in device_tools else
platform.GetHostPlatform().GetArchName())
prebuilt_path = support_binaries.FindPath(executable,
bin_arch_name,
platform_name)
if not prebuilt_path or not os.path.exists(prebuilt_path):
raise NotImplementedError("""
%s must be checked into cloud storage.
Instructions:
http://www.chromium.org/developers/telemetry/upload_to_cloud_storage
""" % t)
shutil.copyfile(prebuilt_path, dest)
os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
return True
def InstallOnDevice(device, profiler_binary):
arch_name = device.GetABI()
host_path = support_binaries.FindPath(profiler_binary, arch_name,
'android')
if not host_path:
logging.error('Profiler binary "%s" not found. Could not be installed',
host_path)
return False
device_binary_path = GetDevicePath(profiler_binary)
device.PushChangedFiles([(host_path, device_binary_path)])
device.RunShellCommand('chmod 777 ' + device_binary_path)
return True
def _StartCrashService(self):
os_name = self.browser.platform.GetOSName()
if os_name != 'win':
return None
arch_name = self.browser.platform.GetArchName()
command = support_binaries.FindPath('crash_service', arch_name,
os_name)
if not command:
logging.warning('crash_service.exe not found for %s %s', arch_name,
os_name)
return None
return subprocess.Popen([
command, '--no-window',
'--dumps-dir=%s' % self._tmp_minidump_dir,
'--pipe-name=%s' % self._GetCrashServicePipeName()
])
def _InstallBinary(self, bin_name, fallback_package=None):
bin_path = support_binaries.FindPath(bin_name, self.GetArchName(),
self.GetOSName())
if not bin_path:
raise Exception('Could not find the binary package %s' % bin_name)
os.environ['PATH'] += os.pathsep + os.path.dirname(bin_path)
try:
cloud_storage.GetIfChanged(bin_path, cloud_storage.INTERNAL_BUCKET)
os.chmod(bin_path, 0755)
except cloud_storage.CloudStorageError, e:
logging.error(str(e))
if fallback_package:
raise Exception(
'You may proceed by manually installing %s via:\n'
'sudo apt-get install %s' % (bin_name, fallback_package))
def __init__(self, dimensions, pixels):
binary = support_binaries.FindPath(
'bitmaptools',
platform.GetHostPlatform().GetArchName(),
platform.GetHostPlatform().GetOSName())
assert binary, 'You must build bitmaptools first!'
self._popen = subprocess.Popen([binary],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# dimensions are: bpp, width, height, boxleft, boxtop, boxwidth, boxheight
packed_dims = struct.pack('iiiiiii', *dimensions)
self._popen.stdin.write(packed_dims)
# If we got a list of ints, we need to convert it into a byte buffer.
if type(pixels) is not bytearray:
pixels = bytearray(pixels)
self._popen.stdin.write(pixels)
def __init__(self, browser_backend, platform_backend, output_path, state):
super(OOMKillerProfiler, self).__init__(
browser_backend, platform_backend, output_path, state)
if not 'mem_consumer_launched' in state:
state['mem_consumer_launched'] = True
arch_name = self._browser_backend.adb.device().GetABI()
mem_consumer_path = support_binaries.FindPath(
os.path.join('apks', 'MemConsumer.apk'), arch_name, 'android')
assert mem_consumer_path, ('Could not find memconsumer app. Please build '
'memconsumer target.')
if not self._platform_backend.CanLaunchApplication(
'org.chromium.memconsumerg'):
self._platform_backend.InstallApplication(mem_consumer_path)
self._browser_backend.adb.device().GoHome()
self._platform_backend.LaunchApplication(
'org.chromium.memconsumer/.MemConsumer',
'--ei memory 20')
# Bring the browser to the foreground after launching the mem consumer
self._browser_backend.adb.device().StartActivity(
intent.Intent(package=browser_backend.package,
activity=browser_backend.activity),
blocking=True)
def _InstallPerfHost():
perfhost_name = android_profiling_helper.GetPerfhostName()
host = platform.GetHostPlatform()
if not host.CanLaunchApplication(perfhost_name):
host.InstallApplication(perfhost_name)
return support_binaries.FindPath(perfhost_name, 'x86_64', 'linux')
def FetchPath(binary_name, platform, arch):
""" Return a path to the appropriate executable for <binary_name>, downloading
from cloud storage if needed, or None if it cannot be found.
"""
return support_binaries.FindPath(binary_name, platform, arch)
def _GetStackFromMinidump(self, minidump):
os_name = self.browser.platform.GetOSName()
if os_name == 'win':
cdb = self._GetCdbPath()
if not cdb:
logging.warning('cdb.exe not found.')
return None
output = subprocess.check_output([
cdb, '-y', self._browser_directory, '-c', '.ecxr;k30;q', '-z',
minidump
])
# cdb output can start the stack with "ChildEBP", "Child-SP", and possibly
# other things we haven't seen yet. If we can't find the start of the
# stack, include output from the beginning.
stack_start = 0
stack_start_match = re.search("^Child(?:EBP|-SP)", output,
re.MULTILINE)
if stack_start_match:
stack_start = stack_start_match.start()
stack_end = output.find('quit:')
return output[stack_start:stack_end]
arch_name = self.browser.platform.GetArchName()
stackwalk = support_binaries.FindPath('minidump_stackwalk', arch_name,
os_name)
if not stackwalk:
logging.warning('minidump_stackwalk binary not found.')
return None
with open(minidump, 'rb') as infile:
minidump += '.stripped'
with open(minidump, 'wb') as outfile:
outfile.write(''.join(infile.read().partition('MDMP')[1:]))
symbols_path = os.path.join(self._tmp_minidump_dir, 'symbols')
symbols = glob.glob(
os.path.join(self._browser_directory, '*.breakpad*'))
if symbols:
for symbol in sorted(symbols, key=os.path.getmtime, reverse=True):
if not os.path.isfile(symbol):
continue
with open(symbol, 'r') as f:
fields = f.readline().split()
if not fields:
continue
sha = fields[3]
binary = ' '.join(fields[4:])
symbol_path = os.path.join(symbols_path, binary, sha)
if os.path.exists(symbol_path):
continue
os.makedirs(symbol_path)
shutil.copyfile(symbol,
os.path.join(symbol_path, binary + '.sym'))
else:
# On some platforms generating the symbol table can be very time
# consuming, skip it if there's nothing to dump.
if self._IsExecutableStripped():
logging.info(
'%s appears to be stripped, skipping symbol dump.' %
(self._executable))
return
logging.info('Dumping breakpad symbols.')
generate_breakpad_symbols_path = os.path.join(
util.GetChromiumSrcDir(), "components", "crash", "tools",
"generate_breakpad_symbols.py")
cmd = [
sys.executable,
generate_breakpad_symbols_path,
'--binary=%s' % self._executable,
'--symbols-dir=%s' % symbols_path,
'--build-dir=%s' % self._browser_directory,
]
try:
subprocess.check_output(cmd, stderr=open(os.devnull, 'w'))
except subprocess.CalledProcessError:
logging.warning('Failed to execute "%s"' % ' '.join(cmd))
return None
return subprocess.check_output([stackwalk, minidump, symbols_path],
stderr=open(os.devnull, 'w'))
def _GetMostRecentCrashpadMinidump(self):
os_name = self.browser.platform.GetOSName()
arch_name = self.browser.platform.GetArchName()
crashpad_database_util = support_binaries.FindPath(
'crashpad_database_util', arch_name, os_name)
if not crashpad_database_util:
return None
report_output = subprocess.check_output([
crashpad_database_util, '--database=' + self._tmp_minidump_dir,
'--show-pending-reports', '--show-completed-reports',
'--show-all-report-info'
])
last_indentation = -1
reports_list = []
report_dict = {}
for report_line in report_output.splitlines():
# Report values are grouped together by the same indentation level.
current_indentation = 0
for report_char in report_line:
if not report_char.isspace():
break
current_indentation += 1
# Decrease in indentation level indicates a new report is being printed.
if current_indentation >= last_indentation:
report_key, report_value = report_line.split(':', 1)
if report_value:
report_dict[report_key.strip()] = report_value.strip()
elif report_dict:
try:
report_time = ParseCrashpadDateTime(
report_dict['Creation time'])
report_path = report_dict['Path'].strip()
reports_list.append((report_time, report_path))
except (ValueError, KeyError) as e:
logging.warning(
'Crashpad report expected valid keys'
' "Path" and "Creation time": %s', e)
finally:
report_dict = {}
last_indentation = current_indentation
# Include the last report.
if report_dict:
try:
report_time = ParseCrashpadDateTime(
report_dict['Creation time'])
report_path = report_dict['Path'].strip()
reports_list.append((report_time, report_path))
except (ValueError, KeyError) as e:
logging.warning(
'Crashpad report expected valid keys'
' "Path" and "Creation time": %s', e)
if reports_list:
_, most_recent_report_path = max(reports_list)
return most_recent_report_path
return None
def _PathForExecutable(executable_name):
"""Fetches the executable from cloud storage, and returns its path."""
arch_name = platform.GetHostPlatform().GetArchName()
return support_binaries.FindPath(executable_name, arch_name, 'mac')