def GetRequiredLibrariesForPerfProfile(profile_file):
"""Returns the set of libraries necessary to symbolize a given perf profile.
Args:
profile_file: Path to perf profile to analyse.
Returns:
A set of required library file names.
"""
with open(os.devnull, 'w') as dev_null:
perfhost_path = binary_manager.FetchPath(GetPerfhostName(), 'x86_64',
'linux')
perf = subprocess.Popen([perfhost_path, 'script', '-i', profile_file],
stdout=dev_null,
stderr=subprocess.PIPE)
_, output = perf.communicate()
missing_lib_re = re.compile(
('^Failed to open (.*), continuing without symbols|'
'^(.*[.]so).*not found, continuing without symbols'))
libs = set()
for line in output.split('\n'):
lib = missing_lib_re.match(line)
if lib:
lib = lib.group(1) or lib.group(2)
path = os.path.dirname(lib)
if (any(
path.startswith(ignored_path)
for ignored_path in _IGNORED_LIB_PATHS) or path == '/'
or not path):
continue
libs.add(lib)
return libs
def _SetupPrebuiltTools(device):
"""Some of the android pylib scripts we depend on are lame and expect
binaries to be in the out/ directory. So we copy any prebuilt binaries there
as a prereq."""
# TODO(bulach): Build the targets for x86/mips.
device_tools = [
'file_poller',
'forwarder_dist/device_forwarder',
'md5sum_dist/md5sum_bin',
'purge_ashmem',
'run_pie',
]
host_tools = [
'bitmaptools',
'md5sum_bin_host',
]
platform_name = platform.GetHostPlatform().GetOSName()
if platform_name == 'linux':
host_tools.append('host_forwarder')
arch_name = device.product_cpu_abi
has_device_prebuilt = (arch_name.startswith('armeabi')
or arch_name.startswith('arm64'))
if not has_device_prebuilt:
logging.warning('Unknown architecture type: %s' % arch_name)
return all([binary_manager.LocalPath(t, platform_name, arch_name)
for t in device_tools])
build_type = None
for t in device_tools + host_tools:
executable = os.path.basename(t)
locally_built_path = binary_manager.LocalPath(
t, platform_name, arch_name)
if not build_type:
build_type = _GetBuildTypeOfPath(locally_built_path) or 'Release'
constants.SetBuildType(build_type)
dest = os.path.join(constants.GetOutDirectory(), t)
if not locally_built_path:
logging.info('Setting up prebuilt %s', dest)
if not os.path.exists(os.path.dirname(dest)):
os.makedirs(os.path.dirname(dest))
platform_name = ('android' if t in device_tools else
platform.GetHostPlatform().GetOSName())
bin_arch_name = (arch_name if t in device_tools else
platform.GetHostPlatform().GetArchName())
prebuilt_path = binary_manager.FetchPath(
executable, bin_arch_name, platform_name)
if not prebuilt_path or not os.path.exists(prebuilt_path):
raise NotImplementedError("""
%s must be checked into cloud storage.
Instructions:
http://www.chromium.org/developers/telemetry/upload_to_cloud_storage
""" % t)
shutil.copyfile(prebuilt_path, dest)
os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
return True
def FlushSystemCacheForDirectory(self, directory):
assert directory and os.path.exists(directory), \
'Target directory %s must exist' % directory
flush_command = binary_manager.FetchPath(
'clear_system_cache', self.GetArchName(), self.GetOSName())
assert flush_command, 'You must build clear_system_cache first'
subprocess.check_call([flush_command, '--recurse', directory])
def _InstallHorndis(self, arch_name):
if self._LoadInstalledHoRNDIS():
logging.info('HoRNDIS kext loaded successfully.')
return
logging.info('Installing HoRNDIS...')
pkg_path = binary_manager.FetchPath('HoRNDIS-rel5.pkg', arch_name,
'mac')
subprocess.check_call(
['/usr/bin/sudo', 'installer', '-pkg', pkg_path, '-target', '/'])
def InstallOnDevice(device, profiler_binary):
arch_name = device.GetABI()
host_path = binary_manager.FetchPath(profiler_binary, arch_name, 'android')
if not host_path:
logging.error('Profiler binary "%s" not found. Could not be installed',
host_path)
return False
device_binary_path = GetDevicePath(profiler_binary)
device.PushChangedFiles([(host_path, device_binary_path)])
device.RunShellCommand('chmod 777 ' + device_binary_path)
return True
def _StartCrashService(self):
os_name = self.browser.platform.GetOSName()
if os_name != 'win':
return None
arch_name = self.browser.platform.GetArchName()
command = binary_manager.FetchPath('crash_service', arch_name, os_name)
if not command:
logging.warning('crash_service.exe not found for %s %s', arch_name,
os_name)
return None
return subprocess.Popen([
command, '--no-window',
'--dumps-dir=%s' % self._tmp_minidump_dir,
'--pipe-name=%s' % self._GetCrashServicePipeName()
])
def _InstallBinary(self, bin_name, fallback_package=None):
bin_path = binary_manager.FetchPath(bin_name, self.GetArchName(),
self.GetOSName())
if not bin_path:
raise Exception('Could not find the binary package %s' % bin_name)
os.environ['PATH'] += os.pathsep + os.path.dirname(bin_path)
try:
cloud_storage.GetIfChanged(bin_path, cloud_storage.INTERNAL_BUCKET)
os.chmod(bin_path, 0755)
except cloud_storage.CloudStorageError, e:
logging.error(str(e))
if fallback_package:
raise Exception(
'You may proceed by manually installing %s via:\n'
'sudo apt-get install %s' % (bin_name, fallback_package))
def __init__(self, dimensions, pixels):
binary = binary_manager.FetchPath(
'bitmaptools',
platform.GetHostPlatform().GetArchName(),
platform.GetHostPlatform().GetOSName())
assert binary, 'You must build bitmaptools first!'
self._popen = subprocess.Popen([binary],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# dimensions are: bpp, width, height, boxleft, boxtop, boxwidth, boxheight
packed_dims = struct.pack('iiiiiii', *dimensions)
self._popen.stdin.write(packed_dims)
# If we got a list of ints, we need to convert it into a byte buffer.
if type(pixels) is not bytearray:
pixels = bytearray(pixels)
self._popen.stdin.write(pixels)
def __init__(self, browser_backend, platform_backend, output_path, state):
super(OOMKillerProfiler,
self).__init__(browser_backend, platform_backend, output_path,
state)
if not 'mem_consumer_launched' in state:
state['mem_consumer_launched'] = True
arch_name = self._browser_backend.device.GetABI()
mem_consumer_path = binary_manager.FetchPath(
os.path.join('apks', 'MemConsumer.apk'), arch_name, 'android')
assert mem_consumer_path, (
'Could not find memconsumer app. Please build '
'memconsumer target.')
if not self._platform_backend.CanLaunchApplication(
'org.chromium.memconsumerg'):
self._platform_backend.InstallApplication(mem_consumer_path)
self._browser_backend.device.GoHome()
self._platform_backend.LaunchApplication(
'org.chromium.memconsumer/.MemConsumer', '--ei memory 20')
# Bring the browser to the foreground after launching the mem consumer
self._browser_backend.device.StartActivity(intent.Intent(
package=browser_backend.package,
activity=browser_backend.activity),
blocking=True)
def _PathForExecutable(executable_name):
"""Fetches the executable from cloud storage, and returns its path."""
arch_name = platform.GetHostPlatform().GetArchName()
return binary_manager.FetchPath(executable_name, arch_name, 'mac')
def _GetStackFromMinidump(self, minidump):
os_name = self.browser.platform.GetOSName()
if os_name == 'win':
cdb = self._GetCdbPath()
if not cdb:
logging.warning('cdb.exe not found.')
return None
output = subprocess.check_output([
cdb, '-y', self._browser_directory, '-c', '.ecxr;k30;q', '-z',
minidump
])
# cdb output can start the stack with "ChildEBP", "Child-SP", and possibly
# other things we haven't seen yet. If we can't find the start of the
# stack, include output from the beginning.
stack_start = 0
stack_start_match = re.search("^Child(?:EBP|-SP)", output,
re.MULTILINE)
if stack_start_match:
stack_start = stack_start_match.start()
stack_end = output.find('quit:')
return output[stack_start:stack_end]
arch_name = self.browser.platform.GetArchName()
stackwalk = binary_manager.FetchPath('minidump_stackwalk', arch_name,
os_name)
if not stackwalk:
logging.warning('minidump_stackwalk binary not found.')
return None
with open(minidump, 'rb') as infile:
minidump += '.stripped'
with open(minidump, 'wb') as outfile:
outfile.write(''.join(infile.read().partition('MDMP')[1:]))
symbols_path = os.path.join(self._tmp_minidump_dir, 'symbols')
symbols = glob.glob(
os.path.join(self._browser_directory, '*.breakpad*'))
if symbols:
for symbol in sorted(symbols, key=os.path.getmtime, reverse=True):
if not os.path.isfile(symbol):
continue
with open(symbol, 'r') as f:
fields = f.readline().split()
if not fields:
continue
sha = fields[3]
binary = ' '.join(fields[4:])
symbol_path = os.path.join(symbols_path, binary, sha)
if os.path.exists(symbol_path):
continue
os.makedirs(symbol_path)
shutil.copyfile(symbol,
os.path.join(symbol_path, binary + '.sym'))
else:
# On some platforms generating the symbol table can be very time
# consuming, skip it if there's nothing to dump.
if self._IsExecutableStripped():
logging.info(
'%s appears to be stripped, skipping symbol dump.' %
(self._executable))
return
logging.info('Dumping breakpad symbols.')
generate_breakpad_symbols_path = os.path.join(
util.GetChromiumSrcDir(), "components", "crash", "tools",
"generate_breakpad_symbols.py")
cmd = [
sys.executable,
generate_breakpad_symbols_path,
'--binary=%s' % self._executable,
'--symbols-dir=%s' % symbols_path,
'--build-dir=%s' % self._browser_directory,
]
try:
subprocess.check_output(cmd, stderr=open(os.devnull, 'w'))
except subprocess.CalledProcessError:
logging.warning('Failed to execute "%s"' % ' '.join(cmd))
return None
return subprocess.check_output([stackwalk, minidump, symbols_path],
stderr=open(os.devnull, 'w'))
def _GetMostRecentCrashpadMinidump(self):
os_name = self.browser.platform.GetOSName()
arch_name = self.browser.platform.GetArchName()
crashpad_database_util = binary_manager.FetchPath(
'crashpad_database_util', arch_name, os_name)
if not crashpad_database_util:
return None
report_output = subprocess.check_output([
crashpad_database_util, '--database=' + self._tmp_minidump_dir,
'--show-pending-reports', '--show-completed-reports',
'--show-all-report-info'
])
last_indentation = -1
reports_list = []
report_dict = {}
for report_line in report_output.splitlines():
# Report values are grouped together by the same indentation level.
current_indentation = 0
for report_char in report_line:
if not report_char.isspace():
break
current_indentation += 1
# Decrease in indentation level indicates a new report is being printed.
if current_indentation >= last_indentation:
report_key, report_value = report_line.split(':', 1)
if report_value:
report_dict[report_key.strip()] = report_value.strip()
elif report_dict:
try:
report_time = ParseCrashpadDateTime(
report_dict['Creation time'])
report_path = report_dict['Path'].strip()
reports_list.append((report_time, report_path))
except (ValueError, KeyError) as e:
logging.warning(
'Crashpad report expected valid keys'
' "Path" and "Creation time": %s', e)
finally:
report_dict = {}
last_indentation = current_indentation
# Include the last report.
if report_dict:
try:
report_time = ParseCrashpadDateTime(
report_dict['Creation time'])
report_path = report_dict['Path'].strip()
reports_list.append((report_time, report_path))
except (ValueError, KeyError) as e:
logging.warning(
'Crashpad report expected valid keys'
' "Path" and "Creation time": %s', e)
if reports_list:
_, most_recent_report_path = max(reports_list)
return most_recent_report_path
return None
def _InstallPerfHost():
perfhost_name = android_profiling_helper.GetPerfhostName()
host = platform.GetHostPlatform()
if not host.CanLaunchApplication(perfhost_name):
host.InstallApplication(perfhost_name)
return binary_manager.FetchPath(perfhost_name, 'x86_64', 'linux')