def _InstallIpfw(self):
ipfw_bin = support_binaries.FindPath('ipfw', self.GetOSName())
ipfw_mod = support_binaries.FindPath('ipfw_mod.ko', self.GetOSName())
try:
changed = cloud_storage.GetIfChanged(
ipfw_bin, cloud_storage.INTERNAL_BUCKET)
changed |= cloud_storage.GetIfChanged(
ipfw_mod, cloud_storage.INTERNAL_BUCKET)
except cloud_storage.CloudStorageError, e:
logging.error(e)
logging.error('You may proceed by manually installing dummynet. See: '
'http://info.iet.unipi.it/~luigi/dummynet/')
sys.exit(1)
def testFindPath(self):
md5sum_path = support_binaries.FindPath(
'md5sum_bin_host', 'x86_64', 'linux')
self.assertNotEquals(md5sum_path, None)
self.assertTrue(os.path.isabs(md5sum_path))
st = os.stat(md5sum_path)
self.assertTrue(st.st_mode & stat.S_IXUSR)
def GetRequiredLibrariesForPerfProfile(profile_file):
"""Returns the set of libraries necessary to symbolize a given perf profile.
Args:
profile_file: Path to perf profile to analyse.
Returns:
A set of required library file names.
"""
with open(os.devnull, 'w') as dev_null:
perfhost_path = support_binaries.FindPath('perfhost', 'linux')
perf = subprocess.Popen([perfhost_path, 'script', '-i', profile_file],
stdout=dev_null,
stderr=subprocess.PIPE)
_, output = perf.communicate()
missing_lib_re = re.compile(
r'^Failed to open (.*), continuing without symbols')
libs = set()
for line in output.split('\n'):
lib = missing_lib_re.match(line)
if lib:
lib = lib.group(1)
path = os.path.dirname(lib)
if any(
path.startswith(ignored_path)
for ignored_path in _IGNORED_LIB_PATHS) or path == '/':
continue
libs.add(lib)
return libs
def SetupPrebuiltTools(adb):
"""Some of the android pylib scripts we depend on are lame and expect
binaries to be in the out/ directory. So we copy any prebuilt binaries there
as a prereq."""
# TODO(bulach): Build the targets for x86/mips.
device_tools = [
'file_poller',
'forwarder_dist/device_forwarder',
'md5sum_dist/md5sum_bin',
'purge_ashmem',
'run_pie',
]
host_tools = [
'bitmaptools',
'md5sum_bin_host',
]
if platform.GetHostPlatform().GetOSName() == 'linux':
host_tools.append('host_forwarder')
arch_name = adb.device().GetABI()
has_device_prebuilt = (arch_name.startswith('armeabi')
or arch_name.startswith('arm64'))
if not has_device_prebuilt:
logging.warning('Unknown architecture type: %s' % arch_name)
return all(
[support_binaries.FindLocallyBuiltPath(t) for t in device_tools])
build_type = None
for t in device_tools + host_tools:
executable = os.path.basename(t)
locally_built_path = support_binaries.FindLocallyBuiltPath(t)
if not build_type:
build_type = GetBuildTypeOfPath(locally_built_path) or 'Release'
constants.SetBuildType(build_type)
dest = os.path.join(constants.GetOutDirectory(), t)
if not locally_built_path:
logging.info('Setting up prebuilt %s', dest)
if not os.path.exists(os.path.dirname(dest)):
os.makedirs(os.path.dirname(dest))
platform_name = ('android' if t in device_tools else
platform.GetHostPlatform().GetOSName())
bin_arch_name = (arch_name if t in device_tools else
platform.GetHostPlatform().GetArchName())
prebuilt_path = support_binaries.FindPath(executable,
bin_arch_name,
platform_name)
if not prebuilt_path or not os.path.exists(prebuilt_path):
raise NotImplementedError("""
%s must be checked into cloud storage.
Instructions:
http://www.chromium.org/developers/telemetry/upload_to_cloud_storage
""" % t)
shutil.copyfile(prebuilt_path, dest)
os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
return True
def _StartCrashService(self):
os_name = self._browser.platform.GetOSName()
if os_name != 'win':
return None
return subprocess.Popen([
support_binaries.FindPath('crash_service', os_name),
'--no-window',
'--dumps-dir=%s' % self._tmp_minidump_dir,
'--pipe-name=%s' % self._GetCrashServicePipeName()])
def _InstallHorndis(self, arch_name):
if self._LoadInstalledHoRNDIS():
logging.info('HoRNDIS kext loaded successfully.')
return
logging.info('Installing HoRNDIS...')
pkg_path = support_binaries.FindPath('HoRNDIS-rel5.pkg', arch_name,
'mac')
subprocess.check_call(
['/usr/bin/sudo', 'installer', '-pkg', pkg_path, '-target', '/'])
def FlushSystemCacheForDirectory(self, directory):
assert directory and os.path.exists(directory), \
'Target directory %s must exist' % directory
flush_command = support_binaries.FindPath('clear_system_cache',
self.GetArchName(),
self.GetOSName())
assert flush_command, 'You must build clear_system_cache first'
subprocess.check_call([flush_command, '--recurse', directory])
def InstallOnDevice(device, profiler_binary):
host_path = support_binaries.FindPath(profiler_binary, 'android')
if not host_path:
logging.error('Profiler binary "%s" not found. Could not be installed',
host_path)
return False
device_binary_path = GetDevicePath(profiler_binary)
device.PushChangedFiles(host_path, device_binary_path)
device.RunShellCommand('chmod 777 ' + device_binary_path)
return True
def _InstallAvconv(self):
avconv_bin = support_binaries.FindPath('avconv', self.GetOSName())
os.environ['PATH'] += os.pathsep + os.path.join(util.GetTelemetryDir(),
'bin')
try:
cloud_storage.GetIfChanged(avconv_bin, cloud_storage.INTERNAL_BUCKET)
except cloud_storage.CloudStorageError, e:
logging.error(e)
logging.error('You may proceed by manually installing avconv via:\n'
'sudo apt-get install libav-tools')
sys.exit(1)
def _InstallBinary(self, bin_name, fallback_package=None):
bin_path = support_binaries.FindPath(bin_name, self.GetOSName())
os.environ['PATH'] += os.pathsep + os.path.dirname(bin_path)
try:
cloud_storage.GetIfChanged(bin_path, cloud_storage.INTERNAL_BUCKET)
os.chmod(bin_path, 0755)
except cloud_storage.CloudStorageError, e:
logging.error(str(e))
if fallback_package:
logging.error(
'You may proceed by manually installing %s via:\n'
'sudo apt-get install %s' % (bin_name, fallback_package))
sys.exit(1)
def _GetStackFromMinidump(self, minidump):
os_name = self._browser.platform.GetOSName()
if os_name == 'win':
cdb = self._GetCdbPath()
if not cdb:
logging.warning('cdb.exe not found.')
return None
output = subprocess.check_output([
cdb, '-y', self._browser_directory, '-c', '.ecxr;k30;q', '-z',
minidump
])
stack_start = output.find('ChildEBP')
stack_end = output.find('quit:')
return output[stack_start:stack_end]
stackwalk = support_binaries.FindPath('minidump_stackwalk', os_name)
if not stackwalk:
logging.warning('minidump_stackwalk binary not found.')
return None
symbols = glob.glob(
os.path.join(self._browser_directory, '*.breakpad*'))
if not symbols:
logging.warning('No breakpad symbols found.')
return None
with open(minidump, 'rb') as infile:
minidump += '.stripped'
with open(minidump, 'wb') as outfile:
outfile.write(''.join(infile.read().partition('MDMP')[1:]))
symbols_path = os.path.join(self._tmp_minidump_dir, 'symbols')
for symbol in sorted(symbols, key=os.path.getmtime, reverse=True):
if not os.path.isfile(symbol):
continue
with open(symbol, 'r') as f:
fields = f.readline().split()
if not fields:
continue
sha = fields[3]
binary = ' '.join(fields[4:])
symbol_path = os.path.join(symbols_path, binary, sha)
if os.path.exists(symbol_path):
continue
os.makedirs(symbol_path)
shutil.copyfile(symbol, os.path.join(symbol_path, binary + '.sym'))
return subprocess.check_output([stackwalk, minidump, symbols_path],
stderr=open(os.devnull, 'w'))
def PullTrace(self):
symfs_dir = os.path.join(tempfile.gettempdir(),
os.path.expandvars('$USER-perf-symfs'))
if not os.path.exists(symfs_dir):
os.makedirs(symfs_dir)
required_libs = set()
# Download the recorded perf profile.
perf_profile = self._perf_instance.PullResult(symfs_dir)
required_libs = \
android_profiling_helper.GetRequiredLibrariesForPerfProfile(
perf_profile)
if not required_libs:
logging.warning(
'No libraries required by perf trace. Most likely there '
'are no samples in the trace.')
# Build a symfs with all the necessary libraries.
kallsyms = android_profiling_helper.CreateSymFs(self._device,
symfs_dir,
required_libs,
use_symlinks=False)
perfhost_path = support_binaries.FindPath(
android_profiling_helper.GetPerfhostName(), 'x86_64', 'linux')
ui.PrintMessage('\nNote: to view the profile in perf, run:')
ui.PrintMessage(' ' + self._GetInteractivePerfCommand(
perfhost_path, perf_profile, symfs_dir, required_libs, kallsyms))
# Convert the perf profile into JSON.
perf_script_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'third_party',
'perf_to_tracing.py')
json_file_name = os.path.basename(perf_profile)
with open(os.devnull, 'w') as dev_null, \
open(json_file_name, 'w') as json_file:
cmd = [
perfhost_path, 'script', '-s', perf_script_path, '-i',
perf_profile, '--symfs', symfs_dir, '--kallsyms', kallsyms
]
if subprocess.call(cmd, stdout=json_file, stderr=dev_null):
logging.warning(
'Perf data to JSON conversion failed. The result will '
'not contain any perf samples. You can still view the '
'perf data manually as shown above.')
return None
return json_file_name
def _StartCrashService(self):
os_name = self.browser.platform.GetOSName()
if os_name != 'win':
return None
arch_name = self.browser.platform.GetArchName()
command = support_binaries.FindPath('crash_service', arch_name,
os_name)
if not command:
logging.warning('crash_service.exe not found for %s %s', arch_name,
os_name)
return None
return subprocess.Popen([
command, '--no-window',
'--dumps-dir=%s' % self._tmp_minidump_dir,
'--pipe-name=%s' % self._GetCrashServicePipeName()
])
def __init__(self, dimensions, pixels):
binary = support_binaries.FindPath(
'bitmaptools', platform.GetHostPlatform().GetOSName())
assert binary, 'You must build bitmaptools first!'
self._popen = subprocess.Popen([binary],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# dimensions are: bpp, width, height, boxleft, boxtop, boxwidth, boxheight
packed_dims = struct.pack('iiiiiii', *dimensions)
self._popen.stdin.write(packed_dims)
# If we got a list of ints, we need to convert it into a byte buffer.
if type(pixels) is not bytearray:
pixels = bytearray(pixels)
self._popen.stdin.write(pixels)
def __init__(self, browser_backend, platform_backend, output_path, state):
super(OOMKillerProfiler, self).__init__(
browser_backend, platform_backend, output_path, state)
if not 'mem_consumer_launched' in state:
state['mem_consumer_launched'] = True
mem_consumer_path = support_binaries.FindPath(
os.path.join('apks', 'MemConsumer.apk'), 'android')
assert mem_consumer_path, ('Could not find memconsumer app. Please build '
'memconsumer target.')
self._browser_backend.adb.Install(mem_consumer_path)
self._browser_backend.adb.device().GoHome()
self._platform_backend.LaunchApplication(
'org.chromium.memconsumer/.MemConsumer',
'--ei memory 20')
# Bring the browser to the foreground after launching the mem consumer
self._browser_backend.adb.device().StartActivity(
intent.Intent(package=browser_backend.package,
activity=browser_backend.activity),
blocking=True)
def FlushSystemCacheForDirectory(self, directory, ignoring=None):
assert directory and os.path.exists(directory), \
'Target directory %s must exist' % directory
flush_command = support_binaries.FindPath('clear_system_cache',
self.GetOSName())
assert flush_command, 'You must build clear_system_cache first'
args = []
directory_contents = os.listdir(directory)
for item in directory_contents:
if not ignoring or item not in ignoring:
args.append(os.path.join(directory, item))
# According to msdn:
# http://msdn.microsoft.com/en-us/library/ms682425%28VS.85%29.aspx
# there's a maximum allowable command line of 32,768 characters on windows.
while args:
# Small note about [:256] and [256:]
# [:N] will return a list with the first N elements, ie.
# with [1,2,3,4,5], [:2] -> [1,2], and [2:] -> [3,4,5]
# with [1,2,3,4,5], [:5] -> [1,2,3,4,5] and [5:] -> []
subprocess.check_call([flush_command, '--recurse'] + args[:256])
args = args[256:]
def _InstallPerfHost():
perfhost_name = android_profiling_helper.GetPerfhostName()
host = platform.GetHostPlatform()
if not host.CanLaunchApplication(perfhost_name):
host.InstallApplication(perfhost_name)
return support_binaries.FindPath(perfhost_name, 'x86_64', 'linux')
def DriverCreator():
ie_driver_exe = support_binaries.FindPath(
'IEDriverServer_%s' % self._architecture, 'win')
return webdriver.Ie(executable_path=ie_driver_exe)
def _GetStackFromMinidump(self, minidump):
os_name = self.browser.platform.GetOSName()
if os_name == 'win':
cdb = self._GetCdbPath()
if not cdb:
logging.warning('cdb.exe not found.')
return None
output = subprocess.check_output([
cdb, '-y', self._browser_directory, '-c', '.ecxr;k30;q', '-z',
minidump
])
stack_start = output.find('ChildEBP')
stack_end = output.find('quit:')
return output[stack_start:stack_end]
arch_name = self.browser.platform.GetArchName()
stackwalk = support_binaries.FindPath('minidump_stackwalk', arch_name,
os_name)
if not stackwalk:
logging.warning('minidump_stackwalk binary not found.')
return None
with open(minidump, 'rb') as infile:
minidump += '.stripped'
with open(minidump, 'wb') as outfile:
outfile.write(''.join(infile.read().partition('MDMP')[1:]))
symbols_path = os.path.join(self._tmp_minidump_dir, 'symbols')
symbols = glob.glob(
os.path.join(self._browser_directory, '*.breakpad*'))
if symbols:
for symbol in sorted(symbols, key=os.path.getmtime, reverse=True):
if not os.path.isfile(symbol):
continue
with open(symbol, 'r') as f:
fields = f.readline().split()
if not fields:
continue
sha = fields[3]
binary = ' '.join(fields[4:])
symbol_path = os.path.join(symbols_path, binary, sha)
if os.path.exists(symbol_path):
continue
os.makedirs(symbol_path)
shutil.copyfile(symbol,
os.path.join(symbol_path, binary + '.sym'))
else:
# On some platforms generating the symbol table can be very time
# consuming, skip it if there's nothing to dump.
if self._IsExecutableStripped():
logging.info(
'%s appears to be stripped, skipping symbol dump.' %
(self._executable))
return
logging.info('Dumping breakpad symbols.')
generate_breakpad_symbols_path = os.path.join(
util.GetChromiumSrcDir(), "components", "crash", "tools",
"generate_breakpad_symbols.py")
cmd = [
sys.executable,
generate_breakpad_symbols_path,
'--binary=%s' % self._executable,
'--symbols-dir=%s' % symbols_path,
'--build-dir=%s' % self._browser_directory,
]
try:
subprocess.check_output(cmd, stderr=open(os.devnull, 'w'))
except subprocess.CalledProcessError:
logging.warning('Failed to execute "%s"' % ' '.join(cmd))
return None
return subprocess.check_output([stackwalk, minidump, symbols_path],
stderr=open(os.devnull, 'w'))
def _GetMostRecentCrashpadMinidump(self):
os_name = self.browser.platform.GetOSName()
arch_name = self.browser.platform.GetArchName()
crashpad_database_util = support_binaries.FindPath(
'crashpad_database_util', arch_name, os_name)
if not crashpad_database_util:
return None
report_output = subprocess.check_output([
crashpad_database_util, '--database=' + self._tmp_minidump_dir,
'--show-pending-reports', '--show-completed-reports',
'--show-all-report-info'
])
last_indentation = -1
reports_list = []
report_dict = {}
for report_line in report_output.splitlines():
# Report values are grouped together by the same indentation level.
current_indentation = 0
for report_char in report_line:
if not report_char.isspace():
break
current_indentation += 1
# Decrease in indentation level indicates a new report is being printed.
if current_indentation >= last_indentation:
report_key, report_value = report_line.split(':', 1)
if report_value:
report_dict[report_key.strip()] = report_value.strip()
elif report_dict:
try:
report_time = ParseCrashpadDateTime(
report_dict['Creation time'])
report_path = report_dict['Path'].strip()
reports_list.append((report_time, report_path))
except (ValueError, KeyError) as e:
logging.warning(
'Crashpad report expected valid keys'
' "Path" and "Creation time": %s', e)
finally:
report_dict = {}
last_indentation = current_indentation
# Include the last report.
if report_dict:
try:
report_time = ParseCrashpadDateTime(
report_dict['Creation time'])
report_path = report_dict['Path'].strip()
reports_list.append((report_time, report_path))
except (ValueError, KeyError) as e:
logging.warning(
'Crashpad report expected valid keys'
' "Path" and "Creation time": %s', e)
if reports_list:
_, most_recent_report_path = max(reports_list)
return most_recent_report_path
return None
def _PathForExecutable(executable_name):
"""Fetches the executable from cloud storage, and returns its path."""
arch_name = platform.GetHostPlatform().GetArchName()
return support_binaries.FindPath(executable_name, arch_name, 'mac')
def _InstallPerfHost():
host = platform.GetHostPlatform()
if not host.CanLaunchApplication('perfhost'):
host.InstallApplication('perfhost')
return support_binaries.FindPath('perfhost', host.GetOSName())