def GetRequiredLibrariesForPerfProfile(profile_file): """Returns the set of libraries necessary to symbolize a given perf profile. Args: profile_file: Path to perf profile to analyse. Returns: A set of required library file names. """ with open(os.devnull, 'w') as dev_null: perfhost_path = support_binaries.FindPath(GetPerfhostName(), 'x86_64', 'linux') perf = subprocess.Popen([perfhost_path, 'script', '-i', profile_file], stdout=dev_null, stderr=subprocess.PIPE) _, output = perf.communicate() missing_lib_re = re.compile( r'^Failed to open (.*), continuing without symbols') libs = set() for line in output.split('\n'): lib = missing_lib_re.match(line) if lib: lib = lib.group(1) path = os.path.dirname(lib) if (any( path.startswith(ignored_path) for ignored_path in _IGNORED_LIB_PATHS) or path == '/' or not path): continue libs.add(lib) return libs
def testFindPath(self): md5sum_path = support_binaries.FindPath( 'md5sum_bin_host', 'x86_64', 'linux') self.assertNotEquals(md5sum_path, None) self.assertTrue(os.path.isabs(md5sum_path)) st = os.stat(md5sum_path) self.assertTrue(st.st_mode & stat.S_IXUSR)
def FetchPath(self, dependency, platform, try_support_binaries=False): """Get a path to an executable for |dependency|, downloading as needed. A path to a default executable may be returned if a platform specific version is not specified in the config(s). Args: dependency: Name of the desired dependency, as given in the config(s) used in this DependencyManager. platform: Name of the platform the dependency will run on. Often of the form 'os_architecture'. Must match those specified in the config(s) used in this DependencyManager. try_support_binaries: True if support_binaries should be queried if the dependency_manager was not initialized with data for |dependency|. Returns: A path to an executable of |dependency| that will run on |platform|, downloading from cloud storage if needed. Raises: NoPathFoundError: If a local copy of the executable cannot be found and a remote path could not be downloaded from cloud_storage. CredentialsError: If cloud_storage credentials aren't configured. PermissionError: If cloud_storage credentials are configured, but not with an account that has permission to download the remote file. NotFoundError: If the remote file does not exist where expected in cloud_storage. ServerError: If an internal server error is hit while downloading the remote file. CloudStorageError: If another error occured while downloading the remote path. FileNotFoundError: If an attempted download was otherwise unsuccessful. """ dependency_info = self._GetDependencyInfo(dependency, platform) if not dependency_info: if not try_support_binaries: raise exceptions.NoPathFoundError(dependency, platform) # TODO(aiolos): Remove the support_binaries call and always raise # NoPathFound once the binary dependencies are moved over to the new # system. # platform should be of the form '%s_%s' % (os_name, arch_name) when # called from the binary_manager. platform_parts = platform.split('_', 1) assert len(platform_parts) == 2 platform_os, platform_arch = platform_parts logging.info( 'Calling into support_binaries with dependency %s, platform ' '%s and arch %s. support_binaries is deprecated.' % (dependency, platform_os, platform_arch)) return support_binaries.FindPath(dependency, platform_arch, platform_os) path = self._LocalPath(dependency_info) if not path or not os.path.exists(path): path = dependency_info.GetRemotePath() if not path or not os.path.exists(path): raise exceptions.NoPathFoundError(dependency, platform) return path
def _InstallIpfw(self): ipfw_bin = support_binaries.FindPath('ipfw', self.GetArchName(), self.GetOSName()) ipfw_mod = support_binaries.FindPath('ipfw_mod.ko', self.GetArchName(), self.GetOSName()) try: changed = cloud_storage.GetIfChanged(ipfw_bin, cloud_storage.INTERNAL_BUCKET) changed |= cloud_storage.GetIfChanged( ipfw_mod, cloud_storage.INTERNAL_BUCKET) except cloud_storage.CloudStorageError, e: logging.error(str(e)) logging.error('You may proceed by manually building and installing' 'dummynet for your kernel. See: ' 'http://info.iet.unipi.it/~luigi/dummynet/') sys.exit(1)
def _InstallHorndis(self, arch_name): if self._LoadInstalledHoRNDIS(): logging.info('HoRNDIS kext loaded successfully.') return logging.info('Installing HoRNDIS...') pkg_path = support_binaries.FindPath('HoRNDIS-rel5.pkg', arch_name, 'mac') subprocess.check_call( ['/usr/bin/sudo', 'installer', '-pkg', pkg_path, '-target', '/'])
def _SetupPrebuiltTools(adb): """Some of the android pylib scripts we depend on are lame and expect binaries to be in the out/ directory. So we copy any prebuilt binaries there as a prereq.""" # TODO(bulach): Build the targets for x86/mips. device_tools = [ 'file_poller', 'forwarder_dist/device_forwarder', 'md5sum_dist/md5sum_bin', 'purge_ashmem', 'run_pie', ] host_tools = [ 'bitmaptools', 'md5sum_bin_host', ] if platform.GetHostPlatform().GetOSName() == 'linux': host_tools.append('host_forwarder') arch_name = adb.device().GetABI() has_device_prebuilt = (arch_name.startswith('armeabi') or arch_name.startswith('arm64')) if not has_device_prebuilt: logging.warning('Unknown architecture type: %s' % arch_name) return all( [support_binaries.FindLocallyBuiltPath(t) for t in device_tools]) build_type = None for t in device_tools + host_tools: executable = os.path.basename(t) locally_built_path = support_binaries.FindLocallyBuiltPath(t) if not build_type: build_type = _GetBuildTypeOfPath(locally_built_path) or 'Release' constants.SetBuildType(build_type) dest = os.path.join(constants.GetOutDirectory(), t) if not locally_built_path: logging.info('Setting up prebuilt %s', dest) if not os.path.exists(os.path.dirname(dest)): os.makedirs(os.path.dirname(dest)) platform_name = ('android' if t in device_tools else platform.GetHostPlatform().GetOSName()) bin_arch_name = (arch_name if t in device_tools else platform.GetHostPlatform().GetArchName()) prebuilt_path = support_binaries.FindPath(executable, bin_arch_name, platform_name) if not prebuilt_path or not os.path.exists(prebuilt_path): raise NotImplementedError(""" %s must be checked into cloud storage. Instructions: http://www.chromium.org/developers/telemetry/upload_to_cloud_storage """ % t) shutil.copyfile(prebuilt_path, dest) os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) return True
def InstallOnDevice(device, profiler_binary): arch_name = device.GetABI() host_path = support_binaries.FindPath(profiler_binary, arch_name, 'android') if not host_path: logging.error('Profiler binary "%s" not found. Could not be installed', host_path) return False device_binary_path = GetDevicePath(profiler_binary) device.PushChangedFiles([(host_path, device_binary_path)]) device.RunShellCommand('chmod 777 ' + device_binary_path) return True
def _StartCrashService(self): os_name = self.browser.platform.GetOSName() if os_name != 'win': return None arch_name = self.browser.platform.GetArchName() command = support_binaries.FindPath('crash_service', arch_name, os_name) if not command: logging.warning('crash_service.exe not found for %s %s', arch_name, os_name) return None return subprocess.Popen([ command, '--no-window', '--dumps-dir=%s' % self._tmp_minidump_dir, '--pipe-name=%s' % self._GetCrashServicePipeName() ])
def _InstallBinary(self, bin_name, fallback_package=None): bin_path = support_binaries.FindPath(bin_name, self.GetArchName(), self.GetOSName()) if not bin_path: raise Exception('Could not find the binary package %s' % bin_name) os.environ['PATH'] += os.pathsep + os.path.dirname(bin_path) try: cloud_storage.GetIfChanged(bin_path, cloud_storage.INTERNAL_BUCKET) os.chmod(bin_path, 0755) except cloud_storage.CloudStorageError, e: logging.error(str(e)) if fallback_package: raise Exception( 'You may proceed by manually installing %s via:\n' 'sudo apt-get install %s' % (bin_name, fallback_package))
def __init__(self, dimensions, pixels): binary = support_binaries.FindPath( 'bitmaptools', platform.GetHostPlatform().GetArchName(), platform.GetHostPlatform().GetOSName()) assert binary, 'You must build bitmaptools first!' self._popen = subprocess.Popen([binary], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # dimensions are: bpp, width, height, boxleft, boxtop, boxwidth, boxheight packed_dims = struct.pack('iiiiiii', *dimensions) self._popen.stdin.write(packed_dims) # If we got a list of ints, we need to convert it into a byte buffer. if type(pixels) is not bytearray: pixels = bytearray(pixels) self._popen.stdin.write(pixels)
def __init__(self, browser_backend, platform_backend, output_path, state): super(OOMKillerProfiler, self).__init__( browser_backend, platform_backend, output_path, state) if not 'mem_consumer_launched' in state: state['mem_consumer_launched'] = True arch_name = self._browser_backend.adb.device().GetABI() mem_consumer_path = support_binaries.FindPath( os.path.join('apks', 'MemConsumer.apk'), arch_name, 'android') assert mem_consumer_path, ('Could not find memconsumer app. Please build ' 'memconsumer target.') if not self._platform_backend.CanLaunchApplication( 'org.chromium.memconsumerg'): self._platform_backend.InstallApplication(mem_consumer_path) self._browser_backend.adb.device().GoHome() self._platform_backend.LaunchApplication( 'org.chromium.memconsumer/.MemConsumer', '--ei memory 20') # Bring the browser to the foreground after launching the mem consumer self._browser_backend.adb.device().StartActivity( intent.Intent(package=browser_backend.package, activity=browser_backend.activity), blocking=True)
def _InstallPerfHost(): perfhost_name = android_profiling_helper.GetPerfhostName() host = platform.GetHostPlatform() if not host.CanLaunchApplication(perfhost_name): host.InstallApplication(perfhost_name) return support_binaries.FindPath(perfhost_name, 'x86_64', 'linux')
def FetchPath(binary_name, platform, arch): """ Return a path to the appropriate executable for <binary_name>, downloading from cloud storage if needed, or None if it cannot be found. """ return support_binaries.FindPath(binary_name, platform, arch)
def _GetStackFromMinidump(self, minidump): os_name = self.browser.platform.GetOSName() if os_name == 'win': cdb = self._GetCdbPath() if not cdb: logging.warning('cdb.exe not found.') return None output = subprocess.check_output([ cdb, '-y', self._browser_directory, '-c', '.ecxr;k30;q', '-z', minidump ]) # cdb output can start the stack with "ChildEBP", "Child-SP", and possibly # other things we haven't seen yet. If we can't find the start of the # stack, include output from the beginning. stack_start = 0 stack_start_match = re.search("^Child(?:EBP|-SP)", output, re.MULTILINE) if stack_start_match: stack_start = stack_start_match.start() stack_end = output.find('quit:') return output[stack_start:stack_end] arch_name = self.browser.platform.GetArchName() stackwalk = support_binaries.FindPath('minidump_stackwalk', arch_name, os_name) if not stackwalk: logging.warning('minidump_stackwalk binary not found.') return None with open(minidump, 'rb') as infile: minidump += '.stripped' with open(minidump, 'wb') as outfile: outfile.write(''.join(infile.read().partition('MDMP')[1:])) symbols_path = os.path.join(self._tmp_minidump_dir, 'symbols') symbols = glob.glob( os.path.join(self._browser_directory, '*.breakpad*')) if symbols: for symbol in sorted(symbols, key=os.path.getmtime, reverse=True): if not os.path.isfile(symbol): continue with open(symbol, 'r') as f: fields = f.readline().split() if not fields: continue sha = fields[3] binary = ' '.join(fields[4:]) symbol_path = os.path.join(symbols_path, binary, sha) if os.path.exists(symbol_path): continue os.makedirs(symbol_path) shutil.copyfile(symbol, os.path.join(symbol_path, binary + '.sym')) else: # On some platforms generating the symbol table can be very time # consuming, skip it if there's nothing to dump. if self._IsExecutableStripped(): logging.info( '%s appears to be stripped, skipping symbol dump.' % (self._executable)) return logging.info('Dumping breakpad symbols.') generate_breakpad_symbols_path = os.path.join( util.GetChromiumSrcDir(), "components", "crash", "tools", "generate_breakpad_symbols.py") cmd = [ sys.executable, generate_breakpad_symbols_path, '--binary=%s' % self._executable, '--symbols-dir=%s' % symbols_path, '--build-dir=%s' % self._browser_directory, ] try: subprocess.check_output(cmd, stderr=open(os.devnull, 'w')) except subprocess.CalledProcessError: logging.warning('Failed to execute "%s"' % ' '.join(cmd)) return None return subprocess.check_output([stackwalk, minidump, symbols_path], stderr=open(os.devnull, 'w'))
def _GetMostRecentCrashpadMinidump(self): os_name = self.browser.platform.GetOSName() arch_name = self.browser.platform.GetArchName() crashpad_database_util = support_binaries.FindPath( 'crashpad_database_util', arch_name, os_name) if not crashpad_database_util: return None report_output = subprocess.check_output([ crashpad_database_util, '--database=' + self._tmp_minidump_dir, '--show-pending-reports', '--show-completed-reports', '--show-all-report-info' ]) last_indentation = -1 reports_list = [] report_dict = {} for report_line in report_output.splitlines(): # Report values are grouped together by the same indentation level. current_indentation = 0 for report_char in report_line: if not report_char.isspace(): break current_indentation += 1 # Decrease in indentation level indicates a new report is being printed. if current_indentation >= last_indentation: report_key, report_value = report_line.split(':', 1) if report_value: report_dict[report_key.strip()] = report_value.strip() elif report_dict: try: report_time = ParseCrashpadDateTime( report_dict['Creation time']) report_path = report_dict['Path'].strip() reports_list.append((report_time, report_path)) except (ValueError, KeyError) as e: logging.warning( 'Crashpad report expected valid keys' ' "Path" and "Creation time": %s', e) finally: report_dict = {} last_indentation = current_indentation # Include the last report. if report_dict: try: report_time = ParseCrashpadDateTime( report_dict['Creation time']) report_path = report_dict['Path'].strip() reports_list.append((report_time, report_path)) except (ValueError, KeyError) as e: logging.warning( 'Crashpad report expected valid keys' ' "Path" and "Creation time": %s', e) if reports_list: _, most_recent_report_path = max(reports_list) return most_recent_report_path return None
def _PathForExecutable(executable_name): """Fetches the executable from cloud storage, and returns its path.""" arch_name = platform.GetHostPlatform().GetArchName() return support_binaries.FindPath(executable_name, arch_name, 'mac')