def _InstallIpfw(self): ipfw_bin = support_binaries.FindPath('ipfw', self.GetOSName()) ipfw_mod = support_binaries.FindPath('ipfw_mod.ko', self.GetOSName()) try: changed = cloud_storage.GetIfChanged( ipfw_bin, cloud_storage.INTERNAL_BUCKET) changed |= cloud_storage.GetIfChanged( ipfw_mod, cloud_storage.INTERNAL_BUCKET) except cloud_storage.CloudStorageError, e: logging.error(e) logging.error('You may proceed by manually installing dummynet. See: ' 'http://info.iet.unipi.it/~luigi/dummynet/') sys.exit(1)
def testFindPath(self): md5sum_path = support_binaries.FindPath( 'md5sum_bin_host', 'x86_64', 'linux') self.assertNotEquals(md5sum_path, None) self.assertTrue(os.path.isabs(md5sum_path)) st = os.stat(md5sum_path) self.assertTrue(st.st_mode & stat.S_IXUSR)
def GetRequiredLibrariesForPerfProfile(profile_file): """Returns the set of libraries necessary to symbolize a given perf profile. Args: profile_file: Path to perf profile to analyse. Returns: A set of required library file names. """ with open(os.devnull, 'w') as dev_null: perfhost_path = support_binaries.FindPath('perfhost', 'linux') perf = subprocess.Popen([perfhost_path, 'script', '-i', profile_file], stdout=dev_null, stderr=subprocess.PIPE) _, output = perf.communicate() missing_lib_re = re.compile( r'^Failed to open (.*), continuing without symbols') libs = set() for line in output.split('\n'): lib = missing_lib_re.match(line) if lib: lib = lib.group(1) path = os.path.dirname(lib) if any( path.startswith(ignored_path) for ignored_path in _IGNORED_LIB_PATHS) or path == '/': continue libs.add(lib) return libs
def SetupPrebuiltTools(adb): """Some of the android pylib scripts we depend on are lame and expect binaries to be in the out/ directory. So we copy any prebuilt binaries there as a prereq.""" # TODO(bulach): Build the targets for x86/mips. device_tools = [ 'file_poller', 'forwarder_dist/device_forwarder', 'md5sum_dist/md5sum_bin', 'purge_ashmem', 'run_pie', ] host_tools = [ 'bitmaptools', 'md5sum_bin_host', ] if platform.GetHostPlatform().GetOSName() == 'linux': host_tools.append('host_forwarder') arch_name = adb.device().GetABI() has_device_prebuilt = (arch_name.startswith('armeabi') or arch_name.startswith('arm64')) if not has_device_prebuilt: logging.warning('Unknown architecture type: %s' % arch_name) return all( [support_binaries.FindLocallyBuiltPath(t) for t in device_tools]) build_type = None for t in device_tools + host_tools: executable = os.path.basename(t) locally_built_path = support_binaries.FindLocallyBuiltPath(t) if not build_type: build_type = GetBuildTypeOfPath(locally_built_path) or 'Release' constants.SetBuildType(build_type) dest = os.path.join(constants.GetOutDirectory(), t) if not locally_built_path: logging.info('Setting up prebuilt %s', dest) if not os.path.exists(os.path.dirname(dest)): os.makedirs(os.path.dirname(dest)) platform_name = ('android' if t in device_tools else platform.GetHostPlatform().GetOSName()) bin_arch_name = (arch_name if t in device_tools else platform.GetHostPlatform().GetArchName()) prebuilt_path = support_binaries.FindPath(executable, bin_arch_name, platform_name) if not prebuilt_path or not os.path.exists(prebuilt_path): raise NotImplementedError(""" %s must be checked into cloud storage. Instructions: http://www.chromium.org/developers/telemetry/upload_to_cloud_storage """ % t) shutil.copyfile(prebuilt_path, dest) os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) return True
def _StartCrashService(self): os_name = self._browser.platform.GetOSName() if os_name != 'win': return None return subprocess.Popen([ support_binaries.FindPath('crash_service', os_name), '--no-window', '--dumps-dir=%s' % self._tmp_minidump_dir, '--pipe-name=%s' % self._GetCrashServicePipeName()])
def _InstallHorndis(self, arch_name): if self._LoadInstalledHoRNDIS(): logging.info('HoRNDIS kext loaded successfully.') return logging.info('Installing HoRNDIS...') pkg_path = support_binaries.FindPath('HoRNDIS-rel5.pkg', arch_name, 'mac') subprocess.check_call( ['/usr/bin/sudo', 'installer', '-pkg', pkg_path, '-target', '/'])
def FlushSystemCacheForDirectory(self, directory): assert directory and os.path.exists(directory), \ 'Target directory %s must exist' % directory flush_command = support_binaries.FindPath('clear_system_cache', self.GetArchName(), self.GetOSName()) assert flush_command, 'You must build clear_system_cache first' subprocess.check_call([flush_command, '--recurse', directory])
def InstallOnDevice(device, profiler_binary): host_path = support_binaries.FindPath(profiler_binary, 'android') if not host_path: logging.error('Profiler binary "%s" not found. Could not be installed', host_path) return False device_binary_path = GetDevicePath(profiler_binary) device.PushChangedFiles(host_path, device_binary_path) device.RunShellCommand('chmod 777 ' + device_binary_path) return True
def _InstallAvconv(self): avconv_bin = support_binaries.FindPath('avconv', self.GetOSName()) os.environ['PATH'] += os.pathsep + os.path.join(util.GetTelemetryDir(), 'bin') try: cloud_storage.GetIfChanged(avconv_bin, cloud_storage.INTERNAL_BUCKET) except cloud_storage.CloudStorageError, e: logging.error(e) logging.error('You may proceed by manually installing avconv via:\n' 'sudo apt-get install libav-tools') sys.exit(1)
def _InstallBinary(self, bin_name, fallback_package=None): bin_path = support_binaries.FindPath(bin_name, self.GetOSName()) os.environ['PATH'] += os.pathsep + os.path.dirname(bin_path) try: cloud_storage.GetIfChanged(bin_path, cloud_storage.INTERNAL_BUCKET) os.chmod(bin_path, 0755) except cloud_storage.CloudStorageError, e: logging.error(str(e)) if fallback_package: logging.error( 'You may proceed by manually installing %s via:\n' 'sudo apt-get install %s' % (bin_name, fallback_package)) sys.exit(1)
def _GetStackFromMinidump(self, minidump): os_name = self._browser.platform.GetOSName() if os_name == 'win': cdb = self._GetCdbPath() if not cdb: logging.warning('cdb.exe not found.') return None output = subprocess.check_output([ cdb, '-y', self._browser_directory, '-c', '.ecxr;k30;q', '-z', minidump ]) stack_start = output.find('ChildEBP') stack_end = output.find('quit:') return output[stack_start:stack_end] stackwalk = support_binaries.FindPath('minidump_stackwalk', os_name) if not stackwalk: logging.warning('minidump_stackwalk binary not found.') return None symbols = glob.glob( os.path.join(self._browser_directory, '*.breakpad*')) if not symbols: logging.warning('No breakpad symbols found.') return None with open(minidump, 'rb') as infile: minidump += '.stripped' with open(minidump, 'wb') as outfile: outfile.write(''.join(infile.read().partition('MDMP')[1:])) symbols_path = os.path.join(self._tmp_minidump_dir, 'symbols') for symbol in sorted(symbols, key=os.path.getmtime, reverse=True): if not os.path.isfile(symbol): continue with open(symbol, 'r') as f: fields = f.readline().split() if not fields: continue sha = fields[3] binary = ' '.join(fields[4:]) symbol_path = os.path.join(symbols_path, binary, sha) if os.path.exists(symbol_path): continue os.makedirs(symbol_path) shutil.copyfile(symbol, os.path.join(symbol_path, binary + '.sym')) return subprocess.check_output([stackwalk, minidump, symbols_path], stderr=open(os.devnull, 'w'))
def PullTrace(self): symfs_dir = os.path.join(tempfile.gettempdir(), os.path.expandvars('$USER-perf-symfs')) if not os.path.exists(symfs_dir): os.makedirs(symfs_dir) required_libs = set() # Download the recorded perf profile. perf_profile = self._perf_instance.PullResult(symfs_dir) required_libs = \ android_profiling_helper.GetRequiredLibrariesForPerfProfile( perf_profile) if not required_libs: logging.warning( 'No libraries required by perf trace. Most likely there ' 'are no samples in the trace.') # Build a symfs with all the necessary libraries. kallsyms = android_profiling_helper.CreateSymFs(self._device, symfs_dir, required_libs, use_symlinks=False) perfhost_path = support_binaries.FindPath( android_profiling_helper.GetPerfhostName(), 'x86_64', 'linux') ui.PrintMessage('\nNote: to view the profile in perf, run:') ui.PrintMessage(' ' + self._GetInteractivePerfCommand( perfhost_path, perf_profile, symfs_dir, required_libs, kallsyms)) # Convert the perf profile into JSON. perf_script_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'third_party', 'perf_to_tracing.py') json_file_name = os.path.basename(perf_profile) with open(os.devnull, 'w') as dev_null, \ open(json_file_name, 'w') as json_file: cmd = [ perfhost_path, 'script', '-s', perf_script_path, '-i', perf_profile, '--symfs', symfs_dir, '--kallsyms', kallsyms ] if subprocess.call(cmd, stdout=json_file, stderr=dev_null): logging.warning( 'Perf data to JSON conversion failed. The result will ' 'not contain any perf samples. You can still view the ' 'perf data manually as shown above.') return None return json_file_name
def _StartCrashService(self): os_name = self.browser.platform.GetOSName() if os_name != 'win': return None arch_name = self.browser.platform.GetArchName() command = support_binaries.FindPath('crash_service', arch_name, os_name) if not command: logging.warning('crash_service.exe not found for %s %s', arch_name, os_name) return None return subprocess.Popen([ command, '--no-window', '--dumps-dir=%s' % self._tmp_minidump_dir, '--pipe-name=%s' % self._GetCrashServicePipeName() ])
def __init__(self, dimensions, pixels): binary = support_binaries.FindPath( 'bitmaptools', platform.GetHostPlatform().GetOSName()) assert binary, 'You must build bitmaptools first!' self._popen = subprocess.Popen([binary], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # dimensions are: bpp, width, height, boxleft, boxtop, boxwidth, boxheight packed_dims = struct.pack('iiiiiii', *dimensions) self._popen.stdin.write(packed_dims) # If we got a list of ints, we need to convert it into a byte buffer. if type(pixels) is not bytearray: pixels = bytearray(pixels) self._popen.stdin.write(pixels)
def __init__(self, browser_backend, platform_backend, output_path, state): super(OOMKillerProfiler, self).__init__( browser_backend, platform_backend, output_path, state) if not 'mem_consumer_launched' in state: state['mem_consumer_launched'] = True mem_consumer_path = support_binaries.FindPath( os.path.join('apks', 'MemConsumer.apk'), 'android') assert mem_consumer_path, ('Could not find memconsumer app. Please build ' 'memconsumer target.') self._browser_backend.adb.Install(mem_consumer_path) self._browser_backend.adb.device().GoHome() self._platform_backend.LaunchApplication( 'org.chromium.memconsumer/.MemConsumer', '--ei memory 20') # Bring the browser to the foreground after launching the mem consumer self._browser_backend.adb.device().StartActivity( intent.Intent(package=browser_backend.package, activity=browser_backend.activity), blocking=True)
def FlushSystemCacheForDirectory(self, directory, ignoring=None): assert directory and os.path.exists(directory), \ 'Target directory %s must exist' % directory flush_command = support_binaries.FindPath('clear_system_cache', self.GetOSName()) assert flush_command, 'You must build clear_system_cache first' args = [] directory_contents = os.listdir(directory) for item in directory_contents: if not ignoring or item not in ignoring: args.append(os.path.join(directory, item)) # According to msdn: # http://msdn.microsoft.com/en-us/library/ms682425%28VS.85%29.aspx # there's a maximum allowable command line of 32,768 characters on windows. while args: # Small note about [:256] and [256:] # [:N] will return a list with the first N elements, ie. # with [1,2,3,4,5], [:2] -> [1,2], and [2:] -> [3,4,5] # with [1,2,3,4,5], [:5] -> [1,2,3,4,5] and [5:] -> [] subprocess.check_call([flush_command, '--recurse'] + args[:256]) args = args[256:]
def _InstallPerfHost(): perfhost_name = android_profiling_helper.GetPerfhostName() host = platform.GetHostPlatform() if not host.CanLaunchApplication(perfhost_name): host.InstallApplication(perfhost_name) return support_binaries.FindPath(perfhost_name, 'x86_64', 'linux')
def DriverCreator(): ie_driver_exe = support_binaries.FindPath( 'IEDriverServer_%s' % self._architecture, 'win') return webdriver.Ie(executable_path=ie_driver_exe)
def _GetStackFromMinidump(self, minidump): os_name = self.browser.platform.GetOSName() if os_name == 'win': cdb = self._GetCdbPath() if not cdb: logging.warning('cdb.exe not found.') return None output = subprocess.check_output([ cdb, '-y', self._browser_directory, '-c', '.ecxr;k30;q', '-z', minidump ]) stack_start = output.find('ChildEBP') stack_end = output.find('quit:') return output[stack_start:stack_end] arch_name = self.browser.platform.GetArchName() stackwalk = support_binaries.FindPath('minidump_stackwalk', arch_name, os_name) if not stackwalk: logging.warning('minidump_stackwalk binary not found.') return None with open(minidump, 'rb') as infile: minidump += '.stripped' with open(minidump, 'wb') as outfile: outfile.write(''.join(infile.read().partition('MDMP')[1:])) symbols_path = os.path.join(self._tmp_minidump_dir, 'symbols') symbols = glob.glob( os.path.join(self._browser_directory, '*.breakpad*')) if symbols: for symbol in sorted(symbols, key=os.path.getmtime, reverse=True): if not os.path.isfile(symbol): continue with open(symbol, 'r') as f: fields = f.readline().split() if not fields: continue sha = fields[3] binary = ' '.join(fields[4:]) symbol_path = os.path.join(symbols_path, binary, sha) if os.path.exists(symbol_path): continue os.makedirs(symbol_path) shutil.copyfile(symbol, os.path.join(symbol_path, binary + '.sym')) else: # On some platforms generating the symbol table can be very time # consuming, skip it if there's nothing to dump. if self._IsExecutableStripped(): logging.info( '%s appears to be stripped, skipping symbol dump.' % (self._executable)) return logging.info('Dumping breakpad symbols.') generate_breakpad_symbols_path = os.path.join( util.GetChromiumSrcDir(), "components", "crash", "tools", "generate_breakpad_symbols.py") cmd = [ sys.executable, generate_breakpad_symbols_path, '--binary=%s' % self._executable, '--symbols-dir=%s' % symbols_path, '--build-dir=%s' % self._browser_directory, ] try: subprocess.check_output(cmd, stderr=open(os.devnull, 'w')) except subprocess.CalledProcessError: logging.warning('Failed to execute "%s"' % ' '.join(cmd)) return None return subprocess.check_output([stackwalk, minidump, symbols_path], stderr=open(os.devnull, 'w'))
def _GetMostRecentCrashpadMinidump(self): os_name = self.browser.platform.GetOSName() arch_name = self.browser.platform.GetArchName() crashpad_database_util = support_binaries.FindPath( 'crashpad_database_util', arch_name, os_name) if not crashpad_database_util: return None report_output = subprocess.check_output([ crashpad_database_util, '--database=' + self._tmp_minidump_dir, '--show-pending-reports', '--show-completed-reports', '--show-all-report-info' ]) last_indentation = -1 reports_list = [] report_dict = {} for report_line in report_output.splitlines(): # Report values are grouped together by the same indentation level. current_indentation = 0 for report_char in report_line: if not report_char.isspace(): break current_indentation += 1 # Decrease in indentation level indicates a new report is being printed. if current_indentation >= last_indentation: report_key, report_value = report_line.split(':', 1) if report_value: report_dict[report_key.strip()] = report_value.strip() elif report_dict: try: report_time = ParseCrashpadDateTime( report_dict['Creation time']) report_path = report_dict['Path'].strip() reports_list.append((report_time, report_path)) except (ValueError, KeyError) as e: logging.warning( 'Crashpad report expected valid keys' ' "Path" and "Creation time": %s', e) finally: report_dict = {} last_indentation = current_indentation # Include the last report. if report_dict: try: report_time = ParseCrashpadDateTime( report_dict['Creation time']) report_path = report_dict['Path'].strip() reports_list.append((report_time, report_path)) except (ValueError, KeyError) as e: logging.warning( 'Crashpad report expected valid keys' ' "Path" and "Creation time": %s', e) if reports_list: _, most_recent_report_path = max(reports_list) return most_recent_report_path return None
def _PathForExecutable(executable_name): """Fetches the executable from cloud storage, and returns its path.""" arch_name = platform.GetHostPlatform().GetArchName() return support_binaries.FindPath(executable_name, arch_name, 'mac')
def _InstallPerfHost(): host = platform.GetHostPlatform() if not host.CanLaunchApplication('perfhost'): host.InstallApplication('perfhost') return support_binaries.FindPath('perfhost', host.GetOSName())