def SymbolizeMinidump(self, minidump): """Gets the stack trace from the given minidump. Args: minidump: the path to the minidump on disk Returns: None if the stack could not be retrieved for some reason, otherwise a string containing the stack trace. """ stackwalk = local_first_binary_manager.GetInstance().FetchPath( 'minidump_stackwalk') if not stackwalk: logging.warning('minidump_stackwalk binary not found.') return None # We only want this logic on linux platforms that are still using breakpad. # See crbug.com/667475 if not self._dump_finder.MinidumpObtainedFromCrashpad(minidump): with open(minidump, 'rb') as infile: minidump += '.stripped' with open(minidump, 'wb') as outfile: outfile.write(''.join(infile.read().partition('MDMP')[1:])) symbols_dir = self._symbols_dir if not symbols_dir: symbols_dir = tempfile.mkdtemp() try: self._GenerateBreakpadSymbols(symbols_dir, minidump) return subprocess.check_output([stackwalk, minidump, symbols_dir], stderr=open(os.devnull, 'w')) finally: if not self._symbols_dir: shutil.rmtree(symbols_dir)
def testSuccessfulInit(self): self.assertTrue( local_first_binary_manager.LocalFirstBinaryManager.NeedsInit()) local_first_binary_manager.LocalFirstBinaryManager.Init( 'build_dir', None, 'os_name', 'arch', ['ignored_dep'], 'os_version') self.assertFalse( local_first_binary_manager.LocalFirstBinaryManager.NeedsInit()) bm = local_first_binary_manager.GetInstance() self.assertEqual(bm._build_dir, 'build_dir') self.assertEqual(bm._os, 'os_name') self.assertEqual(bm._arch, 'arch') self.assertEqual(bm._os_version, 'os_version') self.assertEqual(bm._ignored_dependencies, ['ignored_dep'])
def _GetMinidumpDumpOutput(self, minidump): """Runs minidump_dump on the given minidump. Caches the result for re-use. Args: minidump: The path to the minidump being analyzed. Returns: A string containing the output of minidump_dump, or None if it could not be retrieved for some reason. """ if minidump in self._minidump_dump_output: logging.debug('Returning cached minidump_dump output for %s', minidump) return self._minidump_dump_output[minidump] dumper_path = local_first_binary_manager.GetInstance().FetchPath( 'minidump_dump') if not os.access(dumper_path, os.X_OK): logging.warning( 'Cannot run minidump_dump because %s is not found.', dumper_path) return None # Using subprocess.check_output with stdout/stderr mixed can result in # errors due to log messages showing up in the minidump_dump output. So, # use Popen and combine into a single string afterwards. p = subprocess.Popen([dumper_path, minidump], stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() stdout = stdout + '\n' + stderr if p.returncode != 0: # Dumper errors often do not affect stack walkability, just a warning. # It's possible for the same stack to be symbolized multiple times, so # add a timestamp suffix to prevent artifact collisions. now = datetime.datetime.now() suffix = now.strftime('%Y-%m-%d-%H-%M-%S') artifact_name = 'dumper_errors/%s-%s' % ( os.path.basename(minidump), suffix) logging.warning( 'Reading minidump failed, but likely not actually an issue. Saving ' 'output to artifact %s', artifact_name) artifact_logger.CreateArtifact(artifact_name, stdout) if stdout: self._minidump_dump_output[minidump] = stdout return stdout
def GetSymbolBinaries(self, minidump): """Returns a list of paths to binaries where symbols may be located. Args: minidump: The path to the minidump being symbolized. """ minidump_dump = local_first_binary_manager.GetInstance().FetchPath( 'minidump_dump') assert minidump_dump symbol_binaries = [] minidump_cmd = [minidump_dump, minidump] try: with open(os.devnull, 'wb') as dev_null: minidump_output = subprocess.check_output(minidump_cmd, stderr=dev_null) except subprocess.CalledProcessError as e: # For some reason minidump_dump always fails despite successful dumping. minidump_output = e.output minidump_binary_re = re.compile(r'\W+\(code_file\)\W+=\W\"(.*)\"') for minidump_line in minidump_output.splitlines(): line_match = minidump_binary_re.match(minidump_line) if line_match: binary_path = line_match.group(1) if not os.path.isfile(binary_path): continue # Filter out system binaries. if (binary_path.startswith('/usr/lib/') or binary_path.startswith('/System/Library/') or binary_path.startswith('/lib/')): continue # Filter out other binary file types which have no symbols. if (binary_path.endswith('.pak') or binary_path.endswith('.bin') or binary_path.endswith('.dat') or binary_path.endswith('.ttf')): continue symbol_binaries.append(binary_path) return self._FilterSymbolBinaries(symbol_binaries)
def _GetAllCrashpadMinidumps(self, minidump_dir): if not minidump_dir: self._explanation.append( 'No minidump directory provided. Likely ' 'attempted to retrieve the Crashpad minidumps ' 'after the browser was already closed.') return None try: crashpad_database_util = \ local_first_binary_manager.GetInstance().FetchPath( 'crashpad_database_util') if not crashpad_database_util: self._explanation.append( 'Unable to find crashpad_database_util. This ' 'is likely due to running on a platform that ' 'does not support Crashpad.') return None except dependency_manager.NoPathFoundError: self._explanation.append( 'Could not find a path to look for crashpad_database_util.') return None self._explanation.append('Found crashpad_database_util at %s' % crashpad_database_util) report_output = subprocess.check_output([ crashpad_database_util, '--database=' + minidump_dir, '--show-pending-reports', '--show-completed-reports', '--show-all-report-info' ]) last_indentation = -1 reports_list = [] report_dict = {} for report_line in report_output.splitlines(): # Report values are grouped together by the same indentation level. current_indentation = 0 for report_char in report_line: if not report_char.isspace(): break current_indentation += 1 # Decrease in indentation level indicates a new report is being printed. if current_indentation >= last_indentation: report_key, report_value = report_line.split(':', 1) if report_value: report_dict[report_key.strip()] = report_value.strip() elif report_dict: try: report_time = _ParseCrashpadDateTime( report_dict['Creation time']) report_path = report_dict['Path'].strip() reports_list.append((report_time, report_path)) except (ValueError, KeyError) as e: self._explanation.append( 'Expected to find keys "Path" and "Creation ' 'time" in Crashpad report, but one or both ' 'don\'t exist: %s' % e) finally: report_dict = {} last_indentation = current_indentation # Include the last report. if report_dict: try: report_time = _ParseCrashpadDateTime( report_dict['Creation time']) report_path = report_dict['Path'].strip() reports_list.append((report_time, report_path)) except (ValueError, KeyError) as e: self._explanation.append( 'Expected to find keys "Path" and "Creation ' 'time" in Crashpad report, but one or both ' 'don\'t exist: %s' % e) return reports_list
def testGetInstanceUninitialized(self): with self.assertRaises(exceptions.InitializationError): local_first_binary_manager.GetInstance()
def _GenerateBreakpadSymbols(self, symbols_dir, minidump): """Generates Breakpad symbols for use with stackwalking tools. Args: symbols_dir: The directory where symbols will be written to. minidump: The path to the minidump being symbolized. """ logging.info('Dumping Breakpad symbols.') try: generate_breakpad_symbols_command = \ local_first_binary_manager.GetInstance().FetchPath( 'generate_breakpad_symbols') except dependency_exceptions.NoPathFoundError as e: logging.warning('Failed to get generate_breakpad_symbols: %s', e) generate_breakpad_symbols_command = None if not generate_breakpad_symbols_command: logging.warning( 'generate_breakpad_symbols binary not found, cannot symbolize ' 'minidumps') return symbol_binaries = self.GetSymbolBinaries(minidump) cmds = [] cached_binaries = [] missing_binaries = [] for binary_path in symbol_binaries: if not os.path.exists(binary_path): missing_binaries.append(binary_path) continue # Skip dumping symbols for binaries if they already exist in the symbol # directory, i.e. whatever is using this symbolizer has opted to cache # symbols. The directory will contain a directory with the binary name if # it has already been dumped. cache_path = os.path.join(symbols_dir, os.path.basename(binary_path)) if os.path.exists(cache_path) and os.path.isdir(cache_path): cached_binaries.append(binary_path) continue cmd = [ sys.executable, generate_breakpad_symbols_command, '--binary=%s' % binary_path, '--symbols-dir=%s' % symbols_dir, '--build-dir=%s' % self._build_dir, ] if self.GetBreakpadPlatformOverride(): cmd.append('--platform=%s' % self.GetBreakpadPlatformOverride()) cmds.append(cmd) if missing_binaries: logging.warning( 'Unable to find %d of %d binaries for minidump symbolization. This ' 'is likely not an actual issue, but is worth investigating if the ' 'minidump fails to symbolize properly.', len(missing_binaries), len(symbol_binaries)) # 5 is arbitrary, but a reasonable number of paths to print out. if len(missing_binaries) < 5: logging.warning('Missing binaries: %s', missing_binaries) else: logging.warning( 'Run test with high verbosity to get the list of missing binaries.' ) logging.debug('Missing binaries: %s', missing_binaries) if cached_binaries: logging.info( 'Skipping symbol dumping for %d of %d binaries due to cached symbols ' 'being present.', len(cached_binaries), len(symbol_binaries)) if len(cached_binaries) < 5: logging.info('Skipped binaries: %s', cached_binaries) else: logging.info( 'Run test with high verbosity to get the list of binaries with ' 'cached symbols.') logging.debug('Skipped binaries: %s', cached_binaries) # We need to prevent the number of file handles that we open from reaching # the soft limit set for the current process. This can either be done by # ensuring that the limit is suitably large using the resource module or by # only starting a relatively small number of subprocesses at once. In order # to prevent any potential issues with messing with system limits, that # latter is chosen. # Typically, this would be handled by using the multiprocessing module's # pool functionality, but importing generate_breakpad_symbols and invoking # it directly takes significantly longer than alternatives for whatever # reason, even if they appear to perform more work. Thus, we could either # have each task in the pool create its own subprocess that runs the # command or manually limit the number of subprocesses we have at any # given time. We go with the latter option since it should be less # wasteful. processes = {} # Symbol dumping is somewhat I/O constrained, so use double the number of # logical cores on the system. process_limit = multiprocessing.cpu_count() * 2 while len(cmds) or len(processes): # Clear any processes that have finished. processes_to_delete = [] for p in processes: if p.poll() is not None: stdout, _ = p.communicate() if p.returncode: logging.error(stdout) logging.warning('Failed to execute %s', processes[p]) processes_to_delete.append(p) for p in processes_to_delete: del processes[p] # Add as many more processes as we can. while len(processes) < process_limit and len(cmds): cmd = cmds.pop(-1) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) processes[p] = cmd # 1 second is fairly arbitrary, but strikes a reasonable balance between # spending too many cycles checking the current state of running # processes and letting cores sit idle. time.sleep(1)