def SymbolizeMinidump(self, minidump):
"""Gets the stack trace from the given minidump.
Args:
minidump: the path to the minidump on disk
Returns:
None if the stack could not be retrieved for some reason, otherwise a
string containing the stack trace.
"""
stackwalk = local_first_binary_manager.GetInstance().FetchPath(
'minidump_stackwalk')
if not stackwalk:
logging.warning('minidump_stackwalk binary not found.')
return None
# We only want this logic on linux platforms that are still using breakpad.
# See crbug.com/667475
if not self._dump_finder.MinidumpObtainedFromCrashpad(minidump):
with open(minidump, 'rb') as infile:
minidump += '.stripped'
with open(minidump, 'wb') as outfile:
outfile.write(''.join(infile.read().partition('MDMP')[1:]))
symbols_dir = self._symbols_dir
if not symbols_dir:
symbols_dir = tempfile.mkdtemp()
try:
self._GenerateBreakpadSymbols(symbols_dir, minidump)
return subprocess.check_output([stackwalk, minidump, symbols_dir],
stderr=open(os.devnull, 'w'))
finally:
if not self._symbols_dir:
shutil.rmtree(symbols_dir)
def testSuccessfulInit(self):
self.assertTrue(
local_first_binary_manager.LocalFirstBinaryManager.NeedsInit())
local_first_binary_manager.LocalFirstBinaryManager.Init(
'build_dir', None, 'os_name', 'arch', ['ignored_dep'],
'os_version')
self.assertFalse(
local_first_binary_manager.LocalFirstBinaryManager.NeedsInit())
bm = local_first_binary_manager.GetInstance()
self.assertEqual(bm._build_dir, 'build_dir')
self.assertEqual(bm._os, 'os_name')
self.assertEqual(bm._arch, 'arch')
self.assertEqual(bm._os_version, 'os_version')
self.assertEqual(bm._ignored_dependencies, ['ignored_dep'])
def _GetMinidumpDumpOutput(self, minidump):
"""Runs minidump_dump on the given minidump.
Caches the result for re-use.
Args:
minidump: The path to the minidump being analyzed.
Returns:
A string containing the output of minidump_dump, or None if it could not
be retrieved for some reason.
"""
if minidump in self._minidump_dump_output:
logging.debug('Returning cached minidump_dump output for %s',
minidump)
return self._minidump_dump_output[minidump]
dumper_path = local_first_binary_manager.GetInstance().FetchPath(
'minidump_dump')
if not os.access(dumper_path, os.X_OK):
logging.warning(
'Cannot run minidump_dump because %s is not found.',
dumper_path)
return None
# Using subprocess.check_output with stdout/stderr mixed can result in
# errors due to log messages showing up in the minidump_dump output. So,
# use Popen and combine into a single string afterwards.
p = subprocess.Popen([dumper_path, minidump],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
stdout = stdout + '\n' + stderr
if p.returncode != 0:
# Dumper errors often do not affect stack walkability, just a warning.
# It's possible for the same stack to be symbolized multiple times, so
# add a timestamp suffix to prevent artifact collisions.
now = datetime.datetime.now()
suffix = now.strftime('%Y-%m-%d-%H-%M-%S')
artifact_name = 'dumper_errors/%s-%s' % (
os.path.basename(minidump), suffix)
logging.warning(
'Reading minidump failed, but likely not actually an issue. Saving '
'output to artifact %s', artifact_name)
artifact_logger.CreateArtifact(artifact_name, stdout)
if stdout:
self._minidump_dump_output[minidump] = stdout
return stdout
def GetSymbolBinaries(self, minidump):
"""Returns a list of paths to binaries where symbols may be located.
Args:
minidump: The path to the minidump being symbolized.
"""
minidump_dump = local_first_binary_manager.GetInstance().FetchPath(
'minidump_dump')
assert minidump_dump
symbol_binaries = []
minidump_cmd = [minidump_dump, minidump]
try:
with open(os.devnull, 'wb') as dev_null:
minidump_output = subprocess.check_output(minidump_cmd,
stderr=dev_null)
except subprocess.CalledProcessError as e:
# For some reason minidump_dump always fails despite successful dumping.
minidump_output = e.output
minidump_binary_re = re.compile(r'\W+\(code_file\)\W+=\W\"(.*)\"')
for minidump_line in minidump_output.splitlines():
line_match = minidump_binary_re.match(minidump_line)
if line_match:
binary_path = line_match.group(1)
if not os.path.isfile(binary_path):
continue
# Filter out system binaries.
if (binary_path.startswith('/usr/lib/')
or binary_path.startswith('/System/Library/')
or binary_path.startswith('/lib/')):
continue
# Filter out other binary file types which have no symbols.
if (binary_path.endswith('.pak')
or binary_path.endswith('.bin')
or binary_path.endswith('.dat')
or binary_path.endswith('.ttf')):
continue
symbol_binaries.append(binary_path)
return self._FilterSymbolBinaries(symbol_binaries)
def _GetAllCrashpadMinidumps(self, minidump_dir):
if not minidump_dir:
self._explanation.append(
'No minidump directory provided. Likely '
'attempted to retrieve the Crashpad minidumps '
'after the browser was already closed.')
return None
try:
crashpad_database_util = \
local_first_binary_manager.GetInstance().FetchPath(
'crashpad_database_util')
if not crashpad_database_util:
self._explanation.append(
'Unable to find crashpad_database_util. This '
'is likely due to running on a platform that '
'does not support Crashpad.')
return None
except dependency_manager.NoPathFoundError:
self._explanation.append(
'Could not find a path to look for crashpad_database_util.')
return None
self._explanation.append('Found crashpad_database_util at %s' %
crashpad_database_util)
report_output = subprocess.check_output([
crashpad_database_util, '--database=' + minidump_dir,
'--show-pending-reports', '--show-completed-reports',
'--show-all-report-info'
])
last_indentation = -1
reports_list = []
report_dict = {}
for report_line in report_output.splitlines():
# Report values are grouped together by the same indentation level.
current_indentation = 0
for report_char in report_line:
if not report_char.isspace():
break
current_indentation += 1
# Decrease in indentation level indicates a new report is being printed.
if current_indentation >= last_indentation:
report_key, report_value = report_line.split(':', 1)
if report_value:
report_dict[report_key.strip()] = report_value.strip()
elif report_dict:
try:
report_time = _ParseCrashpadDateTime(
report_dict['Creation time'])
report_path = report_dict['Path'].strip()
reports_list.append((report_time, report_path))
except (ValueError, KeyError) as e:
self._explanation.append(
'Expected to find keys "Path" and "Creation '
'time" in Crashpad report, but one or both '
'don\'t exist: %s' % e)
finally:
report_dict = {}
last_indentation = current_indentation
# Include the last report.
if report_dict:
try:
report_time = _ParseCrashpadDateTime(
report_dict['Creation time'])
report_path = report_dict['Path'].strip()
reports_list.append((report_time, report_path))
except (ValueError, KeyError) as e:
self._explanation.append(
'Expected to find keys "Path" and "Creation '
'time" in Crashpad report, but one or both '
'don\'t exist: %s' % e)
return reports_list
def testGetInstanceUninitialized(self):
with self.assertRaises(exceptions.InitializationError):
local_first_binary_manager.GetInstance()
def _GenerateBreakpadSymbols(self, symbols_dir, minidump):
"""Generates Breakpad symbols for use with stackwalking tools.
Args:
symbols_dir: The directory where symbols will be written to.
minidump: The path to the minidump being symbolized.
"""
logging.info('Dumping Breakpad symbols.')
try:
generate_breakpad_symbols_command = \
local_first_binary_manager.GetInstance().FetchPath(
'generate_breakpad_symbols')
except dependency_exceptions.NoPathFoundError as e:
logging.warning('Failed to get generate_breakpad_symbols: %s', e)
generate_breakpad_symbols_command = None
if not generate_breakpad_symbols_command:
logging.warning(
'generate_breakpad_symbols binary not found, cannot symbolize '
'minidumps')
return
symbol_binaries = self.GetSymbolBinaries(minidump)
cmds = []
cached_binaries = []
missing_binaries = []
for binary_path in symbol_binaries:
if not os.path.exists(binary_path):
missing_binaries.append(binary_path)
continue
# Skip dumping symbols for binaries if they already exist in the symbol
# directory, i.e. whatever is using this symbolizer has opted to cache
# symbols. The directory will contain a directory with the binary name if
# it has already been dumped.
cache_path = os.path.join(symbols_dir,
os.path.basename(binary_path))
if os.path.exists(cache_path) and os.path.isdir(cache_path):
cached_binaries.append(binary_path)
continue
cmd = [
sys.executable,
generate_breakpad_symbols_command,
'--binary=%s' % binary_path,
'--symbols-dir=%s' % symbols_dir,
'--build-dir=%s' % self._build_dir,
]
if self.GetBreakpadPlatformOverride():
cmd.append('--platform=%s' %
self.GetBreakpadPlatformOverride())
cmds.append(cmd)
if missing_binaries:
logging.warning(
'Unable to find %d of %d binaries for minidump symbolization. This '
'is likely not an actual issue, but is worth investigating if the '
'minidump fails to symbolize properly.', len(missing_binaries),
len(symbol_binaries))
# 5 is arbitrary, but a reasonable number of paths to print out.
if len(missing_binaries) < 5:
logging.warning('Missing binaries: %s', missing_binaries)
else:
logging.warning(
'Run test with high verbosity to get the list of missing binaries.'
)
logging.debug('Missing binaries: %s', missing_binaries)
if cached_binaries:
logging.info(
'Skipping symbol dumping for %d of %d binaries due to cached symbols '
'being present.', len(cached_binaries), len(symbol_binaries))
if len(cached_binaries) < 5:
logging.info('Skipped binaries: %s', cached_binaries)
else:
logging.info(
'Run test with high verbosity to get the list of binaries with '
'cached symbols.')
logging.debug('Skipped binaries: %s', cached_binaries)
# We need to prevent the number of file handles that we open from reaching
# the soft limit set for the current process. This can either be done by
# ensuring that the limit is suitably large using the resource module or by
# only starting a relatively small number of subprocesses at once. In order
# to prevent any potential issues with messing with system limits, that
# latter is chosen.
# Typically, this would be handled by using the multiprocessing module's
# pool functionality, but importing generate_breakpad_symbols and invoking
# it directly takes significantly longer than alternatives for whatever
# reason, even if they appear to perform more work. Thus, we could either
# have each task in the pool create its own subprocess that runs the
# command or manually limit the number of subprocesses we have at any
# given time. We go with the latter option since it should be less
# wasteful.
processes = {}
# Symbol dumping is somewhat I/O constrained, so use double the number of
# logical cores on the system.
process_limit = multiprocessing.cpu_count() * 2
while len(cmds) or len(processes):
# Clear any processes that have finished.
processes_to_delete = []
for p in processes:
if p.poll() is not None:
stdout, _ = p.communicate()
if p.returncode:
logging.error(stdout)
logging.warning('Failed to execute %s', processes[p])
processes_to_delete.append(p)
for p in processes_to_delete:
del processes[p]
# Add as many more processes as we can.
while len(processes) < process_limit and len(cmds):
cmd = cmds.pop(-1)
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
processes[p] = cmd
# 1 second is fairly arbitrary, but strikes a reasonable balance between
# spending too many cycles checking the current state of running
# processes and letting cores sit idle.
time.sleep(1)