def test_interesting_processes_contains(self, lister_mock, os_mock):
os_mock.return_value = -1
lister_mock.return_value.dump_processes.return_value = [
(1, "python2"),
(2, "mongo"),
(3, "python3"),
(4, "mongod"),
(5, "python"),
(5, "java") # this should be ignored.
]
process_ids = None
interesting_processes = ['python', 'mongo', 'mongod']
process_match = "contains"
logger = Mock()
processes = get_processes(process_ids, interesting_processes, process_match, logger)
self.assertCountEqual(processes, [
Pinfo(name="python", pidv=[5]),
Pinfo(name="python2", pidv=[1]),
Pinfo(name="python3", pidv=[3]),
Pinfo(name="mongo", pidv=[2]),
Pinfo(name="mongod", pidv=[4])
])
def test_process_ids(self, lister_mock, os_mock):
os_mock.return_value = -1
lister_mock.return_value.dump_processes.return_value = [
(1, "python"),
(2, "mongo"),
(3, "python"),
(4, "mongod"),
(5, "mongod"),
(6, "python"), # rest is ignored
(7, "mongod"),
(8, "mongo"),
(9, "java"),
]
process_ids = [1, 2, 3, 4, 5]
interesting_processes = []
process_match = "exact"
logger = Mock()
processes = get_processes(process_ids, interesting_processes, process_match, logger)
self.assertCountEqual(processes, [
Pinfo(name="python", pidv=[1, 3]),
Pinfo(name="mongo", pidv=[2]),
Pinfo(name="mongod", pidv=[4, 5])
])
def test_interesting_processes_and_process_ids(self, lister_mock, os_mock):
os_mock.return_value = -1
lister_mock.return_value.dump_processes.return_value = [(1, "python"),
(2, "mongo"),
(3, "python"),
(4, "mongod"),
(5, "java")]
process_ids = [1, 2, 5]
interesting_processes = ['python', 'mongo', 'mongod']
process_match = "exact"
logger = Mock()
processes = get_processes(process_ids, interesting_processes,
process_match, logger)
self.assertCountEqual(processes, [
Pinfo(name="python", pidv=[1]),
Pinfo(name="mongo", pidv=[2]),
Pinfo(name="java", pidv=[5]),
])
def execute(self): # pylint: disable=too-many-branches,too-many-locals,too-many-statements
"""
Execute hang analysis.
1. Get a list of interesting processes
2. Dump useful information or take core dumps
"""
self._setup_logging()
self._log_system_info()
extractor.extract_debug_symbols(self.root_logger)
dumpers = dumper.get_dumpers()
processes = process_list.get_processes(self.process_ids, self.interesting_processes,
self.options.process_match, self.root_logger)
max_dump_size_bytes = int(self.options.max_core_dumps_size) * 1024 * 1024
# Dump python processes by signalling them. The resmoke.py process will generate
# the report.json, when signalled, so we do this before attaching to other processes.
for pinfo in [pinfo for pinfo in processes if pinfo.name.startswith("python")]:
signal_python(self.root_logger, pinfo)
trapped_exceptions = []
# Dump all processes, except python & java.
for pinfo in [pinfo for pinfo in processes if not re.match("^(java|python)", pinfo.name)]:
process_logger = self._get_process_logger(pinfo)
try:
dumpers.dbg.dump_info(
self.root_logger, process_logger, pinfo, self.options.dump_core
and _check_dump_quota(max_dump_size_bytes, dumpers.dbg.get_dump_ext()))
except Exception as err: # pylint: disable=broad-except
self.root_logger.info("Error encountered when invoking debugger %s", err)
trapped_exceptions.append(traceback.format_exc())
# Dump java processes using jstack.
for pinfo in [pinfo for pinfo in processes if pinfo.name.startswith("java")]:
process_logger = self._get_process_logger(pinfo)
try:
dumpers.jstack.dump_info(self.root_logger, pinfo.pid)
except Exception as err: # pylint: disable=broad-except
self.root_logger.info("Error encountered when invoking debugger %s", err)
trapped_exceptions.append(traceback.format_exc())
# Signal go processes to ensure they print out stack traces, and die on POSIX OSes.
# On Windows, this will simply kill the process since python emulates SIGABRT as
# TerminateProcess.
# Note: The stacktrace output may be captured elsewhere (i.e. resmoke).
for pinfo in [pinfo for pinfo in processes if pinfo.name in self.go_processes]:
self.root_logger.info("Sending signal SIGABRT to go process %s with PID %d", pinfo.name,
pinfo.pid)
signal_process(self.root_logger, pinfo.pid, signal.SIGABRT)
self.root_logger.info("Done analyzing all processes for hangs")
for exception in trapped_exceptions:
self.root_logger.info(exception)
if trapped_exceptions:
raise RuntimeError(
"Exceptions were thrown while dumping. There may still be some valid dumps.")
def execute(self): # pylint: disable=too-many-branches,too-many-locals,too-many-statements
"""
Execute hang analysis.
1. Get a list of interesting processes
2. Dump useful information or take core dumps
"""
self._log_system_info()
extractor.extract_debug_symbols(self.root_logger, self.debug_symbols_url)
dumpers = dumper.get_dumpers(self.root_logger, self.options.debugger_output)
processes = process_list.get_processes(self.process_ids, self.interesting_processes,
self.options.process_match, self.root_logger)
def is_python_process(pname: str):
# "live-record*" and "python*" are Python processes. Sending SIGUSR1 causes resmoke.py
# to dump its stack and run the hang analyzer on its child processes.
# Sending SIGUSR1 causes live-record to save its recording and terminate.
return pname.startswith("python") or pname.startswith("live-record")
# Suspending all processes, except python, to prevent them from getting unstuck when
# the hang analyzer attaches to them.
for pinfo in [pinfo for pinfo in processes if not is_python_process(pinfo.name)]:
for pid in pinfo.pidv:
process.pause_process(self.root_logger, pinfo.name, pid)
# Dump python processes by signalling them. The resmoke.py process will generate
# the report.json, when signalled, so we do this before attaching to other processes.
for pinfo in [pinfo for pinfo in processes if is_python_process(pinfo.name)]:
for pid in pinfo.pidv:
process.signal_python(self.root_logger, pinfo.name, pid)
trapped_exceptions = []
dump_pids = {}
# Dump core files of all processes, except python & java.
if self.options.dump_core:
for pinfo in [
pinfo for pinfo in processes if not re.match("^(java|python)", pinfo.name)
]:
if self._check_enough_free_space():
try:
dumpers.dbg.dump_info(pinfo, take_dump=True)
except dumper.DumpError as err:
self.root_logger.error(err.message)
dump_pids = {**err.dump_pids, **dump_pids}
except Exception as err: # pylint: disable=broad-except
self.root_logger.info("Error encountered when invoking debugger %s", err)
trapped_exceptions.append(traceback.format_exc())
else:
self.root_logger.info(
"Not enough space for a core dump, skipping %s processes with PIDs %s",
pinfo.name, str(pinfo.pidv))
# Dump info of all processes, except python & java.
for pinfo in [pinfo for pinfo in processes if not re.match("^(java|python)", pinfo.name)]:
try:
dumpers.dbg.dump_info(pinfo, take_dump=False)
except Exception as err: # pylint: disable=broad-except
self.root_logger.info("Error encountered when invoking debugger %s", err)
trapped_exceptions.append(traceback.format_exc())
# Dump java processes using jstack.
for pinfo in [pinfo for pinfo in processes if pinfo.name.startswith("java")]:
for pid in pinfo.pidv:
try:
dumpers.jstack.dump_info(self.root_logger, self.options.debugger_output,
pinfo.name, pid)
except Exception as err: # pylint: disable=broad-except
self.root_logger.info("Error encountered when invoking debugger %s", err)
trapped_exceptions.append(traceback.format_exc())
# Signal go processes to ensure they print out stack traces, and die on POSIX OSes.
# On Windows, this will simply kill the process since python emulates SIGABRT as
# TerminateProcess.
# Note: The stacktrace output may be captured elsewhere (i.e. resmoke).
for pinfo in [pinfo for pinfo in processes if pinfo.name in self.go_processes]:
for pid in pinfo.pidv:
self.root_logger.info("Sending signal SIGABRT to go process %s with PID %d",
pinfo.name, pid)
process.signal_process(self.root_logger, pid, signal.SIGABRT)
self.root_logger.info("Done analyzing all processes for hangs")
# Kill and abort processes if "-k" was specified.
if self.options.kill_processes:
process.teardown_processes(self.root_logger, processes, dump_pids)
else:
# Resuming all suspended processes.
for pinfo in [pinfo for pinfo in processes if not pinfo.name.startswith("python")]:
for pid in pinfo.pidv:
process.resume_process(self.root_logger, pinfo.name, pid)
for exception in trapped_exceptions:
self.root_logger.info(exception)
if trapped_exceptions:
raise RuntimeError(
"Exceptions were thrown while dumping. There may still be some valid dumps.")
