def dump_stacks(cls, futures, verbose=False):
"""
Logs the stack frame for each of the given futures.
The Future objects must have been submitted with ``_submit_execution`` so that they contain
the necessary information.
"""
frames = dict(iter_thread_frames())
for i, future in enumerate(futures, 1):
try:
frame = frames[future.ctx['thread_ident']]
except KeyError:
frame = None # this might happen in race-conditions with a new thread starting
if not verbose or not frame:
if frame:
frame_line = _find_interesting_frame(frame)[:3]
location = " - %s:%s, in %s(..)" % tuple(frame_line)
else:
location = "..."
_logger.info("%3s - %s (DARK_YELLOW<<%s>>)%s", i,
future.funcname, cls._get_context(future),
location)
continue
with _logger.indented("%3s - %s (%s)",
i,
future.funcname,
cls._get_context(future),
footer=False):
lines = format_thread_stack(frame,
skip_modules=[this_module
]).splitlines()
for line in lines:
_logger.info(line.strip())
def get_thread_tree(including_this=True):
from .logging import THREAD_LOGGING_CONTEXT
from .bunch import Bunch
tree = {}
dead_threads = set()
contexts = {}
stacks = {}
def add_to_tree(thread):
contexts[thread.ident] = THREAD_LOGGING_CONTEXT.flatten(thread.uuid)
parent = get_thread_parent(thread)
if isinstance(parent, DeadThread) and parent not in dead_threads:
dead_threads.add(parent)
add_to_tree(parent)
tree.setdefault(parent, []).append(thread)
for thread in threading.enumerate():
add_to_tree(thread)
current_ident = threading.current_thread().ident
main_ident = threading.main_thread().ident
for thread_ident, frame in iter_thread_frames():
if not including_this and thread_ident == current_ident:
formatted = " <this frame>"
else:
# show the entire stack if it's this thread, don't skip ('after_module') anything
show_all = thread_ident in (current_ident, main_ident)
formatted = format_thread_stack(frame,
skip_modules=[] if show_all else
_BOOTSTRAPPERS) if frame else ''
stacks[thread_ident] = formatted, time.time()
def add_thread(parent_thread, parent):
for thread in sorted(tree[parent_thread],
key=lambda thread: thread.name):
ident = thread.ident or 0
stack, ts = stacks.get(ident, ("", 0))
context = contexts.get(ident, {})
context_line = ", ".join("%s: %s" % (k, context[k])
for k in "host context".split()
if context.get(k))
this = Bunch(
name=thread.name,
daemon="[D]" if getattr(thread, "daemon", False) else "",
ident=ident,
context_line="({})".format(context_line)
if context_line else "",
stack=stack,
timestamp=ts,
children=[],
)
parent.children.append(this)
if thread in tree:
add_thread(thread, this)
return parent
return add_thread(None, Bunch(children=[]))
def detect_hogging():
did_switch = True
current_running_greenlet = HUB
def mark_switch(event, args):
nonlocal did_switch
nonlocal current_running_greenlet
if event != 'switch':
return
did_switch = True
current_running_greenlet = args[
1] # args = [origin_greenlet , target_greenlet
global _greenlet_trace_func
_greenlet_trace_func = mark_switch
current_blocker_time = 0
last_warning_time = 0
while True:
non_gevent_sleep(HOGGING_TIMEOUT)
if did_switch:
# all good
pass
elif current_running_greenlet == HUB:
# it's ok for the hub to block if all greenlet wait on async io
pass
else:
current_blocker_time += HOGGING_TIMEOUT
if current_blocker_time < last_warning_time * 2:
continue # dont dump too much warnings - decay exponentialy until exploding after FAIL_BLOCK_TIME_SEC
for thread in threading.enumerate():
if getattr(thread, '_greenlet',
None) == current_running_greenlet:
_logger.info(
'RED<<greenlet hogger detected (%s seconds):>>',
current_blocker_time)
_logger.debug('thread stuck: %s', thread)
break
else:
_logger.info(
'RED<<unknown greenlet hogger detected (%s seconds):>>',
current_blocker_time)
_logger.debug(
'greenlet stuck (no corresponding thread found): %s',
current_running_greenlet)
_logger.debug('hub is: %s', HUB)
func = _logger.debug if current_blocker_time < 5 * HOGGING_TIMEOUT else _logger.info
func(
"Stack:\n%s",
format_thread_stack(
sys._current_frames()[main_thread_ident_before_patching]))
last_warning_time = current_blocker_time
continue
current_blocker_time = 0
last_warning_time = 0
did_switch = False
def dump_stacks(self, futures=None, verbose=False):
futures = futures or self
frames = dict(iter_thread_frames())
for i, future in enumerate(futures, 1):
try:
frame = frames[future.ctx['thread_ident']]
except KeyError:
frame = None # this might happen in race-conditions with a new thread starting
if not verbose or not frame:
if frame:
location = " - %s:%s, in %s(..)" % tuple(extract_stack(frame)[-1][:3])
else:
location = "..."
_logger.info("%3s - %s (DARK_YELLOW<<%s>>)%s",
i, future.funcname, _get_context(future), location)
continue
with _logger.indented("%3s - %s (%s)", i, future.funcname, _get_context(future), footer=False):
lines = format_thread_stack(frame, skip_modules=[this_module]).splitlines()
for line in lines:
_logger.info(line.strip())
