def __init__(self, hub):
self._hub_wref = wref(hub, self._on_hub_gc)
self.should_run = True
# Must be installed in the thread that the hub is running in;
# the trace function is threadlocal
assert get_thread_ident() == hub.thread_ident
self._greenlet_tracer = GreenletTracer()
self._monitoring_functions = [
_MonitorEntry(self.monitor_blocking,
GEVENT_CONFIG.max_blocking_time)
]
self._calculated_sleep_time = GEVENT_CONFIG.max_blocking_time
# Create the actual monitoring thread. This is effectively a "daemon"
# thread.
self.monitor_thread_ident = start_new_thread(self, ())
# We must track the PID to know if your thread has died after a fork
self.pid = os.getpid()
def limit_thread_cpu_usage_by_time() -> None:
"""This will enable Gevent's monitoring thread, and if a Greenlet uses the
CPU for longer than `max_blocking_time` it will be killed.
This will result in the whole process being killed, since exceptions are
propagate to the top-level. The goal here is to detect slow functions that
have to be optimized.
"""
gevent.config.monitor_thread = True
gevent.config.max_blocking_time = 10.0
# The monitoring thread will use the trace api just like the TraceSampler
# and the SwitchMonitoring. Sadly there is no API to uninstall the thread,
# but this should not be a problem.
monitor_thread = gevent.get_hub().start_periodic_monitoring_thread()
# This code must not use the tracer from the monitor_thread because calls
# to `did_block_hub` will reset its internal state. If two threads use the
# same underlying tracer false positives will happen, because the switch
# counter will be artifically reset.
greenlet_tracer = GreenletTracer()
def kill_offender(hub: Hub) -> None:
if greenlet_tracer.did_block_hub(hub):
active_greenlet = greenlet_tracer.active_greenlet
msg = ""
if monitor_thread._tracer.active_greenlet != active_greenlet:
msg = (
f"Mismatch values for the active_greenlet among the "
f"monitor_thread and deubgging tracer, this either means "
f"there is a bug in the trace chain (the wrong values are "
f"forwarded), or that one of the trace functions was wrongly "
f"uninstalled. Active greenlets "
f"monitor_thread={monitor_thread._tracer.active_greenlet} "
f"debug_tracer={active_greenlet}.")
hub.loop.run_callback(lambda: active_greenlet.throw(
RaidenUnrecoverableError(
f"A greenlet used the CPU for longer than "
f"{gevent.config.max_blocking_time} seconds, killing it.{msg}"
)))
monitor_thread.add_monitoring_function(kill_offender,
gevent.config.max_blocking_time)
def __init__(self, hub):
self._hub_wref = wref(hub, self._on_hub_gc)
self.should_run = True
# Must be installed in the thread that the hub is running in;
# the trace function is threadlocal
assert get_thread_ident() == hub.thread_ident
self._greenlet_tracer = GreenletTracer()
self._monitoring_functions = [_MonitorEntry(self.monitor_blocking,
GEVENT_CONFIG.max_blocking_time)]
self._calculated_sleep_time = GEVENT_CONFIG.max_blocking_time
# Create the actual monitoring thread. This is effectively a "daemon"
# thread.
self.monitor_thread_ident = start_new_thread(self, ())
# We must track the PID to know if your thread has died after a fork
self.pid = os.getpid()
class PeriodicMonitoringThread(object):
# This doesn't extend threading.Thread because that gets monkey-patched.
# We use the low-level 'start_new_thread' primitive instead.
# The amount of seconds we will sleep when we think we have nothing
# to do.
inactive_sleep_time = 2.0
# The absolute minimum we will sleep, regardless of
# what particular monitoring functions want to say.
min_sleep_time = 0.005
# The minimum period in seconds at which we will check memory usage.
# Getting memory usage is fairly expensive.
min_memory_monitor_period = 2
# A list of _MonitorEntry objects: [(function(hub), period, last_run_time))]
# The first entry is always our entry for self.monitor_blocking
_monitoring_functions = None
# The calculated min sleep time for the monitoring functions list.
_calculated_sleep_time = None
# A boolean value that also happens to capture the
# memory usage at the time we exceeded the threshold. Reset
# to 0 when we go back below.
_memory_exceeded = 0
# The instance of GreenletTracer we're using
_greenlet_tracer = None
def __init__(self, hub):
self._hub_wref = wref(hub, self._on_hub_gc)
self.should_run = True
# Must be installed in the thread that the hub is running in;
# the trace function is threadlocal
assert get_thread_ident() == hub.thread_ident
self._greenlet_tracer = GreenletTracer()
self._monitoring_functions = [
_MonitorEntry(self.monitor_blocking,
GEVENT_CONFIG.max_blocking_time)
]
self._calculated_sleep_time = GEVENT_CONFIG.max_blocking_time
# Create the actual monitoring thread. This is effectively a "daemon"
# thread.
self.monitor_thread_ident = start_new_thread(self, ())
# We must track the PID to know if your thread has died after a fork
self.pid = os.getpid()
def _on_fork(self):
# Pseudo-standard method that resolver_ares and threadpool
# also have, called by hub.reinit()
pid = os.getpid()
if pid != self.pid:
self.pid = pid
self.monitor_thread_ident = start_new_thread(self, ())
@property
def hub(self):
return self._hub_wref()
def monitoring_functions(self):
# Return a list of _MonitorEntry objects
# Update max_blocking_time each time.
mbt = GEVENT_CONFIG.max_blocking_time # XXX: Events so we know when this changes.
if mbt != self._monitoring_functions[0].period:
self._monitoring_functions[0].period = mbt
self._calculated_sleep_time = min(
x.period for x in self._monitoring_functions)
return self._monitoring_functions
def add_monitoring_function(self, function, period):
if not callable(function):
raise ValueError("function must be callable")
if period is None:
# Remove.
self._monitoring_functions = [
x for x in self._monitoring_functions if x.function != function
]
elif period <= 0:
raise ValueError("Period must be positive.")
else:
# Add or update period
entry = _MonitorEntry(function, period)
self._monitoring_functions = [
x if x.function != function else entry
for x in self._monitoring_functions
]
if entry not in self._monitoring_functions:
self._monitoring_functions.append(entry)
self._calculated_sleep_time = min(x.period
for x in self._monitoring_functions)
def calculate_sleep_time(self):
min_sleep = self._calculated_sleep_time
if min_sleep <= 0:
# Everyone wants to be disabled. Sleep for a longer period of
# time than usual so we don't spin unnecessarily. We might be
# enabled again in the future.
return self.inactive_sleep_time
return max((min_sleep, self.min_sleep_time))
def kill(self):
if not self.should_run:
# Prevent overwriting trace functions.
return
# Stop this monitoring thread from running.
self.should_run = False
# Uninstall our tracing hook
self._greenlet_tracer.kill()
def _on_hub_gc(self, _):
self.kill()
def __call__(self):
# The function that runs in the monitoring thread.
# We cannot use threading.current_thread because it would
# create an immortal DummyThread object.
getcurrent().gevent_monitoring_thread = wref(self)
try:
while self.should_run:
functions = self.monitoring_functions()
assert functions
sleep_time = self.calculate_sleep_time()
thread_sleep(sleep_time)
# Make sure the hub is still around, and still active,
# and keep it around while we are here.
hub = self.hub
if not hub:
self.kill()
if self.should_run:
this_run = perf_counter()
for entry in functions:
f = entry.function
period = entry.period
last_run = entry.last_run_time
if period and last_run + period <= this_run:
entry.last_run_time = this_run
f(hub)
del hub # break our reference to hub while we sleep
except SystemExit:
pass
except: # pylint:disable=bare-except
# We're a daemon thread, so swallow any exceptions that get here
# during interpreter shutdown.
if not sys or not sys.stderr: # pragma: no cover
# Interpreter is shutting down
pass
else:
hub = self.hub
if hub is not None:
# XXX: This tends to do bad things like end the process, because we
# try to switch *threads*, which can't happen. Need something better.
hub.handle_error(self, *sys.exc_info())
def monitor_blocking(self, hub):
# Called periodically to see if the trace function has
# fired to switch greenlets. If not, we will print
# the greenlet tree.
# For tests, we return a true value when we think we found something
# blocking
did_block = self._greenlet_tracer.did_block_hub(hub)
if not did_block:
return
active_greenlet = did_block[1]
report = self._greenlet_tracer.did_block_hub_report(
hub, active_greenlet,
dict(greenlet_stacks=False,
current_thread_ident=self.monitor_thread_ident))
stream = hub.exception_stream
for line in report:
# Printing line by line may interleave with other things,
# but it should also prevent a "reentrant call to print"
# when the report is large.
print(line, file=stream)
notify(
EventLoopBlocked(active_greenlet, GEVENT_CONFIG.max_blocking_time,
report))
return (active_greenlet, report)
def ignore_current_greenlet_blocking(self):
self._greenlet_tracer.ignore_current_greenlet_blocking()
def monitor_current_greenlet_blocking(self):
self._greenlet_tracer.monitor_current_greenlet_blocking()
def _get_process(self): # pylint:disable=method-hidden
proc = get_this_psutil_process()
self._get_process = lambda: proc
return proc
def can_monitor_memory_usage(self):
return self._get_process() is not None
def install_monitor_memory_usage(self):
# Start monitoring memory usage, if possible.
# If not possible, emit a warning.
if not self.can_monitor_memory_usage():
import warnings
warnings.warn("Unable to monitor memory usage. Install psutil.",
MonitorWarning)
return
self.add_monitoring_function(
self.monitor_memory_usage,
max(GEVENT_CONFIG.memory_monitor_period,
self.min_memory_monitor_period))
def monitor_memory_usage(self, _hub):
max_allowed = GEVENT_CONFIG.max_memory_usage
if not max_allowed:
# They disabled it.
return -1 # value for tests
rusage = self._get_process().memory_full_info()
# uss only documented available on Windows, Linux, and OS X.
# If not available, fall back to rss as an aproximation.
mem_usage = getattr(rusage, 'uss', 0) or rusage.rss
event = None # Return value for tests
if mem_usage > max_allowed:
if mem_usage > self._memory_exceeded:
# We're still growing
event = MemoryUsageThresholdExceeded(mem_usage, max_allowed,
rusage)
notify(event)
self._memory_exceeded = mem_usage
else:
# we're below. Were we above it last time?
if self._memory_exceeded:
event = MemoryUsageUnderThreshold(mem_usage, max_allowed,
rusage,
self._memory_exceeded)
notify(event)
self._memory_exceeded = 0
return event
def __repr__(self):
return '<%s at %s in thread %s greenlet %r for %r>' % (
self.__class__.__name__, hex(id(self)),
hex(self.monitor_thread_ident), getcurrent(), self._hub_wref())
class PeriodicMonitoringThread(object):
# This doesn't extend threading.Thread because that gets monkey-patched.
# We use the low-level 'start_new_thread' primitive instead.
# The amount of seconds we will sleep when we think we have nothing
# to do.
inactive_sleep_time = 2.0
# The absolute minimum we will sleep, regardless of
# what particular monitoring functions want to say.
min_sleep_time = 0.005
# The minimum period in seconds at which we will check memory usage.
# Getting memory usage is fairly expensive.
min_memory_monitor_period = 2
# A list of _MonitorEntry objects: [(function(hub), period, last_run_time))]
# The first entry is always our entry for self.monitor_blocking
_monitoring_functions = None
# The calculated min sleep time for the monitoring functions list.
_calculated_sleep_time = None
# A boolean value that also happens to capture the
# memory usage at the time we exceeded the threshold. Reset
# to 0 when we go back below.
_memory_exceeded = 0
# The instance of GreenletTracer we're using
_greenlet_tracer = None
def __init__(self, hub):
self._hub_wref = wref(hub, self._on_hub_gc)
self.should_run = True
# Must be installed in the thread that the hub is running in;
# the trace function is threadlocal
assert get_thread_ident() == hub.thread_ident
self._greenlet_tracer = GreenletTracer()
self._monitoring_functions = [_MonitorEntry(self.monitor_blocking,
GEVENT_CONFIG.max_blocking_time)]
self._calculated_sleep_time = GEVENT_CONFIG.max_blocking_time
# Create the actual monitoring thread. This is effectively a "daemon"
# thread.
self.monitor_thread_ident = start_new_thread(self, ())
# We must track the PID to know if your thread has died after a fork
self.pid = os.getpid()
def _on_fork(self):
# Pseudo-standard method that resolver_ares and threadpool
# also have, called by hub.reinit()
pid = os.getpid()
if pid != self.pid:
self.pid = pid
self.monitor_thread_ident = start_new_thread(self, ())
@property
def hub(self):
return self._hub_wref()
def monitoring_functions(self):
# Return a list of _MonitorEntry objects
# Update max_blocking_time each time.
mbt = GEVENT_CONFIG.max_blocking_time # XXX: Events so we know when this changes.
if mbt != self._monitoring_functions[0].period:
self._monitoring_functions[0].period = mbt
self._calculated_sleep_time = min(x.period for x in self._monitoring_functions)
return self._monitoring_functions
def add_monitoring_function(self, function, period):
if not callable(function):
raise ValueError("function must be callable")
if period is None:
# Remove.
self._monitoring_functions = [
x for x in self._monitoring_functions
if x.function != function
]
elif period <= 0:
raise ValueError("Period must be positive.")
else:
# Add or update period
entry = _MonitorEntry(function, period)
self._monitoring_functions = [
x if x.function != function else entry
for x in self._monitoring_functions
]
if entry not in self._monitoring_functions:
self._monitoring_functions.append(entry)
self._calculated_sleep_time = min(x.period for x in self._monitoring_functions)
def calculate_sleep_time(self):
min_sleep = self._calculated_sleep_time
if min_sleep <= 0:
# Everyone wants to be disabled. Sleep for a longer period of
# time than usual so we don't spin unnecessarily. We might be
# enabled again in the future.
return self.inactive_sleep_time
return max((min_sleep, self.min_sleep_time))
def kill(self):
if not self.should_run:
# Prevent overwriting trace functions.
return
# Stop this monitoring thread from running.
self.should_run = False
# Uninstall our tracing hook
self._greenlet_tracer.kill()
def _on_hub_gc(self, _):
self.kill()
def __call__(self):
# The function that runs in the monitoring thread.
# We cannot use threading.current_thread because it would
# create an immortal DummyThread object.
getcurrent().gevent_monitoring_thread = wref(self)
try:
while self.should_run:
functions = self.monitoring_functions()
assert functions
sleep_time = self.calculate_sleep_time()
thread_sleep(sleep_time)
# Make sure the hub is still around, and still active,
# and keep it around while we are here.
hub = self.hub
if not hub:
self.kill()
if self.should_run:
this_run = perf_counter()
for entry in functions:
f = entry.function
period = entry.period
last_run = entry.last_run_time
if period and last_run + period <= this_run:
entry.last_run_time = this_run
f(hub)
del hub # break our reference to hub while we sleep
except SystemExit:
pass
except: # pylint:disable=bare-except
# We're a daemon thread, so swallow any exceptions that get here
# during interpreter shutdown.
if not sys or not sys.stderr: # pragma: no cover
# Interpreter is shutting down
pass
else:
hub = self.hub
if hub is not None:
# XXX: This tends to do bad things like end the process, because we
# try to switch *threads*, which can't happen. Need something better.
hub.handle_error(self, *sys.exc_info())
def monitor_blocking(self, hub):
# Called periodically to see if the trace function has
# fired to switch greenlets. If not, we will print
# the greenlet tree.
# For tests, we return a true value when we think we found something
# blocking
did_block = self._greenlet_tracer.did_block_hub(hub)
if not did_block:
return
active_greenlet = did_block[1]
report = self._greenlet_tracer.did_block_hub_report(
hub, active_greenlet,
dict(greenlet_stacks=False, current_thread_ident=self.monitor_thread_ident))
stream = hub.exception_stream
for line in report:
# Printing line by line may interleave with other things,
# but it should also prevent a "reentrant call to print"
# when the report is large.
print(line, file=stream)
notify(EventLoopBlocked(active_greenlet, GEVENT_CONFIG.max_blocking_time, report))
return (active_greenlet, report)
def ignore_current_greenlet_blocking(self):
self._greenlet_tracer.ignore_current_greenlet_blocking()
def monitor_current_greenlet_blocking(self):
self._greenlet_tracer.monitor_current_greenlet_blocking()
def _get_process(self): # pylint:disable=method-hidden
proc = get_this_psutil_process()
self._get_process = lambda: proc
return proc
def can_monitor_memory_usage(self):
return self._get_process() is not None
def install_monitor_memory_usage(self):
# Start monitoring memory usage, if possible.
# If not possible, emit a warning.
if not self.can_monitor_memory_usage():
import warnings
warnings.warn("Unable to monitor memory usage. Install psutil.",
MonitorWarning)
return
self.add_monitoring_function(self.monitor_memory_usage,
max(GEVENT_CONFIG.memory_monitor_period,
self.min_memory_monitor_period))
def monitor_memory_usage(self, _hub):
max_allowed = GEVENT_CONFIG.max_memory_usage
if not max_allowed:
# They disabled it.
return -1 # value for tests
rusage = self._get_process().memory_full_info()
# uss only documented available on Windows, Linux, and OS X.
# If not available, fall back to rss as an aproximation.
mem_usage = getattr(rusage, 'uss', 0) or rusage.rss
event = None # Return value for tests
if mem_usage > max_allowed:
if mem_usage > self._memory_exceeded:
# We're still growing
event = MemoryUsageThresholdExceeded(
mem_usage, max_allowed, rusage)
notify(event)
self._memory_exceeded = mem_usage
else:
# we're below. Were we above it last time?
if self._memory_exceeded:
event = MemoryUsageUnderThreshold(
mem_usage, max_allowed, rusage, self._memory_exceeded)
notify(event)
self._memory_exceeded = 0
return event
def __repr__(self):
return '<%s at %s in thread %s greenlet %r for %r>' % (
self.__class__.__name__,
hex(id(self)),
hex(self.monitor_thread_ident),
getcurrent(),
self._hub_wref())