def patch_module(target_module, source_module, items=None,
_warnings=None,
_notify_did_subscribers=True):
"""
patch_module(target_module, source_module, items=None)
Replace attributes in *target_module* with the attributes of the
same name in *source_module*.
The *source_module* can provide some attributes to customize the process:
* ``__implements__`` is a list of attribute names to copy; if not present,
the *items* keyword argument is mandatory.
* ``_gevent_will_monkey_patch(target_module, items, warn, **kwargs)``
* ``_gevent_did_monkey_patch(target_module, items, warn, **kwargs)``
These two functions in the *source_module* are called *if* they exist,
before and after copying attributes, respectively. The "will" function
may modify *items*. The value of *warn* is a function that should be called
with a single string argument to issue a warning to the user. If the "will"
function raises :exc:`gevent.events.DoNotPatch`, no patching will be done. These functions
are called before any event subscribers or plugins.
:keyword list items: A list of attribute names to replace. If
not given, this will be taken from the *source_module* ``__implements__``
attribute.
:return: A true value if patching was done, a false value if patching was canceled.
.. versionadded:: 1.3b1
"""
from gevent import events
if items is None:
items = getattr(source_module, '__implements__', None)
if items is None:
raise AttributeError('%r does not have __implements__' % source_module)
try:
__call_module_hook(source_module, 'will', target_module, items, _warnings)
_notify_patch(
events.GeventWillPatchModuleEvent(target_module.__name__, source_module,
target_module, items),
_warnings)
except events.DoNotPatch:
return False
for attr in items:
patch_item(target_module, attr, getattr(source_module, attr))
__call_module_hook(source_module, 'did', target_module, items, _warnings)
if _notify_did_subscribers:
# We allow turning off the broadcast of the 'did' event for the benefit
# of our internal functions which need to do additional work (besides copying
# attributes) before their patch can be considered complete.
_notify_patch(
events.GeventDidPatchModuleEvent(target_module.__name__, source_module,
target_module)
)
return True
def patch_sys(stdin=True, stdout=True, stderr=True):
"""
Patch sys.std[in,out,err] to use a cooperative IO via a
threadpool.
This is relatively dangerous and can have unintended consequences
such as hanging the process or `misinterpreting control keys`_
when :func:`input` and :func:`raw_input` are used. :func:`patch_all`
does *not* call this function by default.
This method does nothing on Python 3. The Python 3 interpreter
wants to flush the TextIOWrapper objects that make up
stderr/stdout at shutdown time, but using a threadpool at that
time leads to a hang.
.. _`misinterpreting control keys`: https://github.com/gevent/gevent/issues/274
"""
# test__issue6.py demonstrates the hang if these lines are removed;
# strangely enough that test passes even without monkey-patching sys
if PY3:
items = None
else:
items = set([('stdin' if stdin else None),
('stdout' if stdout else None),
('stderr' if stderr else None)])
items.discard(None)
items = list(items)
if not items:
return
from gevent import events
_notify_patch(events.GeventWillPatchModuleEvent('sys', None, sys,
items))
for item in items:
_patch_sys_std(item)
_notify_patch(events.GeventDidPatchModuleEvent('sys', None, sys))
def patch_thread(threading=True,
_threading_local=True,
Event=True,
logging=True,
existing_locks=True,
_warnings=None):
"""
patch_thread(threading=True, _threading_local=True, Event=True, logging=True, existing_locks=True) -> None
Replace the standard :mod:`thread` module to make it greenlet-based.
:keyword bool threading: When True (the default),
also patch :mod:`threading`.
:keyword bool _threading_local: When True (the default),
also patch :class:`_threading_local.local`.
:keyword bool logging: When True (the default), also patch locks
taken if the logging module has been configured.
:keyword bool existing_locks: When True (the default), and the
process is still single threaded, make sure that any
:class:`threading.RLock` (and, under Python 3, :class:`importlib._bootstrap._ModuleLock`)
instances that are currently locked can be properly unlocked. **Important**: This is a
best-effort attempt and, on certain implementations, may not detect all
locks. It is important to monkey-patch extremely early in the startup process.
Setting this to False is not recommended, especially on Python 2.
.. caution::
Monkey-patching :mod:`thread` and using
:class:`multiprocessing.Queue` or
:class:`concurrent.futures.ProcessPoolExecutor` (which uses a
``Queue``) will hang the process.
.. versionchanged:: 1.1b1
Add *logging* and *existing_locks* params.
.. versionchanged:: 1.3a2
``Event`` defaults to True.
"""
# XXX: Simplify
# pylint:disable=too-many-branches,too-many-locals,too-many-statements
# Description of the hang:
# There is an incompatibility with patching 'thread' and the 'multiprocessing' module:
# The problem is that multiprocessing.queues.Queue uses a half-duplex multiprocessing.Pipe,
# which is implemented with os.pipe() and _multiprocessing.Connection. os.pipe isn't patched
# by gevent, as it returns just a fileno. _multiprocessing.Connection is an internal implementation
# class implemented in C, which exposes a 'poll(timeout)' method; under the covers, this issues a
# (blocking) select() call: hence the need for a real thread. Except for that method, we could
# almost replace Connection with gevent.fileobject.SocketAdapter, plus a trivial
# patch to os.pipe (below). Sigh, so close. (With a little work, we could replicate that method)
# import os
# import fcntl
# os_pipe = os.pipe
# def _pipe():
# r, w = os_pipe()
# fcntl.fcntl(r, fcntl.F_SETFL, os.O_NONBLOCK)
# fcntl.fcntl(w, fcntl.F_SETFL, os.O_NONBLOCK)
# return r, w
# os.pipe = _pipe
# The 'threading' module copies some attributes from the
# thread module the first time it is imported. If we patch 'thread'
# before that happens, then we store the wrong values in 'saved',
# So if we're going to patch threading, we either need to import it
# before we patch thread, or manually clean up the attributes that
# are in trouble. The latter is tricky because of the different names
# on different versions.
if threading:
threading_mod = __import__('threading')
# Capture the *real* current thread object before
# we start returning DummyThread objects, for comparison
# to the main thread.
orig_current_thread = threading_mod.current_thread()
else:
threading_mod = None
gevent_threading_mod = None
orig_current_thread = None
gevent_thread_mod, thread_mod = _patch_module(
'thread', _warnings=_warnings, _notify_did_subscribers=False)
if threading:
gevent_threading_mod, _ = _patch_module('threading',
_warnings=_warnings,
_notify_did_subscribers=False)
if Event:
from gevent.event import Event
patch_item(threading_mod, 'Event', Event)
# Python 2 had `Event` as a function returning
# the private class `_Event`. Some code may be relying
# on that.
if hasattr(threading_mod, '_Event'):
patch_item(threading_mod, '_Event', Event)
if existing_locks:
_patch_existing_locks(threading_mod)
if logging and 'logging' in sys.modules:
logging = __import__('logging')
patch_item(logging, '_lock', threading_mod.RLock())
for wr in logging._handlerList:
# In py26, these are actual handlers, not weakrefs
handler = wr() if callable(wr) else wr
if handler is None:
continue
if not hasattr(handler, 'lock'):
raise TypeError("Unknown/unsupported handler %r" % handler)
handler.lock = threading_mod.RLock()
if _threading_local:
_threading_local = __import__('_threading_local')
from gevent.local import local
patch_item(_threading_local, 'local', local)
def make_join_func(thread, thread_greenlet):
from gevent.hub import sleep
from time import time
def join(timeout=None):
end = None
if threading_mod.current_thread() is thread:
raise RuntimeError("Cannot join current thread")
if thread_greenlet is not None and thread_greenlet.dead:
return
# You may ask: Why not call thread_greenlet.join()?
# Well, in the one case we actually have a greenlet, it's the
# low-level greenlet.greenlet object for the main thread, which
# doesn't have a join method.
#
# You may ask: Why not become the main greenlet's *parent*
# so you can get notified when it finishes? Because you can't
# create a greenlet cycle (the current greenlet is a descendent
# of the parent), and nor can you set a greenlet's parent to None,
# so there can only ever be one greenlet with a parent of None: the main
# greenlet, the one we need to watch.
#
# You may ask: why not swizzle out the problematic lock on the main thread
# into a gevent friendly lock? Well, the interpreter actually depends on that
# for the main thread in threading._shutdown; see below.
if not thread.is_alive():
return
if timeout:
end = time() + timeout
while thread.is_alive():
if end is not None and time() > end:
return
sleep(0.01)
return join
if threading:
from gevent.threading import main_native_thread
for thread in threading_mod._active.values():
if thread == main_native_thread():
continue
thread.join = make_join_func(thread, None)
if PY3:
# Issue 18808 changes the nature of Thread.join() to use
# locks. This means that a greenlet spawned in the main thread
# (which is already running) cannot wait for the main thread---it
# hangs forever. We patch around this if possible. See also
# gevent.threading.
greenlet = __import__('greenlet')
already_patched = is_object_patched('threading', '_shutdown')
if orig_current_thread == threading_mod.main_thread(
) and not already_patched:
main_thread = threading_mod.main_thread()
_greenlet = main_thread._greenlet = greenlet.getcurrent()
main_thread.__real_tstate_lock = main_thread._tstate_lock
assert main_thread.__real_tstate_lock is not None
# The interpreter will call threading._shutdown
# when the main thread exits and is about to
# go away. It is called *in* the main thread. This
# is a perfect place to notify other greenlets that
# the main thread is done. We do this by overriding the
# lock of the main thread during operation, and only restoring
# it to the native blocking version at shutdown time
# (the interpreter also has a reference to this lock in a
# C data structure).
main_thread._tstate_lock = threading_mod.Lock()
main_thread._tstate_lock.acquire()
orig_shutdown = threading_mod._shutdown
def _shutdown():
# Release anyone trying to join() me,
# and let us switch to them.
if not main_thread._tstate_lock:
return
main_thread._tstate_lock.release()
from gevent import sleep
try:
sleep()
except: # pylint:disable=bare-except
# A greenlet could have .kill() us
# or .throw() to us. I'm the main greenlet,
# there's no where else for this to go.
from gevent import get_hub
get_hub().print_exception(_greenlet, *sys.exc_info())
# Now, this may have resulted in us getting stopped
# if some other greenlet actually just ran there.
# That's not good, we're not supposed to be stopped
# when we enter _shutdown.
main_thread._is_stopped = False
main_thread._tstate_lock = main_thread.__real_tstate_lock
main_thread.__real_tstate_lock = None
# The only truly blocking native shutdown lock to
# acquire should be our own (hopefully), and the call to
# _stop that orig_shutdown makes will discard it.
orig_shutdown()
patch_item(threading_mod, '_shutdown', orig_shutdown)
patch_item(threading_mod, '_shutdown', _shutdown)
# We create a bit of a reference cycle here,
# so main_thread doesn't get to be collected in a timely way.
# Not good. Take it out of dangling so we don't get
# warned about it.
threading_mod._dangling.remove(main_thread)
# Patch up the ident of the main thread to match. This
# matters if threading was imported before monkey-patching
# thread
oldid = main_thread.ident
main_thread._ident = threading_mod.get_ident()
if oldid in threading_mod._active:
threading_mod._active[
main_thread.ident] = threading_mod._active[oldid]
if oldid != main_thread.ident:
del threading_mod._active[oldid]
elif not already_patched:
_queue_warning(
"Monkey-patching not on the main thread; "
"threading.main_thread().join() will hang from a greenlet",
_warnings)
from gevent import events
_notify_patch(
events.GeventDidPatchModuleEvent('thread', gevent_thread_mod,
thread_mod))
_notify_patch(
events.GeventDidPatchModuleEvent('threading', gevent_threading_mod,
threading_mod))
def patch_select(aggressive=True):
"""
Replace :func:`select.select` with :func:`gevent.select.select`
and :func:`select.poll` with :class:`gevent.select.poll` (where available).
If ``aggressive`` is true (the default), also remove other
blocking functions from :mod:`select` and (on Python 3.4 and
above) :mod:`selectors`:
- :func:`select.epoll`
- :func:`select.kqueue`
- :func:`select.kevent`
- :func:`select.devpoll` (Python 3.5+)
- :class:`selectors.EpollSelector`
- :class:`selectors.KqueueSelector`
- :class:`selectors.DevpollSelector` (Python 3.5+)
"""
source_mod, target_mod = _patch_module('select',
_notify_did_subscribers=False)
if aggressive:
select = target_mod
# since these are blocking we're removing them here. This makes some other
# modules (e.g. asyncore) non-blocking, as they use select that we provide
# when none of these are available.
remove_item(select, 'epoll')
remove_item(select, 'kqueue')
remove_item(select, 'kevent')
remove_item(select, 'devpoll')
if sys.version_info[:2] >= (3, 4):
# Python 3 wants to use `select.select` as a member function,
# leading to this error in selectors.py (because gevent.select.select is
# not a builtin and doesn't get the magic auto-static that they do)
# r, w, _ = self._select(self._readers, self._writers, [], timeout)
# TypeError: select() takes from 3 to 4 positional arguments but 5 were given
# Note that this obviously only happens if selectors was imported after we had patched
# select; but there is a code path that leads to it being imported first (but now we've
# patched select---so we can't compare them identically)
select = target_mod # Should be gevent-patched now
orig_select_select = get_original('select', 'select')
assert select.select is not orig_select_select
selectors = __import__('selectors')
if selectors.SelectSelector._select in (select.select,
orig_select_select):
def _select(self, *args, **kwargs): # pylint:disable=unused-argument
return select.select(*args, **kwargs)
selectors.SelectSelector._select = _select
_select._gevent_monkey = True
# Python 3.7 refactors the poll-like selectors to use a common
# base class and capture a reference to select.poll, etc, at
# import time. selectors tends to get imported early
# (importing 'platform' does it: platform -> subprocess -> selectors),
# so we need to clean that up.
if hasattr(selectors, 'PollSelector') and hasattr(
selectors.PollSelector, '_selector_cls'):
selectors.PollSelector._selector_cls = select.poll
if aggressive:
# If `selectors` had already been imported before we removed
# select.epoll|kqueue|devpoll, these may have been defined in terms
# of those functions. They'll fail at runtime.
remove_item(selectors, 'EpollSelector')
remove_item(selectors, 'KqueueSelector')
remove_item(selectors, 'DevpollSelector')
selectors.DefaultSelector = selectors.SelectSelector
from gevent import events
_notify_patch(
events.GeventDidPatchModuleEvent('select', source_mod, target_mod))
def patch_thread(threading=True,
_threading_local=True,
Event=True,
logging=True,
existing_locks=True,
_warnings=None):
"""
patch_thread(threading=True, _threading_local=True, Event=True, logging=True, existing_locks=True) -> None
Replace the standard :mod:`thread` module to make it greenlet-based.
:keyword bool threading: When True (the default),
also patch :mod:`threading`.
:keyword bool _threading_local: When True (the default),
also patch :class:`_threading_local.local`.
:keyword bool logging: When True (the default), also patch locks
taken if the logging module has been configured.
:keyword bool existing_locks: When True (the default), and the
process is still single threaded, make sure that any
:class:`threading.RLock` (and, under Python 3, :class:`importlib._bootstrap._ModuleLock`)
instances that are currently locked can be properly unlocked.
.. caution::
Monkey-patching :mod:`thread` and using
:class:`multiprocessing.Queue` or
:class:`concurrent.futures.ProcessPoolExecutor` (which uses a
``Queue``) will hang the process.
.. versionchanged:: 1.1b1
Add *logging* and *existing_locks* params.
.. versionchanged:: 1.3a2
``Event`` defaults to True.
"""
# XXX: Simplify
# pylint:disable=too-many-branches,too-many-locals,too-many-statements
# Description of the hang:
# There is an incompatibility with patching 'thread' and the 'multiprocessing' module:
# The problem is that multiprocessing.queues.Queue uses a half-duplex multiprocessing.Pipe,
# which is implemented with os.pipe() and _multiprocessing.Connection. os.pipe isn't patched
# by gevent, as it returns just a fileno. _multiprocessing.Connection is an internal implementation
# class implemented in C, which exposes a 'poll(timeout)' method; under the covers, this issues a
# (blocking) select() call: hence the need for a real thread. Except for that method, we could
# almost replace Connection with gevent.fileobject.SocketAdapter, plus a trivial
# patch to os.pipe (below). Sigh, so close. (With a little work, we could replicate that method)
# import os
# import fcntl
# os_pipe = os.pipe
# def _pipe():
# r, w = os_pipe()
# fcntl.fcntl(r, fcntl.F_SETFL, os.O_NONBLOCK)
# fcntl.fcntl(w, fcntl.F_SETFL, os.O_NONBLOCK)
# return r, w
# os.pipe = _pipe
# The 'threading' module copies some attributes from the
# thread module the first time it is imported. If we patch 'thread'
# before that happens, then we store the wrong values in 'saved',
# So if we're going to patch threading, we either need to import it
# before we patch thread, or manually clean up the attributes that
# are in trouble. The latter is tricky because of the different names
# on different versions.
if threading:
threading_mod = __import__('threading')
# Capture the *real* current thread object before
# we start returning DummyThread objects, for comparison
# to the apis thread.
orig_current_thread = threading_mod.current_thread()
else:
threading_mod = None
gevent_threading_mod = None
orig_current_thread = None
gevent_thread_mod, thread_mod = _patch_module(
'thread', _warnings=_warnings, _notify_did_subscribers=False)
if threading:
gevent_threading_mod, _ = _patch_module('threading',
_warnings=_warnings,
_notify_did_subscribers=False)
if Event:
from gevent.event import Event
patch_item(threading_mod, 'Event', Event)
# Python 2 had `Event` as a function returning
# the private class `_Event`. Some code may be relying
# on that.
if hasattr(threading_mod, '_Event'):
patch_item(threading_mod, '_Event', Event)
if existing_locks:
_patch_existing_locks(threading_mod)
if logging and 'logging' in sys.modules:
logging = __import__('logging')
patch_item(logging, '_lock', threading_mod.RLock())
for wr in logging._handlerList:
# In py26, these are actual handlers, not weakrefs
handler = wr() if callable(wr) else wr
if handler is None:
continue
if not hasattr(handler, 'lock'):
raise TypeError("Unknown/unsupported handler %r" % handler)
handler.lock = threading_mod.RLock()
if _threading_local:
_threading_local = __import__('_threading_local')
from gevent.local import local
patch_item(_threading_local, 'local', local)
def make_join_func(thread, thread_greenlet):
from gevent.hub import sleep
from time import time
def join(timeout=None):
end = None
if threading_mod.current_thread() is thread:
raise RuntimeError("Cannot join current thread")
if thread_greenlet is not None and thread_greenlet.dead:
return
if not thread.is_alive():
return
if timeout:
end = time() + timeout
while thread.is_alive():
if end is not None and time() > end:
return
sleep(0.01)
return join
if threading:
from gevent.threading import main_native_thread
for thread in threading_mod._active.values():
if thread == main_native_thread():
continue
thread.join = make_join_func(thread, None)
if sys.version_info[:2] >= (3, 4):
# Issue 18808 changes the nature of Thread.join() to use
# locks. This means that a greenlet spawned in the apis thread
# (which is already running) cannot wait for the apis thread---it
# hangs forever. We patch around this if possible. See also
# gevent.threading.
greenlet = __import__('greenlet')
if orig_current_thread == threading_mod.main_thread():
main_thread = threading_mod.main_thread()
_greenlet = main_thread._greenlet = greenlet.getcurrent()
main_thread.join = make_join_func(main_thread, _greenlet)
# Patch up the ident of the apis thread to match. This
# matters if threading was imported before monkey-patching
# thread
oldid = main_thread.ident
main_thread._ident = threading_mod.get_ident()
if oldid in threading_mod._active:
threading_mod._active[
main_thread.ident] = threading_mod._active[oldid]
if oldid != main_thread.ident:
del threading_mod._active[oldid]
else:
_queue_warning(
"Monkey-patching not on the apis thread; "
"threading.main_thread().join() will hang from a greenlet",
_warnings)
from gevent import events
_notify_patch(
events.GeventDidPatchModuleEvent('thread', gevent_thread_mod,
thread_mod))
_notify_patch(
events.GeventDidPatchModuleEvent('threading', gevent_threading_mod,
threading_mod))