def patch_thread(threading=True, _threading_local=True, Event=True, logging=True, existing_locks=True, _warnings=None): """ patch_thread(threading=True, _threading_local=True, Event=True, logging=True, existing_locks=True) -> None Replace the standard :mod:`thread` module to make it greenlet-based. :keyword bool threading: When True (the default), also patch :mod:`threading`. :keyword bool _threading_local: When True (the default), also patch :class:`_threading_local.local`. :keyword bool logging: When True (the default), also patch locks taken if the logging module has been configured. :keyword bool existing_locks: When True (the default), and the process is still single threaded, make sure that any :class:`threading.RLock` (and, under Python 3, :class:`importlib._bootstrap._ModuleLock`) instances that are currently locked can be properly unlocked. **Important**: This is a best-effort attempt and, on certain implementations, may not detect all locks. It is important to monkey-patch extremely early in the startup process. .. caution:: Monkey-patching :mod:`thread` and using :class:`multiprocessing.Queue` or :class:`concurrent.futures.ProcessPoolExecutor` (which uses a ``Queue``) will hang the process. .. versionchanged:: 1.1b1 Add *logging* and *existing_locks* params. .. versionchanged:: 1.3a2 ``Event`` defaults to True. """ # XXX: Simplify # pylint:disable=too-many-branches,too-many-locals,too-many-statements # Description of the hang: # There is an incompatibility with patching 'thread' and the 'multiprocessing' module: # The problem is that multiprocessing.queues.Queue uses a half-duplex multiprocessing.Pipe, # which is implemented with os.pipe() and _multiprocessing.Connection. os.pipe isn't patched # by gevent, as it returns just a fileno. _multiprocessing.Connection is an internal implementation # class implemented in C, which exposes a 'poll(timeout)' method; under the covers, this issues a # (blocking) select() call: hence the need for a real thread. Except for that method, we could # almost replace Connection with gevent.fileobject.SocketAdapter, plus a trivial # patch to os.pipe (below). Sigh, so close. (With a little work, we could replicate that method) # import os # import fcntl # os_pipe = os.pipe # def _pipe(): # r, w = os_pipe() # fcntl.fcntl(r, fcntl.F_SETFL, os.O_NONBLOCK) # fcntl.fcntl(w, fcntl.F_SETFL, os.O_NONBLOCK) # return r, w # os.pipe = _pipe # The 'threading' module copies some attributes from the # thread module the first time it is imported. If we patch 'thread' # before that happens, then we store the wrong values in 'saved', # So if we're going to patch threading, we either need to import it # before we patch thread, or manually clean up the attributes that # are in trouble. The latter is tricky because of the different names # on different versions. if threading: threading_mod = __import__('threading') # Capture the *real* current thread object before # we start returning DummyThread objects, for comparison # to the main thread. orig_current_thread = threading_mod.current_thread() else: threading_mod = None gevent_threading_mod = None orig_current_thread = None gevent_thread_mod, thread_mod = _patch_module('thread', _warnings=_warnings, _notify_did_subscribers=False) if threading: gevent_threading_mod, _ = _patch_module('threading', _warnings=_warnings, _notify_did_subscribers=False) if Event: from gevent.event import Event patch_item(threading_mod, 'Event', Event) # Python 2 had `Event` as a function returning # the private class `_Event`. Some code may be relying # on that. if hasattr(threading_mod, '_Event'): patch_item(threading_mod, '_Event', Event) if existing_locks: _patch_existing_locks(threading_mod) if logging and 'logging' in sys.modules: logging = __import__('logging') patch_item(logging, '_lock', threading_mod.RLock()) for wr in logging._handlerList: # In py26, these are actual handlers, not weakrefs handler = wr() if callable(wr) else wr if handler is None: continue if not hasattr(handler, 'lock'): raise TypeError("Unknown/unsupported handler %r" % handler) handler.lock = threading_mod.RLock() if _threading_local: _threading_local = __import__('_threading_local') from gevent.local import local patch_item(_threading_local, 'local', local) def make_join_func(thread, thread_greenlet): from gevent.hub import sleep from time import time def join(timeout=None): end = None if threading_mod.current_thread() is thread: raise RuntimeError("Cannot join current thread") if thread_greenlet is not None and thread_greenlet.dead: return # You may ask: Why not call thread_greenlet.join()? # Well, in the one case we actually have a greenlet, it's the # low-level greenlet.greenlet object for the main thread, which # doesn't have a join method. # # You may ask: Why not become the main greenlet's *parent* # so you can get notified when it finishes? Because you can't # create a greenlet cycle (the current greenlet is a descendent # of the parent), and nor can you set a greenlet's parent to None, # so there can only ever be one greenlet with a parent of None: the main # greenlet, the one we need to watch. # # You may ask: why not swizzle out the problematic lock on the main thread # into a gevent friendly lock? Well, the interpreter actually depends on that # for the main thread in threading._shutdown; see below. if not thread.is_alive(): return if timeout: end = time() + timeout while thread.is_alive(): if end is not None and time() > end: return sleep(0.01) return join if threading: from gevent.threading import main_native_thread for thread in threading_mod._active.values(): if thread == main_native_thread(): continue thread.join = make_join_func(thread, None) if PY3: # Issue 18808 changes the nature of Thread.join() to use # locks. This means that a greenlet spawned in the main thread # (which is already running) cannot wait for the main thread---it # hangs forever. We patch around this if possible. See also # gevent.threading. greenlet = __import__('greenlet') already_patched = is_object_patched('threading', '_shutdown') if orig_current_thread == threading_mod.main_thread() and not already_patched: main_thread = threading_mod.main_thread() _greenlet = main_thread._greenlet = greenlet.getcurrent() main_thread.__real_tstate_lock = main_thread._tstate_lock assert main_thread.__real_tstate_lock is not None # The interpreter will call threading._shutdown # when the main thread exits and is about to # go away. It is called *in* the main thread. This # is a perfect place to notify other greenlets that # the main thread is done. We do this by overriding the # lock of the main thread during operation, and only restoring # it to the native blocking version at shutdown time # (the interpreter also has a reference to this lock in a # C data structure). main_thread._tstate_lock = threading_mod.Lock() main_thread._tstate_lock.acquire() orig_shutdown = threading_mod._shutdown def _shutdown(): # Release anyone trying to join() me, # and let us switch to them. if not main_thread._tstate_lock: return main_thread._tstate_lock.release() from gevent import sleep try: sleep() except: # pylint:disable=bare-except # A greenlet could have .kill() us # or .throw() to us. I'm the main greenlet, # there's no where else for this to go. from gevent import get_hub get_hub().print_exception(_greenlet, *sys.exc_info()) # Now, this may have resulted in us getting stopped # if some other greenlet actually just ran there. # That's not good, we're not supposed to be stopped # when we enter _shutdown. main_thread._is_stopped = False main_thread._tstate_lock = main_thread.__real_tstate_lock main_thread.__real_tstate_lock = None # The only truly blocking native shutdown lock to # acquire should be our own (hopefully), and the call to # _stop that orig_shutdown makes will discard it. orig_shutdown() patch_item(threading_mod, '_shutdown', orig_shutdown) patch_item(threading_mod, '_shutdown', _shutdown) # We create a bit of a reference cycle here, # so main_thread doesn't get to be collected in a timely way. # Not good. Take it out of dangling so we don't get # warned about it. threading_mod._dangling.remove(main_thread) # Patch up the ident of the main thread to match. This # matters if threading was imported before monkey-patching # thread oldid = main_thread.ident main_thread._ident = threading_mod.get_ident() if oldid in threading_mod._active: threading_mod._active[main_thread.ident] = threading_mod._active[oldid] if oldid != main_thread.ident: del threading_mod._active[oldid] elif not already_patched: _queue_warning("Monkey-patching not on the main thread; " "threading.main_thread().join() will hang from a greenlet", _warnings) from gevent import events _notify_patch(events.GeventDidPatchModuleEvent('thread', gevent_thread_mod, thread_mod)) _notify_patch(events.GeventDidPatchModuleEvent('threading', gevent_threading_mod, threading_mod))
def patch_thread(threading=True, _threading_local=True, Event=False, logging=True, existing_locks=True, _warnings=None): """ Replace the standard :mod:`thread` module to make it greenlet-based. - If *threading* is true (the default), also patch ``threading``. - If *_threading_local* is true (the default), also patch ``_threading_local.local``. - If *logging* is True (the default), also patch locks taken if the logging module has been configured. - If *existing_locks* is True (the default), and the process is still single threaded, make sure than any :class:`threading.RLock` (and, under Python 3, :class:`importlib._bootstrap._ModuleLock`) instances that are currently locked can be properly unlocked. .. caution:: Monkey-patching :mod:`thread` and using :class:`multiprocessing.Queue` or :class:`concurrent.futures.ProcessPoolExecutor` (which uses a ``Queue``) will hang the process. .. versionchanged:: 1.1b1 Add *logging* and *existing_locks* params. """ # XXX: Simplify # pylint:disable=too-many-branches,too-many-locals # Description of the hang: # There is an incompatibility with patching 'thread' and the 'multiprocessing' module: # The problem is that multiprocessing.queues.Queue uses a half-duplex multiprocessing.Pipe, # which is implemented with os.pipe() and _multiprocessing.Connection. os.pipe isn't patched # by gevent, as it returns just a fileno. _multiprocessing.Connection is an internal implementation # class implemented in C, which exposes a 'poll(timeout)' method; under the covers, this issues a # (blocking) select() call: hence the need for a real thread. Except for that method, we could # almost replace Connection with gevent.fileobject.SocketAdapter, plus a trivial # patch to os.pipe (below). Sigh, so close. (With a little work, we could replicate that method) # import os # import fcntl # os_pipe = os.pipe # def _pipe(): # r, w = os_pipe() # fcntl.fcntl(r, fcntl.F_SETFL, os.O_NONBLOCK) # fcntl.fcntl(w, fcntl.F_SETFL, os.O_NONBLOCK) # return r, w # os.pipe = _pipe # The 'threading' module copies some attributes from the # thread module the first time it is imported. If we patch 'thread' # before that happens, then we store the wrong values in 'saved', # So if we're going to patch threading, we either need to import it # before we patch thread, or manually clean up the attributes that # are in trouble. The latter is tricky because of the different names # on different versions. if threading: threading_mod = __import__('threading') # Capture the *real* current thread object before # we start returning DummyThread objects, for comparison # to the main thread. orig_current_thread = threading_mod.current_thread() else: threading_mod = None orig_current_thread = None patch_module('thread') if threading: if existing_locks: _patch_existing_locks(threading_mod) patch_module('threading') if Event: from gevent.event import Event patch_item(threading_mod, 'Event', Event) if logging and 'logging' in sys.modules: logging = __import__('logging') patch_item(logging, '_lock', threading_mod.RLock()) for wr in logging._handlerList: # In py26, these are actual handlers, not weakrefs handler = wr() if callable(wr) else wr if handler is None: continue if not hasattr(handler, 'lock'): raise TypeError("Unknown/unsupported handler %r" % handler) handler.lock = threading_mod.RLock() if _threading_local: _threading_local = __import__('_threading_local') from gevent.local import local patch_item(_threading_local, 'local', local) def make_join_func(thread, thread_greenlet): from gevent.hub import sleep from time import time def join(timeout=None): end = None if threading_mod.current_thread() is thread: raise RuntimeError("Cannot join current thread") if thread_greenlet is not None and thread_greenlet.dead: return if not thread.is_alive(): return if timeout: end = time() + timeout while thread.is_alive(): if end is not None and time() > end: return sleep(0.01) return join if threading: from gevent.threading import main_native_thread for thread in threading_mod._active.values(): if thread == main_native_thread(): continue thread.join = make_join_func(thread, None) if sys.version_info[:2] >= (3, 4): # Issue 18808 changes the nature of Thread.join() to use # locks. This means that a greenlet spawned in the main thread # (which is already running) cannot wait for the main thread---it # hangs forever. We patch around this if possible. See also # gevent.threading. greenlet = __import__('greenlet') if orig_current_thread == threading_mod.main_thread(): main_thread = threading_mod.main_thread() _greenlet = main_thread._greenlet = greenlet.getcurrent() from gevent.hub import sleep main_thread.join = make_join_func(main_thread, _greenlet) # Patch up the ident of the main thread to match. This # matters if threading was imported before monkey-patching # thread oldid = main_thread.ident main_thread._ident = threading_mod.get_ident() if oldid in threading_mod._active: threading_mod._active[main_thread.ident] = threading_mod._active[oldid] if oldid != main_thread.ident: del threading_mod._active[oldid] else: _queue_warning("Monkey-patching not on the main thread; " "threading.main_thread().join() will hang from a greenlet", _warnings)
def patch_thread(threading=True, _threading_local=True, Event=True, logging=True, existing_locks=True, _warnings=None): """ patch_thread(threading=True, _threading_local=True, Event=True, logging=True, existing_locks=True) -> None Replace the standard :mod:`thread` module to make it greenlet-based. :keyword bool threading: When True (the default), also patch :mod:`threading`. :keyword bool _threading_local: When True (the default), also patch :class:`_threading_local.local`. :keyword bool logging: When True (the default), also patch locks taken if the logging module has been configured. :keyword bool existing_locks: When True (the default), and the process is still single threaded, make sure that any :class:`threading.RLock` (and, under Python 3, :class:`importlib._bootstrap._ModuleLock`) instances that are currently locked can be properly unlocked. .. caution:: Monkey-patching :mod:`thread` and using :class:`multiprocessing.Queue` or :class:`concurrent.futures.ProcessPoolExecutor` (which uses a ``Queue``) will hang the process. .. versionchanged:: 1.1b1 Add *logging* and *existing_locks* params. .. versionchanged:: 1.3a2 ``Event`` defaults to True. """ # XXX: Simplify # pylint:disable=too-many-branches,too-many-locals,too-many-statements # Description of the hang: # There is an incompatibility with patching 'thread' and the 'multiprocessing' module: # The problem is that multiprocessing.queues.Queue uses a half-duplex multiprocessing.Pipe, # which is implemented with os.pipe() and _multiprocessing.Connection. os.pipe isn't patched # by gevent, as it returns just a fileno. _multiprocessing.Connection is an internal implementation # class implemented in C, which exposes a 'poll(timeout)' method; under the covers, this issues a # (blocking) select() call: hence the need for a real thread. Except for that method, we could # almost replace Connection with gevent.fileobject.SocketAdapter, plus a trivial # patch to os.pipe (below). Sigh, so close. (With a little work, we could replicate that method) # import os # import fcntl # os_pipe = os.pipe # def _pipe(): # r, w = os_pipe() # fcntl.fcntl(r, fcntl.F_SETFL, os.O_NONBLOCK) # fcntl.fcntl(w, fcntl.F_SETFL, os.O_NONBLOCK) # return r, w # os.pipe = _pipe # The 'threading' module copies some attributes from the # thread module the first time it is imported. If we patch 'thread' # before that happens, then we store the wrong values in 'saved', # So if we're going to patch threading, we either need to import it # before we patch thread, or manually clean up the attributes that # are in trouble. The latter is tricky because of the different names # on different versions. if threading: threading_mod = __import__('threading') # Capture the *real* current thread object before # we start returning DummyThread objects, for comparison # to the main thread. orig_current_thread = threading_mod.current_thread() else: threading_mod = None gevent_threading_mod = None orig_current_thread = None gevent_thread_mod, thread_mod = _patch_module('thread', _warnings=_warnings, _notify_did_subscribers=False) if threading: gevent_threading_mod, _ = _patch_module('threading', _warnings=_warnings, _notify_did_subscribers=False) if Event: from gevent.event import Event patch_item(threading_mod, 'Event', Event) # Python 2 had `Event` as a function returning # the private class `_Event`. Some code may be relying # on that. if hasattr(threading_mod, '_Event'): patch_item(threading_mod, '_Event', Event) if existing_locks: _patch_existing_locks(threading_mod) if logging and 'logging' in sys.modules: logging = __import__('logging') patch_item(logging, '_lock', threading_mod.RLock()) for wr in logging._handlerList: # In py26, these are actual handlers, not weakrefs handler = wr() if callable(wr) else wr if handler is None: continue if not hasattr(handler, 'lock'): raise TypeError("Unknown/unsupported handler %r" % handler) handler.lock = threading_mod.RLock() if _threading_local: _threading_local = __import__('_threading_local') from gevent.local import local patch_item(_threading_local, 'local', local) def make_join_func(thread, thread_greenlet): from gevent.hub import sleep from time import time def join(timeout=None): end = None if threading_mod.current_thread() is thread: raise RuntimeError("Cannot join current thread") if thread_greenlet is not None and thread_greenlet.dead: return if not thread.is_alive(): return if timeout: end = time() + timeout while thread.is_alive(): if end is not None and time() > end: return sleep(0.01) return join if threading: from gevent.threading import main_native_thread for thread in threading_mod._active.values(): if thread == main_native_thread(): continue thread.join = make_join_func(thread, None) if sys.version_info[:2] >= (3, 4): # Issue 18808 changes the nature of Thread.join() to use # locks. This means that a greenlet spawned in the main thread # (which is already running) cannot wait for the main thread---it # hangs forever. We patch around this if possible. See also # gevent.threading. greenlet = __import__('greenlet') if orig_current_thread == threading_mod.main_thread(): main_thread = threading_mod.main_thread() _greenlet = main_thread._greenlet = greenlet.getcurrent() main_thread.join = make_join_func(main_thread, _greenlet) # Patch up the ident of the main thread to match. This # matters if threading was imported before monkey-patching # thread oldid = main_thread.ident main_thread._ident = threading_mod.get_ident() if oldid in threading_mod._active: threading_mod._active[main_thread.ident] = threading_mod._active[oldid] if oldid != main_thread.ident: del threading_mod._active[oldid] else: _queue_warning("Monkey-patching not on the main thread; " "threading.main_thread().join() will hang from a greenlet", _warnings) from gevent import events _notify_patch(events.GeventDidPatchModuleEvent('thread', gevent_thread_mod, thread_mod)) _notify_patch(events.GeventDidPatchModuleEvent('threading', gevent_threading_mod, threading_mod))