def test_certificate(self): # Issue 801 from gevent import monkey, ssl # only broken if *not* monkey patched self.assertFalse(monkey.is_module_patched('ssl')) self.assertFalse(monkey.is_module_patched('socket')) self.init_server() server_host, server_port, _family = self.get_server_host_port_family() ssl.get_server_certificate((server_host, server_port))
def test_certificate(self): # Issue 801 from gevent import monkey, ssl # only broken if *not* monkey patched self.assertFalse(monkey.is_module_patched('ssl')) self.assertFalse(monkey.is_module_patched('socket')) self.init_server() server_host, server_port, _family = self.get_server_host_port_family() ssl.get_server_certificate((server_host, server_port)) # pylint:disable=no-member
def test_future_wait_gevent_function(self): # The future object can be waited on with gevent functions. self.assertEqual(monkey.is_module_patched('threading'), self.MONKEY_PATCHED) pool = self.pool def fn(): gevent.sleep(0.5) return 42 future = pool.submit(fn) def spawned(): return 2016 spawned_greenlet = gevent.spawn(spawned) done = gevent.wait((future, )) self.assertEqual(list(done), [future]) self.assertTrue(spawned_greenlet.ready()) self.assertEqual(spawned_greenlet.value, 2016) pool.kill() del future del pool del self.pool
def test_saved(self): self.assertTrue(monkey.saved) for modname, objects in monkey.saved.items(): self.assertTrue(monkey.is_module_patched(modname)) for objname in objects: self.assertTrue(monkey.is_object_patched(modname, objname))
def test_future_wait_gevent_function(self): # The future object can be waited on with gevent functions. self.assertEqual(monkey.is_module_patched('threading'), self.MONKEY_PATCHED) pool = self.pool def fn(): gevent.sleep(0.5) return 42 future = pool.submit(fn) # pylint:disable=no-member def spawned(): return 2016 spawned_greenlet = gevent.spawn(spawned) done = gevent.wait((future,)) self.assertEqual(list(done), [future]) self.assertTrue(spawned_greenlet.ready()) self.assertEqual(spawned_greenlet.value, 2016) pool.kill() del future del pool del self.pool
def test_future_wait_module_function(self): # Instead of waiting on the result, we can wait # on the future using the module functions self.assertEqual(monkey.is_module_patched('threading'), self.MONKEY_PATCHED) pool = self.pool def fn(): gevent.sleep(0.5) return 42 future = pool.submit(fn) # pylint:disable=no-member if self.MONKEY_PATCHED: # Things work as expected when monkey-patched _done, not_done = cf_wait((future,), timeout=0.001) self.assertEqual(list(not_done), [future]) def spawned(): return 2016 spawned_greenlet = gevent.spawn(spawned) done, _not_done = cf_wait((future,)) self.assertEqual(list(done), [future]) self.assertTrue(spawned_greenlet.ready()) self.assertEqual(spawned_greenlet.value, 2016) else: # When not monkey-patched, raises an AttributeError self.assertRaises(AttributeError, cf_wait, (future,)) pool.kill() del future del pool del self.pool
def __init__(self, size=None, greenlet_class=None): super().__init__(size, greenlet_class) if monkey.is_module_patched('socket') is False: raise Exception("""请先打猴子补丁,代码开头加入: from gevent import monkey monkey.patch_all()""") atexit.register(self.shutdown)
def check_gevent_monkey_patch(raise_exc=True): if not monkey.is_module_patched('socket'): # 随便选一个检测标志 if raise_exc: warnings.warn(f'检测到 你还没有打gevent包的猴子补丁,请在所运行的起始脚本第一行写上 【import gevent.monkey;gevent.monkey.patch_all()】 这句话。') raise Exception(f'检测到 你还没有打gevent包的猴子补丁,请在所运行的起始脚本第一行写上 【import gevent.monkey;gevent.monkey.patch_all()】 这句话。') else: return 1
def test_saved(self): self.assertTrue(monkey.saved) for modname in monkey.saved: self.assertTrue(monkey.is_module_patched(modname)) for objname in monkey.saved[modname]: self.assertTrue(monkey.is_object_patched(modname, objname))
def start(): global __started if __started: return __started = True flag = False try: from gevent import monkey flag = monkey.is_module_patched("socket") except ModuleNotFoundError: logger.debug( "it was found that no gevent was used, if you don't use, please ignore." ) if flag: import grpc.experimental.gevent as grpc_gevent grpc_gevent.init_gevent() loggings.init() config.finalize() __init() atexit.register(__fini) if (hasattr(os, 'register_at_fork')): os.register_at_fork(before=__fork_before, after_in_parent=__fork_after_in_parent, after_in_child=__fork_after_in_child)
def test_future_wait_module_function(self): # Instead of waiting on the result, we can wait # on the future using the module functions self.assertEqual(monkey.is_module_patched('threading'), self.MONKEY_PATCHED) pool = self.pool def fn(): gevent.sleep(0.5) return 42 future = pool.submit(fn) if self.MONKEY_PATCHED: # Things work as expected when monkey-patched _done, not_done = cf_wait((future, ), timeout=0.001) self.assertEqual(list(not_done), [future]) def spawned(): return 2016 spawned_greenlet = gevent.spawn(spawned) done, _not_done = cf_wait((future, )) self.assertEqual(list(done), [future]) self.assertTrue(spawned_greenlet.ready()) self.assertEqual(spawned_greenlet.value, 2016) else: # When not monkey-patched, raises an AttributeError self.assertRaises(AttributeError, cf_wait, (future, )) pool.kill() del future del pool del self.pool
def _do_test_acquire_in_one_then_another( self, release=True, require_thread_acquired_to_finish=False, **thread_acquire_kwargs): from gevent import monkey self.assertFalse(monkey.is_module_patched('threading')) import threading thread_running = threading.Event() thread_acquired = threading.Event() sem = self._makeOne() # Make future acquires block sem.acquire() exc_info = [] acquired = [] t = threading.Thread(target=self._makeThreadMain( thread_running, thread_acquired, sem, acquired, exc_info, ** thread_acquire_kwargs)) t.daemon = True t.start() thread_running.wait(10) # implausibly large time if release: sem.release() # Spin the loop to be sure the release gets through. # (Release schedules the notifier to run, and when the # notifier run it sends the async notification to the # other thread. Depending on exactly where we are in the # event loop, and the limit to the number of callbacks # that get run (including time-based) the notifier may or # may not be immediately ready to run, so this can take up # to two iterations.) for _ in range(self.IDLE_ITERATIONS): gevent.idle() if thread_acquired.wait(timing.LARGE_TICK): break self.assertEqual(acquired, [True]) if not release and thread_acquire_kwargs.get("timeout"): # Spin the loop to be sure that the timeout has a chance to # process. Interleave this with something that drops the GIL # so the background thread has a chance to notice that. for _ in range(self.IDLE_ITERATIONS): gevent.idle() if thread_acquired.wait(timing.LARGE_TICK): break thread_acquired.wait(timing.LARGE_TICK * 5) if require_thread_acquired_to_finish: self.assertTrue(thread_acquired.is_set()) try: self.assertEqual(exc_info, []) finally: exc_info = None return sem, acquired
def is_gevent_monkey_patched(): try: from gevent import monkey except ImportError: return False else: return monkey.is_module_patched('os')
def __init__(self, threadpool): # Construct in the main thread (owner of the threadpool) # The parent greenlet and thread identifier will be set once the # new thread begins running. RawGreenlet.__init__(self) self._hub = threadpool.hub # Avoid doing any imports in the background thread if it's not # necessary (monkey.get_original imports if not patched). # Background imports can hang Python 2 (gevent's thread resolver runs in the BG, # and resolving may have to import the idna module, which needs an import lock, so # resolving at module scope) if monkey.is_module_patched('sys'): stderr = monkey.get_original('sys', 'stderr') else: stderr = sys.stderr self._stderr = stderr # We can capture the task_queue; even though it can change if the threadpool # is re-innitted, we won't be running in that case self._task_queue = threadpool.task_queue # type:gevent._threading.Queue self._task_queue_cookie = self._task_queue.allocate_cookie() self._unregister_worker = threadpool._unregister_worker threadpool._register_worker(self) try: start_new_thread(self._begin, ()) except: self._unregister_worker(self) raise
def is_gevent_monkey_patched(): try: from gevent import monkey except ImportError: return False else: return monkey.is_module_patched('__builtin__')
def test_future(self): self.assertEqual(monkey.is_module_patched('threading'), self.MONKEY_PATCHED) pool = self.pool calledback = [] def fn(): gevent.sleep(0.5) return 42 def callback(future): future.calledback += 1 raise greentest.ExpectedException("Expected, ignored") future = pool.submit(fn) # pylint:disable=no-member future.calledback = 0 future.add_done_callback(callback) self.assertRaises(self.FutureTimeoutError, future.result, timeout=0.001) def spawned(): return 2016 spawned_greenlet = gevent.spawn(spawned) # Whether or not we are monkey patched, the background # greenlet we spawned got to run while we waited. self.assertEqual(future.result(), 42) self.assertTrue(future.done()) self.assertFalse(future.cancelled()) # Make sure the notifier has a chance to run so the call back # gets called gevent.sleep() self.assertEqual(future.calledback, 1) self.assertTrue(spawned_greenlet.ready()) self.assertEqual(spawned_greenlet.value, 2016) # Adding the callback again runs immediately future.add_done_callback(lambda f: calledback.append(True)) self.assertEqual(calledback, [True]) # We can wait on the finished future done, _not_done = self.cf_wait((future, )) self.assertEqual(list(done), [future]) self.assertEqual(list(self.cf_as_completed((future, ))), [future]) # Doing so does not call the callback again self.assertEqual(future.calledback, 1) # even after a trip around the event loop gevent.sleep() self.assertEqual(future.calledback, 1) pool.kill() del future del pool del self.pool
def _condition(self): if monkey.is_module_patched('threading') or self.done(): import threading return threading.Condition() # We can only properly work with conditions # when we've been monkey-patched. This is necessary # for the wait/as_completed module functions. raise AttributeError("_condition")
def test_future(self): self.assertEqual(monkey.is_module_patched('threading'), self.MONKEY_PATCHED) pool = self.pool calledback = [] def fn(): gevent.sleep(0.5) return 42 def callback(future): future.calledback += 1 raise greentest.ExpectedException("Expected, ignored") future = pool.submit(fn) # pylint:disable=no-member future.calledback = 0 future.add_done_callback(callback) self.assertRaises(FutureTimeoutError, future.result, timeout=0.001) def spawned(): return 2016 spawned_greenlet = gevent.spawn(spawned) # Whether or not we are monkey patched, the background # greenlet we spawned got to run while we waited. self.assertEqual(future.result(), 42) self.assertTrue(future.done()) self.assertFalse(future.cancelled()) # Make sure the notifier has a chance to run so the call back # gets called gevent.sleep() self.assertEqual(future.calledback, 1) self.assertTrue(spawned_greenlet.ready()) self.assertEqual(spawned_greenlet.value, 2016) # Adding the callback again runs immediately future.add_done_callback(lambda f: calledback.append(True)) self.assertEqual(calledback, [True]) # We can wait on the finished future done, _not_done = cf_wait((future,)) self.assertEqual(list(done), [future]) self.assertEqual(list(cf_as_completed((future,))), [future]) # Doing so does not call the callback again self.assertEqual(future.calledback, 1) # even after a trip around the event loop gevent.sleep() self.assertEqual(future.calledback, 1) pool.kill() del future del pool del self.pool
def can_test_connect_with_gevent(): if os.environ.get('THREADING_BACKEND', 'stdlib') != 'gevent': return True try: from gevent import monkey except ImportError: return False return monkey.is_module_patched('socket')
def _condition(self): from gevent import monkey if monkey.is_module_patched('threading') or self.done(): import threading return threading.Condition() # We can only properly work with conditions # when we've been monkey-patched. This is necessary # for the wait/as_completed module functions. raise AttributeError("_condition")
def _sockets_gevent_monkey_patched(self): # Return whether the socket module has been monkey-patched # by gevent try: from gevent import monkey except ImportError: # pragma: no cover return False else: return monkey.is_module_patched('socket')
def init_app(self, app, **kwargs): super(SocketIOWithLogging, self).init_app(app, **kwargs) self.__logger = logging.getLogger(app.logger.name + ".socketio") if monkey.is_module_patched("os"): self.instance_name_prefix = "throat-socketio-instance-name-" self.instance_name = self.instance_name_prefix + "".join( [chr(random.randrange(0, 26) + ord("a")) for i in range(6)]) gevent.spawn(self.refresh_name_key) gevent.spawn(self.emit_messages)
def is_gevent_monkey_patched(self): try: from gevent import monkey except ImportError: return False else: if hasattr(monkey, "is_module_patched"): return monkey.is_module_patched("threading") else: return bool(monkey.saved)
def initialize(self, server): super(KombuManager, self).initialize(server) monkey_patched = True if server.async_mode == 'eventlet': from eventlet.patcher import is_monkey_patched monkey_patched = is_monkey_patched('socket') elif 'gevent' in server.async_mode: from gevent.monkey import is_module_patched monkey_patched = is_module_patched('socket') if not monkey_patched: raise RuntimeError('Redis requires a monkey patched socket ' 'library to work with ' + server.async_mode)
def _sockets_gevent_monkey_patched(self): # Return whether the socket module has been monkey-patched # by gevent try: from gevent import monkey except ImportError: # pragma: no cover return False else: # some versions of gevent have a bug where if we're monkey-patched # on the command line using python -m gevent.monkey /path/to/testrunner ... # it doesn't report being monkey-patched. import socket return monkey.is_module_patched('socket') or 'gevent' in repr(socket.socket)
def test_acquire_in_one_wait_greenlet_wait_thread_gives_up(self): # The waiter in the thread both arrives and gives up while # the notifier is already running...or at least, that's what # we'd like to arrange, but the _notify_links function doesn't # drop the GIL/object lock, so the other thread is stuck and doesn't # actually get to call into the acquire method. from gevent import monkey self.assertFalse(monkey.is_module_patched('threading')) import threading sem = self._makeOne() # Make future acquires block sem.acquire() def greenlet_one(): ack = sem.acquire() # We're running in the notifier function right now. It switched to # us. thread.start() gevent.sleep(timing.LARGE_TICK) return ack exc_info = [] acquired = [] glet = gevent.spawn(greenlet_one) thread = threading.Thread( target=self._makeThreadMain(threading.Event(), threading.Event(), sem, acquired, exc_info, timeout=timing.LARGE_TICK)) thread.daemon = True gevent.idle() sem.release() glet.join() for _ in range(3): gevent.idle() thread.join(timing.LARGE_TICK) self.assertEqual(glet.value, True) self.assertEqual([], exc_info) self.assertEqual([False], acquired) self.assertTrue(glet.dead, glet) glet = None
def _do_test_acquire_in_one_then_another(self, release=True, **thread_acquire_kwargs): from gevent import monkey self.assertFalse(monkey.is_module_patched('threading')) import threading thread_running = threading.Event() thread_acquired = threading.Event() sem = self._makeOne() # Make future acquires block sem.acquire() exc_info = [] acquired = [] t = threading.Thread(target=self._makeThreadMain( thread_running, thread_acquired, sem, acquired, exc_info, ** thread_acquire_kwargs)) t.start() thread_running.wait(10) # implausibly large time if release: sem.release() # Spin the loop to be sure the release gets through. # (Release schedules the notifier to run, and when the # notifier run it sends the async notification to the # other thread. Depending on exactly where we are in the # event loop, and the limit to the number of callbacks # that get run (including time-based) the notifier may or # may not be immediately ready to run, so this can take up # to two iterations.) for _ in range(3): gevent.idle() if thread_acquired.wait(timing.LARGE_TICK): break self.assertEqual(acquired, [True]) thread_acquired.wait(timing.LARGE_TICK * 5) try: self.assertEqual(exc_info, []) finally: exc_info = None return sem, acquired
def start(): flag = False try: from gevent import monkey flag = monkey.is_module_patched("socket") except ModuleNotFoundError: logger.debug( "it was found that no gevent was used, if you don't use, please ignore." ) if flag: import grpc.experimental.gevent as grpc_gevent grpc_gevent.init_gevent() global __started if __started: raise RuntimeError('the agent can only be started once') loggings.init() config.finalize() __started = True __init() __heartbeat_thread.start() __report_thread.start() atexit.register(__fini)
raise self.target = errors_are_fatal class GreenletThread(BaseThread): def __init__(self, target=None, args=()): BaseThread.__init__(self, target) self.glet = gevent.spawn(self.target, *args) def join(self, *args, **kwargs): return self.glet.join(*args, **kwargs) def is_alive(self): return not self.glet.ready() if not monkey.is_module_patched('threading'): class ThreadThread(BaseThread, _Thread): def __init__(self, **kwargs): target = kwargs.pop('target') BaseThread.__init__(self, target) _Thread.__init__(self, target=self.target, **kwargs) self.start() Thread = ThreadThread else: Thread = GreenletThread class TestTCP(greentest.TestCase): __timeout__ = None TIMEOUT_ERROR = socket.timeout long_data = ", ".join([str(x) for x in range(20000)]) if not isinstance(long_data, bytes):
#! /usr/bin/env python3 """ counterblockd server """ # import before importing other modules import gevent from gevent import monkey # now, import grequests as it will perform monkey_patch_all() # letting grequests do it avoids us double monkey patching... (ugh...) import grequests # this will monkey patch if not monkey.is_module_patched("os"): # if this fails, it's because gevent stopped monkey patching for us monkey.patch_all() # disable urllib3 warnings for requests module (for now at least) # (note that requests is used/imported through grequests only) import requests from requests.packages.urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings(InsecureRequestWarning) import sys import os import argparse import json import logging import datetime import time import tempfile from counterblock.lib import config, config_util, log, blockfeed, util, module, database from counterblock.lib.processor import messages, caughtup, startup # to kick off processors from counterblock.lib.processor import StartUpProcessor
#!/usr/bin/env python3 # -*- coding: UTF-8 -*- from gevent import monkey from flask.wrappers import Response if not monkey.is_module_patched('__builtin__'): monkey.patch_all() # causes problems if done twice from flask import Flask, render_template, request, Blueprint, Response from flask_sockets import Sockets import requests import os, time, sys import psutil, subprocess import threading import socket from websocket import create_connection, WebSocket, ABNF from geventwebsocket.websocket import WebSocket as gWebSocket FlexxBlueprint = Blueprint('FlexxApps', __name__, static_folder='static') app = FlexxBlueprint FlexxWS = Blueprint('FlexxWS', __name__) sockets = FlexxWS SITE_NAME = '127.0.0.1:8081/' RUN_COMMAND = r'python.exe FlexxMain.py' RUN_CWD = os.path.dirname(__file__) if not RUN_CWD: RUN_CWD = None # for subprocess.Popen FIND_STRING = 'FlexxMain.py' STARTUPINFO = subprocess.STARTUPINFO() STARTUPINFO.dwFlags = subprocess.STARTF_USESHOWWINDOW STARTUPINFO.wShowWindow = 6 # SW_HIDE=0, SW_MINIMIZE=6
from gevent import monkey from segment_source_resource.sync import execute if monkey.is_module_patched('os') == False: monkey.patch_all() class MockResource(object): def __init__(self): self.parent = "parent" def fetch(self): return [MockResouce()] def test_execute(): resource = MockResource() execute([resource, resource])
import thread except ImportError: import _thread as thread import threading assert 'built-in' not in repr(thread.start_new_thread), repr( thread.start_new_thread) assert 'built-in' not in repr(threading._start_new_thread), repr( threading._start_new_thread) if sys.version_info[0] == 2: assert 'built-in' not in repr(threading._sleep), repr(threading._sleep) import socket from gevent import socket as gevent_socket assert socket.create_connection is gevent_socket.create_connection import os if hasattr(os, 'fork'): assert 'built-in' not in repr(os.fork), repr(os.fork) assert monkey.saved assert not monkey.is_object_patched('threading', 'Event') monkey.patch_thread(Event=True) assert monkey.is_object_patched('threading', 'Event') for modname in monkey.saved: assert monkey.is_module_patched(modname) for objname in monkey.saved[modname]: assert monkey.is_object_patched(modname, objname)
#! /usr/bin/env python3 """ counterblockd server """ # import before importing other modules import gevent from gevent import monkey # now, import grequests as it will perform monkey_patch_all() # letting grequests do it avoids us double monkey patching... (ugh...) import grequests # this will monkey patch if not monkey.is_module_patched( "os" ): # if this fails, it's because gevent stopped monkey patching for us monkey.patch_all() # disable urllib3 warnings for requests module (for now at least) # (note that requests is used/imported through grequests only) import requests from requests.packages.urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings(InsecureRequestWarning) import sys import os import argparse import json import logging import datetime import time import tempfile from counterblock.lib import config, config_util, log, blockfeed, util, module, database
""" From here we start the app for a production environment. """ from gevent import monkey # If the gunicorn worker is one of the gevent ones, it will have # already done monkey patching, so use that to determine what kind of # worker is running. if monkey.is_module_patched("os"): # Monkey patch the Postgres driver. from psycogreen.gevent import patch_psycopg patch_psycopg() else: # Prefer noisy failure to silent misbehavior, so do our own monkey # patching of gevent calls to raise errors. import app.misc import app.storage import app.tasks class GeventPatch: def sleep(self): raise RuntimeError( "Load balancer misconfiguration: gevent.sleep called from a sync worker" ) def spawn(self, fn, *args): raise RuntimeError( "Load balancer misconfiguration: gevent.spawn called from a sync worker" )
import types for name in ('fork', 'forkpty'): if hasattr(os, name): attr = getattr(os, name) assert 'built-in' not in repr(attr), repr(attr) assert not isinstance(attr, types.BuiltinFunctionType), repr(attr) assert isinstance(attr, types.FunctionType), repr(attr) assert monkey.saved assert not monkey.is_object_patched('threading', 'Event') monkey.patch_thread(Event=True) assert monkey.is_object_patched('threading', 'Event') for modname in monkey.saved: assert monkey.is_module_patched(modname) for objname in monkey.saved[modname]: assert monkey.is_object_patched(modname, objname) orig_saved = {} for k, v in monkey.saved.items(): orig_saved[k] = v.copy() import warnings with warnings.catch_warnings(record=True) as issued_warnings: # Patch again, triggering two warnings, on for os=False/signal=True, # one for repeated monkey-patching. monkey.patch_all(os=False) assert len(issued_warnings) == 2, len(issued_warnings) assert 'SIGCHLD' in str(issued_warnings[-1].message), issued_warnings[-1]
def patch_all(): for module in sys.modules: if monkey.is_module_patched(module.__name__): _patch_no_check(module)
def test_dueling_threads(self, acquire_args=(), create_hub=None): # pylint:disable=too-many-locals,too-many-statements # Threads doing nothing but acquiring and releasing locks, without # having any other greenlets to switch to. # https://github.com/gevent/gevent/issues/1698 from gevent import monkey from gevent._hub_local import get_hub_if_exists self.assertFalse(monkey.is_module_patched('threading')) import threading from time import sleep as native_sleep sem = self._makeOne() self.assertOneHasNoHub(sem) count = 10000 results = [-1, -1] run = True def do_it(ix): if create_hub: gevent.get_hub() try: for i in range(count): if not run: break sem.acquire(*acquire_args) sem.release() results[ix] = i if not create_hub: # We don't artificially create the hub. self.assertIsNone(get_hub_if_exists(), (get_hub_if_exists(), ix, i)) if create_hub and i % 10 == 0: gevent.sleep(timing.SMALLEST_RELIABLE_DELAY) elif i % 100 == 0: native_sleep(timing.SMALLEST_RELIABLE_DELAY) except Exception as ex: # pylint:disable=broad-except import traceback traceback.print_exc() results[ix] = str(ex) ex = None finally: hub = get_hub_if_exists() if hub is not None: hub.join() hub.destroy(destroy_loop=True) t1 = threading.Thread(target=do_it, args=(0, )) t1.daemon = True t2 = threading.Thread(target=do_it, args=(1, )) t2.daemon = True t1.start() t2.start() t1.join(1) t2.join(1) while t1.is_alive() or t2.is_alive(): cur = list(results) t1.join(7) t2.join(7) if cur == results: # Hmm, after two seconds, no progress run = False break self.assertEqual(results, [count - 1, count - 1])