def test_detect_environment_eventlet(self): with patch('eventlet.patcher.is_monkey_patched', create=True) as m: assert sys.modules['eventlet'] m.return_value = True env = compat._detect_environment() m.assert_called_with(socket) assert env == 'eventlet'
def _open(self, scheme='mongodb://'): hostname, dbname, options = self._parse_uri(scheme=scheme) conf = self._prepare_client_options(options) conf['host'] = hostname env = _detect_environment() if env == 'gevent': from gevent import monkey monkey.patch_all() elif env == 'eventlet': from eventlet import monkey_patch monkey_patch() mongoconn = MongoClient(**conf) database = mongoconn[dbname] version_str = mongoconn.server_info()['version'] version = tuple(map(int, version_str.split('.'))) if version < (1, 3): raise VersionMismatch(E_SERVER_VERSION.format(version_str)) elif self.ttl and version < (2, 2): raise VersionMismatch(E_NO_TTL_INDEXES.format(version_str)) return database
def __init__(self, on_task_request, init_callback=noop, hostname=None, pool=None, app=None, timer=None, controller=None, hub=None, amqheartbeat=None, worker_options=None, disable_rate_limits=False, initial_prefetch_count=2, prefetch_multiplier=1, **kwargs): self.app = app self.controller = controller self.init_callback = init_callback self.hostname = hostname or gethostname() self.pid = os.getpid() self.pool = pool self.timer = timer self.strategies = self.Strategies() self.conninfo = self.app.connection_for_read() self.connection_errors = self.conninfo.connection_errors self.channel_errors = self.conninfo.channel_errors self._restart_state = restart_state(maxR=5, maxT=1) self._does_info = logger.isEnabledFor(logging.INFO) self._limit_order = 0 self.on_task_request = on_task_request self.on_task_message = set() self.amqheartbeat_rate = self.app.conf.broker_heartbeat_checkrate self.disable_rate_limits = disable_rate_limits self.initial_prefetch_count = initial_prefetch_count self.prefetch_multiplier = prefetch_multiplier # this contains a tokenbucket for each task type by name, used for # rate limits, or None if rate limits are disabled for that task. self.task_buckets = defaultdict(lambda: None) self.reset_rate_limits() self.gevent_env = _detect_environment() == 'gevent' self.hub = hub if self.hub or getattr(self.pool, 'is_green', False) or self.gevent_env: self.amqheartbeat = amqheartbeat if self.amqheartbeat is None: self.amqheartbeat = self.app.conf.broker_heartbeat else: self.amqheartbeat = 0 if not hasattr(self, 'loop'): self.loop = loops.asynloop if hub else loops.synloop if self.gevent_env: # there's a gevent bug that causes timeouts to not be reset, # so if the connection timeout is exceeded once, it can NEVER # connect again. self.app.conf.broker_connection_timeout = None self._pending_operations = [] self.steps = [] self.blueprint = self.Blueprint( steps=self.app.steps['consumer'], on_close=self.on_close, ) self.blueprint.apply(self, **dict(worker_options or {}, **kwargs))
def test_detect_environment_gevent(self): with patch('gevent.socket', create=True) as m: prev, socket.socket = socket.socket, m.socket try: assert sys.modules['gevent'] env = compat._detect_environment() assert env == 'gevent' finally: socket.socket = prev
def supports_librabbitmq(): """Return true if :pypi:`librabbitmq` can be used.""" if _detect_environment() == 'default': try: import librabbitmq # noqa except ImportError: # pragma: no cover pass else: # pragma: no cover return True
def restore_visible(self, start=0, num=10, interval=10): self._vrestore_count += 1 if (self._vrestore_count - 1) % interval: return with self.channel.conn_or_acquire() as client: ceil = time() - self.visibility_timeout try: with Mutex(client, self.unacked_mutex_key, self.unacked_mutex_expire): env = _detect_environment() if env == 'gevent': ceil = time() visible = client.zrevrangebyscore(self.unacked_index_key, ceil, 0, start=num and start, num=num, withscores=True) for tag, score in visible or []: self.restore_by_tag(tag, client) except MutexHeld: pass
def test_detect_environment_no_eventlet_or_gevent(self): try: sys.modules['eventlet'] = types.ModuleType( bytes_if_py2('eventlet')) sys.modules['eventlet.patcher'] = types.ModuleType( bytes_if_py2('patcher')) assert compat._detect_environment() == 'default' finally: sys.modules.pop('eventlet.patcher', None) sys.modules.pop('eventlet', None) compat._detect_environment() try: sys.modules['gevent'] = types.ModuleType(bytes_if_py2('gevent')) assert compat._detect_environment() == 'default' finally: sys.modules.pop('gevent', None) compat._detect_environment()
def __init__(self, on_task_request, init_callback=noop, hostname=None, pool=None, app=None, timer=None, controller=None, hub=None, amqheartbeat=None, worker_options=None, disable_rate_limits=False, initial_prefetch_count=2, prefetch_multiplier=1, **kwargs): self.app = app self.controller = controller self.init_callback = init_callback self.hostname = hostname or gethostname() self.pid = os.getpid() self.pool = pool self.timer = timer self.strategies = self.Strategies() self.conninfo = self.app.connection_for_read() self.connection_errors = self.conninfo.connection_errors self.channel_errors = self.conninfo.channel_errors self._restart_state = restart_state(maxR=5, maxT=1) self._does_info = logger.isEnabledFor(logging.INFO) self._limit_order = 0 self.on_task_request = on_task_request self.on_task_message = set() self.amqheartbeat_rate = self.app.conf.broker_heartbeat_checkrate self.disable_rate_limits = disable_rate_limits self.initial_prefetch_count = initial_prefetch_count self.prefetch_multiplier = prefetch_multiplier # this contains a tokenbucket for each task type by name, used for # rate limits, or None if rate limits are disabled for that task. self.task_buckets = defaultdict(lambda: None) self.reset_rate_limits() self.hub = hub if self.hub or getattr(self.pool, 'is_green', False): self.amqheartbeat = amqheartbeat if self.amqheartbeat is None: self.amqheartbeat = self.app.conf.broker_heartbeat else: self.amqheartbeat = 0 # 确定循环 if not hasattr(self, 'loop'): self.loop = loops.asynloop if hub else loops.synloop if _detect_environment() == 'gevent': # there's a gevent bug that causes timeouts to not be reset, # so if the connection timeout is exceeded once, it can NEVER # connect again. self.app.conf.broker_connection_timeout = None self._pending_operations = [] self.steps = [] self.blueprint = self.Blueprint( steps=self.app.steps['consumer'], on_close=self.on_close, ) self.blueprint.apply(self, **dict(worker_options or {}, **kwargs))