def _open(self, scheme='mongodb://'): hostname, dbname, options = self._parse_uri(scheme=scheme) conf = self._prepare_client_options(options) conf['host'] = hostname env = _detect_environment() if env == 'gevent': from gevent import monkey monkey.patch_all() elif env == 'eventlet': from eventlet import monkey_patch monkey_patch() mongoconn = MongoClient(**conf) database = mongoconn[dbname] version = mongoconn.server_info()['version'] if tuple(map(int, version.split('.')[:2])) < (1, 3): raise NotImplementedError( 'Kombu requires MongoDB version 1.3+ (server is {0})'.format( version)) self._create_broadcast(database, options) self._client = database
def _open(self, scheme='mongodb://'): hostname, dbname, options = self._parse_uri(scheme=scheme) mongoconn = MongoClient( host=hostname, ssl=options['ssl'], auto_start_request=options['auto_start_request'], connectTimeoutMS=options['connectTimeoutMS'], use_greenlets=_detect_environment() != 'default', ) database = getattr(mongoconn, dbname) version = mongoconn.server_info()['version'] if tuple(map(int, version.split('.')[:2])) < (1, 3): raise NotImplementedError( 'Kombu requires MongoDB version 1.3+ (server is {0})'.format( version)) self.db = database col = database.messages col.ensure_index([('queue', 1), ('_id', 1)], background=True) if 'messages.broadcast' not in database.collection_names(): capsize = options.get('capped_queue_size') or 100000 database.create_collection('messages.broadcast', size=capsize, capped=True) self.bcast = getattr(database, 'messages.broadcast') self.bcast.ensure_index([('queue', 1)]) self.routing = getattr(database, 'messages.routing') self.routing.ensure_index([('queue', 1), ('exchange', 1)]) return database
def _open(self, scheme='mongodb://'): hostname, dbname, options = self._parse_uri(scheme=scheme) conf = self._prepare_client_options(options) conf['host'] = hostname env = _detect_environment() if env == 'gevent': from gevent import monkey monkey.patch_all() elif env == 'eventlet': from eventlet import monkey_patch monkey_patch() mongoconn = MongoClient(**conf) database = mongoconn[dbname] version_str = mongoconn.server_info()['version'] version = tuple(map(int, version_str.split('.'))) if version < (1, 3): raise VersionMismatch(E_SERVER_VERSION.format(version_str)) elif self.ttl and version < (2, 2): raise VersionMismatch(E_NO_TTL_INDEXES.format(version_str)) return database
def test_detect_environment_eventlet(self): with patch('eventlet.patcher.is_monkey_patched', create=True) as m: self.assertTrue(sys.modules['eventlet']) m.return_value = True env = syn._detect_environment() m.assert_called_with(socket) self.assertEqual(env, 'eventlet')
def _open(self): """ See mongodb uri documentation: http://www.mongodb.org/display/DOCS/Connections """ client = self.connection.client options = client.transport_options hostname = client.hostname or DEFAULT_HOST authdb = dbname = client.virtual_host if dbname in ['/', None]: dbname = "kombu_default" authdb = "admin" if not hostname.startswith('mongodb://'): hostname = 'mongodb://' + hostname if not hostname[10:]: hostname = hostname + 'localhost' if '/' in hostname[10:]: if not client.userid: hostname = hostname.replace('/' + client.virtual_host, '/') else: hostname = hostname.replace('/' + client.virtual_host, '/' + authdb) # At this point we expect the hostname to be something like # (considering replica set form too): # # mongodb://[username:password@]host1[:port1][,host2[:port2], # ...[,hostN[:portN]]][/[?options]] options.setdefault('auto_start_request', True) mongoconn = MongoClient( host=hostname, ssl=client.ssl, auto_start_request=options['auto_start_request'], use_greenlets=_detect_environment() != 'default', ) database = getattr(mongoconn, dbname) version = mongoconn.server_info()['version'] if tuple(map(int, version.split('.')[:2])) < (1, 3): raise NotImplementedError( 'Kombu requires MongoDB version 1.3+ (server is {0})'.format( version)) self.db = database col = database.messages col.ensure_index([('queue', 1), ('_id', 1)], background=True) if 'messages.broadcast' not in database.collection_names(): capsize = options.get('capped_queue_size') or 100000 database.create_collection('messages.broadcast', size=capsize, capped=True) self.bcast = getattr(database, 'messages.broadcast') self.bcast.ensure_index([('queue', 1)]) self.routing = getattr(database, 'messages.routing') self.routing.ensure_index([('queue', 1), ('exchange', 1)]) return database
def _kombu_mongo_open(self, scheme='mongodb://'): hostname, dbname, options = self._parse_uri(scheme=scheme) options.pop('auto_start_request', None) env = _detect_environment() if env == 'gevent': from gevent import monkey monkey.patch_all() elif env == 'eventlet': from eventlet import monkey_patch monkey_patch() mongoconn = MongoClient( host=hostname, ssl=options['ssl'], connectTimeoutMS=options['connectTimeoutMS'], ) database = mongoconn[dbname] version = mongoconn.server_info()['version'] if tuple(map(int, version.split('.')[:2])) < (1, 3): raise NotImplementedError( 'Kombu requires MongoDB version 1.3+ (server is {0})'.format( version)) self._create_broadcast(database, options) self._client = database
def __init__(self, handle_task, init_callback=noop, hostname=None, pool=None, app=None, timer=None, controller=None, hub=None, amqheartbeat=None, worker_options=None, disable_rate_limits=False, **kwargs): self.app = app_or_default(app) self.controller = controller self.init_callback = init_callback self.hostname = hostname or socket.gethostname() self.pool = pool self.timer = timer or default_timer self.strategies = {} conninfo = self.app.connection() self.connection_errors = conninfo.connection_errors self.channel_errors = conninfo.channel_errors self._restart_state = restart_state(maxR=5, maxT=1) self._does_info = logger.isEnabledFor(logging.INFO) self.handle_task = handle_task self.amqheartbeat_rate = self.app.conf.BROKER_HEARTBEAT_CHECKRATE self.disable_rate_limits = disable_rate_limits # this contains a tokenbucket for each task type by name, used for # rate limits, or None if rate limits are disabled for that task. self.task_buckets = defaultdict(lambda: None) self.reset_rate_limits() if hub: self.amqheartbeat = amqheartbeat if self.amqheartbeat is None: self.amqheartbeat = self.app.conf.BROKER_HEARTBEAT self.hub = hub self.hub.on_init.append(self.on_poll_init) else: self.hub = None self.amqheartbeat = 0 if not hasattr(self, 'loop'): self.loop = loops.asynloop if hub else loops.synloop if _detect_environment() == 'gevent': # there's a gevent bug that causes timeouts to not be reset, # so if the connection timeout is exceeded once, it can NEVER # connect again. self.app.conf.BROKER_CONNECTION_TIMEOUT = None self.steps = [] self.blueprint = self.Blueprint( app=self.app, on_close=self.on_close, ) self.blueprint.apply(self, **dict(worker_options or {}, **kwargs))
def supports_librabbitmq(): if _detect_environment() == 'default': try: import librabbitmq # noqa return True except ImportError: pass return False
def supports_librabbitmq(): if _detect_environment() == 'default': try: import librabbitmq # noqa except ImportError: # pragma: no cover pass else: # pragma: no cover return True
def test_detect_environment_gevent(self): with patch('gevent.socket', create=True) as m: prev, socket.socket = socket.socket, m.socket try: self.assertTrue(sys.modules['gevent']) env = syn._detect_environment() self.assertEqual(env, 'gevent') finally: socket.socket = prev
def test_detect_environment_gevent(self): with patch('gevent.socket', create=True) as m: prev, socket.socket = socket.socket, m.socket try: assert sys.modules['gevent'] env = syn._detect_environment() assert env == 'gevent' finally: socket.socket = prev
def _open(self, scheme='mongodb://'): # See mongodb uri documentation: # http://www.mongodb.org/display/DOCS/Connections client = self.connection.client options = client.transport_options hostname = client.hostname or DEFAULT_HOST dbname = client.virtual_host if dbname in ['/', None]: dbname = "kombu_default" if not hostname.startswith(scheme): hostname = scheme + hostname if not hostname[len(scheme):]: hostname += 'localhost' # XXX What does this do? [ask] urest = hostname[len(scheme):] if '/' in urest: if not client.userid: urest = urest.replace('/' + client.virtual_host, '/') hostname = ''.join([scheme, urest]) # At this point we expect the hostname to be something like # (considering replica set form too): # # mongodb://[username:password@]host1[:port1][,host2[:port2], # ...[,hostN[:portN]]][/[?options]] options.setdefault('auto_start_request', True) mongoconn = MongoClient( host=hostname, ssl=client.ssl, auto_start_request=options['auto_start_request'], use_greenlets=_detect_environment() != 'default', ) database = getattr(mongoconn, dbname) version = mongoconn.server_info()['version'] if tuple(map(int, version.split('.')[:2])) < (1, 3): raise NotImplementedError( 'Kombu requires MongoDB version 1.3+ (server is {0})'.format( version)) self.db = database col = database.messages col.ensure_index([('queue', 1), ('_id', 1)], background=True) if 'messages.broadcast' not in database.collection_names(): capsize = options.get('capped_queue_size') or 100000 database.create_collection('messages.broadcast', size=capsize, capped=True) self.bcast = getattr(database, 'messages.broadcast') self.bcast.ensure_index([('queue', 1)]) self.routing = getattr(database, 'messages.routing') self.routing.ensure_index([('queue', 1), ('exchange', 1)]) return database
def __init__(self, on_task_request, init_callback=noop, hostname=None, pool=None, app=None, timer=None, controller=None, hub=None, amqheartbeat=None, worker_options=None, disable_rate_limits=False, initial_prefetch_count=2, prefetch_multiplier=1, **kwargs): self.app = app self.controller = controller self.init_callback = init_callback self.hostname = hostname or gethostname() self.pid = os.getpid() self.pool = pool self.timer = timer self.strategies = self.Strategies() self.conninfo = self.app.connection_for_read() self.connection_errors = self.conninfo.connection_errors self.channel_errors = self.conninfo.channel_errors self._restart_state = restart_state(maxR=5, maxT=1) self._does_info = logger.isEnabledFor(logging.INFO) self._limit_order = 0 self.on_task_request = on_task_request self.on_task_message = set() self.amqheartbeat_rate = self.app.conf.broker_heartbeat_checkrate self.disable_rate_limits = disable_rate_limits self.initial_prefetch_count = initial_prefetch_count self.prefetch_multiplier = prefetch_multiplier # this contains a tokenbucket for each task type by name, used for # rate limits, or None if rate limits are disabled for that task. self.task_buckets = defaultdict(lambda: None) self.reset_rate_limits() self.hub = hub if self.hub: self.amqheartbeat = amqheartbeat if self.amqheartbeat is None: self.amqheartbeat = self.app.conf.broker_heartbeat else: self.amqheartbeat = 0 if not hasattr(self, 'loop'): self.loop = loops.asynloop if hub else loops.synloop if _detect_environment() == 'gevent': # there's a gevent bug that causes timeouts to not be reset, # so if the connection timeout is exceeded once, it can NEVER # connect again. self.app.conf.broker_connection_timeout = None self._pending_operations = [] self.steps = [] self.blueprint = self.Blueprint( app=self.app, on_close=self.on_close, ) self.blueprint.apply(self, **dict(worker_options or {}, **kwargs))
def __init__(self, ready_queue, init_callback=noop, send_events=False, hostname=None, initial_prefetch_count=2, pool=None, app=None, timer=None, controller=None, hub=None, amqheartbeat=None, **kwargs): self.app = app_or_default(app) self.connection = None self.task_consumer = None self.controller = controller self.broadcast_consumer = None self.ready_queue = ready_queue self.send_events = send_events self.init_callback = init_callback self.hostname = hostname or socket.gethostname() self.initial_prefetch_count = initial_prefetch_count self.event_dispatcher = None self.heart = None self.pool = pool self.timer = timer or timer2.default_timer pidbox_state = AttributeDict( app=self.app, hostname=self.hostname, listener=self, # pre 2.2 consumer=self) self.pidbox_node = self.app.control.mailbox.Node(self.hostname, state=pidbox_state, handlers=Panel.data) conninfo = self.app.connection() self.connection_errors = conninfo.connection_errors self.channel_errors = conninfo.channel_errors self._does_info = logger.isEnabledFor(logging.INFO) self.strategies = {} if hub: hub.on_init.append(self.on_poll_init) self.hub = hub self._quick_put = self.ready_queue.put self.amqheartbeat = amqheartbeat if self.amqheartbeat is None: self.amqheartbeat = self.app.conf.BROKER_HEARTBEAT if not hub: self.amqheartbeat = 0 if _detect_environment() == 'gevent': # there's a gevent bug that causes timeouts to not be reset, # so if the connection timeout is exceeded once, it can NEVER # connect again. self.app.conf.BROKER_CONNECTION_TIMEOUT = None
def __init__(self, ready_queue, init_callback=noop, hostname=None, pool=None, app=None, timer=None, controller=None, hub=None, amqheartbeat=None, worker_options=None, **kwargs): self.app = app_or_default(app) self.controller = controller self.ready_queue = ready_queue self.init_callback = init_callback self.hostname = hostname or socket.gethostname() self.pool = pool self.timer = timer or default_timer self.strategies = {} conninfo = self.app.connection() self.connection_errors = conninfo.connection_errors self.channel_errors = conninfo.channel_errors self._restart_state = restart_state(maxR=5, maxT=1) self._does_info = logger.isEnabledFor(logging.INFO) self._quick_put = self.ready_queue.put self.amqheartbeat_rate = self.app.conf.BROKER_HEARTBEAT_CHECKRATE if hub: self.amqheartbeat = amqheartbeat if self.amqheartbeat is None: self.amqheartbeat = self.app.conf.BROKER_HEARTBEAT self.hub = hub self.hub.on_init.append(self.on_poll_init) else: self.hub = None self.amqheartbeat = 0 if not hasattr(self, 'loop'): self.loop = loops.asynloop if hub else loops.synloop if _detect_environment() == 'gevent': # there's a gevent bug that causes timeouts to not be reset, # so if the connection timeout is exceeded once, it can NEVER # connect again. self.app.conf.BROKER_CONNECTION_TIMEOUT = None self.steps = [] self.namespace = self.Namespace( app=self.app, on_close=self.on_close, ) self.namespace.apply(self, **dict(worker_options or {}, **kwargs))
def __init__(self, on_task, init_callback=noop, hostname=None, pool=None, app=None, timer=None, controller=None, hub=None, amqheartbeat=None, worker_options=None, disable_rate_limits=False, **kwargs): self.app = app self.controller = controller self.init_callback = init_callback self.hostname = hostname or socket.gethostname() self.pid = os.getpid() self.pool = pool self.timer = timer self.strategies = self.Strategies() conninfo = self.app.connection() self.connection_errors = conninfo.connection_errors self.channel_errors = conninfo.channel_errors self._restart_state = restart_state(maxR=5, maxT=1) self._does_info = logger.isEnabledFor(logging.INFO) self.on_task = on_task self.amqheartbeat_rate = self.app.conf.BROKER_HEARTBEAT_CHECKRATE self.disable_rate_limits = disable_rate_limits # this contains a tokenbucket for each task type by name, used for # rate limits, or None if rate limits are disabled for that task. self.task_buckets = defaultdict(lambda: None) self.reset_rate_limits() if hub: self.amqheartbeat = amqheartbeat if self.amqheartbeat is None: self.amqheartbeat = self.app.conf.BROKER_HEARTBEAT self.hub = hub self.hub.on_init.append(self.on_poll_init) else: self.hub = None self.amqheartbeat = 0 if not hasattr(self, 'loop'): self.loop = loops.asynloop if hub else loops.synloop if _detect_environment() == 'gevent': # there's a gevent bug that causes timeouts to not be reset, # so if the connection timeout is exceeded once, it can NEVER # connect again. self.app.conf.BROKER_CONNECTION_TIMEOUT = None self.steps = [] self.blueprint = self.Blueprint( app=self.app, on_close=self.on_close, ) self.blueprint.apply(self, **dict(worker_options or {}, **kwargs))
def __init__( self, ready_queue, init_callback=noop, send_events=False, hostname=None, initial_prefetch_count=2, pool=None, app=None, timer=None, controller=None, hub=None, amqheartbeat=None, **kwargs ): self.app = app_or_default(app) self.connection = None self.task_consumer = None self.controller = controller self.broadcast_consumer = None self.ready_queue = ready_queue self.send_events = send_events self.init_callback = init_callback self.hostname = hostname or socket.gethostname() self.initial_prefetch_count = initial_prefetch_count self.event_dispatcher = None self.heart = None self.pool = pool self.timer = timer or timer2.default_timer pidbox_state = AttributeDict(app=self.app, hostname=self.hostname, listener=self, consumer=self) # pre 2.2 self.pidbox_node = self.app.control.mailbox.Node(self.hostname, state=pidbox_state, handlers=Panel.data) conninfo = self.app.connection() self.connection_errors = conninfo.connection_errors self.channel_errors = conninfo.channel_errors self._does_info = logger.isEnabledFor(logging.INFO) self.strategies = {} if hub: hub.on_init.append(self.on_poll_init) self.hub = hub self._quick_put = self.ready_queue.put self.amqheartbeat = amqheartbeat if self.amqheartbeat is None: self.amqheartbeat = self.app.conf.BROKER_HEARTBEAT if not hub: self.amqheartbeat = 0 if _detect_environment() == "gevent": # there's a gevent bug that causes timeouts to not be reset, # so if the connection timeout is exceeded once, it can NEVER # connect again. self.app.conf.BROKER_CONNECTION_TIMEOUT = None
def __init__( self, ready_queue, init_callback=noop, hostname=None, pool=None, app=None, timer=None, controller=None, hub=None, amqheartbeat=None, worker_options=None, **kwargs ): self.app = app_or_default(app) self.controller = controller self.ready_queue = ready_queue self.init_callback = init_callback self.hostname = hostname or socket.gethostname() self.pool = pool self.timer = timer or default_timer self.strategies = {} conninfo = self.app.connection() self.connection_errors = conninfo.connection_errors self.channel_errors = conninfo.channel_errors self._does_info = logger.isEnabledFor(logging.INFO) self._quick_put = self.ready_queue.put if hub: self.amqheartbeat = amqheartbeat if self.amqheartbeat is None: self.amqheartbeat = self.app.conf.BROKER_HEARTBEAT self.hub = hub self.hub.on_init.append(self.on_poll_init) else: self.hub = None self.amqheartbeat = 0 if not hasattr(self, "loop"): self.loop = loops.asynloop if hub else loops.synloop if _detect_environment() == "gevent": # there's a gevent bug that causes timeouts to not be reset, # so if the connection timeout is exceeded once, it can NEVER # connect again. self.app.conf.BROKER_CONNECTION_TIMEOUT = None self.steps = [] self.namespace = self.Namespace(app=self.app, on_start=self.on_start, on_close=self.on_close) self.namespace.apply(self, **worker_options or {})
def test_detect_environment_no_eventlet_or_gevent(self): try: sys.modules['eventlet'] = types.ModuleType('eventlet') sys.modules['eventlet.patcher'] = types.ModuleType('eventlet') self.assertEqual(syn._detect_environment(), 'default') finally: sys.modules.pop('eventlet', None) syn._detect_environment() try: sys.modules['gevent'] = types.ModuleType('gevent') self.assertEqual(syn._detect_environment(), 'default') finally: sys.modules.pop('gevent', None) syn._detect_environment()
def _open(self, scheme='mongodb://'): hostname, dbname, options = self._parse_uri(scheme=scheme) mongoconn = MongoClient( host=hostname, ssl=options['ssl'], auto_start_request=options['auto_start_request'], connectTimeoutMS=options['connectTimeoutMS'], use_greenlets=_detect_environment() != 'default', ) database = mongoconn[dbname] version = mongoconn.server_info()['version'] if tuple(map(int, version.split('.')[:2])) < (1, 3): raise NotImplementedError( 'Kombu requires MongoDB version 1.3+ (server is {0})'.format( version)) self._create_broadcast(database, options) self._client = database
def test_detect_environment_no_eventlet_or_gevent(self): try: sys.modules['eventlet'] = types.ModuleType( bytes_if_py2('eventlet')) sys.modules['eventlet.patcher'] = types.ModuleType( bytes_if_py2('patcher')) assert syn._detect_environment() == 'default' finally: sys.modules.pop('eventlet.patcher', None) sys.modules.pop('eventlet', None) syn._detect_environment() try: sys.modules['gevent'] = types.ModuleType(bytes_if_py2('gevent')) assert syn._detect_environment() == 'default' finally: sys.modules.pop('gevent', None) syn._detect_environment()
for local in self.locals: release_local(local) def __repr__(self): return '<{0} storages: {1}>'.format( self.__class__.__name__, len(self.locals)) class _FastLocalStack(threading.local): def __init__(self): self.stack = [] self.push = self.stack.append self.pop = self.stack.pop @property def top(self): try: return self.stack[-1] except (AttributeError, IndexError): return None if _detect_environment() == 'default' and not USE_PURE_LOCALS: LocalStack = _FastLocalStack else: # - See #706 # since each thread has its own greenlet we can just use those as # identifiers for the context. If greenlets are not available we # fall back to the current thread ident. LocalStack = _LocalStack # noqa