def test_detect_environment(self):
try:
syn._environment = None
X = syn.detect_environment()
self.assertEqual(syn._environment, X)
Y = syn.detect_environment()
self.assertEqual(Y, X)
finally:
syn._environment = None
def test_detect_environment(self):
try:
syn._environment = None
X = syn.detect_environment()
self.assertEqual(syn._environment, X)
Y = syn.detect_environment()
self.assertEqual(Y, X)
finally:
syn._environment = None
def test_detect_environment(self):
try:
syn._environment = None
X = syn.detect_environment()
assert syn._environment == X
Y = syn.detect_environment()
assert Y == X
finally:
syn._environment = None
def current(self):
type = detect_environment()
try:
return getattr(self, self.map[type])
except KeyError:
raise KeyError(G_NOT_FOUND % (type,
", ".join(self.map.keys())))
def _get_connection(self):
"""Connect to the MongoDB server."""
if self._connection is None:
from pymongo import MongoClient
host = self.mongo_host
if not host:
# The first pymongo.Connection() argument (host) can be
# a list of ['host:port'] elements or a mongodb connection
# URI. If this is the case, don't use self.port
# but let pymongo get the port(s) from the URI instead.
# This enables the use of replica sets and sharding.
# See pymongo.Connection() for more info.
host = self.host
if isinstance(host, string_t) \
and not host.startswith('mongodb://'):
host = 'mongodb://{0}:{1}'.format(host, self.port)
if host == 'mongodb://':
host += 'localhost'
# don't change self.options
conf = dict(self.options)
conf['host'] = host
if detect_environment() != 'default':
if pymongo.version_tuple < (3, ):
conf['use_greenlets'] = True
self._connection = MongoClient(**conf)
return self._connection
def __init__(self, loglevel=None, hostname=None, ready_callback=noop,
queues=None, app=None, pidfile=None, **kwargs):
self.app = app_or_default(app or self.app)
# all new threads start without a current app, so if an app is not
# passed on to the thread it will fall back to the "default app",
# which then could be the wrong app. So for the worker
# we set this to always return our app. This is a hack,
# and means that only a single app can be used for workers
# running in the same process.
set_default_app(self.app)
self._shutdown_complete = Event()
self.setup_defaults(kwargs, namespace="celeryd")
self.app.select_queues(queues) # select queues subset.
# Options
self.loglevel = loglevel or self.loglevel
self.hostname = hostname or socket.gethostname()
self.ready_callback = ready_callback
self._finalize = Finalize(self, self.stop, exitpriority=1)
self.pidfile = pidfile
self.pidlock = None
self.use_eventloop = (detect_environment() == "default" and
self.app.broker_connection().is_evented)
# Initialize boot steps
self.pool_cls = _concurrency.get_implementation(self.pool_cls)
self.components = []
self.namespace = Namespace(app=self.app).apply(self, **kwargs)
def current(self):
type = detect_environment()
try:
return getattr(self, self.map[type])
except KeyError:
raise KeyError(G_NOT_FOUND.format(
type, ', '.join(keys(self.map))))
def current(self):
type = detect_environment()
try:
return getattr(self, self.map[type])
except KeyError:
raise KeyError(G_NOT_FOUND %
(type, ', '.join(list(self.map.keys()))))
def _get_connection(self):
"""Connect to the MongoDB server."""
if self._connection is None:
from pymongo import MongoClient
host = self.mongo_host
if not host:
# The first pymongo.Connection() argument (host) can be
# a list of ['host:port'] elements or a mongodb connection
# URI. If this is the case, don't use self.port
# but let pymongo get the port(s) from the URI instead.
# This enables the use of replica sets and sharding.
# See pymongo.Connection() for more info.
host = self.host
if isinstance(host, string_t) \
and not host.startswith('mongodb://'):
host = 'mongodb://{0}:{1}'.format(host, self.port)
if host == 'mongodb://':
host += 'localhost'
# don't change self.options
conf = dict(self.options)
conf['host'] = host
if detect_environment() != 'default':
conf['use_greenlets'] = True
self._connection = MongoClient(**conf)
return self._connection
def __init__(self, backend, app, accept, pending_results):
self.backend = backend
self.app = app
self.accept = accept
self._pending_results = pending_results
self.on_message = None
self.buckets = WeakKeyDictionary()
self.drainer = drainers[detect_environment()](self)
def __init__(self, backend, app, accept, pending_results):
self.backend = backend
self.app = app
self.accept = accept
self._pending_results = pending_results
self.on_message = None
self.buckets = WeakKeyDictionary()
self.drainer = drainers[detect_environment()](self)
def _prepare_client_options(self):
if pymongo.version_tuple >= (3, ):
return {'maxPoolSize': self.max_pool_size}
else: # pragma: no cover
options = {
'max_pool_size': self.max_pool_size,
'auto_start_request': False
}
if detect_environment() != 'default':
options['use_greenlets'] = True
return options
def _prepare_client_options(self):
if pymongo.version_tuple >= (3, ):
return {'maxPoolSize': self.max_pool_size}
else: # pragma: no cover
options = {
'max_pool_size': self.max_pool_size,
'auto_start_request': False
}
if detect_environment() != 'default':
options['use_greenlets'] = True
return options
def _get_poller():
if detect_environment() in ("eventlet", "gevent"):
# greenlet
return _select
elif hasattr(select, "epoll"):
# Py2.6+ Linux
return _epoll
elif hasattr(select, "kqueue"):
# Py2.6+ on BSD / Darwin
return _kqueue
else:
return _select
def _get_poller():
if detect_environment() != 'default':
# greenlet
return _select
elif epoll:
# Py2.6+ Linux
return _epoll
elif kqueue:
# Py2.6+ on BSD / Darwin
return _kqueue
else:
return _select
def _get_poller():
if detect_environment() in ("eventlet", "gevent"):
# greenlet
return _select
elif epoll:
# Py2.6+ Linux
return _epoll
elif kqueue:
# Py2.6+ on BSD / Darwin
return _kqueue
else:
return _select
def _get_poller():
if detect_environment() != 'default':
# greenlet
return _select
elif epoll:
# Py2.6+ Linux
return _epoll
elif kqueue:
# Py2.6+ on BSD / Darwin
return _kqueue
else:
return _select
def _get_poller():
if detect_environment() != 'default':
# greenlet
return _select
elif epoll:
# Py2.6+ Linux
return _epoll
elif kqueue and 'netbsd' in sys.platform:
return _kqueue
elif xpoll:
return _poll
else:
return _select
def _get_poller():
if detect_environment() != 'default':
# greenlet
return _select
elif epoll:
# Py2.6+ Linux
return _epoll
elif xpoll:
return _poll
elif kqueue:
# Py2.6+ on BSD / Darwin
# but kqueue has too many bugs
return _poll if xpoll else _select
else:
return _select
def __init__(self,
loglevel=None,
hostname=None,
ready_callback=noop,
queues=None,
app=None,
pidfile=None,
**kwargs):
self.app = app_or_default(app or self.app)
# all new threads start without a current app, so if an app is not
# passed on to the thread it will fall back to the "default app",
# which then could be the wrong app. So for the worker
# we set this to always return our app. This is a hack,
# and means that only a single app can be used for workers
# running in the same process.
set_default_app(self.app)
self.app.finalize()
trace._tasks = self.app._tasks
self._shutdown_complete = Event()
self.setup_defaults(kwargs, namespace='celeryd')
self.app.select_queues(queues) # select queues subset.
# Options
self.loglevel = loglevel or self.loglevel
self.hostname = hostname or socket.gethostname()
self.ready_callback = ready_callback
self._finalize = Finalize(self, self.stop, exitpriority=1)
self.pidfile = pidfile
self.pidlock = None
self.use_eventloop = (detect_environment() == 'default'
and self.app.broker_connection().is_evented
and not self.app.IS_WINDOWS)
# Update celery_include to have all known task modules, so that we
# ensure all task modules are imported in case an execv happens.
task_modules = set(task.__class__.__module__
for task in self.app.tasks.itervalues())
self.app.conf.CELERY_INCLUDE = tuple(
set(self.app.conf.CELERY_INCLUDE) | task_modules, )
# Initialize boot steps
self.pool_cls = _concurrency.get_implementation(self.pool_cls)
self.components = []
self.namespace = Namespace(app=self.app).apply(self, **kwargs)
def _get_connection(self):
"""Connect to the MongoDB server."""
if self._connection is None:
from pymongo import MongoClient
# The first pymongo.Connection() argument (host) can be
# a list of ['host:port'] elements or a mongodb connection
# URI. If this is the case, don't use self.port
# but let pymongo get the port(s) from the URI instead.
# This enables the use of replica sets and sharding.
# See pymongo.Connection() for more info.
url = self.host
if isinstance(url, string_t) and not url.startswith("mongodb://"):
url = "mongodb://{0}:{1}".format(url, self.port)
if url == "mongodb://":
url = url + "localhost"
if detect_environment() != "default":
self.options["use_greenlets"] = True
self._connection = MongoClient(host=url, **self.options)
return self._connection
def __init__(self, loglevel=None, hostname=None, ready_callback=noop,
queues=None, app=None, pidfile=None, **kwargs):
self.app = app_or_default(app or self.app)
# all new threads start without a current app, so if an app is not
# passed on to the thread it will fall back to the "default app",
# which then could be the wrong app. So for the worker
# we set this to always return our app. This is a hack,
# and means that only a single app can be used for workers
# running in the same process.
set_default_app(self.app)
self.app.finalize()
trace._tasks = self.app._tasks
self._shutdown_complete = Event()
self.setup_defaults(kwargs, namespace='celeryd')
self.app.select_queues(queues) # select queues subset.
# Options
self.loglevel = loglevel or self.loglevel
self.hostname = hostname or socket.gethostname()
self.ready_callback = ready_callback
self._finalize = Finalize(self, self.stop, exitpriority=1)
self.pidfile = pidfile
self.pidlock = None
self.use_eventloop = (detect_environment() == 'default' and
self.app.broker_connection().is_evented and
not self.app.IS_WINDOWS)
# Update celery_include to have all known task modules, so that we
# ensure all task modules are imported in case an execv happens.
task_modules = set(task.__class__.__module__
for task in self.app.tasks.itervalues())
self.app.conf.CELERY_INCLUDE = tuple(
set(self.app.conf.CELERY_INCLUDE) | task_modules,
)
# Initialize boot steps
self.pool_cls = _concurrency.get_implementation(self.pool_cls)
self.components = []
self.namespace = Namespace(app=self.app).apply(self, **kwargs)
def _get_connection(self):
"""Connect to the MongoDB server."""
if self._connection is None:
from pymongo import MongoClient
# The first pymongo.Connection() argument (host) can be
# a list of ['host:port'] elements or a mongodb connection
# URI. If this is the case, don't use self.port
# but let pymongo get the port(s) from the URI instead.
# This enables the use of replica sets and sharding.
# See pymongo.Connection() for more info.
url = self.host
if isinstance(url, string_t) \
and not url.startswith('mongodb://'):
url = 'mongodb://{0}:{1}'.format(url, self.port)
if url == 'mongodb://':
url = url + 'localhost'
if detect_environment() != 'default':
self.options['use_greenlets'] = True
self._connection = MongoClient(host=url, **self.options)
return self._connection
def __init__(self,
loglevel=None,
hostname=None,
ready_callback=noop,
queues=None,
app=None,
pidfile=None,
**kwargs):
self.app = app_or_default(app or self.app)
# all new threads start without a current app, so if an app is not
# passed on to the thread it will fall back to the "default app",
# which then could be the wrong app. So for the worker
# we set this to always return our app. This is a hack,
# and means that only a single app can be used for workers
# running in the same process.
set_default_app(self.app)
self.app.finalize()
trace._tasks = self.app._tasks
self._shutdown_complete = Event()
self.setup_defaults(kwargs, namespace="celeryd")
self.app.select_queues(queues) # select queues subset.
# Options
self.loglevel = loglevel or self.loglevel
self.hostname = hostname or socket.gethostname()
self.ready_callback = ready_callback
self._finalize = Finalize(self, self.stop, exitpriority=1)
self.pidfile = pidfile
self.pidlock = None
self.use_eventloop = (detect_environment() == "default"
and self.app.broker_connection().is_evented)
# Initialize boot steps
self.pool_cls = _concurrency.get_implementation(self.pool_cls)
self.components = []
self.namespace = Namespace(app=self.app).apply(self, **kwargs)
def should_use_eventloop(self):
return (detect_environment() == 'default' and
self._conninfo.is_evented and not self.app.IS_WINDOWS)
:copyright: (c) 2009 - 2012 by Ask Solem.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import
import sys
from kombu.syn import detect_environment
DEFAULT_TRANSPORT = 'amqp'
AMQP_TRANSPORT = 'kombu.transport.amqplib.Transport'
AMQP_ALIAS = 'librabbitmq'
if detect_environment() == 'default':
try:
import librabbitmq # noqa
AMQP_TRANSPORT = 'kombu.transport.librabbitmq.Transport' # noqa
AMQP_ALIAS = 'amqp' # noqa
except ImportError:
pass
def _ghettoq(name, new, alias=None):
xxx = new # stupid enclosing
def __inner():
import warnings
_new = callable(xxx) and xxx() or xxx
gtransport = 'ghettoq.taproot.%s' % name
def should_use_eventloop(self):
return (detect_environment() == 'default'
and self.app.connection().is_evented
and not self.app.IS_WINDOWS)
def should_use_eventloop(self):
return (detect_environment() == 'default'
and self._conninfo.transport.implements. async
and not self.app.IS_WINDOWS)
def should_use_eventloop(self):
return detect_environment() == "default" and self.app.connection().is_evented and not self.app.IS_WINDOWS
def _set_stopped(self):
try:
self._is_stopped.set()
except TypeError: # pragma: no cover
# we lost the race at interpreter shutdown,
# so gc collected built-in modules.
pass
def stop(self):
"""Graceful shutdown."""
self._is_shutdown.set()
self._is_stopped.wait()
if self.is_alive():
self.join(1e100)
if detect_environment() == "default":
class LocalStack(threading.local):
def __init__(self):
self.stack = []
self.push = self.stack.append
self.pop = self.stack.pop
@property
def top(self):
try:
return self.stack[-1]
except (AttributeError, IndexError):
return None
else:
# See #706
try:
self._is_stopped.set()
except TypeError: # pragma: no cover
# we lost the race at interpreter shutdown,
# so gc collected built-in modules.
pass
def stop(self):
"""Graceful shutdown."""
self._is_shutdown.set()
self._is_stopped.wait()
if self.is_alive():
self.join(1e100)
if detect_environment() == "default":
class LocalStack(threading.local):
def __init__(self):
self.stack = []
self.push = self.stack.append
self.pop = self.stack.pop
@property
def top(self):
try:
return self.stack[-1]
except (AttributeError, IndexError):
return None
else:
# See #706
def should_use_eventloop(self):
return (detect_environment() == 'default' and
self._conninfo.transport.implements.async and
not self.app.IS_WINDOWS)
for local in self.locals:
release_local(local)
def __repr__(self):
return '<{0} storages: {1}>'.format(self.__class__.__name__,
len(self.locals))
class _FastLocalStack(threading.local):
def __init__(self):
self.stack = []
self.push = self.stack.append
self.pop = self.stack.pop
@property
def top(self):
try:
return self.stack[-1]
except (AttributeError, IndexError):
return None
if detect_environment() == 'default' and not USE_PURE_LOCALS:
LocalStack = _FastLocalStack
else:
# - See #706
# since each thread has its own greenlet we can just use those as
# identifiers for the context. If greenlets are not available we
# fall back to the current thread ident.
LocalStack = _LocalStack # noqa
try:
self._is_stopped.set()
except TypeError: # pragma: no cover
# we lost the race at interpreter shutdown,
# so gc collected built-in modules.
pass
def stop(self):
"""Graceful shutdown."""
self._is_shutdown.set()
self._is_stopped.wait()
if self.is_alive():
self.join(1e100)
if detect_environment() == 'default':
class LocalStack(threading.local):
def __init__(self):
self.stack = []
self.push = self.stack.append
self.pop = self.stack.pop
@property
def top(self):
try:
return self.stack[-1]
except (AttributeError, IndexError):
return None
else:
# See #706
"""
for local in self.locals:
release_local(local)
def __repr__(self):
return "<{0} storages: {1}>".format(self.__class__.__name__, len(self.locals))
class _FastLocalStack(threading.local):
def __init__(self):
self.stack = []
self.push = self.stack.append
self.pop = self.stack.pop
@property
def top(self):
try:
return self.stack[-1]
except (AttributeError, IndexError):
return None
if detect_environment() == "default" and not USE_PURE_LOCALS:
LocalStack = _FastLocalStack
else:
# - See #706
# since each thread has its own greenlet we can just use those as
# identifiers for the context. If greenlets are not available we
# fall back to the current thread ident.
LocalStack = _LocalStack # noqa