def start(self):
"""Starts the workers main loop."""
self.on_start()
self._state = self.RUN
if self.pidfile:
self.pidlock = platforms.create_pidlock(self.pidfile)
try:
for i, component in enumerate(self.components):
logger.debug('Starting %s...', qualname(component))
self._running = i + 1
if component:
component.start()
logger.debug('%s OK!', qualname(component))
except SystemTerminate:
self.terminate()
except Exception as exc:
logger.error('Unrecoverable error: %r', exc,
exc_info=True)
self.stop()
except (KeyboardInterrupt, SystemExit):
self.stop()
try:
# Will only get here if running green,
# makes sure all greenthreads have exited.
self._shutdown_complete.wait()
except IGNORE_ERRORS:
pass
def start(self):
"""Starts the workers main loop."""
self.on_start()
self._state = self.RUN
if self.pidfile:
self.pidlock = platforms.create_pidlock(self.pidfile)
try:
for i, component in enumerate(self.components):
logger.debug('Starting %s...', qualname(component))
self._running = i + 1
if component:
component.start()
logger.debug('%s OK!', qualname(component))
except SystemTerminate:
self.terminate()
except Exception as exc:
logger.error('Unrecoverable error: %r', exc, exc_info=True)
self.stop()
except (KeyboardInterrupt, SystemExit):
self.stop()
try:
# Will only get here if running green,
# makes sure all greenthreads have exited.
self._shutdown_complete.wait()
except IGNORE_ERRORS:
pass
def __init__(self, *args, **kwargs):
super(AppCase, self).__init__(*args, **kwargs)
if self.__class__.__dict__.get('setUp'):
raise RuntimeError(
CASE_REDEFINES_SETUP.format(name=qualname(self)), )
if self.__class__.__dict__.get('tearDown'):
raise RuntimeError(
CASE_REDEFINES_TEARDOWN.format(name=qualname(self)), )
def __new__(cls, *args, **kwargs):
if cls.__dict__.get('setUp'):
raise RuntimeError(CASE_REDEFINES_SETUP.format(name=qualname(cls)))
if cls.__dict__.get('tearDown'):
raise RuntimeError(CASE_REDEFINES_TEARDOWN.format(
name=qualname(cls)),
)
return super(AppCase, cls).__new__(cls, *args, **kwargs)
def start(self, parent):
self.state = RUN
if self.on_start:
self.on_start()
for i, component in enumerate(parent.components):
if component:
logger.debug('Starting %s...', qualname(component))
self.started = i + 1
component.start(parent)
logger.debug('%s OK!', qualname(component))
def __init__(self, *args, **kwargs):
super(AppCase, self).__init__(*args, **kwargs)
if self.__class__.__dict__.get('setUp'):
raise RuntimeError(
CASE_REDEFINES_SETUP.format(name=qualname(self)),
)
if self.__class__.__dict__.get('tearDown'):
raise RuntimeError(
CASE_REDEFINES_TEARDOWN.format(name=qualname(self)),
)
def __init__(self, *args, **kwargs):
super(AppCase, self).__init__(*args, **kwargs)
setUp = self.__class__.__dict__.get('setUp')
tearDown = self.__class__.__dict__.get('tearDown')
if setUp and not hasattr(setUp, '__wrapped__'):
raise RuntimeError(
CASE_REDEFINES_SETUP.format(name=qualname(self)), )
if tearDown and not hasattr(tearDown, '__wrapped__'):
raise RuntimeError(
CASE_REDEFINES_TEARDOWN.format(name=qualname(self)), )
def start(self, parent):
self.state = RUN
if self.on_start:
self.on_start()
for i, component in enumerate(parent.components):
if component:
logger.debug('Starting %s...', qualname(component))
self.started = i + 1
component.start(parent)
logger.debug('%s OK!', qualname(component))
def startup_info(self, beat):
scheduler = beat.get_scheduler(lazy=True)
return STARTUP_INFO_FMT % {
'conninfo': self.app.connection().as_uri(),
'logfile': self.logfile or '[stderr]',
'loglevel': LOG_LEVELS[self.loglevel],
'loader': qualname(self.app.loader),
'scheduler': qualname(scheduler),
'scheduler_info': scheduler.info,
'hmax_interval': humanize_seconds(beat.max_interval),
'max_interval': beat.max_interval,
}
def startup_info(self, beat):
scheduler = beat.get_scheduler(lazy=True)
return STARTUP_INFO_FMT % {
'conninfo': self.app.connection().as_uri(),
'logfile': self.logfile or '[stderr]',
'loglevel': LOG_LEVELS[self.loglevel],
'loader': qualname(self.app.loader),
'scheduler': qualname(scheduler),
'scheduler_info': scheduler.info,
'hmax_interval': humanize_seconds(beat.max_interval),
'max_interval': beat.max_interval,
}
def startup_info(self, beat):
scheduler = beat.get_scheduler(lazy=True)
return STARTUP_INFO_FMT % {
"conninfo": self.app.broker_connection().as_uri(),
"logfile": self.logfile or "[stderr]",
"loglevel": LOG_LEVELS[self.loglevel],
"loader": qualname(self.app.loader),
"scheduler": qualname(scheduler),
"scheduler_info": scheduler.info,
"hmax_interval": humanize_seconds(beat.max_interval),
"max_interval": beat.max_interval,
}
def startup_info(self, beat):
scheduler = beat.get_scheduler(lazy=True)
return STARTUP_INFO_FMT % {
"conninfo": self.app.broker_connection().as_uri(),
"logfile": self.logfile or "[stderr]",
"loglevel": LOG_LEVELS[self.loglevel],
"loader": qualname(self.app.loader),
"scheduler": qualname(scheduler),
"scheduler_info": scheduler.info,
"hmax_interval": humanize_seconds(beat.max_interval),
"max_interval": beat.max_interval,
}
def startup_info(self, beat):
scheduler = beat.get_scheduler(lazy=True)
return STARTUP_INFO_FMT.format(
conninfo=self.app.connection().as_uri(),
logfile=self.logfile or '[stderr]',
loglevel=LOG_LEVELS[self.loglevel],
loader=qualname(self.app.loader),
scheduler=qualname(scheduler),
scheduler_info=scheduler.info,
hmax_interval=humanize_seconds(beat.max_interval),
max_interval=beat.max_interval,
)
def startup_info(self, beat):
scheduler = beat.get_scheduler(lazy=True)
return STARTUP_INFO_FMT.format(
conninfo=self.app.connection().as_uri(),
logfile=self.logfile or '[stderr]',
loglevel=LOG_LEVELS[self.loglevel],
loader=qualname(self.app.loader),
scheduler=qualname(scheduler),
scheduler_info=scheduler.info,
hmax_interval=humanize_seconds(beat.max_interval),
max_interval=beat.max_interval,
)
def __init__(self, *args, **kwargs):
super(AppCase, self).__init__(*args, **kwargs)
setUp = self.__class__.__dict__.get('setUp')
tearDown = self.__class__.__dict__.get('tearDown')
if setUp and not hasattr(setUp, '__wrapped__'):
raise RuntimeError(
CASE_REDEFINES_SETUP.format(name=qualname(self)),
)
if tearDown and not hasattr(tearDown, '__wrapped__'):
raise RuntimeError(
CASE_REDEFINES_TEARDOWN.format(name=qualname(self)),
)
def startup_info(self, service):
scheduler = service.get_scheduler(lazy=True)
return STARTUP_INFO_FMT.format(
conninfo=self.app.connection().as_uri(),
timestamp=datetime.now().replace(microsecond=0),
logfile=self.logfile or '[stderr]',
loglevel=LOG_LEVELS[self.loglevel],
loader=qualname(self.app.loader),
scheduler=qualname(scheduler),
scheduler_info=scheduler.info,
hmax_interval=humanize_seconds(service.max_interval),
max_interval=service.max_interval,
)
def startup_info(self, service):
scheduler = service.get_scheduler(lazy=True)
return STARTUP_INFO_FMT.format(
conninfo=self.app.connection().as_uri(),
timestamp=datetime.now().replace(microsecond=0),
logfile=self.logfile or '[stderr]',
loglevel=LOG_LEVELS[self.loglevel],
loader=qualname(self.app.loader),
scheduler=qualname(scheduler),
scheduler_info=scheduler.info,
hmax_interval=humanize_seconds(scheduler.max_interval),
max_interval=scheduler.max_interval,
)
def start(self):
"""Starts the workers main loop."""
self._state = self.RUN
try:
for i, component in enumerate(self.components):
logger.debug("Starting %s...", qualname(component))
self._running = i + 1
component.start()
logger.debug("%s OK!", qualname(component))
except SystemTerminate:
self.terminate()
except Exception, exc:
logger.error("Unrecoverable error: %r", exc, exc_info=True)
self.stop()
def xxx_subscriber_callback_setting(self,
manager,
event='article.changed'):
event1_ref = 'hardcoded1'
event1_url = manager.event_url('article.changed', ref=event1_ref)
event2_ref = 'hardcoded2'
event2_url = manager.event_url('article.changed', ref=event2_ref)
def callback_subscribers(event, sender=None, **kwargs):
return [
event1_url,
event2_url,
]
with manager.worker_subscribe_to(
'article.changed', callback=qualname(callback_subscribers)):
article = manager.create_article('A brown fox')
article.title = 'A red fox'
article.save()
manager.assert_article_event_received(article,
event,
ref=event1_ref)
manager.assert_article_event_received(article,
event,
ref=event2_ref)
def stop(self, parent, terminate=False):
what = 'Terminating' if terminate else 'Stopping'
socket_timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(SHUTDOWN_SOCKET_TIMEOUT) # Issue 975
if self.state in (CLOSE, TERMINATE):
return
self.close(parent)
if self.state != RUN or self.started != len(parent.components):
# Not fully started, can safely exit.
self.state = TERMINATE
self.shutdown_complete.set()
return
self.state = CLOSE
for component in reversed(parent.components):
if component:
logger.debug('%s %s...', what, qualname(component))
(component.terminate if terminate else component.stop)(parent)
if self.on_stopped:
self.on_stopped()
self.state = TERMINATE
socket.setdefaulttimeout(socket_timeout)
self.shutdown_complete.set()
def bugreport(app):
"""Return a string containing information useful in bug-reports."""
import billiard
import celery
import kombu
try:
conn = app.connection()
driver_v = '{0}:{1}'.format(conn.transport.driver_name,
conn.transport.driver_version())
transport = conn.transport_cls
except Exception: # pylint: disable=broad-except
transport = driver_v = ''
return BUGREPORT_INFO.format(
system=_platform.system(),
arch=', '.join(x for x in _platform.architecture() if x),
py_i=pyimplementation(),
celery_v=celery.VERSION_BANNER,
kombu_v=kombu.__version__,
billiard_v=billiard.__version__,
py_v=_platform.python_version(),
driver_v=driver_v,
transport=transport,
results=maybe_sanitize_url(app.conf.result_backend or 'disabled'),
human_settings=app.conf.humanize(),
loader=qualname(app.loader.__class__),
)
def _shutdown(self, warm=True):
what = "Stopping" if warm else "Terminating"
if self._state in (self.CLOSE, self.TERMINATE):
return
if self.pool:
self.pool.close()
if self._state != self.RUN or self._running != len(self.components):
# Not fully started, can safely exit.
self._state = self.TERMINATE
self._shutdown_complete.set()
return
self._state = self.CLOSE
for component in reversed(self.components):
logger.debug("%s %s...", what, qualname(component))
if component:
stop = component.stop
if not warm:
stop = getattr(component, "terminate", None) or stop
stop()
self.timer.stop()
self.consumer.close_connection()
if self.pidlock:
self.pidlock.release()
self._state = self.TERMINATE
self._shutdown_complete.set()
def stop(self, parent, terminate=False):
what = 'Terminating' if terminate else 'Stopping'
socket_timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(SHUTDOWN_SOCKET_TIMEOUT) # Issue 975
if self.state in (CLOSE, TERMINATE):
return
self.close(parent)
if self.state != RUN or self.started != len(parent.components):
# Not fully started, can safely exit.
self.state = TERMINATE
self.shutdown_complete.set()
return
self.state = CLOSE
for component in reversed(parent.components):
if component:
logger.debug('%s %s...', what, qualname(component))
(component.terminate if terminate else component.stop)(parent)
if self.on_stopped:
self.on_stopped()
self.state = TERMINATE
socket.setdefaulttimeout(socket_timeout)
self.shutdown_complete.set()
def bugreport(app):
import billiard
import celery
import kombu
try:
trans = app.broker_connection().transport
driver_v = "%s:%s" % (trans.driver_name, trans.driver_version())
except Exception:
driver_v = ""
return BUGREPORT_INFO % {
"system": _platform.system(),
"arch": ', '.join(filter(None, _platform.architecture())),
"py_i": platforms.pyimplementation(),
"celery_v": celery.__version__,
"kombu_v": kombu.__version__,
"billiard_v": billiard.__version__,
"py_v": _platform.python_version(),
"driver_v": driver_v,
"transport": app.conf.BROKER_TRANSPORT or "amqp",
"results": app.conf.CELERY_RESULT_BACKEND or "disabled",
"human_settings": app.conf.humanize(),
"loader": qualname(app.loader.__class__),
}
def bugreport(app):
import billiard
import celery
import kombu
try:
trans = app.broker_connection().transport
driver_v = '%s:%s' % (trans.driver_name, trans.driver_version())
except Exception:
driver_v = ''
return BUGREPORT_INFO % {
'system': _platform.system(),
'arch': ', '.join(filter(None, _platform.architecture())),
'py_i': platforms.pyimplementation(),
'celery_v': celery.__version__,
'kombu_v': kombu.__version__,
'billiard_v': billiard.__version__,
'py_v': _platform.python_version(),
'driver_v': driver_v,
'transport': app.conf.BROKER_TRANSPORT or 'amqp',
'results': app.conf.CELERY_RESULT_BACKEND or 'disabled',
'human_settings': app.conf.humanize(),
'loader': qualname(app.loader.__class__),
}
def bugreport(app):
"""Return a string containing information useful in bug reports."""
import billiard
import celery
import kombu
try:
conn = app.connection()
driver_v = '{0}:{1}'.format(conn.transport.driver_name,
conn.transport.driver_version())
transport = conn.transport_cls
except Exception:
transport = driver_v = ''
return BUGREPORT_INFO.format(
system=_platform.system(),
arch=', '.join(x for x in _platform.architecture() if x),
py_i=pyimplementation(),
celery_v=celery.VERSION_BANNER,
kombu_v=kombu.__version__,
billiard_v=billiard.__version__,
py_v=_platform.python_version(),
driver_v=driver_v,
transport=transport,
results=app.conf.CELERY_RESULT_BACKEND or 'disabled',
human_settings=app.conf.humanize(),
loader=qualname(app.loader.__class__),
)
def start(self):
"""Starts the workers main loop."""
self._state = self.RUN
try:
for i, component in enumerate(self.components):
logger.debug("Starting %s...", qualname(component))
self._running = i + 1
component.start()
logger.debug("%s OK!", qualname(component))
except SystemTerminate:
self.terminate()
except Exception, exc:
logger.error("Unrecoverable error: %r", exc,
exc_info=True)
self.stop()
def bugreport(app):
"""Returns a string containing information useful in bug reports."""
import billiard
import celery
import kombu
try:
trans = app.connection().transport
driver_v = '%s:%s' % (trans.driver_name, trans.driver_version())
except Exception:
driver_v = ''
return BUGREPORT_INFO % {
'system': _platform.system(),
'arch': ', '.join(filter(None, _platform.architecture())),
'py_i': platforms.pyimplementation(),
'celery_v': celery.__version__,
'kombu_v': kombu.__version__,
'billiard_v': billiard.__version__,
'py_v': _platform.python_version(),
'driver_v': driver_v,
'transport': app.conf.BROKER_TRANSPORT or 'amqp',
'results': app.conf.CELERY_RESULT_BACKEND or 'disabled',
'human_settings': app.conf.humanize(),
'loader': qualname(app.loader.__class__),
}
def bugreport(app):
"""Return a string containing information useful in bug-reports."""
import billiard
import celery
import kombu
try:
conn = app.connection()
driver_v = "{0}:{1}".format(conn.transport.driver_name, conn.transport.driver_version())
transport = conn.transport_cls
except Exception: # pylint: disable=broad-except
transport = driver_v = ""
return BUGREPORT_INFO.format(
system=_platform.system(),
arch=", ".join(x for x in _platform.architecture() if x),
py_i=pyimplementation(),
celery_v=celery.VERSION_BANNER,
kombu_v=kombu.__version__,
billiard_v=billiard.__version__,
py_v=_platform.python_version(),
driver_v=driver_v,
transport=transport,
results=maybe_sanitize_url(app.conf.result_backend or "disabled"),
human_settings=app.conf.humanize(),
loader=qualname(app.loader.__class__),
)
def bugreport(app):
"""Returns a string containing information useful in bug reports."""
import billiard
import celery
import kombu
try:
conn = app.connection()
driver_v = '%s:%s' % (conn.transport.driver_name,
conn.transport.driver_version())
transport = conn.transport_cls
except Exception:
transport = driver_v = ''
return BUGREPORT_INFO % {
'system': _platform.system(),
'arch': ', '.join(filter(None, _platform.architecture())),
'py_i': platforms.pyimplementation(),
'celery_v': celery.VERSION_BANNER,
'kombu_v': kombu.__version__,
'billiard_v': billiard.__version__,
'py_v': _platform.python_version(),
'driver_v': driver_v,
'transport': transport,
'results': app.conf.CELERY_RESULT_BACKEND or 'disabled',
'human_settings': app.conf.humanize(),
'loader': qualname(app.loader.__class__),
}
def bugreport(app):
"""Returns a string containing information useful in bug reports."""
import billiard
import celery
import kombu
try:
conn = app.connection()
driver_v = '%s:%s' % (conn.transport.driver_name,
conn.transport.driver_version())
transport = conn.transport_cls
except Exception:
transport = driver_v = ''
return BUGREPORT_INFO % {
'system': _platform.system(),
'arch': ', '.join(p for p in _platform.architecture() if p),
'py_i': platforms.pyimplementation(),
'celery_v': celery.VERSION_BANNER,
'kombu_v': kombu.__version__,
'billiard_v': billiard.__version__,
'py_v': _platform.python_version(),
'driver_v': driver_v,
'transport': transport,
'results': app.conf.CELERY_RESULT_BACKEND or 'disabled',
'human_settings': app.conf.humanize(),
'loader': qualname(app.loader.__class__),
}
def _shutdown(self, warm=True):
what = 'Stopping' if warm else 'Terminating'
if self._state in (self.CLOSE, self.TERMINATE):
return
self.app.loader.shutdown_worker()
if self.pool:
self.pool.close()
if self._state != self.RUN or self._running != len(self.components):
# Not fully started, can safely exit.
self._state = self.TERMINATE
self._shutdown_complete.set()
return
self._state = self.CLOSE
for component in reversed(self.components):
logger.debug('%s %s...', what, qualname(component))
if component:
stop = component.stop
if not warm:
stop = getattr(component, 'terminate', None) or stop
stop()
self.timer.stop()
self.consumer.close_connection()
if self.pidlock:
self.pidlock.release()
self._state = self.TERMINATE
self._shutdown_complete.set()
def _shutdown(self, warm=True):
what = "Stopping" if warm else "Terminating"
if self._state in (self.CLOSE, self.TERMINATE):
return
if self._state != self.RUN or self._running != len(self.components):
# Not fully started, can safely exit.
self._state = self.TERMINATE
self._shutdown_complete.set()
return
self._state = self.CLOSE
for component in reversed(self.components):
logger.debug("%s %s...", what, qualname(component))
stop = component.stop
if not warm:
stop = getattr(component, "terminate", None) or stop
stop()
self.priority_timer.stop()
self.consumer.close_connection()
self._state = self.TERMINATE
self._shutdown_complete.set()
def start(self):
"""Starts the workers main loop."""
self._state = self.RUN
if self.pidfile:
self.pidlock = platforms.create_pidlock(self.pidfile)
try:
for i, component in enumerate(self.components):
logger.debug('Starting %s...', qualname(component))
self._running = i + 1
if component:
component.start()
logger.debug('%s OK!', qualname(component))
except SystemTerminate:
self.terminate()
except Exception, exc:
logger.error('Unrecoverable error: %r', exc, exc_info=True)
self.stop()
def subscriber_callback_setting(self, event='article.changed'):
with self.worker_subscribe_to(
'article.changed', callback=qualname(callback_subscribers)):
article = self.create_article('A brown fox')
article.title = 'A red fox'
article.save()
self.assert_article_event_received(article, event, ref=EVENT1_REF)
self.assert_article_event_received(article, event, ref=EVENT2_REF)
def __inner(*args, **kwargs):
from celery.utils.imports import qualname
warn(description=description or qualname(fun),
deprecation=deprecation,
removal=removal,
alternative=alternative,
stacklevel=3)
return fun(*args, **kwargs)
def subscriber_callback_setting(self, event='article.changed'):
with self.worker_subscribe_to(
'article.changed', callback=qualname(callback_subscribers)):
article = self.create_article('A brown fox')
article.title = 'A red fox'
article.save()
self.assert_article_event_received(article, event, ref=EVENT1_REF)
self.assert_article_event_received(article, event, ref=EVENT2_REF)
def start(self):
"""Starts the workers main loop."""
self._state = self.RUN
if self.pidfile:
self.pidlock = platforms.create_pidlock(self.pidfile)
try:
for i, component in enumerate(self.components):
logger.debug('Starting %s...', qualname(component))
self._running = i + 1
if component:
component.start()
logger.debug('%s OK!', qualname(component))
except SystemTerminate:
self.terminate()
except Exception, exc:
logger.error('Unrecoverable error: %r', exc,
exc_info=True)
self.stop()
def bugreport(app):
return BUGREPORT_INFO % {"system": _platform.system(),
"arch": _platform.architecture(),
"py_i": platforms.pyimplementation(),
"celery_v": celery.__version__,
"kombu_v": kombu.__version__,
"py_v": _platform.python_version(),
"transport": app.conf.BROKER_TRANSPORT,
"results": app.conf.CELERY_RESULT_BACKEND,
"human_settings": app.conf.humanize(),
"loader": qualname(app.loader.__class__)}
def test_prepare(self):
o = object()
R = [
{'foo': 'bar'},
qualname(TestRouter),
o,
]
p = routes.prepare(R)
self.assertIsInstance(p[0], routes.MapRoute)
self.assertIsInstance(maybe_evaluate(p[1]), TestRouter)
self.assertIs(p[2], o)
self.assertEqual(routes.prepare(o), [o])
def banner(self, tests):
app = self.app
return BANNER.format(
app='{0}:0x{1:x}'.format(app.main or '__main__', id(app)),
version=cyanide.__version__,
celery_version=celery.VERSION_BANNER,
conninfo=app.connection().as_uri(),
platform=platform.platform(),
toc=self.testlist(tests),
TESTS=pluralize(len(tests), 'test'),
total=len(tests),
suite=':'.join(qualname(self).rsplit('.', 1)),
)
def bugreport(app):
return BUGREPORT_INFO % {
"system": _platform.system(),
"arch": _platform.architecture(),
"py_i": platforms.pyimplementation(),
"celery_v": celery.__version__,
"kombu_v": kombu.__version__,
"py_v": _platform.python_version(),
"transport": app.conf.BROKER_TRANSPORT,
"results": app.conf.CELERY_RESULT_BACKEND,
"human_settings": app.conf.humanize(),
"loader": qualname(app.loader.__class__)
}
def test_prepare(self):
o = object()
R = [
{'foo': 'bar'},
qualname(TestRouter),
o,
]
p = routes.prepare(R)
assert isinstance(p[0], routes.MapRoute)
assert isinstance(maybe_evaluate(p[1]), TestRouter)
assert p[2] is o
assert routes.prepare(o) == [o]
def banner(self, tests):
app = self.app
return BANNER.format(
app='{0}:0x{1:x}'.format(app.main or '__main__', id(app)),
version=cyanide.__version__,
celery_version=celery.VERSION_BANNER,
conninfo=app.connection().as_uri(),
platform=platform.platform(),
toc=self.testlist(tests),
TESTS=pluralize(len(tests), 'test'),
total=len(tests),
suite=':'.join(qualname(self).rsplit('.', 1)),
)
def __init__(self,
steps=None,
name=None,
on_start=None,
on_close=None,
on_stopped=None):
self.name = name or self.name or qualname(type(self))
self.types = set(steps or []) | set(self.default_steps)
self.on_start = on_start
self.on_close = on_close
self.on_stopped = on_stopped
self.shutdown_complete = Event()
self.steps = {}
def managed(cls, hostname=None, caller=None):
hostname = hostname or socket.gethostname()
if caller:
hostname = ".".join([qualname(caller), hostname])
else:
hostname += str(next(cls.worker_ids()))
worker = cls(hostname)
worker.ensure_started()
stack = traceback.format_stack()
@atexit.register
def _ensure_shutdown_once():
if not worker._shutdown_called:
say("-- Found worker not stopped at shutdown: %s\n%s" % (worker.hostname, "\n".join(stack)))
worker.ensure_shutdown()
return worker
def startup_info(self):
app = self.app
concurrency = self.concurrency
if self.autoscale:
concurrency = "{min=%s, max=%s}" % self.autoscale
return BANNER % {
"hostname": self.hostname,
"version": __version__,
"conninfo": self.app.broker_connection().as_uri(),
"concurrency": concurrency,
"loglevel": LOG_LEVELS[self.loglevel],
"logfile": self.logfile or "[stderr]",
"celerybeat": "ON" if self.beat else "OFF",
"events": "ON" if self.send_events else "OFF",
"loader": qualname(self.app.loader),
"queues": app.amqp.queues.format(indent=18, indent_first=False),
}
def startup_info(self):
app = self.app
concurrency = self.concurrency
if self.autoscale:
cmax, cmin = self.autoscale
concurrency = "{min=%s, max=%s}" % (cmin, cmax)
return BANNER % {
"hostname": self.hostname,
"version": __version__,
"conninfo": self.app.broker_connection().as_uri(),
"concurrency": concurrency,
"loglevel": LOG_LEVELS[self.loglevel],
"logfile": self.logfile or "[stderr]",
"celerybeat": "ON" if self.embed_clockservice else "OFF",
"events": "ON" if self.send_events else "OFF",
"loader": qualname(self.loader),
"queues": app.amqp.queues.format(indent=18, indent_first=False),
}
def managed(cls, hostname=None, caller=None):
hostname = hostname or socket.gethostname()
if caller:
hostname = '.'.join([qualname(caller), hostname])
else:
hostname += str(cls.next_worker_id())
worker = cls(hostname)
worker.ensure_started()
stack = traceback.format_stack()
@atexit.register
def _ensure_shutdown_once():
if not worker._shutdown_called:
say('-- Found worker not stopped at shutdown: %s\n%s' %
(worker.hostname, '\n'.join(stack)))
worker.ensure_shutdown()
return worker
def startup_info(self):
app = self.app
concurrency = string(self.concurrency)
appr = '{0}:{1:#x}'.format(app.main or '__main__', id(app))
if not isinstance(app.loader, AppLoader):
loader = qualname(app.loader)
if loader.startswith('celery.loaders'): # pragma: no cover
loader = loader[14:]
appr += ' ({0})'.format(loader)
if self.autoscale:
max, min = self.autoscale
concurrency = '{{min={0}, max={1}}}'.format(min, max)
pool = self.pool_cls
if not isinstance(pool, string_t):
pool = pool.__module__
concurrency += ' ({0})'.format(pool.split('.')[-1])
events = 'ON'
if not self.send_events:
events = 'OFF (enable -E to monitor this worker)'
banner = BANNER.format(
app=appr,
hostname=safe_str(self.hostname),
timestamp=datetime.now().replace(microsecond=0),
version=VERSION_BANNER,
conninfo=self.app.connection().as_uri(),
results=maybe_sanitize_url(
self.app.conf.result_backend or 'disabled',
),
concurrency=concurrency,
platform=safe_str(_platform.platform()),
events=events,
queues=app.amqp.queues.format(indent=0, indent_first=False),
).splitlines()
# integrate the ASCII art.
for i, x in enumerate(banner):
try:
banner[i] = ' '.join([ARTLINES[i], banner[i]])
except IndexError:
banner[i] = ' ' * 16 + banner[i]
return '\n'.join(banner) + '\n'
def startup_info(self, artlines=True):
app = self.app
concurrency = string(self.concurrency)
appr = "{0}:{1:#x}".format(app.main or "__main__", id(app))
if not isinstance(app.loader, AppLoader):
loader = qualname(app.loader)
if loader.startswith("celery.loaders"): # pragma: no cover
loader = loader[14:]
appr += " ({0})".format(loader)
if self.autoscale:
max, min = self.autoscale
concurrency = "{{min={0}, max={1}}}".format(min, max)
pool = self.pool_cls
if not isinstance(pool, string_t):
pool = pool.__module__
concurrency += " ({0})".format(pool.split(".")[-1])
events = "ON"
if not self.task_events:
events = "OFF (enable -E to monitor tasks in this worker)"
banner = BANNER.format(
app=appr,
hostname=safe_str(self.hostname),
timestamp=datetime.now().replace(microsecond=0),
version=VERSION_BANNER,
conninfo=self.app.connection().as_uri(),
results=self.app.backend.as_uri(),
concurrency=concurrency,
platform=safe_str(_platform.platform()),
events=events,
queues=app.amqp.queues.format(indent=0, indent_first=False),
).splitlines()
# integrate the ASCII art.
if artlines:
for i, _ in enumerate(banner):
try:
banner[i] = " ".join([ARTLINES[i], banner[i]])
except IndexError:
banner[i] = " " * 16 + banner[i]
return "\n".join(banner) + "\n"
def startup_info(self):
app = self.app
concurrency = string(self.concurrency)
appr = '{0}:{1:#x}'.format(app.main or '__main__', id(app))
if not isinstance(app.loader, AppLoader):
loader = qualname(app.loader)
if loader.startswith('celery.loaders'): # pragma: no cover
loader = loader[14:]
appr += ' ({0})'.format(loader)
if self.autoscale:
max, min = self.autoscale
concurrency = '{{min={0}, max={1}}}'.format(min, max)
pool = self.pool_cls
if not isinstance(pool, string_t):
pool = pool.__module__
concurrency += ' ({0})'.format(pool.split('.')[-1])
events = 'ON'
if not self.send_events:
events = 'OFF (enable -E to monitor this worker)'
banner = BANNER.format(
app=appr,
hostname=safe_str(self.hostname),
version=VERSION_BANNER,
conninfo=self.app.connection().as_uri(),
results=maybe_sanitize_url(
self.app.conf.result_backend or 'disabled',
),
concurrency=concurrency,
platform=safe_str(_platform.platform()),
events=events,
queues=app.amqp.queues.format(indent=0, indent_first=False),
).splitlines()
# integrate the ASCII art.
for i, x in enumerate(banner):
try:
banner[i] = ' '.join([ARTLINES[i], banner[i]])
except IndexError:
banner[i] = ' ' * 16 + banner[i]
return '\n'.join(banner) + '\n'
def startup_info(self, artlines=True):
app = self.app
concurrency = str(self.concurrency)
appr = '{}:{:#x}'.format(app.main or '__main__', id(app))
if not isinstance(app.loader, AppLoader):
loader = qualname(app.loader)
if loader.startswith('celery.loaders'): # pragma: no cover
loader = loader[14:]
appr += f' ({loader})'
if self.autoscale:
max, min = self.autoscale
concurrency = f'{{min={min}, max={max}}}'
pool = self.pool_cls
if not isinstance(pool, str):
pool = pool.__module__
concurrency += f" ({pool.split('.')[-1]})"
events = 'ON'
if not self.task_events:
events = 'OFF (enable -E to monitor tasks in this worker)'
banner = BANNER.format(
app=appr,
hostname=safe_str(self.hostname),
timestamp=datetime.now().replace(microsecond=0),
version=VERSION_BANNER,
conninfo=self.app.connection().as_uri(),
results=self.app.backend.as_uri(),
concurrency=concurrency,
platform=safe_str(_platform.platform()),
events=events,
queues=app.amqp.queues.format(indent=0, indent_first=False),
).splitlines()
# integrate the ASCII art.
if artlines:
for i, _ in enumerate(banner):
try:
banner[i] = ' '.join([ARTLINES[i], banner[i]])
except IndexError:
banner[i] = ' ' * 16 + banner[i]
return '\n'.join(banner) + '\n'
def startup_info(self):
app = self.app
concurrency = unicode(self.concurrency)
appr = "%s:0x%x" % (app.main or "__main__", id(app))
if not isinstance(app.loader, AppLoader):
loader = qualname(app.loader)
if loader.startswith("celery.loaders"):
loader = loader[14:]
appr += " (%s)" % loader
if self.autoscale:
max, min = self.autoscale
concurrency = "{min=%s, max=%s}" % (min, max)
pool = self.pool_cls
if not isinstance(pool, basestring):
pool = pool.__module__
concurrency += " (%s)" % pool.split(".")[-1]
events = "ON"
if not self.send_events:
events = "OFF (enable -E to monitor this worker)"
banner = (
BANNER
% {
"app": appr,
"hostname": safe_str(self.hostname),
"version": VERSION_BANNER,
"conninfo": self.app.connection().as_uri(),
"concurrency": concurrency,
"platform": safe_str(_platform.platform()),
"events": events,
"queues": app.amqp.queues.format(indent=0, indent_first=False),
}
).splitlines()
# integrate the ASCII art.
for i, x in enumerate(banner):
try:
banner[i] = " ".join([ARTLINES[i], banner[i]])
except IndexError:
banner[i] = " " * 16 + banner[i]
return "\n".join(banner) + "\n"
def xxx_subscriber_callback_setting(self, manager,
event='article.changed'):
event1_ref = 'hardcoded1'
event1_url = manager.event_url('article.changed', ref=event1_ref)
event2_ref = 'hardcoded2'
event2_url = manager.event_url('article.changed', ref=event2_ref)
def callback_subscribers(event, sender=None, **kwargs):
return [
event1_url,
event2_url,
]
with manager.worker_subscribe_to(
'article.changed', callback=qualname(callback_subscribers)):
article = manager.create_article('A brown fox')
article.title = 'A red fox'
article.save()
manager.assert_article_event_received(
article, event, ref=event1_ref)
manager.assert_article_event_received(
article, event, ref=event2_ref)
def startup_info(self):
app = self.app
concurrency = unicode(self.concurrency)
appr = '%s:0x%x' % (app.main or '__main__', id(app))
if not isinstance(app.loader, AppLoader):
loader = qualname(app.loader)
if loader.startswith('celery.loaders'):
loader = loader[14:]
appr += ' (%s)' % loader
if self.autoscale:
max, min = self.autoscale
concurrency = '{min=%s, max=%s}' % (min, max)
pool = self.pool_cls
if not isinstance(pool, basestring):
pool = pool.__module__
concurrency += ' (%s)' % pool.split('.')[-1]
events = 'ON'
if not self.send_events:
events = 'OFF (enable -E to monitor this worker)'
banner = (
BANNER % {
'app': appr,
'hostname': self.hostname,
'version': VERSION_BANNER,
'conninfo': self.app.connection().as_uri(),
'concurrency': concurrency,
'platform': safe_str(_platform.platform()),
'events': events,
'queues': app.amqp.queues.format(indent=0, indent_first=False),
}).splitlines()
# integrate the ASCII art.
for i, x in enumerate(banner):
try:
banner[i] = ' '.join([ARTLINES[i], banner[i]])
except IndexError:
banner[i] = ' ' * 16 + banner[i]
return '\n'.join(banner) + '\n'
def startup_info(self):
app = self.app
concurrency = string(self.concurrency)
appr = "{0}:{1:#x}".format(app.main or "__main__", id(app))
if not isinstance(app.loader, AppLoader):
loader = qualname(app.loader)
if loader.startswith("celery.loaders"):
loader = loader[14:]
appr += " ({0})".format(loader)
if self.autoscale:
max, min = self.autoscale
concurrency = "{{min={0}, max={1}}}".format(min, max)
pool = self.pool_cls
if not isinstance(pool, string_t):
pool = pool.__module__
concurrency += " ({0})".format(pool.split(".")[-1])
events = "ON"
if not self.send_events:
events = "OFF (enable -E to monitor this worker)"
banner = BANNER.format(
app=appr,
hostname=safe_str(self.hostname),
version=VERSION_BANNER,
conninfo=self.app.connection().as_uri(),
results=maybe_sanitize_url(self.app.conf.result_backend or "disabled"),
concurrency=concurrency,
platform=safe_str(_platform.platform()),
events=events,
queues=app.amqp.queues.format(indent=0, indent_first=False),
).splitlines()
# integrate the ASCII art.
for i, x in enumerate(banner):
try:
banner[i] = " ".join([ARTLINES[i], banner[i]])
except IndexError:
banner[i] = " " * 16 + banner[i]
return "\n".join(banner) + "\n"
def startup_info(self):
app = self.app
concurrency = unicode(self.concurrency)
appr = "%s:0x%x" % (app.main or "__main__", id(app))
if not isinstance(app.loader, AppLoader):
loader = qualname(app.loader)
if loader.startswith("celery.loaders"):
loader = loader[14:]
appr += " (%s)" % loader
if self.autoscale:
concurrency = "{min=%s, max=%s}" % tuple(self.autoscale)
pool = self.pool_cls
if not isinstance(pool, basestring):
pool = pool.__module__
concurrency += " (%s)" % pool.split('.')[-1]
events = "ON"
if not self.send_events:
events = "OFF (enable -E to monitor this worker)"
banner = (BANNER % {
"app": appr,
"hostname": self.hostname,
"version": __version__,
"conninfo": self.app.broker_connection().as_uri(),
"concurrency": concurrency,
"events": events,
"queues": app.amqp.queues.format(indent=0, indent_first=False),
}).splitlines()
# integrate the ASCII art.
for i, x in enumerate(banner):
try:
banner[i] = ' '.join([ARTLINES[i], banner[i]])
except IndexError:
banner[i] = ' ' * 16 + banner[i]
return '\n'.join(banner) + '\n'
def _shutdown(self, warm=True):
what = 'Stopping' if warm else 'Terminating'
socket_timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(SHUTDOWN_SOCKET_TIMEOUT) # Issue 975
if self._state in (self.CLOSE, self.TERMINATE):
return
self.app.loader.shutdown_worker()
if self.pool:
self.pool.close()
if self._state != self.RUN or self._running != len(self.components):
# Not fully started, can safely exit.
self._state = self.TERMINATE
self._shutdown_complete.set()
return
self._state = self.CLOSE
for component in reversed(self.components):
logger.debug('%s %s...', what, qualname(component))
if component:
stop = component.stop
if not warm:
stop = getattr(component, 'terminate', None) or stop
stop()
self.timer.stop()
self.consumer.close_connection()
if self.pidlock:
self.pidlock.release()
self._state = self.TERMINATE
socket.setdefaulttimeout(socket_timeout)
self._shutdown_complete.set()
