def on_after_init(self, purge=False, no_color=None, redirect_stdouts=None, redirect_stdouts_level=None, **kwargs):
self.redirect_stdouts = self.app.either("worker_redirect_stdouts", redirect_stdouts)
self.redirect_stdouts_level = self.app.either("worker_redirect_stdouts_level", redirect_stdouts_level)
super(Worker, self).setup_defaults(**kwargs)
self.purge = purge
self.no_color = no_color
self._isatty = isatty(sys.stdout)
self.colored = self.app.log.colored(self.logfile, enabled=not no_color if no_color is not None else no_color)
def __init__(self, concurrency=None, loglevel=None, logfile=None,
hostname=None, discard=False, run_clockservice=False,
schedule=None, task_time_limit=None, task_soft_time_limit=None,
max_tasks_per_child=None, queues=None, events=False, db=None,
include=None, app=None, pidfile=None,
redirect_stdouts=None, redirect_stdouts_level=None,
autoscale=None, scheduler_cls=None, pool=None, **kwargs):
self.app = app = app_or_default(app)
self.concurrency = (concurrency or
app.conf.CELERYD_CONCURRENCY or
multiprocessing.cpu_count())
self.loglevel = loglevel or app.conf.CELERYD_LOG_LEVEL
self.logfile = logfile or app.conf.CELERYD_LOG_FILE
app.conf.CELERYD_LOG_COLOR = not self.logfile and isatty(sys.stderr)
self.hostname = hostname or socket.gethostname()
self.discard = discard
self.run_clockservice = run_clockservice
self.schedule = schedule or app.conf.CELERYBEAT_SCHEDULE_FILENAME
self.scheduler_cls = scheduler_cls or app.conf.CELERYBEAT_SCHEDULER
self.events = events
self.task_time_limit = (task_time_limit or
app.conf.CELERYD_TASK_TIME_LIMIT)
self.task_soft_time_limit = (task_soft_time_limit or
app.conf.CELERYD_TASK_SOFT_TIME_LIMIT)
self.max_tasks_per_child = (max_tasks_per_child or
app.conf.CELERYD_MAX_TASKS_PER_CHILD)
self.redirect_stdouts = (redirect_stdouts or
app.conf.CELERY_REDIRECT_STDOUTS)
self.redirect_stdouts_level = (redirect_stdouts_level or
app.conf.CELERY_REDIRECT_STDOUTS_LEVEL)
self.pool = (pool or app.conf.CELERYD_POOL)
self.db = db
self.use_queues = queues or []
self.queues = None
self.include = include or []
self.pidfile = pidfile
self.autoscale = None
if autoscale:
max_c, _, min_c = partition(autoscale, ",")
self.autoscale = [int(max_c), min_c and int(min_c) or 0]
self._isatty = sys.stdout.isatty()
self.colored = term.colored(enabled=app.conf.CELERYD_LOG_COLOR)
if isinstance(self.use_queues, basestring):
self.use_queues = self.use_queues.split(",")
if isinstance(self.include, basestring):
self.include = self.include.split(",")
if not isinstance(self.loglevel, int):
try:
self.loglevel = LOG_LEVELS[self.loglevel.upper()]
except KeyError:
self.die("Unknown level %r. Please use one of %s." % (
self.loglevel,
"|".join(l for l in LOG_LEVELS.keys()
if isinstance(l, basestring))))
def supports_color(self, logfile=None):
if self.app.IS_WINDOWS:
# Windows does not support ANSI color codes.
return False
if self.colorize is None:
# Only use color if there is no active log file
# and stderr is an actual terminal.
return logfile is None and isatty(sys.stderr)
return self.colorize
def supports_color(self, logfile=None):
if self.app.IS_WINDOWS:
# Windows does not support ANSI color codes.
return False
if self.colorize is None:
# Only use color if there is no active log file
# and stderr is an actual terminal.
return logfile is None and isatty(sys.stderr)
return self.colorize
def __init__(self,
hostname=None,
purge=False,
beat=False,
queues=None,
include=None,
app=None,
pidfile=None,
autoscale=None,
autoreload=False,
no_execv=False,
no_color=None,
**kwargs):
self.app = app = app_or_default(app or self.app)
self.hostname = hostname or socket.gethostname()
# this signal can be used to set up configuration for
# workers by name.
signals.celeryd_init.send(sender=self.hostname,
instance=self,
conf=self.app.conf)
self.setup_defaults(kwargs, namespace='celeryd')
if not self.concurrency:
try:
self.concurrency = cpu_count()
except NotImplementedError:
self.concurrency = 2
self.purge = purge
self.beat = beat
self.use_queues = [] if queues is None else queues
self.queues = None
self.include = include
self.pidfile = pidfile
self.autoscale = None
self.autoreload = autoreload
self.no_color = no_color
self.no_execv = no_execv
if autoscale:
max_c, _, min_c = autoscale.partition(',')
self.autoscale = [int(max_c), min_c and int(min_c) or 0]
self._isatty = isatty(sys.stdout)
self.colored = app.log.colored(
self.logfile,
enabled=not no_color if no_color is not None else no_color)
if isinstance(self.use_queues, basestring):
self.use_queues = self.use_queues.split(',')
if self.include:
if isinstance(self.include, basestring):
self.include = self.include.split(',')
app.conf.CELERY_INCLUDE = (tuple(app.conf.CELERY_INCLUDE) +
tuple(self.include))
self.loglevel = mlevel(self.loglevel)
def __init__(self,
hostname=None,
discard=False,
embed_clockservice=False,
queues=None,
include=None,
app=None,
pidfile=None,
autoscale=None,
autoreload=False,
**kwargs):
self.app = app = app_or_default(app or self.app)
self.hostname = hostname or socket.gethostname()
# this signal can be used to set up configuration for
# workers by name.
signals.celeryd_init.send(sender=self.hostname,
instance=self,
conf=self.app.conf)
self.setup_defaults(kwargs, namespace="celeryd")
if not self.concurrency:
try:
self.concurrency = cpu_count()
except NotImplementedError:
self.concurrency = 2
self.discard = discard
self.embed_clockservice = embed_clockservice
if self.app.IS_WINDOWS and self.embed_clockservice:
self.die("-B option does not work on Windows. "
"Please run celerybeat as a separate service.")
self.use_queues = [] if queues is None else queues
self.queues = None
self.include = [] if include is None else include
self.pidfile = pidfile
self.autoscale = None
self.autoreload = autoreload
if autoscale:
max_c, _, min_c = autoscale.partition(",")
self.autoscale = [int(max_c), min_c and int(min_c) or 0]
self._isatty = isatty(sys.stdout)
self.colored = app.log.colored(self.logfile)
if isinstance(self.use_queues, basestring):
self.use_queues = self.use_queues.split(",")
if isinstance(self.include, basestring):
self.include = self.include.split(",")
try:
self.loglevel = mlevel(self.loglevel)
except KeyError:
self.die("Unknown level %r. Please use one of %s." %
(self.loglevel, "|".join(l for l in LOG_LEVELS.keys()
if isinstance(l, basestring))))
def on_before_init(self, purge=False, no_color=None, **kwargs):
# apply task execution optimizations
trace.setup_worker_optimizations(self.app)
# this signal can be used to set up configuration for
# workers by name.
conf = self.app.conf
signals.celeryd_init.send(sender=self.hostname, instance=self, conf=conf)
self.purge = purge
self.no_color = no_color
self._isatty = isatty(sys.stdout)
self.colored = self.app.log.colored(self.logfile, enabled=not no_color if no_color is not None else no_color)
def __init__(
self,
hostname=None,
purge=False,
beat=False,
queues=None,
include=None,
app=None,
pidfile=None,
autoscale=None,
autoreload=False,
no_execv=False,
no_color=None,
**kwargs
):
self.app = app = app_or_default(app or self.app)
self.hostname = hostname or socket.gethostname()
# this signal can be used to set up configuration for
# workers by name.
signals.celeryd_init.send(sender=self.hostname, instance=self, conf=self.app.conf)
self.setup_defaults(kwargs, namespace="celeryd")
if not self.concurrency:
try:
self.concurrency = cpu_count()
except NotImplementedError:
self.concurrency = 2
self.purge = purge
self.beat = beat
self.use_queues = [] if queues is None else queues
self.queues = None
self.include = include
self.pidfile = pidfile
self.autoscale = None
self.autoreload = autoreload
self.no_color = no_color
self.no_execv = no_execv
if autoscale:
max_c, _, min_c = autoscale.partition(",")
self.autoscale = [int(max_c), min_c and int(min_c) or 0]
self._isatty = isatty(sys.stdout)
self.colored = app.log.colored(self.logfile, enabled=not no_color if no_color is not None else no_color)
if isinstance(self.use_queues, basestring):
self.use_queues = self.use_queues.split(",")
if self.include:
if isinstance(self.include, basestring):
self.include = self.include.split(",")
app.conf.CELERY_INCLUDE = tuple(app.conf.CELERY_INCLUDE) + tuple(self.include)
self.loglevel = mlevel(self.loglevel)
def __init__(self, app,
block_timeout=30 * 60, no_color=False,
stdout=None, stderr=None):
self.app = app
self.stdout = sys.stdout if stdout is None else stdout
self.stderr = sys.stderr if stderr is None else stderr
if not isatty(self.stdout):
no_color = True
self.colored = colored(enabled=not no_color)
self.connerrors = self.app.connection().recoverable_connection_errors
self.block_timeout = block_timeout
self.progress = None
self.speaker = Speaker(file=self.stdout)
self.fbi = FBI(app)
self.init_groups()
def on_before_init(self, purge=False, redirect_stdouts=None,
redirect_stdouts_level=None, **kwargs):
# this signal can be used to set up configuration for
# workers by name.
conf = self.app.conf
signals.celeryd_init.send(sender=self.hostname, instance=self,
conf=conf)
self.purge = purge
self._isatty = isatty(sys.stdout)
self.colored = self.app.log.colored(self.logfile)
if redirect_stdouts is None:
redirect_stdouts = conf.CELERY_REDIRECT_STDOUTS,
if redirect_stdouts_level is None:
redirect_stdouts_level = conf.CELERY_REDIRECT_STDOUTS_LEVEL
self.redirect_stdouts = redirect_stdouts
self.redirect_stdouts_level = redirect_stdouts_level
def on_before_init(self, purge=False, no_color=None, **kwargs):
# apply task execution optimizations
trace.setup_worker_optimizations(self.app)
# this signal can be used to set up configuration for
# workers by name.
conf = self.app.conf
signals.celeryd_init.send(
sender=self.hostname, instance=self, conf=conf,
)
self.purge = purge
self.no_color = no_color
self._isatty = isatty(sys.stdout)
self.colored = self.app.log.colored(self.logfile,
enabled=not no_color if no_color is not None else no_color
)
def on_after_init(self, purge=False, no_color=None,
redirect_stdouts=None, redirect_stdouts_level=None,
**kwargs):
self.redirect_stdouts = self.app.either(
'worker_redirect_stdouts', redirect_stdouts,
)
self.redirect_stdouts_level = self.app.either(
'worker_redirect_stdouts_level', redirect_stdouts_level,
)
super(Worker, self).setup_defaults(**kwargs)
self.purge = purge
self.no_color = no_color
self._isatty = isatty(sys.stdout)
self.colored = self.app.log.colored(
self.logfile,
enabled=not no_color if no_color is not None else no_color
)
def __init__(self,
app,
block_timeout=30 * 60,
no_color=False,
stdout=None,
stderr=None):
self.app = app
self.stdout = sys.stdout if stdout is None else stdout
self.stderr = sys.stderr if stderr is None else stderr
if not isatty(self.stdout):
no_color = True
self.colored = colored(enabled=not no_color)
self.connerrors = self.app.connection().recoverable_connection_errors
self.block_timeout = block_timeout
self.progress = None
self.speaker = Speaker(file=self.stdout)
self.fbi = FBI(app)
self.init_groups()
def __init__(self, hostname=None, discard=False, embed_clockservice=False,
queues=None, include=None, app=None, pidfile=None,
autoscale=None, autoreload=False, **kwargs):
self.app = app = app_or_default(app or self.app)
self.hostname = hostname or socket.gethostname()
# this signal can be used to set up configuration for
# workers by name.
signals.celeryd_init.send(sender=self.hostname, instance=self,
conf=self.app.conf)
self.setup_defaults(kwargs, namespace="celeryd")
if not self.concurrency:
self.concurrency = cpu_count()
self.discard = discard
self.embed_clockservice = embed_clockservice
if self.app.IS_WINDOWS and self.embed_clockservice:
self.die("-B option does not work on Windows. "
"Please run celerybeat as a separate service.")
self.use_queues = [] if queues is None else queues
self.queues = None
self.include = [] if include is None else include
self.pidfile = pidfile
self.autoscale = None
self.autoreload = autoreload
if autoscale:
max_c, _, min_c = autoscale.partition(",")
self.autoscale = [int(max_c), min_c and int(min_c) or 0]
self._isatty = isatty(sys.stdout)
self.colored = app.log.colored(self.logfile)
if isinstance(self.use_queues, basestring):
self.use_queues = self.use_queues.split(",")
if isinstance(self.include, basestring):
self.include = self.include.split(",")
try:
self.loglevel = mlevel(self.loglevel)
except KeyError:
self.die("Unknown level %r. Please use one of %s." % (
self.loglevel,
"|".join(l for l in LOG_LEVELS.keys()
if isinstance(l, basestring))))
def post_config_merge(self, c):
"""Prepare configuration after it has been merged with the
defaults."""
if not c.get("CELERY_QUEUES"):
c["CELERY_QUEUES"] = {
c.CELERY_DEFAULT_QUEUE: {
"exchange": c.CELERY_DEFAULT_EXCHANGE,
"exchange_type": c.CELERY_DEFAULT_EXCHANGE_TYPE,
"binding_key": c.CELERY_DEFAULT_ROUTING_KEY}}
c["CELERY_ROUTES"] = routes.prepare(c.get("CELERY_ROUTES") or {})
if c.get("CELERYD_LOG_COLOR") is None:
c["CELERYD_LOG_COLOR"] = not c.CELERYD_LOG_FILE and \
isatty(sys.stderr)
if self.IS_WINDOWS: # windows console doesn't support ANSI colors
c["CELERYD_LOG_COLOR"] = False
if isinstance(c.CELERY_TASK_RESULT_EXPIRES, int):
c["CELERY_TASK_RESULT_EXPIRES"] = timedelta(
seconds=c.CELERY_TASK_RESULT_EXPIRES)
return c
def on_before_init(self,
purge=False,
redirect_stdouts=None,
redirect_stdouts_level=None,
**kwargs):
# this signal can be used to set up configuration for
# workers by name.
conf = self.app.conf
signals.celeryd_init.send(sender=self.hostname,
instance=self,
conf=conf)
self.purge = purge
self._isatty = isatty(sys.stdout)
self.colored = self.app.log.colored(self.logfile)
if redirect_stdouts is None:
redirect_stdouts = conf.CELERY_REDIRECT_STDOUTS,
if redirect_stdouts_level is None:
redirect_stdouts_level = conf.CELERY_REDIRECT_STDOUTS_LEVEL
self.redirect_stdouts = redirect_stdouts
self.redirect_stdouts_level = redirect_stdouts_level
def test_tty(self):
fh = Mock(name='fh')
self.assertIs(isatty(fh), fh.isatty())
fh.isatty.side_effect = AttributeError()
self.assertFalse(isatty(fh))
def test_tty(self):
fh = Mock(name='fh')
self.assertIs(isatty(fh), fh.isatty())
fh.isatty.side_effect = AttributeError()
self.assertFalse(isatty(fh))