def test_revoke(self):
ready_queue = FastQueue()
l = CarrotListener(ready_queue,
self.eta_schedule,
self.logger,
send_events=False)
backend = MockBackend()
id = gen_unique_id()
c = create_message(backend,
control={
"command": "revoke",
"task_id": id
})
t = create_message(backend,
task=foo_task.name,
args=[2, 4, 8],
kwargs={},
id=id)
l.event_dispatcher = MockEventDispatcher()
l.receive_message(c.decode(), c)
from celery.worker.state import revoked
self.assertIn(id, revoked)
l.receive_message(t.decode(), t)
self.assertTrue(ready_queue.empty())
def __init__(self):
self.ready_queue = FastQueue()
self.ready_queue.put(
TaskRequest(task_name=mytask.name,
task_id=gen_unique_id(),
args=(2, 2),
kwargs={}))
self.eta_schedule = Timer()
self.event_dispatcher = Dispatcher()
def __init__(self):
self.ready_queue = FastQueue()
self.timer = Timer()
self.app = current_app
self.event_dispatcher = Mock()
self.controller = WorkController()
self.task_consumer = Mock()
from celery.concurrency.base import BasePool
self.pool = BasePool(10)
def __init__(self):
self.ready_queue = FastQueue()
self.ready_queue.put(
TaskRequest(mytask.name, uuid(), args=(2, 2), kwargs={}))
self.eta_schedule = Timer()
self.app = current_app
self.event_dispatcher = Mock()
self.controller = WorkController()
from celery.concurrency.base import BasePool
self.pool = BasePool(10)
def test_dump_reserved(self):
consumer = Consumer()
panel = self.create_panel(consumer=consumer)
response = panel.handle("dump_reserved", {"safe": True})
self.assertDictContainsSubset({"name": mytask.name,
"args": (2, 2),
"kwargs": {},
"hostname": socket.gethostname()},
response[0])
consumer.ready_queue = FastQueue()
self.assertFalse(panel.handle("dump_reserved"))
def test_revoke(self):
ready_queue = FastQueue()
l = MyKombuConsumer(ready_queue, timer=self.timer)
backend = Mock()
id = uuid()
t = create_message(backend, task=foo_task.name, args=[2, 4, 8],
kwargs={}, id=id)
from celery.worker.state import revoked
revoked.add(id)
l.receive_message(t.decode(), t)
self.assertTrue(ready_queue.empty())
def __init__(self):
self.ready_queue = FastQueue()
self.ready_queue.put(
TaskRequest(task_name=mytask.name,
task_id=gen_unique_id(),
args=(2, 2),
kwargs={}))
self.eta_schedule = Timer()
self.app = app_or_default()
self.event_dispatcher = Mock()
from celery.concurrency.base import BasePool
self.pool = BasePool(10)
def test_revoke(self):
ready_queue = FastQueue()
l = MyKombuConsumer(ready_queue, self.eta_schedule, self.logger,
send_events=False)
backend = Mock()
id = gen_unique_id()
t = create_message(backend, task=foo_task.name, args=[2, 4, 8],
kwargs={}, id=id)
from celery.worker.state import revoked
revoked.add(id)
l.receive_message(t.decode(), t)
self.assertTrue(ready_queue.empty())
def test_dump_reserved(self):
consumer = Consumer()
panel = self.create_panel(consumer=consumer)
response = panel.handle('dump_reserved', {'safe': True})
self.assertDictContainsSubset(
{
'name': mytask.name,
'args': (2, 2),
'kwargs': {},
'hostname': socket.gethostname()
}, response[0])
consumer.ready_queue = FastQueue()
self.assertFalse(panel.handle('dump_reserved'))
def setUp(self):
self.ready_queue = FastQueue()
self.timer = Timer()
def __init__(self,
concurrency=None,
logfile=None,
loglevel=None,
send_events=None,
hostname=None,
ready_callback=noop,
embed_clockservice=False,
pool_cls=None,
consumer_cls=None,
mediator_cls=None,
eta_scheduler_cls=None,
schedule_filename=None,
task_time_limit=None,
task_soft_time_limit=None,
max_tasks_per_child=None,
pool_putlocks=None,
db=None,
prefetch_multiplier=None,
eta_scheduler_precision=None,
disable_rate_limits=None,
autoscale=None,
autoscaler_cls=None,
scheduler_cls=None,
app=None):
self.app = app_or_default(app)
conf = self.app.conf
# Options
self.loglevel = loglevel or self.loglevel
self.concurrency = concurrency or conf.CELERYD_CONCURRENCY
self.logfile = logfile or conf.CELERYD_LOG_FILE
self.logger = self.app.log.get_default_logger()
if send_events is None:
send_events = conf.CELERY_SEND_EVENTS
self.send_events = send_events
self.pool_cls = _concurrency.get_implementation(pool_cls
or conf.CELERYD_POOL)
self.consumer_cls = consumer_cls or conf.CELERYD_CONSUMER
self.mediator_cls = mediator_cls or conf.CELERYD_MEDIATOR
self.eta_scheduler_cls = eta_scheduler_cls or \
conf.CELERYD_ETA_SCHEDULER
self.autoscaler_cls = autoscaler_cls or \
conf.CELERYD_AUTOSCALER
self.schedule_filename = schedule_filename or \
conf.CELERYBEAT_SCHEDULE_FILENAME
self.scheduler_cls = scheduler_cls or conf.CELERYBEAT_SCHEDULER
self.hostname = hostname or socket.gethostname()
self.embed_clockservice = embed_clockservice
self.ready_callback = ready_callback
self.task_time_limit = task_time_limit or \
conf.CELERYD_TASK_TIME_LIMIT
self.task_soft_time_limit = task_soft_time_limit or \
conf.CELERYD_TASK_SOFT_TIME_LIMIT
self.max_tasks_per_child = max_tasks_per_child or \
conf.CELERYD_MAX_TASKS_PER_CHILD
self.pool_putlocks = pool_putlocks or \
conf.CELERYD_POOL_PUTLOCKS
self.eta_scheduler_precision = eta_scheduler_precision or \
conf.CELERYD_ETA_SCHEDULER_PRECISION
self.prefetch_multiplier = prefetch_multiplier or \
conf.CELERYD_PREFETCH_MULTIPLIER
self.timer_debug = SilenceRepeated(self.logger.debug,
max_iterations=10)
self.db = db or conf.CELERYD_STATE_DB
self.disable_rate_limits = disable_rate_limits or \
conf.CELERY_DISABLE_RATE_LIMITS
self._finalize = Finalize(self, self.stop, exitpriority=1)
self._finalize_db = None
if self.db:
persistence = state.Persistent(self.db)
self._finalize_db = Finalize(persistence,
persistence.save,
exitpriority=5)
# Queues
if self.disable_rate_limits:
self.ready_queue = FastQueue()
self.ready_queue.put = self.process_task
else:
self.ready_queue = TaskBucket(task_registry=registry.tasks)
self.logger.debug("Instantiating thread components...")
# Threads + Pool + Consumer
self.autoscaler = None
max_concurrency = None
min_concurrency = concurrency
if autoscale:
max_concurrency, min_concurrency = autoscale
self.pool = instantiate(self.pool_cls,
min_concurrency,
logger=self.logger,
initializer=process_initializer,
initargs=(self.app, self.hostname),
maxtasksperchild=self.max_tasks_per_child,
timeout=self.task_time_limit,
soft_timeout=self.task_soft_time_limit,
putlocks=self.pool_putlocks)
self.priority_timer = instantiate(self.pool.Timer)
if not self.eta_scheduler_cls:
# Default Timer is set by the pool, as e.g. eventlet
# needs a custom implementation.
self.eta_scheduler_cls = self.pool.Timer
if autoscale:
self.autoscaler = instantiate(self.autoscaler_cls,
self.pool,
max_concurrency=max_concurrency,
min_concurrency=min_concurrency,
logger=self.logger)
self.mediator = None
if not self.disable_rate_limits:
self.mediator = instantiate(self.mediator_cls,
self.ready_queue,
app=self.app,
callback=self.process_task,
logger=self.logger)
self.scheduler = instantiate(self.eta_scheduler_cls,
precision=eta_scheduler_precision,
on_error=self.on_timer_error,
on_tick=self.on_timer_tick)
self.beat = None
if self.embed_clockservice:
self.beat = beat.EmbeddedService(
app=self.app,
logger=self.logger,
schedule_filename=self.schedule_filename,
scheduler_cls=self.scheduler_cls)
prefetch_count = self.concurrency * self.prefetch_multiplier
self.consumer = instantiate(self.consumer_cls,
self.ready_queue,
self.scheduler,
logger=self.logger,
hostname=self.hostname,
send_events=self.send_events,
init_callback=self.ready_callback,
initial_prefetch_count=prefetch_count,
pool=self.pool,
priority_timer=self.priority_timer,
app=self.app)
# The order is important here;
# the first in the list is the first to start,
# and they must be stopped in reverse order.
self.components = filter(None,
(self.pool, self.mediator, self.scheduler,
self.beat, self.autoscaler, self.consumer))
def setUp(self):
self.ready_queue = FastQueue()
self.eta_schedule = Timer()
self.logger = current_app.log.get_default_logger()
self.logger.setLevel(0)
def __init__(self,
concurrency=None,
logfile=None,
loglevel=None,
send_events=conf.SEND_EVENTS,
hostname=None,
ready_callback=noop,
embed_clockservice=False,
pool_cls=conf.CELERYD_POOL,
listener_cls=conf.CELERYD_LISTENER,
mediator_cls=conf.CELERYD_MEDIATOR,
eta_scheduler_cls=conf.CELERYD_ETA_SCHEDULER,
schedule_filename=conf.CELERYBEAT_SCHEDULE_FILENAME,
task_time_limit=conf.CELERYD_TASK_TIME_LIMIT,
task_soft_time_limit=conf.CELERYD_TASK_SOFT_TIME_LIMIT,
max_tasks_per_child=conf.CELERYD_MAX_TASKS_PER_CHILD,
pool_putlocks=conf.CELERYD_POOL_PUTLOCKS,
db=conf.CELERYD_STATE_DB):
# Options
self.loglevel = loglevel or self.loglevel
self.concurrency = concurrency or self.concurrency
self.logfile = logfile or self.logfile
self.logger = setup_logger(loglevel, logfile)
self.hostname = hostname or socket.gethostname()
self.embed_clockservice = embed_clockservice
self.ready_callback = ready_callback
self.send_events = send_events
self.task_time_limit = task_time_limit
self.task_soft_time_limit = task_soft_time_limit
self.max_tasks_per_child = max_tasks_per_child
self.pool_putlocks = pool_putlocks
self.db = db
self._finalize = Finalize(self, self.stop, exitpriority=1)
if self.db:
persistence = state.Persistent(self.db)
Finalize(persistence, persistence.save, exitpriority=5)
# Queues
if conf.DISABLE_RATE_LIMITS:
self.ready_queue = FastQueue()
else:
self.ready_queue = TaskBucket(task_registry=registry.tasks)
self.eta_schedule = Scheduler(self.ready_queue, logger=self.logger)
self.logger.debug("Instantiating thread components...")
# Threads + Pool + Consumer
self.pool = instantiate(pool_cls,
self.concurrency,
logger=self.logger,
initializer=process_initializer,
maxtasksperchild=self.max_tasks_per_child,
timeout=self.task_time_limit,
soft_timeout=self.task_soft_time_limit,
putlocks=self.pool_putlocks)
self.mediator = instantiate(mediator_cls,
self.ready_queue,
callback=self.process_task,
logger=self.logger)
self.scheduler = instantiate(eta_scheduler_cls,
self.eta_schedule,
logger=self.logger)
self.clockservice = None
if self.embed_clockservice:
self.clockservice = EmbeddedClockService(
logger=self.logger, schedule_filename=schedule_filename)
prefetch_count = self.concurrency * conf.CELERYD_PREFETCH_MULTIPLIER
self.listener = instantiate(listener_cls,
self.ready_queue,
self.eta_schedule,
logger=self.logger,
hostname=self.hostname,
send_events=self.send_events,
init_callback=self.ready_callback,
initial_prefetch_count=prefetch_count,
pool=self.pool)
# The order is important here;
# the first in the list is the first to start,
# and they must be stopped in reverse order.
self.components = filter(None,
(self.pool, self.mediator, self.scheduler,
self.clockservice, self.listener))
def setUp(self):
self.ready_queue = FastQueue()
self.eta_schedule = Scheduler(self.ready_queue)
self.logger = get_logger()
self.logger.setLevel(0)
def setUp(self):
self.ready_queue = FastQueue()
self.eta_schedule = Timer()
