def x_start_stop(self):
p = TaskPool(limit=2)
p.start()
self.assertTrue(p._pool)
self.assertTrue(isinstance(p._processes, dict))
p.stop()
self.assertTrue(p._pool is None)
self.assertFalse(p._processes)
def test_execute_using_pool(self):
tid = gen_unique_id()
tw = TaskWrapper("cu.mytask", tid, mytask, [4], {"f": "x"})
p = TaskPool(2)
p.start()
asyncres = tw.execute_using_pool(p)
self.assertTrue(asyncres.get(), 256)
p.stop()
def test_is_full(self):
p = TaskPool(2)
p.start()
self.assertFalse(p.full())
results = [p.apply_async(long_something) for i in xrange(4)]
self.assertTrue(p.full())
p.stop()
def x_apply(self):
p = TaskPool(limit=2)
p.start()
scratchpad = {}
proc_counter = itertools.count().next
def mycallback(ret_value, meta):
process = proc_counter()
scratchpad[process] = {}
scratchpad[process]["ret_value"] = ret_value
scratchpad[process]["meta"] = meta
myerrback = mycallback
res = p.apply_async(do_something, args=[10], callbacks=[mycallback],
meta={"foo": "bar"})
res2 = p.apply_async(raise_something, args=[10], errbacks=[myerrback],
meta={"foo2": "bar2"})
res3 = p.apply_async(do_something, args=[20], callbacks=[mycallback],
meta={"foo3": "bar3"})
self.assertEquals(res.get(), 100)
time.sleep(0.5)
self.assertTrue(scratchpad.get(0))
self.assertEquals(scratchpad[0]["ret_value"], 100)
self.assertEquals(scratchpad[0]["meta"], {"foo": "bar"})
self.assertTrue(isinstance(res2.get(), ExceptionInfo))
self.assertTrue(scratchpad.get(1))
time.sleep(1)
#self.assertEquals(scratchpad[1]["ret_value"], "FOO")
self.assertTrue(isinstance(scratchpad[1]["ret_value"],
ExceptionInfo))
self.assertEquals(scratchpad[1]["ret_value"].exception.args,
("FOO EXCEPTION", ))
self.assertEquals(scratchpad[1]["meta"], {"foo2": "bar2"})
self.assertEquals(res3.get(), 400)
time.sleep(0.5)
self.assertTrue(scratchpad.get(2))
self.assertEquals(scratchpad[2]["ret_value"], 400)
self.assertEquals(scratchpad[2]["meta"], {"foo3": "bar3"})
res3 = p.apply_async(do_something, args=[30], callbacks=[mycallback],
meta={"foo4": "bar4"})
self.assertEquals(res3.get(), 900)
time.sleep(0.5)
self.assertTrue(scratchpad.get(3))
self.assertEquals(scratchpad[3]["ret_value"], 900)
self.assertEquals(scratchpad[3]["meta"], {"foo4": "bar4"})
p.stop()
def __init__(self, concurrency=None, logfile=None, loglevel=None,
queue_wakeup_after=None, is_detached=False):
self.loglevel = loglevel or self.loglevel
self.concurrency = concurrency or self.concurrency
self.logfile = logfile or self.logfile
self.queue_wakeup_after = queue_wakeup_after or \
self.queue_wakeup_after
self.logger = setup_logger(loglevel, logfile)
self.pool = TaskPool(self.concurrency, logger=self.logger)
self.task_consumer = None
self.task_consumer_it = None
self.is_detached = is_detached
self.reset_connection()
def test_get_worker_pids(self):
p = TaskPool(5)
p.start()
self.assertEquals(len(p.get_worker_pids()), 5)
p.stop()
def x_start_stop(self):
p = TaskPool(limit=2)
p.start()
self.assertTrue(p._pool)
p.stop()
self.assertTrue(p._pool is None)
class WorkController(object):
"""Executes tasks waiting in the task queue.
:param concurrency: see :attr:`concurrency`.
:param logfile: see :attr:`logfile`.
:param loglevel: see :attr:`loglevel`.
:param queue_wakeup_after: see :attr:`queue_wakeup_after`.
.. attribute:: concurrency
The number of simultaneous processes doing work (default:
:const:`celery.conf.DAEMON_CONCURRENCY`)
.. attribute:: loglevel
The loglevel used (default: :const:`logging.INFO`)
.. attribute:: logfile
The logfile used, if no logfile is specified it uses ``stderr``
(default: :const:`celery.conf.DAEMON_LOG_FILE`).
.. attribute:: queue_wakeup_after
The time it takes for the daemon to wake up after the queue is empty,
so it can check for more work
(default: :const:`celery.conf.QUEUE_WAKEUP_AFTER`).
.. attribute:: empty_msg_emit_every
How often the daemon emits the ``"Waiting for queue..."`` message.
If this is ``None``, the message will never be logged.
(default: :const:`celery.conf.EMPTY_MSG_EMIT_EVERY`)
.. attribute:: logger
The :class:`logging.Logger` instance used for logging.
.. attribute:: pool
The :class:`multiprocessing.Pool` instance used.
.. attribute:: task_consumer
The :class:`celery.messaging.TaskConsumer` instance used.
"""
loglevel = logging.ERROR
concurrency = DAEMON_CONCURRENCY
logfile = DAEMON_LOG_FILE
queue_wakeup_after = QUEUE_WAKEUP_AFTER
empty_msg_emit_every = EMPTY_MSG_EMIT_EVERY
def __init__(self, concurrency=None, logfile=None, loglevel=None,
queue_wakeup_after=None, is_detached=False):
self.loglevel = loglevel or self.loglevel
self.concurrency = concurrency or self.concurrency
self.logfile = logfile or self.logfile
self.queue_wakeup_after = queue_wakeup_after or \
self.queue_wakeup_after
self.logger = setup_logger(loglevel, logfile)
self.pool = TaskPool(self.concurrency, logger=self.logger)
self.task_consumer = None
self.task_consumer_it = None
self.is_detached = is_detached
self.reset_connection()
def reset_connection(self):
"""Reset the AMQP connection, and reinitialize the
:class:`celery.messaging.TaskConsumer` instance.
Resets the task consumer in :attr:`task_consumer`.
"""
if self.task_consumer:
self.task_consumer.connection.close()
amqp_connection = DjangoAMQPConnection()
self.task_consumer = TaskConsumer(connection=amqp_connection)
self.task_consumer_it = self.task_consumer.iterqueue(infinite=True)
def connection_diagnostics(self):
"""Diagnose the AMQP connection, and reset connection if
necessary."""
connection = self.task_consumer.backend.channel.connection
if not connection:
self.logger.info(
"AMQP Connection has died, restoring connection.")
self.reset_connection()
def receive_message(self):
"""Receive the next message from the message broker.
Tries to reset the AMQP connection if not available.
Returns ``None`` if no message is waiting on the queue.
:rtype: :class:`carrot.messaging.Message` instance.
"""
message = self.task_consumer_it.next()
if not message:
raise EmptyQueue()
return message
def process_task(self, message):
"""Process task message by passing it to the pool of workers."""
task = TaskWrapper.from_message(message, logger=self.logger)
self.logger.info("Got task from broker: %s[%s]" % (
task.task_name, task.task_id))
self.logger.debug("Got a task: %s. Trying to execute it..." % task)
result = task.execute_using_pool(self.pool, self.loglevel,
self.logfile)
self.logger.debug("Task %s has been executed asynchronously." % task)
return result
def execute_next_task(self):
"""Execute the next task on the queue using the multiprocessing pool.
Catches all exceptions and logs them with level
:const:`logging.CRITICAL`.
Raises :exc:`EmptyQueue` exception if there is no message
waiting on the queue.
"""
self.process_task(self.receive_message())
def schedule_retry_tasks(self):
"""Reschedule all requeued tasks waiting for retry."""
pass
def run(self):
"""Starts the workers main loop."""
log_wait = lambda: self.logger.info("Waiting for queue...")
ev_msg_waiting = EventTimer(log_wait, self.empty_msg_emit_every)
self.pool.run()
PeriodicWorkController().start()
# If not running as daemon, and DEBUG logging level is enabled,
# print pool PIDs and sleep for a second before we start.
if self.logger.isEnabledFor(logging.DEBUG):
self.logger.debug("Pool child processes: [%s]" % (
"|".join(map(str, self.pool.get_worker_pids()))))
if not self.is_detached:
time.sleep(1)
while True:
try:
self.execute_next_task()
except ValueError:
# execute_next_task didn't return a r/name/id tuple,
# probably because it got an exception.
continue
except EmptyQueue:
ev_msg_waiting.tick()
time.sleep(self.queue_wakeup_after)
continue
except UnknownTask, exc:
self.logger.info("Unknown task ignored: %s" % (exc))
continue
except Exception, exc:
self.logger.critical("Message queue raised %s: %s\n%s" % (
exc.__class__, exc, traceback.format_exc()))
continue
