def _send_message(routing_key, request):
conn = DjangoAMQPConnection()
publisher = Publisher(connection=conn, exchange='request',
exchange_type='topic', routing_key=routing_key,
serializer='pickle')
publisher.send(request)
publisher.close()
conn.close()
def receive_a_message():
logger = get_logger()
conn = DjangoAMQPConnection()
m = MyMessager(connection=conn).fetch()
if m:
msg = simplejson.loads(m.body)
logger.info("Message receieved: %s" % msg.get("message"))
m.ack()
conn.close()
def setUp(self):
conn = DjangoAMQPConnection()
consumer = StatsConsumer(connection=conn)
consumer.discard_all()
conn.close()
consumer.close()
self.s = StatsCollector()
self.assertEquals(self.s.total_tasks_processed, 0)
self.assertEquals(self.s.total_tasks_processed_by_type, {})
self.assertEquals(self.s.total_task_time_running, 0.0)
self.assertEquals(self.s.total_task_time_running_by_type, {})
def publish(self, **data):
"""Publish statistics to be collected later by
:class:`StatsCollector`.
:param data: An arbitrary Python object containing the statistics
to be published.
"""
if not self.enabled:
return
connection = DjangoAMQPConnection()
publisher = StatsPublisher(connection=connection)
publisher.send({"type": self.type, "data": data})
publisher.close()
connection.close()
def even_time_distribution(task, size, time_window, iterable, **apply_kwargs):
"""With an iterator yielding task args, kwargs tuples, evenly distribute
the processing of its tasks throughout the time window available.
:param task: The kind of task (a :class:`celery.task.base.Task`.)
:param size: Total number of elements the iterator gives.
:param time_window: Total time available, in minutes.
:param iterable: Iterable yielding task args, kwargs tuples.
:param \*\*apply_kwargs: Additional keyword arguments to be passed on to
:func:`celery.execute.apply_async`.
Example
>>> class RefreshAllFeeds(Task):
...
... def run(self, **kwargs):
... feeds = Feed.objects.all()
... total = feeds.count()
...
... time_window = REFRESH_FEEDS_EVERY_INTERVAL_MINUTES
...
... def iter_feed_task_args(iterable):
... for feed in iterable:
... yield ([feed.feed_url], {}) # args, kwargs tuple
...
... it = iter_feed_task_args(feeds.iterator())
...
... even_time_distribution(RefreshFeedTask, total,
... time_window, it)
"""
bucketsize = size / time_window
buckets = chunks(iterable, int(bucketsize))
connection = DjangoAMQPConnection()
try:
for bucket_count, bucket in enumerate(buckets):
# Skew the countdown for items in this bucket by one.
seconds_eta = (60 * bucket_count if bucket_count else None)
for args, kwargs in bucket:
task.apply_async(args=args, kwargs=kwargs,
connection=connection,
countdown=seconds_eta,
**apply_kwargs)
finally:
connection.close()
def discard_all():
"""Discard all waiting tasks.
This will ignore all tasks waiting for execution, and they will
be deleted from the messaging server.
:returns: the number of tasks discarded.
:rtype: int
"""
amqp_connection = DjangoAMQPConnection()
consumer = TaskConsumer(connection=amqp_connection)
discarded_count = consumer.discard_all()
amqp_connection.close()
return discarded_count
def run(self, connect_timeout=AMQP_CONNECTION_TIMEOUT):
"""Run all tasks in the taskset.
:returns: A :class:`celery.result.TaskSetResult` instance.
Example
>>> ts = TaskSet(RefreshFeedTask, [
... ["http://foo.com/rss", {}],
... ["http://bar.com/rss", {}],
... )
>>> result = ts.run()
>>> result.taskset_id
"d2c9b261-8eff-4bfb-8459-1e1b72063514"
>>> result.subtask_ids
["b4996460-d959-49c8-aeb9-39c530dcde25",
"598d2d18-ab86-45ca-8b4f-0779f5d6a3cb"]
>>> result.waiting()
True
>>> time.sleep(10)
>>> result.ready()
True
>>> result.successful()
True
>>> result.failed()
False
>>> result.join()
[True, True]
"""
taskset_id = gen_unique_id()
from celery.conf import ALWAYS_EAGER
if ALWAYS_EAGER:
subtasks = [apply(self.task, args, kwargs)
for args, kwargs in self.arguments]
return TaskSetResult(taskset_id, subtasks)
conn = DjangoAMQPConnection(connect_timeout=connect_timeout)
publisher = TaskPublisher(connection=conn,
exchange=self.task.exchange)
subtasks = [apply_async(self.task, args, kwargs,
taskset_id=taskset_id, publisher=publisher)
for args, kwargs in self.arguments]
publisher.close()
conn.close()
return TaskSetResult(taskset_id, subtasks)
def run(self):
"""Run all tasks in the taskset.
:returns: A :class:`celery.result.TaskSetResult` instance.
Example
>>> ts = TaskSet(RefreshFeedTask, [
... ["http://foo.com/rss", {}],
... ["http://bar.com/rss", {}],
... )
>>> result = ts.run()
>>> result.taskset_id
"d2c9b261-8eff-4bfb-8459-1e1b72063514"
>>> result.subtask_ids
["b4996460-d959-49c8-aeb9-39c530dcde25",
"598d2d18-ab86-45ca-8b4f-0779f5d6a3cb"]
>>> result.waiting()
True
>>> time.sleep(10)
>>> result.ready()
True
>>> result.successful()
True
>>> result.failed()
False
>>> result.join()
[True, True]
"""
taskset_id = str(uuid.uuid4())
conn = DjangoAMQPConnection()
publisher = TaskPublisher(connection=conn)
subtask_ids = [
publisher.delay_task_in_set(
task_name=self.task_name, taskset_id=taskset_id, task_args=arg, task_kwargs=kwarg
)
for arg, kwarg in self.arguments
]
publisher.close()
conn.close()
return TaskSetResult(taskset_id, subtask_ids)
def apply_async(
task, args=None, kwargs=None, routing_key=None, immediate=None, mandatory=None, connect_timeout=None, priority=None
):
"""Run a task asynchronously by the celery daemon(s).
:param task: The task to run (a callable object, or a :class:`Task`
instance
:param args: The positional arguments to pass on to the task (a ``list``).
:param kwargs: The keyword arguments to pass on to the task (a ``dict``)
:keyword routing_key: The routing key used to route the task to a worker
server.
:keyword immediate: Request immediate delivery. Will raise an exception
if the task cannot be routed to a worker immediately.
:keyword mandatory: Mandatory routing. Raises an exception if there's
no running workers able to take on this task.
:keyword connect_timeout: The timeout in seconds, before we give up
on establishing a connection to the AMQP server.
:keyword priority: The task priority, a number between ``0`` and ``9``.
"""
if not args:
args = []
if not kwargs:
kwargs = []
message_opts = {"routing_key": routing_key, "immediate": immediate, "mandatory": mandatory, "priority": priority}
for option_name, option_value in message_opts.items():
message_opts[option_name] = getattr(task, option_name, option_value)
conn = DjangoAMQPConnection(connect_timeout=connect_timeout)
publisher = TaskPublisher(connection=conn)
task_id = publisher.delay_task(task.name, args, kwargs, **message_opts)
publisher.close()
conn.close()
return AsyncResult(task_id)
def save_hits():
conn = DjangoAMQPConnection()
consumer = Consumer(connection=conn, queue='requestrequesthitqueue',
exchange='request',
routing_key='request.*', exchange_type='topic')
count = 0
messages = []
for message in consumer.iterqueue():
messages.append(message)
request = message.decode()
request.path = convert_unicode_to_string(request.path[:255])
request.data = convert_unicode_to_string(request.data[:255])
request.referer = convert_unicode_to_string(request.referer[:255])
request.user_agent =\
convert_unicode_to_string(request.user_agent[:255])
request.language = convert_unicode_to_string(request.language[:255])
request.save()
count += 1
logger.info("Saved {0} requests".format(count))
[m.ack() for m in messages]
logger.debug("Acknowledged all messages")
consumer.close()
conn.close()
def apply_async(task, args=None, kwargs=None, countdown=None, eta=None,
routing_key=None, exchange=None,
immediate=None, mandatory=None, priority=None, connection=None,
connect_timeout=AMQP_CONNECTION_TIMEOUT, **opts):
"""Run a task asynchronously by the celery daemon(s).
:param task: The task to run (a callable object, or a :class:`Task`
instance
:param args: The positional arguments to pass on to the task (a ``list``).
:param kwargs: The keyword arguments to pass on to the task (a ``dict``)
:param countdown: Number of seconds into the future that the task should
execute. Defaults to immediate delivery (Do not confuse that with
the ``immediate`` setting, they are unrelated).
:param eta: A :class:`datetime.datetime` object that describes the
absolute time when the task should execute. May not be specified
if ``countdown`` is also supplied. (Do not confuse this with the
``immediate`` setting, they are unrelated).
:keyword routing_key: The routing key used to route the task to a worker
server.
:keyword exchange: The named exchange to send the task to. Defaults to
:attr:`celery.task.base.Task.exchange`.
:keyword immediate: Request immediate delivery. Will raise an exception
if the task cannot be routed to a worker immediately.
(Do not confuse this parameter with the ``countdown`` and ``eta``
settings, as they are unrelated).
:keyword mandatory: Mandatory routing. Raises an exception if there's
no running workers able to take on this task.
:keyword connection: Re-use existing AMQP connection.
The ``connect_timeout`` argument is not respected if this is set.
:keyword connect_timeout: The timeout in seconds, before we give up
on establishing a connection to the AMQP server.
:keyword priority: The task priority, a number between ``0`` and ``9``.
"""
args = args or []
kwargs = kwargs or {}
routing_key = routing_key or getattr(task, "routing_key", None)
exchange = exchange or getattr(task, "exchange", None)
immediate = immediate or getattr(task, "immediate", None)
mandatory = mandatory or getattr(task, "mandatory", None)
priority = priority or getattr(task, "priority", None)
taskset_id = opts.get("taskset_id")
publisher = opts.get("publisher")
if countdown:
eta = datetime.now() + timedelta(seconds=countdown)
from celery.conf import ALWAYS_EAGER
if ALWAYS_EAGER:
return apply(task, args, kwargs)
need_to_close_connection = False
if not publisher:
if not connection:
connection = DjangoAMQPConnection(connect_timeout=connect_timeout)
need_to_close_connection = True
publisher = TaskPublisher(connection=connection)
delay_task = publisher.delay_task
if taskset_id:
delay_task = curry(publisher.delay_task_in_set, taskset_id)
task_id = delay_task(task.name, args, kwargs,
routing_key=routing_key, exchange=exchange,
mandatory=mandatory, immediate=immediate,
priority=priority, eta=eta)
if need_to_close_connection:
publisher.close()
connection.close()
return AsyncResult(task_id)
def discard_all():
conn = DjangoAMQPConnection()
MyMessager(connection=conn).consumer.discard_all()
conn.close()
def send_a_message(msg):
conn = DjangoAMQPConnection()
MyMessager(connection=conn).send({"message": msg})
conn.close()
