def reply(routing_key, data):
from carrot.connection import DjangoBrokerConnection
from carrot.messaging import Publisher
conn = DjangoBrokerConnection()
publisher = Publisher(connection=conn, exchange="django", routing_key=routing_key, exchange_type="topic")
publisher.send(data)
publisher.close()
conn.close()
def receive_a_message():
logger = get_logger()
conn = DjangoBrokerConnection()
m = MyMessager(connection=conn).fetch()
if m:
msg = simplejson.loads(m.body)
logger.info("Message receieved: %s" % msg.get("message"))
m.ack()
conn.close()
def setUp(self):
conn = DjangoBrokerConnection()
consumer = StatsConsumer(connection=conn)
consumer.discard_all()
conn.close()
consumer.close()
self.s = StatsCollector()
self.assertEquals(self.s.total_tasks_processed, 0)
self.assertEquals(self.s.total_tasks_processed_by_type, {})
self.assertEquals(self.s.total_task_time_running, 0.0)
self.assertEquals(self.s.total_task_time_running_by_type, {})
def sender_callback(self, routing_key, data):
conn = DjangoBrokerConnection()
publisher = Publisher(connection=conn,
exchange="django_send",
routing_key=routing_key,
exchange_type="topic",
)
publisher.send(data)
publisher.close()
conn.close()
print "Sent object change/delete message for %s" % routing_key
def test_amqp(self):
from carrot.connection import DjangoBrokerConnection
from carrot.messaging import Publisher, Consumer
connection = DjangoBrokerConnection()
publisher = Publisher(connection=connection,
exchange="collector",
exchange_type='topic',
routing_key="collector.driver",
serializer='json')
publisher.send("test")
publisher.close()
connection.close()
def test_amqp(self):
from carrot.connection import DjangoBrokerConnection
from carrot.messaging import Publisher, Consumer
connection = DjangoBrokerConnection()
publisher = Publisher(connection=connection,
exchange="collector",
exchange_type='topic',
routing_key="collector.driver",
serializer='json')
publisher.send("test")
publisher.close()
connection.close()
def publish(self, **data):
"""Publish statistics to be collected later by
:class:`StatsCollector`.
:param data: An arbitrary Python object containing the statistics
to be published.
"""
if not self.enabled:
return
connection = DjangoBrokerConnection()
publisher = StatsPublisher(connection=connection)
publisher.send({"type": self.type, "data": data})
publisher.close()
connection.close()
def even_time_distribution(task, size, time_window, iterable, **apply_kwargs):
"""With an iterator yielding task args, kwargs tuples, evenly distribute
the processing of its tasks throughout the time window available.
:param task: The kind of task (a :class:`celery.task.base.Task`.)
:param size: Total number of elements the iterator gives.
:param time_window: Total time available, in minutes.
:param iterable: Iterable yielding task args, kwargs tuples.
:param \*\*apply_kwargs: Additional keyword arguments to be passed on to
:func:`celery.execute.apply_async`.
Example
>>> class RefreshAllFeeds(Task):
...
... def run(self, **kwargs):
... feeds = Feed.objects.all()
... total = feeds.count()
...
... time_window = REFRESH_FEEDS_EVERY_INTERVAL_MINUTES
...
... def iter_feed_task_args(iterable):
... for feed in iterable:
... yield ([feed.feed_url], {}) # args, kwargs tuple
...
... it = iter_feed_task_args(feeds.iterator())
...
... even_time_distribution(RefreshFeedTask, total,
... time_window, it)
"""
bucketsize = size / time_window
buckets = chunks(iterable, int(bucketsize))
connection = DjangoBrokerConnection()
try:
for bucket_count, bucket in enumerate(buckets):
# Skew the countdown for items in this bucket by one.
seconds_eta = (60 * bucket_count if bucket_count else None)
for args, kwargs in bucket:
task.apply_async(args=args, kwargs=kwargs,
connection=connection,
countdown=seconds_eta,
**apply_kwargs)
finally:
connection.close()
def discard_all(connect_timeout=AMQP_CONNECTION_TIMEOUT):
"""Discard all waiting tasks.
This will ignore all tasks waiting for execution, and they will
be deleted from the messaging server.
:returns: the number of tasks discarded.
:rtype: int
"""
amqp_connection = DjangoBrokerConnection(connect_timeout=connect_timeout)
consumer = TaskConsumer(connection=amqp_connection)
discarded_count = consumer.discard_all()
amqp_connection.close()
return discarded_count
def test_DjangoBrokerConnection(self):
try:
from django.conf import settings
except ImportError:
sys.stderr.write("Django is not installed. \
Not testing django specific features.\n")
return
configured_or_configure(settings,
CARROT_BACKEND=CARROT_BACKEND,
BROKER_HOST=BROKER_HOST,
BROKER_PORT=BROKER_PORT,
BROKER_VHOST=BROKER_VHOST,
BROKER_USER=BROKER_USER,
BROKER_PASSWORD=BROKER_PASSWORD)
expected_values = {
"backend_cls": CARROT_BACKEND,
"hostname": BROKER_HOST,
"port": BROKER_PORT,
"virtual_host": BROKER_VHOST,
"userid": BROKER_USER,
"password": BROKER_PASSWORD
}
conn = DjangoBrokerConnection()
self.assertTrue(isinstance(conn, BrokerConnection))
for val_name, val_value in expected_values.items():
self.assertEquals(getattr(conn, val_name, None), val_value)
def run(self, connect_timeout=conf.AMQP_CONNECTION_TIMEOUT):
"""Run all tasks in the taskset.
:returns: A :class:`celery.result.TaskSetResult` instance.
Example
>>> ts = TaskSet(RefreshFeedTask, args=[
... (["http://foo.com/rss"], {}),
... (["http://bar.com/rss"], {}),
... ])
>>> result = ts.run()
>>> result.taskset_id
"d2c9b261-8eff-4bfb-8459-1e1b72063514"
>>> result.subtask_ids
["b4996460-d959-49c8-aeb9-39c530dcde25",
"598d2d18-ab86-45ca-8b4f-0779f5d6a3cb"]
>>> result.waiting()
True
>>> time.sleep(10)
>>> result.ready()
True
>>> result.successful()
True
>>> result.failed()
False
>>> result.join()
[True, True]
"""
taskset_id = gen_unique_id()
from celery.conf import ALWAYS_EAGER
if ALWAYS_EAGER:
subtasks = [apply(self.task, args, kwargs)
for args, kwargs in self.arguments]
return TaskSetResult(taskset_id, subtasks)
conn = DjangoBrokerConnection(connect_timeout=connect_timeout)
publisher = TaskPublisher(connection=conn,
exchange=self.task.exchange)
subtasks = [apply_async(self.task, args, kwargs,
taskset_id=taskset_id, publisher=publisher)
for args, kwargs in self.arguments]
publisher.close()
conn.close()
return TaskSetResult(taskset_id, subtasks)
def get_connection_info():
broker_connection = DjangoBrokerConnection()
carrot_backend = broker_connection.backend_cls
if carrot_backend and not isinstance(carrot_backend, str):
carrot_backend = carrot_backend.__name__
port = broker_connection.port or \
broker_connection.get_backend_cls().default_port
port = port and ":%s" % port or ""
vhost = broker_connection.virtual_host
if not vhost.startswith("/"):
vhost = "/" + vhost
return "%(carrot_backend)s://%(userid)s@%(host)s%(port)s%(vhost)s" % {
"carrot_backend": carrot_backend,
"userid": broker_connection.userid,
"host": broker_connection.hostname,
"port": port,
"vhost": vhost}
def send_requests(requests, **options):
logger = logging
if 'logger' in options.keys():
logger = options['logger']
"""Send a import request message to be picked up by workers."""
connection = DjangoBrokerConnection()
publisher = Publisher(connection=connection,
exchange="collector",
exchange_type='topic',
routing_key="collector.driver",
serializer='json')
for req in requests:
routing_key=req['driver_routing_key']
publisher.send(req, routing_key=routing_key)
logger.debug("Sent request with routing_key %s:%s" %( routing_key,req, ))
publisher.close()
connection.close()
def send_requests(requests, **options):
logger = logging
if 'logger' in options.keys():
logger = options['logger']
"""Send a import request message to be picked up by workers."""
connection = DjangoBrokerConnection()
publisher = Publisher(connection=connection,
exchange="collector",
exchange_type='topic',
routing_key="collector.driver",
serializer='json')
for req in requests:
routing_key = req['driver_routing_key']
publisher.send(req, routing_key=routing_key)
logger.debug("Sent request with routing_key %s:%s" % (
routing_key,
req,
))
publisher.close()
connection.close()
def __init__(self, exchange, queue, routing_key=None, debug=None):
"""
Initialise signal, conection, exchange, queue and run consumer
@return: Django Signal like object.
@param exchange: name of exchange for this signal
@type exchange: string
@param queue: name queue of this signal
@type queue: string
@param routing_key: name of routing_key betwen exchange and queue of this signal
@type routing_key: string
@param debug: debug flag
@type debug: bool
Example:
>>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1")
>>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2")
>>> amqp_signal_1.queue_bind(['q2'])
>>> def amqp_handler(sender, **kwargs):
print "AMPQ handler:", sender, kwargs
>>> amqp_signal_2.connect(amqp_handler, sender=None)
>>> amqp_signal_1.send("Hello world!")
"""
super(SignalAMQP, self).__init__(providing_args=["message"])
self.exchange = exchange
self.queue = queue
self.routing_key = routing_key
self.debug = debug is None and settings.DEBUG or debug
self.conn = DjangoBrokerConnection()
self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\
routing_key=self.routing_key)
self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\
exchange=self.exchange, routing_key=self.routing_key)
self.consumer.register_callback(self.callback)
self.cl = self.listen()
def get_responses(logger=logging):
connection = DjangoBrokerConnection()
consumer = Consumer(connection=connection,
exchange="collector.response",
queue="responses",
routing_key="response")
for message in consumer.iterqueue():
responses = message.payload
for resp in responses:
logger.debug("resp=%s" % resp)
try:
tag = Tag.objects.get(name=resp['name'])
tag.save_with_history(resp['current_value'])
except Exception as ex:
logger.error(ex)
#print "Could have saved '%s' for tag '%s'" % (resp['current_value'], tag.id,)
message.ack()
consumer.close()
connection.close()
def get_responses(logger=logging):
connection = DjangoBrokerConnection()
consumer = Consumer(connection=connection,
exchange="collector.response",
queue="responses",
routing_key="response")
for message in consumer.iterqueue():
responses = message.payload
for resp in responses:
logger.debug("resp=%s" % resp )
try:
tag=Tag.objects.get(name=resp['name'])
tag.save_with_history(resp['current_value'])
except Exception as ex:
logger.error(ex)
#print "Could have saved '%s' for tag '%s'" % (resp['current_value'], tag.id,)
message.ack()
consumer.close()
connection.close()
def __init__(self, exchange, queue, routing_key=None, debug=None):
"""
Initialise signal, conection, exchange, queue and run consumer
@return: Django Signal like object.
@param exchange: name of exchange for this signal
@type exchange: string
@param queue: name queue of this signal
@type queue: string
@param routing_key: name of routing_key betwen exchange and queue of this signal
@type routing_key: string
@param debug: debug flag
@type debug: bool
Example:
>>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1")
>>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2")
>>> amqp_signal_1.queue_bind(['q2'])
>>> def amqp_handler(sender, **kwargs):
print "AMPQ handler:", sender, kwargs
>>> amqp_signal_2.connect(amqp_handler, sender=None)
>>> amqp_signal_1.send("Hello world!")
"""
super(SignalAMQP, self).__init__(providing_args=["message"])
self.exchange = exchange
self.queue = queue
self.routing_key = routing_key
self.debug = debug is None and settings.DEBUG or debug
self.conn = DjangoBrokerConnection()
self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\
routing_key=self.routing_key)
self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\
exchange=self.exchange, routing_key=self.routing_key)
self.consumer.register_callback(self.callback)
self.cl = self.listen()
class SignalAMQP(Signal):
def __init__(self, exchange, queue, routing_key=None, debug=None):
"""
Initialise signal, conection, exchange, queue and run consumer
@return: Django Signal like object.
@param exchange: name of exchange for this signal
@type exchange: string
@param queue: name queue of this signal
@type queue: string
@param routing_key: name of routing_key betwen exchange and queue of this signal
@type routing_key: string
@param debug: debug flag
@type debug: bool
Example:
>>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1")
>>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2")
>>> amqp_signal_1.queue_bind(['q2'])
>>> def amqp_handler(sender, **kwargs):
print "AMPQ handler:", sender, kwargs
>>> amqp_signal_2.connect(amqp_handler, sender=None)
>>> amqp_signal_1.send("Hello world!")
"""
super(SignalAMQP, self).__init__(providing_args=["message"])
self.exchange = exchange
self.queue = queue
self.routing_key = routing_key
self.debug = debug is None and settings.DEBUG or debug
self.conn = DjangoBrokerConnection()
self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\
routing_key=self.routing_key)
self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\
exchange=self.exchange, routing_key=self.routing_key)
self.consumer.register_callback(self.callback)
self.cl = self.listen()
def send(self, message, **kw):
""" Transfer message to bus. Message can be any simple python type. """
self.publisher.send(message, **kw)
def callback(self, message_data, message):
""" Consumer callback function. Send Django singnal."""
try:
sender = message_data['sender']
except:
sender = None
super(SignalAMQP, self).send(sender=sender, message_data=message_data, message=message)
message.ack()
if self.debug:
print "AMPQ CALLBACK: sender=", sender, "messege=", message_data, message
return True
def get_backend(self):
return self.conn.get_backend_cls()(DjangoBrokerConnection())
def queue_bind(self, queue_list):
""" Bind another queues to current exchange """
routing_keys = []
backend = self.get_backend()
for x in queue_list:
backend.queue_bind(queue=x, exchange=self.exchange, \
routing_key='%s_%s' % (self.exchange, x))
routing_keys.append('%s_%s' % (self.exchange, x))
return routing_keys
def listen(self):
""" Run consumer loop thread """
cl = ConsumerLoop()
cl.consumer = self.consumer
cl.start()
return cl
def stop(self):
""" Unactivate this signal """
self.conn.close()
def apply_async(
task,
args=None,
kwargs=None,
countdown=None,
eta=None,
routing_key=None,
exchange=None,
task_id=None,
immediate=None,
mandatory=None,
priority=None,
connection=None,
connect_timeout=AMQP_CONNECTION_TIMEOUT,
serializer=None,
**opts
):
"""Run a task asynchronously by the celery daemon(s).
:param task: The task to run (a callable object, or a :class:`Task`
instance
:param args: The positional arguments to pass on to the task (a ``list``).
:param kwargs: The keyword arguments to pass on to the task (a ``dict``)
:param countdown: Number of seconds into the future that the task should
execute. Defaults to immediate delivery (Do not confuse that with
the ``immediate`` setting, they are unrelated).
:param eta: A :class:`datetime.datetime` object that describes the
absolute time when the task should execute. May not be specified
if ``countdown`` is also supplied. (Do not confuse this with the
``immediate`` setting, they are unrelated).
:keyword routing_key: The routing key used to route the task to a worker
server.
:keyword exchange: The named exchange to send the task to. Defaults to
:attr:`celery.task.base.Task.exchange`.
:keyword immediate: Request immediate delivery. Will raise an exception
if the task cannot be routed to a worker immediately.
(Do not confuse this parameter with the ``countdown`` and ``eta``
settings, as they are unrelated).
:keyword mandatory: Mandatory routing. Raises an exception if there's
no running workers able to take on this task.
:keyword connection: Re-use existing AMQP connection.
The ``connect_timeout`` argument is not respected if this is set.
:keyword connect_timeout: The timeout in seconds, before we give up
on establishing a connection to the AMQP server.
:keyword priority: The task priority, a number between ``0`` and ``9``.
:keyword serializer: A string identifying the default serialization
method to use. Defaults to the ``CELERY_TASK_SERIALIZER`` setting.
Can be ``pickle`` ``json``, ``yaml``, or any custom serialization
methods that have been registered with
:mod:`carrot.serialization.registry`.
"""
args = args or []
kwargs = kwargs or {}
routing_key = routing_key or getattr(task, "routing_key", None)
exchange = exchange or getattr(task, "exchange", None)
immediate = immediate or getattr(task, "immediate", None)
mandatory = mandatory or getattr(task, "mandatory", None)
priority = priority or getattr(task, "priority", None)
serializer = serializer or getattr(task, "serializer", None)
taskset_id = opts.get("taskset_id")
publisher = opts.get("publisher")
retries = opts.get("retries")
if countdown:
eta = datetime.now() + timedelta(seconds=countdown)
from celery.conf import ALWAYS_EAGER
if ALWAYS_EAGER:
return apply(task, args, kwargs)
need_to_close_connection = False
if not publisher:
if not connection:
connection = DjangoBrokerConnection(connect_timeout=connect_timeout)
need_to_close_connection = True
publisher = TaskPublisher(connection=connection)
delay_task = publisher.delay_task
if taskset_id:
delay_task = curry(publisher.delay_task_in_set, taskset_id)
task_id = delay_task(
task.name,
args,
kwargs,
task_id=task_id,
retries=retries,
routing_key=routing_key,
exchange=exchange,
mandatory=mandatory,
immediate=immediate,
serializer=serializer,
priority=priority,
eta=eta,
)
if need_to_close_connection:
publisher.close()
connection.close()
return AsyncResult(task_id)
def get_backend(self):
return self.conn.get_backend_cls()(DjangoBrokerConnection())
class SignalAMQP(Signal):
def __init__(self, exchange, queue, routing_key=None, debug=None):
"""
Initialise signal, conection, exchange, queue and run consumer
@return: Django Signal like object.
@param exchange: name of exchange for this signal
@type exchange: string
@param queue: name queue of this signal
@type queue: string
@param routing_key: name of routing_key betwen exchange and queue of this signal
@type routing_key: string
@param debug: debug flag
@type debug: bool
Example:
>>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1")
>>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2")
>>> amqp_signal_1.queue_bind(['q2'])
>>> def amqp_handler(sender, **kwargs):
print "AMPQ handler:", sender, kwargs
>>> amqp_signal_2.connect(amqp_handler, sender=None)
>>> amqp_signal_1.send("Hello world!")
"""
super(SignalAMQP, self).__init__(providing_args=["message"])
self.exchange = exchange
self.queue = queue
self.routing_key = routing_key
self.debug = debug is None and settings.DEBUG or debug
self.conn = DjangoBrokerConnection()
self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\
routing_key=self.routing_key)
self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\
exchange=self.exchange, routing_key=self.routing_key)
self.consumer.register_callback(self.callback)
self.cl = self.listen()
def send(self, message, **kw):
""" Transfer message to bus. Message can be any simple python type. """
self.publisher.send(message, **kw)
def callback(self, message_data, message):
""" Consumer callback function. Send Django singnal."""
try:
sender = message_data['sender']
except:
sender = None
super(SignalAMQP, self).send(sender=sender,
message_data=message_data,
message=message)
message.ack()
if self.debug:
print "AMPQ CALLBACK: sender=", sender, "messege=", message_data, message
return True
def get_backend(self):
return self.conn.get_backend_cls()(DjangoBrokerConnection())
def queue_bind(self, queue_list):
""" Bind another queues to current exchange """
routing_keys = []
backend = self.get_backend()
for x in queue_list:
backend.queue_bind(queue=x, exchange=self.exchange, \
routing_key='%s_%s' % (self.exchange, x))
routing_keys.append('%s_%s' % (self.exchange, x))
return routing_keys
def listen(self):
""" Run consumer loop thread """
cl = ConsumerLoop()
cl.consumer = self.consumer
cl.start()
return cl
def stop(self):
""" Unactivate this signal """
self.conn.close()
def _get_carrot_object(klass, **kwargs):
"Helper function to create Publisher and Consumer objects."
return klass(connection=DjangoBrokerConnection(),
exchange=settings.EXCHANGE,
routing_key=settings.ROUTING_KEY,
**kwargs)
def send_a_message(msg):
conn = DjangoBrokerConnection()
MyMessager(connection=conn).send({"message": msg})
conn.close()
def discard_all():
conn = DjangoBrokerConnection()
MyMessager(connection=conn).consumer.discard_all()
conn.close()
