def __init__(self, connection, queue=None, exchange=None,
routing_key=None, **kwargs):
self.connection = connection
self.backend = kwargs.get("backend")
self.decoder = kwargs.get("decoder", deserialize)
if not self.backend:
self.backend = DefaultBackend(connection=connection,
decoder=self.decoder)
self.queue = queue or self.queue
# Binding.
self.queue = queue or self.queue
self.exchange = exchange or self.exchange
self.routing_key = routing_key or self.routing_key
self.callbacks = []
# Options
self.durable = kwargs.get("durable", self.durable)
self.exclusive = kwargs.get("exclusive", self.exclusive)
self.auto_delete = kwargs.get("auto_delete", self.auto_delete)
self.exchange_type = kwargs.get("exchange_type", self.exchange_type)
# durable implies auto-delete.
if self.durable:
self.auto_delete = True
self._declare_channel()
def __init__(self, connection, exchange=None, routing_key=None, **kwargs):
self.connection = connection
self.backend = kwargs.get("backend")
self.encoder = kwargs.get("encoder", serialize)
if not self.backend:
self.backend = DefaultBackend(connection=connection,
encoder=self.encoder)
self.exchange = exchange or self.exchange
self.routing_key = routing_key or self.routing_key
self.delivery_mode = kwargs.get("delivery_mode", self.delivery_mode)
class Consumer(object):
"""Message consumer.
:param connection: see :attr:`connection`.
:param queue: see :attr:`queue`.
:param exchange: see :attr:`exchange`.
:param routing_key: see :attr:`routing_key`.
:keyword durable: see :attr:`durable`.
:keyword auto_delete: see :attr:`auto_delete`.
:keyword exclusive: see :attr:`exclusive`.
:keyword exchange_type: see :attr:`exchange_type`.
:keyword decoder: see :attr:`decoder`.
:keyword backend: see :attr:`backend`.
.. attribute:: connection
A :class:`carrot.connection.AMQPConnection` instance.
.. attribute:: queue
Name of the queue.
.. attribute:: exchange
Name of the exchange the queue binds to.
.. attribute:: routing_key
The routing key (if any). The interpretation of the routing key
depends on the value of the :attr:`exchange_type` attribute:
* direct exchange
Matches if the routing key property of the message and
the :attr:`routing_key` attribute are identical.
* fanout exchange
Always matches, even if the binding does not have a key.
* topic exchange
Matches the routing key property of the message by a primitive
pattern matching scheme. The message routing key then consists
of words separated by dots (``"."``, like domain names), and
two special characters are available; star (``"*"``) and hash
(``"#"``). The star matches any word, and the hash matches
zero or more words. For example ``"*.stock.#"`` matches the
routing keys ``"usd.stock"`` and ``"eur.stock.db"`` but not
``"stock.nasdaq"``.
.. attribute:: durable
Durable exchanges remain active when a server restarts. Non-durable
exchanges (transient exchanges) are purged when a server restarts.
Default is ``True``.
.. attribute:: auto_delete
If set, the exchange is deleted when all queues have finished
using it. Default is ``False``.
.. attribute:: exclusive
Exclusive queues may only be consumed from by the current connection.
When :attr:`exclusive` is on, this also implies :attr:`auto_delete`.
Default is ``False``.
.. attribute:: exchange_type
AMQP defines four default exchange types (routing algorithms) that
covers most of the common messaging use cases. An AMQP broker can
also define additional exchange types, so see your message brokers
manual for more information about available exchange types.
* Direct
Direct match between the routing key in the message, and the
routing criteria used when a queue is bound to this exchange.
* Topic
Wildcard match between the routing key and the routing pattern
specified in the binding. The routing key is treated as zero
or more words delimited by ``"."`` and supports special
wildcard characters. ``"*"`` matches a single word and ``"#"``
matches zero or more words.
* Fanout
Queues are bound to this exchange with no arguments. Hence any
message sent to this exchange will be forwarded to all queues
bound to this exchange.
* Headers
Queues are bound to this exchange with a table of arguments
containing headers and values (optional). A special argument
named "x-match" determines the matching algorithm, where
``"all"`` implies an ``AND`` (all pairs must match) and
``"any"`` implies ``OR`` (at least one pair must match).
*NOTE*: carrot has poor support for header exchanges at
this point.
This description of AMQP exchange types was shamelessly stolen
from the blog post `AMQP in 10 minutes: Part 4`_ by
Rajith Attapattu. Recommended reading.
.. _`AMQP in 10 minutes: Part 4`: http://bit.ly/amqp-exchange-types
.. attribute:: decoder
A function able to deserialize the message body.
.. attribute:: backend
The messaging backend used. Defaults to the ``pyamqplib`` backend.
.. attribute:: callbacks
List of registered callbacks to trigger when a message is received
by :meth:`wait`, :meth:`process_next` or :meth:`iterqueue`.
:raises `amqplib.client_0_8.channel.AMQPChannelException`: if the queue is
exclusive and the queue already exists and is owned by another
connection.
Example Usage
>>> consumer = Consumer(connection=DjangoAMQPConnection(),
... queue="foo", exchange="foo", routing_key="foo")
>>> def process_message(message_data, message):
... print("Got message %s: %s" % (
... message.delivery_tag, message_data))
>>> consumer.register_callback(process_message)
>>> consumer.wait() # Go into receive loop
"""
queue = ""
exchange = ""
routing_key = ""
durable = True
exclusive = False
auto_delete = False
exchange_type = "direct"
channel_open = False
def __init__(self, connection, queue=None, exchange=None,
routing_key=None, **kwargs):
self.connection = connection
self.backend = kwargs.get("backend")
self.decoder = kwargs.get("decoder", deserialize)
if not self.backend:
self.backend = DefaultBackend(connection=connection,
decoder=self.decoder)
self.queue = queue or self.queue
# Binding.
self.queue = queue or self.queue
self.exchange = exchange or self.exchange
self.routing_key = routing_key or self.routing_key
self.callbacks = []
# Options
self.durable = kwargs.get("durable", self.durable)
self.exclusive = kwargs.get("exclusive", self.exclusive)
self.auto_delete = kwargs.get("auto_delete", self.auto_delete)
self.exchange_type = kwargs.get("exchange_type", self.exchange_type)
# durable implies auto-delete.
if self.durable:
self.auto_delete = True
self._declare_channel()
def _declare_channel(self):
if self.queue:
self.backend.queue_declare(queue=self.queue, durable=self.durable,
exclusive=self.exclusive,
auto_delete=self.auto_delete)
if self.exchange:
self.backend.exchange_declare(exchange=self.exchange,
type=self.exchange_type,
durable=self.durable,
auto_delete=self.auto_delete)
if self.queue:
self.backend.queue_bind(queue=self.queue, exchange=self.exchange,
routing_key=self.routing_key)
def _receive_callback(self, message):
message_data = self.message_to_python(message)
self.receive(message_data, message)
def message_to_python(self, message):
"""Decode encoded message back to python.
:param message: A :class:`carrot.backends.base.BaseMessage` instance.
"""
return self.decoder(message.body)
def fetch(self):
"""Receive the next message waiting on the queue.
:returns: A :class:`carrot.backends.base.BaseMessage` instance,
or ``None`` if there's no messages to be received.
"""
message = self.backend.get(self.queue)
return message
def receive(self, message_data, message):
"""This method is called when a new message is received by
running :meth:`wait`, :meth:`process_next` or :meth:`iterqueue`.
When a message is received, it passes the message on to the
callbacks listed in the :attr:`callbacks` attribute.
You can register callbacks using :meth:`register_callback`.
:param message_data: The deserialized message data.
:param message: The :class:`carrot.backends.base.BaseMessage` instance.
:raises NotImplementedError: If no callbacks has been registered.
"""
if not self.callbacks:
raise NotImplementError("No consumer callbacks registered")
for callback in self.callbacks:
callback(message_data, message)
def register_callback(self, callback):
"""Register a callback function to be triggered by :meth:`receive`.
The ``callback`` function must take two arguments:
* message_data
The deserialized message data
* message
The :class:`carrot.backends.base.BaseMessage` instance.
"""
self.callbacks.append(callback)
def process_next(self, ack=True):
"""Processes the next pending message on the queue.
This function tries to fetch a message from the queue, and
if successful passes the message on to :meth:`receive`.
:keyword ack: By default, an ack is sent to the server
signifying that the message has been accepted. This means
that the :meth:`carrot.backends.base.BaseMessage.ack` and
:meth:`carrot.backends.base.BaseMessage.reject` methods
on the message object are no longer valid.
If the ack argument is set to ``False``, this behaviour is
disabled and the receiver is required to manually handle
acknowledgment.
:returns: The resulting :class:`carrot.backends.base.BaseMessage`
object.
"""
message = self.fetch()
if message:
self._receive_callback(message)
if ack:
message.ack()
return message
def discard_all(self, filter=None):
"""Discard all waiting messages.
:param filter: A filter function to only discard the messages this
filter returns.
:returns: the number of messages discarded.
*WARNING*: All incoming messages will be ignored and not processed.
Example using filter:
>>> def waiting_feeds_only(message):
... try:
... message_data = message.decode()
... except: # Should probably be more specific.
... pass
...
... if message_data.get("type") == "feed":
... return True
... else:
... return False
"""
discarded_count = 0
while True:
message = self.fetch()
if message is None:
return discarded_count
discard_message = True
if filter and not filter(message):
discard_message = False
if discard_message:
message.ack()
discarded_count = discarded_count + 1
def wait(self):
"""Go into consume mode.
This runs an infinite loop, processing all incoming messages
using :meth:`receive` to apply the message to all registered
callbacks.
"""
self.channel_open = True
self.backend.consume(queue=self.queue, no_ack=True,
callback=self._receive_callback,
consumer_tag=self.__class__.__name__)
def iterqueue(self, limit=None):
"""Infinite iterator yielding pending messages.
Obviously you shouldn't consume the whole iterator at
once, without using a ``limit``.
:keyword limit: If set, the iterator stops when it has processed
this number of messages in total.
"""
for items_since_start in itertools.count():
item = self.process_next()
if item is None or (limit and items_since_start > limit):
raise StopIteration
yield item
def close(self):
"""Close the channel to the queue."""
if self.channel_open:
self.backend.cancel(self.__class__.__name__)
self.backend.close()
class Publisher(object):
"""Message publisher.
:param connection: see :attr:`connection`.
:param exchange: see :attr:`exchange`.
:param routing_key: see :attr:`routing_key`.
:keyword encoder: see :attr:`encoder`.
:keyword backend: see :attr:`backend`.
.. attribute:: connection
The AMQP connection. A :class:`carrot.connection.AMQPConnection`
instance.
.. attribute:: exchange
Name of the exchange we send messages to.
.. attribute:: routing_key
The routing key added to all messages sent using this publisher.
See :attr:`Consumer.routing_key` for more information.
.. attribute:: delivery_mode
The default delivery mode used for messages. The value is an integer.
The following delivery modes are supported by (at least) RabbitMQ:
* 1
The message is non-persistent. Which means it is stored in
memory only, and is lost if the server dies or restarts.
* 2
The message is persistent. Which means the message is
stored both in-memory, and on disk, and therefore
preserved if the server dies or restarts.
The default value is ``2`` (persistent).
.. attribute:: encoder
The function responsible for encoding the message data passed
to :meth:`send`. Note that any consumer of the messages sent
must have a decoder supporting the serialization scheme.
.. attribute:: backend
The backend used. Defaults to the ``pyamqplib`` backend.
"""
exchange = ""
routing_key = ""
delivery_mode = 2 # Persistent
def __init__(self, connection, exchange=None, routing_key=None, **kwargs):
self.connection = connection
self.backend = kwargs.get("backend")
self.encoder = kwargs.get("encoder", serialize)
if not self.backend:
self.backend = DefaultBackend(connection=connection,
encoder=self.encoder)
self.exchange = exchange or self.exchange
self.routing_key = routing_key or self.routing_key
self.delivery_mode = kwargs.get("delivery_mode", self.delivery_mode)
def create_message(self, message_data):
"""With any data, serialize it and encapsulate it in a AMQP
message with the proper headers set."""
message_data = self.encoder(message_data)
return self.backend.prepare_message(message_data, self.delivery_mode)
def send(self, message_data, delivery_mode=None):
"""Send a message.
:param message_data: The message data to send. Can be a list,
dictionary or a string.
:keyword delivery_mode: Override the default :attr:`delivery_mode`.
"""
message = self.create_message(message_data)
self.backend.publish(message, exchange=self.exchange,
routing_key=self.routing_key)
def close(self):
"""Close connection to queue."""
self.backend.close()