class RabbitMQConsumer():
def __init__(self, options={}):
self.options = options
self.connection = SelectConnection(on_open_callback=self.on_connected)
def on_connected(self, connection):
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, new_channel):
self.channel = new_channel
self.channel.queue_declare(queue=self.options['queue'],
durable=self.options['durable'],
exclusive=self.options['exclusive'],
auto_delete=self.options['auto_delete'],
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
self.channel.basic_consume(self.handle_delivery,
queue=self.options['queue'])
def handle_delivery(self, channel, method, header, body):
self.options['on_receive'](body)
self.channel.basic_ack(method.delivery_tag)
def start(self):
self.connection.ioloop.start()
def stop(self):
self.connection.close()
def on_connected(self, connection: SelectConnection):
"""Called when the connection to the message broker is completed
Args:
connection: The SelectConnection instance, representing the achieved connection with the
broker.
"""
connection.channel(on_open_callback=self.on_channel_open)
def channel(self):
x = []
def on_open(ch):
x.append(ch)
SelectConnection.channel(self, on_open)
while not x:
self.ioloop.poller.poll()
return x[0]
def channel(self):
x = []
def on_open(ch):
x.append(ch)
SelectConnection.channel(self, on_open)
while not x:
self.ioloop.poller.poll()
return x[0]
class BasicPublish(object):
"""See also: http://pika.github.com/examples.html"""
def __init__(self, host, port, virtual_host, username, password,
exchange, routing_key, message):
self.channel = None
self.exchange = exchange
self.routing_key = routing_key
self.message = message
credentials = PlainCredentials(
username, password, erase_on_connect=False)
parameters = ConnectionParameters(
host, port, virtual_host, credentials=credentials)
self.connection = SelectConnection(
parameters=parameters, on_open_callback=self.on_connect)
def on_connect(self, connection):
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
self.channel = channel
self.channel.exchange_declare(exchange=self.exchange,
type='direct', durable=True,
auto_delete=False,
callback=self.on_exchange_declared)
def on_exchange_declared(self, frame):
self.channel.queue_declare(queue=self.routing_key, durable=True,
exclusive=False, auto_delete=False,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
self.channel.queue_bind(exchange=self.exchange,
queue=self.routing_key,
routing_key=self.routing_key,
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
properties = BasicProperties(content_type='text/plain',
delivery_mode=1)
self.channel.basic_publish(exchange=self.exchange,
routing_key=self.routing_key,
body=self.message,
properties=properties)
self.connection.close()
class RMQConnection(ABC):
"""
Abstract base class to be implemented.
Wraps a pika.SelectConnection and channel object to provide easy access to
consumer/producer implementers. Allows for a quick start to writing
consumers and producers, without having to re-write the connection logic.
RMQConnection signals implementers with the on_ready and on_close methods
when the connection/channel state changes.
"""
def __init__(self, connection_parameters=None):
"""
:param connection_parameters: pika.ConnectionParameters
"""
# Kept public to allow restart with different parameters.
self.connection_parameters: pika.ConnectionParameters = \
connection_parameters
self._connection_thread = Thread(target=self._connect, daemon=True)
self._connection = None
self._channel = None
self._reconnect_attempts = 0
self._closing = False
self._restarting = False
@abstractmethod
def on_ready(self):
"""
Implementers can handle what should happen when the RMQConnection is
ready for work.
"""
pass
@abstractmethod
def on_close(self, permanent=False):
"""
Implementers can handle what should happen when the RMQConnection is
suddenly closed. An on_close call will normally be followed by an
on_ready call from the RMQConnection unless the connection is
permanently down. It is possible that the RMQConnection will signal
multiple on_close calls for a single connection failure, therefore the
implementation of on_close must be idempotent.
:param permanent: bool
"""
pass
@abstractmethod
def on_error(self):
"""
Implementers can handle what should happen when the RMQConnection has
run into an error, for example an Exchange/Queue declaration failure.
"""
pass
def start(self):
"""
Starts the connection, opens a connection and a channel by running the
underlying connection thread. When ready, callbacks will be invoked on
each step of the way towards establishing the connection and channel,
starting with on_connection_open if all went well.
NOTE! Starting the connection is not a synchronous operation! It is
this way since starting may or may not be successful, so a return
value would be necessary to indicate success/failure. But, since
retries will be made, an initial failure may in fact be successful the
next second, meaning the user is given faulty information. Start
success is therefore indicated via the 'on_ready' hook instead.
"""
if not self._connection_thread.is_alive():
self._connection_thread.start()
def restart(self):
"""
Re-initiates the connection and channel.
"""
LOGGER.info("restarting connection")
try:
self._connection.close()
self._restarting = True # Only if connection could be closed.
except ConnectionWrongStateError:
LOGGER.info("connection closed and could not be restarted")
def stop(self):
"""
Starts a closing procedure for the connection and channel.
"""
LOGGER.info("stopping connection")
if not self._closing:
self._closing = True
try:
self._connection.close()
self._connection_thread.join()
except ConnectionWrongStateError:
LOGGER.info("connection already closed")
def declare_queue(self, queue_params, callback=None):
"""
:param queue_params: rabbitmq_client.QueueParams
:param callback: callable
"""
LOGGER.info(f"declaring queue: {queue_params.queue}")
self._channel.queue_declare(queue_params.queue,
durable=queue_params.durable,
exclusive=queue_params.exclusive,
auto_delete=queue_params.auto_delete,
arguments=queue_params.arguments,
callback=callback)
def declare_exchange(self, exchange_params, callback=None):
"""
:param exchange_params: rabbitmq_client.ExchangeParams
:param callback: callable
"""
LOGGER.info(f"declaring exchange: {exchange_params.exchange}")
self._channel.exchange_declare(
exchange_params.exchange,
exchange_type=exchange_params.exchange_type,
durable=exchange_params.durable,
auto_delete=exchange_params.auto_delete,
internal=exchange_params.internal,
arguments=exchange_params.arguments,
callback=callback)
def bind_queue(self, queue_bind_params, callback=None):
"""
:param queue_bind_params: rabbitmq_client.QueueBindParams
:param callback: callable
"""
LOGGER.info(f"binding queue {queue_bind_params.queue} to exchange "
f"{queue_bind_params.exchange} with routing key: "
f"{queue_bind_params.routing_key}")
self._channel.queue_bind(queue_bind_params.queue,
queue_bind_params.exchange,
routing_key=queue_bind_params.routing_key,
arguments=queue_bind_params.arguments,
callback=callback)
def basic_consume(self,
consume_params,
on_message_callback_override=None,
callback=None):
"""
:param consume_params: rabbitmq_client.ConsumeParams
:param on_message_callback_override: callable
:param callback: callable
"""
LOGGER.info(f"basic consumer starting for queue: "
f"{consume_params.queue}")
self._channel.basic_consume(
consume_params.queue,
(on_message_callback_override if on_message_callback_override else
consume_params.on_message_callback),
auto_ack=consume_params.auto_ack,
exclusive=consume_params.exclusive,
consumer_tag=consume_params.consumer_tag,
arguments=consume_params.arguments,
callback=callback)
def basic_publish(self,
body,
exchange=DEFAULT_EXCHANGE,
routing_key="",
publish_params=None):
"""
:param body: bytes
:param exchange: str
:param routing_key: str
:param publish_params: rabbitmq_client.PublishParams
"""
LOGGER.debug(f"publishing to exchange: {exchange} and routing key: "
f"{routing_key}")
if not publish_params:
publish_params = PublishParams() # Create defaults
self._channel.basic_publish(exchange,
routing_key,
body,
properties=publish_params.properties,
mandatory=publish_params.mandatory)
def confirm_delivery(self, on_delivery_confirmed, callback=None):
"""
:param on_delivery_confirmed: callable
:param callback: callable
"""
LOGGER.info("activating confirm delivery mode")
self._channel.confirm_delivery(on_delivery_confirmed,
callback=callback)
def _connect(self):
"""
Assigns a new pika.SelectConnection to the connection and starts its
ioloop to begin connecting.
"""
if self.connection_parameters is not None:
LOGGER.info(f"starting connection towards: "
f"{self.connection_parameters.host}:"
f"{self.connection_parameters.port}")
else:
LOGGER.info("starting connection towards: 127.0.0.1:5672")
self._connection = SelectConnection(
parameters=self.connection_parameters,
on_open_callback=self.on_connection_open,
on_open_error_callback=self.on_connection_open_error,
on_close_callback=self.on_connection_closed)
self._connection.ioloop.start()
def on_connection_open(self, _connection):
"""
Called when a new connection has been established. Opens a new channel.
:param _connection: pika.SelectConnection
"""
LOGGER.debug("connection opened")
# Do NOT reset reconnect counter here, do it after channel opened.
self._connection.channel(on_open_callback=self.on_channel_open)
def on_connection_open_error(self, _connection, error):
"""
Called when a connection could not be established towards RabbitMQ.
This will lead to attempts to reconnect with an increasing delay
between attempts.
:param _connection: pika.SelectConnection
:param error: pika.exceptions.?
"""
LOGGER.warning(f"error establishing connection, error: {error}")
# This should ensure the current thread runs to completion after
# reconnect handling is done.
self._connection.ioloop.stop()
# No fancy handling, just retry even if credentials are bad.
self._reconnect()
def on_connection_closed(self, _connection, reason):
"""
Called when the connection is closed, either unexpectedly or on
purpose. Reconnection is attempted if the disconnect reason is
something that is possible to recover, like an unexpected broker
shutdown.
:param _connection: pika.SelectConnection
:param reason: pika.exceptions.?
"""
permanent = False
# This should ensure the current thread runs to completion after
# on_connection_closed handling is done.
self._connection.ioloop.stop()
if not self._closing and type(reason) in RECONNECT_REASONS:
LOGGER.debug(f"connection closed: {reason}, attempting reconnect")
self._reconnect()
elif self._restarting:
LOGGER.debug("connection closed, restarting now")
# Reset to prevent reconnect for any reason
self._restarting = False
self._connection_thread = Thread(target=self._connect, daemon=True)
self._connection_thread.start()
else:
permanent = True
LOGGER.warning(f"connection closed: {reason}, will not reconnect")
# Signal subclass that connection is down.
self.on_close(permanent=permanent)
@property
def _reconnect_delay(self):
return self._reconnect_attempts if self._reconnect_attempts < 9 else 30
def _reconnect(self):
"""
Starts up the connection again after a gradually increasing delay.
"""
LOGGER.debug(f"reconnect, attempt no. {self._reconnect_attempts + 1}")
# Reconnect attempt may have been queued up before 'stop' was called.
if self._closing:
LOGGER.debug("skipping reconnect, connection stopped")
return
# _connect will assign a new connection object
self._connection_thread = Thread(target=self._connect, daemon=True)
if self._reconnect_delay == 0: # retry immediately the first time
self._connection_thread.start()
else: # calculate backoff timer
timer = Timer(self._reconnect_delay, self._connection_thread.start)
# Timer daemon exits if program exits, to avoid eternal retries.
timer.daemon = True
timer.start()
self._reconnect_attempts += 1
def on_channel_open(self, channel):
"""
Called when a new channel has been opened, the consumer is now ready
for work.
:param channel: pika.channel.Channel
"""
LOGGER.debug("channel opened")
self._reconnect_attempts = 0
self._channel = channel
self._channel.add_on_close_callback(self.on_channel_closed)
self.on_ready() # Signal subclass that connection is ready.
def on_channel_closed(self, _channel, reason):
"""
Called when a channel is closed by RabbitMQ. Channels are not
attempted to be re-opened since they are either closed from going down
together with the connection, or because input parameters for some
queue/exchange/other establishment was given faulty parameters.
Retrying such an operation will just result in a failure loop.
:param _channel: pika.channel.Channel
:param reason: pika.exceptions.?
"""
LOGGER.warning("channel closed")
permanent = False
try:
# Reply code 406 = PRECONDITION FAILED
permanent = True if reason.reply_code == 406 else False
except AttributeError: # Not all reasons have a reply code
pass
# Signal subclass that connection is down.
self.on_close(permanent=permanent)
if permanent:
LOGGER.critical(f"connection stopping due to permanent channel "
f"closure: {reason}")
self.stop()
def __init__(self, queue: str, connection: pika.SelectConnection):
assert queue is not None and connection is not None
self.channel = connection.channel()
self.queue = queue
