def on_consumer_cancelled(self, method_frame: pika.frame.Method) -> None:
"""Invoked by pika when the broker sends a Basic.Cancel for a consumer
receiving messages.
"""
logger.info(f"Consumer was cancelled remotely, shutting down: {method_frame}")
if self._channel:
self._channel.close()
def close_connection(self):
self._consuming = False
if self._connection.is_closing or self._connection.is_closed:
logger.info("Connection is closing or already closed")
else:
logger.info("Closing connection")
self._connection.close()
def connect(self) -> TornadoConnection:
"""This method connects to the broker, returning the connection handle."""
logger.info(f"Connecting to {self._host}:{self._port}{self._vhost}")
# set amqp credentials
if self._username:
credentials = pika.PlainCredentials(self._username, self._password)
# set amqp connection parameters
parameters = pika.ConnectionParameters(
host=self._host,
port=self._port,
virtual_host=self._vhost,
credentials=credentials,
)
else:
parameters = pika.ConnectionParameters(
host=self._host,
port=self._port,
virtual_host=self._vhost,
)
# connect
connection = TornadoConnection(
parameters=parameters,
on_open_callback=self.on_connection_open,
on_open_error_callback=self.on_connection_open_error,
on_close_callback=self.on_connection_closed,
)
return connection
def open_channel(self):
"""Open a new channel with the broker by issuing the Channel.Open RPC
command. When the broker responds that the channel is open, the
on_channel_open callback will be invoked by pika.
"""
logger.info("Creating a new channel")
self._connection.channel(on_open_callback=self.on_channel_open)
def acknowledge_message(self, delivery_tag) -> None:
"""Acknowledge the message delivery from the broker by sending a
Basic.Ack RPC method for the delivery tag.
:param int delivery_tag: The delivery tag from the Basic.Deliver frame
"""
logger.info("Acknowledging message %s", delivery_tag)
self._channel.basic_ack(delivery_tag)
def add_on_cancel_callback(self) -> None:
"""Add a callback that will be invoked if the broker cancels the consumer
for some reason. If the broker does cancel the consumer,
on_consumer_cancelled will be invoked by pika.
"""
logger.info("Adding consumer cancellation callback")
self._channel.add_on_cancel_callback(self.on_consumer_cancelled)
def stop_consuming(self) -> None:
"""Tell the broker that you would like to stop consuming by sending the
Basic.Cancel RPC command.
"""
if self._channel:
logger.info("Sending a Basic.Cancel command to the broker")
cb = functools.partial(self.on_cancelok, userdata=self._consumer_tag)
self._channel.basic_cancel(self._consumer_tag, cb)
def on_basic_qos_ok(self, _unused_frame) -> None:
"""Invoked by pika when the Basic.QoS method has completed. At this
point we will start consuming messages by calling start_consuming
which will invoke the needed RPC commands to start the process.
:param pika.frame.Method _unused_frame: The Basic.QosOk response frame
"""
logger.info(f"QOS set to: {self._prefetch_count}")
self.start_consuming()
def on_bindok(self, _unused_frame: pika.frame.Method, queue_name: str, routing_key: str = None):
"""Invoked by pika when the Queue.Bind method has completed. At this
point we will set the prefetch count for the channel.
"""
if routing_key:
logger.info(f"Queue bound: {queue_name} with routing key {routing_key}")
else:
logger.info(f"Queue bound: {queue_name}")
self.set_qos()
def on_cancelok(self, _unused_frame: pika.frame.Method, userdata: str) -> None:
"""This method is invoked by pika when the broker acknowledges the
cancellation of a consumer. At this point we will close the channel.
This will invoke the on_channel_closed method once the channel has been
closed, which will in-turn close the connection.
"""
self._consuming = False
logger.info(f"The broker acknowledged the cancellation of the consumer: {userdata}")
self.close_channel()
def setup_queue(self, queue_name: str) -> None:
"""Setup the queue on the broker by invoking the Queue.Declare RPC
command. When it is complete, the on_queue_declareok method will
be invoked by pika.
:param str|unicode queue_name: The name of the queue to declare.
"""
logger.info(f"Declaring queue {queue_name}")
cb = functools.partial(self.on_queue_declareok, userdata=queue_name)
self._channel.queue_declare(queue=queue_name, callback=cb)
def on_channel_open(self, channel: pika.channel.Channel):
"""This method is invoked by pika when the channel has been opened.
The channel object is passed in so we can make use of it.
Since the channel is now open, we'll declare the exchange to use.
"""
logger.info("Channel opened")
self._channel = channel
self.add_on_channel_close_callback()
self.setup_exchange(self._exchange_name)
def start_consuming(self) -> None:
"""This method sets up the consumer by first calling
add_on_cancel_callback so that the object is notified if the broker
cancels the consumer.
"""
logger.info("Issuing consumer")
self.add_on_cancel_callback()
self._consumer_tag = self._channel.basic_consume(self._queue_name, self.on_message)
self.was_consuming = True
self._consuming = True
def setup_exchange(self, exchange_name: str) -> None:
"""Setup the exchange on the broker by invoking the Exchange.Declare RPC
command. When it is complete, the on_exchange_declareok method will
be invoked by pika.
"""
logger.info(f"Declaring exchange: {exchange_name}")
# Note: using functools.partial is not required, it is demonstrating
# how arbitrary data can be passed to the callback when it is called
cb = functools.partial(self.on_exchange_declareok, userdata=exchange_name)
self._channel.exchange_declare(exchange=exchange_name, exchange_type=self.EXCHANGE_TYPE, callback=cb)
def run(self, reconnect: bool = False) -> None:
if reconnect:
logger.info("AMQP consumer running in auto-reconnect mode")
while True:
try:
self._consumer.run()
except KeyboardInterrupt:
self._consumer.stop()
break
self._maybe_reconnect()
else:
self._consumer.run()
logger.info("Terminating Hurricane AMQP client")
loop = tornado.ioloop.IOLoop.current()
loop.stop()
def on_queue_declareok(self, _unused_frame: pika.frame.Method, userdata: str) -> None:
"""Method invoked by pika when the Queue.Declare RPC call made in
setup_queue has completed. In this method we will bind the queue
and exchange together with the routing key by issuing the Queue.Bind
RPC command. When this command is complete, the on_bindok method will
be invoked by pika.
"""
queue_name = userdata
logger.info(f"Binding to {queue_name}")
routing_keys = self.get_routing_keys(queue_name)
if len(routing_keys) == 0:
# no routing key applicable
cb = functools.partial(self.on_bindok, queue_name=queue_name)
self._channel.queue_bind(queue_name, exchange=self._exchange_name, callback=cb)
else:
for routing_key in routing_keys:
cb = functools.partial(self.on_bindok, queue_name=queue_name, routing_key=routing_key)
self._channel.queue_bind(queue_name, routing_key=routing_key, exchange=self._exchange_name, callback=cb)
def run(self, reconnect: bool = False) -> None:
"""
If reconnect is True, AMQP consumer is running in auto-connect mode.
In this case consumer will be executed. If any exception occurs, consumer will be disconnected and after some
delay will be reconnected. Then consumer will be restarted. KeyboardInterrupt exception is handled
differently and stops consumer. In this case IOLoop will be terminated.
If reconnect is false, consumer will be started, but no exceptions and interruptions will be tolerated.
"""
if reconnect:
logger.info("AMQP consumer running in auto-reconnect mode")
while True:
try:
self._consumer.run()
except KeyboardInterrupt:
self._consumer.stop()
break
self._maybe_reconnect()
else:
self._consumer.run()
logger.info("Terminating Hurricane AMQP client")
loop = tornado.ioloop.IOLoop.current()
loop.stop()
def stop(self) -> None:
"""Cleanly shutdown the connection by stopping the consumer."""
if not self._closing:
self._closing = True
logger.info("Stopping")
if self._consuming:
self.stop_consuming()
logger.info("Start called")
self._connection.ioloop.stop()
else:
self._connection.ioloop.stop()
logger.info("Stopped consuming")
def ask_exit(signame):
logger.info(f"Received signal {signame}. Shutting down now.")
loop.stop()
sys.exit(0)
def handle(self, *args, **options):
"""
Defines functionalities for different arguments. After all arguments were processed, it starts the async event
loop.
"""
start_time = time.time()
logger.info("Starting a Tornado-powered Django AMQP consumer")
if options["autoreload"]:
tornado.autoreload.start()
logger.info("Autoreload was performed")
# sanitize probe paths
options["liveness_probe"] = f"/{options['liveness_probe'].lstrip('/')}"
options[
"readiness_probe"] = f"/{options['readiness_probe'].lstrip('/')}"
options["startup_probe"] = f"/{options['startup_probe'].lstrip('/')}"
# set the probe routes
if not options["no_probe"]:
logger.info(
f"Probe application running on port {options['probe_port']}")
probe_application = make_probe_server(options, self.check)
probe_application.listen(options["probe_port"])
else:
logger.info("No probe application running")
# load connection data
connection = {}
if "amqp_host" in options and options["amqp_host"]:
connection["amqp_host"] = options["amqp_host"]
elif hasattr(settings, "AMQP_HOST"):
connection["amqp_host"] = settings.AMQP_HOST
elif os.getenv("AMQP_HOST"):
connection["amqp_host"] = os.getenv("AMQP_HOST")
else:
raise CommandError(
"The amqp host must not be empty: set it either as environment AMQP_HOST, "
"in the django settings as AMQP_HOST or as optional argument --amqp-host"
)
if "amqp_port" in options and options["amqp_port"]:
connection["amqp_port"] = options["amqp_port"]
elif hasattr(settings, "AMQP_PORT"):
connection["amqp_port"] = settings.AMQP_PORT
elif os.getenv("AMQP_PORT"):
connection["amqp_port"] = int(os.getenv("AMQP_PORT"))
else:
raise CommandError(
"The amqp port must not be empty: set it either as environment AMQP_PORT, "
"in the django settings as AMQP_PORT or as optional argument --amqp-port"
)
connection["amqp_vhost"] = "/"
if "amqp_vhost" in options and options["amqp_vhost"]:
connection["amqp_vhost"] = options["amqp_vhost"]
elif hasattr(settings, "AMPQ_VHOST"):
connection["amqp_vhost"] = settings.AMPQ_VHOST
elif os.getenv("AMPQ_VHOST"):
connection["amqp_vhost"] = os.getenv("AMPQ_VHOST")
# load the handler class
_amqp_consumer = import_string(options["handler"])
if not issubclass(_amqp_consumer, _AMQPConsumer):
logger.error(
f"The type {_amqp_consumer} is not subclass of _AMQPConsumer")
raise CommandError(
"Cannot start the consumer due to an implementation error")
worker = AMQPClient(
_amqp_consumer,
queue_name=options["queue"],
exchange_name=options["exchange"],
amqp_host=connection["amqp_host"],
amqp_port=connection["amqp_port"],
amqp_vhost=connection["amqp_vhost"],
)
# prepate the io loop
loop = asyncio.get_event_loop()
def ask_exit(signame):
logger.info(f"Received signal {signame}. Shutting down now.")
loop.stop()
sys.exit(0)
for signame in ("SIGINT", "SIGTERM"):
loop.add_signal_handler(getattr(signal, signame),
functools.partial(ask_exit, signame))
end_time = time.time()
time_elapsed = end_time - start_time
StartupTimeMetric.set(time_elapsed)
worker.run(options["reconnect"])
def on_connection_open(self, _unused_connection: pika.SelectConnection):
"""This method is called by pika once the connection to the broker has
been established.
"""
logger.info("Connection opened")
self.open_channel()
def add_on_channel_close_callback(self):
"""This method tells pika to call the on_channel_closed method if
the broker unexpectedly closes the channel.
"""
logger.info("Adding channel close callback")
self._channel.add_on_close_callback(self.on_channel_closed)
def on_exchange_declareok(self, _unused_frame: pika.frame.Method, userdata: str) -> None:
"""Invoked by pika when the broker has finished the Exchange.Declare RPC
command.
"""
logger.info("Exchange declared: %s", userdata)
self.setup_queue(self._queue_name)
def close_channel(self) -> None:
"""Call to close the channel cleanly by issuing the
Channel.Close RPC command.
"""
logger.info("Closing the channel")
self._channel.close()
def reject_message(self, delivery_tag, requeue: bool = False) -> None:
"""Reject the message delivery from the broker."""
logger.info("Rejecting message %s", delivery_tag)
self._channel.basic_nack(delivery_tag, requeue=requeue)
