def on_consumer_cancelled(self, method_frame: pika.frame.Method) -> None: """Invoked by pika when the broker sends a Basic.Cancel for a consumer receiving messages. """ logger.info(f"Consumer was cancelled remotely, shutting down: {method_frame}") if self._channel: self._channel.close()
def close_connection(self): self._consuming = False if self._connection.is_closing or self._connection.is_closed: logger.info("Connection is closing or already closed") else: logger.info("Closing connection") self._connection.close()
def connect(self) -> TornadoConnection: """This method connects to the broker, returning the connection handle.""" logger.info(f"Connecting to {self._host}:{self._port}{self._vhost}") # set amqp credentials if self._username: credentials = pika.PlainCredentials(self._username, self._password) # set amqp connection parameters parameters = pika.ConnectionParameters( host=self._host, port=self._port, virtual_host=self._vhost, credentials=credentials, ) else: parameters = pika.ConnectionParameters( host=self._host, port=self._port, virtual_host=self._vhost, ) # connect connection = TornadoConnection( parameters=parameters, on_open_callback=self.on_connection_open, on_open_error_callback=self.on_connection_open_error, on_close_callback=self.on_connection_closed, ) return connection
def open_channel(self): """Open a new channel with the broker by issuing the Channel.Open RPC command. When the broker responds that the channel is open, the on_channel_open callback will be invoked by pika. """ logger.info("Creating a new channel") self._connection.channel(on_open_callback=self.on_channel_open)
def acknowledge_message(self, delivery_tag) -> None: """Acknowledge the message delivery from the broker by sending a Basic.Ack RPC method for the delivery tag. :param int delivery_tag: The delivery tag from the Basic.Deliver frame """ logger.info("Acknowledging message %s", delivery_tag) self._channel.basic_ack(delivery_tag)
def add_on_cancel_callback(self) -> None: """Add a callback that will be invoked if the broker cancels the consumer for some reason. If the broker does cancel the consumer, on_consumer_cancelled will be invoked by pika. """ logger.info("Adding consumer cancellation callback") self._channel.add_on_cancel_callback(self.on_consumer_cancelled)
def stop_consuming(self) -> None: """Tell the broker that you would like to stop consuming by sending the Basic.Cancel RPC command. """ if self._channel: logger.info("Sending a Basic.Cancel command to the broker") cb = functools.partial(self.on_cancelok, userdata=self._consumer_tag) self._channel.basic_cancel(self._consumer_tag, cb)
def on_basic_qos_ok(self, _unused_frame) -> None: """Invoked by pika when the Basic.QoS method has completed. At this point we will start consuming messages by calling start_consuming which will invoke the needed RPC commands to start the process. :param pika.frame.Method _unused_frame: The Basic.QosOk response frame """ logger.info(f"QOS set to: {self._prefetch_count}") self.start_consuming()
def on_bindok(self, _unused_frame: pika.frame.Method, queue_name: str, routing_key: str = None): """Invoked by pika when the Queue.Bind method has completed. At this point we will set the prefetch count for the channel. """ if routing_key: logger.info(f"Queue bound: {queue_name} with routing key {routing_key}") else: logger.info(f"Queue bound: {queue_name}") self.set_qos()
def on_cancelok(self, _unused_frame: pika.frame.Method, userdata: str) -> None: """This method is invoked by pika when the broker acknowledges the cancellation of a consumer. At this point we will close the channel. This will invoke the on_channel_closed method once the channel has been closed, which will in-turn close the connection. """ self._consuming = False logger.info(f"The broker acknowledged the cancellation of the consumer: {userdata}") self.close_channel()
def setup_queue(self, queue_name: str) -> None: """Setup the queue on the broker by invoking the Queue.Declare RPC command. When it is complete, the on_queue_declareok method will be invoked by pika. :param str|unicode queue_name: The name of the queue to declare. """ logger.info(f"Declaring queue {queue_name}") cb = functools.partial(self.on_queue_declareok, userdata=queue_name) self._channel.queue_declare(queue=queue_name, callback=cb)
def on_channel_open(self, channel: pika.channel.Channel): """This method is invoked by pika when the channel has been opened. The channel object is passed in so we can make use of it. Since the channel is now open, we'll declare the exchange to use. """ logger.info("Channel opened") self._channel = channel self.add_on_channel_close_callback() self.setup_exchange(self._exchange_name)
def start_consuming(self) -> None: """This method sets up the consumer by first calling add_on_cancel_callback so that the object is notified if the broker cancels the consumer. """ logger.info("Issuing consumer") self.add_on_cancel_callback() self._consumer_tag = self._channel.basic_consume(self._queue_name, self.on_message) self.was_consuming = True self._consuming = True
def setup_exchange(self, exchange_name: str) -> None: """Setup the exchange on the broker by invoking the Exchange.Declare RPC command. When it is complete, the on_exchange_declareok method will be invoked by pika. """ logger.info(f"Declaring exchange: {exchange_name}") # Note: using functools.partial is not required, it is demonstrating # how arbitrary data can be passed to the callback when it is called cb = functools.partial(self.on_exchange_declareok, userdata=exchange_name) self._channel.exchange_declare(exchange=exchange_name, exchange_type=self.EXCHANGE_TYPE, callback=cb)
def run(self, reconnect: bool = False) -> None: if reconnect: logger.info("AMQP consumer running in auto-reconnect mode") while True: try: self._consumer.run() except KeyboardInterrupt: self._consumer.stop() break self._maybe_reconnect() else: self._consumer.run() logger.info("Terminating Hurricane AMQP client") loop = tornado.ioloop.IOLoop.current() loop.stop()
def on_queue_declareok(self, _unused_frame: pika.frame.Method, userdata: str) -> None: """Method invoked by pika when the Queue.Declare RPC call made in setup_queue has completed. In this method we will bind the queue and exchange together with the routing key by issuing the Queue.Bind RPC command. When this command is complete, the on_bindok method will be invoked by pika. """ queue_name = userdata logger.info(f"Binding to {queue_name}") routing_keys = self.get_routing_keys(queue_name) if len(routing_keys) == 0: # no routing key applicable cb = functools.partial(self.on_bindok, queue_name=queue_name) self._channel.queue_bind(queue_name, exchange=self._exchange_name, callback=cb) else: for routing_key in routing_keys: cb = functools.partial(self.on_bindok, queue_name=queue_name, routing_key=routing_key) self._channel.queue_bind(queue_name, routing_key=routing_key, exchange=self._exchange_name, callback=cb)
def run(self, reconnect: bool = False) -> None: """ If reconnect is True, AMQP consumer is running in auto-connect mode. In this case consumer will be executed. If any exception occurs, consumer will be disconnected and after some delay will be reconnected. Then consumer will be restarted. KeyboardInterrupt exception is handled differently and stops consumer. In this case IOLoop will be terminated. If reconnect is false, consumer will be started, but no exceptions and interruptions will be tolerated. """ if reconnect: logger.info("AMQP consumer running in auto-reconnect mode") while True: try: self._consumer.run() except KeyboardInterrupt: self._consumer.stop() break self._maybe_reconnect() else: self._consumer.run() logger.info("Terminating Hurricane AMQP client") loop = tornado.ioloop.IOLoop.current() loop.stop()
def stop(self) -> None: """Cleanly shutdown the connection by stopping the consumer.""" if not self._closing: self._closing = True logger.info("Stopping") if self._consuming: self.stop_consuming() logger.info("Start called") self._connection.ioloop.stop() else: self._connection.ioloop.stop() logger.info("Stopped consuming")
def ask_exit(signame): logger.info(f"Received signal {signame}. Shutting down now.") loop.stop() sys.exit(0)
def handle(self, *args, **options): """ Defines functionalities for different arguments. After all arguments were processed, it starts the async event loop. """ start_time = time.time() logger.info("Starting a Tornado-powered Django AMQP consumer") if options["autoreload"]: tornado.autoreload.start() logger.info("Autoreload was performed") # sanitize probe paths options["liveness_probe"] = f"/{options['liveness_probe'].lstrip('/')}" options[ "readiness_probe"] = f"/{options['readiness_probe'].lstrip('/')}" options["startup_probe"] = f"/{options['startup_probe'].lstrip('/')}" # set the probe routes if not options["no_probe"]: logger.info( f"Probe application running on port {options['probe_port']}") probe_application = make_probe_server(options, self.check) probe_application.listen(options["probe_port"]) else: logger.info("No probe application running") # load connection data connection = {} if "amqp_host" in options and options["amqp_host"]: connection["amqp_host"] = options["amqp_host"] elif hasattr(settings, "AMQP_HOST"): connection["amqp_host"] = settings.AMQP_HOST elif os.getenv("AMQP_HOST"): connection["amqp_host"] = os.getenv("AMQP_HOST") else: raise CommandError( "The amqp host must not be empty: set it either as environment AMQP_HOST, " "in the django settings as AMQP_HOST or as optional argument --amqp-host" ) if "amqp_port" in options and options["amqp_port"]: connection["amqp_port"] = options["amqp_port"] elif hasattr(settings, "AMQP_PORT"): connection["amqp_port"] = settings.AMQP_PORT elif os.getenv("AMQP_PORT"): connection["amqp_port"] = int(os.getenv("AMQP_PORT")) else: raise CommandError( "The amqp port must not be empty: set it either as environment AMQP_PORT, " "in the django settings as AMQP_PORT or as optional argument --amqp-port" ) connection["amqp_vhost"] = "/" if "amqp_vhost" in options and options["amqp_vhost"]: connection["amqp_vhost"] = options["amqp_vhost"] elif hasattr(settings, "AMPQ_VHOST"): connection["amqp_vhost"] = settings.AMPQ_VHOST elif os.getenv("AMPQ_VHOST"): connection["amqp_vhost"] = os.getenv("AMPQ_VHOST") # load the handler class _amqp_consumer = import_string(options["handler"]) if not issubclass(_amqp_consumer, _AMQPConsumer): logger.error( f"The type {_amqp_consumer} is not subclass of _AMQPConsumer") raise CommandError( "Cannot start the consumer due to an implementation error") worker = AMQPClient( _amqp_consumer, queue_name=options["queue"], exchange_name=options["exchange"], amqp_host=connection["amqp_host"], amqp_port=connection["amqp_port"], amqp_vhost=connection["amqp_vhost"], ) # prepate the io loop loop = asyncio.get_event_loop() def ask_exit(signame): logger.info(f"Received signal {signame}. Shutting down now.") loop.stop() sys.exit(0) for signame in ("SIGINT", "SIGTERM"): loop.add_signal_handler(getattr(signal, signame), functools.partial(ask_exit, signame)) end_time = time.time() time_elapsed = end_time - start_time StartupTimeMetric.set(time_elapsed) worker.run(options["reconnect"])
def on_connection_open(self, _unused_connection: pika.SelectConnection): """This method is called by pika once the connection to the broker has been established. """ logger.info("Connection opened") self.open_channel()
def add_on_channel_close_callback(self): """This method tells pika to call the on_channel_closed method if the broker unexpectedly closes the channel. """ logger.info("Adding channel close callback") self._channel.add_on_close_callback(self.on_channel_closed)
def on_exchange_declareok(self, _unused_frame: pika.frame.Method, userdata: str) -> None: """Invoked by pika when the broker has finished the Exchange.Declare RPC command. """ logger.info("Exchange declared: %s", userdata) self.setup_queue(self._queue_name)
def close_channel(self) -> None: """Call to close the channel cleanly by issuing the Channel.Close RPC command. """ logger.info("Closing the channel") self._channel.close()
def reject_message(self, delivery_tag, requeue: bool = False) -> None: """Reject the message delivery from the broker.""" logger.info("Rejecting message %s", delivery_tag) self._channel.basic_nack(delivery_tag, requeue=requeue)