def start(self): """Start worker.""" connected = False # Define connection while (not connected): try: self._creds = self._amqp_url.split('amqp://')[1].split("@")[0] parameters = URLParameters(self._amqp_url) self._host = parameters.host self._port = "15672" self._vhost = parameters.virtual_host self._blocking_connection = BlockingConnection(parameters) self._connection = SelectConnection( URLParameters(self._amqp_url), on_open_callback=self.on_open_connection) self._connection.ioloop.start() connected = True # Catch a Keyboard Interrupt to make sure that the connection is closed cleanly except KeyboardInterrupt: # Gracefully close the connection self._connection.close() # Start the IOLoop again so Pika can communicate, it will stop on its own when the connection is closed self._connection.ioloop.start() except: if (self._debug): print( " [!] RabbitMQ Host Unrecheable. Reconnecting in {} seconds..." .format(self._reconection_time)) time.sleep(self._reconection_time)
class RabbitMQConsumer(): def __init__(self, options={}): self.options = options self.connection = SelectConnection(on_open_callback=self.on_connected) def on_connected(self, connection): self.connection = connection self.connection.channel(self.on_channel_open) def on_channel_open(self, new_channel): self.channel = new_channel self.channel.queue_declare(queue=self.options['queue'], durable=self.options['durable'], exclusive=self.options['exclusive'], auto_delete=self.options['auto_delete'], callback=self.on_queue_declared) def on_queue_declared(self, frame): self.channel.basic_consume(self.handle_delivery, queue=self.options['queue']) def handle_delivery(self, channel, method, header, body): self.options['on_receive'](body) self.channel.basic_ack(method.delivery_tag) def start(self): self.connection.ioloop.start() def stop(self): self.connection.close()
def on_connected(self, connection: SelectConnection): """Called when the connection to the message broker is completed Args: connection: The SelectConnection instance, representing the achieved connection with the broker. """ connection.channel(on_open_callback=self.on_channel_open)
def consume(self): try: self.connection = SelectConnection(self.parameters, self._on_connected) self.connection.ioloop.start() except Exception as e: self.logger.error('{} {}'.format(self.consumer_id, str(e))) self.connection.close() self.connection.ioloop.start()
def channel(self): x = [] def on_open(ch): x.append(ch) SelectConnection.channel(self, on_open) while not x: self.ioloop.poller.poll() return x[0]
def __init__(self, *args, **nargs): x = [] def on_open(connection): x.append(0) nargs['on_open_callback'] = on_open nargs['on_close_callback'] = self._my_on_close SelectConnection.__init__(self, *args, **nargs) while not x: self.ioloop.poller.poll()
def __init__(self, host, port, virtual_host, username, password, exchange, routing_key, message): self.channel = None self.exchange = exchange self.routing_key = routing_key self.message = message credentials = PlainCredentials( username, password, erase_on_connect=False) parameters = ConnectionParameters( host, port, virtual_host, credentials=credentials) self.connection = SelectConnection( parameters=parameters, on_open_callback=self.on_connect)
class BasicPublish(object): """See also: http://pika.github.com/examples.html""" def __init__(self, host, port, virtual_host, username, password, exchange, routing_key, message): self.channel = None self.exchange = exchange self.routing_key = routing_key self.message = message credentials = PlainCredentials( username, password, erase_on_connect=False) parameters = ConnectionParameters( host, port, virtual_host, credentials=credentials) self.connection = SelectConnection( parameters=parameters, on_open_callback=self.on_connect) def on_connect(self, connection): self.connection.channel(self.on_channel_open) def on_channel_open(self, channel): self.channel = channel self.channel.exchange_declare(exchange=self.exchange, type='direct', durable=True, auto_delete=False, callback=self.on_exchange_declared) def on_exchange_declared(self, frame): self.channel.queue_declare(queue=self.routing_key, durable=True, exclusive=False, auto_delete=False, callback=self.on_queue_declared) def on_queue_declared(self, frame): self.channel.queue_bind(exchange=self.exchange, queue=self.routing_key, routing_key=self.routing_key, callback=self.on_queue_bound) def on_queue_bound(self, frame): properties = BasicProperties(content_type='text/plain', delivery_mode=1) self.channel.basic_publish(exchange=self.exchange, routing_key=self.routing_key, body=self.message, properties=properties) self.connection.close()
def __init__(self, config, callback): self.reconnection_delay = 1.0 self.caller_callback = callback self.config = ConnectionConfig() self.config.read(config) self.callbacks = self.set_callbacks() credentials = PlainCredentials(**self.config.credentials) broker_config = self.config.broker_config broker_config['credentials'] = credentials parameters = ConnectionParameters(**broker_config) try: parameters.host = self.config.host except NoOptionError: pass try: parameters.heartbeat = int(self.config.heartbeat) except NoOptionError: pass try: parameters.heartbeat = int(self.config.heartbeat) except NoOptionError: pass self.connection = SelectConnection(parameters, self.on_connected)
def connect(self): """This method connects to RabbitMQ, returning the connection handle. When the connection is established, the on_connection_open method will be invoked by pika. :rtype: pika.SelectConnection """ LOGGER.info("Connecting to %s", self._server_details.host) credentials = PlainCredentials(self._server_details.username, self._server_details.password) connection_params = ConnectionParameters( host=self._server_details.host, port=self._server_details.port, virtual_host=self._server_details.vhost, credentials=credentials, ) if self._server_details.uses_ssl: cafile = os.getenv("REQUESTS_CA_BUNDLE") ssl_context = ssl.create_default_context(cafile=cafile) connection_params.ssl_options = SSLOptions(ssl_context) return SelectConnection( parameters=connection_params, on_open_callback=self.on_connection_open, on_open_error_callback=self.on_connection_open_error, on_close_callback=self.on_connection_closed, )
def run(self): credentials = pika.PlainCredentials(self.username, self.password) ssl_options = None port = self.port if self.use_ssl: port = self.ssl_port if self.use_ssl and self.ssl_auth: ssl_options = { "ca_certs": self.cacertfile, "certfile": self.certfile, "keyfile": self.keyfile, "cert_reqs": CERT_REQUIRED } credentials = ExternalCredentials() try: parameters = pika.ConnectionParameters(host=self.endpoint, port=port, virtual_host=self.vhost, credentials=credentials, ssl=self.use_ssl, ssl_options=ssl_options) except TypeError: parameters = pika.ConnectionParameters(host=self.endpoint, port=port, virtual_host=self.vhost, credentials=credentials) self.logger.debug("Connecting to %s@%s:%s on vhost %s" % (self.username, self.endpoint, port, self.vhost)) try: SelectPoller.TIMEOUT = .1 connection = SelectConnection(parameters) connection.add_on_close_callback(self._on_closed()) connection.callbacks.add(0, '_on_connection_open', self._on_connected(), one_shot=True) self.connection = connection connection.ioloop.start() except IOError, err: if not self.manual_close: self.logger.error("Connection error: {0}".format(err))
def connect(self): """ :param parameters: :return: """ self.connection = SelectConnection(self.parameters, self.on_connected, stop_ioloop_on_close=False)
def run(self): credentials = pika.PlainCredentials(self.username, self.password) ssl_options = None port = self.port if self.use_ssl: port = self.ssl_port if self.use_ssl and self.ssl_auth: ssl_options = { "ca_certs": self.cacertfile, "certfile": self.certfile, "keyfile": self.keyfile, "cert_reqs": CERT_REQUIRED } credentials = ExternalCredentials() try: parameters = pika.ConnectionParameters(host=self.endpoint, port=port, virtual_host=self.vhost, credentials=credentials, ssl=self.use_ssl, ssl_options=ssl_options) except TypeError: parameters = pika.ConnectionParameters(host=self.endpoint, port=port, virtual_host=self.vhost, credentials=credentials) self.logger.debug("Connecting to %s@%s:%s on vhost %s" % ( self.username, self.endpoint, port, self.vhost)) try: SelectPoller.TIMEOUT = .1 connection = SelectConnection(parameters) connection.add_on_close_callback(self._on_closed()) connection.callbacks.add(0, '_on_connection_open', self._on_connected(), one_shot=True) self.connection = connection connection.ioloop.start() except IOError, err: if not self.manual_close: self.logger.error("Connection error: {0}".format(err))
def connect(self): """ Creating connection object """ url = self.get_url() logger.info('Connecting to %s', url) return SelectConnection( URLParameters(url), self.on_connection_open, stop_ioloop_on_close=False )
def _connect(self): """ Assigns a new pika.SelectConnection to the connection and starts its ioloop to begin connecting. """ if self.connection_parameters is not None: LOGGER.info(f"starting connection towards: " f"{self.connection_parameters.host}:" f"{self.connection_parameters.port}") else: LOGGER.info("starting connection towards: 127.0.0.1:5672") self._connection = SelectConnection( parameters=self.connection_parameters, on_open_callback=self.on_connection_open, on_open_error_callback=self.on_connection_open_error, on_close_callback=self.on_connection_closed) self._connection.ioloop.start()
def start(self): """Start worker.""" # Define connection while True: try: self._connection = SelectConnection( URLParameters(self._amqp_url), on_open_callback=self.on_open_connection) self._connection.ioloop.start() # Catch a Keyboard Interrupt to make sure that the connection is closed cleanly except KeyboardInterrupt: # Gracefully close the connection self._connection.close() # Start the IOLoop again so Pika can communicate, it will stop on its own when the connection is closed self._connection.ioloop.start() except: if (self._debug): print( " [!] RabbitMQ Host Unreachable. Reconnecting in {} seconds..." .format(self._reconnection_time)) sleep(self._reconnection_time)
def _init_connection(self, connection_parameters: ConnectionParameters) -> BaseConnection: """Initializes a BaseConnection to be used by the listener. Args: connection_parameters: A set up ConnectionParameters instance. Returns: The set up BaseConnection instance. """ connection = SelectConnection(connection_parameters, on_open_callback=self.on_connected) return connection
def connect(self): """This method connects to RabbitMQ, returning the connection handle. When the connection is established, the on_connection_open method will be invoked by pika. :rtype: pika.SelectConnection """ url = self.get_url() logger.info('Connecting to %s', url) return SelectConnection( parameters=URLParameters(url), on_open_callback=self.on_connection_open, on_open_error_callback=self.on_connection_open_error, on_close_callback=self.on_connection_closed)
def _close_connection(self, connection: SelectConnection): """Closes a given `SelectConnection` instance. It will call the connection's `close` method, and then `ioloop.start`, to try keep looping until it is fully closed, as suggested by pika's documentation: https://pika.readthedocs.io/en/stable/intro.html#io-and-event-looping Args: connection: The connection to close. """ if not connection: return try: # Gracefully close the connection connection.close() # Loop until we're fully closed, will stop on its own connection.ioloop.start() except AMQPError as error: self.logger.error( "Error while closing connection on %s: %s.", self._queue_name, str(error), )
def connect(self, host=None): """ Connection with Rabbit. """ if not host: _, host = list(json.loads(sd_rabbit.rabbit_nodes).items())[0] return SelectConnection(ConnectionParameters( host=host, port=5672, virtual_host=VIRTUAL_HOST, credentials=sd_rabbit.rabbit_credential), on_open_callback=self.on_connected, on_close_callback=self.on_closed)
def open_connection(self): """Opens a connection to RabbitMQ This method immediately returns the connection object. However, whether the connection was successful is not know until a callback is invoked (either on_open_callback or on_open_error_callback). Returns: The SelectConnection object """ return SelectConnection( parameters=self._connection_parameters, on_open_callback=self.on_connection_open, on_close_callback=self.on_connection_closed, on_open_error_callback=self.on_connection_closed, )
def test_publish(publisher, monkeypatch): with monkeypatch.context() as m: publisher._channel = Channel(SelectConnection(), 1, lambda: 1) m.setattr(publisher._channel, '_state', Channel.OPEN) mock_publish = Mock() m.setattr(publisher._channel, 'basic_publish', mock_publish) item = {'url': 'https://www.example.com/page'} publisher._publish(item) mock_publish.assert_called_once_with( exchange='exchange', routing_key='exchange.queue', body=json.dumps(item, ensure_ascii=False), properties=BasicProperties(app_id='app', content_type='application/json', delivery_mode=1), mandatory=True)
def _connect(self): # type: (Consumer) -> SelectConnection """ Connect to RabbitMQ :return pika.adapters.SelectConnection """ param = pika.URLParameters(self.amqp_url) logger.info('Connecting to: {host}:{port}/{vhost}; SSL: {ssl}'.format( host=param.host, port=param.port, vhost=param.virtual_host, ssl=param.ssl)) return SelectConnection( parameters=param, on_open_callback=self._on_connection_open, on_open_error_callback=self._on_open_connection_error, on_close_callback=self._on_connection_closed, stop_ioloop_on_close=False)
def get(self): queue_name = "s_connection" def on_channel(connection): connection.channel(on_declare) def on_declare(channel): channel.queue_declare(publisher(channel), queue_name) def publisher(channel): channel.basic_publish(exchange='', routing_key=queue_name, body='[%s]It is asyn publish') connection = SelectConnection(ConnectionParameters('localhost'), on_channel) # connection = BlockingConnection(ConnectionParameters()) # channel = connection.channel() # channel.queue_declare(queue=queue_name) # channel.basic_publish(exchange='', routing_key=queue_name, body='sss') self.write("[Tornado asyn produce] OK")
class BrokerConnection(object): """Connection class which provides connection to AMQP broker.""" def __init__(self, config, callback): self.reconnection_delay = 1.0 self.caller_callback = callback self.config = ConnectionConfig() self.config.read(config) self.callbacks = self.set_callbacks() credentials = PlainCredentials(**self.config.credentials) broker_config = self.config.broker_config broker_config['credentials'] = credentials parameters = ConnectionParameters(**broker_config) try: parameters.host = self.config.host except NoOptionError: pass try: parameters.heartbeat = int(self.config.heartbeat) except NoOptionError: pass try: parameters.heartbeat = int(self.config.heartbeat) except NoOptionError: pass self.connection = SelectConnection(parameters, self.on_connected) def on_connected(self, connection): """Called when we're connected to AMQP broker.""" logger.info("Connected to AMQP-broker.") connection.channel(self.on_channel_open) def on_channel_open(self, new_channel): """Called when channel has opened""" logger.info("Channel to AMQP-broker opened.") self.channel = new_channel self.channel.add_on_close_callback(self.on_channel_closed) self.generic_callback() def set_callbacks(self): """Set callbacks for queue factory.""" self.exchange_callbacks = [] self.queue_callbacks = [] self.binding_callbacks = [] for exchange in self.config.exchanges: tmp = {} tmp['exchange'] = exchange tmp.update(self.config.exchanges[exchange]) self.exchange_callbacks.append(tmp) for queue in self.config.queues: tmp = {} tmp['queue'] = queue tmp.update(self.config.queues[queue]) self.queue_callbacks.append(tmp) for binding in self.config.bindings: tmp = {} tmp['binding'] = binding tmp.update(self.config.bindings[binding]) self.binding_callbacks.append(tmp) def generic_callback(self, channel=None, frame=None): """Create exchanges, queues and bindings.""" if channel and frame: # You could do error handling here # or add code for anonymous queue handling pass if self.exchange_callbacks: config = self.exchange_callbacks.pop() config['exchange'] = config['exchange'].split(':', 1)[-1] logger.info("Creating exchange %s." % config['exchange']) config.update({"callback": self.generic_callback}) self.channel.exchange_declare(**config) elif self.queue_callbacks: config = self.queue_callbacks.pop() config['queue'] = config['queue'].split(':', 1)[-1] logger.info("Creating queue %s." % config['queue']) config.update({"callback": self.generic_callback}) self.channel.queue_declare(**config) elif self.binding_callbacks: config = self.binding_callbacks.pop() binding = config['binding'].split(':') del config['binding'] config['exchange'] = binding[-1] config['queue'] = binding[1] logger.info("Creating binding (%s -> %s) with routing key %s" % ( config['exchange'], config['queue'], config['routing_key'])) config.update({"callback": self.generic_callback}) self.channel.queue_bind(**config) else: logger.info("RabbitMQ exchanges, queues and bindings are now " "configured.") self.caller_callback() def reset_reconnection_delay(self, *args): self.reconnection_delay = 1.0 def loop(self): """Main loop""" while True: try: logger.info("Starting main loop") self.connection.add_on_open_callback( self.reset_reconnection_delay) self.connection.ioloop.start() except socket.error as e: logger.error("Connection failed or closed unexpectedly: %s", e) except TypeError as e: logger.error("Connection failed or closed unexpectedly: %s", e) except KeyboardInterrupt: self.connection.close() self.connection.ioloop.start() break finally: self.reconnection_delay *= (random.random() * 0.5) + 1 self.reconnection_delay = min(self.reconnection_delay, 60.0) logger.info("Trying reconnection in %s seconds", self.reconnection_delay) time.sleep(self.reconnection_delay) def on_channel_closed(self, channel, code, text): logger.warning("Channel closed with reason '%s %s'", code, text) self.connection.close(code, text)
# Delete the row elif message['operation'] == 'delete': data = {'row_id': message['data']['row_id']} self.cursor.execute(DELETE_QUERY, data) # Ack the message channel.basic_ack(delivery_tag=method_frame.delivery_tag) if __name__ == '__main__': consumer = ExampleConsumer() # Connect to RabbitMQ host = (len(sys.argv) > 1) and sys.argv[1] or '127.0.0.1' connection = SelectConnection(ConnectionParameters(host), consumer._on_connected) # Loop until CTRL-C try: # Start our blocking loop connection.ioloop.start() except KeyboardInterrupt: # Close the connection connection.close() # Loop until the conneciton is closed connection.ioloop.start()
class Updater(object): def __init__(self, debug=True, prefetch_count=1): self._path = environ['DIR_TILES'] self._debug = debug self._res_queue = environ['RES_QUEUE'] self._res_xchg = environ['RES_XCHG'] self._amqp_url = environ['AMQP_URL'] self._prefetch_count = prefetch_count self._reconnection_time = int(environ['TIMEOUT']) def base64_to_img(self, buff, fpath): with open(fpath, 'wb') as f: b64data = buff.split(",")[1] data = b64decode(b64data.encode("utf-8", "ignore")) f.write(data) def callback_res_queue(self, _unused_channel, delivery, properties, body): """ Callback triggered when message is received. """ if (self._debug): print(" [x] Received - Body: {}".format(body[:140])) start_time = time() data = loads(body) path = data['earthImage']['path'] if ('filteredImages' in data): if (self._debug): print(" [x] Saving Images...") for image in data['filteredImages']: new_path = Path(path.replace('RAW', image['filterName'])) dirs = new_path.parts[:-1] # create dirs Path("/".join(dirs)).mkdir(parents=True, exist_ok=True) self.base64_to_img(image['vectorImage'], new_path) self._channel.basic_ack(delivery_tag=delivery.delivery_tag) def on_open_connection(self, _unused_frame): if (self._debug): print(" [x] Connected - Connection state: OPEN. ") self._connection.channel(on_open_callback=self.on_channel_open) def on_channel_open(self, channel): """Callback when we have successfully declared the channel.""" if (self._debug): print(" [x] Channel - Channel state: OPEN. ") self._channel = channel self._channel.exchange_declare(exchange=self._res_xchg, exchange_type='fanout') cb = partial(self.on_queue_declareok, userdata=self._res_queue) self._channel.queue_declare(queue=self._res_queue, durable=True, callback=cb) def on_queue_declareok(self, _unused_frame, userdata): """Callback when we have successfully declared the queue. This call tells the server to send us self._prefetch_count message in advance. This helps overall throughput, but it does require us to deal with the messages we have promptly. """ queue_name = userdata if (self._debug): print(" [x] Queue declared - Queue: {}".format(queue_name)) self._channel.basic_qos(prefetch_count=self._prefetch_count, callback=self.on_qos) self._channel.queue_bind(exchange=self._res_xchg, queue=queue_name) def on_qos(self, _unused_frame): """Callback when Basic.QOS has completed.""" if (self._debug): print(" [x] QOS set to {}".format(self._prefetch_count)) self.start_consuming() def start_consuming(self): """Start consuming from task queue.""" if (self._debug): print(" [x] Waiting for messages...") self._channel.basic_consume(self._res_queue, self.callback_res_queue) def start(self): """Start worker.""" # Define connection while True: try: self._connection = SelectConnection( URLParameters(self._amqp_url), on_open_callback=self.on_open_connection) self._connection.ioloop.start() # Catch a Keyboard Interrupt to make sure that the connection is closed cleanly except KeyboardInterrupt: # Gracefully close the connection self._connection.close() # Start the IOLoop again so Pika can communicate, it will stop on its own when the connection is closed self._connection.ioloop.start() except: if (self._debug): print( " [!] RabbitMQ Host Unreachable. Reconnecting in {} seconds..." .format(self._reconnection_time)) sleep(self._reconnection_time)
class BrokerConnection(object): """Connection class which provides connection to AMQP broker.""" exchange_callbacks = [] queue_callbacks = [] binding_callbacks = [] channel = None def __init__(self, config, callback): self.closing = False self.channel = None self.connection = None self.reconnection_delay = 1.0 self.caller_callback = callback if isinstance(config, ConnectionConfig): self.config = config else: self.config = ConnectionConfig() self.config.read(config) self.set_callbacks() credentials = PlainCredentials(**self.config.credentials) broker_config = self.config.broker_config broker_config['credentials'] = credentials self.parameters = ConnectionParameters(**broker_config) try: self.parameters.host = self.config.host except NoOptionError: pass try: self.parameters.heartbeat_interval = int( self.config.heartbeat_interval) except NoOptionError: pass self.connect() def connect(self): """ :param parameters: :return: """ self.connection = SelectConnection(self.parameters, self.on_connected, stop_ioloop_on_close=False) def on_connected(self, connection): """Called when we're connected to AMQP broker.""" print("Connected to AMQP-broker.") self.add_on_connection_close_callback() connection.channel(self.on_channel_open) def add_on_connection_close_callback(self): """This method adds an on close callback that will be invoked by pika when RabbitMQ closes the connection to the publisher unexpectedly. """ print('Adding connection close callback') self.connection.add_on_close_callback(self.on_connection_closed) def on_connection_closed(self, connection, reply_code, reply_text): """This method is invoked by pika when the connection to RabbitMQ is closed unexpectedly. Since it is unexpected, we will reconnect to RabbitMQ if it disconnects. :param pika.connection.Connection connection: The closed connection obj :param int reply_code: The server provided reply_code if given :param str reply_text: The server provided reply_text if given """ self.channel = None if self.closing: self.connection.ioloop.stop() else: print('Connection closed, reopening in 5 seconds: (%s) %s', reply_code, reply_text) self.connection.add_timeout(5, self.reconnect) def on_channel_open(self, new_channel): """Called when channel has opened""" logger.info("Channel to AMQP-broker opened.") self.channel = new_channel self.channel.add_on_close_callback(self.on_channel_closed) self.generic_callback() def set_callbacks(self): """Set callbacks for queue factory.""" for exchange in self.config.exchanges: tmp = {'exchange': exchange} tmp.update(self.config.exchanges[exchange]) self.exchange_callbacks.append(tmp) for queue in self.config.queues: tmp = {'queue': queue} tmp.update(self.config.queues[queue]) self.queue_callbacks.append(tmp) for binding in self.config.bindings: tmp = {'binding': binding} tmp.update(self.config.bindings[binding]) self.binding_callbacks.append(tmp) def generic_callback(self, channel=None, frame=None): """Create exchanges, queues and bindings.""" if channel and frame: # You could do error handling here # or add code for anonymous queue handling pass if self.exchange_callbacks: config = self.exchange_callbacks.pop() config['exchange'] = config['exchange'].split(':', 1)[-1] logger.info("Creating exchange %s." % config['exchange']) config.update({"callback": self.generic_callback}) self.channel.exchange_declare(**config) elif self.queue_callbacks: config = self.queue_callbacks.pop() config['queue'] = config['queue'].split(':', 1)[-1] logger.info("Creating queue %s." % config['queue']) config.update({"callback": self.generic_callback}) self.channel.queue_declare(**config) elif self.binding_callbacks: config = self.binding_callbacks.pop() binding = config['binding'].split(':') del config['binding'] config['exchange'] = binding[-1] config['queue'] = binding[1] logger.info("Creating binding (%s -> %s) with routing key %s" % ( config['exchange'], config['queue'], config['routing_key'])) config.update({"callback": self.generic_callback}) self.channel.queue_bind(**config) else: logger.info("RabbitMQ exchanges, queues and bindings are now " "configured.") self.caller_callback() def reset_reconnection_delay(self, *args): self.reconnection_delay = 1.0 def loop(self): """Main loop""" while True: try: logger.info("Starting main loop") self.connection.add_on_open_callback( self.reset_reconnection_delay) self.connection.ioloop.start() except socket.error as e: logger.error("Connection failed or closed unexpectedly: %s", e, exc_info=True) except TypeError as e: logger.error("Connection failed or closed unexpectedly: %s", e, exc_info=True) except KeyboardInterrupt: self.connection.close() self.connection.ioloop.start() break finally: self.reconnection_delay *= (random.random() * 0.5) + 1 self.reconnection_delay = min(self.reconnection_delay, 60.0) logger.info("Trying reconnection in %s seconds", self.reconnection_delay) time.sleep(self.reconnection_delay) def on_channel_closed(self, channel, code, text): logger.warning("Channel closed with reason '%s %s'", code, text) self.connection.close(code, text) def reconnect(self): """Will be invoked by the IOLoop timer if the connection is closed. See the on_connection_closed method. """ # This is the old connection IOLoop instance, stop its ioloop self.connection.ioloop.stop() if not self.closing: # Create a new connection self.connect() # There is now a new connection, needs a new ioloop to run self.connection.ioloop.start() def add_on_cancel_callback(self): """Add a callback that will be invoked if RabbitMQ cancels the consumer for some reason. If RabbitMQ does cancel the consumer, on_consumer_cancelled will be invoked by pika. """ print('Adding consumer cancellation callback') self.channel.add_on_cancel_callback(self.on_consumer_cancelled) def on_consumer_cancelled(self, method_frame): """ Invoked by pika when RabbitMQ sends a Basic.Cancel for a consumer receiving messages. :param pika.frame.Method method_frame: The Basic.Cancel frame """ print('Consumer was cancelled remotely, shutting down: %r' % method_frame) if self.channel: self.channel.close()
def __init__(self, queue: str, connection: pika.SelectConnection): assert queue is not None and connection is not None self.channel = connection.channel() self.queue = queue
QUEUE = 'xxx' def on_connection_open(connection): connection.channel(on_channel_open) def on_channel_open(channel): global CHANNEL CHANNEL = channel CHANNEL.queue_declare(queue=QUEUE, callback=on_queue_declared) def on_queue_declared(frame): count = 0 while True: body = str(count) CHANNEL.basic_publish(exchange='', routing_key=QUEUE, body=body) print 'Sent %s' % body sleep(3) count += 1 parameters = ConnectionParameters() connection = SelectConnection(parameters, on_open_callback=on_connection_open) try: connection.ioloop.start() except KeyboardInterrupt: connection.close() connection.ioloop.start()
def __init__(self, options={}): self.options = options self.connection = SelectConnection(on_open_callback=self.on_connected)
class QueueConsumer(object): """The consumer class to manage connections to the AMQP server/queue""" def __init__(self, queue, logger, parameters, thread_id=0): self.channel = None self.connection = None self.queue_name = queue self.logger = logger self.consumer_id = 'Thread: %d' % (thread_id, ) self.parameters = pika.ConnectionParameters(**parameters) def _on_queue_declared(self, frame): self.logger.debug('{} ... declaring queue'.format(self.consumer_id)) self.channel.basic_qos(prefetch_count=1) try: self.channel.basic_consume(self.queue_name, self.handle_delivery, auto_ack=True) self.logger.info("{} Declared queue...".format(self.consumer_id)) except Exception as e: self.logger.error('{} crashing:--> {}'.format( self.consumer_id, str(e))) def _on_channel_open(self, channel): self.channel = channel try: self.channel.queue_declare(queue=self.queue_name, exclusive=False, durable=True, auto_delete=False, callback=self._on_queue_declared) self.logger.info("{} Opened Channel....".format(self.consumer_id)) except Exception as e: self.logger.error('{} {}'.format(self.consumer_id, str(e))) def _on_connected(self, connection): connection.channel(on_open_callback=self._on_channel_open) def consume(self): try: self.connection = SelectConnection(self.parameters, self._on_connected) self.connection.ioloop.start() except Exception as e: self.logger.error('{} {}'.format(self.consumer_id, str(e))) self.connection.close() self.connection.ioloop.start() def decode(self, body): try: _body = body.decode('utf-8') except AttributeError: _body = body return _body def handle_delivery(self, channel, method, header, body): try: start_time = datetime.datetime.now() _logger.info("[x] Received Frame") req = json.loads(self.decode(body)) # Process the frame process_frame(_logger, req) time_taken = datetime.datetime.now() - start_time _logger.info("[x] [Consumer {}] Time Taken: {}.{}".format( self.consumer_id, time_taken.seconds, time_taken.microseconds)) except Exception as err: _logger.exception(err)
class RMQConnection(ABC): """ Abstract base class to be implemented. Wraps a pika.SelectConnection and channel object to provide easy access to consumer/producer implementers. Allows for a quick start to writing consumers and producers, without having to re-write the connection logic. RMQConnection signals implementers with the on_ready and on_close methods when the connection/channel state changes. """ def __init__(self, connection_parameters=None): """ :param connection_parameters: pika.ConnectionParameters """ # Kept public to allow restart with different parameters. self.connection_parameters: pika.ConnectionParameters = \ connection_parameters self._connection_thread = Thread(target=self._connect, daemon=True) self._connection = None self._channel = None self._reconnect_attempts = 0 self._closing = False self._restarting = False @abstractmethod def on_ready(self): """ Implementers can handle what should happen when the RMQConnection is ready for work. """ pass @abstractmethod def on_close(self, permanent=False): """ Implementers can handle what should happen when the RMQConnection is suddenly closed. An on_close call will normally be followed by an on_ready call from the RMQConnection unless the connection is permanently down. It is possible that the RMQConnection will signal multiple on_close calls for a single connection failure, therefore the implementation of on_close must be idempotent. :param permanent: bool """ pass @abstractmethod def on_error(self): """ Implementers can handle what should happen when the RMQConnection has run into an error, for example an Exchange/Queue declaration failure. """ pass def start(self): """ Starts the connection, opens a connection and a channel by running the underlying connection thread. When ready, callbacks will be invoked on each step of the way towards establishing the connection and channel, starting with on_connection_open if all went well. NOTE! Starting the connection is not a synchronous operation! It is this way since starting may or may not be successful, so a return value would be necessary to indicate success/failure. But, since retries will be made, an initial failure may in fact be successful the next second, meaning the user is given faulty information. Start success is therefore indicated via the 'on_ready' hook instead. """ if not self._connection_thread.is_alive(): self._connection_thread.start() def restart(self): """ Re-initiates the connection and channel. """ LOGGER.info("restarting connection") try: self._connection.close() self._restarting = True # Only if connection could be closed. except ConnectionWrongStateError: LOGGER.info("connection closed and could not be restarted") def stop(self): """ Starts a closing procedure for the connection and channel. """ LOGGER.info("stopping connection") if not self._closing: self._closing = True try: self._connection.close() self._connection_thread.join() except ConnectionWrongStateError: LOGGER.info("connection already closed") def declare_queue(self, queue_params, callback=None): """ :param queue_params: rabbitmq_client.QueueParams :param callback: callable """ LOGGER.info(f"declaring queue: {queue_params.queue}") self._channel.queue_declare(queue_params.queue, durable=queue_params.durable, exclusive=queue_params.exclusive, auto_delete=queue_params.auto_delete, arguments=queue_params.arguments, callback=callback) def declare_exchange(self, exchange_params, callback=None): """ :param exchange_params: rabbitmq_client.ExchangeParams :param callback: callable """ LOGGER.info(f"declaring exchange: {exchange_params.exchange}") self._channel.exchange_declare( exchange_params.exchange, exchange_type=exchange_params.exchange_type, durable=exchange_params.durable, auto_delete=exchange_params.auto_delete, internal=exchange_params.internal, arguments=exchange_params.arguments, callback=callback) def bind_queue(self, queue_bind_params, callback=None): """ :param queue_bind_params: rabbitmq_client.QueueBindParams :param callback: callable """ LOGGER.info(f"binding queue {queue_bind_params.queue} to exchange " f"{queue_bind_params.exchange} with routing key: " f"{queue_bind_params.routing_key}") self._channel.queue_bind(queue_bind_params.queue, queue_bind_params.exchange, routing_key=queue_bind_params.routing_key, arguments=queue_bind_params.arguments, callback=callback) def basic_consume(self, consume_params, on_message_callback_override=None, callback=None): """ :param consume_params: rabbitmq_client.ConsumeParams :param on_message_callback_override: callable :param callback: callable """ LOGGER.info(f"basic consumer starting for queue: " f"{consume_params.queue}") self._channel.basic_consume( consume_params.queue, (on_message_callback_override if on_message_callback_override else consume_params.on_message_callback), auto_ack=consume_params.auto_ack, exclusive=consume_params.exclusive, consumer_tag=consume_params.consumer_tag, arguments=consume_params.arguments, callback=callback) def basic_publish(self, body, exchange=DEFAULT_EXCHANGE, routing_key="", publish_params=None): """ :param body: bytes :param exchange: str :param routing_key: str :param publish_params: rabbitmq_client.PublishParams """ LOGGER.debug(f"publishing to exchange: {exchange} and routing key: " f"{routing_key}") if not publish_params: publish_params = PublishParams() # Create defaults self._channel.basic_publish(exchange, routing_key, body, properties=publish_params.properties, mandatory=publish_params.mandatory) def confirm_delivery(self, on_delivery_confirmed, callback=None): """ :param on_delivery_confirmed: callable :param callback: callable """ LOGGER.info("activating confirm delivery mode") self._channel.confirm_delivery(on_delivery_confirmed, callback=callback) def _connect(self): """ Assigns a new pika.SelectConnection to the connection and starts its ioloop to begin connecting. """ if self.connection_parameters is not None: LOGGER.info(f"starting connection towards: " f"{self.connection_parameters.host}:" f"{self.connection_parameters.port}") else: LOGGER.info("starting connection towards: 127.0.0.1:5672") self._connection = SelectConnection( parameters=self.connection_parameters, on_open_callback=self.on_connection_open, on_open_error_callback=self.on_connection_open_error, on_close_callback=self.on_connection_closed) self._connection.ioloop.start() def on_connection_open(self, _connection): """ Called when a new connection has been established. Opens a new channel. :param _connection: pika.SelectConnection """ LOGGER.debug("connection opened") # Do NOT reset reconnect counter here, do it after channel opened. self._connection.channel(on_open_callback=self.on_channel_open) def on_connection_open_error(self, _connection, error): """ Called when a connection could not be established towards RabbitMQ. This will lead to attempts to reconnect with an increasing delay between attempts. :param _connection: pika.SelectConnection :param error: pika.exceptions.? """ LOGGER.warning(f"error establishing connection, error: {error}") # This should ensure the current thread runs to completion after # reconnect handling is done. self._connection.ioloop.stop() # No fancy handling, just retry even if credentials are bad. self._reconnect() def on_connection_closed(self, _connection, reason): """ Called when the connection is closed, either unexpectedly or on purpose. Reconnection is attempted if the disconnect reason is something that is possible to recover, like an unexpected broker shutdown. :param _connection: pika.SelectConnection :param reason: pika.exceptions.? """ permanent = False # This should ensure the current thread runs to completion after # on_connection_closed handling is done. self._connection.ioloop.stop() if not self._closing and type(reason) in RECONNECT_REASONS: LOGGER.debug(f"connection closed: {reason}, attempting reconnect") self._reconnect() elif self._restarting: LOGGER.debug("connection closed, restarting now") # Reset to prevent reconnect for any reason self._restarting = False self._connection_thread = Thread(target=self._connect, daemon=True) self._connection_thread.start() else: permanent = True LOGGER.warning(f"connection closed: {reason}, will not reconnect") # Signal subclass that connection is down. self.on_close(permanent=permanent) @property def _reconnect_delay(self): return self._reconnect_attempts if self._reconnect_attempts < 9 else 30 def _reconnect(self): """ Starts up the connection again after a gradually increasing delay. """ LOGGER.debug(f"reconnect, attempt no. {self._reconnect_attempts + 1}") # Reconnect attempt may have been queued up before 'stop' was called. if self._closing: LOGGER.debug("skipping reconnect, connection stopped") return # _connect will assign a new connection object self._connection_thread = Thread(target=self._connect, daemon=True) if self._reconnect_delay == 0: # retry immediately the first time self._connection_thread.start() else: # calculate backoff timer timer = Timer(self._reconnect_delay, self._connection_thread.start) # Timer daemon exits if program exits, to avoid eternal retries. timer.daemon = True timer.start() self._reconnect_attempts += 1 def on_channel_open(self, channel): """ Called when a new channel has been opened, the consumer is now ready for work. :param channel: pika.channel.Channel """ LOGGER.debug("channel opened") self._reconnect_attempts = 0 self._channel = channel self._channel.add_on_close_callback(self.on_channel_closed) self.on_ready() # Signal subclass that connection is ready. def on_channel_closed(self, _channel, reason): """ Called when a channel is closed by RabbitMQ. Channels are not attempted to be re-opened since they are either closed from going down together with the connection, or because input parameters for some queue/exchange/other establishment was given faulty parameters. Retrying such an operation will just result in a failure loop. :param _channel: pika.channel.Channel :param reason: pika.exceptions.? """ LOGGER.warning("channel closed") permanent = False try: # Reply code 406 = PRECONDITION FAILED permanent = True if reason.reply_code == 406 else False except AttributeError: # Not all reasons have a reply code pass # Signal subclass that connection is down. self.on_close(permanent=permanent) if permanent: LOGGER.critical(f"connection stopping due to permanent channel " f"closure: {reason}") self.stop()
from pika import SelectConnection, ConnectionParameters from time import sleep CHANNEL = None QUEUE = 'xxx' def on_connection_open(connection): connection.channel(on_channel_open) def on_channel_open(channel): global CHANNEL CHANNEL = channel CHANNEL.queue_declare(queue=QUEUE, callback=on_queue_declared) def on_queue_declared(frame): count = 0 while True: body = str(count) CHANNEL.basic_publish(exchange='', routing_key=QUEUE, body=body) print 'Sent %s' % body sleep(3) count += 1 parameters = ConnectionParameters() connection = SelectConnection(parameters, on_open_callback=on_connection_open) try: connection.ioloop.start() except KeyboardInterrupt: connection.close() connection.ioloop.start()
class RabbitMQClient(object): def __init__(self, client, amqp_url, queues={}, reconection_time=10, prefetch_count=1, debug=True): self._amqp_url = amqp_url self._queues = queues self._reconection_time = reconection_time # 10 seconds self._last_timestamps = dict() self._prefetch_count = prefetch_count self._connection = None self._channel = None self._debug = debug self._client = client self._blocking_connection = None def message_count(self, queue_name): try: url = f"http://{self._host}:{self._port}/api/queues/{self._vhost}/{queue_name}?columns=message_stats.publish_details.rate" r = requests.get(url, auth=(self._creds.split(":")[0], self._creds.split(":")[1])) return r.json()["message_stats"]["publish_details"]["rate"] except requests.exceptions.RequestException as e: return None except: return 0 def on_open_connection(self, _unused_frame): if (self._debug): print(" [x] Connected - Connection state: OPEN. ") self._connection.channel(on_open_callback=self.on_channel_open) def on_channel_open(self, channel): """Callback when we have successfully declared the channel.""" if (self._debug): print(" [x] Channel - Channel state: OPEN. ") self._channel = channel for key, queue in self._queues.items(): cb = functools.partial(self.on_queue_declareok, userdata=key) self._channel.queue_declare(queue=key, durable=queue['durable'], exclusive=queue['exclusive'], auto_delete=queue['auto_delete'], callback=cb) def on_queue_declareok(self, _unused_frame, userdata): """Callback when we have successfully declared the queue. This call tells the server to send us self._prefetch_count message in advance. This helps overall throughput, but it does require us to deal with the messages we have promptly. """ queue_name = userdata if (self._debug): print(" [x] Queue declared - Queue: {}".format(queue_name)) cb = functools.partial(self.on_qos, userdata=queue_name) self._channel.basic_qos(prefetch_count=self._prefetch_count, callback=cb) def on_qos(self, _unused_frame, userdata): """Callback when Basic.QOS has completed.""" if (self._debug): print(" [x] QOS set to {}".format(self._prefetch_count)) queue_name = userdata self.start_consuming(queue_name) def on_message(self, channel, method, properties, body): queue_name = method.routing_key if (self._queues[queue_name]['callback']): callback = getattr(self._client, self._queues[queue_name]['callback']) callback(channel, method, properties, body) def start_consuming(self, queue_name): """Start consuming from task queue.""" if (self._debug): print(" [x] Waiting for messages...") self._channel.basic_consume( queue_name, self.on_message, auto_ack=self._queues[queue_name]['auto_ack']) def start(self): """Start worker.""" connected = False # Define connection while (not connected): try: self._creds = self._amqp_url.split('amqp://')[1].split("@")[0] parameters = URLParameters(self._amqp_url) self._host = parameters.host self._port = "15672" self._vhost = parameters.virtual_host self._blocking_connection = BlockingConnection(parameters) self._connection = SelectConnection( URLParameters(self._amqp_url), on_open_callback=self.on_open_connection) self._connection.ioloop.start() connected = True # Catch a Keyboard Interrupt to make sure that the connection is closed cleanly except KeyboardInterrupt: # Gracefully close the connection self._connection.close() # Start the IOLoop again so Pika can communicate, it will stop on its own when the connection is closed self._connection.ioloop.start() except: if (self._debug): print( " [!] RabbitMQ Host Unrecheable. Reconnecting in {} seconds..." .format(self._reconection_time)) time.sleep(self._reconection_time)