class Worker(ConsumerMixin): def __init__(self, **kwargs): self.__callbacks = kwargs.get('callbacks',self.on_message) self.__amqp_url = kwargs.get('amqp_url',AMQP_URL) self.__queue = kwargs.get('queue') self.__conn_retries = kwargs.get('max_retries',2) if self.__queue is None: raise TypeError('invalid worker queue parameter') self.connection = BrokerConnection(self.__amqp_url) self.connection.ensure_connection(max_retries=self.__conn_retries, callback=self.on_conn_retry) def get_consumers(self, consumer, channel): if not isinstance(self.__callbacks,list): self.__callbacks = [ self.__callbacks ] return [consumer(self.__queue, callbacks=self.__callbacks)] def on_message(self, body, message): out = 'Received message: %r' % dumps(body) out += ' properties: %s' % dumps(message.properties) out += ' delivery_info: %s' % dumps(message.delivery_info) LOG.info(out,json=True) message.ack() def on_conn_retry(self): LOG.error('Retrying connection for {0}'.format(self.__amqp_url)) def start(self): LOG.info('Starting AMQP worker {0}'.format(self.__queue)) self.run() def stop(self): LOG.info('Stopping AMQP worker {0}'.format(self.__queue)) self.should_stop = True
class AMQPWorker(ConsumerMixin): def __init__(self, **kwargs): self.__callbacks = kwargs.get('callbacks', self.on_message) self.__amqp_url = kwargs.get('amqp_url', AMQP_URL) self.__queue = kwargs.get('queue') self.__max_retries = kwargs.get('max_retries', 2) self.__max_error = kwargs.get('max_error', 3) if self.__queue is None: raise TypeError('invalid worker queue parameter') self.connection = BrokerConnection(self.__amqp_url) self.connection.ensure_connection(max_retries=self.__max_retries, errback=self.on_connection_error, callback=self.on_conn_retry) def get_consumers(self, consumer, channel): if not isinstance(self.__callbacks, list): self.__callbacks = [self.__callbacks] return [consumer(self.__queue, callbacks=self.__callbacks)] def on_message(self, body, message): out = { 'message': body, 'properties': message.properties, 'delivery_info': message.delivery_info } LOG.info(out, json=True) message.ack() def on_conn_retry(self): LOG.error('Retrying connection for {0}'.format(self.__amqp_url)) def on_connection_error(self, exc, interval): if self.__max_error: LOG.warning( 'Connection error, retrying in {0} seconds (retry={1})'.format( interval, self.__max_error)) self.__max_error -= 1 else: LOG.error('max connection errors exceeded.') stop() def start(self): LOG.info('Starting AMQP worker {0}'.format(self.__queue)) self.should_stop = False self.run() def stop(self): LOG.info('Stopping AMQP worker {0}'.format(self.__queue)) self.should_stop = True
class AMQPWorker(ConsumerMixin): def __init__(self, **kwargs): self.__callbacks = kwargs.get('callbacks',self.on_message) self.__amqp_url = kwargs.get('amqp_url',AMQP_URL) self.__queue = kwargs.get('queue') self.__max_retries = kwargs.get('max_retries',2) self.__max_error = kwargs.get('max_error',3) if self.__queue is None: raise TypeError('invalid worker queue parameter') self.connection = BrokerConnection(self.__amqp_url) self.connection.ensure_connection(max_retries=self.__max_retries, errback=self.on_connection_error, callback=self.on_conn_retry) def get_consumers(self, consumer, channel): if not isinstance(self.__callbacks,list): self.__callbacks = [ self.__callbacks ] return [consumer(self.__queue, callbacks=self.__callbacks)] def on_message(self, body, message): out = { 'message':body, 'properties':message.properties, 'delivery_info': message.delivery_info } LOG.info(out, json=True) message.ack() def on_conn_retry(self): LOG.error('Retrying connection for {0}'.format(self.__amqp_url)) def on_connection_error(self, exc, interval): if self.__max_error: LOG.warning('Connection error, retrying in {0} seconds (retry={1})'.format(interval, self.__max_error)) self.__max_error -= 1 else: LOG.error('max connection errors exceeded.') stop() def start(self): LOG.info('Starting AMQP worker {0}'.format(self.__queue)) self.should_stop = False self.run() def stop(self): LOG.info('Stopping AMQP worker {0}'.format(self.__queue)) self.should_stop = True
def _establish_connection(self, conn: BrokerConnection) -> None: """ We don't use a pool here. We only have one consumer connection per process, so we get no value from a pool, and we want to use a heartbeat to keep the consumer collection alive, which does not work with a pool :return: the connection to the transport """ try: self._logger.debug("Establishing connection.") self._conn = conn.ensure_connection(max_retries=3) self._logger.debug('Got connection: %s', conn.as_uri()) except kombu_exceptions.OperationalError as oe: self._logger.error("Error connecting to RMQ, could not retry %s", oe) # Try to clean up the mess if self._conn is not None: self._conn.close() else: conn.close()
class RabbitMQHandler(object): def __init__(self, connection_string, exchange_name, exchange_type="topic"): self._connection = BrokerConnection(connection_string) self._connections = {self._connection } # set of connection for the heartbeat self._exchange = Exchange(exchange_name, durable=True, delivery_mode=2, type=exchange_type, auto_delete=False, no_declare=False) monitor_heartbeats(self._connections) @retry(wait_fixed=200, stop_max_attempt_number=3) def publish(self, item, contributor_id): with self._connection.channel() as channel: with Producer(channel) as producer: producer.publish( item, exchange=self._exchange, routing_key=contributor_id, declare=[self._exchange], content_type="plain/text", ) def info(self): info = self._connection.info() info.pop("password", None) return info def check_connection(self, force=False): """ Trying to connect is the best way to check that the connection works if force is set to True, we will force the connection again """ try: # we need to refresh the connection to be notified as soon as rabbitmq stopped working if force: self._connection._establish_connection() self._connection.ensure_connection(interval_start=0, max_retries=1) return True except Exception: return False def connect(self): self._connection.connect() def close(self): for c in self._connections: c.release() def listen_load_realtime(self, queue_name, max_retries=10): log = logging.getLogger(__name__) route = "task.load_realtime.*" log.info("listening route {} on exchange {}...".format( route, self._exchange)) rt_queue = Queue(queue_name, routing_key=route, exchange=self._exchange, durable=False) RTReloader(connection=self._connection, rpc_queue=rt_queue, exchange=self._exchange, max_retries=max_retries).run()