class Worker(ConsumerMixin):
def __init__(self, **kwargs):
self.__callbacks = kwargs.get('callbacks',self.on_message)
self.__amqp_url = kwargs.get('amqp_url',AMQP_URL)
self.__queue = kwargs.get('queue')
self.__conn_retries = kwargs.get('max_retries',2)
if self.__queue is None:
raise TypeError('invalid worker queue parameter')
self.connection = BrokerConnection(self.__amqp_url)
self.connection.ensure_connection(max_retries=self.__conn_retries, callback=self.on_conn_retry)
def get_consumers(self, consumer, channel):
if not isinstance(self.__callbacks,list):
self.__callbacks = [ self.__callbacks ]
return [consumer(self.__queue, callbacks=self.__callbacks)]
def on_message(self, body, message):
out = 'Received message: %r' % dumps(body)
out += ' properties: %s' % dumps(message.properties)
out += ' delivery_info: %s' % dumps(message.delivery_info)
LOG.info(out,json=True)
message.ack()
def on_conn_retry(self):
LOG.error('Retrying connection for {0}'.format(self.__amqp_url))
def start(self):
LOG.info('Starting AMQP worker {0}'.format(self.__queue))
self.run()
def stop(self):
LOG.info('Stopping AMQP worker {0}'.format(self.__queue))
self.should_stop = True
class AMQPWorker(ConsumerMixin):
def __init__(self, **kwargs):
self.__callbacks = kwargs.get('callbacks', self.on_message)
self.__amqp_url = kwargs.get('amqp_url', AMQP_URL)
self.__queue = kwargs.get('queue')
self.__max_retries = kwargs.get('max_retries', 2)
self.__max_error = kwargs.get('max_error', 3)
if self.__queue is None:
raise TypeError('invalid worker queue parameter')
self.connection = BrokerConnection(self.__amqp_url)
self.connection.ensure_connection(max_retries=self.__max_retries,
errback=self.on_connection_error,
callback=self.on_conn_retry)
def get_consumers(self, consumer, channel):
if not isinstance(self.__callbacks, list):
self.__callbacks = [self.__callbacks]
return [consumer(self.__queue, callbacks=self.__callbacks)]
def on_message(self, body, message):
out = {
'message': body,
'properties': message.properties,
'delivery_info': message.delivery_info
}
LOG.info(out, json=True)
message.ack()
def on_conn_retry(self):
LOG.error('Retrying connection for {0}'.format(self.__amqp_url))
def on_connection_error(self, exc, interval):
if self.__max_error:
LOG.warning(
'Connection error, retrying in {0} seconds (retry={1})'.format(
interval, self.__max_error))
self.__max_error -= 1
else:
LOG.error('max connection errors exceeded.')
stop()
def start(self):
LOG.info('Starting AMQP worker {0}'.format(self.__queue))
self.should_stop = False
self.run()
def stop(self):
LOG.info('Stopping AMQP worker {0}'.format(self.__queue))
self.should_stop = True
class AMQPWorker(ConsumerMixin):
def __init__(self, **kwargs):
self.__callbacks = kwargs.get('callbacks',self.on_message)
self.__amqp_url = kwargs.get('amqp_url',AMQP_URL)
self.__queue = kwargs.get('queue')
self.__max_retries = kwargs.get('max_retries',2)
self.__max_error = kwargs.get('max_error',3)
if self.__queue is None:
raise TypeError('invalid worker queue parameter')
self.connection = BrokerConnection(self.__amqp_url)
self.connection.ensure_connection(max_retries=self.__max_retries,
errback=self.on_connection_error, callback=self.on_conn_retry)
def get_consumers(self, consumer, channel):
if not isinstance(self.__callbacks,list):
self.__callbacks = [ self.__callbacks ]
return [consumer(self.__queue, callbacks=self.__callbacks)]
def on_message(self, body, message):
out = {
'message':body,
'properties':message.properties,
'delivery_info': message.delivery_info
}
LOG.info(out, json=True)
message.ack()
def on_conn_retry(self):
LOG.error('Retrying connection for {0}'.format(self.__amqp_url))
def on_connection_error(self, exc, interval):
if self.__max_error:
LOG.warning('Connection error, retrying in {0} seconds (retry={1})'.format(interval, self.__max_error))
self.__max_error -= 1
else:
LOG.error('max connection errors exceeded.')
stop()
def start(self):
LOG.info('Starting AMQP worker {0}'.format(self.__queue))
self.should_stop = False
self.run()
def stop(self):
LOG.info('Stopping AMQP worker {0}'.format(self.__queue))
self.should_stop = True
def _establish_connection(self, conn: BrokerConnection) -> None:
"""
We don't use a pool here. We only have one consumer connection per process, so
we get no value from a pool, and we want to use a heartbeat to keep the consumer
collection alive, which does not work with a pool
:return: the connection to the transport
"""
try:
self._logger.debug("Establishing connection.")
self._conn = conn.ensure_connection(max_retries=3)
self._logger.debug('Got connection: %s', conn.as_uri())
except kombu_exceptions.OperationalError as oe:
self._logger.error("Error connecting to RMQ, could not retry %s", oe)
# Try to clean up the mess
if self._conn is not None:
self._conn.close()
else:
conn.close()
class RabbitMQHandler(object):
def __init__(self,
connection_string,
exchange_name,
exchange_type="topic"):
self._connection = BrokerConnection(connection_string)
self._connections = {self._connection
} # set of connection for the heartbeat
self._exchange = Exchange(exchange_name,
durable=True,
delivery_mode=2,
type=exchange_type,
auto_delete=False,
no_declare=False)
monitor_heartbeats(self._connections)
@retry(wait_fixed=200, stop_max_attempt_number=3)
def publish(self, item, contributor_id):
with self._connection.channel() as channel:
with Producer(channel) as producer:
producer.publish(
item,
exchange=self._exchange,
routing_key=contributor_id,
declare=[self._exchange],
content_type="plain/text",
)
def info(self):
info = self._connection.info()
info.pop("password", None)
return info
def check_connection(self, force=False):
"""
Trying to connect is the best way to check that the connection works
if force is set to True, we will force the connection again
"""
try:
# we need to refresh the connection to be notified as soon as rabbitmq stopped working
if force:
self._connection._establish_connection()
self._connection.ensure_connection(interval_start=0, max_retries=1)
return True
except Exception:
return False
def connect(self):
self._connection.connect()
def close(self):
for c in self._connections:
c.release()
def listen_load_realtime(self, queue_name, max_retries=10):
log = logging.getLogger(__name__)
route = "task.load_realtime.*"
log.info("listening route {} on exchange {}...".format(
route, self._exchange))
rt_queue = Queue(queue_name,
routing_key=route,
exchange=self._exchange,
durable=False)
RTReloader(connection=self._connection,
rpc_queue=rt_queue,
exchange=self._exchange,
max_retries=max_retries).run()
