def run_pulse_listener(username, password, timeout, no_send):
"""Run a Pulse message queue listener."""
connection = Connection(
hostname='pulse.mozilla.org',
port=5671,
ssl=True,
userid=username,
password=password,
)
# Connect and pass in our own low value for retries so the connection
# fails fast if there is a problem.
connection.ensure_connection(
max_retries=1
) # Retries must be >=1 or it will retry forever.
with closing(connection):
hgpush_exchange = Exchange(config.PULSE_EXCHANGE, 'topic', channel=connection)
# Pulse queue names need to be prefixed with the username
queue_name = f'queue/{username}/{config.PULSE_QUEUE_NAME}'
queue = Queue(
queue_name,
exchange=hgpush_exchange,
routing_key=config.PULSE_QUEUE_ROUTING_KEY,
durable=True,
exclusive=False,
auto_delete=False,
channel=connection,
)
# Passing passive=True will assert that the exchange exists but won't
# try to declare it. The Pulse server forbids declaring exchanges.
hgpush_exchange.declare(passive=True)
# Queue.declare() also declares the exchange, which isn't allowed by
# the Pulse server. Use the low-level Queue API to only declare the
# queue itself.
queue.queue_declare()
queue.queue_bind()
callback = partial(process_push_message, no_send=no_send)
# Pass auto_declare=False so that Consumer does not try to declare the
# exchange. Declaring exchanges is not allowed by the Pulse server.
with connection.Consumer(
queue, callbacks=[callback], auto_declare=False
) as consumer:
if no_send:
log.info('transmission of ping data has been disabled')
log.info('message acks has been disabled')
log.info('reading messages')
try:
connection.drain_events(timeout=timeout)
except socket.timeout:
log.info('message queue is empty, nothing to do')
log.info('done')
def init(self):
try:
self.channel = channel = self.connection.channel()
queue = Queue(name=self.queueName, durable=False)
queue.queue_declare(channel=channel)
consumer = Consumer(channel=channel,
queues=[queue],
callbacks=[self._consume])
consumer.consume()
return True
except Exception as e:
logger.exception(e)
return False
class RabbitConsumer(ConsumerMixin):
def __init__(self, url, queue, routing_key, qpid):
self.connection = Connection(hostname=url)
self.exchange = Exchange(name='amq.direct',
type='direct',
channel=self.connection)
self.qpid = qpid
self.count = 0
kwargs = {
'exchange': self.exchange,
'routing_key': routing_key,
'channel': self.connection,
'name': queue,
}
# If the requested queue already exists, attach to it
# Otherwise, declare a new queue which will be deleted upon disconnect
try:
self.queue = Queue(**kwargs)
self.queue.queue_declare(passive=True)
except ChannelError:
self.queue = Queue(auto_delete=True, **kwargs)
def get_consumers(self, Consumer, channel):
c = Consumer([self.queue], callbacks=[self.on_message])
c.qos(prefetch_count=100)
return [c]
def on_message(self, body, message):
try:
self.qpid.send(str(body), message.headers)
message.ack()
self.count += 1
except Exception as e:
log.exception(
'Exception while publishing message to QPID, requeueing')
message.requeue()
self.qpid.sender = None
def get_current_queue_depth(self):
try:
result = self.queue.queue_declare(passive=True)
name = result.queue
count = result.message_count
except ChannelError:
if self.count > 0:
log.exception('Exception getting queue count')
name = 'UNK'
count = 0
return name, count, self.count
def init(self):
try:
self.channel = channel = self.connection.channel()
exchange = Exchange(name='policymq.direct', type='direct', durable=False)
exchange.declare(channel=channel)
queue = Queue(name=self.queueName, durable=False)
queue.queue_declare(channel=channel)
queue.bind_to(exchange=exchange, routing_key='ALL_', channel=channel)
queue.bind_to(exchange=exchange, routing_key=self.queueName, channel=channel)
consumer = Consumer(channel=channel, queues=[queue], callbacks=[self._consume])
consumer.consume()
return True
except Exception as e:
logger.exception(e)
return False
def declare_queue(self, connection,
name='',
auto_delete=False, durable=False,
**kwargs):
queue_args = kwargs.pop('queue_arguments', {})
queue_args['x-ha-policy'] = 'all'
queue = Queue(name,
durable=durable, auto_delete=auto_delete,
queue_arguments=queue_args,
**kwargs)
queue.maybe_bind(connection.default_channel)
queue.queue_declare()
return queue
def test_declare_but_no_exchange(self):
q = Queue('a')
q.queue_declare = Mock()
q.queue_bind = Mock()
q.exchange = None
q.declare()
q.queue_declare.assert_called_with(False, passive=False)
def test_declare_but_no_exchange(self):
q = Queue('a')
q.queue_declare = Mock()
q.queue_bind = Mock()
q.exchange = None
q.declare()
q.queue_declare.assert_called_with(False, passive=False)
def test_declare__no_declare(self):
q = Queue('a', no_declare=True)
q.queue_declare = Mock()
q.queue_bind = Mock()
q.exchange = None
q.declare()
q.queue_declare.assert_not_called()
q.queue_bind.assert_not_called()
def test_declare__no_declare(self):
q = Queue('a', no_declare=True)
q.queue_declare = Mock()
q.queue_bind = Mock()
q.exchange = None
q.declare()
self.assertFalse(q.queue_declare.called)
self.assertFalse(q.queue_bind.called)
def test_declare__no_declare(self):
q = Queue('a', no_declare=True)
q.queue_declare = Mock()
q.queue_bind = Mock()
q.exchange = None
q.declare()
q.queue_declare.assert_not_called()
q.queue_bind.assert_not_called()
def declare_queue(self,
connection,
name='',
auto_delete=False,
durable=False,
**kwargs):
queue_args = kwargs.pop('queue_arguments', {})
queue_args['x-ha-policy'] = 'all'
queue = Queue(name,
durable=durable,
auto_delete=auto_delete,
queue_arguments=queue_args,
**kwargs)
queue.maybe_bind(connection.default_channel)
queue.queue_declare()
return queue
def test_declare__no_declare(self):
q = Queue('a', no_declare=True)
q.queue_declare = Mock()
q.queue_bind = Mock()
q.exchange = None
q.declare()
self.assertFalse(q.queue_declare.called)
self.assertFalse(q.queue_bind.called)
def push_sync(self,upload=True,delay=0,dryrun=False,timeout=None): # pragma: no cover
"wait for push messages"
from kombu import Connection, Exchange, Queue, Consumer
import socket, ssl
url, opts, exchange, queue = self.get_broker()
def callback(body, message):
self.process_update(body)
message.ack()
with Connection(url,**opts) as conn:
self.connection = conn
queue = Queue(queue, channel=conn)
queue.queue_declare()
queue.bind_to(exchange)
try: excpt = (socket.error, ssl.SSLZeroReturnError)
except AttributeError: excpt = socket.error
with conn.Consumer(queue, accept=['json'], callbacks=[callback]) as consumer:
while True:
try: conn.drain_events(timeout=timeout)
except socket.timeout: pass
except excpt: break
def test_connection(host, port, user_id, password, virt_host, exchange_name,
queue_name):
"""
Test a connection to an exchange on a virtual host
"""
connection = None
connected = False
success = False
try:
# Connect to the virtual host - will raise exception if it fails.
connection = Connection(host, user_id, password, virt_host, port)
connection.connect()
connected = connection.connected
if connected:
# Check whether exchange exists - will raise exception if it fails.
exchange = Exchange(exchange_name,
channel=connection,
type='topic',
durable=False,
passive=True)
exchange.declare()
# Check whether the queue exists - will raise exception if it
# fails.
rpc_receive_queue = Queue(queue_name,
durable=True,
exchange=exchange,
channel=connection)
rpc_receive_queue.queue_declare(passive=True)
success = True
except Exception as e:
DLOG.info("Unable to connect to virt_host %s, exchange %s, error: %s" %
(virt_host, exchange_name, e))
finally:
if connected:
connection.close()
return success
def run_pulse_listener(
username,
password,
exchange_name,
queue_name,
routing_key,
timeout,
no_send,
worker_args=None,
empty_queue_callback=None,
):
"""Run a Pulse message queue listener."""
connection = build_connection(password, username)
# Connect and pass in our own low value for retries so the connection
# fails fast if there is a problem.
connection.ensure_connection(
max_retries=1) # Retries must be >=1 or it will retry forever.
with closing(connection):
hgpush_exchange = Exchange(exchange_name, "topic", channel=connection)
# Pulse queue names need to be prefixed with the username
queue_name = f"queue/{username}/{queue_name}"
queue = Queue(
queue_name,
exchange=hgpush_exchange,
routing_key=routing_key,
durable=True,
exclusive=False,
auto_delete=False,
channel=connection,
)
# Passing passive=True will assert that the exchange exists but won't
# try to declare it. The Pulse server forbids declaring exchanges.
hgpush_exchange.declare(passive=True)
# Queue.declare() also declares the exchange, which isn't allowed by
# the Pulse server. Use the low-level Queue API to only declare the
# queue itself.
queue.queue_declare()
queue.queue_bind()
callback = partial(process_push_message,
no_send=no_send,
extra_data=worker_args)
# Pass auto_declare=False so that Consumer does not try to declare the
# exchange. Declaring exchanges is not allowed by the Pulse server.
with connection.Consumer(queue,
callbacks=[callback],
auto_declare=False):
if no_send:
log.info("transmission of monitoring data has been disabled")
log.info("message acks has been disabled")
log.info("reading messages")
try:
connection.drain_events(timeout=timeout)
except socket.timeout:
log.info("message queue is empty")
if empty_queue_callback:
empty_queue_callback()
except HaltQueueProcessing:
log.debug("queue processing halted by consumer")
log.info("done")
class ConsumerMixin(KombuConsumer):
consumer_args = {}
def __init__(self, subscribe=None, **kwargs):
super(ConsumerMixin, self).__init__(**kwargs)
name = '{name}.{id}'.format(name=self._name, id=os.urandom(3).hex())
if subscribe:
exchange = Exchange(subscribe['exchange_name'], subscribe['exchange_type'])
self.__connection = Connection(self.url)
self.__exchange = exchange if subscribe else self._default_exchange
self.__subscriptions = defaultdict(list)
self.__queue = Queue(name=name, auto_delete=True, durable=False)
self.__lock = Lock()
self.create_connection()
try:
self._register_thread('consumer', self.__run, on_stop=self.__stop)
except AttributeError:
pass
@property
@contextmanager
def __binding_channel(self):
if not self.__connection.connected:
self.__connection.connect()
yield self.__connection.default_channel
def __create_binding(self, headers, routing_key):
binding = Binding(self.__exchange, routing_key, headers, headers)
self.__queue.bindings.add(binding)
if self.is_running:
try:
with self.__binding_channel as channel:
self.__queue.queue_declare(passive=True, channel=channel)
binding.bind(self.__queue, channel=channel)
except self.__connection.connection_errors as e:
self.log.error('Connection error while creating binding: %s', e)
except NotFound:
self.log.error(
'Queue %s doesn\'t exist on the server', self.__queue.name
)
return binding
def __remove_binding(self, binding):
self.__queue.bindings.remove(binding)
if self.is_running:
try:
with self.__binding_channel as channel:
self.__queue.queue_declare(passive=True, channel=channel)
binding.unbind(self.__queue, channel=channel)
except self.__connection.connection_errors:
self.log.exception('Connection error while removing binding: %s')
except NotFound:
pass
def __dispatch(self, event_name, payload, headers=None):
with self.__lock:
subscriptions = self.__subscriptions[event_name].copy()
for (handler, _) in subscriptions:
try:
handler(payload)
except Exception:
self.log.exception(
'Handler \'%s\' for event \'%s\' failed',
getattr(handler, '__name__', handler),
event_name,
)
continue
def __extract_event_from_message(self, message):
event_name = None
headers = message.headers
payload = message.payload
if 'name' in headers:
event_name = headers['name']
elif isinstance(payload, dict) and 'name' in payload:
event_name = payload['name']
else:
raise ValueError('Received invalid messsage; no event name could be found.')
return event_name, headers, payload
def subscribe(
self,
event_name,
handler,
headers=None,
routing_key=None,
headers_match_all=True,
):
headers = dict(headers or {})
headers.update(name=event_name)
if self.__exchange.type == 'headers':
headers.setdefault('x-match', 'all' if headers_match_all else 'any')
binding = self.__create_binding(headers, routing_key)
subscription = Subscription(handler, binding)
with self.__lock:
self.__subscriptions[event_name].append(subscription)
self.log.debug(
'Registered handler \'%s\' to event \'%s\'',
getattr(handler, '__name__', handler),
event_name,
)
def unsubscribe(self, event_name, handler):
with self.__lock:
subscriptions = self.__subscriptions[event_name].copy()
try:
for subscription in subscriptions:
if subscription.handler == handler:
with self.__lock:
self.__subscriptions[event_name].remove(subscription)
self.__remove_binding(subscription.binding)
self.log.debug(
'Unregistered handler \'%s\' from \'%s\'',
getattr(handler, '__name__', handler),
event_name,
)
return True
return False
finally:
if not self.__subscriptions[event_name]:
with self.__lock:
self.__subscriptions.pop(event_name)
def get_consumers(self, Consumer, channel):
self.__exchange.bind(channel).declare()
return [
Consumer(
queues=[self.__queue],
callbacks=[self.__on_message_received],
auto_declare=True,
)
]
def __on_message_received(self, body, message):
event_name, headers, payload = self.__extract_event_from_message(message)
if event_name not in self.__subscriptions:
return
try:
headers, payload = self._unmarshal(event_name, headers, payload)
except Exception:
raise
else:
self.__dispatch(event_name, payload, headers)
finally:
message.ack()
def on_connection_error(self, exc, interval):
self.log.error(
'Broker connection error: %s, trying to reconnect in %s seconds...',
exc,
interval,
)
if self.should_stop:
# Workaround to force kill the threaded consumer when a stop has been issued
# instead of looping forever to reestablish the connection
raise SystemExit
def create_connection(self):
self.connection = self.__connection.clone()
return self.connection
@property
def is_running(self):
try:
is_running = self.connection.connected
except AttributeError:
is_running = False
return super(ConsumerMixin, self).is_running and is_running
@property
def should_stop(self):
return getattr(self, 'is_stopping', True)
def on_consume_ready(self, connection, channel, consumers, **kwargs):
if 'ready_flag' in kwargs:
ready_flag = kwargs.pop('ready_flag')
ready_flag.set()
def __run(self, ready_flag, **kwargs):
super(ConsumerMixin, self).run(ready_flag=ready_flag, **self.consumer_args)
def __stop(self):
self.__connection.release()
