def run_pulse_listener(username, password, timeout, no_send): """Run a Pulse message queue listener.""" connection = Connection( hostname='pulse.mozilla.org', port=5671, ssl=True, userid=username, password=password, ) # Connect and pass in our own low value for retries so the connection # fails fast if there is a problem. connection.ensure_connection( max_retries=1 ) # Retries must be >=1 or it will retry forever. with closing(connection): hgpush_exchange = Exchange(config.PULSE_EXCHANGE, 'topic', channel=connection) # Pulse queue names need to be prefixed with the username queue_name = f'queue/{username}/{config.PULSE_QUEUE_NAME}' queue = Queue( queue_name, exchange=hgpush_exchange, routing_key=config.PULSE_QUEUE_ROUTING_KEY, durable=True, exclusive=False, auto_delete=False, channel=connection, ) # Passing passive=True will assert that the exchange exists but won't # try to declare it. The Pulse server forbids declaring exchanges. hgpush_exchange.declare(passive=True) # Queue.declare() also declares the exchange, which isn't allowed by # the Pulse server. Use the low-level Queue API to only declare the # queue itself. queue.queue_declare() queue.queue_bind() callback = partial(process_push_message, no_send=no_send) # Pass auto_declare=False so that Consumer does not try to declare the # exchange. Declaring exchanges is not allowed by the Pulse server. with connection.Consumer( queue, callbacks=[callback], auto_declare=False ) as consumer: if no_send: log.info('transmission of ping data has been disabled') log.info('message acks has been disabled') log.info('reading messages') try: connection.drain_events(timeout=timeout) except socket.timeout: log.info('message queue is empty, nothing to do') log.info('done')
def init(self): try: self.channel = channel = self.connection.channel() queue = Queue(name=self.queueName, durable=False) queue.queue_declare(channel=channel) consumer = Consumer(channel=channel, queues=[queue], callbacks=[self._consume]) consumer.consume() return True except Exception as e: logger.exception(e) return False
class RabbitConsumer(ConsumerMixin): def __init__(self, url, queue, routing_key, qpid): self.connection = Connection(hostname=url) self.exchange = Exchange(name='amq.direct', type='direct', channel=self.connection) self.qpid = qpid self.count = 0 kwargs = { 'exchange': self.exchange, 'routing_key': routing_key, 'channel': self.connection, 'name': queue, } # If the requested queue already exists, attach to it # Otherwise, declare a new queue which will be deleted upon disconnect try: self.queue = Queue(**kwargs) self.queue.queue_declare(passive=True) except ChannelError: self.queue = Queue(auto_delete=True, **kwargs) def get_consumers(self, Consumer, channel): c = Consumer([self.queue], callbacks=[self.on_message]) c.qos(prefetch_count=100) return [c] def on_message(self, body, message): try: self.qpid.send(str(body), message.headers) message.ack() self.count += 1 except Exception as e: log.exception( 'Exception while publishing message to QPID, requeueing') message.requeue() self.qpid.sender = None def get_current_queue_depth(self): try: result = self.queue.queue_declare(passive=True) name = result.queue count = result.message_count except ChannelError: if self.count > 0: log.exception('Exception getting queue count') name = 'UNK' count = 0 return name, count, self.count
def init(self): try: self.channel = channel = self.connection.channel() exchange = Exchange(name='policymq.direct', type='direct', durable=False) exchange.declare(channel=channel) queue = Queue(name=self.queueName, durable=False) queue.queue_declare(channel=channel) queue.bind_to(exchange=exchange, routing_key='ALL_', channel=channel) queue.bind_to(exchange=exchange, routing_key=self.queueName, channel=channel) consumer = Consumer(channel=channel, queues=[queue], callbacks=[self._consume]) consumer.consume() return True except Exception as e: logger.exception(e) return False
def declare_queue(self, connection, name='', auto_delete=False, durable=False, **kwargs): queue_args = kwargs.pop('queue_arguments', {}) queue_args['x-ha-policy'] = 'all' queue = Queue(name, durable=durable, auto_delete=auto_delete, queue_arguments=queue_args, **kwargs) queue.maybe_bind(connection.default_channel) queue.queue_declare() return queue
def test_declare_but_no_exchange(self): q = Queue('a') q.queue_declare = Mock() q.queue_bind = Mock() q.exchange = None q.declare() q.queue_declare.assert_called_with(False, passive=False)
def test_declare__no_declare(self): q = Queue('a', no_declare=True) q.queue_declare = Mock() q.queue_bind = Mock() q.exchange = None q.declare() q.queue_declare.assert_not_called() q.queue_bind.assert_not_called()
def test_declare__no_declare(self): q = Queue('a', no_declare=True) q.queue_declare = Mock() q.queue_bind = Mock() q.exchange = None q.declare() self.assertFalse(q.queue_declare.called) self.assertFalse(q.queue_bind.called)
def push_sync(self,upload=True,delay=0,dryrun=False,timeout=None): # pragma: no cover "wait for push messages" from kombu import Connection, Exchange, Queue, Consumer import socket, ssl url, opts, exchange, queue = self.get_broker() def callback(body, message): self.process_update(body) message.ack() with Connection(url,**opts) as conn: self.connection = conn queue = Queue(queue, channel=conn) queue.queue_declare() queue.bind_to(exchange) try: excpt = (socket.error, ssl.SSLZeroReturnError) except AttributeError: excpt = socket.error with conn.Consumer(queue, accept=['json'], callbacks=[callback]) as consumer: while True: try: conn.drain_events(timeout=timeout) except socket.timeout: pass except excpt: break
def test_connection(host, port, user_id, password, virt_host, exchange_name, queue_name): """ Test a connection to an exchange on a virtual host """ connection = None connected = False success = False try: # Connect to the virtual host - will raise exception if it fails. connection = Connection(host, user_id, password, virt_host, port) connection.connect() connected = connection.connected if connected: # Check whether exchange exists - will raise exception if it fails. exchange = Exchange(exchange_name, channel=connection, type='topic', durable=False, passive=True) exchange.declare() # Check whether the queue exists - will raise exception if it # fails. rpc_receive_queue = Queue(queue_name, durable=True, exchange=exchange, channel=connection) rpc_receive_queue.queue_declare(passive=True) success = True except Exception as e: DLOG.info("Unable to connect to virt_host %s, exchange %s, error: %s" % (virt_host, exchange_name, e)) finally: if connected: connection.close() return success
def run_pulse_listener( username, password, exchange_name, queue_name, routing_key, timeout, no_send, worker_args=None, empty_queue_callback=None, ): """Run a Pulse message queue listener.""" connection = build_connection(password, username) # Connect and pass in our own low value for retries so the connection # fails fast if there is a problem. connection.ensure_connection( max_retries=1) # Retries must be >=1 or it will retry forever. with closing(connection): hgpush_exchange = Exchange(exchange_name, "topic", channel=connection) # Pulse queue names need to be prefixed with the username queue_name = f"queue/{username}/{queue_name}" queue = Queue( queue_name, exchange=hgpush_exchange, routing_key=routing_key, durable=True, exclusive=False, auto_delete=False, channel=connection, ) # Passing passive=True will assert that the exchange exists but won't # try to declare it. The Pulse server forbids declaring exchanges. hgpush_exchange.declare(passive=True) # Queue.declare() also declares the exchange, which isn't allowed by # the Pulse server. Use the low-level Queue API to only declare the # queue itself. queue.queue_declare() queue.queue_bind() callback = partial(process_push_message, no_send=no_send, extra_data=worker_args) # Pass auto_declare=False so that Consumer does not try to declare the # exchange. Declaring exchanges is not allowed by the Pulse server. with connection.Consumer(queue, callbacks=[callback], auto_declare=False): if no_send: log.info("transmission of monitoring data has been disabled") log.info("message acks has been disabled") log.info("reading messages") try: connection.drain_events(timeout=timeout) except socket.timeout: log.info("message queue is empty") if empty_queue_callback: empty_queue_callback() except HaltQueueProcessing: log.debug("queue processing halted by consumer") log.info("done")
class ConsumerMixin(KombuConsumer): consumer_args = {} def __init__(self, subscribe=None, **kwargs): super(ConsumerMixin, self).__init__(**kwargs) name = '{name}.{id}'.format(name=self._name, id=os.urandom(3).hex()) if subscribe: exchange = Exchange(subscribe['exchange_name'], subscribe['exchange_type']) self.__connection = Connection(self.url) self.__exchange = exchange if subscribe else self._default_exchange self.__subscriptions = defaultdict(list) self.__queue = Queue(name=name, auto_delete=True, durable=False) self.__lock = Lock() self.create_connection() try: self._register_thread('consumer', self.__run, on_stop=self.__stop) except AttributeError: pass @property @contextmanager def __binding_channel(self): if not self.__connection.connected: self.__connection.connect() yield self.__connection.default_channel def __create_binding(self, headers, routing_key): binding = Binding(self.__exchange, routing_key, headers, headers) self.__queue.bindings.add(binding) if self.is_running: try: with self.__binding_channel as channel: self.__queue.queue_declare(passive=True, channel=channel) binding.bind(self.__queue, channel=channel) except self.__connection.connection_errors as e: self.log.error('Connection error while creating binding: %s', e) except NotFound: self.log.error( 'Queue %s doesn\'t exist on the server', self.__queue.name ) return binding def __remove_binding(self, binding): self.__queue.bindings.remove(binding) if self.is_running: try: with self.__binding_channel as channel: self.__queue.queue_declare(passive=True, channel=channel) binding.unbind(self.__queue, channel=channel) except self.__connection.connection_errors: self.log.exception('Connection error while removing binding: %s') except NotFound: pass def __dispatch(self, event_name, payload, headers=None): with self.__lock: subscriptions = self.__subscriptions[event_name].copy() for (handler, _) in subscriptions: try: handler(payload) except Exception: self.log.exception( 'Handler \'%s\' for event \'%s\' failed', getattr(handler, '__name__', handler), event_name, ) continue def __extract_event_from_message(self, message): event_name = None headers = message.headers payload = message.payload if 'name' in headers: event_name = headers['name'] elif isinstance(payload, dict) and 'name' in payload: event_name = payload['name'] else: raise ValueError('Received invalid messsage; no event name could be found.') return event_name, headers, payload def subscribe( self, event_name, handler, headers=None, routing_key=None, headers_match_all=True, ): headers = dict(headers or {}) headers.update(name=event_name) if self.__exchange.type == 'headers': headers.setdefault('x-match', 'all' if headers_match_all else 'any') binding = self.__create_binding(headers, routing_key) subscription = Subscription(handler, binding) with self.__lock: self.__subscriptions[event_name].append(subscription) self.log.debug( 'Registered handler \'%s\' to event \'%s\'', getattr(handler, '__name__', handler), event_name, ) def unsubscribe(self, event_name, handler): with self.__lock: subscriptions = self.__subscriptions[event_name].copy() try: for subscription in subscriptions: if subscription.handler == handler: with self.__lock: self.__subscriptions[event_name].remove(subscription) self.__remove_binding(subscription.binding) self.log.debug( 'Unregistered handler \'%s\' from \'%s\'', getattr(handler, '__name__', handler), event_name, ) return True return False finally: if not self.__subscriptions[event_name]: with self.__lock: self.__subscriptions.pop(event_name) def get_consumers(self, Consumer, channel): self.__exchange.bind(channel).declare() return [ Consumer( queues=[self.__queue], callbacks=[self.__on_message_received], auto_declare=True, ) ] def __on_message_received(self, body, message): event_name, headers, payload = self.__extract_event_from_message(message) if event_name not in self.__subscriptions: return try: headers, payload = self._unmarshal(event_name, headers, payload) except Exception: raise else: self.__dispatch(event_name, payload, headers) finally: message.ack() def on_connection_error(self, exc, interval): self.log.error( 'Broker connection error: %s, trying to reconnect in %s seconds...', exc, interval, ) if self.should_stop: # Workaround to force kill the threaded consumer when a stop has been issued # instead of looping forever to reestablish the connection raise SystemExit def create_connection(self): self.connection = self.__connection.clone() return self.connection @property def is_running(self): try: is_running = self.connection.connected except AttributeError: is_running = False return super(ConsumerMixin, self).is_running and is_running @property def should_stop(self): return getattr(self, 'is_stopping', True) def on_consume_ready(self, connection, channel, consumers, **kwargs): if 'ready_flag' in kwargs: ready_flag = kwargs.pop('ready_flag') ready_flag.set() def __run(self, ready_flag, **kwargs): super(ConsumerMixin, self).run(ready_flag=ready_flag, **self.consumer_args) def __stop(self): self.__connection.release()