def get_consumers(self, Consumer, channel): assert self.mq_config # Declaring ourselves rather than use auto-declare. log.debug("Declaring %s exchange", self.mq_config.exchange) self.exchange(channel).declare() queues = [] for queue_name, routing_keys in self.mq_config.queues.items(): queue = Queue(name=queue_name, exchange=self.exchange, channel=channel, durable=True) log.debug("Declaring queue %s", queue_name) queue.declare() for routing_key in routing_keys: log.debug("Binding queue %s to %s", queue_name, routing_key) queue.bind_to(exchange=self.exchange, routing_key=routing_key) queues.append(queue) consumer = Consumer(queues=queues, callbacks=[self._callback], auto_declare=False) consumer.qos(prefetch_count=1, apply_global=True) return [consumer]
def declare_exchange(self, name, type='direct', queues=None, **options): """Create or update exchange :param name: name of exchange :type name: str :param type: type of exchange - direct, fanout, topic, match :type type: str :param queues: list of queues with routing keys: [[queue_name, routing_key], [queue_name, routing_key], ...] :type queues: list, None or tuple :param options: additional options for Exchange creation """ if queues is None: queues = [] # pragma: no cover with self.connections[self.connection].acquire() as conn: exchange = Exchange(name, type=type, channel=conn, **options) exchange.declare() self.exchanges[name] = exchange for q_name, routing_key in queues: queue = Queue(name=q_name, channel=conn) queue.declare() queue.bind_to(exchange=name, routing_key=routing_key) self.logger.debug( 'Queue "%s" with routing_key "%s" was bond to exchange "%s"', q_name, routing_key if routing_key else q_name, name)
class JobConsumer(ConsumerMixin): """ Consume jobs from Pulse exchanges """ def __init__(self, connection): self.connection = connection self.consumers = [] self.queue = None config = settings.PULSE_DATA_INGESTION_CONFIG if not config: raise ValueError("PULSE_DATA_INGESTION_CONFIG is required for the " "JobConsumer class.") self.queue_name = "queue/{}/jobs".format(config.username) def get_consumers(self, Consumer, channel): return [ Consumer(**c) for c in self.consumers ] def bind_to(self, exchange, routing_key): if not self.queue: self.queue = Queue( name=self.queue_name, channel=self.connection.channel(), exchange=exchange, routing_key=routing_key, durable=settings.PULSE_DATA_INGESTION_QUEUES_DURABLE, auto_delete=settings.PULSE_DATA_INGESTION_QUEUES_AUTO_DELETE ) self.consumers.append(dict(queues=self.queue, callbacks=[self.on_message])) # just in case the queue does not already exist on Pulse self.queue.declare() else: self.queue.bind_to(exchange=exchange, routing_key=routing_key) def unbind_from(self, exchange, routing_key): self.queue.unbind_from(exchange, routing_key) def on_message(self, body, message): store_pulse_jobs.apply_async( args=[body, message.delivery_info["exchange"], message.delivery_info["routing_key"]], routing_key='store_pulse_jobs' ) message.ack() def close(self): self.connection.release()
def init(self): try: self.channel = channel = self.connection.channel() exchange = Exchange(name='policymq.direct', type='direct', durable=False) exchange.declare(channel=channel) queue = Queue(name=self.queueName, durable=False) queue.queue_declare(channel=channel) queue.bind_to(exchange=exchange, routing_key='ALL_', channel=channel) queue.bind_to(exchange=exchange, routing_key=self.queueName, channel=channel) consumer = Consumer(channel=channel, queues=[queue], callbacks=[self._consume]) consumer.consume() return True except Exception as e: logger.exception(e) return False
def declare_queues(self, queues_names): with self.connection as _conn: _conn.connect() channel = _conn.channel() for queue_name in queues_names: topic = Exchange( name=f"TOPIC/{queue_name}", type="topic", channel=channel ) topic.declare() queue = Queue(name=queue_name, channel=channel) queue.declare() queue.bind_to(exchange=f"TOPIC/{queue_name}")
class JobConsumer(ConsumerMixin): """ Consume jobs from Pulse exchanges """ def __init__(self, connection): self.connection = connection self.consumers = [] self.queue = None config = settings.PULSE_DATA_INGESTION_CONFIG if not config: raise ValueError("PULSE_DATA_INGESTION_CONFIG is required for the " "JobConsumer class.") self.queue_name = "queue/{}/jobs".format(config.username) def get_consumers(self, Consumer, channel): return [Consumer(**c) for c in self.consumers] def bind_to(self, exchange, routing_key): if not self.queue: self.queue = Queue( name=self.queue_name, channel=self.connection.channel(), exchange=exchange, routing_key=routing_key, durable=settings.PULSE_DATA_INGESTION_QUEUES_DURABLE, auto_delete=settings.PULSE_DATA_INGESTION_QUEUES_AUTO_DELETE) self.consumers.append( dict(queues=self.queue, callbacks=[self.on_message])) # just in case the queue does not already exist on Pulse self.queue.declare() else: self.queue.bind_to(exchange=exchange, routing_key=routing_key) def unbind_from(self, exchange, routing_key): self.queue.unbind_from(exchange, routing_key) def on_message(self, body, message): store_pulse_jobs.apply_async(args=[ body, message.delivery_info["exchange"], message.delivery_info["routing_key"] ], routing_key='store_pulse_jobs') message.ack() def close(self): self.connection.release()
def append_to_topic(self, topic_name, queue_name, routing_key=None): with self.connection as _conn: _conn.connect() channel = _conn.channel() if not topic_name.startswith(self.config.get_event_name_prefix()): topic_name = f"{self.config.get_event_name_prefix()}{topic_name}" topic = Exchange(name=f"{topic_name}", type="topic", channel=channel) topic.declare() queue = Queue(name=queue_name, channel=channel, routing_key=routing_key) queue.declare() queue.bind_to(exchange=f"{topic_name}", routing_key=routing_key)
def push_sync(self,upload=True,delay=0,dryrun=False,timeout=None): # pragma: no cover "wait for push messages" from kombu import Connection, Exchange, Queue, Consumer import socket, ssl url, opts, exchange, queue = self.get_broker() def callback(body, message): self.process_update(body) message.ack() with Connection(url,**opts) as conn: self.connection = conn queue = Queue(queue, channel=conn) queue.queue_declare() queue.bind_to(exchange) try: excpt = (socket.error, ssl.SSLZeroReturnError) except AttributeError: excpt = socket.error with conn.Consumer(queue, accept=['json'], callbacks=[callback]) as consumer: while True: try: conn.drain_events(timeout=timeout) except socket.timeout: pass except excpt: break
def start(self): """ Start subscriber. """ with Connection(self.config.broker_url) as conn: data_pipeline_exchange = Exchange(self.config.exchange, 'topic') queue = Queue(self.config.queue_name) queue.maybe_bind(conn) queue.declare() for routing_key in set( [i['routing_key'] for i in self.config.task_mapping]): queue.bind_to(data_pipeline_exchange, routing_key) worker = Worker(self.config, conn, [queue]) logger.info('Starting subscriber...') worker.run()
def declare_exchange(self, name, type='direct', queues=None, **options): """Create or update exchange :param name: name of exchange :type name: str :param type: type of exchange - direct, fanout, topic, match :type type: str :param queues: list of queues with routing keys: [[queue_name, routing_key], [queue_name, routing_key], ...] :type queues: list, None or tuple :param options: additional options for Exchange creation """ if queues is None: queues = [] with connections[self.connection].acquire() as conn: exchange = Exchange(name, type=type, channel=conn, **options) exchange.declare() self.exchanges[name] = exchange for q_name, routing_key in queues: queue = Queue(name=q_name, channel=conn) queue.declare() queue.bind_to(exchange=name, routing_key=routing_key) self.logger.debug('Queue "%s" with routing_key "%s" was bond to exchange "%s"', q_name, routing_key if routing_key else q_name, name)
class PulseConsumer(ConsumerMixin): """ Consume jobs from Pulse exchanges """ def __init__(self, connection, queue_suffix): self.connection = connection self.consumers = [] self.queue = None config = settings.PULSE_DATA_INGESTION_CONFIG if not config: raise ValueError("PULSE_DATA_INGESTION_CONFIG is required for the " "JobConsumer class.") self.queue_name = "queue/{}/{}".format(config.username, queue_suffix) def get_consumers(self, Consumer, channel): return [ Consumer(**c) for c in self.consumers ] def bind_to(self, exchange, routing_key): if not self.queue: self.queue = Queue( name=self.queue_name, channel=self.connection.channel(), exchange=exchange, routing_key=routing_key, durable=settings.PULSE_DATA_INGESTION_QUEUES_DURABLE, auto_delete=settings.PULSE_DATA_INGESTION_QUEUES_AUTO_DELETE ) self.consumers.append(dict(queues=self.queue, callbacks=[self.on_message])) # just in case the queue does not already exist on Pulse self.queue.declare() else: self.queue.bind_to(exchange=exchange, routing_key=routing_key) def unbind_from(self, exchange, routing_key): self.queue.unbind_from(exchange, routing_key) def close(self): self.connection.release() def prune_bindings(self, new_bindings): # get the existing bindings for the queue bindings = [] try: bindings = self.get_bindings(self.queue_name)["bindings"] except Exception: logger.error("Unable to fetch existing bindings for {}".format( self.queue_name)) logger.error("Data ingestion may proceed, " "but no bindings will be pruned") # Now prune any bindings from the queue that were not # established above. # This indicates that they are no longer in the config, and should # therefore be removed from the durable queue bindings list. for binding in bindings: if binding["source"]: binding_str = self.get_binding_str(binding["source"], binding["routing_key"]) if binding_str not in new_bindings: self.unbind_from(Exchange(binding["source"]), binding["routing_key"]) logger.info("Unbound from: {}".format(binding_str)) def get_binding_str(self, exchange, routing_key): """Use consistent string format for binding comparisons""" return "{} {}".format(exchange, routing_key) def get_bindings(self, queue_name): """Get list of bindings from the pulse API""" return fetch_json("{}queue/{}/bindings".format( settings.PULSE_GUARDIAN_URL, queue_name))
class RabbitMQConnection: """ Class handling receiving and publishing message on the RabbitMQ messages bus """ def __init__(self, message_callback): self.message_callback = message_callback self.exchange = Exchange(CFG.exchange) self.connection = Connection(transport='amqp', hostname=CFG.host, port=CFG.port, userid=CFG.username, password=CFG.password, virtual_host=CFG.vhost, ssl=True) self.connection.connect() self.producer = self.connection.Producer(serializer='json', auto_declare=True) self.queue = Queue(channel=self.connection.channel(), name=CFG.queue, routing_key=CFG.routing_key) self.queue.declare() self.queue.bind_to(exchange=Exchange(CFG.exchange), routing_key=CFG.routing_key) self.consumer = self.connection.\ Consumer( queues=self.queue, callbacks=[self._handle_message], prefetch_count= CFG.prefetch_count) self.consuming = True def _handle_message(self, body, message): """ Callback called by consumer. :param body: :param message: :return: """ # body is sometimes dict and sometimes str # make sure it's a json dict before passing it on json_body = dict() if isinstance(body, dict): json_body = body elif isinstance(body, str): json_body = json.loads(body) self.message_callback(json_body) message.ack() def publish_message(self, message): """ Publishes passed message on the RabbitMQ message bus :param message: :return: """ self.producer.publish(message, retry=True, retry_policy={ 'interval_start': 0, 'interval_step': 2, 'interval_max': 30, 'max_retries': 30, }, exchange=self.exchange, routing_key=CFG.routing_key) def read_messages(self): """ Method reading messages from the queue in a while-true loop. Callback is defined in __init__ :return: """ with self.consumer: while self.consuming: self.connection.drain_events() def close_connection(self): """ Closes the channels/connections. :return: """ # for now called when you press Ctrl-C self.consuming = False self.producer.release() self.connection.release()
class PulseConsumer(ConsumerMixin): """ Consume jobs from Pulse exchanges """ def __init__(self, source, build_routing_key): self.connection = Connection(source['pulse_url']) self.consumers = [] self.queue = None self.queue_name = "queue/{}/{}".format(self.connection.userid, self.queue_suffix) self.root_url = source['root_url'] self.source = source self.build_routing_key = build_routing_key def get_consumers(self, Consumer, channel): return [Consumer(**c) for c in self.consumers] def bindings(self): """Get the bindings for this consumer, each of the form `<exchange>.<routing_keys>`, with `<routing_keys>` being `:`-separated.""" return [] def prepare(self): bindings = [] for binding in self.bindings(): # split source string into exchange and routing key sections exchange, _, routing_keys = binding.partition('.') # built an exchange object with our connection and exchange name exchange = get_exchange(self.connection, exchange) # split the routing keys up using the delimiter for routing_key in routing_keys.split(':'): if self.build_routing_key is not None: # build routing key routing_key = self.build_routing_key(routing_key) binding = self.bind_to(exchange, routing_key) bindings.append(binding) # prune stale queues using the binding strings self.prune_bindings(bindings) def bind_to(self, exchange, routing_key): if not self.queue: self.queue = Queue( name=self.queue_name, channel=self.connection.channel(), exchange=exchange, routing_key=routing_key, durable=True, auto_delete=False, ) self.consumers.append( dict(queues=self.queue, callbacks=[self.on_message])) # just in case the queue does not already exist on Pulse self.queue.declare() else: self.queue.bind_to(exchange=exchange, routing_key=routing_key) # get the binding key for this consumer binding = self.get_binding_str(exchange.name, routing_key) logger.info("Pulse queue {} bound to: {}".format( self.queue_name, binding)) return binding def unbind_from(self, exchange, routing_key): self.queue.unbind_from(exchange, routing_key) def close(self): self.connection.release() def prune_bindings(self, new_bindings): # get the existing bindings for the queue bindings = [] try: bindings = self.get_bindings(self.queue_name)["bindings"] except Exception: logger.error( "Unable to fetch existing bindings for %s. Data ingestion may proceed, " "but no bindings will be pruned", self.queue_name) # Now prune any bindings from the queue that were not # established above. # This indicates that they are no longer in the config, and should # therefore be removed from the durable queue bindings list. for binding in bindings: if binding["source"]: binding_str = self.get_binding_str(binding["source"], binding["routing_key"]) if binding_str not in new_bindings: self.unbind_from(Exchange(binding["source"]), binding["routing_key"]) logger.info("Unbound from: %s", binding_str) def get_binding_str(self, exchange, routing_key): """Use consistent string format for binding comparisons""" return "{} {}".format(exchange, routing_key) def get_bindings(self, queue_name): """Get list of bindings from the pulse API""" return fetch_json("{}queue/{}/bindings".format(PULSE_GUARDIAN_URL, queue_name))
class KombuMessageQueue(object): """ A queue based on kombu """ class_name = None def __init__(self, dc, exchange, exchange_type, queue, binding_key, **kwargs): if not isinstance(dc, list): dc = [dc] self.dc = dc self.exchange = exchange self.exchange_type = exchange_type self.exchange_durable = kwargs.get("exchange_durable", True) self.exchange_opts = dict(durable=self.exchange_durable, auto_delete=not self.exchange_durable, type=self.exchange_type) self.queue = queue self.binding_key = binding_key self.queue_durable = kwargs.get("queue_durable", False) self.queue_opts = dict(durable=self.queue_durable, auto_delete=not self.queue_durable, routing_key=self.binding_key) self.amqp_url = kwargs.get("amqp_url") if not self.amqp_url: self.amqp_url = RabbitmqCtx.get_instance().amqp_url self.self_delete = kwargs.get("self_delete", False) self.channel = None self.queue_obj = None self.closed = True if not self.exchange: raise RuntimeError("exchange is required") self.Exchange_objs = set() self.Exchange_dict = dict() self.connect() def _reconnect(self): """Reconnect to rabbitmq server""" if self.channel and self.channel.connection and not self.channel.connection.connected: # and not self.channel.closed: return global connection if connection.connected: connection.close() self._connect() # get_connection(self.amqp_url) def connect(self): self._connect() def _connect(self): connection = get_connection(self.amqp_url) self.channel = connection.channel() # nowait, useful? # arguments .. logger.debug( "{class_name}[{id:#x}] is build Communicate framework..".format( class_name=self.class_name, id=id(self))) for dc in self.dc: full_ex = "{}.{}".format(dc, self.exchange) ex = Exchange(full_ex, channel=self.channel, **self.exchange_opts) # abstract.py line 67, __call__ will do bind ex(self.channel).declare() if not ex.is_bound: logger.error( "Exchange[{id:#x}] {exchange} is not bound to chan {chan_id}" .format(id=id(ex), exchange=ex, chan_id=self.channel.channel_id)) logger.debug( "{class_name}[{id:#x}] create Exchange[{eid:#x}]".format( class_name=self.class_name, id=id(self), eid=id(ex))) logger.debug("Exchange[{id:#x}] build opts: {e_opts}".format( id=id(ex), e_opts=self.exchange_opts)) logger.debug("Exchange[{id:#x}] opts: {e_opts}".format( id=id(ex), e_opts=dict(durable=ex.durable, type=ex.type, auto_delete=ex.auto_delete))) self.Exchange_objs.add(ex) self.Exchange_dict[full_ex] = ex if self.queue: self.Queue_obj = Queue(self.queue, self.Exchange_objs, channel=self.channel, **self.queue_opts) self.Queue_obj.declare() for ex in self.Exchange_objs: self.Queue_obj.bind_to(ex, routing_key=self.binding_key) logger.debug( "{class_name}[{id:#x}] create Queue obj[{qid:#x}]: {q}".format( class_name=self.class_name, id=id(self), qid=id(self.Queue_obj), q=self.Queue_obj)) logger.debug("Queue obj[{id:#x}] build opts: {q_opts}".format( id=id(self.Queue_obj), q_opts=self.queue_opts)) logger.debug("Queue obj[{id:#x}] opts: {q_opts}".format( id=id(self.Queue_obj), q_opts=dict(durable=self.Queue_obj.durable, routing_key=self.Queue_obj.routing_key, auto_delete=self.Queue_obj.auto_delete))) self.closed = False def reconnect(self): try: self._reconnect() except Exception: # try once more logger.warn("----- Babel fail to reconnect : \n%s", sys.exc_info()) self._reconnect() def __del__(self): logger.debug("{cn} default del method".format(cn=self.class_name)) if not self.closed: logger.debug( "=============== {class_name}[{id:#x}] Enter closing... ". format(class_name=self.class_name, id=id(self))) self.close() def close(self): logger.debug( "{cn}[{id:#x}] is calling KombuMessageQueue close method".format( cn=self.class_name, id=id(self))) try: for ex in self.Exchange_objs: if not ex.durable: r = ex.delete() logger.debug( "{cn}[{id:#x}] delete {exchange}[{eid:#x}], return {r}" .format(cn=self.class_name, id=id(self), exchange=ex, eid=id(ex), r=r)) if self.self_delete and self.queue: r = self.channel.queue_delete(queue=self.queue, if_unused=True) logger.debug("delete queue {q} return {r}".format(q=self.queue, r=r)) global connection if self.channel and connection and connection.connected: self.channel.close() self.channel = None except Exception: logger.error( "!!! Babel Error when close {class_name}[{id:#x}]:\n{exc_info}" .format(class_name=self.class_name, id=id(self), exc_info=traceback.format_exc())) finally: self.closed = True
class PulseConsumer(ConsumerMixin): """ Consume jobs from Pulse exchanges """ def __init__(self, connection): self.connection = connection self.consumers = [] self.queue = None self.queue_name = "queue/{}/{}".format(connection.userid, self.queue_suffix) def get_consumers(self, Consumer, channel): return [Consumer(**c) for c in self.consumers] def bind_to(self, exchange, routing_key): if not self.queue: self.queue = Queue( name=self.queue_name, channel=self.connection.channel(), exchange=exchange, routing_key=routing_key, durable=True, auto_delete=False, ) self.consumers.append( dict(queues=self.queue, callbacks=[self.on_message])) # just in case the queue does not already exist on Pulse self.queue.declare() else: self.queue.bind_to(exchange=exchange, routing_key=routing_key) def unbind_from(self, exchange, routing_key): self.queue.unbind_from(exchange, routing_key) def close(self): self.connection.release() def prune_bindings(self, new_bindings): # get the existing bindings for the queue bindings = [] try: bindings = self.get_bindings(self.queue_name)["bindings"] except Exception: logger.error( "Unable to fetch existing bindings for %s. Data ingestion may proceed, " "but no bindings will be pruned", self.queue_name) # Now prune any bindings from the queue that were not # established above. # This indicates that they are no longer in the config, and should # therefore be removed from the durable queue bindings list. for binding in bindings: if binding["source"]: binding_str = self.get_binding_str(binding["source"], binding["routing_key"]) if binding_str not in new_bindings: self.unbind_from(Exchange(binding["source"]), binding["routing_key"]) logger.info("Unbound from: %s", binding_str) def get_binding_str(self, exchange, routing_key): """Use consistent string format for binding comparisons""" return "{} {}".format(exchange, routing_key) def get_bindings(self, queue_name): """Get list of bindings from the pulse API""" return fetch_json("{}queue/{}/bindings".format(PULSE_GUARDIAN_URL, queue_name))
try: queue = Queue(name=queue, channel=channel, **self._queue_kwargs) except Exception, e: logging.exception(e) if type(queue) is not Queue: raise ValueError('No valid queue available') queue.declare() queue.purge() logging.info("Binding queue '%s' to exchange '%s' with:" % (queue.name, exchange)) routing_keys = routing_keys or ['#'] for rk in routing_keys: try: logging.debug("rk: %s" % rk) queue.bind_to(exchange=exchange, routing_key=rk) except Exception, e: logging.exception(str(e)) logging.info('Done: binding') consumer_tag = '%s::%s::consuming_exchange' % (self.name, queue.name) queue.consume(consumer_tag, callback=self._consumer_producer_callback, no_ack=True) self._queues.append(queue) return queue def _consumer_producer_callback(self, message): routing_key = message.delivery_info.get('routing_key') self.publish(message, routing_key) logging.info("Published a message with rk '%s' to '%s'" % (routing_key, self.name))