def get_consumers(self, Consumer, channel): assert self.mq_config # Declaring ourselves rather than use auto-declare. log.debug("Declaring %s exchange", self.mq_config.exchange) self.exchange(channel).declare() queues = [] for queue_name, routing_keys in self.mq_config.queues.items(): queue = Queue(name=queue_name, exchange=self.exchange, channel=channel, durable=True) log.debug("Declaring queue %s", queue_name) queue.declare() for routing_key in routing_keys: log.debug("Binding queue %s to %s", queue_name, routing_key) queue.bind_to(exchange=self.exchange, routing_key=routing_key) queues.append(queue) consumer = Consumer(queues=queues, callbacks=[self._callback], auto_declare=False) consumer.qos(prefetch_count=1, apply_global=True) return [consumer]
class DelayedRetryQueue(Queue): attrs = Queue.attrs + (('retry_delay', int), ) def __init__(self, *args, retry_delay=None, **kwargs): super(DelayedRetryQueue, self).__init__(*args, **kwargs) self.retry_delay = retry_delay or 1 retry_exchange_name = '{}-retry'.format(self.exchange.name) retry_queue_name = '{}-retry'.format(self.name) routing_key = self.routing_key retry_exchange = Exchange(retry_exchange_name, self.exchange.type, durable=self.exchange.durable, auto_delete=self.exchange.auto_delete) self.retry_queue = Queue( name=retry_queue_name, exchange=retry_exchange, routing_key=routing_key, durable=self.durable, auto_delete=self.auto_delete, message_ttl=self.retry_delay, queue_arguments={'x-dead-letter-exchange': self.exchange.name}) if self.queue_arguments is None: self.queue_arguments = {} self.queue_arguments['x-dead-letter-exchange'] = retry_exchange_name def when_bound(self): super(DelayedRetryQueue, self).when_bound() self.retry_queue = self.retry_queue(self.channel) def declare(self, *args, **kwargs): self.retry_queue.declare(*args, **kwargs) return super(DelayedRetryQueue, self).declare(*args, **kwargs)
def _get_or_create_queue(self, queue_name): queue = Queue(name=queue_name, exchange=self.exchange, durable=True, auto_delete=False) queue.declare(channel=self.connection.channel()) return queue
def listen_events(cls, routing_key, exchange=BUS_EXCHANGE_NAME): exchange = Exchange(exchange, type=BUS_EXCHANGE_TYPE) with Connection(BUS_URL) as conn: queue = Queue(BUS_QUEUE_NAME, exchange=exchange, routing_key=routing_key, channel=conn.channel()) queue.declare() queue.purge() cls.bus_queue = queue
def send_address_to_queue(message): logger.info('Starting to send a message to {} queue'.format( ADDRESS_PRODUCER_QUEUE)) with Connection(settings.BROKER_URL) as connection: logger.info('Connected into the broker with success') connection.connect() channel = connection.channel() exchange = Exchange(ADDRESS_EXCHANGE, type='direct') producer = Producer( channel=channel, routing_key=ADDRESS_PRODUCER_ROUTING_KEY, exchange=exchange, ) queue = Queue( name=ADDRESS_PRODUCER_QUEUE, routing_key=ADDRESS_PRODUCER_ROUTING_KEY, exchange=exchange, ) from api.order.serializers import AddressSerializer address_serializer = AddressSerializer(message).data queue.maybe_bind(connection) queue.declare() producer.publish(address_serializer) connection.close()
def run(rabbit_url): print rabbit_url conn = Connection(rabbit_url) conn.ensure_connection() conn.connect() channel = conn.channel() exchange = Exchange(config.EXCHANGE_NAME, type='direct') producer = Producer(exchange=exchange, channel=channel, routing_key=config.ROUTING_KEY) queue = Queue(name=config.QUEUE_NAME, exchange=exchange, routing_key=config.ROUTING_KEY) queue.maybe_bind(conn) queue.declare() index = 0 while True: try: time.sleep(1) print 'producer' index += 1 producer.publish("send message -- %s" % index) except socket.timeout: pass
def declare_exchange(self, name, type='direct', queues=None, **options): """Create or update exchange :param name: name of exchange :type name: str :param type: type of exchange - direct, fanout, topic, match :type type: str :param queues: list of queues with routing keys: [[queue_name, routing_key], [queue_name, routing_key], ...] :type queues: list, None or tuple :param options: additional options for Exchange creation """ if queues is None: queues = [] # pragma: no cover with self.connections[self.connection].acquire() as conn: exchange = Exchange(name, type=type, channel=conn, **options) exchange.declare() self.exchanges[name] = exchange for q_name, routing_key in queues: queue = Queue(name=q_name, channel=conn) queue.declare() queue.bind_to(exchange=name, routing_key=routing_key) self.logger.debug( 'Queue "%s" with routing_key "%s" was bond to exchange "%s"', q_name, routing_key if routing_key else q_name, name)
class EventsPublisher(): def __init__(self): self.rabbit_url = 'amqp://localhost:5672/' self.conn = Connection(self.rabbit_url) self.channel = self.conn.channel() self.exchange = Exchange(name='gateway-exchange', type='fanout') self.producer = Producer(exchange=self.exchange, channel=self.channel, routing_key='gateway') self.queue = Queue(name='gateway-queue', exchange=self.exchange, routing_key='gateway') self.queue.maybe_bind(self.conn) self.queue.declare() def publish(self, body): # body = { # # "id": "bb2cdchl52n4orsopmtg", # "status": 1, # "location": { # "type": "Point", # "coordinates": [2.2861460, 48.8268020], # } # } self.producer.publish(body, serializer='json') logging.info('*** Event published is: {}'.format(body))
def _send_command(self, payload, server_routing_key, properties=None, declare_queue=True): if properties is None: properties = {} self.reply = None logger.debug("Using connection: {!r}".format(CONN_DICT)) logger.info("Declaring queue %s." % self._client_queue) queue = Queue(self._client_queue, channel=self._conn, durable=self._exchange.durable, exchange=self._exchange, routing_key=self._client_queue) if declare_queue: queue.declare() self._queue = queue with producers[self._conn].acquire(block=True) as producer: producer.publish(payload, serializer='json', exchange=self._exchange, declare=[self._exchange], routing_key=server_routing_key, **properties) logger.info("Published {!r} to exchange " "{!r}".format(payload, self._exchange))
def test_task_succeed(self): """What happens when a task succeeds. Expect it to be acked from the queue. """ from kombu import Exchange e = Exchange('', type='direct') # declare queue consumer_queue = Queue('test.task.succeed', e, channel=self._connection, routing_key='test.task.succeed') client_queue = Queue('', e, durable=False, channel=self._connection) consumer_queue.declare() client_queue.declare() self.queues.append(client_queue) q = Qurator(task_exchange=e, queue='test.task.succeed') @q.task def succeed(data): return None client = RpcClient(exchange=e) client.task('succeed', {'x': 1}, server_routing_key='test.task.succeed') curr_queues = q.queues['succeed'] @q.rpc def still_around(body, message): print("Message: {!r}".format(message)) self._connection.drain_events(timeout=1)
def _register_retry_queues(self, **_: Any) -> None: """ Initializes a set of AMQP primitives to implement broker-based delays. """ channel = self.broker_connection().default_channel for queue in self.conf.task_queues: retry_queue = Queue( name=f'{queue.name}.retry', routing_key=f'{queue.routing_key}.retry', exchange=queue.exchange, queue_arguments={ "x-dead-letter-exchange": "", "x-dead-letter-routing-key": queue.name } ) retry_queue.declare(channel=channel) retry_queue.maybe_bind(channel=channel) archived_queue = Queue( name=f'{queue.name}.archived', routing_key=f'{queue.routing_key}.archived', exchange=queue.exchange, queue_arguments={ "x-message-ttl": defaults.AMQP_EVENTS_ARCHIVED_MESSAGE_TTL, "x-max-length": defaults.AMQP_EVENTS_ARCHIVED_QUEUE_LENGTH, "x-queue-mode": "lazy" }) archived_queue.declare(channel=channel) archived_queue.maybe_bind(channel=channel)
def init_adversary(): global __username, __my_type, __connection, __global_exchange, __producer, __backchannel, __backch_producer, __adv_exchange, __adv_queue #, __my_queue __username = '******' __my_type = 'adversary' #adversary has broadcast-only access to the regular exchange __connection = Connection('amqp://') __connection.connect() __global_exchange = Exchange('broadcast', type='fanout', durable=False, delivery_mode=1) __global_exchange.maybe_bind(__connection) __producer = __connection.Producer(__connection) __backchannel = Connection('amqp://') __backchannel.connect() __adv_exchange = Exchange('adversary', durable=False, delivery_mode=1) __adv_exchange.maybe_bind(__backchannel) __backch_producer = __backchannel.Producer(__backchannel) __adv_queue = Queue('adversary', exchange=__adv_exchange, routing_key='adversary', durable=False) __adv_queue = __adv_queue(__backchannel) __adv_queue.declare()
def _register_retry_queues(self, **_: Any) -> None: """ Initializes a set of AMQP primitives to implement broker-based delays. Declares an exchange/queue pair for each delay stage defined by `amqp_events.defaults:AMQP_EVENTS_MAX_RETRIES`. Each exchange has a single bound queue; when task needs to be delayed, it's re-published to new exchange preserving initial routing_key. A queue for each exchange has `message-ttl` set to a power of 2. After message is expired, it is re-routed by broker to `dead-letter-exchange` named `recover`. All queues defined for event handlers are also bound to this exchange with same routing key, thus each message after a retry appears in same incoming queue. `events` -> `demo.my_event_queue` -> Celery worker (retry) | (publishes new message on retry) V `demo:retry.N` -> `demo:retry.N` | (message ttl expires ) V `recover` -> (routing key) -> `demo.my_event_queue` - > Celery worker """ channel = self.broker_connection().default_channel for retry in range(defaults.AMQP_EVENTS_MAX_RETRIES): name = self.get_retry_exchange_name(retry) retry_exchange = Exchange(name=name, type=EXCHANGE_TYPE_FANOUT) retry_queue = Queue( name=name, exchange=retry_exchange, queue_arguments={ X_MESSAGE_TTL: 2**retry * 1000, # ms X_DEAD_LETTER_EXCHANGE: self.recover_exchange_name, }) retry_queue.declare(channel=channel) retry_queue.maybe_bind(channel=channel)
def send_geo_location_to_queue(address_id, geo_location): with Connection(settings.BROKER_URL) as connection: connection.connect() channel = connection.channel() exchange = Exchange(ADDRESS_EXCHANGE, type='direct') producer = Producer( channel=channel, routing_key=ADDRESS_CUSTOMER_PRODUCER_ROUTING_KEY, exchange=exchange, ) queue = Queue( name=ADDRESS_CUSTOMER_PRODUCER_QUEUE, routing_key=ADDRESS_CUSTOMER_PRODUCER_ROUTING_KEY, exchange=exchange, ) geo_location_from_address = geo_location geo_location_from_address.update({'id': address_id}) queue.maybe_bind(connection) queue.declare() producer.publish(geo_location_from_address) connection.close()
def connect(self): """Connect to broker and possibly ensure exchange/queue/routing declared. In case of errors, this will retry connecting several times (controlled by the `CONN_*` class variables), and gracefully recover if possible. If it never succeeds, the exception raised by the final attempt is re-raised. """ self.logger.debug('Connecting to broker at: {}'.format(self.url)) self._connection = Connection(self.url) # Kombu interprets interval_max incorrectly; work around that. interval_max = self.CONN_INTERVAL_MAX - self.CONN_INTERVAL_STEP self._connection.ensure_connection( errback=self.conn_errback, max_retries=self.CONN_MAX_RETRIES, interval_start=self.CONN_INTERVAL_START, interval_step=self.CONN_INTERVAL_STEP, interval_max=interval_max) exchange = Exchange(name=self.exchange_name, type=self.exchange_type) channel = self._connection.channel() self._producer = Producer(channel=channel, exchange=exchange, routing_key=self.routing_key) if self.queue_name: # Bind/declare queue. queue = Queue(name=self.queue_name, exchange=exchange, routing_key=self.routing_key) queue = queue(channel) # Bind queue self.logger.debug('Declaring queue {}, on exchange {} at {}'.format( self.queue_name, self.exchange_name, self.url)) queue.declare()
def test_declare_but_no_exchange(self): q = Queue('a') q.queue_declare = Mock() q.queue_bind = Mock() q.exchange = None q.declare() q.queue_declare.assert_called_with(False, passive=False)
def test_declare(self): chan = get_conn().channel() b = Queue('foo', self.exchange, 'foo', channel=chan) self.assertTrue(b.is_bound) b.declare() self.assertIn('exchange_declare', chan) self.assertIn('queue_declare', chan) self.assertIn('queue_bind', chan)
def test_declare(self): chan = get_conn().channel() b = Queue('foo', self.exchange, 'foo', channel=chan) assert b.is_bound b.declare() assert 'exchange_declare' in chan assert 'queue_declare' in chan assert 'queue_bind' in chan
def __assure_named_queue(self, connection, exchange, queue_name, routing_key_extension='*'): routing_key = '{}.{}'.format(queue_name, routing_key_extension) queue = Queue(queue_name, exchange, routing_key, connection) queue.declare()
def test_declare__no_declare(self): q = Queue('a', no_declare=True) q.queue_declare = Mock() q.queue_bind = Mock() q.exchange = None q.declare() self.assertFalse(q.queue_declare.called) self.assertFalse(q.queue_bind.called)
def test_declare__no_declare(self): q = Queue('a', no_declare=True) q.queue_declare = Mock() q.queue_bind = Mock() q.exchange = None q.declare() q.queue_declare.assert_not_called() q.queue_bind.assert_not_called()
def _declare_queue(glance_api_cfg, routing_key, conn, exchange): queue = Queue(name=routing_key, routing_key=routing_key, exchange=exchange, channel=conn.channel(), durable=False) queue.declare() return queue
def add_binding(queue_name, routing_key, exchange_name=None): exchange_name = exchange_name or world.config['bus']['exchange_name'] exchange = Exchange(exchange_name, type=world.config['bus']['exchange_type']) with Connection(world.config['bus_url']) as conn: queue = Queue(queue_name, exchange=exchange, routing_key=routing_key, channel=conn.channel()) queue.declare() queue.purge() _queues[queue_name] = queue
def listen_events(self, routing_key, exchange=BUS_EXCHANGE_XIVO): with Connection(self._url) as conn: queue = Queue(BUS_QUEUE_NAME, exchange=exchange, routing_key=routing_key, channel=conn.channel()) queue.declare() queue.purge() self.bus_queue = queue
def connect(self): self.close_channel() try: self.channel = self.connection.channel() queue = Queue(name=self.queueName, durable=False) queue.declare(channel=self.channel) self.producer = Producer(channel=self.channel, routing_key='HOST') except Exception as e: logger.exception(e)
def main(): rabbit_url = "amqp://*****:*****@10.5.9.177:5672//" conn = Connection(rabbit_url) channel = conn.channel() exchange = Exchange("test", type="direct") producer = Producer(exchange=exchange, channel=channel, routing_key="vcc") queue = Queue(name="chungpht", exchange=exchange, routing_key="vcc") queue.maybe_bind(conn) queue.declare() producer.publish("Hello from other side 222")
def init_ack(): '''初始化返回exchange和queue''' connection = Connection(CONF.rabbit.connection) channel = connection.channel() exchange = Exchange(CONF.region_id + '_ack', 'direct', channel) exchange.declare() queue = Queue(CONF.region_id + '_ack', exchange=exchange, routing_key=CONF.region_id + '_ack', channel=channel) queue.declare()
def accumulator(self, routing_key, exchange=None): exchange = exchange or self._default_exchange queue_name = 'test-{}'.format(str(uuid.uuid4())) with Connection(self._url) as conn: queue = Queue(name=queue_name, exchange=exchange, routing_key=routing_key, channel=conn.channel()) queue.declare() queue.purge() accumulator = BusMessageAccumulator(self._url, queue) return accumulator
def start_consuming( self, callback: Callable, queue_name: str, prefetch_count: int = 1, no_ack: bool = False, expires: int = None, callback_ready: Callable = None, ): if self._logger is not None: self._logger.debug("Start consuming queue: %s" % queue_name) self._consuming = True while self._consuming: revived_connection = self._connection.clone() revived_connection.ensure_connection() channel = revived_connection.channel() channel.basic_qos(0, prefetch_count, True) queues = [] queue_obj = Queue( channel=channel, name=queue_name, no_ack=no_ack, durable=False, expires=expires, queue_arguments={"x-max-priority": 255}, ) queue_obj.declare() queues.append(queue_obj) consumer = Consumer( revived_connection, queues, callbacks=[callback], accept=["json"], auto_declare=False, prefetch_count=prefetch_count, ) consumer.revive(channel) consumer.consume() while self._consuming: callback_ready is not None and callback_ready() try: revived_connection.drain_events(timeout=2) except socket.timeout: revived_connection.heartbeat_check() except self._connection.connection_errors + ( AMQPError, ConnectionForced, ConnectionError, ): # pragma: no cover if self._logger is not None: self._logger.exception("Connection error", stack_info=True) break
def declare_dead_queue(): channel = connection.channel() dead_exchange = Exchange(name=config.rabbitmq_dead_exchange(), type='direct', channel=channel) dead_queue = Queue(name=config.rabbitmq_dead_queue(), routing_key=config.rabbitmq_routing_key(), exchange=dead_exchange, channel=channel) dead_queue.declare() return dead_exchange
class JobConsumer(ConsumerMixin): """ Consume jobs from Pulse exchanges """ def __init__(self, connection): self.connection = connection self.consumers = [] self.queue = None config = settings.PULSE_DATA_INGESTION_CONFIG if not config: raise ValueError("PULSE_DATA_INGESTION_CONFIG is required for the " "JobConsumer class.") self.queue_name = "queue/{}/jobs".format(config.username) def get_consumers(self, Consumer, channel): return [ Consumer(**c) for c in self.consumers ] def bind_to(self, exchange, routing_key): if not self.queue: self.queue = Queue( name=self.queue_name, channel=self.connection.channel(), exchange=exchange, routing_key=routing_key, durable=settings.PULSE_DATA_INGESTION_QUEUES_DURABLE, auto_delete=settings.PULSE_DATA_INGESTION_QUEUES_AUTO_DELETE ) self.consumers.append(dict(queues=self.queue, callbacks=[self.on_message])) # just in case the queue does not already exist on Pulse self.queue.declare() else: self.queue.bind_to(exchange=exchange, routing_key=routing_key) def unbind_from(self, exchange, routing_key): self.queue.unbind_from(exchange, routing_key) def on_message(self, body, message): store_pulse_jobs.apply_async( args=[body, message.delivery_info["exchange"], message.delivery_info["routing_key"]], routing_key='store_pulse_jobs' ) message.ack() def close(self): self.connection.release()
def producer(msg=None): print("------- in producer") rabbit_url = "amqp://localhost:5672/" conn = Connection(rabbit_url) channel = conn.channel() exchange = Exchange("scrapy", type="direct") producer = Producer(exchange=exchange, channel=channel, routing_key="quotes") queue = Queue(name="quotation", exchange=exchange, routing_key="quotes") queue.maybe_bind(conn) queue.declare() producer.publish(msg) print("published ->")
class Cloner(object): def __init__(self, source_connection, target_connection, source_exchange_name, target_queue_name, intermediate_queue_name=None): self.source_connection = source_connection self.target_connection = target_connection self.source_exchange_name = source_exchange_name self.target_queue_name = target_queue_name self.intermediate_queue_name = ( intermediate_queue_name or source_exchange_name + '_cloner' ) def __str__(self): return ( '{} @ {} > {} @ {}'.format( self.source_exchange_name, self.source_connection, self.target_queue_name, self.target_connection, ) ) def _setup_queues(self): src_channel = self.source_connection.channel() self.intermediate_queue = Queue( name=self.intermediate_queue_name, exchange=Exchange(self.source_exchange_name), routing_key=self.source_exchange_name, channel=src_channel, auto_delete=False, ) self.intermediate_queue.declare() self.target_queue = self.target_connection.SimpleQueue( self.target_queue_name ) def copy_message(self, body, msg): logger.debug('{} - Copying message {}'.format(self, body)) self.target_queue.put(body) msg.ack() def clone(self): self._setup_queues() src_channel = self.source_connection.channel() Consumer( src_channel, self.intermediate_queue, callbacks=[self.copy_message], ).consume()
def test_basic_get(self): chan1 = self.connection.channel() producer = Producer(chan1, self.exchange) producer.publish({"basic.get": "this"}, routing_key="basic_get") chan1.close() chan2 = self.connection.channel() queue = Queue("amqplib_basic_get", self.exchange, "basic_get") queue = queue(chan2) queue.declare() for i in range(50): m = queue.get() if m: break time.sleep(0.1) self.assertEqual(m.payload, {"basic.get": "this"}) chan2.close()
def pre_declare_queues(self, queues): """Pre-declare any queues that will be used in tests. :queues: list of names of queues """ declared_queues = [] for queue_name in queues: q = Queue(queue_name, self._exchange, channel=self._connection, durable=self._exchange.durable, routing_key=queue_name) q.declare() declared_queues.append(q) self.queues = declared_queues
def test_basic_get(self): if not self.verify_alive(): return chan1 = self.connection.channel() producer = Producer(chan1, self.exchange) chan2 = self.connection.channel() queue = Queue(self.P("basic_get"), self.exchange, "basic_get") queue = queue(chan2) queue.declare() producer.publish({"basic.get": "this"}, routing_key="basic_get") chan1.close() for i in range(self.event_loop_max): m = queue.get() if m: break time.sleep(0.1) self.assertEqual(m.payload, {"basic.get": "this"}) chan2.close()
def test_basic_get(self): if not self.verify_alive(): return chan1 = self.connection.channel() producer = chan1.Producer(self.exchange) chan2 = self.connection.channel() queue = Queue(self.P('basic_get'), self.exchange, 'basic_get') queue = queue(chan2) queue.declare() producer.publish({'basic.get': 'this'}, routing_key='basic_get') chan1.close() for i in range(self.event_loop_max): m = queue.get() if m: break time.sleep(0.1) self.assertEqual(m.payload, {'basic.get': 'this'}) self.purge([queue.name]) chan2.close()
def test_task_fail(self): """What happens when a task fails. Expected behaviour is that it does not ack and is left in the queue. """ from kombu import Exchange e = Exchange('', type='direct') # declare queue consumer_queue = Queue('test.task.fail', e, channel=self._connection, routing_key='test.task.fail') client_queue = Queue('', e, durable=False, channel=self._connection) consumer_queue.declare() client_queue.declare() self.queues.append(consumer_queue) self.queues.append(client_queue) q = Qurator(task_exchange=e) @q.task(queue_name='test.task.fail') def fail(data): raise Exception('YOU FAIL!') client = RpcClient(exchange=e) client.task('fail', {'x': 1}, server_routing_key='test.task.fail') curr_queues = q.queues['fail'] curr_callbacks = q.callbacks['fail'] def still_around(body, message): self.assertFalse(message.acknowledged) message.ack() curr_callbacks.append(still_around) with Consumer(self._connection, curr_queues, callbacks=curr_callbacks): self._connection.drain_events(timeout=1)
def send(self): try: # Connection conn = Connection(self.broker) # Channel channel = conn.channel() # Exchange task_exchange = Exchange(self._exchange_name, type=self._queue_type) # Queues if self._queue_name: queue = Queue(name=self._queue_name, channel=channel, exchange=task_exchange, routing_key=self._routing_key) queue.declare() # Producer producer = Producer(exchange=task_exchange, channel=channel, routing_key=self._routing_key) # Send message for message in self._msgs: serialized_message = json.dumps(message, ensure_ascii=False) producer.publish(serialized_message) conn.close() except Exception, e: self.log.error( u'QueueManagerError - Error on sending objects from queue.') self.log.debug(e) raise Exception( 'QueueManagerError - Error on sending objects to queue.')
def declare_queues(): channel = connection.channel() almanach_exchange = Exchange(name=config.rabbitmq_retry_return_exchange(), type='direct', channel=channel) retry_exchange = Exchange(name=config.rabbitmq_retry_exchange(), type='direct', channel=channel) retry_queue = Queue(name=config.rabbitmq_retry_queue(), exchange=retry_exchange, routing_key=config.rabbitmq_routing_key(), queue_arguments=self._get_queue_arguments(), channel=channel) almanach_queue = Queue(name=config.rabbitmq_queue(), exchange=almanach_exchange, durable=False, routing_key=config.rabbitmq_routing_key(), channel=channel) retry_queue.declare() almanach_queue.declare() return retry_exchange
def declare_exchange(self, name, type='direct', queues=None, **options): """Create or update exchange :param name: name of exchange :type name: str :param type: type of exchange - direct, fanout, topic, match :type type: str :param queues: list of queues with routing keys: [[queue_name, routing_key], [queue_name, routing_key], ...] :type queues: list, None or tuple :param options: additional options for Exchange creation """ if queues is None: queues = [] with connections[self.connection].acquire() as conn: exchange = Exchange(name, type=type, channel=conn, **options) exchange.declare() self.exchanges[name] = exchange for q_name, routing_key in queues: queue = Queue(name=q_name, channel=conn) queue.declare() queue.bind_to(exchange=name, routing_key=routing_key) self.logger.debug('Queue "%s" with routing_key "%s" was bond to exchange "%s"', q_name, routing_key if routing_key else q_name, name)
class Publisher(KombuConfReader): def __init__(self, config): self._log = log.getLogger() KombuConfReader.__init__(self, config) self.connection = Connection(self.broker_url) try: self._init_amqp() except Exception as exc: self._log.error('Publisher fail in init connection: %s' % exc) raise def _init_amqp(self): """Init AMQP objects after connection""" self.producer = self.connection.Producer() self.exchange = Exchange( self.exchange_name, channel=self.connection.channel(), type=self.exchange_type, durable=self.exchange_is_durable) self.queue = Queue( self.queue_name, self.exchange, channel=self.connection.channel(), durable=self.queue_is_durable, routing_key=self.routing_key, queue_arguments=self.queue_args) # We declare object to broker. this way only we can # ensure to publish to an existing queue and routing_key # AMQP work this way, not a library principle self.exchange.declare() self.queue.declare() def switch_connection(self): """Switch AMQP connection from url to backup_url and vice versa""" self._log.warn('Switching AMQP connection from %s' % self.connection.as_uri()) if (self.connection.hostname in self.broker_url and self.broker_backup_url): self.connection = Connection(self.broker_backup_url) elif self.connection.hostname in self.broker_backup_url: self.connection = Connection(self.broker_url) else: raise URLError('Invalid current URI to switch connection : %s' % self.connection.as_uri()) self._init_amqp() def _publish(self, msg): """Publish message ensuring connection is available""" publish = self.connection.ensure( self.producer, self.producer.publish, max_retries=3) publish(msg, exchange=self.exchange, routing_key=self.routing_key, serializer=self.serializer, compression=self.compression) return True def publish(self, msg): """ must return True/False if message is well publish or not """ try: return self._publish(msg) except Exception as exc: try: self.switch_connection() return self._publish(msg) except Exception as exc: self._log.error('Publish fail when switching connection: %s' % exc) return False
task_exchange.declare() # 在这里进行了exchange和queue的绑定,并且指定了这个queue的routing_key task_queue = Queue('piap', task_exchange, channel=channel, routing_key='suo_piao.#', durable=False) task_queue2 = Queue('piap2.*abc.#', task_exchange, channel=channel, routing_key='suo_piao.#', durable=False) task_queue3 = Queue('piap3', task_exchange, channel=channel, routing_key='suo_piao.abc.#', durable=False) task_queue4 = Queue('piap4', task_exchange, channel=channel, routing_key='abc.#', durable=False) task_queues = [] for x in xrange(1,10): tmpQueue = Queue('testFlood'+str(x), task_exchange, channel=channel, routing_key='abc.*.'+str(x), durable=False) tmpQueue.declare() task_queues.append(tmpQueue) task_queue.declare() task_queue2.declare() task_queue3.declare() task_queue4.declare() message = Message(channel, body= {'state_strings': [u'pending'], 'stepid': 1, 'complete_at': None, 'name': u'git', 'buildid': 1, 'results': None, 'number': 0, 'urls': [], 'complete': False}) # produce producer = Producer(channel, exchange=task_exchange, auto_declare=False, serializer="json") producer.publish(message.body, routing_key='abc.suo_piao')
class PulseConsumer(ConsumerMixin): """ Consume jobs from Pulse exchanges """ def __init__(self, connection, queue_suffix): self.connection = connection self.consumers = [] self.queue = None config = settings.PULSE_DATA_INGESTION_CONFIG if not config: raise ValueError("PULSE_DATA_INGESTION_CONFIG is required for the " "JobConsumer class.") self.queue_name = "queue/{}/{}".format(config.username, queue_suffix) def get_consumers(self, Consumer, channel): return [ Consumer(**c) for c in self.consumers ] def bind_to(self, exchange, routing_key): if not self.queue: self.queue = Queue( name=self.queue_name, channel=self.connection.channel(), exchange=exchange, routing_key=routing_key, durable=settings.PULSE_DATA_INGESTION_QUEUES_DURABLE, auto_delete=settings.PULSE_DATA_INGESTION_QUEUES_AUTO_DELETE ) self.consumers.append(dict(queues=self.queue, callbacks=[self.on_message])) # just in case the queue does not already exist on Pulse self.queue.declare() else: self.queue.bind_to(exchange=exchange, routing_key=routing_key) def unbind_from(self, exchange, routing_key): self.queue.unbind_from(exchange, routing_key) def close(self): self.connection.release() def prune_bindings(self, new_bindings): # get the existing bindings for the queue bindings = [] try: bindings = self.get_bindings(self.queue_name)["bindings"] except Exception: logger.error("Unable to fetch existing bindings for {}".format( self.queue_name)) logger.error("Data ingestion may proceed, " "but no bindings will be pruned") # Now prune any bindings from the queue that were not # established above. # This indicates that they are no longer in the config, and should # therefore be removed from the durable queue bindings list. for binding in bindings: if binding["source"]: binding_str = self.get_binding_str(binding["source"], binding["routing_key"]) if binding_str not in new_bindings: self.unbind_from(Exchange(binding["source"]), binding["routing_key"]) logger.info("Unbound from: {}".format(binding_str)) def get_binding_str(self, exchange, routing_key): """Use consistent string format for binding comparisons""" return "{} {}".format(exchange, routing_key) def get_bindings(self, queue_name): """Get list of bindings from the pulse API""" return fetch_json("{}queue/{}/bindings".format( settings.PULSE_GUARDIAN_URL, queue_name))
import boto from kombu import BrokerConnection, Exchange, Queue, Consumer connection = BrokerConnection() connection.connect() channel = connection.channel() exchange = Exchange(name="android", type="fanout", channel=channel, durable=True) exchange.declare() channel = connection.channel() queue = Queue(name="filr", exchange=exchange, durable=True, auto_delete=False, channel=channel, routing_key="filr") queue.declare() def fetch(b, m): print b, m consumer = Consumer(channel=connection.channel(), queues=queue, auto_declare=False, callbacks=[fetch]) consumer.consume(no_ack=False) while True: connection.drain_events() pass # execfile('.private-settings') # sdb = boto.connect_sdb(key_id, sec_key) # domain = sdb.create_domain('android') # item = domain.new_item('kral_step1')