def publish(self, message: Union[dict, str] = "", headers: dict = {}, exchange: str = EXCHANGE, routing_key: str = ROUTING_KEY): """ Publish a message to th AMQP Queue :param message: message to be published :param headers: header key-values to publish with the message :param exchange: specifies the top level specifier for message publish :param routing_key: determines which queue the message is published to """ self._conn.connect() queue = kombu.Queue(routing_key, kombu.Exchange(exchange, type="topic"), routing_key=routing_key) queue.maybe_bind(self._conn) queue.declare() producer = kombu.Producer(self._conn.channel()) producer.publish(message, headers=headers, exchange=queue.exchange, routing_key=queue.routing_key, declare=[queue]) producer.close() self._conn.release()
def main(): parser = ArgumentParser('Send message to RabbitMQ exchange xivo') parser.add_argument('-n', '--hostname', help='RabbitMQ server', default='localhost') parser.add_argument('-p', '--port', help='Port of RabbitMQ', default='5672') parser.add_argument('-r', '--routing-key', help='Routing key to bind on bus', dest='routing_key', required=True) parser.add_argument('-i', '--input-event-file', help='JSON file containing the event to send. Default: stdin.', dest='input_event_file', default='-') args = parser.parse_args() config = _DEFAULT_CONFIG config['host'] = args.hostname config['port'] = args.port bus_url = 'amqp://{username}:{password}@{host}:{port}/{vhost}'.format(**config) with kombu.Connection(bus_url) as connection: exchange = kombu.Exchange(config['exchange_name'], type=config['exchange_type']) producer = kombu.Producer(connection, exchange=exchange) with sys.stdin if args.input_event_file == '-' else open(args.input_event_file, 'r') as f: event_data = json.load(f) headers = { 'name': event_data['name'], 'required_acl': event_data['required_acl'], } producer.publish(event_data, routing_key=args.routing_key, headers=headers)
def _reconnect(self): if self._conn_lock.locked(): # either connection-monitor or publisher should have taken # the lock. The one who acquired the lock would re-establish # the connection and releases the lock, so the other one can # just wait on the lock, till it gets released self._conn_lock.wait() return self._conn_lock.acquire() msg = "RabbitMQ connection down" self._logger(msg, level=SandeshLevel.SYS_ERR) self._update_sandesh_status(ConnectionStatus.DOWN) self._conn_state = ConnectionStatus.DOWN self._conn.close() self._conn.ensure_connection() self._conn.connect() self._update_sandesh_status(ConnectionStatus.UP) self._conn_state = ConnectionStatus.UP msg = 'RabbitMQ connection ESTABLISHED %s' % repr(self._conn) self._logger(msg, level=SandeshLevel.SYS_NOTICE) self._channel = self._conn.channel() self._consumer = kombu.Consumer(self._channel, queues=self._update_queue_obj, callbacks=[self._subscribe]) self._producer = kombu.Producer(self._channel, exchange=self.obj_upd_exchange) self._conn_lock.release()
def run(self): self._is_running = True while self._is_running: if self.consumer.is_connected(): producer = kombu.Producer( self.consumer._channel, on_return=self.consumer._handle_return) try: queued_request = self._out_queue.get(timeout=0.5) if True: # with kombu.producers[self.consumer.get_connection()].acquire(block=True) as producer: # producer.on_return = print try: self._dispatch_request(queued_request, producer) except Exception as e: # except ConnectionResetError: log.debug( 'Failed to dispatch request, re-enqueueing again, error was: {}' .format(str(e))) self.enqueue(queued_request) except Empty: continue else: sleep(0.5) log.debug('Waiting for consumer to be ready...')
def __init__(self, url, queue, headers, allowed, username='******', password='******', max_events=None, **kwargs): super(KombuPublisher, self).__init__(allowed, max_events, **kwargs) self._url = url self.queue = queue self._headers = headers self.username = username self.password = password self.exchange = kombu.Exchange(name='amq.direct', type='direct') self._queue = kombu.Queue(name=queue, exchange=self.exchange, routing_key=queue) self.connection = kombu.Connection(self._url, userid=self.username, password=self.password) self.producer = kombu.Producer(self.connection, routing_key=self.queue, exchange=self.exchange)
def _start_publishing(self): errors = (self._connection.connection_errors + self._connection.channel_errors) payload = None while self._running: try: self._ensure_connection() while self._running and self._connection.connected: if payload is None: payload = self._publisher_queue.get() if self._producer_changed: self._producer = kombu.Producer(self._channel) self._producer_changed = False exchange = self.get_exchange(payload.exchange) self._producer.publish(payload.message, exchange=exchange, routing_key=payload.routing_key, **payload.kwargs) payload = None except errors as e: msg = 'Connection error in Kombu amqp publisher greenlet: %s' % str( e) self._logger(msg, level=SandeshLevel.SYS_WARN) except Exception as e: msg = 'Error in Kombu amqp publisher greenlet: %s' % str(e) self._logger(msg, level=SandeshLevel.SYS_ERR) payload = None
def archive(self, body, message, reason=''): """ Put the message onto the archive queue """ _logger.warning(reason) try: kombu.Producer( self.channel, exchange=self.archive_queue.exchange, routing_key=self.archive_queue.routing_key, serializer=settings.SERIALIZER, ).publish( body, headers=message.headers, retry=True, declares=[self.archive_queue], ) except Exception as e: message.requeue() _logger.error("Archive failure: retry-reason='{reason}' " "exception='{cls}, {error}'\n" "{traceback}".format( reason=reason, cls=e.__class__.__name__, error=e, traceback=traceback.format_exc(), )) else: message.ack() _logger.debug("Archive: {reason}".format(reason=reason))
def setUp(self): super().setUp() self.tenant_uuid = SUB_TENANT self.tenant_name = 'mytenant' bus_port = self.service_port(5672, 'rabbitmq') bus = BusClient.from_connection_fields(host='localhost', port=bus_port) until.true(bus.is_up, timeout=5) bus_url = 'amqp://{username}:{password}@{host}:{port}//'.format( username='******', password='******', host='localhost', port=bus_port) connection = kombu.Connection(bus_url) connection.connect() marshaler = Marshaler('the-xivo-uuid') exchange = kombu.Exchange('xivo', type='topic') producer = kombu.Producer(connection, exchange=exchange, auto_declare=True) self.publisher = Publisher(producer, marshaler) self.mock_auth_client = MockAuthClient('localhost', self.service_port(9497, 'auth')) def wait_for_dird_bus_connection(): response = self.client.status.get() assert_that(response, has_entries(bus_consumer={'status': 'ok'})) until.assert_(wait_for_dird_bus_connection, timeout=6)
def _generate_publish(self, name, exchange): # Create producer for the exchange exchange_path = "exchange/%s/%s%s" % ( self.namespace, self.exchange_prefix, exchange.exchange) producer = kombu.Producer(channel=self.connection, exchange=kombu.Exchange( name=exchange_path, type='topic', durable=True, delivery_mode='persistent'), auto_declare=True) publish_message = self.connection.ensure(producer, producer.publish, max_retries=3) # Create publication method for the exchange def publish(**kwargs): message = exchange.message(kwargs) jsonschema.validate(message, self.schemas[exchange.schema]) publish_message(body=json.dumps(message), routing_key=exchange.routing(**kwargs), content_type='application/json') return publish
def producer_single(self, msg_body_str): try: self.producer.publish(msg_body_str, \ exchange=self.exchange, \ routing_key=self.routing_key, \ serializer=self.serializer, \ compression=self.compression) #print 'before drain_events' #self.connection.drain_events() #print 'after drain_events' except self.connection.connection_errors + self.connection.channel_errors: self.connection.close() print("[%s:%s] host breakdown, switch to next one." % \ (self.connection.hostname, self.connection.port)) self.connection.ensure_connection() self.channel = self.connection.channel() self.producer = kombu.Producer(self.channel, \ self.exchange, routing_key=self.routing_key, \ serializer=self.serializer, \ compression=self.compression, \ on_return=on_return_callback) self.producer.publish(msg_body_str, \ exchange=self.exchange, \ routing_key=self.routing_key, \ serializer=self.serializer, \ compression=self.compression)
def setup_producer(cfg=outgoing_cfg, serializer='json', set_queue=True): """ Create a Producer, with a corresponding queue if required. """ assert type(set_queue) is bool logger.debug("cfg: {}".format(cfg)) channel = setup_channel(cfg.hostname, exchange=cfg.exchange) # Publishing is to an exchange but we need a queue to store messages *before* publication. # Note that Consumers should really be responsible for (their) queues. queue = None if set_queue: queue = setup_queue(channel, cfg=cfg) # Publish message; the default message *routing* key is the outgoing queue name. producer = kombu.Producer(channel, exchange=cfg.exchange, routing_key=cfg.queue, serializer=serializer) logger.debug('channel_id: {}'.format(producer.channel.channel_id)) logger.debug('exchange: {}'.format(producer.exchange.name)) logger.debug('routing_key: {}'.format(producer.routing_key)) logger.debug('serializer: {}'.format(producer.serializer)) # Track queue, for debugging purposes. producer._queue = queue return producer
def _start_publishing(self): errors = (self._connection.connection_errors + self._connection.channel_errors) payload = None connection = self._connection.clone() msg = 'KombuAmqpClient: Starting publisher greenlet' self._logger(msg, level=SandeshLevel.SYS_DEBUG) while self._running: try: self._ensure_connection(connection, "Publisher") producer = kombu.Producer(connection) while self._running: if payload is None: payload = self._publisher_queue.get() exchange = self.get_exchange(payload.exchange) producer.publish(payload.message, exchange=exchange, routing_key=payload.routing_key, **payload.kwargs) payload = None except errors as e: msg = 'KombuAmqpClient: Connection error in Kombu amqp publisher greenlet: %s' % str(e) self._logger(msg, level=SandeshLevel.SYS_WARN) except Exception as e: msg = 'KombuAmqpClient: Error in Kombu amqp publisher greenlet: %s' % str(e) self._logger(msg, level=SandeshLevel.SYS_ERR) msg = 'KombuAmqpClient: Exiting publisher greenlet' self._logger(msg, level=SandeshLevel.SYS_DEBUG)
def amqp_publish_user(owner, routing_key, data): with kombu.Connection(config.BROKER_URL) as connection: channel = connection.channel() try: kombu.Producer(channel).publish(data, exchange=kombu.Exchange( _amqp_owner_exchange(owner)), routing_key=routing_key, serializer='json', retry=True) started_at = time() while True: try: connection.drain_events(timeout=0.5) except AmqpNotFound: raise except: pass if time() - started_at >= 0.5: break except AmqpNotFound: return False else: return True finally: channel.close()
def main(): parser = argparse.ArgumentParser( description='read a file and send its lines to an amqp broker') parser.add_argument('--file', help='the file to read', required=True) parser.add_argument('--url', help='the amqp broker url', required=True) parser.add_argument('--name', help='the service name of the log', required=True) parser.add_argument('--tail', help='only send new lines appended to file', action='store_true') parser.add_argument('--queue', help='the amqp queue name to publish on ' '(default: sparkhara)', default='sparkhara') args = parser.parse_args() logfile = open(args.file, 'r') if args.tail: logfile.seek(0, os.SEEK_END) conn = kombu.Connection(args.url) producer = kombu.Producer(conn) done = False while not done: pos1 = logfile.tell() l = logfile.readline() pos2 = logfile.tell() if pos1 != pos2: message = {args.name: l} producer.publish(message, routing_key=args.queue, expiration=60) print('sent: {}'.format(message)) else: time.sleep(1)
def test_publish_consume(self, connection): test_queue = kombu.Queue('test', routing_key='test') def callback(body, message): assert body == {'hello': 'world'} assert message.content_type == 'application/x-python-serialize' message.delivery_info['routing_key'] == 'test' message.delivery_info['exchange'] == '' message.ack() assert message.payload == body with connection as conn: with conn.channel() as channel: producer = kombu.Producer(channel) producer.publish({'hello': 'world'}, retry=True, exchange=test_queue.exchange, routing_key=test_queue.routing_key, declare=[test_queue], serializer='pickle') consumer = kombu.Consumer(conn, [test_queue], accept=['pickle']) consumer.register_callback(callback) with consumer: conn.drain_events(timeout=1)
def test_publish_consume(self, connection): test_queue = kombu.Queue('ttl_test', routing_key='ttl_test') def callback(body, message): assert False, 'Callback should not be called' with connection as conn: with conn.channel() as channel: producer = kombu.Producer(channel) producer.publish( {'hello': 'world'}, retry=True, exchange=test_queue.exchange, routing_key=test_queue.routing_key, declare=[test_queue], serializer='pickle', expiration=2 ) consumer = kombu.Consumer( conn, [test_queue], accept=['pickle'] ) consumer.register_callback(callback) sleep(3) with consumer: with pytest.raises(socket.timeout): conn.drain_events(timeout=1)
def retry(self, body, message, reason=''): """ Put the message onto the retry queue """ _logger.warning(reason) try: retry_count = self.retry_count(message) headers = message.headers.copy() headers.update({settings.RETRY_HEADER: retry_count + 1}) kombu.Producer( self.channel, exchange=self.retry_queue.exchange, routing_key=self.retry_queue.routing_key, serializer=settings.SERIALIZER, ).publish(body, headers=headers, retry=True, declares=[self.retry_queue], expiration=self.backoff_func(retry_count)) except Exception as e: message.requeue() _logger.error("Retry failure: retry-reason='{reason}' " "exception='{cls}, {error}'\n" "{traceback}".format( reason=reason, cls=e.__class__.__name__, error=e, traceback=traceback.format_exc(), )) else: message.ack() _logger.debug("Retry: {reason}".format(reason=reason))
def test_passing_a_conn(channel, random_queue, connection): processor: missive.Processor[missive.JSONMessage] = missive.Processor() flag = False @processor.handle_for(always) def catch_all(message, ctx): nonlocal flag flag = message.get_json() ctx.ack() adapted.shutdown_handler.set_flag() adapted = RabbitMQAdapter( missive.JSONMessage, processor, [random_queue.name], url_or_conn=connection, disable_shutdown_handler=True, ) test_event = {"test-event": True} producer = kombu.Producer(channel) producer.publish(json.dumps(test_event).encode("utf-8"), routing_key=random_queue.name) thread = threading.Thread(target=adapted.run) thread.start() thread.join(1) assert flag == test_event # Assert nothing left on the queue assert random_queue.get() is None
def _new_publisher(self, uuid, url, exchange_name, exchange_type): bus_connection = kombu.Connection(url) bus_exchange = kombu.Exchange(exchange_name, type=exchange_type) bus_producer = kombu.Producer(bus_connection, exchange=bus_exchange, auto_declare=True) bus_marshaler = xivo_bus.Marshaler(uuid) return xivo_bus.Publisher(bus_producer, bus_marshaler)
def pulse_producer(pulse_conn): exchange = kombu.Exchange('exchange/mrp/', type='topic') producer = kombu.Producer(pulse_conn, exchange=exchange) # Ensure the exchange is declared so that consumers # can start listening before a message is published. producer.maybe_declare(producer.exchange) return producer
def run(connection, args): channel = connection.channel() exchange = kombu.Exchange('villas', type='headers', durable=True) producer = kombu.Producer(channel, exchange=exchange) message = {'action': 'reset'} producer.publish(message, headers=SimulatorCommand.get_headers(args))
def connect(): conn = kombu.Connection('amqp://*****:*****@localhost:5670//', connect_timeout=0.1, transport_options=dict( confirm_publish=True, read_timeout=1.0, write_timeout=1.0, )) chan = conn.channel() return kombu.Producer(chan)
def _get_publisher(self): if not self._publisher: bus_connection = kombu.Connection(self._bus_url) bus_exchange = kombu.Exchange(self._exchange_name, type=self._exchange_type) producer = kombu.Producer(bus_connection, exchange=bus_exchange, auto_declare=True) self._publisher = Publisher(producer, self._marshaler) return self._publisher
def publisher(limeapp, message_queue_connection): """A publisher function that posts to the in-memory queue""" exchange = kombu.Exchange( 'lime.{}'.format(limeapp.identifier), type='topic') producer = kombu.Producer(message_queue_connection, exchange) def publish(routing_key, body): producer.publish(body, routing_key) return publish
def send(self, message, key, exchange=''): self._check_exception() if not self._connected: raise RuntimeError('Not connected to RabbitMQ') producer = kombu.Producer(self._connection) producer.publish(exchange=str(exchange), routing_key=str(key), body=anyjson.dumps(message.body), message_id=str(message.id))
def _publish(self, channel, exchange, queues, routing_key=None): producer = kombu.Producer(channel, exchange=exchange) if routing_key: producer.publish({'hello': 'world'}, declare=list(queues), serializer='pickle', routing_key=routing_key) else: producer.publish({'hello': 'world'}, declare=list(queues), serializer='pickle')
def test_publish_consume(self, connection): # py-amqp transport has higher numbers higher priority # redis transport has lower numbers higher priority if self.PRIORITY_ORDER == 'asc': prio_high = 6 prio_low = 3 else: prio_high = 3 prio_low = 6 test_queue = kombu.Queue('priority_test', routing_key='priority_test', max_priority=10) received_messages = [] def callback(body, message): received_messages.append(body) message.ack() with connection as conn: with conn.channel() as channel: producer = kombu.Producer(channel) for msg, prio in [ [{ 'msg': 'first' }, prio_low], [{ 'msg': 'second' }, prio_high], [{ 'msg': 'third' }, prio_low], ]: producer.publish(msg, retry=True, exchange=test_queue.exchange, routing_key=test_queue.routing_key, declare=[test_queue], serializer='pickle', priority=prio) # Sleep to make sure that queue sorted based on priority sleep(0.5) consumer = kombu.Consumer(conn, [test_queue], accept=['pickle']) consumer.register_callback(callback) with consumer: conn.drain_events(timeout=1) # Second message must be received first assert received_messages[0] == {'msg': 'second'} assert received_messages[1] == {'msg': 'first'} assert received_messages[2] == {'msg': 'third'}
def from_config(cls, bus_config, wazo_uuid): bus_url = 'amqp://{username}:{password}@{host}:{port}//'.format( **bus_config) bus_connection = kombu.Connection(bus_url) bus_exchange = kombu.Exchange(bus_config['exchange_name'], type=bus_config['exchange_type']) bus_producer = kombu.Producer(bus_connection, exchange=bus_exchange, auto_declare=True) bus_marshaler = Marshaler(wazo_uuid) bus_publisher = FailFastPublisher(bus_producer, bus_marshaler) return cls(bus_publisher)
def run(connection, args): channel = connection.channel() exchange = kombu.Exchange('villas', type='headers', durable=True) producer = kombu.Producer(channel, exchange=exchange) message = { 'action': 'delete', 'parameters': _get_parameters(args.parameters, args.parameters_file) } producer.publish(message, headers=SimulatorCommand.get_headers(args))
def _send_event(event): config = _load_config()['bus'] bus_url = 'amqp://{username}:{password}@{host}:{port}/{vhost}'.format( **config) with kombu.Connection(bus_url) as connection: exchange = kombu.Exchange(config['exchange_name'], type=config['exchange_type']) producer = kombu.Producer(connection, exchange=exchange) event_data = { 'name': event.name, 'origin_uuid': os.getenv('XIVO_UUID'), 'data': event.marshal(), } producer.publish(event_data, routing_key=event.routing_key)