def setUp(self): super(NotificationTest, self).setUp() self.exchange = kombu.Exchange('heat', 'topic', durable=False) queue = kombu.Queue(exchange=self.exchange, routing_key='notifications.info', exclusive=True) self.conn = kombu.Connection( get_url(transport.get_transport(cfg.CONF).conf)) self.ch = self.conn.channel() self.queue = queue(self.ch) self.queue.declare()
def _test_messaging(url): with kombu.Connection(url, connect_timeout=1) as conn: simple_queue = conn.SimpleQueue('simple_queue') message = 'Healthcheck sent' simple_queue.put(message) LOG.debug('Sent: %s' % message) message = simple_queue.get(block=True, timeout=1) LOG.debug("Received: %s" % message.payload) message.ack() simple_queue.close()
def get_connection(hostname, port, vhost, user_name=None, password=None, transport_options=None): credentials = f'{user_name}:{password}@' if user_name else '' return kombu.Connection(f'redis://{credentials}{hostname}:{port}', transport_options=transport_options)
def _init_rabbitmq(self): """ connection to rabbitmq and initialize queues """ try: self.connection = kombu.Connection(self.broker_url) exchange = kombu.Exchange(self.exchange_name, type="topic") self.producer = self.connection.Producer(exchange=exchange) except: logging.getLogger(__name__).exception( 'Unable to activate the producer of stat')
def __init__(self, host, port, user, password, virtual_host='/', ssl=True, connect_timeout=5): self.connection = kombu.Connection( hostname=host, port=port, userid=user, password=password, virtual_host=virtual_host, ssl=ssl, connect_timeout=connect_timeout, )
def get_rabbit_status(): client = kombu.Connection(settings.BROKER_URL) try: client.connect() except client.connection_errors: return 'error' except Exception: logger.exception('Unexpected error when trying to connect to Rabbit') return 'error' finally: client.close() return 'ok'
def clear(self) -> None: """ Clear a queue of messages. If the queue does not exist in the exchange, it will be created. If the queue doesn't exist, this is the same as creating an empty queue with no listener. """ with kombu.Connection(self.connection_str) as conn: with kombu.Consumer(conn, [self.queue], no_ack=True): try: conn.drain_events(timeout=2) except (socket.timeout, NotImplementedError): print("\nQueue has been drained\n")
def __init__(self, uri=None, queue='logging', level=NOTSET, filter=None, bubble=False, context=None): Handler.__init__(self, level, filter, bubble) try: import kombu except ImportError: raise RuntimeError('The kombu library is required for ' 'the RabbitMQSubscriber.') if uri: connection = kombu.Connection(uri) self.queue = connection.SimpleQueue(queue)
def _connect(self): self._conn = kombu.Connection(self._hostname, connect_timeout=self._connect_timeout) self._channel = self._conn.channel() self._exchange = kombu.Exchange(name=self._exchange_name, channel=self._channel, durable=False) self._callback_queue = self._create_callback_queue( self._channel, self._exchange)
def setUp(self): super(TestKombuPatch, self).setUp() conn = kombu.Connection('amqp://*****:*****@127.0.0.1:{p}//'.format(p=self.TEST_PORT)) conn.connect() producer = conn.Producer() Pin.override(producer, service=self.TEST_SERVICE, tracer=self.tracer) self.conn = conn self.producer = producer patch()
def from_config(cls, bus_config, wazo_uuid): bus_url = 'amqp://{username}:{password}@{host}:{port}//'.format( **bus_config) bus_connection = kombu.Connection(bus_url) bus_exchange = kombu.Exchange(bus_config['exchange_name'], type=bus_config['exchange_type']) bus_producer = kombu.Producer(bus_connection, exchange=bus_exchange, auto_declare=True) bus_marshaler = Marshaler(wazo_uuid) bus_publisher = FailFastPublisher(bus_producer, bus_marshaler) return cls(bus_publisher)
def __init__(self, url='amqp://*****:*****@localhost:5672//', channel='socketio', write_only=False): if kombu is None: raise RuntimeError('Kombu package is not installed ' '(Run "pip install kombu" in your ' 'virtualenv).') super(KombuManager, self).__init__(channel=channel) self.url = url self.writer_conn = kombu.Connection(self.url) self.writer_queue = self._queue(self.writer_conn)
def __init__(self): transport = 'amqp' transport_options = dict() url = 'amqp://*****:*****@localhost:5672/' self._exchange_name = 'media' self._topic = 'media' self._running = threading.Event() self._drain_events_timeout = 1 self._conn = kombu.Connection(url, transport=transport, transport_options=transport_options) self._exchange = kombu.Exchange(name=self._exchange_name)
def _init_thread(self): if not hasattr(self.tdata, 'connection'): self.tdata.connection = kombu.Connection(self.url) self.tdata.connection.default_channel.basic_qos(0, 1, False) self.tdata.consumer = self.tdata.connection.Consumer( [], callbacks=[self.on_message], no_ack=False) if not hasattr(self.tdata, 'buffer'): self.tdata.buffer = deque() if not hasattr(self.tdata, 'routing_keys'): self.tdata.routing_keys = set()
def __init__(self, host_name, port, userid, password, virtual_host, encoder_class): self.urls_seen = set() # rabbit mq self.q_connection = kombu.Connection('amqp://' + userid + ':' + password + '@' + host_name + ':' + str(port) + '/' + virtual_host) self.exchange = kombu.entity.Exchange(name='JobAds', durable=True) self.encoder = encoder_class() dispatcher.connect(self.spider_opened, signals.spider_opened) dispatcher.connect(self.spider_closed, signals.spider_closed)
def __init__(self, topic, exchange, type_handlers=None, on_wait=None, url=None, transport=None, transport_options=None, retry_options=None): self._topic = topic self._exchange_name = exchange self._on_wait = on_wait self._running = threading.Event() self._dispatcher = dispatcher.TypeDispatcher( # NOTE(skudriashev): Process all incoming messages only if proxy is # running, otherwise requeue them. requeue_filters=[lambda data, message: not self.is_running], type_handlers=type_handlers) ensure_options = self.DEFAULT_RETRY_OPTIONS.copy() if retry_options is not None: # Override the defaults with any user provided values... for k in set(six.iterkeys(ensure_options)): if k in retry_options: # Ensure that the right type is passed in... val = retry_options[k] if k in self._RETRY_INT_OPTS: tmp_val = int(val) else: tmp_val = float(val) if tmp_val < 0: raise ValueError("Expected value greater or equal to" " zero for 'retry_options' %s; got" " %s instead" % (k, val)) ensure_options[k] = tmp_val self._ensure_options = ensure_options self._drain_events_timeout = DRAIN_EVENTS_PERIOD if transport == 'memory' and transport_options: polling_interval = transport_options.get('polling_interval') if polling_interval is not None: self._drain_events_timeout = polling_interval # create connection self._conn = kombu.Connection(url, transport=transport, transport_options=transport_options) # create exchange self._exchange = kombu.Exchange(name=self._exchange_name, durable=False, auto_delete=True)
def send_pulse_message(config, exchange, repo_url, payload): """Send a pulse message for a repository event. The Pulse host configured by the config object will be connected to. The routing key will be constructed from the repository URL. The Pulse message will be constructed from the specified payload and sent to the requested exchange. """ c = config.c routing_key_strip_prefix = config.get('pulse', 'routing_key_strip_prefix') if not repo_url.startswith(routing_key_strip_prefix): raise Exception('repo URL does not begin with %s: %s' % (routing_key_strip_prefix, repo_url)) routing_key = repo_url[len(routing_key_strip_prefix):] hostname = config.get('pulse', 'hostname') port = c.getint('pulse', 'port') userid = config.get('pulse', 'userid') logger.warn('connecting to pulse at %s:%d as %s' % (hostname, port, userid)) conn = kombu.Connection(hostname=hostname, port=port, userid=userid, password=config.get('pulse', 'password'), virtual_host=config.get('pulse', 'virtual_host'), ssl=c.getboolean('pulse', 'ssl'), connect_timeout=c.getint('pulse', 'connect_timeout')) conn.connect() with conn: ex = kombu.Exchange(exchange, type='topic') producer = conn.Producer(exchange=ex, routing_key=routing_key, serializer='json') data = { 'payload': payload, '_meta': { 'exchange': exchange, 'routing_key': routing_key, 'serializer': 'json', 'sent': datetime.datetime.utcnow().isoformat(), } } logger.warn('publishing message to %s#%s' % (exchange, routing_key)) logger.warn('payload: %s' % payload) producer.publish(data)
def connect(): """Create the connection the AMQP. :return: ``object`` """ if not RPC_CFG: return False return kombu.Connection(hostname=RPC_CFG.get('host', '127.0.0.1'), port=RPC_CFG.get('port', 5672), userid=RPC_CFG.get('userid', 'guest'), password=RPC_CFG.get('password', 'guest'), virtual_host=RPC_CFG.get('virtual_host', '/'))
def print_rabbitmq_status(user, password, host): """Print status of RabbitMQ server.""" amqp_url = "amqp://{user}:{password}@{host}:5672//".format( user=user, password=password, host=host) logger.debug("amqp_url: {}".format(amqp_url)) try: conn = kombu.Connection(amqp_url) conn.ensure_connection(max_retries=3) print(("RabbitMQ: ", highlight("RUNNING"))) except Exception as e: print(("RabbitMQ: ", blink(highlight("NOT RUNNING", 'red', True)))) print(e)
def __init__(self): self.handlers = named.NamedExtensionManager( namespace='dnh.handler', names=cfg.CONF[CFG_GRP].handlers, invoke_on_load=True) # Some handlers may have required options. They're loaded by stevedore # after the initial config parsing, so we reparse the config file here # to make sure handlers get all their (required) options parsed. cfg.CONF() self.connection = kombu.Connection( 'amqp://%s:%d' % (cfg.CONF[CFG_GRP].host, cfg.CONF[CFG_GRP].port)) signal.signal(signal.SIGTERM, self.on_signal_stop) signal.signal(signal.SIGINT, self.on_signal_stop)
def _init_rabbitmq(self): """ connect to rabbitmq and init the queues """ self.connection = kombu.Connection(self.config.broker_url) exchange_name = self.config.exchange_name exchange = kombu.Exchange(exchange_name, type="direct") logging.getLogger('stat_persistor').info( "listen following exchange: %s", self.config.exchange_name) queue_name = self.config.queue_name queue = kombu.Queue(queue_name, exchange=exchange, durable=True) self.queues.append(queue)
def amqp_publish(): wait_for(f"http://{config.amqp_host}:{AMQP_MANAGEMENT_PORT}/") connection = kombu.Connection(AMQP_URI) queue = connection.SimpleQueue("test_queue") def publisher(message: dict) -> None: queue.put(message, serializer="json") try: yield publisher finally: queue.clear() queue.close()
def setUp(self): super(BaseConsumerIntegrationTest, self).setUp() # NOTE: # must be a real rabbitmq instance, we rely on rabbitmq # features (dead-letter exchange) for our retry queue logic self.connection = kombu.Connection( settings.BROKER_URL, connect_timeout=1, ) self.connection.ensure_connection() self.connection.connect() self.channel = self.connection.channel()
def __init__(self, url='amqp://*****:*****@localhost:5672//', channel='socketio', write_only=False): if kombu is None: raise RuntimeError('Kombu package is not installed ' '(Run "pip install kombu" in your ' 'virtualenv).') self.kombu = kombu.Connection(url) self.exchange = kombu.Exchange(channel, type='fanout', durable=False) self.queue = kombu.Queue(str(uuid.uuid4()), self.exchange) super(KombuManager, self).__init__(channel=channel, write_only=write_only)
def setUp(self): super(TestKombuSettings, self).setUp() conn = kombu.Connection('amqp://*****:*****@127.0.0.1:{p}//'.format( p=RABBITMQ_CONFIG['port'])) conn.connect() producer = conn.Producer() Pin.override(producer, tracer=self.tracer) self.conn = conn self.producer = producer patch()
def _init_rabbitmq(self): """ connect to rabbitmq and init the queues """ self.connection = kombu.Connection( self.config['rabbitmq']['broker-url']) exchange_name = self.config['rabbitmq']['exchange-name'] exchange = kombu.Exchange(exchange_name, type="direct") logging.getLogger('stat_logger').info("listen following exchange: %s", exchange_name) print("listen following exchange: {}".format(exchange_name)) queue = kombu.Queue(exchange=exchange, durable=False, auto_delete=True) self.queues.append(queue)
def run(self): with kombu.Connection() as conn: self.on_start(conn) with closing(BaseQueue(conn, self.queue_name, self.name)) as queue: try: while True: # TODO: Add correct way to exit run loop obj = queue.get() if obj is not None: self.process(obj) queue.ack() else: self.on_idle() finally: self.on_finish()
def __init__(self, rabbit_ip, rabbit_port, rabbit_user, rabbit_password, rabbit_vhost, rabbit_ha_mode, q_name, subscribe_cb, logger): super(VncKombuClientV1, self).__init__(rabbit_ip, rabbit_port, rabbit_user, rabbit_password, rabbit_vhost, rabbit_ha_mode, q_name, subscribe_cb, logger) self._conn = kombu.Connection(hostname=self._rabbit_ip, port=self._rabbit_port, userid=self._rabbit_user, password=self._rabbit_password, virtual_host=self._rabbit_vhost) self._update_queue_obj = kombu.Queue(q_name, self.obj_upd_exchange, durable=False) self._start()
def run_pulse_listener(config): """ Configures Pulse connection and triggers events from Pulse messages. Connection details are managed at https://pulseguardian.mozilla.org/. """ exchanges = [] queues = {} for queue_name, queue_props in config['pulse'].iteritems(): if (isinstance(queue_props, dict) and set(queue_props.keys()) == {"queue", "exchange", "routing_key"}): queues[queue_name] = queue_props for queue in queues.itervalues(): logger.info("Connecting to pulse queue:%(queue)s exchange:%(exchange)s" " route:%(routing_key)s" % queue) exchanges.append( (queue['queue'], queue['exchange'], queue['routing_key'])) conn = kombu.Connection(hostname=config['pulse']['host'], port=config['pulse']['port'], ssl=config['pulse']['ssl'], userid=config['pulse']['username'], password=config['pulse']['password']) listen_logger = get_listen_logger(config) filter_map = { 'github': GitHubFilter, 'hgmo': PushFilter, 'taskcluster-taskgroup': TaskGroupFilter, 'taskcluster-try-completed': TaskFilter, 'taskcluster-try-failed': TaskFilter, 'taskcluster-try-exception': TaskFilter, } with conn: try: listener = get_listener(conn, userid=config['pulse']['username'], exchanges=exchanges, logger=listen_logger) for queue_name, queue in queues.iteritems(): queue_filter = filter_map[queue_name](config, listen_logger) listener.add_callback(queue['exchange'], queue_filter) listener.run() except KeyboardInterrupt: pass
def heartbeat(): """ send a heartbeat to all kraken """ logging.info('ping krakens!!') with kombu.Connection(current_app.config['CELERY_BROKER_URL']) as connection: instances = models.Instance.query.all() task = task_pb2.Task() task.action = task_pb2.HEARTBEAT for instance in instances: config = load_instance_config(instance.name) exchange = kombu.Exchange(config.exchange, 'topic', durable=True) producer = connection.Producer(exchange=exchange) producer.publish(task.SerializeToString(), routing_key='{}.task.heartbeat'.format(instance.name))