def setUp(self):
super(NotificationTest, self).setUp()
self.exchange = kombu.Exchange('heat', 'topic', durable=False)
queue = kombu.Queue(exchange=self.exchange,
routing_key='notifications.info',
exclusive=True)
self.conn = kombu.Connection(
get_url(transport.get_transport(cfg.CONF).conf))
self.ch = self.conn.channel()
self.queue = queue(self.ch)
self.queue.declare()
def _test_messaging(url):
with kombu.Connection(url, connect_timeout=1) as conn:
simple_queue = conn.SimpleQueue('simple_queue')
message = 'Healthcheck sent'
simple_queue.put(message)
LOG.debug('Sent: %s' % message)
message = simple_queue.get(block=True, timeout=1)
LOG.debug("Received: %s" % message.payload)
message.ack()
simple_queue.close()
def get_connection(hostname,
port,
vhost,
user_name=None,
password=None,
transport_options=None):
credentials = f'{user_name}:{password}@' if user_name else ''
return kombu.Connection(f'redis://{credentials}{hostname}:{port}',
transport_options=transport_options)
def _init_rabbitmq(self):
"""
connection to rabbitmq and initialize queues
"""
try:
self.connection = kombu.Connection(self.broker_url)
exchange = kombu.Exchange(self.exchange_name, type="topic")
self.producer = self.connection.Producer(exchange=exchange)
except:
logging.getLogger(__name__).exception(
'Unable to activate the producer of stat')
def __init__(self, host, port, user, password, virtual_host='/', ssl=True,
connect_timeout=5):
self.connection = kombu.Connection(
hostname=host,
port=port,
userid=user,
password=password,
virtual_host=virtual_host,
ssl=ssl,
connect_timeout=connect_timeout,
)
def get_rabbit_status():
client = kombu.Connection(settings.BROKER_URL)
try:
client.connect()
except client.connection_errors:
return 'error'
except Exception:
logger.exception('Unexpected error when trying to connect to Rabbit')
return 'error'
finally:
client.close()
return 'ok'
def clear(self) -> None:
"""
Clear a queue of messages. If the queue does not exist in the exchange,
it will be created. If the queue doesn't exist, this is the same as
creating an empty queue with no listener.
"""
with kombu.Connection(self.connection_str) as conn:
with kombu.Consumer(conn, [self.queue], no_ack=True):
try:
conn.drain_events(timeout=2)
except (socket.timeout, NotImplementedError):
print("\nQueue has been drained\n")
def __init__(self, uri=None, queue='logging', level=NOTSET,
filter=None, bubble=False, context=None):
Handler.__init__(self, level, filter, bubble)
try:
import kombu
except ImportError:
raise RuntimeError('The kombu library is required for '
'the RabbitMQSubscriber.')
if uri:
connection = kombu.Connection(uri)
self.queue = connection.SimpleQueue(queue)
def _connect(self):
self._conn = kombu.Connection(self._hostname,
connect_timeout=self._connect_timeout)
self._channel = self._conn.channel()
self._exchange = kombu.Exchange(name=self._exchange_name,
channel=self._channel,
durable=False)
self._callback_queue = self._create_callback_queue(
self._channel, self._exchange)
def setUp(self):
super(TestKombuPatch, self).setUp()
conn = kombu.Connection('amqp://*****:*****@127.0.0.1:{p}//'.format(p=self.TEST_PORT))
conn.connect()
producer = conn.Producer()
Pin.override(producer, service=self.TEST_SERVICE, tracer=self.tracer)
self.conn = conn
self.producer = producer
patch()
def from_config(cls, bus_config, wazo_uuid):
bus_url = 'amqp://{username}:{password}@{host}:{port}//'.format(
**bus_config)
bus_connection = kombu.Connection(bus_url)
bus_exchange = kombu.Exchange(bus_config['exchange_name'],
type=bus_config['exchange_type'])
bus_producer = kombu.Producer(bus_connection,
exchange=bus_exchange,
auto_declare=True)
bus_marshaler = Marshaler(wazo_uuid)
bus_publisher = FailFastPublisher(bus_producer, bus_marshaler)
return cls(bus_publisher)
def __init__(self,
url='amqp://*****:*****@localhost:5672//',
channel='socketio',
write_only=False):
if kombu is None:
raise RuntimeError('Kombu package is not installed '
'(Run "pip install kombu" in your '
'virtualenv).')
super(KombuManager, self).__init__(channel=channel)
self.url = url
self.writer_conn = kombu.Connection(self.url)
self.writer_queue = self._queue(self.writer_conn)
def __init__(self):
transport = 'amqp'
transport_options = dict()
url = 'amqp://*****:*****@localhost:5672/'
self._exchange_name = 'media'
self._topic = 'media'
self._running = threading.Event()
self._drain_events_timeout = 1
self._conn = kombu.Connection(url,
transport=transport,
transport_options=transport_options)
self._exchange = kombu.Exchange(name=self._exchange_name)
def _init_thread(self):
if not hasattr(self.tdata, 'connection'):
self.tdata.connection = kombu.Connection(self.url)
self.tdata.connection.default_channel.basic_qos(0, 1, False)
self.tdata.consumer = self.tdata.connection.Consumer(
[], callbacks=[self.on_message], no_ack=False)
if not hasattr(self.tdata, 'buffer'):
self.tdata.buffer = deque()
if not hasattr(self.tdata, 'routing_keys'):
self.tdata.routing_keys = set()
def __init__(self, host_name, port, userid, password, virtual_host,
encoder_class):
self.urls_seen = set()
# rabbit mq
self.q_connection = kombu.Connection('amqp://' + userid + ':' +
password + '@' + host_name + ':' +
str(port) + '/' + virtual_host)
self.exchange = kombu.entity.Exchange(name='JobAds', durable=True)
self.encoder = encoder_class()
dispatcher.connect(self.spider_opened, signals.spider_opened)
dispatcher.connect(self.spider_closed, signals.spider_closed)
def __init__(self,
topic,
exchange,
type_handlers=None,
on_wait=None,
url=None,
transport=None,
transport_options=None,
retry_options=None):
self._topic = topic
self._exchange_name = exchange
self._on_wait = on_wait
self._running = threading.Event()
self._dispatcher = dispatcher.TypeDispatcher(
# NOTE(skudriashev): Process all incoming messages only if proxy is
# running, otherwise requeue them.
requeue_filters=[lambda data, message: not self.is_running],
type_handlers=type_handlers)
ensure_options = self.DEFAULT_RETRY_OPTIONS.copy()
if retry_options is not None:
# Override the defaults with any user provided values...
for k in set(six.iterkeys(ensure_options)):
if k in retry_options:
# Ensure that the right type is passed in...
val = retry_options[k]
if k in self._RETRY_INT_OPTS:
tmp_val = int(val)
else:
tmp_val = float(val)
if tmp_val < 0:
raise ValueError("Expected value greater or equal to"
" zero for 'retry_options' %s; got"
" %s instead" % (k, val))
ensure_options[k] = tmp_val
self._ensure_options = ensure_options
self._drain_events_timeout = DRAIN_EVENTS_PERIOD
if transport == 'memory' and transport_options:
polling_interval = transport_options.get('polling_interval')
if polling_interval is not None:
self._drain_events_timeout = polling_interval
# create connection
self._conn = kombu.Connection(url,
transport=transport,
transport_options=transport_options)
# create exchange
self._exchange = kombu.Exchange(name=self._exchange_name,
durable=False,
auto_delete=True)
def send_pulse_message(config, exchange, repo_url, payload):
"""Send a pulse message for a repository event.
The Pulse host configured by the config object will be connected to.
The routing key will be constructed from the repository URL.
The Pulse message will be constructed from the specified payload
and sent to the requested exchange.
"""
c = config.c
routing_key_strip_prefix = config.get('pulse', 'routing_key_strip_prefix')
if not repo_url.startswith(routing_key_strip_prefix):
raise Exception('repo URL does not begin with %s: %s' %
(routing_key_strip_prefix, repo_url))
routing_key = repo_url[len(routing_key_strip_prefix):]
hostname = config.get('pulse', 'hostname')
port = c.getint('pulse', 'port')
userid = config.get('pulse', 'userid')
logger.warn('connecting to pulse at %s:%d as %s' %
(hostname, port, userid))
conn = kombu.Connection(hostname=hostname,
port=port,
userid=userid,
password=config.get('pulse', 'password'),
virtual_host=config.get('pulse', 'virtual_host'),
ssl=c.getboolean('pulse', 'ssl'),
connect_timeout=c.getint('pulse',
'connect_timeout'))
conn.connect()
with conn:
ex = kombu.Exchange(exchange, type='topic')
producer = conn.Producer(exchange=ex,
routing_key=routing_key,
serializer='json')
data = {
'payload': payload,
'_meta': {
'exchange': exchange,
'routing_key': routing_key,
'serializer': 'json',
'sent': datetime.datetime.utcnow().isoformat(),
}
}
logger.warn('publishing message to %s#%s' % (exchange, routing_key))
logger.warn('payload: %s' % payload)
producer.publish(data)
def connect():
"""Create the connection the AMQP.
:return: ``object``
"""
if not RPC_CFG:
return False
return kombu.Connection(hostname=RPC_CFG.get('host', '127.0.0.1'),
port=RPC_CFG.get('port', 5672),
userid=RPC_CFG.get('userid', 'guest'),
password=RPC_CFG.get('password', 'guest'),
virtual_host=RPC_CFG.get('virtual_host', '/'))
def print_rabbitmq_status(user, password, host):
"""Print status of RabbitMQ server."""
amqp_url = "amqp://{user}:{password}@{host}:5672//".format(
user=user, password=password, host=host)
logger.debug("amqp_url: {}".format(amqp_url))
try:
conn = kombu.Connection(amqp_url)
conn.ensure_connection(max_retries=3)
print(("RabbitMQ: ", highlight("RUNNING")))
except Exception as e:
print(("RabbitMQ: ", blink(highlight("NOT RUNNING", 'red', True))))
print(e)
def __init__(self):
self.handlers = named.NamedExtensionManager(
namespace='dnh.handler',
names=cfg.CONF[CFG_GRP].handlers,
invoke_on_load=True)
# Some handlers may have required options. They're loaded by stevedore
# after the initial config parsing, so we reparse the config file here
# to make sure handlers get all their (required) options parsed.
cfg.CONF()
self.connection = kombu.Connection(
'amqp://%s:%d' % (cfg.CONF[CFG_GRP].host, cfg.CONF[CFG_GRP].port))
signal.signal(signal.SIGTERM, self.on_signal_stop)
signal.signal(signal.SIGINT, self.on_signal_stop)
def _init_rabbitmq(self):
"""
connect to rabbitmq and init the queues
"""
self.connection = kombu.Connection(self.config.broker_url)
exchange_name = self.config.exchange_name
exchange = kombu.Exchange(exchange_name, type="direct")
logging.getLogger('stat_persistor').info(
"listen following exchange: %s", self.config.exchange_name)
queue_name = self.config.queue_name
queue = kombu.Queue(queue_name, exchange=exchange, durable=True)
self.queues.append(queue)
def amqp_publish():
wait_for(f"http://{config.amqp_host}:{AMQP_MANAGEMENT_PORT}/")
connection = kombu.Connection(AMQP_URI)
queue = connection.SimpleQueue("test_queue")
def publisher(message: dict) -> None:
queue.put(message, serializer="json")
try:
yield publisher
finally:
queue.clear()
queue.close()
def setUp(self):
super(BaseConsumerIntegrationTest, self).setUp()
# NOTE:
# must be a real rabbitmq instance, we rely on rabbitmq
# features (dead-letter exchange) for our retry queue logic
self.connection = kombu.Connection(
settings.BROKER_URL,
connect_timeout=1,
)
self.connection.ensure_connection()
self.connection.connect()
self.channel = self.connection.channel()
def __init__(self,
url='amqp://*****:*****@localhost:5672//',
channel='socketio',
write_only=False):
if kombu is None:
raise RuntimeError('Kombu package is not installed '
'(Run "pip install kombu" in your '
'virtualenv).')
self.kombu = kombu.Connection(url)
self.exchange = kombu.Exchange(channel, type='fanout', durable=False)
self.queue = kombu.Queue(str(uuid.uuid4()), self.exchange)
super(KombuManager, self).__init__(channel=channel,
write_only=write_only)
def setUp(self):
super(TestKombuSettings, self).setUp()
conn = kombu.Connection('amqp://*****:*****@127.0.0.1:{p}//'.format(
p=RABBITMQ_CONFIG['port']))
conn.connect()
producer = conn.Producer()
Pin.override(producer, tracer=self.tracer)
self.conn = conn
self.producer = producer
patch()
def _init_rabbitmq(self):
"""
connect to rabbitmq and init the queues
"""
self.connection = kombu.Connection(
self.config['rabbitmq']['broker-url'])
exchange_name = self.config['rabbitmq']['exchange-name']
exchange = kombu.Exchange(exchange_name, type="direct")
logging.getLogger('stat_logger').info("listen following exchange: %s",
exchange_name)
print("listen following exchange: {}".format(exchange_name))
queue = kombu.Queue(exchange=exchange, durable=False, auto_delete=True)
self.queues.append(queue)
def run(self):
with kombu.Connection() as conn:
self.on_start(conn)
with closing(BaseQueue(conn, self.queue_name, self.name)) as queue:
try:
while True: # TODO: Add correct way to exit run loop
obj = queue.get()
if obj is not None:
self.process(obj)
queue.ack()
else:
self.on_idle()
finally:
self.on_finish()
def __init__(self, rabbit_ip, rabbit_port, rabbit_user, rabbit_password,
rabbit_vhost, rabbit_ha_mode, q_name, subscribe_cb, logger):
super(VncKombuClientV1, self).__init__(rabbit_ip, rabbit_port,
rabbit_user, rabbit_password,
rabbit_vhost, rabbit_ha_mode,
q_name, subscribe_cb, logger)
self._conn = kombu.Connection(hostname=self._rabbit_ip,
port=self._rabbit_port,
userid=self._rabbit_user,
password=self._rabbit_password,
virtual_host=self._rabbit_vhost)
self._update_queue_obj = kombu.Queue(q_name, self.obj_upd_exchange, durable=False)
self._start()
def run_pulse_listener(config):
"""
Configures Pulse connection and triggers events from Pulse messages.
Connection details are managed at https://pulseguardian.mozilla.org/.
"""
exchanges = []
queues = {}
for queue_name, queue_props in config['pulse'].iteritems():
if (isinstance(queue_props, dict) and set(queue_props.keys())
== {"queue", "exchange", "routing_key"}):
queues[queue_name] = queue_props
for queue in queues.itervalues():
logger.info("Connecting to pulse queue:%(queue)s exchange:%(exchange)s"
" route:%(routing_key)s" % queue)
exchanges.append(
(queue['queue'], queue['exchange'], queue['routing_key']))
conn = kombu.Connection(hostname=config['pulse']['host'],
port=config['pulse']['port'],
ssl=config['pulse']['ssl'],
userid=config['pulse']['username'],
password=config['pulse']['password'])
listen_logger = get_listen_logger(config)
filter_map = {
'github': GitHubFilter,
'hgmo': PushFilter,
'taskcluster-taskgroup': TaskGroupFilter,
'taskcluster-try-completed': TaskFilter,
'taskcluster-try-failed': TaskFilter,
'taskcluster-try-exception': TaskFilter,
}
with conn:
try:
listener = get_listener(conn,
userid=config['pulse']['username'],
exchanges=exchanges,
logger=listen_logger)
for queue_name, queue in queues.iteritems():
queue_filter = filter_map[queue_name](config, listen_logger)
listener.add_callback(queue['exchange'], queue_filter)
listener.run()
except KeyboardInterrupt:
pass
def heartbeat():
"""
send a heartbeat to all kraken
"""
logging.info('ping krakens!!')
with kombu.Connection(current_app.config['CELERY_BROKER_URL']) as connection:
instances = models.Instance.query.all()
task = task_pb2.Task()
task.action = task_pb2.HEARTBEAT
for instance in instances:
config = load_instance_config(instance.name)
exchange = kombu.Exchange(config.exchange, 'topic', durable=True)
producer = connection.Producer(exchange=exchange)
producer.publish(task.SerializeToString(), routing_key='{}.task.heartbeat'.format(instance.name))
