def publish(self,
message: Union[dict, str] = "",
headers: dict = {},
exchange: str = EXCHANGE,
routing_key: str = ROUTING_KEY):
"""
Publish a message to th AMQP Queue
:param message: message to be published
:param headers: header key-values to publish with the message
:param exchange: specifies the top level specifier for message publish
:param routing_key: determines which queue the message is published to
"""
self._conn.connect()
queue = kombu.Queue(routing_key,
kombu.Exchange(exchange, type="topic"),
routing_key=routing_key)
queue.maybe_bind(self._conn)
queue.declare()
producer = kombu.Producer(self._conn.channel())
producer.publish(message,
headers=headers,
exchange=queue.exchange,
routing_key=queue.routing_key,
declare=[queue])
producer.close()
self._conn.release()
def main():
parser = ArgumentParser('Send message to RabbitMQ exchange xivo')
parser.add_argument('-n', '--hostname', help='RabbitMQ server',
default='localhost')
parser.add_argument('-p', '--port', help='Port of RabbitMQ',
default='5672')
parser.add_argument('-r', '--routing-key', help='Routing key to bind on bus',
dest='routing_key', required=True)
parser.add_argument('-i', '--input-event-file', help='JSON file containing the event to send. Default: stdin.',
dest='input_event_file', default='-')
args = parser.parse_args()
config = _DEFAULT_CONFIG
config['host'] = args.hostname
config['port'] = args.port
bus_url = 'amqp://{username}:{password}@{host}:{port}/{vhost}'.format(**config)
with kombu.Connection(bus_url) as connection:
exchange = kombu.Exchange(config['exchange_name'], type=config['exchange_type'])
producer = kombu.Producer(connection, exchange=exchange)
with sys.stdin if args.input_event_file == '-' else open(args.input_event_file, 'r') as f:
event_data = json.load(f)
headers = {
'name': event_data['name'],
'required_acl': event_data['required_acl'],
}
producer.publish(event_data, routing_key=args.routing_key, headers=headers)
def _reconnect(self):
if self._conn_lock.locked():
# either connection-monitor or publisher should have taken
# the lock. The one who acquired the lock would re-establish
# the connection and releases the lock, so the other one can
# just wait on the lock, till it gets released
self._conn_lock.wait()
return
self._conn_lock.acquire()
msg = "RabbitMQ connection down"
self._logger(msg, level=SandeshLevel.SYS_ERR)
self._update_sandesh_status(ConnectionStatus.DOWN)
self._conn_state = ConnectionStatus.DOWN
self._conn.close()
self._conn.ensure_connection()
self._conn.connect()
self._update_sandesh_status(ConnectionStatus.UP)
self._conn_state = ConnectionStatus.UP
msg = 'RabbitMQ connection ESTABLISHED %s' % repr(self._conn)
self._logger(msg, level=SandeshLevel.SYS_NOTICE)
self._channel = self._conn.channel()
self._consumer = kombu.Consumer(self._channel,
queues=self._update_queue_obj,
callbacks=[self._subscribe])
self._producer = kombu.Producer(self._channel,
exchange=self.obj_upd_exchange)
self._conn_lock.release()
def run(self):
self._is_running = True
while self._is_running:
if self.consumer.is_connected():
producer = kombu.Producer(
self.consumer._channel,
on_return=self.consumer._handle_return)
try:
queued_request = self._out_queue.get(timeout=0.5)
if True:
# with kombu.producers[self.consumer.get_connection()].acquire(block=True) as producer:
# producer.on_return = print
try:
self._dispatch_request(queued_request, producer)
except Exception as e:
# except ConnectionResetError:
log.debug(
'Failed to dispatch request, re-enqueueing again, error was: {}'
.format(str(e)))
self.enqueue(queued_request)
except Empty:
continue
else:
sleep(0.5)
log.debug('Waiting for consumer to be ready...')
def __init__(self,
url,
queue,
headers,
allowed,
username='******',
password='******',
max_events=None,
**kwargs):
super(KombuPublisher, self).__init__(allowed, max_events, **kwargs)
self._url = url
self.queue = queue
self._headers = headers
self.username = username
self.password = password
self.exchange = kombu.Exchange(name='amq.direct', type='direct')
self._queue = kombu.Queue(name=queue,
exchange=self.exchange,
routing_key=queue)
self.connection = kombu.Connection(self._url,
userid=self.username,
password=self.password)
self.producer = kombu.Producer(self.connection,
routing_key=self.queue,
exchange=self.exchange)
def _start_publishing(self):
errors = (self._connection.connection_errors +
self._connection.channel_errors)
payload = None
while self._running:
try:
self._ensure_connection()
while self._running and self._connection.connected:
if payload is None:
payload = self._publisher_queue.get()
if self._producer_changed:
self._producer = kombu.Producer(self._channel)
self._producer_changed = False
exchange = self.get_exchange(payload.exchange)
self._producer.publish(payload.message,
exchange=exchange,
routing_key=payload.routing_key,
**payload.kwargs)
payload = None
except errors as e:
msg = 'Connection error in Kombu amqp publisher greenlet: %s' % str(
e)
self._logger(msg, level=SandeshLevel.SYS_WARN)
except Exception as e:
msg = 'Error in Kombu amqp publisher greenlet: %s' % str(e)
self._logger(msg, level=SandeshLevel.SYS_ERR)
payload = None
def archive(self, body, message, reason=''):
"""
Put the message onto the archive queue
"""
_logger.warning(reason)
try:
kombu.Producer(
self.channel,
exchange=self.archive_queue.exchange,
routing_key=self.archive_queue.routing_key,
serializer=settings.SERIALIZER,
).publish(
body,
headers=message.headers,
retry=True,
declares=[self.archive_queue],
)
except Exception as e:
message.requeue()
_logger.error("Archive failure: retry-reason='{reason}' "
"exception='{cls}, {error}'\n"
"{traceback}".format(
reason=reason,
cls=e.__class__.__name__,
error=e,
traceback=traceback.format_exc(),
))
else:
message.ack()
_logger.debug("Archive: {reason}".format(reason=reason))
def setUp(self):
super().setUp()
self.tenant_uuid = SUB_TENANT
self.tenant_name = 'mytenant'
bus_port = self.service_port(5672, 'rabbitmq')
bus = BusClient.from_connection_fields(host='localhost', port=bus_port)
until.true(bus.is_up, timeout=5)
bus_url = 'amqp://{username}:{password}@{host}:{port}//'.format(
username='******',
password='******',
host='localhost',
port=bus_port)
connection = kombu.Connection(bus_url)
connection.connect()
marshaler = Marshaler('the-xivo-uuid')
exchange = kombu.Exchange('xivo', type='topic')
producer = kombu.Producer(connection,
exchange=exchange,
auto_declare=True)
self.publisher = Publisher(producer, marshaler)
self.mock_auth_client = MockAuthClient('localhost',
self.service_port(9497, 'auth'))
def wait_for_dird_bus_connection():
response = self.client.status.get()
assert_that(response, has_entries(bus_consumer={'status': 'ok'}))
until.assert_(wait_for_dird_bus_connection, timeout=6)
def _generate_publish(self, name, exchange):
# Create producer for the exchange
exchange_path = "exchange/%s/%s%s" % (
self.namespace, self.exchange_prefix, exchange.exchange)
producer = kombu.Producer(channel=self.connection,
exchange=kombu.Exchange(
name=exchange_path,
type='topic',
durable=True,
delivery_mode='persistent'),
auto_declare=True)
publish_message = self.connection.ensure(producer,
producer.publish,
max_retries=3)
# Create publication method for the exchange
def publish(**kwargs):
message = exchange.message(kwargs)
jsonschema.validate(message, self.schemas[exchange.schema])
publish_message(body=json.dumps(message),
routing_key=exchange.routing(**kwargs),
content_type='application/json')
return publish
def producer_single(self, msg_body_str):
try:
self.producer.publish(msg_body_str, \
exchange=self.exchange, \
routing_key=self.routing_key, \
serializer=self.serializer, \
compression=self.compression)
#print 'before drain_events'
#self.connection.drain_events()
#print 'after drain_events'
except self.connection.connection_errors + self.connection.channel_errors:
self.connection.close()
print("[%s:%s] host breakdown, switch to next one." % \
(self.connection.hostname, self.connection.port))
self.connection.ensure_connection()
self.channel = self.connection.channel()
self.producer = kombu.Producer(self.channel, \
self.exchange, routing_key=self.routing_key, \
serializer=self.serializer, \
compression=self.compression, \
on_return=on_return_callback)
self.producer.publish(msg_body_str, \
exchange=self.exchange, \
routing_key=self.routing_key, \
serializer=self.serializer, \
compression=self.compression)
def setup_producer(cfg=outgoing_cfg, serializer='json', set_queue=True):
""" Create a Producer, with a corresponding queue if required. """
assert type(set_queue) is bool
logger.debug("cfg: {}".format(cfg))
channel = setup_channel(cfg.hostname, exchange=cfg.exchange)
# Publishing is to an exchange but we need a queue to store messages *before* publication.
# Note that Consumers should really be responsible for (their) queues.
queue = None
if set_queue:
queue = setup_queue(channel, cfg=cfg)
# Publish message; the default message *routing* key is the outgoing queue name.
producer = kombu.Producer(channel,
exchange=cfg.exchange,
routing_key=cfg.queue,
serializer=serializer)
logger.debug('channel_id: {}'.format(producer.channel.channel_id))
logger.debug('exchange: {}'.format(producer.exchange.name))
logger.debug('routing_key: {}'.format(producer.routing_key))
logger.debug('serializer: {}'.format(producer.serializer))
# Track queue, for debugging purposes.
producer._queue = queue
return producer
def _start_publishing(self):
errors = (self._connection.connection_errors +
self._connection.channel_errors)
payload = None
connection = self._connection.clone()
msg = 'KombuAmqpClient: Starting publisher greenlet'
self._logger(msg, level=SandeshLevel.SYS_DEBUG)
while self._running:
try:
self._ensure_connection(connection, "Publisher")
producer = kombu.Producer(connection)
while self._running:
if payload is None:
payload = self._publisher_queue.get()
exchange = self.get_exchange(payload.exchange)
producer.publish(payload.message, exchange=exchange,
routing_key=payload.routing_key, **payload.kwargs)
payload = None
except errors as e:
msg = 'KombuAmqpClient: Connection error in Kombu amqp publisher greenlet: %s' % str(e)
self._logger(msg, level=SandeshLevel.SYS_WARN)
except Exception as e:
msg = 'KombuAmqpClient: Error in Kombu amqp publisher greenlet: %s' % str(e)
self._logger(msg, level=SandeshLevel.SYS_ERR)
msg = 'KombuAmqpClient: Exiting publisher greenlet'
self._logger(msg, level=SandeshLevel.SYS_DEBUG)
def amqp_publish_user(owner, routing_key, data):
with kombu.Connection(config.BROKER_URL) as connection:
channel = connection.channel()
try:
kombu.Producer(channel).publish(data,
exchange=kombu.Exchange(
_amqp_owner_exchange(owner)),
routing_key=routing_key,
serializer='json',
retry=True)
started_at = time()
while True:
try:
connection.drain_events(timeout=0.5)
except AmqpNotFound:
raise
except:
pass
if time() - started_at >= 0.5:
break
except AmqpNotFound:
return False
else:
return True
finally:
channel.close()
def main():
parser = argparse.ArgumentParser(
description='read a file and send its lines to an amqp broker')
parser.add_argument('--file', help='the file to read', required=True)
parser.add_argument('--url', help='the amqp broker url', required=True)
parser.add_argument('--name',
help='the service name of the log',
required=True)
parser.add_argument('--tail',
help='only send new lines appended to file',
action='store_true')
parser.add_argument('--queue',
help='the amqp queue name to publish on '
'(default: sparkhara)',
default='sparkhara')
args = parser.parse_args()
logfile = open(args.file, 'r')
if args.tail:
logfile.seek(0, os.SEEK_END)
conn = kombu.Connection(args.url)
producer = kombu.Producer(conn)
done = False
while not done:
pos1 = logfile.tell()
l = logfile.readline()
pos2 = logfile.tell()
if pos1 != pos2:
message = {args.name: l}
producer.publish(message, routing_key=args.queue, expiration=60)
print('sent: {}'.format(message))
else:
time.sleep(1)
def test_publish_consume(self, connection):
test_queue = kombu.Queue('test', routing_key='test')
def callback(body, message):
assert body == {'hello': 'world'}
assert message.content_type == 'application/x-python-serialize'
message.delivery_info['routing_key'] == 'test'
message.delivery_info['exchange'] == ''
message.ack()
assert message.payload == body
with connection as conn:
with conn.channel() as channel:
producer = kombu.Producer(channel)
producer.publish({'hello': 'world'},
retry=True,
exchange=test_queue.exchange,
routing_key=test_queue.routing_key,
declare=[test_queue],
serializer='pickle')
consumer = kombu.Consumer(conn, [test_queue],
accept=['pickle'])
consumer.register_callback(callback)
with consumer:
conn.drain_events(timeout=1)
def test_publish_consume(self, connection):
test_queue = kombu.Queue('ttl_test', routing_key='ttl_test')
def callback(body, message):
assert False, 'Callback should not be called'
with connection as conn:
with conn.channel() as channel:
producer = kombu.Producer(channel)
producer.publish(
{'hello': 'world'},
retry=True,
exchange=test_queue.exchange,
routing_key=test_queue.routing_key,
declare=[test_queue],
serializer='pickle',
expiration=2
)
consumer = kombu.Consumer(
conn, [test_queue], accept=['pickle']
)
consumer.register_callback(callback)
sleep(3)
with consumer:
with pytest.raises(socket.timeout):
conn.drain_events(timeout=1)
def retry(self, body, message, reason=''):
"""
Put the message onto the retry queue
"""
_logger.warning(reason)
try:
retry_count = self.retry_count(message)
headers = message.headers.copy()
headers.update({settings.RETRY_HEADER: retry_count + 1})
kombu.Producer(
self.channel,
exchange=self.retry_queue.exchange,
routing_key=self.retry_queue.routing_key,
serializer=settings.SERIALIZER,
).publish(body,
headers=headers,
retry=True,
declares=[self.retry_queue],
expiration=self.backoff_func(retry_count))
except Exception as e:
message.requeue()
_logger.error("Retry failure: retry-reason='{reason}' "
"exception='{cls}, {error}'\n"
"{traceback}".format(
reason=reason,
cls=e.__class__.__name__,
error=e,
traceback=traceback.format_exc(),
))
else:
message.ack()
_logger.debug("Retry: {reason}".format(reason=reason))
def test_passing_a_conn(channel, random_queue, connection):
processor: missive.Processor[missive.JSONMessage] = missive.Processor()
flag = False
@processor.handle_for(always)
def catch_all(message, ctx):
nonlocal flag
flag = message.get_json()
ctx.ack()
adapted.shutdown_handler.set_flag()
adapted = RabbitMQAdapter(
missive.JSONMessage,
processor,
[random_queue.name],
url_or_conn=connection,
disable_shutdown_handler=True,
)
test_event = {"test-event": True}
producer = kombu.Producer(channel)
producer.publish(json.dumps(test_event).encode("utf-8"),
routing_key=random_queue.name)
thread = threading.Thread(target=adapted.run)
thread.start()
thread.join(1)
assert flag == test_event
# Assert nothing left on the queue
assert random_queue.get() is None
def _new_publisher(self, uuid, url, exchange_name, exchange_type):
bus_connection = kombu.Connection(url)
bus_exchange = kombu.Exchange(exchange_name, type=exchange_type)
bus_producer = kombu.Producer(bus_connection,
exchange=bus_exchange,
auto_declare=True)
bus_marshaler = xivo_bus.Marshaler(uuid)
return xivo_bus.Publisher(bus_producer, bus_marshaler)
def pulse_producer(pulse_conn):
exchange = kombu.Exchange('exchange/mrp/', type='topic')
producer = kombu.Producer(pulse_conn, exchange=exchange)
# Ensure the exchange is declared so that consumers
# can start listening before a message is published.
producer.maybe_declare(producer.exchange)
return producer
def run(connection, args):
channel = connection.channel()
exchange = kombu.Exchange('villas', type='headers', durable=True)
producer = kombu.Producer(channel, exchange=exchange)
message = {'action': 'reset'}
producer.publish(message, headers=SimulatorCommand.get_headers(args))
def connect():
conn = kombu.Connection('amqp://*****:*****@localhost:5670//',
connect_timeout=0.1,
transport_options=dict(
confirm_publish=True,
read_timeout=1.0,
write_timeout=1.0,
))
chan = conn.channel()
return kombu.Producer(chan)
def _get_publisher(self):
if not self._publisher:
bus_connection = kombu.Connection(self._bus_url)
bus_exchange = kombu.Exchange(self._exchange_name,
type=self._exchange_type)
producer = kombu.Producer(bus_connection,
exchange=bus_exchange,
auto_declare=True)
self._publisher = Publisher(producer, self._marshaler)
return self._publisher
def publisher(limeapp, message_queue_connection):
"""A publisher function that posts to the in-memory queue"""
exchange = kombu.Exchange(
'lime.{}'.format(limeapp.identifier), type='topic')
producer = kombu.Producer(message_queue_connection, exchange)
def publish(routing_key, body):
producer.publish(body, routing_key)
return publish
def send(self, message, key, exchange=''):
self._check_exception()
if not self._connected:
raise RuntimeError('Not connected to RabbitMQ')
producer = kombu.Producer(self._connection)
producer.publish(exchange=str(exchange),
routing_key=str(key),
body=anyjson.dumps(message.body),
message_id=str(message.id))
def _publish(self, channel, exchange, queues, routing_key=None):
producer = kombu.Producer(channel, exchange=exchange)
if routing_key:
producer.publish({'hello': 'world'},
declare=list(queues),
serializer='pickle',
routing_key=routing_key)
else:
producer.publish({'hello': 'world'},
declare=list(queues),
serializer='pickle')
def test_publish_consume(self, connection):
# py-amqp transport has higher numbers higher priority
# redis transport has lower numbers higher priority
if self.PRIORITY_ORDER == 'asc':
prio_high = 6
prio_low = 3
else:
prio_high = 3
prio_low = 6
test_queue = kombu.Queue('priority_test',
routing_key='priority_test',
max_priority=10)
received_messages = []
def callback(body, message):
received_messages.append(body)
message.ack()
with connection as conn:
with conn.channel() as channel:
producer = kombu.Producer(channel)
for msg, prio in [
[{
'msg': 'first'
}, prio_low],
[{
'msg': 'second'
}, prio_high],
[{
'msg': 'third'
}, prio_low],
]:
producer.publish(msg,
retry=True,
exchange=test_queue.exchange,
routing_key=test_queue.routing_key,
declare=[test_queue],
serializer='pickle',
priority=prio)
# Sleep to make sure that queue sorted based on priority
sleep(0.5)
consumer = kombu.Consumer(conn, [test_queue],
accept=['pickle'])
consumer.register_callback(callback)
with consumer:
conn.drain_events(timeout=1)
# Second message must be received first
assert received_messages[0] == {'msg': 'second'}
assert received_messages[1] == {'msg': 'first'}
assert received_messages[2] == {'msg': 'third'}
def from_config(cls, bus_config, wazo_uuid):
bus_url = 'amqp://{username}:{password}@{host}:{port}//'.format(
**bus_config)
bus_connection = kombu.Connection(bus_url)
bus_exchange = kombu.Exchange(bus_config['exchange_name'],
type=bus_config['exchange_type'])
bus_producer = kombu.Producer(bus_connection,
exchange=bus_exchange,
auto_declare=True)
bus_marshaler = Marshaler(wazo_uuid)
bus_publisher = FailFastPublisher(bus_producer, bus_marshaler)
return cls(bus_publisher)
def run(connection, args):
channel = connection.channel()
exchange = kombu.Exchange('villas', type='headers', durable=True)
producer = kombu.Producer(channel, exchange=exchange)
message = {
'action': 'delete',
'parameters': _get_parameters(args.parameters,
args.parameters_file)
}
producer.publish(message, headers=SimulatorCommand.get_headers(args))
def _send_event(event):
config = _load_config()['bus']
bus_url = 'amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
**config)
with kombu.Connection(bus_url) as connection:
exchange = kombu.Exchange(config['exchange_name'],
type=config['exchange_type'])
producer = kombu.Producer(connection, exchange=exchange)
event_data = {
'name': event.name,
'origin_uuid': os.getenv('XIVO_UUID'),
'data': event.marshal(),
}
producer.publish(event_data, routing_key=event.routing_key)
