def given_published_messages(self):
self.message1 = asynqp.Message('one')
self.message2 = asynqp.Message('one')
self.exchange.publish(self.message1, 'routingkey')
self.exchange.publish(self.message2, 'routingkey')
self.received = []
def hello_world():
"""
Sends a 'hello world' message and then reads it from the queue.
"""
# connect to the RabbitMQ broker
connection = yield from asynqp.connect('localhost',
5672,
username='******',
password='******')
# Open a communications channel
channel = yield from connection.open_channel()
# Create a queue and an exchange on the broker
exchange = yield from channel.declare_exchange('test.exchange', 'direct')
queue = yield from channel.declare_queue('test.queue')
# Bind the queue to the exchange, so the queue will get messages published to the exchange
yield from queue.bind(exchange, 'routing.key')
# If you pass in a dict it will be automatically converted to JSON
msg = asynqp.Message({'hello': 'world'})
exchange.publish(msg, 'routing.key')
# Synchronously get a message from the queue
received_message = yield from queue.get()
print(received_message.json()) # get JSON from incoming messages easily
# Acknowledge a delivered message
received_message.ack()
yield from channel.close()
yield from connection.close()
def test():
with async_tracer.start_active_span('test'):
msg1 = asynqp.Message({'consume': 'this'})
self.exchange.publish(msg1, 'routing.key')
self.consumer = yield from self.queue.consume(handle_message)
yield from asyncio.sleep(0.5)
async def send_bulk_messages_to_exchange(rabbitmq_config: dict,
queue_config: dict,
messages: list) -> None:
"""
Send Messages to rabbit mq exchange. Don't use directly.
:param rabbitmq_config: dict config for rabbitmq instance config: {'host': '', 'port': '', 'username': '',
'password': ''}
:param queue_config: dict config of consumer queues: {'exchange': '', 'queues': [], 'routing_key': '',
'exchange_type': '', 'error_messaging': dict}
:param messages: list of dict object(messages) to be sent
:return: None
"""
connection, channel = None, None
try:
# connect to the RabbitMQ broker
connection = await asynqp.connect(rabbitmq_config['host'],
rabbitmq_config['port'],
rabbitmq_config['username'],
rabbitmq_config['password'])
# Open a communications channel
channel = await connection.open_channel()
# Create an exchange and QUEUE on the broker
amqp_exchange = await channel.declare_exchange(
queue_config['exchange'], queue_config['exchange_type'])
for queue in queue_config['queues']:
amqp_queue = await channel.declare_queue(queue)
await amqp_queue.bind(amqp_exchange, queue_config['routing_key'])
for message in messages:
# If you pass in a dict it will be automatically converted to JSON
msg = asynqp.Message(message, content_type="application/json")
amqp_exchange.publish(msg, queue_config['routing_key'])
print("Published bulk messages to {exchange}-{routing_key}".format(
exchange=queue_config['exchange'],
routing_key=queue_config['routing_key']))
except asynqp.AMQPError:
print(
"Unable to publish bulk messages to exchange: {exchange} with routing key: {routing_key}"
.format(exchange=queue_config['exchange'],
routing_key=queue_config['routing_key']))
if 'error_messaging' in queue_config.keys():
error_message = dict(messages=messages,
exchange=queue_config['exchange'],
queues=queue_config['queues'],
routing_key=queue_config['routing_key'],
exchange_type=queue_config['exchange_type'])
await send_request_to_error_queue(rabbitmq_config,
queue_config['error_messaging'],
error_message)
except asyncio.CancelledError:
print("Asyncio error : cancelled routine")
finally:
if channel is not None:
await channel.close()
if connection is not None:
await connection.close()
print("Queue channel and connection closed")
async def server_to_server_broadcasts(app):
connection = await asynqp.connect(
host=app.config.AMQP_HOST,
port=app.config.AMQP_PORT,
username=app.config.AMQP_USERNAME,
password=app.config.AMQP_PASSWORD)
channel = await connection.open_channel()
@app.listener('after_server_stop')
async def teardown(*_):
await channel.close()
await connection.close()
exchange = await channel.declare_exchange('test.exchange', 'direct')
queue = await channel.declare_queue('test.queue')
await queue.bind(exchange, 'routing.key')
msg = asynqp.Message({'hello': 'world'})
exchange.publish(msg, 'routing.key')
while True:
message = await queue.get()
print(message.json())
message.ack()
async def execute(self, rpc_service: str, rpc_method: str, *args: list, **kwargs: dict):
"""
Parameters
----------
rpc_service : str
...
rpc_method : str
...
args : list
...
kwargs : dict
...
"""
creation_time = time.time()
request = dict(method=rpc_method, args=args, kwargs=kwargs, timings=dict(creation_time=creation_time))
msg = asynqp.Message(
body=self._dumper(request, cls=self._encoder),
content_type='application/json',
# delivery_mode=2, # make message persistent
)
exchange = await self._channel.declare_exchange('', 'direct')
exchange.publish(msg, routing_key=rpc_service)
logger.info("The request to execute the function successfully sent <method='{}'>.".format(rpc_method))
def given_a_multi_frame_message_and_a_consumer(self):
frame_max = self.connection.connection_info['frame_max']
body1 = "a" * (frame_max - 8)
body2 = "b" * (frame_max - 8)
body3 = "c" * (frame_max - 8)
body = body1 + body2 + body3
self.msg = asynqp.Message(body)
def given_a_consumer_has_thrown_an_exception(self):
self.loop.set_exception_handler(lambda l, c: None)
self.expected_message = asynqp.Message('body',
timestamp=datetime(2014, 5, 5))
self.consumer.callback.side_effect = Exception
self.deliver_msg() # cause the exception to be thrown
def given_I_am_listening_for_asyncio_exceptions(self):
self.expected_message = asynqp.Message('body')
self.exception = None
self.loop.set_exception_handler(lambda l, c: setattr(self, "exception", c["exception"]))
self.return_msg() # cause basic_return exception to be thrown
self.exception = None # reset self.exception
def publish_msg(request):
msg = asynqp.Message({'hello': 'world'})
rabbit_util.exchange.publish(msg, 'routing.key')
rabbit_util.exchange.publish(msg, 'routing.key')
rabbit_util.exchange.publish(msg, 'routing.key')
rabbit_util.exchange.publish(msg, 'routing.key')
msg = yield from rabbit_util.queue.get()
return web.Response(text='Published 4 messages. Got 1. %s' % str(msg))
async def message(self, mess):
connection = await asynqp.connect()
channel = await connection.open_channel()
exchange = await channel.declare_exchange('test.exchange', 'direct')
queue = await channel.declare_queue('test.queue')
await queue.bind(exchange, 'routing.key')
msg = asynqp.Message(mess)
exchange.publish(msg, 'routing.key')
await channel.close()
await connection.close()
def given_a_multi_frame_message(self):
frame_max = self.connection.connection_info['frame_max']
body1 = "a" * (frame_max - 8)
body2 = "b" * (frame_max - 8)
body3 = "c" * (frame_max - 8)
body = body1 + body2 + body3
self.msg = asynqp.Message(body)
self.message_received = asyncio.Future()
self.loop.run_until_complete(asyncio.wait_for(self.queue.consume(self.message_received.set_result), 0.2))
async def messages(mess, types):
connection = await asynqp.connect()
channel = await connection.open_channel()
exchange = await channel.declare_exchange(
'douban_{}.exchange'.format(types), 'direct')
queue = await channel.declare_queue('douban_{}.queue'.format(types))
await queue.bind(exchange, 'routing.key')
msg = asynqp.Message(mess)
exchange.publish(msg, 'routing.key')
await channel.close()
await connection.close()
def publish(self, mesg, outgoing_key=None):
"""Route publish packet from client to message queue"""
try:
key = outgoing_key or self.outgoing_key or ''
msg = asynqp.Message(mesg, content_encoding='utf-8')
if self.exchange:
self.exchange.publish(msg, key)
log.debug("To %s: %s", key, mesg)
else:
log.error("Could not publish, because exchange is not exist")
except Exception as err:
log.error('Could not publish, because some error: {}'.format(err))
def setup_producer(connection):
'''
The producer will live as an asyncio.Task
to stop it call Task.cancel()
'''
exchange, _ = yield from setup_exchange_and_queue(connection)
count = 0
while True:
msg = asynqp.Message('Message #{}'.format(count))
exchange.publish(msg, 'routing.key')
yield from asyncio.sleep(1)
count += 1
def test():
@asyncio.coroutine
def publish_a_bunch(msg):
for _ in range(20):
self.exchange.publish(msg, 'routing.key')
with async_tracer.start_active_span('test'):
msg = asynqp.Message({'hello': 'world'})
yield from publish_a_bunch(msg)
for _ in range(10):
msg = yield from self.queue.get()
self.assertIsNotNone(msg)
async def call(self, rpc_service: str, rpc_method: str, *args: list, **kwargs: dict):
"""
Performs the function on the remote server and returns the execution result or raises error.
Parameters
----------
rpc_service : str
...
rpc_method : str
...
args : list
...
kwargs : dict
...
"""
creation_time = time.time()
correlation_id = str(uuid.uuid4())
self._waiter[correlation_id] = asyncio.Event()
# self._response = None
request = dict(method=rpc_method, args=args, kwargs=kwargs, timings=dict(creation_time=creation_time))
msg = asynqp.Message(
body=self._dumper(request, cls=self._encoder),
reply_to=self._response_queue.name,
correlation_id=correlation_id, # If notify not send it
content_type='application/json',
# delivery_mode=2, # make message persistent
)
exchange = await self._channel.declare_exchange('', 'direct')
exchange.publish(msg, routing_key=rpc_service)
logger.info("The request to call the function successfully sent <id='{}', method='{}'>.".format(
correlation_id,
rpc_method,
))
await self._waiter[correlation_id].wait()
response = self._response[correlation_id]
del self._response[correlation_id]
del self._waiter[correlation_id]
if 'result' in response:
return response['result']
# TODO: Raise error with same type?
raise WorkerException(response['error']['message'], code=response['error']['code'])
async def monitor_queue(connection):
exchange, zigbee_queue, api_worker_queue = await setup_exchange_and_queue(
connection)
while True:
if rx_queue.qsize():
print("RX Queue length: {}".format(rx_queue.qsize()))
# If a queue length is detected then we received something from the ZigBee
# and we need to send that message to RabbitMQ and remove it from the queue.
if not rx_queue.empty():
msg = asynqp.Message(rx_queue.get())
exchange.publish(msg, 'route.api-worker')
await asyncio.sleep(0.25)
def given_I_published_a_message(self):
@asyncio.coroutine
def set_up():
self.connection = yield from asynqp.connect(loop=self.loop)
self.channel = yield from self.connection.open_channel()
self.exchange = yield from self.channel.declare_exchange(
'', 'direct')
self.queue = yield from self.channel.declare_queue(
durable=False, exclusive=True, auto_delete=True)
self.loop = asyncio.get_event_loop()
asyncio.set_event_loop(None)
self.loop.run_until_complete(set_up())
message = asynqp.Message(b"Test message")
self.exchange.publish(message, self.queue.name)
def given_I_received_a_message(self):
self.delivery_tag = 12487
msg = asynqp.Message('body', timestamp=datetime(2014, 5, 5))
task = asyncio.async(self.queue.get())
self.tick()
method = spec.BasicGetOK(self.delivery_tag, False, 'my.exchange', 'routing.key', 0)
self.server.send_method(self.channel.id, method)
header = message.get_header_payload(msg, spec.BasicGet.method_type[0])
self.server.send_frame(frames.ContentHeaderFrame(self.channel.id, header))
body = message.get_frame_payloads(msg, 100)[0]
self.server.send_frame(frames.ContentBodyFrame(self.channel.id, body))
self.tick()
self.msg = task.result()
def handle_pub_packet(self, client_id, payload):
"""Route publish packet from client to message queue"""
if client_id in self.publish_channels:
channel_id, = unpack('B', payload[:1])
channel = next((c for c in self.publish_channels[client_id] if c['channel_id'] == channel_id), None)
if channel:
transform = self.transforms[channel['transform_id']]
routing_key = channel['routing_key']
msg = asynqp.Message(transform.to_message(payload[1:]))
self.exchange.publish(msg, routing_key)
else:
LOGGER.warning('Client {0} tried to publish message with unknown channel {1}'.format(
client_id,
channel_id
))
else:
LOGGER.warning('Client {0} tried to publish, but has no channels set up'.format(client_id))
def given_a_message(self):
self.correlation_id = str(uuid.uuid4())
self.message_id = str(uuid.uuid4())
self.timestamp = datetime.fromtimestamp(12345)
self.message = asynqp.Message('body',
content_type='application/json',
content_encoding='utf-8',
headers={},
delivery_mode=2,
priority=5,
correlation_id=self.correlation_id,
reply_to='me',
expiration='tomorrow',
message_id=self.message_id,
timestamp=self.timestamp,
type='telegram',
user_id='benjamin',
app_id='asynqptests')
def produce():
"""
Sends a 'hello world' message and then reads it from the queue.
"""
# connect to the RabbitMQ broker
connection = yield from asynqp.connect('localhost', 5672, username='******', password='******')
# Open a communications channel
channel = yield from connection.open_channel()
# Create a queue and an exchange on the broker
exchange = yield from channel.declare_exchange('test.exchange', 'direct')
queue = yield from channel.declare_queue('test.queue')
# Bind the queue to the exchange, so the queue will get messages published to the exchange
yield from queue.bind(exchange, 'routing.key')
for i in range(1,50000):
# If you pass in a dict it will be automatically converted to JSON
msg = asynqp.Message({'hello': 'world'})
exchange.publish(msg, 'routing.key')
yield from channel.close()
yield from connection.close()
def given_I_asked_for_a_message(self):
self.expected_message = asynqp.Message('body', timestamp=datetime(2014, 5, 5))
self.task = asyncio.async(self.queue.get(no_ack=False))
self.tick()
def given_a_message(self):
self.expected_message = asynqp.Message('body', timestamp=datetime(2014, 5, 5))
def test():
with async_tracer.start_active_span('test'):
msg = asynqp.Message({'hello': 'world'}, content_type='application/json')
self.exchange.publish(msg, 'routing.key')
def when_I_publish_the_message(self):
message = asynqp.Message(b"Test message")
self.exchange.publish(message, self.queue.name)
def handle_message(msg):
self.assertIsNotNone(msg)
msg.ack()
msg2 = asynqp.Message({'handled': 'msg1'})
self.exchange.publish(msg2, 'another.key')
def publish():
with async_tracer.start_active_span('test'):
msg1 = asynqp.Message({'consume': 'this'})
self.exchange.publish(msg1, 'routing.key')
msg = yield from self.queue.get()
self.assertIsNotNone(msg)
def sender(payload):
msg = asynqp.Message(payload)
exchange.publish(msg, 'routing.key')