async def test_event_hubs_multiple_receiver_async(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth_a = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
sas_auth_b = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'])
partition_0 = uamqp.ReceiveClientAsync(source + "0",
debug=False,
auth=sas_auth_a,
timeout=3000,
prefetch=10)
partition_1 = uamqp.ReceiveClientAsync(source + "1",
debug=False,
auth=sas_auth_b,
timeout=3000,
prefetch=10)
try:
await partition_0.open_async()
await partition_1.open_async()
tasks = [receive_ten("0", partition_0), receive_ten("1", partition_1)]
messages = await asyncio.gather(*tasks)
assert len(messages) == 2
assert len(messages[0]) >= 10
assert len(messages[1]) >= 10
print(messages)
finally:
await partition_0.close_async()
await partition_1.close_async()
async def test_event_hubs_shared_connection_async(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'], live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/".format(
live_eventhub_config['hostname'],
live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'])
async with uamqp.ConnectionAsync(live_eventhub_config['hostname'], sas_auth, debug=False) as conn:
partition_0 = uamqp.ReceiveClientAsync(source + "0", debug=False, auth=sas_auth, timeout=3000, prefetch=10)
partition_1 = uamqp.ReceiveClientAsync(source + "1", debug=False, auth=sas_auth, timeout=3000, prefetch=10)
await partition_0.open_async(connection=conn)
await partition_1.open_async(connection=conn)
tasks = [
partition_0.receive_message_batch_async(1),
partition_1.receive_message_batch_async(1)
]
try:
messages = await asyncio.gather(*tasks)
assert len(messages[0]) == 1 and len(messages[1]) == 1
except:
raise
finally:
await partition_0.close_async()
await partition_1.close_async()
async def test_event_hubs_batch_receive_async_no_shutdown_after_timeout_sync(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'], live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'],
live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
source = address.Source(source)
source.set_filter(b"amqp.annotation.x-opt-offset > '@latest'")
async with uamqp.ReceiveClientAsync(source, auth=sas_auth, timeout=3000, prefetch=10, shutdown_after_timeout=False) as receive_client:
received_cnt = 0
received_cnt += len(await receive_client.receive_message_batch_async(10))
assert received_cnt == 0
message_handler_before = receive_client.message_handler
send_single_message(live_eventhub_config, live_eventhub_config['partition'], 'message')
received_cnt += len(await receive_client.receive_message_batch_async(10))
assert received_cnt == 1
received_cnt += len(await receive_client.receive_message_batch_async(10))
assert received_cnt == 1
send_single_message(live_eventhub_config, live_eventhub_config['partition'], 'message')
received_cnt += len(await receive_client.receive_message_batch_async(10))
message_handler_after = receive_client.message_handler
assert message_handler_before == message_handler_after
assert received_cnt == 2
async def test_event_hubs_not_receive_events_during_connection_establishment_async(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'], live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'],
live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
receive_client = uamqp.ReceiveClientAsync(source, auth=sas_auth, timeout=1000, debug=False, prefetch=10)
try:
await receive_client.open_async()
while not await receive_client.client_ready_async():
await asyncio.sleep(0.05)
await asyncio.sleep(1) # sleep for 1s
await receive_client._connection.work_async() # do a single connection iteration to see if there're incoming transfers
# make sure no messages are received
assert not receive_client._was_message_received
assert receive_client._received_messages.empty()
messages_0 = await receive_client.receive_message_batch_async()
assert len(messages_0) > 0
finally:
await receive_client.close_async()
async def test_event_hubs_iter_receive_async(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
receive_client = uamqp.ReceiveClientAsync(source,
debug=False,
auth=sas_auth,
timeout=3000,
prefetch=10)
count = 0
message_generator = receive_client.receive_messages_iter_async()
async for message in message_generator:
log.info("No. {} : {}".format(
message.annotations.get(b'x-opt-sequence-number'), message))
count += 1
if count >= 10:
log.info("Got {} messages. Breaking.".format(count))
message.accept()
break
count = 0
async for message in message_generator:
count += 1
if count >= 10:
log.info("Got {} more messages. Shutting down.".format(count))
message.accept()
break
await receive_client.close_async()
async def test_event_hubs_batch_receive_async(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
async with uamqp.ReceiveClientAsync(source,
debug=False,
auth=sas_auth,
timeout=3000,
prefetch=10) as receive_client:
message_batch = await receive_client.receive_message_batch_async(10)
log.info("got batch: {}".format(len(message_batch)))
for message in message_batch:
annotations = message.annotations
log.info("Sequence Number: {}".format(
annotations.get(b'x-opt-sequence-number')))
next_batch = await receive_client.receive_message_batch_async(10)
log.info("got another batch: {}".format(len(next_batch)))
for message in next_batch:
annotations = message.annotations
log.info("Sequence Number: {}".format(
annotations.get(b'x-opt-sequence-number')))
next_batch = await receive_client.receive_message_batch_async(10)
log.info("got another batch: {}".format(len(next_batch)))
for message in next_batch:
annotations = message.annotations
log.info("Sequence Number: {}".format(
annotations.get(b'x-opt-sequence-number')))
async def test_event_hubs_receive_with_runtime_metric_async(
live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
receiver_runtime_metric_symbol = b'com.microsoft:enable-receiver-runtime-metric'
symbol_array = [types.AMQPSymbol(receiver_runtime_metric_symbol)]
desired_capabilities = utils.data_factory(types.AMQPArray(symbol_array))
async with uamqp.ReceiveClientAsync(
source,
debug=False,
auth=sas_auth,
timeout=1000,
prefetch=10,
desired_capabilities=desired_capabilities) as receive_client:
message_batch = await receive_client.receive_message_batch_async(10)
log.info("got batch: {}".format(len(message_batch)))
for message in message_batch:
annotations = message.annotations
delivery_annotations = message.delivery_annotations
log.info("Sequence Number: {}".format(
annotations.get(b'x-opt-sequence-number')))
assert b'last_enqueued_sequence_number' in delivery_annotations
assert b'last_enqueued_offset' in delivery_annotations
assert b'last_enqueued_time_utc' in delivery_annotations
assert b'runtime_info_retrieval_time_utc' in delivery_annotations
async def test_event_hubs_iter_receive_no_shutdown_after_timeout_async(
live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
source = address.Source(source)
source.set_filter(b"amqp.annotation.x-opt-offset > '@latest'")
receive_client = uamqp.ReceiveClientAsync(source,
auth=sas_auth,
timeout=5000,
debug=False,
shutdown_after_timeout=False)
count = 0
try:
await receive_client.open_async()
while not await receive_client.client_ready_async():
await asyncio.sleep(0.05)
await asyncio.sleep(1) # sleep for 1s
await receive_client._connection.work_async(
) # do a single connection iteration to see if there're incoming transfers
# make sure no messages are received
assert not receive_client._was_message_received
assert receive_client._received_messages.empty()
gen = receive_client.receive_messages_iter_async()
send_single_message(live_eventhub_config,
live_eventhub_config['partition'], 'message')
async for message in gen:
log.info(message.annotations.get(b'x-opt-sequence-number'))
log.info(str(message))
count += 1
assert count == 1
count = 0
message_handler_before = receive_client.message_handler
send_single_message(live_eventhub_config,
live_eventhub_config['partition'], 'message')
gen = receive_client.receive_messages_iter_async()
async for message in gen:
log.info(message.annotations.get(b'x-opt-sequence-number'))
log.info(str(message))
count += 1
assert count == 1
message_handler_after = receive_client.message_handler
assert message_handler_before == message_handler_after
finally:
await receive_client.close_async()
async def monitor_events(endpoint, connection, path, auth, partition, consumer_group, enqueuedtimeutc,
properties, device_id=None, timeout=0):
source = uamqp.address.Source('amqps://{}/{}/ConsumerGroups/{}/Partitions/{}'.format(endpoint, path,
consumer_group, partition))
source.set_filter(
bytes('amqp.annotation.x-opt-enqueuedtimeutc > ' + str(enqueuedtimeutc), 'utf8'))
def _output_msg_kpi(msg):
# TODO: Determine if amqp filters can support boolean operators for multiple conditions
origin = str(msg.annotations.get(b'iothub-connection-device-id'), 'utf8')
if device_id and origin != device_id:
return
event_source = {'event': {}}
event_source['event']['origin'] = origin
event_source['event']['payload'] = str(next(msg.get_data()), 'utf8')
if 'anno' in properties or 'all' in properties:
event_source['event']['annotations'] = unicode_binary_map(msg.annotations)
if 'sys' in properties or 'all' in properties:
if not event_source['event'].get('properties'):
event_source['event']['properties'] = {}
event_source['event']['properties']['system'] = unicode_binary_map(parse_entity(msg.properties, True))
if 'app' in properties or 'all' in properties:
if not event_source['event'].get('properties'):
event_source['event']['properties'] = {}
app_prop = msg.application_properties if msg.application_properties else None
if app_prop:
event_source['event']['properties']['application'] = unicode_binary_map(app_prop)
six.print_(yaml.dump(event_source, default_flow_style=False), flush=True)
exp_cancelled = False
receive_client = uamqp.ReceiveClientAsync(source, auth=auth, timeout=timeout, prefetch=0, debug=DEBUG)
try:
if connection:
await receive_client.open_async(connection=connection)
async for msg in receive_client.receive_messages_iter_async():
_output_msg_kpi(msg)
except asyncio.CancelledError:
exp_cancelled = True
await receive_client.close_async()
except uamqp.errors.LinkDetach as ld:
if isinstance(ld.description, bytes):
ld.description = str(ld.description, 'utf8')
raise RuntimeError(ld.description)
except KeyboardInterrupt:
logger.info("Keyboard interrupt, closing monitor on partition %s", partition)
exp_cancelled = True
await receive_client.close_async()
raise
finally:
if not exp_cancelled:
await receive_client.close_async()
logger.info("Closed monitor on partition %s", partition)
async def receive_async(uri, key_name, key_value, conn_str):
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, key_name, key_value)
receive_client = uamqp.ReceiveClientAsync(conn_str, auth=sas_auth)
print("Created client, receiving...")
await receive_client.receive_messages_async(on_message_received)
print("Finished receiving")
async def test_event_hubs_callback_async_receive_no_shutdown_after_timeout(
live_eventhub_config):
received_cnt = {'cnt': 0}
def on_message_received_internal(message):
annotations = message.annotations
log.info("Sequence Number: {}".format(
annotations.get(b'x-opt-sequence-number')))
log.info(str(message))
message.accept()
received_cnt['cnt'] += 1
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
source = address.Source(source)
source.set_filter(b"amqp.annotation.x-opt-offset > '@latest'")
receive_client = uamqp.ReceiveClientAsync(source,
auth=sas_auth,
timeout=3000,
prefetch=10,
shutdown_after_timeout=False)
log.info("Created client, receiving...")
await receive_client.open_async()
while not await receive_client.client_ready_async():
await receive_client.do_work_async()
send_single_message(live_eventhub_config,
live_eventhub_config['partition'], 'message')
await receive_client.receive_messages_async(on_message_received_internal)
message_handler_before = receive_client.message_handler
assert received_cnt['cnt'] == 1
send_single_message(live_eventhub_config,
live_eventhub_config['partition'], 'message')
await receive_client.receive_messages_async(on_message_received_internal)
message_handler_after = receive_client.message_handler
assert message_handler_before == message_handler_after
assert received_cnt['cnt'] == 2
log.info("Finished receiving")
await receive_client.close_async()
async def test_event_hubs_iter_receive_no_shutdown_after_timeout_async(
live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
source = address.Source(source)
source.set_filter(b"amqp.annotation.x-opt-offset > '@latest'")
receive_client = uamqp.ReceiveClientAsync(source,
auth=sas_auth,
timeout=2000,
debug=False,
shutdown_after_timeout=False)
count = 0
await receive_client.open_async()
while not await receive_client.client_ready_async():
await receive_client.do_work_async()
gen = receive_client.receive_messages_iter_async()
send_single_message(live_eventhub_config,
live_eventhub_config['partition'], 'message')
async for message in gen:
log.info(message.annotations.get(b'x-opt-sequence-number'))
log.info(str(message))
count += 1
assert count == 1
count = 0
message_handler_before = receive_client.message_handler
send_single_message(live_eventhub_config,
live_eventhub_config['partition'], 'message')
gen = receive_client.receive_messages_iter_async()
async for message in gen:
log.info(message.annotations.get(b'x-opt-sequence-number'))
log.info(str(message))
count += 1
assert count == 1
message_handler_after = receive_client.message_handler
assert message_handler_before == message_handler_after
await receive_client.close_async()
async def test_event_hubs_callback_async_receive(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'], live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'],
live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
receive_client = uamqp.ReceiveClientAsync(source, auth=sas_auth, timeout=1000, prefetch=10)
log.info("Created client, receiving...")
await receive_client.receive_messages_async(on_message_received)
log.info("Finished receiving")
async def _monitor_events(
target: Target,
connection,
partition,
enqueued_time_utc,
on_message_received,
timeout=0,
):
source = uamqp.address.Source(
"amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
target.hostname, target.path, target.consumer_group, partition))
source.set_filter(
bytes(
"amqp.annotation.x-opt-enqueuedtimeutc > " +
str(enqueued_time_utc), "utf8"))
exp_cancelled = False
receive_client = uamqp.ReceiveClientAsync(
source,
auth=target.auth,
timeout=timeout,
prefetch=0,
client_name=_get_container_id(),
debug=DEBUG,
)
try:
if connection:
await receive_client.open_async(connection=connection)
async for msg in receive_client.receive_messages_iter_async():
on_message_received(msg)
except asyncio.CancelledError:
exp_cancelled = True
await receive_client.close_async()
except uamqp.errors.LinkDetach as ld:
if isinstance(ld.description, bytes):
ld.description = str(ld.description, "utf8")
raise RuntimeError(ld.description)
except KeyboardInterrupt:
logger.info("Keyboard interrupt, closing monitor on partition %s",
partition)
exp_cancelled = True
await receive_client.close_async()
raise
finally:
if not exp_cancelled:
await receive_client.close_async()
logger.info("Closed monitor on partition %s", partition)
async def test_event_hubs_client_web_socket_async(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'], live_eventhub_config['access_key'],
transport_type=uamqp.TransportType.AmqpOverWebsocket)
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'],
live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
async with uamqp.ReceiveClientAsync(source, auth=sas_auth, debug=False, timeout=5000, prefetch=50) as receive_client:
receive_client.receive_message_batch(max_batch_size=10)
async def test_event_hubs_dynamic_issue_link_credit_async(
live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
msg_sent_cnt = 200
send_multiple_message(live_eventhub_config, msg_sent_cnt)
def message_received_callback(message):
message_received_callback.received_msg_cnt += 1
message_received_callback.received_msg_cnt = 0
async with uamqp.ReceiveClientAsync(source,
debug=True,
auth=sas_auth,
prefetch=1) as receive_client:
receive_client._message_received_callback = message_received_callback
while not await receive_client.client_ready_async():
await asyncio.sleep(0.05)
await asyncio.sleep(1) # sleep for 1s
await receive_client._connection.work_async(
) # do a single connection iteration to see if there're incoming transfers
# make sure no messages are received
assert not receive_client._was_message_received
assert receive_client._received_messages.empty()
await receive_client.message_handler.reset_link_credit_async(
msg_sent_cnt)
now = start = time.time()
wait_time = 5
while now - start <= wait_time:
await receive_client._connection.work_async()
now = time.time()
assert message_received_callback.received_msg_cnt == msg_sent_cnt
log.info("Finished receiving")
async def test_event_hubs_filter_receive_async(live_eventhub_config):
plain_auth = authentication.SASLPlain(live_eventhub_config['hostname'],
live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source_url = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
source = address.Source(source_url)
source.set_filter(b"amqp.annotation.x-opt-enqueuedtimeutc > 1518731960545")
receive_client = uamqp.ReceiveClientAsync(source,
auth=plain_auth,
timeout=5000)
await receive_client.receive_messages_async(on_message_received)
async def _receive_mesages(conn, source, auth):
receive_client = uamqp.ReceiveClientAsync(source,
auth=auth,
debug=True,
timeout=1000,
prefetch=1)
try:
await receive_client.open_async(connection=conn)
batch = await receive_client.receive_message_batch_async(
max_batch_size=1)
except errors.LinkRedirect as redirect:
return redirect
else:
return batch
finally:
await receive_client.close_async()
async def evaluate_redirect(endpoint):
source = uamqp.address.Source('amqps://{}/messages/events/$management'.format(endpoint))
receive_client = uamqp.ReceiveClientAsync(source, timeout=30000, prefetch=1, debug=DEBUG)
try:
await receive_client.open_async()
await receive_client.receive_message_batch_async(max_batch_size=1)
except uamqp.errors.LinkRedirect as redirect:
redirect = unicode_binary_map(parse_entity(redirect))
result = {}
result['events'] = {}
result['events']['endpoint'] = redirect['hostname']
result['events']['path'] = redirect['address'].replace('amqps://', '').split('/')[1]
result['events']['address'] = redirect['address']
return redirect, result
finally:
await receive_client.close_async()
async def query_meta_data(endpoint, path, auth):
source = uamqp.address.Source(endpoint)
receive_client = uamqp.ReceiveClientAsync(source, auth=auth, timeout=30000, debug=DEBUG)
try:
await receive_client.open_async()
message = uamqp.Message(application_properties={'name': path})
response = await receive_client.mgmt_request_async(
message,
b'READ',
op_type=b'com.microsoft:eventhub',
status_code_field=b'status-code',
description_fields=b'status-description',
timeout=30000
)
test = response.get_data()
return test
finally:
await receive_client.close_async()
async def async_create_and_open_receive_client(args, partition):
print('Creating and opening receiver:{}'.format(partition))
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
'$default', partition)
receive_client = uamqp.ReceiveClientAsync(source,
auth=sas_auth,
debug=True,
timeout=5000,
prefetch=args.link_credit)
await receive_client.open_async()
while not await receive_client.client_ready_async():
asyncio.sleep(0.05)
print('Receiver:{} is ready to receive'.format(partition))
return receive_client
async def query_meta_data(address, path, auth):
source = uamqp.address.Source(address)
receive_client = uamqp.ReceiveClientAsync(source,
auth=auth,
timeout=30000,
debug=DEBUG)
try:
await receive_client.open_async()
message = uamqp.Message(application_properties={"name": path})
response = await receive_client.mgmt_request_async(
message,
b"READ",
op_type=b"com.microsoft:eventhub",
status_code_field=b"status-code",
description_fields=b"status-description",
timeout=30000,
)
test = response.get_data()
return test
finally:
await receive_client.close_async()
async def _evaluate_redirect(self, endpoint):
source = uamqp.address.Source(
"amqps://{}/messages/events/$management".format(endpoint))
receive_client = uamqp.ReceiveClientAsync(source,
timeout=30000,
prefetch=1,
debug=DEBUG)
try:
await receive_client.open_async()
await receive_client.receive_message_batch_async(max_batch_size=1)
except uamqp.errors.LinkRedirect as redirect:
redirect = unicode_binary_map(parse_entity(redirect))
result = {}
result["events"] = {}
result["events"]["endpoint"] = redirect["hostname"]
result["events"]["path"] = (redirect["address"].replace(
"amqps://", "").split("/")[1])
result["events"]["address"] = redirect["address"]
return redirect, result
finally:
await receive_client.close_async()
async def monitor_events(endpoint,
connection,
path,
auth,
partition,
consumer_group,
enqueuedtimeutc,
properties,
device_id=None,
timeout=0,
output=None,
content_type=None,
devices=None,
interface_id=None,
pnp_context=None):
source = uamqp.address.Source(
'amqps://{}/{}/ConsumerGroups/{}/Partitions/{}'.format(
endpoint, path, consumer_group, partition))
source.set_filter(
bytes(
'amqp.annotation.x-opt-enqueuedtimeutc > ' + str(enqueuedtimeutc),
'utf8'))
def _output_msg_kpi(msg):
origin = str(msg.annotations.get(b'iothub-connection-device-id'),
'utf8')
if device_id and device_id != origin:
if '*' in device_id or '?' in device_id:
regex = re.escape(device_id).replace("\\*", ".*").replace(
'\\?', ".") + "$"
if not re.match(regex, origin):
return
else:
return
if devices and origin not in devices:
return
if pnp_context:
msg_interface_id = str(msg.annotations.get(b'iothub-interface-id'),
'utf8')
if not msg_interface_id:
return
if interface_id:
if msg_interface_id != interface_id:
return
event_source = {'event': {}}
event_source['event']['origin'] = origin
payload = ''
data = msg.get_data()
if data:
payload = str(next(data), 'utf8')
system_props = unicode_binary_map(parse_entity(msg.properties, True))
ct = content_type
if not ct:
ct = system_props[
'content_type'] if 'content_type' in system_props else ''
if ct and ct.lower() == 'application/json':
try:
payload = json.loads(
re.compile(r'(\\r\\n)+|\\r+|\\n+').sub('', payload))
except Exception: # pylint: disable=broad-except
# We don't want to crash the monitor if JSON parsing fails
pass
event_source['event']['payload'] = payload
if pnp_context:
event_source['event']['interface'] = msg_interface_id
msg_schema = str(
msg.application_properties.get(b'iothub-message-schema'),
'utf8')
interface_context = pnp_context['interface'].get(msg_interface_id)
if interface_context:
msg_schema_context = interface_context.get(msg_schema)
if msg_schema_context:
msg_context_display = msg_schema_context.get('display')
msg_context_unit = msg_schema_context.get('unit')
if msg_context_display:
event_source['event']['payload'] = {}
event_source['event']['payload'][
msg_context_display] = payload
if msg_context_unit:
event_source['event']['payload'][
'unit'] = msg_context_unit
if 'anno' in properties or 'all' in properties:
event_source['event']['annotations'] = unicode_binary_map(
msg.annotations)
if 'sys' in properties or 'all' in properties:
if not event_source['event'].get('properties'):
event_source['event']['properties'] = {}
event_source['event']['properties']['system'] = system_props
if 'app' in properties or 'all' in properties:
if not event_source['event'].get('properties'):
event_source['event']['properties'] = {}
app_prop = msg.application_properties if msg.application_properties else None
if app_prop:
event_source['event']['properties'][
'application'] = unicode_binary_map(app_prop)
if output.lower() == 'json':
dump = json.dumps(event_source, indent=4)
else:
dump = yaml.safe_dump(event_source, default_flow_style=False)
six.print_(dump, flush=True)
exp_cancelled = False
receive_client = uamqp.ReceiveClientAsync(source,
auth=auth,
timeout=timeout,
prefetch=0,
debug=DEBUG)
try:
if connection:
await receive_client.open_async(connection=connection)
async for msg in receive_client.receive_messages_iter_async():
_output_msg_kpi(msg)
except asyncio.CancelledError:
exp_cancelled = True
await receive_client.close_async()
except uamqp.errors.LinkDetach as ld:
if isinstance(ld.description, bytes):
ld.description = str(ld.description, 'utf8')
raise RuntimeError(ld.description)
except KeyboardInterrupt:
logger.info("Keyboard interrupt, closing monitor on partition %s",
partition)
exp_cancelled = True
await receive_client.close_async()
raise
finally:
if not exp_cancelled:
await receive_client.close_async()
logger.info("Closed monitor on partition %s", partition)
async def event_hubs_send_different_amqp_body_type_async(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
target = "amqps://{}/{}/Partitions/0".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
send_client = uamqp.SendClientAsync(target, auth=sas_auth, debug=False)
data_body_1 = [b'data1', b'data2']
data_body_message_1 = uamqp.message.Message(body=data_body_1)
data_body_message_1.application_properties = {'body_type': 'data_body_1'}
send_client.queue_message(data_body_message_1)
data_body_2 = b'data1'
data_body_message_2 = uamqp.message.Message(body=data_body_2,
body_type=MessageBodyType.Data)
data_body_message_2.application_properties = {'body_type': 'data_body_2'}
send_client.queue_message(data_body_message_2)
value_body_1 = [
b'data1', -1.23, True, {
b'key': b'value'
}, [1, False, 1.23, b'4']
]
value_body_message_1 = uamqp.message.Message(body=value_body_1)
value_body_message_1.application_properties = {'body_type': 'value_body_1'}
send_client.queue_message(value_body_message_1)
value_body_2 = {
b'key1': {
b'sub_key': b'sub_value'
},
b'key2': b'value',
3: -1.23
}
value_body_message_2 = uamqp.message.Message(
body=value_body_2, body_type=MessageBodyType.Value)
value_body_message_2.application_properties = {'body_type': 'value_body_2'}
send_client.queue_message(value_body_message_2)
sequence_body_1 = [
b'data1', -1.23, True, {
b'key': b'value'
}, [b'a', 1.23, True]
]
sequence_body_message_1 = uamqp.message.Message(
body=sequence_body_1, body_type=MessageBodyType.Sequence)
sequence_body_message_1.application_properties = {
'body_type': 'sequence_body_1'
}
send_client.queue_message(sequence_body_message_1)
sequence_body_2 = [[1, 2, 3], [b'aa', b'bb', b'cc'], [True, False, True],
[{
b'key1': b'value'
}, {
b'key2': 123
}]]
sequence_body_message_2 = uamqp.message.Message(
body=sequence_body_2, body_type=MessageBodyType.Sequence)
sequence_body_message_2.application_properties = {
'body_type': 'sequence_body_2'
}
send_client.queue_message(sequence_body_message_2)
results = await send_client.send_all_messages_async(close_on_done=False)
assert not [
m for m in results if m == uamqp.constants.MessageState.SendFailed
]
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'], 0)
result_dic = {}
receive_client = uamqp.ReceiveClientAsync(source,
auth=sas_auth,
timeout=5000,
debug=False,
prefetch=10)
gen = receive_client.receive_messages_iter_async()
async for message in gen:
if message.application_properties and message.application_properties.get(
b'body_type'):
if message.application_properties.get(
b'body_type') == b'data_body_1':
check_list = [data for data in message.get_data()]
assert isinstance(message._body, DataBody)
assert check_list == data_body_1
result_dic['data_body_1'] = 1
elif message.application_properties.get(
b'body_type') == b'data_body_2':
check_list = [data for data in message.get_data()]
assert isinstance(message._body, DataBody)
assert check_list == [data_body_2]
result_dic['data_body_2'] = 1
elif message.application_properties.get(
b'body_type') == b'value_body_1':
assert message.get_data() == value_body_1
assert isinstance(message._body, ValueBody)
result_dic['value_body_1'] = 1
elif message.application_properties.get(
b'body_type') == b'value_body_2':
assert message.get_data() == value_body_2
assert isinstance(message._body, ValueBody)
result_dic['value_body_2'] = 1
elif message.application_properties.get(
b'body_type') == b'sequence_body_1':
check_list = [data for data in message.get_data()]
assert check_list == [sequence_body_1]
assert isinstance(message._body, SequenceBody)
result_dic['sequence_body_1'] = 1
elif message.application_properties.get(
b'body_type') == b'sequence_body_2':
check_list = [data for data in message.get_data()]
assert check_list == sequence_body_2
assert isinstance(message._body, SequenceBody)
result_dic['sequence_body_2'] = 1
log.info(message.annotations.get(b'x-opt-sequence-number'))
log.info(str(message))
await send_client.close_async()
await receive_client.close_async()
assert len(results) == 6
async def monitor_events(target,
partition,
consumer_group,
enqueuedtimeutc,
properties,
device_id=None,
timeout=0,
debug=False):
import uamqp
import yaml
from azext_iot.common.utility import unicode_binary_map, parse_entity
source = uamqp.address.Source(
"amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
target['events']['endpoint'], target['events']['path'],
consumer_group, partition))
source.set_filter(
bytes(
'amqp.annotation.x-opt-enqueuedtimeutc > ' + str(enqueuedtimeutc),
'utf8'))
def _output_msg_kpi(msg):
# TODO: Determine if amqp filters can support boolean operators for multiple conditions
if device_id and msg.message_annotations[
b'iothub-connection-device-id'] != device_id:
return
event_source = {'event': {}}
event_source['event']['origin'] = msg.message_annotations.get(
b'iothub-connection-device-id')
event_source['event']['payload'] = str(next(msg.get_data()), 'utf8')
if 'anno' in properties or 'all' in properties:
event_source['event']['annotations'] = unicode_binary_map(
msg.message_annotations)
if 'sys' in properties or 'all' in properties:
if not event_source['event'].get('properties'):
event_source['event']['properties'] = {}
event_source['event']['properties']['system'] = parse_entity(
msg.properties)
if 'app' in properties or 'all' in properties:
if not event_source['event'].get('properties'):
event_source['event']['properties'] = {}
app_prop = None
msg_handle = msg.get_message()
app_prop = msg_handle.application_properties.value.value
del msg_handle
if app_prop:
event_source['event']['properties']['application'] = app_prop
six.print_(yaml.dump(event_source, default_flow_style=False),
flush=True)
exp_cancelled = False
async_client = uamqp.ReceiveClientAsync(source,
auth=_create_sas_auth(target),
timeout=timeout,
prefetch=0,
debug=debug)
try:
# await async_client.open_async(connection=connection)
# Callback method
# await async_client.receive_messages_async(_output_msg_kpi)
async for msg in async_client.receive_messages_iter_async():
_output_msg_kpi(msg)
except asyncio.CancelledError:
exp_cancelled = True
await async_client.close_async()
finally:
if not exp_cancelled:
await async_client.close_async()
async def monitor_events(
endpoint,
connection,
path,
auth,
partition,
consumer_group,
enqueuedtimeutc,
properties,
device_id=None,
timeout=0,
output=None,
content_type=None,
devices=None,
interface_name=None,
pnp_context=None,
validate_messages=False,
simulate_errors=False,
):
source = uamqp.address.Source(
"amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
endpoint, path, consumer_group, partition
)
)
source.set_filter(
bytes("amqp.annotation.x-opt-enqueuedtimeutc > " + str(enqueuedtimeutc), "utf8")
)
exp_cancelled = False
receive_client = uamqp.ReceiveClientAsync(
source,
auth=auth,
timeout=timeout,
prefetch=0,
client_name=_get_container_id(),
debug=DEBUG,
)
try:
if connection:
await receive_client.open_async(connection=connection)
async for msg in receive_client.receive_messages_iter_async():
_output_msg_kpi(
msg,
device_id,
devices,
pnp_context,
interface_name,
content_type,
properties,
output,
validate_messages,
simulate_errors,
)
except asyncio.CancelledError:
exp_cancelled = True
await receive_client.close_async()
except uamqp.errors.LinkDetach as ld:
if isinstance(ld.description, bytes):
ld.description = str(ld.description, "utf8")
raise RuntimeError(ld.description)
except KeyboardInterrupt:
logger.info("Keyboard interrupt, closing monitor on partition %s", partition)
exp_cancelled = True
await receive_client.close_async()
raise
finally:
if not exp_cancelled:
await receive_client.close_async()
logger.info("Closed monitor on partition %s", partition)
