def test_event_hubs_shared_connection(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'], live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/".format(
live_eventhub_config['hostname'],
live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'])
send_single_message(live_eventhub_config, "0", "Message")
send_single_message(live_eventhub_config, "1", "Message")
with uamqp.Connection(live_eventhub_config['hostname'], sas_auth, debug=False) as conn:
partition_0 = uamqp.ReceiveClient(source + "0", debug=False, auth=sas_auth, timeout=3000, prefetch=10)
partition_1 = uamqp.ReceiveClient(source + "1", debug=False, auth=sas_auth, timeout=3000, prefetch=10)
partition_0.open(connection=conn)
partition_1.open(connection=conn)
try:
messages_0 = partition_0.receive_message_batch(1)
messages_1 = partition_1.receive_message_batch(1)
assert len(messages_0) == 1 and len(messages_1) == 1
except:
raise
finally:
partition_0.close()
partition_1.close()
def test_event_hubs_filter_receive(live_eventhub_config):
plain_auth = authentication.SASLPlain(
live_eventhub_config['hostname'],
live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source_url = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'],
live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
source = address.Source(source_url)
source.set_filter(b"amqp.annotation.x-opt-enqueuedtimeutc > 1518731960545")
with uamqp.ReceiveClient(source, auth=plain_auth, timeout=50, prefetch=50) as receive_client:
log.info("Created client, receiving...")
batch = receive_client.receive_message_batch(max_batch_size=10)
while batch:
for message in batch:
annotations = message.annotations
log.info("Partition Key: {}".format(annotations.get(b'x-opt-partition-key')))
log.info("Sequence Number: {}".format(annotations.get(b'x-opt-sequence-number')))
log.info("Offset: {}".format(annotations.get(b'x-opt-offset')))
log.info("Enqueued Time: {}".format(annotations.get(b'x-opt-enqueued-time')))
log.info("Message format: {}".format(message._message.message_format))
log.info("{}".format(list(message.get_data())))
batch = receive_client.receive_message_batch(max_batch_size=10)
log.info("Finished receiving")
def create_and_open_receive_client(args, partition, clients_arr=None):
print("Creating and opening receive client:{}".format(partition))
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
'$default', str(partition))
global_msg_cnt_dict[partition] = 0
receive_client = uamqp.ReceiveClient(source,
auth=sas_auth,
debug=True,
timeout=5000,
prefetch=args.link_credit)
receive_client.open()
while not receive_client.client_ready():
time.sleep(0.05)
if clients_arr:
clients_arr[partition] = receive_client
print("Receive client:{} is ready to receive".format(partition))
return receive_client
def test_event_hubs_callback_receive_sync(live_eventhub_config):
def on_message_received(message):
annotations = message.annotations
log.info("Sequence Number: {}".format(
annotations.get(b'x-opt-sequence-number')))
log.info(str(message))
message.accept()
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
receive_client = uamqp.ReceiveClient(source,
auth=sas_auth,
timeout=10,
debug=False)
log.info("Created client, receiving...")
receive_client.receive_messages(on_message_received)
log.info("Finished receiving")
def test_event_hubs_client_receive_sync(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
with uamqp.ReceiveClient(source,
auth=sas_auth,
debug=False,
timeout=50,
prefetch=50) as receive_client:
log.info("Created client, receiving...")
with pytest.raises(ValueError):
batch = receive_client.receive_message_batch(max_batch_size=100)
batch = receive_client.receive_message_batch(max_batch_size=10)
while batch:
log.info("Got batch: {}".format(len(batch)))
assert len(batch) <= 10
for message in batch:
annotations = message.annotations
log.info("Sequence Number: {}".format(
annotations.get(b'x-opt-sequence-number')))
batch = receive_client.receive_message_batch(max_batch_size=10)
log.info("Finished receiving")
def test_event_hubs_iter_receive_sync(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
receive_client = uamqp.ReceiveClient(source,
auth=sas_auth,
timeout=10,
debug=False,
prefetch=10)
count = 0
gen = receive_client.receive_messages_iter()
for message in gen:
log.info(message.annotations.get(b'x-opt-sequence-number'))
log.info(str(message))
count += 1
if count >= 10:
log.info("Got {} messages. Breaking.".format(count))
message.accept()
break
count = 0
for message in gen:
count += 1
if count >= 10:
log.info("Got {} more messages. Shutting down.".format(count))
message.accept()
break
receive_client.close()
def test_event_hubs_not_receive_events_during_connection_establishment(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'], live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'],
live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
receive_client = uamqp.ReceiveClient(source, auth=sas_auth, timeout=1000, debug=False, prefetch=10)
try:
receive_client.open()
while not receive_client.client_ready():
time.sleep(0.05)
time.sleep(1) # sleep for 1s
receive_client._connection.work() # do a single connection iteration to see if there're incoming transfers
# make sure no messages are received
assert not receive_client._was_message_received
assert receive_client._received_messages.empty()
messages_0 = receive_client.receive_message_batch()
assert len(messages_0) > 0
finally:
receive_client.close()
def test_event_hubs_client_receive_with_runtime_metric_sync(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'], live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'],
live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
receiver_runtime_metric_symbol = b'com.microsoft:enable-receiver-runtime-metric'
symbol_array = [types.AMQPSymbol(receiver_runtime_metric_symbol)]
desired_capabilities = utils.data_factory(types.AMQPArray(symbol_array))
with uamqp.ReceiveClient(source, auth=sas_auth, debug=False, timeout=50, prefetch=50,
desired_capabilities=desired_capabilities) as receive_client:
log.info("Created client, receiving...")
with pytest.raises(ValueError):
batch = receive_client.receive_message_batch(max_batch_size=100)
batch = receive_client.receive_message_batch(max_batch_size=10)
log.info("Got batch: {}".format(len(batch)))
assert len(batch) <= 10
for message in batch:
annotations = message.annotations
delivery_annotations = message.delivery_annotations
log.info("Sequence Number: {}".format(annotations.get(b'x-opt-sequence-number')))
assert b'last_enqueued_sequence_number' in delivery_annotations
assert b'last_enqueued_offset' in delivery_annotations
assert b'last_enqueued_time_utc' in delivery_annotations
assert b'runtime_info_retrieval_time_utc' in delivery_annotations
log.info("Finished receiving")
async def test_send_with_long_interval_async(live_eventhub, sleep):
test_partition = "0"
sender = EventHubProducerClient(
live_eventhub['hostname'], live_eventhub['event_hub'],
EventHubSharedKeyCredential(live_eventhub['key_name'],
live_eventhub['access_key']))
async with sender:
batch = await sender.create_batch(partition_id=test_partition)
batch.add(EventData(b"A single event"))
await sender.send_batch(batch)
if sleep:
await asyncio.sleep(250
) # EH server side idle timeout is 240 second
else:
await sender._producers[test_partition
]._handler._connection._conn.destroy()
batch = await sender.create_batch(partition_id=test_partition)
batch.add(EventData(b"A single event"))
await sender.send_batch(batch)
received = []
uri = "sb://{}/{}".format(live_eventhub['hostname'],
live_eventhub['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub['key_name'], live_eventhub['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub['hostname'], live_eventhub['event_hub'],
live_eventhub['consumer_group'], test_partition)
receiver = uamqp.ReceiveClient(source,
auth=sas_auth,
debug=False,
timeout=10000,
prefetch=10)
try:
receiver.open()
# receive_message_batch() returns immediately once it receives any messages before the max_batch_size
# and timeout reach. Could be 1, 2, or any number between 1 and max_batch_size.
# So call it twice to ensure the two events are received.
received.extend([
EventData._from_message(x)
for x in receiver.receive_message_batch(max_batch_size=1,
timeout=5000)
])
received.extend([
EventData._from_message(x)
for x in receiver.receive_message_batch(max_batch_size=1,
timeout=5000)
])
finally:
receiver.close()
assert len(received) == 2
assert list(received[0].body)[0] == b"A single event"
def test_event_hubs_iter_receive_no_shutdown_after_timeout_sync(
live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
source = address.Source(source)
source.set_filter(b"amqp.annotation.x-opt-offset > '@latest'")
receive_client = uamqp.ReceiveClient(source,
auth=sas_auth,
timeout=2000,
debug=False,
shutdown_after_timeout=False)
count = 0
receive_client.open()
while not receive_client.client_ready():
receive_client.do_work()
gen = receive_client.receive_messages_iter()
send_single_message(live_eventhub_config,
live_eventhub_config['partition'], 'message')
for message in gen:
log.info(message.annotations.get(b'x-opt-sequence-number'))
log.info(str(message))
count += 1
assert count == 1
count = 0
message_handler_before = receive_client.message_handler
send_single_message(live_eventhub_config,
live_eventhub_config['partition'], 'message')
gen = receive_client.receive_messages_iter()
for message in gen:
log.info(message.annotations.get(b'x-opt-sequence-number'))
log.info(str(message))
count += 1
assert count == 1
message_handler_after = receive_client.message_handler
assert message_handler_before == message_handler_after
receive_client.close()
def test_event_hubs_callback_receive_no_shutdown_after_timeout_sync(
live_eventhub_config):
received_cnt = {'cnt': 0}
def on_message_received(message):
annotations = message.annotations
log.info("Sequence Number: {}".format(
annotations.get(b'x-opt-sequence-number')))
log.info(str(message))
message.accept()
received_cnt['cnt'] += 1
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
source = address.Source(source)
source.set_filter(b"amqp.annotation.x-opt-offset > '@latest'")
receive_client = uamqp.ReceiveClient(source,
auth=sas_auth,
timeout=2000,
debug=False,
shutdown_after_timeout=False)
log.info("Created client, receiving...")
receive_client.open()
while not receive_client.client_ready():
receive_client.do_work()
send_single_message(live_eventhub_config,
live_eventhub_config['partition'], 'message')
receive_client.receive_messages(on_message_received)
message_handler_before = receive_client.message_handler
assert received_cnt['cnt'] == 1
send_single_message(live_eventhub_config,
live_eventhub_config['partition'], 'message')
receive_client.receive_messages(on_message_received)
message_handler_after = receive_client.message_handler
assert message_handler_before == message_handler_after
assert received_cnt['cnt'] == 2
log.info("Finished receiving")
receive_client.close()
def monitor_feedback(target, device_id, wait_on_id=None, token_duration=3600):
def handle_msg(msg):
payload = next(msg.get_data())
if isinstance(payload, bytes):
payload = str(payload, "utf8")
# assume json [] based on spec
payload = json.loads(payload)
for p in payload:
if (
device_id
and p.get("deviceId")
and p["deviceId"].lower() != device_id.lower()
):
return None
six.print_(yaml.dump({"feedback": p}, default_flow_style=False), flush=True)
if wait_on_id:
msg_id = p["originalMessageId"]
if msg_id == wait_on_id:
return msg_id
return None
operation = "/messages/servicebound/feedback"
endpoint = AmqpBuilder.build_iothub_amqp_endpoint_from_target(
target, duration=token_duration
)
endpoint = endpoint + operation
device_filter_txt = None
if device_id:
device_filter_txt = " filtering on device: {},".format(device_id)
six.print_(
"Starting C2D feedback monitor,{} use ctrl-c to stop...".format(
device_filter_txt if device_filter_txt else ""
)
)
try:
client = uamqp.ReceiveClient(
"amqps://" + endpoint, client_name=_get_container_id(), debug=DEBUG
)
message_generator = client.receive_messages_iter()
for msg in message_generator:
match = handle_msg(msg)
if match:
logger.info("Requested message Id has been matched...")
msg.accept()
return match
except uamqp.errors.AMQPConnectionError:
logger.debug("AMQPS connection has expired...")
finally:
client.close()
def test_event_hubs_client_web_socket(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'], live_eventhub_config['access_key'],
transport_type=uamqp.TransportType.AmqpOverWebsocket)
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'],
live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
with uamqp.ReceiveClient(source, auth=sas_auth, debug=False, timeout=5000, prefetch=50) as receive_client:
receive_client.receive_message_batch(max_batch_size=10)
def test_event_hubs_iter_receive_sync(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'], live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'],
live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
receive_client = uamqp.ReceiveClient(source, auth=sas_auth, timeout=10, debug=False, prefetch=10)
for message in receive_client.receive_messages_iter():
annotations = message.annotations
log.info("Sequence Number: {}".format(annotations.get(b'x-opt-sequence-number')))
def _receive_message(conn, source, auth):
batch = []
receive_client = uamqp.ReceiveClient(source,
auth=auth,
debug=True,
timeout=5,
prefetch=50)
try:
receive_client.open(connection=conn)
batch = receive_client.receive_message_batch(max_batch_size=10)
except errors.LinkRedirect as redirect:
return redirect
finally:
receive_client.close()
return batch
def test_event_hubs_client_proxy_settings(live_eventhub_config):
pytest.skip("skipping the test in CI due to no proxy server")
proxy_settings={'proxy_hostname': '127.0.0.1', 'proxy_port': 12345}
uri = "sb://{}/{}".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'], live_eventhub_config['access_key'], http_proxy=proxy_settings)
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'],
live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
with uamqp.ReceiveClient(source, auth=sas_auth, debug=False, timeout=5000, prefetch=50) as receive_client:
receive_client.receive_message_batch(max_batch_size=10)
def test_event_hubs_client_receive_no_shutdown_after_timeout_sync(
live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
source = address.Source(source)
source.set_filter(b"amqp.annotation.x-opt-offset > '@latest'")
received_cnt = 0
with uamqp.ReceiveClient(source,
auth=sas_auth,
timeout=2000,
debug=False,
shutdown_after_timeout=False) as receive_client:
log.info("Created client, receiving...")
received_cnt += len(
receive_client.receive_message_batch(max_batch_size=10))
assert received_cnt == 0
message_handler_before = receive_client.message_handler
send_single_message(live_eventhub_config,
live_eventhub_config['partition'], 'message')
received_cnt += len(
receive_client.receive_message_batch(max_batch_size=10))
assert received_cnt == 1
received_cnt += len(
receive_client.receive_message_batch(max_batch_size=10))
assert received_cnt == 1
send_single_message(live_eventhub_config,
live_eventhub_config['partition'], 'message')
received_cnt += len(
receive_client.receive_message_batch(max_batch_size=10))
message_handler_after = receive_client.message_handler
assert message_handler_before == message_handler_after
assert received_cnt == 2
async def test_send_with_long_interval_async(live_eventhub, sleep):
sender = EventHubProducerClient(
live_eventhub['hostname'], live_eventhub['event_hub'],
EventHubSharedKeyCredential(live_eventhub['key_name'],
live_eventhub['access_key']))
async with sender:
batch = await sender.create_batch()
batch.add(EventData(b"A single event"))
await sender.send_batch(batch)
for _ in range(1):
if sleep:
await asyncio.sleep(300)
else:
await sender._producers[-1]._handler._connection._conn.destroy(
)
batch = await sender.create_batch()
batch.add(EventData(b"A single event"))
await sender.send_batch(batch)
partition_ids = await sender.get_partition_ids()
received = []
for p in partition_ids:
uri = "sb://{}/{}".format(live_eventhub['hostname'],
live_eventhub['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub['key_name'], live_eventhub['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub['hostname'], live_eventhub['event_hub'],
live_eventhub['consumer_group'], p)
receiver = uamqp.ReceiveClient(source,
auth=sas_auth,
debug=False,
timeout=5000,
prefetch=500)
try:
receiver.open()
received.extend([
EventData._from_message(x)
for x in receiver.receive_message_batch(timeout=5000)
])
finally:
receiver.close()
assert len(received) == 2
assert list(received[0].body)[0] == b"A single event"
def test_send_with_long_interval_sync(live_eventhub, sleep):
test_partition = "0"
sender = EventHubProducerClient(
live_eventhub['hostname'], live_eventhub['event_hub'],
EventHubSharedKeyCredential(live_eventhub['key_name'],
live_eventhub['access_key']))
with sender:
batch = sender.create_batch(partition_id=test_partition)
batch.add(EventData(b"A single event"))
sender.send_batch(batch)
if sleep:
time.sleep(250)
else:
sender._producers[
test_partition]._handler._connection._conn.destroy()
batch = sender.create_batch(partition_id=test_partition)
batch.add(EventData(b"A single event"))
sender.send_batch(batch)
received = []
uri = "sb://{}/{}".format(live_eventhub['hostname'],
live_eventhub['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub['key_name'], live_eventhub['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub['hostname'], live_eventhub['event_hub'],
live_eventhub['consumer_group'], test_partition)
receiver = uamqp.ReceiveClient(source,
auth=sas_auth,
debug=False,
timeout=5000,
prefetch=500)
try:
receiver.open()
received.extend([
EventData._from_message(x)
for x in receiver.receive_message_batch(max_batch_size=2,
timeout=10000)
])
finally:
receiver.close()
assert len(received) == 2
assert list(received[0].body)[0] == b"A single event"
def monitor_feedback(target, device_id, wait_on_id=None, token_duration=3600):
def handle_msg(msg):
payload = next(msg.get_data())
if isinstance(payload, bytes):
payload = str(payload, 'utf8')
# assume json [] based on spec
payload = json.loads(payload)
for p in payload:
if device_id and p.get(
'deviceId') and p['deviceId'].lower() != device_id.lower():
return None
six.print_(yaml.dump({'feedback': p}, default_flow_style=False),
flush=True)
if wait_on_id:
msg_id = p['originalMessageId']
if msg_id == wait_on_id:
return msg_id
return None
operation = '/messages/servicebound/feedback'
endpoint = _build_iothub_amqp_endpoint_from_target(target,
duration=token_duration)
endpoint = endpoint + operation
device_filter_txt = None
if device_id:
device_filter_txt = ' filtering on device: {},'.format(device_id)
six.print_('Starting C2D feedback monitor,{} use ctrl-c to stop...'.format(
device_filter_txt if device_filter_txt else ''))
try:
client = uamqp.ReceiveClient('amqps://' + endpoint, debug=DEBUG)
message_generator = client.receive_messages_iter()
for msg in message_generator:
match = handle_msg(msg)
if match:
logger.info('requested msg id has been matched...')
msg.accept()
return match
except uamqp.errors.AMQPConnectionError:
logger.debug('amqp connection has expired...')
finally:
client.close()
def test_event_hubs_dynamic_issue_link_credit(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'], live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'],
live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
msg_sent_cnt = 200
send_multiple_message(live_eventhub_config, msg_sent_cnt)
def message_received_callback(message):
message_received_callback.received_msg_cnt += 1
message_received_callback.received_msg_cnt = 0
with uamqp.ReceiveClient(source, auth=sas_auth, debug=True, prefetch=1) as receive_client:
receive_client._message_received_callback = message_received_callback
while not receive_client.client_ready():
time.sleep(0.05)
time.sleep(1)
receive_client._connection.work()
assert not receive_client._was_message_received
assert receive_client._received_messages.empty()
receive_client.message_handler.reset_link_credit(msg_sent_cnt)
now = start = time.time()
wait_time = 5
while now - start <= wait_time:
receive_client._connection.work()
now = time.time()
assert message_received_callback.received_msg_cnt == msg_sent_cnt
log.info("Finished receiving")
def test_iothub_client_receive_sync(live_iothub_config):
operation = '/messages/events/ConsumerGroups/{}/Partitions/{}'.format(
live_iothub_config['consumer_group'], live_iothub_config['partition'])
endpoint = _build_iothub_amqp_endpoint_from_target(live_iothub_config)
source = 'amqps://' + endpoint + operation
log.info("Source: {}".format(source))
receive_client = uamqp.ReceiveClient(source,
debug=False,
timeout=5000,
prefetch=50)
try:
log.info("Created client, receiving...")
_receive_message(receive_client, live_iothub_config)
except Exception as e:
print(e)
raise
finally:
receive_client.close()
log.info("Finished receiving")
def test_event_hubs_client_proxy_settings(live_eventhub_config):
proxy_settings = {'proxy_hostname': '127.0.0.1', 'proxy_port': 12345}
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri,
live_eventhub_config['key_name'],
live_eventhub_config['access_key'],
http_proxy=proxy_settings)
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
with pytest.raises(errors.AMQPConnectionError):
receive_client = uamqp.ReceiveClient(source,
auth=sas_auth,
debug=False,
timeout=50,
prefetch=50)
receive_client.receive_message_batch(max_batch_size=10)
receive_client.close()
def test_event_hubs_client_proxy_settings(live_eventhub_config):
#pytest.skip("")
proxy_settings = {'proxy_hostname': '127.0.0.1', 'proxy_port': 12345}
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri,
live_eventhub_config['key_name'],
live_eventhub_config['access_key'],
http_proxy=proxy_settings)
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
#if not sys.platform.startswith('darwin'): # Not sure why this passes for OSX:
# with pytest.raises(errors.AMQPConnectionError):
with uamqp.ReceiveClient(source,
auth=sas_auth,
debug=False,
timeout=50,
prefetch=50) as receive_client:
receive_client.receive_message_batch(max_batch_size=10)
def connstr_receivers(connection_str, live_eventhub_config):
partitions = [str(i) for i in range(PARTITION_COUNT)]
receivers = []
for p in partitions:
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'],
live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'], p)
receiver = uamqp.ReceiveClient(source,
auth=sas_auth,
debug=False,
timeout=0,
prefetch=500)
receiver.open()
receivers.append(receiver)
yield connection_str, receivers
for r in receivers:
r.close()
def event_hubs_send_different_amqp_body_type_sync(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'], live_eventhub_config['access_key'])
target = "amqps://{}/{}/Partitions/0".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
send_client = uamqp.SendClient(target, auth=sas_auth, debug=False)
data_body_1 = [b'data1', b'data2']
data_body_message_1 = uamqp.message.Message(body=data_body_1)
data_body_message_1.application_properties = {'body_type': 'data_body_1'}
send_client.queue_message(data_body_message_1)
data_body_2 = b'data1'
data_body_message_2 = uamqp.message.Message(body=data_body_2, body_type=MessageBodyType.Data)
data_body_message_2.application_properties = {'body_type': 'data_body_2'}
send_client.queue_message(data_body_message_2)
value_body_1 = [b'data1', -1.23, True, {b'key': b'value'}, [1, False, 1.23, b'4']]
value_body_message_1 = uamqp.message.Message(body=value_body_1)
value_body_message_1.application_properties = {'body_type': 'value_body_1'}
send_client.queue_message(value_body_message_1)
value_body_2 = {b'key1': {b'sub_key': b'sub_value'}, b'key2': b'value', 3: -1.23}
value_body_message_2 = uamqp.message.Message(body=value_body_2, body_type=MessageBodyType.Value)
value_body_message_2.application_properties = {'body_type': 'value_body_2'}
send_client.queue_message(value_body_message_2)
sequence_body_1 = [b'data1', -1.23, True, {b'key': b'value'}, [b'a', 1.23, True]]
sequence_body_message_1 = uamqp.message.Message(body=sequence_body_1, body_type=MessageBodyType.Sequence)
sequence_body_message_1.application_properties = {'body_type': 'sequence_body_1'}
send_client.queue_message(sequence_body_message_1)
sequence_body_2 = [[1, 2, 3], [b'aa', b'bb', b'cc'], [True, False, True], [{b'key1': b'value'}, {b'key2': 123}]]
sequence_body_message_2 = uamqp.message.Message(body=sequence_body_2, body_type=MessageBodyType.Sequence)
sequence_body_message_2.application_properties = {'body_type': 'sequence_body_2'}
send_client.queue_message(sequence_body_message_2)
results = send_client.send_all_messages(close_on_done=False)
assert not [m for m in results if m == uamqp.constants.MessageState.SendFailed]
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'], live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'],
live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
0)
result_dic = {}
receive_client = uamqp.ReceiveClient(source, auth=sas_auth, timeout=5000, debug=False, prefetch=10)
gen = receive_client.receive_messages_iter()
for message in gen:
if message.application_properties and message.application_properties.get(b'body_type'):
if message.application_properties.get(b'body_type') == b'data_body_1':
check_list = [data for data in message.get_data()]
assert isinstance(message._body, DataBody)
assert check_list == data_body_1
result_dic['data_body_1'] = 1
elif message.application_properties.get(b'body_type') == b'data_body_2':
check_list = [data for data in message.get_data()]
assert isinstance(message._body, DataBody)
assert check_list == [data_body_2]
result_dic['data_body_2'] = 1
elif message.application_properties.get(b'body_type') == b'value_body_1':
assert message.get_data() == value_body_1
assert isinstance(message._body, ValueBody)
result_dic['value_body_1'] = 1
elif message.application_properties.get(b'body_type') == b'value_body_2':
assert message.get_data() == value_body_2
assert isinstance(message._body, ValueBody)
result_dic['value_body_2'] = 1
elif message.application_properties.get(b'body_type') == b'sequence_body_1':
check_list = [data for data in message.get_data()]
assert check_list == [sequence_body_1]
assert isinstance(message._body, SequenceBody)
result_dic['sequence_body_1'] = 1
elif message.application_properties.get(b'body_type') == b'sequence_body_2':
check_list = [data for data in message.get_data()]
assert check_list == sequence_body_2
assert isinstance(message._body, SequenceBody)
result_dic['sequence_body_2'] = 1
log.info(message.annotations.get(b'x-opt-sequence-number'))
log.info(str(message))
send_client.close()
receive_client.close()
assert len(result_dic) == 6
def test_event_hubs_send_event_with_amqp_attributes_sync(live_eventhub_config):
def data_generator():
for i in range(2):
msg = uamqp.message.Message(body='Data')
# header is only used on received msg, not set on messages being sent
msg.application_properties = {'msg_type': 'rich_batch'}
msg.properties = uamqp.message.MessageProperties(message_id='richid', user_id=b'!@qwe\0?123')
msg.footer = {'footerkey':'footervalue'}
msg.delivery_annotations = {'deliveryannkey':'deliveryannvalue'}
msg.annotations = {'annkey':'annvalue'}
yield msg
uri = "sb://{}/{}".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'], live_eventhub_config['access_key'])
target = "amqps://{}/{}/Partitions/0".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
send_client = uamqp.SendClient(target, auth=sas_auth, debug=False)
message = uamqp.message.Message(body='Data')
# header is only used on received msg, not set on messages being sent
message.application_properties = {'msg_type': 'rich_single'}
message.properties = uamqp.message.MessageProperties(message_id='richid')
message.footer = {'footerkey':'footervalue'}
message.delivery_annotations = {'deliveryannkey':'deliveryannvalue'}
message.annotations = {'annkey':'annvalue'}
send_client.queue_message(message)
message_batch = uamqp.message.BatchMessage(data_generator())
send_client.queue_message(message_batch)
results = send_client.send_all_messages(close_on_done=False)
assert not [m for m in results if m == uamqp.constants.MessageState.SendFailed]
rich_single_received = rich_batch_received = False
uri = "sb://{}/{}".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'], live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'],
live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
0)
receive_client = uamqp.ReceiveClient(source, auth=sas_auth, timeout=5000, debug=False, prefetch=10)
gen = receive_client.receive_messages_iter()
for message in gen:
if message.application_properties and message.application_properties.get(b'msg_type'):
if not rich_single_received:
rich_single_received = message.application_properties.get(b'msg_type') == b'rich_single'
if not rich_batch_received:
rich_batch_received = message.application_properties.get(b'msg_type') == b'rich_batch'
if message.application_properties.get(b'msg_type') == b'rich_single':
assert message.properties.user_id is None
elif message.application_properties.get(b'msg_type') == b'rich_batch':
assert message.properties.user_id == b'!@qwe\0?123'
assert message.properties.message_id == b'richid'
assert message.delivery_annotations
assert message.delivery_annotations.get(b'deliveryannkey') == b'deliveryannvalue'
assert message.footer
assert message.footer.get(b'footerkey') == b'footervalue'
assert message.annotations
assert message.annotations.get(b'annkey') == b'annvalue'
log.info(message.annotations.get(b'x-opt-sequence-number'))
log.info(str(message))
send_client.close()
receive_client.close()
assert rich_single_received
assert rich_batch_received
# Valid predicates include:
# - amqp.annotation.x-opt-sequence-number
# - amqp.annotation.x-opt-offset
# - amqp.annotation.x-opt-enqueued-time
# Set endpoint_filter variable to None if no filter is needed
endpoint_filter = b'amqp.annotation.x-opt-sequence-number > -1'
# Helper function to set the filtering predicate on the source URI
def set_endpoint_filter(uri, endpoint_filter=''):
source_uri = uamqp.address.Source(uri)
source_uri.set_filter(endpoint_filter)
return source_uri
receive_client = uamqp.ReceiveClient(set_endpoint_filter(uri, endpoint_filter),
debug=True)
try:
batch = receive_client.receive_message_batch(max_batch_size=5)
except uamqp.errors.LinkRedirect as redirect:
# Once a redirect error is received, close the original client and recreate a new one to the re-directed address
receive_client.close()
sas_auth = uamqp.authentication.SASTokenAuth.from_shared_access_key(
redirect.address, policy_name, access_key)
receive_client = uamqp.ReceiveClient(set_endpoint_filter(
redirect.address, endpoint_filter),
auth=sas_auth,
debug=True)
# Start receiving messages in batches
batch = receive_client.receive_message_batch(max_batch_size=5)
with open(os.path.abspath(os.path.join(os.path.dirname(__file__),
'../config.json')
), 'r') as read_file:
config = json.load(read_file)
TOPIC_NAME = config['topic_name']
SUBSCRIPTION_NAME = config['subscription_name']
SERVICE_NAMESPACE = config['service_namespace']
KEY_NAME = config['key_name']
KEY_VALUE = config['key_value']
CONN_STR = 'amqps://{}:{}@{}.servicebus.windows.net/{}/Subscriptions/{}'.format(
KEY_NAME, quote_plus(KEY_VALUE, safe=''), SERVICE_NAMESPACE, TOPIC_NAME,
SUBSCRIPTION_NAME)
hostname = '{}.servicebus.windows.net'.format(SERVICE_NAMESPACE)
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
CONN_STR, KEY_NAME, KEY_VALUE)
receive_client = uamqp.ReceiveClient(
CONN_STR, auth=sas_auth, debug=False, timeout=0, prefetch=300)
consumer = perftest.PerfConsumerSync(receive_client)
consumer.start()
for i in range(1, 30):
time.sleep(1)
print("consumer received " + consumer.rate.print_rate())
consumer.stop()
def amqpReceive(iot_hub_name,policy_name,consumer_group,partition_id,access_key,since_seq=None,max_age_sec=None,since_ts=None,device_id=None,max_batch_size=60,timeout=1000,debug_network=False):
'''
Receive messages from from AMQP
The optional parameters `since_seq`, `max_age_sec`, `since_ts` are optional and mutually exclusive, in this precedence order
'''
import uamqp
filter=make_fiter(since_seq,max_age_sec,since_ts,device_id)
source_uri=endpoint_uri(iot_hub_name,consumer_group,partition_id,policy_name,access_key,filter)
logger.info(f"Receiving from {source_uri}")
batch=[]
try:
receive_client = uamqp.ReceiveClient(source_uri, debug=debug_network)
logger.info("Start receiving messages batch")
# receive_client.receive_messages(processMessage)
batch=receive_client.receive_message_batch(max_batch_size=max_batch_size,timeout=timeout)
except uamqp.errors.LinkRedirect as redirect:
logger.info("Redirect exception, following")
# receive_client.redirect(redirect,uamqp.authentication.SASTokenAuth.from_shared_access_key(redirect.address.decode(), policy_name, access_key))
receive_client.close()
sas_auth = uamqp.authentication.SASTokenAuth.from_shared_access_key(redirect.address.decode(), policy_name, access_key)
receive_client = uamqp.ReceiveClient(uri_filter(redirect.address,filter), auth=sas_auth, debug=True)
except uamqp.errors.ConnectionClose as connClose:
logger.info(f"Connection close {connClose}")
batch=None
messages=[]
while batch!=None:
logger.info(f"Got {len(batch)} messages")
for msg in batch:
messages.append(processMessage(msg))
try:
# Receiving next messages in batch
logger.debug("Receiving next messages")
# receive_client.receive_messages(processMessage)
batch=receive_client.receive_message_batch(max_batch_size=max_batch_size,timeout=timeout)
if len(batch)==0:
batch=None
except uamqp.errors.LinkDetach as detach:
logger.info(f"Link Detach {detach}")
logger.info(f"detach.condition={detach.condition}")
logger.info(f"detach.info={detach.info}")
logger.info(f"detach.description={detach.description}")
batch=None
except uamqp.errors.ConnectionClose as connClose:
logger.info(f"Connection close {connClose}")
batch=None
receive_client.close()
return messages