async def test_client_secret_credential_async(aad_credential, live_eventhub):
try:
from azure.identity.aio import ClientSecretCredential
except ImportError:
pytest.skip("No azure identity library")
client_id, secret, tenant_id = aad_credential
credential = ClientSecretCredential(client_id=client_id,
client_secret=secret,
tenant_id=tenant_id)
client = EventHubClient(host=live_eventhub['hostname'],
event_hub_path=live_eventhub['event_hub'],
credential=credential,
user_agent='customized information')
sender = client.create_producer(partition_id='0')
receiver = client.create_consumer(consumer_group="$default",
partition_id='0',
event_position=EventPosition("@latest"))
async with receiver:
received = await receiver.receive(timeout=3)
assert len(received) == 0
async with sender:
event = EventData(body='A single message')
await sender.send(event)
await asyncio.sleep(1)
received = await receiver.receive(timeout=3)
assert len(received) == 1
assert list(received[0].body)[0] == 'A single message'.encode('utf-8')
async def open_clients_async(self):
"""
Responsible for establishing connection to event hub client
throws EventHubsException, IOException, InterruptedException, ExecutionException.
"""
await self.partition_context.get_initial_offset_async()
# Create event hub client and receive handler and set options
hostname = "{}.{}".format(self.host.eh_config.sb_name,
self.host.eh_config.namespace_suffix)
event_hub_path = self.host.eh_config.eh_name
shared_key_cred = EventHubSharedKeyCredential(
self.host.eh_config.policy, self.host.eh_config.sas_key)
self.eh_client = EventHubClient(
hostname,
event_hub_path,
shared_key_cred,
network_tracing=self.host.eph_options.debug_trace,
http_proxy=self.host.eph_options.http_proxy)
self.partition_receive_handler = self.eh_client.create_consumer(
partition_id=self.partition_context.partition_id,
consumer_group=self.partition_context.consumer_group_name,
event_position=EventPosition(self.partition_context.offset),
prefetch=self.host.eph_options.prefetch_count,
loop=self.loop)
self.partition_receiver = PartitionReceiver(self)
async def main():
client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY),
network_tracing=False)
consumer = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EVENT_POSITION)
async with consumer:
async for item in consumer:
print(item)
async def test_long_running_receive_async(connection_str):
parser = argparse.ArgumentParser()
parser.add_argument("--duration",
help="Duration in seconds of the test",
type=int,
default=30)
parser.add_argument("--consumer",
help="Consumer group name",
default="$default")
parser.add_argument("--partitions", help="Comma seperated partition IDs")
parser.add_argument("--offset", help="Starting offset", default="-1")
parser.add_argument("--conn-str",
help="EventHub connection string",
default=connection_str)
parser.add_argument("--eventhub", help="Name of EventHub")
parser.add_argument("--address", help="Address URI to the EventHub entity")
parser.add_argument(
"--sas-policy",
help="Name of the shared access policy to authenticate with")
parser.add_argument("--sas-key", help="Shared access key")
loop = asyncio.get_event_loop()
args, _ = parser.parse_known_args()
if args.conn_str:
client = EventHubClient.from_connection_string(
args.conn_str,
event_hub_path=args.eventhub,
auth_timeout=240,
network_tracing=False)
elif args.address:
client = EventHubClient(host=args.address,
event_hub_path=args.eventhub,
credential=EventHubSharedKeyCredential(
args.sas_policy, args.sas_key),
auth_timeout=240,
network_tracing=False)
else:
try:
import pytest
pytest.skip("Must specify either '--conn-str' or '--address'")
except ImportError:
raise ValueError("Must specify either '--conn-str' or '--address'")
if not args.partitions:
partitions = await client.get_partition_ids()
else:
partitions = args.partitions.split(",")
pumps = []
for pid in partitions:
receiver = client.create_consumer(consumer_group="$default",
partition_id=pid,
event_position=EventPosition(
args.offset),
prefetch=300,
loop=loop)
pumps.append(pump(pid, receiver, args, args.duration))
await asyncio.gather(*pumps)
async def main():
if not HOSTNAME:
raise ValueError("No EventHubs URL supplied.")
client = EventHubClient(host=HOSTNAME,
event_hub_path=EVENT_HUB,
credential=EventHubSharedKeyCredential(USER, KEY),
network_tracing=False)
consumer = client.create_consumer(consumer_group="$default",
partition_id="0",
event_position=EVENT_POSITION)
await iter_consumer(consumer)
async def test_example_eventhub_async_send_and_receive(live_eventhub_config):
# [START create_eventhub_client_async]
from azure.eventhub.aio import EventHubClient
import os
connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format(
os.environ['EVENT_HUB_HOSTNAME'],
os.environ['EVENT_HUB_SAS_POLICY'],
os.environ['EVENT_HUB_SAS_KEY'],
os.environ['EVENT_HUB_NAME'])
client = EventHubClient.from_connection_string(connection_str)
# [END create_eventhub_client_async]
from azure.eventhub import EventData, EventPosition
# [START create_eventhub_client_async_sender]
client = EventHubClient.from_connection_string(connection_str)
# Create an async producer.
producer = client.create_producer(partition_id="0")
# [END create_eventhub_client_async_sender]
# [START create_eventhub_client_async_receiver]
client = EventHubClient.from_connection_string(connection_str)
# Create an async consumer.
receiver = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition('@latest'))
# Create an exclusive async consumer.
receiver = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition('@latest'), owner_level=1)
# [END create_eventhub_client_async_receiver]
client = EventHubClient.from_connection_string(connection_str)
producer = client.create_producer(partition_id="0")
consumer = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition('@latest'))
await consumer.receive(timeout=1)
# [START eventhub_client_async_send]
async with producer:
event_data = EventData(b"A single event")
await producer.send(event_data)
# [END eventhub_client_async_send]
await asyncio.sleep(1)
# [START eventhub_client_async_receive]
logger = logging.getLogger("azure.eventhub")
async with consumer:
received = await consumer.receive(timeout=5)
for event_data in received:
logger.info("Message received:{}".format(event_data.body_as_str()))
# [END eventhub_client_async_receive]
assert len(received) > 0
assert received[0].body_as_str() == "A single event"
assert list(received[-1].body)[0] == b"A single event"
async def test_long_running_partition_send_async(connection_str):
parser = argparse.ArgumentParser()
parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30)
parser.add_argument("--payload", help="payload size", type=int, default=1024)
parser.add_argument("--partitions", help="Comma separated partition IDs")
parser.add_argument("--conn-str", help="EventHub connection string", default=connection_str)
parser.add_argument("--eventhub", help="Name of EventHub")
parser.add_argument("--address", help="Address URI to the EventHub entity")
parser.add_argument("--sas-policy", help="Name of the shared access policy to authenticate with")
parser.add_argument("--sas-key", help="Shared access key")
parser.add_argument("--logger-name", help="Unique log file ID")
loop = asyncio.get_event_loop()
args, _ = parser.parse_known_args()
if args.conn_str:
client = EventHubClient.from_connection_string(
args.conn_str,
event_hub_path=args.eventhub, network_tracing=False)
elif args.address:
client = EventHubClient(host=args.address,
event_hub_path=args.eventhub,
credential=EventHubSharedKeyCredential(args.sas_policy, args.sas_key),
network_tracing=False)
else:
try:
import pytest
pytest.skip("Must specify either '--conn-str' or '--address'")
except ImportError:
raise ValueError("Must specify either '--conn-str' or '--address'")
try:
if not args.partitions:
partitions = await client.get_partition_ids()
else:
pid_range = args.partitions.split("-")
if len(pid_range) > 1:
partitions = [str(i) for i in range(int(pid_range[0]), int(pid_range[1]) + 1)]
else:
partitions = args.partitions.split(",")
pumps = []
for pid in partitions:
sender = client.create_producer(partition_id=pid, send_timeout=0)
pumps.append(pump(pid, sender, args, args.duration))
results = await asyncio.gather(*pumps, return_exceptions=True)
assert not results
except Exception as e:
logger.error("EventHubProducer failed: {}".format(e))
async def test_send_with_invalid_key_async(invalid_key, connstr_receivers):
_, receivers = connstr_receivers
client = EventHubClient.from_connection_string(invalid_key,
network_tracing=False)
sender = client.create_producer()
with pytest.raises(AuthenticationError):
await sender._open()
async def test_non_existing_entity_sender_async(connection_str):
client = EventHubClient.from_connection_string(connection_str,
event_hub_path="nemo",
network_tracing=False)
sender = client.create_producer(partition_id="1")
with pytest.raises(AuthenticationError):
await sender._open()
def __init__(self):
# This test requires a previusly created Event Hub.
# In this example the name is "myeventhub", but it could be change below
connection_string = os.environ["EVENT_HUBS_CONNECTION_STRING"]
event_hub_name = "myeventhub"
self.client = EventHubClient.from_connection_string(
connection_string, event_hub_name)
async def test_receive_run_time_metric_async(connstr_senders):
from uamqp import __version__ as uamqp_version
from distutils.version import StrictVersion
if StrictVersion(uamqp_version) < StrictVersion('1.2.3'):
pytest.skip("Disabled for uamqp 1.2.2. Will enable after uamqp 1.2.3 is released.")
connection_str, senders = connstr_senders
client = EventHubClient.from_connection_string(connection_str, transport_type=TransportType.AmqpOverWebsocket,
network_tracing=False)
receiver = client.create_consumer(consumer_group="$default", partition_id="0",
event_position=EventPosition('@latest'), prefetch=500,
track_last_enqueued_event_properties=True)
event_list = []
for i in range(20):
event_list.append(EventData("Event Number {}".format(i)))
async with receiver:
received = await receiver.receive(timeout=5)
assert len(received) == 0
senders[0].send(event_list)
await asyncio.sleep(1)
received = await receiver.receive(max_batch_size=50, timeout=5)
assert len(received) == 20
assert receiver.last_enqueued_event_properties
assert receiver.last_enqueued_event_properties.get('sequence_number', None)
assert receiver.last_enqueued_event_properties.get('offset', None)
assert receiver.last_enqueued_event_properties.get('enqueued_time', None)
assert receiver.last_enqueued_event_properties.get('retrieval_time', None)
async def test_send_with_partition_key_async(connstr_receivers):
connection_str, receivers = connstr_receivers
client = EventHubClient.from_connection_string(connection_str,
network_tracing=False)
sender = client.create_producer()
async with sender:
data_val = 0
for partition in [b"a", b"b", b"c", b"d", b"e", b"f"]:
partition_key = b"test_partition_" + partition
for i in range(50):
data = EventData(str(data_val))
# data.partition_key = partition_key
data_val += 1
await sender.send(data, partition_key=partition_key)
found_partition_keys = {}
for index, partition in enumerate(receivers):
received = partition.receive(timeout=5)
for message in received:
try:
existing = found_partition_keys[message.partition_key]
assert existing == index
except KeyError:
found_partition_keys[message.partition_key] = index
async def test_exclusive_receiver_after_non_exclusive_receiver_async(
connstr_senders):
connection_str, senders = connstr_senders
senders[0].send(EventData(b"Receiving only a single event"))
client = EventHubClient.from_connection_string(connection_str,
network_tracing=False)
receiver1 = client.create_consumer(consumer_group="$default",
partition_id="0",
event_position=EventPosition("-1"),
prefetch=10)
receiver2 = client.create_consumer(consumer_group="$default",
partition_id="0",
event_position=EventPosition("-1"),
owner_level=15,
prefetch=10)
try:
await pump(receiver1)
output2 = await pump(receiver2)
with pytest.raises(ConnectionLostError):
await receiver1.receive(timeout=1)
assert output2 == 1
finally:
await receiver1.close()
await receiver2.close()
async def test_send_with_forced_conn_close_async(connstr_receivers, sleep):
pytest.skip("This test is similar to the above one")
connection_str, receivers = connstr_receivers
client = EventHubClient.from_connection_string(connection_str,
network_tracing=False)
sender = client.create_producer()
try:
await sender.send(EventData(b"A single event"))
if sleep:
await asyncio.sleep(300)
else:
sender._handler._connection._conn.destroy()
await sender.send(EventData(b"A single event"))
await sender.send(EventData(b"A single event"))
if sleep:
await asyncio.sleep(300)
else:
sender._handler._connection._conn.destroy()
await sender.send(EventData(b"A single event"))
await sender.send(EventData(b"A single event"))
finally:
await sender.close()
received = []
for r in receivers:
if not sleep:
r._handler._connection._conn.destroy()
received.extend(pump(r))
assert len(received) == 5
assert list(received[0].body)[0] == b"A single event"
async def test_receive_over_websocket_async(connstr_senders):
connection_str, senders = connstr_senders
client = EventHubClient.from_connection_string(
connection_str,
transport_type=TransportType.AmqpOverWebsocket,
network_tracing=False)
receiver = client.create_consumer(consumer_group="$default",
partition_id="0",
event_position=EventPosition('@latest'),
prefetch=500)
event_list = []
for i in range(20):
event_list.append(EventData("Event Number {}".format(i)))
async with receiver:
received = await receiver.receive(timeout=5)
assert len(received) == 0
senders[0].send(event_list)
time.sleep(1)
received = await receiver.receive(max_batch_size=50, timeout=5)
assert len(received) == 20
async def get_partition_ids_async(self):
"""
Returns a list of all the event hub partition IDs.
:rtype: list[str]
"""
if not self.partition_ids:
hostname = "{}.{}".format(self.host.eh_config.sb_name,
self.host.eh_config.namespace_suffix)
event_hub_path = self.host.eh_config.eh_name
shared_key_cred = EventHubSharedKeyCredential(
self.host.eh_config.policy, self.host.eh_config.sas_key)
eh_client = EventHubClient(
hostname,
event_hub_path,
shared_key_cred,
network_tracing=self.host.eph_options.debug_trace,
# http_proxy=self.host.eph_options.http_proxy,
)
try:
eh_info = await eh_client.get_properties()
self.partition_ids = eh_info['partition_ids']
except Exception as err: # pylint: disable=broad-except
raise Exception("Failed to get partition ids", repr(err))
return self.partition_ids
async def test_multiple_receiver_async(connstr_senders):
connection_str, senders = connstr_senders
senders[0].send(EventData(b"Receiving only a single event"))
client = EventHubClient.from_connection_string(connection_str,
network_tracing=False)
partitions = await client.get_properties()
assert partitions["partition_ids"] == ["0", "1"]
receivers = []
for i in range(2):
receivers.append(
client.create_consumer(consumer_group="$default",
partition_id="0",
event_position=EventPosition("-1"),
prefetch=10))
try:
more_partitions = await client.get_properties()
assert more_partitions["partition_ids"] == ["0", "1"]
outputs = [0, 0]
outputs[0] = await pump(receivers[0])
outputs[1] = await pump(receivers[1])
assert isinstance(outputs[0], int) and outputs[0] == 1
assert isinstance(outputs[1], int) and outputs[1] == 1
finally:
for r in receivers:
await r.close()
async def test_receive_batch_with_app_prop_async(connstr_senders):
connection_str, senders = connstr_senders
app_prop_key = "raw_prop"
app_prop_value = "raw_value"
app_prop = {app_prop_key: app_prop_value}
def batched():
for i in range(10):
ed = EventData("Event Data {}".format(i))
ed.application_properties = app_prop
yield ed
for i in range(10, 20):
ed = EventData("Event Data {}".format(i))
ed.application_properties = app_prop
yield ed
client = EventHubClient.from_connection_string(connection_str, network_tracing=False)
receiver = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition('@latest'), prefetch=500)
async with receiver:
received = await receiver.receive(timeout=5)
assert len(received) == 0
senders[0].send(batched())
await asyncio.sleep(1)
received = await receiver.receive(max_batch_size=15, timeout=5)
assert len(received) == 15
for index, message in enumerate(received):
assert list(message.body)[0] == "Event Data {}".format(index).encode('utf-8')
assert (app_prop_key.encode('utf-8') in message.application_properties) \
and (dict(message.application_properties)[app_prop_key.encode('utf-8')] == app_prop_value.encode('utf-8'))
async def test_send_batch_with_app_prop_async(connstr_receivers):
connection_str, receivers = connstr_receivers
app_prop_key = "raw_prop"
app_prop_value = "raw_value"
app_prop = {app_prop_key: app_prop_value}
def batched():
for i in range(10):
ed = EventData("Event number {}".format(i))
ed.application_properties = app_prop
yield ed
for i in range(10, 20):
ed = EventData("Event number {}".format(i))
ed.application_properties = app_prop
yield ed
client = EventHubClient.from_connection_string(connection_str,
network_tracing=False)
sender = client.create_producer()
async with sender:
await sender.send(batched())
time.sleep(1)
received = []
for r in receivers:
received.extend(r.receive(timeout=3))
assert len(received) == 20
for index, message in enumerate(received):
assert list(
message.body)[0] == "Event number {}".format(index).encode('utf-8')
assert (app_prop_key.encode('utf-8') in message.application_properties) \
and (dict(message.application_properties)[app_prop_key.encode('utf-8')] == app_prop_value.encode('utf-8'))
async def test_receive_with_datetime_async(connstr_senders):
connection_str, senders = connstr_senders
client = EventHubClient.from_connection_string(connection_str,
network_tracing=False)
receiver = client.create_consumer(consumer_group="$default",
partition_id="0",
event_position=EventPosition('@latest'))
async with receiver:
received = await receiver.receive(timeout=5)
assert len(received) == 0
senders[0].send(EventData(b"Data"))
received = await receiver.receive(timeout=5)
assert len(received) == 1
offset = received[0].enqueued_time
offset_receiver = client.create_consumer(
consumer_group="$default",
partition_id="0",
event_position=EventPosition(offset))
async with offset_receiver:
received = await offset_receiver.receive(timeout=5)
assert len(received) == 0
senders[0].send(EventData(b"Message after timestamp"))
time.sleep(1)
received = await offset_receiver.receive(timeout=5)
assert len(received) == 1
async def test_receive_with_invalid_policy_async(invalid_policy):
client = EventHubClient.from_connection_string(invalid_policy,
network_tracing=False)
sender = client.create_consumer(consumer_group="$default",
partition_id="0",
event_position=EventPosition("-1"))
with pytest.raises(AuthenticationError):
await sender._open()
async def test_send_to_invalid_partitions_async(connection_str):
partitions = ["XYZ", "-1", "1000", "-"]
for p in partitions:
client = EventHubClient.from_connection_string(connection_str,
network_tracing=False)
sender = client.create_producer(partition_id=p)
with pytest.raises(ConnectError):
await sender.send(EventData("test data"))
await sender.close()
async def test_receive_with_invalid_key_async(invalid_key):
client = EventHubClient.from_connection_string(invalid_key,
network_tracing=False)
receiver = client.create_consumer(consumer_group="$default",
partition_id="0",
event_position=EventPosition("-1"))
with pytest.raises(AuthenticationError):
await receiver.receive(timeout=3)
await receiver.close()
async def test_send_with_invalid_policy_async(invalid_policy,
connstr_receivers):
_, receivers = connstr_receivers
client = EventHubClient.from_connection_string(invalid_policy,
network_tracing=False)
sender = client.create_producer()
with pytest.raises(AuthenticationError):
await sender.send(EventData("test data"))
await sender.close()
async def test_non_existing_entity_receiver_async(connection_str):
client = EventHubClient.from_connection_string(connection_str,
event_hub_path="nemo",
network_tracing=False)
receiver = client.create_consumer(consumer_group="$default",
partition_id="0",
event_position=EventPosition("-1"))
with pytest.raises(AuthenticationError):
await receiver._open()
async def test_send_null_body_async(connection_str):
client = EventHubClient.from_connection_string(connection_str,
network_tracing=False)
sender = client.create_producer()
try:
with pytest.raises(ValueError):
data = EventData(None)
await sender.send(data)
finally:
await sender.close()
def __enter__(self):
self._client = EventHubClient.from_connection_string(
self._eh_connection_string)
self._storage_client = ContainerClient.from_connection_string(
conn_str=self._sa_connection_string,
container=self._sa_container_name)
self._partition_manager = BlobPartitionManager(self._storage_client)
self._event_processor = EventProcessor(self._client, "$default",
self._partition_processor,
self._partition_manager)
async def test_create_batch_with_invalid_hostname_async(invalid_hostname):
client = EventHubClient.from_connection_string(invalid_hostname,
network_tracing=False)
sender = client.create_producer()
try:
with pytest.raises(AuthenticationError):
batch_event_data = await sender.create_batch(max_size=300,
partition_key="key")
finally:
await sender.close()
async def test_partition_processor(connstr_senders):
partition_processor1 = None
partition_processor2 = None
class TestPartitionProcessor(PartitionProcessor):
def __init__(self):
self.initialize_called = False
self.error = None
self.close_reason = None
self.received_events = []
self.checkpoint = None
async def initialize(self, partition_context):
nonlocal partition_processor1, partition_processor2
if partition_context.partition_id == "1":
partition_processor1 = self
else:
partition_processor2 = self
async def process_events(self, events, partition_context):
self.received_events.extend(events)
if events:
offset, sn = events[-1].offset, events[-1].sequence_number
await partition_context.update_checkpoint(offset, sn)
self.checkpoint = (offset, sn)
async def process_error(self, error, partition_context):
self.error = error
assert partition_context is not None
async def close(self, reason, partition_context):
self.close_reason = reason
assert partition_context is not None
connection_str, senders = connstr_senders
for sender in senders:
sender.send(EventData("EventProcessor Test"))
eventhub_client = EventHubClient.from_connection_string(connection_str,
receive_timeout=3)
partition_manager = SamplePartitionManager()
event_processor = EventProcessor(eventhub_client,
"$default",
TestPartitionProcessor,
partition_manager,
polling_interval=1)
asyncio.ensure_future(event_processor.start())
await asyncio.sleep(10)
await event_processor.stop()
assert partition_processor1 is not None and partition_processor2 is not None
assert len(partition_processor1.received_events) == 1 and len(
partition_processor2.received_events) == 1
assert partition_processor1.checkpoint is not None
assert partition_processor1.close_reason == CloseReason.SHUTDOWN
assert partition_processor1.error is None
async def test_create_batch_with_too_large_size_async(connection_str):
client = EventHubClient.from_connection_string(connection_str,
network_tracing=False)
sender = client.create_producer()
try:
with pytest.raises(ValueError):
batch_event_data = await sender.create_batch(max_size=5 * 1024 *
1024,
partition_key="key")
finally:
await sender.close()
