async def test_example_eventhub_async_sender_ops(live_eventhub_config, connection_str):
import os
# [START create_eventhub_client_async_sender_instance]
from azure.eventhub import EventHubClientAsync
client = EventHubClientAsync.from_connection_string(connection_str)
sender = client.add_async_sender(partition="0")
# [END create_eventhub_client_async_sender_instance]
# [START eventhub_client_async_sender_open]
client = EventHubClientAsync.from_connection_string(connection_str)
sender = client.add_async_sender(partition="0")
try:
# Open the Async Sender using the supplied conneciton.
await sender.open_async()
# Start sending
except:
raise
finally:
# Close down the send handler.
await sender.close_async()
# [END eventhub_client_async_sender_open]
# [START eventhub_client_async_sender_close]
client = EventHubClientAsync.from_connection_string(connection_str)
sender = client.add_async_sender(partition="0")
try:
# Open the Async Sender using the supplied conneciton.
await sender.open_async()
# Start sending
except:
raise
finally:
# Close down the send handler.
await sender.close_async()
def test_long_running_partition_send_async(connection_str):
parser = argparse.ArgumentParser()
parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30)
parser.add_argument("--payload", help="payload size", type=int, default=1024)
parser.add_argument("--batch", help="Number of events to send and wait", type=int, default=200)
parser.add_argument("--partitions", help="Comma seperated partition IDs")
parser.add_argument("--conn-str", help="EventHub connection string", default=connection_str)
parser.add_argument("--eventhub", help="Name of EventHub")
parser.add_argument("--address", help="Address URI to the EventHub entity")
parser.add_argument("--sas-policy", help="Name of the shared access policy to authenticate with")
parser.add_argument("--sas-key", help="Shared access key")
parser.add_argument("--logger-name", help="Unique log file ID")
loop = asyncio.get_event_loop()
args, _ = parser.parse_known_args()
if args.conn_str:
client = EventHubClientAsync.from_connection_string(
args.conn_str,
eventhub=args.eventhub, debug=True)
elif args.address:
client = EventHubClientAsync(
args.address,
username=args.sas_policy,
password=args.sas_key,
auth_timeout=500)
else:
try:
import pytest
pytest.skip("Must specify either '--conn-str' or '--address'")
except ImportError:
raise ValueError("Must specify either '--conn-str' or '--address'")
try:
if not args.partitions:
partitions = loop.run_until_complete(get_partitions(client))
else:
pid_range = args.partitions.split("-")
if len(pid_range) > 1:
partitions = [str(i) for i in range(int(pid_range[0]), int(pid_range[1]) + 1)]
else:
partitions = args.partitions.split(",")
pumps = []
for pid in partitions:
sender = client.add_async_sender(partition=pid, send_timeout=0, keep_alive=False)
pumps.append(pump(pid, sender, args, args.duration))
loop.run_until_complete(client.run_async())
results = loop.run_until_complete(asyncio.gather(*pumps, return_exceptions=True))
assert not results
except Exception as e:
logger.error("Sender failed: {}".format(e))
finally:
logger.info("Shutting down sender")
loop.run_until_complete(client.stop_async())
async def test_send_with_partition_key_async(connstr_receivers):
connection_str, receivers = connstr_receivers
client = EventHubClientAsync.from_connection_string(connection_str, debug=False)
sender = client.add_async_sender()
await client.run_async()
data_val = 0
for partition in [b"a", b"b", b"c", b"d", b"e", b"f"]:
partition_key = b"test_partition_" + partition
for i in range(50):
data = EventData(str(data_val))
data.partition_key = partition_key
data_val += 1
await sender.send(data)
await client.stop_async()
found_partition_keys = {}
for index, partition in enumerate(receivers):
received = partition.receive(timeout=5)
for message in received:
try:
existing = found_partition_keys[message.partition_key]
assert existing == index
except KeyError:
found_partition_keys[message.partition_key] = index
async def get_partitions(iot_connection_str):
try:
client = EventHubClientAsync.from_iothub_connection_string(iot_connection_str, debug=True)
client.add_async_receiver("$default", "0", prefetch=1000, operation='/messages/events')
await client.run_async()
partitions = await client.get_eventhub_info_async()
return partitions["partition_ids"]
finally:
await client.stop_async()
async def test_send_to_invalid_partitions_async(connection_str):
partitions = ["XYZ", "-1", "1000", "-" ]
for p in partitions:
client = EventHubClientAsync.from_connection_string(connection_str, debug=False)
sender = client.add_async_sender(partition=p)
try:
with pytest.raises(EventHubError):
await client.run_async()
finally:
await client.stop_async()
async def test_send_null_body_async(connection_str):
client = EventHubClientAsync.from_connection_string(connection_str, debug=False)
sender = client.add_async_sender()
try:
await client.run_async()
with pytest.raises(ValueError):
data = EventData(None)
await sender.send(data)
finally:
await client.stop_async()
async def test_send_partition_key_with_partition_async(connection_str):
client = EventHubClientAsync.from_connection_string(connection_str, debug=True)
sender = client.add_async_sender(partition="1")
try:
await client.run_async()
data = EventData(b"Data")
data.partition_key = b"PKey"
with pytest.raises(ValueError):
await sender.send(data)
finally:
await client.stop_async()
async def test_receive_from_invalid_partitions_async(connection_str):
partitions = ["XYZ", "-1", "1000", "-" ]
for p in partitions:
client = EventHubClientAsync.from_connection_string(connection_str, debug=True)
receiver = client.add_async_receiver("$default", p)
try:
with pytest.raises(EventHubError):
await client.run_async()
await receiver.receive(timeout=10)
finally:
await client.stop_async()
async def test_send_too_large_message_async(connection_str):
if sys.platform.startswith('darwin'):
pytest.skip("Skipping on OSX - open issue regarding message size")
client = EventHubClientAsync.from_connection_string(connection_str, debug=False)
sender = client.add_async_sender()
try:
await client.run_async()
data = EventData(b"A" * 300000)
with pytest.raises(EventHubError):
await sender.send(data)
finally:
await client.stop_async()
async def test_iothub_receive_multiple_async(iot_connection_str):
partitions = await get_partitions(iot_connection_str)
client = EventHubClientAsync.from_iothub_connection_string(iot_connection_str, debug=True)
try:
receivers = []
for p in partitions:
receivers.append(client.add_async_receiver("$default", p, prefetch=10, operation='/messages/events'))
await client.run_async()
outputs = await asyncio.gather(*[pump(r) for r in receivers])
assert isinstance(outputs[0], int) and outputs[0] <= 10
assert isinstance(outputs[1], int) and outputs[1] <= 10
finally:
await client.stop_async()
async def test_example_eventhub_async_receiver_ops(live_eventhub_config, connection_str):
import os
# [START create_eventhub_client_async_receiver_instance]
from azure.eventhub import EventHubClientAsync, Offset
client = EventHubClientAsync.from_connection_string(connection_str)
receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest'))
# [END create_eventhub_client_async_receiver_instance]
# [START eventhub_client_async_receiver_open]
client = EventHubClientAsync.from_connection_string(connection_str)
receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest'))
try:
# Open the Async Receiver using the supplied conneciton.
await receiver.open_async()
# Start receiving
except:
raise
finally:
# Close down the receive handler.
await receiver.close_async()
# [END eventhub_client_async_receiver_open]
# [START eventhub_client_async_receiver_close]
client = EventHubClientAsync.from_connection_string(connection_str)
receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest'))
try:
# Open the Async Receiver using the supplied conneciton.
await receiver.open_async()
# Start receiving
except:
raise
finally:
# Close down the receive handler.
await receiver.close_async()
# [END eventhub_client_async_receiver_close]
async def test_send_partition_async(connstr_receivers):
connection_str, receivers = connstr_receivers
client = EventHubClientAsync.from_connection_string(connection_str, debug=False)
sender = client.add_async_sender(partition="1")
try:
await client.run_async()
await sender.send(EventData(b"Data"))
except:
raise
finally:
await client.stop_async()
partition_0 = receivers[0].receive(timeout=2)
assert len(partition_0) == 0
partition_1 = receivers[1].receive(timeout=2)
assert len(partition_1) == 1
async def test_send_non_ascii_async(connstr_receivers):
connection_str, receivers = connstr_receivers
client = EventHubClientAsync.from_connection_string(connection_str, debug=False)
sender = client.add_async_sender(partition="0")
try:
await client.run_async()
await sender.send(EventData("é,è,à,ù,â,ê,î,ô,û"))
await sender.send(EventData(json.dumps({"foo": "漢字"})))
except:
raise
finally:
await client.stop_async()
partition_0 = receivers[0].receive(timeout=2)
assert len(partition_0) == 2
assert partition_0[0].body_as_str() == "é,è,à,ù,â,ê,î,ô,û"
assert partition_0[1].body_as_json() == {"foo": "漢字"}
async def test_send_with_long_interval_async(connstr_receivers):
connection_str, receivers = connstr_receivers
client = EventHubClientAsync.from_connection_string(connection_str, debug=True)
sender = client.add_async_sender()
try:
await client.run_async()
await sender.send(EventData(b"A single event"))
for _ in range(2):
await asyncio.sleep(300)
await sender.send(EventData(b"A single event"))
finally:
await client.stop_async()
received = []
for r in receivers:
received.extend(r.receive(timeout=1))
assert len(received) == 3
assert list(received[0].body)[0] == b"A single event"
async def test_send_single_event_async(connstr_receivers):
connection_str, receivers = connstr_receivers
client = EventHubClientAsync.from_connection_string(connection_str, debug=False)
sender = client.add_async_sender()
try:
await client.run_async()
await sender.send(EventData(b"A single event"))
except:
raise
finally:
await client.stop_async()
received = []
for r in receivers:
received.extend(r.receive(timeout=1))
assert len(received) == 1
assert list(received[0].body)[0] == b"A single event"
async def test_send_partition_batch_async(connection_str, receivers):
def batched():
for i in range(10):
yield "Event number {}".format(i)
client = EventHubClientAsync.from_connection_string(connection_str,
debug=False)
sender = client.add_async_sender(partition="1")
try:
await client.run_async()
await sender.send(EventData(batch=batched()))
except:
raise
finally:
await client.stop_async()
partition_0 = receivers[0].receive(timeout=2)
assert len(partition_0) == 0
partition_1 = receivers[1].receive(timeout=2)
assert len(partition_1) == 10
async def test_send_partition_batch_async(connstr_receivers):
connection_str, receivers = connstr_receivers
def batched():
for i in range(10):
yield "Event number {}".format(i)
client = EventHubClientAsync.from_connection_string(connection_str, debug=False)
sender = client.add_async_sender(partition="1")
try:
await client.run_async()
await sender.send(EventData(batch=batched()))
except:
raise
finally:
await client.stop_async()
partition_0 = receivers[0].receive(timeout=2)
assert len(partition_0) == 0
partition_1 = receivers[1].receive(timeout=2)
assert len(partition_1) == 10
async def open_clients_async(self):
"""
Responsible for establishing connection to event hub client
throws EventHubsException, IOException, InterruptedException, ExecutionException.
"""
await self.partition_context.get_initial_offset_async()
# Create event hub client and receive handler and set options
self.eh_client = EventHubClientAsync(
self.host.eh_config.client_address,
debug=self.host.eph_options.debug_trace,
http_proxy=self.host.eph_options.http_proxy)
self.partition_receive_handler = self.eh_client.add_async_receiver(
self.partition_context.consumer_group_name,
self.partition_context.partition_id,
Offset(self.partition_context.offset),
prefetch=self.host.eph_options.prefetch_count,
keep_alive=self.host.eph_options.keep_alive_interval,
auto_reconnect=self.host.eph_options.auto_reconnect_on_error,
loop=self.loop)
self.partition_receiver = PartitionReceiver(self)
async def get_partition_ids_async(self):
"""
Returns a list of all the event hub partition IDs.
:rtype: list[str]
"""
if not self.partition_ids:
try:
eh_client = EventHubClientAsync(
self.host.eh_config.client_address,
debug=self.host.eph_options.debug_trace,
http_proxy=self.host.eph_options.http_proxy)
try:
eh_info = await eh_client.get_eventhub_info_async()
self.partition_ids = eh_info['partition_ids']
except Exception as err: # pylint: disable=broad-except
raise Exception("Failed to get partition ids", repr(err))
finally:
await eh_client.stop_async()
return self.partition_ids
def test_long_running_receive_async(connection_str):
parser = argparse.ArgumentParser()
parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30)
parser.add_argument("--consumer", help="Consumer group name", default="$default")
parser.add_argument("--partitions", help="Comma seperated partition IDs")
parser.add_argument("--offset", help="Starting offset", default="-1")
parser.add_argument("--conn-str", help="EventHub connection string", default=connection_str)
parser.add_argument("--eventhub", help="Name of EventHub")
parser.add_argument("--address", help="Address URI to the EventHub entity")
parser.add_argument("--sas-policy", help="Name of the shared access policy to authenticate with")
parser.add_argument("--sas-key", help="Shared access key")
loop = asyncio.get_event_loop()
args, _ = parser.parse_known_args()
if args.conn_str:
client = EventHubClientAsync.from_connection_string(
args.conn_str,
eventhub=args.eventhub, auth_timeout=240, debug=False)
elif args.address:
client = EventHubClientAsync(
args.address,
auth_timeout=240,
username=args.sas_policy,
password=args.sas_key)
else:
try:
import pytest
pytest.skip("Must specify either '--conn-str' or '--address'")
except ImportError:
raise ValueError("Must specify either '--conn-str' or '--address'")
try:
if not args.partitions:
partitions = loop.run_until_complete(get_partitions(client))
else:
partitions = args.partitions.split(",")
pumps = []
for pid in partitions:
receiver = client.add_async_receiver(
consumer_group=args.consumer,
partition=pid,
offset=Offset(args.offset),
prefetch=50)
pumps.append(pump(pid, receiver, args, args.duration))
loop.run_until_complete(client.run_async())
loop.run_until_complete(asyncio.gather(*pumps))
finally:
loop.run_until_complete(client.stop_async())
async def test_non_epoch_receiver_after_epoch_receiver_async(connstr_senders):
connection_str, senders = connstr_senders
senders[0].send(EventData(b"Receiving only a single event"))
client = EventHubClientAsync.from_connection_string(connection_str,
debug=False)
receivers = []
receivers.append(
client.add_async_epoch_receiver("$default", "0", 15, prefetch=10))
receivers.append(client.add_async_receiver("$default", "0", prefetch=10))
try:
await client.run_async()
outputs = await asyncio.gather(pump(receivers[0]),
pump(receivers[1]),
return_exceptions=True)
assert isinstance(outputs[1], EventHubError)
assert isinstance(outputs[0], int) and outputs[0] == 1
except:
raise
finally:
await client.stop_async()
async def test_max_receivers_async(connection_str, senders):
client = EventHubClientAsync.from_connection_string(connection_str, debug=True)
receivers = []
for i in range(6):
receivers.append(client.add_async_receiver("$default", "0", prefetch=1000, offset=Offset('@latest')))
try:
await client.run_async()
outputs = await asyncio.gather(
pump(receivers[0]),
pump(receivers[1]),
pump(receivers[2]),
pump(receivers[3]),
pump(receivers[4]),
pump(receivers[5]),
return_exceptions=True)
print(outputs)
failed = [o for o in outputs if isinstance(o, EventHubError)]
assert len(failed) == 1
print(failed[0].message)
finally:
await client.stop_async()
async def test_receive_with_inclusive_offset_async(connection_str, senders):
client = EventHubClientAsync.from_connection_string(connection_str, debug=False)
receiver = client.add_async_receiver("$default", "0", offset=Offset('@latest'))
await client.run_async()
try:
received = await receiver.receive(timeout=5)
assert len(received) == 0
senders[0].send(EventData(b"Data"))
time.sleep(1)
received = await receiver.receive(timeout=5)
assert len(received) == 1
offset = received[0].offset
offset_receiver = client.add_async_receiver("$default", "0", offset=Offset(offset, inclusive=True))
await client.run_async()
received = await offset_receiver.receive(timeout=5)
assert len(received) == 1
except:
raise
finally:
await client.stop_async()
async def test_send_batch_with_app_prop_async(connstr_receivers):
pytest.skip("Waiting on uAMQP release")
connection_str, receivers = connstr_receivers
def batched():
for i in range(10):
yield "Event number {}".format(i)
for i in range(10, 20):
yield EventData("Event number {}".format(i))
client = EventHubClientAsync.from_connection_string(connection_str,
debug=False)
sender = client.add_async_sender()
try:
await client.run_async()
app_prop_key = "raw_prop"
app_prop_value = "raw_value"
batch_app_prop = {app_prop_key: app_prop_value}
batch_event = EventData(batch=batched())
batch_event.application_properties = batch_app_prop
await sender.send(batch_event)
except:
raise
finally:
await client.stop_async()
time.sleep(1)
received = []
for r in receivers:
received.extend(r.receive(timeout=3))
assert len(received) == 20
for index, message in enumerate(received):
assert list(
message.body)[0] == "Event number {}".format(index).encode('utf-8')
assert (app_prop_key.encode('utf-8') in message.application_properties) \
and (dict(message.application_properties)[app_prop_key.encode('utf-8')] == app_prop_value.encode('utf-8'))
async def test_epoch_receiver_after_non_epoch_receiver_async(connection_str, senders):
client = EventHubClientAsync.from_connection_string(connection_str, debug=False)
receivers = []
receivers.append(client.add_async_receiver("$default", "0", prefetch=1000))
receivers.append(client.add_async_epoch_receiver("$default", "0", 15, prefetch=1000))
await client.run_async()
try:
outputs = await asyncio.gather(
pump(receivers[0]),
pump(receivers[1], sleep=5),
return_exceptions=True)
# Depending on how many messages are present and how long the test
# runs, one receiver may not throw and error - in which case it should
# still not have received any messages.
print(outputs)
assert isinstance(outputs[0], EventHubError) or outputs[0] == 0
assert isinstance(outputs[1], int) and outputs[1] >= 1
except:
raise
finally:
await client.stop_async()
async def test_max_receivers_async(connstr_senders):
connection_str, senders = connstr_senders
client = EventHubClientAsync.from_connection_string(connection_str, debug=True)
receivers = []
for i in range(6):
receivers.append(client.add_async_receiver("$default", "0", prefetch=1000, offset=Offset('@latest')))
try:
await client.run_async()
outputs = await asyncio.gather(
pump(receivers[0]),
pump(receivers[1]),
pump(receivers[2]),
pump(receivers[3]),
pump(receivers[4]),
pump(receivers[5]),
return_exceptions=True)
print(outputs)
failed = [o for o in outputs if isinstance(o, EventHubError)]
assert len(failed) == 1
print(failed[0].message)
finally:
await client.stop_async()
async def test_send_batch_with_app_prop_async(connstr_receivers):
pytest.skip("Waiting on uAMQP release")
connection_str, receivers = connstr_receivers
def batched():
for i in range(10):
yield "Event number {}".format(i)
for i in range(10, 20):
yield EventData("Event number {}".format(i))
client = EventHubClientAsync.from_connection_string(connection_str, debug=False)
sender = client.add_async_sender()
try:
await client.run_async()
app_prop_key = "raw_prop"
app_prop_value = "raw_value"
batch_app_prop = {app_prop_key:app_prop_value}
batch_event = EventData(batch=batched())
batch_event.application_properties = batch_app_prop
await sender.send(batch_event)
except:
raise
finally:
await client.stop_async()
time.sleep(1)
received = []
for r in receivers:
received.extend(r.receive(timeout=3))
assert len(received) == 20
for index, message in enumerate(received):
assert list(message.body)[0] == "Event number {}".format(index).encode('utf-8')
assert (app_prop_key.encode('utf-8') in message.application_properties) \
and (dict(message.application_properties)[app_prop_key.encode('utf-8')] == app_prop_value.encode('utf-8'))
async def test_send_with_forced_conn_close_async(connstr_receivers):
connection_str, receivers = connstr_receivers
client = EventHubClientAsync.from_connection_string(connection_str, debug=True)
sender = client.add_async_sender()
try:
await client.run_async()
await sender.send(EventData(b"A single event"))
sender._handler._message_sender.destroy()
await asyncio.sleep(300)
await sender.send(EventData(b"A single event"))
await sender.send(EventData(b"A single event"))
sender._handler._message_sender.destroy()
await asyncio.sleep(300)
await sender.send(EventData(b"A single event"))
await sender.send(EventData(b"A single event"))
finally:
await client.stop_async()
received = []
for r in receivers:
received.extend(pump(r))
assert len(received) == 5
assert list(received[0].body)[0] == b"A single event"
async def test_send_batch_async(connstr_receivers):
connection_str, receivers = connstr_receivers
def batched():
for i in range(10):
yield "Event number {}".format(i)
client = EventHubClientAsync.from_connection_string(connection_str, debug=False)
sender = client.add_async_sender()
try:
await client.run_async()
await sender.send(EventData(batch=batched()))
except:
raise
finally:
await client.stop_async()
time.sleep(1)
received = []
for r in receivers:
received.extend(r.receive(timeout=3))
assert len(received) == 10
for index, message in enumerate(received):
assert list(message.body)[0] == "Event number {}".format(index).encode('utf-8')
async def test_non_existing_entity_sender_async(connection_str):
client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False)
sender = client.add_async_sender(partition="1")
with pytest.raises(EventHubError):
await client.run_async()
async def pump(client, partition):
receiver = client.add_async_receiver(CONSUMER_GROUP, partition, OFFSET, prefetch=5)
await client.run_async()
total = 0
start_time = time.time()
for event_data in await receiver.receive(timeout=10):
last_offset = event_data.offset
last_sn = event_data.sequence_number
print("Received: {}, {}".format(last_offset.value, last_sn))
total += 1
end_time = time.time()
run_time = end_time - start_time
print("Received {} messages in {} seconds".format(total, run_time))
try:
if not ADDRESS:
raise ValueError("No EventHubs URL supplied.")
loop = asyncio.get_event_loop()
client = EventHubClientAsync(ADDRESS, debug=False, username=USER, password=KEY)
tasks = [
asyncio.ensure_future(pump(client, "0")),
asyncio.ensure_future(pump(client, "1"))]
loop.run_until_complete(asyncio.wait(tasks))
loop.run_until_complete(client.stop_async())
loop.close()
except KeyboardInterrupt:
pass
start_time = time.time()
for event_data in await receiver.receive(timeout=10):
last_offset = event_data.offset
last_sn = event_data.sequence_number
print("Received: {}, {}".format(last_offset, last_sn))
total += 1
end_time = time.time()
run_time = end_time - start_time
print("Received {} messages in {} seconds".format(total, run_time))
try:
if not ADDRESS:
raise ValueError("No EventHubs URL supplied.")
loop = asyncio.get_event_loop()
client = EventHubClientAsync(ADDRESS,
debug=False,
username=USER,
password=KEY)
tasks = [
asyncio.ensure_future(pump(client, "0")),
asyncio.ensure_future(pump(client, "1"))
]
loop.run_until_complete(asyncio.wait(tasks))
loop.run_until_complete(client.stop_async())
loop.close()
except KeyboardInterrupt:
pass
async def test_non_existing_entity_receiver_async(connection_str):
client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False)
receiver = client.add_async_receiver("$default", "0")
with pytest.raises(EventHubError):
await client.run_async()
class EventHubPartitionPump(PartitionPump):
"""
Pulls and messages from lease partition from eventhub and sends them to processor
"""
def __init__(self, host, lease):
PartitionPump.__init__(self, host, lease)
self.eh_client = None
self.partition_receiver = None
self.partition_receive_handler = None
self.running = None
async def on_open_async(self):
"""
Eventhub Override for on_open_async
"""
_opened_ok = False
_retry_count = 0
while (not _opened_ok) and (_retry_count < 5):
try:
await self.open_clients_async()
_opened_ok = True
except Exception as err: # pylint: disable=broad-except
_logger.warning(
"{},{} PartitionPumpWarning: Failure creating client or receiver, "
"retrying: {!r}".format(
self.host.guid, self.partition_context.partition_id,
err))
last_exception = err
_retry_count += 1
if not _opened_ok:
await self.processor.process_error_async(self.partition_context,
last_exception)
self.set_pump_status("OpenFailed")
if self.pump_status == "Opening":
loop = asyncio.get_event_loop()
self.set_pump_status("Running")
await self.eh_client.run_async()
self.running = loop.create_task(self.partition_receiver.run())
if self.pump_status in ["OpenFailed", "Errored"]:
self.set_pump_status("Closing")
await self.clean_up_clients_async()
self.set_pump_status("Closed")
async def open_clients_async(self):
"""
Responsible for establishing connection to event hub client
throws EventHubsException, IOException, InterruptedException, ExecutionException
"""
await self.partition_context.get_initial_offset_async()
# Create event hub client and receive handler and set options
self.eh_client = EventHubClientAsync(
self.host.eh_config.client_address,
debug=self.host.eph_options.debug_trace)
self.partition_receive_handler = self.eh_client.add_async_receiver(
self.partition_context.consumer_group_name,
self.partition_context.partition_id,
Offset(self.partition_context.offset),
prefetch=self.host.eph_options.prefetch_count,
loop=self.loop)
self.partition_receiver = PartitionReceiver(self)
async def clean_up_clients_async(self):
"""
Resets the pump swallows all exceptions
"""
if self.partition_receiver:
if self.eh_client:
await self.eh_client.stop_async()
self.partition_receiver = None
self.partition_receive_handler = None
self.eh_client = None
async def on_closing_async(self, reason):
"""
Overides partition pump on cleasing
:param reason: The reason for the shutdown.
:type reason: str
"""
self.partition_receiver.eh_partition_pump.set_pump_status("Errored")
await self.running
await self.clean_up_clients_async()
async def test_example_eventhub_async_send_and_receive(live_eventhub_config):
# [START create_eventhub_client_async]
from azure.eventhub import EventHubClientAsync
import os
connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format(
os.environ['EVENT_HUB_HOSTNAME'], os.environ['EVENT_HUB_SAS_POLICY'],
os.environ['EVENT_HUB_SAS_KEY'], os.environ['EVENT_HUB_NAME'])
client = EventHubClientAsync.from_connection_string(connection_str)
# [END create_eventhub_client_async]
from azure.eventhub import EventData, Offset
# [START create_eventhub_client_async_sender]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add a async sender to the async client object.
sender = client.add_async_sender(partition="0")
# [END create_eventhub_client_async_sender]
# [START create_eventhub_client_async_receiver]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add a async receiver to the async client object.
receiver = client.add_async_receiver(consumer_group="$default",
partition="0",
offset=Offset('@latest'))
# [END create_eventhub_client_async_receiver]
# [START create_eventhub_client_async_epoch_receiver]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add a async receiver to the async client object.
epoch_receiver = client.add_async_epoch_receiver(consumer_group="$default",
partition="0",
epoch=42)
# [END create_eventhub_client_async_epoch_receiver]
# [START eventhub_client_run_async]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add AsyncSenders/AsyncReceivers
try:
# Opens the connection and starts running all AsyncSender/AsyncReceiver clients.
await client.run_async()
# Start sending and receiving
except:
raise
finally:
await client.stop_async()
# [END eventhub_client_run_async]
client = EventHubClientAsync.from_connection_string(connection_str)
sender = client.add_async_sender(partition="0")
receiver = client.add_async_receiver(consumer_group="$default",
partition="0",
offset=Offset('@latest'))
try:
# Opens the connection and starts running all AsyncSender/AsyncReceiver clients.
await client.run_async()
# [START eventhub_client_async_send]
event_data = EventData(b"A single event")
await sender.send(event_data)
# [END eventhub_client_async_send]
time.sleep(1)
# [START eventhub_client_async_receive]
logger = logging.getLogger("azure.eventhub")
received = await receiver.receive(timeout=5)
for event_data in received:
logger.info("Message received:{}".format(event_data.body_as_str()))
# [END eventhub_client_async_receive]
assert len(received) == 1
assert received[0].body_as_str() == "A single event"
assert list(received[-1].body)[0] == b"A single event"
except:
raise
finally:
await client.stop_async()
# [START eventhub_client_async_stop]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add AsyncSenders/AsyncReceivers
try:
# Opens the connection and starts running all AsyncSender/AsyncReceiver clients.
await client.run_async()
# Start sending and receiving
except:
raise
finally:
await client.stop_async()
async def test_send_with_invalid_policy_async(invalid_policy, receivers):
client = EventHubClientAsync.from_connection_string(invalid_policy, debug=False)
sender = client.add_async_sender()
with pytest.raises(EventHubError):
await client.run_async()
async def test_non_existing_entity_receiver_async(connection_str):
client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False)
receiver = client.add_async_receiver("$default", "0")
with pytest.raises(EventHubError):
await client.run_async()
async def test_non_existing_entity_sender_async(connection_str):
client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False)
sender = client.add_async_sender(partition="1")
with pytest.raises(EventHubError):
await client.run_async()
async def test_send_with_invalid_policy_async(invalid_policy, connstr_receivers):
_, receivers = connstr_receivers
client = EventHubClientAsync.from_connection_string(invalid_policy, debug=False)
sender = client.add_async_sender()
with pytest.raises(EventHubError):
await client.run_async()
async def test_example_eventhub_async_send_and_receive(live_eventhub_config):
# [START create_eventhub_client_async]
from azure.eventhub import EventHubClientAsync
import os
connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format(
os.environ['EVENT_HUB_HOSTNAME'],
os.environ['EVENT_HUB_SAS_POLICY'],
os.environ['EVENT_HUB_SAS_KEY'],
os.environ['EVENT_HUB_NAME'])
client = EventHubClientAsync.from_connection_string(connection_str)
# [END create_eventhub_client_async]
from azure.eventhub import EventData, Offset
# [START create_eventhub_client_async_sender]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add a async sender to the async client object.
sender = client.add_async_sender(partition="0")
# [END create_eventhub_client_async_sender]
# [START create_eventhub_client_async_receiver]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add a async receiver to the async client object.
receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest'))
# [END create_eventhub_client_async_receiver]
# [START create_eventhub_client_async_epoch_receiver]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add a async receiver to the async client object.
epoch_receiver = client.add_async_epoch_receiver(consumer_group="$default", partition="0", epoch=42)
# [END create_eventhub_client_async_epoch_receiver]
# [START eventhub_client_run_async]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add AsyncSenders/AsyncReceivers
try:
# Opens the connection and starts running all AsyncSender/AsyncReceiver clients.
await client.run_async()
# Start sending and receiving
except:
raise
finally:
await client.stop_async()
# [END eventhub_client_run_async]
client = EventHubClientAsync.from_connection_string(connection_str)
sender = client.add_async_sender(partition="0")
receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest'))
try:
# Opens the connection and starts running all AsyncSender/AsyncReceiver clients.
await client.run_async()
# [START eventhub_client_async_send]
event_data = EventData(b"A single event")
await sender.send(event_data)
# [END eventhub_client_async_send]
time.sleep(1)
# [START eventhub_client_async_receive]
logger = logging.getLogger("azure.eventhub")
received = await receiver.receive(timeout=5)
for event_data in received:
logger.info("Message received:{}".format(event_data.body_as_str()))
# [END eventhub_client_async_receive]
assert len(received) == 1
assert received[0].body_as_str() == "A single event"
assert list(received[-1].body)[0] == b"A single event"
except:
raise
finally:
await client.stop_async()
# [START eventhub_client_async_stop]
client = EventHubClientAsync.from_connection_string(connection_str)
# Add AsyncSenders/AsyncReceivers
try:
# Opens the connection and starts running all AsyncSender/AsyncReceiver clients.
await client.run_async()
# Start sending and receiving
except:
raise
finally:
await client.stop_async()
def test_long_running_eph(live_eventhub):
parser = argparse.ArgumentParser()
parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30)
parser.add_argument("--storage-account", help="Storage account name", default=os.environ.get('AZURE_STORAGE_ACCOUNT'))
parser.add_argument("--storage-key", help="Storage account access key", default=os.environ.get('AZURE_STORAGE_ACCESS_KEY'))
parser.add_argument("--container", help="Lease container name", default="nocontextleases")
parser.add_argument("--eventhub", help="Name of EventHub", default=live_eventhub['event_hub'])
parser.add_argument("--namespace", help="Namespace of EventHub", default=live_eventhub['namespace'])
parser.add_argument("--suffix", help="Namespace of EventHub", default="servicebus.windows.net")
parser.add_argument("--sas-policy", help="Name of the shared access policy to authenticate with", default=live_eventhub['key_name'])
parser.add_argument("--sas-key", help="Shared access key", default=live_eventhub['access_key'])
loop = asyncio.get_event_loop()
args, _ = parser.parse_known_args()
if not args.namespace or not args.eventhub:
try:
import pytest
pytest.skip("Must specify '--namespace' and '--eventhub'")
except ImportError:
raise ValueError("Must specify '--namespace' and '--eventhub'")
# Queue up some events in the Eventhub
conn_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format(
live_eventhub['hostname'],
live_eventhub['key_name'],
live_eventhub['access_key'],
live_eventhub['event_hub'])
send_client = EventHubClientAsync.from_connection_string(conn_str)
pumps = []
for pid in ["0", "1"]:
sender = send_client.add_async_sender(partition=pid, send_timeout=0, keep_alive=False)
pumps.append(pump(pid, sender, 15))
loop.run_until_complete(send_client.run_async())
results = loop.run_until_complete(asyncio.gather(*pumps, return_exceptions=True))
loop.run_until_complete(send_client.stop_async())
assert not any(results)
# Eventhub config and storage manager
eh_config = EventHubConfig(
args.namespace,
args.eventhub,
args.sas_policy,
args.sas_key,
consumer_group="$default",
namespace_suffix=args.suffix)
eh_options = EPHOptions()
eh_options.release_pump_on_timeout = True
eh_options.debug_trace = False
eh_options.receive_timeout = 120
storage_manager = AzureStorageCheckpointLeaseManager(
storage_account_name=args.storage_account,
storage_account_key=args.storage_key,
lease_renew_interval=30,
lease_container_name=args.container,
lease_duration=60)
# Event loop and host
host = EventProcessorHost(
EventProcessor,
eh_config,
storage_manager,
ep_params=["param1","param2"],
eph_options=eh_options,
loop=loop)
tasks = asyncio.gather(
host.open_async(),
wait_and_close(host, args.duration), return_exceptions=True)
results = loop.run_until_complete(tasks)
assert not any(results)
async def test_receive_with_invalid_policy_async(invalid_policy):
client = EventHubClientAsync.from_connection_string(invalid_policy, debug=True)
sender = client.add_async_receiver("$default", "0")
with pytest.raises(EventHubError):
await client.run_async()
class EventHubPartitionPump(PartitionPump):
"""
Pulls and messages from lease partition from eventhub and sends them to processor.
"""
def __init__(self, host, lease):
PartitionPump.__init__(self, host, lease)
self.eh_client = None
self.partition_receiver = None
self.partition_receive_handler = None
self.running = None
async def on_open_async(self):
"""
Eventhub Override for on_open_async.
"""
_opened_ok = False
_retry_count = 0
while (not _opened_ok) and (_retry_count < 5):
try:
await self.open_clients_async()
_opened_ok = True
except Exception as err: # pylint: disable=broad-except
_logger.warning(
"%r,%r PartitionPumpWarning: Failure creating client or receiver, retrying: %r",
self.host.guid, self.partition_context.partition_id, err)
last_exception = err
_retry_count += 1
if not _opened_ok:
await self.processor.process_error_async(self.partition_context, last_exception)
self.set_pump_status("OpenFailed")
if self.pump_status == "Opening":
loop = asyncio.get_event_loop()
self.set_pump_status("Running")
await self.eh_client.run_async()
self.running = loop.create_task(self.partition_receiver.run())
if self.pump_status in ["OpenFailed", "Errored"]:
self.set_pump_status("Closing")
await self.clean_up_clients_async()
self.set_pump_status("Closed")
async def open_clients_async(self):
"""
Responsible for establishing connection to event hub client
throws EventHubsException, IOException, InterruptedException, ExecutionException.
"""
await self.partition_context.get_initial_offset_async()
# Create event hub client and receive handler and set options
self.eh_client = EventHubClientAsync(
self.host.eh_config.client_address,
debug=self.host.eph_options.debug_trace,
http_proxy=self.host.eph_options.http_proxy)
self.partition_receive_handler = self.eh_client.add_async_receiver(
self.partition_context.consumer_group_name,
self.partition_context.partition_id,
Offset(self.partition_context.offset),
prefetch=self.host.eph_options.prefetch_count,
keep_alive=self.host.eph_options.keep_alive_interval,
auto_reconnect=self.host.eph_options.auto_reconnect_on_error,
loop=self.loop)
self.partition_receiver = PartitionReceiver(self)
async def clean_up_clients_async(self):
"""
Resets the pump swallows all exceptions.
"""
if self.partition_receiver:
if self.eh_client:
await self.eh_client.stop_async()
self.partition_receiver = None
self.partition_receive_handler = None
self.eh_client = None
async def on_closing_async(self, reason):
"""
Overides partition pump on closing.
:param reason: The reason for the shutdown.
:type reason: str
"""
self.partition_receiver.eh_partition_pump.set_pump_status("Errored")
try:
await self.running
except TypeError:
_logger.debug("No partition pump running.")
except Exception as err: # pylint: disable=broad-except
_logger.info("Error on closing partition pump: %r", err)
await self.clean_up_clients_async()
CONSUMER_GROUP = "$default"
EPOCH = 42
PARTITION = "0"
async def pump(client, epoch):
receiver = client.add_async_epoch_receiver(CONSUMER_GROUP, PARTITION, epoch=epoch)
await client.run_async()
total = 0
start_time = time.time()
for event_data in await receiver.receive(timeout=5):
last_offset = event_data.offset
last_sn = event_data.sequence_number
total += 1
end_time = time.time()
run_time = end_time - start_time
await client.stop_async()
print("Received {} messages in {} seconds".format(total, run_time))
try:
if not ADDRESS:
raise ValueError("No EventHubs URL supplied.")
loop = asyncio.get_event_loop()
client = EventHubClientAsync(ADDRESS, debug=False, username=USER, password=KEY)
loop.run_until_complete(pump(client, 20))
loop.close()
except KeyboardInterrupt:
pass
namespace_name = os.environ["EVENTHUB_NAMESPACE_NAME"]
saspolicy_name = os.environ["EVENTHUB_KEY_NAME"]
saspolicy_secret = os.environ["EVENTHUB_KEY_VALUE"]
hub_name = os.environ["EVENTHUB_HUB_NAME"]
conn_string = ("Endpoint=sb://{}.servicebus.windows.net/;"
"SharedAccessKeyName={};SharedAccessKey={}").format(
namespace_name, saspolicy_name, saspolicy_secret)
consumer_group = "$default"
if __name__ == '__main__':
readEnv()
loop = asyncio.get_event_loop()
offset = Offset("-1")
client = EventHubClientAsync.from_connection_string(conn_string,
eventhub=hub_name)
try:
eh_info = loop.run_until_complete(client.get_eventhub_info_async())
partition_ids = eh_info['partition_ids']
pumps = []
for pid in partition_ids:
receiver = client.add_async_receiver(consumer_group=consumer_group,
partition=pid,
offset=offset,
prefetch=5000)
pumps.append(pump(receiver))
loop.run_until_complete(client.run_async())
loop.run_until_complete(asyncio.gather(*pumps))
except:
async def test_receive_with_invalid_policy_async(invalid_policy):
client = EventHubClientAsync.from_connection_string(invalid_policy,
debug=True)
sender = client.add_async_receiver("$default", "0")
with pytest.raises(EventHubError):
await client.run_async()
def test_long_running_partition_send_async(connection_str):
parser = argparse.ArgumentParser()
parser.add_argument("--duration",
help="Duration in seconds of the test",
type=int,
default=30)
parser.add_argument("--payload",
help="payload size",
type=int,
default=1024)
parser.add_argument("--batch",
help="Number of events to send and wait",
type=int,
default=200)
parser.add_argument("--partitions", help="Comma seperated partition IDs")
parser.add_argument("--conn-str",
help="EventHub connection string",
default=connection_str)
parser.add_argument("--eventhub", help="Name of EventHub")
parser.add_argument("--address", help="Address URI to the EventHub entity")
parser.add_argument(
"--sas-policy",
help="Name of the shared access policy to authenticate with")
parser.add_argument("--sas-key", help="Shared access key")
parser.add_argument("--logger-name", help="Unique log file ID")
loop = asyncio.get_event_loop()
args, _ = parser.parse_known_args()
if args.conn_str:
client = EventHubClientAsync.from_connection_string(
args.conn_str, eventhub=args.eventhub, debug=True)
elif args.address:
client = EventHubClientAsync(args.address,
username=args.sas_policy,
password=args.sas_key,
auth_timeout=500)
else:
try:
import pytest
pytest.skip("Must specify either '--conn-str' or '--address'")
except ImportError:
raise ValueError("Must specify either '--conn-str' or '--address'")
try:
if not args.partitions:
partitions = loop.run_until_complete(get_partitions(client))
else:
pid_range = args.partitions.split("-")
if len(pid_range) > 1:
partitions = [
str(i) for i in range(int(pid_range[0]),
int(pid_range[1]) + 1)
]
else:
partitions = args.partitions.split(",")
pumps = []
for pid in partitions:
sender = client.add_async_sender(partition=pid,
send_timeout=0,
keep_alive=False)
pumps.append(pump(pid, sender, args, args.duration))
loop.run_until_complete(client.run_async())
results = loop.run_until_complete(
asyncio.gather(*pumps, return_exceptions=True))
assert not results
except Exception as e:
logger.error("Sender failed: {}".format(e))
finally:
logger.info("Shutting down sender")
loop.run_until_complete(client.stop_async())