async def test_example_eventhub_async_sender_ops(live_eventhub_config, connection_str): import os # [START create_eventhub_client_async_sender_instance] from azure.eventhub import EventHubClientAsync client = EventHubClientAsync.from_connection_string(connection_str) sender = client.add_async_sender(partition="0") # [END create_eventhub_client_async_sender_instance] # [START eventhub_client_async_sender_open] client = EventHubClientAsync.from_connection_string(connection_str) sender = client.add_async_sender(partition="0") try: # Open the Async Sender using the supplied conneciton. await sender.open_async() # Start sending except: raise finally: # Close down the send handler. await sender.close_async() # [END eventhub_client_async_sender_open] # [START eventhub_client_async_sender_close] client = EventHubClientAsync.from_connection_string(connection_str) sender = client.add_async_sender(partition="0") try: # Open the Async Sender using the supplied conneciton. await sender.open_async() # Start sending except: raise finally: # Close down the send handler. await sender.close_async()
async def test_non_existing_entity_receiver_async(connection_str): client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False) receiver = client.add_async_receiver("$default", "0") with pytest.raises(EventHubError): await client.run_async()
async def test_max_receivers_async(connstr_senders): connection_str, senders = connstr_senders client = EventHubClientAsync.from_connection_string(connection_str, debug=True) receivers = [] for i in range(6): receivers.append( client.add_async_receiver("$default", "0", prefetch=1000, offset=Offset('@latest'))) try: await client.run_async() outputs = await asyncio.gather(pump(receivers[0]), pump(receivers[1]), pump(receivers[2]), pump(receivers[3]), pump(receivers[4]), pump(receivers[5]), return_exceptions=True) print(outputs) failed = [o for o in outputs if isinstance(o, EventHubError)] assert len(failed) == 1 print(failed[0].message) finally: await client.stop_async()
async def test_send_with_invalid_key_async(invalid_key, connstr_receivers): _, receivers = connstr_receivers client = EventHubClientAsync.from_connection_string(invalid_key, debug=False) sender = client.add_async_sender() with pytest.raises(EventHubError): await client.run_async()
async def test_send_with_partition_key_async(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender() await client.run_async() data_val = 0 for partition in [b"a", b"b", b"c", b"d", b"e", b"f"]: partition_key = b"test_partition_" + partition for i in range(50): data = EventData(str(data_val)) data.partition_key = partition_key data_val += 1 await sender.send(data) await client.stop_async() found_partition_keys = {} for index, partition in enumerate(receivers): received = partition.receive(timeout=5) for message in received: try: existing = found_partition_keys[message.partition_key] assert existing == index except KeyError: found_partition_keys[message.partition_key] = index
async def test_max_receivers_async(connection_str, senders): client = EventHubClientAsync.from_connection_string(connection_str, debug=False) receivers = [] for i in range(6): receivers.append( client.add_async_receiver("$default", "0", prefetch=1000, offset=Offset('@latest'))) await client.run_async() try: outputs = await asyncio.gather(pump(receivers[0]), pump(receivers[1]), pump(receivers[2]), pump(receivers[3]), pump(receivers[4]), pump(receivers[5]), return_exceptions=True) assert len([o for o in outputs if isinstance(o, EventHubError)]) == 1 except: raise finally: await client.stop_async()
async def test_non_existing_entity_sender_async(connection_str): client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False) sender = client.add_async_sender(partition="1") with pytest.raises(EventHubError): await client.run_async()
async def test_multiple_receiver_async(connstr_senders): connection_str, senders = connstr_senders senders[0].send(EventData(b"Receiving only a single event")) client = EventHubClientAsync.from_connection_string(connection_str, debug=True) partitions = await client.get_eventhub_info_async() assert partitions["partition_ids"] == ["0", "1"] receivers = [] for i in range(2): receivers.append( client.add_async_receiver("$default", "0", prefetch=10)) try: await client.run_async() more_partitions = await client.get_eventhub_info_async() assert more_partitions["partition_ids"] == ["0", "1"] outputs = await asyncio.gather(pump(receivers[0]), pump(receivers[1]), return_exceptions=True) assert isinstance(outputs[0], int) and outputs[0] == 1 assert isinstance(outputs[1], int) and outputs[1] == 1 except: raise finally: await client.stop_async()
async def test_send_batch_async(connstr_receivers): connection_str, receivers = connstr_receivers def batched(): for i in range(10): yield "Event number {}".format(i) client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender() try: await client.run_async() await sender.send(EventData(batch=batched())) except: raise finally: await client.stop_async() time.sleep(1) received = [] for r in receivers: received.extend(r.receive(timeout=3)) assert len(received) == 10 for index, message in enumerate(received): assert list( message.body)[0] == "Event number {}".format(index).encode('utf-8')
async def test_receive_with_datetime_async(connection_str, senders): client = EventHubClientAsync.from_connection_string(connection_str, debug=False) receiver = client.add_async_receiver("$default", "0", offset=Offset('@latest')) await client.run_async() try: received = await receiver.receive(timeout=5) assert len(received) == 0 senders[0].send(EventData(b"Data")) received = await receiver.receive(timeout=5) assert len(received) == 1 offset = received[0].enqueued_time offset_receiver = client.add_async_receiver("$default", "0", offset=Offset(offset)) await client.run_async() received = await offset_receiver.receive(timeout=5) assert len(received) == 0 senders[0].send(EventData(b"Message after timestamp")) time.sleep(1) received = await offset_receiver.receive(timeout=5) assert len(received) == 1 except: raise finally: await client.stop_async()
async def start_event_generation_async_impl(cancellation_token): random.seed(int(time.time())) # use ticks as seed client = EventHubClientAsync.from_connection_string(conn_str=EVENTHUB_CONNECTION_STRING, eventhub=EVENTHUB_NAME) sender = client.add_async_sender() client.run() while cancellation_token.is_set(): # Simulate sending data from 100 weather sensors devices_data = [] for i in range(0, 100): scale_factor = random.randrange(0,25) windturbine_measure = generate_turbine_measure("Python_Turbine_" + str(i), scale_factor) ev_data = serialize_windturbine_to_eventdata(windturbine_measure) devices_data.append(ev_data) await sender.send(EventData(batch=[event for event in devices_data])) print(".", end='', flush=True) logger.info("100 events sent!") client.stop() return
async def test_receive_with_inclusive_offset_async(connstr_senders): connection_str, senders = connstr_senders client = EventHubClientAsync.from_connection_string(connection_str, debug=False) receiver = client.add_async_receiver("$default", "0", offset=Offset('@latest')) await client.run_async() try: received = await receiver.receive(timeout=5) assert len(received) == 0 senders[0].send(EventData(b"Data")) time.sleep(1) received = await receiver.receive(timeout=5) assert len(received) == 1 offset = received[0].offset offset_receiver = client.add_async_receiver("$default", "0", offset=Offset( offset.value, inclusive=True)) await client.run_async() received = await offset_receiver.receive(timeout=5) assert len(received) == 1 except: raise finally: await client.stop_async()
async def test_send_with_partition_key_async(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender() await client.run_async() data_val = 0 for partition in [b"a", b"b", b"c", b"d", b"e", b"f"]: partition_key = b"test_partition_" + partition for i in range(50): data = EventData(str(data_val)) data.partition_key = partition_key data_val += 1 await sender.send(data) await client.stop_async() found_partition_keys = {} for index, partition in enumerate(receivers): received = partition.receive(timeout=5) for message in received: try: existing = found_partition_keys[message.partition_key] assert existing == index except KeyError: found_partition_keys[message.partition_key] = index
def test_long_running_receive_async(connection_str): parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--consumer", help="Consumer group name", default="$default") parser.add_argument("--partitions", help="Comma seperated partition IDs") parser.add_argument("--offset", help="Starting offset", default="-1") parser.add_argument("--conn-str", help="EventHub connection string", default=connection_str) parser.add_argument("--eventhub", help="Name of EventHub") parser.add_argument("--address", help="Address URI to the EventHub entity") parser.add_argument( "--sas-policy", help="Name of the shared access policy to authenticate with") parser.add_argument("--sas-key", help="Shared access key") loop = asyncio.get_event_loop() args, _ = parser.parse_known_args() if args.conn_str: client = EventHubClientAsync.from_connection_string( args.conn_str, eventhub=args.eventhub, auth_timeout=240, debug=False) elif args.address: client = EventHubClientAsync(args.address, auth_timeout=240, username=args.sas_policy, password=args.sas_key) else: try: import pytest pytest.skip("Must specify either '--conn-str' or '--address'") except ImportError: raise ValueError("Must specify either '--conn-str' or '--address'") try: if not args.partitions: partitions = loop.run_until_complete(get_partitions(client)) else: partitions = args.partitions.split(",") pumps = [] for pid in partitions: receiver = client.add_async_receiver(consumer_group=args.consumer, partition=pid, offset=Offset(args.offset), prefetch=50) pumps.append(pump(pid, receiver, args, args.duration)) loop.run_until_complete(client.run_async()) loop.run_until_complete(asyncio.gather(*pumps)) finally: loop.run_until_complete(client.stop_async())
async def test_send_null_body_async(connection_str): client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender() try: await client.run_async() with pytest.raises(ValueError): data = EventData(None) await sender.send(data) finally: await client.stop_async()
def __init__(self, arguments): super().__init__(arguments) connection_string = self.get_from_env( "AZURE_EVENTHUB_CONNECTION_STRING") eventhub_name = self.get_from_env("AZURE_EVENTHUB_NAME") if self.args.no_client_share: self.eventhub_client = EventHubClient.from_connection_string( connection_string, eventhub=eventhub_name) self.async_eventhub_client = EventHubClientAsync.from_connection_string( connection_string, eventhub=eventhub_name) else: if not _EventHubTest.eventhub_client: _EventHubTest.eventhub_client = EventHubClient.from_connection_string( connection_string, eventhub=eventhub_name) _EventHubTest.async_eventhub_client = EventHubClientAsync.from_connection_string( connection_string, eventhub=eventhub_name) self.eventhub_client = _EventHubTest.eventhub_client self.async_eventhub_client = _EventHubTest.async_eventhub_client
async def test_send_to_invalid_partitions_async(connection_str): partitions = ["XYZ", "-1", "1000", "-" ] for p in partitions: client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender(partition=p) try: with pytest.raises(EventHubError): await client.run_async() finally: await client.stop_async()
async def test_send_too_large_message_async(connection_str): client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender() try: await client.run_async() data = EventData(b"A" * 300000) with pytest.raises(EventHubError): await sender.send(data) finally: await client.stop_async()
async def test_send_null_body_async(connection_str): client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender() try: await client.run_async() with pytest.raises(ValueError): data = EventData(None) await sender.send(data) finally: await client.stop_async()
async def test_example_eventhub_async_receiver_ops(live_eventhub_config, connection_str): import os # [START create_eventhub_client_async_receiver_instance] from azure.eventhub import EventHubClientAsync, Offset client = EventHubClientAsync.from_connection_string(connection_str) receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) # [END create_eventhub_client_async_receiver_instance] # [START eventhub_client_async_receiver_open] client = EventHubClientAsync.from_connection_string(connection_str) receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) try: # Open the Async Receiver using the supplied conneciton. await receiver.open_async() # Start receiving except: raise finally: # Close down the receive handler. await receiver.close_async() # [END eventhub_client_async_receiver_open] # [START eventhub_client_async_receiver_close] client = EventHubClientAsync.from_connection_string(connection_str) receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) try: # Open the Async Receiver using the supplied conneciton. await receiver.open_async() # Start receiving except: raise finally: # Close down the receive handler. await receiver.close_async()
async def test_send_partition_key_with_partition_async(connection_str): client = EventHubClientAsync.from_connection_string(connection_str, debug=True) sender = client.add_async_sender(partition="1") try: await client.run_async() data = EventData(b"Data") data.partition_key = b"PKey" with pytest.raises(ValueError): await sender.send(data) finally: await client.stop_async()
def test_long_running_partition_send_async(connection_str): parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--payload", help="payload size", type=int, default=1024) parser.add_argument("--batch", help="Number of events to send and wait", type=int, default=200) parser.add_argument("--partitions", help="Comma seperated partition IDs") parser.add_argument("--conn-str", help="EventHub connection string", default=connection_str) parser.add_argument("--eventhub", help="Name of EventHub") parser.add_argument("--address", help="Address URI to the EventHub entity") parser.add_argument("--sas-policy", help="Name of the shared access policy to authenticate with") parser.add_argument("--sas-key", help="Shared access key") parser.add_argument("--logger-name", help="Unique log file ID") loop = asyncio.get_event_loop() args, _ = parser.parse_known_args() if args.conn_str: client = EventHubClientAsync.from_connection_string( args.conn_str, eventhub=args.eventhub, debug=True) elif args.address: client = EventHubClientAsync( args.address, username=args.sas_policy, password=args.sas_key, auth_timeout=500) else: try: import pytest pytest.skip("Must specify either '--conn-str' or '--address'") except ImportError: raise ValueError("Must specify either '--conn-str' or '--address'") try: if not args.partitions: partitions = loop.run_until_complete(get_partitions(client)) else: pid_range = args.partitions.split("-") if len(pid_range) > 1: partitions = [str(i) for i in range(int(pid_range[0]), int(pid_range[1]) + 1)] else: partitions = args.partitions.split(",") pumps = [] for pid in partitions: sender = client.add_async_sender(partition=pid, send_timeout=0, keep_alive=False) pumps.append(pump(pid, sender, args, args.duration)) loop.run_until_complete(client.run_async()) results = loop.run_until_complete(asyncio.gather(*pumps, return_exceptions=True)) assert not results except Exception as e: logger.error("Sender failed: {}".format(e)) finally: logger.info("Shutting down sender") loop.run_until_complete(client.stop_async())
async def test_receive_from_invalid_partitions_async(connection_str): partitions = ["XYZ", "-1", "1000", "-" ] for p in partitions: client = EventHubClientAsync.from_connection_string(connection_str, debug=True) receiver = client.add_async_receiver("$default", p) try: with pytest.raises(EventHubError): await client.run_async() await receiver.receive(timeout=10) finally: await client.stop_async()
async def test_send_partition_key_with_partition_async(connection_str): client = EventHubClientAsync.from_connection_string(connection_str, debug=True) sender = client.add_async_sender(partition="1") try: await client.run_async() data = EventData(b"Data") data.partition_key = b"PKey" with pytest.raises(ValueError): await sender.send(data) finally: await client.stop_async()
async def test_receive_from_invalid_partitions_async(connection_str): partitions = ["XYZ", "-1", "1000", "-" ] for p in partitions: client = EventHubClientAsync.from_connection_string(connection_str, debug=True) receiver = client.add_async_receiver("$default", p) try: with pytest.raises(EventHubError): await client.run_async() await receiver.receive(timeout=10) finally: await client.stop_async()
async def test_send_to_invalid_partitions_async(connection_str): partitions = ["XYZ", "-1", "1000", "-"] for p in partitions: client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender(partition=p) try: with pytest.raises(EventHubError): await client.run_async() finally: await client.stop_async()
def test_long_running_partition_send_async(connection_str): parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--payload", help="payload size", type=int, default=1024) parser.add_argument("--batch", help="Number of events to send and wait", type=int, default=200) parser.add_argument("--partitions", help="Comma seperated partition IDs") parser.add_argument("--conn-str", help="EventHub connection string", default=connection_str) parser.add_argument("--eventhub", help="Name of EventHub") parser.add_argument("--address", help="Address URI to the EventHub entity") parser.add_argument("--sas-policy", help="Name of the shared access policy to authenticate with") parser.add_argument("--sas-key", help="Shared access key") parser.add_argument("--logger-name", help="Unique log file ID") loop = asyncio.get_event_loop() args, _ = parser.parse_known_args() if args.conn_str: client = EventHubClientAsync.from_connection_string( args.conn_str, eventhub=args.eventhub, debug=True) elif args.address: client = EventHubClientAsync( args.address, username=args.sas_policy, password=args.sas_key, auth_timeout=500) else: try: import pytest pytest.skip("Must specify either '--conn-str' or '--address'") except ImportError: raise ValueError("Must specify either '--conn-str' or '--address'") try: if not args.partitions: partitions = loop.run_until_complete(get_partitions(client)) else: pid_range = args.partitions.split("-") if len(pid_range) > 1: partitions = [str(i) for i in range(int(pid_range[0]), int(pid_range[1]) + 1)] else: partitions = args.partitions.split(",") pumps = [] for pid in partitions: sender = client.add_async_sender(partition=pid, send_timeout=0, keep_alive=False) pumps.append(pump(pid, sender, args, args.duration)) loop.run_until_complete(client.run_async()) results = loop.run_until_complete(asyncio.gather(*pumps, return_exceptions=True)) assert not results except Exception as e: logger.error("Sender failed: {}".format(e)) finally: logger.info("Shutting down sender") loop.run_until_complete(client.stop_async())
async def test_send_too_large_message_async(connection_str): if sys.platform.startswith('darwin'): pytest.skip("Skipping on OSX - open issue regarding message size") client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender() try: await client.run_async() data = EventData(b"A" * 300000) with pytest.raises(EventHubError): await sender.send(data) finally: await client.stop_async()
async def test_non_existing_entity_receiver_async(connection_str): client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False) receiver = client.add_async_receiver("$default", "0") try: await client.run_async() with pytest.raises(EventHubError): await receiver.receive(timeout=5) except: raise finally: await client.stop_async()
async def test_send_too_large_message_async(connection_str): if sys.platform.startswith('darwin'): pytest.skip("Skipping on OSX - open issue regarding message size") client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender() try: await client.run_async() data = EventData(b"A" * 300000) with pytest.raises(EventHubError): await sender.send(data) finally: await client.stop_async()
async def test_non_existing_entity_sender_async(connection_str): client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False) sender = client.add_async_sender(partition="1") try: await client.run_async() data = EventData(b"Data") with pytest.raises(EventHubError): await sender.send(data) except: raise finally: await client.stop_async()
async def test_receive_batch_async(connection_str, senders): client = EventHubClientAsync.from_connection_string(connection_str, debug=False) receiver = client.add_async_receiver("$default", "0", prefetch=500, offset=Offset('@latest')) await client.run_async() try: received = await receiver.receive(timeout=5) assert len(received) == 0 for i in range(10): senders[0].send(EventData(b"Data")) received = await receiver.receive(max_batch_size=5, timeout=5) assert len(received) == 5 except: raise finally: await client.stop_async()
async def test_example_eventhub_async_sender_ops(live_eventhub_config, connection_str): import os # [START create_eventhub_client_async_sender_instance] from azure.eventhub import EventHubClientAsync client = EventHubClientAsync.from_connection_string(connection_str) sender = client.add_async_sender(partition="0") # [END create_eventhub_client_async_sender_instance] # [START eventhub_client_async_sender_open] client = EventHubClientAsync.from_connection_string(connection_str) sender = client.add_async_sender(partition="0") try: # Open the Async Sender using the supplied conneciton. await sender.open_async() # Start sending except: raise finally: # Close down the send handler. await sender.close_async() # [END eventhub_client_async_sender_open] # [START eventhub_client_async_sender_close] client = EventHubClientAsync.from_connection_string(connection_str) sender = client.add_async_sender(partition="0") try: # Open the Async Sender using the supplied conneciton. await sender.open_async() # Start sending except: raise finally: # Close down the send handler. await sender.close_async()
async def test_example_eventhub_async_receiver_ops(live_eventhub_config, connection_str): import os # [START create_eventhub_client_async_receiver_instance] from azure.eventhub import EventHubClientAsync, Offset client = EventHubClientAsync.from_connection_string(connection_str) receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) # [END create_eventhub_client_async_receiver_instance] # [START eventhub_client_async_receiver_open] client = EventHubClientAsync.from_connection_string(connection_str) receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) try: # Open the Async Receiver using the supplied conneciton. await receiver.open_async() # Start receiving except: raise finally: # Close down the receive handler. await receiver.close_async() # [END eventhub_client_async_receiver_open] # [START eventhub_client_async_receiver_close] client = EventHubClientAsync.from_connection_string(connection_str) receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) try: # Open the Async Receiver using the supplied conneciton. await receiver.open_async() # Start receiving except: raise finally: # Close down the receive handler. await receiver.close_async() # [END eventhub_client_async_receiver_close]
def test_long_running_receive_async(connection_str): parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--consumer", help="Consumer group name", default="$default") parser.add_argument("--partitions", help="Comma seperated partition IDs") parser.add_argument("--offset", help="Starting offset", default="-1") parser.add_argument("--conn-str", help="EventHub connection string", default=connection_str) parser.add_argument("--eventhub", help="Name of EventHub") parser.add_argument("--address", help="Address URI to the EventHub entity") parser.add_argument("--sas-policy", help="Name of the shared access policy to authenticate with") parser.add_argument("--sas-key", help="Shared access key") loop = asyncio.get_event_loop() args, _ = parser.parse_known_args() if args.conn_str: client = EventHubClientAsync.from_connection_string( args.conn_str, eventhub=args.eventhub, auth_timeout=240, debug=False) elif args.address: client = EventHubClientAsync( args.address, auth_timeout=240, username=args.sas_policy, password=args.sas_key) else: try: import pytest pytest.skip("Must specify either '--conn-str' or '--address'") except ImportError: raise ValueError("Must specify either '--conn-str' or '--address'") try: if not args.partitions: partitions = loop.run_until_complete(get_partitions(client)) else: partitions = args.partitions.split(",") pumps = [] for pid in partitions: receiver = client.add_async_receiver( consumer_group=args.consumer, partition=pid, offset=Offset(args.offset), prefetch=50) pumps.append(pump(pid, receiver, args, args.duration)) loop.run_until_complete(client.run_async()) loop.run_until_complete(asyncio.gather(*pumps)) finally: loop.run_until_complete(client.stop_async())
async def test_receive_end_of_stream_async(connection_str, senders): client = EventHubClientAsync.from_connection_string(connection_str, debug=False) receiver = client.add_async_receiver("$default", "0", offset=Offset('@latest')) await client.run_async() try: received = await receiver.receive(timeout=5) assert len(received) == 0 senders[0].send(EventData(b"Receiving only a single event")) received = await receiver.receive(timeout=5) assert len(received) == 1 assert list(received[-1].body)[0] == b"Receiving only a single event" except: raise finally: await client.stop_async()
async def test_send_partition_async(connection_str, receivers): client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender(partition="1") try: await client.run_async() await sender.send(EventData(b"Data")) except: raise finally: await client.stop_async() partition_0 = receivers[0].receive(timeout=2) assert len(partition_0) == 0 partition_1 = receivers[1].receive(timeout=2) assert len(partition_1) == 1
async def test_send_partition_async(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender(partition="1") try: await client.run_async() await sender.send(EventData(b"Data")) except: raise finally: await client.stop_async() partition_0 = receivers[0].receive(timeout=2) assert len(partition_0) == 0 partition_1 = receivers[1].receive(timeout=2) assert len(partition_1) == 1
async def test_send_non_ascii_async(connection_str, receivers): client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender(partition="0") try: await client.run_async() await sender.send(EventData("é,è,à,ù,â,ê,î,ô,û")) await sender.send(EventData(json.dumps({"foo": "漢字"}))) except: raise finally: await client.stop_async() partition_0 = receivers[0].receive(timeout=2) assert len(partition_0) == 2 assert partition_0[0].body_as_str() == "é,è,à,ù,â,ê,î,ô,û" assert partition_0[1].body_as_json() == {"foo": "漢字"}
async def test_send_non_ascii_async(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender(partition="0") try: await client.run_async() await sender.send(EventData("é,è,à,ù,â,ê,î,ô,û")) await sender.send(EventData(json.dumps({"foo": "漢字"}))) except: raise finally: await client.stop_async() partition_0 = receivers[0].receive(timeout=2) assert len(partition_0) == 2 assert partition_0[0].body_as_str() == "é,è,à,ù,â,ê,î,ô,û" assert partition_0[1].body_as_json() == {"foo": "漢字"}
async def test_multiple_receiver_async(connection_str, senders): client = EventHubClientAsync.from_connection_string(connection_str, debug=True) receivers = [] for i in range(2): receivers.append( client.add_async_receiver("$default", "0", prefetch=10)) try: await client.run_async() outputs = await asyncio.gather(pump(receivers[0]), pump(receivers[1]), return_exceptions=True) assert isinstance(outputs[0], int) and outputs[0] >= 1 assert isinstance(outputs[1], int) and outputs[1] >= 1 except: raise finally: await client.stop_async()
async def test_send_with_long_interval_async(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubClientAsync.from_connection_string(connection_str, debug=True) sender = client.add_async_sender() try: await client.run_async() await sender.send(EventData(b"A single event")) for _ in range(2): await asyncio.sleep(300) await sender.send(EventData(b"A single event")) finally: await client.stop_async() received = [] for r in receivers: received.extend(r.receive(timeout=1)) assert len(received) == 3 assert list(received[0].body)[0] == b"A single event"
async def test_send_single_event_async(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender() try: await client.run_async() await sender.send(EventData(b"A single event")) except: raise finally: await client.stop_async() received = [] for r in receivers: received.extend(r.receive(timeout=1)) assert len(received) == 1 assert list(received[0].body)[0] == b"A single event"
async def test_send_partition_batch_async(connstr_receivers): connection_str, receivers = connstr_receivers def batched(): for i in range(10): yield "Event number {}".format(i) client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender(partition="1") try: await client.run_async() await sender.send(EventData(batch=batched())) except: raise finally: await client.stop_async() partition_0 = receivers[0].receive(timeout=2) assert len(partition_0) == 0 partition_1 = receivers[1].receive(timeout=2) assert len(partition_1) == 10
async def test_max_receivers_async(connstr_senders): connection_str, senders = connstr_senders client = EventHubClientAsync.from_connection_string(connection_str, debug=True) receivers = [] for i in range(6): receivers.append(client.add_async_receiver("$default", "0", prefetch=1000, offset=Offset('@latest'))) try: await client.run_async() outputs = await asyncio.gather( pump(receivers[0]), pump(receivers[1]), pump(receivers[2]), pump(receivers[3]), pump(receivers[4]), pump(receivers[5]), return_exceptions=True) print(outputs) failed = [o for o in outputs if isinstance(o, EventHubError)] assert len(failed) == 1 print(failed[0].message) finally: await client.stop_async()
async def test_send_batch_with_app_prop_async(connstr_receivers): pytest.skip("Waiting on uAMQP release") connection_str, receivers = connstr_receivers def batched(): for i in range(10): yield "Event number {}".format(i) for i in range(10, 20): yield EventData("Event number {}".format(i)) client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender() try: await client.run_async() app_prop_key = "raw_prop" app_prop_value = "raw_value" batch_app_prop = {app_prop_key:app_prop_value} batch_event = EventData(batch=batched()) batch_event.application_properties = batch_app_prop await sender.send(batch_event) except: raise finally: await client.stop_async() time.sleep(1) received = [] for r in receivers: received.extend(r.receive(timeout=3)) assert len(received) == 20 for index, message in enumerate(received): assert list(message.body)[0] == "Event number {}".format(index).encode('utf-8') assert (app_prop_key.encode('utf-8') in message.application_properties) \ and (dict(message.application_properties)[app_prop_key.encode('utf-8')] == app_prop_value.encode('utf-8'))
async def test_send_with_forced_conn_close_async(connstr_receivers): connection_str, receivers = connstr_receivers client = EventHubClientAsync.from_connection_string(connection_str, debug=True) sender = client.add_async_sender() try: await client.run_async() await sender.send(EventData(b"A single event")) sender._handler._message_sender.destroy() await asyncio.sleep(300) await sender.send(EventData(b"A single event")) await sender.send(EventData(b"A single event")) sender._handler._message_sender.destroy() await asyncio.sleep(300) await sender.send(EventData(b"A single event")) await sender.send(EventData(b"A single event")) finally: await client.stop_async() received = [] for r in receivers: received.extend(pump(r)) assert len(received) == 5 assert list(received[0].body)[0] == b"A single event"
async def test_send_batch_async(connstr_receivers): connection_str, receivers = connstr_receivers def batched(): for i in range(10): yield "Event number {}".format(i) client = EventHubClientAsync.from_connection_string(connection_str, debug=False) sender = client.add_async_sender() try: await client.run_async() await sender.send(EventData(batch=batched())) except: raise finally: await client.stop_async() time.sleep(1) received = [] for r in receivers: received.extend(r.receive(timeout=3)) assert len(received) == 10 for index, message in enumerate(received): assert list(message.body)[0] == "Event number {}".format(index).encode('utf-8')
def test_long_running_eph(live_eventhub): parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--storage-account", help="Storage account name", default=os.environ.get('AZURE_STORAGE_ACCOUNT')) parser.add_argument("--storage-key", help="Storage account access key", default=os.environ.get('AZURE_STORAGE_ACCESS_KEY')) parser.add_argument("--container", help="Lease container name", default="nocontextleases") parser.add_argument("--eventhub", help="Name of EventHub", default=live_eventhub['event_hub']) parser.add_argument("--namespace", help="Namespace of EventHub", default=live_eventhub['namespace']) parser.add_argument("--suffix", help="Namespace of EventHub", default="servicebus.windows.net") parser.add_argument("--sas-policy", help="Name of the shared access policy to authenticate with", default=live_eventhub['key_name']) parser.add_argument("--sas-key", help="Shared access key", default=live_eventhub['access_key']) loop = asyncio.get_event_loop() args, _ = parser.parse_known_args() if not args.namespace or not args.eventhub: try: import pytest pytest.skip("Must specify '--namespace' and '--eventhub'") except ImportError: raise ValueError("Must specify '--namespace' and '--eventhub'") # Queue up some events in the Eventhub conn_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format( live_eventhub['hostname'], live_eventhub['key_name'], live_eventhub['access_key'], live_eventhub['event_hub']) send_client = EventHubClientAsync.from_connection_string(conn_str) pumps = [] for pid in ["0", "1"]: sender = send_client.add_async_sender(partition=pid, send_timeout=0, keep_alive=False) pumps.append(pump(pid, sender, 15)) loop.run_until_complete(send_client.run_async()) results = loop.run_until_complete(asyncio.gather(*pumps, return_exceptions=True)) loop.run_until_complete(send_client.stop_async()) assert not any(results) # Eventhub config and storage manager eh_config = EventHubConfig( args.namespace, args.eventhub, args.sas_policy, args.sas_key, consumer_group="$default", namespace_suffix=args.suffix) eh_options = EPHOptions() eh_options.release_pump_on_timeout = True eh_options.debug_trace = False eh_options.receive_timeout = 120 storage_manager = AzureStorageCheckpointLeaseManager( storage_account_name=args.storage_account, storage_account_key=args.storage_key, lease_renew_interval=30, lease_container_name=args.container, lease_duration=60) # Event loop and host host = EventProcessorHost( EventProcessor, eh_config, storage_manager, ep_params=["param1","param2"], eph_options=eh_options, loop=loop) tasks = asyncio.gather( host.open_async(), wait_and_close(host, args.duration), return_exceptions=True) results = loop.run_until_complete(tasks) assert not any(results)
async def test_example_eventhub_async_send_and_receive(live_eventhub_config): # [START create_eventhub_client_async] from azure.eventhub import EventHubClientAsync import os connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format( os.environ['EVENT_HUB_HOSTNAME'], os.environ['EVENT_HUB_SAS_POLICY'], os.environ['EVENT_HUB_SAS_KEY'], os.environ['EVENT_HUB_NAME']) client = EventHubClientAsync.from_connection_string(connection_str) # [END create_eventhub_client_async] from azure.eventhub import EventData, Offset # [START create_eventhub_client_async_sender] client = EventHubClientAsync.from_connection_string(connection_str) # Add a async sender to the async client object. sender = client.add_async_sender(partition="0") # [END create_eventhub_client_async_sender] # [START create_eventhub_client_async_receiver] client = EventHubClientAsync.from_connection_string(connection_str) # Add a async receiver to the async client object. receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) # [END create_eventhub_client_async_receiver] # [START create_eventhub_client_async_epoch_receiver] client = EventHubClientAsync.from_connection_string(connection_str) # Add a async receiver to the async client object. epoch_receiver = client.add_async_epoch_receiver(consumer_group="$default", partition="0", epoch=42) # [END create_eventhub_client_async_epoch_receiver] # [START eventhub_client_run_async] client = EventHubClientAsync.from_connection_string(connection_str) # Add AsyncSenders/AsyncReceivers try: # Opens the connection and starts running all AsyncSender/AsyncReceiver clients. await client.run_async() # Start sending and receiving except: raise finally: await client.stop_async() # [END eventhub_client_run_async] client = EventHubClientAsync.from_connection_string(connection_str) sender = client.add_async_sender(partition="0") receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) try: # Opens the connection and starts running all AsyncSender/AsyncReceiver clients. await client.run_async() # [START eventhub_client_async_send] event_data = EventData(b"A single event") await sender.send(event_data) # [END eventhub_client_async_send] time.sleep(1) # [START eventhub_client_async_receive] logger = logging.getLogger("azure.eventhub") received = await receiver.receive(timeout=5) for event_data in received: logger.info("Message received:{}".format(event_data.body_as_str())) # [END eventhub_client_async_receive] assert len(received) == 1 assert received[0].body_as_str() == "A single event" assert list(received[-1].body)[0] == b"A single event" except: raise finally: await client.stop_async() # [START eventhub_client_async_stop] client = EventHubClientAsync.from_connection_string(connection_str) # Add AsyncSenders/AsyncReceivers try: # Opens the connection and starts running all AsyncSender/AsyncReceiver clients. await client.run_async() # Start sending and receiving except: raise finally: await client.stop_async()
async def test_non_existing_entity_sender_async(connection_str): client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False) sender = client.add_async_sender(partition="1") with pytest.raises(EventHubError): await client.run_async()
async def test_receive_with_invalid_policy_async(invalid_policy): client = EventHubClientAsync.from_connection_string(invalid_policy, debug=True) sender = client.add_async_receiver("$default", "0") with pytest.raises(EventHubError): await client.run_async()
async def test_non_existing_entity_receiver_async(connection_str): client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False) receiver = client.add_async_receiver("$default", "0") with pytest.raises(EventHubError): await client.run_async()
async def test_send_with_invalid_policy_async(invalid_policy, connstr_receivers): _, receivers = connstr_receivers client = EventHubClientAsync.from_connection_string(invalid_policy, debug=False) sender = client.add_async_sender() with pytest.raises(EventHubError): await client.run_async()