def test_long_running_receive_async(connection_str):
parser = argparse.ArgumentParser()
parser.add_argument("--duration",
help="Duration in seconds of the test",
type=int,
default=30)
parser.add_argument("--consumer",
help="Consumer group name",
default="$default")
parser.add_argument("--partitions", help="Comma seperated partition IDs")
parser.add_argument("--offset", help="Starting offset", default="-1")
parser.add_argument("--conn-str",
help="EventHub connection string",
default=connection_str)
parser.add_argument("--eventhub", help="Name of EventHub")
parser.add_argument("--address", help="Address URI to the EventHub entity")
parser.add_argument(
"--sas-policy",
help="Name of the shared access policy to authenticate with")
parser.add_argument("--sas-key", help="Shared access key")
loop = asyncio.get_event_loop()
args, _ = parser.parse_known_args()
if args.conn_str:
client = EventHubClientAsync.from_connection_string(
args.conn_str,
eventhub=args.eventhub,
auth_timeout=240,
debug=False)
elif args.address:
client = EventHubClientAsync(args.address,
auth_timeout=240,
username=args.sas_policy,
password=args.sas_key)
else:
try:
import pytest
pytest.skip("Must specify either '--conn-str' or '--address'")
except ImportError:
raise ValueError("Must specify either '--conn-str' or '--address'")
try:
if not args.partitions:
partitions = loop.run_until_complete(get_partitions(client))
else:
partitions = args.partitions.split(",")
pumps = []
for pid in partitions:
receiver = client.add_async_receiver(consumer_group=args.consumer,
partition=pid,
offset=Offset(args.offset),
prefetch=50)
pumps.append(pump(pid, receiver, args, args.duration))
loop.run_until_complete(client.run_async())
loop.run_until_complete(asyncio.gather(*pumps))
finally:
loop.run_until_complete(client.stop_async())
def test_long_running_receive_async(connection_str):
parser = argparse.ArgumentParser()
parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30)
parser.add_argument("--consumer", help="Consumer group name", default="$default")
parser.add_argument("--partitions", help="Comma seperated partition IDs")
parser.add_argument("--offset", help="Starting offset", default="-1")
parser.add_argument("--conn-str", help="EventHub connection string", default=connection_str)
parser.add_argument("--eventhub", help="Name of EventHub")
parser.add_argument("--address", help="Address URI to the EventHub entity")
parser.add_argument("--sas-policy", help="Name of the shared access policy to authenticate with")
parser.add_argument("--sas-key", help="Shared access key")
loop = asyncio.get_event_loop()
args, _ = parser.parse_known_args()
if args.conn_str:
client = EventHubClientAsync.from_connection_string(
args.conn_str,
eventhub=args.eventhub, auth_timeout=240, debug=False)
elif args.address:
client = EventHubClientAsync(
args.address,
auth_timeout=240,
username=args.sas_policy,
password=args.sas_key)
else:
try:
import pytest
pytest.skip("Must specify either '--conn-str' or '--address'")
except ImportError:
raise ValueError("Must specify either '--conn-str' or '--address'")
try:
if not args.partitions:
partitions = loop.run_until_complete(get_partitions(client))
else:
partitions = args.partitions.split(",")
pumps = []
for pid in partitions:
receiver = client.add_async_receiver(
consumer_group=args.consumer,
partition=pid,
offset=Offset(args.offset),
prefetch=50)
pumps.append(pump(pid, receiver, args, args.duration))
loop.run_until_complete(client.run_async())
loop.run_until_complete(asyncio.gather(*pumps))
finally:
loop.run_until_complete(client.stop_async())
class EventHubPartitionPump(PartitionPump):
"""
Pulls and messages from lease partition from eventhub and sends them to processor
"""
def __init__(self, host, lease):
PartitionPump.__init__(self, host, lease)
self.eh_client = None
self.partition_receiver = None
self.partition_receive_handler = None
self.running = None
async def on_open_async(self):
"""
Eventhub Override for on_open_async
"""
_opened_ok = False
_retry_count = 0
while (not _opened_ok) and (_retry_count < 5):
try:
await self.open_clients_async()
_opened_ok = True
except Exception as err: # pylint: disable=broad-except
_logger.warning(
"{},{} PartitionPumpWarning: Failure creating client or receiver, "
"retrying: {!r}".format(
self.host.guid, self.partition_context.partition_id,
err))
last_exception = err
_retry_count += 1
if not _opened_ok:
await self.processor.process_error_async(self.partition_context,
last_exception)
self.set_pump_status("OpenFailed")
if self.pump_status == "Opening":
loop = asyncio.get_event_loop()
self.set_pump_status("Running")
await self.eh_client.run_async()
self.running = loop.create_task(self.partition_receiver.run())
if self.pump_status in ["OpenFailed", "Errored"]:
self.set_pump_status("Closing")
await self.clean_up_clients_async()
self.set_pump_status("Closed")
async def open_clients_async(self):
"""
Responsible for establishing connection to event hub client
throws EventHubsException, IOException, InterruptedException, ExecutionException
"""
await self.partition_context.get_initial_offset_async()
# Create event hub client and receive handler and set options
self.eh_client = EventHubClientAsync(
self.host.eh_config.client_address,
debug=self.host.eph_options.debug_trace)
self.partition_receive_handler = self.eh_client.add_async_receiver(
self.partition_context.consumer_group_name,
self.partition_context.partition_id,
Offset(self.partition_context.offset),
prefetch=self.host.eph_options.prefetch_count,
loop=self.loop)
self.partition_receiver = PartitionReceiver(self)
async def clean_up_clients_async(self):
"""
Resets the pump swallows all exceptions
"""
if self.partition_receiver:
if self.eh_client:
await self.eh_client.stop_async()
self.partition_receiver = None
self.partition_receive_handler = None
self.eh_client = None
async def on_closing_async(self, reason):
"""
Overides partition pump on cleasing
:param reason: The reason for the shutdown.
:type reason: str
"""
self.partition_receiver.eh_partition_pump.set_pump_status("Errored")
await self.running
await self.clean_up_clients_async()
class EventHubPartitionPump(PartitionPump):
"""
Pulls and messages from lease partition from eventhub and sends them to processor.
"""
def __init__(self, host, lease):
PartitionPump.__init__(self, host, lease)
self.eh_client = None
self.partition_receiver = None
self.partition_receive_handler = None
self.running = None
async def on_open_async(self):
"""
Eventhub Override for on_open_async.
"""
_opened_ok = False
_retry_count = 0
while (not _opened_ok) and (_retry_count < 5):
try:
await self.open_clients_async()
_opened_ok = True
except Exception as err: # pylint: disable=broad-except
_logger.warning(
"%r,%r PartitionPumpWarning: Failure creating client or receiver, retrying: %r",
self.host.guid, self.partition_context.partition_id, err)
last_exception = err
_retry_count += 1
if not _opened_ok:
await self.processor.process_error_async(self.partition_context, last_exception)
self.set_pump_status("OpenFailed")
if self.pump_status == "Opening":
loop = asyncio.get_event_loop()
self.set_pump_status("Running")
await self.eh_client.run_async()
self.running = loop.create_task(self.partition_receiver.run())
if self.pump_status in ["OpenFailed", "Errored"]:
self.set_pump_status("Closing")
await self.clean_up_clients_async()
self.set_pump_status("Closed")
async def open_clients_async(self):
"""
Responsible for establishing connection to event hub client
throws EventHubsException, IOException, InterruptedException, ExecutionException.
"""
await self.partition_context.get_initial_offset_async()
# Create event hub client and receive handler and set options
self.eh_client = EventHubClientAsync(
self.host.eh_config.client_address,
debug=self.host.eph_options.debug_trace,
http_proxy=self.host.eph_options.http_proxy)
self.partition_receive_handler = self.eh_client.add_async_receiver(
self.partition_context.consumer_group_name,
self.partition_context.partition_id,
Offset(self.partition_context.offset),
prefetch=self.host.eph_options.prefetch_count,
keep_alive=self.host.eph_options.keep_alive_interval,
auto_reconnect=self.host.eph_options.auto_reconnect_on_error,
loop=self.loop)
self.partition_receiver = PartitionReceiver(self)
async def clean_up_clients_async(self):
"""
Resets the pump swallows all exceptions.
"""
if self.partition_receiver:
if self.eh_client:
await self.eh_client.stop_async()
self.partition_receiver = None
self.partition_receive_handler = None
self.eh_client = None
async def on_closing_async(self, reason):
"""
Overides partition pump on closing.
:param reason: The reason for the shutdown.
:type reason: str
"""
self.partition_receiver.eh_partition_pump.set_pump_status("Errored")
try:
await self.running
except TypeError:
_logger.debug("No partition pump running.")
except Exception as err: # pylint: disable=broad-except
_logger.info("Error on closing partition pump: %r", err)
await self.clean_up_clients_async()