def example_create_async_eventhub_consumer_client():
# [START create_eventhub_consumer_client_from_conn_str_async]
import os
from azure.eventhub.aio import EventHubConsumerClient
event_hub_connection_str = os.environ['EVENT_HUB_CONN_STR']
eventhub_name = os.environ['EVENT_HUB_NAME']
consumer = EventHubConsumerClient.from_connection_string(
conn_str=event_hub_connection_str,
consumer_group='$Default',
eventhub_name=
eventhub_name # EventHub name should be specified if it doesn't show up in connection string.
)
# [END create_eventhub_consumer_client_from_conn_str_async]
# [START create_eventhub_consumer_client_async]
import os
from azure.eventhub.aio import EventHubConsumerClient, EventHubSharedKeyCredential
fully_qualified_namespace = os.environ['EVENT_HUB_HOSTNAME']
eventhub_name = os.environ['EVENT_HUB_NAME']
shared_access_policy = os.environ['EVENT_HUB_SAS_POLICY']
shared_access_key = os.environ['EVENT_HUB_SAS_KEY']
consumer = EventHubConsumerClient(
fully_qualified_namespace=fully_qualified_namespace,
consumer_group='$Default',
eventhub_name=eventhub_name,
credential=EventHubSharedKeyCredential(shared_access_policy,
shared_access_key))
# [END create_eventhub_consumer_client_async]
return consumer
async def test_receive_load_balancing_async(connstr_senders):
connection_str, senders = connstr_senders
cs = InMemoryCheckpointStore()
client1 = EventHubConsumerClient.from_connection_string(
connection_str,
consumer_group='$default',
checkpoint_store=cs,
load_balancing_interval=1)
client2 = EventHubConsumerClient.from_connection_string(
connection_str,
consumer_group='$default',
checkpoint_store=cs,
load_balancing_interval=1)
async def on_event(partition_context, event):
pass
async with client1, client2:
task1 = asyncio.ensure_future(
client1.receive(on_event, starting_position="-1"))
await asyncio.sleep(3.3)
task2 = asyncio.ensure_future(
client2.receive(on_event, starting_position="-1"))
await asyncio.sleep(10)
assert len(client1._event_processors[("$default",
ALL_PARTITIONS)]._tasks) == 1
assert len(client2._event_processors[("$default",
ALL_PARTITIONS)]._tasks) == 1
task1.cancel()
task2.cancel()
async def test_client_secret_credential_async(aad_credential, live_eventhub):
try:
from azure.identity.aio import EnvironmentCredential
except ImportError:
pytest.skip("No azure identity library")
credential = EnvironmentCredential()
producer_client = EventHubProducerClient(host=live_eventhub['hostname'],
event_hub_path=live_eventhub['event_hub'],
credential=credential,
user_agent='customized information')
consumer_client = EventHubConsumerClient(host=live_eventhub['hostname'],
event_hub_path=live_eventhub['event_hub'],
credential=credential,
user_agent='customized information')
async with producer_client:
await producer_client.send(EventData(body='A single message'))
async def event_handler(partition_context, events):
assert partition_context.partition_id == '0'
assert len(events) == 1
assert list(events[0].body)[0] == 'A single message'.encode('utf-8')
async with consumer_client:
task = asyncio.ensure_future(
consumer_client.receive(event_handler=event_handler, consumer_group='$default', partition_id='0'))
await asyncio.sleep(2)
task.cancel()
async def test_receive_owner_level_async(connstr_senders):
app_prop = {"raw_prop": "raw_value"}
async def on_event(partition_context, event):
pass
async def on_error(partition_context, error):
on_error.error = error
on_error.error = None
connection_str, senders = connstr_senders
client1 = EventHubConsumerClient.from_connection_string(connection_str, consumer_group='$default')
client2 = EventHubConsumerClient.from_connection_string(connection_str, consumer_group='$default')
async with client1, client2:
task1 = asyncio.ensure_future(client1.receive(on_event,
partition_id="0", starting_position="-1",
on_error=on_error))
for i in range(5):
ed = EventData("Event Number {}".format(i))
senders[0].send(ed)
await asyncio.sleep(10)
task2 = asyncio.ensure_future(client2.receive(on_event,
partition_id="0", starting_position="-1",
owner_level=1))
for i in range(5):
ed = EventData("Event Number {}".format(i))
senders[0].send(ed)
await asyncio.sleep(10)
task1.cancel()
task2.cancel()
assert isinstance(on_error.error, EventHubError)
async def test_receive_load_balancing_async(connstr_senders):
connection_str, senders = connstr_senders
pm = InMemoryPartitionManager()
client1 = EventHubConsumerClient.from_connection_string(
connection_str, partition_manager=pm, load_balancing_interval=1)
client2 = EventHubConsumerClient.from_connection_string(
connection_str, partition_manager=pm, load_balancing_interval=1)
async def on_event(partition_context, event):
pass
async with client1, client2:
task1 = asyncio.ensure_future(
client1.receive(on_event,
consumer_group="$default",
initial_event_position="-1"))
task2 = asyncio.ensure_future(
client2.receive(on_event,
consumer_group="$default",
initial_event_position="-1"))
await asyncio.sleep(10)
assert len(client1._event_processors[("$default",
ALL_PARTITIONS)]._tasks) == 1
assert len(client2._event_processors[("$default",
ALL_PARTITIONS)]._tasks) == 1
task1.cancel()
task2.cancel()
async def test_client_secret_credential_async(live_eventhub):
credential = EnvironmentCredential()
producer_client = EventHubProducerClient(
fully_qualified_namespace=live_eventhub['hostname'],
eventhub_name=live_eventhub['event_hub'],
credential=credential,
user_agent='customized information')
consumer_client = EventHubConsumerClient(
fully_qualified_namespace=live_eventhub['hostname'],
eventhub_name=live_eventhub['event_hub'],
consumer_group='$default',
credential=credential,
user_agent='customized information')
async with producer_client:
batch = await producer_client.create_batch(partition_id='0')
batch.add(EventData(body='A single message'))
await producer_client.send_batch(batch)
def on_event(partition_context, event):
on_event.called = True
on_event.partition_id = partition_context.partition_id
on_event.event = event
on_event.called = False
async with consumer_client:
task = asyncio.ensure_future(
consumer_client.receive(on_event,
partition_id='0',
starting_position='-1'))
await asyncio.sleep(13)
await task
assert on_event.called is True
assert on_event.partition_id == "0"
assert list(on_event.event.body)[0] == 'A single message'.encode('utf-8')
def create_async_eventhub_consumer_client():
# [START create_eventhub_consumer_client_from_conn_str_async]
import os
from azure.eventhub.aio import EventHubConsumerClient
event_hub_connection_str = os.environ['EVENT_HUB_CONN_STR']
event_hub = os.environ['EVENT_HUB_NAME']
consumer = EventHubConsumerClient.from_connection_string(conn_str=event_hub_connection_str,
event_hub_path=event_hub)
# [END create_eventhub_consumer_client_from_conn_str_async]
# [START create_eventhub_consumer_client_async]
import os
from azure.eventhub import EventHubSharedKeyCredential
from azure.eventhub.aio import EventHubConsumerClient
hostname = os.environ['EVENT_HUB_HOSTNAME']
event_hub = os.environ['EVENT_HUB_NAME']
shared_access_policy = os.environ['EVENT_HUB_SAS_POLICY']
shared_access_key = os.environ['EVENT_HUB_SAS_KEY']
consumer = EventHubConsumerClient(host=hostname,
event_hub_path=event_hub,
credential=EventHubSharedKeyCredential(shared_access_policy, shared_access_key))
# [END create_eventhub_consumer_client_async]
return consumer
async def test_get_properties_with_auth_error_async(live_eventhub):
client = EventHubConsumerClient(
live_eventhub['hostname'], live_eventhub['event_hub'], '$default',
EventHubSharedKeyCredential(live_eventhub['key_name'],
"AaBbCcDdEeFf="))
async with client:
with pytest.raises(AuthenticationError) as e:
await client.get_eventhub_properties()
client = EventHubConsumerClient(
live_eventhub['hostname'], live_eventhub['event_hub'], '$default',
EventHubSharedKeyCredential("invalid", live_eventhub['access_key']))
async with client:
with pytest.raises(AuthenticationError) as e:
await client.get_eventhub_properties()
async def test_receive_end_of_stream_async(connstr_senders):
async def on_event(partition_context, event):
if partition_context.partition_id == "0":
assert event.body_as_str() == "Receiving only a single event"
assert list(event.body)[0] == b"Receiving only a single event"
on_event.called = True
assert event.partition_key == b'0'
event_str = str(event)
assert ", offset: " in event_str
assert ", sequence_number: " in event_str
assert ", enqueued_time: " in event_str
assert ", partition_key: 0" in event_str
on_event.called = False
connection_str, senders = connstr_senders
client = EventHubConsumerClient.from_connection_string(
connection_str, consumer_group='$default')
async with client:
task = asyncio.ensure_future(
client.receive(on_event,
partition_id="0",
starting_position="@latest"))
await asyncio.sleep(10)
assert on_event.called is False
senders[0].send(EventData(b"Receiving only a single event"),
partition_key='0')
await asyncio.sleep(10)
assert on_event.called is True
await task
async def example_eventhub_async_consumer_ops():
# [START eventhub_consumer_client_close_async]
import os
event_hub_connection_str = os.environ['EVENT_HUB_CONN_STR']
eventhub_name = os.environ['EVENT_HUB_NAME']
from azure.eventhub.aio import EventHubConsumerClient
consumer = EventHubConsumerClient.from_connection_string(
conn_str=event_hub_connection_str,
consumer_group='$Default',
eventhub_name=eventhub_name)
logger = logging.getLogger("azure.eventhub")
async def on_event(partition_context, event):
logger.info("Received event from partition: {}".format(
partition_context.partition_id))
# Do asynchronous ops on the received event
# The receive method is a coroutine which will be blocking when awaited.
# It can be executed in an async task for non-blocking behavior, and combined with the 'close' method.
recv_task = asyncio.ensure_future(consumer.receive(on_event=on_event))
await asyncio.sleep(3) # keep receiving for 3 seconds
recv_task.cancel() # stop receiving
# Close down the consumer handler explicitly.
await consumer.close()
async def connect(self, offset=None):
logger("EventHubApi: connect: thread={} {} loop={}".format(
threading.current_thread(),
id(threading.current_thread()),
id(asyncio.get_running_loop()),
))
if not offset:
offset = datetime.datetime.utcnow() - datetime.timedelta(
seconds=10)
self.received_events = asyncio.Queue()
# Create a consumer client for the event hub.
self.consumer_client = EventHubConsumerClient.from_connection_string(
self.eventhub_connection_string, consumer_group="$Default")
async def on_event(partition_context, event):
# this receives all events. they get filtered by device_id (if necessary) when
# pulled from the queue
await self.received_events.put(event)
async def listener():
await self.consumer_client.receive(on_event,
starting_position=offset)
self.listener_future = asyncio.ensure_future(listener())
logger("EventHubApi: Listener Created")
async def begin_processing(
self,
processed_messages: List[int],
):
"""
The function the concurrent tasks will be running.
Starts the shared checkpoint blob and the Event Hub client.
:param processed_messages: Counter of processed events.
:return:
"""
# Initialize the checkpoint store mechanism.
checkpoint_store = BlobCheckpointStore.from_connection_string(
self.checkpoint_conn_str,
self.checkpoint_container,
)
# Initialize the asynchronous Consumer Client.
client = EventHubConsumerClient.from_connection_string(
conn_str=self.hub_connection_str,
consumer_group=self.consumer_group,
eventhub_name=self.hub_name,
checkpoint_store=checkpoint_store,
idle_timeout=self.idle_timeout,
)
# Begin the pull.
# (See parent class.)
await self.time_constrained_pull(
client,
processed_messages,
)
def __init__(self, arguments):
super().__init__(arguments)
connection_string = self.get_from_env("AZURE_EVENTHUB_CONNECTION_STRING")
eventhub_name = self.get_from_env("AZURE_EVENTHUB_NAME")
self.async_producer = AsyncEventHubProducerClient.from_connection_string(connection_string, eventhub_name=eventhub_name)
self.consumer = EventHubConsumerClient.from_connection_string(connection_string, _EventHubTest.consumer_group, eventhub_name=eventhub_name)
self.async_consumer = AsyncEventHubConsumerClient.from_connection_string(connection_string, _EventHubTest.consumer_group, eventhub_name=eventhub_name)
async def test_loadbalancer_list_ownership_error(connstr_senders):
class ErrorCheckpointStore(InMemoryCheckpointStore):
async def list_ownership(self, fully_qualified_namespace,
eventhub_name, consumer_group):
raise RuntimeError("Test runtime error")
connection_str, senders = connstr_senders
for sender in senders:
sender.send(EventData("EventProcessor Test"))
eventhub_client = EventHubConsumerClient.from_connection_string(
connection_str, consumer_group='$default')
checkpoint_store = ErrorCheckpointStore()
event_processor = EventProcessor(eventhub_client=eventhub_client,
consumer_group='$default',
checkpoint_store=checkpoint_store,
event_handler=event_handler,
error_handler=None,
partition_initialize_handler=None,
partition_close_handler=None,
polling_interval=1)
task = asyncio.ensure_future(event_processor.start())
await asyncio.sleep(5)
assert event_processor._running is True
assert len(event_processor._tasks) == 0
await event_processor.stop()
# task.cancel()
await eventhub_client.close()
async def test_loadbalancer_balance(connstr_senders):
connection_str, senders = connstr_senders
for sender in senders:
sender.send(EventData("EventProcessor Test"))
eventhub_client = EventHubConsumerClient.from_connection_string(
connection_str, consumer_group='$default')
checkpoint_store = InMemoryCheckpointStore()
tasks = []
event_processor1 = EventProcessor(eventhub_client=eventhub_client,
consumer_group='$default',
checkpoint_store=checkpoint_store,
event_handler=event_handler,
error_handler=None,
partition_initialize_handler=None,
partition_close_handler=None,
polling_interval=1)
tasks.append(asyncio.ensure_future(event_processor1.start()))
await asyncio.sleep(3)
assert len(
event_processor1._tasks) == 2 # event_processor1 claims two partitions
event_processor2 = EventProcessor(eventhub_client=eventhub_client,
consumer_group='$default',
checkpoint_store=checkpoint_store,
event_handler=event_handler,
error_handler=None,
partition_initialize_handler=None,
partition_close_handler=None,
polling_interval=1)
tasks.append(asyncio.ensure_future(event_processor2.start()))
await asyncio.sleep(3)
assert len(event_processor1._tasks
) == 1 # two event processors balance. So each has 1 task
assert len(event_processor2._tasks) == 1
event_processor3 = EventProcessor(eventhub_client=eventhub_client,
consumer_group='$default',
checkpoint_store=checkpoint_store,
event_handler=event_handler,
error_handler=None,
partition_initialize_handler=None,
partition_close_handler=None,
polling_interval=1)
tasks.append(asyncio.ensure_future(event_processor3.start()))
await asyncio.sleep(3)
assert len(event_processor3._tasks) == 0
await event_processor3.stop()
await event_processor1.stop()
await asyncio.sleep(3)
assert len(
event_processor2._tasks
) == 2 # event_procesor2 takes another one after event_processor1 stops
await event_processor2.stop()
await eventhub_client.close()
def start_azureiothub():
global globMQTTClient
# send complete config list for all devices (as gateway, not only single PLC device)
globMQTTClient = MQTTClient(host=MQTT_HOST,
port=MQTT_PORT,
user=MQTT_USERNAME,
password=MQTT_PASSWORD,
tls_cert=MQTT_TLS_CERT)
# connect to MQTT
globMQTTClient.connect()
loop = asyncio.get_event_loop()
client = EventHubConsumerClient.from_connection_string(
conn_str=CONNECTION_STR,
consumer_group="$default",
# transport_type=TransportType.AmqpOverWebsocket, # uncomment it if you want to use web socket
# http_proxy={ # uncomment if you want to use proxy
# 'proxy_hostname': '127.0.0.1', # proxy hostname.
# 'proxy_port': 3128, # proxy port.
# 'username': '******',
# 'password': '******'
# }
)
try:
loop.run_until_complete(
client.receive_batch(on_event_batch=on_event_MQTTPayload,
on_error=on_error))
except KeyboardInterrupt:
print("Receiving has stopped.")
finally:
loop.run_until_complete(client.close())
loop.stop()
async def main():
# Connection string for namespace and name for event hub
VAR_CONN_STR = "Connection string for namespace"
VAR_EVENTHUB_NAME = "Name for event hub"
# Consumer group name
VAR_CONSUMER_GROUP = "$Default"
# Connectionstring for storage account and blob container name
VAR_STORAGE_CONN_STR = "Connectionstring for storage account"
VAR_BLOB_CONTAINER = "Blob container name"
# Storage service API version for Azure Stack Hub
STORAGE_SERVICE_API_VERSION = "2017-11-09"
# Create an Azure blob checkpoint store to store the checkpoints.
checkpoint_store = BlobCheckpointStore.from_connection_string(
VAR_STORAGE_CONN_STR,
container_name=VAR_BLOB_CONTAINER,
api_version=STORAGE_SERVICE_API_VERSION)
# Create a consumer client for the event hub.
client = EventHubConsumerClient.from_connection_string(
conn_str=VAR_CONN_STR,
consumer_group=VAR_CONSUMER_GROUP,
eventhub_name=VAR_EVENTHUB_NAME,
checkpoint_store=checkpoint_store)
async with client:
# Call the receive method.
await client.receive(on_event=on_event)
async def example_eventhub_async_consumer_receive_and_close():
# [START eventhub_consumer_client_close_async]
import os
event_hub_connection_str = os.environ['EVENT_HUB_CONN_STR']
eventhub_name = os.environ['EVENT_HUB_NAME']
from azure.eventhub.aio import EventHubConsumerClient
consumer = EventHubConsumerClient.from_connection_string(
conn_str=event_hub_connection_str,
consumer_group='$Default',
eventhub_name=
eventhub_name # EventHub name should be specified if it doesn't show up in connection string.
)
logger = logging.getLogger("azure.eventhub")
async def on_event(partition_context, event):
# Put your code here.
# If the operation is i/o intensive, async will have better performance.
logger.info("Received event from partition: {}".format(
partition_context.partition_id))
# The receive method is a coroutine which will be blocking when awaited.
# It can be executed in an async task for non-blocking behavior, and combined with the 'close' method.
recv_task = asyncio.ensure_future(consumer.receive(on_event=on_event))
await asyncio.sleep(3) # keep receiving for 3 seconds
recv_task.cancel() # stop receiving
# Close down the consumer handler explicitly.
await consumer.close()
async def example_eventhub_async_consumer_ops():
# [START eventhub_consumer_client_close_async]
import os
event_hub_connection_str = os.environ['EVENT_HUB_CONN_STR']
event_hub = os.environ['EVENT_HUB_NAME']
from azure.eventhub.aio import EventHubConsumerClient
consumer = EventHubConsumerClient.from_connection_string(
conn_str=event_hub_connection_str,
event_hub_path=event_hub
)
logger = logging.getLogger("azure.eventhub")
async def on_event(partition_context, event):
logger.info("Received event from partition: {}".format(partition_context.partition_id))
# Do asynchronous ops on received events
# The receive method is a coroutine method which can be called by `await consumer.receive(...)` and it will block.
# so execute it in an async task to better demonstrate how to stop the receiving by calling he close method.
recv_task = asyncio.ensure_future(consumer.receive(on_event=on_event, consumer_group='$Default'))
await asyncio.sleep(3) # keep receiving for 3 seconds
recv_task.cancel() # stop receiving
# Close down the consumer handler explicitly.
await consumer.close()
async def main():
consumer_client = EventHubConsumerClient.from_connection_string(
conn_str=CONNECTION_STR,
consumer_group='$Default',
eventhub_name=EVENTHUB_NAME,
http_proxy=HTTP_PROXY)
producer_client = EventHubProducerClient.from_connection_string(
conn_str=CONNECTION_STR,
eventhub_name=EVENTHUB_NAME,
http_proxy=HTTP_PROXY)
async with producer_client:
event_data_batch = await producer_client.create_batch(
max_size_in_bytes=10000)
while True:
try:
event_data_batch.add(
EventData('Message inside EventBatchData'))
except ValueError:
# EventDataBatch object reaches max_size.
# New EventDataBatch object can be created here to send more data.
break
await producer_client.send_batch(event_data_batch)
print('Finished sending.')
async with consumer_client:
await consumer_client.receive(on_event=on_event)
print('Finished receiving.')
async def main():
# Create an Azure blob checkpoint store to store the checkpoints.
checkpoint_store = BlobCheckpointStore.from_connection_string(checkpoint_conn_str, checkpoint_container_name)
# Create a consumer client for the event hub.
client = EventHubConsumerClient.from_connection_string(conn_str, consumer_group="$Default", eventhub_name=event_hub_name, checkpoint_store=checkpoint_store)
client_future = client.receive(on_event=on_event, starting_position=-1)
print('client is listening')
producer = EventHubProducerClient.from_connection_string(conn_str=conn_str, eventhub_name=event_hub_name)
async with producer:
# Create a batch.
event_data_batch = await producer.create_batch()
# Add events to the batch.
event_data_batch.add(EventData('First event '))
event_data_batch.add(EventData('Second event'))
event_data_batch.add(EventData('Third event'))
# Send the batch of events to the event hub.
await producer.send_batch(event_data_batch)
print('sent events')
await client_future
def consume_preprocessed_message():
consumer = Consumer2()
consumer_client = EventHubConsumerClient.from_connection_string(
conn_str=consumer.EVENTHUB_CONNECTION_STRING,
consumer_group='$Default',
eventhub_name=consumer.EVENTHUB_NAME,
)
print('Consumer will keep receiving for {} seconds, start time is {}.'.
format(consumer.RECEIVE_DURATION, time.time()))
try:
thread = threading.Thread(
target=consumer_client.receive,
kwargs={
"on_event": consumer.on_event,
"on_partition_initialize": consumer.on_partition_initialize,
"on_partition_close": consumer.on_partition_close,
"on_error": consumer.on_error,
"starting_position":
"-1", # "-1" is from the beginning of the partition.
})
thread.daemon = True
thread.start()
time.sleep(consumer.RECEIVE_DURATION)
consumer_client.close()
thread.join()
except KeyboardInterrupt:
print('Stop receiving.')
print('Consumer2 has stopped receiving, end time is {}.'.format(
time.time()))
async def main():
client = EventHubConsumerClient.from_connection_string(
conn_str=CONNECTION_STR,
eventhub_name=EVENTHUB_NAME,
)
async with client:
await receive(client)
async def test_receive_with_invalid_param_async(live_eventhub_config,
invalid_place):
eventhub_config = live_eventhub_config.copy()
if invalid_place != "partition":
eventhub_config[invalid_place] = "invalid " + invalid_place
conn_str = live_eventhub_config["connection_str"].format(
eventhub_config['hostname'], eventhub_config['key_name'],
eventhub_config['access_key'], eventhub_config['event_hub'])
client = EventHubConsumerClient.from_connection_string(
conn_str, consumer_group='$default', retry_total=0)
async def on_event(partition_context, event):
pass
async with client:
if invalid_place == "partition":
task = asyncio.ensure_future(
client.receive(on_event,
partition_id=invalid_place,
starting_position="-1"))
else:
task = asyncio.ensure_future(
client.receive(on_event,
partition_id="0",
starting_position="-1"))
await asyncio.sleep(10)
assert len(client._event_processors) == 1
await task
async def test_receive_connection_idle_timeout_and_reconnect_async(
connstr_senders):
connection_str, senders = connstr_senders
client = EventHubConsumerClient.from_connection_string(
conn_str=connection_str, consumer_group='$default', idle_timeout=10)
async def on_event_received(event):
on_event_received.event = event
async with client:
consumer = client._create_consumer("$default", "0", "-1",
on_event_received)
async with consumer:
await consumer._open_with_retry()
time.sleep(11)
ed = EventData("Event")
senders[0].send(ed)
await consumer._handler.do_work_async()
assert consumer._handler._connection._state == c_uamqp.ConnectionState.DISCARDING
await consumer.receive(batch=False,
max_batch_size=1,
max_wait_time=10)
assert on_event_received.event.body_as_str() == "Event"
async def test_receive_over_websocket_async(connstr_senders):
app_prop = {"raw_prop": "raw_value"}
async def on_event(partition_context, event):
on_event.received.append(event)
on_event.app_prop = event.properties
on_event.received = []
on_event.app_prop = None
connection_str, senders = connstr_senders
client = EventHubConsumerClient.from_connection_string(
connection_str,
consumer_group='$default',
transport_type=TransportType.AmqpOverWebsocket)
event_list = []
for i in range(5):
ed = EventData("Event Number {}".format(i))
ed.properties = app_prop
event_list.append(ed)
senders[0].send(event_list)
async with client:
task = asyncio.ensure_future(
client.receive(on_event, partition_id="0", starting_position="-1"))
await asyncio.sleep(10)
await task
assert len(on_event.received) == 5
for ed in on_event.received:
assert ed.properties[b"raw_prop"] == b"raw_value"
async def test_receive_connection_idle_timeout_and_reconnect_async(
connstr_senders):
connection_str, senders = connstr_senders
client = EventHubConsumerClient.from_connection_string(
conn_str=connection_str, consumer_group='$default', idle_timeout=10)
async def on_event_received(event):
on_event_received.event = event
async with client:
consumer = client._create_consumer("$default",
"0",
"-1",
on_event_received=on_event_received)
async with consumer:
await consumer._open_with_retry()
time.sleep(11)
ed = EventData("Event")
senders[0].send(ed)
await consumer._handler.do_work_async()
assert consumer._handler._connection._state == c_uamqp.ConnectionState.DISCARDING
duration = 10
now_time = time.time()
end_time = now_time + duration
while now_time < end_time:
await consumer.receive()
await asyncio.sleep(0.01)
now_time = time.time()
assert on_event_received.event.body_as_str() == "Event"
async def test_partition_processor(connstr_senders):
lock = asyncio.Lock()
event_map = {}
checkpoint = None
close_reason = None
error = None
async def partition_initialize_handler(partition_context):
partition_initialize_handler.partition_context = partition_context
async def event_handler(partition_context, event):
async with lock:
if event:
nonlocal checkpoint, event_map
event_map[partition_context.partition_id] = event_map.get(
partition_context.partition_id, 0) + 1
offset, sn = event.offset, event.sequence_number
checkpoint = (offset, sn)
await partition_context.update_checkpoint(event)
async def partition_close_handler(partition_context, reason):
assert partition_context and reason
nonlocal close_reason
close_reason = reason
async def error_handler(partition_context, err):
assert partition_context and err
nonlocal error
error = err
connection_str, senders = connstr_senders
for sender in senders:
sender.send(EventData("EventProcessor Test"))
eventhub_client = EventHubConsumerClient.from_connection_string(
connection_str, consumer_group='$default')
checkpoint_store = InMemoryCheckpointStore()
event_processor = EventProcessor(
eventhub_client=eventhub_client,
consumer_group='$default',
checkpoint_store=checkpoint_store,
event_handler=event_handler,
error_handler=error_handler,
partition_initialize_handler=partition_initialize_handler,
partition_close_handler=partition_close_handler,
polling_interval=1)
task = asyncio.ensure_future(event_processor.start())
await asyncio.sleep(10)
assert len(event_processor._tasks) == 2
await event_processor.stop()
task.cancel()
await eventhub_client.close()
assert event_map['0'] == 1 and event_map['1'] == 1
assert checkpoint is not None
assert close_reason == CloseReason.SHUTDOWN
assert error is None
assert partition_initialize_handler.partition_context
async def test_receive_with_event_position_async(connstr_senders, position,
inclusive, expected_result):
async def on_event(partition_context, event):
assert partition_context.last_enqueued_event_properties.get(
'sequence_number') == event.sequence_number
assert partition_context.last_enqueued_event_properties.get(
'offset') == event.offset
assert partition_context.last_enqueued_event_properties.get(
'enqueued_time') == event.enqueued_time
assert partition_context.last_enqueued_event_properties.get(
'retrieval_time') is not None
if position == "offset":
on_event.event_position = event.offset
elif position == "sequence":
on_event.event_position = event.sequence_number
else:
on_event.event_position = event.enqueued_time
on_event.event = event
on_event.event_position = None
connection_str, senders = connstr_senders
senders[0].send(EventData(b"Inclusive"))
client = EventHubConsumerClient.from_connection_string(
connection_str, consumer_group='$default')
async with client:
task = asyncio.ensure_future(
client.receive(on_event,
starting_position="-1",
starting_position_inclusive=inclusive,
track_last_enqueued_event_properties=True))
await asyncio.sleep(10)
assert on_event.event_position is not None
await task
senders[0].send(EventData(expected_result))
client2 = EventHubConsumerClient.from_connection_string(
connection_str, consumer_group='$default')
async with client2:
task = asyncio.ensure_future(
client2.receive(on_event,
starting_position=on_event.event_position,
starting_position_inclusive=inclusive,
track_last_enqueued_event_properties=True))
await asyncio.sleep(10)
assert on_event.event.body_as_str() == expected_result
await task
async def test_get_partition_ids(live_eventhub):
client = EventHubConsumerClient(
live_eventhub['hostname'], live_eventhub['event_hub'], '$default',
EventHubSharedKeyCredential(live_eventhub['key_name'],
live_eventhub['access_key']))
async with client:
partition_ids = await client.get_partition_ids()
assert partition_ids == ['0', '1']
