class EventHubConsumer(ConsumerProducerMixin): # pylint:disable=too-many-instance-attributes
"""
A consumer responsible for reading EventData from a specific Event Hub
partition and as a member of a specific consumer group.
A consumer may be exclusive, which asserts ownership over the partition for the consumer
group to ensure that only one consumer from that group is reading the from the partition.
These exclusive consumers are sometimes referred to as "Epoch Consumers."
A consumer may also be non-exclusive, allowing multiple consumers from the same consumer
group to be actively reading events from the partition. These non-exclusive consumers are
sometimes referred to as "Non-Epoch Consumers."
Please use the method `create_consumer` on `EventHubClient` for creating `EventHubConsumer`.
"""
_timeout = 0
_epoch_symbol = b'com.microsoft:epoch'
_timeout_symbol = b'com.microsoft:timeout'
_receiver_runtime_metric_symbol = b'com.microsoft:enable-receiver-runtime-metric'
def __init__( # pylint: disable=super-init-not-called
self, client, source, **kwargs):
"""
Instantiate an async consumer. EventHubConsumer should be instantiated by calling the `create_consumer` method
in EventHubClient.
:param client: The parent EventHubClientAsync.
:type client: ~azure.eventhub.aio.EventHubClientAsync
:param source: The source EventHub from which to receive events.
:type source: ~uamqp.address.Source
:param event_position: The position from which to start receiving.
:type event_position: ~azure.eventhub.common.EventPosition
:param prefetch: The number of events to prefetch from the service
for processing. Default is 300.
:type prefetch: int
:param owner_level: The priority of the exclusive consumer. An exclusive
consumer will be created if owner_level is set.
:type owner_level: int
:param track_last_enqueued_event_properties: Indicates whether or not the consumer should request information
on the last enqueued event on its associated partition, and track that information as events are received.
When information about the partition's last enqueued event is being tracked, each event received from the
Event Hubs service will carry metadata about the partition. This results in a small amount of additional
network bandwidth consumption that is generally a favorable trade-off when considered against periodically
making requests for partition properties using the Event Hub client.
It is set to `False` by default.
:type track_last_enqueued_event_properties: bool
:param loop: An event loop.
"""
event_position = kwargs.get("event_position", None)
prefetch = kwargs.get("prefetch", 300)
owner_level = kwargs.get("owner_level", None)
keep_alive = kwargs.get("keep_alive", None)
auto_reconnect = kwargs.get("auto_reconnect", True)
track_last_enqueued_event_properties = kwargs.get(
"track_last_enqueued_event_properties", False)
loop = kwargs.get("loop", None)
super(EventHubConsumer, self).__init__()
self._loop = loop or asyncio.get_event_loop()
self._client = client
self._source = source
self._offset = event_position
self._messages_iter = None
self._prefetch = prefetch
self._owner_level = owner_level
self._keep_alive = keep_alive
self._auto_reconnect = auto_reconnect
self._retry_policy = errors.ErrorPolicy(
max_retries=self._client._config.max_retries,
on_error=_error_handler) # pylint:disable=protected-access
self._reconnect_backoff = 1
self._link_properties = {}
partition = self._source.split('/')[-1]
self._partition = partition
self._name = "EHReceiver-{}-partition{}".format(
uuid.uuid4(), partition)
if owner_level:
self._link_properties[types.AMQPSymbol(
self._epoch_symbol)] = types.AMQPLong(int(owner_level))
link_property_timeout_ms = (self._client._config.receive_timeout
or self._timeout) * 1000 # pylint:disable=protected-access
self._link_properties[types.AMQPSymbol(
self._timeout_symbol)] = types.AMQPLong(
int(link_property_timeout_ms))
self._handler = None
self._track_last_enqueued_event_properties = track_last_enqueued_event_properties
self._last_enqueued_event_properties = {}
def __aiter__(self):
return self
async def __anext__(self):
retried_times = 0
last_exception = None
while retried_times < self._client._config.max_retries: # pylint:disable=protected-access
try:
await self._open()
if not self._messages_iter:
self._messages_iter = self._handler.receive_messages_iter_async(
)
message = await self._messages_iter.__anext__()
event_data = EventData._from_message(message) # pylint:disable=protected-access
event_data._trace_link_message() # pylint:disable=protected-access
self._offset = EventPosition(event_data.offset,
inclusive=False)
retried_times = 0
if self._track_last_enqueued_event_properties:
self._last_enqueued_event_properties = event_data._get_last_enqueued_event_properties(
) # pylint:disable=protected-access
return event_data
except Exception as exception: # pylint:disable=broad-except
last_exception = await self._handle_exception(exception)
await self._client._try_delay(
retried_times=retried_times,
last_exception=last_exception, # pylint:disable=protected-access
entity_name=self._name)
retried_times += 1
log.info("%r operation has exhausted retry. Last exception: %r.",
self._name, last_exception)
raise last_exception
def _create_handler(self):
source = Source(self._source)
if self._offset is not None:
source.set_filter(self._offset._selector()) # pylint:disable=protected-access
if StrictVersion(uamqp.__version__) < StrictVersion(
"1.2.3"): # backward compatible until uamqp 1.2.3 released
desired_capabilities = {}
elif self._track_last_enqueued_event_properties:
symbol_array = [
types.AMQPSymbol(self._receiver_runtime_metric_symbol)
]
desired_capabilities = {
"desired_capabilities":
utils.data_factory(types.AMQPArray(symbol_array))
}
else:
desired_capabilities = {"desired_capabilities": None}
self._handler = ReceiveClientAsync(
source,
auth=self._client._create_auth(), # pylint:disable=protected-access
debug=self._client._config.network_tracing, # pylint:disable=protected-access
prefetch=self._prefetch,
link_properties=self._link_properties,
timeout=self._timeout,
error_policy=self._retry_policy,
keep_alive_interval=self._keep_alive,
client_name=self._name,
receive_settle_mode=uamqp.constants.ReceiverSettleMode.
ReceiveAndDelete,
auto_complete=False,
properties=self._client._create_properties( # pylint:disable=protected-access
self._client._config.user_agent), # pylint:disable=protected-access
**desired_capabilities, # pylint:disable=protected-access
loop=self._loop)
self._messages_iter = None
async def _open_with_retry(self):
return await self._do_retryable_operation(self._open,
operation_need_param=False)
async def _receive(self, timeout_time=None, max_batch_size=None, **kwargs):
last_exception = kwargs.get("last_exception")
data_batch = []
await self._open()
remaining_time = timeout_time - time.time()
if remaining_time <= 0.0:
if last_exception:
log.info("%r receive operation timed out. (%r)", self._name,
last_exception)
raise last_exception
return data_batch
remaining_time_ms = 1000 * remaining_time
message_batch = await self._handler.receive_message_batch_async(
max_batch_size=max_batch_size, timeout=remaining_time_ms)
for message in message_batch:
event_data = EventData._from_message(message) # pylint:disable=protected-access
data_batch.append(event_data)
event_data._trace_link_message() # pylint:disable=protected-access
if data_batch:
self._offset = EventPosition(data_batch[-1].offset)
if self._track_last_enqueued_event_properties and data_batch:
self._last_enqueued_event_properties = data_batch[
-1]._get_last_enqueued_event_properties() # pylint:disable=protected-access
return data_batch
async def _receive_with_retry(self,
timeout=None,
max_batch_size=None,
**kwargs):
return await self._do_retryable_operation(
self._receive,
timeout=timeout,
max_batch_size=max_batch_size,
**kwargs)
@property
def last_enqueued_event_properties(self):
"""
The latest enqueued event information. This property will be updated each time an event is received when
the receiver is created with `track_last_enqueued_event_properties` being `True`.
The dict includes following information of the partition:
- `sequence_number`
- `offset`
- `enqueued_time`
- `retrieval_time`
:rtype: dict or None
"""
return self._last_enqueued_event_properties if self._track_last_enqueued_event_properties else None
@property
def queue_size(self):
# type: () -> int
"""
The current size of the unprocessed Event queue.
:rtype: int
"""
# pylint: disable=protected-access
if self._handler._received_messages:
return self._handler._received_messages.qsize()
return 0
async def receive(self, *, max_batch_size=None, timeout=None):
# type: (Any, int, float) -> List[EventData]
"""
Receive events asynchronously from the EventHub.
:param max_batch_size: Receive a batch of events. Batch size will
be up to the maximum specified, but will return as soon as service
returns no new events. If combined with a timeout and no events are
retrieve before the time, the result will be empty. If no batch
size is supplied, the prefetch size will be the maximum.
:type max_batch_size: int
:param timeout: The maximum wait time to build up the requested message count for the batch.
If not specified, the default wait time specified when the consumer was created will be used.
:type timeout: float
:rtype: list[~azure.eventhub.common.EventData]
:raises: ~azure.eventhub.AuthenticationError, ~azure.eventhub.ConnectError, ~azure.eventhub.ConnectionLostError,
~azure.eventhub.EventHubError
Example:
.. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py
:start-after: [START eventhub_client_async_receive]
:end-before: [END eventhub_client_async_receive]
:language: python
:dedent: 4
:caption: Receives events asynchronously
"""
self._check_closed()
timeout = timeout or self._client._config.receive_timeout # pylint:disable=protected-access
max_batch_size = max_batch_size or min(
self._client._config.max_batch_size, self._prefetch) # pylint:disable=protected-access
return await self._receive_with_retry(timeout=timeout,
max_batch_size=max_batch_size)
async def close(self):
# type: () -> None
"""
Close down the handler. If the handler has already closed,
this will be a no op.
Example:
.. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py
:start-after: [START eventhub_client_async_receiver_close]
:end-before: [END eventhub_client_async_receiver_close]
:language: python
:dedent: 4
:caption: Close down the handler.
"""
await super(EventHubConsumer, self).close()
class EventHubConsumer(ConsumerProducerMixin):
"""
A consumer responsible for reading EventData from a specific Event Hub
partition and as a member of a specific consumer group.
A consumer may be exclusive, which asserts ownership over the partition for the consumer
group to ensure that only one consumer from that group is reading the from the partition.
These exclusive consumers are sometimes referred to as "Epoch Consumers."
A consumer may also be non-exclusive, allowing multiple consumers from the same consumer
group to be actively reading events from the partition. These non-exclusive consumers are
sometimes referred to as "Non-Epoch Consumers."
"""
timeout = 0
_epoch = b'com.microsoft:epoch'
_timeout = b'com.microsoft:timeout'
def __init__( # pylint: disable=super-init-not-called
self, client, source, **kwargs):
"""
Instantiate an async consumer. EventHubConsumer should be instantiated by calling the `create_consumer` method
in EventHubClient.
:param client: The parent EventHubClientAsync.
:type client: ~azure.eventhub.aio.EventHubClientAsync
:param source: The source EventHub from which to receive events.
:type source: ~uamqp.address.Source
:param event_position: The position from which to start receiving.
:type event_position: ~azure.eventhub.common.EventPosition
:param prefetch: The number of events to prefetch from the service
for processing. Default is 300.
:type prefetch: int
:param owner_level: The priority of the exclusive consumer. It will an exclusive
consumer if owner_level is set.
:type owner_level: int
:param loop: An event loop.
"""
event_position = kwargs.get("event_position", None)
prefetch = kwargs.get("prefetch", 300)
owner_level = kwargs.get("owner_level", None)
keep_alive = kwargs.get("keep_alive", None)
auto_reconnect = kwargs.get("auto_reconnect", True)
loop = kwargs.get("loop", None)
super(EventHubConsumer, self).__init__()
self.loop = loop or asyncio.get_event_loop()
self.running = False
self.client = client
self.source = source
self.offset = event_position
self.messages_iter = None
self.prefetch = prefetch
self.owner_level = owner_level
self.keep_alive = keep_alive
self.auto_reconnect = auto_reconnect
self.retry_policy = errors.ErrorPolicy(
max_retries=self.client.config.max_retries,
on_error=_error_handler)
self.reconnect_backoff = 1
self.redirected = None
self.error = None
self._link_properties = {}
partition = self.source.split('/')[-1]
self.partition = partition
self.name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition)
if owner_level:
self._link_properties[types.AMQPSymbol(
self._epoch)] = types.AMQPLong(int(owner_level))
link_property_timeout_ms = (self.client.config.receive_timeout
or self.timeout) * 1000
self._link_properties[types.AMQPSymbol(
self._timeout)] = types.AMQPLong(int(link_property_timeout_ms))
self._handler = None
def __aiter__(self):
return self
async def __anext__(self):
max_retries = self.client.config.max_retries
retry_count = 0
while True:
try:
await self._open()
if not self.messages_iter:
self.messages_iter = self._handler.receive_messages_iter_async(
)
message = await self.messages_iter.__anext__()
event_data = EventData._from_message(message)
self.offset = EventPosition(event_data.offset, inclusive=False)
retry_count = 0
return event_data
except Exception as exception:
await self._handle_exception(exception, retry_count,
max_retries)
retry_count += 1
def _create_handler(self):
alt_creds = {
"username": self.client._auth_config.get("iot_username"),
"password": self.client._auth_config.get("iot_password")
}
source = Source(self.source)
if self.offset is not None:
source.set_filter(self.offset._selector())
self._handler = ReceiveClientAsync(
source,
auth=self.client.get_auth(**alt_creds),
debug=self.client.config.network_tracing,
prefetch=self.prefetch,
link_properties=self._link_properties,
timeout=self.timeout,
error_policy=self.retry_policy,
keep_alive_interval=self.keep_alive,
client_name=self.name,
properties=self.client._create_properties(
self.client.config.user_agent), # pylint: disable=protected-access
loop=self.loop)
self.messages_iter = None
async def _redirect(self, redirect):
self.messages_iter = None
await super(EventHubConsumer, self)._redirect(redirect)
async def _open(self, timeout_time=None):
"""
Open the EventHubConsumer using the supplied connection.
If the handler has previously been redirected, the redirect
context will be used to create a new handler before opening it.
"""
# pylint: disable=protected-access
if not self.running and self.redirected:
self.client._process_redirect_uri(self.redirected)
self.source = self.redirected.address
await super(EventHubConsumer, self)._open(timeout_time)
async def _receive(self, timeout_time=None, max_batch_size=None, **kwargs):
last_exception = kwargs.get("last_exception")
data_batch = kwargs.get("data_batch")
await self._open(timeout_time)
remaining_time = timeout_time - time.time()
if remaining_time <= 0.0:
if last_exception:
log.info("%r receive operation timed out. (%r)", self.name,
last_exception)
raise last_exception
return data_batch
remaining_time_ms = 1000 * remaining_time
message_batch = await self._handler.receive_message_batch_async(
max_batch_size=max_batch_size, timeout=remaining_time_ms)
for message in message_batch:
event_data = EventData._from_message(message)
self.offset = EventPosition(event_data.offset)
data_batch.append(event_data)
return data_batch
@_retry_decorator
async def _receive_with_try(self,
timeout_time=None,
max_batch_size=None,
**kwargs):
return await self._receive(timeout_time=timeout_time,
max_batch_size=max_batch_size,
**kwargs)
@property
def queue_size(self):
# type: () -> int
"""
The current size of the unprocessed Event queue.
:rtype: int
"""
# pylint: disable=protected-access
if self._handler._received_messages:
return self._handler._received_messages.qsize()
return 0
async def receive(self, *, max_batch_size=None, timeout=None):
# type: (int, float) -> List[EventData]
"""
Receive events asynchronously from the EventHub.
:param max_batch_size: Receive a batch of events. Batch size will
be up to the maximum specified, but will return as soon as service
returns no new events. If combined with a timeout and no events are
retrieve before the time, the result will be empty. If no batch
size is supplied, the prefetch size will be the maximum.
:type max_batch_size: int
:param timeout: The maximum wait time to build up the requested message count for the batch.
If not specified, the default wait time specified when the consumer was created will be used.
:type timeout: float
:rtype: list[~azure.eventhub.common.EventData]
:raises: ~azure.eventhub.AuthenticationError, ~azure.eventhub.ConnectError, ~azure.eventhub.ConnectionLostError,
~azure.eventhub.EventHubError
Example:
.. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py
:start-after: [START eventhub_client_async_receive]
:end-before: [END eventhub_client_async_receive]
:language: python
:dedent: 4
:caption: Receives events asynchronously
"""
self._check_closed()
timeout = timeout or self.client.config.receive_timeout
max_batch_size = max_batch_size or min(
self.client.config.max_batch_size, self.prefetch)
data_batch = [] # type: List[EventData]
return await self._receive_with_try(timeout=timeout,
max_batch_size=max_batch_size,
data_batch=data_batch)
async def close(self, exception=None):
# type: (Exception) -> None
"""
Close down the handler. If the handler has already closed,
this will be a no op. An optional exception can be passed in to
indicate that the handler was shutdown due to error.
:param exception: An optional exception if the handler is closing
due to an error.
:type exception: Exception
Example:
.. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py
:start-after: [START eventhub_client_async_receiver_close]
:end-before: [END eventhub_client_async_receiver_close]
:language: python
:dedent: 4
:caption: Close down the handler.
"""
self.running = False
if self.error:
return
if isinstance(exception, errors.LinkRedirect):
self.redirected = exception
elif isinstance(exception, EventHubError):
self.error = exception
elif isinstance(exception,
(errors.LinkDetach, errors.ConnectionClose)):
self.error = ConnectError(str(exception), exception)
elif exception:
self.error = EventHubError(str(exception))
else:
self.error = EventHubError("This receive handler is now closed.")
await self._handler.close_async()
