def convert_iothub_to_eventhub_conn_str(iothub_conn_str):
hostname, shared_access_key_name, shared_access_key = parse_iot_conn_str(iothub_conn_str)
iot_hub_name = hostname.split(".")[0]
operation = '/messages/events/ConsumerGroups/{}/Partitions/{}'.format('$Default', 0)
username = '******'.format(shared_access_key_name, iot_hub_name)
sas_token = generate_sas_token(hostname, shared_access_key_name, shared_access_key)
uri = 'amqps://{}:{}@{}{}'.format(quote_plus(username),
quote_plus(sas_token), hostname, operation)
source_uri = Source(uri)
receive_client = ReceiveClient(source_uri)
try:
receive_client.receive_message_batch(max_batch_size=1)
except LinkRedirect as redirect:
# Once a redirect error is received, close the original client and recreate a new one to the re-directed address
receive_client.close()
fully_qualified_name = redirect.hostname.decode("utf-8")
return "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format(
fully_qualified_name,
shared_access_key_name,
shared_access_key,
iot_hub_name
)
except Exception as exp:
raise ValueError(
"{} is not an invalid IoT Hub connection string. The underlying exception is {}".format(
iothub_conn_str,
exp,
)
)
def convert_iothub_to_eventhub_conn_str(iothub_conn_str):
hostname, shared_access_key_name, shared_access_key = parse_iot_conn_str(
iothub_conn_str)
iot_hub_name = hostname.split(".")[0]
operation = '/messages/events/ConsumerGroups/{}/Partitions/{}'.format(
'$Default', 0)
username = '******'.format(shared_access_key_name, iot_hub_name)
sas_token = generate_sas_token(hostname, shared_access_key_name,
shared_access_key)
uri = 'amqps://{}:{}@{}{}'.format(quote_plus(username),
quote_plus(sas_token), hostname,
operation)
source_uri = Source(uri)
receive_client = ReceiveClient(source_uri)
try:
receive_client.receive_message_batch(max_batch_size=1)
except LinkRedirect as redirect:
# Once a redirect error is received, close the original client and recreate a new one to the re-directed address
receive_client.close()
fully_qualified_name = redirect.hostname.decode("utf-8")
# Use regular expression to parse the Event Hub name from the IoT Hub redirection address
if redirect.address:
# The regex searches for the Event Hub compatible name in the redirection address. The name is nested in
# between the port and 'ConsumerGroups'.
# (ex. "...servicebus.windows.net:12345/<Event Hub name>/ConsumerGroups/...").
# The regex matches string ':<digits>/', then any characters, then the string '/ConsumerGroups'.
iot_hub_name = re.search(
":\d+\/.*/ConsumerGroups",
str(redirect.address)).group(0).split("/")[1]
return "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format(
fully_qualified_name, shared_access_key_name, shared_access_key,
iot_hub_name)
except Exception as exp:
raise ValueError(
"{} is not an invalid IoT Hub connection string. The underlying exception is {}"
.format(
iothub_conn_str,
exp,
))
class Receiver(object):
"""
Implements a Receiver.
"""
timeout = 0
_epoch = b'com.microsoft:epoch'
def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_alive=None, auto_reconnect=True):
"""
Instantiate a receiver.
:param client: The parent EventHubClient.
:type client: ~azure.eventhub.client.EventHubClient
:param source: The source EventHub from which to receive events.
:type source: str
:param prefetch: The number of events to prefetch from the service
for processing. Default is 300.
:type prefetch: int
:param epoch: An optional epoch value.
:type epoch: int
"""
self.running = False
self.client = client
self.source = source
self.offset = offset
self.prefetch = prefetch
self.epoch = epoch
self.keep_alive = keep_alive
self.auto_reconnect = auto_reconnect
self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler)
self.reconnect_backoff = 1
self.properties = None
self.redirected = None
self.error = None
partition = self.source.split('/')[-1]
self.name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition)
source = Source(self.source)
if self.offset is not None:
source.set_filter(self.offset.selector())
if epoch:
self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))}
self._handler = ReceiveClient(
source,
auth=self.client.get_auth(),
debug=self.client.debug,
prefetch=self.prefetch,
link_properties=self.properties,
timeout=self.timeout,
error_policy=self.retry_policy,
keep_alive_interval=self.keep_alive,
client_name=self.name,
properties=self.client.create_properties())
def open(self):
"""
Open the Receiver using the supplied conneciton.
If the handler has previously been redirected, the redirect
context will be used to create a new handler before opening it.
:param connection: The underlying client shared connection.
:type: connection: ~uamqp.connection.Connection
"""
# pylint: disable=protected-access
self.running = True
if self.redirected:
self.source = self.redirected.address
source = Source(self.source)
if self.offset is not None:
source.set_filter(self.offset.selector())
alt_creds = {
"username": self.client._auth_config.get("iot_username"),
"password":self.client._auth_config.get("iot_password")}
self._handler = ReceiveClient(
source,
auth=self.client.get_auth(**alt_creds),
debug=self.client.debug,
prefetch=self.prefetch,
link_properties=self.properties,
timeout=self.timeout,
error_policy=self.retry_policy,
keep_alive_interval=self.keep_alive,
client_name=self.name,
properties=self.client.create_properties())
self._handler.open()
while not self._handler.client_ready():
time.sleep(0.05)
def _reconnect(self): # pylint: disable=too-many-statements
# pylint: disable=protected-access
alt_creds = {
"username": self.client._auth_config.get("iot_username"),
"password": self.client._auth_config.get("iot_password")}
self._handler.close()
source = Source(self.source)
if self.offset is not None:
source.set_filter(self.offset.selector())
self._handler = ReceiveClient(
source,
auth=self.client.get_auth(**alt_creds),
debug=self.client.debug,
prefetch=self.prefetch,
link_properties=self.properties,
timeout=self.timeout,
error_policy=self.retry_policy,
keep_alive_interval=self.keep_alive,
client_name=self.name,
properties=self.client.create_properties())
try:
self._handler.open()
while not self._handler.client_ready():
time.sleep(0.05)
return True
except errors.TokenExpired as shutdown:
log.info("Receiver disconnected due to token expiry. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
self.close(exception=error)
raise error
except (errors.LinkDetach, errors.ConnectionClose) as shutdown:
if shutdown.action.retry and self.auto_reconnect:
log.info("Receiver detached. Attempting reconnect.")
return False
log.info("Receiver detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
self.close(exception=error)
raise error
except errors.MessageHandlerError as shutdown:
if self.auto_reconnect:
log.info("Receiver detached. Attempting reconnect.")
return False
log.info("Receiver detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
self.close(exception=error)
raise error
except errors.AMQPConnectionError as shutdown:
if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect:
log.info("Receiver couldn't authenticate. Attempting reconnect.")
return False
log.info("Receiver connection error (%r). Shutting down.", shutdown)
error = EventHubError(str(shutdown))
self.close(exception=error)
raise error
except Exception as e:
log.info("Unexpected error occurred (%r). Shutting down.", e)
error = EventHubError("Receiver reconnect failed: {}".format(e))
self.close(exception=error)
raise error
def reconnect(self):
"""If the Receiver was disconnected from the service with
a retryable error - attempt to reconnect."""
while not self._reconnect():
time.sleep(self.reconnect_backoff)
def get_handler_state(self):
"""
Get the state of the underlying handler with regards to start
up processes.
:rtype: ~uamqp.constants.MessageReceiverState
"""
# pylint: disable=protected-access
return self._handler._message_receiver.get_state()
def has_started(self):
"""
Whether the handler has completed all start up processes such as
establishing the connection, session, link and authentication, and
is not ready to process messages.
**This function is now deprecated and will be removed in v2.0+.**
:rtype: bool
"""
# pylint: disable=protected-access
timeout = False
auth_in_progress = False
if self._handler._connection.cbs:
timeout, auth_in_progress = self._handler._auth.handle_token()
if timeout:
raise EventHubError("Authorization timeout.")
if auth_in_progress:
return False
if not self._handler._client_ready():
return False
return True
def close(self, exception=None):
"""
Close down the handler. If the handler has already closed,
this will be a no op. An optional exception can be passed in to
indicate that the handler was shutdown due to error.
:param exception: An optional exception if the handler is closing
due to an error.
:type exception: Exception
"""
self.running = False
if self.error:
return
if isinstance(exception, errors.LinkRedirect):
self.redirected = exception
elif isinstance(exception, EventHubError):
self.error = exception
elif exception:
self.error = EventHubError(str(exception))
else:
self.error = EventHubError("This receive handler is now closed.")
self._handler.close()
@property
def queue_size(self):
"""
The current size of the unprocessed Event queue.
:rtype: int
"""
# pylint: disable=protected-access
if self._handler._received_messages:
return self._handler._received_messages.qsize()
return 0
def receive(self, max_batch_size=None, timeout=None):
"""
Receive events from the EventHub.
:param max_batch_size: Receive a batch of events. Batch size will
be up to the maximum specified, but will return as soon as service
returns no new events. If combined with a timeout and no events are
retrieve before the time, the result will be empty. If no batch
size is supplied, the prefetch size will be the maximum.
:type max_batch_size: int
:rtype: list[~azure.eventhub.common.EventData]
"""
if self.error:
raise self.error
if not self.running:
raise ValueError("Unable to receive until client has been started.")
data_batch = []
try:
timeout_ms = 1000 * timeout if timeout else 0
message_batch = self._handler.receive_message_batch(
max_batch_size=max_batch_size,
timeout=timeout_ms)
for message in message_batch:
event_data = EventData(message=message)
self.offset = event_data.offset
data_batch.append(event_data)
return data_batch
except (errors.TokenExpired, errors.AuthenticationException):
log.info("Receiver disconnected due to token error. Attempting reconnect.")
self.reconnect()
return data_batch
except (errors.LinkDetach, errors.ConnectionClose) as shutdown:
if shutdown.action.retry and self.auto_reconnect:
log.info("Receiver detached. Attempting reconnect.")
self.reconnect()
return data_batch
log.info("Receiver detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
self.close(exception=error)
raise error
except errors.MessageHandlerError as shutdown:
if self.auto_reconnect:
log.info("Receiver detached. Attempting reconnect.")
self.reconnect()
return data_batch
log.info("Receiver detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
self.close(exception=error)
raise error
except Exception as e:
log.info("Unexpected error occurred (%r). Shutting down.", e)
error = EventHubError("Receive failed: {}".format(e))
self.close(exception=error)
raise error
class EventHubConsumer(object):
"""
A consumer responsible for reading EventData from a specific Event Hub
partition and as a member of a specific consumer group.
A consumer may be exclusive, which asserts ownership over the partition for the consumer
group to ensure that only one consumer from that group is reading the from the partition.
These exclusive consumers are sometimes referred to as "Epoch Consumers."
A consumer may also be non-exclusive, allowing multiple consumers from the same consumer
group to be actively reading events from the partition. These non-exclusive consumers are
sometimes referred to as "Non-Epoch Consumers."
"""
timeout = 0
_epoch = b'com.microsoft:epoch'
def __init__(self,
client,
source,
event_position=None,
prefetch=300,
owner_level=None,
keep_alive=None,
auto_reconnect=True):
"""
Instantiate a consumer. EventHubConsumer should be instantiated by calling the `create_consumer` method
in EventHubClient.
:param client: The parent EventHubClient.
:type client: ~azure.eventhub.client.EventHubClient
:param source: The source EventHub from which to receive events.
:type source: str
:param prefetch: The number of events to prefetch from the service
for processing. Default is 300.
:type prefetch: int
:param owner_level: The priority of the exclusive consumer. It will an exclusive
consumer if owner_level is set.
:type owner_level: int
"""
self.running = False
self.client = client
self.source = source
self.offset = event_position
self.messages_iter = None
self.prefetch = prefetch
self.owner_level = owner_level
self.keep_alive = keep_alive
self.auto_reconnect = auto_reconnect
self.retry_policy = errors.ErrorPolicy(
max_retries=self.client.config.max_retries,
on_error=_error_handler)
self.reconnect_backoff = 1
self.properties = None
self.redirected = None
self.error = None
partition = self.source.split('/')[-1]
self.name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition)
source = Source(self.source)
if self.offset is not None:
source.set_filter(self.offset._selector()) # pylint: disable=protected-access
if owner_level:
self.properties = {
types.AMQPSymbol(self._epoch): types.AMQPLong(int(owner_level))
}
self._handler = ReceiveClient(
source,
auth=self.client.get_auth(),
debug=self.client.config.network_tracing,
prefetch=self.prefetch,
link_properties=self.properties,
timeout=self.timeout,
error_policy=self.retry_policy,
keep_alive_interval=self.keep_alive,
client_name=self.name,
properties=self.client._create_properties(
self.client.config.user_agent)) # pylint: disable=protected-access
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close(exc_val)
def __iter__(self):
return self
def __next__(self):
self._open()
max_retries = self.client.config.max_retries
connecting_count = 0
while True:
connecting_count += 1
try:
if not self.messages_iter:
self.messages_iter = self._handler.receive_messages_iter()
message = next(self.messages_iter)
event_data = EventData(message=message)
self.offset = EventPosition(event_data.offset, inclusive=False)
return event_data
except errors.AuthenticationException as auth_error:
if connecting_count < max_retries:
log.info(
"EventHubConsumer disconnected due to token error. Attempting reconnect."
)
self._reconnect()
else:
log.info(
"EventHubConsumer authentication failed. Shutting down."
)
error = AuthenticationError(str(auth_error), auth_error)
self.close(auth_error)
raise error
except (errors.LinkDetach, errors.ConnectionClose) as shutdown:
if shutdown.action.retry and self.auto_reconnect:
log.info(
"EventHubConsumer detached. Attempting reconnect.")
self._reconnect()
else:
log.info("EventHubConsumer detached. Shutting down.")
error = ConnectionLostError(str(shutdown), shutdown)
self.close(exception=error)
raise error
except errors.MessageHandlerError as shutdown:
if connecting_count < max_retries:
log.info(
"EventHubConsumer detached. Attempting reconnect.")
self._reconnect()
else:
log.info("EventHubConsumer detached. Shutting down.")
error = ConnectionLostError(str(shutdown), shutdown)
self.close(error)
raise error
except errors.AMQPConnectionError as shutdown:
if connecting_count < max_retries:
log.info(
"EventHubConsumer connection lost. Attempting reconnect."
)
self._reconnect()
else:
log.info(
"EventHubConsumer connection lost. Shutting down.")
error = ConnectionLostError(str(shutdown), shutdown)
self.close(error)
raise error
except compat.TimeoutException as shutdown:
if connecting_count < max_retries:
log.info(
"EventHubConsumer timed out receiving event data. Attempting reconnect."
)
self._reconnect()
else:
log.info("EventHubConsumer timed out. Shutting down.")
self.close(shutdown)
raise ConnectionLostError(str(shutdown), shutdown)
except StopIteration:
raise
except KeyboardInterrupt:
log.info("EventHubConsumer stops due to keyboard interrupt")
self.close()
raise
except Exception as e:
log.error("Unexpected error occurred (%r). Shutting down.", e)
error = EventHubError("Receive failed: {}".format(e), e)
self.close(exception=error)
raise error
def _check_closed(self):
if self.error:
raise EventHubError(
"This consumer has been closed. Please create a new consumer to receive event data.",
self.error)
def _redirect(self, redirect):
self.redirected = redirect
self.running = False
self.messages_iter = None
self._open()
def _open(self):
"""
Open the EventHubConsumer using the supplied connection.
If the handler has previously been redirected, the redirect
context will be used to create a new handler before opening it.
"""
# pylint: disable=protected-access
self._check_closed()
if self.redirected:
self.client._process_redirect_uri(self.redirected)
self.source = self.redirected.address
source = Source(self.source)
if self.offset is not None:
source.set_filter(self.offset._selector())
alt_creds = {
"username": self.client._auth_config.get("iot_username"),
"password": self.client._auth_config.get("iot_password")
}
self._handler = ReceiveClient(
source,
auth=self.client.get_auth(**alt_creds),
debug=self.client.config.network_tracing,
prefetch=self.prefetch,
link_properties=self.properties,
timeout=self.timeout,
error_policy=self.retry_policy,
keep_alive_interval=self.keep_alive,
client_name=self.name,
properties=self.client._create_properties(
self.client.config.user_agent)) # pylint: disable=protected-access
if not self.running:
self._connect()
self.running = True
def _connect(self):
connected = self._build_connection()
if not connected:
time.sleep(self.reconnect_backoff)
while not self._build_connection(is_reconnect=True):
time.sleep(self.reconnect_backoff)
def _build_connection(self, is_reconnect=False):
"""
:param is_reconnect: True - trying to reconnect after fail to connect or a connection is lost.
False - the 1st time to connect
:return: True - connected. False - not connected
"""
# pylint: disable=protected-access
if is_reconnect:
alt_creds = {
"username": self.client._auth_config.get("iot_username"),
"password": self.client._auth_config.get("iot_password")
}
self._handler.close()
source = Source(self.source)
if self.offset is not None:
source.set_filter(self.offset._selector())
self._handler = ReceiveClient(
source,
auth=self.client.get_auth(**alt_creds),
debug=self.client.config.network_tracing,
prefetch=self.prefetch,
link_properties=self.properties,
timeout=self.timeout,
error_policy=self.retry_policy,
keep_alive_interval=self.keep_alive,
client_name=self.name,
properties=self.client._create_properties(
self.client.config.user_agent)) # pylint: disable=protected-access
self.messages_iter = None
try:
self._handler.open()
while not self._handler.client_ready():
time.sleep(0.05)
return True
except errors.AuthenticationException as shutdown:
if is_reconnect:
log.info(
"EventHubConsumer couldn't authenticate. Shutting down. (%r)",
shutdown)
error = AuthenticationError(str(shutdown), shutdown)
self.close(exception=error)
raise error
else:
log.info(
"EventHubConsumer couldn't authenticate. Attempting reconnect."
)
return False
except errors.LinkRedirect as redirect:
self._redirect(redirect)
return True
except (errors.LinkDetach, errors.ConnectionClose) as shutdown:
if shutdown.action.retry:
log.info("EventHubConsumer detached. Attempting reconnect.")
return False
else:
log.info("EventHubConsumer detached. Shutting down.")
error = ConnectError(str(shutdown), shutdown)
self.close(exception=error)
raise error
except errors.MessageHandlerError as shutdown:
if is_reconnect:
log.info("EventHubConsumer detached. Shutting down.")
error = ConnectError(str(shutdown), shutdown)
self.close(exception=error)
raise error
else:
log.info("EventHubConsumer detached. Attempting reconnect.")
return False
except errors.AMQPConnectionError as shutdown:
if is_reconnect:
log.info(
"EventHubConsumer connection error (%r). Shutting down.",
shutdown)
error = AuthenticationError(str(shutdown), shutdown)
self.close(exception=error)
raise error
else:
log.info(
"EventHubConsumer couldn't authenticate. Attempting reconnect."
)
return False
except compat.TimeoutException as shutdown:
if is_reconnect:
log.info(
"EventHubConsumer authentication timed out. Shutting down."
)
error = AuthenticationError(str(shutdown), shutdown)
self.close(exception=error)
raise error
else:
log.info(
"EventHubConsumer authentication timed out. Attempting reconnect."
)
return False
except Exception as e:
log.error(
"Unexpected error occurred when building connection (%r). Shutting down.",
e)
error = EventHubError(
"Unexpected error occurred when building connection", e)
self.close(exception=error)
raise error
def _reconnect(self):
return self._build_connection(is_reconnect=True)
@property
def queue_size(self):
# type:() -> int
"""
The current size of the unprocessed Event queue.
:rtype: int
"""
# pylint: disable=protected-access
if self._handler._received_messages:
return self._handler._received_messages.qsize()
return 0
def receive(self, max_batch_size=None, timeout=None):
# type:(int, float) -> List[EventData]
"""
Receive events from the EventHub.
:param max_batch_size: Receive a batch of events. Batch size will
be up to the maximum specified, but will return as soon as service
returns no new events. If combined with a timeout and no events are
retrieve before the time, the result will be empty. If no batch
size is supplied, the prefetch size will be the maximum.
:type max_batch_size: int
:param timeout: The maximum wait time to build up the requested message count for the batch.
If not specified, the default wait time specified when the consumer was created will be used.
:type timeout: float
:rtype: list[~azure.eventhub.common.EventData]
:raises: ~azure.eventhub.AuthenticationError, ~azure.eventhub.ConnectError, ~azure.eventhub.ConnectionLostError,
~azure.eventhub.EventHubError
Example:
.. literalinclude:: ../examples/test_examples_eventhub.py
:start-after: [START eventhub_client_sync_receive]
:end-before: [END eventhub_client_sync_receive]
:language: python
:dedent: 4
:caption: Receive events from the EventHub.
"""
self._check_closed()
self._open()
max_batch_size = min(
self.client.config.max_batch_size,
self.prefetch) if max_batch_size is None else max_batch_size
timeout = self.client.config.receive_timeout if timeout is None else timeout
data_batch = [] # type: List[EventData]
max_retries = self.client.config.max_retries
connecting_count = 0
while True:
connecting_count += 1
try:
timeout_ms = 1000 * timeout if timeout else 0
message_batch = self._handler.receive_message_batch(
max_batch_size=max_batch_size -
(len(data_batch) if data_batch else 0),
timeout=timeout_ms)
for message in message_batch:
event_data = EventData(message=message)
self.offset = EventPosition(event_data.offset)
data_batch.append(event_data)
return data_batch
except errors.AuthenticationException as auth_error:
if connecting_count < max_retries:
log.info(
"EventHubConsumer disconnected due to token error. Attempting reconnect."
)
self._reconnect()
else:
log.info(
"EventHubConsumer authentication failed. Shutting down."
)
error = AuthenticationError(str(auth_error), auth_error)
self.close(auth_error)
raise error
except (errors.LinkDetach, errors.ConnectionClose) as shutdown:
if shutdown.action.retry and self.auto_reconnect:
log.info(
"EventHubConsumer detached. Attempting reconnect.")
self._reconnect()
else:
log.info("EventHubConsumer detached. Shutting down.")
error = ConnectionLostError(str(shutdown), shutdown)
self.close(exception=error)
raise error
except errors.MessageHandlerError as shutdown:
if connecting_count < max_retries:
log.info(
"EventHubConsumer detached. Attempting reconnect.")
self._reconnect()
else:
log.info("EventHubConsumer detached. Shutting down.")
error = ConnectionLostError(str(shutdown), shutdown)
self.close(error)
raise error
except errors.AMQPConnectionError as shutdown:
if connecting_count < max_retries:
log.info(
"EventHubConsumer connection lost. Attempting reconnect."
)
self._reconnect()
else:
log.info(
"EventHubConsumer connection lost. Shutting down.")
error = ConnectionLostError(str(shutdown), shutdown)
self.close(error)
raise error
except compat.TimeoutException as shutdown:
if connecting_count < max_retries:
log.info(
"EventHubConsumer timed out receiving event data. Attempting reconnect."
)
self._reconnect()
else:
log.info("EventHubConsumer timed out. Shutting down.")
self.close(shutdown)
raise ConnectionLostError(str(shutdown), shutdown)
except KeyboardInterrupt:
log.info("EventHubConsumer stops due to keyboard interrupt")
self.close()
raise
except Exception as e:
log.error("Unexpected error occurred (%r). Shutting down.", e)
error = EventHubError("Receive failed: {}".format(e), e)
self.close(exception=error)
raise error
def close(self, exception=None):
# type:(Exception) -> None
"""
Close down the handler. If the handler has already closed,
this will be a no op. An optional exception can be passed in to
indicate that the handler was shutdown due to error.
:param exception: An optional exception if the handler is closing
due to an error.
:type exception: Exception
Example:
.. literalinclude:: ../examples/test_examples_eventhub.py
:start-after: [START eventhub_client_receiver_close]
:end-before: [END eventhub_client_receiver_close]
:language: python
:dedent: 4
:caption: Close down the handler.
"""
if self.messages_iter:
self.messages_iter.close()
self.messages_iter = None
self.running = False
if self.error:
return
if isinstance(exception, errors.LinkRedirect):
self.redirected = exception
elif isinstance(exception, EventHubError):
self.error = exception
elif exception:
self.error = EventHubError(str(exception))
else:
self.error = EventHubError("This receive handler is now closed.")
self._handler.close()
next = __next__ # for python2.7
class EventHubConsumer(ConsumerProducerMixin): # pylint:disable=too-many-instance-attributes
"""
A consumer responsible for reading EventData from a specific Event Hub
partition and as a member of a specific consumer group.
A consumer may be exclusive, which asserts ownership over the partition for the consumer
group to ensure that only one consumer from that group is reading the from the partition.
These exclusive consumers are sometimes referred to as "Epoch Consumers."
A consumer may also be non-exclusive, allowing multiple consumers from the same consumer
group to be actively reading events from the partition. These non-exclusive consumers are
sometimes referred to as "Non-Epoch Consumers."
Please use the method `create_consumer` on `EventHubClient` for creating `EventHubConsumer`.
"""
_timeout = 0
_epoch_symbol = b'com.microsoft:epoch'
_timeout_symbol = b'com.microsoft:timeout'
def __init__(self, client, source, **kwargs):
"""
Instantiate a consumer. EventHubConsumer should be instantiated by calling the `create_consumer` method
in EventHubClient.
:param client: The parent EventHubClient.
:type client: ~azure.eventhub.client.EventHubClient
:param source: The source EventHub from which to receive events.
:type source: str
:param prefetch: The number of events to prefetch from the service
for processing. Default is 300.
:type prefetch: int
:param owner_level: The priority of the exclusive consumer. An exclusive
consumer will be created if owner_level is set.
:type owner_level: int
"""
event_position = kwargs.get("event_position", None)
prefetch = kwargs.get("prefetch", 300)
owner_level = kwargs.get("owner_level", None)
keep_alive = kwargs.get("keep_alive", None)
auto_reconnect = kwargs.get("auto_reconnect", True)
super(EventHubConsumer, self).__init__()
self._running = False
self._client = client
self._source = source
self._offset = event_position
self._messages_iter = None
self._prefetch = prefetch
self._owner_level = owner_level
self._keep_alive = keep_alive
self._auto_reconnect = auto_reconnect
self._retry_policy = errors.ErrorPolicy(
max_retries=self._client._config.max_retries,
on_error=_error_handler) # pylint:disable=protected-access
self._reconnect_backoff = 1
self._link_properties = {}
self._redirected = None
self._error = None
partition = self._source.split('/')[-1]
self._partition = partition
self._name = "EHConsumer-{}-partition{}".format(
uuid.uuid4(), partition)
if owner_level:
self._link_properties[types.AMQPSymbol(
self._epoch_symbol)] = types.AMQPLong(int(owner_level))
link_property_timeout_ms = (self._client._config.receive_timeout
or self._timeout) * 1000 # pylint:disable=protected-access
self._link_properties[types.AMQPSymbol(
self._timeout_symbol)] = types.AMQPLong(
int(link_property_timeout_ms))
self._handler = None
def __iter__(self):
return self
def __next__(self):
retried_times = 0
last_exception = None
while retried_times < self._client._config.max_retries: # pylint:disable=protected-access
try:
self._open()
if not self._messages_iter:
self._messages_iter = self._handler.receive_messages_iter()
message = next(self._messages_iter)
event_data = EventData._from_message(message) # pylint:disable=protected-access
self._offset = EventPosition(event_data.offset,
inclusive=False)
retried_times = 0
return event_data
except Exception as exception: # pylint:disable=broad-except
last_exception = self._handle_exception(exception)
self._client._try_delay(
retried_times=retried_times,
last_exception=last_exception, # pylint:disable=protected-access
entity_name=self._name)
retried_times += 1
log.info("%r operation has exhausted retry. Last exception: %r.",
self._name, last_exception)
raise last_exception
def _create_handler(self):
alt_creds = {
"username":
self._client._auth_config.get("iot_username")
if self._redirected else None, # pylint:disable=protected-access
"password":
self._client._auth_config.get("iot_password")
if self._redirected else None # pylint:disable=protected-access
}
source = Source(self._source)
if self._offset is not None:
source.set_filter(self._offset._selector()) # pylint:disable=protected-access
self._handler = ReceiveClient(
source,
auth=self._client._get_auth(**alt_creds), # pylint:disable=protected-access
debug=self._client._config.network_tracing, # pylint:disable=protected-access
prefetch=self._prefetch,
link_properties=self._link_properties,
timeout=self._timeout,
error_policy=self._retry_policy,
keep_alive_interval=self._keep_alive,
client_name=self._name,
properties=self._client._create_properties( # pylint:disable=protected-access
self._client._config.user_agent)) # pylint:disable=protected-access
self._messages_iter = None
def _redirect(self, redirect):
self._messages_iter = None
super(EventHubConsumer, self)._redirect(redirect)
def _open(self):
"""
Open the EventHubConsumer using the supplied connection.
If the handler has previously been redirected, the redirect
context will be used to create a new handler before opening it.
"""
# pylint: disable=protected-access
self._redirected = self._redirected or self._client._iothub_redirect_info
if not self._running and self._redirected:
self._client._process_redirect_uri(self._redirected)
self._source = self._redirected.address
super(EventHubConsumer, self)._open()
def _open_with_retry(self):
return self._do_retryable_operation(self._open,
operation_need_param=False)
def _receive(self, timeout_time=None, max_batch_size=None, **kwargs):
last_exception = kwargs.get("last_exception")
data_batch = []
self._open()
remaining_time = timeout_time - time.time()
if remaining_time <= 0.0:
if last_exception:
log.info("%r receive operation timed out. (%r)", self._name,
last_exception)
raise last_exception
return data_batch
remaining_time_ms = 1000 * remaining_time
message_batch = self._handler.receive_message_batch(
max_batch_size=max_batch_size, timeout=remaining_time_ms)
for message in message_batch:
event_data = EventData._from_message(message) # pylint:disable=protected-access
self._offset = EventPosition(event_data.offset)
data_batch.append(event_data)
return data_batch
def _receive_with_retry(self, timeout=None, max_batch_size=None, **kwargs):
return self._do_retryable_operation(self._receive,
timeout=timeout,
max_batch_size=max_batch_size,
**kwargs)
@property
def queue_size(self):
# type:() -> int
"""
The current size of the unprocessed Event queue.
:rtype: int
"""
# pylint: disable=protected-access
if self._handler._received_messages:
return self._handler._received_messages.qsize()
return 0
def receive(self, max_batch_size=None, timeout=None):
# type: (int, float) -> List[EventData]
"""
Receive events from the EventHub.
:param max_batch_size: Receive a batch of events. Batch size will
be up to the maximum specified, but will return as soon as service
returns no new events. If combined with a timeout and no events are
retrieve before the time, the result will be empty. If no batch
size is supplied, the prefetch size will be the maximum.
:type max_batch_size: int
:param timeout: The maximum wait time to build up the requested message count for the batch.
If not specified, the default wait time specified when the consumer was created will be used.
:type timeout: float
:rtype: list[~azure.eventhub.common.EventData]
:raises: ~azure.eventhub.AuthenticationError, ~azure.eventhub.ConnectError, ~azure.eventhub.ConnectionLostError,
~azure.eventhub.EventHubError
Example:
.. literalinclude:: ../examples/test_examples_eventhub.py
:start-after: [START eventhub_client_sync_receive]
:end-before: [END eventhub_client_sync_receive]
:language: python
:dedent: 4
:caption: Receive events from the EventHub.
"""
self._check_closed()
timeout = timeout or self._client._config.receive_timeout # pylint:disable=protected-access
max_batch_size = max_batch_size or min(
self._client._config.max_batch_size, self._prefetch) # pylint:disable=protected-access
return self._receive_with_retry(timeout=timeout,
max_batch_size=max_batch_size)
def close(self, exception=None):
# type:(Exception) -> None
"""
Close down the handler. If the handler has already closed,
this will be a no op. An optional exception can be passed in to
indicate that the handler was shutdown due to error.
:param exception: An optional exception if the handler is closing
due to an error.
:type exception: Exception
Example:
.. literalinclude:: ../examples/test_examples_eventhub.py
:start-after: [START eventhub_client_receiver_close]
:end-before: [END eventhub_client_receiver_close]
:language: python
:dedent: 4
:caption: Close down the handler.
"""
if self._messages_iter:
self._messages_iter.close()
self._messages_iter = None
super(EventHubConsumer, self).close(exception)
next = __next__ # for python2.7
class ServiceBusReceiver(BaseHandler, ReceiverMixin): # pylint: disable=too-many-instance-attributes
"""The ServiceBusReceiver class defines a high level interface for
receiving messages from the Azure Service Bus Queue or Topic Subscription.
The two primary channels for message receipt are `receive()` to make a single request for messages,
and `for message in receiver:` to continuously receive incoming messages in an ongoing fashion.
:ivar fully_qualified_namespace: The fully qualified host name for the Service Bus namespace.
The namespace format is: `<yournamespace>.servicebus.windows.net`.
:vartype fully_qualified_namespace: str
:ivar entity_path: The path of the entity that the client connects to.
:vartype entity_path: str
:param str fully_qualified_namespace: The fully qualified host name for the Service Bus namespace.
The namespace format is: `<yournamespace>.servicebus.windows.net`.
:param ~azure.core.credentials.TokenCredential credential: The credential object used for authentication which
implements a particular interface for getting tokens. It accepts
:class:`ServiceBusSharedKeyCredential<azure.servicebus.ServiceBusSharedKeyCredential>`, or credential objects
generated by the azure-identity library and objects that implement the `get_token(self, *scopes)` method.
:keyword str queue_name: The path of specific Service Bus Queue the client connects to.
:keyword str topic_name: The path of specific Service Bus Topic which contains the Subscription
the client connects to.
:keyword str subscription_name: The path of specific Service Bus Subscription under the
specified Topic the client connects to.
:keyword int prefetch: The maximum number of messages to cache with each request to the service.
The default value is 0, meaning messages will be received from the service and processed
one at a time. Increasing this value will improve message throughput performance but increase
the change that messages will expire while they are cached if they're not processed fast enough.
:keyword float idle_timeout: The timeout in seconds between received messages after which the receiver will
automatically shutdown. The default value is 0, meaning no timeout.
:keyword mode: The mode with which messages will be retrieved from the entity. The two options
are PeekLock and ReceiveAndDelete. Messages received with PeekLock must be settled within a given
lock period before they will be removed from the queue. Messages received with ReceiveAndDelete
will be immediately removed from the queue, and cannot be subsequently abandoned or re-received
if the client fails to process the message.
The default mode is PeekLock.
:paramtype mode: ~azure.servicebus.ReceiveSettleMode
:keyword bool logging_enable: Whether to output network trace logs to the logger. Default is `False`.
:keyword int retry_total: The total number of attempts to redo a failed operation when an error occurs.
Default value is 3.
:keyword transport_type: The type of transport protocol that will be used for communicating with
the Service Bus service. Default is `TransportType.Amqp`.
:paramtype transport_type: ~azure.servicebus.TransportType
:keyword dict http_proxy: HTTP proxy settings. This must be a dictionary with the following
keys: `'proxy_hostname'` (str value) and `'proxy_port'` (int value).
Additionally the following keys may also be present: `'username', 'password'`.
.. admonition:: Example:
.. literalinclude:: ../samples/sync_samples/sample_code_servicebus.py
:start-after: [START create_servicebus_receiver_sync]
:end-before: [END create_servicebus_receiver_sync]
:language: python
:dedent: 4
:caption: Create a new instance of the ServiceBusReceiver.
"""
def __init__(self, fully_qualified_namespace, credential, **kwargs):
# type: (str, TokenCredential, Any) -> None
if kwargs.get("entity_name"):
super(ServiceBusReceiver, self).__init__(
fully_qualified_namespace=fully_qualified_namespace,
credential=credential,
**kwargs)
else:
queue_name = kwargs.get("queue_name") # type: Optional[str]
topic_name = kwargs.get("topic_name") # type: Optional[str]
subscription_name = kwargs.get("subscription_name")
if queue_name and topic_name:
raise ValueError(
"Queue/Topic name can not be specified simultaneously.")
if topic_name and not subscription_name:
raise ValueError(
"Subscription name is missing for the topic. Please specify subscription_name."
)
entity_name = queue_name or topic_name
if not entity_name:
raise ValueError(
"Queue/Topic name is missing. Please specify queue_name/topic_name."
)
super(ServiceBusReceiver, self).__init__(
fully_qualified_namespace=fully_qualified_namespace,
credential=credential,
entity_name=entity_name,
**kwargs)
self._populate_attributes(**kwargs)
def __iter__(self):
return self
def __next__(self):
self._check_live()
while True:
try:
return self._do_retryable_operation(self._iter_next)
except StopIteration:
self.close()
raise
next = __next__ # for python2.7
def _iter_next(self):
self._open()
uamqp_message = next(self._message_iter)
message = self._build_message(uamqp_message)
return message
def _create_handler(self, auth):
# type: (AMQPAuth) -> None
self._handler = ReceiveClient(
self._get_source(),
auth=auth,
debug=self._config.logging_enable,
properties=self._properties,
error_policy=self._error_policy,
client_name=self._name,
on_attach=self._on_attach,
auto_complete=False,
encoding=self._config.encoding,
receive_settle_mode=self._mode.value,
send_settle_mode=SenderSettleMode.Settled
if self._mode == ReceiveSettleMode.ReceiveAndDelete else None,
timeout=self._idle_timeout * 1000 if self._idle_timeout else 0,
prefetch=self._prefetch)
def _open(self):
if self._running:
return
if self._handler:
self._handler.close()
auth = None if self._connection else create_authentication(self)
self._create_handler(auth)
try:
self._handler.open(connection=self._connection)
self._message_iter = self._handler.receive_messages_iter() # pylint: disable=attribute-defined-outside-init
while not self._handler.client_ready():
time.sleep(0.05)
self._running = True
except:
self.close()
raise
def _receive(self, max_batch_size=None, timeout=None):
# type: (Optional[int], Optional[float]) -> List[ReceivedMessage]
self._open()
max_batch_size = max_batch_size or self._handler._prefetch # pylint: disable=protected-access
timeout_ms = 1000 * (timeout or self._idle_timeout) if (
timeout or self._idle_timeout) else 0
batch = self._handler.receive_message_batch(
max_batch_size=max_batch_size, timeout=timeout_ms)
return [self._build_message(message) for message in batch]
def _settle_message(self,
settlement,
lock_tokens,
dead_letter_details=None):
# type: (bytes, List[str], Optional[Dict[str, Any]]) -> Any
message = {
MGMT_REQUEST_DISPOSITION_STATUS: settlement,
MGMT_REQUEST_LOCK_TOKENS: types.AMQPArray(lock_tokens)
}
self._populate_message_properties(message)
if dead_letter_details:
message.update(dead_letter_details)
return self._mgmt_request_response_with_retry(
REQUEST_RESPONSE_UPDATE_DISPOSTION_OPERATION, message,
mgmt_handlers.default)
def _renew_locks(self, *lock_tokens):
# type: (*str) -> Any
message = {MGMT_REQUEST_LOCK_TOKENS: types.AMQPArray(lock_tokens)}
return self._mgmt_request_response_with_retry(
REQUEST_RESPONSE_RENEWLOCK_OPERATION, message,
mgmt_handlers.lock_renew_op)
@classmethod
def from_connection_string(cls, conn_str, **kwargs):
# type: (str, Any) -> ServiceBusReceiver
"""Create a ServiceBusReceiver from a connection string.
:param conn_str: The connection string of a Service Bus.
:keyword str queue_name: The path of specific Service Bus Queue the client connects to.
:keyword str topic_name: The path of specific Service Bus Topic which contains the Subscription
the client connects to.
:keyword str subscription_name: The path of specific Service Bus Subscription under the
specified Topic the client connects to.
:keyword mode: The mode with which messages will be retrieved from the entity. The two options
are PeekLock and ReceiveAndDelete. Messages received with PeekLock must be settled within a given
lock period before they will be removed from the queue. Messages received with ReceiveAndDelete
will be immediately removed from the queue, and cannot be subsequently abandoned or re-received
if the client fails to process the message.
The default mode is PeekLock.
:paramtype mode: ~azure.servicebus.ReceiveSettleMode
:keyword int prefetch: The maximum number of messages to cache with each request to the service.
The default value is 0, meaning messages will be received from the service and processed
one at a time. Increasing this value will improve message throughput performance but increase
the change that messages will expire while they are cached if they're not processed fast enough.
:keyword float idle_timeout: The timeout in seconds between received messages after which the receiver will
automatically shutdown. The default value is 0, meaning no timeout.
:keyword bool logging_enable: Whether to output network trace logs to the logger. Default is `False`.
:keyword int retry_total: The total number of attempts to redo a failed operation when an error occurs.
Default value is 3.
:keyword transport_type: The type of transport protocol that will be used for communicating with
the Service Bus service. Default is `TransportType.Amqp`.
:paramtype transport_type: ~azure.servicebus.TransportType
:keyword dict http_proxy: HTTP proxy settings. This must be a dictionary with the following
keys: `'proxy_hostname'` (str value) and `'proxy_port'` (int value).
Additionally the following keys may also be present: `'username', 'password'`.
:rtype: ~azure.servicebus.ServiceBusReceiver
.. admonition:: Example:
.. literalinclude:: ../samples/sync_samples/sample_code_servicebus.py
:start-after: [START create_servicebus_receiver_from_conn_str_sync]
:end-before: [END create_servicebus_receiver_from_conn_str_sync]
:language: python
:dedent: 4
:caption: Create a new instance of the ServiceBusReceiver from connection string.
"""
constructor_args = _convert_connection_string_to_kwargs(
conn_str, ServiceBusSharedKeyCredential, **kwargs)
if kwargs.get("queue_name") and kwargs.get("subscription_name"):
raise ValueError("Queue entity does not have subscription.")
if kwargs.get("topic_name") and not kwargs.get("subscription_name"):
raise ValueError(
"Subscription name is missing for the topic. Please specify subscription_name."
)
return cls(**constructor_args)
def receive_messages(self, max_batch_size=None, max_wait_time=None):
# type: (int, float) -> List[ReceivedMessage]
"""Receive a batch of messages at once.
This approach is optimal if you wish to process multiple messages simultaneously, or
perform an ad-hoc receive as a single call.
Note that the number of messages retrieved in a single batch will be dependent on
whether `prefetch` was set for the receiver. This call will prioritize returning
quickly over meeting a specified batch size, and so will return as soon as at least
one message is received and there is a gap in incoming messages regardless
of the specified batch size.
:param int max_batch_size: Maximum number of messages in the batch. Actual number
returned will depend on prefetch size and incoming stream rate.
:param float max_wait_time: Maximum time to wait in seconds for the first message to arrive.
If no messages arrive, and no timeout is specified, this call will not return
until the connection is closed. If specified, an no messages arrive within the
timeout period, an empty list will be returned.
:rtype: list[~azure.servicebus.ReceivedMessage]
.. admonition:: Example:
.. literalinclude:: ../samples/sync_samples/sample_code_servicebus.py
:start-after: [START receive_sync]
:end-before: [END receive_sync]
:language: python
:dedent: 4
:caption: Receive messages from ServiceBus.
"""
self._check_live()
if max_batch_size and self._prefetch < max_batch_size:
raise ValueError(
"max_batch_size should be less than or equal to prefetch of ServiceBusReceiver, or you "
"could set a larger prefetch value when you're constructing the ServiceBusReceiver."
)
return self._do_retryable_operation(self._receive,
max_batch_size=max_batch_size,
timeout=max_wait_time,
require_timeout=True)
def receive_deferred_messages(self, sequence_numbers):
# type: (List[int]) -> List[ReceivedMessage]
"""Receive messages that have previously been deferred.
When receiving deferred messages from a partitioned entity, all of the supplied
sequence numbers must be messages from the same partition.
:param list[int] sequence_numbers: A list of the sequence numbers of messages that have been
deferred.
:rtype: list[~azure.servicebus.ReceivedMessage]
.. admonition:: Example:
.. literalinclude:: ../samples/sync_samples/sample_code_servicebus.py
:start-after: [START receive_defer_sync]
:end-before: [END receive_defer_sync]
:language: python
:dedent: 4
:caption: Receive deferred messages from ServiceBus.
"""
self._check_live()
if not sequence_numbers:
raise ValueError("At least one sequence number must be specified.")
self._open()
try:
receive_mode = self._mode.value.value
except AttributeError:
receive_mode = int(self._mode)
message = {
MGMT_REQUEST_SEQUENCE_NUMBERS:
types.AMQPArray([types.AMQPLong(s) for s in sequence_numbers]),
MGMT_REQUEST_RECEIVER_SETTLE_MODE:
types.AMQPuInt(receive_mode)
}
self._populate_message_properties(message)
handler = functools.partial(mgmt_handlers.deferred_message_op,
mode=self._mode)
messages = self._mgmt_request_response_with_retry(
REQUEST_RESPONSE_RECEIVE_BY_SEQUENCE_NUMBER, message, handler)
for m in messages:
m._receiver = self # pylint: disable=protected-access
return messages
def peek_messages(self, message_count=1, sequence_number=None):
# type: (int, Optional[int]) -> List[PeekMessage]
"""Browse messages currently pending in the queue.
Peeked messages are not removed from queue, nor are they locked. They cannot be completed,
deferred or dead-lettered.
:param int message_count: The maximum number of messages to try and peek. The default
value is 1.
:param int sequence_number: A message sequence number from which to start browsing messages.
:rtype: list[~azure.servicebus.PeekMessage]
.. admonition:: Example:
.. literalinclude:: ../samples/sync_samples/sample_code_servicebus.py
:start-after: [START peek_messages_sync]
:end-before: [END peek_messages_sync]
:language: python
:dedent: 4
:caption: Look at pending messages in the queue.
"""
self._check_live()
if not sequence_number:
sequence_number = self._last_received_sequenced_number or 1
if int(message_count) < 1:
raise ValueError("count must be 1 or greater.")
if int(sequence_number) < 1:
raise ValueError("start_from must be 1 or greater.")
self._open()
message = {
MGMT_REQUEST_FROM_SEQUENCE_NUMBER: types.AMQPLong(sequence_number),
MGMT_REQUEST_MESSAGE_COUNT: message_count
}
self._populate_message_properties(message)
return self._mgmt_request_response_with_retry(
REQUEST_RESPONSE_PEEK_OPERATION, message, mgmt_handlers.peek_op)
class Receiver:
"""
Implements a Receiver.
"""
timeout = 0
_epoch = b'com.microsoft:epoch'
def __init__(self, client, source, prefetch=300, epoch=None):
"""
Instantiate a receiver.
:param client: The parent EventHubClient.
:type client: ~azure.eventhub.client.EventHubClient
:param source: The source EventHub from which to receive events.
:type source: ~uamqp.address.Source
:param prefetch: The number of events to prefetch from the service
for processing. Default is 300.
:type prefetch: int
:param epoch: An optional epoch value.
:type epoch: int
"""
self.offset = None
self.prefetch = prefetch
self.epoch = epoch
self.properties = None
self.redirected = None
self.debug = client.debug
self.error = None
if epoch:
self.properties = {
types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))
}
self._handler = ReceiveClient(source,
auth=client.auth,
debug=self.debug,
prefetch=self.prefetch,
link_properties=self.properties,
timeout=self.timeout)
def open(self, connection):
"""
Open the Receiver using the supplied conneciton.
If the handler has previously been redirected, the redirect
context will be used to create a new handler before opening it.
:param connection: The underlying client shared connection.
:type: connection: ~uamqp.connection.Connection
"""
if self.redirected:
self._handler = ReceiveClient(self.redirected.address,
auth=None,
debug=self.debug,
prefetch=self.prefetch,
link_properties=self.properties,
timeout=self.timeout)
self._handler.open(connection)
def get_handler_state(self):
"""
Get the state of the underlying handler with regards to start
up processes.
:rtype: ~uamqp.constants.MessageReceiverState
"""
# pylint: disable=protected-access
return self._handler._message_receiver.get_state()
def has_started(self):
"""
Whether the handler has completed all start up processes such as
establishing the connection, session, link and authentication, and
is not ready to process messages.
:rtype: bool
"""
# pylint: disable=protected-access
timeout = False
auth_in_progress = False
if self._handler._connection.cbs:
timeout, auth_in_progress = self._handler._auth.handle_token()
if timeout:
raise EventHubError("Authorization timeout.")
elif auth_in_progress:
return False
elif not self._handler._client_ready():
return False
else:
return True
def close(self, exception=None):
"""
Close down the handler. If the handler has already closed,
this will be a no op. An optional exception can be passed in to
indicate that the handler was shutdown due to error.
:param exception: An optional exception if the handler is closing
due to an error.
:type exception: Exception
"""
if self.error:
return
elif isinstance(exception, errors.LinkRedirect):
self.redirected = exception
elif isinstance(exception, EventHubError):
self.error = exception
elif exception:
self.error = EventHubError(str(exception))
else:
self.error = EventHubError("This receive handler is now closed.")
self._handler.close()
@property
def queue_size(self):
"""
The current size of the unprocessed Event queue.
:rtype: int
"""
# pylint: disable=protected-access
if self._handler._received_messages:
return self._handler._received_messages.qsize()
return 0
def receive(self, max_batch_size=None, timeout=None):
"""
Receive events from the EventHub.
:param max_batch_size: Receive a batch of events. Batch size will
be up to the maximum specified, but will return as soon as service
returns no new events. If combined with a timeout and no events are
retrieve before the time, the result will be empty. If no batch
size is supplied, the prefetch size will be the maximum.
:type max_batch_size: int
:rtype: list[~azure.eventhub.common.EventData]
"""
if self.error:
raise self.error
try:
timeout_ms = 1000 * timeout if timeout else 0
message_batch = self._handler.receive_message_batch(
max_batch_size=max_batch_size, timeout=timeout_ms)
data_batch = []
for message in message_batch:
event_data = EventData(message=message)
self.offset = event_data.offset
data_batch.append(event_data)
return data_batch
except errors.LinkDetach as detach:
error = EventHubError(str(detach))
self.close(exception=error)
raise error
except Exception as e:
error = EventHubError("Receive failed: {}".format(e))
self.close(exception=error)
raise error
def selector(self, default):
"""
Create a selector for the current offset if it is set.
:param default: The fallback receive offset.
:type default: ~azure.eventhub.common.Offset
:rtype: ~azure.eventhub.common.Offset
"""
if self.offset is not None:
return Offset(self.offset).selector()
return default
class Receiver(BaseHandler): # pylint: disable=too-many-instance-attributes
"""A message receiver.
This receive handler acts as an iterable message stream for retrieving
messages for a Service Bus entity. It operates a single connection that must be opened and
closed on completion. The service connection will remain open for the entirety of the iterator.
If you find yourself only partially iterating the message stream, you should run the receiver
in a `with` statement to ensure the connection is closed.
The Receiver should not be instantiated directly, and should be accessed from a `QueueClient` or
`SubscriptionClient` using the `get_receiver()` method.
.. note:: This object is not thread-safe.
:param handler_id: The ID used as the connection name for the Receiver.
:type handler_id: str
:param source: The endpoint from which to receive messages.
:type source: ~uamqp.Source
:param auth_config: The SASL auth credentials.
:type auth_config: dict[str, str]
:param connection: A shared connection [not yet supported].
:type connection: ~uamqp.Connection
:param mode: The receive connection mode. Value must be either PeekLock or ReceiveAndDelete.
:type mode: ~azure.servicebus.common.constants.ReceiveSettleMode
:param encoding: The encoding used for string properties. Default is 'UTF-8'.
:type encoding: str
:param debug: Whether to enable network trace debug logs.
:type debug: bool
Example:
.. literalinclude:: ../examples/test_examples.py
:start-after: [START get_receiver]
:end-before: [END get_receiver]
:language: python
:dedent: 4
:caption: Get the receiver client from Service Bus client
"""
def __init__(
self, handler_id, source, auth_config, connection=None,
mode=ReceiveSettleMode.PeekLock, encoding='UTF-8', debug=False, **kwargs):
self._used = threading.Event()
self.name = "SBReceiver-{}".format(handler_id)
self.last_received = None
self.mode = mode
self.message_iter = None
super(Receiver, self).__init__(
source, auth_config, connection=connection, encoding=encoding, debug=debug, **kwargs)
def __iter__(self):
return self
def __next__(self):
self._can_run()
while True:
if self.receiver_shutdown:
self.close()
raise StopIteration
try:
received = next(self.message_iter)
wrapped = self._build_message(received)
return wrapped
except StopIteration:
self.close()
raise
except Exception as e: # pylint: disable=broad-except
self._handle_exception(e)
def _build_handler(self):
auth = None if self.connection else authentication.SASTokenAuth.from_shared_access_key(**self.auth_config)
self._handler = ReceiveClient(
self.endpoint,
auth=auth,
debug=self.debug,
properties=self.properties,
error_policy=self.error_policy,
client_name=self.name,
auto_complete=False,
encoding=self.encoding,
**self.handler_kwargs)
def _build_message(self, received):
message = Message(None, message=received)
message._receiver = self # pylint: disable=protected-access
self.last_received = message.sequence_number
return message
def _can_run(self):
if self._used.is_set():
raise InvalidHandlerState("Receiver has already closed.")
if self.receiver_shutdown:
self.close()
raise InvalidHandlerState("Receiver has already closed.")
if not self.running:
self.open()
def _renew_locks(self, *lock_tokens):
message = {'lock-tokens': types.AMQPArray(lock_tokens)}
return self._mgmt_request_response(
REQUEST_RESPONSE_RENEWLOCK_OPERATION,
message,
mgmt_handlers.lock_renew_op)
def _settle_deferred(self, settlement, lock_tokens, dead_letter_details=None):
message = {
'disposition-status': settlement,
'lock-tokens': types.AMQPArray(lock_tokens)}
if dead_letter_details:
message.update(dead_letter_details)
return self._mgmt_request_response(
REQUEST_RESPONSE_UPDATE_DISPOSTION_OPERATION,
message,
mgmt_handlers.default)
def _build_receiver(self):
"""This is a temporary patch pending a fix in uAMQP."""
# pylint: disable=protected-access
self._handler.message_handler = self._handler.receiver_type(
self._handler._session,
self._handler._remote_address,
self._handler._name,
on_message_received=self._handler._message_received,
name='receiver-link-{}'.format(uuid.uuid4()),
debug=self._handler._debug_trace,
prefetch=self._handler._prefetch,
max_message_size=self._handler._max_message_size,
properties=self._handler._link_properties,
error_policy=self._handler._error_policy,
encoding=self._handler._encoding)
if self.mode != ReceiveSettleMode.PeekLock:
self._handler.message_handler.send_settle_mode = constants.SenderSettleMode.Settled
self._handler.message_handler.receive_settle_mode = constants.ReceiverSettleMode.ReceiveAndDelete
self._handler.message_handler._settle_mode = constants.ReceiverSettleMode.ReceiveAndDelete
self._handler.message_handler.open()
def next(self):
return self.__next__()
@property
def receiver_shutdown(self):
if self._handler:
return self._handler._shutdown # pylint: disable=protected-access
return True
@receiver_shutdown.setter
def receiver_shutdown(self, value):
if self._handler:
self._handler._shutdown = value # pylint: disable=protected-access
else:
raise ValueError("Receiver has no AMQP handler")
@property
def queue_size(self):
"""The current size of the unprocessed message queue.
:rtype: int
Example:
.. literalinclude:: ../examples/test_examples.py
:start-after: [START queue_size]
:end-before: [END queue_size]
:language: python
:dedent: 4
:caption: Get the number of unprocessed messages in the queue
"""
# pylint: disable=protected-access
if self._handler._received_messages:
return self._handler._received_messages.qsize()
return 0
def peek(self, count=1, start_from=None):
"""Browse messages currently pending in the queue.
Peeked messages are not removed from queue, nor are they locked. They cannot be completed,
deferred or dead-lettered.
:param count: The maximum number of messages to try and peek. The default
value is 1.
:type count: int
:param start_from: A message sequence number from which to start browsing messages.
:type start_from: int
:rtype: list[~azure.servicebus.common.message.PeekMessage]
Example:
.. literalinclude:: ../examples/test_examples.py
:start-after: [START peek_messages]
:end-before: [END peek_messages]
:language: python
:dedent: 4
:caption: Look at pending messages in the queue
"""
if not start_from:
start_from = self.last_received or 1
if int(count) < 1:
raise ValueError("count must be 1 or greater.")
if int(start_from) < 1:
raise ValueError("start_from must be 1 or greater.")
self._can_run()
message = {
'from-sequence-number': types.AMQPLong(start_from),
'message-count': count
}
return self._mgmt_request_response(
REQUEST_RESPONSE_PEEK_OPERATION,
message,
mgmt_handlers.peek_op)
def receive_deferred_messages(self, sequence_numbers, mode=ReceiveSettleMode.PeekLock):
"""Receive messages that have previously been deferred.
When receiving deferred messages from a partitioned entity, all of the supplied
sequence numbers must be messages from the same partition.
:param sequence_numbers: A list of the sequence numbers of messages that have been
deferred.
:type sequence_numbers: list[int]
:param mode: The receive mode, default value is PeekLock.
:type mode: ~azure.servicebus.common.constants.ReceiveSettleMode
:rtype: list[~azure.servicebus.common.message.DeferredMessage]
Example:
.. literalinclude:: ../examples/test_examples.py
:start-after: [START receive_deferred_messages]
:end-before: [END receive_deferred_messages]
:language: python
:dedent: 8
:caption: Get the messages which were previously deferred
"""
if not sequence_numbers:
raise ValueError("At least one sequence number must be specified.")
self._can_run()
try:
receive_mode = mode.value.value
except AttributeError:
receive_mode = int(mode)
message = {
'sequence-numbers': types.AMQPArray([types.AMQPLong(s) for s in sequence_numbers]),
'receiver-settle-mode': types.AMQPuInt(receive_mode)
}
handler = functools.partial(mgmt_handlers.deferred_message_op, mode=receive_mode)
messages = self._mgmt_request_response(
REQUEST_RESPONSE_RECEIVE_BY_SEQUENCE_NUMBER,
message,
handler)
for m in messages:
m._receiver = self # pylint: disable=protected-access
return messages
def open(self):
"""Open receiver connection and authenticate session.
If the receiver is already open, this operation will do nothing.
This method will be called automatically when one starts to iterate
messages in the receiver, so there should be no need to call it directly.
A receiver opened with this method must be explicitly closed.
It is recommended to open a handler within a context manager as
opposed to calling the method directly.
.. note:: This operation is not thread-safe.
"""
if self.running:
return
self.running = True
try:
self._handler.open(connection=self.connection)
self.message_iter = self._handler.receive_messages_iter()
while not self._handler.auth_complete():
time.sleep(0.05)
self._build_receiver()
while not self._handler.client_ready():
time.sleep(0.05)
except Exception as e: # pylint: disable=broad-except
try:
self._handle_exception(e)
except:
self.running = False
raise
def close(self, exception=None):
"""Close down the receiver connection.
If the receiver has already closed, this operation will do nothing. An optional exception can be passed in to
indicate that the handler was shutdown due to error.
It is recommended to open a handler within a context manager as
opposed to calling the method directly.
The receiver will be implicitly closed on completion of the message iterator,
however this method will need to be called explicitly if the message iterator is not run
to completion.
.. note:: This operation is not thread-safe.
:param exception: An optional exception if the handler is closing
due to an error.
:type exception: Exception
Example:
.. literalinclude:: ../examples/test_examples.py
:start-after: [START open_close_receiver_connection]
:end-before: [END open_close_receiver_connection]
:language: python
:dedent: 4
:caption: Close the connection and shutdown the receiver
"""
if not self.running:
return
self.running = False
self.receiver_shutdown = True
self._used.set()
super(Receiver, self).close(exception=exception)
def fetch_next(self, max_batch_size=None, timeout=None):
"""Receive a batch of messages at once.
This approach it optimal if you wish to process multiple messages simultaneously. Note that the
number of messages retrieved in a single batch will be dependent on
whether `prefetch` was set for the receiver. This call will prioritize returning
quickly over meeting a specified batch size, and so will return as soon as at least
one message is received and there is a gap in incoming messages regardless
of the specified batch size.
:param max_batch_size: Maximum number of messages in the batch. Actual number
returned will depend on prefetch size and incoming stream rate.
:type max_batch_size: int
:param timeout: The time to wait in seconds for the first message to arrive.
If no messages arrive, and no timeout is specified, this call will not return
until the connection is closed. If specified, an no messages arrive within the
timeout period, an empty list will be returned.
:rtype: list[~azure.servicebus.common.message.Message]
Example:
.. literalinclude:: ../examples/test_examples.py
:start-after: [START fetch_next_messages]
:end-before: [END fetch_next_messages]
:language: python
:dedent: 4
:caption: Get the messages in batch from the receiver
"""
self._can_run()
wrapped_batch = []
max_batch_size = max_batch_size or self._handler._prefetch # pylint: disable=protected-access
try:
timeout_ms = 1000 * timeout if timeout else 0
batch = self._handler.receive_message_batch(
max_batch_size=max_batch_size,
timeout=timeout_ms)
for received in batch:
message = self._build_message(received)
wrapped_batch.append(message)
except Exception as e: # pylint: disable=broad-except
self._handle_exception(e)
return wrapped_batch
class Receiver(BaseHandler): # pylint: disable=too-many-instance-attributes
"""A message receiver.
This receive handler acts as an iterable message stream for retrieving
messages for a Service Bus entity. It operates a single connection that must be opened and
closed on completion. The service connection will remain open for the entirety of the iterator.
If you find yourself only partially iterating the message stream, you should run the receiver
in a `with` statement to ensure the connection is closed.
The Receiver should not be instantiated directly, and should be accessed from a `QueueClient` or
`SubscriptionClient` using the `get_receiver()` method.
.. note:: This object is not thread-safe.
:param handler_id: The ID used as the connection name for the Receiver.
:type handler_id: str
:param source: The endpoint from which to receive messages.
:type source: ~uamqp.Source
:param auth_config: The SASL auth credentials.
:type auth_config: dict[str, str]
:param connection: A shared connection [not yet supported].
:type connection: ~uamqp.Connection
:param mode: The receive connection mode. Value must be either PeekLock or ReceiveAndDelete.
:type mode: ~azure.servicebus.common.constants.ReceiveSettleMode
:param encoding: The encoding used for string properties. Default is 'UTF-8'.
:type encoding: str
:param debug: Whether to enable network trace debug logs.
:type debug: bool
.. admonition:: Example:
.. literalinclude:: ../samples/sync_samples/test_examples.py
:start-after: [START get_receiver]
:end-before: [END get_receiver]
:language: python
:dedent: 4
:caption: Get the receiver client from Service Bus client
"""
def __init__(self,
handler_id,
source,
auth_config,
connection=None,
mode=ReceiveSettleMode.PeekLock,
encoding='UTF-8',
debug=False,
**kwargs):
self._used = threading.Event()
self.name = "SBReceiver-{}".format(handler_id)
self.last_received = None
self.mode = mode
self.message_iter = None
super(Receiver, self).__init__(source,
auth_config,
connection=connection,
encoding=encoding,
debug=debug,
**kwargs)
def __iter__(self):
return self
def __next__(self):
self._can_run()
while True:
if self.receiver_shutdown:
self.close()
raise StopIteration
try:
received = next(self.message_iter)
wrapped = self._build_message(received)
return wrapped
except StopIteration:
self.close()
raise
except Exception as e: # pylint: disable=broad-except
self._handle_exception(e)
def _build_handler(self):
auth = None if self.connection else authentication.SASTokenAuth.from_shared_access_key(
**self.auth_config)
self._handler = ReceiveClient(self.endpoint,
auth=auth,
debug=self.debug,
properties=self.properties,
error_policy=self.error_policy,
client_name=self.name,
auto_complete=False,
encoding=self.encoding,
**self.handler_kwargs)
def _build_message(self, received):
message = Message(None, message=received)
message._receiver = self # pylint: disable=protected-access
self.last_received = message.sequence_number
return message
def _can_run(self):
if self._used.is_set():
raise InvalidHandlerState("Receiver has already closed.")
if self.receiver_shutdown:
self.close()
raise InvalidHandlerState("Receiver has already closed.")
if not self.running:
self.open()
def _renew_locks(self, *lock_tokens):
message = {'lock-tokens': types.AMQPArray(lock_tokens)}
return self._mgmt_request_response(
REQUEST_RESPONSE_RENEWLOCK_OPERATION, message,
mgmt_handlers.lock_renew_op)
def _settle_deferred(self,
settlement,
lock_tokens,
dead_letter_details=None):
message = {
'disposition-status': settlement,
'lock-tokens': types.AMQPArray(lock_tokens)
}
if dead_letter_details:
message.update(dead_letter_details)
return self._mgmt_request_response(
REQUEST_RESPONSE_UPDATE_DISPOSTION_OPERATION, message,
mgmt_handlers.default)
def _build_receiver(self):
"""This is a temporary patch pending a fix in uAMQP."""
# pylint: disable=protected-access
self._handler.message_handler = self._handler.receiver_type(
self._handler._session,
self._handler._remote_address,
self._handler._name,
on_message_received=self._handler._message_received,
name='receiver-link-{}'.format(uuid.uuid4()),
debug=self._handler._debug_trace,
prefetch=self._handler._prefetch,
max_message_size=self._handler._max_message_size,
properties=self._handler._link_properties,
error_policy=self._handler._error_policy,
encoding=self._handler._encoding)
if self.mode != ReceiveSettleMode.PeekLock:
self._handler.message_handler.send_settle_mode = constants.SenderSettleMode.Settled
self._handler.message_handler.receive_settle_mode = constants.ReceiverSettleMode.ReceiveAndDelete
self._handler.message_handler._settle_mode = constants.ReceiverSettleMode.ReceiveAndDelete
self._handler.message_handler.open()
def next(self):
return self.__next__()
@property
def receiver_shutdown(self):
if self._handler:
return self._handler._shutdown # pylint: disable=protected-access
return True
@receiver_shutdown.setter
def receiver_shutdown(self, value):
if self._handler:
self._handler._shutdown = value # pylint: disable=protected-access
else:
raise ValueError("Receiver has no AMQP handler")
@property
def queue_size(self):
"""The current size of the unprocessed message queue.
:rtype: int
.. admonition:: Example:
.. literalinclude:: ../samples/sync_samples/test_examples.py
:start-after: [START queue_size]
:end-before: [END queue_size]
:language: python
:dedent: 4
:caption: Get the number of unprocessed messages in the queue
"""
# pylint: disable=protected-access
if self._handler._received_messages:
return self._handler._received_messages.qsize()
return 0
def peek(self, count=1, start_from=None):
"""Browse messages currently pending in the queue.
Peeked messages are not removed from queue, nor are they locked. They cannot be completed,
deferred or dead-lettered.
:param count: The maximum number of messages to try and peek. The default
value is 1.
:type count: int
:param start_from: A message sequence number from which to start browsing messages.
:type start_from: int
:rtype: list[~azure.servicebus.common.message.PeekMessage]
.. admonition:: Example:
.. literalinclude:: ../samples/sync_samples/test_examples.py
:start-after: [START peek_messages]
:end-before: [END peek_messages]
:language: python
:dedent: 4
:caption: Look at pending messages in the queue
"""
if not start_from:
start_from = self.last_received or 1
if int(count) < 1:
raise ValueError("count must be 1 or greater.")
if int(start_from) < 1:
raise ValueError("start_from must be 1 or greater.")
self._can_run()
message = {
'from-sequence-number': types.AMQPLong(start_from),
'message-count': count
}
return self._mgmt_request_response(REQUEST_RESPONSE_PEEK_OPERATION,
message, mgmt_handlers.peek_op)
def receive_deferred_messages(self,
sequence_numbers,
mode=ReceiveSettleMode.PeekLock):
"""Receive messages that have previously been deferred.
When receiving deferred messages from a partitioned entity, all of the supplied
sequence numbers must be messages from the same partition.
:param sequence_numbers: A list of the sequence numbers of messages that have been
deferred.
:type sequence_numbers: list[int]
:param mode: The receive mode, default value is PeekLock.
:type mode: ~azure.servicebus.common.constants.ReceiveSettleMode
:rtype: list[~azure.servicebus.common.message.DeferredMessage]
.. admonition:: Example:
.. literalinclude:: ../samples/sync_samples/test_examples.py
:start-after: [START receive_deferred_messages]
:end-before: [END receive_deferred_messages]
:language: python
:dedent: 8
:caption: Get the messages which were previously deferred
"""
if not sequence_numbers:
raise ValueError("At least one sequence number must be specified.")
self._can_run()
try:
receive_mode = mode.value.value
except AttributeError:
receive_mode = int(mode)
message = {
'sequence-numbers':
types.AMQPArray([types.AMQPLong(s) for s in sequence_numbers]),
'receiver-settle-mode':
types.AMQPuInt(receive_mode)
}
handler = functools.partial(mgmt_handlers.deferred_message_op,
mode=receive_mode)
messages = self._mgmt_request_response(
REQUEST_RESPONSE_RECEIVE_BY_SEQUENCE_NUMBER, message, handler)
for m in messages:
m._receiver = self # pylint: disable=protected-access
return messages
def open(self):
"""Open receiver connection and authenticate session.
If the receiver is already open, this operation will do nothing.
This method will be called automatically when one starts to iterate
messages in the receiver, so there should be no need to call it directly.
A receiver opened with this method must be explicitly closed.
It is recommended to open a handler within a context manager as
opposed to calling the method directly.
.. note:: This operation is not thread-safe.
"""
if self.running:
return
self.running = True
try:
self._handler.open(connection=self.connection)
self.message_iter = self._handler.receive_messages_iter()
while not self._handler.auth_complete():
time.sleep(0.05)
self._build_receiver()
while not self._handler.client_ready():
time.sleep(0.05)
except Exception as e: # pylint: disable=broad-except
try:
self._handle_exception(e)
except:
self.running = False
raise
def close(self, exception=None):
"""Close down the receiver connection.
If the receiver has already closed, this operation will do nothing. An optional exception can be passed in to
indicate that the handler was shutdown due to error.
It is recommended to open a handler within a context manager as
opposed to calling the method directly.
The receiver will be implicitly closed on completion of the message iterator,
however this method will need to be called explicitly if the message iterator is not run
to completion.
.. note:: This operation is not thread-safe.
:param exception: An optional exception if the handler is closing
due to an error.
:type exception: Exception
.. admonition:: Example:
.. literalinclude:: ../samples/sync_samples/test_examples.py
:start-after: [START open_close_receiver_connection]
:end-before: [END open_close_receiver_connection]
:language: python
:dedent: 4
:caption: Close the connection and shutdown the receiver
"""
if not self.running:
return
self.running = False
self.receiver_shutdown = True
self._used.set()
super(Receiver, self).close(exception=exception)
def fetch_next(self, max_batch_size=None, timeout=None):
"""Receive a batch of messages at once.
This approach it optimal if you wish to process multiple messages simultaneously. Note that the
number of messages retrieved in a single batch will be dependent on
whether `prefetch` was set for the receiver. This call will prioritize returning
quickly over meeting a specified batch size, and so will return as soon as at least
one message is received and there is a gap in incoming messages regardless
of the specified batch size.
:param max_batch_size: Maximum number of messages in the batch. Actual number
returned will depend on prefetch size and incoming stream rate.
:type max_batch_size: int
:param timeout: The time to wait in seconds for the first message to arrive.
If no messages arrive, and no timeout is specified, this call will not return
until the connection is closed. If specified, an no messages arrive within the
timeout period, an empty list will be returned.
:rtype: list[~azure.servicebus.common.message.Message]
.. admonition:: Example:
.. literalinclude:: ../samples/sync_samples/test_examples.py
:start-after: [START fetch_next_messages]
:end-before: [END fetch_next_messages]
:language: python
:dedent: 4
:caption: Get the messages in batch from the receiver
"""
self._can_run()
wrapped_batch = []
max_batch_size = max_batch_size or self._handler._prefetch # pylint: disable=protected-access
try:
timeout_ms = 1000 * timeout if timeout else 0
batch = self._handler.receive_message_batch(
max_batch_size=max_batch_size, timeout=timeout_ms)
for received in batch:
message = self._build_message(received)
wrapped_batch.append(message)
except Exception as e: # pylint: disable=broad-except
self._handle_exception(e)
return wrapped_batch
class Receiver:
"""
Implements a Receiver.
"""
timeout = 0
_epoch = b'com.microsoft:epoch'
def __init__(self, client, source, prefetch=300, epoch=None):
"""
Instantiate a receiver.
:param client: The parent EventHubClient.
:type client: ~azure.eventhub.EventHubClient
:param source: The source EventHub from which to receive events.
:type source: ~uamqp.Source
:param prefetch: The number of events to prefetch from the service
for processing. Default is 300.
:type prefetch: int
:param epoch: An optional epoch value.
:type epoch: int
"""
self.offset = None
self._callback = None
self.prefetch = prefetch
self.epoch = epoch
properties = None
if epoch:
properties = {
types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))
}
self._handler = ReceiveClient(source,
auth=client.auth,
debug=client.debug,
prefetch=self.prefetch,
link_properties=properties,
timeout=self.timeout)
@property
def queue_size(self):
"""
The current size of the unprocessed message queue.
:returns: int
"""
# pylint: disable=protected-access
if self._handler._received_messages:
return self._handler._received_messages.qsize()
return 0
def on_message(self, event):
"""
Callback to process a received message and wrap it in EventData.
Will also call a user supplied callback.
:param event: The received message.
:type event: ~uamqp.Message
:returns: ~azure.eventhub.EventData.
"""
event_data = EventData(message=event)
if self._callback:
self._callback(event_data)
self.offset = event_data.offset
return event_data
def receive(self, max_batch_size=None, callback=None, timeout=None):
"""
Receive events from the EventHub.
:param max_batch_size: Receive a batch of events. Batch size will
be up to the maximum specified, but will return as soon as service
returns no new events. If combined with a timeout and no events are
retrieve before the time, the result will be empty. If no batch
size is supplied, the prefetch size will be the maximum.
:type max_batch_size: int
:param callback: A callback to be run for each received event. This must
be a function that accepts a single argument - the event data. This callback
will be run before the message is returned in the result generator.
:type callback: func[~azure.eventhub.EventData]
:returns: list[~azure.eventhub.EventData]
"""
try:
timeout_ms = 1000 * timeout if timeout else 0
self._callback = callback
batch = self._handler.receive_message_batch(
max_batch_size=max_batch_size,
on_message_received=self.on_message,
timeout=timeout_ms)
return batch
except errors.AMQPConnectionError as e:
message = "Failed to open receiver: {}".format(e)
message += "\nPlease check that the partition key is valid "
if self.epoch:
message += ("and that a higher epoch receiver is not "
"already running for this partition.")
else:
message += ("and whether an epoch receiver is "
"already running for this partition.")
raise EventHubError(message)
except Exception as e:
raise EventHubError("Receive failed: {}".format(e))
def selector(self, default):
"""
Create a selector for the current offset if it is set.
:param default: The fallback receive offset.
:type default: ~azure.eventhub.Offset
:returns: ~azure.eventhub.Offset
"""
if self.offset is not None:
return Offset(self.offset).selector()
return default
class EventHubConsumer(ConsumerProducerMixin): # pylint:disable=too-many-instance-attributes
"""
A consumer responsible for reading EventData from a specific Event Hub
partition and as a member of a specific consumer group.
A consumer may be exclusive, which asserts ownership over the partition for the consumer
group to ensure that only one consumer from that group is reading the from the partition.
These exclusive consumers are sometimes referred to as "Epoch Consumers."
A consumer may also be non-exclusive, allowing multiple consumers from the same consumer
group to be actively reading events from the partition. These non-exclusive consumers are
sometimes referred to as "Non-Epoch Consumers."
Please use the method `create_consumer` on `EventHubClient` for creating `EventHubConsumer`.
"""
_timeout = 0
_epoch_symbol = b'com.microsoft:epoch'
_timeout_symbol = b'com.microsoft:timeout'
_receiver_runtime_metric_symbol = b'com.microsoft:enable-receiver-runtime-metric'
def __init__(self, client, source, **kwargs):
"""
Instantiate a consumer. EventHubConsumer should be instantiated by calling the `create_consumer` method
in EventHubClient.
:param client: The parent EventHubClient.
:type client: ~azure.eventhub.client.EventHubClient
:param source: The source EventHub from which to receive events.
:type source: str
:param prefetch: The number of events to prefetch from the service
for processing. Default is 300.
:type prefetch: int
:param owner_level: The priority of the exclusive consumer. An exclusive
consumer will be created if owner_level is set.
:type owner_level: int
:param track_last_enqueued_event_properties: Indicates whether or not the consumer should request information
on the last enqueued event on its associated partition, and track that information as events are received.
When information about the partition's last enqueued event is being tracked, each event received from the
Event Hubs service will carry metadata about the partition. This results in a small amount of additional
network bandwidth consumption that is generally a favorable trade-off when considered against periodically
making requests for partition properties using the Event Hub client.
It is set to `False` by default.
:type track_last_enqueued_event_properties: bool
"""
event_position = kwargs.get("event_position", None)
prefetch = kwargs.get("prefetch", 300)
owner_level = kwargs.get("owner_level", None)
keep_alive = kwargs.get("keep_alive", None)
auto_reconnect = kwargs.get("auto_reconnect", True)
track_last_enqueued_event_properties = kwargs.get("track_last_enqueued_event_properties", False)
super(EventHubConsumer, self).__init__()
self._client = client
self._source = source
self._offset = event_position
self._messages_iter = None
self._prefetch = prefetch
self._owner_level = owner_level
self._keep_alive = keep_alive
self._auto_reconnect = auto_reconnect
self._retry_policy = errors.ErrorPolicy(max_retries=self._client._config.max_retries, on_error=_error_handler) # pylint:disable=protected-access
self._reconnect_backoff = 1
self._link_properties = {}
self._error = None
partition = self._source.split('/')[-1]
self._partition = partition
self._name = "EHConsumer-{}-partition{}".format(uuid.uuid4(), partition)
if owner_level:
self._link_properties[types.AMQPSymbol(self._epoch_symbol)] = types.AMQPLong(int(owner_level))
link_property_timeout_ms = (self._client._config.receive_timeout or self._timeout) * 1000 # pylint:disable=protected-access
self._link_properties[types.AMQPSymbol(self._timeout_symbol)] = types.AMQPLong(int(link_property_timeout_ms))
self._handler = None
self._track_last_enqueued_event_properties = track_last_enqueued_event_properties
self._last_enqueued_event_properties = {}
def __iter__(self):
return self
def __next__(self):
retried_times = 0
last_exception = None
while retried_times < self._client._config.max_retries: # pylint:disable=protected-access
try:
self._open()
if not self._messages_iter:
self._messages_iter = self._handler.receive_messages_iter()
message = next(self._messages_iter)
event_data = EventData._from_message(message) # pylint:disable=protected-access
event_data._trace_link_message() # pylint:disable=protected-access
self._offset = EventPosition(event_data.offset, inclusive=False)
retried_times = 0
if self._track_last_enqueued_event_properties:
self._last_enqueued_event_properties = event_data._get_last_enqueued_event_properties() # pylint:disable=protected-access
return event_data
except Exception as exception: # pylint:disable=broad-except
last_exception = self._handle_exception(exception)
self._client._try_delay(retried_times=retried_times, last_exception=last_exception, # pylint:disable=protected-access
entity_name=self._name)
retried_times += 1
log.info("%r operation has exhausted retry. Last exception: %r.", self._name, last_exception)
raise last_exception
def _create_handler(self):
source = Source(self._source)
if self._offset is not None:
source.set_filter(self._offset._selector()) # pylint:disable=protected-access
if StrictVersion(uamqp.__version__) < StrictVersion("1.2.3"): # backward compatible until uamqp 1.2.3 released
desired_capabilities = {}
elif self._track_last_enqueued_event_properties:
symbol_array = [types.AMQPSymbol(self._receiver_runtime_metric_symbol)]
desired_capabilities = {"desired_capabilities": utils.data_factory(types.AMQPArray(symbol_array))}
else:
desired_capabilities = {"desired_capabilities": None}
self._handler = ReceiveClient(
source,
auth=self._client._create_auth(), # pylint:disable=protected-access
debug=self._client._config.network_tracing, # pylint:disable=protected-access
prefetch=self._prefetch,
link_properties=self._link_properties,
timeout=self._timeout,
error_policy=self._retry_policy,
keep_alive_interval=self._keep_alive,
client_name=self._name,
receive_settle_mode=uamqp.constants.ReceiverSettleMode.ReceiveAndDelete,
auto_complete=False,
properties=self._client._create_properties( # pylint:disable=protected-access
self._client._config.user_agent), # pylint:disable=protected-access
**desired_capabilities) # pylint:disable=protected-access
self._messages_iter = None
def _open_with_retry(self):
return self._do_retryable_operation(self._open, operation_need_param=False)
def _receive(self, timeout_time=None, max_batch_size=None, **kwargs):
last_exception = kwargs.get("last_exception")
data_batch = []
self._open()
remaining_time = timeout_time - time.time()
if remaining_time <= 0.0:
if last_exception:
log.info("%r receive operation timed out. (%r)", self._name, last_exception)
raise last_exception
return data_batch
remaining_time_ms = 1000 * remaining_time
message_batch = self._handler.receive_message_batch(
max_batch_size=max_batch_size,
timeout=remaining_time_ms)
for message in message_batch:
event_data = EventData._from_message(message) # pylint:disable=protected-access
data_batch.append(event_data)
event_data._trace_link_message() # pylint:disable=protected-access
if data_batch:
self._offset = EventPosition(data_batch[-1].offset)
if self._track_last_enqueued_event_properties and data_batch:
self._last_enqueued_event_properties = data_batch[-1]._get_last_enqueued_event_properties() # pylint:disable=protected-access
return data_batch
def _receive_with_retry(self, timeout=None, max_batch_size=None, **kwargs):
return self._do_retryable_operation(self._receive, timeout=timeout,
max_batch_size=max_batch_size, **kwargs)
@property
def last_enqueued_event_properties(self):
"""
The latest enqueued event information. This property will be updated each time an event is received when
the receiver is created with `track_last_enqueued_event_properties` being `True`.
The dict includes following information of the partition:
- `sequence_number`
- `offset`
- `enqueued_time`
- `retrieval_time`
:rtype: dict or None
"""
return self._last_enqueued_event_properties if self._track_last_enqueued_event_properties else None
@property
def queue_size(self):
# type:() -> int
"""
The current size of the unprocessed Event queue.
:rtype: int
"""
# pylint: disable=protected-access
if self._handler._received_messages:
return self._handler._received_messages.qsize()
return 0
def receive(self, max_batch_size=None, timeout=None):
# type: (int, float) -> List[EventData]
"""
Receive events from the EventHub.
:param max_batch_size: Receive a batch of events. Batch size will
be up to the maximum specified, but will return as soon as service
returns no new events. If combined with a timeout and no events are
retrieve before the time, the result will be empty. If no batch
size is supplied, the prefetch size will be the maximum.
:type max_batch_size: int
:param timeout: The maximum wait time to build up the requested message count for the batch.
If not specified, the default wait time specified when the consumer was created will be used.
:type timeout: float
:rtype: list[~azure.eventhub.common.EventData]
:raises: ~azure.eventhub.AuthenticationError, ~azure.eventhub.ConnectError, ~azure.eventhub.ConnectionLostError,
~azure.eventhub.EventHubError
"""
self._check_closed()
timeout = timeout or self._client._config.receive_timeout # pylint:disable=protected-access
max_batch_size = max_batch_size or min(self._client._config.max_batch_size, self._prefetch) # pylint:disable=protected-access
return self._receive_with_retry(timeout=timeout, max_batch_size=max_batch_size)
def close(self): # pylint:disable=useless-super-delegation
# type:() -> None
"""
Close down the handler. If the handler has already closed,
this will be a no op.
"""
super(EventHubConsumer, self).close()
next = __next__ # for python2.7
class Receiver(object):
"""
Implements a Receiver.
Example:
.. literalinclude:: ../examples/test_examples_eventhub.py
:start-after: [START create_eventhub_client_receiver_instance]
:end-before: [END create_eventhub_client_receiver_instance]
:language: python
:dedent: 4
:caption: Create a new instance of the Receiver.
"""
timeout = 0
_epoch = b'com.microsoft:epoch'
def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_alive=None, auto_reconnect=True):
"""
Instantiate a receiver.
:param client: The parent EventHubClient.
:type client: ~azure.eventhub.client.EventHubClient
:param source: The source EventHub from which to receive events.
:type source: str
:param prefetch: The number of events to prefetch from the service
for processing. Default is 300.
:type prefetch: int
:param epoch: An optional epoch value.
:type epoch: int
"""
self.running = False
self.client = client
self.source = source
self.offset = offset
self.prefetch = prefetch
self.epoch = epoch
self.keep_alive = keep_alive
self.auto_reconnect = auto_reconnect
self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler)
self.reconnect_backoff = 1
self.properties = None
self.redirected = None
self.error = None
partition = self.source.split('/')[-1]
self.name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition)
source = Source(self.source)
if self.offset is not None:
source.set_filter(self.offset.selector())
if epoch:
self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))}
self._handler = ReceiveClient(
source,
auth=self.client.get_auth(),
debug=self.client.debug,
prefetch=self.prefetch,
link_properties=self.properties,
timeout=self.timeout,
error_policy=self.retry_policy,
keep_alive_interval=self.keep_alive,
client_name=self.name,
properties=self.client.create_properties())
def open(self):
"""
Open the Receiver using the supplied conneciton.
If the handler has previously been redirected, the redirect
context will be used to create a new handler before opening it.
:param connection: The underlying client shared connection.
:type: connection: ~uamqp.connection.Connection
Example:
.. literalinclude:: ../examples/test_examples_eventhub.py
:start-after: [START eventhub_client_receiver_open]
:end-before: [END eventhub_client_receiver_open]
:language: python
:dedent: 4
:caption: Open the Receiver using the supplied conneciton.
"""
# pylint: disable=protected-access
self.running = True
if self.redirected:
self.source = self.redirected.address
source = Source(self.source)
if self.offset is not None:
source.set_filter(self.offset.selector())
alt_creds = {
"username": self.client._auth_config.get("iot_username"),
"password":self.client._auth_config.get("iot_password")}
self._handler = ReceiveClient(
source,
auth=self.client.get_auth(**alt_creds),
debug=self.client.debug,
prefetch=self.prefetch,
link_properties=self.properties,
timeout=self.timeout,
error_policy=self.retry_policy,
keep_alive_interval=self.keep_alive,
client_name=self.name,
properties=self.client.create_properties())
self._handler.open()
while not self._handler.client_ready():
time.sleep(0.05)
def _reconnect(self): # pylint: disable=too-many-statements
# pylint: disable=protected-access
alt_creds = {
"username": self.client._auth_config.get("iot_username"),
"password": self.client._auth_config.get("iot_password")}
self._handler.close()
source = Source(self.source)
if self.offset is not None:
source.set_filter(self.offset.selector())
self._handler = ReceiveClient(
source,
auth=self.client.get_auth(**alt_creds),
debug=self.client.debug,
prefetch=self.prefetch,
link_properties=self.properties,
timeout=self.timeout,
error_policy=self.retry_policy,
keep_alive_interval=self.keep_alive,
client_name=self.name,
properties=self.client.create_properties())
try:
self._handler.open()
while not self._handler.client_ready():
time.sleep(0.05)
return True
except errors.TokenExpired as shutdown:
log.info("Receiver disconnected due to token expiry. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
self.close(exception=error)
raise error
except (errors.LinkDetach, errors.ConnectionClose) as shutdown:
if shutdown.action.retry and self.auto_reconnect:
log.info("Receiver detached. Attempting reconnect.")
return False
log.info("Receiver detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
self.close(exception=error)
raise error
except errors.MessageHandlerError as shutdown:
if self.auto_reconnect:
log.info("Receiver detached. Attempting reconnect.")
return False
log.info("Receiver detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
self.close(exception=error)
raise error
except errors.AMQPConnectionError as shutdown:
if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect:
log.info("Receiver couldn't authenticate. Attempting reconnect.")
return False
log.info("Receiver connection error (%r). Shutting down.", shutdown)
error = EventHubError(str(shutdown))
self.close(exception=error)
raise error
except Exception as e:
log.info("Unexpected error occurred (%r). Shutting down.", e)
error = EventHubError("Receiver reconnect failed: {}".format(e))
self.close(exception=error)
raise error
def reconnect(self):
"""If the Receiver was disconnected from the service with
a retryable error - attempt to reconnect."""
while not self._reconnect():
time.sleep(self.reconnect_backoff)
def get_handler_state(self):
"""
Get the state of the underlying handler with regards to start
up processes.
:rtype: ~uamqp.constants.MessageReceiverState
"""
# pylint: disable=protected-access
return self._handler._message_receiver.get_state()
def has_started(self):
"""
Whether the handler has completed all start up processes such as
establishing the connection, session, link and authentication, and
is not ready to process messages.
**This function is now deprecated and will be removed in v2.0+.**
:rtype: bool
"""
# pylint: disable=protected-access
timeout = False
auth_in_progress = False
if self._handler._connection.cbs:
timeout, auth_in_progress = self._handler._auth.handle_token()
if timeout:
raise EventHubError("Authorization timeout.")
if auth_in_progress:
return False
if not self._handler._client_ready():
return False
return True
def close(self, exception=None):
"""
Close down the handler. If the handler has already closed,
this will be a no op. An optional exception can be passed in to
indicate that the handler was shutdown due to error.
:param exception: An optional exception if the handler is closing
due to an error.
:type exception: Exception
Example:
.. literalinclude:: ../examples/test_examples_eventhub.py
:start-after: [START eventhub_client_receiver_close]
:end-before: [END eventhub_client_receiver_close]
:language: python
:dedent: 4
:caption: Close down the handler.
"""
self.running = False
if self.error:
return
if isinstance(exception, errors.LinkRedirect):
self.redirected = exception
elif isinstance(exception, EventHubError):
self.error = exception
elif exception:
self.error = EventHubError(str(exception))
else:
self.error = EventHubError("This receive handler is now closed.")
self._handler.close()
@property
def queue_size(self):
"""
The current size of the unprocessed Event queue.
:rtype: int
"""
# pylint: disable=protected-access
if self._handler._received_messages:
return self._handler._received_messages.qsize()
return 0
def receive(self, max_batch_size=None, timeout=None):
"""
Receive events from the EventHub.
:param max_batch_size: Receive a batch of events. Batch size will
be up to the maximum specified, but will return as soon as service
returns no new events. If combined with a timeout and no events are
retrieve before the time, the result will be empty. If no batch
size is supplied, the prefetch size will be the maximum.
:type max_batch_size: int
:rtype: list[~azure.eventhub.common.EventData]
Example:
.. literalinclude:: ../examples/test_examples_eventhub.py
:start-after: [START eventhub_client_sync_receive]
:end-before: [END eventhub_client_sync_receive]
:language: python
:dedent: 4
:caption: Receive events from the EventHub.
"""
if self.error:
raise self.error
if not self.running:
raise ValueError("Unable to receive until client has been started.")
data_batch = []
try:
timeout_ms = 1000 * timeout if timeout else 0
message_batch = self._handler.receive_message_batch(
max_batch_size=max_batch_size,
timeout=timeout_ms)
for message in message_batch:
event_data = EventData(message=message)
self.offset = event_data.offset
data_batch.append(event_data)
return data_batch
except (errors.TokenExpired, errors.AuthenticationException):
log.info("Receiver disconnected due to token error. Attempting reconnect.")
self.reconnect()
return data_batch
except (errors.LinkDetach, errors.ConnectionClose) as shutdown:
if shutdown.action.retry and self.auto_reconnect:
log.info("Receiver detached. Attempting reconnect.")
self.reconnect()
return data_batch
log.info("Receiver detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
self.close(exception=error)
raise error
except errors.MessageHandlerError as shutdown:
if self.auto_reconnect:
log.info("Receiver detached. Attempting reconnect.")
self.reconnect()
return data_batch
log.info("Receiver detached. Shutting down.")
error = EventHubError(str(shutdown), shutdown)
self.close(exception=error)
raise error
except Exception as e:
log.info("Unexpected error occurred (%r). Shutting down.", e)
error = EventHubError("Receive failed: {}".format(e))
self.close(exception=error)
raise error