def create_properties(user_agent=None):
# type: (Optional[str]) -> Dict[types.AMQPSymbol, str]
"""
Format the properties with which to instantiate the connection.
This acts like a user agent over HTTP.
:rtype: dict
"""
properties = {}
properties[types.AMQPSymbol("product")] = USER_AGENT_PREFIX
properties[types.AMQPSymbol("version")] = VERSION
framework = "Python/{}.{}.{}".format(
sys.version_info[0], sys.version_info[1], sys.version_info[2]
)
properties[types.AMQPSymbol("framework")] = framework
platform_str = platform.platform()
properties[types.AMQPSymbol("platform")] = platform_str
final_user_agent = "{}/{} {} ({})".format(
USER_AGENT_PREFIX, VERSION, framework, platform_str
)
if user_agent:
final_user_agent = "{} {}".format(final_user_agent, user_agent)
if len(final_user_agent) > MAX_USER_AGENT_LENGTH:
raise ValueError(
"The user-agent string cannot be more than {} in length."
"Current user_agent string is: {} with length: {}".format(
MAX_USER_AGENT_LENGTH, final_user_agent, len(final_user_agent)
)
)
properties[types.AMQPSymbol("user-agent")] = final_user_agent
return properties
def _create_properties(self, user_agent=None): # pylint: disable=no-self-use
"""
Format the properties with which to instantiate the connection.
This acts like a user agent over HTTP.
:rtype: dict
"""
properties = {}
product = "azsdk-python-eventhubs"
properties[types.AMQPSymbol("product")] = product
properties[types.AMQPSymbol("version")] = __version__
framework = "Python {}.{}.{}, {}".format(
sys.version_info[0], sys.version_info[1], sys.version_info[2],
platform.python_implementation())
properties[types.AMQPSymbol("framework")] = framework
platform_str = platform.platform()
properties[types.AMQPSymbol("platform")] = platform_str
final_user_agent = '{}/{} ({}, {})'.format(product, __version__,
framework, platform_str)
if user_agent:
final_user_agent = '{}, {}'.format(final_user_agent, user_agent)
if len(final_user_agent) > MAX_USER_AGENT_LENGTH:
raise ValueError(
"The user-agent string cannot be more than {} in length."
"Current user_agent string is: {} with length: {}".format(
MAX_USER_AGENT_LENGTH, final_user_agent,
len(final_user_agent)))
properties[types.AMQPSymbol("user-agent")] = final_user_agent
return properties
def create_properties(user_agent=None):
# type: (Optional[str]) -> Dict[types.AMQPSymbol, str]
"""
Format the properties with which to instantiate the connection.
This acts like a user agent over HTTP.
:param str user_agent: If specified,
this will be added in front of the built-in user agent string.
:rtype: dict
"""
properties = {}
properties[types.AMQPSymbol("product")] = USER_AGENT_PREFIX
properties[types.AMQPSymbol("version")] = VERSION
framework = "Python/{}.{}.{}".format(sys.version_info[0],
sys.version_info[1],
sys.version_info[2])
properties[types.AMQPSymbol("framework")] = framework
platform_str = platform.platform()
properties[types.AMQPSymbol("platform")] = platform_str
final_user_agent = "{}/{} {} ({})".format(USER_AGENT_PREFIX, VERSION,
framework, platform_str)
if user_agent:
final_user_agent = "{} {}".format(user_agent, final_user_agent)
properties[types.AMQPSymbol("user-agent")] = final_user_agent
return properties
def __init__( # pylint: disable=super-init-not-called
self, client, source, **kwargs):
"""
Instantiate an async consumer. EventHubConsumer should be instantiated by calling the `create_consumer` method
in EventHubClient.
:param client: The parent EventHubClientAsync.
:type client: ~azure.eventhub.aio.EventHubClientAsync
:param source: The source EventHub from which to receive events.
:type source: ~uamqp.address.Source
:param event_position: The position from which to start receiving.
:type event_position: ~azure.eventhub.common.EventPosition
:param prefetch: The number of events to prefetch from the service
for processing. Default is 300.
:type prefetch: int
:param owner_level: The priority of the exclusive consumer. An exclusive
consumer will be created if owner_level is set.
:type owner_level: int
:param track_last_enqueued_event_properties: Indicates whether or not the consumer should request information
on the last enqueued event on its associated partition, and track that information as events are received.
When information about the partition's last enqueued event is being tracked, each event received from the
Event Hubs service will carry metadata about the partition. This results in a small amount of additional
network bandwidth consumption that is generally a favorable trade-off when considered against periodically
making requests for partition properties using the Event Hub client.
It is set to `False` by default.
:type track_last_enqueued_event_properties: bool
:param loop: An event loop.
"""
event_position = kwargs.get("event_position", None)
prefetch = kwargs.get("prefetch", 300)
owner_level = kwargs.get("owner_level", None)
keep_alive = kwargs.get("keep_alive", None)
auto_reconnect = kwargs.get("auto_reconnect", True)
track_last_enqueued_event_properties = kwargs.get("track_last_enqueued_event_properties", False)
loop = kwargs.get("loop", None)
super(EventHubConsumer, self).__init__()
self._loop = loop or asyncio.get_event_loop()
self._client = client
self._source = source
self._offset = event_position
self._messages_iter = None
self._prefetch = prefetch
self._owner_level = owner_level
self._keep_alive = keep_alive
self._auto_reconnect = auto_reconnect
self._retry_policy = errors.ErrorPolicy(max_retries=self._client._config.max_retries, on_error=_error_handler) # pylint:disable=protected-access
self._reconnect_backoff = 1
self._link_properties = {}
partition = self._source.split('/')[-1]
self._partition = partition
self._name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition)
if owner_level:
self._link_properties[types.AMQPSymbol(self._epoch_symbol)] = types.AMQPLong(int(owner_level))
link_property_timeout_ms = (self._client._config.receive_timeout or self._timeout) * 1000 # pylint:disable=protected-access
self._link_properties[types.AMQPSymbol(self._timeout_symbol)] = types.AMQPLong(int(link_property_timeout_ms))
self._handler = None
self._track_last_enqueued_event_properties = track_last_enqueued_event_properties
self._last_enqueued_event_properties = {}
def __init__( # pylint: disable=super-init-not-called
self, client, source, **kwargs):
"""
Instantiate an async consumer. EventHubConsumer should be instantiated by calling the `create_consumer` method
in EventHubClient.
:param client: The parent EventHubClientAsync.
:type client: ~azure.eventhub.aio.EventHubClientAsync
:param source: The source EventHub from which to receive events.
:type source: ~uamqp.address.Source
:param event_position: The position from which to start receiving.
:type event_position: ~azure.eventhub.common.EventPosition
:param prefetch: The number of events to prefetch from the service
for processing. Default is 300.
:type prefetch: int
:param owner_level: The priority of the exclusive consumer. An exclusive
consumer will be created if owner_level is set.
:type owner_level: int
:param loop: An event loop.
"""
event_position = kwargs.get("event_position", None)
prefetch = kwargs.get("prefetch", 300)
owner_level = kwargs.get("owner_level", None)
keep_alive = kwargs.get("keep_alive", None)
auto_reconnect = kwargs.get("auto_reconnect", True)
loop = kwargs.get("loop", None)
super(EventHubConsumer, self).__init__()
self._loop = loop or asyncio.get_event_loop()
self._running = False
self._client = client
self._source = source
self._offset = event_position
self._messages_iter = None
self._prefetch = prefetch
self._owner_level = owner_level
self._keep_alive = keep_alive
self._auto_reconnect = auto_reconnect
self._retry_policy = errors.ErrorPolicy(
max_retries=self._client._config.max_retries,
on_error=_error_handler) # pylint:disable=protected-access
self._reconnect_backoff = 1
self._redirected = None
self._error = None
self._link_properties = {}
partition = self._source.split('/')[-1]
self._partition = partition
self._name = "EHReceiver-{}-partition{}".format(
uuid.uuid4(), partition)
if owner_level:
self._link_properties[types.AMQPSymbol(
self._epoch_symbol)] = types.AMQPLong(int(owner_level))
link_property_timeout_ms = (self._client._config.receive_timeout
or self._timeout) * 1000 # pylint:disable=protected-access
self._link_properties[types.AMQPSymbol(
self._timeout_symbol)] = types.AMQPLong(
int(link_property_timeout_ms))
self._handler = None
def __init__(self, client, source, **kwargs):
# type: (EventHubConsumerClient, str, Any) -> None
event_position = kwargs.get("event_position", None)
prefetch = kwargs.get("prefetch", 300)
owner_level = kwargs.get("owner_level", None)
keep_alive = kwargs.get("keep_alive", None)
auto_reconnect = kwargs.get("auto_reconnect", True)
track_last_enqueued_event_properties = kwargs.get(
"track_last_enqueued_event_properties", False
)
idle_timeout = kwargs.get("idle_timeout", None)
self.running = False
self.closed = False
self.stop = False # used by event processor
self.handler_ready = False
self._on_event_received = kwargs[
"on_event_received"
] # type: Callable[[EventData], None]
self._client = client
self._source = source
self._offset = event_position
self._offset_inclusive = kwargs.get("event_position_inclusive", False)
self._prefetch = prefetch
self._owner_level = owner_level
self._keep_alive = keep_alive
self._auto_reconnect = auto_reconnect
self._retry_policy = errors.ErrorPolicy(
max_retries=self._client._config.max_retries, on_error=_error_handler # pylint:disable=protected-access
)
self._reconnect_backoff = 1
self._link_properties = {} # type: Dict[types.AMQPType, types.AMQPType]
self._error = None
self._timeout = 0
self._idle_timeout = (idle_timeout * 1000) if idle_timeout else None
partition = self._source.split("/")[-1]
self._partition = partition
self._name = "EHConsumer-{}-partition{}".format(uuid.uuid4(), partition)
if owner_level is not None:
self._link_properties[types.AMQPSymbol(EPOCH_SYMBOL)] = types.AMQPLong(
int(owner_level)
)
link_property_timeout_ms = (
self._client._config.receive_timeout or self._timeout # pylint:disable=protected-access
) * 1000
self._link_properties[types.AMQPSymbol(TIMEOUT_SYMBOL)] = types.AMQPLong(
int(link_property_timeout_ms)
)
self._handler = None # type: Optional[ReceiveClient]
self._track_last_enqueued_event_properties = (
track_last_enqueued_event_properties
)
self._message_buffer = deque() # type: ignore
self._last_received_event = None # type: Optional[EventData]
def __init__(self, client: "EventHubConsumerClient", source: str,
**kwargs) -> None:
super().__init__()
event_position = kwargs.get("event_position", None)
prefetch = kwargs.get("prefetch", 300)
owner_level = kwargs.get("owner_level", None)
keep_alive = kwargs.get("keep_alive", None)
auto_reconnect = kwargs.get("auto_reconnect", True)
track_last_enqueued_event_properties = kwargs.get(
"track_last_enqueued_event_properties", False)
idle_timeout = kwargs.get("idle_timeout", None)
loop = kwargs.get("loop", None)
self.running = False
self.closed = False
self._on_event_received = kwargs[
"on_event_received"] # type: Callable[[EventData], Awaitable[None]]
self._loop = loop or get_running_loop()
self._client = client
self._source = source
self._offset = event_position
self._offset_inclusive = kwargs.get("event_position_inclusive", False)
self._prefetch = prefetch
self._owner_level = owner_level
self._keep_alive = keep_alive
self._auto_reconnect = auto_reconnect
self._retry_policy = errors.ErrorPolicy(
max_retries=self._client._config.max_retries,
on_error=_error_handler # pylint:disable=protected-access
)
self._reconnect_backoff = 1
self._timeout = 0
self._idle_timeout = (idle_timeout * 1000) if idle_timeout else None
self._link_properties = {
} # type: Dict[types.AMQPType, types.AMQPType]
partition = self._source.split("/")[-1]
self._partition = partition
self._name = "EHReceiver-{}-partition{}".format(
uuid.uuid4(), partition)
if owner_level:
self._link_properties[types.AMQPSymbol(
EPOCH_SYMBOL)] = types.AMQPLong(int(owner_level))
link_property_timeout_ms = (
self._client._config.receive_timeout or self._timeout # pylint:disable=protected-access
) * 1000
self._link_properties[types.AMQPSymbol(
TIMEOUT_SYMBOL)] = types.AMQPLong(int(link_property_timeout_ms))
self._handler = None # type: Optional[ReceiveClientAsync]
self._track_last_enqueued_event_properties = (
track_last_enqueued_event_properties)
self._event_queue = queue.Queue()
self._last_received_event = None # type: Optional[EventData]
def __init__(self, client, source, **kwargs):
"""
Instantiate a consumer. EventHubConsumer should be instantiated by calling the `create_consumer` method
in EventHubClient.
:param client: The parent EventHubClient.
:type client: ~azure.eventhub.client.EventHubClient
:param source: The source EventHub from which to receive events.
:type source: str
:param prefetch: The number of events to prefetch from the service
for processing. Default is 300.
:type prefetch: int
:param owner_level: The priority of the exclusive consumer. It will an exclusive
consumer if owner_level is set.
:type owner_level: int
"""
event_position = kwargs.get("event_position", None)
prefetch = kwargs.get("prefetch", 300)
owner_level = kwargs.get("owner_level", None)
keep_alive = kwargs.get("keep_alive", None)
auto_reconnect = kwargs.get("auto_reconnect", True)
super(EventHubConsumer, self).__init__()
self.running = False
self.client = client
self.source = source
self.offset = event_position
self.messages_iter = None
self.prefetch = prefetch
self.owner_level = owner_level
self.keep_alive = keep_alive
self.auto_reconnect = auto_reconnect
self.retry_policy = errors.ErrorPolicy(
max_retries=self.client.config.max_retries,
on_error=_error_handler)
self.reconnect_backoff = 1
self._link_properties = {}
self.redirected = None
self.error = None
partition = self.source.split('/')[-1]
self.partition = partition
self.name = "EHConsumer-{}-partition{}".format(uuid.uuid4(), partition)
if owner_level:
self._link_properties[types.AMQPSymbol(
self._epoch)] = types.AMQPLong(int(owner_level))
link_property_timeout_ms = (self.client.config.receive_timeout
or self.timeout) * 1000
self._link_properties[types.AMQPSymbol(
self._timeout)] = types.AMQPLong(int(link_property_timeout_ms))
self._handler = None
def __init__(self, client, source, prefetch=300, epoch=None, loop=None): # pylint: disable=super-init-not-called
"""
Instantiate an async receiver.
:param client: The parent EventHubClient.
:type client: ~azure.eventhub.EventHubClient
:param source: The source EventHub from which to receive events.
:type source: ~uamqp.Source
:param prefetch: The number of events to prefetch from the service
for processing. Default is 300.
:type prefetch: int
:param epoch: An optional epoch value.
:type epoch: int
:param loop: An event loop.
"""
self.loop = loop or asyncio.get_event_loop()
self.offset = None
self._callback = None
self.prefetch = prefetch
self.epoch = epoch
properties = None
if epoch:
properties = {
types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))
}
self._handler = ReceiveClientAsync(source,
auth=client.auth,
debug=client.debug,
prefetch=self.prefetch,
link_properties=properties,
timeout=self.timeout,
loop=self.loop)
def _create_handler(self):
source = Source(self._source)
if self._offset is not None:
source.set_filter(self._offset._selector()) # pylint:disable=protected-access
if StrictVersion(uamqp.__version__) < StrictVersion("1.2.3"): # backward compatible until uamqp 1.2.3 released
desired_capabilities = {}
elif self._track_last_enqueued_event_properties:
symbol_array = [types.AMQPSymbol(self._receiver_runtime_metric_symbol)]
desired_capabilities = {"desired_capabilities": utils.data_factory(types.AMQPArray(symbol_array))}
else:
desired_capabilities = {"desired_capabilities": None}
self._handler = ReceiveClientAsync(
source,
auth=self._client._create_auth(), # pylint:disable=protected-access
debug=self._client._config.network_tracing, # pylint:disable=protected-access
prefetch=self._prefetch,
link_properties=self._link_properties,
timeout=self._timeout,
error_policy=self._retry_policy,
keep_alive_interval=self._keep_alive,
client_name=self._name,
receive_settle_mode=uamqp.constants.ReceiverSettleMode.ReceiveAndDelete,
auto_complete=False,
properties=self._client._create_properties( # pylint:disable=protected-access
self._client._config.user_agent), # pylint:disable=protected-access
**desired_capabilities, # pylint:disable=protected-access
loop=self._loop)
self._messages_iter = None
def test_event_hubs_client_receive_with_runtime_metric_sync(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAuth.from_shared_access_key(
uri, live_eventhub_config['key_name'], live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'],
live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
receiver_runtime_metric_symbol = b'com.microsoft:enable-receiver-runtime-metric'
symbol_array = [types.AMQPSymbol(receiver_runtime_metric_symbol)]
desired_capabilities = utils.data_factory(types.AMQPArray(symbol_array))
with uamqp.ReceiveClient(source, auth=sas_auth, debug=False, timeout=50, prefetch=50,
desired_capabilities=desired_capabilities) as receive_client:
log.info("Created client, receiving...")
with pytest.raises(ValueError):
batch = receive_client.receive_message_batch(max_batch_size=100)
batch = receive_client.receive_message_batch(max_batch_size=10)
log.info("Got batch: {}".format(len(batch)))
assert len(batch) <= 10
for message in batch:
annotations = message.annotations
delivery_annotations = message.delivery_annotations
log.info("Sequence Number: {}".format(annotations.get(b'x-opt-sequence-number')))
assert b'last_enqueued_sequence_number' in delivery_annotations
assert b'last_enqueued_offset' in delivery_annotations
assert b'last_enqueued_time_utc' in delivery_annotations
assert b'runtime_info_retrieval_time_utc' in delivery_annotations
log.info("Finished receiving")
def scheduled_enqueue_time_utc(self, value):
# type: (datetime.datetime) -> None
if not self.properties.message_id:
self.properties.message_id = str(uuid.uuid4())
if not self.message.annotations:
self.message.annotations = {}
self.message.annotations[types.AMQPSymbol(_X_OPT_SCHEDULED_ENQUEUE_TIME)] = value
def __init__(self, body=None, batch=None, to_device=None, message=None):
"""
Initialize EventData.
:param body: The data to send in a single message.
:type body: str, bytes or list
:param batch: A data generator to send batched messages.
:type batch: Generator
:param message: The received message.
:type message: ~uamqp.message.Message
"""
self._partition_key = types.AMQPSymbol(EventData.PROP_PARTITION_KEY)
self._annotations = {}
self._app_properties = {}
self.msg_properties = MessageProperties()
if to_device:
self.msg_properties.to = '/devices/{}/messages/devicebound'.format(to_device)
if batch:
self.message = BatchMessage(data=batch, multi_messages=True, properties=self.msg_properties)
elif message:
self.message = message
self.msg_properties = message.properties
self._annotations = message.annotations
self._app_properties = message.application_properties
else:
if isinstance(body, list) and body:
self.message = Message(body[0], properties=self.msg_properties)
for more in body[1:]:
self.message._body.append(more) # pylint: disable=protected-access
elif body is None:
raise ValueError("EventData cannot be None.")
else:
self.message = Message(body, properties=self.msg_properties)
def __init__(self, body=None, batch=None, message=None):
"""
Initialize EventData
:param body: The data to send in a single message.
:type body: str, bytes or list
:param batch: A data generator to send batched messages.
:type batch: Generator
:param message: The received message.
:type message: ~uamqp.Message
"""
self._partition_key = types.AMQPSymbol(EventData.PROP_PARTITION_KEY)
self._annotations = {}
self._properties = {}
self._header = MessageHeader()
self._header.durable = True
if batch:
self.message = BatchMessage(data=batch, multi_messages=True)
elif message:
self.message = message
self._annotations = message.annotations
self._properties = message.application_properties
self._header = message.header
else:
if isinstance(body, list) and body:
self.message = Message(body[0])
for more in body[1:]:
self.message._body.append(more) # pylint: disable=protected-access
elif body is None:
raise ValueError("EventData cannot be None.")
else:
self.message = Message(body)
def __init__(self,
client,
source,
event_position=None,
prefetch=300,
owner_level=None,
keep_alive=None,
auto_reconnect=True):
"""
Instantiate a consumer. EventHubConsumer should be instantiated by calling the `create_consumer` method
in EventHubClient.
:param client: The parent EventHubClient.
:type client: ~azure.eventhub.client.EventHubClient
:param source: The source EventHub from which to receive events.
:type source: str
:param prefetch: The number of events to prefetch from the service
for processing. Default is 300.
:type prefetch: int
:param owner_level: The priority of the exclusive consumer. It will an exclusive
consumer if owner_level is set.
:type owner_level: int
"""
self.running = False
self.client = client
self.source = source
self.offset = event_position
self.messages_iter = None
self.prefetch = prefetch
self.owner_level = owner_level
self.keep_alive = keep_alive
self.auto_reconnect = auto_reconnect
self.retry_policy = errors.ErrorPolicy(
max_retries=self.client.config.max_retries,
on_error=_error_handler)
self.reconnect_backoff = 1
self.properties = None
self.redirected = None
self.error = None
partition = self.source.split('/')[-1]
self.name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition)
source = Source(self.source)
if self.offset is not None:
source.set_filter(self.offset._selector()) # pylint: disable=protected-access
if owner_level:
self.properties = {
types.AMQPSymbol(self._epoch): types.AMQPLong(int(owner_level))
}
self._handler = ReceiveClient(
source,
auth=self.client.get_auth(),
debug=self.client.config.network_tracing,
prefetch=self.prefetch,
link_properties=self.properties,
timeout=self.timeout,
error_policy=self.retry_policy,
keep_alive_interval=self.keep_alive,
client_name=self.name,
properties=self.client._create_properties(
self.client.config.user_agent)) # pylint: disable=protected-access
async def test_event_hubs_receive_with_runtime_metric_async(
live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'],
live_eventhub_config['consumer_group'],
live_eventhub_config['partition'])
receiver_runtime_metric_symbol = b'com.microsoft:enable-receiver-runtime-metric'
symbol_array = [types.AMQPSymbol(receiver_runtime_metric_symbol)]
desired_capabilities = utils.data_factory(types.AMQPArray(symbol_array))
async with uamqp.ReceiveClientAsync(
source,
debug=False,
auth=sas_auth,
timeout=1000,
prefetch=10,
desired_capabilities=desired_capabilities) as receive_client:
message_batch = await receive_client.receive_message_batch_async(10)
log.info("got batch: {}".format(len(message_batch)))
for message in message_batch:
annotations = message.annotations
delivery_annotations = message.delivery_annotations
log.info("Sequence Number: {}".format(
annotations.get(b'x-opt-sequence-number')))
assert b'last_enqueued_sequence_number' in delivery_annotations
assert b'last_enqueued_offset' in delivery_annotations
assert b'last_enqueued_time_utc' in delivery_annotations
assert b'runtime_info_retrieval_time_utc' in delivery_annotations
def __init__(self, body=None, to_device=None):
"""
Initialize EventData.
:param body: The data to send in a single message.
:type body: str, bytes or list
:param to_device: An IoT device to route to.
:type to_device: str
"""
self._partition_key = types.AMQPSymbol(EventData.PROP_PARTITION_KEY)
self._annotations = {}
self._app_properties = {}
self.msg_properties = MessageProperties()
if to_device:
self.msg_properties.to = '/devices/{}/messages/devicebound'.format(
to_device)
if body and isinstance(body, list):
self.message = Message(body[0], properties=self.msg_properties)
for more in body[1:]:
self.message._body.append(more) # pylint: disable=protected-access
elif body is None:
raise ValueError("EventData cannot be None.")
else:
self.message = Message(body, properties=self.msg_properties)
def __init__(self, client, source, prefetch=300, epoch=None):
"""
Instantiate a receiver.
:param client: The parent EventHubClient.
:type client: ~azure.eventhub.client.EventHubClient
:param source: The source EventHub from which to receive events.
:type source: ~uamqp.address.Source
:param prefetch: The number of events to prefetch from the service
for processing. Default is 300.
:type prefetch: int
:param epoch: An optional epoch value.
:type epoch: int
"""
self.offset = None
self.prefetch = prefetch
self.epoch = epoch
self.properties = None
self.redirected = None
self.debug = client.debug
self.error = None
if epoch:
self.properties = {
types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))
}
self._handler = ReceiveClient(source,
auth=client.auth,
debug=self.debug,
prefetch=self.prefetch,
link_properties=self.properties,
timeout=self.timeout)
def _create_handler(self, auth: "JWTTokenAsync") -> None:
source = Source(self._source)
if self._offset is not None:
source.set_filter(
event_position_selector(self._offset, self._offset_inclusive))
desired_capabilities = None
if self._track_last_enqueued_event_properties:
symbol_array = [types.AMQPSymbol(RECEIVER_RUNTIME_METRIC_SYMBOL)]
desired_capabilities = utils.data_factory(
types.AMQPArray(symbol_array))
properties = create_properties(self._client._config.user_agent # pylint:disable=protected-access
)
self._handler = ReceiveClientAsync(
source,
auth=auth,
debug=self._client._config.network_tracing, # pylint:disable=protected-access
prefetch=self._prefetch,
link_properties=self._link_properties,
timeout=self._timeout,
idle_timeout=self._idle_timeout,
error_policy=self._retry_policy,
keep_alive_interval=self._keep_alive,
client_name=self._name,
receive_settle_mode=uamqp.constants.ReceiverSettleMode.
ReceiveAndDelete,
auto_complete=False,
properties=properties,
desired_capabilities=desired_capabilities,
loop=self._loop,
)
self._handler._streaming_receive = True # pylint:disable=protected-access
self._handler._message_received_callback = ( # pylint:disable=protected-access
self._message_received)
def __init__( # pylint: disable=super-init-not-called
self,
client,
source,
offset=None,
prefetch=300,
epoch=None,
keep_alive=None,
auto_reconnect=True,
loop=None):
"""
Instantiate an async receiver.
:param client: The parent EventHubClientAsync.
:type client: ~azure.eventhub.async_ops.EventHubClientAsync
:param source: The source EventHub from which to receive events.
:type source: ~uamqp.address.Source
:param prefetch: The number of events to prefetch from the service
for processing. Default is 300.
:type prefetch: int
:param epoch: An optional epoch value.
:type epoch: int
:param loop: An event loop.
"""
self.loop = loop or asyncio.get_event_loop()
self.running = False
self.client = client
self.source = source
self.offset = offset
self.prefetch = prefetch
self.epoch = epoch
self.keep_alive = keep_alive
self.auto_reconnect = auto_reconnect
self.retry_policy = errors.ErrorPolicy(max_retries=3,
on_error=_error_handler)
self.reconnect_backoff = 1
self.redirected = None
self.error = None
self.properties = None
partition = self.source.split('/')[-1]
self.name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition)
source = Source(self.source)
if self.offset is not None:
source.set_filter(self.offset.selector())
if epoch:
self.properties = {
types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))
}
self._handler = ReceiveClientAsync(
source,
auth=self.client.get_auth(),
debug=self.client.debug,
prefetch=self.prefetch,
link_properties=self.properties,
timeout=self.timeout,
error_policy=self.retry_policy,
keep_alive_interval=self.keep_alive,
client_name=self.name,
properties=self.client.create_properties(),
loop=self.loop)
async def test_event_hubs_idempotent_producer(live_eventhub_config):
uri = "sb://{}/{}".format(live_eventhub_config['hostname'],
live_eventhub_config['event_hub'])
sas_auth = authentication.SASTokenAsync.from_shared_access_key(
uri, live_eventhub_config['key_name'],
live_eventhub_config['access_key'])
target = "amqps://{}/{}/Partitions/0".format(
live_eventhub_config['hostname'], live_eventhub_config['event_hub'])
symbol_array = [
uamqp_types.AMQPSymbol(b"com.microsoft:idempotent-producer")
]
desired_capabilities = utils.data_factory(
uamqp_types.AMQPArray(symbol_array))
link_properties = {
uamqp_types.AMQPSymbol(b"com.microsoft:timeout"):
uamqp_types.AMQPLong(int(60 * 1000))
}
def on_attach(attach_source, attach_target, properties, error):
if str(attach_target) == target:
on_attach.owner_level = properties.get(
b"com.microsoft:producer-epoch")
on_attach.producer_group_id = properties.get(
b"com.microsoft:producer-id")
on_attach.starting_sequence_number = properties.get(
b"com.microsoft:producer-sequence-number")
send_client = uamqp.SendClientAsync(
target,
auth=sas_auth,
desired_capabilities=desired_capabilities,
link_properties=link_properties,
on_attach=on_attach,
debug=True)
await send_client.open_async()
while not await send_client.client_ready_async():
await asyncio.sleep(0.05)
assert on_attach.owner_level is not None
assert on_attach.producer_group_id is not None
assert on_attach.starting_sequence_number is not None
await send_client.close_async()
def __init__( # pylint: disable=super-init-not-called
self, client, target, **kwargs):
"""
Instantiate an async EventHubProducer. EventHubProducer should be instantiated by calling the `create_producer`
method in EventHubClient.
:param client: The parent EventHubClientAsync.
:type client: ~azure.eventhub.aio.EventHubClientAsync
:param target: The URI of the EventHub to send to.
:type target: str
:param partition: The specific partition ID to send to. Default is `None`, in which case the service
will assign to all partitions using round-robin.
:type partition: str
:param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is
queued. Default value is 60 seconds. If set to 0, there will be no timeout.
:type send_timeout: float
:param keep_alive: The time interval in seconds between pinging the connection to keep it alive during
periods of inactivity. The default value is `None`, i.e. no keep alive pings.
:type keep_alive: float
:param auto_reconnect: Whether to automatically reconnect the producer if a retryable error occurs.
Default value is `True`.
:type auto_reconnect: bool
:param loop: An event loop. If not specified the default event loop will be used.
"""
partition = kwargs.get("partition", None)
send_timeout = kwargs.get("send_timeout", 60)
keep_alive = kwargs.get("keep_alive", None)
auto_reconnect = kwargs.get("auto_reconnect", True)
loop = kwargs.get("loop", None)
super(EventHubProducer, self).__init__()
self._loop = loop or asyncio.get_event_loop()
self._max_message_size_on_link = None
self._running = False
self._client = client
self._target = target
self._partition = partition
self._keep_alive = keep_alive
self._auto_reconnect = auto_reconnect
self._timeout = send_timeout
self._retry_policy = errors.ErrorPolicy(
max_retries=self._client._config.max_retries,
on_error=_error_handler) # pylint:disable=protected-access
self._reconnect_backoff = 1
self._name = "EHProducer-{}".format(uuid.uuid4())
self._unsent_events = None
self._redirected = None
self._error = None
if partition:
self._target += "/Partitions/" + partition
self._name += "-partition{}".format(partition)
self._handler = None
self._outcome = None
self._condition = None
self._link_properties = {
types.AMQPSymbol(self._timeout_symbol):
types.AMQPLong(int(self._timeout * 1000))
}
def _set_partition_key(self, value):
if value:
annotations = self.message.annotations
if annotations is None:
annotations = dict()
annotations[types.AMQPSymbol(EventData.PROP_PARTITION_KEY)] = value
header = MessageHeader()
header.durable = True
self.message.annotations = annotations
self.message.header = header
def schedule(self, schedule_time):
"""Add a specific enqueue time to the message.
:param schedule_time: The scheduled time to enqueue the message.
:type schedule_time: ~datetime.datetime
"""
if not self.properties.message_id:
self.properties.message_id = str(uuid.uuid4())
if not self.message.annotations:
self.message.annotations = {}
self.message.annotations[types.AMQPSymbol(self._x_OPT_SCHEDULED_ENQUEUE_TIME)] = schedule_time
def schedule(self, schedule_time_utc):
# type: (datetime.datetime) -> None
"""Add a specific utc enqueue time to the message.
:param schedule_time_utc: The scheduled utc time to enqueue the message.
:type schedule_time_utc: ~datetime.datetime
:rtype: None
"""
if not self.properties.message_id:
self.properties.message_id = str(uuid.uuid4())
if not self.message.annotations:
self.message.annotations = {}
self.message.annotations[types.AMQPSymbol(
_X_OPT_SCHEDULED_ENQUEUE_TIME)] = schedule_time_utc
def __init__(self, client: "EventHubProducerClient", target: str,
**kwargs) -> None:
super().__init__()
partition = kwargs.get("partition", None)
send_timeout = kwargs.get("send_timeout", 60)
keep_alive = kwargs.get("keep_alive", None)
auto_reconnect = kwargs.get("auto_reconnect", True)
loop = kwargs.get("loop", None)
idle_timeout = kwargs.get("idle_timeout", None)
self.running = False
self.closed = False
self._loop = loop or get_running_loop()
self._max_message_size_on_link = None
self._client = client
self._target = target
self._partition = partition
self._keep_alive = keep_alive
self._auto_reconnect = auto_reconnect
self._timeout = send_timeout
self._idle_timeout = (idle_timeout * 1000) if idle_timeout else None
self._retry_policy = errors.ErrorPolicy(
max_retries=self._client._config.max_retries,
on_error=_error_handler # pylint:disable=protected-access
)
self._reconnect_backoff = 1
self._name = "EHProducer-{}".format(uuid.uuid4())
self._unsent_events = [] # type: List[Any]
self._error = None
if partition:
self._target += "/Partitions/" + partition
self._name += "-partition{}".format(partition)
self._handler = None # type: Optional[SendClientAsync]
self._outcome = None # type: Optional[constants.MessageSendResult]
self._condition = None # type: Optional[Exception]
self._lock = asyncio.Lock(loop=self._loop)
self._link_properties = {
types.AMQPSymbol(TIMEOUT_SYMBOL):
types.AMQPLong(int(self._timeout * 1000))
}
DEAD_LETTER_QUEUE_SUFFIX = '/$DeadLetterQueue'
TRANSFER_DEAD_LETTER_QUEUE_SUFFIX = '/$Transfer' + DEAD_LETTER_QUEUE_SUFFIX
class ReceiveMode(Enum):
PeekLock = constants.ReceiverSettleMode.PeekLock
ReceiveAndDelete = constants.ReceiverSettleMode.ReceiveAndDelete
class SessionFilter(Enum):
NextAvailable = 0
class SubQueue(Enum):
DeadLetter = 1
TransferDeadLetter = 2
ANNOTATION_SYMBOL_PARTITION_KEY = types.AMQPSymbol(_X_OPT_PARTITION_KEY)
ANNOTATION_SYMBOL_VIA_PARTITION_KEY = types.AMQPSymbol(_X_OPT_VIA_PARTITION_KEY)
ANNOTATION_SYMBOL_SCHEDULED_ENQUEUE_TIME = types.AMQPSymbol(_X_OPT_SCHEDULED_ENQUEUE_TIME)
ANNOTATION_SYMBOL_KEY_MAP = {
_X_OPT_PARTITION_KEY: ANNOTATION_SYMBOL_PARTITION_KEY,
_X_OPT_VIA_PARTITION_KEY: ANNOTATION_SYMBOL_VIA_PARTITION_KEY,
_X_OPT_SCHEDULED_ENQUEUE_TIME: ANNOTATION_SYMBOL_SCHEDULED_ENQUEUE_TIME
}
NEXT_AVAILABLE = SessionFilter.NextAvailable
def via_partition_key(self, value):
if not self.message.annotations:
self.message.annotations = {}
self.message.annotations[types.AMQPSymbol(self._X_OPT_VIA_PARTITION_KEY)] = value
def enqueue_sequence_number(self, value):
if not self.message.annotations:
self.message.annotations = {}
self.message.annotations[types.AMQPSymbol(self._X_OPT_ENQUEUE_SEQUENCE_NUMBER)] = value
def __init__(self, client, source, **kwargs):
"""
Instantiate a consumer. EventHubConsumer should be instantiated by calling the `create_consumer` method
in EventHubClient.
:param client: The parent EventHubClient.
:type client: ~azure.eventhub.client.EventHubClient
:param source: The source EventHub from which to receive events.
:type source: str
:param prefetch: The number of events to prefetch from the service
for processing. Default is 300.
:type prefetch: int
:param owner_level: The priority of the exclusive consumer. An exclusive
consumer will be created if owner_level is set.
:type owner_level: int
:param track_last_enqueued_event_properties: Indicates whether or not the consumer should request information
on the last enqueued event on its associated partition, and track that information as events are received.
When information about the partition's last enqueued event is being tracked, each event received from the
Event Hubs service will carry metadata about the partition. This results in a small amount of additional
network bandwidth consumption that is generally a favorable trade-off when considered against periodically
making requests for partition properties using the Event Hub client.
It is set to `False` by default.
:type track_last_enqueued_event_properties: bool
"""
event_position = kwargs.get("event_position", None)
prefetch = kwargs.get("prefetch", 300)
owner_level = kwargs.get("owner_level", None)
keep_alive = kwargs.get("keep_alive", None)
auto_reconnect = kwargs.get("auto_reconnect", True)
track_last_enqueued_event_properties = kwargs.get(
"track_last_enqueued_event_properties", False)
idle_timeout = kwargs.get("idle_timeout", None)
self.running = False
self.closed = False
self.stop = False # used by event processor
self.handler_ready = False
self._on_event_received = kwargs.get("on_event_received")
self._client = client
self._source = source
self._offset = event_position
self._offset_inclusive = kwargs.get("event_position_inclusive", False)
self._prefetch = prefetch
self._owner_level = owner_level
self._keep_alive = keep_alive
self._auto_reconnect = auto_reconnect
self._retry_policy = errors.ErrorPolicy(
max_retries=self._client._config.max_retries,
on_error=_error_handler) # pylint:disable=protected-access
self._reconnect_backoff = 1
self._link_properties = {}
self._error = None
self._timeout = 0
self._idle_timeout = (idle_timeout * 1000) if idle_timeout else None
partition = self._source.split('/')[-1]
self._partition = partition
self._name = "EHConsumer-{}-partition{}".format(
uuid.uuid4(), partition)
if owner_level:
self._link_properties[types.AMQPSymbol(
EPOCH_SYMBOL)] = types.AMQPLong(int(owner_level))
link_property_timeout_ms = (self._client._config.receive_timeout
or self._timeout) * 1000 # pylint:disable=protected-access
self._link_properties[types.AMQPSymbol(
TIMEOUT_SYMBOL)] = types.AMQPLong(int(link_property_timeout_ms))
self._handler = None
self._track_last_enqueued_event_properties = track_last_enqueued_event_properties
self._last_received_event = None
