class EventHubProducer(ConsumerProducerMixin): # pylint: disable=too-many-instance-attributes """ A producer responsible for transmitting EventData to a specific Event Hub, grouped together in batches. Depending on the options specified at creation, the producer may be created to allow event data to be automatically routed to an available partition or specific to a partition. Please use the method `create_producer` on `EventHubClient` for creating `EventHubProducer`. """ _timeout_symbol = b'com.microsoft:timeout' def __init__( # pylint: disable=super-init-not-called self, client, target, **kwargs): """ Instantiate an async EventHubProducer. EventHubProducer should be instantiated by calling the `create_producer` method in EventHubClient. :param client: The parent EventHubClientAsync. :type client: ~azure.eventhub.aio.EventHubClientAsync :param target: The URI of the EventHub to send to. :type target: str :param partition: The specific partition ID to send to. Default is `None`, in which case the service will assign to all partitions using round-robin. :type partition: str :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. :type send_timeout: float :param keep_alive: The time interval in seconds between pinging the connection to keep it alive during periods of inactivity. The default value is `None`, i.e. no keep alive pings. :type keep_alive: float :param auto_reconnect: Whether to automatically reconnect the producer if a retryable error occurs. Default value is `True`. :type auto_reconnect: bool :param loop: An event loop. If not specified the default event loop will be used. """ partition = kwargs.get("partition", None) send_timeout = kwargs.get("send_timeout", 60) keep_alive = kwargs.get("keep_alive", None) auto_reconnect = kwargs.get("auto_reconnect", True) loop = kwargs.get("loop", None) super(EventHubProducer, self).__init__() self._loop = loop or asyncio.get_event_loop() self._max_message_size_on_link = None self._client = client self._target = target self._partition = partition self._keep_alive = keep_alive self._auto_reconnect = auto_reconnect self._timeout = send_timeout self._retry_policy = errors.ErrorPolicy(max_retries=self._client._config.max_retries, on_error=_error_handler) # pylint:disable=protected-access self._reconnect_backoff = 1 self._name = "EHProducer-{}".format(uuid.uuid4()) self._unsent_events = None self._error = None if partition: self._target += "/Partitions/" + partition self._name += "-partition{}".format(partition) self._handler = None self._outcome = None self._condition = None self._link_properties = {types.AMQPSymbol(self._timeout_symbol): types.AMQPLong(int(self._timeout * 1000))} def _create_handler(self): self._handler = SendClientAsync( self._target, auth=self._client._create_auth(), # pylint:disable=protected-access debug=self._client._config.network_tracing, # pylint:disable=protected-access msg_timeout=self._timeout, error_policy=self._retry_policy, keep_alive_interval=self._keep_alive, client_name=self._name, link_properties=self._link_properties, properties=self._client._create_properties( # pylint: disable=protected-access self._client._config.user_agent), # pylint:disable=protected-access loop=self._loop) async def _open_with_retry(self): return await self._do_retryable_operation(self._open, operation_need_param=False) async def _send_event_data(self, timeout_time=None, last_exception=None): if self._unsent_events: await self._open() remaining_time = timeout_time - time.time() if remaining_time <= 0.0: if last_exception: error = last_exception else: error = OperationTimeoutError("send operation timed out") log.info("%r send operation timed out. (%r)", self._name, error) raise error self._handler._msg_timeout = remaining_time * 1000 # pylint: disable=protected-access self._handler.queue_message(*self._unsent_events) await self._handler.wait_async() self._unsent_events = self._handler.pending_messages if self._outcome != constants.MessageSendResult.Ok: if self._outcome == constants.MessageSendResult.Timeout: self._condition = OperationTimeoutError("send operation timed out") _error(self._outcome, self._condition) return async def _send_event_data_with_retry(self, timeout=None): return await self._do_retryable_operation(self._send_event_data, timeout=timeout) def _on_outcome(self, outcome, condition): """ Called when the outcome is received for a delivery. :param outcome: The outcome of the message delivery - success or failure. :type outcome: ~uamqp.constants.MessageSendResult :param condition: Detail information of the outcome. """ self._outcome = outcome self._condition = condition async def create_batch(self, max_size=None): # type:(int) -> EventDataBatch """ Create an EventDataBatch object with max size being max_size. The max_size should be no greater than the max allowed message size defined by the service side. :param max_size: The maximum size of bytes data that an EventDataBatch object can hold. :type max_size: int :return: an EventDataBatch instance :rtype: ~azure.eventhub.EventDataBatch """ if not self._max_message_size_on_link: await self._open_with_retry() if max_size and max_size > self._max_message_size_on_link: raise ValueError('Max message size: {} is too large, acceptable max batch size is: {} bytes.' .format(max_size, self._max_message_size_on_link)) return EventDataBatch(max_size=(max_size or self._max_message_size_on_link)) async def send( self, event_data: Union[EventData, EventDataBatch, Iterable[EventData]], *, partition_key: Union[str, bytes] = None, timeout: float = None): """ Sends an event data and blocks until acknowledgement is received or operation times out. :param event_data: The event to be sent. It can be an EventData object, or iterable of EventData objects :type event_data: ~azure.eventhub.common.EventData, Iterator, Generator, list :param partition_key: With the given partition_key, event data will land to a particular partition of the Event Hub decided by the service. partition_key could be omitted if event_data is of type ~azure.eventhub.EventDataBatch. :type partition_key: str :param timeout: The maximum wait time to send the event data. If not specified, the default wait time specified when the producer was created will be used. :type timeout: float :raises: ~azure.eventhub.AuthenticationError, ~azure.eventhub.ConnectError, ~azure.eventhub.ConnectionLostError, ~azure.eventhub.EventDataError, ~azure.eventhub.EventDataSendError, ~azure.eventhub.EventHubError :return: None :rtype: None """ # Tracing code span_impl_type = settings.tracing_implementation() # type: Type[AbstractSpan] child = None if span_impl_type is not None: child = span_impl_type(name="Azure.EventHubs.send") child.kind = SpanKind.CLIENT # Should be PRODUCER self._check_closed() if isinstance(event_data, EventData): if partition_key: event_data._set_partition_key(partition_key) # pylint: disable=protected-access wrapper_event_data = event_data wrapper_event_data._trace_message(child) # pylint: disable=protected-access else: if isinstance(event_data, EventDataBatch): if partition_key and partition_key != event_data._partition_key: # pylint: disable=protected-access raise EventDataError('The partition_key does not match the one of the EventDataBatch') wrapper_event_data = event_data #type: ignore else: if partition_key: event_data = _set_partition_key(event_data, partition_key) event_data = _set_trace_message(event_data) wrapper_event_data = EventDataBatch._from_batch(event_data, partition_key) # pylint: disable=protected-access wrapper_event_data.message.on_send_complete = self._on_outcome self._unsent_events = [wrapper_event_data.message] if span_impl_type is not None: with child: self._client._add_span_request_attributes(child) # pylint: disable=protected-access await self._send_event_data_with_retry(timeout=timeout) # pylint:disable=unexpected-keyword-arg # TODO: to refactor else: await self._send_event_data_with_retry(timeout=timeout) # pylint:disable=unexpected-keyword-arg # TODO: to refactor async def close(self): # type: () -> None """ Close down the handler. If the handler has already closed, this will be a no op. """ await super(EventHubProducer, self).close()
class Sender(BaseHandler, mixins.SenderMixin): """This handler is for sending messages to a Service Bus entity. It operates a single connection that must be opened and closed on completion. The Sender can be run within a context manager to ensure that the connection is closed on exit. The Sender should not be instantiated directly, and should be accessed from a `QueueClient` or `TopicClient` using the `get_sender()` method. .. note:: This object is not thread-safe. :param handler_id: The ID used as the connection name for the Sender. :type handler_id: str :param target: The endpoint to send messages to. :type target: ~uamqp.Target :param auth_config: The SASL auth credentials. :type auth_config: dict[str, str] :param session: An optional session ID. If supplied, all outgoing messages will have this session ID added (unless they already have one specified). :type session: str :param loop: An async event loop :type loop: ~asyncio.EventLoop :param connection: A shared connection [not yet supported]. :type connection: ~uamqp.Connection :param encoding: The encoding used for string properties. Default is 'UTF-8'. :type encoding: str :param debug: Whether to enable network trace debug logs. :type debug: bool """ def __init__( self, handler_id, target, auth_config, *, session=None, loop=None, connection=None, encoding='UTF-8', debug=False, **kwargs): self.name = "SBSender-{}".format(handler_id) self.session_id = session super(Sender, self).__init__( target, auth_config, loop=loop, connection=connection, encoding=encoding, debug=debug, **kwargs) def _build_handler(self): auth = None if self.connection else authentication.SASTokenAsync.from_shared_access_key(**self.auth_config) self._handler = SendClientAsync( self.endpoint, auth=auth, debug=self.debug, properties=self.properties, error_policy=self.error_policy, client_name=self.name, encoding=self.encoding, loop=self.loop, **self.handler_kwargs) async def send(self, message): """Send a message and blocks until acknowledgement is received or the operation fails. :param message: The message to be sent. :type message: ~azure.servicebus.aio.async_message.Message :raises: ~azure.servicebus.common.errors.MessageSendFailed if the message fails to send. .. admonition:: Example: .. literalinclude:: ../samples/async_samples/test_examples_async.py :start-after: [START open_close_sender_context] :end-before: [END open_close_sender_context] :language: python :dedent: 4 :caption: Open a Sender and send messages. """ if not isinstance(message, Message): raise TypeError("Value of message must be of type 'Message'.") if not self.running: await self.open() if self.session_id and not message.properties.group_id: message.properties.group_id = self.session_id try: await self._handler.send_message_async(message.message) except Exception as e: # pylint: disable=broad-except raise MessageSendFailed(e) async def schedule(self, schedule_time, *messages): """Send one or more messages to be enqueued at a specific time. Returns a list of the sequence numbers of the enqueued messages. :param schedule_time: The date and time to enqueue the messages. :type schedule_time: ~datetime.datetime :param messages: The messages to schedule. :type messages: ~azure.servicebus.aio.async_message.Message :rtype: list[int] .. admonition:: Example: .. literalinclude:: ../samples/async_samples/test_examples_async.py :start-after: [START schedule_messages] :end-before: [END schedule_messages] :language: python :dedent: 4 :caption: Schedule messages. """ if not self.running: await self.open() request_body = self._build_schedule_request(schedule_time, *messages) return await self._mgmt_request_response( REQUEST_RESPONSE_SCHEDULE_MESSAGE_OPERATION, request_body, mgmt_handlers.schedule_op) async def cancel_scheduled_messages(self, *sequence_numbers): """Cancel one or more messages that have previsouly been scheduled and are still pending. :param sequence_numbers: The seqeuence numbers of the scheduled messages. :type sequence_numbers: int .. admonition:: Example: .. literalinclude:: ../samples/async_samples/test_examples_async.py :start-after: [START cancel_schedule_messages] :end-before: [END cancel_schedule_messages] :language: python :dedent: 4 :caption: Schedule messages. """ if not self.running: await self.open() numbers = [types.AMQPLong(s) for s in sequence_numbers] request_body = {'sequence-numbers': types.AMQPArray(numbers)} return await self._mgmt_request_response( REQUEST_RESPONSE_CANCEL_SCHEDULED_MESSAGE_OPERATION, request_body, mgmt_handlers.default) async def send_pending_messages(self): """Wait until all pending messages have been sent. :returns: A list of the send results of all the pending messages. Each send result is a tuple with two values. The first is a boolean, indicating `True` if the message sent, or `False` if it failed. The second is an error if the message failed, otherwise it will be `None`. :rtype: list[tuple[bool, ~azure.servicebus.common.errors.MessageSendFailed]] .. admonition:: Example: .. literalinclude:: ../samples/async_samples/test_examples_async.py :start-after: [START queue_sender_messages] :end-before: [END queue_sender_messages] :language: python :dedent: 4 :caption: Schedule messages. """ if not self.running: await self.open() try: pending = self._handler._pending_messages[:] # pylint: disable=protected-access await self._handler.wait_async() results = [] for m in pending: if m.state == constants.MessageState.SendFailed: results.append((False, MessageSendFailed(m._response))) # pylint: disable=protected-access else: results.append((True, None)) return results except Exception as e: # pylint: disable=broad-except raise MessageSendFailed(e) async def reconnect(self): """Reconnect the handler. If the handler was disconnected from the service with a retryable error - attempt to reconnect. This method will be called automatically for most retryable errors. Also attempts to re-queue any messages that were pending before the reconnect. """ unsent_events = self._handler.pending_messages await super(Sender, self).reconnect() try: self._handler.queue_message(*unsent_events) await self._handler.wait_async() except Exception as e: # pylint: disable=broad-except await self._handle_exception(e)
class EventHubProducer(ConsumerProducerMixin): # pylint: disable=too-many-instance-attributes """A producer responsible for transmitting batches of EventData to a specific Event Hub. Depending on the options specified at creation, the producer may be created to allow event data to be automatically routed to an available partition or specific to a partition. Please use the method `_create_producer` on `EventHubClient` for creating `EventHubProducer`. :param client: The parent EventHubProducerClient. :type client: ~azure.eventhub.aio.EventHubProducerClient :param target: The URI of the EventHub to send to. :type target: str :keyword str partition: The specific partition ID to send to. Default is `None`, in which case the service will assign to all partitions using round-robin. :keyword float send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. :keyword int keep_alive: The time interval in seconds between pinging the connection to keep it alive during periods of inactivity. The default value is `None`, i.e. no keep alive pings. :keyword bool auto_reconnect: Whether to automatically reconnect the producer if a retryable error occurs. Default value is `True`. :keyword ~asyncio.AbstractEventLoop loop: An event loop. If not specified the default event loop will be used. """ def __init__(self, client: "EventHubProducerClient", target: str, **kwargs) -> None: super().__init__() partition = kwargs.get("partition", None) send_timeout = kwargs.get("send_timeout", 60) keep_alive = kwargs.get("keep_alive", None) auto_reconnect = kwargs.get("auto_reconnect", True) idle_timeout = kwargs.get("idle_timeout", None) self.running = False self.closed = False self._loop = kwargs.get("loop", None) self._max_message_size_on_link = None self._client = client self._target = target self._partition = partition self._keep_alive = keep_alive self._auto_reconnect = auto_reconnect self._timeout = send_timeout self._idle_timeout = (idle_timeout * 1000) if idle_timeout else None self._retry_policy = errors.ErrorPolicy( max_retries=self._client._config.max_retries, on_error=_error_handler # pylint:disable=protected-access ) self._reconnect_backoff = 1 self._name = "EHProducer-{}".format(uuid.uuid4()) self._unsent_events = [] # type: List[Any] self._error = None if partition: self._target += "/Partitions/" + partition self._name += "-partition{}".format(partition) self._handler = None # type: Optional[SendClientAsync] self._outcome = None # type: Optional[constants.MessageSendResult] self._condition = None # type: Optional[Exception] self._lock = asyncio.Lock(loop=self._loop) self._link_properties = { types.AMQPSymbol(TIMEOUT_SYMBOL): types.AMQPLong(int(self._timeout * 1000)) } def _create_handler(self, auth: "JWTTokenAsync") -> None: self._handler = SendClientAsync( self._target, auth=auth, debug=self._client._config.network_tracing, # pylint:disable=protected-access msg_timeout=self._timeout * 1000, idle_timeout=self._idle_timeout, error_policy=self._retry_policy, keep_alive_interval=self._keep_alive, client_name=self._name, link_properties=self._link_properties, properties=create_properties(self._client._config.user_agent # pylint:disable=protected-access ), loop=self._loop, ) async def _open_with_retry(self) -> Any: return await self._do_retryable_operation(self._open, operation_need_param=False) def _set_msg_timeout(self, timeout_time: Optional[float], last_exception: Optional[Exception]) -> None: if not timeout_time: return remaining_time = timeout_time - time.time() if remaining_time <= 0.0: if last_exception: error = last_exception else: error = OperationTimeoutError("Send operation timed out") _LOGGER.info("%r send operation timed out. (%r)", self._name, error) raise error self._handler._msg_timeout = remaining_time * 1000 # type: ignore # pylint: disable=protected-access async def _send_event_data( self, timeout_time: Optional[float] = None, last_exception: Optional[Exception] = None, ) -> None: # TODO: Correct uAMQP type hints if self._unsent_events: await self._open() self._set_msg_timeout(timeout_time, last_exception) self._handler.queue_message(*self._unsent_events) # type: ignore await self._handler.wait_async() # type: ignore self._unsent_events = self._handler.pending_messages # type: ignore if self._outcome != constants.MessageSendResult.Ok: if self._outcome == constants.MessageSendResult.Timeout: self._condition = OperationTimeoutError( "Send operation timed out") if self._condition: raise self._condition async def _send_event_data_with_retry(self, timeout: Optional[float] = None ) -> None: await self._do_retryable_operation(self._send_event_data, timeout=timeout) def _on_outcome(self, outcome: constants.MessageSendResult, condition: Optional[Exception]) -> None: """ Called when the outcome is received for a delivery. :param outcome: The outcome of the message delivery - success or failure. :type outcome: ~uamqp.constants.MessageSendResult :param condition: Detail information of the outcome. """ self._outcome = outcome self._condition = condition def _wrap_eventdata( self, event_data: Union[EventData, EventDataBatch, Iterable[EventData]], span: Optional[AbstractSpan], partition_key: Optional[AnyStr], ) -> Union[EventData, EventDataBatch]: if isinstance(event_data, EventData): if partition_key: set_message_partition_key(event_data.message, partition_key) wrapper_event_data = event_data trace_message(wrapper_event_data, span) else: if isinstance(event_data, EventDataBatch ): # The partition_key in the param will be omitted. if (partition_key and partition_key != event_data._partition_key # pylint: disable=protected-access ): raise ValueError( "The partition_key does not match the one of the EventDataBatch" ) for event in event_data.message._body_gen: # pylint: disable=protected-access trace_message(event, span) wrapper_event_data = event_data # type:ignore else: if partition_key: event_data = _set_partition_key(event_data, partition_key) event_data = _set_trace_message(event_data, span) wrapper_event_data = EventDataBatch._from_batch( event_data, partition_key) # type: ignore # pylint: disable=protected-access wrapper_event_data.message.on_send_complete = self._on_outcome return wrapper_event_data async def send(self, event_data: Union[EventData, EventDataBatch, Iterable[EventData]], *, partition_key: Optional[AnyStr] = None, timeout: Optional[float] = None) -> None: """ Sends an event data and blocks until acknowledgement is received or operation times out. :param event_data: The event to be sent. It can be an EventData object, or iterable of EventData objects :type event_data: ~azure.eventhub.common.EventData, Iterator, Generator, list :param partition_key: With the given partition_key, event data will land to a particular partition of the Event Hub decided by the service. partition_key could be omitted if event_data is of type ~azure.eventhub.EventDataBatch. :type partition_key: str :param timeout: The maximum wait time to send the event data. If not specified, the default wait time specified when the producer was created will be used. :type timeout: float :raises: ~azure.eventhub.exceptions.AuthenticationError, ~azure.eventhub.exceptions.ConnectError, ~azure.eventhub.exceptions.ConnectionLostError, ~azure.eventhub.exceptions.EventDataError, ~azure.eventhub.exceptions.EventDataSendError, ~azure.eventhub.exceptions.EventHubError :return: None :rtype: None """ # Tracing code async with self._lock: with send_context_manager() as child: self._check_closed() wrapper_event_data = self._wrap_eventdata( event_data, child, partition_key) self._unsent_events = [wrapper_event_data.message] if child: self._client._add_span_request_attributes( # pylint: disable=protected-access child) await self._send_event_data_with_retry(timeout=timeout) # pylint:disable=unexpected-keyword-arg async def close(self) -> None: """ Close down the handler. If the handler has already closed, this will be a no op. """ async with self._lock: await super(EventHubProducer, self).close()
class AsyncSender(Sender): """ Implements the async API of a Sender. """ def __init__( # pylint: disable=super-init-not-called self, client, target, partition=None, send_timeout=60, keep_alive=None, auto_reconnect=True, loop=None): """ Instantiate an EventHub event SenderAsync handler. :param client: The parent EventHubClientAsync. :type client: ~azure.eventhub.async_ops.EventHubClientAsync :param target: The URI of the EventHub to send to. :type target: str :param partition: The specific partition ID to send to. Default is `None`, in which case the service will assign to all partitions using round-robin. :type partition: str :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. :type send_timeout: int :param keep_alive: The time interval in seconds between pinging the connection to keep it alive during periods of inactivity. The default value is `None`, i.e. no keep alive pings. :type keep_alive: int :param auto_reconnect: Whether to automatically reconnect the sender if a retryable error occurs. Default value is `True`. :type auto_reconnect: bool :param loop: An event loop. If not specified the default event loop will be used. """ self.loop = loop or asyncio.get_event_loop() self.client = client self.target = target self.partition = partition self.keep_alive = keep_alive self.auto_reconnect = auto_reconnect self.timeout = send_timeout self.retry_policy = errors.ErrorPolicy(max_retries=3, on_error=_error_handler) self.name = "EHSender-{}".format(uuid.uuid4()) self.redirected = None self.error = None if partition: self.target += "/Partitions/" + partition self.name += "-partition{}".format(partition) self._handler = SendClientAsync( self.target, auth=self.client.get_auth(), debug=self.client.debug, msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, properties=self.client.create_properties(), loop=self.loop) self._outcome = None self._condition = None async def open_async(self): """ Open the Sender using the supplied conneciton. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. :param connection: The underlying client shared connection. :type: connection: ~uamqp.async_ops.connection_async.ConnectionAsync """ if self.redirected: self.target = self.redirected.address self._handler = SendClientAsync( self.target, auth=self.client.get_auth(), debug=self.client.debug, msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, properties=self.client.create_properties(), loop=self.loop) await self._handler.open_async() while not await self.has_started(): await self._handler._connection.work_async() # pylint: disable=protected-access async def reconnect_async(self): """If the Receiver was disconnected from the service with a retryable error - attempt to reconnect.""" await self._handler.close_async() unsent_events = self._handler.pending_messages self._handler = SendClientAsync( self.target, auth=self.client.get_auth(), debug=self.client.debug, msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, properties=self.client.create_properties(), loop=self.loop) try: await self._handler.open_async() self._handler.queue_message(*unsent_events) await self._handler.wait_async() except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: log.info("AsyncSender detached. Attempting reconnect.") await self.reconnect_async() else: log.info("AsyncSender reconnect failed. Shutting down.") error = EventHubError(str(shutdown), shutdown) await self.close_async(exception=error) raise error except errors.MessageHandlerError as shutdown: if self.auto_reconnect: log.info("AsyncSender detached. Attempting reconnect.") await self.reconnect_async() else: log.info("AsyncSender reconnect failed. Shutting down.") error = EventHubError(str(shutdown), shutdown) await self.close_async(exception=error) raise error except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Sender reconnect failed: {}".format(e)) await self.close_async(exception=error) raise error async def has_started(self): """ Whether the handler has completed all start up processes such as establishing the connection, session, link and authentication, and is not ready to process messages. :rtype: bool """ # pylint: disable=protected-access timeout = False auth_in_progress = False if self._handler._connection.cbs: timeout, auth_in_progress = await self._handler._auth.handle_token_async( ) if timeout: raise EventHubError("Authorization timeout.") elif auth_in_progress: return False elif not await self._handler._client_ready_async(): return False else: return True async def close_async(self, exception=None): """ Close down the handler. If the handler has already closed, this will be a no op. An optional exception can be passed in to indicate that the handler was shutdown due to error. :param exception: An optional exception if the handler is closing due to an error. :type exception: Exception """ if self.error: return elif isinstance(exception, errors.LinkRedirect): self.redirected = exception elif isinstance(exception, EventHubError): self.error = exception elif isinstance(exception, (errors.LinkDetach, errors.ConnectionClose)): self.error = EventHubError(str(exception), exception) elif exception: self.error = EventHubError(str(exception)) else: self.error = EventHubError("This send handler is now closed.") await self._handler.close_async() async def send(self, event_data): """ Sends an event data and asynchronously waits until acknowledgement is received or operation times out. :param event_data: The event to be sent. :type event_data: ~azure.eventhub.common.EventData :raises: ~azure.eventhub.common.EventHubError if the message fails to send. """ if self.error: raise self.error if event_data.partition_key and self.partition: raise ValueError( "EventData partition key cannot be used with a partition sender." ) event_data.message.on_send_complete = self._on_outcome try: await self._handler.send_message_async(event_data.message) if self._outcome != constants.MessageSendResult.Ok: raise Sender._error(self._outcome, self._condition) except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: log.info("AsyncSender detached. Attempting reconnect.") await self.reconnect_async() else: log.info("AsyncSender detached. Shutting down.") error = EventHubError(str(shutdown), shutdown) await self.close_async(exception=error) raise error except errors.MessageHandlerError as shutdown: if self.auto_reconnect: log.info("AsyncSender detached. Attempting reconnect.") await self.reconnect_async() else: log.info("AsyncSender detached. Shutting down.") error = EventHubError(str(shutdown), shutdown) await self.close_async(exception=error) raise error except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Send failed: {}".format(e)) await self.close_async(exception=error) raise error else: return self._outcome async def wait_async(self): """ Wait until all transferred events have been sent. """ if self.error: raise self.error try: await self._handler.wait_async() except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: log.info("AsyncSender detached. Attempting reconnect.") await self.reconnect_async() else: log.info("AsyncSender detached. Shutting down.") error = EventHubError(str(shutdown), shutdown) await self.close_async(exception=error) raise error except errors.MessageHandlerError as shutdown: if self.auto_reconnect: log.info("AsyncSender detached. Attempting reconnect.") await self.reconnect_async() else: log.info("AsyncSender detached. Shutting down.") error = EventHubError(str(shutdown), shutdown) await self.close_async(exception=error) raise error except Exception as e: log.info("Unexpected error occurred (%r).", e) raise EventHubError("Send failed: {}".format(e))
class EventHubProducer(object): """ A producer responsible for transmitting EventData to a specific Event Hub, grouped together in batches. Depending on the options specified at creation, the producer may be created to allow event data to be automatically routed to an available partition or specific to a partition. """ def __init__( # pylint: disable=super-init-not-called self, client, target, partition=None, send_timeout=60, keep_alive=None, auto_reconnect=True, loop=None): """ Instantiate an async EventHubProducer. EventHubProducer should be instantiated by calling the `create_producer` method in EventHubClient. :param client: The parent EventHubClientAsync. :type client: ~azure.eventhub.aio.EventHubClientAsync :param target: The URI of the EventHub to send to. :type target: str :param partition: The specific partition ID to send to. Default is `None`, in which case the service will assign to all partitions using round-robin. :type partition: str :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. :type send_timeout: float :param keep_alive: The time interval in seconds between pinging the connection to keep it alive during periods of inactivity. The default value is `None`, i.e. no keep alive pings. :type keep_alive: float :param auto_reconnect: Whether to automatically reconnect the producer if a retryable error occurs. Default value is `True`. :type auto_reconnect: bool :param loop: An event loop. If not specified the default event loop will be used. """ self.loop = loop or asyncio.get_event_loop() self.running = False self.client = client self.target = target self.partition = partition self.keep_alive = keep_alive self.auto_reconnect = auto_reconnect self.timeout = send_timeout self.retry_policy = errors.ErrorPolicy( max_retries=self.client.config.max_retries, on_error=_error_handler) self.reconnect_backoff = 1 self.name = "EHProducer-{}".format(uuid.uuid4()) self.unsent_events = None self.redirected = None self.error = None if partition: self.target += "/Partitions/" + partition self.name += "-partition{}".format(partition) self._handler = SendClientAsync( self.target, auth=self.client.get_auth(), debug=self.client.config.network_tracing, msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, properties=self.client._create_properties( self.client.config.user_agent), # pylint: disable=protected-access loop=self.loop) self._outcome = None self._condition = None async def __aenter__(self): return self async def __aexit__(self, exc_type, exc_val, exc_tb): await self.close(exc_val) async def _open(self): """ Open the EventHubProducer using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. """ if self.redirected: self.target = self.redirected.address self._handler = SendClientAsync( self.target, auth=self.client.get_auth(), debug=self.client.config.network_tracing, msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, properties=self.client._create_properties( self.client.config.user_agent), # pylint: disable=protected-access loop=self.loop) if not self.running: await self._connect() self.running = True async def _connect(self): connected = await self._build_connection() if not connected: await asyncio.sleep(self.reconnect_backoff) while not await self._build_connection(is_reconnect=True): await asyncio.sleep(self.reconnect_backoff) async def _build_connection(self, is_reconnect=False): """ :param is_reconnect: True - trying to reconnect after fail to connect or a connection is lost. False - the 1st time to connect :return: True - connected. False - not connected """ # pylint: disable=protected-access if is_reconnect: await self._handler.close_async() self._handler = SendClientAsync( self.target, auth=self.client.get_auth(), debug=self.client.config.network_tracing, msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, properties=self.client._create_properties( self.client.config.user_agent), loop=self.loop) try: await self._handler.open_async() while not await self._handler.client_ready_async(): await asyncio.sleep(0.05) return True except errors.AuthenticationException as shutdown: if is_reconnect: log.info( "EventHubProducer couldn't authenticate. Shutting down. (%r)", shutdown) error = AuthenticationError(str(shutdown), shutdown) await self.close(exception=error) raise error else: log.info( "EventHubProducer couldn't authenticate. Attempting reconnect." ) return False except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry: log.info("EventHubProducer detached. Attempting reconnect.") return False else: log.info("EventHubProducer detached. Shutting down.") error = ConnectError(str(shutdown), shutdown) await self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: if is_reconnect: log.info("EventHubProducer detached. Shutting down.") error = ConnectError(str(shutdown), shutdown) await self.close(exception=error) raise error else: log.info("EventHubProducer detached. Attempting reconnect.") return False except errors.AMQPConnectionError as shutdown: if is_reconnect: log.info( "EventHubProducer connection error (%r). Shutting down.", shutdown) error = AuthenticationError(str(shutdown), shutdown) await self.close(exception=error) raise error else: log.info( "EventHubProducer couldn't authenticate. Attempting reconnect." ) return False except compat.TimeoutException as shutdown: if is_reconnect: log.info( "EventHubProducer authentication timed out. Shutting down." ) error = AuthenticationError(str(shutdown), shutdown) await self.close(exception=error) raise error else: log.info( "EventHubProducer authentication timed out. Attempting reconnect." ) return False except Exception as e: log.info( "Unexpected error occurred when building connection (%r). Shutting down.", e) error = EventHubError( "Unexpected error occurred when building connection", e) await self.close(exception=error) raise error async def _reconnect(self): return await self._build_connection(is_reconnect=True) async def _send_event_data(self): await self._open() max_retries = self.client.config.max_retries connecting_count = 0 while True: connecting_count += 1 try: if self.unsent_events: self._handler.queue_message(*self.unsent_events) await self._handler.wait_async() self.unsent_events = self._handler.pending_messages if self._outcome != constants.MessageSendResult.Ok: EventHubProducer._error(self._outcome, self._condition) return except (errors.MessageAccepted, errors.MessageAlreadySettled, errors.MessageModified, errors.MessageRejected, errors.MessageReleased, errors.MessageContentTooLarge) as msg_error: raise EventDataError(str(msg_error), msg_error) except errors.MessageException as failed: log.error("Send event data error (%r)", failed) error = EventDataSendError(str(failed), failed) await self.close(exception=error) raise error except errors.AuthenticationException as auth_error: if connecting_count < max_retries: log.info( "EventHubProducer disconnected due to token error. Attempting reconnect." ) await self._reconnect() else: log.info( "EventHubProducer authentication failed. Shutting down." ) error = AuthenticationError(str(auth_error), auth_error) await self.close(auth_error) raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry: log.info( "EventHubProducer detached. Attempting reconnect.") await self._reconnect() else: log.info("EventHubProducer detached. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) await self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: if connecting_count < max_retries: log.info( "EventHubProducer detached. Attempting reconnect.") await self._reconnect() else: log.info("EventHubProducer detached. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) await self.close(error) raise error except errors.AMQPConnectionError as shutdown: if connecting_count < max_retries: log.info( "EventHubProducer connection lost. Attempting reconnect." ) await self._reconnect() else: log.info( "EventHubProducer connection lost. Shutting down.") error = ConnectionLostError(str(shutdown), shutdown) await self.close(error) raise error except compat.TimeoutException as shutdown: if connecting_count < max_retries: log.info( "EventHubProducer timed out sending event data. Attempting reconnect." ) await self._reconnect() else: log.info("EventHubProducer timed out. Shutting down.") await self.close(shutdown) raise ConnectionLostError(str(shutdown), shutdown) except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Send failed: {}".format(e), e) await self.close(exception=error) raise error def _check_closed(self): if self.error: raise EventHubError( "This producer has been closed. Please create a new producer to send event data.", self.error) def _on_outcome(self, outcome, condition): """ Called when the outcome is received for a delivery. :param outcome: The outcome of the message delivery - success or failure. :type outcome: ~uamqp.constants.MessageSendResult :param condition: Detail information of the outcome. """ self._outcome = outcome self._condition = condition @staticmethod def _error(outcome, condition): if outcome != constants.MessageSendResult.Ok: raise condition @staticmethod def _set_partition_key(event_datas, partition_key): ed_iter = iter(event_datas) for ed in ed_iter: ed._set_partition_key(partition_key) yield ed async def send(self, event_data, partition_key=None): # type:(Union[EventData, Union[List[EventData], Iterator[EventData], Generator[EventData]]], Union[str, bytes]) -> None """ Sends an event data and blocks until acknowledgement is received or operation times out. :param event_data: The event to be sent. It can be an EventData object, or iterable of EventData objects :type event_data: ~azure.eventhub.common.EventData, Iterator, Generator, list :param partition_key: With the given partition_key, event data will land to a particular partition of the Event Hub decided by the service. :type partition_key: str :raises: ~azure.eventhub.AuthenticationError, ~azure.eventhub.ConnectError, ~azure.eventhub.ConnectionLostError, ~azure.eventhub.EventDataError, ~azure.eventhub.EventDataSendError, ~azure.eventhub.EventHubError :return: None :rtype: None Example: .. literalinclude:: ../examples/test_examples_eventhub.py :start-after: [START eventhub_client_async_send] :end-before: [END eventhub_client_async_send] :language: python :dedent: 4 :caption: Sends an event data and blocks until acknowledgement is received or operation times out. """ self._check_closed() if isinstance(event_data, EventData): if partition_key: event_data._set_partition_key(partition_key) wrapper_event_data = event_data else: event_data_with_pk = self._set_partition_key( event_data, partition_key) wrapper_event_data = _BatchSendEventData( event_data_with_pk, partition_key=partition_key ) if partition_key else _BatchSendEventData(event_data) wrapper_event_data.message.on_send_complete = self._on_outcome self.unsent_events = [wrapper_event_data.message] await self._send_event_data() async def close(self, exception=None): # type: (Exception) -> None """ Close down the handler. If the handler has already closed, this will be a no op. An optional exception can be passed in to indicate that the handler was shutdown due to error. :param exception: An optional exception if the handler is closing due to an error. :type exception: Exception Example: .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py :start-after: [START eventhub_client_async_sender_close] :end-before: [END eventhub_client_async_sender_close] :language: python :dedent: 4 :caption: Close down the handler. """ self.running = False if self.error: return if isinstance(exception, errors.LinkRedirect): self.redirected = exception elif isinstance(exception, EventHubError): self.error = exception elif isinstance(exception, (errors.LinkDetach, errors.ConnectionClose)): self.error = ConnectError(str(exception), exception) elif exception: self.error = EventHubError(str(exception)) else: self.error = EventHubError("This send handler is now closed.") await self._handler.close_async()
class EventHubProducer(ConsumerProducerMixin): """ A producer responsible for transmitting EventData to a specific Event Hub, grouped together in batches. Depending on the options specified at creation, the producer may be created to allow event data to be automatically routed to an available partition or specific to a partition. """ _timeout = b'com.microsoft:timeout' def __init__( # pylint: disable=super-init-not-called self, client, target, **kwargs): """ Instantiate an async EventHubProducer. EventHubProducer should be instantiated by calling the `create_producer` method in EventHubClient. :param client: The parent EventHubClientAsync. :type client: ~azure.eventhub.aio.EventHubClientAsync :param target: The URI of the EventHub to send to. :type target: str :param partition: The specific partition ID to send to. Default is `None`, in which case the service will assign to all partitions using round-robin. :type partition: str :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. :type send_timeout: float :param keep_alive: The time interval in seconds between pinging the connection to keep it alive during periods of inactivity. The default value is `None`, i.e. no keep alive pings. :type keep_alive: float :param auto_reconnect: Whether to automatically reconnect the producer if a retryable error occurs. Default value is `True`. :type auto_reconnect: bool :param loop: An event loop. If not specified the default event loop will be used. """ partition = kwargs.get("partition", None) send_timeout = kwargs.get("send_timeout", 60) keep_alive = kwargs.get("keep_alive", None) auto_reconnect = kwargs.get("auto_reconnect", True) loop = kwargs.get("loop", None) super(EventHubProducer, self).__init__() self.loop = loop or asyncio.get_event_loop() self._max_message_size_on_link = None self.running = False self.client = client self.target = target self.partition = partition self.keep_alive = keep_alive self.auto_reconnect = auto_reconnect self.timeout = send_timeout self.retry_policy = errors.ErrorPolicy( max_retries=self.client.config.max_retries, on_error=_error_handler) self.reconnect_backoff = 1 self.name = "EHProducer-{}".format(uuid.uuid4()) self.unsent_events = None self.redirected = None self.error = None if partition: self.target += "/Partitions/" + partition self.name += "-partition{}".format(partition) self._handler = None self._outcome = None self._condition = None self._link_properties = { types.AMQPSymbol(self._timeout): types.AMQPLong(int(self.timeout * 1000)) } def _create_handler(self): self._handler = SendClientAsync( self.target, auth=self.client.get_auth(), debug=self.client.config.network_tracing, msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, link_properties=self._link_properties, properties=self.client._create_properties( self.client.config.user_agent), # pylint: disable=protected-access loop=self.loop) async def _open(self, timeout_time=None, **kwargs): """ Open the EventHubProducer using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. """ if not self.running and self.redirected: self.client._process_redirect_uri(self.redirected) self.target = self.redirected.address await super(EventHubProducer, self)._open(timeout_time) @_retry_decorator async def _open_with_retry(self, timeout_time=None, **kwargs): return await self._open(timeout_time=timeout_time, **kwargs) async def _send_event_data(self, timeout_time=None, last_exception=None): if self.unsent_events: await self._open(timeout_time) remaining_time = timeout_time - time.time() if remaining_time <= 0.0: if last_exception: error = last_exception else: error = OperationTimeoutError("send operation timed out") log.info("%r send operation timed out. (%r)", self.name, error) raise error self._handler._msg_timeout = remaining_time # pylint: disable=protected-access self._handler.queue_message(*self.unsent_events) await self._handler.wait_async() self.unsent_events = self._handler.pending_messages if self._outcome != constants.MessageSendResult.Ok: if self._outcome == constants.MessageSendResult.Timeout: self._condition = OperationTimeoutError( "send operation timed out") _error(self._outcome, self._condition) return @_retry_decorator async def _send_event_data_with_retry(self, timeout_time=None, last_exception=None): return await self._send_event_data(timeout_time=timeout_time, last_exception=last_exception) def _on_outcome(self, outcome, condition): """ Called when the outcome is received for a delivery. :param outcome: The outcome of the message delivery - success or failure. :type outcome: ~uamqp.constants.MessageSendResult :param condition: Detail information of the outcome. """ self._outcome = outcome self._condition = condition async def create_batch(self, max_size=None, partition_key=None): # type:(int, str) -> EventDataBatch """ Create an EventDataBatch object with max size being max_size. The max_size should be no greater than the max allowed message size defined by the service side. :param max_size: The maximum size of bytes data that an EventDataBatch object can hold. :type max_size: int :param partition_key: With the given partition_key, event data will land to a particular partition of the Event Hub decided by the service. :type partition_key: str :return: an EventDataBatch instance :rtype: ~azure.eventhub.EventDataBatch Example: .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py :start-after: [START eventhub_client_async_create_batch] :end-before: [END eventhub_client_async_create_batch] :language: python :dedent: 4 :caption: Create EventDataBatch object within limited size """ if not self._max_message_size_on_link: await self._open_with_retry(timeout=self.client.config.send_timeout ) if max_size and max_size > self._max_message_size_on_link: raise ValueError( 'Max message size: {} is too large, acceptable max batch size is: {} bytes.' .format(max_size, self._max_message_size_on_link)) return EventDataBatch(max_size=(max_size or self._max_message_size_on_link), partition_key=partition_key) async def send(self, event_data, *, partition_key=None, timeout=None): # type:(Union[EventData, EventDataBatch, Iterable[EventData]], Union[str, bytes], float) -> None """ Sends an event data and blocks until acknowledgement is received or operation times out. :param event_data: The event to be sent. It can be an EventData object, or iterable of EventData objects :type event_data: ~azure.eventhub.common.EventData, Iterator, Generator, list :param partition_key: With the given partition_key, event data will land to a particular partition of the Event Hub decided by the service. partition_key could be omitted if event_data is of type ~azure.eventhub.EventDataBatch. :type partition_key: str :param timeout: The maximum wait time to send the event data. If not specified, the default wait time specified when the producer was created will be used. :type timeout:float :raises: ~azure.eventhub.AuthenticationError, ~azure.eventhub.ConnectError, ~azure.eventhub.ConnectionLostError, ~azure.eventhub.EventDataError, ~azure.eventhub.EventDataSendError, ~azure.eventhub.EventHubError :return: None :rtype: None Example: .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py :start-after: [START eventhub_client_async_send] :end-before: [END eventhub_client_async_send] :language: python :dedent: 4 :caption: Sends an event data and blocks until acknowledgement is received or operation times out. """ self._check_closed() if isinstance(event_data, EventData): if partition_key: event_data._set_partition_key(partition_key) # pylint: disable=protected-access wrapper_event_data = event_data else: if isinstance(event_data, EventDataBatch): if partition_key and not (partition_key == event_data._partition_key): # pylint: disable=protected-access raise EventDataError( 'The partition_key does not match the one of the EventDataBatch' ) wrapper_event_data = event_data else: if partition_key: event_data = _set_partition_key(event_data, partition_key) wrapper_event_data = EventDataBatch._from_batch( event_data, partition_key) # pylint: disable=protected-access wrapper_event_data.message.on_send_complete = self._on_outcome self.unsent_events = [wrapper_event_data.message] await self._send_event_data_with_retry(timeout=timeout) async def close(self, exception=None): # type: (Exception) -> None """ Close down the handler. If the handler has already closed, this will be a no op. An optional exception can be passed in to indicate that the handler was shutdown due to error. :param exception: An optional exception if the handler is closing due to an error. :type exception: Exception Example: .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py :start-after: [START eventhub_client_async_sender_close] :end-before: [END eventhub_client_async_sender_close] :language: python :dedent: 4 :caption: Close down the handler. """ await super(EventHubProducer, self).close(exception)