def _terminate_internal_only(self, outcome): """Terminates the operation. Args: outcome: A base.Outcome describing the outcome of the operation. """ self.outcome = outcome callbacks = list(self._callbacks) self._callbacks = None act = callable_util.with_exceptions_logged( self._action, _constants.INTERNAL_ERROR_LOG_MESSAGE) # TODO(issue 3202): Don't call the local application's callbacks if it has # previously shown a programming defect. if False and outcome.kind is base.Outcome.Kind.LOCAL_FAILURE: self._pool.submit(act, base.Outcome.Kind.LOCAL_FAILURE) else: def call_callbacks_and_act(callbacks, outcome): for callback in callbacks: callback_outcome = callable_util.call_logging_exceptions( callback, _constants.TERMINATION_CALLBACK_EXCEPTION_LOG_MESSAGE, outcome) if callback_outcome.exception is not None: act_outcome_kind = base.Outcome.Kind.LOCAL_FAILURE break else: act_outcome_kind = outcome.kind act(act_outcome_kind) self._pool.submit( callable_util.with_exceptions_logged( call_callbacks_and_act, _constants.INTERNAL_ERROR_LOG_MESSAGE), callbacks, outcome)
def _terminate_internal_only(self, outcome): """Terminates the operation. Args: outcome: A base.Outcome describing the outcome of the operation. """ self.outcome = outcome callbacks = list(self._callbacks) self._callbacks = None act = callable_util.with_exceptions_logged( self._action, _constants.INTERNAL_ERROR_LOG_MESSAGE) if outcome is base.Outcome.LOCAL_FAILURE: self._pool.submit(act, outcome) else: def call_callbacks_and_act(callbacks, outcome): for callback in callbacks: callback_outcome = callable_util.call_logging_exceptions( callback, _constants.TERMINATION_CALLBACK_EXCEPTION_LOG_MESSAGE, outcome) if callback_outcome.exception is not None: outcome = base.Outcome.LOCAL_FAILURE break act(outcome) self._pool.submit( callable_util.with_exceptions_logged( call_callbacks_and_act, _constants.INTERNAL_ERROR_LOG_MESSAGE), callbacks, outcome)
def _terminate(self, outcome): """Terminates the operation. Args: outcome: An interfaces.Outcome describing the outcome of the operation. """ self._expiration_manager.abort() self._outstanding_requirements = None callbacks = list(self._callbacks) self._callbacks = None self._outcome = outcome act = callable_util.with_exceptions_logged( self._action, _constants.INTERNAL_ERROR_LOG_MESSAGE) if self._has_locally_failed: self._utility_pool.submit(act, outcome) else: def call_callbacks_and_act(callbacks, outcome): for callback in callbacks: callback_outcome = callable_util.call_logging_exceptions( callback, _CALLBACK_EXCEPTION_LOG_MESSAGE, outcome) if callback_outcome.exception is not None: outcome = self._local_failure break self._utility_pool.submit(act, outcome) self._work_pool.submit( callable_util.with_exceptions_logged( call_callbacks_and_act, _constants.INTERNAL_ERROR_LOG_MESSAGE), callbacks, outcome)
def _terminate_internal_only(self, outcome): """Terminates the operation. Args: outcome: A base.Outcome describing the outcome of the operation. """ self.outcome = outcome callbacks = list(self._callbacks) self._callbacks = None act = callable_util.with_exceptions_logged( self._action, _constants.INTERNAL_ERROR_LOG_MESSAGE) if outcome is base.Outcome.LOCAL_FAILURE: self._pool.submit(act, outcome) else: def call_callbacks_and_act(callbacks, outcome): for callback in callbacks: callback_outcome = callable_util.call_logging_exceptions( callback, _constants.TERMINATION_CALLBACK_EXCEPTION_LOG_MESSAGE, outcome) if callback_outcome.exception is not None: outcome = base.Outcome.LOCAL_FAILURE break act(outcome) self._pool.submit( callable_util.with_exceptions_logged( call_callbacks_and_act, _constants.INTERNAL_ERROR_LOG_MESSAGE), callbacks, outcome)
def _terminate_internal_only(self, outcome): """Terminates the operation. Args: outcome: A base.Outcome describing the outcome of the operation. """ self.outcome = outcome callbacks = list(self._callbacks) self._callbacks = None act = callable_util.with_exceptions_logged( self._action, _constants.INTERNAL_ERROR_LOG_MESSAGE) # TODO(issue 3202): Don't call the local application's callbacks if it has # previously shown a programming defect. if False and outcome.kind is base.Outcome.Kind.LOCAL_FAILURE: self._pool.submit(act, base.Outcome.Kind.LOCAL_FAILURE) else: def call_callbacks_and_act(callbacks, outcome): for callback in callbacks: callback_outcome = callable_util.call_logging_exceptions( callback, _constants.TERMINATION_CALLBACK_EXCEPTION_LOG_MESSAGE, outcome) if callback_outcome.exception is not None: act_outcome_kind = base.Outcome.Kind.LOCAL_FAILURE break else: act_outcome_kind = outcome.kind act(act_outcome_kind) self._pool.submit( callable_util.with_exceptions_logged( call_callbacks_and_act, _constants.INTERNAL_ERROR_LOG_MESSAGE), callbacks, outcome)
def _terminate(self, outcome): """Terminates the operation. Args: outcome: An interfaces.Outcome describing the outcome of the operation. """ self._expiration_manager.abort() self._outstanding_requirements = None callbacks = list(self._callbacks) self._callbacks = None self._outcome = outcome act = callable_util.with_exceptions_logged( self._action, _constants.INTERNAL_ERROR_LOG_MESSAGE) if self._has_locally_failed: self._utility_pool.submit(act, outcome) else: def call_callbacks_and_act(callbacks, outcome): for callback in callbacks: callback_outcome = callable_util.call_logging_exceptions( callback, _CALLBACK_EXCEPTION_LOG_MESSAGE, outcome) if callback_outcome.exception is not None: outcome = self._local_failure break self._utility_pool.submit(act, outcome) self._work_pool.submit(callable_util.with_exceptions_logged( call_callbacks_and_act, _constants.INTERNAL_ERROR_LOG_MESSAGE), callbacks, outcome)
def _terminate(self, kind): """Terminates the operation. Args: kind: One of packets.Kind.COMPLETION, packets.Kind.CANCELLATION, packets.Kind.EXPIRATION, packets.Kind.RECEPTION_FAILURE, packets.Kind.TRANSMISSION_FAILURE, packets.Kind.SERVICER_FAILURE, or packets.Kind.SERVICED_FAILURE. """ self._expiration_manager.abort() self._outstanding_requirements = None callbacks = list(self._callbacks) self._callbacks = None self._kind = kind outcome = _KINDS_TO_OUTCOMES[kind] act = callable_util.with_exceptions_logged( self._action, _constants.INTERNAL_ERROR_LOG_MESSAGE) if self._has_locally_failed: self._utility_pool.submit(act, outcome) else: def call_callbacks_and_act(callbacks, outcome): for callback in callbacks: callback_outcome = callable_util.call_logging_exceptions( callback, _CALLBACK_EXCEPTION_LOG_MESSAGE, outcome) if callback_outcome.exception is not None: outcome = _KINDS_TO_OUTCOMES[self._local_failure] break self._utility_pool.submit(act, outcome) self._work_pool.submit(callable_util.with_exceptions_logged( call_callbacks_and_act, _constants.INTERNAL_ERROR_LOG_MESSAGE), callbacks, outcome)
def start(self, requirement): if self._pending_ingestion is not None: def initialize(): consumer_creation_outcome = callable_util.call_logging_exceptions( self._consumer_creator.create_consumer, _CREATE_CONSUMER_EXCEPTION_LOG_MESSAGE, requirement) if consumer_creation_outcome.return_value is None: with self._lock: self._abort_and_notify(self._failure_kind) self._processing = False elif consumer_creation_outcome.return_value.remote_error: with self._lock: self._abort_and_notify(packets.Kind.RECEPTION_FAILURE) self._processing = False elif consumer_creation_outcome.return_value.abandoned: with self._lock: if self._pending_ingestion is not None: self._abort_and_notify(self._failure_kind) self._processing = False else: wrapped_ingestion_consumer = _WrappedConsumer( consumer_creation_outcome.return_value.consumer) with self._lock: self._wrapped_ingestion_consumer = wrapped_ingestion_consumer payload, complete, moar = self._next() if not moar: self._processing = False return self._process(wrapped_ingestion_consumer, payload, complete) self._pool.submit( callable_util.with_exceptions_logged( initialize, _constants.INTERNAL_ERROR_LOG_MESSAGE)) self._processing = True
def pool_wrap(behavior, operation_context): """Wraps an operation-related behavior so that it may be called in a pool. Args: behavior: A callable related to carrying out an operation. operation_context: A base_interfaces.OperationContext for the operation. Returns: A callable that when called carries out the behavior of the given callable and handles whatever exceptions it raises appropriately. """ def translation(*args): try: behavior(*args) except ( abandonment.Abandoned, face.CancellationError, face.ExpirationError, face.LocalShutdownError, face.RemoteShutdownError, face.NetworkError, face.RemoteError, ) as e: if operation_context.outcome() is None: operation_context.fail(e) except Exception as e: operation_context.fail(e) return callable_util.with_exceptions_logged(translation, _INTERNAL_ERROR_LOG_MESSAGE)
def pool_wrap(behavior, operation_context): """Wraps an operation-related behavior so that it may be called in a pool. Args: behavior: A callable related to carrying out an operation. operation_context: A base_interfaces.OperationContext for the operation. Returns: A callable that when called carries out the behavior of the given callable and handles whatever exceptions it raises appropriately. """ def translation(*args): try: behavior(*args) except ( abandonment.Abandoned, face.CancellationError, face.ExpirationError, face.LocalShutdownError, face.RemoteShutdownError, face.NetworkError, face.RemoteError, ) as e: if operation_context.outcome() is None: operation_context.fail(e) except Exception as e: operation_context.fail(e) return callable_util.with_exceptions_logged( translation, _INTERNAL_ERROR_LOG_MESSAGE)
def _transmit(self, packet): """Commences the transmission loop sending packets. Args: packet: A packet to be sent to the other side of the operation. """ def transmit(packet): while True: transmission_outcome = callable_util.call_logging_exceptions( self._callback, _TRANSMISSION_EXCEPTION_LOG_MESSAGE, packet ) if transmission_outcome.exception is None: with self._lock: complete, packet = self._next_packet() if packet is None: if complete: self._termination_manager.transmission_complete() self._transmitting = False return else: with self._lock: self._emissions = None self._termination_manager.abort(packets.Kind.TRANSMISSION_FAILURE) self._ingestion_manager.abort() self._expiration_manager.abort() self._transmitting = False return self._pool.submit(callable_util.with_exceptions_logged(transmit, _constants.INTERNAL_ERROR_LOG_MESSAGE), packet) self._transmitting = True
def start(self, requirement): if self._pending_ingestion is not None: def initialize(): consumer_creation_outcome = callable_util.call_logging_exceptions( self._consumer_creator.create_consumer, _CREATE_CONSUMER_EXCEPTION_LOG_MESSAGE, requirement) if consumer_creation_outcome.return_value is None: with self._lock: self._abort_and_notify(self._failure_outcome) self._processing = False elif consumer_creation_outcome.return_value.remote_error: with self._lock: self._abort_and_notify(interfaces.Outcome.RECEPTION_FAILURE) self._processing = False elif consumer_creation_outcome.return_value.abandoned: with self._lock: if self._pending_ingestion is not None: self._abort_and_notify(self._failure_outcome) self._processing = False else: wrapped_ingestion_consumer = _WrappedConsumer( consumer_creation_outcome.return_value.consumer) with self._lock: self._wrapped_ingestion_consumer = wrapped_ingestion_consumer payload, complete, moar = self._next() if not moar: self._processing = False return self._process(wrapped_ingestion_consumer, payload, complete) self._pool.submit( callable_util.with_exceptions_logged( initialize, _constants.INTERNAL_ERROR_LOG_MESSAGE)) self._processing = True
def _operator_advance(self, initial_metadata, payload, completion, allowance): if self._processing: self._store_advance(initial_metadata, payload, completion, allowance) else: action = False if initial_metadata is not None: action = True if payload is not None: if 0 < self._local_allowance: self._local_allowance -= 1 action = True else: self._pending_payloads.append(payload) payload = False if completion is not None: if self._pending_payloads: self._pending_completion = completion else: action = True if allowance is not None and self._remote_allowance is not None: allowance += self._remote_allowance self._remote_allowance = 0 action = True if action: self._pool.submit( callable_util.with_exceptions_logged( self._operator_process, _constants.INTERNAL_ERROR_LOG_MESSAGE), self._wrapped_operator, initial_metadata, payload, completion, allowance)
def _transmit(self, packet): """Commences the transmission loop sending packets. Args: packet: A packet to be sent to the other side of the operation. """ def transmit(packet): while True: transmission_outcome = callable_util.call_logging_exceptions( self._callback, _TRANSMISSION_EXCEPTION_LOG_MESSAGE, packet) if transmission_outcome.exception is None: with self._lock: complete, packet = self._next_packet() if packet is None: if complete: self._termination_manager.transmission_complete( ) self._transmitting = False return else: with self._lock: self._emissions = None self._termination_manager.abort( packets.Kind.TRANSMISSION_FAILURE) self._ingestion_manager.abort() self._expiration_manager.abort() self._transmitting = False return self._pool.submit( callable_util.with_exceptions_logged( transmit, _constants.INTERNAL_ERROR_LOG_MESSAGE), packet) self._transmitting = True
def _transmit(self, ticket): """Commences the transmission loop sending tickets. Args: ticket: A links.Ticket to be sent to the other side of the operation. """ def transmit(ticket): while True: transmission_outcome = callable_util.call_logging_exceptions( self._ticket_sink, _TRANSMISSION_EXCEPTION_LOG_MESSAGE, ticket) if transmission_outcome.exception is None: with self._lock: if ticket.termination is links.Ticket.Termination.COMPLETION: self._termination_manager.transmission_complete() ticket = self._next_ticket() if ticket is None: self._transmitting = False return else: with self._lock: self._abort = _ABORTED_NO_NOTIFY if self._termination_manager.outcome is None: self._termination_manager.abort(base.Outcome.TRANSMISSION_FAILURE) self._expiration_manager.terminate() return self._pool.submit(callable_util.with_exceptions_logged( transmit, _constants.INTERNAL_ERROR_LOG_MESSAGE), ticket) self._transmitting = True
def _operator_advance(self, initial_metadata, payload, completion, allowance): if self._processing: self._store_advance(initial_metadata, payload, completion, allowance) else: action = False if initial_metadata is not None: action = True if payload is not None: if 0 < self._local_allowance: self._local_allowance -= 1 action = True else: self._pending_payloads.append(payload) payload = False if completion is not None: if self._pending_payloads: self._pending_completion = completion else: action = True if allowance is not None and self._remote_allowance is not None: allowance += self._remote_allowance self._remote_allowance = 0 action = True if action: self._pool.submit( callable_util.with_exceptions_logged( self._operator_process, _constants.INTERNAL_ERROR_LOG_MESSAGE), self._wrapped_operator, initial_metadata, payload, completion, allowance)
def add_idle_action(self, action): with self._lock: if self._operations: self._idle_actions.append(action) else: self._pool.submit(callable_util.with_exceptions_logged( action, _IDLE_ACTION_EXCEPTION_LOG_MESSAGE))
def _transmit(self, ticket): """Commences the transmission loop sending tickets. Args: ticket: A links.Ticket to be sent to the other side of the operation. """ def transmit(ticket): while True: transmission_outcome = callable_util.call_logging_exceptions( self._ticket_sink, _TRANSMISSION_EXCEPTION_LOG_MESSAGE, ticket) if transmission_outcome.exception is None: with self._lock: if ticket.termination is links.Ticket.Termination.COMPLETION: self._termination_manager.transmission_complete() ticket = self._next_ticket() if ticket is None: self._transmitting = False return else: with self._lock: self._abort = _ABORTED_NO_NOTIFY if self._termination_manager.outcome is None: self._termination_manager.abort( base.Outcome.TRANSMISSION_FAILURE) self._expiration_manager.terminate() return self._pool.submit( callable_util.with_exceptions_logged( transmit, _constants.INTERNAL_ERROR_LOG_MESSAGE), ticket) self._transmitting = True
def set_group_and_method(self, group, method): """See _interfaces.IngestionManager.set_group_and_method for spec.""" if self._subscription_creator is not None and not self._processing: self._pool.submit( callable_util.with_exceptions_logged( self._create, _constants.INTERNAL_ERROR_LOG_MESSAGE), self._subscription_creator, group, method) self._processing = True
def set_group_and_method(self, group, method): """See _interfaces.IngestionManager.set_group_and_method for spec.""" if self._subscription_creator is not None and not self._processing: self._pool.submit( callable_util.with_exceptions_logged( self._create, _constants.INTERNAL_ERROR_LOG_MESSAGE), self._subscription_creator, group, method) self._processing = True
def add_idle_action(self, action): with self._lock: if self._operations: self._idle_actions.append(action) else: self._pool.submit( callable_util.with_exceptions_logged( action, _IDLE_ACTION_EXCEPTION_LOG_MESSAGE))
def add_callback(self, callback): """See _interfaces.TerminationManager.add_callback for specification.""" if not self._has_locally_failed: if self._outstanding_requirements is None: self._work_pool.submit( callable_util.with_exceptions_logged( callback, _CALLBACK_EXCEPTION_LOG_MESSAGE), self._outcome) else: self._callbacks.append(callback)
def set_sink(self, sink): with self._lock: self._sink = sink if sink is not None and self._values and not self._spinning: self._spinning = True self._pool.submit( callable_util.with_exceptions_logged( self._spin, _constants.INTERNAL_ERROR_LOG_MESSAGE), sink, self._values.pop(0))
def termination_action(outcome): with self._lock: self._stats[outcome] += 1 self._operations.pop(operation_id, None) if not self._operations: for action in self._idle_actions: self._pool.submit(callable_util.with_exceptions_logged( action, _IDLE_ACTION_EXCEPTION_LOG_MESSAGE)) self._idle_actions = []
def add_callback(self, callback): """See _interfaces.TerminationManager.add_callback for specification.""" if not self._has_locally_failed: if self._outstanding_requirements is None: self._work_pool.submit( callable_util.with_exceptions_logged( callback, _CALLBACK_EXCEPTION_LOG_MESSAGE), _KINDS_TO_OUTCOMES[self._kind]) else: self._callbacks.append(callback)
def termination_action(outcome): with self._lock: self._stats[outcome] += 1 self._operations.pop(operation_id, None) if not self._operations: for action in self._idle_actions: self._pool.submit( callable_util.with_exceptions_logged( action, _IDLE_ACTION_EXCEPTION_LOG_MESSAGE)) self._idle_actions = []
def add_value(self, value): with self._lock: if self._sink and not self._spinning: self._spinning = True self._pool.submit( callable_util.with_exceptions_logged( self._spin, _constants.INTERNAL_ERROR_LOG_MESSAGE), self._sink, value) else: self._values.append(value)
def _deliver(self, behavior, value): def deliver(): delivery_outcome = callable_util.call_logging_exceptions( behavior, _EXCEPTION_LOG_MESSAGE, value) if delivery_outcome.kind is callable_util.Outcome.Kind.RAISED: with self._lock: self._abort_and_notify(_LOCAL_FAILURE_OUTCOME) self._pool.submit( callable_util.with_exceptions_logged( deliver, _constants.INTERNAL_ERROR_LOG_MESSAGE))
def add_idle_action(self, action): """See base.End.add_idle_action for specification.""" with self._lock: if self._cycle is None: raise ValueError('Can\'t add idle action to stopped End!') action_with_exceptions_logged = callable_util.with_exceptions_logged( action, _IDLE_ACTION_EXCEPTION_LOG_MESSAGE) if self._cycle.operations: self._cycle.idle_actions.append(action_with_exceptions_logged) else: self._cycle.pool.submit(action_with_exceptions_logged)
def add_idle_action(self, action): """See base.End.add_idle_action for specification.""" with self._lock: if self._cycle is None: raise ValueError('Can\'t add idle action to stopped End!') action_with_exceptions_logged = callable_util.with_exceptions_logged( action, _IDLE_ACTION_EXCEPTION_LOG_MESSAGE) if self._cycle.operations: self._cycle.idle_actions.append(action_with_exceptions_logged) else: self._cycle.pool.submit(action_with_exceptions_logged)
def terminate(self): if self._ingestion_complete: self._abort_and_notify(self._failure_outcome) else: self._ingestion_complete = True if self._pending_ingestion is not None and not self._processing: self._pool.submit( callable_util.with_exceptions_logged( self._process, _constants.INTERNAL_ERROR_LOG_MESSAGE), self._wrapped_ingestion_consumer, None, True) self._processing = True
def _deliver(self, behavior, value): def deliver(): delivery_outcome = callable_util.call_logging_exceptions( behavior, _EXCEPTION_LOG_MESSAGE, value) if delivery_outcome.kind is callable_util.Outcome.Kind.RAISED: with self._lock: self._abort_and_notify(_LOCAL_FAILURE_OUTCOME) self._pool.submit( callable_util.with_exceptions_logged( deliver, _constants.INTERNAL_ERROR_LOG_MESSAGE))
def terminate(self): if self._ingestion_complete: self._abort_and_notify(self._failure_outcome) else: self._ingestion_complete = True if self._pending_ingestion is not None and not self._processing: self._pool.submit( callable_util.with_exceptions_logged( self._process, _constants.INTERNAL_ERROR_LOG_MESSAGE), self._wrapped_ingestion_consumer, None, True) self._processing = True
def consume(self, payload): if self._ingestion_complete: self._abort_and_notify(self._failure_outcome) elif self._pending_ingestion is not None: if self._processing: self._pending_ingestion.append(payload) else: self._pool.submit( callable_util.with_exceptions_logged( self._process, _constants.INTERNAL_ERROR_LOG_MESSAGE), self._wrapped_ingestion_consumer, payload, False) self._processing = True
def inline_stream_in_stream_out( front, name, payload_iterator, timeout, trace_id, pool): """Services a stream-in stream-out servicer method.""" rendezvous = _control.Rendezvous() subscription = _rendezvous_subscription(rendezvous) operation = front.operate(name, None, False, timeout, subscription, trace_id) operation.context.add_termination_callback(rendezvous.set_outcome) pool.submit( callable_util.with_exceptions_logged( _control.pipe_iterator_to_consumer, _ITERATOR_EXCEPTION_LOG_MESSAGE), payload_iterator, operation.consumer, lambda: True, True) return _OperationCancellableIterator(rendezvous, operation)
def consume(self, payload): if self._ingestion_complete: self._abort_and_notify(self._failure_outcome) elif self._pending_ingestion is not None: if self._processing: self._pending_ingestion.append(payload) else: self._pool.submit( callable_util.with_exceptions_logged( self._process, _constants.INTERNAL_ERROR_LOG_MESSAGE), self._wrapped_ingestion_consumer, payload, False) self._processing = True
def add_local_allowance(self, allowance): """See _interfaces.IngestionManager.add_local_allowance for spec.""" if any((self._subscription_creator, self._wrapped_operator,)): self._local_allowance += allowance if not self._processing: initial_metadata, payload, completion, allowance, moar = ( self._operator_next()) if moar: self._pool.submit( callable_util.with_exceptions_logged( self._operator_process, _constants.INTERNAL_ERROR_LOG_MESSAGE), initial_metadata, payload, completion, allowance)
def add_local_allowance(self, allowance): """See _interfaces.IngestionManager.add_local_allowance for spec.""" if any((self._subscription_creator, self._wrapped_operator,)): self._local_allowance += allowance if not self._processing: initial_metadata, payload, completion, allowance, moar = ( self._operator_next()) if moar: self._pool.submit( callable_util.with_exceptions_logged( self._operator_process, _constants.INTERNAL_ERROR_LOG_MESSAGE), initial_metadata, payload, completion, allowance)
def inline_stream_in_stream_out(front, name, payload_iterator, timeout, trace_id, pool): """Services a stream-in stream-out servicer method.""" rendezvous = _control.Rendezvous() subscription = _rendezvous_subscription(rendezvous) operation = front.operate(name, None, False, timeout, subscription, trace_id) operation.context.add_termination_callback(rendezvous.set_outcome) pool.submit( callable_util.with_exceptions_logged( _control.pipe_iterator_to_consumer, _ITERATOR_EXCEPTION_LOG_MESSAGE), payload_iterator, operation.consumer, lambda: True, True) return _OperationCancellableIterator(rendezvous, operation)
def _terminate(self, kind): """Terminates the operation. Args: kind: One of packets.Kind.COMPLETION, packets.Kind.CANCELLATION, packets.Kind.EXPIRATION, packets.Kind.RECEPTION_FAILURE, packets.Kind.TRANSMISSION_FAILURE, packets.Kind.SERVICER_FAILURE, or packets.Kind.SERVICED_FAILURE. """ self._expiration_manager.abort() self._outstanding_requirements = None callbacks = list(self._callbacks) self._callbacks = None self._kind = kind outcome = _KINDS_TO_OUTCOMES[kind] act = callable_util.with_exceptions_logged( self._action, _constants.INTERNAL_ERROR_LOG_MESSAGE) if self._has_locally_failed: self._utility_pool.submit(act, outcome) else: def call_callbacks_and_act(callbacks, outcome): for callback in callbacks: callback_outcome = callable_util.call_logging_exceptions( callback, _CALLBACK_EXCEPTION_LOG_MESSAGE, outcome) if callback_outcome.exception is not None: outcome = _KINDS_TO_OUTCOMES[self._local_failure] break self._utility_pool.submit(act, outcome) self._work_pool.submit( callable_util.with_exceptions_logged( call_callbacks_and_act, _constants.INTERNAL_ERROR_LOG_MESSAGE), callbacks, outcome)