def _on_mqtt_connection_failure(self, cause):
"""
Handler that gets called by the transport when a connection fails.
:param Exception cause: The Exception that caused the connection failure.
"""
logger.info("{}: _on_mqtt_connection_failure called: {}".format(
self.name, cause))
if isinstance(self._pending_connection_op,
pipeline_ops_base.ConnectOperation) or isinstance(
self._pending_connection_op,
pipeline_ops_base.ReauthorizeConnectionOperation):
logger.debug("{}: failing connect op".format(self.name))
op = self._pending_connection_op
self._pending_connection_op = None
op.complete(error=cause)
else:
logger.info("{}: Connection failure was unexpected".format(
self.name))
handle_exceptions.swallow_unraised_exception(
cause,
log_msg="Unexpected connection failure. Safe to ignore.",
log_lvl="info")
def _on_mqtt_disconnected(self, cause=None):
"""
Handler that gets called by the transport when the transport disconnects.
:param Exception cause: The Exception that caused the disconnection, if any (optional)
"""
if cause:
logger.error("{}: _on_mqtt_disconnect called: {}".format(self.name, cause))
else:
logger.info("{}: _on_mqtt_disconnect called".format(self.name))
# self.on_disconnected() tells other pipeilne stages that we're disconnected. Do this before
# we do anything else (in case upper stages have any "are we connected" logic.
self.on_disconnected()
if isinstance(self._pending_connection_op, pipeline_ops_base.DisconnectOperation):
logger.debug("{}: completing disconnect op".format(self.name))
op = self._pending_connection_op
self._pending_connection_op = None
# Swallow any errors, because we intended to disconnect - even if something went wrong, we
# got to the state we wanted to be in!
if cause:
handle_exceptions.swallow_unraised_exception(
cause,
log_msg="Unexpected disconnect with error while disconnecting - swallowing error",
)
self._complete_op(op)
else:
logger.warning("{}: disconnection was unexpected".format(self.name))
# Regardless of cause, it is now a ConnectionDroppedError
e = transport_exceptions.ConnectionDroppedError(cause=cause)
handle_exceptions.handle_background_exception(e)
def _on_mqtt_disconnected(self, cause=None):
"""
Handler that gets called by the transport when the transport disconnects.
:param Exception cause: The Exception that caused the disconnection, if any (optional)
"""
if cause:
logger.info("{}: _on_mqtt_disconnect called: {}".format(
self.name, cause))
else:
logger.info("{}: _on_mqtt_disconnect called".format(self.name))
# Send an event to tell other pipeilne stages that we're disconnected. Do this before
# we do anything else (in case upper stages have any "are we connected" logic.)
self.send_event_up(pipeline_events_base.DisconnectedEvent())
if self._pending_connection_op:
# on_mqtt_disconnected will cause any pending connect op to complete. This is how Paho
# behaves when there is a connection error, and it also makes sense that on_mqtt_disconnected
# would cause a pending connection op to fail.
logger.debug("{}: completing pending {} op".format(
self.name, self._pending_connection_op.name))
op = self._pending_connection_op
self._cancel_connection_watchdog(op)
self._pending_connection_op = None
if isinstance(
op, pipeline_ops_base.DisconnectOperation) or isinstance(
op, pipeline_ops_base.ReauthorizeConnectionOperation):
# Swallow any errors if we intended to disconnect - even if something went wrong, we
# got to the state we wanted to be in!
if cause:
handle_exceptions.swallow_unraised_exception(
cause,
log_msg=
"Unexpected disconnect with error while disconnecting - swallowing error",
)
op.complete()
else:
if cause:
op.complete(error=cause)
else:
op.complete(
error=transport_exceptions.ConnectionDroppedError(
"transport disconnected"))
else:
logger.info("{}: disconnection was unexpected".format(self.name))
# Regardless of cause, it is now a ConnectionDroppedError. log it and swallow it.
# Higher layers will see that we're disconencted and reconnect as necessary.
e = transport_exceptions.ConnectionDroppedError(cause=cause)
handle_exceptions.swallow_unraised_exception(
e,
log_msg=
"Unexpected disconnection. Safe to ignore since other stages will reconnect.",
log_lvl="info",
)
def _on_mqtt_disconnected(self, cause=None):
"""
Handler that gets called by the transport when the transport disconnects.
:param Exception cause: The Exception that caused the disconnection, if any (optional)
"""
if cause:
logger.error("{}: _on_mqtt_disconnect called: {}".format(
self.name, cause))
else:
logger.info("{}: _on_mqtt_disconnect called".format(self.name))
# self.on_disconnected() tells other pipeilne stages that we're disconnected. Do this before
# we do anything else (in case upper stages have any "are we connected" logic.
self.on_disconnected()
if self._pending_connection_op:
# on_mqtt_disconnected will cause any pending connect op to complete. This is how Paho
# behaves when there is a connection error, and it also makes sense that on_mqtt_disconnected
# would cause a pending connection op to fail.
logger.debug("{}: completing pending {} op".format(
self.name, self._pending_connection_op.name))
op = self._pending_connection_op
self._pending_connection_op = None
if isinstance(op, pipeline_ops_base.DisconnectOperation):
# Swallow any errors if we intended to disconnect - even if something went wrong, we
# got to the state we wanted to be in!
if cause:
handle_exceptions.swallow_unraised_exception(
cause,
log_msg=
"Unexpected disconnect with error while disconnecting - swallowing error",
)
op.complete()
else:
if cause:
op.complete(error=cause)
else:
op.complete(
error=transport_exceptions.ConnectionDroppedError(
"transport disconnected"))
else:
logger.warning("{}: disconnection was unexpected".format(
self.name))
# Regardless of cause, it is now a ConnectionDroppedError
e = transport_exceptions.ConnectionDroppedError(cause=cause)
handle_exceptions.handle_background_exception(e)
def _on_mqtt_disconnected(self, cause=None):
"""
Handler that gets called by the transport when the transport disconnects.
:param Exception cause: The Exception that caused the disconnection, if any (optional)
"""
if cause:
logger.info("{}: _on_mqtt_disconnect called: {}".format(
self.name, cause))
else:
logger.info("{}: _on_mqtt_disconnect called".format(self.name))
# Send an event to tell other pipeline stages that we're disconnected. Do this before
# we do anything else (in case upper stages have any "are we connected" logic.)
# NOTE: Other stages rely on the fact that this occurs before any op that may be in
# progress is completed. Be careful with changing the order things occur here.
self.send_event_up(pipeline_events_base.DisconnectedEvent())
if self._pending_connection_op:
op = self._pending_connection_op
if isinstance(op, pipeline_ops_base.DisconnectOperation):
logger.debug(
"{}: Expected disconnect - completing pending disconnect op"
.format(self.name))
# Swallow any errors if we intended to disconnect - even if something went wrong, we
# got to the state we wanted to be in!
if cause:
handle_exceptions.swallow_unraised_exception(
cause,
log_msg=
"Unexpected error while disconnecting - swallowing error",
)
# Disconnect complete, no longer pending
self._pending_connection_op = None
op.complete()
else:
logger.debug(
"{}: Unexpected disconnect - completing pending {} operation"
.format(self.name, op.name))
# Cancel any potential connection watchdog, and clear the pending op
self._cancel_connection_watchdog(op)
self._pending_connection_op = None
# Complete
if cause:
op.complete(error=cause)
else:
op.complete(
error=transport_exceptions.ConnectionDroppedError(
"transport disconnected"))
else:
logger.info(
"{}: Unexpected disconnect (no pending connection op)".format(
self.name))
# If there is no connection retry, cancel any transport operations waiting on response
# so that they do not get stuck there.
if not self.pipeline_root.pipeline_configuration.connection_retry:
logger.debug(
"{}: Connection Retry disabled - cancelling in-flight operations"
.format(self.name))
# TODO: Remove private access to the op manager (this layer shouldn't know about it)
# This is a stopgap. I didn't want to invest too much infrastructure into a cancel flow
# given that future development of individual operation cancels might affect the
# approach to cancelling inflight ops waiting in the transport.
self.transport._op_manager.cancel_all_operations()
# Regardless of cause, it is now a ConnectionDroppedError. Log it and swallow it.
# Higher layers will see that we're disconencted and may reconnect as necessary.
e = transport_exceptions.ConnectionDroppedError(cause=cause)
handle_exceptions.swallow_unraised_exception(
e,
log_msg="Unexpected disconnection",
log_lvl="info",
)
def _on_mqtt_disconnected(self, cause=None):
"""
Handler that gets called by the transport when the transport disconnects.
:param Exception cause: The Exception that caused the disconnection, if any (optional)
"""
if cause:
logger.info("{}: _on_mqtt_disconnect called: {}".format(
self.name, cause))
else:
logger.info("{}: _on_mqtt_disconnect called".format(self.name))
# Send an event to tell other pipeline stages that we're disconnected. Do this before
# we do anything else (in case upper stages have any "are we connected" logic.)
self.send_event_up(pipeline_events_base.DisconnectedEvent())
# Complete any pending connection ops
if self._pending_connection_op:
# on_mqtt_disconnected will cause any pending connect op to complete. This is how Paho
# behaves when there is a connection error, and it also makes sense that on_mqtt_disconnected
# would cause a pending connection op to fail.
logger.debug("{}: completing pending {} op".format(
self.name, self._pending_connection_op.name))
op = self._pending_connection_op
self._cancel_connection_watchdog(op)
self._pending_connection_op = None
if isinstance(
op, pipeline_ops_base.DisconnectOperation) or isinstance(
op, pipeline_ops_base.ReauthorizeConnectionOperation):
# Swallow any errors if we intended to disconnect - even if something went wrong, we
# got to the state we wanted to be in!
#
# NOTE: ReauthorizeConnectionOperation currently completes on disconnect, not when
# the connection is re-established (it leaves connection retry to automatically
# re-establish). This needs to change because it is inaccurate and means that if
# a SASToken expires while connection retry is disabled, a reauthorization cannot
# complete (!!!!)
# TODO: Fix that!
if cause:
handle_exceptions.swallow_unraised_exception(
cause,
log_msg=
"Unexpected disconnect with error while disconnecting - swallowing error",
)
op.complete()
else:
if cause:
op.complete(error=cause)
else:
op.complete(
error=transport_exceptions.ConnectionDroppedError(
"transport disconnected"))
else:
logger.info("{}: disconnection was unexpected".format(self.name))
# If there is no connection retry, cancel any transport operations waiting on response
# so that they do not get stuck there.
if not self.pipeline_root.pipeline_configuration.connection_retry:
logger.debug(
"{}: Connection Retry disabled - cancelling in-flight operations"
.format(self.name))
# TODO: Remove private access to the op manager (this layer shouldn't know about it)
# This is a stopgap. I didn't want to invest too much infrastructure into a cancel flow
# given that future development of individual operation cancels might affect the
# approach to cancelling inflight ops waiting in the transport.
self.transport._op_manager.cancel_all_operations()
# Regardless of cause, it is now a ConnectionDroppedError. Log it and swallow it.
# Higher layers will see that we're disconencted and may reconnect as necessary.
e = transport_exceptions.ConnectionDroppedError(cause=cause)
handle_exceptions.swallow_unraised_exception(
e,
log_msg="Unexpected disconnection",
log_lvl="info",
)
