def _on_mqtt_disconnected(self, cause):
"""
Handler that gets called by the transport when the transport disconnects.
"""
logger.error("{}: _on_mqtt_disconnect called: {}".format(
self.name, cause))
# self.on_disconnected() tells other pipeilne stages that we're disconnected. Do this before
# we do anything else (in case upper stages have any "are we connected" logic.
self.on_disconnected()
# regardless of the cause, we wrap it in a ConnectionDroppedError object because that's
# the real problem at this point.
if cause:
try:
six.raise_from(errors.ConnectionDroppedError, cause)
except errors.ConnectionDroppedError as e:
cause = e
if self._active_disconnect_op:
logger.info("{}: completing disconnect op".format(self.name))
op = self._active_disconnect_op
self._active_disconnect_op = None
op.error = cause
operation_flow.complete_op(stage=self, op=op)
else:
logger.warning("{}: disconnection was unexpected".format(
self.name))
unhandled_exceptions.exception_caught_in_background_thread(cause)
def _on_mqtt_disconnected(self, cause=None):
"""
Handler that gets called by the transport when the transport disconnects.
:param Exception cause: The Exception that caused the disconnection, if any (optional)
"""
logger.error("{}: _on_mqtt_disconnect called: {}".format(
self.name, cause))
# self.on_disconnected() tells other pipeilne stages that we're disconnected. Do this before
# we do anything else (in case upper stages have any "are we connected" logic.
self.on_disconnected()
if isinstance(self._pending_connection_op,
pipeline_ops_base.DisconnectOperation):
logger.info("{}: completing disconnect op".format(self.name))
op = self._pending_connection_op
self._pending_connection_op = None
if cause:
# Only create a ConnnectionDroppedError if there is a cause,
# i.e. unexpected disconnect.
try:
six.raise_from(errors.ConnectionDroppedError, cause)
except errors.ConnectionDroppedError as e:
op.error = e
operation_flow.complete_op(stage=self, op=op)
else:
logger.warning("{}: disconnection was unexpected".format(
self.name))
# Regardless of cause, it is now a ConnectionDroppedError
try:
six.raise_from(errors.ConnectionDroppedError, cause)
except errors.ConnectionDroppedError as e:
unhandled_exceptions.exception_caught_in_background_thread(e)
def on_token_update_complete(op):
logger.info(
"{}({}): token update operation is complete. Error={}".format(
self.name, op.name, op.error))
if op.error:
unhandled_exceptions.exception_caught_in_background_thread(
op.error)
def complete_op(stage, op):
"""
Helper function to complete an operation by calling its callback function thus
returning the result of the operation back up the pipeline. This is perferred to
calling the operation's callback directly as it provides several layers of protection
(such as a try/except wrapper) which are strongly advised.
"""
if op.error:
logger.error("{}({}): completing with error {}".format(
stage.name, op.name, op.error))
else:
logger.info("{}({}): completing without error".format(
stage.name, op.name))
try:
op.callback(op)
except Exception as e:
_, e, _ = sys.exc_info()
logger.error(
msg=
"Unhandled error calling back inside {}.complete_op() after {} complete"
.format(stage.name, op.name),
exc_info=e,
)
unhandled_exceptions.exception_caught_in_background_thread(e)
def on_token_update_complete(op):
if op.error:
logger.error(
"{}({}): token update operation failed. Error={}".format(
self.name, op.name, op.error))
unhandled_exceptions.exception_caught_in_background_thread(
op.error)
else:
logger.debug(
"{}({}): token update operation is complete".format(
self.name, op.name))
def handle_pipeline_event(self, event):
"""
Handle a pipeline event that arrives from the stage below this stage. Derived
classes should not override this function. Any stage-specific handling of
PipelineEvent objects should be implemented by overriding the private
_handle_pipeline_event function in the derived stage.
:param PipelineEvent event: The event that is being passed back up the pipeline
"""
try:
self._handle_pipeline_event(event)
except Exception as e:
unhandled_exceptions.exception_caught_in_background_thread(e)
def _on_mqtt_connection_failure(self, cause):
"""
Handler that gets called by the transport when a connection fails.
"""
logger.error("{}: _on_mqtt_connection_failure called: {}".format(
self.name, cause))
if self._active_connect_op:
logger.info("{}: failing connect op".format(self.name))
op = self._active_connect_op
self._active_connect_op = None
op.error = cause
operation_flow.complete_op(stage=self, op=op)
else:
logger.warning("{}: Connection failure was unexpected".format(
self.name))
unhandled_exceptions.exception_caught_in_background_thread(cause)
def pass_event_to_previous_stage(stage, event):
"""
Helper function to pass an event to the previous stage of the pipeline. This is the default
behavior of events while traveling through the pipeline. They start somewhere (maybe the
bottom) and move up the pipeline until they're handled or until they error out.
"""
if stage.previous:
logger.debug("{}({}): pushing event up to {}".format(
stage.name, event.name, stage.previous.name))
stage.previous.handle_pipeline_event(event)
else:
logger.error("{}({}): Error: unhandled event".format(
stage.name, event.name))
error = NotImplementedError(
"{} unhandled at {} stage with no previous stage".format(
event.name, stage.name))
unhandled_exceptions.exception_caught_in_background_thread(error)
def thread_proc():
threading.current_thread().name = thread_name
try:
return func(*args, **kwargs)
except Exception as e:
if not block:
unhandled_exceptions.exception_caught_in_background_thread(
e)
except BaseException:
if not block:
logger.error(
"Unhandled exception in background thread")
logger.error(
"This may cause the background thread to abort and may result in system instability."
)
traceback.print_exc()
raise
def on_last_op_done(last_op):
if finally_op:
@pipeline_thread.runs_on_pipeline_thread
def on_finally_done(finally_op):
logger.info("{}({}):run_ops_serial: finally_op done.".format(
stage.name, finally_op.name))
if last_op.error:
logger.info(
"{}({}):run_ops_serial: copying error from {}.".format(
stage.name, finally_op.name, last_op.name))
finally_op.error = last_op.error
try:
logger.info(
"{}({}):run_ops_serial: calling back after finally_op."
.format(stage.name, finally_op.name))
callback(finally_op)
except Exception as e:
logger.error(
msg="{}({}):run_ops_serial: Unhandled error in callback"
.format(stage.name, finally_op.name),
exc_info=e,
)
unhandled_exceptions.exception_caught_in_background_thread(
e)
finally_op.callback = on_finally_done
logger.info("{}({}):run_ops_serial: running finally_op.".format(
stage.name, last_op.name))
pass_op_to_next_stage(stage, finally_op)
else:
try:
logger.info(
"{}({}):run_ops_serial: no finally_op. calling back.".
format(stage.name, last_op.name))
callback(last_op)
except Exception as e:
logger.error(
msg="{}({}):run_ops_serial: Unhandled error in callback".
format(stage.name, last_op.name),
exc_info=e,
)
unhandled_exceptions.exception_caught_in_background_thread(e)
def on_finally_done(finally_op):
logger.info("{}({}):run_ops_serial: finally_op done.".format(
stage.name, finally_op.name))
if last_op.error:
logger.info(
"{}({}):run_ops_serial: copying error from {}.".format(
stage.name, finally_op.name, last_op.name))
finally_op.error = last_op.error
try:
logger.info(
"{}({}):run_ops_serial: calling back after finally_op."
.format(stage.name, finally_op.name))
callback(finally_op)
except Exception as e:
logger.error(
msg="{}({}):run_ops_serial: Unhandled error in callback"
.format(stage.name, finally_op.name),
exc_info=e,
)
unhandled_exceptions.exception_caught_in_background_thread(
e)
def _on_mqtt_connection_failure(self, cause):
"""
Handler that gets called by the transport when a connection fails.
:param Exception cause: The Exception that caused the connection failure.
"""
logger.error("{}: _on_mqtt_connection_failure called: {}".format(
self.name, cause))
if isinstance(self._pending_connection_op,
pipeline_ops_base.ConnectOperation) or isinstance(
self._pending_connection_op,
pipeline_ops_base.ReconnectOperation):
logger.info("{}: failing connect op".format(self.name))
op = self._pending_connection_op
self._pending_connection_op = None
op.error = cause
operation_flow.complete_op(stage=self, op=op)
else:
logger.warning("{}: Connection failure was unexpected".format(
self.name))
unhandled_exceptions.exception_caught_in_background_thread(cause)
