def handle_uncaught_exception(self,
request,
resolver,
exc_info,
reraise=True):
"""Override `BaseHandler.handle_uncaught_exception`.
If a retryable failure is detected, a retry is requested. It's up
to ``get_response`` to actually do the retry.
"""
exc_type, exc_value, exc_traceback = exc_info
exc = exc_type(exc_value)
exc.__traceback__ = exc_traceback
exc.__cause__ = exc_value.__cause__
if reraise:
raise exc from exc.__cause__
# Re-perform the exception so the process_exception_by_middlware
# can process the exception. Any exceptions that cause a retry to
# occur place it in the __retry so the `get_response` can handle
# performing the retry.
try:
try:
raise exc from exc.__cause__
except Exception as exc:
return self.process_exception_by_middleware(exc, request)
except SystemExit:
raise
except RetryTransaction:
response = HttpResponseConflict()
self.__retry.add(response)
return response
except Exception as exc:
if is_retryable_failure(exc):
response = HttpResponseConflict()
self.__retry.add(response)
return response
else:
logger.error(
"500 Internal Server Error @ %s" % request.path,
exc_info=sys.exc_info(),
)
return HttpResponse(
content=str(exc).encode("utf-8"),
status=int(http.client.INTERNAL_SERVER_ERROR),
content_type="text/plain; charset=utf-8",
)
def handle_uncaught_exception(self, request, resolver, exc_info):
"""Override `BaseHandler.handle_uncaught_exception`.
If a retryable failure is detected, a retry is requested. It's up
to ``get_response`` to actually do the retry.
"""
upcall = super(WebApplicationHandler, self).handle_uncaught_exception
response = upcall(request, resolver, exc_info)
# Add it to the retry set if this response was caused by a
# retryable failure.
exc_type, exc_value, exc_traceback = exc_info
if isinstance(exc_value, RetryTransaction):
self.__retry.add(response)
elif is_retryable_failure(exc_value):
self.__retry.add(response)
elif isinstance(exc_value, MAASAPIException):
return exc_value.make_http_response()
else:
logger.error("500 Internal Server Error @ %s" % request.path,
exc_info=exc_info)
# Return the response regardless. This means that we'll get Django's
# error page when there's a persistent retryable failure.
return response
def try_or_log_event(machine, signal_status, error_message, func, *args,
**kwargs):
"""
Attempts to run the specified function, related to the specified node and
signal status. Will log the specified error, and create a node event,
if the function fails.
If the function raises a retryable failure, will re-raise the exception so
that a retry can be attempted.
:param machine: The machine related to the attempted action. Will be used
in order to log an event, if the function raises an exception.
:param signal_status: The initial SIGNAL_STATUS, which will be returned
as-is if no exception occurs. If an exception occurs,
SIGNAL_STATUS.FAILED will be returned.
:param error_message: The error message for the log (and node event log) if
an exception occurs.
:param func: The function which will be attempted
:param args: Arguments to pass to the function to be attempted.
:param kwargs: Keyword arguments to pass to the function to be attempted.
:return:
"""
try:
func(*args, **kwargs)
except BaseException as e:
if is_retryable_failure(e):
# Not the fault of the post-processing function, so
# re-raise so that the retry mechanism does its job.
raise
log.err(None, error_message)
Event.objects.create_node_event(
system_id=machine.system_id,
event_type=EVENT_TYPES.SCRIPT_RESULT_ERROR,
event_description=error_message)
signal_status = SIGNAL_STATUS.FAILED
return signal_status
def process_exception(self, request, exception):
encoding = "utf-8"
if isinstance(exception, MAASAPIException):
# Print a traceback if this is a 500 error.
if (settings.DEBUG or exception.api_error
== http.client.INTERNAL_SERVER_ERROR):
self.log_exception(exception)
# This type of exception knows how to translate itself into
# an http response.
return exception.make_http_response()
elif isinstance(exception, ValidationError):
if settings.DEBUG:
self.log_exception(exception)
if hasattr(exception, "message_dict"):
# Complex validation error with multiple fields:
# return a json version of the message_dict.
return HttpResponseBadRequest(
json.dumps(exception.message_dict),
content_type="application/json",
)
else:
# Simple validation error: return the error message.
return HttpResponseBadRequest(
str("".join(exception.messages)).encode(encoding),
content_type="text/plain; charset=%s" % encoding,
)
elif isinstance(exception, PermissionDenied):
if settings.DEBUG:
self.log_exception(exception)
return HttpResponseForbidden(
content=str(exception).encode(encoding),
content_type="text/plain; charset=%s" % encoding,
)
elif isinstance(exception, ExternalProcessError):
# Catch problems interacting with processes that the
# appserver spawns, e.g. rndc.
#
# While this is a serious error, it should be a temporary
# one as the admin should be checking and fixing, or it
# could be spurious. There's no way of knowing, so the best
# course of action is to ask the caller to repeat.
if settings.DEBUG:
self.log_exception(exception)
response = HttpResponse(
content=str(exception).encode(encoding),
status=int(http.client.SERVICE_UNAVAILABLE),
content_type="text/plain; charset=%s" % encoding,
)
response["Retry-After"] = RETRY_AFTER_SERVICE_UNAVAILABLE
return response
elif isinstance(exception, Http404):
if settings.DEBUG:
self.log_exception(exception)
return get_exception_response(request, get_resolver(get_urlconf()),
404, exception)
elif is_retryable_failure(exception):
# We never handle retryable failures.
return None
elif isinstance(exception, SystemExit):
return None
else:
# Print a traceback.
self.log_exception(exception)
# Return an API-readable "Internal Server Error" response.
return HttpResponse(
content=str(exception).encode(encoding),
status=int(http.client.INTERNAL_SERVER_ERROR),
content_type="text/plain; charset=%s" % encoding,
)
def process_exception(self, request, exception):
"""Django middleware callback."""
if not request.path_info.startswith(self.path_prefix):
# Not a path we're handling exceptions for.
return None
if is_retryable_failure(exception):
# We never handle retryable failures.
return None
encoding = 'utf-8'
if isinstance(exception, MAASAPIException):
# Print a traceback if this is a 500 error.
if (settings.DEBUG or exception.api_error
== http.client.INTERNAL_SERVER_ERROR):
self.log_exception(exception)
# This type of exception knows how to translate itself into
# an http response.
return exception.make_http_response()
elif isinstance(exception, ValidationError):
if settings.DEBUG:
self.log_exception(exception)
if hasattr(exception, 'message_dict'):
# Complex validation error with multiple fields:
# return a json version of the message_dict.
return HttpResponseBadRequest(json.dumps(
exception.message_dict),
content_type='application/json')
else:
# Simple validation error: return the error message.
return HttpResponseBadRequest(
str(''.join(exception.messages)).encode(encoding),
content_type="text/plain; charset=%s" % encoding)
elif isinstance(exception, PermissionDenied):
if settings.DEBUG:
self.log_exception(exception)
return HttpResponseForbidden(
content=str(exception).encode(encoding),
content_type="text/plain; charset=%s" % encoding)
elif isinstance(exception, ExternalProcessError):
# Catch problems interacting with processes that the
# appserver spawns, e.g. rndc.
#
# While this is a serious error, it should be a temporary
# one as the admin should be checking and fixing, or it
# could be spurious. There's no way of knowing, so the best
# course of action is to ask the caller to repeat.
if settings.DEBUG:
self.log_exception(exception)
response = HttpResponse(
content=str(exception).encode(encoding),
status=int(http.client.SERVICE_UNAVAILABLE),
content_type="text/plain; charset=%s" % encoding)
response['Retry-After'] = (RETRY_AFTER_SERVICE_UNAVAILABLE)
return response
else:
# Print a traceback.
self.log_exception(exception)
# Return an API-readable "Internal Server Error" response.
return HttpResponse(content=str(exception).encode(encoding),
status=int(http.client.INTERNAL_SERVER_ERROR),
content_type="text/plain; charset=%s" %
encoding)
