def logStatus(e: grpc.RpcError) -> None:
logging.info("----------------------------------")
# logging.info("gRPC returned error: {}".format(e))
# logging.info(" trailing_metadata: {}".format(e.trailing_metadata()))
print("----")
code = e.code()
print(" code: {} ({})".format(code.name, code.value[0]))
details = e.details()
print(" details: {}".format(details))
err_string = e.debug_error_string()
print(" err str: {}".format(err_string))
status = rpc_status.from_call(e)
if status is not None:
for detail in status.details:
if detail.Is(error_details_pb2.ErrorInfo.DESCRIPTOR):
errinfo = error_details_pb2.ErrorInfo()
detail.Unpack(errinfo)
print(" ErrorInfo:")
print(" Reason: {}".format(errinfo.reason))
print(" Domain: {}".format(errinfo.domain))
print(" Metadata:")
for k, v in errinfo.metadata.items():
print(" {}: {}".format(k, v))
# logging.info(" ErrorInfo: {}".format(errinfo))
else:
print(" -- unknown details type.")
async def handle_rpc_error(request: Request, exc: RpcError):
return JSONResponse(
status_code=get_http_status_code_from_grpc_status(exc.code()),
content={
'detail': exc.details(),
},
)
def raise_not_found():
err = RpcError()
err.code = MagicMock(return_value=StatusCode.NOT_FOUND)
err.details = MagicMock(return_value='does not exist')
err.trailing_metadata = MagicMock()
raise err
def _should_terminate_polling(e: grpc.RpcError) -> None:
# Caller only expects polling to be terminated after deadline exceeded.
if e.code() == grpc.StatusCode.DEADLINE_EXCEEDED:
return True
# Could be a temporary connection issue. Suppress error.
# TODO: reconnect GRPC channel?
if e.code() == grpc.StatusCode.UNAVAILABLE:
return True
return False
def is_grpc_error_retryable(error: grpc.RpcError) -> bool:
status_code = error.code()
error_details = error.details()
if status_code == grpc.StatusCode.UNAVAILABLE and \
any(err_msg.value in error_details for err_msg in
RetryableGrpcErrorDetails):
# server end closed connection.
return True
return False
def decode_exception(e: grpc.RpcError) -> Exception:
if e.code() != grpc.StatusCode.ABORTED:
# The ABORTED status code is used by the server when an application
# error is serialized into the the exception details. If the code
# isn't ABORTED, then return the original error since there's no
# serialized error to decode.
# See server.py::return_exception_in_context for details
return ConnectionError(f"GRPC connection failed: {e}")
data = base64.standard_b64decode(e.details())
return loads_from_server(data)
def _raise_grpc_error(error: grpc.RpcError) -> Any:
"""Convert an error message from gRPC into a GraknError or a ConnectionError"""
assert isinstance(error, grpc.Call)
error_type = next(
(value
for (key, value) in error.trailing_metadata() if key == 'errortype'),
None)
if error_type is not None:
raise GraknError(error.details()) from error
else:
raise ConnectionError from error
def get_foo():
nonlocal num_calls
if num_calls < 2:
err = RpcError()
err.code = MagicMock(return_value=StatusCode.CANCELLED)
err.details = MagicMock(return_value='Server broke')
num_calls += 1
raise err
return 'success'
def foo():
err = RpcError()
err.code = MagicMock(return_value=StatusCode.INVALID_ARGUMENT)
err.details = MagicMock(return_value=error_msg)
# format is a list/tuple of 2-tuples (pairs)
trailing_metadata = [
(_GRPC_DETAILS_METADATA_KEY, status_proto.SerializeToString()),
]
err.trailing_metadata = MagicMock(return_value=trailing_metadata)
raise err
def _log_grpc_error(err: grpc.RpcError) -> None:
if err.code() == grpc.StatusCode.INTERNAL or err.code() == grpc.StatusCode.UNIMPLEMENTED:
logging.error(
'Internal Error in subscriberdb-Orc8r communication. '
'If AGW and Orc8r are deployed at a commit hash, '
'upgrade AGW and Orc8r to latest commit hash.',
)
else:
logging.error(
# inspect.stack() indices: [1] -> frame record of penultimate caller; [3] -> function name
"%s request error! [%s] %s", inspect.stack(context=0)[1][3], err.code(),
err.details(),
)
def handle_grpc_error(exn: grpc.RpcError):
# gRPC-python gets creative with their exceptions. grpc.RpcError as a type is useless;
# the usefulness come from the fact that it is polymorphically also a grpc.Call and thus has
# the .code() member. Pylint doesn't know this because it's not known statically.
#
# Neither pylint nor I are the only ones who find this confusing:
# https://github.com/grpc/grpc/issues/10885#issuecomment-302581315
# pylint: disable=no-member
if exn.code() == grpc.StatusCode.UNAVAILABLE:
# If the monitor is unavailable, it is in the process of shutting down or has already
# shut down. Don't emit an error if this is the case.
return
details = exn.details()
raise Exception(details)
def grpc_error_detail(err: grpc.RpcError) -> Dict[str, Any]:
"""
This function takes a `grpc.RpcError` and returns a dict containing the
error details.
Usage example:
```python
import vegaapiclient as vac
grpc_client = vac.VegaTradingClient(node_grpc_url)
grpc_request = ...
try:
grpc_response = grpc_client.NameOfEndpoint(grpc_request)
except grpc.RpcError as exc:
print(json.dumps(vac.grpc_error_detail(exc), indent=2, sort_keys=True))
exit(1)
```
"""
if not isinstance(err, grpc.RpcError):
raise ValueError(f"Not a gRPC error: {err}")
e: Dict[str, Any] = {}
for arg in err.args:
for attr in ["code", "details"]:
if hasattr(arg, attr):
e[attr] = str(getattr(arg, attr))
if hasattr(arg, "debug_error_string"):
try:
parsed = json.loads(arg.debug_error_string)
e["debug_error_string"] = parsed
except json.decoder.JSONDecodeError:
e["debug_error_string_raw"] = arg.debug_error_string
md: List[Dict[str, Any]] = []
for tmd in err.trailing_metadata():
if not (hasattr(tmd, "key") and hasattr(tmd, "value")
and tmd.key.startswith("grpc-status-details")):
continue
status = goog_status.Status()
status.MergeFromString(tmd.value)
md.append({
"code":
str(status.code),
"message":
status.message,
"details": [{
"type":
d.type_url,
"value":
remove_control_chars(
d.value.decode("utf-8", errors="backslashreplace")),
} for d in status.details],
})
e["metadata"] = md
return {"gRPCerror": e}
def _is_migration_error(self, e: grpc.RpcError) -> bool:
if e.code(
) == grpc.StatusCode.FAILED_PRECONDITION and self.get_blockchain_version(
) > self._kin_version:
return True
return False
def try_from(
cls,
error: RpcError) -> Union["AuthServiceNotActivated", RpcError]:
if isinstance(error, _InactiveRpcError):
details = error.details()
if "the auth service is not activated" in details:
return cls(details)
return error
def test_client_grpc_retry_non_retryable(mocker):
err = RpcError()
err.code = MagicMock(return_value=StatusCode.INVALID_ARGUMENT)
err.details = MagicMock(return_value='bad value')
def get_foo():
raise err
stub = make_stub_with_func(get_foo)
make_stub_methods_retryable(stub)
with pytest.raises(RpcError) as exc_info:
stub.GetFoo()
# Should raise original error
assert exc_info.value.code() == StatusCode.INVALID_ARGUMENT
assert exc_info.value == err
def handle_exception(self, trace: str, first_try: float, e: RpcError, retry_delay: float):
if e.code() == StatusCode.UNAVAILABLE and self.timeout is not None and (time.time() - first_try) < self.timeout:
# Server is not available, and timeout didn't expired yet: sleep and retry
self.logger.debug(f"<RPC> << {self.s_name}.{self.m_name} (will retry in {retry_delay}s because of 'unavailable' error; details: '{e.details()}')")
time.sleep(retry_delay)
self.logger.debug(f"<RPC> >> {self.s_name}.{self.m_name}... (retry)")
else:
# Timed out or any other reason: raise exception
self.logger.debug(f"<RPC> >> {self.s_name}.{self.m_name} error: {str(e)}")
raise RpcException(f"RPC error (on {trace}): {e}", rc=ResultCode.ERROR_RPC)
def test_list_images_exc(self):
self.images.stub.ListImages.side_effect = err = RpcError()
err.code = MagicMock(return_value=StatusCode.UNKNOWN)
err.details = MagicMock(return_value="these are error details")
with self.assertRaisesRegex(ImageServiceException,
"these are error details"):
self.images.list_images()
self.images.stub.ListImages.assert_called_with(ListImagesRequest())
def show_grpc_exception(self,
message: str,
e: grpc.RpcError,
blocking: bool = False) -> None:
logger.exception("app grpc exception", exc_info=e)
dialog = ErrorDialog(self, "GRPC Exception", message, e.details())
if blocking:
dialog.show()
else:
self.after(0, lambda: dialog.show())
def indicates_connection_error(error: grpc.RpcError) -> bool:
"""Try to determine if an RpcError is caused by a connection error
to the RPC endpoint, which means that it's likely not caused by a bug in the code
Args:
error (grpc.RpcError): The RpcError
Returns:
bool: Is this a network error
"""
return error.code() in {grpc.StatusCode.UNAVAILABLE, grpc.StatusCode.DEADLINE_EXCEEDED}
def connection_error(rpc_error: RpcError) -> VectorConnectionException:
"""Translates grpc-specific errors to user-friendly :class:`VectorConnectionException`."""
code = rpc_error.code()
if code is StatusCode.UNAUTHENTICATED:
return VectorUnauthenticatedException(rpc_error)
if code is StatusCode.UNAVAILABLE:
return VectorUnavailableException(rpc_error)
if code is StatusCode.UNIMPLEMENTED:
return VectorUnimplementedException(rpc_error)
if code is StatusCode.DEADLINE_EXCEEDED:
return VectorTimeoutException(rpc_error)
return VectorConnectionException(rpc_error)
def _log_grpc_error(err: grpc.RpcError) -> None:
if err.code() in {grpc.StatusCode.INTERNAL, grpc.StatusCode.UNIMPLEMENTED}:
logging.error(
'Internal Error in subscriberdb-Orc8r communication. '
'If AGW and Orc8r are deployed at a commit hash, '
'upgrade AGW and Orc8r to latest commit hash.',
extra=EXCLUDE_FROM_ERROR_MONITORING,
)
elif err.code() in {
grpc.StatusCode.UNAVAILABLE, grpc.StatusCode.DEADLINE_EXCEEDED
}:
logging.error(
'Internal Error in subscriberdb-Orc8r communication. '
'Cannot connect to server: [%s] %s',
err.code(),
err.details(),
extra=EXCLUDE_FROM_ERROR_MONITORING,
)
else:
logging.error(
# inspect.stack() indices: [1] -> frame record of penultimate caller; [3] -> function name
"%s request error! [%s] %s",
inspect.stack(context=0)[1][3],
err.code(),
err.details(),
)
def _can_reconnect(self, e: grpc.RpcError) -> bool:
"""
Returns True if the RPC error can be recovered from and a retry is
appropriate, false otherwise.
"""
if not self._reconnect_enabled:
return False
if self._in_shutdown:
# Channel is being shutdown, don't try to reconnect
return False
if e.code() in GRPC_UNRECOVERABLE_ERRORS:
# Unrecoverable error -- These errors are specifically raised
# by the server's application logic
return False
if e.code() == grpc.StatusCode.INTERNAL:
details = e.details()
if details == "Exception serializing request!":
# The client failed tried to send a bad request (for example,
# passing "None" instead of a valid grpc message). Don't
# try to reconnect/retry.
return False
# All other errors can be treated as recoverable
return True
def test_client_grpc_retry_fail(mocker):
num_calls = 0
err = RpcError()
err.code = MagicMock(return_value=StatusCode.CANCELLED)
err.details = MagicMock(return_value='Server broke')
def get_foo():
nonlocal num_calls
num_calls += 1
raise err
stub = make_stub_with_func(get_foo)
make_stub_methods_retryable(stub)
with pytest.raises(RpcError) as exc_info:
stub.GetFoo()
# Should raise original error
assert exc_info.value.code() == StatusCode.CANCELLED
assert exc_info.value == err
# num calls should be 1(orig call) + MAX_RETRIES
assert num_calls == DEFAULT_MAX_RETRIES + 1
def raise_native(err: RpcError):
"""Convert the given gRPC error into a native exception and raise it."""
status = err.code()
if status == StatusCode.DEADLINE_EXCEEDED:
raise TimeoutError(err.details())
elif status == StatusCode.UNIMPLEMENTED:
raise NotImplementedError(err.details())
elif status == StatusCode.INVALID_ARGUMENT:
raise ValueError(err.details())
elif status == StatusCode.NOT_FOUND:
raise LookupError(err.details())
elif status == StatusCode.ALREADY_EXISTS:
raise FileExistsError(err.details())
else:
raise RuntimeError(err.details())
def _handle_grpc_error(
self,
cancel_handler: CancelHandler,
retry_handler: Optional[RetryHandler],
grpc_error: grpc.RpcError,
on_success: Callable[[X509Context], None],
on_error: Callable[[Exception], None],
):
grpc_error_code = grpc_error.code()
if retry_handler and grpc_error_code not in _NON_RETRYABLE_CODES:
_logger.error('Error connecting to the Workload API: {}'.format(
str(grpc_error_code)))
retry_handler.do_retry(
self._call_watch_x509_context,
[cancel_handler, retry_handler, on_success, on_error],
)
else:
# don't retry, instead report error to user on the on_error callback
error = FetchX509SvidError(str(grpc_error_code))
on_error(error)
def _propagate_grpc_code_err(context: grpc.ServicerContext,
err: grpc.RpcError):
logging.error(traceback.format_exc())
context.set_code(err.code())
context.set_details(str(err))
def from_exception(e: grpc.RpcError):
code, msg = e.code().value
ctor = MAP.get(code, ClientError)
return ctor(code, str(e))
def show_grpc_exception(self, title: str, e: grpc.RpcError) -> None:
logging.exception("app grpc exception", exc_info=e)
message = e.details()
self.show_error(title, message)
def bblfsh_parse_raise_rpc_error(*args, **kwargs):
raise RpcError()
def bblfsh_parse_raise_rpc_error(*args, **kwargs):
from grpc import RpcError
raise RpcError()
