def validate_against_openapi_schema(
content: Dict[str, Any],
path: str,
method: str,
status_code: str,
display_brief_error: bool = False,
) -> bool:
"""Compare a "content" dict with the defined schema for a specific method
in an endpoint. Return true if validated and false if skipped.
"""
# This first set of checks are primarily training wheels that we
# hope to eliminate over time as we improve our API documentation.
# No 500 responses have been documented, so skip them
if status_code.startswith("5"):
return False
if path not in openapi_spec.openapi()["paths"].keys():
endpoint = find_openapi_endpoint(path)
# If it doesn't match it hasn't been documented yet.
if endpoint is None:
return False
else:
endpoint = path
# Excluded endpoint/methods
if (endpoint, method) in EXCLUDE_UNDOCUMENTED_ENDPOINTS:
return False
# Return true for endpoints with only response documentation remaining
if (endpoint, method) in EXCLUDE_DOCUMENTED_ENDPOINTS:
return True
# Check if the response matches its code
if status_code.startswith("2") and (content.get(
"result", "success").lower() != "success"):
raise SchemaError(
"Response is not 200 but is validating against 200 schema")
# Code is not declared but appears in various 400 responses. If
# common, it can be added to 400 response schema
if status_code.startswith("4"):
# This return statement should ideally be not here. But since
# we have not defined 400 responses for various paths this has
# been added as all 400 have the same schema. When all 400
# response have been defined this should be removed.
return True
# The actual work of validating that the response matches the
# schema is done via the third-party OAS30Validator.
schema = get_schema(endpoint, method, status_code)
if endpoint == "/events" and method == "get":
# This a temporary function for checking only documented events
# as all events haven't been documented yet.
# TODO: Remove this after all events have been documented.
fix_events(content)
validator = OAS30Validator(schema)
try:
validator.validate(content)
except JsonSchemaValidationError as error:
if not display_brief_error:
raise error
# display_brief_error is designed to avoid printing 1000 lines
# of output when the schema to validate is extremely large
# (E.g. the several dozen format variants for individual
# events returned by GET /events) and instead just display the
# specific variant we expect to match the response.
brief_error_display_schema = {"nullable": False, "oneOf": list()}
brief_error_display_schema_oneOf = []
brief_error_validator_value = []
for validator_value in error.validator_value:
if validator_value["example"]["type"] == error.instance["type"]:
brief_error_validator_value.append(validator_value)
for i_schema in error.schema["oneOf"]:
if i_schema["example"]["type"] == error.instance["type"]:
brief_error_display_schema_oneOf.append(i_schema)
brief_error_display_schema["oneOf"] = brief_error_display_schema_oneOf
# Field list from https://python-jsonschema.readthedocs.io/en/stable/errors/
raise JsonSchemaValidationError(
message=error.message,
validator=error.validator,
path=error.path,
instance=error.instance,
schema_path=error.schema_path,
schema=brief_error_display_schema,
validator_value=brief_error_validator_value,
cause=error.cause,
)
return True
def validate_against_openapi_schema(
content: Dict[str, Any],
path: str,
method: str,
status_code: str,
display_brief_error: bool = False,
) -> bool:
"""Compare a "content" dict with the defined schema for a specific method
in an endpoint. Return true if validated and false if skipped.
"""
# This first set of checks are primarily training wheels that we
# hope to eliminate over time as we improve our API documentation.
# No 500 responses have been documented, so skip them
if status_code.startswith("5"):
return False
if path not in openapi_spec.openapi()["paths"].keys():
endpoint = find_openapi_endpoint(path)
# If it doesn't match it hasn't been documented yet.
if endpoint is None:
return False
else:
endpoint = path
# Excluded endpoint/methods
if (endpoint, method) in EXCLUDE_UNDOCUMENTED_ENDPOINTS:
return False
# Return true for endpoints with only response documentation remaining
if (endpoint, method) in EXCLUDE_DOCUMENTED_ENDPOINTS:
return True
# Check if the response matches its code
if status_code.startswith("2") and (content.get(
"result", "success").lower() != "success"):
raise SchemaError(
"Response is not 200 but is validating against 200 schema")
# Code is not declared but appears in various 400 responses. If
# common, it can be added to 400 response schema
if status_code.startswith("4"):
# This return statement should ideally be not here. But since
# we have not defined 400 responses for various paths this has
# been added as all 400 have the same schema. When all 400
# response have been defined this should be removed.
return True
if endpoint == "/events" and method == "get":
# This a temporary function for checking only documented events
# as all events haven't been documented yet.
# TODO: Remove this after all events have been documented.
fix_events(content)
mock_request = MockRequest("http://localhost:9991/", method,
"/api/v1" + path)
mock_response = MockResponse(
# TODO: Use original response content instead of re-serializing it.
orjson.dumps(content),
status_code=status_code,
)
result = openapi_spec.response_validator().validate(
mock_request, mock_response)
try:
result.raise_for_errors()
except InvalidSchemaValue as isv:
message = f"{len(isv.schema_errors)} response validation error(s) at {method} /api/v1{path} ({status_code}):"
for error in isv.schema_errors:
if display_brief_error:
# display_brief_error is designed to avoid printing 1000 lines
# of output when the schema to validate is extremely large
# (E.g. the several dozen format variants for individual
# events returned by GET /events) and instead just display the
# specific variant we expect to match the response.
brief_error_validator_value = [
validator_value
for validator_value in error.validator_value
if not prune_schema_by_type(validator_value,
error.instance["type"])
]
brief_error_display_schema = error.schema.copy()
if "oneOf" in brief_error_display_schema:
brief_error_display_schema["oneOf"] = [
i_schema for i_schema in error.schema["oneOf"]
if not prune_schema_by_type(i_schema,
error.instance["type"])
]
# Field list from https://python-jsonschema.readthedocs.io/en/stable/errors/
error = JsonSchemaValidationError(
message=error.message,
validator=error.validator,
path=error.path,
instance=error.instance,
schema_path=error.schema_path,
schema=brief_error_display_schema,
validator_value=brief_error_validator_value,
cause=error.cause,
)
message += f"\n\n{type(error).__name__}: {error}"
raise SchemaError(message) from None
return True