def check_for_non_existant_openapi_endpoints(self) -> None:
""" Here, we check to see if every endpoint documented in the openapi
documentation actually exists in urls.py and thus in actual code.
Note: We define this as a helper called at the end of
test_openapi_arguments instead of as a separate test to ensure that
this test is only executed after test_openapi_arguments so that it's
results can be used here in the set operations. """
openapi_paths = set(get_openapi_paths())
undocumented_paths = openapi_paths - self.checked_endpoints
undocumented_paths -= self.buggy_documentation_endpoints
undocumented_paths -= self.pending_endpoints
try:
self.assertEqual(len(undocumented_paths), 0)
except AssertionError: # nocoverage
msg = "The following endpoints have been documented but can't be found in urls.py:"
for undocumented_path in undocumented_paths:
msg += "\n + {}".format(undocumented_path)
raise AssertionError(msg)
def test_openapi_arguments(self) -> None:
"""This end-to-end API documentation test compares the arguments
defined in the actual code using @has_request_variables and
REQ(), with the arguments declared in our API documentation
for every API endpoint in Zulip.
First, we import the fancy-Django version of zproject/urls.py
by doing this, each has_request_variables wrapper around each
imported view function gets called to generate the wrapped
view function and thus filling the global arguments_map variable.
Basically, we're exploiting code execution during import.
Then we need to import some view modules not already imported in
urls.py. We use this different syntax because of the linters complaining
of an unused import (which is correct, but we do this for triggering the
has_request_variables decorator).
At the end, we perform a reverse mapping test that verifies that
every url pattern defined in the openapi documentation actually exists
in code.
"""
urlconf = __import__(getattr(settings, "ROOT_URLCONF"), {}, {}, [''])
__import__('zerver.views.typing')
__import__('zerver.views.events_register')
__import__('zerver.views.realm_emoji')
# We loop through all the API patterns, looking in particular
# for those using the rest_dispatch decorator; we then parse
# its mapping of (HTTP_METHOD -> FUNCTION).
for p in urlconf.v1_api_and_json_patterns:
if p.lookup_str != 'zerver.lib.rest.rest_dispatch':
continue
for method, value in p.default_args.items():
if isinstance(value, str):
function = value
tags = set() # type: Set[str]
else:
function, tags = value
# Our accounting logic in the `has_request_variables()`
# code means we have the list of all arguments
# accepted by every view function in arguments_map.
#
# TODO: Probably with a bit more work, we could get
# the types, too; `check_int` -> `int`, etc., and
# verify those too!
accepted_arguments = set(arguments_map[function])
# Convert regular expressions style URL patterns to their
# corresponding OpenAPI style formats.
# E.G.
# /messages/{message_id} <-> r'^messages/(?P<message_id>[0-9]+)$'
# /events <-> r'^events$'
regex_pattern = p.regex.pattern
self.assertTrue(regex_pattern.startswith("^"))
self.assertTrue(regex_pattern.endswith("$"))
url_pattern = '/' + regex_pattern[1:][:-1]
# Deal with the conversion of named capturing groups:
# Two possible ways to denote variables in urls exist.
# {var_name} and <var_name>. So we need to consider both.
url_patterns = [
re.sub(r"\(\?P<(\w+)>[^/]+\)", r"<\1>", url_pattern),
re.sub(r"\(\?P<(\w+)>[^/]+\)", r"{\1}", url_pattern)
]
if any([
url_patterns[0] in self.pending_endpoints,
url_patterns[1] in self.pending_endpoints
]):
continue
if "intentionally_undocumented" in tags:
error = AssertionError(
"We found some OpenAPI \
documentation for %s %s, so maybe we shouldn't mark it as intentionally \
undocumented in the urls." %
(method, url_patterns[0] + " or " + url_patterns[1]))
try:
get_openapi_parameters(url_patterns[0], method)
raise error # nocoverage
except KeyError:
pass
try:
get_openapi_parameters(url_patterns[1], method)
raise error # nocoverage
except KeyError:
pass
continue # nocoverage # although, this *is* covered.
try:
openapi_parameters = get_openapi_parameters(
url_patterns[0], method)
except Exception: # nocoverage
try:
openapi_parameters = get_openapi_parameters(
url_patterns[1], method)
except Exception: # nocoverage
raise AssertionError(
"Could not find OpenAPI docs for %s %s" %
(method,
url_patterns[0] + " or " + url_patterns[1]))
# We now have everything we need to understand the
# function as defined in our urls.py:
#
# * method is the HTTP method, e.g. GET, POST, or PATCH
#
# * p.regex.pattern is the URL pattern; might require
# some processing to match with OpenAPI rules
#
# * accepted_arguments is the full set of arguments
# this method accepts (from the REQ declarations in
# code).
#
# * The documented parameters for the endpoint as recorded in our
# OpenAPI data in zerver/openapi/zulip.yaml.
#
# We now compare these to confirm that the documented
# argument list matches what actually appears in the
# codebase.
openapi_parameter_names = set(
[parameter['name'] for parameter in openapi_parameters])
if len(openapi_parameter_names - accepted_arguments) > 0:
print("Undocumented parameters for",
url_patterns[0] + " or " + url_patterns[1], method,
function)
print(" +", openapi_parameter_names)
print(" -", accepted_arguments)
assert (any([
url_patterns[0] in self.buggy_documentation_endpoints,
url_patterns[1] in self.buggy_documentation_endpoints
]))
elif len(accepted_arguments - openapi_parameter_names) > 0:
print("Documented invalid parameters for",
url_patterns[0] + " or " + url_patterns[1], method,
function)
print(" -", openapi_parameter_names)
print(" +", accepted_arguments)
assert (any([
url_patterns[0] in self.buggy_documentation_endpoints,
url_patterns[1] in self.buggy_documentation_endpoints
]))
else:
self.assertEqual(openapi_parameter_names,
accepted_arguments)
self.checked_endpoints.add(url_patterns[0])
self.checked_endpoints.add(url_patterns[1])
openapi_paths = set(get_openapi_paths())
undocumented_paths = openapi_paths - self.checked_endpoints
undocumented_paths -= self.buggy_documentation_endpoints
undocumented_paths -= self.pending_endpoints
try:
self.assertEqual(len(undocumented_paths), 0)
except AssertionError: # nocoverage
msg = "The following endpoints have been documented but can't be found in urls.py:"
for undocumented_path in undocumented_paths:
msg += "\n + {}".format(undocumented_path)
raise AssertionError(msg)