def get_context_data(self, **kwargs: Any) -> Dict[str, Any]:
article = kwargs["article"]
context: Dict[str, Any] = super().get_context_data()
(context["article"], http_status_ignored) = self.get_path(article)
# For disabling the "Back to home" on the homepage
context["not_index_page"] = not context["article"].endswith(
"/index.md")
if self.path_template == "/zerver/help/%s.md":
context["page_is_help_center"] = True
context["doc_root"] = "/help/"
(sidebar_index,
http_status_ignored) = self.get_path("include/sidebar_index")
title_base = "Zulip Help Center"
else:
context["page_is_api_center"] = True
context["doc_root"] = "/api/"
(sidebar_index,
http_status_ignored) = self.get_path("sidebar_index")
title_base = "Zulip API documentation"
# The following is a somewhat hacky approach to extract titles from articles.
# Hack: `context["article"] has a leading `/`, so we use + to add directories.
article_path = os.path.join(settings.DEPLOY_ROOT,
"templates") + context["article"]
if os.path.exists(article_path):
with open(article_path) as article_file:
first_line = article_file.readlines()[0]
# Strip the header and then use the first line to get the article title
if self.path_template == "/zerver/api/%s.md" and first_line[
0] != "#":
api_operation = context["OPEN_GRAPH_URL"].split(
"/api/")[1].replace("-", "_")
endpoint_path, endpoint_method = get_endpoint_from_operationid(
api_operation)
article_title = get_openapi_summary(endpoint_path,
endpoint_method)
else:
article_title = first_line.lstrip("#").strip()
if context["not_index_page"]:
context["OPEN_GRAPH_TITLE"] = f"{article_title} ({title_base})"
else:
context["OPEN_GRAPH_TITLE"] = title_base
self.request.placeholder_open_graph_description = (
f"REPLACMENT_OPEN_GRAPH_DESCRIPTION_{int(2**24 * random.random())}"
)
context[
"OPEN_GRAPH_DESCRIPTION"] = self.request.placeholder_open_graph_description
context["sidebar_index"] = sidebar_index
# An "article" might require the api_uri_context to be rendered
api_uri_context: Dict[str, Any] = {}
add_api_uri_context(api_uri_context, self.request)
api_uri_context["run_content_validators"] = True
context["api_uri_context"] = api_uri_context
add_google_analytics_context(context)
return context
def get_path(self, article: str) -> DocumentationArticle:
http_status = 200
if article == "":
article = "index"
elif article == "include/sidebar_index":
pass
elif "/" in article:
article = "missing"
http_status = 404
elif len(article) > 100 or not re.match("^[0-9a-zA-Z_-]+$", article):
article = "missing"
http_status = 404
path = self.path_template % (article, )
endpoint_name = None
endpoint_method = None
# The following is a somewhat hacky approach to extract titles from articles.
# Hack: `context["article"] has a leading `/`, so we use + to add directories.
article_path = os.path.join(settings.DEPLOY_ROOT, "templates") + path
if (not os.path.exists(article_path)
) and self.path_template == "/zerver/api/%s.md":
endpoint_path = article.replace("-", "_")
try:
endpoint_name, endpoint_method = get_endpoint_from_operationid(
endpoint_path)
path = "/zerver/api/api-doc-template.md"
except AssertionError:
return DocumentationArticle(
article_path=self.path_template % ("missing", ),
article_http_status=404,
endpoint_path=None,
endpoint_method=None,
)
try:
loader.get_template(path)
return DocumentationArticle(
article_path=path,
article_http_status=http_status,
endpoint_path=endpoint_name,
endpoint_method=endpoint_method,
)
except loader.TemplateDoesNotExist:
return DocumentationArticle(
article_path=self.path_template % ("missing", ),
article_http_status=404,
endpoint_path=None,
endpoint_method=None,
)
def test_generated_curl_examples_for_success(client: Client) -> None:
default_authentication_line = f"{client.email}:{client.api_key}"
# A limited Markdown engine that just processes the code example syntax.
realm = get_realm("zulip")
md_engine = markdown.Markdown(extensions=[
markdown_extension.makeExtension(api_url=realm.uri + "/api")
])
# We run our curl tests in alphabetical order (except that we
# delay the deactivate-user test to the very end), since we depend
# on "add" tests coming before "remove" tests in some cases. We
# should try to either avoid ordering dependencies or make them
# very explicit.
rest_endpoints_path = os.path.join(
settings.DEPLOY_ROOT,
"templates/zerver/help/include/rest-endpoints.md")
rest_endpoints_raw = open(rest_endpoints_path, "r").read()
ENDPOINT_REGEXP = re.compile(r"/api/\s*(.*?)\)")
endpoint_list = sorted(set(re.findall(ENDPOINT_REGEXP,
rest_endpoints_raw)))
for endpoint in endpoint_list:
article_name = endpoint + ".md"
file_name = os.path.join(settings.DEPLOY_ROOT, "templates/zerver/api/",
article_name)
curl_commands_to_test = []
if os.path.exists(file_name):
f = open(file_name, "r")
for line in f:
# A typical example from the Markdown source looks like this:
# {generate_code_example(curl, ...}
if line.startswith("{generate_code_example(curl"):
curl_commands_to_test.append(line)
else:
# If the file doesn't exist, then it has been
# deleted and its page is generated by the
# template. Thus, the curl example would just
# a single one following the template's pattern.
endpoint_path, endpoint_method = get_endpoint_from_operationid(
endpoint.replace("-", "_"))
endpoint_string = endpoint_path + ":" + endpoint_method
command = f"{{generate_code_example(curl)|{endpoint_string}|example}}"
curl_commands_to_test.append(command)
for line in curl_commands_to_test:
# To do an end-to-end test on the documentation examples
# that will be actually shown to users, we use the
# Markdown rendering pipeline to compute the user-facing
# example, and then run that to test it.
# Set AUTHENTICATION_LINE to default_authentication_line.
# Set this every iteration, because deactivate_own_user
# will override this for its test.
AUTHENTICATION_LINE[0] = default_authentication_line
curl_command_html = md_engine.convert(line.strip())
unescaped_html = html.unescape(curl_command_html)
curl_regex = re.compile(r"<code>curl\n(.*?)</code>", re.DOTALL)
commands = re.findall(curl_regex, unescaped_html)
for curl_command_text in commands:
curl_command_text = curl_command_text.replace(
"BOT_EMAIL_ADDRESS:BOT_API_KEY", AUTHENTICATION_LINE[0])
print("Testing {} ...".format(
curl_command_text.split("\n")[0]))
# Turn the text into an arguments list.
generated_curl_command = [
x for x in shlex.split(curl_command_text) if x != "\n"
]
response_json = None
response = None
try:
# We split this across two lines so if curl fails and
# returns non-JSON output, we'll still print it.
response_json = subprocess.check_output(
generated_curl_command, universal_newlines=True)
response = json.loads(response_json)
assert response["result"] == "success"
except (AssertionError, Exception):
error_template = """
Error verifying the success of the API documentation curl example.
File: {file_name}
Line: {line}
Curl command:
{curl_command}
Response:
{response}
This test is designed to check each generate_code_example(curl) instance in the
API documentation for success. If this fails then it means that the curl example
that was generated was faulty and when tried, it resulted in an unsuccessful
response.
Common reasons for why this could occur:
1. One or more example values in zerver/openapi/zulip.yaml for this endpoint
do not line up with the values in the test database.
2. One or more mandatory parameters were included in the "exclude" list.
To learn more about the test itself, see zerver/openapi/test_curl_examples.py.
"""
print(
error_template.format(
file_name=file_name,
line=line,
curl_command=generated_curl_command,
response=response_json if response is None else
json.dumps(response, indent=4),
))
raise
assert_all_helper_functions_called()
def get_path(self, article: str) -> DocumentationArticle:
http_status = 200
if article == "":
article = "index"
elif article == "include/sidebar_index":
pass
elif "/" in article:
article = "missing"
http_status = 404
elif len(article) > 100 or not re.match("^[0-9a-zA-Z_-]+$", article):
article = "missing"
http_status = 404
path = self.path_template % (article,)
endpoint_name = None
endpoint_method = None
if self.policies_view and self.path_template.startswith("/"):
# This block is required because neither the Django
# template loader nor the article_path logic below support
# settings.POLICIES_DIRECTORY being an absolute path.
if not os.path.exists(path):
article = "missing"
http_status = 404
path = self.path_template % (article,)
return DocumentationArticle(
article_path=path,
article_http_status=http_status,
endpoint_path=None,
endpoint_method=None,
)
# The following is a somewhat hacky approach to extract titles from articles.
# Hack: `context["article"] has a leading `/`, so we use + to add directories.
article_path = os.path.join(settings.DEPLOY_ROOT, "templates") + path
if (not os.path.exists(article_path)) and self.path_template == "/zerver/api/%s.md":
try:
endpoint_name, endpoint_method = get_endpoint_from_operationid(article)
path = "/zerver/api/api-doc-template.md"
except AssertionError:
return DocumentationArticle(
article_path=self.path_template % ("missing",),
article_http_status=404,
endpoint_path=None,
endpoint_method=None,
)
try:
loader.get_template(path)
return DocumentationArticle(
article_path=path,
article_http_status=http_status,
endpoint_path=endpoint_name,
endpoint_method=endpoint_method,
)
except loader.TemplateDoesNotExist:
return DocumentationArticle(
article_path=self.path_template % ("missing",),
article_http_status=404,
endpoint_path=None,
endpoint_method=None,
)
def get_context_data(self, **kwargs: Any) -> Dict[str, Any]:
article = kwargs["article"]
context: Dict[str, Any] = super().get_context_data()
documentation_article = self.get_path(article)
context["article"] = documentation_article.article_path
if documentation_article.article_path.startswith("/") and os.path.exists(
documentation_article.article_path
):
# Absolute path case
article_path = documentation_article.article_path
elif documentation_article.article_path.startswith("/"):
# Hack: `context["article"] has a leading `/`, so we use + to add directories.
article_path = (
os.path.join(settings.DEPLOY_ROOT, "templates") + documentation_article.article_path
)
else:
article_path = os.path.join(
settings.DEPLOY_ROOT, "templates", documentation_article.article_path
)
# For disabling the "Back to home" on the homepage
context["not_index_page"] = not context["article"].endswith("/index.md")
if self.path_template == "/zerver/help/%s.md":
context["page_is_help_center"] = True
context["doc_root"] = "/help/"
context["doc_root_title"] = "Help center"
sidebar_article = self.get_path("include/sidebar_index")
sidebar_index = sidebar_article.article_path
title_base = "Zulip Help Center"
elif self.path_template == f"{settings.POLICIES_DIRECTORY}/%s.md":
context["page_is_policy_center"] = True
context["doc_root"] = "/policies/"
context["doc_root_title"] = "Terms and policies"
sidebar_article = self.get_path("sidebar_index")
sidebar_index = sidebar_article.article_path
title_base = "Zulip terms and policies"
else:
context["page_is_api_center"] = True
context["doc_root"] = "/api/"
context["doc_root_title"] = "API documentation"
sidebar_article = self.get_path("sidebar_index")
sidebar_index = sidebar_article.article_path
title_base = "Zulip API documentation"
# The following is a somewhat hacky approach to extract titles from articles.
endpoint_name = None
endpoint_method = None
if os.path.exists(article_path):
with open(article_path) as article_file:
first_line = article_file.readlines()[0]
# Strip the header and then use the first line to get the article title
if context["article"] == "/zerver/api/api-doc-template.md":
endpoint_name, endpoint_method = (
documentation_article.endpoint_path,
documentation_article.endpoint_method,
)
assert endpoint_name is not None
assert endpoint_method is not None
article_title = get_openapi_summary(endpoint_name, endpoint_method)
elif self.path_template == "/zerver/api/%s.md" and "{generate_api_title(" in first_line:
api_operation = context["OPEN_GRAPH_URL"].split("/api/")[1]
endpoint_name, endpoint_method = get_endpoint_from_operationid(api_operation)
article_title = get_openapi_summary(endpoint_name, endpoint_method)
else:
article_title = first_line.lstrip("#").strip()
endpoint_name = endpoint_method = None
if context["not_index_page"]:
context["OPEN_GRAPH_TITLE"] = f"{article_title} ({title_base})"
else:
context["OPEN_GRAPH_TITLE"] = title_base
request_notes = RequestNotes.get_notes(self.request)
request_notes.placeholder_open_graph_description = (
f"REPLACMENT_OPEN_GRAPH_DESCRIPTION_{int(2**24 * random.random())}"
)
context["OPEN_GRAPH_DESCRIPTION"] = request_notes.placeholder_open_graph_description
context["sidebar_index"] = sidebar_index
# An "article" might require the api_uri_context to be rendered
api_uri_context: Dict[str, Any] = {}
add_api_uri_context(api_uri_context, self.request)
api_uri_context["run_content_validators"] = True
context["api_uri_context"] = api_uri_context
if endpoint_name and endpoint_method:
context["api_uri_context"]["API_ENDPOINT_NAME"] = endpoint_name + ":" + endpoint_method
add_google_analytics_context(context)
return context