def get_context_data(self, **kwargs: Any) -> Dict[str, Any]: article = kwargs["article"] context: Dict[str, Any] = super().get_context_data() (context["article"], http_status_ignored) = self.get_path(article) # For disabling the "Back to home" on the homepage context["not_index_page"] = not context["article"].endswith( "/index.md") if self.path_template == "/zerver/help/%s.md": context["page_is_help_center"] = True context["doc_root"] = "/help/" (sidebar_index, http_status_ignored) = self.get_path("include/sidebar_index") title_base = "Zulip Help Center" else: context["page_is_api_center"] = True context["doc_root"] = "/api/" (sidebar_index, http_status_ignored) = self.get_path("sidebar_index") title_base = "Zulip API documentation" # The following is a somewhat hacky approach to extract titles from articles. # Hack: `context["article"] has a leading `/`, so we use + to add directories. article_path = os.path.join(settings.DEPLOY_ROOT, "templates") + context["article"] if os.path.exists(article_path): with open(article_path) as article_file: first_line = article_file.readlines()[0] # Strip the header and then use the first line to get the article title if self.path_template == "/zerver/api/%s.md" and first_line[ 0] != "#": api_operation = context["OPEN_GRAPH_URL"].split( "/api/")[1].replace("-", "_") endpoint_path, endpoint_method = get_endpoint_from_operationid( api_operation) article_title = get_openapi_summary(endpoint_path, endpoint_method) else: article_title = first_line.lstrip("#").strip() if context["not_index_page"]: context["OPEN_GRAPH_TITLE"] = f"{article_title} ({title_base})" else: context["OPEN_GRAPH_TITLE"] = title_base self.request.placeholder_open_graph_description = ( f"REPLACMENT_OPEN_GRAPH_DESCRIPTION_{int(2**24 * random.random())}" ) context[ "OPEN_GRAPH_DESCRIPTION"] = self.request.placeholder_open_graph_description context["sidebar_index"] = sidebar_index # An "article" might require the api_uri_context to be rendered api_uri_context: Dict[str, Any] = {} add_api_uri_context(api_uri_context, self.request) api_uri_context["run_content_validators"] = True context["api_uri_context"] = api_uri_context add_google_analytics_context(context) return context
def get_path(self, article: str) -> DocumentationArticle: http_status = 200 if article == "": article = "index" elif article == "include/sidebar_index": pass elif "/" in article: article = "missing" http_status = 404 elif len(article) > 100 or not re.match("^[0-9a-zA-Z_-]+$", article): article = "missing" http_status = 404 path = self.path_template % (article, ) endpoint_name = None endpoint_method = None # The following is a somewhat hacky approach to extract titles from articles. # Hack: `context["article"] has a leading `/`, so we use + to add directories. article_path = os.path.join(settings.DEPLOY_ROOT, "templates") + path if (not os.path.exists(article_path) ) and self.path_template == "/zerver/api/%s.md": endpoint_path = article.replace("-", "_") try: endpoint_name, endpoint_method = get_endpoint_from_operationid( endpoint_path) path = "/zerver/api/api-doc-template.md" except AssertionError: return DocumentationArticle( article_path=self.path_template % ("missing", ), article_http_status=404, endpoint_path=None, endpoint_method=None, ) try: loader.get_template(path) return DocumentationArticle( article_path=path, article_http_status=http_status, endpoint_path=endpoint_name, endpoint_method=endpoint_method, ) except loader.TemplateDoesNotExist: return DocumentationArticle( article_path=self.path_template % ("missing", ), article_http_status=404, endpoint_path=None, endpoint_method=None, )
def test_generated_curl_examples_for_success(client: Client) -> None: default_authentication_line = f"{client.email}:{client.api_key}" # A limited Markdown engine that just processes the code example syntax. realm = get_realm("zulip") md_engine = markdown.Markdown(extensions=[ markdown_extension.makeExtension(api_url=realm.uri + "/api") ]) # We run our curl tests in alphabetical order (except that we # delay the deactivate-user test to the very end), since we depend # on "add" tests coming before "remove" tests in some cases. We # should try to either avoid ordering dependencies or make them # very explicit. rest_endpoints_path = os.path.join( settings.DEPLOY_ROOT, "templates/zerver/help/include/rest-endpoints.md") rest_endpoints_raw = open(rest_endpoints_path, "r").read() ENDPOINT_REGEXP = re.compile(r"/api/\s*(.*?)\)") endpoint_list = sorted(set(re.findall(ENDPOINT_REGEXP, rest_endpoints_raw))) for endpoint in endpoint_list: article_name = endpoint + ".md" file_name = os.path.join(settings.DEPLOY_ROOT, "templates/zerver/api/", article_name) curl_commands_to_test = [] if os.path.exists(file_name): f = open(file_name, "r") for line in f: # A typical example from the Markdown source looks like this: # {generate_code_example(curl, ...} if line.startswith("{generate_code_example(curl"): curl_commands_to_test.append(line) else: # If the file doesn't exist, then it has been # deleted and its page is generated by the # template. Thus, the curl example would just # a single one following the template's pattern. endpoint_path, endpoint_method = get_endpoint_from_operationid( endpoint.replace("-", "_")) endpoint_string = endpoint_path + ":" + endpoint_method command = f"{{generate_code_example(curl)|{endpoint_string}|example}}" curl_commands_to_test.append(command) for line in curl_commands_to_test: # To do an end-to-end test on the documentation examples # that will be actually shown to users, we use the # Markdown rendering pipeline to compute the user-facing # example, and then run that to test it. # Set AUTHENTICATION_LINE to default_authentication_line. # Set this every iteration, because deactivate_own_user # will override this for its test. AUTHENTICATION_LINE[0] = default_authentication_line curl_command_html = md_engine.convert(line.strip()) unescaped_html = html.unescape(curl_command_html) curl_regex = re.compile(r"<code>curl\n(.*?)</code>", re.DOTALL) commands = re.findall(curl_regex, unescaped_html) for curl_command_text in commands: curl_command_text = curl_command_text.replace( "BOT_EMAIL_ADDRESS:BOT_API_KEY", AUTHENTICATION_LINE[0]) print("Testing {} ...".format( curl_command_text.split("\n")[0])) # Turn the text into an arguments list. generated_curl_command = [ x for x in shlex.split(curl_command_text) if x != "\n" ] response_json = None response = None try: # We split this across two lines so if curl fails and # returns non-JSON output, we'll still print it. response_json = subprocess.check_output( generated_curl_command, universal_newlines=True) response = json.loads(response_json) assert response["result"] == "success" except (AssertionError, Exception): error_template = """ Error verifying the success of the API documentation curl example. File: {file_name} Line: {line} Curl command: {curl_command} Response: {response} This test is designed to check each generate_code_example(curl) instance in the API documentation for success. If this fails then it means that the curl example that was generated was faulty and when tried, it resulted in an unsuccessful response. Common reasons for why this could occur: 1. One or more example values in zerver/openapi/zulip.yaml for this endpoint do not line up with the values in the test database. 2. One or more mandatory parameters were included in the "exclude" list. To learn more about the test itself, see zerver/openapi/test_curl_examples.py. """ print( error_template.format( file_name=file_name, line=line, curl_command=generated_curl_command, response=response_json if response is None else json.dumps(response, indent=4), )) raise assert_all_helper_functions_called()
def get_path(self, article: str) -> DocumentationArticle: http_status = 200 if article == "": article = "index" elif article == "include/sidebar_index": pass elif "/" in article: article = "missing" http_status = 404 elif len(article) > 100 or not re.match("^[0-9a-zA-Z_-]+$", article): article = "missing" http_status = 404 path = self.path_template % (article,) endpoint_name = None endpoint_method = None if self.policies_view and self.path_template.startswith("/"): # This block is required because neither the Django # template loader nor the article_path logic below support # settings.POLICIES_DIRECTORY being an absolute path. if not os.path.exists(path): article = "missing" http_status = 404 path = self.path_template % (article,) return DocumentationArticle( article_path=path, article_http_status=http_status, endpoint_path=None, endpoint_method=None, ) # The following is a somewhat hacky approach to extract titles from articles. # Hack: `context["article"] has a leading `/`, so we use + to add directories. article_path = os.path.join(settings.DEPLOY_ROOT, "templates") + path if (not os.path.exists(article_path)) and self.path_template == "/zerver/api/%s.md": try: endpoint_name, endpoint_method = get_endpoint_from_operationid(article) path = "/zerver/api/api-doc-template.md" except AssertionError: return DocumentationArticle( article_path=self.path_template % ("missing",), article_http_status=404, endpoint_path=None, endpoint_method=None, ) try: loader.get_template(path) return DocumentationArticle( article_path=path, article_http_status=http_status, endpoint_path=endpoint_name, endpoint_method=endpoint_method, ) except loader.TemplateDoesNotExist: return DocumentationArticle( article_path=self.path_template % ("missing",), article_http_status=404, endpoint_path=None, endpoint_method=None, )
def get_context_data(self, **kwargs: Any) -> Dict[str, Any]: article = kwargs["article"] context: Dict[str, Any] = super().get_context_data() documentation_article = self.get_path(article) context["article"] = documentation_article.article_path if documentation_article.article_path.startswith("/") and os.path.exists( documentation_article.article_path ): # Absolute path case article_path = documentation_article.article_path elif documentation_article.article_path.startswith("/"): # Hack: `context["article"] has a leading `/`, so we use + to add directories. article_path = ( os.path.join(settings.DEPLOY_ROOT, "templates") + documentation_article.article_path ) else: article_path = os.path.join( settings.DEPLOY_ROOT, "templates", documentation_article.article_path ) # For disabling the "Back to home" on the homepage context["not_index_page"] = not context["article"].endswith("/index.md") if self.path_template == "/zerver/help/%s.md": context["page_is_help_center"] = True context["doc_root"] = "/help/" context["doc_root_title"] = "Help center" sidebar_article = self.get_path("include/sidebar_index") sidebar_index = sidebar_article.article_path title_base = "Zulip Help Center" elif self.path_template == f"{settings.POLICIES_DIRECTORY}/%s.md": context["page_is_policy_center"] = True context["doc_root"] = "/policies/" context["doc_root_title"] = "Terms and policies" sidebar_article = self.get_path("sidebar_index") sidebar_index = sidebar_article.article_path title_base = "Zulip terms and policies" else: context["page_is_api_center"] = True context["doc_root"] = "/api/" context["doc_root_title"] = "API documentation" sidebar_article = self.get_path("sidebar_index") sidebar_index = sidebar_article.article_path title_base = "Zulip API documentation" # The following is a somewhat hacky approach to extract titles from articles. endpoint_name = None endpoint_method = None if os.path.exists(article_path): with open(article_path) as article_file: first_line = article_file.readlines()[0] # Strip the header and then use the first line to get the article title if context["article"] == "/zerver/api/api-doc-template.md": endpoint_name, endpoint_method = ( documentation_article.endpoint_path, documentation_article.endpoint_method, ) assert endpoint_name is not None assert endpoint_method is not None article_title = get_openapi_summary(endpoint_name, endpoint_method) elif self.path_template == "/zerver/api/%s.md" and "{generate_api_title(" in first_line: api_operation = context["OPEN_GRAPH_URL"].split("/api/")[1] endpoint_name, endpoint_method = get_endpoint_from_operationid(api_operation) article_title = get_openapi_summary(endpoint_name, endpoint_method) else: article_title = first_line.lstrip("#").strip() endpoint_name = endpoint_method = None if context["not_index_page"]: context["OPEN_GRAPH_TITLE"] = f"{article_title} ({title_base})" else: context["OPEN_GRAPH_TITLE"] = title_base request_notes = RequestNotes.get_notes(self.request) request_notes.placeholder_open_graph_description = ( f"REPLACMENT_OPEN_GRAPH_DESCRIPTION_{int(2**24 * random.random())}" ) context["OPEN_GRAPH_DESCRIPTION"] = request_notes.placeholder_open_graph_description context["sidebar_index"] = sidebar_index # An "article" might require the api_uri_context to be rendered api_uri_context: Dict[str, Any] = {} add_api_uri_context(api_uri_context, self.request) api_uri_context["run_content_validators"] = True context["api_uri_context"] = api_uri_context if endpoint_name and endpoint_method: context["api_uri_context"]["API_ENDPOINT_NAME"] = endpoint_name + ":" + endpoint_method add_google_analytics_context(context) return context