def test_example_lint(self): class_page_info = parser.docs_for_object( full_name='TestClass', py_object=TestClass, parser_config=self.parser_config) test_api_report = utils.ApiReport() test_api_report.fill_metrics(class_page_info) for test_report in test_api_report.api_report.symbol_metric: if (test_report.symbol_name == 'TestClass' and test_report.object_type == api_report_pb2.ObjectType.CLASS): self.assertEqual(test_report.usage_example_lint.num_doctest, 2) self.assertEqual( test_report.usage_example_lint.num_untested_examples, 1) self.assertEqual(test_report.package_group, 'TestClass') if (test_report.symbol_name == 'TestClass.method_one' and test_report.object_type == api_report_pb2.ObjectType.METHOD): self.assertEqual(test_report.usage_example_lint.num_doctest, 0) self.assertEqual( test_report.usage_example_lint.num_untested_examples, 1) self.assertEqual(test_report.package_group, 'TestClass')
def test_parameter_lint(self): class_page_info = parser.docs_for_object( full_name='TestClass', py_object=TestClass, parser_config=self.parser_config) test_api_report = utils.ApiReport() test_api_report.fill_metrics(class_page_info) for test_report in test_api_report.api_report.symbol_metric: if (test_report.symbol_name == 'TestClass' and test_report.object_type == api_report_pb2.ObjectType.CLASS): self.assertEqual( test_report.parameter_lint.num_empty_param_desc_args, 2) self.assertEqual(test_report.parameter_lint.num_args_in_doc, 4) self.assertEqual(test_report.parameter_lint.num_args_in_code, 3) self.assertEqual( test_report.parameter_lint.num_empty_param_desc_attr, 1) self.assertEqual(test_report.parameter_lint.total_attr_param, 2) if (test_report.symbol_name == 'TestClass.method_one' and test_report.object_type == api_report_pb2.ObjectType.METHOD): self.assertEqual( test_report.parameter_lint.num_empty_param_desc_args, 0) self.assertEqual(test_report.parameter_lint.num_args_in_doc, 1) self.assertEqual(test_report.parameter_lint.num_args_in_code, 1) self.assertEqual( test_report.parameter_lint.num_empty_param_desc_attr, 0) self.assertEqual(test_report.parameter_lint.total_attr_param, 0)
def test_method_return_lint(self): class_page_info = parser.docs_for_object( full_name='TestClass', py_object=TestClass, parser_config=self.parser_config) test_api_report = utils.ApiReport() test_api_report.fill_metrics(class_page_info) for test_report in test_api_report.api_report.symbol_metric: if (test_report.symbol_name == 'TestClass.method_one' and test_report.object_type == api_report_pb2.ObjectType.METHOD): self.assertTrue(test_report.return_lint.returns_defined)
def test_fill_report_doesnt_edit_page(self): page1 = self._build_page_info() page2 = self._build_page_info() test_api_report = utils.ApiReport() test_api_report.fill_metrics(page2) page1.api_node = None page1.parser_config = None page2.api_node = None page2.parser_config = None self.assertEqual(page1, page2)
def test_class_raises_lint(self): class_page_info = parser.docs_for_object( full_name='TestClass', py_object=TestClass, parser_config=self.parser_config) test_api_report = utils.ApiReport() test_api_report.fill_metrics(class_page_info) for test_report in test_api_report.api_report.symbol_metric: if (test_report.symbol_name == 'TestClass' and test_report.object_type == api_report_pb2.ObjectType.CLASS): self.assertEqual(test_report.raises_lint.num_raises_defined, 2) self.assertEqual(test_report.raises_lint.total_raises_in_code, 2)
def test_description_lint(self): class_page_info = parser.docs_for_object( full_name='TestClass', py_object=TestClass, parser_config=self.parser_config) test_api_report = utils.ApiReport() test_api_report.fill_metrics(class_page_info) for test_report in test_api_report.api_report.symbol_metric: if (test_report.symbol_name == 'TestClass' and test_report.object_type == api_report_pb2.ObjectType.CLASS): self.assertEqual(test_report.desc_lint.len_brief, 2) self.assertEqual(test_report.desc_lint.len_long_desc, 54) if (test_report.symbol_name == 'TestClass.method_one' and test_report.object_type == api_report_pb2.ObjectType.METHOD): self.assertEqual(test_report.desc_lint.len_brief, 4) self.assertEqual(test_report.desc_lint.len_long_desc, 10)
def write_docs( *, output_dir: Union[str, pathlib.Path], parser_config: parser.ParserConfig, yaml_toc: bool, root_module_name: str, root_title: str = 'TensorFlow', search_hints: bool = True, site_path: str = 'api_docs/python', gen_redirects: bool = True, table_view: bool = True, gen_report: bool = False, ): """Write previously extracted docs to disk. Write a docs page for each symbol included in the indices of parser_config to a tree of docs at `output_dir`. Symbols with multiple aliases will have only one page written about them, which is referenced for all aliases. Args: output_dir: Directory to write documentation markdown files to. Will be created if it doesn't exist. parser_config: A `parser.ParserConfig` object, containing all the necessary indices. yaml_toc: Set to `True` to generate a "_toc.yaml" file. root_module_name: (str) the name of the root module (`tf` for tensorflow). root_title: The title name for the root level index.md. search_hints: (bool) include meta-data search hints at the top of each output file. site_path: The output path relative to the site root. Used in the `_toc.yaml` and `_redirects.yaml` files. gen_redirects: Bool which decides whether to generate _redirects.yaml file or not. table_view: If True, `Args`, `Returns`, `Raises` or `Attributes` will be converted to a tabular format while generating markdown. If False, they will be converted to a markdown List view. gen_report: If True, a report for the library is generated by linting the docstrings of its public API symbols. Raises: ValueError: if `output_dir` is not an absolute path """ output_dir = pathlib.Path(output_dir) site_path = pathlib.Path('/', site_path) # Make output_dir. if not output_dir.is_absolute(): raise ValueError("'output_dir' must be an absolute path.\n" f" output_dir='{output_dir}'") output_dir.mkdir(parents=True, exist_ok=True) # These dictionaries are used for table-of-contents generation below # They will contain, after the for-loop below:: # - module name(string):classes and functions the module contains(list) module_children = {} # Collect redirects for an api _redirects.yaml file. redirects = [] if gen_report: api_report_obj = utils.ApiReport() # Parse and write Markdown pages, resolving cross-links (`tf.symbol`). for full_name in sorted(parser_config.index.keys(), key=lambda k: k.lower()): py_object = parser_config.index[full_name] if full_name in parser_config.duplicate_of: continue # Methods constants are only documented only as part of their parent's page. if parser_config.reference_resolver.is_fragment(full_name): continue # Remove the extension from the path. docpath, _ = os.path.splitext(parser.documentation_path(full_name)) # For a module, remember the module for the table-of-contents if inspect.ismodule(py_object): if full_name in parser_config.tree: mod_obj = Module(module=full_name, py_object=py_object, path=str(site_path / docpath)) module_children[full_name] = mod_obj # For something else that's documented, # figure out what module it lives in else: subname = str(full_name) while True: subname = subname[:subname.rindex('.')] if inspect.ismodule(parser_config.index[subname]): module_name = parser_config.duplicate_of.get( subname, subname) child_mod = ModuleChild(name=full_name, py_object=py_object, parent=module_name, path=str(site_path / docpath)) module_children[module_name].add_children(child_mod) break # Generate docs for `py_object`, resolving references. try: page_info = parser.docs_for_object(full_name, py_object, parser_config) except: raise ValueError( f'Failed to generate docs for symbol: `{full_name}`') if gen_report and not full_name.startswith( ('tf.compat.v', 'tf.keras.backend')): api_report_obj.fill_metrics(page_info) continue path = output_dir / parser.documentation_path(full_name) try: path.parent.mkdir(exist_ok=True, parents=True) # This function returns unicode in PY3. hidden = doc_controls.should_hide_from_search(page_info.py_object) brief_no_backticks = page_info.doc.brief.replace('`', '').strip() content = [] if brief_no_backticks: content.append(f'description: {brief_no_backticks}\n') if search_hints and not hidden: content.append(page_info.get_metadata_html()) else: content.append('robots: noindex\n') content.append(pretty_docs.build_md_page(page_info, table_view)) text = '\n'.join(content) path.write_text(text, encoding='utf-8') except OSError: raise OSError('Cannot write documentation for ' f'{full_name} to {path.parent}') duplicates = parser_config.duplicates.get(full_name, []) if not duplicates: continue duplicates = [item for item in duplicates if item != full_name] if gen_redirects: for dup in duplicates: from_path = site_path / dup.replace('.', '/') to_path = site_path / full_name.replace('.', '/') redirects.append({'from': str(from_path), 'to': str(to_path)}) if gen_report: serialized_proto = api_report_obj.api_report.SerializeToString() raw_proto = output_dir / 'api_report.pb' raw_proto.write_bytes(serialized_proto) return if yaml_toc: toc_gen = GenerateToc(module_children) toc_dict = toc_gen.generate() # Replace the overview path *only* for 'TensorFlow' to # `/api_docs/python/tf_overview`. This will be redirected to # `/api_docs/python/tf`. toc_values = toc_dict['toc'][0] if toc_values['title'] == 'tf': section = toc_values['section'][0] section['path'] = str(site_path / 'tf_overview') leftnav_toc = output_dir / root_module_name / '_toc.yaml' with open(leftnav_toc, 'w') as toc_file: yaml.dump(toc_dict, toc_file, default_flow_style=False) if redirects and gen_redirects: if yaml_toc and toc_values['title'] == 'tf': redirects.append({ 'from': str(site_path / 'tf_overview'), 'to': str(site_path / 'tf'), }) redirects_dict = { 'redirects': sorted(redirects, key=lambda redirect: redirect['from']) } api_redirects_path = output_dir / root_module_name / '_redirects.yaml' with open(api_redirects_path, 'w') as redirect_file: yaml.dump(redirects_dict, redirect_file, default_flow_style=False) # Write a global index containing all full names with links. with open(output_dir / root_module_name / 'all_symbols.md', 'w') as f: global_index = parser.generate_global_index( root_title, parser_config.index, parser_config.reference_resolver) if not search_hints: global_index = 'robots: noindex\n' + global_index f.write(global_index)
def write_docs( *, output_dir: Union[str, pathlib.Path], parser_config: config.ParserConfig, yaml_toc: Union[bool, Type[toc_lib.TocBuilder]], root_module_name: str, root_title: str = 'TensorFlow', search_hints: bool = True, site_path: str = 'api_docs/python', gen_redirects: bool = True, gen_report: bool = True, extra_docs: Optional[Dict[int, str]] = None, page_builder_classes: Optional[docs_for_object.PageBuilderDict] = None, ): """Write previously extracted docs to disk. Write a docs page for each symbol included in the indices of parser_config to a tree of docs at `output_dir`. Symbols with multiple aliases will have only one page written about them, which is referenced for all aliases. Args: output_dir: Directory to write documentation markdown files to. Will be created if it doesn't exist. parser_config: A `config.ParserConfig` object, containing all the necessary indices. yaml_toc: Set to `True` to generate a "_toc.yaml" file. root_module_name: (str) the name of the root module (`tf` for tensorflow). root_title: The title name for the root level index.md. search_hints: (bool) include meta-data search hints at the top of each output file. site_path: The output path relative to the site root. Used in the `_toc.yaml` and `_redirects.yaml` files. gen_redirects: Bool which decides whether to generate _redirects.yaml file or not. gen_report: If True, a report for the library is generated by linting the docstrings of its public API symbols. extra_docs: To add docs for a particular object instance set it's __doc__ attribute. For some classes (list, tuple, etc) __doc__ is not writable. Pass those docs like: `extra_docs={id(obj): "docs"}` page_builder_classes: A optional dict of `{ObjectType:Type[PageInfo]}` for overriding the default page builder classes. Raises: ValueError: if `output_dir` is not an absolute path """ output_dir = pathlib.Path(output_dir) site_path = pathlib.Path('/', site_path) # Make output_dir. if not output_dir.is_absolute(): raise ValueError("'output_dir' must be an absolute path.\n" f" output_dir='{output_dir}'") output_dir.mkdir(parents=True, exist_ok=True) # Collect redirects for an api _redirects.yaml file. redirects = [] api_report = None if gen_report: api_report = utils.ApiReport() # Parse and write Markdown pages, resolving cross-links (`tf.symbol`). num_docs_output = 0 for api_node in parser_config.api_tree.iter_nodes(): full_name = api_node.full_name if api_node.output_type() is api_node.OutputType.FRAGMENT: continue # Generate docs for `py_object`, resolving references. try: page_info = docs_for_object.docs_for_object( api_node=api_node, parser_config=parser_config, extra_docs=extra_docs, search_hints=search_hints, page_builder_classes=page_builder_classes) if api_report is not None and not full_name.startswith( ('tf.compat.v', 'tf.keras.backend', 'tf.numpy', 'tf.experimental.numpy')): api_report.fill_metrics(page_info) except Exception as e: raise ValueError( f'Failed to generate docs for symbol: `{full_name}`') from e path = output_dir / parser.documentation_path(full_name) try: path.parent.mkdir(exist_ok=True, parents=True) path.write_text(page_info.page_text, encoding='utf-8') num_docs_output += 1 except OSError as e: raise OSError('Cannot write documentation for ' f'{full_name} to {path.parent}') from e duplicates = parser_config.duplicates.get(full_name, []) if not duplicates: continue duplicates = [item for item in duplicates if item != full_name] if gen_redirects: for dup in duplicates: from_path = site_path / dup.replace('.', '/') to_path = site_path / full_name.replace('.', '/') redirects.append({'from': str(from_path), 'to': str(to_path)}) if api_report is not None: api_report.write(output_dir / root_module_name / 'api_report.pb') if num_docs_output <= 1: raise ValueError( 'The `DocGenerator` failed to generate any docs. Verify ' 'your arguments (`base_dir` and `callbacks`). ' 'Everything you want documented should be within ' '`base_dir`.') if yaml_toc: if isinstance(yaml_toc, bool): yaml_toc = toc_lib.FlatModulesTocBuilder toc = yaml_toc(site_path).build(parser_config.api_tree) toc_path = output_dir / root_module_name / '_toc.yaml' toc.write(toc_path) if redirects and gen_redirects: redirects_dict = { 'redirects': sorted(redirects, key=lambda redirect: redirect['from']) } api_redirects_path = output_dir / root_module_name / '_redirects.yaml' with open(api_redirects_path, 'w') as redirect_file: yaml.dump(redirects_dict, redirect_file, default_flow_style=False) # Write a global index containing all full names with links. with open(output_dir / root_module_name / 'all_symbols.md', 'w') as f: global_index = parser.generate_global_index( root_title, parser_config.index, parser_config.reference_resolver) if not search_hints: global_index = 'robots: noindex\n' + global_index f.write(global_index)
def _make_report(self): page_info = self._build_page_info() test_api_report = utils.ApiReport() test_api_report.fill_metrics(page_info) return test_api_report
def write_docs( *, output_dir: Union[str, pathlib.Path], parser_config: config.ParserConfig, yaml_toc: bool, root_module_name: str, root_title: str = 'TensorFlow', search_hints: bool = True, site_path: str = 'api_docs/python', gen_redirects: bool = True, gen_report: bool = True, extra_docs: Optional[Dict[int, str]] = None, page_builder_classes: Optional[docs_for_object.PageBuilderDict] = None, ): """Write previously extracted docs to disk. Write a docs page for each symbol included in the indices of parser_config to a tree of docs at `output_dir`. Symbols with multiple aliases will have only one page written about them, which is referenced for all aliases. Args: output_dir: Directory to write documentation markdown files to. Will be created if it doesn't exist. parser_config: A `config.ParserConfig` object, containing all the necessary indices. yaml_toc: Set to `True` to generate a "_toc.yaml" file. root_module_name: (str) the name of the root module (`tf` for tensorflow). root_title: The title name for the root level index.md. search_hints: (bool) include meta-data search hints at the top of each output file. site_path: The output path relative to the site root. Used in the `_toc.yaml` and `_redirects.yaml` files. gen_redirects: Bool which decides whether to generate _redirects.yaml file or not. gen_report: If True, a report for the library is generated by linting the docstrings of its public API symbols. extra_docs: To add docs for a particular object instance set it's __doc__ attribute. For some classes (list, tuple, etc) __doc__ is not writable. Pass those docs like: `extra_docs={id(obj): "docs"}` page_builder_classes: A optional dict of `{ObjectType:Type[PageInfo]}` for overriding the default page builder classes. Raises: ValueError: if `output_dir` is not an absolute path """ output_dir = pathlib.Path(output_dir) site_path = pathlib.Path('/', site_path) # Make output_dir. if not output_dir.is_absolute(): raise ValueError("'output_dir' must be an absolute path.\n" f" output_dir='{output_dir}'") output_dir.mkdir(parents=True, exist_ok=True) # These dictionaries are used for table-of-contents generation below # They will contain, after the for-loop below:: # - module name(string):classes and functions the module contains(list) module_children = {} # Collect redirects for an api _redirects.yaml file. redirects = [] if gen_report: api_report_obj = utils.ApiReport() # Parse and write Markdown pages, resolving cross-links (`tf.symbol`). num_docs_output = 0 for full_name in sorted(parser_config.index.keys(), key=lambda k: k.lower()): py_object = parser_config.index[full_name] if full_name in parser_config.duplicate_of: continue # Methods constants are only documented only as part of their parent's page. if parser_config.reference_resolver.is_fragment(full_name): continue # Remove the extension from the path. docpath, _ = os.path.splitext(parser.documentation_path(full_name)) # For a module, remember the module for the table-of-contents if inspect.ismodule(py_object): if full_name in parser_config.tree: mod_obj = Module(module=full_name, py_object=py_object, path=str(site_path / docpath)) module_children[full_name] = mod_obj # For something else that's documented, # figure out what module it lives in else: subname = str(full_name) while True: subname = subname[:subname.rindex('.')] if inspect.ismodule(parser_config.index[subname]): module_name = parser_config.duplicate_of.get( subname, subname) child_mod = ModuleChild(name=full_name, py_object=py_object, parent=module_name, path=str(site_path / docpath)) module_children[module_name].add_children(child_mod) break # Generate docs for `py_object`, resolving references. try: page_info = docs_for_object.docs_for_object( full_name=full_name, py_object=py_object, parser_config=parser_config, extra_docs=extra_docs, search_hints=search_hints, page_builder_classes=page_builder_classes) if gen_report and not full_name.startswith( ('tf.compat.v', 'tf.keras.backend', 'tf.numpy', 'tf.experimental.numpy')): api_report_obj.fill_metrics(page_info) except Exception as e: raise ValueError( f'Failed to generate docs for symbol: `{full_name}`') from e path = output_dir / parser.documentation_path(full_name) text = page_info.build() try: path.parent.mkdir(exist_ok=True, parents=True) path.write_text(text, encoding='utf-8') num_docs_output += 1 except OSError: raise OSError('Cannot write documentation for ' f'{full_name} to {path.parent}') duplicates = parser_config.duplicates.get(full_name, []) if not duplicates: continue duplicates = [item for item in duplicates if item != full_name] if gen_redirects: for dup in duplicates: from_path = site_path / dup.replace('.', '/') to_path = site_path / full_name.replace('.', '/') redirects.append({'from': str(from_path), 'to': str(to_path)}) if gen_report: serialized_proto = api_report_obj.api_report.SerializeToString() raw_proto = output_dir / root_module_name / 'api_report.pb' raw_proto.write_bytes(serialized_proto) if num_docs_output <= 1: raise ValueError( 'The `DocGenerator` failed to generate any docs. Verify ' 'your arguments (`base_dir` and `callbacks`). ' 'Everything you want documented should be within ' '`base_dir`.') if yaml_toc: toc_gen = GenerateToc(module_children) toc_dict = toc_gen.generate() # Replace the overview path *only* for 'TensorFlow' to # `/api_docs/python/tf_overview`. This will be redirected to # `/api_docs/python/tf`. toc_values = toc_dict['toc'][0] if toc_values['title'] == 'tf': section = toc_values['section'][0] section['path'] = str(site_path / 'tf_overview') leftnav_toc = output_dir / root_module_name / '_toc.yaml' with open(leftnav_toc, 'w') as toc_file: yaml.dump(toc_dict, toc_file, default_flow_style=False) if redirects and gen_redirects: if yaml_toc and toc_values['title'] == 'tf': redirects.append({ 'from': str(site_path / 'tf_overview'), 'to': str(site_path / 'tf'), }) redirects_dict = { 'redirects': sorted(redirects, key=lambda redirect: redirect['from']) } api_redirects_path = output_dir / root_module_name / '_redirects.yaml' with open(api_redirects_path, 'w') as redirect_file: yaml.dump(redirects_dict, redirect_file, default_flow_style=False) # Write a global index containing all full names with links. with open(output_dir / root_module_name / 'all_symbols.md', 'w') as f: global_index = parser.generate_global_index( root_title, parser_config.index, parser_config.reference_resolver) if not search_hints: global_index = 'robots: noindex\n' + global_index f.write(global_index)