def test_generate_index(self):
m = types.ModuleType('m')
m.__file__ = __file__
m.TestClass = TestClass
m.test_function = test_function
m.submodule = types.ModuleType('submodule')
m.submodule.test_function = test_function
generator = generate_lib.DocGenerator(
root_title='test',
py_modules=[('m', m)],
code_url_prefix='https://tensorflow.org')
parser_config = generator.run_extraction()
docs = parser.generate_global_index(
'TestLibrary',
index=parser_config.index,
reference_resolver=parser_config.reference_resolver)
# Make sure duplicates and non-top-level symbols are in the index, but
# methods and properties are not.
self.assertNotIn('a_method', docs)
self.assertNotIn('a_property', docs)
self.assertIn('m.TestClass', docs)
self.assertIn('m.TestClass.ChildClass', docs)
self.assertIn('m.submodule.test_function', docs)
self.assertIn('<code>m.submodule.test_function', docs)
def test_generate_index(self):
index = {
'tf': test_module,
'tf.TestModule': test_module,
'tf.test_function': test_function,
'tf.TestModule.test_function': test_function,
'tf.TestModule.TestClass': TestClass,
'tf.TestModule.TestClass.a_method': TestClass.a_method,
'tf.TestModule.TestClass.a_property': TestClass.a_property,
'tf.TestModule.TestClass.ChildClass': TestClass.ChildClass,
}
duplicate_of = {'tf.TestModule.test_function': 'tf.test_function'}
visitor = DummyVisitor(index=index, duplicate_of=duplicate_of)
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
docs = parser.generate_global_index(
'TestLibrary', index=index, reference_resolver=reference_resolver)
# Make sure duplicates and non-top-level symbols are in the index, but
# methods and properties are not.
self.assertNotIn('a_method', docs)
self.assertNotIn('a_property', docs)
self.assertIn('TestModule.TestClass', docs)
self.assertIn('TestModule.TestClass.ChildClass', docs)
self.assertIn('TestModule.test_function', docs)
# Leading backtick to make sure it's included top-level.
# This depends on formatting, but should be stable.
self.assertIn('<code>tf.test_function', docs)
def test_generate_index(self):
index = {
'tf': test_module,
'tf.TestModule': test_module,
'tf.test_function': test_function,
'tf.TestModule.test_function': test_function,
'tf.TestModule.TestClass': TestClass,
'tf.TestModule.TestClass.a_method': TestClass.a_method,
'tf.TestModule.TestClass.a_property': TestClass.a_property,
'tf.TestModule.TestClass.ChildClass': TestClass.ChildClass,
}
duplicate_of = {'tf.TestModule.test_function': 'tf.test_function'}
visitor = DummyVisitor(index=index, duplicate_of=duplicate_of)
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
docs = parser.generate_global_index('TestLibrary', index=index,
reference_resolver=reference_resolver)
# Make sure duplicates and non-top-level symbols are in the index, but
# methods and properties are not.
self.assertNotIn('a_method', docs)
self.assertNotIn('a_property', docs)
self.assertIn('TestModule.TestClass', docs)
self.assertIn('TestModule.TestClass.ChildClass', docs)
self.assertIn('TestModule.test_function', docs)
# Leading backtick to make sure it's included top-level.
# This depends on formatting, but should be stable.
self.assertIn('<code>tf.test_function', docs)
def write_docs(output_dir,
parser_config,
yaml_toc,
root_title='TensorFlow',
search_hints=True,
site_path=''):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `parser.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
root_title: The title name for the root level index.md.
search_hints: (bool) include meta-data search hints at the top of each
output file.
site_path: The output path relative to the site root. Used in the
`_toc.yaml` and `_redirects.yaml` files.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
# Make output_dir.
if not os.path.isabs(output_dir):
raise ValueError("'output_dir' must be an absolute path.\n"
" output_dir='%s'" % output_dir)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# These dictionaries are used for table-of-contents generation below
# They will contain, after the for-loop below::
# - module name(string):classes and functions the module contains(list)
module_children = {}
# - symbol name(string):pathname (string)
symbol_to_file = {}
# Collect redirects for an api _redirects.yaml file.
redirects = []
# Parse and write Markdown pages, resolving cross-links (`tf.symbol`).
for full_name, py_object in six.iteritems(parser_config.index):
parser_config.reference_resolver.current_doc_full_name = full_name
if full_name in parser_config.duplicate_of:
continue
# Methods and some routines are documented only as part of their class.
if not (tf_inspect.ismodule(py_object)
or tf_inspect.isclass(py_object) or parser.is_free_function(
py_object, full_name, parser_config.index)):
continue
sitepath = os.path.join('api_docs/python',
parser.documentation_path(full_name)[:-3])
# For TOC, we need to store a mapping from full_name to the file
# we're generating
symbol_to_file[full_name] = sitepath
# For a module, remember the module for the table-of-contents
if tf_inspect.ismodule(py_object):
if full_name in parser_config.tree:
module_children.setdefault(full_name, [])
# For something else that's documented,
# figure out what module it lives in
else:
subname = str(full_name)
while True:
subname = subname[:subname.rindex('.')]
if tf_inspect.ismodule(parser_config.index[subname]):
module_name = parser_config.duplicate_of.get(
subname, subname)
module_children.setdefault(module_name,
[]).append(full_name)
break
# Generate docs for `py_object`, resolving references.
page_info = parser.docs_for_object(full_name, py_object, parser_config)
path = os.path.join(output_dir, parser.documentation_path(full_name))
directory = os.path.dirname(path)
try:
if not os.path.exists(directory):
os.makedirs(directory)
# This function returns raw bytes in PY2 or unicode in PY3.
if search_hints:
content = [page_info.get_metadata_html()]
else:
content = ['']
content.append(pretty_docs.build_md_page(page_info))
text = '\n'.join(content)
if six.PY3:
text = text.encode('utf-8')
with open(path, 'wb') as f:
f.write(text)
except OSError:
raise OSError('Cannot write documentation for %s to %s' %
(full_name, directory))
duplicates = parser_config.duplicates.get(full_name, [])
if not duplicates:
continue
duplicates = [item for item in duplicates if item != full_name]
for dup in duplicates:
from_path = os.path.join(site_path, 'api_docs/python',
dup.replace('.', '/'))
to_path = os.path.join(site_path, 'api_docs/python',
full_name.replace('.', '/'))
redirects.append(
(os.path.join('/', from_path), os.path.join('/', to_path)))
if redirects:
redirects = sorted(redirects)
template = ('- from: {}\n' ' to: {}\n')
redirects = [template.format(f, t) for f, t in redirects]
api_redirects_path = os.path.join(output_dir, '_redirects.yaml')
with open(api_redirects_path, 'w') as redirect_file:
redirect_file.write('redirects:\n')
redirect_file.write(''.join(redirects))
if yaml_toc:
# Generate table of contents
# Put modules in alphabetical order, case-insensitive
modules = sorted(module_children.keys(), key=lambda a: a.upper())
leftnav_path = os.path.join(output_dir, '_toc.yaml')
with open(leftnav_path, 'w') as f:
# Generate header
f.write(
'# Automatically generated file; please do not edit\ntoc:\n')
for module in modules:
indent_num = module.count('.')
# Don't list `tf.submodule` inside `tf`
indent_num = max(indent_num, 1)
indent = ' ' * indent_num
if indent_num > 1:
# tf.contrib.baysflow.entropy will be under
# tf.contrib->baysflow->entropy
title = module.split('.')[-1]
else:
title = module
header = [
'- title: ' + title, ' section:', ' - title: Overview',
' path: ' +
os.path.join('/', site_path, symbol_to_file[module])
]
header = ''.join([indent + line + '\n' for line in header])
f.write(header)
symbols_in_module = module_children.get(module, [])
# Sort case-insensitive, if equal sort case sensitive (upper first)
symbols_in_module.sort(key=lambda a: (a.upper(), a))
for full_name in symbols_in_module:
item = [
' - title: ' + full_name[len(module) + 1:],
' path: ' +
os.path.join('/', site_path, symbol_to_file[full_name])
]
item = ''.join([indent + line + '\n' for line in item])
f.write(item)
# Write a global index containing all full names with links.
with open(os.path.join(output_dir, 'index.md'), 'w') as f:
f.write(
parser.generate_global_index(root_title, parser_config.index,
parser_config.reference_resolver))
def write_docs(
*,
output_dir: Union[str, pathlib.Path],
parser_config: parser.ParserConfig,
yaml_toc: bool,
root_module_name: str,
root_title: str = 'TensorFlow',
search_hints: bool = True,
site_path: str = 'api_docs/python',
gen_redirects: bool = True,
table_view: bool = True,
gen_report: bool = False,
):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `parser.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
root_module_name: (str) the name of the root module (`tf` for tensorflow).
root_title: The title name for the root level index.md.
search_hints: (bool) include meta-data search hints at the top of each
output file.
site_path: The output path relative to the site root. Used in the
`_toc.yaml` and `_redirects.yaml` files.
gen_redirects: Bool which decides whether to generate _redirects.yaml file
or not.
table_view: If True, `Args`, `Returns`, `Raises` or `Attributes` will be
converted to a tabular format while generating markdown. If False, they
will be converted to a markdown List view.
gen_report: If True, a report for the library is generated by linting the
docstrings of its public API symbols.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
output_dir = pathlib.Path(output_dir)
site_path = pathlib.Path('/', site_path)
# Make output_dir.
if not output_dir.is_absolute():
raise ValueError("'output_dir' must be an absolute path.\n"
f" output_dir='{output_dir}'")
output_dir.mkdir(parents=True, exist_ok=True)
# These dictionaries are used for table-of-contents generation below
# They will contain, after the for-loop below::
# - module name(string):classes and functions the module contains(list)
module_children = {}
# Collect redirects for an api _redirects.yaml file.
redirects = []
if gen_report:
api_report_obj = utils.ApiReport()
# Parse and write Markdown pages, resolving cross-links (`tf.symbol`).
for full_name in sorted(parser_config.index.keys(),
key=lambda k: k.lower()):
py_object = parser_config.index[full_name]
if full_name in parser_config.duplicate_of:
continue
# Methods constants are only documented only as part of their parent's page.
if parser_config.reference_resolver.is_fragment(full_name):
continue
# Remove the extension from the path.
docpath, _ = os.path.splitext(parser.documentation_path(full_name))
# For a module, remember the module for the table-of-contents
if inspect.ismodule(py_object):
if full_name in parser_config.tree:
mod_obj = Module(module=full_name,
py_object=py_object,
path=str(site_path / docpath))
module_children[full_name] = mod_obj
# For something else that's documented,
# figure out what module it lives in
else:
subname = str(full_name)
while True:
subname = subname[:subname.rindex('.')]
if inspect.ismodule(parser_config.index[subname]):
module_name = parser_config.duplicate_of.get(
subname, subname)
child_mod = ModuleChild(name=full_name,
py_object=py_object,
parent=module_name,
path=str(site_path / docpath))
module_children[module_name].add_children(child_mod)
break
# Generate docs for `py_object`, resolving references.
try:
page_info = parser.docs_for_object(full_name, py_object,
parser_config)
except:
raise ValueError(
f'Failed to generate docs for symbol: `{full_name}`')
if gen_report and not full_name.startswith(
('tf.compat.v', 'tf.keras.backend')):
api_report_obj.fill_metrics(page_info)
continue
path = output_dir / parser.documentation_path(full_name)
try:
path.parent.mkdir(exist_ok=True, parents=True)
# This function returns unicode in PY3.
hidden = doc_controls.should_hide_from_search(page_info.py_object)
brief_no_backticks = page_info.doc.brief.replace('`', '').strip()
content = []
if brief_no_backticks:
content.append(f'description: {brief_no_backticks}\n')
if search_hints and not hidden:
content.append(page_info.get_metadata_html())
else:
content.append('robots: noindex\n')
content.append(pretty_docs.build_md_page(page_info, table_view))
text = '\n'.join(content)
path.write_text(text, encoding='utf-8')
except OSError:
raise OSError('Cannot write documentation for '
f'{full_name} to {path.parent}')
duplicates = parser_config.duplicates.get(full_name, [])
if not duplicates:
continue
duplicates = [item for item in duplicates if item != full_name]
if gen_redirects:
for dup in duplicates:
from_path = site_path / dup.replace('.', '/')
to_path = site_path / full_name.replace('.', '/')
redirects.append({'from': str(from_path), 'to': str(to_path)})
if gen_report:
serialized_proto = api_report_obj.api_report.SerializeToString()
raw_proto = output_dir / 'api_report.pb'
raw_proto.write_bytes(serialized_proto)
return
if yaml_toc:
toc_gen = GenerateToc(module_children)
toc_dict = toc_gen.generate()
# Replace the overview path *only* for 'TensorFlow' to
# `/api_docs/python/tf_overview`. This will be redirected to
# `/api_docs/python/tf`.
toc_values = toc_dict['toc'][0]
if toc_values['title'] == 'tf':
section = toc_values['section'][0]
section['path'] = str(site_path / 'tf_overview')
leftnav_toc = output_dir / root_module_name / '_toc.yaml'
with open(leftnav_toc, 'w') as toc_file:
yaml.dump(toc_dict, toc_file, default_flow_style=False)
if redirects and gen_redirects:
if yaml_toc and toc_values['title'] == 'tf':
redirects.append({
'from': str(site_path / 'tf_overview'),
'to': str(site_path / 'tf'),
})
redirects_dict = {
'redirects': sorted(redirects,
key=lambda redirect: redirect['from'])
}
api_redirects_path = output_dir / root_module_name / '_redirects.yaml'
with open(api_redirects_path, 'w') as redirect_file:
yaml.dump(redirects_dict, redirect_file, default_flow_style=False)
# Write a global index containing all full names with links.
with open(output_dir / root_module_name / 'all_symbols.md', 'w') as f:
global_index = parser.generate_global_index(
root_title, parser_config.index, parser_config.reference_resolver)
if not search_hints:
global_index = 'robots: noindex\n' + global_index
f.write(global_index)
def write_docs(output_dir,
parser_config,
yaml_toc,
root_title='TensorFlow',
search_hints=True,
site_path='api_docs/python'):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `parser.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
root_title: The title name for the root level index.md.
search_hints: (bool) include meta-data search hints at the top of each
output file.
site_path: The output path relative to the site root. Used in the
`_toc.yaml` and `_redirects.yaml` files.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
# Make output_dir.
if not os.path.isabs(output_dir):
raise ValueError("'output_dir' must be an absolute path.\n"
" output_dir='%s'" % output_dir)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# These dictionaries are used for table-of-contents generation below
# They will contain, after the for-loop below::
# - module name(string):classes and functions the module contains(list)
module_children = {}
# Collect redirects for an api _redirects.yaml file.
redirects = []
# Parse and write Markdown pages, resolving cross-links (`tf.symbol`).
for full_name in sorted(parser_config.index.keys(),
key=lambda k: k.lower()):
py_object = parser_config.index[full_name]
if full_name in parser_config.duplicate_of:
continue
# Methods and some routines are documented only as part of their class.
if not (tf_inspect.ismodule(py_object)
or tf_inspect.isclass(py_object) or parser.is_free_function(
py_object, full_name, parser_config.index)):
continue
# Remove the extension from the path.
docpath, _ = os.path.splitext(parser.documentation_path(full_name))
# For a module, remember the module for the table-of-contents
if tf_inspect.ismodule(py_object):
if full_name in parser_config.tree:
mod_obj = Module(module=full_name,
py_object=py_object,
path=os.path.join('/', site_path, docpath))
module_children[full_name] = mod_obj
# For something else that's documented,
# figure out what module it lives in
else:
subname = str(full_name)
while True:
subname = subname[:subname.rindex('.')]
if tf_inspect.ismodule(parser_config.index[subname]):
module_name = parser_config.duplicate_of.get(
subname, subname)
child_mod = ModuleChild(name=full_name,
py_object=py_object,
parent=module_name,
path=os.path.join(
'/', site_path, docpath))
module_children[module_name].add_children(child_mod)
break
# Generate docs for `py_object`, resolving references.
try:
page_info = parser.docs_for_object(full_name, py_object,
parser_config)
except:
raise ValueError(
'Failed to generate docs for symbol: `{}`'.format(full_name))
path = os.path.join(output_dir, parser.documentation_path(full_name))
directory = os.path.dirname(path)
try:
if not os.path.exists(directory):
os.makedirs(directory)
# This function returns raw bytes in PY2 or unicode in PY3.
if search_hints:
content = [page_info.get_metadata_html()]
else:
content = ['']
content.append(pretty_docs.build_md_page(page_info))
text = '\n'.join(content)
if six.PY3:
text = text.encode('utf-8')
with open(path, 'wb') as f:
f.write(text)
except OSError:
raise OSError('Cannot write documentation for %s to %s' %
(full_name, directory))
duplicates = parser_config.duplicates.get(full_name, [])
if not duplicates:
continue
duplicates = [item for item in duplicates if item != full_name]
for dup in duplicates:
from_path = os.path.join(site_path, dup.replace('.', '/'))
to_path = os.path.join(site_path, full_name.replace('.', '/'))
redirects.append({
'from': os.path.join('/', from_path),
'to': os.path.join('/', to_path)
})
if redirects:
redirects_dict = {
'redirects': sorted(redirects,
key=lambda redirect: redirect['from'])
}
api_redirects_path = os.path.join(output_dir, '_redirects.yaml')
with open(api_redirects_path, 'w') as redirect_file:
yaml.dump(redirects_dict, redirect_file, default_flow_style=False)
if yaml_toc:
toc_gen = GenerateToc(module_children)
toc_dict = toc_gen.generate()
leftnav_toc = os.path.join(output_dir, '_toc.yaml')
with open(leftnav_toc, 'w') as toc_file:
yaml.dump(toc_dict, toc_file, default_flow_style=False)
# Write a global index containing all full names with links.
with open(os.path.join(output_dir, 'index.md'), 'w') as f:
f.write(
parser.generate_global_index(root_title, parser_config.index,
parser_config.reference_resolver))
def write_docs(
*,
output_dir: Union[str, pathlib.Path],
parser_config: config.ParserConfig,
yaml_toc: Union[bool, Type[toc_lib.TocBuilder]],
root_module_name: str,
root_title: str = 'TensorFlow',
search_hints: bool = True,
site_path: str = 'api_docs/python',
gen_redirects: bool = True,
gen_report: bool = True,
extra_docs: Optional[Dict[int, str]] = None,
page_builder_classes: Optional[docs_for_object.PageBuilderDict] = None,
):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `config.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
root_module_name: (str) the name of the root module (`tf` for tensorflow).
root_title: The title name for the root level index.md.
search_hints: (bool) include meta-data search hints at the top of each
output file.
site_path: The output path relative to the site root. Used in the
`_toc.yaml` and `_redirects.yaml` files.
gen_redirects: Bool which decides whether to generate _redirects.yaml file
or not.
gen_report: If True, a report for the library is generated by linting the
docstrings of its public API symbols.
extra_docs: To add docs for a particular object instance set it's __doc__
attribute. For some classes (list, tuple, etc) __doc__ is not writable.
Pass those docs like: `extra_docs={id(obj): "docs"}`
page_builder_classes: A optional dict of `{ObjectType:Type[PageInfo]}` for
overriding the default page builder classes.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
output_dir = pathlib.Path(output_dir)
site_path = pathlib.Path('/', site_path)
# Make output_dir.
if not output_dir.is_absolute():
raise ValueError("'output_dir' must be an absolute path.\n"
f" output_dir='{output_dir}'")
output_dir.mkdir(parents=True, exist_ok=True)
# Collect redirects for an api _redirects.yaml file.
redirects = []
api_report = None
if gen_report:
api_report = utils.ApiReport()
# Parse and write Markdown pages, resolving cross-links (`tf.symbol`).
num_docs_output = 0
for api_node in parser_config.api_tree.iter_nodes():
full_name = api_node.full_name
if api_node.output_type() is api_node.OutputType.FRAGMENT:
continue
# Generate docs for `py_object`, resolving references.
try:
page_info = docs_for_object.docs_for_object(
api_node=api_node,
parser_config=parser_config,
extra_docs=extra_docs,
search_hints=search_hints,
page_builder_classes=page_builder_classes)
if api_report is not None and not full_name.startswith(
('tf.compat.v', 'tf.keras.backend', 'tf.numpy',
'tf.experimental.numpy')):
api_report.fill_metrics(page_info)
except Exception as e:
raise ValueError(
f'Failed to generate docs for symbol: `{full_name}`') from e
path = output_dir / parser.documentation_path(full_name)
try:
path.parent.mkdir(exist_ok=True, parents=True)
path.write_text(page_info.page_text, encoding='utf-8')
num_docs_output += 1
except OSError as e:
raise OSError('Cannot write documentation for '
f'{full_name} to {path.parent}') from e
duplicates = parser_config.duplicates.get(full_name, [])
if not duplicates:
continue
duplicates = [item for item in duplicates if item != full_name]
if gen_redirects:
for dup in duplicates:
from_path = site_path / dup.replace('.', '/')
to_path = site_path / full_name.replace('.', '/')
redirects.append({'from': str(from_path), 'to': str(to_path)})
if api_report is not None:
api_report.write(output_dir / root_module_name / 'api_report.pb')
if num_docs_output <= 1:
raise ValueError(
'The `DocGenerator` failed to generate any docs. Verify '
'your arguments (`base_dir` and `callbacks`). '
'Everything you want documented should be within '
'`base_dir`.')
if yaml_toc:
if isinstance(yaml_toc, bool):
yaml_toc = toc_lib.FlatModulesTocBuilder
toc = yaml_toc(site_path).build(parser_config.api_tree)
toc_path = output_dir / root_module_name / '_toc.yaml'
toc.write(toc_path)
if redirects and gen_redirects:
redirects_dict = {
'redirects': sorted(redirects,
key=lambda redirect: redirect['from'])
}
api_redirects_path = output_dir / root_module_name / '_redirects.yaml'
with open(api_redirects_path, 'w') as redirect_file:
yaml.dump(redirects_dict, redirect_file, default_flow_style=False)
# Write a global index containing all full names with links.
with open(output_dir / root_module_name / 'all_symbols.md', 'w') as f:
global_index = parser.generate_global_index(
root_title, parser_config.index, parser_config.reference_resolver)
if not search_hints:
global_index = 'robots: noindex\n' + global_index
f.write(global_index)
def write_docs(
*,
output_dir: Union[str, pathlib.Path],
parser_config: config.ParserConfig,
yaml_toc: bool,
root_module_name: str,
root_title: str = 'TensorFlow',
search_hints: bool = True,
site_path: str = 'api_docs/python',
gen_redirects: bool = True,
gen_report: bool = True,
extra_docs: Optional[Dict[int, str]] = None,
page_builder_classes: Optional[docs_for_object.PageBuilderDict] = None,
):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `config.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
root_module_name: (str) the name of the root module (`tf` for tensorflow).
root_title: The title name for the root level index.md.
search_hints: (bool) include meta-data search hints at the top of each
output file.
site_path: The output path relative to the site root. Used in the
`_toc.yaml` and `_redirects.yaml` files.
gen_redirects: Bool which decides whether to generate _redirects.yaml file
or not.
gen_report: If True, a report for the library is generated by linting the
docstrings of its public API symbols.
extra_docs: To add docs for a particular object instance set it's __doc__
attribute. For some classes (list, tuple, etc) __doc__ is not writable.
Pass those docs like: `extra_docs={id(obj): "docs"}`
page_builder_classes: A optional dict of `{ObjectType:Type[PageInfo]}` for
overriding the default page builder classes.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
output_dir = pathlib.Path(output_dir)
site_path = pathlib.Path('/', site_path)
# Make output_dir.
if not output_dir.is_absolute():
raise ValueError("'output_dir' must be an absolute path.\n"
f" output_dir='{output_dir}'")
output_dir.mkdir(parents=True, exist_ok=True)
# These dictionaries are used for table-of-contents generation below
# They will contain, after the for-loop below::
# - module name(string):classes and functions the module contains(list)
module_children = {}
# Collect redirects for an api _redirects.yaml file.
redirects = []
if gen_report:
api_report_obj = utils.ApiReport()
# Parse and write Markdown pages, resolving cross-links (`tf.symbol`).
num_docs_output = 0
for full_name in sorted(parser_config.index.keys(),
key=lambda k: k.lower()):
py_object = parser_config.index[full_name]
if full_name in parser_config.duplicate_of:
continue
# Methods constants are only documented only as part of their parent's page.
if parser_config.reference_resolver.is_fragment(full_name):
continue
# Remove the extension from the path.
docpath, _ = os.path.splitext(parser.documentation_path(full_name))
# For a module, remember the module for the table-of-contents
if inspect.ismodule(py_object):
if full_name in parser_config.tree:
mod_obj = Module(module=full_name,
py_object=py_object,
path=str(site_path / docpath))
module_children[full_name] = mod_obj
# For something else that's documented,
# figure out what module it lives in
else:
subname = str(full_name)
while True:
subname = subname[:subname.rindex('.')]
if inspect.ismodule(parser_config.index[subname]):
module_name = parser_config.duplicate_of.get(
subname, subname)
child_mod = ModuleChild(name=full_name,
py_object=py_object,
parent=module_name,
path=str(site_path / docpath))
module_children[module_name].add_children(child_mod)
break
# Generate docs for `py_object`, resolving references.
try:
page_info = docs_for_object.docs_for_object(
full_name=full_name,
py_object=py_object,
parser_config=parser_config,
extra_docs=extra_docs,
search_hints=search_hints,
page_builder_classes=page_builder_classes)
if gen_report and not full_name.startswith(
('tf.compat.v', 'tf.keras.backend', 'tf.numpy',
'tf.experimental.numpy')):
api_report_obj.fill_metrics(page_info)
except Exception as e:
raise ValueError(
f'Failed to generate docs for symbol: `{full_name}`') from e
path = output_dir / parser.documentation_path(full_name)
text = page_info.build()
try:
path.parent.mkdir(exist_ok=True, parents=True)
path.write_text(text, encoding='utf-8')
num_docs_output += 1
except OSError:
raise OSError('Cannot write documentation for '
f'{full_name} to {path.parent}')
duplicates = parser_config.duplicates.get(full_name, [])
if not duplicates:
continue
duplicates = [item for item in duplicates if item != full_name]
if gen_redirects:
for dup in duplicates:
from_path = site_path / dup.replace('.', '/')
to_path = site_path / full_name.replace('.', '/')
redirects.append({'from': str(from_path), 'to': str(to_path)})
if gen_report:
serialized_proto = api_report_obj.api_report.SerializeToString()
raw_proto = output_dir / root_module_name / 'api_report.pb'
raw_proto.write_bytes(serialized_proto)
if num_docs_output <= 1:
raise ValueError(
'The `DocGenerator` failed to generate any docs. Verify '
'your arguments (`base_dir` and `callbacks`). '
'Everything you want documented should be within '
'`base_dir`.')
if yaml_toc:
toc_gen = GenerateToc(module_children)
toc_dict = toc_gen.generate()
# Replace the overview path *only* for 'TensorFlow' to
# `/api_docs/python/tf_overview`. This will be redirected to
# `/api_docs/python/tf`.
toc_values = toc_dict['toc'][0]
if toc_values['title'] == 'tf':
section = toc_values['section'][0]
section['path'] = str(site_path / 'tf_overview')
leftnav_toc = output_dir / root_module_name / '_toc.yaml'
with open(leftnav_toc, 'w') as toc_file:
yaml.dump(toc_dict, toc_file, default_flow_style=False)
if redirects and gen_redirects:
if yaml_toc and toc_values['title'] == 'tf':
redirects.append({
'from': str(site_path / 'tf_overview'),
'to': str(site_path / 'tf'),
})
redirects_dict = {
'redirects': sorted(redirects,
key=lambda redirect: redirect['from'])
}
api_redirects_path = output_dir / root_module_name / '_redirects.yaml'
with open(api_redirects_path, 'w') as redirect_file:
yaml.dump(redirects_dict, redirect_file, default_flow_style=False)
# Write a global index containing all full names with links.
with open(output_dir / root_module_name / 'all_symbols.md', 'w') as f:
global_index = parser.generate_global_index(
root_title, parser_config.index, parser_config.reference_resolver)
if not search_hints:
global_index = 'robots: noindex\n' + global_index
f.write(global_index)
def write_docs(output_dir,
parser_config,
yaml_toc,
root_title='TensorFlow',
search_hints=True,
site_path=''):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `parser.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
root_title: The title name for the root level index.md.
search_hints: (bool) include meta-data search hints at the top of each
output file.
site_path: The output path relative to the site root. Used in the
`_toc.yaml` and `_redirects.yaml` files.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
# Make output_dir.
if not os.path.isabs(output_dir):
raise ValueError("'output_dir' must be an absolute path.\n"
" output_dir='%s'" % output_dir)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# These dictionaries are used for table-of-contents generation below
# They will contain, after the for-loop below::
# - module name(string):classes and functions the module contains(list)
module_children = {}
# - symbol name(string):pathname (string)
symbol_to_file = {}
# Collect redirects for an api _redirects.yaml file.
redirects = []
# Parse and write Markdown pages, resolving cross-links (`tf.symbol`).
for full_name, py_object in six.iteritems(parser_config.index):
parser_config.reference_resolver.current_doc_full_name = full_name
if full_name in parser_config.duplicate_of:
continue
# Methods and some routines are documented only as part of their class.
if not (tf_inspect.ismodule(py_object) or tf_inspect.isclass(py_object) or
parser.is_free_function(py_object, full_name, parser_config.index)):
continue
sitepath = os.path.join('api_docs/python',
parser.documentation_path(full_name)[:-3])
# For TOC, we need to store a mapping from full_name to the file
# we're generating
symbol_to_file[full_name] = sitepath
# For a module, remember the module for the table-of-contents
if tf_inspect.ismodule(py_object):
if full_name in parser_config.tree:
module_children.setdefault(full_name, [])
# For something else that's documented,
# figure out what module it lives in
else:
subname = str(full_name)
while True:
subname = subname[:subname.rindex('.')]
if tf_inspect.ismodule(parser_config.index[subname]):
module_name = parser_config.duplicate_of.get(subname, subname)
module_children.setdefault(module_name, []).append(full_name)
break
# Generate docs for `py_object`, resolving references.
page_info = parser.docs_for_object(full_name, py_object, parser_config)
path = os.path.join(output_dir, parser.documentation_path(full_name))
directory = os.path.dirname(path)
try:
if not os.path.exists(directory):
os.makedirs(directory)
# This function returns raw bytes in PY2 or unicode in PY3.
if search_hints:
content = [page_info.get_metadata_html()]
else:
content = ['']
content.append(pretty_docs.build_md_page(page_info))
text = '\n'.join(content)
if six.PY3:
text = text.encode('utf-8')
with open(path, 'wb') as f:
f.write(text)
except OSError:
raise OSError(
'Cannot write documentation for %s to %s' % (full_name, directory))
duplicates = parser_config.duplicates.get(full_name, [])
if not duplicates:
continue
duplicates = [item for item in duplicates if item != full_name]
for dup in duplicates:
from_path = os.path.join(site_path, 'api_docs/python',
dup.replace('.', '/'))
to_path = os.path.join(site_path, 'api_docs/python',
full_name.replace('.', '/'))
redirects.append((
os.path.join('/', from_path),
os.path.join('/', to_path)))
if redirects:
redirects = sorted(redirects)
template = ('- from: {}\n'
' to: {}\n')
redirects = [template.format(f, t) for f, t in redirects]
api_redirects_path = os.path.join(output_dir, '_redirects.yaml')
with open(api_redirects_path, 'w') as redirect_file:
redirect_file.write('redirects:\n')
redirect_file.write(''.join(redirects))
if yaml_toc:
# Generate table of contents
# Put modules in alphabetical order, case-insensitive
modules = sorted(module_children.keys(), key=lambda a: a.upper())
leftnav_path = os.path.join(output_dir, '_toc.yaml')
with open(leftnav_path, 'w') as f:
# Generate header
f.write('# Automatically generated file; please do not edit\ntoc:\n')
for module in modules:
indent_num = module.count('.')
# Don't list `tf.submodule` inside `tf`
indent_num = max(indent_num, 1)
indent = ' '*indent_num
if indent_num > 1:
# tf.contrib.baysflow.entropy will be under
# tf.contrib->baysflow->entropy
title = module.split('.')[-1]
else:
title = module
header = [
'- title: ' + title, ' section:', ' - title: Overview',
' path: ' + os.path.join('/', site_path, symbol_to_file[module])
]
header = ''.join([indent+line+'\n' for line in header])
f.write(header)
symbols_in_module = module_children.get(module, [])
# Sort case-insensitive, if equal sort case sensitive (upper first)
symbols_in_module.sort(key=lambda a: (a.upper(), a))
for full_name in symbols_in_module:
item = [
' - title: ' + full_name[len(module) + 1:], ' path: ' +
os.path.join('/', site_path, symbol_to_file[full_name])
]
item = ''.join([indent+line+'\n' for line in item])
f.write(item)
# Write a global index containing all full names with links.
with open(os.path.join(output_dir, 'index.md'), 'w') as f:
f.write(
parser.generate_global_index(root_title, parser_config.index,
parser_config.reference_resolver))
def write_docs(output_dir,
parser_config,
yaml_toc,
root_title='TensorFlow',
search_hints=True,
site_path='api_docs/python'):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `parser.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
root_title: The title name for the root level index.md.
search_hints: (bool) include meta-data search hints at the top of each
output file.
site_path: The output path relative to the site root. Used in the
`_toc.yaml` and `_redirects.yaml` files.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
output_dir = pathlib.Path(output_dir)
site_path = pathlib.Path('/', site_path)
# Make output_dir.
if not output_dir.is_absolute():
raise ValueError("'output_dir' must be an absolute path.\n"
f" output_dir='{output_dir}'")
output_dir.mkdir(parents=True, exist_ok=True)
# These dictionaries are used for table-of-contents generation below
# They will contain, after the for-loop below::
# - module name(string):classes and functions the module contains(list)
module_children = {}
# Collect redirects for an api _redirects.yaml file.
redirects = []
# Parse and write Markdown pages, resolving cross-links (`tf.symbol`).
for full_name in sorted(parser_config.index.keys(),
key=lambda k: k.lower()):
py_object = parser_config.index[full_name]
if full_name in parser_config.duplicate_of:
continue
# Methods and some routines are documented only as part of their class.
if not (inspect.ismodule(py_object)
or inspect.isclass(py_object) or parser.is_free_function(
py_object, full_name, parser_config.index)):
continue
# Remove the extension from the path.
docpath, _ = os.path.splitext(parser.documentation_path(full_name))
# For a module, remember the module for the table-of-contents
if inspect.ismodule(py_object):
if full_name in parser_config.tree:
mod_obj = Module(module=full_name,
py_object=py_object,
path=str(site_path / docpath))
module_children[full_name] = mod_obj
# For something else that's documented,
# figure out what module it lives in
else:
subname = str(full_name)
while True:
subname = subname[:subname.rindex('.')]
if inspect.ismodule(parser_config.index[subname]):
module_name = parser_config.duplicate_of.get(
subname, subname)
child_mod = ModuleChild(name=full_name,
py_object=py_object,
parent=module_name,
path=str(site_path / docpath))
module_children[module_name].add_children(child_mod)
break
# Generate docs for `py_object`, resolving references.
try:
page_info = parser.docs_for_object(full_name, py_object,
parser_config)
except:
raise ValueError(
f'Failed to generate docs for symbol: `{full_name}`')
path = output_dir / parser.documentation_path(full_name)
try:
path.parent.mkdir(exist_ok=True, parents=True)
# This function returns unicode in PY3.
hidden = doc_controls.should_hide_from_search(page_info.py_object)
if search_hints and not hidden:
content = [page_info.get_metadata_html()]
else:
content = ['<meta name="robots" content="noindex">\n']
content.append(pretty_docs.build_md_page(page_info))
text = '\n'.join(content)
path.write_text(text, encoding='utf-8')
except OSError:
raise OSError('Cannot write documentation for '
f'{full_name} to {path.parent}')
duplicates = parser_config.duplicates.get(full_name, [])
if not duplicates:
continue
duplicates = [item for item in duplicates if item != full_name]
for dup in duplicates:
from_path = site_path / dup.replace('.', '/')
to_path = site_path / full_name.replace('.', '/')
redirects.append({'from': str(from_path), 'to': str(to_path)})
if yaml_toc:
toc_gen = GenerateToc(module_children)
toc_dict = toc_gen.generate()
# Replace the overview path *only* for 'TensorFlow' to
# `/api_docs/python/tf_overview`. This will be redirected to
# `/api_docs/python/tf`.
toc_values = toc_dict['toc'][0]
if toc_values['title'] == 'tf':
section = toc_values['section'][0]
section['path'] = str(site_path / 'tf_overview')
leftnav_toc = output_dir / '_toc.yaml'
with open(leftnav_toc, 'w') as toc_file:
yaml.dump(toc_dict, toc_file, default_flow_style=False)
if redirects:
if yaml_toc and toc_values['title'] == 'tf':
redirects.append({
'from': str(site_path / 'tf_overview'),
'to': str(site_path / 'tf'),
})
redirects_dict = {
'redirects': sorted(redirects,
key=lambda redirect: redirect['from'])
}
api_redirects_path = output_dir / '_redirects.yaml'
with open(api_redirects_path, 'w') as redirect_file:
yaml.dump(redirects_dict, redirect_file, default_flow_style=False)
# Write a global index containing all full names with links.
with open(output_dir / 'index.md', 'w') as f:
f.write(
parser.generate_global_index(root_title, parser_config.index,
parser_config.reference_resolver))