def test_strips_default_arg_memory_address(self):
"""Validates that parser strips memory addresses out out default argspecs.
argspec.defaults can contain object memory addresses, which can change
between invocations. It's desirable to strip these out to reduce churn.
See: `help(collections.MutableMapping.pop)`
"""
m = types.ModuleType('m')
m.__file__ = __file__
m.fun = lambda x=object(): x
generator = generate_lib.DocGenerator(
root_title='test',
py_modules=[('m', m)],
code_url_prefix='https://tensorflow.org')
parser_config = generator.run_extraction()
api_node = doc_generator_visitor.ApiTreeNode(path=('m', 'fun'),
py_object=m.fun)
page_info = docs_for_object.docs_for_object(
api_node=api_node, parser_config=parser_config)
output = str(page_info.signature)
self.assertNotIn('object at 0x', output)
self.assertIn('<object object>', output)
def test_dataclass_attributes_table(self):
index = {
'ExampleDataclass': ExampleDataclass,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = reference_resolver_lib.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {'ExampleDataclass': []}
parser_config = config.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = docs_for_object.docs_for_object(
full_name='ExampleDataclass',
py_object=ExampleDataclass,
parser_config=parser_config)
self.assertCountEqual(
['a', 'b', 'c', 'x', 'y', 'z'],
[name for name, value in page_info.attr_block.items])
def test_docs_for_function_with_kwargs(self):
m = types.ModuleType('m')
m.__file__ = __file__
m.test_function_with_args_kwargs = test_function_with_args_kwargs
generator = generate_lib.DocGenerator(
root_title='test',
py_modules=[('m', m)],
code_url_prefix='https://tensorflow.org')
parser_config = generator.run_extraction()
api_node = doc_generator_visitor.ApiTreeNode(
path=('test_function_with_args_kwargs', ),
py_object=test_function_with_args_kwargs)
page_info = docs_for_object.docs_for_object(
api_node=api_node, parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
inspect.getdoc(test_function_with_args_kwargs).split('\n')[0],
page_info.doc.brief)
# Make sure the extracted signature is good.
self.assertEqual('(\n unused_arg, *unused_args, **unused_kwargs\n)',
str(page_info.signature))
def test_docs_for_function_with_kwargs(self):
index = {
'test_function_with_args_kwargs': test_function_with_args_kwargs
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = reference_resolver_lib.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {'': ['test_function_with_args_kwargs']}
parser_config = config.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = docs_for_object.docs_for_object(
full_name='test_function_with_args_kwargs',
py_object=test_function_with_args_kwargs,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
inspect.getdoc(test_function_with_args_kwargs).split('\n')[0],
page_info.doc.brief)
# Make sure the extracted signature is good.
self.assertEqual(['unused_arg', '*unused_args', '**unused_kwargs'],
page_info.signature.arguments)
def test_docs_for_class_should_skip(self):
class Parent(object):
@doc_controls.do_not_doc_inheritable
def a_method(self, arg='default'):
pass
class Child(Parent):
def a_method(self, arg='default'):
pass
m = types.ModuleType('m')
m.__file__ = __file__
m.Child = Child
generator = generate_lib.DocGenerator(
root_title='test',
py_modules=[('m', m)],
code_url_prefix='https://tensorflow.org')
parser_config = generator.run_extraction()
api_node = doc_generator_visitor.ApiTreeNode(path=(
'm',
'Child',
),
py_object=Child)
page_info = docs_for_object.docs_for_object(
api_node=api_node, parser_config=parser_config)
# Make sure the `a_method` is not present
self.assertEmpty(page_info.methods)
def test_docs_for_module(self):
m = types.ModuleType('m')
m.__file__ = __file__
m.test_function = test_function
m.test_function_with_args_kwargs = test_function_with_args_kwargs
m.TestClass = TestClass
generator = generate_lib.DocGenerator(
root_title='test',
py_modules=[('m', m)],
code_url_prefix='https://tensorflow.org')
parser_config = generator.run_extraction()
api_node = doc_generator_visitor.ApiTreeNode(path=('m', ),
py_object=test_module)
page_info = docs_for_object.docs_for_object(
api_node=api_node, parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
inspect.getdoc(test_module).split('\n')[0], page_info.doc.brief)
# Make sure that the members are there
funcs = {f_info.py_object for f_info in page_info.functions}
self.assertEqual({test_function, test_function_with_args_kwargs},
funcs)
classes = {cls_info.py_object for cls_info in page_info.classes}
self.assertEqual({TestClass}, classes)
def test_strips_default_arg_memory_address(self):
"""Validates that parser strips memory addresses out out default argspecs.
argspec.defaults can contain object memory addresses, which can change
between invocations. It's desirable to strip these out to reduce churn.
See: `help(collections.MutableMapping.pop)`
"""
index = {
'ConcreteMutableMapping': ConcreteMutableMapping,
'ConcreteMutableMapping.pop': ConcreteMutableMapping.pop
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = reference_resolver_lib.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {'ConcreteMutableMapping': ['pop']}
parser_config = config.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = docs_for_object.docs_for_object(
full_name='ConcreteMutableMapping',
py_object=ConcreteMutableMapping,
parser_config=parser_config)
pop_default_arg = page_info.methods[0].signature.arguments[1]
self.assertNotIn('object at 0x', pop_default_arg)
self.assertIn('<object>', pop_default_arg)
def test_getsource_indexerror_resilience(self):
"""Validates that parser gracefully handles IndexErrors.
getsource() can raise an IndexError in some cases. It's unclear
why this happens, but it consistently repros on the `get` method of
collections.MutableMapping subclasses.
"""
m = types.ModuleType('m')
m.__file__ = __file__
m.ConcreteMutableMapping = ConcreteMutableMapping
generator = generate_lib.DocGenerator(
root_title='test',
py_modules=[('m', m)],
code_url_prefix='https://tensorflow.org')
parser_config = generator.run_extraction()
api_node = doc_generator_visitor.ApiTreeNode(
path=('m', 'ConcreteMutableMapping'),
py_object=ConcreteMutableMapping)
page_info = docs_for_object.docs_for_object(
api_node=api_node, parser_config=parser_config)
self.assertIn(ConcreteMutableMapping.get,
[m.py_object for m in page_info.methods])
def test_docs_for_message_class(self):
class CMessage(object):
def hidden(self):
pass
class Message(object):
def hidden2(self):
pass
class MessageMeta(object):
def hidden3(self):
pass
class ChildMessage(CMessage, Message, MessageMeta):
def my_method(self):
pass
index = {
'ChildMessage': ChildMessage,
'ChildMessage.hidden': ChildMessage.hidden,
'ChildMessage.hidden2': ChildMessage.hidden2,
'ChildMessage.hidden3': ChildMessage.hidden3,
'ChildMessage.my_method': ChildMessage.my_method,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = reference_resolver_lib.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {'ChildMessage': ['hidden', 'hidden2', 'hidden3', 'my_method']}
parser_config = config.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = docs_for_object.docs_for_object(
full_name='ChildMessage',
py_object=ChildMessage,
parser_config=parser_config)
self.assertLen(page_info.methods, 1)
self.assertEqual('my_method', page_info.methods[0].short_name)
def test_docs_for_class(self):
m = types.ModuleType('m')
m.__file__ = __file__
m.TestClass = TestClass
generator = generate_lib.DocGenerator(
root_title='test',
py_modules=[('m', m)],
code_url_prefix='https://tensorflow.org')
parser_config = generator.run_extraction()
api_node = doc_generator_visitor.ApiTreeNode(path=(
'm',
'TestClass',
),
py_object=TestClass)
page_info = docs_for_object.docs_for_object(
api_node=api_node, parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
inspect.getdoc(TestClass).split('\n')[0], page_info.doc.brief)
# Make sure the method is present
method_infos = {
method_info.short_name: method_info
for method_info in page_info.methods
}
self.assertIs(method_infos['a_method'].py_object, TestClass.a_method)
# Make sure that the signature is extracted properly and omits self.
self.assertEqual('(\n arg='default'\n)',
str(method_infos['a_method'].signature))
self.assertEqual(method_infos['static_method'].decorators,
['staticmethod'])
self.assertEqual(method_infos['class_method'].decorators,
['classmethod'])
# Make sure the property is present
attrs = page_info.attr_block
self.assertIsInstance(attrs, parser.TitleBlock)
self.assertIn('a_property', [name for name, desc in attrs.items])
# Make sure there is a link to the child class and it points the right way.
self.assertIs(TestClass.ChildClass, page_info.classes[0].py_object)
def _build_page_info(self):
m = types.ModuleType('m')
m.__file__ = __file__
m.TestClass = TestClass
generator = generate_lib.DocGenerator(
root_title='test',
py_modules=[('m', m)],
code_url_prefix='https://tensorflow.org')
parser_config = generator.run_extraction()
api_node = doc_generator_visitor.ApiTreeNode(path=('m', 'TestClass'),
py_object=TestClass)
return docs_for_object.docs_for_object(api_node=api_node,
parser_config=parser_config)
def test_getsource_indexerror_resilience(self):
"""Validates that parser gracefully handles IndexErrors.
inspect.getsource() can raise an IndexError in some cases. It's unclear
why this happens, but it consistently repros on the `get` method of
collections.MutableMapping subclasses.
"""
# This isn't the full set of APIs from MutableMapping, but sufficient for
# testing.
index = {
'ConcreteMutableMapping': ConcreteMutableMapping,
'ConcreteMutableMapping.__init__': ConcreteMutableMapping.__init__,
'ConcreteMutableMapping.__getitem__':
ConcreteMutableMapping.__getitem__,
'ConcreteMutableMapping.__setitem__':
ConcreteMutableMapping.__setitem__,
'ConcreteMutableMapping.values': ConcreteMutableMapping.values,
'ConcreteMutableMapping.get': ConcreteMutableMapping.get
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = reference_resolver_lib.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {
'ConcreteMutableMapping':
['__init__', '__getitem__', '__setitem__', 'values', 'get']
}
parser_config = config.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = docs_for_object.docs_for_object(
full_name='ConcreteMutableMapping',
py_object=ConcreteMutableMapping,
parser_config=parser_config)
self.assertIn(ConcreteMutableMapping.get,
[m.py_object for m in page_info.methods])
def test_namedtuple_field_order(self):
namedtupleclass = collections.namedtuple(
'namedtupleclass', ['z', 'y', 'x', 'hidden', 'w', 'v', 'u'])
index = {
'namedtupleclass': namedtupleclass,
'namedtupleclass.u': namedtupleclass.u,
'namedtupleclass.v': namedtupleclass.v,
'namedtupleclass.w': namedtupleclass.w,
'namedtupleclass.x': namedtupleclass.x,
'namedtupleclass.y': namedtupleclass.y,
'namedtupleclass.z': namedtupleclass.z,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = reference_resolver_lib.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {'namedtupleclass': {'u', 'v', 'w', 'x', 'y', 'z'}}
parser_config = config.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = docs_for_object.docs_for_object(
full_name='namedtupleclass',
py_object=namedtupleclass,
parser_config=parser_config)
self.assertIsNone(page_info._namedtuplefields['hidden'])
# Each namedtiple field has a docstring of the form:
# 'Alias for field number ##'. These props are returned sorted.
def field_number(desc):
return int(desc.split(' ')[-1])
self.assertSequenceEqual(
[0, 1, 2, 4, 5, 6],
[field_number(desc) for name, desc in page_info.attr_block.items])
def test_docs_for_module(self):
index = {
'TestModule': test_module,
'TestModule.test_function': test_function,
'TestModule.test_function_with_args_kwargs':
test_function_with_args_kwargs,
'TestModule.TestClass': TestClass,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = reference_resolver_lib.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {
'TestModule':
['TestClass', 'test_function', 'test_function_with_args_kwargs']
}
parser_config = config.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = docs_for_object.docs_for_object(
full_name='TestModule',
py_object=test_module,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
inspect.getdoc(test_module).split('\n')[0], page_info.doc.brief)
# Make sure that the members are there
funcs = {f_info.py_object for f_info in page_info.functions}
self.assertEqual({test_function, test_function_with_args_kwargs},
funcs)
classes = {cls_info.py_object for cls_info in page_info.classes}
self.assertEqual({TestClass}, classes)
def test_empty_defined_in(self, cls, method, py_object):
"""Validates that the parser omits the defined_in location properly.
This test covers two cases where the parser should omit the defined_in
location:
1. built-ins.
2. methods automatically generated by python attr library.
For built-ins, if without special handling, the defined-in URL ends up like:
http://prefix/<embedded stdlib>/_collections_abc.py
For methods automatically generated by python attr library, if without
special handling, the defined-in URL ends up like:
http://prefix/<attrs generated eq ...>
Args:
cls: The class name to generate docs for.
method: The class method name to generate docs for.
py_object: The python object for the specified cls.method.
"""
visitor = DummyVisitor(index={}, duplicate_of={})
reference_resolver = reference_resolver_lib.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {cls: [method]}
parser_config = config.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index={},
reverse_index={},
base_dir='/',
code_url_prefix='/')
function_info = docs_for_object.docs_for_object(
full_name='%s.%s' % (cls, method),
py_object=py_object,
parser_config=parser_config)
self.assertIsNone(function_info.defined_in)
def test_dataclass_attributes_table(self):
m = types.ModuleType('m')
m.__file__ = __file__
m.ExampleDataclass = ExampleDataclass
generator = generate_lib.DocGenerator(
root_title='test',
py_modules=[('m', m)],
code_url_prefix='https://tensorflow.org')
parser_config = generator.run_extraction()
api_node = doc_generator_visitor.ApiTreeNode(
path=('m', 'ExampleDataclass'), py_object=ExampleDataclass)
page_info = docs_for_object.docs_for_object(
api_node=api_node, parser_config=parser_config)
self.assertCountEqual(
['a', 'b', 'c', 'x', 'y', 'z'],
[name for name, value in page_info.attr_block.items])
def test_docs_for_class_should_skip(self):
class Parent(object):
@doc_controls.do_not_doc_inheritable
def a_method(self, arg='default'):
pass
class Child(Parent):
def a_method(self, arg='default'):
pass
index = {
'Child': Child,
'Child.a_method': Child.a_method,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = reference_resolver_lib.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {
'Child': ['a_method'],
}
parser_config = config.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = docs_for_object.docs_for_object(
full_name='Child', py_object=Child, parser_config=parser_config)
# Make sure the `a_method` is not present
self.assertEmpty(page_info.methods)
def test_empty_defined_in(self, cls, method, py_object):
"""Validates that the parser omits the defined_in location properly.
This test covers two cases where the parser should omit the defined_in
location:
1. built-ins.
2. methods automatically generated by python attr library.
For built-ins, if without special handling, the defined-in URL ends up like:
http://prefix/<embedded stdlib>/_collections_abc.py
For methods automatically generated by python attr library, if without
special handling, the defined-in URL ends up like:
http://prefix/<attrs generated eq ...>
Args:
cls: The class name to generate docs for.
method: The class method name to generate docs for.
py_object: The python object for the specified cls.method.
"""
m = types.ModuleType('m')
m.__file__ = __file__
m.ConcreteMutableMapping = ConcreteMutableMapping
generator = generate_lib.DocGenerator(
root_title='test',
py_modules=[('m', m)],
code_url_prefix='https://tensorflow.org')
parser_config = generator.run_extraction()
api_node = doc_generator_visitor.ApiTreeNode(path=(cls, method),
py_object=py_object)
function_info = docs_for_object.docs_for_object(
api_node=api_node, parser_config=parser_config)
self.assertIsNone(function_info.defined_in)
def test_namedtuple_field_order_respects_hidden(self):
namedtupleclass = collections.namedtuple(
'namedtupleclass', ['z', 'y', 'x', 'hidden', 'w', 'v', 'u'])
m = types.ModuleType('m')
m.__file__ = __file__
m.namedtupleclass = namedtupleclass
def hide(path, parent, children):
return [(name, value) for name, value in children
if name != 'hidden']
generator = generate_lib.DocGenerator(
root_title='test',
py_modules=[('m', m)],
code_url_prefix='https://tensorflow.org',
callbacks=[hide])
parser_config = generator.run_extraction()
api_node = doc_generator_visitor.ApiTreeNode(path=('m',
'namedtupleclass'),
py_object=namedtupleclass)
page_info = docs_for_object.docs_for_object(
api_node=api_node, parser_config=parser_config)
self.assertIsNone(page_info._namedtuplefields['hidden'])
# Each namedtiple field has a docstring of the form:
# 'Alias for field number ##'. These props are returned sorted.
def field_number(desc):
return int(desc.split(' ')[-1])
self.assertSequenceEqual(
[0, 1, 2, 4, 5, 6],
[field_number(desc) for name, desc in page_info.attr_block.items])
def test_docs_for_message_class(self):
class CMessage(object):
def hidden(self):
pass
class Message(object):
def hidden2(self):
pass
class MessageMeta(object):
def hidden3(self):
pass
class ChildMessage(CMessage, Message, MessageMeta):
def my_method(self):
pass
m = types.ModuleType('m')
m.__file__ = __file__
m.ChildMessage = ChildMessage
generator = generate_lib.DocGenerator(
root_title='test',
py_modules=[('m', m)],
code_url_prefix='https://tensorflow.org')
parser_config = generator.run_extraction()
api_node = doc_generator_visitor.ApiTreeNode(path=('m',
'ChildMessage'),
py_object=ChildMessage)
page_info = docs_for_object.docs_for_object(
api_node=api_node, parser_config=parser_config)
self.assertLen(page_info.methods, 1)
self.assertEqual('my_method', page_info.methods[0].short_name)
def test_docs_for_class(self):
index = {
'TestClass': TestClass,
'TestClass.a_method': TestClass.a_method,
'TestClass.a_property': TestClass.a_property,
'TestClass.ChildClass': TestClass.ChildClass,
'TestClass.static_method': TestClass.static_method,
'TestClass.class_method': TestClass.class_method,
'TestClass.CLASS_MEMBER': TestClass.CLASS_MEMBER,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = reference_resolver_lib.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {
'TestClass': [
'a_method', 'class_method', 'static_method', 'a_property',
'ChildClass', 'CLASS_MEMBER'
]
}
parser_config = config.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = docs_for_object.docs_for_object(
full_name='TestClass',
py_object=TestClass,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
inspect.getdoc(TestClass).split('\n')[0], page_info.doc.brief)
# Make sure the method is present
method_infos = {
method_info.short_name: method_info
for method_info in page_info.methods
}
self.assertIs(method_infos['a_method'].py_object, TestClass.a_method)
# Make sure that the signature is extracted properly and omits self.
self.assertEqual(['arg='default''],
method_infos['a_method'].signature.arguments)
self.assertEqual(method_infos['static_method'].decorators,
['staticmethod'])
self.assertEqual(method_infos['class_method'].decorators,
['classmethod'])
# Make sure the property is present
attrs = page_info.attr_block
self.assertIsInstance(attrs, parser.TitleBlock)
self.assertIn('a_property', [name for name, desc in attrs.items])
# Make sure there is a link to the child class and it points the right way.
self.assertIs(TestClass.ChildClass, page_info.classes[0].py_object)
def write_docs(
*,
output_dir: Union[str, pathlib.Path],
parser_config: config.ParserConfig,
yaml_toc: Union[bool, Type[toc_lib.TocBuilder]],
root_module_name: str,
root_title: str = 'TensorFlow',
search_hints: bool = True,
site_path: str = 'api_docs/python',
gen_redirects: bool = True,
gen_report: bool = True,
extra_docs: Optional[Dict[int, str]] = None,
page_builder_classes: Optional[docs_for_object.PageBuilderDict] = None,
):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `config.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
root_module_name: (str) the name of the root module (`tf` for tensorflow).
root_title: The title name for the root level index.md.
search_hints: (bool) include meta-data search hints at the top of each
output file.
site_path: The output path relative to the site root. Used in the
`_toc.yaml` and `_redirects.yaml` files.
gen_redirects: Bool which decides whether to generate _redirects.yaml file
or not.
gen_report: If True, a report for the library is generated by linting the
docstrings of its public API symbols.
extra_docs: To add docs for a particular object instance set it's __doc__
attribute. For some classes (list, tuple, etc) __doc__ is not writable.
Pass those docs like: `extra_docs={id(obj): "docs"}`
page_builder_classes: A optional dict of `{ObjectType:Type[PageInfo]}` for
overriding the default page builder classes.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
output_dir = pathlib.Path(output_dir)
site_path = pathlib.Path('/', site_path)
# Make output_dir.
if not output_dir.is_absolute():
raise ValueError("'output_dir' must be an absolute path.\n"
f" output_dir='{output_dir}'")
output_dir.mkdir(parents=True, exist_ok=True)
# Collect redirects for an api _redirects.yaml file.
redirects = []
api_report = None
if gen_report:
api_report = utils.ApiReport()
# Parse and write Markdown pages, resolving cross-links (`tf.symbol`).
num_docs_output = 0
for api_node in parser_config.api_tree.iter_nodes():
full_name = api_node.full_name
if api_node.output_type() is api_node.OutputType.FRAGMENT:
continue
# Generate docs for `py_object`, resolving references.
try:
page_info = docs_for_object.docs_for_object(
api_node=api_node,
parser_config=parser_config,
extra_docs=extra_docs,
search_hints=search_hints,
page_builder_classes=page_builder_classes)
if api_report is not None and not full_name.startswith(
('tf.compat.v', 'tf.keras.backend', 'tf.numpy',
'tf.experimental.numpy')):
api_report.fill_metrics(page_info)
except Exception as e:
raise ValueError(
f'Failed to generate docs for symbol: `{full_name}`') from e
path = output_dir / parser.documentation_path(full_name)
try:
path.parent.mkdir(exist_ok=True, parents=True)
path.write_text(page_info.page_text, encoding='utf-8')
num_docs_output += 1
except OSError as e:
raise OSError('Cannot write documentation for '
f'{full_name} to {path.parent}') from e
duplicates = parser_config.duplicates.get(full_name, [])
if not duplicates:
continue
duplicates = [item for item in duplicates if item != full_name]
if gen_redirects:
for dup in duplicates:
from_path = site_path / dup.replace('.', '/')
to_path = site_path / full_name.replace('.', '/')
redirects.append({'from': str(from_path), 'to': str(to_path)})
if api_report is not None:
api_report.write(output_dir / root_module_name / 'api_report.pb')
if num_docs_output <= 1:
raise ValueError(
'The `DocGenerator` failed to generate any docs. Verify '
'your arguments (`base_dir` and `callbacks`). '
'Everything you want documented should be within '
'`base_dir`.')
if yaml_toc:
if isinstance(yaml_toc, bool):
yaml_toc = toc_lib.FlatModulesTocBuilder
toc = yaml_toc(site_path).build(parser_config.api_tree)
toc_path = output_dir / root_module_name / '_toc.yaml'
toc.write(toc_path)
if redirects and gen_redirects:
redirects_dict = {
'redirects': sorted(redirects,
key=lambda redirect: redirect['from'])
}
api_redirects_path = output_dir / root_module_name / '_redirects.yaml'
with open(api_redirects_path, 'w') as redirect_file:
yaml.dump(redirects_dict, redirect_file, default_flow_style=False)
# Write a global index containing all full names with links.
with open(output_dir / root_module_name / 'all_symbols.md', 'w') as f:
global_index = parser.generate_global_index(
root_title, parser_config.index, parser_config.reference_resolver)
if not search_hints:
global_index = 'robots: noindex\n' + global_index
f.write(global_index)
def write_docs(
*,
output_dir: Union[str, pathlib.Path],
parser_config: config.ParserConfig,
yaml_toc: bool,
root_module_name: str,
root_title: str = 'TensorFlow',
search_hints: bool = True,
site_path: str = 'api_docs/python',
gen_redirects: bool = True,
gen_report: bool = True,
extra_docs: Optional[Dict[int, str]] = None,
page_builder_classes: Optional[docs_for_object.PageBuilderDict] = None,
):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `config.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
root_module_name: (str) the name of the root module (`tf` for tensorflow).
root_title: The title name for the root level index.md.
search_hints: (bool) include meta-data search hints at the top of each
output file.
site_path: The output path relative to the site root. Used in the
`_toc.yaml` and `_redirects.yaml` files.
gen_redirects: Bool which decides whether to generate _redirects.yaml file
or not.
gen_report: If True, a report for the library is generated by linting the
docstrings of its public API symbols.
extra_docs: To add docs for a particular object instance set it's __doc__
attribute. For some classes (list, tuple, etc) __doc__ is not writable.
Pass those docs like: `extra_docs={id(obj): "docs"}`
page_builder_classes: A optional dict of `{ObjectType:Type[PageInfo]}` for
overriding the default page builder classes.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
output_dir = pathlib.Path(output_dir)
site_path = pathlib.Path('/', site_path)
# Make output_dir.
if not output_dir.is_absolute():
raise ValueError("'output_dir' must be an absolute path.\n"
f" output_dir='{output_dir}'")
output_dir.mkdir(parents=True, exist_ok=True)
# These dictionaries are used for table-of-contents generation below
# They will contain, after the for-loop below::
# - module name(string):classes and functions the module contains(list)
module_children = {}
# Collect redirects for an api _redirects.yaml file.
redirects = []
if gen_report:
api_report_obj = utils.ApiReport()
# Parse and write Markdown pages, resolving cross-links (`tf.symbol`).
num_docs_output = 0
for full_name in sorted(parser_config.index.keys(),
key=lambda k: k.lower()):
py_object = parser_config.index[full_name]
if full_name in parser_config.duplicate_of:
continue
# Methods constants are only documented only as part of their parent's page.
if parser_config.reference_resolver.is_fragment(full_name):
continue
# Remove the extension from the path.
docpath, _ = os.path.splitext(parser.documentation_path(full_name))
# For a module, remember the module for the table-of-contents
if inspect.ismodule(py_object):
if full_name in parser_config.tree:
mod_obj = Module(module=full_name,
py_object=py_object,
path=str(site_path / docpath))
module_children[full_name] = mod_obj
# For something else that's documented,
# figure out what module it lives in
else:
subname = str(full_name)
while True:
subname = subname[:subname.rindex('.')]
if inspect.ismodule(parser_config.index[subname]):
module_name = parser_config.duplicate_of.get(
subname, subname)
child_mod = ModuleChild(name=full_name,
py_object=py_object,
parent=module_name,
path=str(site_path / docpath))
module_children[module_name].add_children(child_mod)
break
# Generate docs for `py_object`, resolving references.
try:
page_info = docs_for_object.docs_for_object(
full_name=full_name,
py_object=py_object,
parser_config=parser_config,
extra_docs=extra_docs,
search_hints=search_hints,
page_builder_classes=page_builder_classes)
if gen_report and not full_name.startswith(
('tf.compat.v', 'tf.keras.backend', 'tf.numpy',
'tf.experimental.numpy')):
api_report_obj.fill_metrics(page_info)
except Exception as e:
raise ValueError(
f'Failed to generate docs for symbol: `{full_name}`') from e
path = output_dir / parser.documentation_path(full_name)
text = page_info.build()
try:
path.parent.mkdir(exist_ok=True, parents=True)
path.write_text(text, encoding='utf-8')
num_docs_output += 1
except OSError:
raise OSError('Cannot write documentation for '
f'{full_name} to {path.parent}')
duplicates = parser_config.duplicates.get(full_name, [])
if not duplicates:
continue
duplicates = [item for item in duplicates if item != full_name]
if gen_redirects:
for dup in duplicates:
from_path = site_path / dup.replace('.', '/')
to_path = site_path / full_name.replace('.', '/')
redirects.append({'from': str(from_path), 'to': str(to_path)})
if gen_report:
serialized_proto = api_report_obj.api_report.SerializeToString()
raw_proto = output_dir / root_module_name / 'api_report.pb'
raw_proto.write_bytes(serialized_proto)
if num_docs_output <= 1:
raise ValueError(
'The `DocGenerator` failed to generate any docs. Verify '
'your arguments (`base_dir` and `callbacks`). '
'Everything you want documented should be within '
'`base_dir`.')
if yaml_toc:
toc_gen = GenerateToc(module_children)
toc_dict = toc_gen.generate()
# Replace the overview path *only* for 'TensorFlow' to
# `/api_docs/python/tf_overview`. This will be redirected to
# `/api_docs/python/tf`.
toc_values = toc_dict['toc'][0]
if toc_values['title'] == 'tf':
section = toc_values['section'][0]
section['path'] = str(site_path / 'tf_overview')
leftnav_toc = output_dir / root_module_name / '_toc.yaml'
with open(leftnav_toc, 'w') as toc_file:
yaml.dump(toc_dict, toc_file, default_flow_style=False)
if redirects and gen_redirects:
if yaml_toc and toc_values['title'] == 'tf':
redirects.append({
'from': str(site_path / 'tf_overview'),
'to': str(site_path / 'tf'),
})
redirects_dict = {
'redirects': sorted(redirects,
key=lambda redirect: redirect['from'])
}
api_redirects_path = output_dir / root_module_name / '_redirects.yaml'
with open(api_redirects_path, 'w') as redirect_file:
yaml.dump(redirects_dict, redirect_file, default_flow_style=False)
# Write a global index containing all full names with links.
with open(output_dir / root_module_name / 'all_symbols.md', 'w') as f:
global_index = parser.generate_global_index(
root_title, parser_config.index, parser_config.reference_resolver)
if not search_hints:
global_index = 'robots: noindex\n' + global_index
f.write(global_index)
