def test_example_lint(self):
class_page_info = parser.docs_for_object(
full_name='TestClass',
py_object=TestClass,
parser_config=self.parser_config)
test_api_report = utils.ApiReport()
test_api_report.fill_metrics(class_page_info)
for test_report in test_api_report.api_report.symbol_metric:
if (test_report.symbol_name == 'TestClass'
and test_report.object_type
== api_report_pb2.ObjectType.CLASS):
self.assertEqual(test_report.usage_example_lint.num_doctest, 2)
self.assertEqual(
test_report.usage_example_lint.num_untested_examples, 1)
self.assertEqual(test_report.package_group, 'TestClass')
if (test_report.symbol_name == 'TestClass.method_one'
and test_report.object_type
== api_report_pb2.ObjectType.METHOD):
self.assertEqual(test_report.usage_example_lint.num_doctest, 0)
self.assertEqual(
test_report.usage_example_lint.num_untested_examples, 1)
self.assertEqual(test_report.package_group, 'TestClass')
def test_parameter_lint(self):
class_page_info = parser.docs_for_object(
full_name='TestClass',
py_object=TestClass,
parser_config=self.parser_config)
test_api_report = utils.ApiReport()
test_api_report.fill_metrics(class_page_info)
for test_report in test_api_report.api_report.symbol_metric:
if (test_report.symbol_name == 'TestClass'
and test_report.object_type
== api_report_pb2.ObjectType.CLASS):
self.assertEqual(
test_report.parameter_lint.num_empty_param_desc_args, 2)
self.assertEqual(test_report.parameter_lint.num_args_in_doc, 4)
self.assertEqual(test_report.parameter_lint.num_args_in_code,
3)
self.assertEqual(
test_report.parameter_lint.num_empty_param_desc_attr, 1)
self.assertEqual(test_report.parameter_lint.total_attr_param,
2)
if (test_report.symbol_name == 'TestClass.method_one'
and test_report.object_type
== api_report_pb2.ObjectType.METHOD):
self.assertEqual(
test_report.parameter_lint.num_empty_param_desc_args, 0)
self.assertEqual(test_report.parameter_lint.num_args_in_doc, 1)
self.assertEqual(test_report.parameter_lint.num_args_in_code,
1)
self.assertEqual(
test_report.parameter_lint.num_empty_param_desc_attr, 0)
self.assertEqual(test_report.parameter_lint.total_attr_param,
0)
def test_docs_for_function_with_kwargs(self):
index = {
'test_function_with_args_kwargs': test_function_with_args_kwargs
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {'': ['test_function_with_args_kwargs']}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(
full_name='test_function_with_args_kwargs',
py_object=test_function_with_args_kwargs,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
inspect.getdoc(test_function_with_args_kwargs).split('\n')[0],
page_info.doc.brief)
# Make sure the extracted signature is good.
self.assertEqual(['unused_arg', '*unused_args', '**unused_kwargs'],
page_info.signature)
def test_dataclass_attributes_table(self):
index = {
'ExampleDataclass': ExampleDataclass,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {'ExampleDataclass': []}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(full_name='ExampleDataclass',
py_object=ExampleDataclass,
parser_config=parser_config)
self.assertCountEqual(
['a', 'b', 'c', 'x', 'y', 'z'],
[name for name, value in page_info.attr_block.items])
def test_docs_for_function(self):
index = {'test_function': test_function}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {'': ['test_function']}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(full_name='test_function',
py_object=test_function,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
inspect.getdoc(test_function).split('\n')[0], page_info.doc.brief)
# Make sure the extracted signature is good.
self.assertEqual(['unused_arg', "unused_kwarg='default'"],
page_info.signature)
# Make sure this file is contained as the definition location.
self.assertEqual(os.path.relpath(__file__, '/'),
page_info.defined_in.rel_path)
def test_strips_default_arg_memory_address(self):
"""Validates that parser strips memory addresses out out default argspecs.
argspec.defaults can contain object memory addresses, which can change
between invocations. It's desirable to strip these out to reduce churn.
See: `help(collections.MutableMapping.pop)`
"""
index = {
'ConcreteMutableMapping': ConcreteMutableMapping,
'ConcreteMutableMapping.pop': ConcreteMutableMapping.pop
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {'ConcreteMutableMapping': ['pop']}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(full_name='ConcreteMutableMapping',
py_object=ConcreteMutableMapping,
parser_config=parser_config)
pop_default_arg = page_info.methods[0].signature[1]
self.assertNotIn('object at 0x', pop_default_arg)
self.assertIn('<object>', pop_default_arg)
def test_builtins_defined_in(self, cls, method, py_object):
"""Validates that the parser omits the defined_in location for built-ins.
Without special handling, the defined-in URL ends up like:
http://prefix/<embedded stdlib>/_collections_abc.py
Args:
cls: The class name to generate docs for.
method: The class method name to generate docs for.
py_object: The python object for the specified cls.method.
"""
visitor = DummyVisitor(index={}, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {cls: [method]}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index={},
reverse_index={},
base_dir='/',
code_url_prefix='/')
function_info = parser.docs_for_object(full_name='%s.%s' %
(cls, method),
py_object=py_object,
parser_config=parser_config)
self.assertIsNone(function_info.defined_in)
def test_builtins_defined_in(self):
"""Validates that the parser omits the defined_in location for built-ins.
Without special handling, the defined-in URL ends up like:
http://prefix/<embedded stdlib>/_collections_abc.py
"""
visitor = DummyVisitor(index={}, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {'ConcreteMutableMapping': ['__contains__']}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index={},
reverse_index={},
base_dir='/',
code_url_prefix='/')
function_info = parser.docs_for_object(
full_name='ConcreteMutableMapping.__contains__',
py_object=ConcreteMutableMapping.__contains__,
parser_config=parser_config)
self.assertIsNone(function_info.defined_in)
def test_docs_for_message_class(self):
class CMessage(object):
def hidden(self):
pass
class Message(object):
def hidden2(self):
pass
class MessageMeta(object):
def hidden3(self):
pass
class ChildMessage(CMessage, Message, MessageMeta):
def my_method(self):
pass
index = {
'ChildMessage': ChildMessage,
'ChildMessage.hidden': ChildMessage.hidden,
'ChildMessage.hidden2': ChildMessage.hidden2,
'ChildMessage.hidden3': ChildMessage.hidden3,
'ChildMessage.my_method': ChildMessage.my_method,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, doc_index={}, py_module_names=['tf'])
tree = {'ChildMessage': ['hidden', 'hidden2', 'hidden3', 'my_method']}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
guide_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(
full_name='ChildMessage',
py_object=ChildMessage,
parser_config=parser_config)
self.assertEqual(1, len(page_info.methods))
self.assertEqual('my_method', page_info.methods[0].short_name)
def test_docs_for_message_class(self):
class CMessage(object):
def hidden(self):
pass
class Message(object):
def hidden2(self):
pass
class MessageMeta(object):
def hidden3(self):
pass
class ChildMessage(CMessage, Message, MessageMeta):
def my_method(self):
pass
index = {
'ChildMessage': ChildMessage,
'ChildMessage.hidden': ChildMessage.hidden,
'ChildMessage.hidden2': ChildMessage.hidden2,
'ChildMessage.hidden3': ChildMessage.hidden3,
'ChildMessage.my_method': ChildMessage.my_method,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {'ChildMessage': ['hidden', 'hidden2', 'hidden3', 'my_method']}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(
full_name='ChildMessage',
py_object=ChildMessage,
parser_config=parser_config)
self.assertLen(page_info.methods, 1)
self.assertEqual('my_method', page_info.methods[0].short_name)
def test_docs_for_module(self):
index = {
'TestModule':
test_module,
'TestModule.test_function':
test_function,
'TestModule.test_function_with_args_kwargs':
test_function_with_args_kwargs,
'TestModule.TestClass':
TestClass,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {
'TestModule': [
'TestClass', 'test_function', 'test_function_with_args_kwargs'
]
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(
full_name='TestModule',
py_object=test_module,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
inspect.getdoc(test_module).split('\n')[0], page_info.doc.brief)
# Make sure that the members are there
funcs = {f_info.obj for f_info in page_info.functions}
self.assertEqual({test_function, test_function_with_args_kwargs}, funcs)
classes = {cls_info.obj for cls_info in page_info.classes}
self.assertEqual({TestClass}, classes)
# Make sure the module's file is contained as the definition location.
self.assertEqual(
os.path.relpath(test_module.__file__.rstrip('c'), '/'),
page_info.defined_in.rel_path)
def test_docs_for_class(self):
index = {
'TestClass': TestClass,
'TestClass.a_method': TestClass.a_method,
'TestClass.a_property': TestClass.a_property,
'TestClass.ChildClass': TestClass.ChildClass,
'TestClass.CLASS_MEMBER': TestClass.CLASS_MEMBER
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {
'TestClass':
['a_method', 'a_property', 'ChildClass', 'CLASS_MEMBER']
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(full_name='TestClass',
py_object=TestClass,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
inspect.getdoc(TestClass).split('\n')[0], page_info.doc.brief)
# Make sure the method is present
self.assertEqual(TestClass.a_method, page_info.methods[0].obj)
# Make sure that the signature is extracted properly and omits self.
self.assertEqual(["arg='default'"], page_info.methods[0].signature)
# Make sure the property is present
self.assertIs(TestClass.a_property, page_info.properties[0].obj)
# Make sure there is a link to the child class and it points the right way.
self.assertIs(TestClass.ChildClass, page_info.classes[0].obj)
# Make sure this file is contained as the definition location.
self.assertEqual(os.path.relpath(__file__, '/'),
page_info.defined_in.rel_path)
def test_method_return_lint(self):
class_page_info = parser.docs_for_object(
full_name='TestClass',
py_object=TestClass,
parser_config=self.parser_config)
test_api_report = utils.ApiReport()
test_api_report.fill_metrics(class_page_info)
for test_report in test_api_report.api_report.symbol_metric:
if (test_report.symbol_name == 'TestClass.method_one' and
test_report.object_type == api_report_pb2.ObjectType.METHOD):
self.assertTrue(test_report.return_lint.returns_defined)
def test_docs_for_module(self):
index = {
'TestModule':
test_module,
'TestModule.test_function':
test_function,
'TestModule.test_function_with_args_kwargs':
test_function_with_args_kwargs,
'TestModule.TestClass':
TestClass,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {
'TestModule': ['TestClass', 'test_function',
'test_function_with_args_kwargs']
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(
full_name='TestModule',
py_object=test_module,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
tf_inspect.getdoc(test_module).split('\n')[0], page_info.doc.brief)
# Make sure that the members are there
funcs = {f_info.obj for f_info in page_info.functions}
self.assertEqual({test_function, test_function_with_args_kwargs}, funcs)
classes = {cls_info.obj for cls_info in page_info.classes}
self.assertEqual({TestClass}, classes)
# Make sure the module's file is contained as the definition location.
self.assertEqual(
os.path.relpath(test_module.__file__, '/'), page_info.defined_in.path)
def test_class_raises_lint(self):
class_page_info = parser.docs_for_object(
full_name='TestClass',
py_object=TestClass,
parser_config=self.parser_config)
test_api_report = utils.ApiReport()
test_api_report.fill_metrics(class_page_info)
for test_report in test_api_report.api_report.symbol_metric:
if (test_report.symbol_name == 'TestClass' and
test_report.object_type == api_report_pb2.ObjectType.CLASS):
self.assertEqual(test_report.raises_lint.num_raises_defined, 2)
self.assertEqual(test_report.raises_lint.total_raises_in_code, 2)
def test_getsource_indexerror_resilience(self):
"""Validates that parser gracefully handles IndexErrors.
tf_inspect.getsource() can raise an IndexError in some cases. It's unclear
why this happens, but it consistently repros on the `get` method of
collections.MutableMapping subclasses.
"""
# This isn't the full set of APIs from MutableMapping, but sufficient for
# testing.
index = {
'ConcreteMutableMapping':
ConcreteMutableMapping,
'ConcreteMutableMapping.__init__':
ConcreteMutableMapping.__init__,
'ConcreteMutableMapping.__getitem__':
ConcreteMutableMapping.__getitem__,
'ConcreteMutableMapping.__setitem__':
ConcreteMutableMapping.__setitem__,
'ConcreteMutableMapping.values':
ConcreteMutableMapping.values,
'ConcreteMutableMapping.get':
ConcreteMutableMapping.get
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {
'ConcreteMutableMapping': [
'__init__', '__getitem__', '__setitem__', 'values', 'get'
]
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(
full_name='ConcreteMutableMapping',
py_object=ConcreteMutableMapping,
parser_config=parser_config)
self.assertIn(ConcreteMutableMapping.get,
[m.obj for m in page_info.methods])
def test_docs_for_class(self):
index = {
'TestClass': TestClass,
'TestClass.a_method': TestClass.a_method,
'TestClass.a_property': TestClass.a_property,
'TestClass.ChildClass': TestClass.ChildClass,
'TestClass.CLASS_MEMBER': TestClass.CLASS_MEMBER
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {
'TestClass': ['a_method', 'a_property', 'ChildClass', 'CLASS_MEMBER']
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(
full_name='TestClass', py_object=TestClass, parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
tf_inspect.getdoc(TestClass).split('\n')[0], page_info.doc.brief)
# Make sure the method is present
self.assertEqual(TestClass.a_method, page_info.methods[0].obj)
# Make sure that the signature is extracted properly and omits self.
self.assertEqual(["arg='default'"], page_info.methods[0].signature)
# Make sure the property is present
self.assertIs(TestClass.a_property, page_info.properties[0].obj)
# Make sure there is a link to the child class and it points the right way.
self.assertIs(TestClass.ChildClass, page_info.classes[0].obj)
# Make sure this file is contained as the definition location.
self.assertEqual(os.path.relpath(__file__, '/'), page_info.defined_in.path)
def test_namedtuple_field_order(self):
namedtupleclass = collections.namedtuple(
'namedtupleclass', ['z', 'y', 'x', 'hidden', 'w', 'v', 'u'])
index = {
'namedtupleclass': namedtupleclass,
'namedtupleclass.u': namedtupleclass.u,
'namedtupleclass.v': namedtupleclass.v,
'namedtupleclass.w': namedtupleclass.w,
'namedtupleclass.x': namedtupleclass.x,
'namedtupleclass.y': namedtupleclass.y,
'namedtupleclass.z': namedtupleclass.z,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {'namedtupleclass': {'u', 'v', 'w', 'x', 'y', 'z'}}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(
full_name='namedtupleclass',
py_object=namedtupleclass,
parser_config=parser_config)
self.assertIsNone(page_info._namedtuplefields['hidden'])
# Each namedtiple field has a docstring of the form:
# 'Alias for field number ##'. These props are returned sorted.
def field_number(desc):
return int(desc.split(' ')[-1])
self.assertSequenceEqual(
[0, 1, 2, 4, 5, 6],
[field_number(desc) for name, desc in page_info.attr_block.items])
def test_namedtuple_field_order(self):
namedtupleclass = collections.namedtuple('namedtupleclass',
{'z', 'y', 'x', 'w', 'v', 'u'})
index = {
'namedtupleclass': namedtupleclass,
'namedtupleclass.u': namedtupleclass.u,
'namedtupleclass.v': namedtupleclass.v,
'namedtupleclass.w': namedtupleclass.w,
'namedtupleclass.x': namedtupleclass.x,
'namedtupleclass.y': namedtupleclass.y,
'namedtupleclass.z': namedtupleclass.z,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, doc_index={}, py_module_names=['tf'])
tree = {'namedtupleclass': {'u', 'v', 'w', 'x', 'y', 'z'}}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
guide_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(
full_name='namedtupleclass',
py_object=namedtupleclass,
parser_config=parser_config)
# Each namedtiple field has a docstring of the form:
# 'Alias for field number ##'. These props are returned sorted.
def sort_key(prop_info):
return int(prop_info.obj.__doc__.split(' ')[-1])
self.assertSequenceEqual(page_info.properties,
sorted(page_info.properties, key=sort_key))
def test_description_lint(self):
class_page_info = parser.docs_for_object(
full_name='TestClass',
py_object=TestClass,
parser_config=self.parser_config)
test_api_report = utils.ApiReport()
test_api_report.fill_metrics(class_page_info)
for test_report in test_api_report.api_report.symbol_metric:
if (test_report.symbol_name == 'TestClass' and
test_report.object_type == api_report_pb2.ObjectType.CLASS):
self.assertEqual(test_report.desc_lint.len_brief, 2)
self.assertEqual(test_report.desc_lint.len_long_desc, 54)
if (test_report.symbol_name == 'TestClass.method_one' and
test_report.object_type == api_report_pb2.ObjectType.METHOD):
self.assertEqual(test_report.desc_lint.len_brief, 4)
self.assertEqual(test_report.desc_lint.len_long_desc, 10)
def test_docs_for_class_should_skip(self):
class Parent(object):
@doc_controls.do_not_doc_inheritable
def a_method(self, arg='default'):
pass
class Child(Parent):
def a_method(self, arg='default'):
pass
index = {
'Child': Child,
'Child.a_method': Child.a_method,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, doc_index={}, py_module_names=['tf'])
tree = {
'Child': ['a_method'],
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
guide_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(
full_name='Child', py_object=Child, parser_config=parser_config)
# Make sure the `a_method` is not present
self.assertEqual(0, len(page_info.methods))
def test_namedtuple_field_order(self):
namedtupleclass = collections.namedtuple('namedtupleclass',
{'z', 'y', 'x', 'w', 'v', 'u'})
index = {
'namedtupleclass': namedtupleclass,
'namedtupleclass.u': namedtupleclass.u,
'namedtupleclass.v': namedtupleclass.v,
'namedtupleclass.w': namedtupleclass.w,
'namedtupleclass.x': namedtupleclass.x,
'namedtupleclass.y': namedtupleclass.y,
'namedtupleclass.z': namedtupleclass.z,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {'namedtupleclass': {'u', 'v', 'w', 'x', 'y', 'z'}}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(
full_name='namedtupleclass',
py_object=namedtupleclass,
parser_config=parser_config)
# Each namedtiple field has a docstring of the form:
# 'Alias for field number ##'. These props are returned sorted.
def sort_key(prop_info):
return int(prop_info.obj.__doc__.split(' ')[-1])
self.assertSequenceEqual(page_info.properties,
sorted(page_info.properties, key=sort_key))
def test_docs_for_class_should_skip(self):
class Parent(object):
@doc_controls.do_not_doc_inheritable
def a_method(self, arg='default'):
pass
class Child(Parent):
def a_method(self, arg='default'):
pass
index = {
'Child': Child,
'Child.a_method': Child.a_method,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {
'Child': ['a_method'],
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(
full_name='Child', py_object=Child, parser_config=parser_config)
# Make sure the `a_method` is not present
self.assertEmpty(page_info.methods)
def test_docs_for_function(self):
index = {
'test_function': test_function
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {
'': ['test_function']
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(
full_name='test_function',
py_object=test_function,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
tf_inspect.getdoc(test_function).split('\n')[0], page_info.doc.brief)
# Make sure the extracted signature is good.
self.assertEqual(['unused_arg', "unused_kwarg='default'"],
page_info.signature)
# Make sure this file is contained as the definition location.
self.assertEqual(os.path.relpath(__file__, '/'), page_info.defined_in.path)
def test_docs_for_function_with_kwargs(self):
index = {
'test_function_with_args_kwargs': test_function_with_args_kwargs
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {
'': ['test_function_with_args_kwargs']
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(
full_name='test_function_with_args_kwargs',
py_object=test_function_with_args_kwargs,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
tf_inspect.getdoc(test_function_with_args_kwargs).split('\n')[0],
page_info.doc.brief)
# Make sure the extracted signature is good.
self.assertEqual(['unused_arg', '*unused_args', '**unused_kwargs'],
page_info.signature)
def write_docs(output_dir,
parser_config,
yaml_toc,
root_title='TensorFlow',
search_hints=True,
site_path=''):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `parser.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
root_title: The title name for the root level index.md.
search_hints: (bool) include meta-data search hints at the top of each
output file.
site_path: The output path relative to the site root. Used in the
`_toc.yaml` and `_redirects.yaml` files.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
# Make output_dir.
if not os.path.isabs(output_dir):
raise ValueError("'output_dir' must be an absolute path.\n"
" output_dir='%s'" % output_dir)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# These dictionaries are used for table-of-contents generation below
# They will contain, after the for-loop below::
# - module name(string):classes and functions the module contains(list)
module_children = {}
# - symbol name(string):pathname (string)
symbol_to_file = {}
# Collect redirects for an api _redirects.yaml file.
redirects = []
# Parse and write Markdown pages, resolving cross-links (`tf.symbol`).
for full_name, py_object in six.iteritems(parser_config.index):
parser_config.reference_resolver.current_doc_full_name = full_name
if full_name in parser_config.duplicate_of:
continue
# Methods and some routines are documented only as part of their class.
if not (tf_inspect.ismodule(py_object)
or tf_inspect.isclass(py_object) or parser.is_free_function(
py_object, full_name, parser_config.index)):
continue
sitepath = os.path.join('api_docs/python',
parser.documentation_path(full_name)[:-3])
# For TOC, we need to store a mapping from full_name to the file
# we're generating
symbol_to_file[full_name] = sitepath
# For a module, remember the module for the table-of-contents
if tf_inspect.ismodule(py_object):
if full_name in parser_config.tree:
module_children.setdefault(full_name, [])
# For something else that's documented,
# figure out what module it lives in
else:
subname = str(full_name)
while True:
subname = subname[:subname.rindex('.')]
if tf_inspect.ismodule(parser_config.index[subname]):
module_name = parser_config.duplicate_of.get(
subname, subname)
module_children.setdefault(module_name,
[]).append(full_name)
break
# Generate docs for `py_object`, resolving references.
page_info = parser.docs_for_object(full_name, py_object, parser_config)
path = os.path.join(output_dir, parser.documentation_path(full_name))
directory = os.path.dirname(path)
try:
if not os.path.exists(directory):
os.makedirs(directory)
# This function returns raw bytes in PY2 or unicode in PY3.
if search_hints:
content = [page_info.get_metadata_html()]
else:
content = ['']
content.append(pretty_docs.build_md_page(page_info))
text = '\n'.join(content)
if six.PY3:
text = text.encode('utf-8')
with open(path, 'wb') as f:
f.write(text)
except OSError:
raise OSError('Cannot write documentation for %s to %s' %
(full_name, directory))
duplicates = parser_config.duplicates.get(full_name, [])
if not duplicates:
continue
duplicates = [item for item in duplicates if item != full_name]
for dup in duplicates:
from_path = os.path.join(site_path, 'api_docs/python',
dup.replace('.', '/'))
to_path = os.path.join(site_path, 'api_docs/python',
full_name.replace('.', '/'))
redirects.append(
(os.path.join('/', from_path), os.path.join('/', to_path)))
if redirects:
redirects = sorted(redirects)
template = ('- from: {}\n' ' to: {}\n')
redirects = [template.format(f, t) for f, t in redirects]
api_redirects_path = os.path.join(output_dir, '_redirects.yaml')
with open(api_redirects_path, 'w') as redirect_file:
redirect_file.write('redirects:\n')
redirect_file.write(''.join(redirects))
if yaml_toc:
# Generate table of contents
# Put modules in alphabetical order, case-insensitive
modules = sorted(module_children.keys(), key=lambda a: a.upper())
leftnav_path = os.path.join(output_dir, '_toc.yaml')
with open(leftnav_path, 'w') as f:
# Generate header
f.write(
'# Automatically generated file; please do not edit\ntoc:\n')
for module in modules:
indent_num = module.count('.')
# Don't list `tf.submodule` inside `tf`
indent_num = max(indent_num, 1)
indent = ' ' * indent_num
if indent_num > 1:
# tf.contrib.baysflow.entropy will be under
# tf.contrib->baysflow->entropy
title = module.split('.')[-1]
else:
title = module
header = [
'- title: ' + title, ' section:', ' - title: Overview',
' path: ' +
os.path.join('/', site_path, symbol_to_file[module])
]
header = ''.join([indent + line + '\n' for line in header])
f.write(header)
symbols_in_module = module_children.get(module, [])
# Sort case-insensitive, if equal sort case sensitive (upper first)
symbols_in_module.sort(key=lambda a: (a.upper(), a))
for full_name in symbols_in_module:
item = [
' - title: ' + full_name[len(module) + 1:],
' path: ' +
os.path.join('/', site_path, symbol_to_file[full_name])
]
item = ''.join([indent + line + '\n' for line in item])
f.write(item)
# Write a global index containing all full names with links.
with open(os.path.join(output_dir, 'index.md'), 'w') as f:
f.write(
parser.generate_global_index(root_title, parser_config.index,
parser_config.reference_resolver))
def write_docs(
*,
output_dir: Union[str, pathlib.Path],
parser_config: parser.ParserConfig,
yaml_toc: bool,
root_module_name: str,
root_title: str = 'TensorFlow',
search_hints: bool = True,
site_path: str = 'api_docs/python',
gen_redirects: bool = True,
table_view: bool = True,
gen_report: bool = False,
):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `parser.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
root_module_name: (str) the name of the root module (`tf` for tensorflow).
root_title: The title name for the root level index.md.
search_hints: (bool) include meta-data search hints at the top of each
output file.
site_path: The output path relative to the site root. Used in the
`_toc.yaml` and `_redirects.yaml` files.
gen_redirects: Bool which decides whether to generate _redirects.yaml file
or not.
table_view: If True, `Args`, `Returns`, `Raises` or `Attributes` will be
converted to a tabular format while generating markdown. If False, they
will be converted to a markdown List view.
gen_report: If True, a report for the library is generated by linting the
docstrings of its public API symbols.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
output_dir = pathlib.Path(output_dir)
site_path = pathlib.Path('/', site_path)
# Make output_dir.
if not output_dir.is_absolute():
raise ValueError("'output_dir' must be an absolute path.\n"
f" output_dir='{output_dir}'")
output_dir.mkdir(parents=True, exist_ok=True)
# These dictionaries are used for table-of-contents generation below
# They will contain, after the for-loop below::
# - module name(string):classes and functions the module contains(list)
module_children = {}
# Collect redirects for an api _redirects.yaml file.
redirects = []
if gen_report:
api_report_obj = utils.ApiReport()
# Parse and write Markdown pages, resolving cross-links (`tf.symbol`).
for full_name in sorted(parser_config.index.keys(),
key=lambda k: k.lower()):
py_object = parser_config.index[full_name]
if full_name in parser_config.duplicate_of:
continue
# Methods constants are only documented only as part of their parent's page.
if parser_config.reference_resolver.is_fragment(full_name):
continue
# Remove the extension from the path.
docpath, _ = os.path.splitext(parser.documentation_path(full_name))
# For a module, remember the module for the table-of-contents
if inspect.ismodule(py_object):
if full_name in parser_config.tree:
mod_obj = Module(module=full_name,
py_object=py_object,
path=str(site_path / docpath))
module_children[full_name] = mod_obj
# For something else that's documented,
# figure out what module it lives in
else:
subname = str(full_name)
while True:
subname = subname[:subname.rindex('.')]
if inspect.ismodule(parser_config.index[subname]):
module_name = parser_config.duplicate_of.get(
subname, subname)
child_mod = ModuleChild(name=full_name,
py_object=py_object,
parent=module_name,
path=str(site_path / docpath))
module_children[module_name].add_children(child_mod)
break
# Generate docs for `py_object`, resolving references.
try:
page_info = parser.docs_for_object(full_name, py_object,
parser_config)
except:
raise ValueError(
f'Failed to generate docs for symbol: `{full_name}`')
if gen_report and not full_name.startswith(
('tf.compat.v', 'tf.keras.backend')):
api_report_obj.fill_metrics(page_info)
continue
path = output_dir / parser.documentation_path(full_name)
try:
path.parent.mkdir(exist_ok=True, parents=True)
# This function returns unicode in PY3.
hidden = doc_controls.should_hide_from_search(page_info.py_object)
brief_no_backticks = page_info.doc.brief.replace('`', '').strip()
content = []
if brief_no_backticks:
content.append(f'description: {brief_no_backticks}\n')
if search_hints and not hidden:
content.append(page_info.get_metadata_html())
else:
content.append('robots: noindex\n')
content.append(pretty_docs.build_md_page(page_info, table_view))
text = '\n'.join(content)
path.write_text(text, encoding='utf-8')
except OSError:
raise OSError('Cannot write documentation for '
f'{full_name} to {path.parent}')
duplicates = parser_config.duplicates.get(full_name, [])
if not duplicates:
continue
duplicates = [item for item in duplicates if item != full_name]
if gen_redirects:
for dup in duplicates:
from_path = site_path / dup.replace('.', '/')
to_path = site_path / full_name.replace('.', '/')
redirects.append({'from': str(from_path), 'to': str(to_path)})
if gen_report:
serialized_proto = api_report_obj.api_report.SerializeToString()
raw_proto = output_dir / 'api_report.pb'
raw_proto.write_bytes(serialized_proto)
return
if yaml_toc:
toc_gen = GenerateToc(module_children)
toc_dict = toc_gen.generate()
# Replace the overview path *only* for 'TensorFlow' to
# `/api_docs/python/tf_overview`. This will be redirected to
# `/api_docs/python/tf`.
toc_values = toc_dict['toc'][0]
if toc_values['title'] == 'tf':
section = toc_values['section'][0]
section['path'] = str(site_path / 'tf_overview')
leftnav_toc = output_dir / root_module_name / '_toc.yaml'
with open(leftnav_toc, 'w') as toc_file:
yaml.dump(toc_dict, toc_file, default_flow_style=False)
if redirects and gen_redirects:
if yaml_toc and toc_values['title'] == 'tf':
redirects.append({
'from': str(site_path / 'tf_overview'),
'to': str(site_path / 'tf'),
})
redirects_dict = {
'redirects': sorted(redirects,
key=lambda redirect: redirect['from'])
}
api_redirects_path = output_dir / root_module_name / '_redirects.yaml'
with open(api_redirects_path, 'w') as redirect_file:
yaml.dump(redirects_dict, redirect_file, default_flow_style=False)
# Write a global index containing all full names with links.
with open(output_dir / root_module_name / 'all_symbols.md', 'w') as f:
global_index = parser.generate_global_index(
root_title, parser_config.index, parser_config.reference_resolver)
if not search_hints:
global_index = 'robots: noindex\n' + global_index
f.write(global_index)
def write_docs(output_dir,
parser_config,
yaml_toc,
root_title='TensorFlow',
search_hints=True,
site_path='api_docs/python'):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `parser.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
root_title: The title name for the root level index.md.
search_hints: (bool) include meta-data search hints at the top of each
output file.
site_path: The output path relative to the site root. Used in the
`_toc.yaml` and `_redirects.yaml` files.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
# Make output_dir.
if not os.path.isabs(output_dir):
raise ValueError("'output_dir' must be an absolute path.\n"
" output_dir='%s'" % output_dir)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# These dictionaries are used for table-of-contents generation below
# They will contain, after the for-loop below::
# - module name(string):classes and functions the module contains(list)
module_children = {}
# Collect redirects for an api _redirects.yaml file.
redirects = []
# Parse and write Markdown pages, resolving cross-links (`tf.symbol`).
for full_name in sorted(parser_config.index.keys(),
key=lambda k: k.lower()):
py_object = parser_config.index[full_name]
if full_name in parser_config.duplicate_of:
continue
# Methods and some routines are documented only as part of their class.
if not (tf_inspect.ismodule(py_object)
or tf_inspect.isclass(py_object) or parser.is_free_function(
py_object, full_name, parser_config.index)):
continue
# Remove the extension from the path.
docpath, _ = os.path.splitext(parser.documentation_path(full_name))
# For a module, remember the module for the table-of-contents
if tf_inspect.ismodule(py_object):
if full_name in parser_config.tree:
mod_obj = Module(module=full_name,
py_object=py_object,
path=os.path.join('/', site_path, docpath))
module_children[full_name] = mod_obj
# For something else that's documented,
# figure out what module it lives in
else:
subname = str(full_name)
while True:
subname = subname[:subname.rindex('.')]
if tf_inspect.ismodule(parser_config.index[subname]):
module_name = parser_config.duplicate_of.get(
subname, subname)
child_mod = ModuleChild(name=full_name,
py_object=py_object,
parent=module_name,
path=os.path.join(
'/', site_path, docpath))
module_children[module_name].add_children(child_mod)
break
# Generate docs for `py_object`, resolving references.
try:
page_info = parser.docs_for_object(full_name, py_object,
parser_config)
except:
raise ValueError(
'Failed to generate docs for symbol: `{}`'.format(full_name))
path = os.path.join(output_dir, parser.documentation_path(full_name))
directory = os.path.dirname(path)
try:
if not os.path.exists(directory):
os.makedirs(directory)
# This function returns raw bytes in PY2 or unicode in PY3.
if search_hints:
content = [page_info.get_metadata_html()]
else:
content = ['']
content.append(pretty_docs.build_md_page(page_info))
text = '\n'.join(content)
if six.PY3:
text = text.encode('utf-8')
with open(path, 'wb') as f:
f.write(text)
except OSError:
raise OSError('Cannot write documentation for %s to %s' %
(full_name, directory))
duplicates = parser_config.duplicates.get(full_name, [])
if not duplicates:
continue
duplicates = [item for item in duplicates if item != full_name]
for dup in duplicates:
from_path = os.path.join(site_path, dup.replace('.', '/'))
to_path = os.path.join(site_path, full_name.replace('.', '/'))
redirects.append({
'from': os.path.join('/', from_path),
'to': os.path.join('/', to_path)
})
if redirects:
redirects_dict = {
'redirects': sorted(redirects,
key=lambda redirect: redirect['from'])
}
api_redirects_path = os.path.join(output_dir, '_redirects.yaml')
with open(api_redirects_path, 'w') as redirect_file:
yaml.dump(redirects_dict, redirect_file, default_flow_style=False)
if yaml_toc:
toc_gen = GenerateToc(module_children)
toc_dict = toc_gen.generate()
leftnav_toc = os.path.join(output_dir, '_toc.yaml')
with open(leftnav_toc, 'w') as toc_file:
yaml.dump(toc_dict, toc_file, default_flow_style=False)
# Write a global index containing all full names with links.
with open(os.path.join(output_dir, 'index.md'), 'w') as f:
f.write(
parser.generate_global_index(root_title, parser_config.index,
parser_config.reference_resolver))
def test_docs_for_class(self):
index = {
'TestClass': TestClass,
'TestClass.a_method': TestClass.a_method,
'TestClass.a_property': TestClass.a_property,
'TestClass.ChildClass': TestClass.ChildClass,
'TestClass.static_method': TestClass.static_method,
'TestClass.class_method': TestClass.class_method,
'TestClass.CLASS_MEMBER': TestClass.CLASS_MEMBER,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, py_module_names=['tf'])
tree = {
'TestClass': [
'a_method', 'class_method', 'static_method', 'a_property',
'ChildClass', 'CLASS_MEMBER'
]
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
base_dir='/',
code_url_prefix='/')
page_info = parser.docs_for_object(full_name='TestClass',
py_object=TestClass,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
inspect.getdoc(TestClass).split('\n')[0], page_info.doc.brief)
# Make sure the method is present
method_infos = {
method_info.short_name: method_info
for method_info in page_info.methods
}
self.assertIs(method_infos['a_method'].py_object, TestClass.a_method)
# Make sure that the signature is extracted properly and omits self.
self.assertEqual(["arg='default'"],
method_infos['a_method'].signature.arguments)
self.assertEqual(method_infos['static_method'].decorators,
['staticmethod'])
self.assertEqual(method_infos['class_method'].decorators,
['classmethod'])
# Make sure the property is present
attrs = page_info.attr_block
self.assertIsInstance(attrs, parser.TitleBlock)
self.assertIn('a_property', [name for name, desc in attrs.items])
# Make sure there is a link to the child class and it points the right way.
self.assertIs(TestClass.ChildClass, page_info.classes[0].py_object)
# Make sure this file is contained as the definition location.
self.assertEqual(os.path.relpath(__file__, '/'),
page_info.defined_in.rel_path)
def write_docs(output_dir,
parser_config,
yaml_toc,
root_title='TensorFlow',
search_hints=True,
site_path='api_docs/python'):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `parser.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
root_title: The title name for the root level index.md.
search_hints: (bool) include meta-data search hints at the top of each
output file.
site_path: The output path relative to the site root. Used in the
`_toc.yaml` and `_redirects.yaml` files.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
output_dir = pathlib.Path(output_dir)
site_path = pathlib.Path('/', site_path)
# Make output_dir.
if not output_dir.is_absolute():
raise ValueError("'output_dir' must be an absolute path.\n"
f" output_dir='{output_dir}'")
output_dir.mkdir(parents=True, exist_ok=True)
# These dictionaries are used for table-of-contents generation below
# They will contain, after the for-loop below::
# - module name(string):classes and functions the module contains(list)
module_children = {}
# Collect redirects for an api _redirects.yaml file.
redirects = []
# Parse and write Markdown pages, resolving cross-links (`tf.symbol`).
for full_name in sorted(parser_config.index.keys(),
key=lambda k: k.lower()):
py_object = parser_config.index[full_name]
if full_name in parser_config.duplicate_of:
continue
# Methods and some routines are documented only as part of their class.
if not (inspect.ismodule(py_object)
or inspect.isclass(py_object) or parser.is_free_function(
py_object, full_name, parser_config.index)):
continue
# Remove the extension from the path.
docpath, _ = os.path.splitext(parser.documentation_path(full_name))
# For a module, remember the module for the table-of-contents
if inspect.ismodule(py_object):
if full_name in parser_config.tree:
mod_obj = Module(module=full_name,
py_object=py_object,
path=str(site_path / docpath))
module_children[full_name] = mod_obj
# For something else that's documented,
# figure out what module it lives in
else:
subname = str(full_name)
while True:
subname = subname[:subname.rindex('.')]
if inspect.ismodule(parser_config.index[subname]):
module_name = parser_config.duplicate_of.get(
subname, subname)
child_mod = ModuleChild(name=full_name,
py_object=py_object,
parent=module_name,
path=str(site_path / docpath))
module_children[module_name].add_children(child_mod)
break
# Generate docs for `py_object`, resolving references.
try:
page_info = parser.docs_for_object(full_name, py_object,
parser_config)
except:
raise ValueError(
f'Failed to generate docs for symbol: `{full_name}`')
path = output_dir / parser.documentation_path(full_name)
try:
path.parent.mkdir(exist_ok=True, parents=True)
# This function returns unicode in PY3.
hidden = doc_controls.should_hide_from_search(page_info.py_object)
if search_hints and not hidden:
content = [page_info.get_metadata_html()]
else:
content = ['<meta name="robots" content="noindex">\n']
content.append(pretty_docs.build_md_page(page_info))
text = '\n'.join(content)
path.write_text(text, encoding='utf-8')
except OSError:
raise OSError('Cannot write documentation for '
f'{full_name} to {path.parent}')
duplicates = parser_config.duplicates.get(full_name, [])
if not duplicates:
continue
duplicates = [item for item in duplicates if item != full_name]
for dup in duplicates:
from_path = site_path / dup.replace('.', '/')
to_path = site_path / full_name.replace('.', '/')
redirects.append({'from': str(from_path), 'to': str(to_path)})
if yaml_toc:
toc_gen = GenerateToc(module_children)
toc_dict = toc_gen.generate()
# Replace the overview path *only* for 'TensorFlow' to
# `/api_docs/python/tf_overview`. This will be redirected to
# `/api_docs/python/tf`.
toc_values = toc_dict['toc'][0]
if toc_values['title'] == 'tf':
section = toc_values['section'][0]
section['path'] = str(site_path / 'tf_overview')
leftnav_toc = output_dir / '_toc.yaml'
with open(leftnav_toc, 'w') as toc_file:
yaml.dump(toc_dict, toc_file, default_flow_style=False)
if redirects:
if yaml_toc and toc_values['title'] == 'tf':
redirects.append({
'from': str(site_path / 'tf_overview'),
'to': str(site_path / 'tf'),
})
redirects_dict = {
'redirects': sorted(redirects,
key=lambda redirect: redirect['from'])
}
api_redirects_path = output_dir / '_redirects.yaml'
with open(api_redirects_path, 'w') as redirect_file:
yaml.dump(redirects_dict, redirect_file, default_flow_style=False)
# Write a global index containing all full names with links.
with open(output_dir / 'index.md', 'w') as f:
f.write(
parser.generate_global_index(root_title, parser_config.index,
parser_config.reference_resolver))
def write_docs(output_dir,
parser_config,
yaml_toc,
root_title='TensorFlow',
search_hints=True,
site_path=''):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `parser.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
root_title: The title name for the root level index.md.
search_hints: (bool) include meta-data search hints at the top of each
output file.
site_path: The output path relative to the site root. Used in the
`_toc.yaml` and `_redirects.yaml` files.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
# Make output_dir.
if not os.path.isabs(output_dir):
raise ValueError("'output_dir' must be an absolute path.\n"
" output_dir='%s'" % output_dir)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# These dictionaries are used for table-of-contents generation below
# They will contain, after the for-loop below::
# - module name(string):classes and functions the module contains(list)
module_children = {}
# - symbol name(string):pathname (string)
symbol_to_file = {}
# Collect redirects for an api _redirects.yaml file.
redirects = []
# Parse and write Markdown pages, resolving cross-links (`tf.symbol`).
for full_name, py_object in six.iteritems(parser_config.index):
parser_config.reference_resolver.current_doc_full_name = full_name
if full_name in parser_config.duplicate_of:
continue
# Methods and some routines are documented only as part of their class.
if not (tf_inspect.ismodule(py_object) or tf_inspect.isclass(py_object) or
parser.is_free_function(py_object, full_name, parser_config.index)):
continue
sitepath = os.path.join('api_docs/python',
parser.documentation_path(full_name)[:-3])
# For TOC, we need to store a mapping from full_name to the file
# we're generating
symbol_to_file[full_name] = sitepath
# For a module, remember the module for the table-of-contents
if tf_inspect.ismodule(py_object):
if full_name in parser_config.tree:
module_children.setdefault(full_name, [])
# For something else that's documented,
# figure out what module it lives in
else:
subname = str(full_name)
while True:
subname = subname[:subname.rindex('.')]
if tf_inspect.ismodule(parser_config.index[subname]):
module_name = parser_config.duplicate_of.get(subname, subname)
module_children.setdefault(module_name, []).append(full_name)
break
# Generate docs for `py_object`, resolving references.
page_info = parser.docs_for_object(full_name, py_object, parser_config)
path = os.path.join(output_dir, parser.documentation_path(full_name))
directory = os.path.dirname(path)
try:
if not os.path.exists(directory):
os.makedirs(directory)
# This function returns raw bytes in PY2 or unicode in PY3.
if search_hints:
content = [page_info.get_metadata_html()]
else:
content = ['']
content.append(pretty_docs.build_md_page(page_info))
text = '\n'.join(content)
if six.PY3:
text = text.encode('utf-8')
with open(path, 'wb') as f:
f.write(text)
except OSError:
raise OSError(
'Cannot write documentation for %s to %s' % (full_name, directory))
duplicates = parser_config.duplicates.get(full_name, [])
if not duplicates:
continue
duplicates = [item for item in duplicates if item != full_name]
for dup in duplicates:
from_path = os.path.join(site_path, 'api_docs/python',
dup.replace('.', '/'))
to_path = os.path.join(site_path, 'api_docs/python',
full_name.replace('.', '/'))
redirects.append((
os.path.join('/', from_path),
os.path.join('/', to_path)))
if redirects:
redirects = sorted(redirects)
template = ('- from: {}\n'
' to: {}\n')
redirects = [template.format(f, t) for f, t in redirects]
api_redirects_path = os.path.join(output_dir, '_redirects.yaml')
with open(api_redirects_path, 'w') as redirect_file:
redirect_file.write('redirects:\n')
redirect_file.write(''.join(redirects))
if yaml_toc:
# Generate table of contents
# Put modules in alphabetical order, case-insensitive
modules = sorted(module_children.keys(), key=lambda a: a.upper())
leftnav_path = os.path.join(output_dir, '_toc.yaml')
with open(leftnav_path, 'w') as f:
# Generate header
f.write('# Automatically generated file; please do not edit\ntoc:\n')
for module in modules:
indent_num = module.count('.')
# Don't list `tf.submodule` inside `tf`
indent_num = max(indent_num, 1)
indent = ' '*indent_num
if indent_num > 1:
# tf.contrib.baysflow.entropy will be under
# tf.contrib->baysflow->entropy
title = module.split('.')[-1]
else:
title = module
header = [
'- title: ' + title, ' section:', ' - title: Overview',
' path: ' + os.path.join('/', site_path, symbol_to_file[module])
]
header = ''.join([indent+line+'\n' for line in header])
f.write(header)
symbols_in_module = module_children.get(module, [])
# Sort case-insensitive, if equal sort case sensitive (upper first)
symbols_in_module.sort(key=lambda a: (a.upper(), a))
for full_name in symbols_in_module:
item = [
' - title: ' + full_name[len(module) + 1:], ' path: ' +
os.path.join('/', site_path, symbol_to_file[full_name])
]
item = ''.join([indent+line+'\n' for line in item])
f.write(item)
# Write a global index containing all full names with links.
with open(os.path.join(output_dir, 'index.md'), 'w') as f:
f.write(
parser.generate_global_index(root_title, parser_config.index,
parser_config.reference_resolver))
