def test_docs_for_function_with_kwargs(self):
index = {
'test_function_with_args_kwargs': test_function_with_args_kwargs
}
reference_resolver = parser.ReferenceResolver(
duplicate_of={}, doc_index={}, index=index, py_module_names=['tf'])
tree = {
'': ['test_function_with_args_kwargs']
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver, duplicates={}, tree=tree,
reverse_index={}, guide_index={}, base_dir='/')
page_info = parser.docs_for_object(
full_name='test_function_with_args_kwargs',
py_object=test_function_with_args_kwargs,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
inspect.getdoc(test_function_with_args_kwargs).split('\n')[0],
page_info.doc.brief)
# Make sure the extracted signature is good.
self.assertEqual(['unused_arg', '*unused_args', '**unused_kwargs'],
page_info.signature)
def test_docs_for_function(self):
index = {'test_function': test_function}
reference_resolver = parser.ReferenceResolver(duplicate_of={},
doc_index={},
index=index,
py_module_names=['tf'])
tree = {'': ['test_function']}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
tree=tree,
reverse_index={},
guide_index={},
base_dir='/')
page_info = parser.docs_for_object(full_name='test_function',
py_object=test_function,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
inspect.getdoc(test_function).split('\n')[0], page_info.doc.brief)
# Make sure the extracted signature is good.
self.assertEqual(['unused_arg', "unused_kwarg='default'"],
page_info.signature)
# Make sure this file is contained as the definition location.
self.assertEqual(os.path.relpath(__file__, '/'),
page_info.defined_in.path)
def test_docs_for_function_with_kwargs(self):
index = {
'test_function_with_args_kwargs': test_function_with_args_kwargs
}
reference_resolver = parser.ReferenceResolver(duplicate_of={},
doc_index={},
index=index,
py_module_names=['tf'])
tree = {'': ['test_function_with_args_kwargs']}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
tree=tree,
reverse_index={},
guide_index={},
base_dir='/')
page_info = parser.docs_for_object(
full_name='test_function_with_args_kwargs',
py_object=test_function_with_args_kwargs,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
inspect.getdoc(test_function_with_args_kwargs).split('\n')[0],
page_info.doc.brief)
# Make sure the extracted signature is good.
self.assertEqual(['unused_arg', '*unused_args', '**unused_kwargs'],
page_info.signature)
def test_docs_for_function(self):
index = {
'test_function': test_function
}
reference_resolver = parser.ReferenceResolver(
duplicate_of={}, doc_index={}, index=index, py_module_names=['tf'])
tree = {
'': ['test_function']
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver, duplicates={}, tree=tree,
reverse_index={}, guide_index={}, base_dir='/')
page_info = parser.docs_for_object(
full_name='test_function',
py_object=test_function,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
inspect.getdoc(test_function).split('\n')[0], page_info.doc.brief)
# Make sure the extracted signature is good.
self.assertEqual(['unused_arg', "unused_kwarg='default'"],
page_info.signature)
# Make sure this file is contained as the definition location.
self.assertEqual(os.path.relpath(__file__, '/'), page_info.defined_in.path)
def test_docs_for_message_class(self):
class CMessage(object):
def hidden(self):
pass
class Message(object):
def hidden2(self):
pass
class MessageMeta(object):
def hidden3(self):
pass
class ChildMessage(CMessage, Message, MessageMeta):
def my_method(self):
pass
index = {
'ChildMessage': ChildMessage,
'ChildMessage.hidden': ChildMessage.hidden,
'ChildMessage.hidden2': ChildMessage.hidden2,
'ChildMessage.hidden3': ChildMessage.hidden3,
'ChildMessage.my_method': ChildMessage.my_method,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, doc_index={}, py_module_names=['tf'])
tree = {'ChildMessage': ['hidden', 'hidden2', 'hidden3', 'my_method']}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
guide_index={},
base_dir='/')
page_info = parser.docs_for_object(
full_name='ChildMessage',
py_object=ChildMessage,
parser_config=parser_config)
self.assertEqual(1, len(page_info.methods))
self.assertEqual('my_method', page_info.methods[0].short_name)
def test_docs_for_class(self):
index = {
'TestClass': TestClass,
'TestClass.a_method': TestClass.a_method,
'TestClass.a_property': TestClass.a_property,
'TestClass.ChildClass': TestClass.ChildClass,
'TestClass.CLASS_MEMBER': TestClass.CLASS_MEMBER
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, doc_index={}, py_module_names=['tf'])
tree = {
'TestClass':
['a_method', 'a_property', 'ChildClass', 'CLASS_MEMBER']
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
guide_index={},
base_dir='/')
page_info = parser.docs_for_object(full_name='TestClass',
py_object=TestClass,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
six.ensure_str(tf_inspect.getdoc(TestClass)).split('\n')[0],
page_info.doc.brief)
# Make sure the method is present
self.assertEqual(TestClass.a_method, page_info.methods[0].obj)
# Make sure that the signature is extracted properly and omits self.
self.assertEqual(["arg='default'"], page_info.methods[0].signature)
# Make sure the property is present
self.assertIs(TestClass.a_property, page_info.properties[0].obj)
# Make sure there is a link to the child class and it points the right way.
self.assertIs(TestClass.ChildClass, page_info.classes[0].obj)
# Make sure this file is contained as the definition location.
self.assertEqual(os.path.relpath(__file__, '/'),
page_info.defined_in.path)
def test_docs_for_module(self):
index = {
'TestModule':
test_module,
'TestModule.test_function':
test_function,
'TestModule.test_function_with_args_kwargs':
test_function_with_args_kwargs,
'TestModule.TestClass':
TestClass,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, doc_index={}, py_module_names=['tf'])
tree = {
'TestModule': ['TestClass', 'test_function',
'test_function_with_args_kwargs']
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
guide_index={},
base_dir='/')
page_info = parser.docs_for_object(
full_name='TestModule',
py_object=test_module,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
tf_inspect.getdoc(test_module).split('\n')[0], page_info.doc.brief)
# Make sure that the members are there
funcs = {f_info.obj for f_info in page_info.functions}
self.assertEqual({test_function, test_function_with_args_kwargs}, funcs)
classes = {cls_info.obj for cls_info in page_info.classes}
self.assertEqual({TestClass}, classes)
# Make sure the module's file is contained as the definition location.
self.assertEqual(
os.path.relpath(test_module.__file__.rstrip('c'), '/'),
page_info.defined_in.path)
def test_docs_for_module(self):
# Get the current module.
module = sys.modules[__name__]
index = {
'TestModule': module,
'TestModule.test_function': test_function,
'TestModule.test_function_with_args_kwargs':
test_function_with_args_kwargs,
'TestModule.TestClass': TestClass,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, doc_index={}, py_module_names=['tf'])
tree = {
'TestModule':
['TestClass', 'test_function', 'test_function_with_args_kwargs']
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
guide_index={},
base_dir='/')
page_info = parser.docs_for_object(full_name='TestModule',
py_object=module,
parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
tf_inspect.getdoc(module).split('\n')[0], page_info.doc.brief)
# Make sure that the members are there
funcs = {f_info.obj for f_info in page_info.functions}
self.assertEqual({test_function, test_function_with_args_kwargs},
funcs)
classes = {cls_info.obj for cls_info in page_info.classes}
self.assertEqual({TestClass}, classes)
# Make sure this file is contained as the definition location.
self.assertEqual(os.path.relpath(__file__, '/'),
page_info.defined_in.path)
def test_docs_for_class(self):
index = {
'TestClass': TestClass,
'TestClass.a_method': TestClass.a_method,
'TestClass.a_property': TestClass.a_property,
'TestClass.ChildClass': TestClass.ChildClass,
'TestClass.CLASS_MEMBER': TestClass.CLASS_MEMBER
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, doc_index={}, py_module_names=['tf'])
tree = {
'TestClass': ['a_method', 'a_property', 'ChildClass', 'CLASS_MEMBER']
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
guide_index={},
base_dir='/')
page_info = parser.docs_for_object(
full_name='TestClass', py_object=TestClass, parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(
tf_inspect.getdoc(TestClass).split('\n')[0], page_info.doc.brief)
# Make sure the method is present
self.assertEqual(TestClass.a_method, page_info.methods[0].obj)
# Make sure that the signature is extracted properly and omits self.
self.assertEqual(["arg='default'"], page_info.methods[0].signature)
# Make sure the property is present
self.assertIs(TestClass.a_property, page_info.properties[0].obj)
# Make sure there is a link to the child class and it points the right way.
self.assertIs(TestClass.ChildClass, page_info.classes[0].obj)
# Make sure this file is contained as the definition location.
self.assertEqual(os.path.relpath(__file__, '/'), page_info.defined_in.path)
def test_docs_for_message_class(self):
class CMessage(object):
def hidden(self):
pass
class Message(object):
def hidden2(self):
pass
class MessageMeta(object):
def hidden3(self):
pass
class ChildMessage(CMessage, Message, MessageMeta):
def my_method(self):
pass
index = {
'ChildMessage': ChildMessage,
'ChildMessage.hidden': ChildMessage.hidden,
'ChildMessage.hidden2': ChildMessage.hidden2,
'ChildMessage.hidden3': ChildMessage.hidden3,
'ChildMessage.my_method': ChildMessage.my_method,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, doc_index={}, py_module_names=['tf'])
tree = {'ChildMessage': ['hidden', 'hidden2', 'hidden3', 'my_method']}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
guide_index={},
base_dir='/')
page_info = parser.docs_for_object(full_name='ChildMessage',
py_object=ChildMessage,
parser_config=parser_config)
self.assertEqual(1, len(page_info.methods))
self.assertEqual('my_method', page_info.methods[0].short_name)
def test_namedtuple_field_order(self):
namedtupleclass = collections.namedtuple('namedtupleclass',
{'z', 'y', 'x', 'w', 'v', 'u'})
index = {
'namedtupleclass': namedtupleclass,
'namedtupleclass.u': namedtupleclass.u,
'namedtupleclass.v': namedtupleclass.v,
'namedtupleclass.w': namedtupleclass.w,
'namedtupleclass.x': namedtupleclass.x,
'namedtupleclass.y': namedtupleclass.y,
'namedtupleclass.z': namedtupleclass.z,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, doc_index={}, py_module_names=['tf'])
tree = {'namedtupleclass': {'u', 'v', 'w', 'x', 'y', 'z'}}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
guide_index={},
base_dir='/')
page_info = parser.docs_for_object(
full_name='namedtupleclass',
py_object=namedtupleclass,
parser_config=parser_config)
# Each namedtiple field has a docstring of the form:
# 'Alias for field number ##'. These props are returned sorted.
def sort_key(prop_info):
return int(prop_info.obj.__doc__.split(' ')[-1])
self.assertSequenceEqual(page_info.properties,
sorted(page_info.properties, key=sort_key))
def test_namedtuple_field_order(self):
namedtupleclass = collections.namedtuple(
'namedtupleclass', {'z', 'y', 'x', 'w', 'v', 'u'})
index = {
'namedtupleclass': namedtupleclass,
'namedtupleclass.u': namedtupleclass.u,
'namedtupleclass.v': namedtupleclass.v,
'namedtupleclass.w': namedtupleclass.w,
'namedtupleclass.x': namedtupleclass.x,
'namedtupleclass.y': namedtupleclass.y,
'namedtupleclass.z': namedtupleclass.z,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, doc_index={}, py_module_names=['tf'])
tree = {'namedtupleclass': {'u', 'v', 'w', 'x', 'y', 'z'}}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
guide_index={},
base_dir='/')
page_info = parser.docs_for_object(full_name='namedtupleclass',
py_object=namedtupleclass,
parser_config=parser_config)
# Each namedtiple field has a docstring of the form:
# 'Alias for field number ##'. These props are returned sorted.
def sort_key(prop_info):
return int(prop_info.obj.__doc__.split(' ')[-1])
self.assertSequenceEqual(page_info.properties,
sorted(page_info.properties, key=sort_key))
def test_docs_for_class_should_skip(self):
class Parent(object):
@doc_controls.do_not_doc_inheritable
def a_method(self, arg='default'):
pass
class Child(Parent):
def a_method(self, arg='default'):
pass
index = {
'Child': Child,
'Child.a_method': Child.a_method,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, doc_index={}, py_module_names=['tf'])
tree = {
'Child': ['a_method'],
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
guide_index={},
base_dir='/')
page_info = parser.docs_for_object(
full_name='Child', py_object=Child, parser_config=parser_config)
# Make sure the `a_method` is not present
self.assertEqual(0, len(page_info.methods))
def test_docs_for_class_should_skip(self):
class Parent(object):
@doc_controls.do_not_doc_inheritable
def a_method(self, arg='default'):
pass
class Child(Parent):
def a_method(self, arg='default'):
pass
index = {
'Child': Child,
'Child.a_method': Child.a_method,
}
visitor = DummyVisitor(index=index, duplicate_of={})
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, doc_index={}, py_module_names=['tf'])
tree = {
'Child': ['a_method'],
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates={},
duplicate_of={},
tree=tree,
index=index,
reverse_index={},
guide_index={},
base_dir='/')
page_info = parser.docs_for_object(full_name='Child',
py_object=Child,
parser_config=parser_config)
# Make sure the `a_method` is not present
self.assertEqual(0, len(page_info.methods))
def test_docs_for_module(self):
# Get the current module.
module = sys.modules[__name__]
index = {
'TestModule': module,
'TestModule.test_function': test_function,
'TestModule.test_function_with_args_kwargs':
test_function_with_args_kwargs,
'TestModule.TestClass': TestClass,
}
reference_resolver = parser.ReferenceResolver(
duplicate_of={}, doc_index={}, index=index, py_module_names=['tf'])
tree = {
'TestModule': ['TestClass', 'test_function',
'test_function_with_args_kwargs']
}
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver, duplicates={}, tree=tree,
reverse_index={}, guide_index={}, base_dir='/')
page_info = parser.docs_for_object(
full_name='TestModule', py_object=module, parser_config=parser_config)
# Make sure the brief docstring is present
self.assertEqual(inspect.getdoc(module).split('\n')[0], page_info.doc.brief)
# Make sure that the members are there
members = [member_info.obj for member_info in page_info.members]
self.assertIn(test_function, members)
self.assertIn(test_function_with_args_kwargs, members)
self.assertIn(TestClass, members)
# Make sure this file is contained as the definition location.
self.assertEqual(os.path.relpath(__file__, '/'), page_info.defined_in.path)
def write_docs(output_dir, parser_config, yaml_toc, root_title='TensorFlow'):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `parser.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
root_title: The title name for the root level index.md.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
# Make output_dir.
if not os.path.isabs(output_dir):
raise ValueError("'output_dir' must be an absolute path.\n"
" output_dir='%s'" % output_dir)
try:
if not os.path.exists(output_dir):
os.makedirs(output_dir)
except OSError as e:
print('Creating output dir "%s" failed: %s' % (output_dir, e))
raise
# These dictionaries are used for table-of-contents generation below
# They will contain, after the for-loop below::
# - module name(string):classes and functions the module contains(list)
module_children = {}
# - symbol name(string):pathname (string)
symbol_to_file = {}
# Parse and write Markdown pages, resolving cross-links (@{symbol}).
for full_name, py_object in six.iteritems(parser_config.index):
parser_config.reference_resolver.current_doc_full_name = full_name
if full_name in parser_config.duplicate_of:
continue
# Methods and some routines are documented only as part of their class.
if not (tf_inspect.ismodule(py_object) or tf_inspect.isclass(py_object) or
_is_free_function(py_object, full_name, parser_config.index)):
continue
sitepath = os.path.join('api_docs/python',
parser.documentation_path(full_name)[:-3])
# For TOC, we need to store a mapping from full_name to the file
# we're generating
symbol_to_file[full_name] = sitepath
# For a module, remember the module for the table-of-contents
if tf_inspect.ismodule(py_object):
if full_name in parser_config.tree:
module_children.setdefault(full_name, [])
# For something else that's documented,
# figure out what module it lives in
else:
subname = str(full_name)
while True:
subname = subname[:subname.rindex('.')]
if tf_inspect.ismodule(parser_config.index[subname]):
module_children.setdefault(subname, []).append(full_name)
break
print('Writing docs for %s (%r).' % (full_name, py_object))
# Generate docs for `py_object`, resolving references.
page_info = parser.docs_for_object(full_name, py_object, parser_config)
path = os.path.join(output_dir, parser.documentation_path(full_name))
directory = os.path.dirname(path)
try:
if not os.path.exists(directory):
os.makedirs(directory)
with open(path, 'w') as f:
f.write(pretty_docs.build_md_page(page_info))
except OSError as e:
print('Cannot write documentation for %s to %s: %s' % (full_name,
directory, e))
raise
if yaml_toc:
# Generate table of contents
# Put modules in alphabetical order, case-insensitive
modules = sorted(module_children.keys(), key=lambda a: a.upper())
leftnav_path = os.path.join(output_dir, '_toc.yaml')
with open(leftnav_path, 'w') as f:
# Generate header
f.write('# Automatically generated file; please do not edit\ntoc:\n')
for module in modules:
indent_num = module.count('.')
# Don't list `tf.submodule` inside `tf`
indent_num = max(indent_num, 1)
indent = ' '*indent_num
if indent_num > 1:
# tf.contrib.baysflow.entropy will be under
# tf.contrib->baysflow->entropy
title = module.split('.')[-1]
else:
title = module
header = [
'- title: ' + title,
' section:',
' - title: Overview',
' path: /TARGET_DOC_ROOT/VERSION/' + symbol_to_file[module]]
header = ''.join([indent+line+'\n' for line in header])
f.write(header)
symbols_in_module = module_children.get(module, [])
# Sort case-insensitive, if equal sort case sensitive (upper first)
symbols_in_module.sort(key=lambda a: (a.upper(), a))
for full_name in symbols_in_module:
item = [
' - title: ' + full_name[len(module) + 1:],
' path: /TARGET_DOC_ROOT/VERSION/' + symbol_to_file[full_name]]
item = ''.join([indent+line+'\n' for line in item])
f.write(item)
# Write a global index containing all full names with links.
with open(os.path.join(output_dir, 'index.md'), 'w') as f:
f.write(
parser.generate_global_index(root_title, parser_config.index,
parser_config.reference_resolver))
def write_docs(output_dir, parser_config, duplicate_of, index, yaml_toc):
"""Write previously extracted docs to disk.
Write a docs page for each symbol in `index` to a tree of docs at
`output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `parser.ParserConfig` object.
duplicate_of: A `dict` mapping fully qualified names to "master" names.
Used to determine which docs pages to write.
index: A `dict` mapping fully qualified names to the corresponding Python
objects. Used to produce docs for child objects.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
"""
# Make output_dir.
try:
if not os.path.exists(output_dir):
os.makedirs(output_dir)
except OSError as e:
print('Creating output dir "%s" failed: %s' % (output_dir, e))
raise
# These dictionaries are used for table-of-contents generation below
# They will contain, after the for-loop below::
# - module name(string):classes and functions the module contains(list)
module_children = {}
# - symbol name(string):pathname (string)
symbol_to_file = {}
# Parse and write Markdown pages, resolving cross-links (@{symbol}).
for full_name, py_object in six.iteritems(index):
if full_name in duplicate_of:
continue
# Methods and some routines are documented only as part of their class.
if not (inspect.ismodule(py_object) or
inspect.isclass(py_object) or
_is_free_function(py_object, full_name, index)):
continue
sitepath = os.path.join('api_docs/python',
parser.documentation_path(full_name)[:-3])
# For TOC, we need to store a mapping from full_name to the file
# we're generating
symbol_to_file[full_name] = sitepath
# For a module, remember the module for the table-of-contents
if inspect.ismodule(py_object):
if full_name in parser_config.tree:
module_children.setdefault(full_name, [])
# For something else that's documented,
# figure out what module it lives in
else:
subname = str(full_name)
while True:
subname = subname[:subname.rindex('.')]
if inspect.ismodule(index[subname]):
module_children.setdefault(subname, []).append(full_name)
break
print('Writing docs for %s (%r).' % (full_name, py_object))
# Generate docs for `py_object`, resolving references.
page_info = parser.docs_for_object(full_name, py_object, parser_config)
path = os.path.join(output_dir, parser.documentation_path(full_name))
directory = os.path.dirname(path)
try:
if not os.path.exists(directory):
os.makedirs(directory)
with open(path, 'w') as f:
f.write(pretty_docs.build_md_page(page_info))
except OSError as e:
print('Cannot write documentation for %s to %s: %s' % (full_name,
directory, e))
raise
if yaml_toc:
# Generate table of contents
# Put modules in alphabetical order, case-insensitive
modules = sorted(module_children.keys(), key=lambda a: a.upper())
leftnav_path = os.path.join(output_dir, '_toc.yaml')
with open(leftnav_path, 'w') as f:
# Generate header
f.write('# Automatically generated file; please do not edit\ntoc:\n')
for module in modules:
f.write(' - title: ' + module + '\n'
' section:\n' +
' - title: Overview\n' +
' path: /TARGET_DOC_ROOT/' + symbol_to_file[module] + '\n')
symbols_in_module = module_children.get(module, [])
symbols_in_module.sort(key=lambda a: a.upper())
for full_name in symbols_in_module:
f.write(' - title: ' + full_name[len(module)+1:] + '\n'
' path: /TARGET_DOC_ROOT/' +
symbol_to_file[full_name] + '\n')
# Write a global index containing all full names with links.
with open(os.path.join(output_dir, 'index.md'), 'w') as f:
f.write(parser.generate_global_index(
'TensorFlow', index, parser_config.reference_resolver))
def write_docs(output_dir,
parser_config,
yaml_toc,
root_title='TensorFlow',
search_hints=True,
site_api_path=''):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `parser.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
root_title: The title name for the root level index.md.
search_hints: (bool) include meta-data search hints at the top of each
output file.
site_api_path: The output path relative to the site root. Used in the
`_toc.yaml` and `_redirects.yaml` files.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
# Make output_dir.
if not os.path.isabs(output_dir):
raise ValueError("'output_dir' must be an absolute path.\n"
" output_dir='%s'" % output_dir)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# These dictionaries are used for table-of-contents generation below
# They will contain, after the for-loop below::
# - module name(string):classes and functions the module contains(list)
module_children = {}
# - symbol name(string):pathname (string)
symbol_to_file = {}
# Collect redirects for an api _redirects.yaml file.
redirects = []
# Parse and write Markdown pages, resolving cross-links (@{symbol}).
for full_name, py_object in six.iteritems(parser_config.index):
parser_config.reference_resolver.current_doc_full_name = full_name
if full_name in parser_config.duplicate_of:
continue
# Methods and some routines are documented only as part of their class.
if not (tf_inspect.ismodule(py_object) or tf_inspect.isclass(py_object) or
parser.is_free_function(py_object, full_name, parser_config.index)):
continue
sitepath = os.path.join('api_docs/python',
parser.documentation_path(full_name)[:-3])
# For TOC, we need to store a mapping from full_name to the file
# we're generating
symbol_to_file[full_name] = sitepath
# For a module, remember the module for the table-of-contents
if tf_inspect.ismodule(py_object):
if full_name in parser_config.tree:
module_children.setdefault(full_name, [])
# For something else that's documented,
# figure out what module it lives in
else:
subname = str(full_name)
while True:
subname = subname[:subname.rindex('.')]
if tf_inspect.ismodule(parser_config.index[subname]):
module_children.setdefault(subname, []).append(full_name)
break
# Generate docs for `py_object`, resolving references.
page_info = parser.docs_for_object(full_name, py_object, parser_config)
path = os.path.join(output_dir, parser.documentation_path(full_name))
directory = os.path.dirname(path)
try:
if not os.path.exists(directory):
os.makedirs(directory)
# This function returns raw bytes in PY2 or unicode in PY3.
if search_hints:
content = [page_info.get_metadata_html()]
else:
content = ['']
content.append(pretty_docs.build_md_page(page_info))
text = '\n'.join(content)
if six.PY3:
text = text.encode('utf-8')
with open(path, 'wb') as f:
f.write(text)
except OSError:
raise OSError(
'Cannot write documentation for %s to %s' % (full_name, directory))
duplicates = parser_config.duplicates.get(full_name, [])
if not duplicates:
continue
duplicates = [item for item in duplicates if item != full_name]
for dup in duplicates:
from_path = os.path.join(site_api_path, dup.replace('.', '/'))
to_path = os.path.join(site_api_path, full_name.replace('.', '/'))
redirects.append((
os.path.join('/', from_path),
os.path.join('/', to_path)))
if redirects:
redirects = sorted(redirects)
template = ('- from: {}\n'
' to: {}\n')
redirects = [template.format(f, t) for f, t in redirects]
api_redirects_path = os.path.join(output_dir, '_redirects.yaml')
with open(api_redirects_path, 'w') as redirect_file:
redirect_file.write('redirects:\n')
redirect_file.write(''.join(redirects))
if yaml_toc:
# Generate table of contents
# Put modules in alphabetical order, case-insensitive
modules = sorted(module_children.keys(), key=lambda a: a.upper())
leftnav_path = os.path.join(output_dir, '_toc.yaml')
with open(leftnav_path, 'w') as f:
# Generate header
f.write('# Automatically generated file; please do not edit\ntoc:\n')
for module in modules:
indent_num = module.count('.')
# Don't list `tf.submodule` inside `tf`
indent_num = max(indent_num, 1)
indent = ' '*indent_num
if indent_num > 1:
# tf.contrib.baysflow.entropy will be under
# tf.contrib->baysflow->entropy
title = module.split('.')[-1]
else:
title = module
header = [
'- title: ' + title,
' section:',
' - title: Overview',
' path: ' + os.path.join('/', site_api_path,
symbol_to_file[module])]
header = ''.join([indent+line+'\n' for line in header])
f.write(header)
symbols_in_module = module_children.get(module, [])
# Sort case-insensitive, if equal sort case sensitive (upper first)
symbols_in_module.sort(key=lambda a: (a.upper(), a))
for full_name in symbols_in_module:
item = [
' - title: ' + full_name[len(module) + 1:],
' path: ' + os.path.join('/', site_api_path,
symbol_to_file[full_name])]
item = ''.join([indent+line+'\n' for line in item])
f.write(item)
# Write a global index containing all full names with links.
with open(os.path.join(output_dir, 'index.md'), 'w') as f:
f.write(
parser.generate_global_index(root_title, parser_config.index,
parser_config.reference_resolver))
def write_docs(output_dir,
parser_config,
yaml_toc,
root_title='TensorFlow',
search_hints=True,
site_api_path=None):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `parser.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
root_title: The title name for the root level index.md.
search_hints: (bool) include meta-data search hints at the top of each
output file.
site_api_path: Used to write the api-duplicates _redirects.yaml file. if
None (the default) the file is not generated.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
# Make output_dir.
if not os.path.isabs(output_dir):
raise ValueError("'output_dir' must be an absolute path.\n"
" output_dir='%s'" % output_dir)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# These dictionaries are used for table-of-contents generation below
# They will contain, after the for-loop below::
# - module name(string):classes and functions the module contains(list)
module_children = {}
# - symbol name(string):pathname (string)
symbol_to_file = {}
# Collect redirects for an api _redirects.yaml file.
redirects = ['redirects:\n']
# Parse and write Markdown pages, resolving cross-links (@{symbol}).
for full_name, py_object in six.iteritems(parser_config.index):
parser_config.reference_resolver.current_doc_full_name = full_name
if full_name in parser_config.duplicate_of:
continue
# Methods and some routines are documented only as part of their class.
if not (tf_inspect.ismodule(py_object)
or tf_inspect.isclass(py_object) or _is_free_function(
py_object, full_name, parser_config.index)):
continue
sitepath = os.path.join('api_docs/python',
parser.documentation_path(full_name)[:-3])
# For TOC, we need to store a mapping from full_name to the file
# we're generating
symbol_to_file[full_name] = sitepath
# For a module, remember the module for the table-of-contents
if tf_inspect.ismodule(py_object):
if full_name in parser_config.tree:
module_children.setdefault(full_name, [])
# For something else that's documented,
# figure out what module it lives in
else:
subname = str(full_name)
while True:
subname = subname[:subname.rindex('.')]
if tf_inspect.ismodule(parser_config.index[subname]):
module_children.setdefault(subname, []).append(full_name)
break
# Generate docs for `py_object`, resolving references.
page_info = parser.docs_for_object(full_name, py_object, parser_config)
path = os.path.join(output_dir, parser.documentation_path(full_name))
directory = os.path.dirname(path)
try:
if not os.path.exists(directory):
os.makedirs(directory)
# This function returns raw bytes in PY2 or unicode in PY3.
if search_hints:
content = [page_info.get_metadata_html()]
else:
content = ['']
content.append(pretty_docs.build_md_page(page_info))
text = '\n'.join(content)
if six.PY3:
text = text.encode('utf-8')
with open(path, 'wb') as f:
f.write(text)
except OSError:
raise OSError('Cannot write documentation for %s to %s' %
(full_name, directory))
if site_api_path:
duplicates = parser_config.duplicates.get(full_name, [])
if not duplicates:
continue
duplicates = [item for item in duplicates if item != full_name]
template = ('- from: /{}\n' ' to: /{}\n')
for dup in duplicates:
from_path = os.path.join(site_api_path, dup.replace('.', '/'))
to_path = os.path.join(site_api_path,
full_name.replace('.', '/'))
redirects.append(template.format(from_path, to_path))
if site_api_path:
api_redirects_path = os.path.join(output_dir, '_redirects.yaml')
with open(api_redirects_path, 'w') as redirect_file:
redirect_file.write(''.join(redirects))
if yaml_toc:
# Generate table of contents
# Put modules in alphabetical order, case-insensitive
modules = sorted(module_children.keys(), key=lambda a: a.upper())
leftnav_path = os.path.join(output_dir, '_toc.yaml')
with open(leftnav_path, 'w') as f:
# Generate header
f.write(
'# Automatically generated file; please do not edit\ntoc:\n')
for module in modules:
indent_num = module.count('.')
# Don't list `tf.submodule` inside `tf`
indent_num = max(indent_num, 1)
indent = ' ' * indent_num
if indent_num > 1:
# tf.contrib.baysflow.entropy will be under
# tf.contrib->baysflow->entropy
title = module.split('.')[-1]
else:
title = module
header = [
'- title: ' + title, ' section:', ' - title: Overview',
' path: /TARGET_DOC_ROOT/VERSION/' +
symbol_to_file[module]
]
header = ''.join([indent + line + '\n' for line in header])
f.write(header)
symbols_in_module = module_children.get(module, [])
# Sort case-insensitive, if equal sort case sensitive (upper first)
symbols_in_module.sort(key=lambda a: (a.upper(), a))
for full_name in symbols_in_module:
item = [
' - title: ' + full_name[len(module) + 1:],
' path: /TARGET_DOC_ROOT/VERSION/' +
symbol_to_file[full_name]
]
item = ''.join([indent + line + '\n' for line in item])
f.write(item)
# Write a global index containing all full names with links.
with open(os.path.join(output_dir, 'index.md'), 'w') as f:
f.write(
parser.generate_global_index(root_title, parser_config.index,
parser_config.reference_resolver))
def write_docs(output_dir, parser_config, yaml_toc):
"""Write previously extracted docs to disk.
Write a docs page for each symbol included in the indices of parser_config to
a tree of docs at `output_dir`.
Symbols with multiple aliases will have only one page written about
them, which is referenced for all aliases.
Args:
output_dir: Directory to write documentation markdown files to. Will be
created if it doesn't exist.
parser_config: A `parser.ParserConfig` object, containing all the necessary
indices.
yaml_toc: Set to `True` to generate a "_toc.yaml" file.
Raises:
ValueError: if `output_dir` is not an absolute path
"""
# Make output_dir.
if not os.path.isabs(output_dir):
raise ValueError(
"'output_dir' must be an absolute path.\n"
" output_dir='%s'" % output_dir)
try:
if not os.path.exists(output_dir):
os.makedirs(output_dir)
except OSError as e:
print('Creating output dir "%s" failed: %s' % (output_dir, e))
raise
# These dictionaries are used for table-of-contents generation below
# They will contain, after the for-loop below::
# - module name(string):classes and functions the module contains(list)
module_children = {}
# - symbol name(string):pathname (string)
symbol_to_file = {}
# Parse and write Markdown pages, resolving cross-links (@{symbol}).
for full_name, py_object in six.iteritems(parser_config.index):
if full_name in parser_config.duplicate_of:
continue
# Methods and some routines are documented only as part of their class.
if not (tf_inspect.ismodule(py_object) or tf_inspect.isclass(py_object) or
_is_free_function(py_object, full_name, parser_config.index)):
continue
sitepath = os.path.join('api_docs/python',
parser.documentation_path(full_name)[:-3])
# For TOC, we need to store a mapping from full_name to the file
# we're generating
symbol_to_file[full_name] = sitepath
# For a module, remember the module for the table-of-contents
if tf_inspect.ismodule(py_object):
if full_name in parser_config.tree:
module_children.setdefault(full_name, [])
# For something else that's documented,
# figure out what module it lives in
else:
subname = str(full_name)
while True:
subname = subname[:subname.rindex('.')]
if tf_inspect.ismodule(parser_config.index[subname]):
module_children.setdefault(subname, []).append(full_name)
break
print('Writing docs for %s (%r).' % (full_name, py_object))
# Generate docs for `py_object`, resolving references.
page_info = parser.docs_for_object(full_name, py_object, parser_config)
path = os.path.join(output_dir, parser.documentation_path(full_name))
directory = os.path.dirname(path)
try:
if not os.path.exists(directory):
os.makedirs(directory)
with open(path, 'w') as f:
f.write(pretty_docs.build_md_page(page_info))
except OSError as e:
print('Cannot write documentation for %s to %s: %s' % (full_name,
directory, e))
raise
if yaml_toc:
# Generate table of contents
# Put modules in alphabetical order, case-insensitive
modules = sorted(module_children.keys(), key=lambda a: a.upper())
leftnav_path = os.path.join(output_dir, '_toc.yaml')
with open(leftnav_path, 'w') as f:
# Generate header
f.write('# Automatically generated file; please do not edit\ntoc:\n')
for module in modules:
f.write(' - title: ' + module + '\n'
' section:\n' + ' - title: Overview\n' +
' path: /TARGET_DOC_ROOT/VERSION/' + symbol_to_file[module]
+ '\n')
symbols_in_module = module_children.get(module, [])
# Sort case-insensitive, if equal sort case sensitive (upper first)
symbols_in_module.sort(key=lambda a: (a.upper(), a))
for full_name in symbols_in_module:
f.write(' - title: ' + full_name[len(module) + 1:] + '\n'
' path: /TARGET_DOC_ROOT/VERSION/' +
symbol_to_file[full_name] + '\n')
# Write a global index containing all full names with links.
with open(os.path.join(output_dir, 'index.md'), 'w') as f:
f.write(
parser.generate_global_index('TensorFlow', parser_config.index,
parser_config.reference_resolver))