def test_docs_for_function_with_kwargs(self): index = { 'test_function_with_args_kwargs': test_function_with_args_kwargs } reference_resolver = parser.ReferenceResolver( duplicate_of={}, doc_index={}, index=index, py_module_names=['tf']) tree = { '': ['test_function_with_args_kwargs'] } parser_config = parser.ParserConfig( reference_resolver=reference_resolver, duplicates={}, tree=tree, reverse_index={}, guide_index={}, base_dir='/') page_info = parser.docs_for_object( full_name='test_function_with_args_kwargs', py_object=test_function_with_args_kwargs, parser_config=parser_config) # Make sure the brief docstring is present self.assertEqual( inspect.getdoc(test_function_with_args_kwargs).split('\n')[0], page_info.doc.brief) # Make sure the extracted signature is good. self.assertEqual(['unused_arg', '*unused_args', '**unused_kwargs'], page_info.signature)
def test_docs_for_function(self): index = {'test_function': test_function} reference_resolver = parser.ReferenceResolver(duplicate_of={}, doc_index={}, index=index, py_module_names=['tf']) tree = {'': ['test_function']} parser_config = parser.ParserConfig( reference_resolver=reference_resolver, duplicates={}, tree=tree, reverse_index={}, guide_index={}, base_dir='/') page_info = parser.docs_for_object(full_name='test_function', py_object=test_function, parser_config=parser_config) # Make sure the brief docstring is present self.assertEqual( inspect.getdoc(test_function).split('\n')[0], page_info.doc.brief) # Make sure the extracted signature is good. self.assertEqual(['unused_arg', "unused_kwarg='default'"], page_info.signature) # Make sure this file is contained as the definition location. self.assertEqual(os.path.relpath(__file__, '/'), page_info.defined_in.path)
def test_docs_for_function_with_kwargs(self): index = { 'test_function_with_args_kwargs': test_function_with_args_kwargs } reference_resolver = parser.ReferenceResolver(duplicate_of={}, doc_index={}, index=index, py_module_names=['tf']) tree = {'': ['test_function_with_args_kwargs']} parser_config = parser.ParserConfig( reference_resolver=reference_resolver, duplicates={}, tree=tree, reverse_index={}, guide_index={}, base_dir='/') page_info = parser.docs_for_object( full_name='test_function_with_args_kwargs', py_object=test_function_with_args_kwargs, parser_config=parser_config) # Make sure the brief docstring is present self.assertEqual( inspect.getdoc(test_function_with_args_kwargs).split('\n')[0], page_info.doc.brief) # Make sure the extracted signature is good. self.assertEqual(['unused_arg', '*unused_args', '**unused_kwargs'], page_info.signature)
def test_docs_for_function(self): index = { 'test_function': test_function } reference_resolver = parser.ReferenceResolver( duplicate_of={}, doc_index={}, index=index, py_module_names=['tf']) tree = { '': ['test_function'] } parser_config = parser.ParserConfig( reference_resolver=reference_resolver, duplicates={}, tree=tree, reverse_index={}, guide_index={}, base_dir='/') page_info = parser.docs_for_object( full_name='test_function', py_object=test_function, parser_config=parser_config) # Make sure the brief docstring is present self.assertEqual( inspect.getdoc(test_function).split('\n')[0], page_info.doc.brief) # Make sure the extracted signature is good. self.assertEqual(['unused_arg', "unused_kwarg='default'"], page_info.signature) # Make sure this file is contained as the definition location. self.assertEqual(os.path.relpath(__file__, '/'), page_info.defined_in.path)
def test_docs_for_message_class(self): class CMessage(object): def hidden(self): pass class Message(object): def hidden2(self): pass class MessageMeta(object): def hidden3(self): pass class ChildMessage(CMessage, Message, MessageMeta): def my_method(self): pass index = { 'ChildMessage': ChildMessage, 'ChildMessage.hidden': ChildMessage.hidden, 'ChildMessage.hidden2': ChildMessage.hidden2, 'ChildMessage.hidden3': ChildMessage.hidden3, 'ChildMessage.my_method': ChildMessage.my_method, } visitor = DummyVisitor(index=index, duplicate_of={}) reference_resolver = parser.ReferenceResolver.from_visitor( visitor=visitor, doc_index={}, py_module_names=['tf']) tree = {'ChildMessage': ['hidden', 'hidden2', 'hidden3', 'my_method']} parser_config = parser.ParserConfig( reference_resolver=reference_resolver, duplicates={}, duplicate_of={}, tree=tree, index=index, reverse_index={}, guide_index={}, base_dir='/') page_info = parser.docs_for_object( full_name='ChildMessage', py_object=ChildMessage, parser_config=parser_config) self.assertEqual(1, len(page_info.methods)) self.assertEqual('my_method', page_info.methods[0].short_name)
def test_docs_for_class(self): index = { 'TestClass': TestClass, 'TestClass.a_method': TestClass.a_method, 'TestClass.a_property': TestClass.a_property, 'TestClass.ChildClass': TestClass.ChildClass, 'TestClass.CLASS_MEMBER': TestClass.CLASS_MEMBER } visitor = DummyVisitor(index=index, duplicate_of={}) reference_resolver = parser.ReferenceResolver.from_visitor( visitor=visitor, doc_index={}, py_module_names=['tf']) tree = { 'TestClass': ['a_method', 'a_property', 'ChildClass', 'CLASS_MEMBER'] } parser_config = parser.ParserConfig( reference_resolver=reference_resolver, duplicates={}, duplicate_of={}, tree=tree, index=index, reverse_index={}, guide_index={}, base_dir='/') page_info = parser.docs_for_object(full_name='TestClass', py_object=TestClass, parser_config=parser_config) # Make sure the brief docstring is present self.assertEqual( six.ensure_str(tf_inspect.getdoc(TestClass)).split('\n')[0], page_info.doc.brief) # Make sure the method is present self.assertEqual(TestClass.a_method, page_info.methods[0].obj) # Make sure that the signature is extracted properly and omits self. self.assertEqual(["arg='default'"], page_info.methods[0].signature) # Make sure the property is present self.assertIs(TestClass.a_property, page_info.properties[0].obj) # Make sure there is a link to the child class and it points the right way. self.assertIs(TestClass.ChildClass, page_info.classes[0].obj) # Make sure this file is contained as the definition location. self.assertEqual(os.path.relpath(__file__, '/'), page_info.defined_in.path)
def test_docs_for_module(self): index = { 'TestModule': test_module, 'TestModule.test_function': test_function, 'TestModule.test_function_with_args_kwargs': test_function_with_args_kwargs, 'TestModule.TestClass': TestClass, } visitor = DummyVisitor(index=index, duplicate_of={}) reference_resolver = parser.ReferenceResolver.from_visitor( visitor=visitor, doc_index={}, py_module_names=['tf']) tree = { 'TestModule': ['TestClass', 'test_function', 'test_function_with_args_kwargs'] } parser_config = parser.ParserConfig( reference_resolver=reference_resolver, duplicates={}, duplicate_of={}, tree=tree, index=index, reverse_index={}, guide_index={}, base_dir='/') page_info = parser.docs_for_object( full_name='TestModule', py_object=test_module, parser_config=parser_config) # Make sure the brief docstring is present self.assertEqual( tf_inspect.getdoc(test_module).split('\n')[0], page_info.doc.brief) # Make sure that the members are there funcs = {f_info.obj for f_info in page_info.functions} self.assertEqual({test_function, test_function_with_args_kwargs}, funcs) classes = {cls_info.obj for cls_info in page_info.classes} self.assertEqual({TestClass}, classes) # Make sure the module's file is contained as the definition location. self.assertEqual( os.path.relpath(test_module.__file__.rstrip('c'), '/'), page_info.defined_in.path)
def test_docs_for_module(self): # Get the current module. module = sys.modules[__name__] index = { 'TestModule': module, 'TestModule.test_function': test_function, 'TestModule.test_function_with_args_kwargs': test_function_with_args_kwargs, 'TestModule.TestClass': TestClass, } visitor = DummyVisitor(index=index, duplicate_of={}) reference_resolver = parser.ReferenceResolver.from_visitor( visitor=visitor, doc_index={}, py_module_names=['tf']) tree = { 'TestModule': ['TestClass', 'test_function', 'test_function_with_args_kwargs'] } parser_config = parser.ParserConfig( reference_resolver=reference_resolver, duplicates={}, duplicate_of={}, tree=tree, index=index, reverse_index={}, guide_index={}, base_dir='/') page_info = parser.docs_for_object(full_name='TestModule', py_object=module, parser_config=parser_config) # Make sure the brief docstring is present self.assertEqual( tf_inspect.getdoc(module).split('\n')[0], page_info.doc.brief) # Make sure that the members are there funcs = {f_info.obj for f_info in page_info.functions} self.assertEqual({test_function, test_function_with_args_kwargs}, funcs) classes = {cls_info.obj for cls_info in page_info.classes} self.assertEqual({TestClass}, classes) # Make sure this file is contained as the definition location. self.assertEqual(os.path.relpath(__file__, '/'), page_info.defined_in.path)
def test_docs_for_class(self): index = { 'TestClass': TestClass, 'TestClass.a_method': TestClass.a_method, 'TestClass.a_property': TestClass.a_property, 'TestClass.ChildClass': TestClass.ChildClass, 'TestClass.CLASS_MEMBER': TestClass.CLASS_MEMBER } visitor = DummyVisitor(index=index, duplicate_of={}) reference_resolver = parser.ReferenceResolver.from_visitor( visitor=visitor, doc_index={}, py_module_names=['tf']) tree = { 'TestClass': ['a_method', 'a_property', 'ChildClass', 'CLASS_MEMBER'] } parser_config = parser.ParserConfig( reference_resolver=reference_resolver, duplicates={}, duplicate_of={}, tree=tree, index=index, reverse_index={}, guide_index={}, base_dir='/') page_info = parser.docs_for_object( full_name='TestClass', py_object=TestClass, parser_config=parser_config) # Make sure the brief docstring is present self.assertEqual( tf_inspect.getdoc(TestClass).split('\n')[0], page_info.doc.brief) # Make sure the method is present self.assertEqual(TestClass.a_method, page_info.methods[0].obj) # Make sure that the signature is extracted properly and omits self. self.assertEqual(["arg='default'"], page_info.methods[0].signature) # Make sure the property is present self.assertIs(TestClass.a_property, page_info.properties[0].obj) # Make sure there is a link to the child class and it points the right way. self.assertIs(TestClass.ChildClass, page_info.classes[0].obj) # Make sure this file is contained as the definition location. self.assertEqual(os.path.relpath(__file__, '/'), page_info.defined_in.path)
def test_docs_for_message_class(self): class CMessage(object): def hidden(self): pass class Message(object): def hidden2(self): pass class MessageMeta(object): def hidden3(self): pass class ChildMessage(CMessage, Message, MessageMeta): def my_method(self): pass index = { 'ChildMessage': ChildMessage, 'ChildMessage.hidden': ChildMessage.hidden, 'ChildMessage.hidden2': ChildMessage.hidden2, 'ChildMessage.hidden3': ChildMessage.hidden3, 'ChildMessage.my_method': ChildMessage.my_method, } visitor = DummyVisitor(index=index, duplicate_of={}) reference_resolver = parser.ReferenceResolver.from_visitor( visitor=visitor, doc_index={}, py_module_names=['tf']) tree = {'ChildMessage': ['hidden', 'hidden2', 'hidden3', 'my_method']} parser_config = parser.ParserConfig( reference_resolver=reference_resolver, duplicates={}, duplicate_of={}, tree=tree, index=index, reverse_index={}, guide_index={}, base_dir='/') page_info = parser.docs_for_object(full_name='ChildMessage', py_object=ChildMessage, parser_config=parser_config) self.assertEqual(1, len(page_info.methods)) self.assertEqual('my_method', page_info.methods[0].short_name)
def test_namedtuple_field_order(self): namedtupleclass = collections.namedtuple('namedtupleclass', {'z', 'y', 'x', 'w', 'v', 'u'}) index = { 'namedtupleclass': namedtupleclass, 'namedtupleclass.u': namedtupleclass.u, 'namedtupleclass.v': namedtupleclass.v, 'namedtupleclass.w': namedtupleclass.w, 'namedtupleclass.x': namedtupleclass.x, 'namedtupleclass.y': namedtupleclass.y, 'namedtupleclass.z': namedtupleclass.z, } visitor = DummyVisitor(index=index, duplicate_of={}) reference_resolver = parser.ReferenceResolver.from_visitor( visitor=visitor, doc_index={}, py_module_names=['tf']) tree = {'namedtupleclass': {'u', 'v', 'w', 'x', 'y', 'z'}} parser_config = parser.ParserConfig( reference_resolver=reference_resolver, duplicates={}, duplicate_of={}, tree=tree, index=index, reverse_index={}, guide_index={}, base_dir='/') page_info = parser.docs_for_object( full_name='namedtupleclass', py_object=namedtupleclass, parser_config=parser_config) # Each namedtiple field has a docstring of the form: # 'Alias for field number ##'. These props are returned sorted. def sort_key(prop_info): return int(prop_info.obj.__doc__.split(' ')[-1]) self.assertSequenceEqual(page_info.properties, sorted(page_info.properties, key=sort_key))
def test_namedtuple_field_order(self): namedtupleclass = collections.namedtuple( 'namedtupleclass', {'z', 'y', 'x', 'w', 'v', 'u'}) index = { 'namedtupleclass': namedtupleclass, 'namedtupleclass.u': namedtupleclass.u, 'namedtupleclass.v': namedtupleclass.v, 'namedtupleclass.w': namedtupleclass.w, 'namedtupleclass.x': namedtupleclass.x, 'namedtupleclass.y': namedtupleclass.y, 'namedtupleclass.z': namedtupleclass.z, } visitor = DummyVisitor(index=index, duplicate_of={}) reference_resolver = parser.ReferenceResolver.from_visitor( visitor=visitor, doc_index={}, py_module_names=['tf']) tree = {'namedtupleclass': {'u', 'v', 'w', 'x', 'y', 'z'}} parser_config = parser.ParserConfig( reference_resolver=reference_resolver, duplicates={}, duplicate_of={}, tree=tree, index=index, reverse_index={}, guide_index={}, base_dir='/') page_info = parser.docs_for_object(full_name='namedtupleclass', py_object=namedtupleclass, parser_config=parser_config) # Each namedtiple field has a docstring of the form: # 'Alias for field number ##'. These props are returned sorted. def sort_key(prop_info): return int(prop_info.obj.__doc__.split(' ')[-1]) self.assertSequenceEqual(page_info.properties, sorted(page_info.properties, key=sort_key))
def test_docs_for_class_should_skip(self): class Parent(object): @doc_controls.do_not_doc_inheritable def a_method(self, arg='default'): pass class Child(Parent): def a_method(self, arg='default'): pass index = { 'Child': Child, 'Child.a_method': Child.a_method, } visitor = DummyVisitor(index=index, duplicate_of={}) reference_resolver = parser.ReferenceResolver.from_visitor( visitor=visitor, doc_index={}, py_module_names=['tf']) tree = { 'Child': ['a_method'], } parser_config = parser.ParserConfig( reference_resolver=reference_resolver, duplicates={}, duplicate_of={}, tree=tree, index=index, reverse_index={}, guide_index={}, base_dir='/') page_info = parser.docs_for_object( full_name='Child', py_object=Child, parser_config=parser_config) # Make sure the `a_method` is not present self.assertEqual(0, len(page_info.methods))
def test_docs_for_class_should_skip(self): class Parent(object): @doc_controls.do_not_doc_inheritable def a_method(self, arg='default'): pass class Child(Parent): def a_method(self, arg='default'): pass index = { 'Child': Child, 'Child.a_method': Child.a_method, } visitor = DummyVisitor(index=index, duplicate_of={}) reference_resolver = parser.ReferenceResolver.from_visitor( visitor=visitor, doc_index={}, py_module_names=['tf']) tree = { 'Child': ['a_method'], } parser_config = parser.ParserConfig( reference_resolver=reference_resolver, duplicates={}, duplicate_of={}, tree=tree, index=index, reverse_index={}, guide_index={}, base_dir='/') page_info = parser.docs_for_object(full_name='Child', py_object=Child, parser_config=parser_config) # Make sure the `a_method` is not present self.assertEqual(0, len(page_info.methods))
def test_docs_for_module(self): # Get the current module. module = sys.modules[__name__] index = { 'TestModule': module, 'TestModule.test_function': test_function, 'TestModule.test_function_with_args_kwargs': test_function_with_args_kwargs, 'TestModule.TestClass': TestClass, } reference_resolver = parser.ReferenceResolver( duplicate_of={}, doc_index={}, index=index, py_module_names=['tf']) tree = { 'TestModule': ['TestClass', 'test_function', 'test_function_with_args_kwargs'] } parser_config = parser.ParserConfig( reference_resolver=reference_resolver, duplicates={}, tree=tree, reverse_index={}, guide_index={}, base_dir='/') page_info = parser.docs_for_object( full_name='TestModule', py_object=module, parser_config=parser_config) # Make sure the brief docstring is present self.assertEqual(inspect.getdoc(module).split('\n')[0], page_info.doc.brief) # Make sure that the members are there members = [member_info.obj for member_info in page_info.members] self.assertIn(test_function, members) self.assertIn(test_function_with_args_kwargs, members) self.assertIn(TestClass, members) # Make sure this file is contained as the definition location. self.assertEqual(os.path.relpath(__file__, '/'), page_info.defined_in.path)
def write_docs(output_dir, parser_config, yaml_toc, root_title='TensorFlow'): """Write previously extracted docs to disk. Write a docs page for each symbol included in the indices of parser_config to a tree of docs at `output_dir`. Symbols with multiple aliases will have only one page written about them, which is referenced for all aliases. Args: output_dir: Directory to write documentation markdown files to. Will be created if it doesn't exist. parser_config: A `parser.ParserConfig` object, containing all the necessary indices. yaml_toc: Set to `True` to generate a "_toc.yaml" file. root_title: The title name for the root level index.md. Raises: ValueError: if `output_dir` is not an absolute path """ # Make output_dir. if not os.path.isabs(output_dir): raise ValueError("'output_dir' must be an absolute path.\n" " output_dir='%s'" % output_dir) try: if not os.path.exists(output_dir): os.makedirs(output_dir) except OSError as e: print('Creating output dir "%s" failed: %s' % (output_dir, e)) raise # These dictionaries are used for table-of-contents generation below # They will contain, after the for-loop below:: # - module name(string):classes and functions the module contains(list) module_children = {} # - symbol name(string):pathname (string) symbol_to_file = {} # Parse and write Markdown pages, resolving cross-links (@{symbol}). for full_name, py_object in six.iteritems(parser_config.index): parser_config.reference_resolver.current_doc_full_name = full_name if full_name in parser_config.duplicate_of: continue # Methods and some routines are documented only as part of their class. if not (tf_inspect.ismodule(py_object) or tf_inspect.isclass(py_object) or _is_free_function(py_object, full_name, parser_config.index)): continue sitepath = os.path.join('api_docs/python', parser.documentation_path(full_name)[:-3]) # For TOC, we need to store a mapping from full_name to the file # we're generating symbol_to_file[full_name] = sitepath # For a module, remember the module for the table-of-contents if tf_inspect.ismodule(py_object): if full_name in parser_config.tree: module_children.setdefault(full_name, []) # For something else that's documented, # figure out what module it lives in else: subname = str(full_name) while True: subname = subname[:subname.rindex('.')] if tf_inspect.ismodule(parser_config.index[subname]): module_children.setdefault(subname, []).append(full_name) break print('Writing docs for %s (%r).' % (full_name, py_object)) # Generate docs for `py_object`, resolving references. page_info = parser.docs_for_object(full_name, py_object, parser_config) path = os.path.join(output_dir, parser.documentation_path(full_name)) directory = os.path.dirname(path) try: if not os.path.exists(directory): os.makedirs(directory) with open(path, 'w') as f: f.write(pretty_docs.build_md_page(page_info)) except OSError as e: print('Cannot write documentation for %s to %s: %s' % (full_name, directory, e)) raise if yaml_toc: # Generate table of contents # Put modules in alphabetical order, case-insensitive modules = sorted(module_children.keys(), key=lambda a: a.upper()) leftnav_path = os.path.join(output_dir, '_toc.yaml') with open(leftnav_path, 'w') as f: # Generate header f.write('# Automatically generated file; please do not edit\ntoc:\n') for module in modules: indent_num = module.count('.') # Don't list `tf.submodule` inside `tf` indent_num = max(indent_num, 1) indent = ' '*indent_num if indent_num > 1: # tf.contrib.baysflow.entropy will be under # tf.contrib->baysflow->entropy title = module.split('.')[-1] else: title = module header = [ '- title: ' + title, ' section:', ' - title: Overview', ' path: /TARGET_DOC_ROOT/VERSION/' + symbol_to_file[module]] header = ''.join([indent+line+'\n' for line in header]) f.write(header) symbols_in_module = module_children.get(module, []) # Sort case-insensitive, if equal sort case sensitive (upper first) symbols_in_module.sort(key=lambda a: (a.upper(), a)) for full_name in symbols_in_module: item = [ ' - title: ' + full_name[len(module) + 1:], ' path: /TARGET_DOC_ROOT/VERSION/' + symbol_to_file[full_name]] item = ''.join([indent+line+'\n' for line in item]) f.write(item) # Write a global index containing all full names with links. with open(os.path.join(output_dir, 'index.md'), 'w') as f: f.write( parser.generate_global_index(root_title, parser_config.index, parser_config.reference_resolver))
def write_docs(output_dir, parser_config, duplicate_of, index, yaml_toc): """Write previously extracted docs to disk. Write a docs page for each symbol in `index` to a tree of docs at `output_dir`. Symbols with multiple aliases will have only one page written about them, which is referenced for all aliases. Args: output_dir: Directory to write documentation markdown files to. Will be created if it doesn't exist. parser_config: A `parser.ParserConfig` object. duplicate_of: A `dict` mapping fully qualified names to "master" names. Used to determine which docs pages to write. index: A `dict` mapping fully qualified names to the corresponding Python objects. Used to produce docs for child objects. yaml_toc: Set to `True` to generate a "_toc.yaml" file. """ # Make output_dir. try: if not os.path.exists(output_dir): os.makedirs(output_dir) except OSError as e: print('Creating output dir "%s" failed: %s' % (output_dir, e)) raise # These dictionaries are used for table-of-contents generation below # They will contain, after the for-loop below:: # - module name(string):classes and functions the module contains(list) module_children = {} # - symbol name(string):pathname (string) symbol_to_file = {} # Parse and write Markdown pages, resolving cross-links (@{symbol}). for full_name, py_object in six.iteritems(index): if full_name in duplicate_of: continue # Methods and some routines are documented only as part of their class. if not (inspect.ismodule(py_object) or inspect.isclass(py_object) or _is_free_function(py_object, full_name, index)): continue sitepath = os.path.join('api_docs/python', parser.documentation_path(full_name)[:-3]) # For TOC, we need to store a mapping from full_name to the file # we're generating symbol_to_file[full_name] = sitepath # For a module, remember the module for the table-of-contents if inspect.ismodule(py_object): if full_name in parser_config.tree: module_children.setdefault(full_name, []) # For something else that's documented, # figure out what module it lives in else: subname = str(full_name) while True: subname = subname[:subname.rindex('.')] if inspect.ismodule(index[subname]): module_children.setdefault(subname, []).append(full_name) break print('Writing docs for %s (%r).' % (full_name, py_object)) # Generate docs for `py_object`, resolving references. page_info = parser.docs_for_object(full_name, py_object, parser_config) path = os.path.join(output_dir, parser.documentation_path(full_name)) directory = os.path.dirname(path) try: if not os.path.exists(directory): os.makedirs(directory) with open(path, 'w') as f: f.write(pretty_docs.build_md_page(page_info)) except OSError as e: print('Cannot write documentation for %s to %s: %s' % (full_name, directory, e)) raise if yaml_toc: # Generate table of contents # Put modules in alphabetical order, case-insensitive modules = sorted(module_children.keys(), key=lambda a: a.upper()) leftnav_path = os.path.join(output_dir, '_toc.yaml') with open(leftnav_path, 'w') as f: # Generate header f.write('# Automatically generated file; please do not edit\ntoc:\n') for module in modules: f.write(' - title: ' + module + '\n' ' section:\n' + ' - title: Overview\n' + ' path: /TARGET_DOC_ROOT/' + symbol_to_file[module] + '\n') symbols_in_module = module_children.get(module, []) symbols_in_module.sort(key=lambda a: a.upper()) for full_name in symbols_in_module: f.write(' - title: ' + full_name[len(module)+1:] + '\n' ' path: /TARGET_DOC_ROOT/' + symbol_to_file[full_name] + '\n') # Write a global index containing all full names with links. with open(os.path.join(output_dir, 'index.md'), 'w') as f: f.write(parser.generate_global_index( 'TensorFlow', index, parser_config.reference_resolver))
def write_docs(output_dir, parser_config, yaml_toc, root_title='TensorFlow', search_hints=True, site_api_path=''): """Write previously extracted docs to disk. Write a docs page for each symbol included in the indices of parser_config to a tree of docs at `output_dir`. Symbols with multiple aliases will have only one page written about them, which is referenced for all aliases. Args: output_dir: Directory to write documentation markdown files to. Will be created if it doesn't exist. parser_config: A `parser.ParserConfig` object, containing all the necessary indices. yaml_toc: Set to `True` to generate a "_toc.yaml" file. root_title: The title name for the root level index.md. search_hints: (bool) include meta-data search hints at the top of each output file. site_api_path: The output path relative to the site root. Used in the `_toc.yaml` and `_redirects.yaml` files. Raises: ValueError: if `output_dir` is not an absolute path """ # Make output_dir. if not os.path.isabs(output_dir): raise ValueError("'output_dir' must be an absolute path.\n" " output_dir='%s'" % output_dir) if not os.path.exists(output_dir): os.makedirs(output_dir) # These dictionaries are used for table-of-contents generation below # They will contain, after the for-loop below:: # - module name(string):classes and functions the module contains(list) module_children = {} # - symbol name(string):pathname (string) symbol_to_file = {} # Collect redirects for an api _redirects.yaml file. redirects = [] # Parse and write Markdown pages, resolving cross-links (@{symbol}). for full_name, py_object in six.iteritems(parser_config.index): parser_config.reference_resolver.current_doc_full_name = full_name if full_name in parser_config.duplicate_of: continue # Methods and some routines are documented only as part of their class. if not (tf_inspect.ismodule(py_object) or tf_inspect.isclass(py_object) or parser.is_free_function(py_object, full_name, parser_config.index)): continue sitepath = os.path.join('api_docs/python', parser.documentation_path(full_name)[:-3]) # For TOC, we need to store a mapping from full_name to the file # we're generating symbol_to_file[full_name] = sitepath # For a module, remember the module for the table-of-contents if tf_inspect.ismodule(py_object): if full_name in parser_config.tree: module_children.setdefault(full_name, []) # For something else that's documented, # figure out what module it lives in else: subname = str(full_name) while True: subname = subname[:subname.rindex('.')] if tf_inspect.ismodule(parser_config.index[subname]): module_children.setdefault(subname, []).append(full_name) break # Generate docs for `py_object`, resolving references. page_info = parser.docs_for_object(full_name, py_object, parser_config) path = os.path.join(output_dir, parser.documentation_path(full_name)) directory = os.path.dirname(path) try: if not os.path.exists(directory): os.makedirs(directory) # This function returns raw bytes in PY2 or unicode in PY3. if search_hints: content = [page_info.get_metadata_html()] else: content = [''] content.append(pretty_docs.build_md_page(page_info)) text = '\n'.join(content) if six.PY3: text = text.encode('utf-8') with open(path, 'wb') as f: f.write(text) except OSError: raise OSError( 'Cannot write documentation for %s to %s' % (full_name, directory)) duplicates = parser_config.duplicates.get(full_name, []) if not duplicates: continue duplicates = [item for item in duplicates if item != full_name] for dup in duplicates: from_path = os.path.join(site_api_path, dup.replace('.', '/')) to_path = os.path.join(site_api_path, full_name.replace('.', '/')) redirects.append(( os.path.join('/', from_path), os.path.join('/', to_path))) if redirects: redirects = sorted(redirects) template = ('- from: {}\n' ' to: {}\n') redirects = [template.format(f, t) for f, t in redirects] api_redirects_path = os.path.join(output_dir, '_redirects.yaml') with open(api_redirects_path, 'w') as redirect_file: redirect_file.write('redirects:\n') redirect_file.write(''.join(redirects)) if yaml_toc: # Generate table of contents # Put modules in alphabetical order, case-insensitive modules = sorted(module_children.keys(), key=lambda a: a.upper()) leftnav_path = os.path.join(output_dir, '_toc.yaml') with open(leftnav_path, 'w') as f: # Generate header f.write('# Automatically generated file; please do not edit\ntoc:\n') for module in modules: indent_num = module.count('.') # Don't list `tf.submodule` inside `tf` indent_num = max(indent_num, 1) indent = ' '*indent_num if indent_num > 1: # tf.contrib.baysflow.entropy will be under # tf.contrib->baysflow->entropy title = module.split('.')[-1] else: title = module header = [ '- title: ' + title, ' section:', ' - title: Overview', ' path: ' + os.path.join('/', site_api_path, symbol_to_file[module])] header = ''.join([indent+line+'\n' for line in header]) f.write(header) symbols_in_module = module_children.get(module, []) # Sort case-insensitive, if equal sort case sensitive (upper first) symbols_in_module.sort(key=lambda a: (a.upper(), a)) for full_name in symbols_in_module: item = [ ' - title: ' + full_name[len(module) + 1:], ' path: ' + os.path.join('/', site_api_path, symbol_to_file[full_name])] item = ''.join([indent+line+'\n' for line in item]) f.write(item) # Write a global index containing all full names with links. with open(os.path.join(output_dir, 'index.md'), 'w') as f: f.write( parser.generate_global_index(root_title, parser_config.index, parser_config.reference_resolver))
def write_docs(output_dir, parser_config, yaml_toc, root_title='TensorFlow', search_hints=True, site_api_path=None): """Write previously extracted docs to disk. Write a docs page for each symbol included in the indices of parser_config to a tree of docs at `output_dir`. Symbols with multiple aliases will have only one page written about them, which is referenced for all aliases. Args: output_dir: Directory to write documentation markdown files to. Will be created if it doesn't exist. parser_config: A `parser.ParserConfig` object, containing all the necessary indices. yaml_toc: Set to `True` to generate a "_toc.yaml" file. root_title: The title name for the root level index.md. search_hints: (bool) include meta-data search hints at the top of each output file. site_api_path: Used to write the api-duplicates _redirects.yaml file. if None (the default) the file is not generated. Raises: ValueError: if `output_dir` is not an absolute path """ # Make output_dir. if not os.path.isabs(output_dir): raise ValueError("'output_dir' must be an absolute path.\n" " output_dir='%s'" % output_dir) if not os.path.exists(output_dir): os.makedirs(output_dir) # These dictionaries are used for table-of-contents generation below # They will contain, after the for-loop below:: # - module name(string):classes and functions the module contains(list) module_children = {} # - symbol name(string):pathname (string) symbol_to_file = {} # Collect redirects for an api _redirects.yaml file. redirects = ['redirects:\n'] # Parse and write Markdown pages, resolving cross-links (@{symbol}). for full_name, py_object in six.iteritems(parser_config.index): parser_config.reference_resolver.current_doc_full_name = full_name if full_name in parser_config.duplicate_of: continue # Methods and some routines are documented only as part of their class. if not (tf_inspect.ismodule(py_object) or tf_inspect.isclass(py_object) or _is_free_function( py_object, full_name, parser_config.index)): continue sitepath = os.path.join('api_docs/python', parser.documentation_path(full_name)[:-3]) # For TOC, we need to store a mapping from full_name to the file # we're generating symbol_to_file[full_name] = sitepath # For a module, remember the module for the table-of-contents if tf_inspect.ismodule(py_object): if full_name in parser_config.tree: module_children.setdefault(full_name, []) # For something else that's documented, # figure out what module it lives in else: subname = str(full_name) while True: subname = subname[:subname.rindex('.')] if tf_inspect.ismodule(parser_config.index[subname]): module_children.setdefault(subname, []).append(full_name) break # Generate docs for `py_object`, resolving references. page_info = parser.docs_for_object(full_name, py_object, parser_config) path = os.path.join(output_dir, parser.documentation_path(full_name)) directory = os.path.dirname(path) try: if not os.path.exists(directory): os.makedirs(directory) # This function returns raw bytes in PY2 or unicode in PY3. if search_hints: content = [page_info.get_metadata_html()] else: content = [''] content.append(pretty_docs.build_md_page(page_info)) text = '\n'.join(content) if six.PY3: text = text.encode('utf-8') with open(path, 'wb') as f: f.write(text) except OSError: raise OSError('Cannot write documentation for %s to %s' % (full_name, directory)) if site_api_path: duplicates = parser_config.duplicates.get(full_name, []) if not duplicates: continue duplicates = [item for item in duplicates if item != full_name] template = ('- from: /{}\n' ' to: /{}\n') for dup in duplicates: from_path = os.path.join(site_api_path, dup.replace('.', '/')) to_path = os.path.join(site_api_path, full_name.replace('.', '/')) redirects.append(template.format(from_path, to_path)) if site_api_path: api_redirects_path = os.path.join(output_dir, '_redirects.yaml') with open(api_redirects_path, 'w') as redirect_file: redirect_file.write(''.join(redirects)) if yaml_toc: # Generate table of contents # Put modules in alphabetical order, case-insensitive modules = sorted(module_children.keys(), key=lambda a: a.upper()) leftnav_path = os.path.join(output_dir, '_toc.yaml') with open(leftnav_path, 'w') as f: # Generate header f.write( '# Automatically generated file; please do not edit\ntoc:\n') for module in modules: indent_num = module.count('.') # Don't list `tf.submodule` inside `tf` indent_num = max(indent_num, 1) indent = ' ' * indent_num if indent_num > 1: # tf.contrib.baysflow.entropy will be under # tf.contrib->baysflow->entropy title = module.split('.')[-1] else: title = module header = [ '- title: ' + title, ' section:', ' - title: Overview', ' path: /TARGET_DOC_ROOT/VERSION/' + symbol_to_file[module] ] header = ''.join([indent + line + '\n' for line in header]) f.write(header) symbols_in_module = module_children.get(module, []) # Sort case-insensitive, if equal sort case sensitive (upper first) symbols_in_module.sort(key=lambda a: (a.upper(), a)) for full_name in symbols_in_module: item = [ ' - title: ' + full_name[len(module) + 1:], ' path: /TARGET_DOC_ROOT/VERSION/' + symbol_to_file[full_name] ] item = ''.join([indent + line + '\n' for line in item]) f.write(item) # Write a global index containing all full names with links. with open(os.path.join(output_dir, 'index.md'), 'w') as f: f.write( parser.generate_global_index(root_title, parser_config.index, parser_config.reference_resolver))
def write_docs(output_dir, parser_config, yaml_toc): """Write previously extracted docs to disk. Write a docs page for each symbol included in the indices of parser_config to a tree of docs at `output_dir`. Symbols with multiple aliases will have only one page written about them, which is referenced for all aliases. Args: output_dir: Directory to write documentation markdown files to. Will be created if it doesn't exist. parser_config: A `parser.ParserConfig` object, containing all the necessary indices. yaml_toc: Set to `True` to generate a "_toc.yaml" file. Raises: ValueError: if `output_dir` is not an absolute path """ # Make output_dir. if not os.path.isabs(output_dir): raise ValueError( "'output_dir' must be an absolute path.\n" " output_dir='%s'" % output_dir) try: if not os.path.exists(output_dir): os.makedirs(output_dir) except OSError as e: print('Creating output dir "%s" failed: %s' % (output_dir, e)) raise # These dictionaries are used for table-of-contents generation below # They will contain, after the for-loop below:: # - module name(string):classes and functions the module contains(list) module_children = {} # - symbol name(string):pathname (string) symbol_to_file = {} # Parse and write Markdown pages, resolving cross-links (@{symbol}). for full_name, py_object in six.iteritems(parser_config.index): if full_name in parser_config.duplicate_of: continue # Methods and some routines are documented only as part of their class. if not (tf_inspect.ismodule(py_object) or tf_inspect.isclass(py_object) or _is_free_function(py_object, full_name, parser_config.index)): continue sitepath = os.path.join('api_docs/python', parser.documentation_path(full_name)[:-3]) # For TOC, we need to store a mapping from full_name to the file # we're generating symbol_to_file[full_name] = sitepath # For a module, remember the module for the table-of-contents if tf_inspect.ismodule(py_object): if full_name in parser_config.tree: module_children.setdefault(full_name, []) # For something else that's documented, # figure out what module it lives in else: subname = str(full_name) while True: subname = subname[:subname.rindex('.')] if tf_inspect.ismodule(parser_config.index[subname]): module_children.setdefault(subname, []).append(full_name) break print('Writing docs for %s (%r).' % (full_name, py_object)) # Generate docs for `py_object`, resolving references. page_info = parser.docs_for_object(full_name, py_object, parser_config) path = os.path.join(output_dir, parser.documentation_path(full_name)) directory = os.path.dirname(path) try: if not os.path.exists(directory): os.makedirs(directory) with open(path, 'w') as f: f.write(pretty_docs.build_md_page(page_info)) except OSError as e: print('Cannot write documentation for %s to %s: %s' % (full_name, directory, e)) raise if yaml_toc: # Generate table of contents # Put modules in alphabetical order, case-insensitive modules = sorted(module_children.keys(), key=lambda a: a.upper()) leftnav_path = os.path.join(output_dir, '_toc.yaml') with open(leftnav_path, 'w') as f: # Generate header f.write('# Automatically generated file; please do not edit\ntoc:\n') for module in modules: f.write(' - title: ' + module + '\n' ' section:\n' + ' - title: Overview\n' + ' path: /TARGET_DOC_ROOT/VERSION/' + symbol_to_file[module] + '\n') symbols_in_module = module_children.get(module, []) # Sort case-insensitive, if equal sort case sensitive (upper first) symbols_in_module.sort(key=lambda a: (a.upper(), a)) for full_name in symbols_in_module: f.write(' - title: ' + full_name[len(module) + 1:] + '\n' ' path: /TARGET_DOC_ROOT/VERSION/' + symbol_to_file[full_name] + '\n') # Write a global index containing all full names with links. with open(os.path.join(output_dir, 'index.md'), 'w') as f: f.write( parser.generate_global_index('TensorFlow', parser_config.index, parser_config.reference_resolver))