def _score_name(self, name): """Return a tuple of scores indicating how to sort for the best name. This function is meant to be used as the `key` to the `sorted` function. This returns a score tuple composed of the following scores: defining_class: Prefers method names pointing into the defining class, over a subclass (`ParentClass.method` over `Subclass.method`, if it referrs to the same method implementation). experimental: Prefers names that are not in "contrib" or "experimental". keras: Prefers keras names to non-keras names. module_length: Prefers submodules (tf.sub.thing) over the root namespace (tf.thing) over deeply nested paths (tf.a.b.c.thing) name: Fallback, sorts lexicographically on the full_name. Args: name: the full name to score, for example `tf.estimator.Estimator` Returns: A tuple of scores. When sorted the preferred name will have the lowest value. """ parts = name.split('.') short_name = parts[-1] if len(parts) == 1: return (-99, -99, -99, -99, short_name) container = self._index['.'.join(parts[:-1])] defining_class_score = 1 if tf_inspect.isclass(container): if short_name in container.__dict__: # prefer the defining class defining_class_score = -1 experimental_score = -1 if 'contrib' in parts or any('experimental' in part for part in parts): experimental_score = 1 keras_score = 1 if 'keras' in parts: keras_score = -1 while parts: container = self._index['.'.join(parts)] if tf_inspect.ismodule(container): break parts.pop() module_length = len(parts) if len(parts) == 2: # `tf.submodule.thing` is better than `tf.thing` module_length_score = -1 else: # shorter is better module_length_score = module_length return (defining_class_score, experimental_score, keras_score, module_length_score, name)
def __call__(self, parent_name, parent, children): """Visitor interface, see `tensorflow/tools/common:traverse` for details. This method is called for each symbol found in a traversal using `tensorflow/tools/common:traverse`. It should not be called directly in user code. Args: parent_name: The fully qualified name of a symbol found during traversal. parent: The Python object referenced by `parent_name`. children: A list of `(name, py_object)` pairs enumerating, in alphabetical order, the children (as determined by `tf_inspect.getmembers`) of `parent`. `name` is the local name of `py_object` in `parent`. Raises: RuntimeError: If this visitor is called with a `parent` that is not a class or module. """ parent_name = self._add_prefix(parent_name) self._index[parent_name] = parent self._tree[parent_name] = [] if not (tf_inspect.ismodule(parent) or tf_inspect.isclass(parent)): raise RuntimeError('Unexpected type in visitor -- %s: %r' % (parent_name, parent)) for i, (name, child) in enumerate(list(children)): # Don't document __metaclass__ if name in ['__metaclass__']: del children[i] continue full_name = '.'.join([parent_name, name]) if parent_name else name self._index[full_name] = child self._tree[parent_name].append(name)
def _score_name(self, name): """Return a tuple of scores indicating how to sort for the best name. This function is meant to be used as the `key` to the `sorted` function. This sorting in order: Prefers names refering to the defining class, over a subclass. Prefers names that are not in "contrib". prefers submodules to the root namespace. Prefers short names `tf.thing` over `tf.a.b.c.thing` Sorts lexicographically on name parts. Args: name: the full name to score, for example `tf.estimator.Estimator` Returns: A tuple of scores. When sorted the preferred name will have the lowest value. """ parts = name.split('.') short_name = parts[-1] if len(parts) == 1: return (-99, -99, -99, short_name) container = self._index['.'.join(parts[:-1])] defining_class_score = 1 if tf_inspect.isclass(container): if short_name in container.__dict__: # prefer the defining class defining_class_score = -1 contrib_score = -1 if 'contrib' in parts: contrib_score = 1 while parts: container = self._index['.'.join(parts)] if tf_inspect.ismodule(container): break parts.pop() module_length = len(parts) if len(parts) == 2: # `tf.submodule.thing` is better than `tf.thing` module_length_score = -1 else: # shorter is better module_length_score = module_length return (defining_class_score, contrib_score, module_length_score, name)
def _score_name(self, name): """Return a tuple of scores indicating how to sort for the best name. This function is meant to be used as the `key` to the `sorted` function. This sorting in order: Prefers names refering to the defining class, over a subclass. Prefers names that are not in "contrib". prefers submodules to the root namespace. Prefers short names `tf.thing` over `tf.a.b.c.thing` Sorts lexicographically on name parts. Args: name: the full name to score, for example `tf.estimator.Estimator` Returns: A tuple of scores. When sorted the preferred name will have the lowest value. """ parts = name.split('.') short_name = parts[-1] if len(parts) == 1: return (-99, -99, -99, short_name) container = self._index['.'.join(parts[:-1])] defining_class_score = 1 if tf_inspect.isclass(container): if short_name in container.__dict__: # prefer the defining class defining_class_score = -1 contrib_score = -1 if 'contrib' in parts: contrib_score = 1 while parts: container = self._index['.'.join(parts)] if tf_inspect.ismodule(container): break parts.pop() module_length = len(parts) if len(parts) == 2: # `tf.submodule.thing` is better than `tf.thing` module_length_score = -1 else: # shorter is better module_length_score = module_length return (defining_class_score, contrib_score, module_length_score, name)
def _get_raw_docstring(py_object): """Get the docs for a given python object. Args: py_object: A python object to retrieve the docs for (class, function/method, or module). Returns: The docstring, or the empty string if no docstring was found. """ # For object instances, tf_inspect.getdoc does give us the docstring of their # type, which is not what we want. Only return the docstring if it is useful. if (tf_inspect.isclass(py_object) or tf_inspect.ismethod(py_object) or tf_inspect.isfunction(py_object) or tf_inspect.ismodule(py_object) or isinstance(py_object, property)): return tf_inspect.getdoc(py_object) or '' else: return ''
def __call__(self, parent_path, parent, children): """Visitor interface, see `tensorflow/tools/common:traverse` for details. This method is called for each symbol found in a traversal using `tensorflow/tools/common:traverse`. It should not be called directly in user code. Args: parent_path: A tuple of strings. The fully qualified path to a symbol found during traversal. parent: The Python object referenced by `parent_name`. children: A list of `(name, py_object)` pairs enumerating, in alphabetical order, the children (as determined by `tf_inspect.getmembers`) of `parent`. `name` is the local name of `py_object` in `parent`. Returns: The list of children, with any __metaclass__ removed. Raises: RuntimeError: If this visitor is called with a `parent` that is not a class or module. """ parent_name = '.'.join(parent_path) self._index[parent_name] = parent self._tree[parent_name] = [] if not (tf_inspect.ismodule(parent) or tf_inspect.isclass(parent)): raise RuntimeError('Unexpected type in visitor -- %s: %r' % (parent_name, parent)) for i, (name, child) in enumerate(list(children)): # Don't document __metaclass__ if name in ['__metaclass__']: del children[i] continue full_name = '.'.join([parent_name, name]) if parent_name else name self._index[full_name] = child self._tree[parent_name].append(name) return children
def collect_docs_for_module(self, parser_config): """Collect information necessary specifically for a module's doc page. Mainly this is information about the members of the module. Args: parser_config: An instance of ParserConfig. """ relative_path = os.path.relpath( path='.', start=os.path.dirname(documentation_path(self.full_name)) or '.') member_names = parser_config.tree.get(self.full_name, []) for name in member_names: if name in ['__builtins__', '__doc__', '__file__', '__name__', '__path__', '__package__', '__cached__', '__loader__', '__spec__', 'absolute_import', 'division', 'print_function', 'unicode_literals']: continue member_full_name = self.full_name + '.' + name if self.full_name else name member = parser_config.py_name_to_object(member_full_name) member_doc = _parse_md_docstring(member, relative_path, parser_config.reference_resolver) url = parser_config.reference_resolver.reference_to_url( member_full_name, relative_path) if tf_inspect.ismodule(member): self._add_module(name, member_full_name, member, member_doc, url) elif tf_inspect.isclass(member): self._add_class(name, member_full_name, member, member_doc, url) elif tf_inspect.isfunction(member): self._add_function(name, member_full_name, member, member_doc, url) else: self._add_other_member(name, member_full_name, member, member_doc)
def from_visitor(cls, visitor, **kwargs): """A factory function for building a ReferenceResolver from a visitor. Args: visitor: an instance of `DocGeneratorVisitor` **kwargs: all remaining args are passed to the constructor Returns: an instance of `ReferenceResolver` () """ is_fragment = {} for name, obj in visitor.index.items(): has_page = ( tf_inspect.isclass(obj) or tf_inspect.ismodule(obj) or is_free_function(obj, name, visitor.index)) is_fragment[name] = not has_page return cls( duplicate_of=visitor.duplicate_of, is_fragment=is_fragment, **kwargs)
def collect_docs_for_module(self, parser_config): """Collect information necessary specifically for a module's doc page. Mainly this is information about the members of the module. Args: parser_config: An instance of ParserConfig. """ relative_path = os.path.relpath( path='.', start=os.path.dirname(documentation_path(self.full_name)) or '.') member_names = parser_config.tree.get(self.full_name, []) for name in member_names: if name in ['__builtins__', '__doc__', '__file__', '__name__', '__path__', '__package__', '__cached__', '__loader__', '__spec__']: continue member_full_name = self.full_name + '.' + name if self.full_name else name member = parser_config.py_name_to_object(member_full_name) member_doc = _parse_md_docstring(member, relative_path, parser_config.reference_resolver) url = parser_config.reference_resolver.reference_to_url( member_full_name, relative_path) if tf_inspect.ismodule(member): self._add_module(name, member_full_name, member, member_doc, url) elif tf_inspect.isclass(member): self._add_class(name, member_full_name, member, member_doc, url) elif tf_inspect.isfunction(member): self._add_function(name, member_full_name, member, member_doc, url) else: self._add_other_member(name, member_full_name, member, member_doc)
def __call__(self, parent_path, parent, children): """Visitor interface, see `tensorflow/tools/common:traverse` for details. This method is called for each symbol found in a traversal using `tensorflow/tools/common:traverse`. It should not be called directly in user code. Args: parent_path: A tuple of strings. The fully qualified path to a symbol found during traversal. parent: The Python object referenced by `parent_name`. children: A list of `(name, py_object)` pairs enumerating, in alphabetical order, the children (as determined by `tf_inspect.getmembers`) of `parent`. `name` is the local name of `py_object` in `parent`. Returns: The list of children, with any __metaclass__ removed. Raises: RuntimeError: If this visitor is called with a `parent` that is not a class or module. """ parent_name = '.'.join(parent_path) self._index[parent_name] = parent self._tree[parent_name] = [] if parent_path not in self._api_tree: self._api_tree[parent_path] = parent if not (tf_inspect.ismodule(parent) or tf_inspect.isclass(parent)): raise RuntimeError('Unexpected type in visitor -- %s: %r' % (parent_name, parent)) for (name, child) in children: self._api_tree[parent_path + (name, )] = child full_name = '.'.join([parent_name, name]) if parent_name else name self._index[full_name] = child self._tree[parent_name].append(name) return children
def generate_global_index(library_name, index, reference_resolver): """Given a dict of full names to python objects, generate an index page. The index page generated contains a list of links for all symbols in `index` that have their own documentation page. Args: library_name: The name for the documented library to use in the title. index: A dict mapping full names to python objects. reference_resolver: An instance of ReferenceResolver. Returns: A string containing an index page as Markdown. """ symbol_links = [] for full_name, py_object in six.iteritems(index): if (tf_inspect.ismodule(py_object) or tf_inspect.isfunction(py_object) or tf_inspect.isclass(py_object)): # In Python 3, unbound methods are functions, so eliminate those. if tf_inspect.isfunction(py_object): if full_name.count('.') == 0: parent_name = '' else: parent_name = full_name[:full_name.rfind('.')] if parent_name in index and tf_inspect.isclass(index[parent_name]): # Skip methods (=functions with class parents). continue symbol_links.append(( full_name, reference_resolver.python_link(full_name, full_name, '.'))) lines = ['# All symbols in %s' % library_name, ''] for _, link in sorted(symbol_links, key=lambda x: x[0]): lines.append('* %s' % link) # TODO(markdaoust): use a _ModulePageInfo -> prety_docs.build_md_page() return '\n'.join(lines)
def write_docs(output_dir, parser_config, yaml_toc, root_title='TensorFlow', search_hints=True, site_path=''): """Write previously extracted docs to disk. Write a docs page for each symbol included in the indices of parser_config to a tree of docs at `output_dir`. Symbols with multiple aliases will have only one page written about them, which is referenced for all aliases. Args: output_dir: Directory to write documentation markdown files to. Will be created if it doesn't exist. parser_config: A `parser.ParserConfig` object, containing all the necessary indices. yaml_toc: Set to `True` to generate a "_toc.yaml" file. root_title: The title name for the root level index.md. search_hints: (bool) include meta-data search hints at the top of each output file. site_path: The output path relative to the site root. Used in the `_toc.yaml` and `_redirects.yaml` files. Raises: ValueError: if `output_dir` is not an absolute path """ # Make output_dir. if not os.path.isabs(output_dir): raise ValueError("'output_dir' must be an absolute path.\n" " output_dir='%s'" % output_dir) if not os.path.exists(output_dir): os.makedirs(output_dir) # These dictionaries are used for table-of-contents generation below # They will contain, after the for-loop below:: # - module name(string):classes and functions the module contains(list) module_children = {} # - symbol name(string):pathname (string) symbol_to_file = {} # Collect redirects for an api _redirects.yaml file. redirects = [] # Parse and write Markdown pages, resolving cross-links (`tf.symbol`). for full_name, py_object in six.iteritems(parser_config.index): parser_config.reference_resolver.current_doc_full_name = full_name if full_name in parser_config.duplicate_of: continue # Methods and some routines are documented only as part of their class. if not (tf_inspect.ismodule(py_object) or tf_inspect.isclass(py_object) or parser.is_free_function( py_object, full_name, parser_config.index)): continue sitepath = os.path.join('api_docs/python', parser.documentation_path(full_name)[:-3]) # For TOC, we need to store a mapping from full_name to the file # we're generating symbol_to_file[full_name] = sitepath # For a module, remember the module for the table-of-contents if tf_inspect.ismodule(py_object): if full_name in parser_config.tree: module_children.setdefault(full_name, []) # For something else that's documented, # figure out what module it lives in else: subname = str(full_name) while True: subname = subname[:subname.rindex('.')] if tf_inspect.ismodule(parser_config.index[subname]): module_name = parser_config.duplicate_of.get( subname, subname) module_children.setdefault(module_name, []).append(full_name) break # Generate docs for `py_object`, resolving references. page_info = parser.docs_for_object(full_name, py_object, parser_config) path = os.path.join(output_dir, parser.documentation_path(full_name)) directory = os.path.dirname(path) try: if not os.path.exists(directory): os.makedirs(directory) # This function returns raw bytes in PY2 or unicode in PY3. if search_hints: content = [page_info.get_metadata_html()] else: content = [''] content.append(pretty_docs.build_md_page(page_info)) text = '\n'.join(content) if six.PY3: text = text.encode('utf-8') with open(path, 'wb') as f: f.write(text) except OSError: raise OSError('Cannot write documentation for %s to %s' % (full_name, directory)) duplicates = parser_config.duplicates.get(full_name, []) if not duplicates: continue duplicates = [item for item in duplicates if item != full_name] for dup in duplicates: from_path = os.path.join(site_path, 'api_docs/python', dup.replace('.', '/')) to_path = os.path.join(site_path, 'api_docs/python', full_name.replace('.', '/')) redirects.append( (os.path.join('/', from_path), os.path.join('/', to_path))) if redirects: redirects = sorted(redirects) template = ('- from: {}\n' ' to: {}\n') redirects = [template.format(f, t) for f, t in redirects] api_redirects_path = os.path.join(output_dir, '_redirects.yaml') with open(api_redirects_path, 'w') as redirect_file: redirect_file.write('redirects:\n') redirect_file.write(''.join(redirects)) if yaml_toc: # Generate table of contents # Put modules in alphabetical order, case-insensitive modules = sorted(module_children.keys(), key=lambda a: a.upper()) leftnav_path = os.path.join(output_dir, '_toc.yaml') with open(leftnav_path, 'w') as f: # Generate header f.write( '# Automatically generated file; please do not edit\ntoc:\n') for module in modules: indent_num = module.count('.') # Don't list `tf.submodule` inside `tf` indent_num = max(indent_num, 1) indent = ' ' * indent_num if indent_num > 1: # tf.contrib.baysflow.entropy will be under # tf.contrib->baysflow->entropy title = module.split('.')[-1] else: title = module header = [ '- title: ' + title, ' section:', ' - title: Overview', ' path: ' + os.path.join('/', site_path, symbol_to_file[module]) ] header = ''.join([indent + line + '\n' for line in header]) f.write(header) symbols_in_module = module_children.get(module, []) # Sort case-insensitive, if equal sort case sensitive (upper first) symbols_in_module.sort(key=lambda a: (a.upper(), a)) for full_name in symbols_in_module: item = [ ' - title: ' + full_name[len(module) + 1:], ' path: ' + os.path.join('/', site_path, symbol_to_file[full_name]) ] item = ''.join([indent + line + '\n' for line in item]) f.write(item) # Write a global index containing all full names with links. with open(os.path.join(output_dir, 'index.md'), 'w') as f: f.write( parser.generate_global_index(root_title, parser_config.index, parser_config.reference_resolver))
def write_docs(output_dir, parser_config, yaml_toc, root_title='TensorFlow', search_hints=True, site_path='api_docs/python'): """Write previously extracted docs to disk. Write a docs page for each symbol included in the indices of parser_config to a tree of docs at `output_dir`. Symbols with multiple aliases will have only one page written about them, which is referenced for all aliases. Args: output_dir: Directory to write documentation markdown files to. Will be created if it doesn't exist. parser_config: A `parser.ParserConfig` object, containing all the necessary indices. yaml_toc: Set to `True` to generate a "_toc.yaml" file. root_title: The title name for the root level index.md. search_hints: (bool) include meta-data search hints at the top of each output file. site_path: The output path relative to the site root. Used in the `_toc.yaml` and `_redirects.yaml` files. Raises: ValueError: if `output_dir` is not an absolute path """ # Make output_dir. if not os.path.isabs(output_dir): raise ValueError("'output_dir' must be an absolute path.\n" " output_dir='%s'" % output_dir) if not os.path.exists(output_dir): os.makedirs(output_dir) # These dictionaries are used for table-of-contents generation below # They will contain, after the for-loop below:: # - module name(string):classes and functions the module contains(list) module_children = {} # Collect redirects for an api _redirects.yaml file. redirects = [] # Parse and write Markdown pages, resolving cross-links (`tf.symbol`). for full_name in sorted(parser_config.index.keys(), key=lambda k: k.lower()): py_object = parser_config.index[full_name] if full_name in parser_config.duplicate_of: continue # Methods and some routines are documented only as part of their class. if not (tf_inspect.ismodule(py_object) or tf_inspect.isclass(py_object) or parser.is_free_function( py_object, full_name, parser_config.index)): continue # Remove the extension from the path. docpath, _ = os.path.splitext(parser.documentation_path(full_name)) # For a module, remember the module for the table-of-contents if tf_inspect.ismodule(py_object): if full_name in parser_config.tree: mod_obj = Module(module=full_name, py_object=py_object, path=os.path.join('/', site_path, docpath)) module_children[full_name] = mod_obj # For something else that's documented, # figure out what module it lives in else: subname = str(full_name) while True: subname = subname[:subname.rindex('.')] if tf_inspect.ismodule(parser_config.index[subname]): module_name = parser_config.duplicate_of.get( subname, subname) child_mod = ModuleChild(name=full_name, py_object=py_object, parent=module_name, path=os.path.join( '/', site_path, docpath)) module_children[module_name].add_children(child_mod) break # Generate docs for `py_object`, resolving references. try: page_info = parser.docs_for_object(full_name, py_object, parser_config) except: raise ValueError( 'Failed to generate docs for symbol: `{}`'.format(full_name)) path = os.path.join(output_dir, parser.documentation_path(full_name)) directory = os.path.dirname(path) try: if not os.path.exists(directory): os.makedirs(directory) # This function returns raw bytes in PY2 or unicode in PY3. if search_hints: content = [page_info.get_metadata_html()] else: content = [''] content.append(pretty_docs.build_md_page(page_info)) text = '\n'.join(content) if six.PY3: text = text.encode('utf-8') with open(path, 'wb') as f: f.write(text) except OSError: raise OSError('Cannot write documentation for %s to %s' % (full_name, directory)) duplicates = parser_config.duplicates.get(full_name, []) if not duplicates: continue duplicates = [item for item in duplicates if item != full_name] for dup in duplicates: from_path = os.path.join(site_path, dup.replace('.', '/')) to_path = os.path.join(site_path, full_name.replace('.', '/')) redirects.append({ 'from': os.path.join('/', from_path), 'to': os.path.join('/', to_path) }) if redirects: redirects_dict = { 'redirects': sorted(redirects, key=lambda redirect: redirect['from']) } api_redirects_path = os.path.join(output_dir, '_redirects.yaml') with open(api_redirects_path, 'w') as redirect_file: yaml.dump(redirects_dict, redirect_file, default_flow_style=False) if yaml_toc: toc_gen = GenerateToc(module_children) toc_dict = toc_gen.generate() leftnav_toc = os.path.join(output_dir, '_toc.yaml') with open(leftnav_toc, 'w') as toc_file: yaml.dump(toc_dict, toc_file, default_flow_style=False) # Write a global index containing all full names with links. with open(os.path.join(output_dir, 'index.md'), 'w') as f: f.write( parser.generate_global_index(root_title, parser_config.index, parser_config.reference_resolver))
def docs_for_object(full_name, py_object, parser_config): """Return a PageInfo object describing a given object from the TF API. This function uses _parse_md_docstring to parse the docs pertaining to `object`. This function resolves '`tf.symbol`' references in the docstrings into links to the appropriate location. It also adds a list of alternative names for the symbol automatically. It assumes that the docs for each object live in a file given by `documentation_path`, and that relative links to files within the documentation are resolvable. Args: full_name: The fully qualified name of the symbol to be documented. py_object: The Python object to be documented. Its documentation is sourced from `py_object`'s docstring. parser_config: A ParserConfig object. Returns: Either a `_FunctionPageInfo`, `_ClassPageInfo`, or a `_ModulePageInfo` depending on the type of the python object being documented. Raises: RuntimeError: If an object is encountered for which we don't know how to make docs. """ # Which other aliases exist for the object referenced by full_name? master_name = parser_config.reference_resolver.py_master_name(full_name) duplicate_names = parser_config.duplicates.get(master_name, [full_name]) # TODO(wicke): Once other pieces are ready, enable this also for partials. if (tf_inspect.ismethod(py_object) or tf_inspect.isfunction(py_object) or # Some methods in classes from extensions come in as routines. tf_inspect.isroutine(py_object)): page_info = _FunctionPageInfo(master_name) page_info.set_signature(py_object, parser_config.reverse_index) elif tf_inspect.isclass(py_object): page_info = _ClassPageInfo(master_name) page_info.collect_docs_for_class(py_object, parser_config) elif tf_inspect.ismodule(py_object): page_info = _ModulePageInfo(master_name) page_info.collect_docs_for_module(parser_config) else: raise RuntimeError('Cannot make docs for object %s: %r' % (full_name, py_object)) relative_path = os.path.relpath( path='.', start=os.path.dirname(documentation_path(full_name)) or '.') page_info.set_doc(_parse_md_docstring( py_object, relative_path, parser_config.reference_resolver)) page_info.set_aliases(duplicate_names) page_info.set_defined_in(_get_defined_in(py_object, parser_config)) return page_info
def write_docs(output_dir, parser_config, yaml_toc, root_title='TensorFlow', search_hints=True, site_path=''): """Write previously extracted docs to disk. Write a docs page for each symbol included in the indices of parser_config to a tree of docs at `output_dir`. Symbols with multiple aliases will have only one page written about them, which is referenced for all aliases. Args: output_dir: Directory to write documentation markdown files to. Will be created if it doesn't exist. parser_config: A `parser.ParserConfig` object, containing all the necessary indices. yaml_toc: Set to `True` to generate a "_toc.yaml" file. root_title: The title name for the root level index.md. search_hints: (bool) include meta-data search hints at the top of each output file. site_path: The output path relative to the site root. Used in the `_toc.yaml` and `_redirects.yaml` files. Raises: ValueError: if `output_dir` is not an absolute path """ # Make output_dir. if not os.path.isabs(output_dir): raise ValueError("'output_dir' must be an absolute path.\n" " output_dir='%s'" % output_dir) if not os.path.exists(output_dir): os.makedirs(output_dir) # These dictionaries are used for table-of-contents generation below # They will contain, after the for-loop below:: # - module name(string):classes and functions the module contains(list) module_children = {} # - symbol name(string):pathname (string) symbol_to_file = {} # Collect redirects for an api _redirects.yaml file. redirects = [] # Parse and write Markdown pages, resolving cross-links (`tf.symbol`). for full_name, py_object in six.iteritems(parser_config.index): parser_config.reference_resolver.current_doc_full_name = full_name if full_name in parser_config.duplicate_of: continue # Methods and some routines are documented only as part of their class. if not (tf_inspect.ismodule(py_object) or tf_inspect.isclass(py_object) or parser.is_free_function(py_object, full_name, parser_config.index)): continue sitepath = os.path.join('api_docs/python', parser.documentation_path(full_name)[:-3]) # For TOC, we need to store a mapping from full_name to the file # we're generating symbol_to_file[full_name] = sitepath # For a module, remember the module for the table-of-contents if tf_inspect.ismodule(py_object): if full_name in parser_config.tree: module_children.setdefault(full_name, []) # For something else that's documented, # figure out what module it lives in else: subname = str(full_name) while True: subname = subname[:subname.rindex('.')] if tf_inspect.ismodule(parser_config.index[subname]): module_name = parser_config.duplicate_of.get(subname, subname) module_children.setdefault(module_name, []).append(full_name) break # Generate docs for `py_object`, resolving references. page_info = parser.docs_for_object(full_name, py_object, parser_config) path = os.path.join(output_dir, parser.documentation_path(full_name)) directory = os.path.dirname(path) try: if not os.path.exists(directory): os.makedirs(directory) # This function returns raw bytes in PY2 or unicode in PY3. if search_hints: content = [page_info.get_metadata_html()] else: content = [''] content.append(pretty_docs.build_md_page(page_info)) text = '\n'.join(content) if six.PY3: text = text.encode('utf-8') with open(path, 'wb') as f: f.write(text) except OSError: raise OSError( 'Cannot write documentation for %s to %s' % (full_name, directory)) duplicates = parser_config.duplicates.get(full_name, []) if not duplicates: continue duplicates = [item for item in duplicates if item != full_name] for dup in duplicates: from_path = os.path.join(site_path, 'api_docs/python', dup.replace('.', '/')) to_path = os.path.join(site_path, 'api_docs/python', full_name.replace('.', '/')) redirects.append(( os.path.join('/', from_path), os.path.join('/', to_path))) if redirects: redirects = sorted(redirects) template = ('- from: {}\n' ' to: {}\n') redirects = [template.format(f, t) for f, t in redirects] api_redirects_path = os.path.join(output_dir, '_redirects.yaml') with open(api_redirects_path, 'w') as redirect_file: redirect_file.write('redirects:\n') redirect_file.write(''.join(redirects)) if yaml_toc: # Generate table of contents # Put modules in alphabetical order, case-insensitive modules = sorted(module_children.keys(), key=lambda a: a.upper()) leftnav_path = os.path.join(output_dir, '_toc.yaml') with open(leftnav_path, 'w') as f: # Generate header f.write('# Automatically generated file; please do not edit\ntoc:\n') for module in modules: indent_num = module.count('.') # Don't list `tf.submodule` inside `tf` indent_num = max(indent_num, 1) indent = ' '*indent_num if indent_num > 1: # tf.contrib.baysflow.entropy will be under # tf.contrib->baysflow->entropy title = module.split('.')[-1] else: title = module header = [ '- title: ' + title, ' section:', ' - title: Overview', ' path: ' + os.path.join('/', site_path, symbol_to_file[module]) ] header = ''.join([indent+line+'\n' for line in header]) f.write(header) symbols_in_module = module_children.get(module, []) # Sort case-insensitive, if equal sort case sensitive (upper first) symbols_in_module.sort(key=lambda a: (a.upper(), a)) for full_name in symbols_in_module: item = [ ' - title: ' + full_name[len(module) + 1:], ' path: ' + os.path.join('/', site_path, symbol_to_file[full_name]) ] item = ''.join([indent+line+'\n' for line in item]) f.write(item) # Write a global index containing all full names with links. with open(os.path.join(output_dir, 'index.md'), 'w') as f: f.write( parser.generate_global_index(root_title, parser_config.index, parser_config.reference_resolver))