def mkModuleRst(self, classname, fullclassname, buildtype='full'): """Create rst file for module.""" LOG.info('Creating rst file for %r, aka %r', classname, fullclassname) filename = classname + '.rst' lines = [] lines.append('%s' % classname) lines.append('=' * len(classname)) lines.append('.. automodule:: %s' % fullclassname) if buildtype == 'full': lines.append(' :members:') if not any(x in self.config.code_noInherited for x in [classname, fullclassname]): lines.append(' :inherited-members:') lines.append(' :undoc-members:') lines.append(' :show-inheritance:') if classname in self.config.code_privateMembers: lines.append(' :special-members:') lines.append(' :private-members:') else: lines.append(' :special-members: __init__') if classname.startswith('_'): lines.append(' :private-members:') if fullclassname in CUSTOMIZED_DOCSTRINGS: ds = CUSTOMIZED_DOCSTRINGS[fullclassname] if ds.replace: lines = ds.doc_string else: lines.append(ds.doc_string) writeLinesToFile(filename, lines)
def mkModuleRst(self, classname, fullclassname, buildtype="full"): """Create rst file for module.""" LOG.info("Creating rst file for %r, aka %r", classname, fullclassname) filename = classname + ".rst" lines = [] lines.append("%s" % classname) lines.append("=" * len(classname)) lines.append(".. automodule:: %s" % fullclassname) if buildtype == "full": lines.append(" :members:") if not any(x in self.config.code_noInherited for x in [classname, fullclassname]): lines.append(" :inherited-members:") lines.append(" :undoc-members:") lines.append(" :show-inheritance:") if classname in self.config.code_privateMembers: lines.append(" :special-members:") lines.append(" :private-members:") else: lines.append(" :special-members: __init__") if classname.startswith("_"): lines.append(" :private-members:") if fullclassname in CUSTOMIZED_DOCSTRINGS: ds = CUSTOMIZED_DOCSTRINGS[fullclassname] if ds.replace: lines = ds.doc_string else: lines.append(ds.doc_string) writeLinesToFile(filename, lines)
def createCodeDocIndex(self, subpackages, modules, buildtype="full"): """create the main index file""" LOG.info('Creating base index file') filename = 'index.rst' lines = [] lines.append('.. _code_documentation:') lines.append('') lines.append('Code Documentation (|release|)') lines.append('------------------------------') # for limited builds we only create the most basic code documentation so # we let users know there is more elsewhere if buildtype == 'limited': lines.append('') lines.append('.. warning::') lines.append( ' This a limited build of the code documentation, for the full code documentation ' 'please look at the website') lines.append('') else: if subpackages or modules: lines.append('.. toctree::') lines.append(' :maxdepth: 1') lines.append('') if subpackages: systemPackages = sorted( [pck for pck in subpackages if pck.endswith('System')]) otherPackages = sorted( [pck for pck in subpackages if not pck.endswith('System')]) lines.append('=======') lines.append('Systems') lines.append('=======') lines.append('') lines.append('.. toctree::') lines.append(' :maxdepth: 1') lines.append('') for package in systemPackages: lines.append(' %s/%s_Module.rst' % (package, package.split('/')[-1])) lines.append('') lines.append('=====') lines.append('Other') lines.append('=====') lines.append('') lines.append('.. toctree::') lines.append(' :maxdepth: 1') lines.append('') for package in otherPackages: lines.append(' %s/%s_Module.rst' % (package, package.split('/')[-1])) if modules: for module in sorted(modules): lines.append(' %s.rst' % (module.split('/')[-1], )) writeLinesToFile(filename, lines)
def createCodeDocIndex(self, subpackages, modules, buildtype="full"): """create the main index file""" LOG.info("Creating base index file") filename = "index.rst" lines = [] lines.append(".. _code_documentation:") lines.append("") lines.append("Code Documentation (|release|)") lines.append("------------------------------") # for limited builds we only create the most basic code documentation so # we let users know there is more elsewhere if buildtype == "limited": lines.append("") lines.append(".. warning::") lines.append( " This a limited build of the code documentation, for the full code documentation " "please look at the website") lines.append("") else: if subpackages or modules: lines.append(".. toctree::") lines.append(" :maxdepth: 1") lines.append("") if subpackages: systemPackages = sorted( [pck for pck in subpackages if pck.endswith("System")]) otherPackages = sorted( [pck for pck in subpackages if not pck.endswith("System")]) lines.append("=======") lines.append("Systems") lines.append("=======") lines.append("") lines.append(".. toctree::") lines.append(" :maxdepth: 1") lines.append("") for package in systemPackages: lines.append(" %s/%s_Module.rst" % (package, package.split("/")[-1])) lines.append("") lines.append("=====") lines.append("Other") lines.append("=====") lines.append("") lines.append(".. toctree::") lines.append(" :maxdepth: 1") lines.append("") for package in otherPackages: lines.append(" %s/%s_Module.rst" % (package, package.split("/")[-1])) if modules: for module in sorted(modules): lines.append(" %s.rst" % (module.split("/")[-1], )) writeLinesToFile(filename, lines)
def mkPackageRst(self, filename, modulename, fullmodulename, subpackages=None, modules=None): """Make a rst file for module containing other modules.""" if modulename == 'scripts': return else: modulefinal = modulename lines = [] lines.append('%s' % modulefinal) lines.append('=' * len(modulefinal)) lines.append('.. automodule:: %s ' % fullmodulename) lines.append(' :members:') lines.append('') if subpackages or modules: lines.append('.. toctree::') lines.append(' :maxdepth: 1') lines.append('') subpackages = [s for s in subpackages if not s.endswith(('scripts', ))] if subpackages: LOG.info('Module %r with subpackages: %r', fullmodulename, ', '.join(subpackages)) lines.append('SubPackages') lines.append('...........') lines.append('') lines.append('.. toctree::') lines.append(' :maxdepth: 1') lines.append('') for package in sorted(subpackages): lines.append(' %s/%s_Module.rst' % (package, package.split('/')[-1])) lines.append('') # remove CLI etc. because we drop them earlier modules = [ m for m in modules if not m.endswith('CLI') and '-' not in m ] if modules: lines.append('Modules') lines.append('.......') lines.append('') lines.append('.. toctree::') lines.append(' :maxdepth: 1') lines.append('') for module in sorted(modules): lines.append(' %s.rst' % (module.split('/')[-1], )) lines.append('') writeLinesToFile(filename, lines)
def mkPackageRst(self, filename, modulename, fullmodulename, subpackages=None, modules=None): """Make a rst file for module containing other modules.""" if modulename == "scripts": return else: modulefinal = modulename lines = [] lines.append("%s" % modulefinal) lines.append("=" * len(modulefinal)) lines.append(".. automodule:: %s " % fullmodulename) lines.append(" :members:") lines.append("") if subpackages or modules: lines.append(".. toctree::") lines.append(" :maxdepth: 1") lines.append("") subpackages = [s for s in subpackages if not s.endswith(("scripts", ))] if subpackages: LOG.info("Module %r with subpackages: %r", fullmodulename, ", ".join(subpackages)) lines.append("SubPackages") lines.append("...........") lines.append("") lines.append(".. toctree::") lines.append(" :maxdepth: 1") lines.append("") for package in sorted(subpackages): lines.append(" %s/%s_Module.rst" % (package, package.split("/")[-1])) lines.append("") # remove CLI etc. because we drop them earlier modules = [ m for m in modules if not m.endswith("CLI") and "-" not in m ] if modules: lines.append("Modules") lines.append(".......") lines.append("") lines.append(".. toctree::") lines.append(" :maxdepth: 1") lines.append("") for module in sorted(modules): lines.append(" %s.rst" % (module.split("/")[-1], )) lines.append("") writeLinesToFile(filename, lines)
def mkDummyRest(self, classname, fullclassname): """Create a dummy rst file for files that behave badly.""" filename = classname + '.rst' lines = [] lines.append('%s' % classname) lines.append('=' * len(classname)) lines.append('') lines.append('.. py:module:: %s' % fullclassname) lines.append('') lines.append('This is an empty file, because we cannot parse this file correctly or it causes problems.') lines.append('Please look at the source code directly') writeLinesToFile(filename, lines)
def createFilesAndIndex(self, sectionDict): """Create the index file and folder where the RST files will go e.g.: source/UserGuide/CommandReference/DataManagement """ sectionPath = os.path.join(self.config.docsPath, sectionDict[SECTION_PATH]) mkdir(sectionPath) systemName = sectionDict[TITLE] systemHeader = systemName + " Command Reference" systemHeader = "%s\n%s\n%s\n" % ("=" * len(systemHeader), systemHeader, "=" * len(systemHeader)) sectionIndexRST = systemHeader + textwrap.dedent( """ In this subsection the %s commands are collected .. this page automatically is created in %s .. toctree:: :maxdepth: 2 """ % (systemName, __name__)) listOfScripts = [] # these scripts use pre-existing rst files, cannot re-create them automatically listOfScripts.extend(sectionDict[MANUAL]) sectionPath = os.path.join(self.config.docsPath, sectionDict[SECTION_PATH]) futures = [] with ThreadPoolExecutor() as pool: for script in sectionDict[SCRIPTS]: futures.append( pool.submit(self.createScriptDocFiles, script, sectionPath, sectionDict)) for future in futures: scriptName, createdScriptDocs = future.result() if createdScriptDocs: listOfScripts.append(scriptName) for scriptName in sorted(listOfScripts): sectionIndexRST += " %s\n" % scriptName writeLinesToFile( os.path.join(self.config.docsPath, sectionDict[SECTION_PATH], 'index.rst'), sectionIndexRST)
def createFilesAndIndex(self, sectionDict): """Create the index file and folder where the RST files will go e.g.: source/UserGuide/CommandReference/DataManagement """ sectionPath = os.path.join(self.config.docsPath, sectionDict[SECTION_PATH]) mkdir(sectionPath) systemName = sectionDict[TITLE] systemHeader = systemName + " Command Reference" systemHeader = "%s\n%s\n%s\n" % ("=" * len(systemHeader), systemHeader, "=" * len(systemHeader)) sectionIndexRST = systemHeader + textwrap.dedent( """ In this subsection the %s commands are collected .. this page automatically is created in %s .. toctree:: :maxdepth: 2 """ % (systemName, __name__)) listOfScripts = [] # these scripts use pre-existing rst files, cannot re-create them automatically listOfScripts.extend(sectionDict[MANUAL]) sectionPath = os.path.join(self.config.docsPath, sectionDict[SECTION_PATH]) for script in sectionDict[SCRIPTS]: scriptName = os.path.basename(script) if scriptName.endswith('.py'): scriptName = scriptName[:-3] prefix = sectionDict[PREFIX].lower() prefix = prefix + '_' if prefix else '' if self.createScriptDocFiles(script, sectionPath, scriptName, referencePrefix=prefix): listOfScripts.append(scriptName) for scriptName in sorted(listOfScripts): sectionIndexRST += " %s\n" % scriptName writeLinesToFile( os.path.join(self.config.docsPath, sectionDict[SECTION_PATH], 'index.rst'), sectionIndexRST)
def createAllScriptsDocsAndWriteToRST(self): """Get all scripts and write it to RST file.""" # Use `:orphan:` in case you do not need a reference to this document in doctree sectionIndexRST = textwrap.dedent(f""" :orphan: .. this page automatically is created in {__name__} .. _cmd: Command Reference In this subsection all commands are collected: """) futures = [] # Call all scripts help with ThreadPoolExecutor() as pool: for script in self.config.allScripts: futures.append(pool.submit(self.createScriptDoc, script)) systems = [] # Collect all scripts help messages for future in futures: script = future.result() if script: self.scriptDocs[script.name] = script script.system not in systems and systems.append(script.system) # Write all commands in one RST for each system for system in sorted(systems): # Write system head sectionIndexRST += textwrap.dedent(f""" .. _{system}_cmd: {"=" * len(system)} {system} {"=" * len(system)} """) # Write each system command description for script in sorted(self.scriptDocs): if self.scriptDocs[script].system == system: sectionIndexRST += self.scriptDocs[script].description writeLinesToFile( os.path.join(self.config.docsPath, self.config.com_rst_path), sectionIndexRST)
def mkDummyRest(self, classname, fullclassname): """Create a dummy rst file for files that behave badly.""" filename = classname + ".rst" lines = [] lines.append("%s" % classname) lines.append("=" * len(classname)) lines.append("") lines.append(".. py:module:: %s" % fullclassname) lines.append("") lines.append( "This is an empty file, because we cannot parse this file correctly or it causes problems." ) lines.append("Please look at the source code directly") writeLinesToFile(filename, lines)
def createSectionAndIndex(self, sectionDict: dict): """Create the index file and folder where the RST files will go. :param sectionDict: section description """ reference = f".. _{sectionDict[PREFIX]}_cmd:" if sectionDict[ PREFIX] else "" title = f"{sectionDict[TITLE]} Command Reference" # Add description sectionIndexRST = textwrap.dedent(f""" {reference} {"=" * len(title)} {title} {"=" * len(title)} .. this page automatically is created in {__name__} In this subsection the {title} commands are collected """) # Write commands that were not included in the subgroups for name in sectionDict[SCRIPTS]: if name in self.scriptDocs: sectionIndexRST += f"- :ref:`{name}<{name}>`\n" # Write commands included in the subgroups for group in sectionDict["subgroups"]: groupDict = sectionDict[group] # Add subgroup reference ref = f".. _{groupDict[PREFIX]}_cmd:" if groupDict[PREFIX] else "" # Add subgroup header sectionIndexRST += textwrap.dedent(f""" {ref} {"-" * len(groupDict[TITLE])} {groupDict[TITLE]} {"-" * len(groupDict[TITLE])} """) for name in groupDict[SCRIPTS]: if name in self.scriptDocs: sectionIndexRST += f" - :ref:`{name}<{name}>`\n" writeLinesToFile( os.path.join(self.config.docsPath, sectionDict[RST_PATH]), sectionIndexRST)
def createScriptDocFiles(self, script, sectionPath, scriptName, referencePrefix=''): """Create the RST files for all the scripts. Folders and indices already exist, just call the scripts and get the help messages. Format the help message. """ if scriptName in self.config.com_ignore_commands: return False LOG.info('Creating Doc for %r in %r', scriptName, sectionPath) helpMessage = runCommand('python %s -h' % script) if not helpMessage: LOG.warning('NO DOC for %s', scriptName) return False rstLines = [] rstLines.append(' .. _%s%s:' % (referencePrefix, scriptName)) rstLines.append('') rstLines.append('=' * len(scriptName)) rstLines.append('%s' % scriptName) rstLines.append('=' * len(scriptName)) rstLines.append('') lineIndented = False genOptions = False for line in helpMessage.splitlines(): line = line.rstrip() if not line: pass # strip general options from documentation elif line.lower().strip() == 'general options:': LOG.debug("Found general options in line %r", line) if self.debug else None genOptions = True continue elif genOptions and line.startswith(' '): LOG.debug("Skipping General options line %r", line) if self.debug else None continue elif genOptions and not line.startswith(' '): LOG.debug("General options done") if self.debug else None genOptions = False newLine = '\n' + line + ':\n' if line.endswith(':') else line # ensure dedented lines are separated by newline from previous block if lineIndented and not newLine.startswith(' '): newLine = '\n' + newLine rstLines.append(newLine) lineIndented = newLine.startswith(' ') scriptRSTPath = os.path.join(sectionPath, scriptName + '.rst') fileContent = '\n'.join(rstLines).strip() + '\n' for index, marker in enumerate(['example', '.. note::']): if scriptName in self.config.com_module_docstring: if index == 0: content = self.getContentFromModuleDocstring(script) fileContent += '\n' + content.strip() + '\n' else: content = self.getContentFromScriptDoc(scriptRSTPath, marker) if not content: break # nothing in content, files probably does not exist if content and marker not in fileContent.lower(): fileContent += '\n' + content.strip() + '\n' LOG.debug('\n' + '*' * 88 + '\n' + fileContent + '\n' + '*' * 88) if self.debug else None while '\n\n\n' in fileContent: fileContent = fileContent.replace('\n\n\n', '\n\n') # remove the standalone '-' when no short option exists fileContent = fileContent.replace('- --', '--') writeLinesToFile(scriptRSTPath, fileContent) return True
def createScriptDocFiles(self, script, sectionPath, sectionDict): """Create the RST files for all the scripts. Folders and indices already exist, just call the scripts and get the help messages. Format the help message. """ scriptName = os.path.basename(script) if scriptName.endswith(".py"): scriptName = scriptName[:-3] scriptName = scriptName.replace("_", "-") referencePrefix = sectionDict[PREFIX].lower() referencePrefix = referencePrefix + "_" if referencePrefix else "" if scriptName in self.config.com_ignore_commands: return scriptName, False LOG.info("Creating Doc for %r in %r", scriptName, sectionPath) helpMessage = runCommand("python %s -h" % script) if not helpMessage: LOG.warning("NO DOC for %s", scriptName) return scriptName, False rstLines = [] rstLines.append(" .. _%s%s:" % (referencePrefix, scriptName)) rstLines.append("") rstLines.append("=" * len(scriptName)) rstLines.append("%s" % scriptName) rstLines.append("=" * len(scriptName)) rstLines.append("") lineIndented = False genOptions = False for line in helpMessage.splitlines(): line = line.rstrip() if not line: pass # strip general options from documentation elif line.lower().strip() == "general options:": LOG.debug("Found general options in line %r", line) if self.debug else None genOptions = True continue elif genOptions and line.startswith(" "): LOG.debug("Skipping General options line %r", line) if self.debug else None continue elif genOptions and not line.startswith(" "): LOG.debug("General options done") if self.debug else None genOptions = False newLine = "\n" + line + ":\n" if line.endswith(":") else line # ensure dedented lines are separated by newline from previous block if lineIndented and not newLine.startswith(" "): newLine = "\n" + newLine rstLines.append(newLine) lineIndented = newLine.startswith(" ") scriptRSTPath = os.path.join(sectionPath, scriptName + ".rst") fileContent = "\n".join(rstLines).strip() + "\n" for index, marker in enumerate(["example", ".. note::"]): if scriptName in self.config.com_module_docstring: if index == 0: content = self.getContentFromModuleDocstring(script) fileContent += "\n" + content.strip() + "\n" else: content = self.getContentFromScriptDoc(scriptRSTPath, marker) if not content: break # nothing in content, files probably does not exist if content and marker not in fileContent.lower(): fileContent += "\n" + content.strip() + "\n" LOG.debug("\n" + "*" * 88 + "\n" + fileContent + "\n" + "*" * 88) if self.debug else None while "\n\n\n" in fileContent: fileContent = fileContent.replace("\n\n\n", "\n\n") # remove the standalone '-' when no short option exists fileContent = fileContent.replace("- --", "--") writeLinesToFile(scriptRSTPath, fileContent) return scriptName, True