def mkModuleRst(self, classname, fullclassname, buildtype='full'):
"""Create rst file for module."""
LOG.info('Creating rst file for %r, aka %r', classname, fullclassname)
filename = classname + '.rst'
lines = []
lines.append('%s' % classname)
lines.append('=' * len(classname))
lines.append('.. automodule:: %s' % fullclassname)
if buildtype == 'full':
lines.append(' :members:')
if not any(x in self.config.code_noInherited
for x in [classname, fullclassname]):
lines.append(' :inherited-members:')
lines.append(' :undoc-members:')
lines.append(' :show-inheritance:')
if classname in self.config.code_privateMembers:
lines.append(' :special-members:')
lines.append(' :private-members:')
else:
lines.append(' :special-members: __init__')
if classname.startswith('_'):
lines.append(' :private-members:')
if fullclassname in CUSTOMIZED_DOCSTRINGS:
ds = CUSTOMIZED_DOCSTRINGS[fullclassname]
if ds.replace:
lines = ds.doc_string
else:
lines.append(ds.doc_string)
writeLinesToFile(filename, lines)
def mkModuleRst(self, classname, fullclassname, buildtype="full"):
"""Create rst file for module."""
LOG.info("Creating rst file for %r, aka %r", classname, fullclassname)
filename = classname + ".rst"
lines = []
lines.append("%s" % classname)
lines.append("=" * len(classname))
lines.append(".. automodule:: %s" % fullclassname)
if buildtype == "full":
lines.append(" :members:")
if not any(x in self.config.code_noInherited
for x in [classname, fullclassname]):
lines.append(" :inherited-members:")
lines.append(" :undoc-members:")
lines.append(" :show-inheritance:")
if classname in self.config.code_privateMembers:
lines.append(" :special-members:")
lines.append(" :private-members:")
else:
lines.append(" :special-members: __init__")
if classname.startswith("_"):
lines.append(" :private-members:")
if fullclassname in CUSTOMIZED_DOCSTRINGS:
ds = CUSTOMIZED_DOCSTRINGS[fullclassname]
if ds.replace:
lines = ds.doc_string
else:
lines.append(ds.doc_string)
writeLinesToFile(filename, lines)
def createCodeDocIndex(self, subpackages, modules, buildtype="full"):
"""create the main index file"""
LOG.info('Creating base index file')
filename = 'index.rst'
lines = []
lines.append('.. _code_documentation:')
lines.append('')
lines.append('Code Documentation (|release|)')
lines.append('------------------------------')
# for limited builds we only create the most basic code documentation so
# we let users know there is more elsewhere
if buildtype == 'limited':
lines.append('')
lines.append('.. warning::')
lines.append(
' This a limited build of the code documentation, for the full code documentation '
'please look at the website')
lines.append('')
else:
if subpackages or modules:
lines.append('.. toctree::')
lines.append(' :maxdepth: 1')
lines.append('')
if subpackages:
systemPackages = sorted(
[pck for pck in subpackages if pck.endswith('System')])
otherPackages = sorted(
[pck for pck in subpackages if not pck.endswith('System')])
lines.append('=======')
lines.append('Systems')
lines.append('=======')
lines.append('')
lines.append('.. toctree::')
lines.append(' :maxdepth: 1')
lines.append('')
for package in systemPackages:
lines.append(' %s/%s_Module.rst' %
(package, package.split('/')[-1]))
lines.append('')
lines.append('=====')
lines.append('Other')
lines.append('=====')
lines.append('')
lines.append('.. toctree::')
lines.append(' :maxdepth: 1')
lines.append('')
for package in otherPackages:
lines.append(' %s/%s_Module.rst' %
(package, package.split('/')[-1]))
if modules:
for module in sorted(modules):
lines.append(' %s.rst' % (module.split('/')[-1], ))
writeLinesToFile(filename, lines)
def createCodeDocIndex(self, subpackages, modules, buildtype="full"):
"""create the main index file"""
LOG.info("Creating base index file")
filename = "index.rst"
lines = []
lines.append(".. _code_documentation:")
lines.append("")
lines.append("Code Documentation (|release|)")
lines.append("------------------------------")
# for limited builds we only create the most basic code documentation so
# we let users know there is more elsewhere
if buildtype == "limited":
lines.append("")
lines.append(".. warning::")
lines.append(
" This a limited build of the code documentation, for the full code documentation "
"please look at the website")
lines.append("")
else:
if subpackages or modules:
lines.append(".. toctree::")
lines.append(" :maxdepth: 1")
lines.append("")
if subpackages:
systemPackages = sorted(
[pck for pck in subpackages if pck.endswith("System")])
otherPackages = sorted(
[pck for pck in subpackages if not pck.endswith("System")])
lines.append("=======")
lines.append("Systems")
lines.append("=======")
lines.append("")
lines.append(".. toctree::")
lines.append(" :maxdepth: 1")
lines.append("")
for package in systemPackages:
lines.append(" %s/%s_Module.rst" %
(package, package.split("/")[-1]))
lines.append("")
lines.append("=====")
lines.append("Other")
lines.append("=====")
lines.append("")
lines.append(".. toctree::")
lines.append(" :maxdepth: 1")
lines.append("")
for package in otherPackages:
lines.append(" %s/%s_Module.rst" %
(package, package.split("/")[-1]))
if modules:
for module in sorted(modules):
lines.append(" %s.rst" % (module.split("/")[-1], ))
writeLinesToFile(filename, lines)
def mkPackageRst(self,
filename,
modulename,
fullmodulename,
subpackages=None,
modules=None):
"""Make a rst file for module containing other modules."""
if modulename == 'scripts':
return
else:
modulefinal = modulename
lines = []
lines.append('%s' % modulefinal)
lines.append('=' * len(modulefinal))
lines.append('.. automodule:: %s ' % fullmodulename)
lines.append(' :members:')
lines.append('')
if subpackages or modules:
lines.append('.. toctree::')
lines.append(' :maxdepth: 1')
lines.append('')
subpackages = [s for s in subpackages if not s.endswith(('scripts', ))]
if subpackages:
LOG.info('Module %r with subpackages: %r', fullmodulename,
', '.join(subpackages))
lines.append('SubPackages')
lines.append('...........')
lines.append('')
lines.append('.. toctree::')
lines.append(' :maxdepth: 1')
lines.append('')
for package in sorted(subpackages):
lines.append(' %s/%s_Module.rst' %
(package, package.split('/')[-1]))
lines.append('')
# remove CLI etc. because we drop them earlier
modules = [
m for m in modules if not m.endswith('CLI') and '-' not in m
]
if modules:
lines.append('Modules')
lines.append('.......')
lines.append('')
lines.append('.. toctree::')
lines.append(' :maxdepth: 1')
lines.append('')
for module in sorted(modules):
lines.append(' %s.rst' % (module.split('/')[-1], ))
lines.append('')
writeLinesToFile(filename, lines)
def mkPackageRst(self,
filename,
modulename,
fullmodulename,
subpackages=None,
modules=None):
"""Make a rst file for module containing other modules."""
if modulename == "scripts":
return
else:
modulefinal = modulename
lines = []
lines.append("%s" % modulefinal)
lines.append("=" * len(modulefinal))
lines.append(".. automodule:: %s " % fullmodulename)
lines.append(" :members:")
lines.append("")
if subpackages or modules:
lines.append(".. toctree::")
lines.append(" :maxdepth: 1")
lines.append("")
subpackages = [s for s in subpackages if not s.endswith(("scripts", ))]
if subpackages:
LOG.info("Module %r with subpackages: %r", fullmodulename,
", ".join(subpackages))
lines.append("SubPackages")
lines.append("...........")
lines.append("")
lines.append(".. toctree::")
lines.append(" :maxdepth: 1")
lines.append("")
for package in sorted(subpackages):
lines.append(" %s/%s_Module.rst" %
(package, package.split("/")[-1]))
lines.append("")
# remove CLI etc. because we drop them earlier
modules = [
m for m in modules if not m.endswith("CLI") and "-" not in m
]
if modules:
lines.append("Modules")
lines.append(".......")
lines.append("")
lines.append(".. toctree::")
lines.append(" :maxdepth: 1")
lines.append("")
for module in sorted(modules):
lines.append(" %s.rst" % (module.split("/")[-1], ))
lines.append("")
writeLinesToFile(filename, lines)
def mkDummyRest(self, classname, fullclassname):
"""Create a dummy rst file for files that behave badly."""
filename = classname + '.rst'
lines = []
lines.append('%s' % classname)
lines.append('=' * len(classname))
lines.append('')
lines.append('.. py:module:: %s' % fullclassname)
lines.append('')
lines.append('This is an empty file, because we cannot parse this file correctly or it causes problems.')
lines.append('Please look at the source code directly')
writeLinesToFile(filename, lines)
def createFilesAndIndex(self, sectionDict):
"""Create the index file and folder where the RST files will go
e.g.:
source/UserGuide/CommandReference/DataManagement
"""
sectionPath = os.path.join(self.config.docsPath,
sectionDict[SECTION_PATH])
mkdir(sectionPath)
systemName = sectionDict[TITLE]
systemHeader = systemName + " Command Reference"
systemHeader = "%s\n%s\n%s\n" % ("=" * len(systemHeader), systemHeader,
"=" * len(systemHeader))
sectionIndexRST = systemHeader + textwrap.dedent(
"""
In this subsection the %s commands are collected
.. this page automatically is created in %s
.. toctree::
:maxdepth: 2
""" %
(systemName, __name__))
listOfScripts = []
# these scripts use pre-existing rst files, cannot re-create them automatically
listOfScripts.extend(sectionDict[MANUAL])
sectionPath = os.path.join(self.config.docsPath,
sectionDict[SECTION_PATH])
futures = []
with ThreadPoolExecutor() as pool:
for script in sectionDict[SCRIPTS]:
futures.append(
pool.submit(self.createScriptDocFiles, script, sectionPath,
sectionDict))
for future in futures:
scriptName, createdScriptDocs = future.result()
if createdScriptDocs:
listOfScripts.append(scriptName)
for scriptName in sorted(listOfScripts):
sectionIndexRST += " %s\n" % scriptName
writeLinesToFile(
os.path.join(self.config.docsPath, sectionDict[SECTION_PATH],
'index.rst'), sectionIndexRST)
def createFilesAndIndex(self, sectionDict):
"""Create the index file and folder where the RST files will go
e.g.:
source/UserGuide/CommandReference/DataManagement
"""
sectionPath = os.path.join(self.config.docsPath,
sectionDict[SECTION_PATH])
mkdir(sectionPath)
systemName = sectionDict[TITLE]
systemHeader = systemName + " Command Reference"
systemHeader = "%s\n%s\n%s\n" % ("=" * len(systemHeader), systemHeader,
"=" * len(systemHeader))
sectionIndexRST = systemHeader + textwrap.dedent(
"""
In this subsection the %s commands are collected
.. this page automatically is created in %s
.. toctree::
:maxdepth: 2
""" %
(systemName, __name__))
listOfScripts = []
# these scripts use pre-existing rst files, cannot re-create them automatically
listOfScripts.extend(sectionDict[MANUAL])
sectionPath = os.path.join(self.config.docsPath,
sectionDict[SECTION_PATH])
for script in sectionDict[SCRIPTS]:
scriptName = os.path.basename(script)
if scriptName.endswith('.py'):
scriptName = scriptName[:-3]
prefix = sectionDict[PREFIX].lower()
prefix = prefix + '_' if prefix else ''
if self.createScriptDocFiles(script,
sectionPath,
scriptName,
referencePrefix=prefix):
listOfScripts.append(scriptName)
for scriptName in sorted(listOfScripts):
sectionIndexRST += " %s\n" % scriptName
writeLinesToFile(
os.path.join(self.config.docsPath, sectionDict[SECTION_PATH],
'index.rst'), sectionIndexRST)
def createAllScriptsDocsAndWriteToRST(self):
"""Get all scripts and write it to RST file."""
# Use `:orphan:` in case you do not need a reference to this document in doctree
sectionIndexRST = textwrap.dedent(f"""
:orphan:
.. this page automatically is created in {__name__}
.. _cmd:
Command Reference
In this subsection all commands are collected:
""")
futures = []
# Call all scripts help
with ThreadPoolExecutor() as pool:
for script in self.config.allScripts:
futures.append(pool.submit(self.createScriptDoc, script))
systems = []
# Collect all scripts help messages
for future in futures:
script = future.result()
if script:
self.scriptDocs[script.name] = script
script.system not in systems and systems.append(script.system)
# Write all commands in one RST for each system
for system in sorted(systems):
# Write system head
sectionIndexRST += textwrap.dedent(f"""
.. _{system}_cmd:
{"=" * len(system)}
{system}
{"=" * len(system)}
""")
# Write each system command description
for script in sorted(self.scriptDocs):
if self.scriptDocs[script].system == system:
sectionIndexRST += self.scriptDocs[script].description
writeLinesToFile(
os.path.join(self.config.docsPath, self.config.com_rst_path),
sectionIndexRST)
def mkDummyRest(self, classname, fullclassname):
"""Create a dummy rst file for files that behave badly."""
filename = classname + ".rst"
lines = []
lines.append("%s" % classname)
lines.append("=" * len(classname))
lines.append("")
lines.append(".. py:module:: %s" % fullclassname)
lines.append("")
lines.append(
"This is an empty file, because we cannot parse this file correctly or it causes problems."
)
lines.append("Please look at the source code directly")
writeLinesToFile(filename, lines)
def createSectionAndIndex(self, sectionDict: dict):
"""Create the index file and folder where the RST files will go.
:param sectionDict: section description
"""
reference = f".. _{sectionDict[PREFIX]}_cmd:" if sectionDict[
PREFIX] else ""
title = f"{sectionDict[TITLE]} Command Reference"
# Add description
sectionIndexRST = textwrap.dedent(f"""
{reference}
{"=" * len(title)}
{title}
{"=" * len(title)}
.. this page automatically is created in {__name__}
In this subsection the {title} commands are collected
""")
# Write commands that were not included in the subgroups
for name in sectionDict[SCRIPTS]:
if name in self.scriptDocs:
sectionIndexRST += f"- :ref:`{name}<{name}>`\n"
# Write commands included in the subgroups
for group in sectionDict["subgroups"]:
groupDict = sectionDict[group]
# Add subgroup reference
ref = f".. _{groupDict[PREFIX]}_cmd:" if groupDict[PREFIX] else ""
# Add subgroup header
sectionIndexRST += textwrap.dedent(f"""
{ref}
{"-" * len(groupDict[TITLE])}
{groupDict[TITLE]}
{"-" * len(groupDict[TITLE])}
""")
for name in groupDict[SCRIPTS]:
if name in self.scriptDocs:
sectionIndexRST += f" - :ref:`{name}<{name}>`\n"
writeLinesToFile(
os.path.join(self.config.docsPath, sectionDict[RST_PATH]),
sectionIndexRST)
def createScriptDocFiles(self, script, sectionPath, scriptName, referencePrefix=''):
"""Create the RST files for all the scripts.
Folders and indices already exist, just call the scripts and get the help messages. Format the help message.
"""
if scriptName in self.config.com_ignore_commands:
return False
LOG.info('Creating Doc for %r in %r', scriptName, sectionPath)
helpMessage = runCommand('python %s -h' % script)
if not helpMessage:
LOG.warning('NO DOC for %s', scriptName)
return False
rstLines = []
rstLines.append(' .. _%s%s:' % (referencePrefix, scriptName))
rstLines.append('')
rstLines.append('=' * len(scriptName))
rstLines.append('%s' % scriptName)
rstLines.append('=' * len(scriptName))
rstLines.append('')
lineIndented = False
genOptions = False
for line in helpMessage.splitlines():
line = line.rstrip()
if not line:
pass
# strip general options from documentation
elif line.lower().strip() == 'general options:':
LOG.debug("Found general options in line %r", line) if self.debug else None
genOptions = True
continue
elif genOptions and line.startswith(' '):
LOG.debug("Skipping General options line %r", line) if self.debug else None
continue
elif genOptions and not line.startswith(' '):
LOG.debug("General options done") if self.debug else None
genOptions = False
newLine = '\n' + line + ':\n' if line.endswith(':') else line
# ensure dedented lines are separated by newline from previous block
if lineIndented and not newLine.startswith(' '):
newLine = '\n' + newLine
rstLines.append(newLine)
lineIndented = newLine.startswith(' ')
scriptRSTPath = os.path.join(sectionPath, scriptName + '.rst')
fileContent = '\n'.join(rstLines).strip() + '\n'
for index, marker in enumerate(['example', '.. note::']):
if scriptName in self.config.com_module_docstring:
if index == 0:
content = self.getContentFromModuleDocstring(script)
fileContent += '\n' + content.strip() + '\n'
else:
content = self.getContentFromScriptDoc(scriptRSTPath, marker)
if not content:
break # nothing in content, files probably does not exist
if content and marker not in fileContent.lower():
fileContent += '\n' + content.strip() + '\n'
LOG.debug('\n' + '*' * 88 + '\n' + fileContent + '\n' + '*' * 88) if self.debug else None
while '\n\n\n' in fileContent:
fileContent = fileContent.replace('\n\n\n', '\n\n')
# remove the standalone '-' when no short option exists
fileContent = fileContent.replace('- --', '--')
writeLinesToFile(scriptRSTPath, fileContent)
return True
def createScriptDocFiles(self, script, sectionPath, sectionDict):
"""Create the RST files for all the scripts.
Folders and indices already exist, just call the scripts and get the help messages. Format the help message.
"""
scriptName = os.path.basename(script)
if scriptName.endswith(".py"):
scriptName = scriptName[:-3]
scriptName = scriptName.replace("_", "-")
referencePrefix = sectionDict[PREFIX].lower()
referencePrefix = referencePrefix + "_" if referencePrefix else ""
if scriptName in self.config.com_ignore_commands:
return scriptName, False
LOG.info("Creating Doc for %r in %r", scriptName, sectionPath)
helpMessage = runCommand("python %s -h" % script)
if not helpMessage:
LOG.warning("NO DOC for %s", scriptName)
return scriptName, False
rstLines = []
rstLines.append(" .. _%s%s:" % (referencePrefix, scriptName))
rstLines.append("")
rstLines.append("=" * len(scriptName))
rstLines.append("%s" % scriptName)
rstLines.append("=" * len(scriptName))
rstLines.append("")
lineIndented = False
genOptions = False
for line in helpMessage.splitlines():
line = line.rstrip()
if not line:
pass
# strip general options from documentation
elif line.lower().strip() == "general options:":
LOG.debug("Found general options in line %r", line) if self.debug else None
genOptions = True
continue
elif genOptions and line.startswith(" "):
LOG.debug("Skipping General options line %r", line) if self.debug else None
continue
elif genOptions and not line.startswith(" "):
LOG.debug("General options done") if self.debug else None
genOptions = False
newLine = "\n" + line + ":\n" if line.endswith(":") else line
# ensure dedented lines are separated by newline from previous block
if lineIndented and not newLine.startswith(" "):
newLine = "\n" + newLine
rstLines.append(newLine)
lineIndented = newLine.startswith(" ")
scriptRSTPath = os.path.join(sectionPath, scriptName + ".rst")
fileContent = "\n".join(rstLines).strip() + "\n"
for index, marker in enumerate(["example", ".. note::"]):
if scriptName in self.config.com_module_docstring:
if index == 0:
content = self.getContentFromModuleDocstring(script)
fileContent += "\n" + content.strip() + "\n"
else:
content = self.getContentFromScriptDoc(scriptRSTPath, marker)
if not content:
break # nothing in content, files probably does not exist
if content and marker not in fileContent.lower():
fileContent += "\n" + content.strip() + "\n"
LOG.debug("\n" + "*" * 88 + "\n" + fileContent + "\n" + "*" * 88) if self.debug else None
while "\n\n\n" in fileContent:
fileContent = fileContent.replace("\n\n\n", "\n\n")
# remove the standalone '-' when no short option exists
fileContent = fileContent.replace("- --", "--")
writeLinesToFile(scriptRSTPath, fileContent)
return scriptName, True
