def gen_categories():
"""Generate categories listing."""
categories = glob('*-*') + ['virtual', ]
dep(['profiles/categories', ], categories)
with open('profiles/categories', 'w') as file:
for cat in sorted(categories):
if not os.path.isdir(cat):
print(warn('Category match on %s, may cause problems with '
'portage' % cat))
continue
file.write(cat + '\n')
print(success('categories list generated!'))
def task_doc_check():
"""Check tasks are documented."""
# This should be far easier to write, if only we could rely on the Sphinx
# cache or mock the Sphinx extensions simply and use the docutils parser
lines = open('doc/maintenance.rst').readlines()
commands = []
for n, line in enumerate(lines):
if match("^'+\n$", line):
commands.append(lines[n - 1][2:-3])
for name in sorted(cli.commands):
if name not in commands:
print(warn('%s task undocumented' % name))
def eclass_doc_check():
"""Check eclass documentation syntax."""
p = Popen(['portageq', 'repos_config', '/'], stdout=PIPE)
p.wait()
conf = ConfigParser()
conf.readfp(p.stdout)
portdir = conf.get(conf.defaults()['main-repo'], 'location')
awk_file = portdir + '/' + \
'app-portage/eclass-manpages/files/eclass-to-manpage.awk'
eclasses = glob('eclass/*.eclass')
for eclass in eclasses:
proc = Popen(['gawk', '-f', awk_file], stdin=PIPE, stdout=PIPE,
stderr=PIPE)
_, err = proc.communicate(open(eclass, 'rb').read())
if err:
print(warn('>>> %s' % eclass))
print(err)
def gen_manifests():
"""Generate Manifest files."""
packages = glob('*-*/*/*') + glob('virtual/*/*')
dep(glob('*-*/*/Manifest') + glob('virtual/*/Manifest'), packages)
base_dir = os.path.abspath(os.curdir)
if not SIGN_KEY:
print(warn('No GnuPG key set!'))
for package in sorted(packages):
try:
os.chdir(package)
if not any(map(lambda f: newer(f, 'Manifest'), glob('*'))):
break
check_call(['repoman', 'manifest'])
if SIGN_KEY:
check_call(['gpg', '--local-user', SIGN_KEY, '--clearsign',
'Manifest'])
os.rename('Manifest.asc', 'Manifest')
finally:
os.chdir(base_dir)
def check(ctx,
files,
check_proper_formatting=False,
error_on_known_failures=False):
"""
Check salt's docstrings
"""
# CD into Salt's repo root directory
ctx.cd(CODE_DIR)
# Unfortunately invoke does not support nargs.
# We migth have been passed --files="foo.py bar.py"
# Turn that into a list of paths
_files = []
for path in files:
if not path:
continue
_files.extend(path.split())
if not _files:
_files = SALT_CODE_DIR.rglob("*.py")
else:
_files = [pathlib.Path(fname) for fname in _files]
_files = [path.resolve() for path in _files]
errors = 0
exitcode = 0
warnings = 0
for path in _files:
contents = path.read_text()
try:
module = ast.parse(path.read_text(), filename=str(path))
module_docstring = ast.get_docstring(module, clean=False)
if module_docstring:
error = _check_valid_versions_on_docstrings(module_docstring)
if error:
errors += 1
exitcode = 1
utils.error(
"The module '{}' does not provide a proper `{}` version: {!r} is not valid.",
path.relative_to(CODE_DIR),
*error,
)
for funcdef in [
node for node in module.body
if isinstance(node, ast.FunctionDef)
]:
docstring = ast.get_docstring(funcdef, clean=False)
if docstring:
error = _check_valid_versions_on_docstrings(docstring)
if error:
errors += 1
exitcode = 1
utils.error(
"The module '{}' does not provide a proper `{}` version: {!r} is not valid.",
path.relative_to(CODE_DIR),
*error,
)
if not str(path).startswith(SALT_INTERNAL_LOADERS_PATHS):
# No further docstrings checks are needed
continue
funcname = funcdef.name
relpath = str(path.relative_to(CODE_DIR))
# We're dealing with a salt loader module
if funcname.startswith("_"):
# We're not interested in internal functions
continue
if not docstring:
if (funcname in MISSING_DOCSTRINGS.get(relpath, ())
and error_on_known_failures is False):
warnings += 1
utils.warn(
"The function '{}' on '{}' does not have a docstring",
funcname,
relpath,
)
continue
errors += 1
exitcode = 1
utils.error(
"The function '{}' on '{}' does not have a docstring",
funcname,
relpath,
)
continue
elif funcname in MISSING_DOCSTRINGS.get(relpath, ()):
# This was previously a know function with a missing docstring.
# Warn about it so that it get's removed from this list
warnings += 1
utils.warn(
"The function '{}' on '{}' was previously known to not have a docstring, "
"which is no longer the case. Please remove it from 'MISSING_DOCSTRINGS' ."
"in '{}'",
funcname,
relpath,
THIS_FILE,
)
try:
salt_modules_relpath = path.relative_to(SALT_MODULES_PATH)
if str(salt_modules_relpath.parent) != ".":
# We don't want to check nested packages
continue
# But this is a module under salt/modules, let's check
# the CLI examples
except ValueError:
# We're not checking CLI examples in any other salt loader modules
continue
if _check_cli_example_present(docstring) is False:
if (funcname in MISSING_EXAMPLES.get(relpath, ())
and error_on_known_failures is False):
warnings += 1
utils.warn(
"The function '{}' on '{}' does not have a 'CLI Example:' in it's docstring",
funcname,
relpath,
)
continue
errors += 1
exitcode = 1
utils.error(
"The function '{}' on '{}' does not have a 'CLI Example:' in it's docstring",
funcname,
relpath,
)
continue
elif funcname in MISSING_EXAMPLES.get(relpath, ()):
# This was previously a know function with a missing CLI example
# Warn about it so that it get's removed from this list
warnings += 1
utils.warn(
"The function '{}' on '{}' was previously known to not have a CLI Example, "
"which is no longer the case. Please remove it from 'MISSING_EXAMPLES'. "
"in '{}'",
funcname,
relpath,
THIS_FILE,
)
if check_proper_formatting is False:
continue
# By now we now this function has a docstring and it has a CLI Example section
# Let's now check if it's properly formatted
if _check_cli_example_proper_formatting(docstring) is False:
errors += 1
exitcode = 1
utils.error(
"The function {!r} on '{}' does not have a proper 'CLI Example:' section in "
"it's docstring. The proper format is:\n"
"CLI Example:\n"
"\n"
".. code-block:: bash\n"
"\n"
" salt '*' <insert example here>\n",
funcdef.name,
path.relative_to(CODE_DIR),
)
continue
finally:
if contents != path.read_text():
path.write_text(contents)
if warnings:
utils.warn("Found {} warnings", warnings)
if exitcode:
utils.error("Found {} errors", errors)
utils.exit_invoke(exitcode)
