def check_virtual(ctx, files):
"""
Check if .rst files for each module contains the text ".. _virtual"
indicating it is a virtual doc page, and, in case a module exists by
the same name, it's going to be shaddowed and not accessible
"""
exitcode = 0
files = build_docs_paths(files)
for path in files:
if path.name == "index.rst":
continue
contents = path.read_text()
if ".. _virtual-" in contents:
try:
python_module = doc_path_to_python_module[path]
utils.error(
"The doc file at {} indicates that it's virtual, yet, there's a python module "
"at {} that will shaddow it.",
path,
python_module,
)
exitcode += 1
except KeyError:
# This is what we're expecting
continue
utils.exit_invoke(exitcode)
def check_inline_markup(ctx, files):
"""
Check docstring for :doc: usage
We should not be using the ``:doc:`` inline markup option when
cross-referencing locations. Use ``:ref:`` or ``:mod:`` instead.
This task checks for reference to ``:doc:`` usage.
See Issue #12788 for more information.
https://github.com/saltstack/salt/issues/12788
"""
# CD into Salt's repo root directory
ctx.cd(CODE_DIR)
files = build_python_module_paths(files)
exitcode = 0
for path in files:
module = ast.parse(path.read_text(), filename=str(path))
funcdefs = [node for node in module.body if isinstance(node, ast.FunctionDef)]
for funcdef in funcdefs:
docstring = ast.get_docstring(funcdef, clean=True)
if not docstring:
continue
if ":doc:" in docstring:
utils.error(
"The {} function in {} contains ':doc:' usage", funcdef.name, path
)
exitcode += 1
utils.exit_invoke(exitcode)
def check_stray(ctx, files):
exitcode = 0
exclude_paths = (
DOCS_DIR / "_inc",
DOCS_DIR / "ref" / "cli" / "_includes",
DOCS_DIR / "ref" / "cli",
DOCS_DIR / "ref" / "configuration",
DOCS_DIR / "ref" / "file_server" / "backends.rst",
DOCS_DIR / "ref" / "file_server" / "environments.rst",
DOCS_DIR / "ref" / "file_server" / "file_roots.rst",
DOCS_DIR / "ref" / "internals",
DOCS_DIR / "ref" / "modules" / "all" / "salt.modules.inspectlib.rst",
DOCS_DIR / "ref" / "peer.rst",
DOCS_DIR / "ref" / "publisheracl.rst",
DOCS_DIR / "ref" / "python-api.rst",
DOCS_DIR / "ref" / "states" / "aggregate.rst",
DOCS_DIR / "ref" / "states" / "altering_states.rst",
DOCS_DIR / "ref" / "states" / "backup_mode.rst",
DOCS_DIR / "ref" / "states" / "compiler_ordering.rst",
DOCS_DIR / "ref" / "states" / "extend.rst",
DOCS_DIR / "ref" / "states" / "failhard.rst",
DOCS_DIR / "ref" / "states" / "global_state_arguments.rst",
DOCS_DIR / "ref" / "states" / "highstate.rst",
DOCS_DIR / "ref" / "states" / "include.rst",
DOCS_DIR / "ref" / "states" / "layers.rst",
DOCS_DIR / "ref" / "states" / "master_side.rst",
DOCS_DIR / "ref" / "states" / "ordering.rst",
DOCS_DIR / "ref" / "states" / "parallel.rst",
DOCS_DIR / "ref" / "states" / "providers.rst",
DOCS_DIR / "ref" / "states" / "requisites.rst",
DOCS_DIR / "ref" / "states" / "startup.rst",
DOCS_DIR / "ref" / "states" / "testing.rst",
DOCS_DIR / "ref" / "states" / "top.rst",
DOCS_DIR / "ref" / "states" / "vars.rst",
DOCS_DIR / "ref" / "states" / "writing.rst",
DOCS_DIR / "topics",
)
exclude_paths = tuple([str(p.relative_to(CODE_DIR)) for p in exclude_paths])
files = build_docs_paths(files)
for path in files:
if not str(path).startswith(str((DOCS_DIR / "ref").relative_to(CODE_DIR))):
continue
if str(path).startswith(exclude_paths):
continue
if path.name in ("index.rst", "glossary.rst", "faq.rst", "README.rst"):
continue
try:
python_module = doc_path_to_python_module[path]
except KeyError:
contents = path.read_text()
if ".. _virtual-" in contents:
continue
exitcode += 1
utils.error(
"The doc at {} doesn't have a corresponding python module an is considered a stray "
"doc. Please remove it.",
path,
)
utils.exit_invoke(exitcode)
def check(ctx):
exitcode = 0
excludes = ("tasks/", "templates/", ".nox/")
full_filelist = [
path.relative_to(CODE_DIR) for path in CODE_DIR.rglob("*.py")
]
filelist = [
str(path) for path in full_filelist
if not str(path).startswith(excludes)
]
filename_map = yaml.safe_load(FILENAME_MAP_PATH.read_text())
checked = set()
for rule, matches in filename_map.items():
if rule == "*":
exitcode += _check_matches(rule, matches)
elif "|" in rule:
# This is regex
for filepath in filelist:
if re.match(rule, filepath):
# Found at least one match, stop looking
break
else:
utils.error(
"Could not find a matching file in the salt repo for the rule '{}'",
rule,
)
exitcode += 1
continue
exitcode += _check_matches(rule, matches)
elif "*" in rule or "\\" in rule:
# Glob matching
process_matches = True
for filerule in CODE_DIR.glob(rule):
if not filerule.exists():
utils.error(
"The rule '{}' points to a non existing path: {}",
rule,
filerule,
)
exitcode += 1
process_matches = False
if process_matches:
exitcode += _check_matches(rule, matches)
else:
# Direct file paths as rules
filerule = pathlib.Path(rule)
if not filerule.exists():
utils.error("The rule '{}' points to a non existing path: {}",
rule, filerule)
exitcode += 1
continue
exitcode += _check_matches(rule, matches)
if exitcode:
utils.error("Found {} errors", exitcode)
utils.exit_invoke(exitcode)
def check(ctx, files):
exitcode = 0
utils.info("Checking inline :doc: markup")
exitcode += check_inline_markup(ctx, files)
utils.info("Checking python module stubs")
exitcode += check_stubs(ctx, files)
utils.info("Checking virtual modules")
exitcode += check_virtual(ctx, files)
utils.info("Checking stray docs")
exitcode += check_stray(ctx, files)
utils.info("Checking doc module indexes")
exitcode += check_module_indexes(ctx, files)
utils.exit_invoke(exitcode)
def check_stubs(ctx, files):
# CD into Salt's repo root directory
ctx.cd(CODE_DIR)
files = build_python_module_paths(files)
exitcode = 0
for path in files:
strpath = str(path)
if strpath.endswith("__init__.py"):
continue
if not strpath.startswith(check_paths):
continue
if strpath.startswith(exclude_paths):
continue
stub_path = python_module_to_doc_path[path]
if not stub_path.exists():
exitcode += 1
utils.error("The module at {} does not have a sphinx stub at {}",
path, stub_path)
utils.exit_invoke(exitcode)
def remove_comments(ctx, files):
"""
Remove import comments, 'Import Python libs', 'Import salt libs', etc
"""
# CD into Salt's repo root directory
ctx.cd(CODE_DIR)
# Unfortunately invoke does not support nargs.
# We migth have been passed --files="foo.py bar.py"
# Turn that into a list of paths
_files = []
for path in files:
if not path:
continue
_files.extend(path.split())
if not _files:
utils.exit_invoke(0)
_files = [
pathlib.Path(fname).resolve() for fname in _files if fname.endswith(".py")
]
fixes = 0
exitcode = 0
comments_regex = re.compile(r"^# ([I|i])mports? .*(([L|l])ibs?)?\n", re.MULTILINE)
for path in _files:
contents = path.read_text()
fixed = comments_regex.sub("", contents)
if fixed == contents:
continue
fixes += 1
exitcode = 1
path.write_text(fixed)
if exitcode:
utils.error("Fixed {} files", fixes)
utils.exit_invoke(exitcode)
def check_module_indexes(ctx, files):
exitcode = 0
files = build_docs_paths(files)
for path in files:
if path.name != "index.rst":
continue
contents = path.read_text()
if ".. autosummary::" not in contents:
continue
module_index_block = re.search(
r"""
\.\.\s+autosummary::\s*\n
(\s+:[a-z]+:.*\n)*
(\s*\n)+
(?P<mods>(\s*[a-z0-9_\.]+\s*\n)+)
""",
contents,
flags=re.VERBOSE,
)
if not module_index_block:
continue
module_index = re.findall(
r"""\s*([a-z0-9_\.]+)\s*\n""", module_index_block.group("mods")
)
if module_index != sorted(module_index):
exitcode += 1
utils.error(
"The autosummary mods in {} are not properly sorted. Please sort them.",
path,
)
module_index_duplicates = [
mod for mod, count in collections.Counter(module_index).items() if count > 1
]
if module_index_duplicates:
exitcode += 1
utils.error(
"Module index {} contains duplicates: {}", path, module_index_duplicates
)
# Let's check if all python modules are included in the index
path_parts = list(path.parts)
# drop doc
path_parts.pop(0)
# drop ref
path_parts.pop(0)
# drop "index.rst"
path_parts.pop()
# drop "all"
path_parts.pop()
package = path_parts.pop(0)
if package == "clouds":
package = "cloud"
if package == "file_server":
package = "fileserver"
if package == "configuration":
package = "log"
path_parts = ["handlers"]
python_package = SALT_CODE_DIR.joinpath(package, *path_parts).relative_to(
CODE_DIR
)
modules = set()
for module in python_package.rglob("*.py"):
if package == "netapi":
if module.stem == "__init__":
continue
if len(module.parts) > 4:
continue
if len(module.parts) > 3:
modules.add(module.parent.stem)
else:
modules.add(module.stem)
elif package == "cloud":
if len(module.parts) < 4:
continue
if module.name == "__init__.py":
continue
modules.add(module.stem)
elif package == "modules":
if len(module.parts) > 3:
# salt.modules.inspeclib
if module.name == "__init__.py":
modules.add(module.parent.stem)
continue
modules.add("{}.{}".format(module.parent.stem, module.stem))
continue
if module.name == "__init__.py":
continue
modules.add(module.stem)
elif module.name == "__init__.py":
continue
elif module.name != "__init__.py":
modules.add(module.stem)
missing_modules_in_index = set(modules) - set(module_index)
if missing_modules_in_index:
exitcode += 1
utils.error(
"The module index at {} is missing the following modules: {}",
path,
", ".join(missing_modules_in_index),
)
extra_modules_in_index = set(module_index) - set(modules)
if extra_modules_in_index:
exitcode += 1
utils.error(
"The module index at {} has extra modules(non existing): {}",
path,
", ".join(extra_modules_in_index),
)
utils.exit_invoke(exitcode)
def check(ctx,
files,
check_proper_formatting=False,
error_on_known_failures=False):
"""
Check salt's docstrings
"""
# CD into Salt's repo root directory
ctx.cd(CODE_DIR)
# Unfortunately invoke does not support nargs.
# We migth have been passed --files="foo.py bar.py"
# Turn that into a list of paths
_files = []
for path in files:
if not path:
continue
_files.extend(path.split())
if not _files:
_files = SALT_CODE_DIR.rglob("*.py")
else:
_files = [pathlib.Path(fname) for fname in _files]
_files = [path.resolve() for path in _files]
errors = 0
exitcode = 0
warnings = 0
for path in _files:
contents = path.read_text()
try:
module = ast.parse(path.read_text(), filename=str(path))
module_docstring = ast.get_docstring(module, clean=False)
if module_docstring:
error = _check_valid_versions_on_docstrings(module_docstring)
if error:
errors += 1
exitcode = 1
utils.error(
"The module '{}' does not provide a proper `{}` version: {!r} is not valid.",
path.relative_to(CODE_DIR),
*error,
)
for funcdef in [
node for node in module.body
if isinstance(node, ast.FunctionDef)
]:
docstring = ast.get_docstring(funcdef, clean=False)
if docstring:
error = _check_valid_versions_on_docstrings(docstring)
if error:
errors += 1
exitcode = 1
utils.error(
"The module '{}' does not provide a proper `{}` version: {!r} is not valid.",
path.relative_to(CODE_DIR),
*error,
)
if not str(path).startswith(SALT_INTERNAL_LOADERS_PATHS):
# No further docstrings checks are needed
continue
funcname = funcdef.name
relpath = str(path.relative_to(CODE_DIR))
# We're dealing with a salt loader module
if funcname.startswith("_"):
# We're not interested in internal functions
continue
if not docstring:
if (funcname in MISSING_DOCSTRINGS.get(relpath, ())
and error_on_known_failures is False):
warnings += 1
utils.warn(
"The function '{}' on '{}' does not have a docstring",
funcname,
relpath,
)
continue
errors += 1
exitcode = 1
utils.error(
"The function '{}' on '{}' does not have a docstring",
funcname,
relpath,
)
continue
elif funcname in MISSING_DOCSTRINGS.get(relpath, ()):
# This was previously a know function with a missing docstring.
# Warn about it so that it get's removed from this list
warnings += 1
utils.warn(
"The function '{}' on '{}' was previously known to not have a docstring, "
"which is no longer the case. Please remove it from 'MISSING_DOCSTRINGS' ."
"in '{}'",
funcname,
relpath,
THIS_FILE,
)
try:
salt_modules_relpath = path.relative_to(SALT_MODULES_PATH)
if str(salt_modules_relpath.parent) != ".":
# We don't want to check nested packages
continue
# But this is a module under salt/modules, let's check
# the CLI examples
except ValueError:
# We're not checking CLI examples in any other salt loader modules
continue
if _check_cli_example_present(docstring) is False:
if (funcname in MISSING_EXAMPLES.get(relpath, ())
and error_on_known_failures is False):
warnings += 1
utils.warn(
"The function '{}' on '{}' does not have a 'CLI Example:' in it's docstring",
funcname,
relpath,
)
continue
errors += 1
exitcode = 1
utils.error(
"The function '{}' on '{}' does not have a 'CLI Example:' in it's docstring",
funcname,
relpath,
)
continue
elif funcname in MISSING_EXAMPLES.get(relpath, ()):
# This was previously a know function with a missing CLI example
# Warn about it so that it get's removed from this list
warnings += 1
utils.warn(
"The function '{}' on '{}' was previously known to not have a CLI Example, "
"which is no longer the case. Please remove it from 'MISSING_EXAMPLES'. "
"in '{}'",
funcname,
relpath,
THIS_FILE,
)
if check_proper_formatting is False:
continue
# By now we now this function has a docstring and it has a CLI Example section
# Let's now check if it's properly formatted
if _check_cli_example_proper_formatting(docstring) is False:
errors += 1
exitcode = 1
utils.error(
"The function {!r} on '{}' does not have a proper 'CLI Example:' section in "
"it's docstring. The proper format is:\n"
"CLI Example:\n"
"\n"
".. code-block:: bash\n"
"\n"
" salt '*' <insert example here>\n",
funcdef.name,
path.relative_to(CODE_DIR),
)
continue
finally:
if contents != path.read_text():
path.write_text(contents)
if warnings:
utils.warn("Found {} warnings", warnings)
if exitcode:
utils.error("Found {} errors", errors)
utils.exit_invoke(exitcode)
def check_virtual(ctx, files, enforce_virtualname=False):
"""
Check Salt loader modules for a defined `__virtualname__` attribute and `__virtual__` function.
This is meant to replace:
https://github.com/saltstack/salt/blob/27ae8260983b11fe6e32a18e777d550be9fe1dc2/tests/unit/test_virtualname.py
"""
# CD into Salt's repo root directory
ctx.cd(CODE_DIR)
# Unfortunately invoke does not support nargs.
# We migth have been passed --files="foo.py bar.py"
# Turn that into a list of paths
_files = []
for path in files:
if not path:
continue
_files.extend(path.split())
if not _files:
_files = SALT_CODE_DIR.rglob("*.py")
else:
_files = [pathlib.Path(fname) for fname in _files]
_files = [path.resolve() for path in _files]
errors = 0
exitcode = 0
for path in _files:
strpath = str(path)
if path.name == "__init__.py":
continue
for loader in SALT_INTERNAL_LOADERS_PATHS:
try:
path.relative_to(loader)
break
except ValueError:
# Path doesn't start with the loader path, carry on
continue
module = ast.parse(path.read_text(), filename=str(path))
found_virtual_func = False
for funcdef in [
node for node in module.body
if isinstance(node, ast.FunctionDef)
]:
if funcdef.name == "__virtual__":
found_virtual_func = True
break
if not found_virtual_func:
# If the module does not define a __virtual__() function, we don't require a __virtualname__ attribute
continue
found_virtualname_attr = False
for node in module.body:
if isinstance(node, ast.Assign):
if not found_virtualname_attr:
for target in node.targets:
if not isinstance(target, ast.Name):
continue
if target.id == "__virtualname__":
found_virtualname_attr = True
if node.value.s not in path.name:
errors += 1
exitcode = 1
utils.error(
'The value of the __virtualname__ attribute, "{}"'
" is not part of {}",
node.value.s,
path.name,
)
if found_virtualname_attr:
break
if not found_virtualname_attr and enforce_virtualname:
errors += 1
exitcode = 1
utils.error(
"The salt loader module {} defines a __virtual__() function but does"
" not define a __virtualname__ attribute",
path.relative_to(CODE_DIR),
)
if exitcode:
utils.error("Found {} errors", errors)
utils.exit_invoke(exitcode)
def check_virtual(ctx, files):
"""
Check Salt loader modules for a defined `__virtualname__` attribute and `__virtual__` function.
This is meant to replace:
https://github.com/saltstack/salt/blob/27ae8260983b11fe6e32a18e777d550be9fe1dc2/tests/unit/test_virtualname.py
"""
# CD into Salt's repo root directory
ctx.cd(CODE_DIR)
# Unfortunately invoke does not support nargs.
# We migth have been passed --files="foo.py bar.py"
# Turn that into a list of paths
_files = []
for path in files:
if not path:
continue
_files.extend(path.split())
if not _files:
_files = SALT_CODE_DIR.rglob("*.py")
else:
_files = [pathlib.Path(fname) for fname in _files]
_files = [path.resolve() for path in _files]
salt_loaders = (
CODE_DIR / "salt" / "modules",
CODE_DIR / "salt" / "metaproxy",
CODE_DIR / "salt" / "matchers",
CODE_DIR / "salt" / "engines",
CODE_DIR / "salt" / "proxy",
CODE_DIR / "salt" / "returners",
CODE_DIR / "salt" / "utils",
CODE_DIR / "salt" / "pillar",
CODE_DIR / "salt" / "tops",
CODE_DIR / "salt" / "wheel",
CODE_DIR / "salt" / "output",
CODE_DIR / "salt" / "serializers",
CODE_DIR / "salt" / "tokens",
CODE_DIR / "salt" / "auth",
CODE_DIR / "salt" / "fileserver",
CODE_DIR / "salt" / "roster",
CODE_DIR / "salt" / "thorium",
CODE_DIR / "salt" / "states",
CODE_DIR / "salt" / "beacons",
CODE_DIR / "salt" / "log" / "handlers",
CODE_DIR / "salt" / "client" / "ssh",
CODE_DIR / "salt" / "renderers",
CODE_DIR / "salt" / "grains",
CODE_DIR / "salt" / "runners",
CODE_DIR / "salt" / "queues",
CODE_DIR / "salt" / "sdb",
CODE_DIR / "salt" / "spm" / "pkgdb",
CODE_DIR / "salt" / "spm" / "pkgfiles",
CODE_DIR / "salt" / "cloud" / "clouds",
CODE_DIR / "salt" / "netapi",
CODE_DIR / "salt" / "executors",
CODE_DIR / "salt" / "cache",
)
# This is just internal task checking
for loader in salt_loaders:
if not pathlib.Path(loader).is_dir():
utils.error("The {} path is not a directory", loader)
errors = 0
exitcode = 0
for path in _files:
strpath = str(path)
if strpath.endswith("__init__.py"):
continue
for loader in salt_loaders:
try:
path.relative_to(loader)
break
except ValueError:
# Path doesn't start with the loader path, carry on
continue
module = ast.parse(path.read_text(), filename=strpath)
found_virtual_func = False
for funcdef in [
node for node in module.body
if isinstance(node, ast.FunctionDef)
]:
if funcdef.name == "__virtual__":
found_virtual_func = True
break
if not found_virtual_func:
# If the module does not define a __virtual__() function, we don't require a __virtualname__ attribute
continue
found_virtualname_attr = False
for node in module.body:
if isinstance(node, ast.Assign):
if not found_virtualname_attr:
for target in node.targets:
if not isinstance(target, ast.Name):
continue
if target.id == "__virtualname__":
found_virtualname_attr = True
if node.value.s not in path.name:
errors += 1
exitcode = 1
utils.error(
'The value of the __virtualname__ attribute, "{}" is not part of {}',
node.value.s,
path.name,
)
if found_virtualname_attr:
break
if not found_virtualname_attr:
errors += 1
exitcode = 1
utils.error(
"The salt loader module {} defines a __virtual__() function but does not define a "
"__virtualname__ attribute",
path.relative_to(CODE_DIR),
)
if exitcode:
utils.error("Found {} errors", errors)
utils.exit_invoke(exitcode)
