def check_virtual(ctx, files): """ Check if .rst files for each module contains the text ".. _virtual" indicating it is a virtual doc page, and, in case a module exists by the same name, it's going to be shaddowed and not accessible """ exitcode = 0 files = build_docs_paths(files) for path in files: if path.name == "index.rst": continue contents = path.read_text() if ".. _virtual-" in contents: try: python_module = doc_path_to_python_module[path] utils.error( "The doc file at {} indicates that it's virtual, yet, there's a python module " "at {} that will shaddow it.", path, python_module, ) exitcode += 1 except KeyError: # This is what we're expecting continue utils.exit_invoke(exitcode)
def check_inline_markup(ctx, files): """ Check docstring for :doc: usage We should not be using the ``:doc:`` inline markup option when cross-referencing locations. Use ``:ref:`` or ``:mod:`` instead. This task checks for reference to ``:doc:`` usage. See Issue #12788 for more information. https://github.com/saltstack/salt/issues/12788 """ # CD into Salt's repo root directory ctx.cd(CODE_DIR) files = build_python_module_paths(files) exitcode = 0 for path in files: module = ast.parse(path.read_text(), filename=str(path)) funcdefs = [node for node in module.body if isinstance(node, ast.FunctionDef)] for funcdef in funcdefs: docstring = ast.get_docstring(funcdef, clean=True) if not docstring: continue if ":doc:" in docstring: utils.error( "The {} function in {} contains ':doc:' usage", funcdef.name, path ) exitcode += 1 utils.exit_invoke(exitcode)
def check_stray(ctx, files): exitcode = 0 exclude_paths = ( DOCS_DIR / "_inc", DOCS_DIR / "ref" / "cli" / "_includes", DOCS_DIR / "ref" / "cli", DOCS_DIR / "ref" / "configuration", DOCS_DIR / "ref" / "file_server" / "backends.rst", DOCS_DIR / "ref" / "file_server" / "environments.rst", DOCS_DIR / "ref" / "file_server" / "file_roots.rst", DOCS_DIR / "ref" / "internals", DOCS_DIR / "ref" / "modules" / "all" / "salt.modules.inspectlib.rst", DOCS_DIR / "ref" / "peer.rst", DOCS_DIR / "ref" / "publisheracl.rst", DOCS_DIR / "ref" / "python-api.rst", DOCS_DIR / "ref" / "states" / "aggregate.rst", DOCS_DIR / "ref" / "states" / "altering_states.rst", DOCS_DIR / "ref" / "states" / "backup_mode.rst", DOCS_DIR / "ref" / "states" / "compiler_ordering.rst", DOCS_DIR / "ref" / "states" / "extend.rst", DOCS_DIR / "ref" / "states" / "failhard.rst", DOCS_DIR / "ref" / "states" / "global_state_arguments.rst", DOCS_DIR / "ref" / "states" / "highstate.rst", DOCS_DIR / "ref" / "states" / "include.rst", DOCS_DIR / "ref" / "states" / "layers.rst", DOCS_DIR / "ref" / "states" / "master_side.rst", DOCS_DIR / "ref" / "states" / "ordering.rst", DOCS_DIR / "ref" / "states" / "parallel.rst", DOCS_DIR / "ref" / "states" / "providers.rst", DOCS_DIR / "ref" / "states" / "requisites.rst", DOCS_DIR / "ref" / "states" / "startup.rst", DOCS_DIR / "ref" / "states" / "testing.rst", DOCS_DIR / "ref" / "states" / "top.rst", DOCS_DIR / "ref" / "states" / "vars.rst", DOCS_DIR / "ref" / "states" / "writing.rst", DOCS_DIR / "topics", ) exclude_paths = tuple([str(p.relative_to(CODE_DIR)) for p in exclude_paths]) files = build_docs_paths(files) for path in files: if not str(path).startswith(str((DOCS_DIR / "ref").relative_to(CODE_DIR))): continue if str(path).startswith(exclude_paths): continue if path.name in ("index.rst", "glossary.rst", "faq.rst", "README.rst"): continue try: python_module = doc_path_to_python_module[path] except KeyError: contents = path.read_text() if ".. _virtual-" in contents: continue exitcode += 1 utils.error( "The doc at {} doesn't have a corresponding python module an is considered a stray " "doc. Please remove it.", path, ) utils.exit_invoke(exitcode)
def check(ctx): exitcode = 0 excludes = ("tasks/", "templates/", ".nox/") full_filelist = [ path.relative_to(CODE_DIR) for path in CODE_DIR.rglob("*.py") ] filelist = [ str(path) for path in full_filelist if not str(path).startswith(excludes) ] filename_map = yaml.safe_load(FILENAME_MAP_PATH.read_text()) checked = set() for rule, matches in filename_map.items(): if rule == "*": exitcode += _check_matches(rule, matches) elif "|" in rule: # This is regex for filepath in filelist: if re.match(rule, filepath): # Found at least one match, stop looking break else: utils.error( "Could not find a matching file in the salt repo for the rule '{}'", rule, ) exitcode += 1 continue exitcode += _check_matches(rule, matches) elif "*" in rule or "\\" in rule: # Glob matching process_matches = True for filerule in CODE_DIR.glob(rule): if not filerule.exists(): utils.error( "The rule '{}' points to a non existing path: {}", rule, filerule, ) exitcode += 1 process_matches = False if process_matches: exitcode += _check_matches(rule, matches) else: # Direct file paths as rules filerule = pathlib.Path(rule) if not filerule.exists(): utils.error("The rule '{}' points to a non existing path: {}", rule, filerule) exitcode += 1 continue exitcode += _check_matches(rule, matches) if exitcode: utils.error("Found {} errors", exitcode) utils.exit_invoke(exitcode)
def check(ctx, files): exitcode = 0 utils.info("Checking inline :doc: markup") exitcode += check_inline_markup(ctx, files) utils.info("Checking python module stubs") exitcode += check_stubs(ctx, files) utils.info("Checking virtual modules") exitcode += check_virtual(ctx, files) utils.info("Checking stray docs") exitcode += check_stray(ctx, files) utils.info("Checking doc module indexes") exitcode += check_module_indexes(ctx, files) utils.exit_invoke(exitcode)
def check_stubs(ctx, files): # CD into Salt's repo root directory ctx.cd(CODE_DIR) files = build_python_module_paths(files) exitcode = 0 for path in files: strpath = str(path) if strpath.endswith("__init__.py"): continue if not strpath.startswith(check_paths): continue if strpath.startswith(exclude_paths): continue stub_path = python_module_to_doc_path[path] if not stub_path.exists(): exitcode += 1 utils.error("The module at {} does not have a sphinx stub at {}", path, stub_path) utils.exit_invoke(exitcode)
def remove_comments(ctx, files): """ Remove import comments, 'Import Python libs', 'Import salt libs', etc """ # CD into Salt's repo root directory ctx.cd(CODE_DIR) # Unfortunately invoke does not support nargs. # We migth have been passed --files="foo.py bar.py" # Turn that into a list of paths _files = [] for path in files: if not path: continue _files.extend(path.split()) if not _files: utils.exit_invoke(0) _files = [ pathlib.Path(fname).resolve() for fname in _files if fname.endswith(".py") ] fixes = 0 exitcode = 0 comments_regex = re.compile(r"^# ([I|i])mports? .*(([L|l])ibs?)?\n", re.MULTILINE) for path in _files: contents = path.read_text() fixed = comments_regex.sub("", contents) if fixed == contents: continue fixes += 1 exitcode = 1 path.write_text(fixed) if exitcode: utils.error("Fixed {} files", fixes) utils.exit_invoke(exitcode)
def check_module_indexes(ctx, files): exitcode = 0 files = build_docs_paths(files) for path in files: if path.name != "index.rst": continue contents = path.read_text() if ".. autosummary::" not in contents: continue module_index_block = re.search( r""" \.\.\s+autosummary::\s*\n (\s+:[a-z]+:.*\n)* (\s*\n)+ (?P<mods>(\s*[a-z0-9_\.]+\s*\n)+) """, contents, flags=re.VERBOSE, ) if not module_index_block: continue module_index = re.findall( r"""\s*([a-z0-9_\.]+)\s*\n""", module_index_block.group("mods") ) if module_index != sorted(module_index): exitcode += 1 utils.error( "The autosummary mods in {} are not properly sorted. Please sort them.", path, ) module_index_duplicates = [ mod for mod, count in collections.Counter(module_index).items() if count > 1 ] if module_index_duplicates: exitcode += 1 utils.error( "Module index {} contains duplicates: {}", path, module_index_duplicates ) # Let's check if all python modules are included in the index path_parts = list(path.parts) # drop doc path_parts.pop(0) # drop ref path_parts.pop(0) # drop "index.rst" path_parts.pop() # drop "all" path_parts.pop() package = path_parts.pop(0) if package == "clouds": package = "cloud" if package == "file_server": package = "fileserver" if package == "configuration": package = "log" path_parts = ["handlers"] python_package = SALT_CODE_DIR.joinpath(package, *path_parts).relative_to( CODE_DIR ) modules = set() for module in python_package.rglob("*.py"): if package == "netapi": if module.stem == "__init__": continue if len(module.parts) > 4: continue if len(module.parts) > 3: modules.add(module.parent.stem) else: modules.add(module.stem) elif package == "cloud": if len(module.parts) < 4: continue if module.name == "__init__.py": continue modules.add(module.stem) elif package == "modules": if len(module.parts) > 3: # salt.modules.inspeclib if module.name == "__init__.py": modules.add(module.parent.stem) continue modules.add("{}.{}".format(module.parent.stem, module.stem)) continue if module.name == "__init__.py": continue modules.add(module.stem) elif module.name == "__init__.py": continue elif module.name != "__init__.py": modules.add(module.stem) missing_modules_in_index = set(modules) - set(module_index) if missing_modules_in_index: exitcode += 1 utils.error( "The module index at {} is missing the following modules: {}", path, ", ".join(missing_modules_in_index), ) extra_modules_in_index = set(module_index) - set(modules) if extra_modules_in_index: exitcode += 1 utils.error( "The module index at {} has extra modules(non existing): {}", path, ", ".join(extra_modules_in_index), ) utils.exit_invoke(exitcode)
def check(ctx, files, check_proper_formatting=False, error_on_known_failures=False): """ Check salt's docstrings """ # CD into Salt's repo root directory ctx.cd(CODE_DIR) # Unfortunately invoke does not support nargs. # We migth have been passed --files="foo.py bar.py" # Turn that into a list of paths _files = [] for path in files: if not path: continue _files.extend(path.split()) if not _files: _files = SALT_CODE_DIR.rglob("*.py") else: _files = [pathlib.Path(fname) for fname in _files] _files = [path.resolve() for path in _files] errors = 0 exitcode = 0 warnings = 0 for path in _files: contents = path.read_text() try: module = ast.parse(path.read_text(), filename=str(path)) module_docstring = ast.get_docstring(module, clean=False) if module_docstring: error = _check_valid_versions_on_docstrings(module_docstring) if error: errors += 1 exitcode = 1 utils.error( "The module '{}' does not provide a proper `{}` version: {!r} is not valid.", path.relative_to(CODE_DIR), *error, ) for funcdef in [ node for node in module.body if isinstance(node, ast.FunctionDef) ]: docstring = ast.get_docstring(funcdef, clean=False) if docstring: error = _check_valid_versions_on_docstrings(docstring) if error: errors += 1 exitcode = 1 utils.error( "The module '{}' does not provide a proper `{}` version: {!r} is not valid.", path.relative_to(CODE_DIR), *error, ) if not str(path).startswith(SALT_INTERNAL_LOADERS_PATHS): # No further docstrings checks are needed continue funcname = funcdef.name relpath = str(path.relative_to(CODE_DIR)) # We're dealing with a salt loader module if funcname.startswith("_"): # We're not interested in internal functions continue if not docstring: if (funcname in MISSING_DOCSTRINGS.get(relpath, ()) and error_on_known_failures is False): warnings += 1 utils.warn( "The function '{}' on '{}' does not have a docstring", funcname, relpath, ) continue errors += 1 exitcode = 1 utils.error( "The function '{}' on '{}' does not have a docstring", funcname, relpath, ) continue elif funcname in MISSING_DOCSTRINGS.get(relpath, ()): # This was previously a know function with a missing docstring. # Warn about it so that it get's removed from this list warnings += 1 utils.warn( "The function '{}' on '{}' was previously known to not have a docstring, " "which is no longer the case. Please remove it from 'MISSING_DOCSTRINGS' ." "in '{}'", funcname, relpath, THIS_FILE, ) try: salt_modules_relpath = path.relative_to(SALT_MODULES_PATH) if str(salt_modules_relpath.parent) != ".": # We don't want to check nested packages continue # But this is a module under salt/modules, let's check # the CLI examples except ValueError: # We're not checking CLI examples in any other salt loader modules continue if _check_cli_example_present(docstring) is False: if (funcname in MISSING_EXAMPLES.get(relpath, ()) and error_on_known_failures is False): warnings += 1 utils.warn( "The function '{}' on '{}' does not have a 'CLI Example:' in it's docstring", funcname, relpath, ) continue errors += 1 exitcode = 1 utils.error( "The function '{}' on '{}' does not have a 'CLI Example:' in it's docstring", funcname, relpath, ) continue elif funcname in MISSING_EXAMPLES.get(relpath, ()): # This was previously a know function with a missing CLI example # Warn about it so that it get's removed from this list warnings += 1 utils.warn( "The function '{}' on '{}' was previously known to not have a CLI Example, " "which is no longer the case. Please remove it from 'MISSING_EXAMPLES'. " "in '{}'", funcname, relpath, THIS_FILE, ) if check_proper_formatting is False: continue # By now we now this function has a docstring and it has a CLI Example section # Let's now check if it's properly formatted if _check_cli_example_proper_formatting(docstring) is False: errors += 1 exitcode = 1 utils.error( "The function {!r} on '{}' does not have a proper 'CLI Example:' section in " "it's docstring. The proper format is:\n" "CLI Example:\n" "\n" ".. code-block:: bash\n" "\n" " salt '*' <insert example here>\n", funcdef.name, path.relative_to(CODE_DIR), ) continue finally: if contents != path.read_text(): path.write_text(contents) if warnings: utils.warn("Found {} warnings", warnings) if exitcode: utils.error("Found {} errors", errors) utils.exit_invoke(exitcode)
def check_virtual(ctx, files, enforce_virtualname=False): """ Check Salt loader modules for a defined `__virtualname__` attribute and `__virtual__` function. This is meant to replace: https://github.com/saltstack/salt/blob/27ae8260983b11fe6e32a18e777d550be9fe1dc2/tests/unit/test_virtualname.py """ # CD into Salt's repo root directory ctx.cd(CODE_DIR) # Unfortunately invoke does not support nargs. # We migth have been passed --files="foo.py bar.py" # Turn that into a list of paths _files = [] for path in files: if not path: continue _files.extend(path.split()) if not _files: _files = SALT_CODE_DIR.rglob("*.py") else: _files = [pathlib.Path(fname) for fname in _files] _files = [path.resolve() for path in _files] errors = 0 exitcode = 0 for path in _files: strpath = str(path) if path.name == "__init__.py": continue for loader in SALT_INTERNAL_LOADERS_PATHS: try: path.relative_to(loader) break except ValueError: # Path doesn't start with the loader path, carry on continue module = ast.parse(path.read_text(), filename=str(path)) found_virtual_func = False for funcdef in [ node for node in module.body if isinstance(node, ast.FunctionDef) ]: if funcdef.name == "__virtual__": found_virtual_func = True break if not found_virtual_func: # If the module does not define a __virtual__() function, we don't require a __virtualname__ attribute continue found_virtualname_attr = False for node in module.body: if isinstance(node, ast.Assign): if not found_virtualname_attr: for target in node.targets: if not isinstance(target, ast.Name): continue if target.id == "__virtualname__": found_virtualname_attr = True if node.value.s not in path.name: errors += 1 exitcode = 1 utils.error( 'The value of the __virtualname__ attribute, "{}"' " is not part of {}", node.value.s, path.name, ) if found_virtualname_attr: break if not found_virtualname_attr and enforce_virtualname: errors += 1 exitcode = 1 utils.error( "The salt loader module {} defines a __virtual__() function but does" " not define a __virtualname__ attribute", path.relative_to(CODE_DIR), ) if exitcode: utils.error("Found {} errors", errors) utils.exit_invoke(exitcode)
def check_virtual(ctx, files): """ Check Salt loader modules for a defined `__virtualname__` attribute and `__virtual__` function. This is meant to replace: https://github.com/saltstack/salt/blob/27ae8260983b11fe6e32a18e777d550be9fe1dc2/tests/unit/test_virtualname.py """ # CD into Salt's repo root directory ctx.cd(CODE_DIR) # Unfortunately invoke does not support nargs. # We migth have been passed --files="foo.py bar.py" # Turn that into a list of paths _files = [] for path in files: if not path: continue _files.extend(path.split()) if not _files: _files = SALT_CODE_DIR.rglob("*.py") else: _files = [pathlib.Path(fname) for fname in _files] _files = [path.resolve() for path in _files] salt_loaders = ( CODE_DIR / "salt" / "modules", CODE_DIR / "salt" / "metaproxy", CODE_DIR / "salt" / "matchers", CODE_DIR / "salt" / "engines", CODE_DIR / "salt" / "proxy", CODE_DIR / "salt" / "returners", CODE_DIR / "salt" / "utils", CODE_DIR / "salt" / "pillar", CODE_DIR / "salt" / "tops", CODE_DIR / "salt" / "wheel", CODE_DIR / "salt" / "output", CODE_DIR / "salt" / "serializers", CODE_DIR / "salt" / "tokens", CODE_DIR / "salt" / "auth", CODE_DIR / "salt" / "fileserver", CODE_DIR / "salt" / "roster", CODE_DIR / "salt" / "thorium", CODE_DIR / "salt" / "states", CODE_DIR / "salt" / "beacons", CODE_DIR / "salt" / "log" / "handlers", CODE_DIR / "salt" / "client" / "ssh", CODE_DIR / "salt" / "renderers", CODE_DIR / "salt" / "grains", CODE_DIR / "salt" / "runners", CODE_DIR / "salt" / "queues", CODE_DIR / "salt" / "sdb", CODE_DIR / "salt" / "spm" / "pkgdb", CODE_DIR / "salt" / "spm" / "pkgfiles", CODE_DIR / "salt" / "cloud" / "clouds", CODE_DIR / "salt" / "netapi", CODE_DIR / "salt" / "executors", CODE_DIR / "salt" / "cache", ) # This is just internal task checking for loader in salt_loaders: if not pathlib.Path(loader).is_dir(): utils.error("The {} path is not a directory", loader) errors = 0 exitcode = 0 for path in _files: strpath = str(path) if strpath.endswith("__init__.py"): continue for loader in salt_loaders: try: path.relative_to(loader) break except ValueError: # Path doesn't start with the loader path, carry on continue module = ast.parse(path.read_text(), filename=strpath) found_virtual_func = False for funcdef in [ node for node in module.body if isinstance(node, ast.FunctionDef) ]: if funcdef.name == "__virtual__": found_virtual_func = True break if not found_virtual_func: # If the module does not define a __virtual__() function, we don't require a __virtualname__ attribute continue found_virtualname_attr = False for node in module.body: if isinstance(node, ast.Assign): if not found_virtualname_attr: for target in node.targets: if not isinstance(target, ast.Name): continue if target.id == "__virtualname__": found_virtualname_attr = True if node.value.s not in path.name: errors += 1 exitcode = 1 utils.error( 'The value of the __virtualname__ attribute, "{}" is not part of {}', node.value.s, path.name, ) if found_virtualname_attr: break if not found_virtualname_attr: errors += 1 exitcode = 1 utils.error( "The salt loader module {} defines a __virtual__() function but does not define a " "__virtualname__ attribute", path.relative_to(CODE_DIR), ) if exitcode: utils.error("Found {} errors", errors) utils.exit_invoke(exitcode)