def lint_orphan_secrets(compiled_path, secrets_path):
""" Checks your secrets/ folder for unused secrets files by:
- iterating the secrets_path/ dir and extracting all secrets names from the file paths
- does a text search over the entire compiled_path/ to find usages of those secrets
Args:
compiled_path (string): path to your compiled/ folder
secrets_path (string): path to your secrets/ folder
Yields:
checks_sum (int): the number of orphan secrets found
"""
logger.debug("Find secret paths for " + secrets_path)
secrets_paths = set()
for path in list_all_paths(secrets_path):
if os.path.isfile(path):
path = path[len(secrets_path) + 1:]
secrets_paths.add(path)
logger.debug("Collected # of paths: {}".format(len(secrets_paths)))
logger.debug("Checking if all secrets are declared in " + compiled_path)
for path in list_all_paths(compiled_path):
if os.path.isfile(path):
with open(path, "r") as compiled_file:
file_contents = compiled_file.read()
for secret_path in list(secrets_paths):
if secret_path in file_contents:
secrets_paths.discard(secret_path)
checks_sum = len(secrets_paths)
if checks_sum > 0:
logger.info(
"No usage found for the following {} secrets files:\n{}\n".format(
len(secrets_paths), pformat(secrets_paths)))
return checks_sum
def lint_unused_classes(inventory_path):
""" Checks your inventory for unused classes by:
- iterating the inventory_path/classes/ dir and extracting all class names from the file paths
- converting those file paths to class references (e.g. component/mysql -> component.mysql)
- does a text search over the entire inventory_path/ to find usages of those classes
Args:
inventory_path (string): path to your inventory folder
Yields:
checks_sum (int): the number of unused classes found
"""
classes_dir = os.path.join(inventory_path, "classes/")
if not os.path.isdir(classes_dir):
raise KapitanError(
"{} is not a valid directory or does not exist".format(
classes_dir))
logger.debug("Find unused classes from {}".format(classes_dir))
class_paths = set()
for path in list_all_paths(classes_dir):
if os.path.isfile(path) and (path.endswith('.yml')
or path.endswith('.yaml')):
path = path[len(classes_dir):]
path = path.replace(".yml", "").replace(".yaml",
"").replace("/", ".")
class_paths.add(path)
logger.debug("Collected # of paths: {}".format(len(class_paths)))
logger.debug("Checking if all classes are declared in " + classes_dir)
for path in list_all_paths(inventory_path):
if os.path.isfile(path):
with open(path, "r") as compiled_file:
file_contents = compiled_file.read()
for class_path in list(class_paths):
exists = class_path in file_contents
"""
Classes files may reside in subdirectories, which act as namespaces.
For instance, a class ssh.server will result in the class definition to be read from ssh/server.yml.
Specifying just ssh will cause the class data to be read from ssh/init.yml or ssh.yml.
Note, however, that only one of those two may be present.
Thus we also check for ".init" being used in the class here to cover that case.
https://reclass.pantsfullofunix.net/operations.html
"""
if class_path.endswith(".init"):
exists = (class_path[:-5] in file_contents) or (exists)
if exists:
class_paths.discard(class_path)
checks_sum = len(class_paths)
if checks_sum > 0:
logger.info(
"No usage found for the following {} classes:\n{}\n".format(
len(class_paths), pformat(class_paths)))
return checks_sum
def lint_yamllint(inventory_path):
""" Run yamllint on all yaml files in inventory
Args:
inventory_path (string): path to your inventory/ folder
Yields:
checks_sum (int): the number of yaml lint issues found
"""
logger.debug("Running yamllint for " + inventory_path)
if os.path.isfile('.yamllint'):
logger.info("Loading values from .yamllint found.")
conf = YamlLintConfig(file='.yamllint')
else:
logger.info(".yamllint not found. Using default values")
conf = YamlLintConfig(yamllint_config)
checks_sum = 0
for path in list_all_paths(inventory_path):
if os.path.isfile(path) and (path.endswith(".yml")
or path.endswith(".yaml")):
with open(path, "r") as yaml_file:
file_contents = yaml_file.read()
try:
problems = list(
linter.run(file_contents, conf, filepath=path))
except EnvironmentError as e:
logger.error(e)
sys.exit(-1)
if len(problems) > 0:
checks_sum += len(problems)
logger.info(
"File {} has the following issues:".format(path))
for problem in problems:
logger.info("\t{}".format(problem))
if checks_sum > 0:
logger.info("\nTotal yamllint issues found: {}".format(checks_sum))
return checks_sum
def _reveal_dir(self, dirname):
"""
returns tuple with yaml, json, and raw concatenated output for revealed file types
recurses through subdirectories in dirname
"""
out_yaml = ""
out_json = ""
out_raw = ""
# find yaml/json/raw files and concatenate output per type
for fpath in list_all_paths(dirname):
if not os.path.isfile(fpath):
continue
if fpath.endswith(".yml") or fpath.endswith(".yaml"):
out, _ = self._reveal_file(fpath)
out_yaml += out
elif fpath.endswith(".json"):
out, _ = self._reveal_file(fpath)
out_json += self._reveal_file(fpath)
else:
out, _ = self._reveal_file(fpath)
out_raw += out
return out_yaml, out_json, out_raw
def __iter__(self):
for full_path in list_all_paths(self.path):
ref_path = full_path[len(self.path):]
yield ref_path
