def get_graph_checks_report(self, root_folder: str,
runner_filter: RunnerFilter) -> Report:
report = Report(self.check_type)
checks_results = self.run_graph_checks_results(runner_filter)
for check, check_results in checks_results.items():
for check_result in check_results:
entity = check_result["entity"]
entity_file_path = entity.get(CustomAttributes.FILE_PATH)
entity_file_abs_path = _get_entity_abs_path(
root_folder, entity_file_path)
entity_id = entity.get(CustomAttributes.ID)
entity_context = self.context[entity_file_path][entity_id]
record = Record(check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_context.get("code_lines"),
file_path=entity_file_path,
file_line_range=[
entity_context.get("start_line"),
entity_context.get("end_line")
],
resource=entity.get(CustomAttributes.ID),
evaluations={},
check_class=check.__class__.__module__,
file_abs_path=entity_file_abs_path)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def mutateKubernetesResults(self, results, report, k8_file=None, k8_file_path=None, file_abs_path=None, entity_conf=None, variable_evaluations=None, reportMutatorData=None):
# Moves report generation logic out of checkov.kubernetes.runner.run() def.
# Allows us to overriding report file information for "child" frameworks such as Kustomize, Helm
# Where Kubernetes CHECKS are needed, but the specific file references are to another framework for the user output (or a mix of both).
kustomizeMetadata = reportMutatorData['kustomizeMetadata'],
kustomizeFileMappings = reportMutatorData['kustomizeFileMappings']
for check, check_result in results.items():
resource_id = get_resource_id(entity_conf)
entity_context = self.context[k8_file][resource_id]
if file_abs_path in kustomizeFileMappings:
realKustomizeEnvMetadata = kustomizeMetadata[0][kustomizeFileMappings[file_abs_path]]
if 'overlay' in realKustomizeEnvMetadata["type"]:
kustomizeResourceID = f'{realKustomizeEnvMetadata["type"]}:{str(realKustomizeEnvMetadata["overlay_name"])}:{resource_id}'
else:
kustomizeResourceID = f'{realKustomizeEnvMetadata["type"]}:{resource_id}'
else:
kustomizeResourceID = "Unknown error. This is a bug."
record = Record(
check_id=check.id, bc_check_id=check.bc_id, check_name=check.name,
check_result=check_result, code_block=entity_context.get("code_lines"), file_path=realKustomizeEnvMetadata['filePath'],
file_line_range=[0,0],
resource=kustomizeResourceID, evaluations=variable_evaluations,
check_class=check.__class__.__module__, file_abs_path=realKustomizeEnvMetadata['filePath'])
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def get_graph_checks_report(self, root_folder: str,
runner_filter: RunnerFilter) -> Report:
report = Report(self.check_type)
checks_results = self.run_graph_checks_results(runner_filter)
for check, check_results in checks_results.items():
for check_result in check_results:
entity = check_result["entity"]
entity_file_abs_path = entity.get(CustomAttributes.FILE_PATH)
entity_file_path = f"/{os.path.relpath(entity_file_abs_path, root_folder)}"
start_line = entity['__startline__']
end_line = entity['__endline__']
if start_line == end_line:
entity_lines_range = [start_line, end_line]
entity_code_lines = self.definitions_raw[entity_file_path][
start_line - 1:end_line]
else:
entity_lines_range = [start_line, end_line - 1]
entity_code_lines = self.definitions_raw[entity_file_path][
start_line - 1:end_line - 1]
record = Record(check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=entity_file_path,
file_line_range=entity_lines_range,
resource=entity.get(CustomAttributes.ID),
evaluations={},
check_class=check.__class__.__module__,
file_abs_path=entity_file_abs_path)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def mutateKubernetesGraphResults(self,
root_folder: str,
runner_filter: RunnerFilter,
report: Report,
checks_results,
reportMutatorData=None) -> Report:
# Moves report generation logic out of run() method in Runner class.
# Allows function overriding of a much smaller function than run() for other "child" frameworks such as Kustomize, Helm
# Where Kubernetes CHECKS are needed, but the specific file references are to another framework for the user output (or a mix of both).
for check, check_results in checks_results.items():
for check_result in check_results:
entity = check_result["entity"]
entity_file_path = entity.get(CustomAttributes.FILE_PATH)
entity_file_abs_path = _get_entity_abs_path(
root_folder, entity_file_path)
entity_id = entity.get(CustomAttributes.ID)
entity_context = self.context[entity_file_path][entity_id]
record = Record(check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_context.get("code_lines"),
file_path=entity_file_path,
file_line_range=[
entity_context.get("start_line"),
entity_context.get("end_line")
],
resource=entity.get(CustomAttributes.ID),
evaluations={},
check_class=check.__class__.__module__,
file_abs_path=entity_file_abs_path)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def mutateKubernetesResults(self,
results,
report,
k8_file=None,
k8_file_path=None,
file_abs_path=None,
entity_conf=None,
variable_evaluations=None,
reportMutatorData=None):
# Moves report generation logic out of run() method in Runner class.
# Allows function overriding of a much smaller function than run() for other "child" frameworks such as Kustomize, Helm
# Where Kubernetes CHECKS are needed, but the specific file references are to another framework for the user output (or a mix of both).
for check, check_result in results.items():
resource_id = get_resource_id(entity_conf)
entity_context = self.context[k8_file][resource_id]
record = Record(check_id=check.id,
bc_check_id=check.bc_id,
check_name=check.name,
check_result=check_result,
code_block=entity_context.get("code_lines"),
file_path=k8_file_path,
file_line_range=[
entity_context.get("start_line"),
entity_context.get("end_line")
],
resource=resource_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def check_definitions(self, root_folder, runner_filter, report):
for k8_file in self.definitions.keys():
# There are a few cases here. If -f was used, there could be a leading / because it's an absolute path,
# or there will be no leading slash; root_folder will always be none.
# If -d is used, root_folder will be the value given, and -f will start with a / (hardcoded above).
# The goal here is simply to get a valid path to the file (which sls_file does not always give).
if k8_file[0] == '/':
path_to_convert = (root_folder +
k8_file) if root_folder else k8_file
else:
path_to_convert = (os.path.join(
root_folder, k8_file)) if root_folder else k8_file
file_abs_path = os.path.abspath(path_to_convert)
# Run for each definition
for entity_conf in self.definitions[k8_file]:
entity_type = entity_conf.get("kind")
# Skip List and Kustomization Templates (for now)
if entity_type == "Kustomization":
continue
skipped_checks = get_skipped_checks(entity_conf)
results = registry.scan(k8_file, entity_conf, skipped_checks,
runner_filter)
# TODO? - Variable Eval Message!
variable_evaluations = {}
for check, check_result in results.items():
resource_id = get_resource_id(entity_conf)
entity_context = self.context[k8_file][resource_id]
record = Record(
check_id=check.id,
bc_check_id=check.bc_id,
check_name=check.name,
check_result=check_result,
code_block=entity_context.get("code_lines"),
file_path=k8_file,
file_line_range=[
entity_context.get("start_line"),
entity_context.get("end_line")
],
resource=resource_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def check_definitions(self, root_folder, runner_filter, report):
for file_abs_path, definition in self.definitions.items():
cf_file = f"/{os.path.relpath(file_abs_path, root_folder)}"
if isinstance(
definition,
dict) and TemplateSections.RESOURCES in definition.keys():
for resource_name, resource in definition[
TemplateSections.RESOURCES].items():
resource_id = ContextParser.extract_cf_resource_id(
resource, resource_name)
# check that the resource can be parsed as a CF resource
if resource_id:
resource_context = self.context[file_abs_path][
TemplateSections.RESOURCES][resource_name]
entity_lines_range = [
resource_context['start_line'],
resource_context['end_line']
]
entity_code_lines = resource_context['code_lines']
if entity_lines_range and entity_code_lines:
# TODO - Variable Eval Message!
variable_evaluations = {}
skipped_checks = ContextParser.collect_skip_comments(
entity_code_lines)
entity = {resource_name: resource}
results = cfn_registry.scan(
cf_file, entity, skipped_checks, runner_filter)
tags = cfn_utils.get_resource_tags(entity)
for check, check_result in results.items():
record = Record(
check_id=check.id,
bc_check_id=check.bc_id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=cf_file,
file_line_range=entity_lines_range,
resource=resource_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path,
entity_tags=tags)
breadcrumb = self.breadcrumbs.get(
record.file_path, {}).get(record.resource)
if breadcrumb:
record = GraphRecord(record, breadcrumb)
record.set_guideline(check.guideline)
report.add_record(record=record)
def get_graph_checks_report(self, root_folder,
runner_filter: RunnerFilter):
report = Report(self.check_type)
checks_results = self.run_graph_checks_results(runner_filter)
for check, check_results in checks_results.items():
for check_result in check_results:
entity = check_result['entity']
entity_context, entity_evaluations = self.get_entity_context_and_evaluations(
entity)
if entity_context:
full_file_path = entity[CustomAttributes.FILE_PATH]
copy_of_check_result = copy.deepcopy(check_result)
for skipped_check in entity_context.get(
'skipped_checks', []):
if skipped_check['id'] == check.id:
copy_of_check_result[
'result'] = CheckResult.SKIPPED
copy_of_check_result[
'suppress_comment'] = skipped_check[
'suppress_comment']
break
copy_of_check_result['entity'] = entity.get(
CustomAttributes.CONFIG)
record = Record(
check_id=check.id,
bc_check_id=check.bc_id,
check_name=check.name,
check_result=copy_of_check_result,
code_block=entity_context.get('code_lines'),
file_path=
f"/{os.path.relpath(full_file_path, root_folder)}",
file_line_range=[
entity_context.get('start_line'),
entity_context.get('end_line')
],
resource=".".join(entity_context['definition_path']),
entity_tags=entity.get('tags', {}),
evaluations=entity_evaluations,
check_class=check.__class__.__module__,
file_abs_path=os.path.abspath(full_file_path),
resource_address=entity_context.get('address'))
if self.breadcrumbs:
breadcrumb = self.breadcrumbs.get(
record.file_path, {}).get(record.resource)
if breadcrumb:
record = GraphRecord(record, breadcrumb)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def run_block(self, entities, full_file_path, report, scanned_file, block_type, runner_filter=None):
registry = self.block_type_registries[block_type]
if registry:
for entity in entities:
context_parser = parser_registry.context_parsers[block_type]
definition_path = context_parser.get_entity_context_path(entity)
entity_id = ".".join(definition_path)
# Entity can exist only once per dir, for file as well
entity_context = self.get_entity_context(definition_path, full_file_path)
entity_lines_range = [entity_context.get('start_line'), entity_context.get('end_line')]
entity_code_lines = entity_context.get('code_lines')
results = registry.scan(scanned_file, entity, [], runner_filter)
for check, check_result in results.items():
record = Record(check_id=check.id, bc_check_id=check.bc_id, check_name=check.name, check_result=check_result,
code_block=entity_code_lines, file_path=scanned_file,
file_line_range=entity_lines_range,
resource=entity_id, evaluations=None,
check_class=check.__class__.__module__, file_abs_path=full_file_path)
record.set_guideline(check.guideline)
report.add_record(record=record)
def mutateKubernetesGraphResults(self, root_folder: str, runner_filter: RunnerFilter, report: Report, checks_results, reportMutatorData=None) -> Report:
# Moves report generation logic out of run() method in Runner class.
# Allows function overriding of a much smaller function than run() for other "child" frameworks such as Kustomize, Helm
# Where Kubernetes CHECKS are needed, but the specific file references are to another framework for the user output (or a mix of both).
kustomizeMetadata = reportMutatorData['kustomizeMetadata'],
kustomizeFileMappings = reportMutatorData['kustomizeFileMappings']
for check, check_results in checks_results.items():
for check_result in check_results:
entity = check_result["entity"]
entity_file_path = entity.get(CustomAttributes.FILE_PATH)
entity_file_abs_path = _get_entity_abs_path(root_folder, entity_file_path)
entity_id = entity.get(CustomAttributes.ID)
entity_context = self.context[entity_file_path][entity_id]
if entity_file_abs_path in kustomizeFileMappings:
realKustomizeEnvMetadata = kustomizeMetadata[0][kustomizeFileMappings[entity_file_abs_path]]
if 'overlay' in realKustomizeEnvMetadata["type"]:
kustomizeResourceID = f'{realKustomizeEnvMetadata["type"]}:{str(realKustomizeEnvMetadata["overlay_name"])}:{entity_id}'
else:
kustomizeResourceID = f'{realKustomizeEnvMetadata["type"]}:{entity_id}'
else:
kustomizeResourceID = "Unknown error. This is a bug."
record = Record(
check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_context.get("code_lines"),
file_path=realKustomizeEnvMetadata['filePath'],
file_line_range=[0,0],
resource=kustomizeResourceID, # entity.get(CustomAttributes.ID),
evaluations={},
check_class=check.__class__.__module__,
file_abs_path=entity_file_abs_path
)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def get_graph_checks_report(self, root_folder: str,
runner_filter: RunnerFilter) -> Report:
report = Report(self.check_type)
checks_results = self.run_graph_checks_results(runner_filter)
for check, check_results in checks_results.items():
for check_result in check_results:
entity = check_result["entity"]
entity_file_abs_path = entity.get(CustomAttributes.FILE_PATH)
entity_file_path = scanned_file = f"/{os.path.relpath(entity_file_abs_path, root_folder)}"
entity_name = entity.get(
CustomAttributes.BLOCK_NAME).split(".")[1]
entity_context = self.context[entity_file_abs_path][
TemplateSections.RESOURCES][entity_name]
record = Record(
check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_context.get("code_lines"),
file_path=entity_file_path,
file_line_range=[
entity_context.get("start_line"),
entity_context.get("end_line")
],
resource=entity.get(CustomAttributes.ID),
evaluations={},
check_class=check.__class__.__module__,
file_abs_path=entity_file_abs_path,
entity_tags={} if not entity.get("Tags") else
cfn_utils.parse_entity_tags(entity.get("Tags")))
record.set_guideline(check.guideline)
if self.breadcrumbs:
breadcrumb = self.breadcrumbs.get(record.file_path,
{}).get(record.resource)
if breadcrumb:
record = GraphRecord(record, breadcrumb)
report.add_record(record=record)
return report
def run(self,
root_folder=None,
external_checks_dir=None,
files=None,
runner_filter=RunnerFilter(),
collect_skip_comments=True):
report = Report(self.check_type)
files_list = []
filepath_fn = None
if external_checks_dir:
for directory in external_checks_dir:
registry.load_external_checks(directory)
if files:
files_list = [
file for file in files
if Runner._is_docker_file(os.path.basename(file))
]
if root_folder:
filepath_fn = lambda f: f'/{os.path.relpath(f, os.path.commonprefix((root_folder, f)))}'
for root, d_names, f_names in os.walk(root_folder):
filter_ignored_paths(root, d_names,
runner_filter.excluded_paths)
filter_ignored_paths(root, f_names,
runner_filter.excluded_paths)
for file in f_names:
if Runner._is_docker_file(file):
file_path = os.path.join(root, file)
files_list.append(file_path)
definitions, definitions_raw = get_files_definitions(
files_list, filepath_fn)
for docker_file_path in definitions.keys():
# There are a few cases here. If -f was used, there could be a leading / because it's an absolute path,
# or there will be no leading slash; root_folder will always be none.
# If -d is used, root_folder will be the value given, and -f will start with a / (hardcoded above).
# The goal here is simply to get a valid path to the file (which docker_file_path does not always give).
if docker_file_path[0] == '/':
path_to_convert = (
root_folder +
docker_file_path) if root_folder else docker_file_path
else:
path_to_convert = (os.path.join(
root_folder,
docker_file_path)) if root_folder else docker_file_path
file_abs_path = os.path.abspath(path_to_convert)
report.add_resource(file_abs_path)
skipped_checks = collect_skipped_checks(
definitions[docker_file_path])
instructions = definitions[docker_file_path]
results = registry.scan(docker_file_path, instructions,
skipped_checks, runner_filter)
for check, check_result in results.items():
result_configuration = check_result['results_configuration']
startline = 0
endline = len(definitions_raw[docker_file_path]) - 1
result_instruction = ""
if result_configuration:
startline = result_configuration['startline']
endline = result_configuration['endline']
result_instruction = result_configuration["instruction"]
codeblock = []
self.calc_record_codeblock(codeblock, definitions_raw,
docker_file_path, endline,
startline)
record = Record(check_id=check.id,
bc_check_id=check.bc_id,
check_name=check.name,
check_result=check_result,
code_block=codeblock,
file_path=docker_file_path,
file_line_range=[startline + 1, endline + 1],
resource="{}.{}".format(
docker_file_path, result_instruction,
startline),
evaluations=None,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path,
entity_tags=None)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def run(
self,
root_folder: str,
external_checks_dir: Optional[List[str]] = None,
files: Optional[List[str]] = None,
runner_filter: RunnerFilter = RunnerFilter(),
collect_skip_comments: bool = True,
) -> Report:
report = Report(self.check_type)
files_list = []
filepath_fn = None
if external_checks_dir:
for directory in external_checks_dir:
arm_resource_registry.load_external_checks(directory)
if files:
files_list = files.copy()
if root_folder:
filepath_fn = lambda f: f'/{os.path.relpath(f, os.path.commonprefix((root_folder, f)))}'
for root, d_names, f_names in os.walk(root_folder):
filter_ignored_paths(root, d_names,
runner_filter.excluded_paths)
filter_ignored_paths(root, f_names,
runner_filter.excluded_paths)
for file in f_names:
file_ending = os.path.splitext(file)[1]
if file_ending in ARM_POSSIBLE_ENDINGS:
files_list.append(os.path.join(root, file))
definitions, definitions_raw = get_files_definitions(
files_list, filepath_fn)
# Filter out empty files that have not been parsed successfully, and filter out non-CF template files
definitions = {
k: v
for k, v in definitions.items()
if v and v.__contains__("resources")
}
definitions_raw = {
k: v
for k, v in definitions_raw.items() if k in definitions.keys()
}
for arm_file in definitions.keys():
# There are a few cases here. If -f was used, there could be a leading / because it's an absolute path,
# or there will be no leading slash; root_folder will always be none.
# If -d is used, root_folder will be the value given, and -f will start with a / (hardcoded above).
# The goal here is simply to get a valid path to the file (which arm_file does not always give).
if arm_file[0] == '/':
path_to_convert = (root_folder +
arm_file) if root_folder else arm_file
else:
path_to_convert = (os.path.join(
root_folder, arm_file)) if root_folder else arm_file
file_abs_path = os.path.abspath(path_to_convert)
if isinstance(definitions[arm_file], DictNode):
arm_context_parser = ContextParser(arm_file,
definitions[arm_file],
definitions_raw[arm_file])
logging.debug(
f"Template Dump for {arm_file}: {definitions[arm_file]}")
if 'resources' in definitions[arm_file].keys():
arm_context_parser.evaluate_default_parameters()
# Split out nested resources from base resource
for resource in definitions[arm_file]['resources']:
if isinstance(
resource,
dict) and "parent_name" in resource.keys():
continue
nested_resources = []
nested_resources = arm_context_parser.search_deep_keys(
"resources", resource, [])
if nested_resources:
for nr in nested_resources:
nr_element = nr.pop()
if nr_element:
for element in nr_element:
new_resource = {}
new_resource = element
if isinstance(new_resource, dict):
new_resource[
"parent_name"] = resource[
"name"]
new_resource[
"parent_type"] = resource[
"type"]
definitions[arm_file][
'resources'].append(
new_resource)
for resource in definitions[arm_file]['resources']:
resource_id = arm_context_parser.extract_arm_resource_id(
resource)
report.add_resource(f'{arm_file}:{resource_id}')
resource_name = arm_context_parser.extract_arm_resource_name(
resource)
entity_lines_range, entity_code_lines = arm_context_parser.extract_arm_resource_code_lines(
resource)
if entity_lines_range and entity_code_lines:
# TODO - Variable Eval Message!
variable_evaluations = {}
skipped_checks = ContextParser.collect_skip_comments(
resource)
results = arm_resource_registry.scan(
arm_file, {resource_name: resource},
skipped_checks, runner_filter)
for check, check_result in results.items():
record = Record(
check_id=check.id,
bc_check_id=check.bc_id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=arm_file,
file_line_range=entity_lines_range,
resource=resource_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path)
record.set_guideline(check.guideline)
report.add_record(record=record)
if 'parameters' in definitions[arm_file].keys():
parameters = definitions[arm_file]['parameters']
for parameter_name, parameter_details in parameters.items(
):
# TODO - Variable Eval Message!
variable_evaluations = {}
resource_id = f'parameter.{parameter_name}'
resource_name = parameter_name
entity_lines_range, entity_code_lines = arm_context_parser.extract_arm_resource_code_lines(
parameter_details)
if entity_lines_range and entity_code_lines:
skipped_checks = ContextParser.collect_skip_comments(
parameter_details)
results = arm_parameter_registry.scan(
arm_file, {resource_name: parameter_details},
skipped_checks, runner_filter)
for check, check_result in results.items():
record = Record(
check_id=check.id,
bc_check_id=check.bc_id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=arm_file,
file_line_range=entity_lines_range,
resource=resource_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def run_block(self, entities,
definition_context,
full_file_path, root_folder, report, scanned_file,
block_type, runner_filter=None, entity_context_path_header=None,
module_referrer: Optional[str] = None):
registry = self.block_type_registries[block_type]
if not registry:
return
for entity in entities:
entity_evaluations = None
context_parser = parser_registry.context_parsers[block_type]
definition_path = context_parser.get_entity_context_path(entity)
entity_id = ".".join(definition_path) # example: aws_s3_bucket.my_bucket
caller_file_path = None
caller_file_line_range = None
if module_referrer is not None:
referrer_id = self._find_id_for_referrer(full_file_path,
self.definitions)
if referrer_id:
entity_id = f"{referrer_id}.{entity_id}" # ex: module.my_module.aws_s3_bucket.my_bucket
abs_caller_file = module_referrer[:module_referrer.rindex("#")]
caller_file_path = f"/{os.path.relpath(abs_caller_file, root_folder)}"
try:
caller_context = definition_context[abs_caller_file]
# HACK ALERT: module data is currently double-nested in
# definition context. If fixed, remove the
# addition of "module." at the beginning.
for part in f"module.{referrer_id}".split("."):
caller_context = caller_context[part]
except KeyError:
logging.debug("Unable to find caller context for: %s", abs_caller_file)
caller_context = None
if caller_context:
caller_file_line_range = [caller_context.get('start_line'), caller_context.get('end_line')]
else:
logging.debug(f"Unable to find referrer ID for full path: {full_file_path}")
if entity_context_path_header is None:
entity_context_path = [block_type] + definition_path
else:
entity_context_path = entity_context_path_header + block_type + definition_path
# Entity can exist only once per dir, for file as well
try:
entity_context = data_structures_utils.get_inner_dict(definition_context[full_file_path], entity_context_path)
entity_lines_range = [entity_context.get('start_line'), entity_context.get('end_line')]
entity_code_lines = entity_context.get('code_lines')
skipped_checks = entity_context.get('skipped_checks')
except KeyError:
# TODO: Context info isn't working for modules
entity_lines_range = None
entity_code_lines = None
skipped_checks = None
if block_type == "module":
self.push_skipped_checks_down(self, definition_context, full_file_path, skipped_checks)
if full_file_path in self.evaluations_context:
variables_evaluations = {}
for var_name, context_info in self.evaluations_context.get(full_file_path, {}).items():
variables_evaluations[var_name] = dataclasses.asdict(context_info)
entity_evaluations = BaseVariableEvaluation.reduce_entity_evaluations(variables_evaluations,
entity_context_path)
results = registry.scan(scanned_file, entity, skipped_checks, runner_filter)
absolut_scanned_file_path, _ = self._strip_module_referrer(file_path=full_file_path)
# This duplicates a call at the start of scan, but adding this here seems better than kludging with some tuple return type
(entity_type, entity_name, entity_config) = registry.extract_entity_details(entity)
tags = get_resource_tags(entity_type, entity_config)
for check, check_result in results.items():
record = Record(check_id=check.id, bc_check_id=check.bc_id, check_name=check.name, check_result=check_result,
code_block=entity_code_lines, file_path=scanned_file,
file_line_range=entity_lines_range,
resource=entity_id, evaluations=entity_evaluations,
check_class=check.__class__.__module__, file_abs_path=absolut_scanned_file_path,
entity_tags=tags,
caller_file_path=caller_file_path,
caller_file_line_range=caller_file_line_range)
breadcrumb = self.breadcrumbs.get(record.file_path, {}).get('.'.join([entity_type, entity_name]))
if breadcrumb:
record = GraphRecord(record, breadcrumb)
record.set_guideline(check.guideline)
report.add_record(record=record)
def run(self,
root_folder,
external_checks_dir=None,
files=None,
runner_filter=RunnerFilter(),
collect_skip_comments=True,
helmChart=None):
report = Report(self.check_type)
if external_checks_dir:
for directory in external_checks_dir:
registry.load_external_checks(directory)
self.graph_registry.load_external_checks(directory)
if files:
definitions, self.definitions_raw = get_files_definitions(files)
elif root_folder:
definitions, self.definitions_raw = get_folder_definitions(
root_folder, runner_filter.excluded_paths)
else:
return report
# TODO: uncomment it in order to build the graph
# logging.info("creating kubernetes graph")
# local_graph = self.graph_manager.build_graph_from_definitions(definitions)
# for vertex in local_graph.vertices:
# report.add_resource(f'{vertex.path}:{vertex.id}')
# self.graph_manager.save_graph(local_graph)
for k8_file in definitions.keys():
# There are a few cases here. If -f was used, there could be a leading / because it's an absolute path,
# or there will be no leading slash; root_folder will always be none.
# If -d is used, root_folder will be the value given, and -f will start with a / (hardcoded above).
# The goal here is simply to get a valid path to the file (which sls_file does not always give).
if k8_file[0] == '/':
path_to_convert = (root_folder +
k8_file) if root_folder else k8_file
else:
path_to_convert = (os.path.join(
root_folder, k8_file)) if root_folder else k8_file
file_abs_path = os.path.abspath(path_to_convert)
if definitions[k8_file]:
for i in range(len(definitions[k8_file])):
if (not 'apiVersion' in definitions[k8_file][i].keys()
) and (not 'kind' in definitions[k8_file][i].keys()):
continue
logging.debug("Template Dump for {}: {}".format(
k8_file, definitions[k8_file][i], indent=2))
entity_conf = definitions[k8_file][i]
if entity_conf is None:
continue
# Split out resources if entity kind is List
if isinstance(entity_conf,
dict) and entity_conf["kind"] == "List":
for item in entity_conf.get("items", []):
definitions[k8_file].append(item)
for i in range(len(definitions[k8_file])):
if _is_invalid_k8_definition(definitions[k8_file][i]):
continue
logging.debug("Template Dump for {}: {}".format(
k8_file, definitions[k8_file][i], indent=2))
entity_conf = definitions[k8_file][i]
if isinstance(entity_conf,
dict) and entity_conf.get("kind") == "List":
continue
# Skip entity without metadata["name"]
if isinstance(entity_conf,
dict) and entity_conf.get("metadata"):
if isinstance(
entity_conf["metadata"],
int) or "name" not in entity_conf["metadata"]:
continue
else:
continue
# Skip entity with parent (metadata["ownerReferences"]) in runtime
# We will alert in runtime only
if "ownerReferences" in entity_conf["metadata"] and \
entity_conf["metadata"]["ownerReferences"] is not None:
continue
# Append containers and initContainers to definitions list
for type in ["containers", "initContainers"]:
containers = []
if entity_conf["kind"] == "CustomResourceDefinition":
continue
containers = search_deep_keys(type, entity_conf, [])
if not containers:
continue
containers = containers.pop()
#containers.insert(0,entity_conf['kind'])
containerDef = {}
if "namespace" in entity_conf["metadata"]:
namespace = entity_conf["metadata"]["namespace"]
else:
namespace = "default"
containerDef["containers"] = containers.pop()
if containerDef["containers"] is not None:
containerDef["containers"] = force_list(
containerDef["containers"])
for cd in containerDef["containers"]:
i = containerDef["containers"].index(cd)
containerDef["containers"][i][
"apiVersion"] = entity_conf["apiVersion"]
containerDef["containers"][i]["kind"] = type
containerDef["containers"][i][
"parent"] = "{}.{}.{} (container {})".format(
entity_conf["kind"],
entity_conf["metadata"]["name"],
namespace, str(i))
containerDef["containers"][i][
"parent_metadata"] = entity_conf[
"metadata"]
definitions[k8_file].extend(
containerDef["containers"])
# Run for each definition included added container definitions
for i in range(len(definitions[k8_file])):
if _is_invalid_k8_definition(definitions[k8_file][i]):
continue
logging.debug("Template Dump for {}: {}".format(
k8_file, definitions[k8_file][i], indent=2))
entity_conf = definitions[k8_file][i]
if entity_conf is None:
continue
if isinstance(entity_conf,
dict) and (entity_conf["kind"] == "List"
or not entity_conf.get("kind")):
continue
if isinstance(entity_conf, dict) and isinstance(
entity_conf.get("kind"), int):
continue
# Skip entity without metadata["name"] or parent_metadata["name"]
if not any(x in entity_conf["kind"]
for x in ["containers", "initContainers"]):
if entity_conf.get("metadata"):
if isinstance(
entity_conf["metadata"], int
) or not "name" in entity_conf["metadata"]:
continue
else:
continue
# Skip entity with parent (metadata["ownerReferences"]) in runtime
# We will alert in runtime only
if "metadata" in entity_conf:
if "ownerReferences" in entity_conf["metadata"] and \
entity_conf["metadata"]["ownerReferences"] is not None:
continue
# Skip Kustomization Templates (for now)
if entity_conf["kind"] == "Kustomization":
continue
skipped_checks = get_skipped_checks(entity_conf)
results = registry.scan(k8_file, entity_conf,
skipped_checks, runner_filter)
start_line = entity_conf["__startline__"]
end_line = entity_conf["__endline__"]
if start_line == end_line:
entity_lines_range = [start_line, end_line]
entity_code_lines = self.definitions_raw[k8_file][
start_line - 1:end_line]
else:
entity_lines_range = [start_line, end_line - 1]
entity_code_lines = self.definitions_raw[k8_file][
start_line - 1:end_line - 1]
# TODO? - Variable Eval Message!
variable_evaluations = {}
for check, check_result in results.items():
resource_id = check.get_resource_id(entity_conf)
report.add_resource(f'{k8_file}:{resource_id}')
record = Record(check_id=check.id,
bc_check_id=check.bc_id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=k8_file,
file_line_range=entity_lines_range,
resource=resource_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def run(self, root_folder, external_checks_dir=None, files=None, runner_filter=RunnerFilter(), collect_skip_comments=True):
report = Report(self.check_type)
files_list = []
filepath_fn = None
if external_checks_dir:
for directory in external_checks_dir:
function_registry.load_external_checks(directory)
if files:
files_list = [file for file in files if os.path.basename(file) in SLS_FILE_MASK]
if root_folder:
filepath_fn = lambda f: f'/{os.path.relpath(f, os.path.commonprefix((root_folder, f)))}'
for root, d_names, f_names in os.walk(root_folder):
# Don't walk in to "node_modules" directories under the root folder. If –for some reason–
# scanning one of these is desired, it can be directly specified.
if "node_modules" in d_names:
d_names.remove("node_modules")
filter_ignored_paths(root, d_names, runner_filter.excluded_paths)
filter_ignored_paths(root, f_names, runner_filter.excluded_paths)
for file in f_names:
if file in SLS_FILE_MASK:
full_path = os.path.join(root, file)
if "/." not in full_path:
# skip temp directories
files_list.append(full_path)
definitions, definitions_raw = get_files_definitions(files_list, filepath_fn)
# Filter out empty files that have not been parsed successfully
definitions = {k: v for k, v in definitions.items() if v}
definitions_raw = {k: v for k, v in definitions_raw.items() if k in definitions.keys()}
for sls_file, sls_file_data in definitions.items():
# There are a few cases here. If -f was used, there could be a leading / because it's an absolute path,
# or there will be no leading slash; root_folder will always be none.
# If -d is used, root_folder will be the value given, and -f will start with a / (hardcoded above).
# The goal here is simply to get a valid path to the file (which sls_file does not always give).
if sls_file[0] == '/':
path_to_convert = (root_folder + sls_file) if root_folder else sls_file
else:
path_to_convert = (os.path.join(root_folder, sls_file)) if root_folder else sls_file
file_abs_path = os.path.abspath(path_to_convert)
if not isinstance(sls_file_data, DictNode):
continue
if CFN_RESOURCES_TOKEN in sls_file_data and isinstance(sls_file_data[CFN_RESOURCES_TOKEN], DictNode):
cf_sub_template = sls_file_data[CFN_RESOURCES_TOKEN]
if not cf_sub_template.get('Resources'):
continue
cf_context_parser = CfnContextParser(sls_file, cf_sub_template, definitions_raw[sls_file])
logging.debug("Template Dump for {}: {}".format(sls_file, sls_file_data, indent=2))
cf_context_parser.evaluate_default_refs()
for resource_name, resource in cf_sub_template['Resources'].items():
if not isinstance(resource, DictNode):
continue
cf_resource_id = cf_context_parser.extract_cf_resource_id(resource, resource_name)
if not cf_resource_id:
# Not Type attribute for resource
continue
report.add_resource(f'{file_abs_path}:{cf_resource_id}')
entity_lines_range, entity_code_lines = cf_context_parser.extract_cf_resource_code_lines(
resource)
if entity_lines_range and entity_code_lines:
skipped_checks = CfnContextParser.collect_skip_comments(entity_code_lines)
# TODO - Variable Eval Message!
variable_evaluations = {}
entity = {resource_name: resource}
results = cfn_registry.scan(sls_file, entity, skipped_checks, runner_filter)
tags = cfn_utils.get_resource_tags(entity, cfn_registry)
for check, check_result in results.items():
record = Record(check_id=check.id, bc_check_id=check.bc_id, check_name=check.name, check_result=check_result,
code_block=entity_code_lines, file_path=sls_file,
file_line_range=entity_lines_range,
resource=cf_resource_id, evaluations=variable_evaluations,
check_class=check.__class__.__module__, file_abs_path=file_abs_path,
entity_tags=tags)
record.set_guideline(check.guideline)
report.add_record(record=record)
sls_context_parser = SlsContextParser(sls_file, sls_file_data, definitions_raw[sls_file])
# Sub-sections that have multiple items under them
for token, registry in MULTI_ITEM_SECTIONS:
template_items = sls_file_data.get(token)
if not template_items or not isinstance(template_items, dict):
continue
for item_name, item_content in template_items.items():
if not isinstance(item_content, DictNode):
continue
entity_lines_range, entity_code_lines = sls_context_parser.extract_code_lines(item_content)
if entity_lines_range and entity_code_lines:
skipped_checks = CfnContextParser.collect_skip_comments(entity_code_lines)
variable_evaluations = {}
if token == "functions": #nosec
# "Enriching" copies things like "environment" and "stackTags" down into the
# function data from the provider block since logically that's what serverless
# does. This allows checks to see what the complete data would be.
sls_context_parser.enrich_function_with_provider(item_name)
entity = EntityDetails(sls_context_parser.provider_type, item_content)
results = registry.scan(sls_file, entity, skipped_checks, runner_filter)
tags = cfn_utils.get_resource_tags(entity, registry)
for check, check_result in results.items():
record = Record(check_id=check.id, check_name=check.name, check_result=check_result,
code_block=entity_code_lines, file_path=sls_file,
file_line_range=entity_lines_range,
resource=item_name, evaluations=variable_evaluations,
check_class=check.__class__.__module__, file_abs_path=file_abs_path,
entity_tags=tags)
record.set_guideline(check.guideline)
report.add_record(record=record)
# Sub-sections that are a single item
for token, registry in SINGLE_ITEM_SECTIONS:
item_content = sls_file_data.get(token)
if not item_content:
continue
entity_lines_range, entity_code_lines = sls_context_parser.extract_code_lines(item_content)
if not entity_lines_range:
entity_lines_range, entity_code_lines = sls_context_parser.extract_code_lines(sls_file_data)
skipped_checks = CfnContextParser.collect_skip_comments(entity_code_lines)
variable_evaluations = {}
entity = EntityDetails(sls_context_parser.provider_type, item_content)
results = registry.scan(sls_file, entity, skipped_checks, runner_filter)
tags = cfn_utils.get_resource_tags(entity, registry)
for check, check_result in results.items():
record = Record(check_id=check.id, check_name=check.name, check_result=check_result,
code_block=entity_code_lines, file_path=sls_file,
file_line_range=entity_lines_range,
resource=token, evaluations=variable_evaluations,
check_class=check.__class__.__module__, file_abs_path=file_abs_path,
entity_tags=tags)
record.set_guideline(check.guideline)
report.add_record(record=record)
# "Complete" checks
# NOTE: Ignore code content, no point in showing (could be long)
entity_lines_range, entity_code_lines = sls_context_parser.extract_code_lines(sls_file_data)
if entity_lines_range:
skipped_checks = CfnContextParser.collect_skip_comments(entity_code_lines)
variable_evaluations = {}
entity = EntityDetails(sls_context_parser.provider_type, sls_file_data)
results = complete_registry.scan(sls_file, entity, skipped_checks, runner_filter)
tags = cfn_utils.get_resource_tags(entity, complete_registry)
for check, check_result in results.items():
record = Record(check_id=check.id, check_name=check.name, check_result=check_result,
code_block=[], # Don't show, could be large
file_path=sls_file,
file_line_range=entity_lines_range,
resource="complete", # Weird, not sure what to put where
evaluations=variable_evaluations,
check_class=check.__class__.__module__, file_abs_path=file_abs_path,
entity_tags=tags)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
