def mutateKubernetesResults(self,
results,
report,
k8_file=None,
k8_file_path=None,
file_abs_path=None,
entity_conf=None,
variable_evaluations=None,
reportMutatorData=None):
# Moves report generation logic out of run() method in Runner class.
# Allows function overriding of a much smaller function than run() for other "child" frameworks such as Kustomize, Helm
# Where Kubernetes CHECKS are needed, but the specific file references are to another framework for the user output (or a mix of both).
for check, check_result in results.items():
resource_id = get_resource_id(entity_conf)
entity_context = self.context[k8_file][resource_id]
record = Record(check_id=check.id,
bc_check_id=check.bc_id,
check_name=check.name,
check_result=check_result,
code_block=entity_context.get("code_lines"),
file_path=k8_file_path,
file_line_range=[
entity_context.get("start_line"),
entity_context.get("end_line")
],
resource=resource_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def get_graph_checks_report(self, root_folder: str,
runner_filter: RunnerFilter) -> Report:
report = Report(self.check_type)
checks_results = self.run_graph_checks_results(runner_filter)
for check, check_results in checks_results.items():
for check_result in check_results:
entity = check_result["entity"]
entity_file_abs_path = entity.get(CustomAttributes.FILE_PATH)
entity_file_path = f"/{os.path.relpath(entity_file_abs_path, root_folder)}"
start_line = entity['__startline__']
end_line = entity['__endline__']
if start_line == end_line:
entity_lines_range = [start_line, end_line]
entity_code_lines = self.definitions_raw[entity_file_path][
start_line - 1:end_line]
else:
entity_lines_range = [start_line, end_line - 1]
entity_code_lines = self.definitions_raw[entity_file_path][
start_line - 1:end_line - 1]
record = Record(check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=entity_file_path,
file_line_range=entity_lines_range,
resource=entity.get(CustomAttributes.ID),
evaluations={},
check_class=check.__class__.__module__,
file_abs_path=entity_file_abs_path)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def test_policy_suppression(self):
instance = BcPlatformIntegration()
suppressions_integration = SuppressionsIntegration(instance)
suppression = {
"suppressionType": "Policy",
"id": "7caab873-7400-47f9-8b3f-82b33d0463ed",
"policyId": "BC_AWS_GENERAL_31",
"comment": "No justification comment provided.",
"checkovPolicyId": "CKV_AWS_79",
}
record1 = Record(check_id='CKV_AWS_79', check_name=None, check_result=None,
code_block=None, file_path=None,
file_line_range=None,
resource=None, evaluations=None,
check_class=None, file_abs_path='.', entity_tags=None)
record2 = Record(check_id='CKV_AWS_1', check_name=None, check_result=None,
code_block=None, file_path=None,
file_line_range=None,
resource=None, evaluations=None,
check_class=None, file_abs_path='.', entity_tags=None)
self.assertTrue(suppressions_integration._check_suppression(record1, suppression))
self.assertFalse(suppressions_integration._check_suppression(record2, suppression))
def mutateKubernetesResults(self, results, report, k8_file=None, k8_file_path=None, file_abs_path=None, entity_conf=None, variable_evaluations=None, reportMutatorData=None):
# Moves report generation logic out of checkov.kubernetes.runner.run() def.
# Allows us to overriding report file information for "child" frameworks such as Kustomize, Helm
# Where Kubernetes CHECKS are needed, but the specific file references are to another framework for the user output (or a mix of both).
kustomizeMetadata = reportMutatorData['kustomizeMetadata'],
kustomizeFileMappings = reportMutatorData['kustomizeFileMappings']
for check, check_result in results.items():
resource_id = get_resource_id(entity_conf)
entity_context = self.context[k8_file][resource_id]
if file_abs_path in kustomizeFileMappings:
realKustomizeEnvMetadata = kustomizeMetadata[0][kustomizeFileMappings[file_abs_path]]
if 'overlay' in realKustomizeEnvMetadata["type"]:
kustomizeResourceID = f'{realKustomizeEnvMetadata["type"]}:{str(realKustomizeEnvMetadata["overlay_name"])}:{resource_id}'
else:
kustomizeResourceID = f'{realKustomizeEnvMetadata["type"]}:{resource_id}'
else:
kustomizeResourceID = "Unknown error. This is a bug."
record = Record(
check_id=check.id, bc_check_id=check.bc_id, check_name=check.name,
check_result=check_result, code_block=entity_context.get("code_lines"), file_path=realKustomizeEnvMetadata['filePath'],
file_line_range=[0,0],
resource=kustomizeResourceID, evaluations=variable_evaluations,
check_class=check.__class__.__module__, file_abs_path=realKustomizeEnvMetadata['filePath'])
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def get_graph_checks_report(self, root_folder: str,
runner_filter: RunnerFilter) -> Report:
report = Report(self.check_type)
checks_results = self.run_graph_checks_results(runner_filter)
for check, check_results in checks_results.items():
for check_result in check_results:
entity = check_result["entity"]
entity_file_path = entity.get(CustomAttributes.FILE_PATH)
entity_file_abs_path = _get_entity_abs_path(
root_folder, entity_file_path)
entity_id = entity.get(CustomAttributes.ID)
entity_context = self.context[entity_file_path][entity_id]
record = Record(check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_context.get("code_lines"),
file_path=entity_file_path,
file_line_range=[
entity_context.get("start_line"),
entity_context.get("end_line")
],
resource=entity.get(CustomAttributes.ID),
evaluations={},
check_class=check.__class__.__module__,
file_abs_path=entity_file_abs_path)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def test_valid_passing_valid_testcases(self):
record1 = Record(check_id='CKV_AWS_21',
check_name="Some Check", check_result={"result": CheckResult.FAILED},
code_block=None, file_path="./s3.tf",
file_line_range='1:3',
resource='aws_s3_bucket.operations', evaluations=None,
check_class=None, file_abs_path=',.',
entity_tags={
'tag1': 'value1'
})
record2 = Record(check_id='CKV_AWS_3',
check_name="Ensure all data stored in the EBS is securely encrypted",
check_result={"result": CheckResult.FAILED},
code_block=None, file_path="./ec2.tf",
file_line_range='1:3',
resource='aws_ebs_volume.web_host_storage', evaluations=None,
check_class=None, file_abs_path=',.',
entity_tags={
'tag1': 'value1'
})
r = Report("terraform")
r.add_record(record=record1)
r.add_record(record=record2)
ts = r.get_test_suites()
xml_string = r.get_junit_xml_string(ts)
root = ET.fromstring(xml_string)
self.assertEqual(root.attrib['errors'], '0')
def mutateKubernetesGraphResults(self,
root_folder: str,
runner_filter: RunnerFilter,
report: Report,
checks_results,
reportMutatorData=None) -> Report:
# Moves report generation logic out of run() method in Runner class.
# Allows function overriding of a much smaller function than run() for other "child" frameworks such as Kustomize, Helm
# Where Kubernetes CHECKS are needed, but the specific file references are to another framework for the user output (or a mix of both).
for check, check_results in checks_results.items():
for check_result in check_results:
entity = check_result["entity"]
entity_file_path = entity.get(CustomAttributes.FILE_PATH)
entity_file_abs_path = _get_entity_abs_path(
root_folder, entity_file_path)
entity_id = entity.get(CustomAttributes.ID)
entity_context = self.context[entity_file_path][entity_id]
record = Record(check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_context.get("code_lines"),
file_path=entity_file_path,
file_line_range=[
entity_context.get("start_line"),
entity_context.get("end_line")
],
resource=entity.get(CustomAttributes.ID),
evaluations={},
check_class=check.__class__.__module__,
file_abs_path=entity_file_abs_path)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def test_account_suppression(self):
instance = BcPlatformIntegration()
instance.repo_id = 'org/repo'
suppressions_integration = SuppressionsIntegration(instance)
suppression = {
"suppressionType": "Accounts",
"policyId": "BC_AWS_S3_13",
"comment": "testing checkov",
"accountIds": ["org/repo", "not/valid"],
"checkovPolicyId": "CKV_AWS_18",
}
record1 = Record(check_id='CKV_AWS_18', check_name=None, check_result=None,
code_block=None, file_path=None,
file_line_range=None,
resource=None, evaluations=None,
check_class=None, file_abs_path='.', entity_tags=None)
record2 = Record(check_id='CKV_AWS_1', check_name=None, check_result=None,
code_block=None, file_path=None,
file_line_range=None,
resource=None, evaluations=None,
check_class=None, file_abs_path='.', entity_tags=None)
self.assertTrue(suppressions_integration._check_suppression(record1, suppression))
self.assertFalse(suppressions_integration._check_suppression(record2, suppression))
def check_definitions(self, root_folder, runner_filter, report):
for k8_file in self.definitions.keys():
# There are a few cases here. If -f was used, there could be a leading / because it's an absolute path,
# or there will be no leading slash; root_folder will always be none.
# If -d is used, root_folder will be the value given, and -f will start with a / (hardcoded above).
# The goal here is simply to get a valid path to the file (which sls_file does not always give).
if k8_file[0] == '/':
path_to_convert = (root_folder +
k8_file) if root_folder else k8_file
else:
path_to_convert = (os.path.join(
root_folder, k8_file)) if root_folder else k8_file
file_abs_path = os.path.abspath(path_to_convert)
# Run for each definition
for entity_conf in self.definitions[k8_file]:
entity_type = entity_conf.get("kind")
# Skip List and Kustomization Templates (for now)
if entity_type == "Kustomization":
continue
skipped_checks = get_skipped_checks(entity_conf)
results = registry.scan(k8_file, entity_conf, skipped_checks,
runner_filter)
# TODO? - Variable Eval Message!
variable_evaluations = {}
for check, check_result in results.items():
resource_id = get_resource_id(entity_conf)
entity_context = self.context[k8_file][resource_id]
record = Record(
check_id=check.id,
bc_check_id=check.bc_id,
check_name=check.name,
check_result=check_result,
code_block=entity_context.get("code_lines"),
file_path=k8_file,
file_line_range=[
entity_context.get("start_line"),
entity_context.get("end_line")
],
resource=resource_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def check_definitions(self, root_folder, runner_filter, report):
for file_abs_path, definition in self.definitions.items():
cf_file = f"/{os.path.relpath(file_abs_path, root_folder)}"
if isinstance(
definition,
dict) and TemplateSections.RESOURCES in definition.keys():
for resource_name, resource in definition[
TemplateSections.RESOURCES].items():
resource_id = ContextParser.extract_cf_resource_id(
resource, resource_name)
# check that the resource can be parsed as a CF resource
if resource_id:
resource_context = self.context[file_abs_path][
TemplateSections.RESOURCES][resource_name]
entity_lines_range = [
resource_context['start_line'],
resource_context['end_line']
]
entity_code_lines = resource_context['code_lines']
if entity_lines_range and entity_code_lines:
# TODO - Variable Eval Message!
variable_evaluations = {}
skipped_checks = ContextParser.collect_skip_comments(
entity_code_lines)
entity = {resource_name: resource}
results = cfn_registry.scan(
cf_file, entity, skipped_checks, runner_filter)
tags = cfn_utils.get_resource_tags(entity)
for check, check_result in results.items():
record = Record(
check_id=check.id,
bc_check_id=check.bc_id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=cf_file,
file_line_range=entity_lines_range,
resource=resource_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path,
entity_tags=tags)
breadcrumb = self.breadcrumbs.get(
record.file_path, {}).get(record.resource)
if breadcrumb:
record = GraphRecord(record, breadcrumb)
record.set_guideline(check.guideline)
report.add_record(record=record)
def get_graph_checks_report(self, root_folder,
runner_filter: RunnerFilter):
report = Report(self.check_type)
checks_results = self.run_graph_checks_results(runner_filter)
for check, check_results in checks_results.items():
for check_result in check_results:
entity = check_result['entity']
entity_context, entity_evaluations = self.get_entity_context_and_evaluations(
entity)
if entity_context:
full_file_path = entity[CustomAttributes.FILE_PATH]
copy_of_check_result = copy.deepcopy(check_result)
for skipped_check in entity_context.get(
'skipped_checks', []):
if skipped_check['id'] == check.id:
copy_of_check_result[
'result'] = CheckResult.SKIPPED
copy_of_check_result[
'suppress_comment'] = skipped_check[
'suppress_comment']
break
copy_of_check_result['entity'] = entity.get(
CustomAttributes.CONFIG)
record = Record(
check_id=check.id,
bc_check_id=check.bc_id,
check_name=check.name,
check_result=copy_of_check_result,
code_block=entity_context.get('code_lines'),
file_path=
f"/{os.path.relpath(full_file_path, root_folder)}",
file_line_range=[
entity_context.get('start_line'),
entity_context.get('end_line')
],
resource=".".join(entity_context['definition_path']),
entity_tags=entity.get('tags', {}),
evaluations=entity_evaluations,
check_class=check.__class__.__module__,
file_abs_path=os.path.abspath(full_file_path),
resource_address=entity_context.get('address'))
if self.breadcrumbs:
breadcrumb = self.breadcrumbs.get(
record.file_path, {}).get(record.resource)
if breadcrumb:
record = GraphRecord(record, breadcrumb)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def test_post_runner_with_cloned_checks(self):
instance = BcPlatformIntegration()
instance.skip_policy_download = False
instance.platform_integration_configured = True
custom_policies_integration = CustomPoliciesIntegration(instance)
# mock _get_policies_from_platform method
custom_policies_integration._get_policies_from_platform = types.MethodType(_get_policies_from_platform,
custom_policies_integration)
custom_policies_integration.pre_scan()
scan_reports = Report("terraform")
record = Record(
check_id="CKV_AWS_5",
check_name="Ensure all data stored in the Elasticsearch is securely encrypted at rest",
check_result={"result": CheckResult.FAILED},
code_block=[],
file_path="./main.tf",
file_line_range=[7, 10],
resource="aws_elasticsearch_domain.enabled",
evaluations=None,
check_class='',
file_abs_path=",.",
entity_tags={"tag1": "value1"},
bc_check_id="BC_AWS_ELASTICSEARCH_3"
)
scan_reports.failed_checks.append(record)
custom_policies_integration.post_runner(scan_reports)
self.assertEqual(2, len(scan_reports.failed_checks))
self.assertEqual('mikepolicies_cloned_AWS_1625063607541', scan_reports.failed_checks[1].check_id)
def run_block(self, entities, definition_context, full_file_path, report,
scanned_file, block_type):
for entity in entities:
entity_type = list(entity.keys())[0]
entity_name = list(list(entity.values())[0].keys())[0]
entity_id = "{}.{}".format(entity_type, entity_name)
if dpath.search(definition_context[full_file_path],
f'{block_type}/{entity_type}/{entity_name}'):
entity_context = dpath.get(
definition_context[full_file_path],
f'{block_type}/{entity_type}/{entity_name}')
entity_lines_range = [
entity_context.get('start_line'),
entity_context.get('end_line')
]
entity_code_lines = entity_context.get('code_lines')
skipped_checks = entity_context.get('skipped_checks')
variable_evaluations = definition_context[full_file_path].get(
'evaluations')
registry = self.block_type_registries[block_type]
if registry:
results = registry.scan(scanned_file, entity,
skipped_checks)
for check, check_result in results.items():
record = Record(check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=scanned_file,
file_line_range=entity_lines_range,
resource=entity_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__)
report.add_record(record=record)
def create_report_from_graph_checks_results(self, checks_results, check):
report = Report("cloudformation")
first_results_key = list(checks_results.keys())[0]
for check_result in checks_results[first_results_key]:
entity = check_result["entity"]
record = Record(check_id=check['id'],
check_name=check['name'],
check_result=copy.deepcopy(check_result),
code_block="",
file_path=entity.get(CustomAttributes.FILE_PATH),
file_line_range=[
entity.get('__startline__'),
entity.get('__endline__')
],
resource=entity.get(CustomAttributes.BLOCK_NAME),
entity_tags=entity.get('tags', {}),
evaluations=None,
check_class=None,
file_abs_path=entity.get(
CustomAttributes.FILE_PATH))
if check_result["result"] == CheckResult.PASSED:
report.passed_checks.append(record)
if check_result["result"] == CheckResult.FAILED:
report.failed_checks.append(record)
return report
def run_block(self, entities, definition_context, full_file_path, report, scanned_file, block_type,
runner_filter=None):
registry = self.block_type_registries[block_type]
if registry:
for entity in entities:
entity_evaluations = None
context_parser = parser_registry.context_parsers[block_type]
definition_path = context_parser.get_entity_context_path(entity)
entity_id = ".".join(definition_path)
entity_context_path = [block_type] + definition_path
for _, entity_context in dpath.search(definition_context[full_file_path], entity_context_path, yielded=True):
entity_lines_range = [entity_context.get('start_line'), entity_context.get('end_line')]
entity_code_lines = entity_context.get('code_lines')
skipped_checks = entity_context.get('skipped_checks')
variables_evaluations = definition_context[full_file_path].get('evaluations')
if variables_evaluations:
entity_evaluations = BaseVariableEvaluation.reduce_entity_evaluations(variables_evaluations,
entity_context_path)
results = registry.scan(scanned_file, entity, skipped_checks, runner_filter)
for check, check_result in results.items():
record = Record(check_id=check.id, check_name=check.name, check_result=check_result,
code_block=entity_code_lines, file_path=scanned_file,
file_line_range=entity_lines_range,
resource=entity_id, evaluations=entity_evaluations,
check_class=check.__class__.__module__)
report.add_record(record=record)
def run(self, root_folder=None, external_checks_dir=None, files=None,
runner_filter=RunnerFilter(), collect_skip_comments=True) -> Report:
registry = self.import_registry()
definitions = {}
definitions_raw = {}
report = Report(self.check_type)
if not files and not root_folder:
logging.debug("No resources to scan.")
return report
if not external_checks_dir and self.require_external_checks():
logging.debug("The json runner requires that external checks are defined.")
return report
if external_checks_dir:
for directory in external_checks_dir:
registry.load_external_checks(directory)
if files:
self._load_files(files, definitions, definitions_raw)
if root_folder:
for root, d_names, f_names in os.walk(root_folder):
filter_ignored_paths(root, d_names, runner_filter.excluded_paths)
filter_ignored_paths(root, f_names, runner_filter.excluded_paths)
self._load_files(
f_names,
definitions,
definitions_raw,
lambda f: os.path.join(root, f)
)
for json_file_path in definitions.keys():
results = registry.scan(
json_file_path, definitions[json_file_path], [], runner_filter
)
for check, result in results.items():
result_config = result["results_configuration"]
start = result_config.start_mark.line
end = result_config.end_mark.line
record = Record(
check_id=check.id,
bc_check_id=check.bc_id,
check_name=check.name,
check_result=result,
code_block=definitions_raw[json_file_path][start:end + 1],
file_path=json_file_path,
file_line_range=[start + 1, end + 1],
resource=f"{json_file_path}",
evaluations=None,
check_class=check.__class__.__module__,
file_abs_path=os.path.abspath(json_file_path),
entity_tags=None
)
report.add_record(record)
return report
def run(self, root_folder, external_checks_dir=None, files=None, runner_filter=RunnerFilter()):
report = Report(self.check_type)
definitions = {}
definitions_raw = {}
parsing_errors = {}
files_list = []
if external_checks_dir:
for directory in external_checks_dir:
cfn_registry.load_external_checks(directory)
if files:
for file in files:
(definitions[file], definitions_raw[file]) = parse(file)
if root_folder:
for root, d_names, f_names in os.walk(root_folder):
filter_ignored_directories(d_names)
for file in f_names:
file_ending = os.path.splitext(file)[1]
if file_ending in CF_POSSIBLE_ENDINGS:
files_list.append(os.path.join(root, file))
for file in files_list:
relative_file_path = f'/{os.path.relpath(file, os.path.commonprefix((root_folder, file)))}'
(definitions[relative_file_path], definitions_raw[relative_file_path]) = parse(file)
# Filter out empty files that have not been parsed successfully, and filter out non-CF template files
definitions = {k: v for k, v in definitions.items() if v and v.__contains__("Resources")}
definitions_raw = {k: v for k, v in definitions_raw.items() if k in definitions.keys()}
for cf_file in definitions.keys():
if isinstance(definitions[cf_file], dict_node) and 'Resources' in definitions[cf_file].keys():
cf_context_parser = ContextParser(cf_file, definitions[cf_file], definitions_raw[cf_file])
logging.debug("Template Dump for {}: {}".format(cf_file, definitions[cf_file], indent=2))
cf_context_parser.evaluate_default_refs()
for resource_name, resource in definitions[cf_file]['Resources'].items():
resource_id = cf_context_parser.extract_cf_resource_id(resource, resource_name)
# check that the resource can be parsed as a CF resource
if resource_id:
entity_lines_range, entity_code_lines = cf_context_parser.extract_cf_resource_code_lines(resource)
if entity_lines_range and entity_code_lines:
# TODO - Variable Eval Message!
variable_evaluations = {}
skipped_checks = ContextParser.collect_skip_comments(entity_code_lines)
results = cfn_registry.scan(cf_file, {resource_name: resource}, skipped_checks,
runner_filter)
for check, check_result in results.items():
record = Record(check_id=check.id, check_name=check.name, check_result=check_result,
code_block=entity_code_lines, file_path=cf_file,
file_line_range=entity_lines_range,
resource=resource_id, evaluations=variable_evaluations,
check_class=check.__class__.__module__)
report.add_record(record=record)
return report
def run_block(self, entities, full_file_path, report, scanned_file, block_type, runner_filter=None):
registry = self.block_type_registries[block_type]
if registry:
for entity in entities:
context_parser = parser_registry.context_parsers[block_type]
definition_path = context_parser.get_entity_context_path(entity)
entity_id = ".".join(definition_path)
# Entity can exist only once per dir, for file as well
entity_context = self.get_entity_context(definition_path, full_file_path)
entity_lines_range = [entity_context.get('start_line'), entity_context.get('end_line')]
entity_code_lines = entity_context.get('code_lines')
results = registry.scan(scanned_file, entity, [], runner_filter)
for check, check_result in results.items():
record = Record(check_id=check.id, bc_check_id=check.bc_id, check_name=check.name, check_result=check_result,
code_block=entity_code_lines, file_path=scanned_file,
file_line_range=entity_lines_range,
resource=entity_id, evaluations=None,
check_class=check.__class__.__module__, file_abs_path=full_file_path)
record.set_guideline(check.guideline)
report.add_record(record=record)
def test_valid_passing_valid_testcases(self):
record1 = Record(
check_id="CKV_AWS_21",
check_name="Some Check",
check_result={"result": CheckResult.FAILED},
code_block=None,
file_path="./s3.tf",
file_line_range=[1, 3],
resource="aws_s3_bucket.operations",
evaluations=None,
check_class=None,
file_abs_path=",.",
entity_tags={"tag1": "value1"},
)
record2 = Record(
check_id="CKV_AWS_3",
check_name=
"Ensure all data stored in the EBS is securely encrypted",
check_result={"result": CheckResult.FAILED},
code_block=None,
file_path="./ec2.tf",
file_line_range=[1, 3],
resource="aws_ebs_volume.web_host_storage",
evaluations=None,
check_class=None,
file_abs_path=",.",
entity_tags={"tag1": "value1"},
)
r = Report("terraform")
r.add_record(record=record1)
r.add_record(record=record2)
r.get_test_suites()
json_structure = r.get_sarif_json("")
print(json.dumps(json_structure))
self.assertEqual(
None,
jsonschema.validate(instance=json_structure,
schema=get_sarif_schema()),
)
def mutateKubernetesGraphResults(self, root_folder: str, runner_filter: RunnerFilter, report: Report, checks_results, reportMutatorData=None) -> Report:
# Moves report generation logic out of run() method in Runner class.
# Allows function overriding of a much smaller function than run() for other "child" frameworks such as Kustomize, Helm
# Where Kubernetes CHECKS are needed, but the specific file references are to another framework for the user output (or a mix of both).
kustomizeMetadata = reportMutatorData['kustomizeMetadata'],
kustomizeFileMappings = reportMutatorData['kustomizeFileMappings']
for check, check_results in checks_results.items():
for check_result in check_results:
entity = check_result["entity"]
entity_file_path = entity.get(CustomAttributes.FILE_PATH)
entity_file_abs_path = _get_entity_abs_path(root_folder, entity_file_path)
entity_id = entity.get(CustomAttributes.ID)
entity_context = self.context[entity_file_path][entity_id]
if entity_file_abs_path in kustomizeFileMappings:
realKustomizeEnvMetadata = kustomizeMetadata[0][kustomizeFileMappings[entity_file_abs_path]]
if 'overlay' in realKustomizeEnvMetadata["type"]:
kustomizeResourceID = f'{realKustomizeEnvMetadata["type"]}:{str(realKustomizeEnvMetadata["overlay_name"])}:{entity_id}'
else:
kustomizeResourceID = f'{realKustomizeEnvMetadata["type"]}:{entity_id}'
else:
kustomizeResourceID = "Unknown error. This is a bug."
record = Record(
check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_context.get("code_lines"),
file_path=realKustomizeEnvMetadata['filePath'],
file_line_range=[0,0],
resource=kustomizeResourceID, # entity.get(CustomAttributes.ID),
evaluations={},
check_class=check.__class__.__module__,
file_abs_path=entity_file_abs_path
)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def get_graph_checks_report(self, root_folder: str,
runner_filter: RunnerFilter) -> Report:
report = Report(self.check_type)
checks_results = self.run_graph_checks_results(runner_filter)
for check, check_results in checks_results.items():
for check_result in check_results:
entity = check_result["entity"]
entity_file_abs_path = entity.get(CustomAttributes.FILE_PATH)
entity_file_path = scanned_file = f"/{os.path.relpath(entity_file_abs_path, root_folder)}"
entity_name = entity.get(
CustomAttributes.BLOCK_NAME).split(".")[1]
entity_context = self.context[entity_file_abs_path][
TemplateSections.RESOURCES][entity_name]
record = Record(
check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_context.get("code_lines"),
file_path=entity_file_path,
file_line_range=[
entity_context.get("start_line"),
entity_context.get("end_line")
],
resource=entity.get(CustomAttributes.ID),
evaluations={},
check_class=check.__class__.__module__,
file_abs_path=entity_file_abs_path,
entity_tags={} if not entity.get("Tags") else
cfn_utils.parse_entity_tags(entity.get("Tags")))
record.set_guideline(check.guideline)
if self.breadcrumbs:
breadcrumb = self.breadcrumbs.get(record.file_path,
{}).get(record.resource)
if breadcrumb:
record = GraphRecord(record, breadcrumb)
report.add_record(record=record)
return report
def run_block(self, entities,
definition_context,
full_file_path, root_folder, report, scanned_file,
block_type, runner_filter=None, entity_context_path_header=None):
registry = self.block_type_registries[block_type]
if not registry:
return
for entity in entities:
entity_evaluations = None
context_parser = parser_registry.context_parsers[block_type]
definition_path = context_parser.get_entity_context_path(entity)
entity_id = ".".join(definition_path)
if entity_context_path_header is None:
entity_context_path = [block_type] + definition_path
else:
entity_context_path = entity_context_path_header + block_type + definition_path
# Entity can exist only once per dir, for file as well
try:
entity_context = dict_utils.getInnerDict(definition_context[full_file_path], entity_context_path)
entity_lines_range = [entity_context.get('start_line'), entity_context.get('end_line')]
entity_code_lines = entity_context.get('code_lines')
skipped_checks = entity_context.get('skipped_checks')
except KeyError:
# TODO: Context info isn't working for modules
entity_lines_range = None
entity_code_lines = None
skipped_checks = None
if full_file_path in self.evaluations_context:
variables_evaluations = {}
for var_name, context_info in self.evaluations_context.get(full_file_path, {}).items():
variables_evaluations[var_name] = dataclasses.asdict(context_info)
entity_evaluations = BaseVariableEvaluation.reduce_entity_evaluations(variables_evaluations,
entity_context_path)
results = registry.scan(scanned_file, entity, skipped_checks, runner_filter)
absolut_scanned_file_path = self._strip_module_referrer(file_path=full_file_path)
# This duplicates a call at the start of scan, but adding this here seems better than kludging with some tuple return type
(entity_type, _, entity_config) = registry.extract_entity_details(entity)
tags = get_resource_tags(entity_type, entity_config)
for check, check_result in results.items():
record = Record(check_id=check.id, check_name=check.name, check_result=check_result,
code_block=entity_code_lines, file_path=scanned_file,
file_line_range=entity_lines_range,
resource=entity_id, evaluations=entity_evaluations,
check_class=check.__class__.__module__, file_abs_path=absolut_scanned_file_path, entity_tags=tags)
report.add_record(record=record)
def test_resource_suppression(self):
instance = BcPlatformIntegration()
instance.repo_id = 'org/repo'
suppressions_integration = SuppressionsIntegration(instance)
suppression = {
"suppressionType": "Resources",
"policyId": "BC_AWS_S3_13",
"comment": "No justification comment provided.",
"resources": [
{
"accountId": "org/repo",
"resourceId": "/terraform/aws/s3.tf:aws_s3_bucket.operations",
}
],
"checkovPolicyId": "CKV_AWS_18",
}
record1 = Record(check_id='CKV_AWS_18', check_name=None, check_result=None,
code_block=None, file_path=None,
file_line_range=None,
resource='aws_s3_bucket.operations', evaluations=None,
check_class=None, file_abs_path=',.', entity_tags=None)
record1.repo_file_path = '/terraform/aws/s3.tf'
record2 = Record(check_id='CKV_AWS_13', check_name=None, check_result=None,
code_block=None, file_path=None,
file_line_range=None,
resource='aws_s3_bucket.no', evaluations=None,
check_class=None, file_abs_path='.', entity_tags=None)
record2.repo_file_path = '/terraform/aws/s3.tf'
record3 = Record(check_id='CKV_AWS_1', check_name=None, check_result=None,
code_block=None, file_path=None,
file_line_range=None,
resource='aws_s3_bucket.operations', evaluations=None,
check_class=None, file_abs_path='.', entity_tags=None)
record3.repo_file_path = '/terraform/aws/s3.tf'
self.assertTrue(suppressions_integration._check_suppression(record1, suppression))
self.assertFalse(suppressions_integration._check_suppression(record2, suppression))
self.assertFalse(suppressions_integration._check_suppression(record3, suppression))
def run(self,
root_folder,
external_checks_dir=None,
files=None,
runner_filter=RunnerFilter(),
collect_skip_comments=True,
helmChart=None):
report = Report(self.check_type)
definitions = {}
definitions_raw = {}
parsing_errors = {}
files_list = []
if external_checks_dir:
for directory in external_checks_dir:
registry.load_external_checks(directory, runner_filter)
if files:
for file in files:
parse_result = parse(file)
if parse_result:
(definitions[file], definitions_raw[file]) = parse_result
if root_folder:
for root, d_names, f_names in os.walk(root_folder):
filter_ignored_directories(d_names)
for file in f_names:
file_ending = os.path.splitext(file)[1]
if file_ending in K8_POSSIBLE_ENDINGS:
full_path = os.path.join(root, file)
if "/." not in full_path and file not in [
'package.json', 'package-lock.json'
]:
# skip temp directories
files_list.append(full_path)
for file in files_list:
relative_file_path = f'/{os.path.relpath(file, os.path.commonprefix((root_folder, file)))}'
parse_result = parse(file)
if parse_result:
(definitions[relative_file_path],
definitions_raw[relative_file_path]) = parse_result
for k8_file in definitions.keys():
# There are a few cases here. If -f was used, there could be a leading / because it's an absolute path,
# or there will be no leading slash; root_folder will always be none.
# If -d is used, root_folder will be the value given, and -f will start with a / (hardcoded above).
# The goal here is simply to get a valid path to the file (which sls_file does not always give).
if k8_file[0] == '/':
path_to_convert = (root_folder +
k8_file) if root_folder else k8_file
else:
path_to_convert = (os.path.join(
root_folder, k8_file)) if root_folder else k8_file
file_abs_path = os.path.abspath(path_to_convert)
if definitions[k8_file]:
for i in range(len(definitions[k8_file])):
if (not 'apiVersion' in definitions[k8_file][i].keys()
) and (not 'kind' in definitions[k8_file][i].keys()):
continue
logging.debug("Template Dump for {}: {}".format(
k8_file, definitions[k8_file][i], indent=2))
entity_conf = definitions[k8_file][i]
# Split out resources if entity kind is List
if entity_conf["kind"] == "List":
for item in entity_conf["items"]:
definitions[k8_file].append(item)
for i in range(len(definitions[k8_file])):
if (not 'apiVersion' in definitions[k8_file][i].keys()
) and (not 'kind' in definitions[k8_file][i].keys()):
continue
logging.debug("Template Dump for {}: {}".format(
k8_file, definitions[k8_file][i], indent=2))
entity_conf = definitions[k8_file][i]
if entity_conf["kind"] == "List":
continue
# Skip entity without metadata["name"]
if "metadata" in entity_conf:
if isinstance(
entity_conf["metadata"],
int) or not "name" in entity_conf["metadata"]:
continue
else:
continue
# Skip entity with parent (metadata["ownerReferences"]) in runtime
# We will alert in runtime only
if "ownerReferences" in entity_conf["metadata"] and \
entity_conf["metadata"]["ownerReferences"] is not None:
continue
# Append containers and initContainers to definitions list
for type in ["containers", "initContainers"]:
containers = []
if entity_conf["kind"] == "CustomResourceDefinition":
continue
containers = self._search_deep_keys(
type, entity_conf, [])
if not containers:
continue
containers = containers.pop()
#containers.insert(0,entity_conf['kind'])
containerDef = {}
namespace = ""
if "namespace" in entity_conf["metadata"]:
namespace = entity_conf["metadata"]["namespace"]
else:
namespace = "default"
containerDef["containers"] = containers.pop()
if containerDef["containers"] is not None:
for cd in containerDef["containers"]:
i = containerDef["containers"].index(cd)
containerDef["containers"][i][
"apiVersion"] = entity_conf["apiVersion"]
containerDef["containers"][i]["kind"] = type
containerDef["containers"][i][
"parent"] = "{}.{}.{} (container {})".format(
entity_conf["kind"],
entity_conf["metadata"]["name"],
namespace, str(i))
containerDef["containers"][i][
"parent_metadata"] = entity_conf[
"metadata"]
definitions[k8_file].extend(
containerDef["containers"])
# Run for each definition included added container definitions
for i in range(len(definitions[k8_file])):
if (not 'apiVersion' in definitions[k8_file][i].keys()
) and (not 'kind' in definitions[k8_file][i].keys()):
continue
logging.debug("Template Dump for {}: {}".format(
k8_file, definitions[k8_file][i], indent=2))
entity_conf = definitions[k8_file][i]
if entity_conf["kind"] == "List":
continue
if isinstance(entity_conf["kind"], int):
continue
# Skip entity without metadata["name"] or parent_metadata["name"]
if not any(x in entity_conf["kind"]
for x in ["containers", "initContainers"]):
if "metadata" in entity_conf:
if isinstance(
entity_conf["metadata"], int
) or not "name" in entity_conf["metadata"]:
continue
else:
continue
# Skip entity with parent (metadata["ownerReferences"]) in runtime
# We will alert in runtime only
if "metadata" in entity_conf:
if "ownerReferences" in entity_conf["metadata"] and \
entity_conf["metadata"]["ownerReferences"] is not None:
continue
# Skip Kustomization Templates (for now)
if entity_conf["kind"] == "Kustomization":
continue
skipped_checks = get_skipped_checks(entity_conf)
results = registry.scan(k8_file, entity_conf,
skipped_checks, runner_filter)
# TODO refactor into context parsing
find_lines_result_list = list(
find_lines(entity_conf, '__startline__'))
start_line = entity_conf["__startline__"]
end_line = entity_conf["__endline__"]
if start_line == end_line:
entity_lines_range = [start_line, end_line]
entity_code_lines = definitions_raw[k8_file][
start_line - 1:end_line]
else:
entity_lines_range = [start_line, end_line - 1]
entity_code_lines = definitions_raw[k8_file][
start_line - 1:end_line - 1]
# TODO? - Variable Eval Message!
variable_evaluations = {}
for check, check_result in results.items():
record = Record(
check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=k8_file,
file_line_range=entity_lines_range,
resource=check.get_resource_id(entity_conf),
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path)
report.add_record(record=record)
return report
def run(self,
root_folder,
external_checks_dir=None,
files=None,
runner_filter=RunnerFilter(),
collect_skip_comments=True):
report = Report(self.check_type)
definitions = {}
definitions_raw = {}
parsing_errors = {}
files_list = []
if external_checks_dir:
for directory in external_checks_dir:
cfn_registry.load_external_checks(directory, runner_filter)
if files:
for file in files:
(definitions[file], definitions_raw[file]) = parse(file)
if root_folder:
for root, d_names, f_names in os.walk(root_folder):
filter_ignored_directories(d_names)
for file in f_names:
file_ending = os.path.splitext(file)[1]
if file_ending in CF_POSSIBLE_ENDINGS:
files_list.append(os.path.join(root, file))
for file in files_list:
relative_file_path = f'/{os.path.relpath(file, os.path.commonprefix((root_folder, file)))}'
try:
(definitions[relative_file_path],
definitions_raw[relative_file_path]) = parse(file)
except TypeError:
logging.info(
f'CloudFormation skipping {file} as it is not a valid CF template'
)
# Filter out empty files that have not been parsed successfully, and filter out non-CF template files
definitions = {
k: v
for k, v in definitions.items()
if v and isinstance(v, dict_node) and v.__contains__("Resources")
and isinstance(v["Resources"], dict_node)
}
definitions_raw = {
k: v
for k, v in definitions_raw.items() if k in definitions.keys()
}
for cf_file in definitions.keys():
# There are a few cases here. If -f was used, there could be a leading / because it's an absolute path,
# or there will be no leading slash; root_folder will always be none.
# If -d is used, root_folder will be the value given, and -f will start with a / (hardcoded above).
# The goal here is simply to get a valid path to the file (which cf_file does not always give).
if cf_file[0] == '/':
path_to_convert = (root_folder +
cf_file) if root_folder else cf_file
else:
path_to_convert = (os.path.join(
root_folder, cf_file)) if root_folder else cf_file
file_abs_path = os.path.abspath(path_to_convert)
if isinstance(
definitions[cf_file],
dict_node) and 'Resources' in definitions[cf_file].keys():
cf_context_parser = ContextParser(cf_file,
definitions[cf_file],
definitions_raw[cf_file])
logging.debug("Template Dump for {}: {}".format(
cf_file, definitions[cf_file], indent=2))
cf_context_parser.evaluate_default_refs()
for resource_name, resource in definitions[cf_file][
'Resources'].items():
resource_id = cf_context_parser.extract_cf_resource_id(
resource, resource_name)
# check that the resource can be parsed as a CF resource
if resource_id:
entity_lines_range, entity_code_lines = cf_context_parser.extract_cf_resource_code_lines(
resource)
if entity_lines_range and entity_code_lines:
# TODO - Variable Eval Message!
variable_evaluations = {}
skipped_checks = ContextParser.collect_skip_comments(
entity_code_lines)
results = cfn_registry.scan(
cf_file, {resource_name: resource},
skipped_checks, runner_filter)
for check, check_result in results.items():
record = Record(
check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=cf_file,
file_line_range=entity_lines_range,
resource=resource_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path)
report.add_record(record=record)
return report
def run(self,
root_folder=None,
external_checks_dir=None,
files=None,
runner_filter=RunnerFilter(),
collect_skip_comments=True):
report = Report(self.check_type)
files_list = []
filepath_fn = None
if external_checks_dir:
for directory in external_checks_dir:
registry.load_external_checks(directory)
if files:
files_list = [
file for file in files
if Runner._is_docker_file(os.path.basename(file))
]
if root_folder:
filepath_fn = lambda f: f'/{os.path.relpath(f, os.path.commonprefix((root_folder, f)))}'
for root, d_names, f_names in os.walk(root_folder):
filter_ignored_paths(root, d_names,
runner_filter.excluded_paths)
filter_ignored_paths(root, f_names,
runner_filter.excluded_paths)
for file in f_names:
if Runner._is_docker_file(file):
file_path = os.path.join(root, file)
files_list.append(file_path)
definitions, definitions_raw = get_files_definitions(
files_list, filepath_fn)
for docker_file_path in definitions.keys():
# There are a few cases here. If -f was used, there could be a leading / because it's an absolute path,
# or there will be no leading slash; root_folder will always be none.
# If -d is used, root_folder will be the value given, and -f will start with a / (hardcoded above).
# The goal here is simply to get a valid path to the file (which docker_file_path does not always give).
if docker_file_path[0] == '/':
path_to_convert = (
root_folder +
docker_file_path) if root_folder else docker_file_path
else:
path_to_convert = (os.path.join(
root_folder,
docker_file_path)) if root_folder else docker_file_path
file_abs_path = os.path.abspath(path_to_convert)
report.add_resource(file_abs_path)
skipped_checks = collect_skipped_checks(
definitions[docker_file_path])
instructions = definitions[docker_file_path]
results = registry.scan(docker_file_path, instructions,
skipped_checks, runner_filter)
for check, check_result in results.items():
result_configuration = check_result['results_configuration']
startline = 0
endline = len(definitions_raw[docker_file_path]) - 1
result_instruction = ""
if result_configuration:
startline = result_configuration['startline']
endline = result_configuration['endline']
result_instruction = result_configuration["instruction"]
codeblock = []
self.calc_record_codeblock(codeblock, definitions_raw,
docker_file_path, endline,
startline)
record = Record(check_id=check.id,
bc_check_id=check.bc_id,
check_name=check.name,
check_result=check_result,
code_block=codeblock,
file_path=docker_file_path,
file_line_range=[startline + 1, endline + 1],
resource="{}.{}".format(
docker_file_path, result_instruction,
startline),
evaluations=None,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path,
entity_tags=None)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def run(
self,
root_folder: str,
external_checks_dir: Optional[List[str]] = None,
files: Optional[List[str]] = None,
runner_filter: RunnerFilter = RunnerFilter(),
collect_skip_comments: bool = True,
) -> Report:
report = Report(self.check_type)
files_list = []
filepath_fn = None
if external_checks_dir:
for directory in external_checks_dir:
arm_resource_registry.load_external_checks(directory)
if files:
files_list = files.copy()
if root_folder:
filepath_fn = lambda f: f'/{os.path.relpath(f, os.path.commonprefix((root_folder, f)))}'
for root, d_names, f_names in os.walk(root_folder):
filter_ignored_paths(root, d_names,
runner_filter.excluded_paths)
filter_ignored_paths(root, f_names,
runner_filter.excluded_paths)
for file in f_names:
file_ending = os.path.splitext(file)[1]
if file_ending in ARM_POSSIBLE_ENDINGS:
files_list.append(os.path.join(root, file))
definitions, definitions_raw = get_files_definitions(
files_list, filepath_fn)
# Filter out empty files that have not been parsed successfully, and filter out non-CF template files
definitions = {
k: v
for k, v in definitions.items()
if v and v.__contains__("resources")
}
definitions_raw = {
k: v
for k, v in definitions_raw.items() if k in definitions.keys()
}
for arm_file in definitions.keys():
# There are a few cases here. If -f was used, there could be a leading / because it's an absolute path,
# or there will be no leading slash; root_folder will always be none.
# If -d is used, root_folder will be the value given, and -f will start with a / (hardcoded above).
# The goal here is simply to get a valid path to the file (which arm_file does not always give).
if arm_file[0] == '/':
path_to_convert = (root_folder +
arm_file) if root_folder else arm_file
else:
path_to_convert = (os.path.join(
root_folder, arm_file)) if root_folder else arm_file
file_abs_path = os.path.abspath(path_to_convert)
if isinstance(definitions[arm_file], DictNode):
arm_context_parser = ContextParser(arm_file,
definitions[arm_file],
definitions_raw[arm_file])
logging.debug(
f"Template Dump for {arm_file}: {definitions[arm_file]}")
if 'resources' in definitions[arm_file].keys():
arm_context_parser.evaluate_default_parameters()
# Split out nested resources from base resource
for resource in definitions[arm_file]['resources']:
if isinstance(
resource,
dict) and "parent_name" in resource.keys():
continue
nested_resources = []
nested_resources = arm_context_parser.search_deep_keys(
"resources", resource, [])
if nested_resources:
for nr in nested_resources:
nr_element = nr.pop()
if nr_element:
for element in nr_element:
new_resource = {}
new_resource = element
if isinstance(new_resource, dict):
new_resource[
"parent_name"] = resource[
"name"]
new_resource[
"parent_type"] = resource[
"type"]
definitions[arm_file][
'resources'].append(
new_resource)
for resource in definitions[arm_file]['resources']:
resource_id = arm_context_parser.extract_arm_resource_id(
resource)
report.add_resource(f'{arm_file}:{resource_id}')
resource_name = arm_context_parser.extract_arm_resource_name(
resource)
entity_lines_range, entity_code_lines = arm_context_parser.extract_arm_resource_code_lines(
resource)
if entity_lines_range and entity_code_lines:
# TODO - Variable Eval Message!
variable_evaluations = {}
skipped_checks = ContextParser.collect_skip_comments(
resource)
results = arm_resource_registry.scan(
arm_file, {resource_name: resource},
skipped_checks, runner_filter)
for check, check_result in results.items():
record = Record(
check_id=check.id,
bc_check_id=check.bc_id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=arm_file,
file_line_range=entity_lines_range,
resource=resource_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path)
record.set_guideline(check.guideline)
report.add_record(record=record)
if 'parameters' in definitions[arm_file].keys():
parameters = definitions[arm_file]['parameters']
for parameter_name, parameter_details in parameters.items(
):
# TODO - Variable Eval Message!
variable_evaluations = {}
resource_id = f'parameter.{parameter_name}'
resource_name = parameter_name
entity_lines_range, entity_code_lines = arm_context_parser.extract_arm_resource_code_lines(
parameter_details)
if entity_lines_range and entity_code_lines:
skipped_checks = ContextParser.collect_skip_comments(
parameter_details)
results = arm_parameter_registry.scan(
arm_file, {resource_name: parameter_details},
skipped_checks, runner_filter)
for check, check_result in results.items():
record = Record(
check_id=check.id,
bc_check_id=check.bc_id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=arm_file,
file_line_range=entity_lines_range,
resource=resource_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path)
record.set_guideline(check.guideline)
report.add_record(record=record)
return report
def run(self,
root_folder,
external_checks_dir=None,
files=None,
runner_filter=RunnerFilter()):
report = Report(self.check_type)
definitions = {}
definitions_raw = {}
parsing_errors = {}
files_list = []
if external_checks_dir:
for directory in external_checks_dir:
resource_registry.load_external_checks(directory)
if files:
for file in files:
(definitions[file], definitions_raw[file]) = parse(file)
if root_folder:
for root, d_names, f_names in os.walk(root_folder):
for file in f_names:
file_ending = os.path.splitext(file)[1]
if file_ending in CF_POSSIBLE_ENDINGS:
files_list.append(os.path.join(root, file))
for file in files_list:
relative_file_path = f'/{os.path.relpath(file, os.path.commonprefix((root_folder, file)))}'
(definitions[relative_file_path],
definitions_raw[relative_file_path]) = parse(file)
# Filter out empty files that have not been parsed successfully, and filter out non-CF template files
definitions = {
k: v
for k, v in definitions.items()
if v and v.__contains__("Resources")
}
definitions_raw = {
k: v
for k, v in definitions_raw.items() if k in definitions.keys()
}
for cf_file in definitions.keys():
if isinstance(
definitions[cf_file],
dict_node) and 'Resources' in definitions[cf_file].keys():
logging.debug("Template Dump for {}: {}".format(
cf_file, definitions[cf_file], indent=2))
# Get Parameter Defaults - Locate Refs in Template
refs = []
refs.extend(
self._search_deep_keys('Ref', definitions[cf_file], []))
for ref in refs:
refname = ref.pop()
ref.pop() # Get rid of the 'Ref' dict key
if 'Parameters' in definitions[cf_file].keys(
) and refname in definitions[cf_file]['Parameters'].keys():
# TODO refactor into evaluations
if 'Default' in definitions[cf_file]['Parameters'][
refname].keys():
logging.debug(
"Replacing Ref {} in file {} with default parameter value: {}"
.format(
refname, cf_file, definitions[cf_file]
['Parameters'][refname]['Default']))
_set_in_dict(
definitions[cf_file], ref, definitions[cf_file]
['Parameters'][refname]['Default'])
## TODO - Add Variable Eval Message for Output
# Output in Checkov looks like this:
# Variable versioning (of /.) evaluated to value "True" in expression: enabled = ${var.versioning}
for resource_name, resource in definitions[cf_file][
'Resources'].items():
if resource_name == '__startline__' or resource_name == '__endline__':
continue
resource_id = f"{resource['Type']}.{resource_name}"
# TODO refactor into context parsing
find_lines_result_list = list(
find_lines(resource, '__startline__'))
if len(find_lines_result_list) >= 1:
start_line = min(find_lines_result_list)
end_line = max(
list(find_lines(resource, '__endline__')))
entity_lines_range = [start_line, end_line - 1]
entity_code_lines = definitions_raw[cf_file][
start_line - 1:end_line - 1]
# TODO - Variable Eval Message!
variable_evaluations = {}
skipped_checks = []
for line in entity_code_lines:
skip_search = re.search(COMMENT_REGEX, str(line))
if skip_search:
skipped_checks.append({
'id':
skip_search.group(2),
'suppress_comment':
skip_search.group(3)[1:]
if skip_search.group(3) else
"No comment provided"
})
results = resource_registry.scan(
cf_file, {resource_name: resource}, skipped_checks,
runner_filter)
for check, check_result in results.items():
### TODO - Need to get entity_code_lines and entity_lines_range
record = Record(
check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=cf_file,
file_line_range=entity_lines_range,
resource=resource_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__)
report.add_record(record=record)
return report
def run(self,
root_folder,
external_checks_dir=None,
files=None,
runner_filter=RunnerFilter()):
report = Report(self.check_type)
definitions = {}
definitions_raw = {}
parsing_errors = {}
files_list = []
if external_checks_dir:
for directory in external_checks_dir:
sls_registry.load_external_checks(directory, runner_filter)
if files:
for file in files:
if os.path.basename(file) in SLS_FILE_MASK:
parse_result = parse(file)
if parse_result:
(definitions[file],
definitions_raw[file]) = parse_result
if root_folder:
for root, d_names, f_names in os.walk(root_folder):
# Don't walk in to "node_modules" directories under the root folder. If –for some reason–
# scanning one of these is desired, it can be directly specified.
if "node_modules" in d_names:
d_names.remove("node_modules")
filter_ignored_directories(d_names)
for file in f_names:
if file in SLS_FILE_MASK:
full_path = os.path.join(root, file)
if "/." not in full_path:
# skip temp directories
files_list.append(full_path)
for file in files_list:
relative_file_path = f'/{os.path.relpath(file, os.path.commonprefix((root_folder, file)))}'
parse_result = parse(file)
if parse_result:
(definitions[relative_file_path],
definitions_raw[relative_file_path]) = parse_result
# Filter out empty files that have not been parsed successfully
definitions = {k: v for k, v in definitions.items() if v}
definitions_raw = {
k: v
for k, v in definitions_raw.items() if k in definitions.keys()
}
for sls_file in definitions.keys():
if not isinstance(definitions[sls_file], dict_node):
continue
if CFN_RESOURCES_TOKEN in definitions[sls_file] and isinstance(
definitions[sls_file][CFN_RESOURCES_TOKEN], dict_node):
cf_sub_template = definitions[sls_file][CFN_RESOURCES_TOKEN]
cf_context_parser = CfnContextParser(sls_file, cf_sub_template,
definitions_raw[sls_file])
logging.debug("Template Dump for {}: {}".format(
sls_file, definitions[sls_file], indent=2))
cf_context_parser.evaluate_default_refs()
for resource_name, resource in cf_sub_template[
'Resources'].items():
if not isinstance(resource, dict_node):
continue
cf_resource_id = cf_context_parser.extract_cf_resource_id(
resource, resource_name)
entity_lines_range, entity_code_lines = cf_context_parser.extract_cf_resource_code_lines(
resource)
if entity_lines_range and entity_code_lines:
skipped_checks = CfnContextParser.collect_skip_comments(
entity_code_lines)
# TODO - Variable Eval Message!
variable_evaluations = {}
results = cfn_registry.scan(sls_file,
{resource_name: resource},
skipped_checks,
runner_filter)
for check, check_result in results.items():
record = Record(
check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=sls_file,
file_line_range=entity_lines_range,
resource=cf_resource_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__)
report.add_record(record=record)
if FUNCTIONS_TOKEN in definitions[sls_file]:
template_functions = definitions[sls_file][FUNCTIONS_TOKEN]
sls_context_parser = SlsContextParser(
sls_file, definitions[sls_file], definitions_raw[sls_file])
for sls_function_name, sls_function in template_functions.items(
):
if not isinstance(sls_function, dict_node):
continue
entity_lines_range, entity_code_lines = sls_context_parser.extract_function_code_lines(
sls_function)
if entity_lines_range and entity_code_lines:
skipped_checks = CfnContextParser.collect_skip_comments(
entity_code_lines)
variable_evaluations = {}
sls_context_parser.enrich_function_with_provider(
sls_function_name)
results = sls_registry.scan(
sls_file, {
'function': sls_function,
'provider_type':
sls_context_parser.provider_type
}, skipped_checks, runner_filter)
for check, check_result in results.items():
record = Record(
check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=sls_file,
file_line_range=entity_lines_range,
resource=sls_function_name,
evaluations=variable_evaluations,
check_class=check.__class__.__module__)
report.add_record(record=record)
return report
def run(self,
root_folder,
external_checks_dir=None,
files=None,
runner_filter=RunnerFilter(),
collect_skip_comments=True):
report = Report(self.check_type)
definitions = {}
definitions_raw = {}
parsing_errors = {}
files_list = []
if external_checks_dir:
for directory in external_checks_dir:
function_registry.load_external_checks(directory)
if files:
for file in files:
if os.path.basename(file) in SLS_FILE_MASK:
parse_result = parse(file)
if parse_result:
(definitions[file],
definitions_raw[file]) = parse_result
if root_folder:
for root, d_names, f_names in os.walk(root_folder):
# Don't walk in to "node_modules" directories under the root folder. If –for some reason–
# scanning one of these is desired, it can be directly specified.
if "node_modules" in d_names:
d_names.remove("node_modules")
filter_ignored_directories(d_names)
for file in f_names:
if file in SLS_FILE_MASK:
full_path = os.path.join(root, file)
if "/." not in full_path:
# skip temp directories
files_list.append(full_path)
for file in files_list:
relative_file_path = f'/{os.path.relpath(file, os.path.commonprefix((root_folder, file)))}'
parse_result = parse(file)
if parse_result:
(definitions[relative_file_path],
definitions_raw[relative_file_path]) = parse_result
# Filter out empty files that have not been parsed successfully
definitions = {k: v for k, v in definitions.items() if v}
definitions_raw = {
k: v
for k, v in definitions_raw.items() if k in definitions.keys()
}
for sls_file, sls_file_data in definitions.items():
# There are a few cases here. If -f was used, there could be a leading / because it's an absolute path,
# or there will be no leading slash; root_folder will always be none.
# If -d is used, root_folder will be the value given, and -f will start with a / (hardcoded above).
# The goal here is simply to get a valid path to the file (which sls_file does not always give).
if sls_file[0] == '/':
path_to_convert = (root_folder +
sls_file) if root_folder else sls_file
else:
path_to_convert = (os.path.join(
root_folder, sls_file)) if root_folder else sls_file
file_abs_path = os.path.abspath(path_to_convert)
if not isinstance(sls_file_data, dict_node):
continue
if CFN_RESOURCES_TOKEN in sls_file_data and isinstance(
sls_file_data[CFN_RESOURCES_TOKEN], dict_node):
cf_sub_template = sls_file_data[CFN_RESOURCES_TOKEN]
cf_context_parser = CfnContextParser(sls_file, cf_sub_template,
definitions_raw[sls_file])
logging.debug("Template Dump for {}: {}".format(sls_file,
sls_file_data,
indent=2))
cf_context_parser.evaluate_default_refs()
for resource_name, resource in cf_sub_template[
'Resources'].items():
if not isinstance(resource, dict_node):
continue
cf_resource_id = cf_context_parser.extract_cf_resource_id(
resource, resource_name)
if not cf_resource_id:
# Not Type attribute for resource
continue
entity_lines_range, entity_code_lines = cf_context_parser.extract_cf_resource_code_lines(
resource)
if entity_lines_range and entity_code_lines:
skipped_checks = CfnContextParser.collect_skip_comments(
entity_code_lines)
# TODO - Variable Eval Message!
variable_evaluations = {}
entity = {resource_name: resource}
results = cfn_registry.scan(sls_file, entity,
skipped_checks,
runner_filter)
tags = cfn_utils.get_resource_tags(
entity, cfn_registry)
for check, check_result in results.items():
record = Record(
check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=sls_file,
file_line_range=entity_lines_range,
resource=cf_resource_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path,
entity_tags=tags)
report.add_record(record=record)
sls_context_parser = SlsContextParser(sls_file, sls_file_data,
definitions_raw[sls_file])
# Sub-sections that have multiple items under them
for token, registry in MULTI_ITEM_SECTIONS:
template_items = sls_file_data.get(token)
if not template_items or not isinstance(template_items, dict):
continue
for item_name, item_content in template_items.items():
if not isinstance(item_content, dict_node):
continue
entity_lines_range, entity_code_lines = sls_context_parser.extract_code_lines(
item_content)
if entity_lines_range and entity_code_lines:
skipped_checks = CfnContextParser.collect_skip_comments(
entity_code_lines)
variable_evaluations = {}
if token == "functions": #nosec
# "Enriching" copies things like "environment" and "stackTags" down into the
# function data from the provider block since logically that's what serverless
# does. This allows checks to see what the complete data would be.
sls_context_parser.enrich_function_with_provider(
item_name)
entity = EntityDetails(
sls_context_parser.provider_type, item_content)
results = registry.scan(sls_file, entity,
skipped_checks, runner_filter)
tags = cfn_utils.get_resource_tags(entity, registry)
for check, check_result in results.items():
record = Record(
check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=sls_file,
file_line_range=entity_lines_range,
resource=item_name,
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path,
entity_tags=tags)
report.add_record(record=record)
# Sub-sections that are a single item
for token, registry in SINGLE_ITEM_SECTIONS:
item_content = sls_file_data.get(token)
if not item_content:
continue
entity_lines_range, entity_code_lines = sls_context_parser.extract_code_lines(
item_content)
if not entity_lines_range:
entity_lines_range, entity_code_lines = sls_context_parser.extract_code_lines(
sls_file_data)
skipped_checks = CfnContextParser.collect_skip_comments(
entity_code_lines)
variable_evaluations = {}
entity = EntityDetails(sls_context_parser.provider_type,
item_content)
results = registry.scan(sls_file, entity, skipped_checks,
runner_filter)
tags = cfn_utils.get_resource_tags(entity, registry)
for check, check_result in results.items():
record = Record(check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=sls_file,
file_line_range=entity_lines_range,
resource=token,
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path,
entity_tags=tags)
report.add_record(record=record)
# "Complete" checks
# NOTE: Ignore code content, no point in showing (could be long)
entity_lines_range, entity_code_lines = sls_context_parser.extract_code_lines(
sls_file_data)
if entity_lines_range:
skipped_checks = CfnContextParser.collect_skip_comments(
entity_code_lines)
variable_evaluations = {}
entity = EntityDetails(sls_context_parser.provider_type,
sls_file_data)
results = complete_registry.scan(sls_file, entity,
skipped_checks, runner_filter)
tags = cfn_utils.get_resource_tags(entity, complete_registry)
for check, check_result in results.items():
record = Record(
check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=[], # Don't show, could be large
file_path=sls_file,
file_line_range=entity_lines_range,
resource="complete", # Weird, not sure what to put where
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path,
entity_tags=tags)
report.add_record(record=record)
return report
