def test_build_graph_with_params_outputs(self):
relative_file_path = "../../checks/resource/aws/example_IAMRoleAllowAssumeFromAccount/example_IAMRoleAllowAssumeFromAccount-PASSED-2.yml"
definitions = {}
file = os.path.realpath(os.path.join(TEST_DIRNAME, relative_file_path))
(definitions[relative_file_path], definitions_raw) = parse(file)
local_graph = CloudformationLocalGraph(definitions)
local_graph.build_graph(render_variables=False)
self.assertEqual(len(local_graph.vertices), 57)
self.assertEqual(
len([
v for v in local_graph.vertices
if v.block_type == BlockType.CONDITIONS
]), 2)
self.assertEqual(
len([
v for v in local_graph.vertices
if v.block_type == BlockType.RESOURCE
]), 16)
self.assertEqual(
len([
v for v in local_graph.vertices
if v.block_type == BlockType.PARAMETERS
]), 30)
self.assertEqual(
len([
v for v in local_graph.vertices
if v.block_type == BlockType.OUTPUTS
]), 8)
self.assertEqual(
len([
v for v in local_graph.vertices
if v.block_type == BlockType.MAPPINGS
]), 1)
def test_code_line_extraction(self):
current_dir = os.path.dirname(os.path.realpath(__file__))
# the test data that we'll evaluate against
# line ranges are 1-based
# mapping is file name, to resource index, to resource details
# checking the resource index helps make sure that we are testing what we think we are testing
files = [
f'{current_dir}/cfn_newline_at_end.yaml',
f'{current_dir}/cfn_nonewline_at_end.yaml'
]
resource_properties_mapping = {
files[0]: {
0: {
'name': 'MyDB',
'line_range': [2, 9]
},
1: {
'name': 'MyBucket',
'line_range': [10, 13]
}
},
files[1]: {
0: {
'name': 'MyDB',
'line_range': [2, 9]
},
1: {
'name': 'MyBucket',
'line_range': [11, 14]
}
}
}
for file in files:
cfn_dict, cfn_str = parse(file)
cf_context_parser = ContextParser(file, cfn_dict, cfn_str)
for index, (resource_name,
resource) in enumerate(cfn_dict['Resources'].items()):
# this filters out __startline__ and __endline__ markers
resource_id = cf_context_parser.extract_cf_resource_id(
resource, resource_name)
if resource_id:
# make sure we are checking the right resource
self.assertEqual(
resource_name,
resource_properties_mapping[file][index]['name'])
entity_lines_range, entity_code_lines = cf_context_parser.extract_cf_resource_code_lines(
resource)
self.assertEqual(entity_lines_range[0],
entity_code_lines[0][0])
self.assertEqual(entity_lines_range[1],
entity_code_lines[-1][0])
self.assertEqual(
entity_lines_range,
resource_properties_mapping[file][index]['line_range'])
def run(self, root_folder, external_checks_dir=None, files=None, runner_filter=RunnerFilter()):
report = Report(self.check_type)
definitions = {}
definitions_raw = {}
parsing_errors = {}
files_list = []
if external_checks_dir:
for directory in external_checks_dir:
cfn_registry.load_external_checks(directory)
if files:
for file in files:
(definitions[file], definitions_raw[file]) = parse(file)
if root_folder:
for root, d_names, f_names in os.walk(root_folder):
filter_ignored_directories(d_names)
for file in f_names:
file_ending = os.path.splitext(file)[1]
if file_ending in CF_POSSIBLE_ENDINGS:
files_list.append(os.path.join(root, file))
for file in files_list:
relative_file_path = f'/{os.path.relpath(file, os.path.commonprefix((root_folder, file)))}'
(definitions[relative_file_path], definitions_raw[relative_file_path]) = parse(file)
# Filter out empty files that have not been parsed successfully, and filter out non-CF template files
definitions = {k: v for k, v in definitions.items() if v and v.__contains__("Resources")}
definitions_raw = {k: v for k, v in definitions_raw.items() if k in definitions.keys()}
for cf_file in definitions.keys():
if isinstance(definitions[cf_file], dict_node) and 'Resources' in definitions[cf_file].keys():
cf_context_parser = ContextParser(cf_file, definitions[cf_file], definitions_raw[cf_file])
logging.debug("Template Dump for {}: {}".format(cf_file, definitions[cf_file], indent=2))
cf_context_parser.evaluate_default_refs()
for resource_name, resource in definitions[cf_file]['Resources'].items():
resource_id = cf_context_parser.extract_cf_resource_id(resource, resource_name)
# check that the resource can be parsed as a CF resource
if resource_id:
entity_lines_range, entity_code_lines = cf_context_parser.extract_cf_resource_code_lines(resource)
if entity_lines_range and entity_code_lines:
# TODO - Variable Eval Message!
variable_evaluations = {}
skipped_checks = ContextParser.collect_skip_comments(entity_code_lines)
results = cfn_registry.scan(cf_file, {resource_name: resource}, skipped_checks,
runner_filter)
for check, check_result in results.items():
record = Record(check_id=check.id, check_name=check.name, check_result=check_result,
code_block=entity_code_lines, file_path=cf_file,
file_line_range=entity_lines_range,
resource=resource_id, evaluations=variable_evaluations,
check_class=check.__class__.__module__)
report.add_record(record=record)
return report
def test_parameter_import_lines(self):
# check that when a parameter is imported into a resource, the line numbers of the resource are preserved
current_dir = os.path.dirname(os.path.realpath(__file__))
file = f'{current_dir}/cfn_with_ref.yaml'
definitions, definitions_raw = parse(file)
cf_context_parser = ContextParser(file, definitions, definitions_raw)
resource = definitions['Resources']['ElasticsearchDomain']
entity_lines_range, entity_code_lines = cf_context_parser.extract_cf_resource_code_lines(resource)
self.assertEqual(entity_lines_range[0], 10)
self.assertEqual(entity_lines_range[1], 20)
def test_get_tags(self):
current_dir = os.path.dirname(os.path.realpath(__file__))
scan_file_path = os.path.join(current_dir, "resources", "tags.yaml")
definitions, _ = parse(scan_file_path)
resource_name = 'DataBucket'
resource = definitions['Resources'][resource_name]
entity = {resource_name: resource}
entity_tags = cfn_utils.get_resource_tags(entity)
self.assertEqual(len(entity_tags), 4)
tags = {
'Simple': 'Value',
'Name': '${AWS::AccountId}-data',
'Environment': 'long-form-sub-${account}',
'Account': 'long-form-sub-${account}'
}
for name, value in tags.items():
self.assertEqual(entity_tags[name], value)
resource_name = 'NoTags'
resource = definitions['Resources'][resource_name]
entity = {resource_name: resource}
entity_tags = cfn_utils.get_resource_tags(entity)
self.assertIsNone(entity_tags)
'TerraformServerAutoScalingGroup'
resource_name = 'TerraformServerAutoScalingGroup'
resource = definitions['Resources'][resource_name]
entity = {resource_name: resource}
entity_tags = cfn_utils.get_resource_tags(entity)
self.assertIsNone(entity_tags)
resource_name = 'EKSClusterNodegroup'
resource = definitions['Resources'][resource_name]
entity = {resource_name: resource}
entity_tags = cfn_utils.get_resource_tags(entity)
self.assertEqual(len(entity_tags), 1)
tags = {
'Name':
'{\'Ref\': \'ClusterName\'}-EKS-{\'Ref\': \'NodeGroupName\'}'
}
for name, value in tags.items():
self.assertEqual(entity_tags[name], value)
def test_build_graph_from_definitions(self):
relative_file_path = "./checks/resource/aws/example_APIGatewayXray/APIGatewayXray-PASSED.yaml"
definitions = {}
file = os.path.realpath(os.path.join(TEST_DIRNAME, relative_file_path))
(definitions[relative_file_path], definitions_raw) = parse(file)
graph_manager = CloudformationGraphManager(db_connector=NetworkxConnector())
local_graph = graph_manager.build_graph_from_definitions(definitions)
self.assertEqual(1, len(local_graph.vertices))
resource_vertex = local_graph.vertices[0]
self.assertEqual("AWS::ApiGateway::Stage.Enabled", resource_vertex.name)
self.assertEqual("AWS::ApiGateway::Stage.Enabled", resource_vertex.id)
self.assertEqual(BlockType.RESOURCE, resource_vertex.block_type)
self.assertEqual("CloudFormation", resource_vertex.source)
self.assertDictEqual(definitions[relative_file_path]["Resources"]["Enabled"]["Properties"], resource_vertex.attributes)
def test_build_graph_with_single_resource(self):
relative_file_path = "../../checks/resource/aws/example_APIGatewayXray/APIGatewayXray-PASSED.yaml"
definitions = {}
file = os.path.realpath(os.path.join(TEST_DIRNAME, relative_file_path))
(definitions[relative_file_path], definitions_raw) = parse(file)
local_graph = CloudformationLocalGraph(definitions)
local_graph.build_graph(render_variables=False)
self.assertEqual(1, len(local_graph.vertices))
self.assertEqual(0, len(local_graph.edges))
resource_vertex = local_graph.vertices[0]
self.assertEqual("AWS::ApiGateway::Stage.MyStage", resource_vertex.name)
self.assertEqual("AWS::ApiGateway::Stage.MyStage", resource_vertex.id)
self.assertEqual(BlockType.RESOURCE, resource_vertex.block_type)
self.assertEqual("CloudFormation", resource_vertex.source)
self.assertDictEqual(definitions[relative_file_path]["Resources"]["MyStage"]["Properties"],
resource_vertex.attributes)
def run(self,
root_folder,
external_checks_dir=None,
files=None,
runner_filter=RunnerFilter(),
collect_skip_comments=True):
report = Report(self.check_type)
definitions = {}
definitions_raw = {}
parsing_errors = {}
files_list = []
if external_checks_dir:
for directory in external_checks_dir:
cfn_registry.load_external_checks(directory, runner_filter)
if files:
for file in files:
(definitions[file], definitions_raw[file]) = parse(file)
if root_folder:
for root, d_names, f_names in os.walk(root_folder):
filter_ignored_directories(d_names)
for file in f_names:
file_ending = os.path.splitext(file)[1]
if file_ending in CF_POSSIBLE_ENDINGS:
files_list.append(os.path.join(root, file))
for file in files_list:
relative_file_path = f'/{os.path.relpath(file, os.path.commonprefix((root_folder, file)))}'
try:
(definitions[relative_file_path],
definitions_raw[relative_file_path]) = parse(file)
except TypeError:
logging.info(
f'CloudFormation skipping {file} as it is not a valid CF template'
)
# Filter out empty files that have not been parsed successfully, and filter out non-CF template files
definitions = {
k: v
for k, v in definitions.items()
if v and isinstance(v, dict_node) and v.__contains__("Resources")
and isinstance(v["Resources"], dict_node)
}
definitions_raw = {
k: v
for k, v in definitions_raw.items() if k in definitions.keys()
}
for cf_file in definitions.keys():
# There are a few cases here. If -f was used, there could be a leading / because it's an absolute path,
# or there will be no leading slash; root_folder will always be none.
# If -d is used, root_folder will be the value given, and -f will start with a / (hardcoded above).
# The goal here is simply to get a valid path to the file (which cf_file does not always give).
if cf_file[0] == '/':
path_to_convert = (root_folder +
cf_file) if root_folder else cf_file
else:
path_to_convert = (os.path.join(
root_folder, cf_file)) if root_folder else cf_file
file_abs_path = os.path.abspath(path_to_convert)
if isinstance(
definitions[cf_file],
dict_node) and 'Resources' in definitions[cf_file].keys():
cf_context_parser = ContextParser(cf_file,
definitions[cf_file],
definitions_raw[cf_file])
logging.debug("Template Dump for {}: {}".format(
cf_file, definitions[cf_file], indent=2))
cf_context_parser.evaluate_default_refs()
for resource_name, resource in definitions[cf_file][
'Resources'].items():
resource_id = cf_context_parser.extract_cf_resource_id(
resource, resource_name)
# check that the resource can be parsed as a CF resource
if resource_id:
entity_lines_range, entity_code_lines = cf_context_parser.extract_cf_resource_code_lines(
resource)
if entity_lines_range and entity_code_lines:
# TODO - Variable Eval Message!
variable_evaluations = {}
skipped_checks = ContextParser.collect_skip_comments(
entity_code_lines)
results = cfn_registry.scan(
cf_file, {resource_name: resource},
skipped_checks, runner_filter)
for check, check_result in results.items():
record = Record(
check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=cf_file,
file_line_range=entity_lines_range,
resource=resource_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__,
file_abs_path=file_abs_path)
report.add_record(record=record)
return report
def run(self,
root_folder,
external_checks_dir=None,
files=None,
runner_filter=RunnerFilter()):
report = Report(self.check_type)
definitions = {}
definitions_raw = {}
parsing_errors = {}
files_list = []
if external_checks_dir:
for directory in external_checks_dir:
resource_registry.load_external_checks(directory)
if files:
for file in files:
(definitions[file], definitions_raw[file]) = parse(file)
if root_folder:
for root, d_names, f_names in os.walk(root_folder):
for file in f_names:
file_ending = os.path.splitext(file)[1]
if file_ending in CF_POSSIBLE_ENDINGS:
files_list.append(os.path.join(root, file))
for file in files_list:
relative_file_path = f'/{os.path.relpath(file, os.path.commonprefix((root_folder, file)))}'
(definitions[relative_file_path],
definitions_raw[relative_file_path]) = parse(file)
# Filter out empty files that have not been parsed successfully, and filter out non-CF template files
definitions = {
k: v
for k, v in definitions.items()
if v and v.__contains__("Resources")
}
definitions_raw = {
k: v
for k, v in definitions_raw.items() if k in definitions.keys()
}
for cf_file in definitions.keys():
if isinstance(
definitions[cf_file],
dict_node) and 'Resources' in definitions[cf_file].keys():
logging.debug("Template Dump for {}: {}".format(
cf_file, definitions[cf_file], indent=2))
# Get Parameter Defaults - Locate Refs in Template
refs = []
refs.extend(
self._search_deep_keys('Ref', definitions[cf_file], []))
for ref in refs:
refname = ref.pop()
ref.pop() # Get rid of the 'Ref' dict key
if 'Parameters' in definitions[cf_file].keys(
) and refname in definitions[cf_file]['Parameters'].keys():
# TODO refactor into evaluations
if 'Default' in definitions[cf_file]['Parameters'][
refname].keys():
logging.debug(
"Replacing Ref {} in file {} with default parameter value: {}"
.format(
refname, cf_file, definitions[cf_file]
['Parameters'][refname]['Default']))
_set_in_dict(
definitions[cf_file], ref, definitions[cf_file]
['Parameters'][refname]['Default'])
## TODO - Add Variable Eval Message for Output
# Output in Checkov looks like this:
# Variable versioning (of /.) evaluated to value "True" in expression: enabled = ${var.versioning}
for resource_name, resource in definitions[cf_file][
'Resources'].items():
if resource_name == '__startline__' or resource_name == '__endline__':
continue
resource_id = f"{resource['Type']}.{resource_name}"
# TODO refactor into context parsing
find_lines_result_list = list(
find_lines(resource, '__startline__'))
if len(find_lines_result_list) >= 1:
start_line = min(find_lines_result_list)
end_line = max(
list(find_lines(resource, '__endline__')))
entity_lines_range = [start_line, end_line - 1]
entity_code_lines = definitions_raw[cf_file][
start_line - 1:end_line - 1]
# TODO - Variable Eval Message!
variable_evaluations = {}
skipped_checks = []
for line in entity_code_lines:
skip_search = re.search(COMMENT_REGEX, str(line))
if skip_search:
skipped_checks.append({
'id':
skip_search.group(2),
'suppress_comment':
skip_search.group(3)[1:]
if skip_search.group(3) else
"No comment provided"
})
results = resource_registry.scan(
cf_file, {resource_name: resource}, skipped_checks,
runner_filter)
for check, check_result in results.items():
### TODO - Need to get entity_code_lines and entity_lines_range
record = Record(
check_id=check.id,
check_name=check.name,
check_result=check_result,
code_block=entity_code_lines,
file_path=cf_file,
file_line_range=entity_lines_range,
resource=resource_id,
evaluations=variable_evaluations,
check_class=check.__class__.__module__)
report.add_record(record=record)
return report
def _parse_file(file):
parsing_errors = {}
result = parse(file, parsing_errors)
return (file, result), parsing_errors
