def run(self,
root_folder,
external_checks_dir=None,
files=None,
runner_filter=RunnerFilter(),
collect_skip_comments=True):
report = Report(self.check_type)
parsing_errors = {}
self.load_external_checks(external_checks_dir)
scan_hcl = should_scan_hcl_files()
if self.context is None or self.definitions is None or self.breadcrumbs is None:
self.definitions = {}
logging.info(
"Scanning root folder and producing fresh tf_definitions and context"
)
if root_folder:
root_folder = os.path.abspath(root_folder)
local_graph, tf_definitions = self.graph_manager.build_graph_from_source_directory(
source_dir=root_folder,
local_graph_class=self.graph_class,
download_external_modules=runner_filter.
download_external_modules,
external_modules_download_path=runner_filter.
external_modules_download_path,
parsing_errors=parsing_errors,
excluded_paths=runner_filter.excluded_paths,
vars_files=runner_filter.var_files)
elif files:
files = [os.path.abspath(file) for file in files]
root_folder = os.path.split(os.path.commonprefix(files))[0]
self.parser.evaluate_variables = False
self._parse_files(files, scan_hcl, parsing_errors)
local_graph = self.graph_manager.build_graph_from_definitions(
self.definitions)
else:
raise Exception(
"Root directory was not specified, files were not specified"
)
for vertex in local_graph.vertices:
if vertex.block_type == BlockType.RESOURCE:
report.add_resource(f'{vertex.path}:{vertex.id}')
self.graph_manager.save_graph(local_graph)
self.definitions, self.breadcrumbs = convert_graph_vertices_to_tf_definitions(
local_graph.vertices, root_folder)
else:
logging.info("Scanning root folder using existing tf_definitions")
self.check_tf_definition(report, root_folder, runner_filter,
collect_skip_comments)
report.add_parsing_errors(list(parsing_errors.keys()))
graph_report = self.get_graph_checks_report(root_folder, runner_filter)
merge_reports(report, graph_report)
report = remove_duplicate_results(report)
return report
def run(self,
root_folder,
external_checks_dir=None,
files=None,
runner_filter=RunnerFilter(),
collect_skip_comments=True):
report = Report(self.check_type)
parsing_errors = {}
self.load_external_checks(external_checks_dir)
if self.definitions_context is None or self.tf_definitions is None or self.breadcrumbs is None:
self.tf_definitions = {}
logging.info(
"Scanning root folder and producing fresh tf_definitions and context"
)
if root_folder:
root_folder = os.path.abspath(root_folder)
local_graph, tf_definitions = \
self.graph_manager.build_graph_from_source_directory(root_folder,
local_graph_class=self.graph_class,
download_external_modules=runner_filter.download_external_modules,
parsing_errors=parsing_errors, excluded_paths=runner_filter.excluded_paths)
elif files:
files = [os.path.abspath(file) for file in files]
root_folder = os.path.split(os.path.commonprefix(files))[0]
self.parser.evaluate_variables = False
for file in files:
if file.endswith(".tf"):
file_parsing_errors = {}
parse_result = self.parser.parse_file(
file=file, parsing_errors=file_parsing_errors)
if parse_result is not None:
self.tf_definitions[file] = parse_result
if file_parsing_errors:
parsing_errors.update(file_parsing_errors)
continue
local_graph = self.graph_manager.build_graph_from_tf_definitions(
self.tf_definitions)
else:
raise Exception(
"Root directory was not specified, files were not specified"
)
self.graph_manager.save_graph(local_graph)
self.tf_definitions, self.breadcrumbs = convert_graph_vertices_to_tf_definitions(
local_graph.vertices, root_folder)
else:
logging.info(f"Scanning root folder using existing tf_definitions")
self.check_tf_definition(report, root_folder, runner_filter,
collect_skip_comments)
report.add_parsing_errors(parsing_errors.keys())
graph_report = self.get_graph_checks_report(root_folder, runner_filter)
merge_reports(report, graph_report)
return report
def test_vertices_from_local_graph(self):
resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME,
'../resources/variable_rendering/render_from_module_vpc'))
hcl_config_parser = Parser()
module, module_dependency_map, _ = hcl_config_parser.parse_hcl_module(resources_dir,
self.source)
local_graph = LocalGraph(module, module_dependency_map)
local_graph._create_vertices()
tf_definitions, breadcrumbs = convert_graph_vertices_to_tf_definitions(local_graph.vertices, resources_dir)
self.assertIsNotNone(tf_definitions)
self.assertIsNotNone(breadcrumbs)
def test_blocks_from_local_graph_module(self):
resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/modules/stacks'))
graph_manager = TerraformGraphManager(NetworkxConnector())
local_graph, tf = graph_manager.build_graph_from_source_directory(resources_dir, render_variables=True)
tf, _ = convert_graph_vertices_to_tf_definitions(local_graph.vertices, resources_dir)
found_results = 0
for key, value in tf.items():
if key.startswith(os.path.join(os.path.dirname(resources_dir), 's3_inner_modules', 'inner', 'main.tf')):
conf = value['resource'][0]['aws_s3_bucket']['inner_s3']
if 'stage/main' in key or 'prod/main' in key:
self.assertTrue(conf['versioning'][0]['enabled'][0])
found_results += 1
elif 'test/main' in key:
self.assertFalse(conf['versioning'][0]['enabled'][0])
found_results += 1
self.assertEqual(found_results, 3)
def go(self,
dir_name,
different_expected=None,
replace_expected=False,
vars_files=None):
os.environ['RENDER_VARIABLES_ASYNC'] = 'False'
os.environ['LOG_LEVEL'] = 'INFO'
different_expected = {} if not different_expected else different_expected
resources_dir = os.path.realpath(
os.path.join(TEST_DIRNAME,
'../../parser/resources/parser_scenarios', dir_name))
if vars_files:
vars_files = [os.path.join(resources_dir, f) for f in vars_files]
graph_manager = TerraformGraphManager(dir_name, [dir_name])
local_graph, _ = graph_manager.build_graph_from_source_directory(
resources_dir, render_variables=True, vars_files=vars_files)
got_tf_definitions, _ = convert_graph_vertices_to_tf_definitions(
local_graph.vertices, resources_dir)
expected = load_expected(replace_expected, dir_name, resources_dir)
for expected_file, expected_block_type_dict in expected.items():
module_removed_path = expected_file
got_file = got_tf_definitions.get(module_removed_path)
self.assertIsNotNone(got_file)
for expected_block_type, expected_block_type_list in expected_block_type_dict.items(
):
got_block_type_list = got_file.get(expected_block_type)
self.assertIsNotNone(got_block_type_list)
for expected_block_dict in expected_block_type_list:
for expected_block_name, expected_block_val in expected_block_dict.items(
):
if expected_block_type != BlockType.RESOURCE:
found = self.match_blocks(expected_block_val,
different_expected,
got_block_type_list,
expected_block_name)
else:
found = self.match_resources(
expected_block_val, different_expected,
got_block_type_list, expected_block_name)
self.assertTrue(
found,
f"expected to find block {expected_block_dict} from file {expected_file} in graph"
)
