def run(self, root_folder, external_checks_dir=None, files=None, runner_filter=RunnerFilter(), collect_skip_comments=True): report = Report(self.check_type) parsing_errors = {} self.load_external_checks(external_checks_dir) scan_hcl = should_scan_hcl_files() if self.context is None or self.definitions is None or self.breadcrumbs is None: self.definitions = {} logging.info( "Scanning root folder and producing fresh tf_definitions and context" ) if root_folder: root_folder = os.path.abspath(root_folder) local_graph, tf_definitions = self.graph_manager.build_graph_from_source_directory( source_dir=root_folder, local_graph_class=self.graph_class, download_external_modules=runner_filter. download_external_modules, external_modules_download_path=runner_filter. external_modules_download_path, parsing_errors=parsing_errors, excluded_paths=runner_filter.excluded_paths, vars_files=runner_filter.var_files) elif files: files = [os.path.abspath(file) for file in files] root_folder = os.path.split(os.path.commonprefix(files))[0] self.parser.evaluate_variables = False self._parse_files(files, scan_hcl, parsing_errors) local_graph = self.graph_manager.build_graph_from_definitions( self.definitions) else: raise Exception( "Root directory was not specified, files were not specified" ) for vertex in local_graph.vertices: if vertex.block_type == BlockType.RESOURCE: report.add_resource(f'{vertex.path}:{vertex.id}') self.graph_manager.save_graph(local_graph) self.definitions, self.breadcrumbs = convert_graph_vertices_to_tf_definitions( local_graph.vertices, root_folder) else: logging.info("Scanning root folder using existing tf_definitions") self.check_tf_definition(report, root_folder, runner_filter, collect_skip_comments) report.add_parsing_errors(list(parsing_errors.keys())) graph_report = self.get_graph_checks_report(root_folder, runner_filter) merge_reports(report, graph_report) report = remove_duplicate_results(report) return report
def run(self, root_folder, external_checks_dir=None, files=None, runner_filter=RunnerFilter(), collect_skip_comments=True): report = Report(self.check_type) parsing_errors = {} self.load_external_checks(external_checks_dir) if self.definitions_context is None or self.tf_definitions is None or self.breadcrumbs is None: self.tf_definitions = {} logging.info( "Scanning root folder and producing fresh tf_definitions and context" ) if root_folder: root_folder = os.path.abspath(root_folder) local_graph, tf_definitions = \ self.graph_manager.build_graph_from_source_directory(root_folder, local_graph_class=self.graph_class, download_external_modules=runner_filter.download_external_modules, parsing_errors=parsing_errors, excluded_paths=runner_filter.excluded_paths) elif files: files = [os.path.abspath(file) for file in files] root_folder = os.path.split(os.path.commonprefix(files))[0] self.parser.evaluate_variables = False for file in files: if file.endswith(".tf"): file_parsing_errors = {} parse_result = self.parser.parse_file( file=file, parsing_errors=file_parsing_errors) if parse_result is not None: self.tf_definitions[file] = parse_result if file_parsing_errors: parsing_errors.update(file_parsing_errors) continue local_graph = self.graph_manager.build_graph_from_tf_definitions( self.tf_definitions) else: raise Exception( "Root directory was not specified, files were not specified" ) self.graph_manager.save_graph(local_graph) self.tf_definitions, self.breadcrumbs = convert_graph_vertices_to_tf_definitions( local_graph.vertices, root_folder) else: logging.info(f"Scanning root folder using existing tf_definitions") self.check_tf_definition(report, root_folder, runner_filter, collect_skip_comments) report.add_parsing_errors(parsing_errors.keys()) graph_report = self.get_graph_checks_report(root_folder, runner_filter) merge_reports(report, graph_report) return report
def test_vertices_from_local_graph(self): resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/variable_rendering/render_from_module_vpc')) hcl_config_parser = Parser() module, module_dependency_map, _ = hcl_config_parser.parse_hcl_module(resources_dir, self.source) local_graph = LocalGraph(module, module_dependency_map) local_graph._create_vertices() tf_definitions, breadcrumbs = convert_graph_vertices_to_tf_definitions(local_graph.vertices, resources_dir) self.assertIsNotNone(tf_definitions) self.assertIsNotNone(breadcrumbs)
def test_blocks_from_local_graph_module(self): resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/modules/stacks')) graph_manager = TerraformGraphManager(NetworkxConnector()) local_graph, tf = graph_manager.build_graph_from_source_directory(resources_dir, render_variables=True) tf, _ = convert_graph_vertices_to_tf_definitions(local_graph.vertices, resources_dir) found_results = 0 for key, value in tf.items(): if key.startswith(os.path.join(os.path.dirname(resources_dir), 's3_inner_modules', 'inner', 'main.tf')): conf = value['resource'][0]['aws_s3_bucket']['inner_s3'] if 'stage/main' in key or 'prod/main' in key: self.assertTrue(conf['versioning'][0]['enabled'][0]) found_results += 1 elif 'test/main' in key: self.assertFalse(conf['versioning'][0]['enabled'][0]) found_results += 1 self.assertEqual(found_results, 3)
def go(self, dir_name, different_expected=None, replace_expected=False, vars_files=None): os.environ['RENDER_VARIABLES_ASYNC'] = 'False' os.environ['LOG_LEVEL'] = 'INFO' different_expected = {} if not different_expected else different_expected resources_dir = os.path.realpath( os.path.join(TEST_DIRNAME, '../../parser/resources/parser_scenarios', dir_name)) if vars_files: vars_files = [os.path.join(resources_dir, f) for f in vars_files] graph_manager = TerraformGraphManager(dir_name, [dir_name]) local_graph, _ = graph_manager.build_graph_from_source_directory( resources_dir, render_variables=True, vars_files=vars_files) got_tf_definitions, _ = convert_graph_vertices_to_tf_definitions( local_graph.vertices, resources_dir) expected = load_expected(replace_expected, dir_name, resources_dir) for expected_file, expected_block_type_dict in expected.items(): module_removed_path = expected_file got_file = got_tf_definitions.get(module_removed_path) self.assertIsNotNone(got_file) for expected_block_type, expected_block_type_list in expected_block_type_dict.items( ): got_block_type_list = got_file.get(expected_block_type) self.assertIsNotNone(got_block_type_list) for expected_block_dict in expected_block_type_list: for expected_block_name, expected_block_val in expected_block_dict.items( ): if expected_block_type != BlockType.RESOURCE: found = self.match_blocks(expected_block_val, different_expected, got_block_type_list, expected_block_name) else: found = self.match_resources( expected_block_val, different_expected, got_block_type_list, expected_block_name) self.assertTrue( found, f"expected to find block {expected_block_dict} from file {expected_file} in graph" )