def run(self, root_folder, external_checks_dir=None, files=None, runner_filter=RunnerFilter()): report = Report(self.check_type) definitions = {} definitions_raw = {} parsing_errors = {} files_list = [] if external_checks_dir: for directory in external_checks_dir: cfn_registry.load_external_checks(directory) if files: for file in files: (definitions[file], definitions_raw[file]) = parse(file) if root_folder: for root, d_names, f_names in os.walk(root_folder): filter_ignored_directories(d_names) for file in f_names: file_ending = os.path.splitext(file)[1] if file_ending in CF_POSSIBLE_ENDINGS: files_list.append(os.path.join(root, file)) for file in files_list: relative_file_path = f'/{os.path.relpath(file, os.path.commonprefix((root_folder, file)))}' (definitions[relative_file_path], definitions_raw[relative_file_path]) = parse(file) # Filter out empty files that have not been parsed successfully, and filter out non-CF template files definitions = {k: v for k, v in definitions.items() if v and v.__contains__("Resources")} definitions_raw = {k: v for k, v in definitions_raw.items() if k in definitions.keys()} for cf_file in definitions.keys(): if isinstance(definitions[cf_file], dict_node) and 'Resources' in definitions[cf_file].keys(): cf_context_parser = ContextParser(cf_file, definitions[cf_file], definitions_raw[cf_file]) logging.debug("Template Dump for {}: {}".format(cf_file, definitions[cf_file], indent=2)) cf_context_parser.evaluate_default_refs() for resource_name, resource in definitions[cf_file]['Resources'].items(): resource_id = cf_context_parser.extract_cf_resource_id(resource, resource_name) # check that the resource can be parsed as a CF resource if resource_id: entity_lines_range, entity_code_lines = cf_context_parser.extract_cf_resource_code_lines(resource) if entity_lines_range and entity_code_lines: # TODO - Variable Eval Message! variable_evaluations = {} skipped_checks = ContextParser.collect_skip_comments(entity_code_lines) results = cfn_registry.scan(cf_file, {resource_name: resource}, skipped_checks, runner_filter) for check, check_result in results.items(): record = Record(check_id=check.id, check_name=check.name, check_result=check_result, code_block=entity_code_lines, file_path=cf_file, file_line_range=entity_lines_range, resource=resource_id, evaluations=variable_evaluations, check_class=check.__class__.__module__) report.add_record(record=record) return report
def run( self, root_folder: str, external_checks_dir: Optional[List[str]] = None, files: Optional[List[str]] = None, runner_filter: RunnerFilter = RunnerFilter(), collect_skip_comments: bool = True, ) -> Report: report = Report(self.check_type) parsing_errors = {} if self.context is None or self.definitions is None or self.breadcrumbs is None: self.definitions, self.definitions_raw = create_definitions( root_folder, files, runner_filter, parsing_errors) if external_checks_dir: for directory in external_checks_dir: cfn_registry.load_external_checks(directory) self.graph_registry.load_external_checks(directory) self.context = build_definitions_context(self.definitions, self.definitions_raw, root_folder) logging.info("creating cloudformation graph") local_graph = self.graph_manager.build_graph_from_definitions( self.definitions) for vertex in local_graph.vertices: if vertex.block_type == BlockType.RESOURCE: report.add_resource(f'{vertex.path}:{vertex.id}') self.graph_manager.save_graph(local_graph) self.definitions, self.breadcrumbs = convert_graph_vertices_to_definitions( local_graph.vertices, root_folder) # TODO: replace with real graph rendering for cf_file in self.definitions.keys(): file_definition = self.definitions.get(cf_file, None) file_definition_raw = self.definitions_raw.get(cf_file, None) if file_definition is not None and file_definition_raw is not None: cf_context_parser = ContextParser(cf_file, file_definition, file_definition_raw) logging.debug("Template Dump for {}: {}".format( cf_file, json.dumps(file_definition, indent=2, default=str))) cf_context_parser.evaluate_default_refs() report.add_parsing_errors(list(parsing_errors.keys())) # run checks self.check_definitions(root_folder, runner_filter, report) # run graph checks graph_report = self.get_graph_checks_report(root_folder, runner_filter) merge_reports(report, graph_report) return report
def run(self, root_folder, external_checks_dir=None, files=None, runner_filter=RunnerFilter(), collect_skip_comments=True): report = Report(self.check_type) definitions = {} definitions_raw = {} parsing_errors = {} files_list = [] if external_checks_dir: for directory in external_checks_dir: cfn_registry.load_external_checks(directory, runner_filter) if files: for file in files: (definitions[file], definitions_raw[file]) = parse(file) if root_folder: for root, d_names, f_names in os.walk(root_folder): filter_ignored_directories(d_names) for file in f_names: file_ending = os.path.splitext(file)[1] if file_ending in CF_POSSIBLE_ENDINGS: files_list.append(os.path.join(root, file)) for file in files_list: relative_file_path = f'/{os.path.relpath(file, os.path.commonprefix((root_folder, file)))}' try: (definitions[relative_file_path], definitions_raw[relative_file_path]) = parse(file) except TypeError: logging.info( f'CloudFormation skipping {file} as it is not a valid CF template' ) # Filter out empty files that have not been parsed successfully, and filter out non-CF template files definitions = { k: v for k, v in definitions.items() if v and isinstance(v, dict_node) and v.__contains__("Resources") and isinstance(v["Resources"], dict_node) } definitions_raw = { k: v for k, v in definitions_raw.items() if k in definitions.keys() } for cf_file in definitions.keys(): # There are a few cases here. If -f was used, there could be a leading / because it's an absolute path, # or there will be no leading slash; root_folder will always be none. # If -d is used, root_folder will be the value given, and -f will start with a / (hardcoded above). # The goal here is simply to get a valid path to the file (which cf_file does not always give). if cf_file[0] == '/': path_to_convert = (root_folder + cf_file) if root_folder else cf_file else: path_to_convert = (os.path.join( root_folder, cf_file)) if root_folder else cf_file file_abs_path = os.path.abspath(path_to_convert) if isinstance( definitions[cf_file], dict_node) and 'Resources' in definitions[cf_file].keys(): cf_context_parser = ContextParser(cf_file, definitions[cf_file], definitions_raw[cf_file]) logging.debug("Template Dump for {}: {}".format( cf_file, definitions[cf_file], indent=2)) cf_context_parser.evaluate_default_refs() for resource_name, resource in definitions[cf_file][ 'Resources'].items(): resource_id = cf_context_parser.extract_cf_resource_id( resource, resource_name) # check that the resource can be parsed as a CF resource if resource_id: entity_lines_range, entity_code_lines = cf_context_parser.extract_cf_resource_code_lines( resource) if entity_lines_range and entity_code_lines: # TODO - Variable Eval Message! variable_evaluations = {} skipped_checks = ContextParser.collect_skip_comments( entity_code_lines) results = cfn_registry.scan( cf_file, {resource_name: resource}, skipped_checks, runner_filter) for check, check_result in results.items(): record = Record( check_id=check.id, check_name=check.name, check_result=check_result, code_block=entity_code_lines, file_path=cf_file, file_line_range=entity_lines_range, resource=resource_id, evaluations=variable_evaluations, check_class=check.__class__.__module__, file_abs_path=file_abs_path) report.add_record(record=record) return report