def _parse_file(filename): try: return filename, parse(filename) except (TypeError, ValueError) as e: logging.warning( f"Kubernetes skipping {filename} as it is not a valid Kubernetes template\n{e}" )
def test_build_graph_with_multi_resources(self): relative_file_path = "../checks/example_DefaultNamespace/default-k8s-service-and-sa-PASSED2.yaml" definitions = {} file = os.path.realpath(os.path.join(TEST_DIRNAME, relative_file_path)) (definitions[relative_file_path], definitions_raw) = parse(file) local_graph = KubernetesLocalGraph(definitions) local_graph.build_graph(render_variables=False) self.assertEqual(4, len(local_graph.vertices))
def test_build_graph_with_single_resource(self): relative_file_path = "../checks/example_AllowedCapabilities/cronjob-PASSED.yaml" definitions = {} file = os.path.realpath(os.path.join(TEST_DIRNAME, relative_file_path)) (definitions[relative_file_path], definitions_raw) = parse(file) resource = definitions[relative_file_path][0] local_graph = KubernetesLocalGraph(definitions) local_graph.build_graph(render_variables=False) self.assertEqual(1, len(local_graph.vertices)) self.assert_vertex(local_graph.vertices[0], resource)
def test_build_graph_from_definitions(self): relative_file_path = "../checks/example_AllowedCapabilities/cronjob-PASSED.yaml" definitions = {} file = os.path.realpath(os.path.join(TEST_DIRNAME, relative_file_path)) (definitions[relative_file_path], definitions_raw) = parse(file) resource = definitions[relative_file_path][0] graph_manager = KubernetesGraphManager( db_connector=NetworkxConnector()) local_graph = graph_manager.build_graph_from_definitions(definitions) self.assertEqual(1, len(local_graph.vertices)) self.assert_vertex(local_graph.vertices[0], resource)
def run(self, root_folder, external_checks_dir=None, files=None, runner_filter=RunnerFilter(), collect_skip_comments=True, helmChart=None): report = Report(self.check_type) definitions = {} definitions_raw = {} parsing_errors = {} files_list = [] if external_checks_dir: for directory in external_checks_dir: registry.load_external_checks(directory, runner_filter) if files: for file in files: parse_result = parse(file) if parse_result: (definitions[file], definitions_raw[file]) = parse_result if root_folder: for root, d_names, f_names in os.walk(root_folder): filter_ignored_directories(d_names) for file in f_names: file_ending = os.path.splitext(file)[1] if file_ending in K8_POSSIBLE_ENDINGS: full_path = os.path.join(root, file) if "/." not in full_path and file not in [ 'package.json', 'package-lock.json' ]: # skip temp directories files_list.append(full_path) for file in files_list: relative_file_path = f'/{os.path.relpath(file, os.path.commonprefix((root_folder, file)))}' parse_result = parse(file) if parse_result: (definitions[relative_file_path], definitions_raw[relative_file_path]) = parse_result for k8_file in definitions.keys(): # There are a few cases here. If -f was used, there could be a leading / because it's an absolute path, # or there will be no leading slash; root_folder will always be none. # If -d is used, root_folder will be the value given, and -f will start with a / (hardcoded above). # The goal here is simply to get a valid path to the file (which sls_file does not always give). if k8_file[0] == '/': path_to_convert = (root_folder + k8_file) if root_folder else k8_file else: path_to_convert = (os.path.join( root_folder, k8_file)) if root_folder else k8_file file_abs_path = os.path.abspath(path_to_convert) if definitions[k8_file]: for i in range(len(definitions[k8_file])): if (not 'apiVersion' in definitions[k8_file][i].keys() ) and (not 'kind' in definitions[k8_file][i].keys()): continue logging.debug("Template Dump for {}: {}".format( k8_file, definitions[k8_file][i], indent=2)) entity_conf = definitions[k8_file][i] # Split out resources if entity kind is List if entity_conf["kind"] == "List": for item in entity_conf["items"]: definitions[k8_file].append(item) for i in range(len(definitions[k8_file])): if (not 'apiVersion' in definitions[k8_file][i].keys() ) and (not 'kind' in definitions[k8_file][i].keys()): continue logging.debug("Template Dump for {}: {}".format( k8_file, definitions[k8_file][i], indent=2)) entity_conf = definitions[k8_file][i] if entity_conf["kind"] == "List": continue # Skip entity without metadata["name"] if "metadata" in entity_conf: if isinstance( entity_conf["metadata"], int) or not "name" in entity_conf["metadata"]: continue else: continue # Skip entity with parent (metadata["ownerReferences"]) in runtime # We will alert in runtime only if "ownerReferences" in entity_conf["metadata"] and \ entity_conf["metadata"]["ownerReferences"] is not None: continue # Append containers and initContainers to definitions list for type in ["containers", "initContainers"]: containers = [] if entity_conf["kind"] == "CustomResourceDefinition": continue containers = self._search_deep_keys( type, entity_conf, []) if not containers: continue containers = containers.pop() #containers.insert(0,entity_conf['kind']) containerDef = {} namespace = "" if "namespace" in entity_conf["metadata"]: namespace = entity_conf["metadata"]["namespace"] else: namespace = "default" containerDef["containers"] = containers.pop() if containerDef["containers"] is not None: for cd in containerDef["containers"]: i = containerDef["containers"].index(cd) containerDef["containers"][i][ "apiVersion"] = entity_conf["apiVersion"] containerDef["containers"][i]["kind"] = type containerDef["containers"][i][ "parent"] = "{}.{}.{} (container {})".format( entity_conf["kind"], entity_conf["metadata"]["name"], namespace, str(i)) containerDef["containers"][i][ "parent_metadata"] = entity_conf[ "metadata"] definitions[k8_file].extend( containerDef["containers"]) # Run for each definition included added container definitions for i in range(len(definitions[k8_file])): if (not 'apiVersion' in definitions[k8_file][i].keys() ) and (not 'kind' in definitions[k8_file][i].keys()): continue logging.debug("Template Dump for {}: {}".format( k8_file, definitions[k8_file][i], indent=2)) entity_conf = definitions[k8_file][i] if entity_conf["kind"] == "List": continue if isinstance(entity_conf["kind"], int): continue # Skip entity without metadata["name"] or parent_metadata["name"] if not any(x in entity_conf["kind"] for x in ["containers", "initContainers"]): if "metadata" in entity_conf: if isinstance( entity_conf["metadata"], int ) or not "name" in entity_conf["metadata"]: continue else: continue # Skip entity with parent (metadata["ownerReferences"]) in runtime # We will alert in runtime only if "metadata" in entity_conf: if "ownerReferences" in entity_conf["metadata"] and \ entity_conf["metadata"]["ownerReferences"] is not None: continue # Skip Kustomization Templates (for now) if entity_conf["kind"] == "Kustomization": continue skipped_checks = get_skipped_checks(entity_conf) results = registry.scan(k8_file, entity_conf, skipped_checks, runner_filter) # TODO refactor into context parsing find_lines_result_list = list( find_lines(entity_conf, '__startline__')) start_line = entity_conf["__startline__"] end_line = entity_conf["__endline__"] if start_line == end_line: entity_lines_range = [start_line, end_line] entity_code_lines = definitions_raw[k8_file][ start_line - 1:end_line] else: entity_lines_range = [start_line, end_line - 1] entity_code_lines = definitions_raw[k8_file][ start_line - 1:end_line - 1] # TODO? - Variable Eval Message! variable_evaluations = {} for check, check_result in results.items(): record = Record( check_id=check.id, check_name=check.name, check_result=check_result, code_block=entity_code_lines, file_path=k8_file, file_line_range=entity_lines_range, resource=check.get_resource_id(entity_conf), evaluations=variable_evaluations, check_class=check.__class__.__module__, file_abs_path=file_abs_path) report.add_record(record=record) return report
def run(self, root_folder, external_checks_dir=None, files=None, runner_filter=RunnerFilter()): report = Report(self.check_type) definitions = {} definitions_raw = {} parsing_errors = {} files_list = [] if external_checks_dir: for directory in external_checks_dir: registry.load_external_checks(directory) if files: for file in files: parse_result = parse(file) if parse_result: (definitions[file], definitions_raw[file]) = parse_result if root_folder: for root, d_names, f_names in os.walk(root_folder): for file in f_names: file_ending = os.path.splitext(file)[1] if file_ending in K8_POSSIBLE_ENDINGS: full_path = os.path.join(root, file) if 'node_modules' not in full_path and "/." not in full_path and file not in [ 'package.json', 'package-lock.json' ]: # skip temp directories files_list.append(full_path) for file in files_list: relative_file_path = f'/{os.path.relpath(file, os.path.commonprefix((root_folder, file)))}' parse_result = parse(file) if parse_result: (definitions[relative_file_path], definitions_raw[relative_file_path]) = parse_result # Filter out empty files that have not been parsed successfully, and filter out non-K8 template files #definitions = {k: v for k, v in definitions.items() if # v and (v.__contains__("apiVersion") and v.__contains__("kind"))} #definitions_raw = {k: v for k, v in definitions_raw.items() if k in definitions.keys()} for k8_file in definitions.keys(): if definitions[k8_file]: for i in range(len(definitions[k8_file])): if (not 'apiVersion' in definitions[k8_file][i].keys() ) and (not 'kind' in definitions[k8_file][i].keys()): continue logging.debug("Template Dump for {}: {}".format( k8_file, definitions[k8_file][i], indent=2)) entity_conf = definitions[k8_file][i] # Split out resources if entity kind is List if entity_conf["kind"] == "List": for item in entity_conf["items"]: definitions[k8_file].append(item) for i in range(len(definitions[k8_file])): if (not 'apiVersion' in definitions[k8_file][i].keys() ) and (not 'kind' in definitions[k8_file][i].keys()): continue logging.debug("Template Dump for {}: {}".format( k8_file, definitions[k8_file][i], indent=2)) entity_conf = definitions[k8_file][i] if entity_conf["kind"] == "List": continue # Append containers and initContainers to definitions list for type in ["containers", "initContainers"]: containers = [] if entity_conf["kind"] == "CustomResourceDefinition": continue containers = self._search_deep_keys( type, entity_conf, []) if not containers: continue containers = containers.pop() #containers.insert(0,entity_conf['kind']) containerDef = {} namespace = "" if "namespace" in entity_conf["metadata"]: namespace = entity_conf["metadata"]["namespace"] else: namespace = "default" containerDef["containers"] = containers.pop() for cd in containerDef["containers"]: i = containerDef["containers"].index(cd) containerDef["containers"][i][ "apiVersion"] = entity_conf["apiVersion"] containerDef["containers"][i]["kind"] = type containerDef["containers"][i][ "parent"] = "{}.{}.{} (container {})".format( entity_conf["kind"], entity_conf["metadata"]["name"], namespace, str(i)) containerDef["containers"][i][ "parent_metadata"] = entity_conf["metadata"] definitions[k8_file].extend(containerDef["containers"]) # Run for each definition included added container definitions for i in range(len(definitions[k8_file])): if (not 'apiVersion' in definitions[k8_file][i].keys() ) and (not 'kind' in definitions[k8_file][i].keys()): continue logging.debug("Template Dump for {}: {}".format( k8_file, definitions[k8_file][i], indent=2)) entity_conf = definitions[k8_file][i] if entity_conf["kind"] == "List": continue # Skip Kustomization Templates (for now) if entity_conf["kind"] == "Kustomization": continue skipped_checks = get_skipped_checks(entity_conf) results = registry.scan(k8_file, entity_conf, skipped_checks, runner_filter) # TODO refactor into context parsing find_lines_result_list = list( find_lines(entity_conf, '__startline__')) start_line = entity_conf["__startline__"] end_line = entity_conf["__endline__"] if start_line == end_line: entity_lines_range = [start_line, end_line] entity_code_lines = definitions_raw[k8_file][ start_line - 1:end_line] else: entity_lines_range = [start_line, end_line - 1] entity_code_lines = definitions_raw[k8_file][ start_line - 1:end_line - 1] # TODO? - Variable Eval Message! variable_evaluations = {} for check, check_result in results.items(): record = Record( check_id=check.id, check_name=check.name, check_result=check_result, code_block=entity_code_lines, file_path=k8_file, file_line_range=entity_lines_range, resource=check.get_resource_id(entity_conf), evaluations=variable_evaluations, check_class=check.__class__.__module__) report.add_record(record=record) return report