def check_rbac_cb(): """Checks whether RBAC is enabled on all the kube-apiservers. Args: None Returns: True if successful, False otherwise. """ ret_code = True for name, ready_status, status, node_name in \ utils.get_pods_status_iterator("kube-apiserver-", False): cmd = "kubectl describe pod " + name + \ " -n kube-system" try: output = subprocess.check_output(cmd, shell=True) except subprocess.CalledProcessError as exc: log.error( "command to check kube-apiserver pod configuration has failed." "error code: {} {}".format(exc.returncode, exc.output)) ret_code = False else: if "--authorization-mode=RBAC" in output or \ "--authorization-mode=ABAC" in output: log.info("RBAC is enabled on the cluster") else: log.info("RBAC has been disabled on this cluster") return ret_code
def check_access_log_config_cb(): """Checks cilium access log parameter. Args: None Returns: True if successful, False otherwise. """ ret_code = True for name, ready_status, status, node_name in \ utils.get_pods_status_iterator("cilium-"): # TODO: Add volume checks. config = utils.get_pod_config(name) if not config: log.warn('could not check access log configuration on cilium' ' pod {} on node {}'.format(name, node_name)) ret_code = False continue if re.search('^.*--access-log.*/var/run/cilium/access.log.*', config, re.DOTALL) is None: log.warn('cilium pod {} on node {} ' 'has not been configured with ' '--access-log parameter or the ' 'access log filename is incorrect.' ' Fix this if you would like' ' to see Layer 7 proxy logs.'.format(name, node_name)) ret_code = False else: log.info('cilium pod {} on node {} ' 'has been configured with ' '--access-log parameter and the ' 'access log filename is correct'.format(name, node_name)) return ret_code
def check_cilium_version_cb(): """Checks whether cilium version is >= minimum supported version. Args: None Returns: True if successful, False otherwise. """ ret_code = True for name, ready_status, status, node_name in\ utils.get_pods_status_iterator("cilium-"): cmd = "kubectl describe pod " + name + \ " -n kube-system | grep Image: | " \ "awk '{print $2}' | awk -F ':' '{print $2}'" output = "" try: encoded_output = subprocess.check_output(cmd, shell=True) except subprocess.CalledProcessError as grepexc: log.error("command to fetch cilium version has failed." "error code: {} {}".format(grepexc.returncode, grepexc.output)) ret_code = False break output = encoded_output.decode().strip(' \t\n\r') m = re.match(r"v(\d+).(\d+).(\d+)", output) if not m: log.error( "cilium version {} not in the expected format vX.Y.Z".format( output)) ret_code = False break major = int(m.group(1)) minor = int(m.group(2)) patch = int(m.group(3)) print_error = True if major > MINIMUM_SUPPORTED_CILIUM_VERSION_MAJOR: print_error = False elif major == MINIMUM_SUPPORTED_CILIUM_VERSION_MAJOR: if minor > MINIMUM_SUPPORTED_CILIUM_VERSION_MINOR: print_error = False elif minor == MINIMUM_SUPPORTED_CILIUM_VERSION_MINOR: if patch >= MINIMUM_SUPPORTED_CILIUM_VERSION_PATCH: print_error = False if print_error: log.error( 'cilium version is {}. Minimum supported version is: v{}.{}.{}' .format(output, MINIMUM_SUPPORTED_CILIUM_VERSION_MAJOR, MINIMUM_SUPPORTED_CILIUM_VERSION_MINOR, MINIMUM_SUPPORTED_CILIUM_VERSION_PATCH)) ret_code = False else: log.info('cilium version is {}'.format(output)) break # We do not need to inspect every cilium pod. Break here. return ret_code
def collect_logs(self, pod_name_prefix): for name, _, _, _ in \ utils.get_pods_status_iterator(pod_name_prefix): log_file_name = "{}-{}.log".format(name, datetime.datetime. utcnow().isoformat()) cmd = "kubectl logs --since={} --limit-bytes={} " \ "-n kube-system {} > {}/{}".format( self.since, self.size_limit, name, self.sysdump_dir_name, log_file_name) try: subprocess.check_output(cmd, shell=True) except subprocess.CalledProcessError as exc: if exc.returncode != 0: log.error("Error: {}. Could not collect log file: {}" .format(exc, log_file_name)) else: log.info("collected log file: {}".format(log_file_name))
def collect_gops(self, pod_name_prefix, type_of_stat): for name, _, _, _ in \ utils.get_pods_status_iterator(pod_name_prefix): file_name = "{}-{}-{}.txt".format(name, datetime.datetime. utcnow().isoformat(), type_of_stat) cmd = "kubectl exec -it -n kube-system {} -- " \ "/bin/gops {} 1 > {}/{}".format( name, type_of_stat, self.sysdump_dir_name, file_name) try: subprocess.check_output(cmd, shell=True) except subprocess.CalledProcessError as exc: if exc.returncode != 0: log.error("Error: {}. Could not collect gops {}: {}" .format(exc, type_of_stat, file_name)) else: log.info("collected gops {} file: {}".format( type_of_stat, file_name))
def collect_cilium_bugtool_output(self): for name, _, _, _ in \ utils.get_pods_status_iterator("cilium-"): bugtool_output_file_name = "bugtool-{}-{}.tar".format( name, time.strftime("%Y%m%d-%H%M%S")) cmd = "kubectl exec -n kube-system -it {} cilium-bugtool".format( name) try: encoded_output = subprocess.check_output(cmd, shell=True) except subprocess.CalledProcessError as exc: if exc.returncode != 0: log.error( "Error: {}. Could not run cilium-bugtool on {}" .format(exc, name)) else: output = encoded_output.decode() p = re.compile( "^ARCHIVE at (.*)$") output_file_name = "" for line in output.splitlines(): match = p.search(line) if match: output_file_name = match.group(1) if output_file_name == "": log.error( "Error: {}. Could not find cilium-bugtool output" " file name".format(exc)) cmd = "kubectl cp kube-system/{}:{} ./{}/{}".format( name, output_file_name, self.sysdump_dir_name, bugtool_output_file_name) try: subprocess.check_output(cmd, shell=True) except subprocess.CalledProcessError as exc: if exc.returncode != 0: log.error( "Error: {} Could not collect cilium-bugtool" " output: {}".format( exc, bugtool_output_file_name)) else: log.info("collected cilium-bugtool output: {}".format( bugtool_output_file_name))
def check_trace_notifications_enabled_cb(): """Checks whether TraceNotification is enabled Args: None Returns: True if successful, False otherwise. """ ret_code = True for name, ready_status, status, node_name in\ utils.get_pods_status_iterator("cilium-"): cmd = "kubectl exec -it " + name + \ " -n kube-system cilium config " \ "| grep TraceNotification | awk '{print $2}'" output = "" try: encoded_output = subprocess.check_output(cmd, shell=True) except subprocess.CalledProcessError as grepexc: log.error("command to fetch cilium config has failed." "error code: {} {}".format(grepexc.returncode, grepexc.output)) ret_code = False continue output = encoded_output.decode() if "Enabled" not in output.strip(' \t\n\r'): # Can't use exact match/string comparison as the output has # ASCII-encoded characters. log.error('cilium pod {} on ' 'node {} has TraceNotifications {}'.format( name, node_name, repr(output.strip(' \t\n\r')))) ret_code = False else: log.info('cilium pod {} on ' 'node {} has TraceNotifications {}'.format( name, node_name, output.strip(' \t\n\r'))) return ret_code
def check_pod_running_cb(nodes): """Checks whether the Cilium container is running on all the nodes. Args: nodes -- list of nodes where the cilium pod is expected to be running Returns: True if successful, False otherwise. """ ret_code = True pod_not_seen_on_nodes = nodes[:] for name, ready_status, status, node_name in \ utils.get_pods_status_iterator("cilium-"): pod_not_seen_on_nodes.remove(node_name) if status != "Running" or ready_status != "1/1": log.error("pod {} running on {} has ready status" " {} and status {}".format(name, node_name, ready_status, status)) # Check the log for common errors. cmd = "kubectl logs -n kube-system " + name output = "" try: output = subprocess.check_output(cmd, shell=True) except subprocess.CalledProcessError as exc: log.error( "command to get the Cilium logs has failed. " "error code: ", exc.returncode, exc.output) ret_code = False continue ret_code = False # Check whether the KV store configuration is bad. if "Unable to setup kvstore" in output: log.error("Hint: Check KV-store configuration") # Print the exact log message log.error(output.splitlines()[-1]) # Check whether the kernel is too old. elif "kernel version: NOT OK: minimal supported" \ " kernel version is >= 4.8" in output: log.error("Hint: Unsupported kernel version; Please " "install Cilium on Linux kernel >= 4.8") # Print the exact log message log.error(output.splitlines()[-1]) elif "Unable to contact kubernetes api-server" in output: log.error("Hint: Cilium is unable to connect to " "the kube-apiserver") # Print the exact log message log.error(output.splitlines()[-1]) elif "linking environment: OK!" not in output: log.error("Hint: linking environment error. Check" " Cilium logs for more information.") elif "bpf_requirements check: OK!" not in output: log.error("Hint: BPF requirements have not been met. Check" " Cilium logs for more information.") else: log.info("pod {} running on {} has {} pods ready" " and status {}".format(name, node_name, ready_status, status)) if len(pod_not_seen_on_nodes) != 0: log.warning("could not find cilium pod on node(s): " "{}".format(pod_not_seen_on_nodes)) return ret_code