def check_rbac_cb():
"""Checks whether RBAC is enabled on all the kube-apiservers.
Args:
None
Returns:
True if successful, False otherwise.
"""
ret_code = True
for name, ready_status, status, node_name in \
utils.get_pods_status_iterator("kube-apiserver-", False):
cmd = "kubectl describe pod " + name + \
" -n kube-system"
try:
output = subprocess.check_output(cmd, shell=True)
except subprocess.CalledProcessError as exc:
log.error(
"command to check kube-apiserver pod configuration has failed."
"error code: {} {}".format(exc.returncode, exc.output))
ret_code = False
else:
if "--authorization-mode=RBAC" in output or \
"--authorization-mode=ABAC" in output:
log.info("RBAC is enabled on the cluster")
else:
log.info("RBAC has been disabled on this cluster")
return ret_code
def check_access_log_config_cb():
"""Checks cilium access log parameter.
Args:
None
Returns:
True if successful, False otherwise.
"""
ret_code = True
for name, ready_status, status, node_name in \
utils.get_pods_status_iterator("cilium-"):
# TODO: Add volume checks.
config = utils.get_pod_config(name)
if not config:
log.warn('could not check access log configuration on cilium'
' pod {} on node {}'.format(name, node_name))
ret_code = False
continue
if re.search('^.*--access-log.*/var/run/cilium/access.log.*', config,
re.DOTALL) is None:
log.warn('cilium pod {} on node {} '
'has not been configured with '
'--access-log parameter or the '
'access log filename is incorrect.'
' Fix this if you would like'
' to see Layer 7 proxy logs.'.format(name, node_name))
ret_code = False
else:
log.info('cilium pod {} on node {} '
'has been configured with '
'--access-log parameter and the '
'access log filename is correct'.format(name, node_name))
return ret_code
def check_cilium_version_cb():
"""Checks whether cilium version is >= minimum supported version.
Args:
None
Returns:
True if successful, False otherwise.
"""
ret_code = True
for name, ready_status, status, node_name in\
utils.get_pods_status_iterator("cilium-"):
cmd = "kubectl describe pod " + name + \
" -n kube-system | grep Image: | " \
"awk '{print $2}' | awk -F ':' '{print $2}'"
output = ""
try:
encoded_output = subprocess.check_output(cmd, shell=True)
except subprocess.CalledProcessError as grepexc:
log.error("command to fetch cilium version has failed."
"error code: {} {}".format(grepexc.returncode,
grepexc.output))
ret_code = False
break
output = encoded_output.decode().strip(' \t\n\r')
m = re.match(r"v(\d+).(\d+).(\d+)", output)
if not m:
log.error(
"cilium version {} not in the expected format vX.Y.Z".format(
output))
ret_code = False
break
major = int(m.group(1))
minor = int(m.group(2))
patch = int(m.group(3))
print_error = True
if major > MINIMUM_SUPPORTED_CILIUM_VERSION_MAJOR:
print_error = False
elif major == MINIMUM_SUPPORTED_CILIUM_VERSION_MAJOR:
if minor > MINIMUM_SUPPORTED_CILIUM_VERSION_MINOR:
print_error = False
elif minor == MINIMUM_SUPPORTED_CILIUM_VERSION_MINOR:
if patch >= MINIMUM_SUPPORTED_CILIUM_VERSION_PATCH:
print_error = False
if print_error:
log.error(
'cilium version is {}. Minimum supported version is: v{}.{}.{}'
.format(output, MINIMUM_SUPPORTED_CILIUM_VERSION_MAJOR,
MINIMUM_SUPPORTED_CILIUM_VERSION_MINOR,
MINIMUM_SUPPORTED_CILIUM_VERSION_PATCH))
ret_code = False
else:
log.info('cilium version is {}'.format(output))
break # We do not need to inspect every cilium pod. Break here.
return ret_code
def collect_logs(self, pod_name_prefix):
for name, _, _, _ in \
utils.get_pods_status_iterator(pod_name_prefix):
log_file_name = "{}-{}.log".format(name,
datetime.datetime.
utcnow().isoformat())
cmd = "kubectl logs --since={} --limit-bytes={} " \
"-n kube-system {} > {}/{}".format(
self.since, self.size_limit, name, self.sysdump_dir_name,
log_file_name)
try:
subprocess.check_output(cmd, shell=True)
except subprocess.CalledProcessError as exc:
if exc.returncode != 0:
log.error("Error: {}. Could not collect log file: {}"
.format(exc, log_file_name))
else:
log.info("collected log file: {}".format(log_file_name))
def collect_gops(self, pod_name_prefix, type_of_stat):
for name, _, _, _ in \
utils.get_pods_status_iterator(pod_name_prefix):
file_name = "{}-{}-{}.txt".format(name,
datetime.datetime.
utcnow().isoformat(),
type_of_stat)
cmd = "kubectl exec -it -n kube-system {} -- " \
"/bin/gops {} 1 > {}/{}".format(
name, type_of_stat, self.sysdump_dir_name, file_name)
try:
subprocess.check_output(cmd, shell=True)
except subprocess.CalledProcessError as exc:
if exc.returncode != 0:
log.error("Error: {}. Could not collect gops {}: {}"
.format(exc, type_of_stat, file_name))
else:
log.info("collected gops {} file: {}".format(
type_of_stat, file_name))
def collect_cilium_bugtool_output(self):
for name, _, _, _ in \
utils.get_pods_status_iterator("cilium-"):
bugtool_output_file_name = "bugtool-{}-{}.tar".format(
name, time.strftime("%Y%m%d-%H%M%S"))
cmd = "kubectl exec -n kube-system -it {} cilium-bugtool".format(
name)
try:
encoded_output = subprocess.check_output(cmd, shell=True)
except subprocess.CalledProcessError as exc:
if exc.returncode != 0:
log.error(
"Error: {}. Could not run cilium-bugtool on {}"
.format(exc, name))
else:
output = encoded_output.decode()
p = re.compile(
"^ARCHIVE at (.*)$")
output_file_name = ""
for line in output.splitlines():
match = p.search(line)
if match:
output_file_name = match.group(1)
if output_file_name == "":
log.error(
"Error: {}. Could not find cilium-bugtool output"
" file name".format(exc))
cmd = "kubectl cp kube-system/{}:{} ./{}/{}".format(
name, output_file_name, self.sysdump_dir_name,
bugtool_output_file_name)
try:
subprocess.check_output(cmd, shell=True)
except subprocess.CalledProcessError as exc:
if exc.returncode != 0:
log.error(
"Error: {} Could not collect cilium-bugtool"
" output: {}".format(
exc, bugtool_output_file_name))
else:
log.info("collected cilium-bugtool output: {}".format(
bugtool_output_file_name))
def check_trace_notifications_enabled_cb():
"""Checks whether TraceNotification is enabled
Args:
None
Returns:
True if successful, False otherwise.
"""
ret_code = True
for name, ready_status, status, node_name in\
utils.get_pods_status_iterator("cilium-"):
cmd = "kubectl exec -it " + name + \
" -n kube-system cilium config " \
"| grep TraceNotification | awk '{print $2}'"
output = ""
try:
encoded_output = subprocess.check_output(cmd, shell=True)
except subprocess.CalledProcessError as grepexc:
log.error("command to fetch cilium config has failed."
"error code: {} {}".format(grepexc.returncode,
grepexc.output))
ret_code = False
continue
output = encoded_output.decode()
if "Enabled" not in output.strip(' \t\n\r'):
# Can't use exact match/string comparison as the output has
# ASCII-encoded characters.
log.error('cilium pod {} on '
'node {} has TraceNotifications {}'.format(
name, node_name, repr(output.strip(' \t\n\r'))))
ret_code = False
else:
log.info('cilium pod {} on '
'node {} has TraceNotifications {}'.format(
name, node_name, output.strip(' \t\n\r')))
return ret_code
def check_pod_running_cb(nodes):
"""Checks whether the Cilium container is running on all the nodes.
Args:
nodes -- list of nodes where the cilium pod is expected
to be running
Returns:
True if successful, False otherwise.
"""
ret_code = True
pod_not_seen_on_nodes = nodes[:]
for name, ready_status, status, node_name in \
utils.get_pods_status_iterator("cilium-"):
pod_not_seen_on_nodes.remove(node_name)
if status != "Running" or ready_status != "1/1":
log.error("pod {} running on {} has ready status"
" {} and status {}".format(name, node_name, ready_status,
status))
# Check the log for common errors.
cmd = "kubectl logs -n kube-system " + name
output = ""
try:
output = subprocess.check_output(cmd, shell=True)
except subprocess.CalledProcessError as exc:
log.error(
"command to get the Cilium logs has failed. "
"error code: ", exc.returncode, exc.output)
ret_code = False
continue
ret_code = False
# Check whether the KV store configuration is bad.
if "Unable to setup kvstore" in output:
log.error("Hint: Check KV-store configuration")
# Print the exact log message
log.error(output.splitlines()[-1])
# Check whether the kernel is too old.
elif "kernel version: NOT OK: minimal supported" \
" kernel version is >= 4.8" in output:
log.error("Hint: Unsupported kernel version; Please "
"install Cilium on Linux kernel >= 4.8")
# Print the exact log message
log.error(output.splitlines()[-1])
elif "Unable to contact kubernetes api-server" in output:
log.error("Hint: Cilium is unable to connect to "
"the kube-apiserver")
# Print the exact log message
log.error(output.splitlines()[-1])
elif "linking environment: OK!" not in output:
log.error("Hint: linking environment error. Check"
" Cilium logs for more information.")
elif "bpf_requirements check: OK!" not in output:
log.error("Hint: BPF requirements have not been met. Check"
" Cilium logs for more information.")
else:
log.info("pod {} running on {} has {} pods ready"
" and status {}".format(name, node_name, ready_status,
status))
if len(pod_not_seen_on_nodes) != 0:
log.warning("could not find cilium pod on node(s): "
"{}".format(pod_not_seen_on_nodes))
return ret_code
