def application_deployable_names(cluster_name: str, namespace: str,
app_name: str) -> List[str]:
"""
Returns the names of the Deployables that belong the specified Application.
:param (str) cluster_name: name of cluster where the Application resides
:param (str) namespace: namespace where the Application resides
:param (str) app_name
:return: (List[str]) list of Deployable names
"""
# find the Application using the given arguments
api_client = k8s_api.api_client(cluster_name, "CustomObjectsApi")
field_selector = "metadata.name=" + app_name
results = api_client.list_namespaced_custom_object(
namespace=namespace,
group=APP_CRD_GROUP,
version=APP_CRD_VERSION,
plural=APP_CRD_PLURAL,
field_selector=field_selector)["items"]
# if we couldn't find the Application, return None
if len(results) == 0:
print("No application found with name", app_name, "in cluster",
cluster_name, "and ns", namespace + ".")
return
# return the list of deployables specified in the Application's annotations
app = results[0]
dpb_list = app["metadata"]["annotations"][
"apps.ibm.com/deployables"].split(",")
return dpb_list
def cluster_namespaces(cluster_name: str) -> List[V1Namespace]:
"""
Returns all the Namespaces that exist under the given cluster.
:param (str) cluster_name
:return: (List[V1Namespace]) list of namespace objects
"""
api_client = k8s_api.api_client(cluster_name, "CoreV1Api")
namespaces = api_client.list_namespace()
return namespaces.items
def stateful_set_pods(sset_name: str, namespace: str,
cluster_name: str) -> List[V1Pod]:
"""
Returns the Pods that exist under the given cluster, namespace, and stateful set.
:param (str) namespace
:param (str) cluster_name: cluster that the namespace of interest is in
:param (str) sset_name: name of stateful set of interest
:return: (List[V1Pod]) list of pod objects
"""
AppsV1Api_client = k8s_api.api_client(cluster_name, "AppsV1Api")
CoreV1Api_client = k8s_api.api_client(cluster_name, "CoreV1Api")
sset = AppsV1Api_client.list_namespaced_stateful_set(
namespace, field_selector="metadata.name=" + sset_name).items[0]
selector_labels = sset.spec.selector.match_labels # dict
selector_str = ",".join(
[key + "=" + val for key, val in selector_labels.items()])
selected_pods = CoreV1Api_client.list_namespaced_pod(
namespace, label_selector=selector_str)
return selected_pods.items
def namespace_services(namespace: str, cluster_name: str) -> List[V1Service]:
"""
Returns the Services that exist under the given cluster and namespace.
:param (str) namespace
:param (str) cluster_name: cluster that the namespace of interest is in
:return: (List[V1Service]) list of service objects
"""
api_client = k8s_api.api_client(cluster_name, "CoreV1Api")
if api_client == None:
print("Cluster", cluster_name,
"could not be accessed using your kube-config credentials.")
return
svcs = api_client.list_namespaced_service(namespace)
return svcs.items
def namespace_deployments(namespace: str,
cluster_name: str) -> List[V1Deployment]:
"""
Returns the Deployments that exist under the given cluster and namespace.
:param (str) namespace
:param (str) cluster_name: cluster that the namespace of interest is in
:return: (List[V1Deployment]) list of deployment objects
"""
api_client = k8s_api.api_client(cluster_name, "AppsV1Api")
if api_client == None:
print("Cluster", cluster_name,
"could not be accessed using your kube-config credentials.")
return
deploys = api_client.list_namespaced_deployment(namespace)
return deploys.items
def service_pods(svc_name: str, namespace: str,
cluster_name: str) -> List[V1Pod]:
"""
Returns the Pods that exist under the given cluster and namespace, and are selected by the given service.
:param (str) namespace
:param (str) cluster_name: cluster that the namespace of interest is in
:param (str) svc_name: name of service of interest
:return: (List[V1Pod]) list of pod objects
"""
CoreV1Api_client = k8s_api.api_client(cluster_name, "CoreV1Api")
svc = CoreV1Api_client.list_namespaced_service(
namespace, field_selector="metadata.name=" + svc_name).items[0]
selector_labels = svc.spec.selector # dict
if selector_labels == None:
return []
selector_str = ",".join(
[key + "=" + val for key, val in selector_labels.items()])
selected_pods = CoreV1Api_client.list_namespaced_pod(
namespace, label_selector=selector_str)
return selected_pods.items
def cluster_deployables(cluster_name: str) -> List[Dict]:
"""
Returns a list of all the Deployables that belong to the given cluster.
:param (str) cluster_name
:return: (List[Dict]) list of dicts, where each dict represents a Deployable
"""
# retrieve the cluster's Deployables
api_client = k8s_api.api_client(cluster_name, "CustomObjectsApi")
try:
results = api_client.list_cluster_custom_object(
group=DPB_CRD_GROUP,
version=DPB_CRD_VERSION,
plural=DPB_CRD_PLURAL)["items"]
except k8s.client.rest.ApiException as e:
return []
# add cluster name to metadata of Deployables
for result in results:
result["metadata"]["cluster_name"] = cluster_name
return results
def cluster_applications(cluster_name: str) -> List[Dict]:
"""
Returns all the applications that belong to the given cluster.
:param (str) cluster_name
:return: (List[Dict]) list of dicts, where each dict represents an Application
"""
# retrieve the cluster's Applications
api_client = k8s_api.api_client(cluster_name, "CustomObjectsApi")
try:
apps = api_client.list_cluster_custom_object(
group=APP_CRD_GROUP,
version=APP_CRD_VERSION,
plural=APP_CRD_PLURAL)["items"]
except k8s.client.rest.ApiException:
return []
# insert cluster attribute into metadata
for app in apps:
app["metadata"]["cluster_name"] = cluster_name
return apps
def deployable_resource_name(cluster_name: str, namespace: str,
deployable_name: str) -> Dict:
"""
Returns name of resource (deployer) deployed by the specified Deployable.
(Same logic as first part of deployable_resource() above)
:param (str) cluster_name: name of cluster where Deployable resides
:param (str) namespace: namespace where Deployable resides
:param (str) deployable_name
:return: (str) name of deployer, or empty string if deployer kind is helm
"""
# find the Deployable using the arguments given
api_client = k8s_api.api_client(cluster_name, "CustomObjectsApi")
field_selector = "metadata.name=" + deployable_name
results = api_client.list_namespaced_custom_object(
namespace=namespace,
group=DPB_CRD_GROUP,
version=DPB_CRD_VERSION,
plural=DPB_CRD_PLURAL,
field_selector=field_selector)["items"]
# if we couldn't find the Deployable, return None
if len(results) == 0:
print("No Deployable found with name", deployable_name, "in cluster",
cluster_name, "and ns", namespace + ".")
return
# extract the managed resource from the Deployable dict
deployable = results[0]
kind = deployable["spec"]["deployer"]["kind"]
if kind != 'helm':
deployer_name = deployable["spec"]["deployer"]["kube"]["template"][
"metadata"]["name"]
return deployer_name
return ""
def get_pod_container_limits(cluster_name, namespace, pod_name):
"""
Helper method for getting limits for containers in a pod
:return: Dict(container_name : (cpu, mem, (bool) is_init_container))), where None for cpu and mem mean no limits specified
"""
api_client = k8s_api.api_client(cluster_name=cluster_name, api_class="CoreV1Api")
pod_object = api_client.read_namespaced_pod(pod_name, namespace)
limits_by_container = {}
containers = pod_object.spec.containers
for ct in containers:
name = ct.name
try:
ct_cpu_limit = ct.resources.limits.get('cpu')
except:
ct_cpu_limit = None
try:
ct_mem_limit = ct.resources.limits.get('memory')
except:
ct_mem_limit = None
limits_by_container[name] = (ct_cpu_limit, ct_mem_limit, False)
if pod_object.spec.init_containers is not None:
for ct in pod_object.spec.init_containers:
name = ct.name
try:
ct_cpu_limit = ct.resources.limits.get('cpu')
except:
ct_cpu_limit = None
try:
ct_mem_limit = ct.resources.limits.get('memory')
except:
ct_mem_limit = None
limits_by_container[name] = (ct_cpu_limit, ct_mem_limit, True)
return limits_by_container
def deployable_resource(cluster_name: str, namespace: str,
deployable_name: str) -> Dict:
"""
Returns info on the resource (deployer) deployed by the specified Deployable.
:param (str) cluster_name: name of cluster where Deployable resides
:param (str) namespace: namespace where Deployable resides
:param (str) deployable_name
:return: (Dict) dict with info about the deployer, or empty dict if deployer kind (e.g. helm) is not accessible
"""
# find the Deployable using the arguments given
api_client = k8s_api.api_client(cluster_name, "CustomObjectsApi")
field_selector = "metadata.name=" + deployable_name
results = api_client.list_namespaced_custom_object(
namespace=namespace,
group=DPB_CRD_GROUP,
version=DPB_CRD_VERSION,
plural=DPB_CRD_PLURAL,
field_selector=field_selector)["items"]
# if we couldn't find the Deployable, return None
if len(results) == 0:
print("No Deployable found with name", deployable_name, "in cluster",
cluster_name, "and ns", namespace + ".")
return
# extract the managed resource from the Deployable dict
deployable = results[0]
kind = deployable["spec"]["deployer"]["kind"]
if kind != 'helm':
deployer_name = deployable["spec"]["deployer"]["kube"]["template"][
"metadata"]["name"]
deployer_namespace = deployable["spec"]["deployer"]["kube"][
"namespace"]
else:
pass
deployer_dict = {}
# go through given namespace in all clusters until we find a resource that has name = deployer_name
cluster_names = k8s_config.all_cluster_names()
for cluster_name in cluster_names:
candidates = []
if kind == 'Deployment':
candidates = cmb.namespace_deployments(deployer_namespace,
cluster_name)
elif kind == 'Service':
candidates = cmb.namespace_services(deployer_namespace,
cluster_name)
for res in candidates:
if res.metadata.name == deployer_name:
deployer_dict = res.to_dict()
deployer_dict["metadata"]["cluster_name"] = cluster_name
deployer_dict["kind"] = kind
break
# if already matched
if deployer_dict != {}:
break
return deployer_dict
import k8s_api
import k8s_config
# example script that print all namespaces for each cluster
# the user has access to
k8s_config.update_available_clusters()
clusters = k8s_config.all_cluster_names()
for cluster in clusters:
api_client = k8s_api.api_client(cluster_name=cluster,
api_class="CoreV1Api")
namespaces = api_client.list_namespace()
ns_names = [ns.metadata.name for ns in namespaces.items]
print("The cluster", cluster, "has the following namespaces:", ns_names)
def get_unhealthy_pods():
"""
Gets unhealthy pods
(follows same logic as https://github.ibm.com/IBMPrivateCloud/search-collector/blob/master/pkg/transforms/pod.go)
:return: ((List(tuple)) skipper_uid, rtype, name, reason, message,
(List(V1Pod)) pod object)
"""
bad_pods = []
table_rows = []
pod_list = []
# getting all pods
clusters = k8s_config.all_cluster_names()
for cluster in clusters:
CoreV1Api_client = k8s_api.api_client(cluster, "CoreV1Api")
namespaces = cmb.cluster_namespace_names(cluster)
for ns in namespaces:
pods = CoreV1Api_client.list_namespaced_pod(ns).items
for pod in pods:
pod_list.append((pod, ns, cluster))
for pod, pod_ns, pod_cluster in pod_list:
containers = []
for ct in pod.spec.containers:
containers.append(ct.name)
reason = pod.status.phase
if pod.status.reason is not None:
reason = pod.status.reason
initializing = False
restarts = 0
# loop through the containers
if pod.status.init_container_statuses != None:
for i, ct in enumerate(pod.status.init_container_statuses):
restarts += ct.restart_count
if ct.state.terminated != None and ct.state.terminated.exit_code == 0:
continue
elif ct.state.terminated != None:
# initialization failed
if len(ct.state.terminated.reason) == 0:
if ct.state.terminated.signal != 0:
reason = "Init:Signal:{}".format(
ct.state.terminated.signal)
else:
reason = "Init:ExitCode:{}".format(
ct.state.terminated.exit_code)
else:
reason = "Init:" + ct.state.terminated.reason
initializing = True
elif ct.state.waiting != None and len(
ct.state.waiting.reason
) > 0 and ct.state.waiting.reason != "PodInitializing":
reason = "Init:" + ct.state.waiting.reason
else:
reason = "Init:{}/{}".format(i,
len(pod.spec.init_containers))
initializing = True
break
if not initializing:
# clear and sum the restarts
restarts = 0
hasRunning = False
if pod.status.container_statuses != None:
for ct in pod.status.container_statuses[::-1]:
restarts += ct.restart_count
if ct.state.waiting != None and ct.state.waiting.reason != None:
reason = ct.state.waiting.reason
elif ct.state.terminated != None and ct.state.terminated.reason != None:
reason = ct.state.terminated.reason
elif ct.state.terminated != None and ct.state.terminated.reason == None:
if ct.state.terminated.signal != 0:
reason = "Signal:{}".format(
ct.state.terminated.signal)
else:
reason = "ExitCode:{}".format(
ct.state.terminated.exit_code)
elif ct.ready and ct.state.running != None:
hasRunning = True
# change pod status back to Running if there is at least one container still reporting as "Running" status
if reason == "Completed" and hasRunning:
reason = "Running"
if pod.metadata.deletion_timestamp != None and pod.status.reason == "NodeLost":
reason = "Unknown"
elif pod.metadata.deletion_timestamp != None:
reason = "Terminating"
message = pod.status.message if pod.status.message != None else ''
if reason not in ['Running', 'Succeeded', 'Completed']:
skipper_uid = pod_cluster + "_" + pod.metadata.uid
pod.metadata.cluster_name = pod_cluster
pod.metadata.sev_reason = reason
bad_pods.append(pod)
table_rows.append(
(skipper_uid, 'Pod', pod.metadata.name, reason, message))
return (table_rows, bad_pods)
def mcm_clusters(cluster_names):
"""
Returns all MCM clusters (clusters defined using MCM cluster CRD).
:param (List[str]) cluster_names: List of (local) cluster names
:return: (Dict(local cluster name, cluster object)) where each item represents an MCM cluster
"""
myconfig = client.Configuration()
remotes = {} # cluster_name : [ remote addresse(s) ]
locals = {} # cluster_name : local cluster server
cluster_objects = {} # [ uids : cluster object ]
remotes_by_uids = {} # cluster uid : [ remote addresses ]
for cluster in cluster_names:
desired_context = k8s_config.context_for_cluster(cluster)
config.load_kube_config(context=desired_context,
client_configuration=myconfig)
# getting local address for cluster
host = myconfig.host
token = myconfig.api_key
locals[cluster] = host
response = requests.get(host + '/api', headers=token, verify=False)
if response.status_code == 200:
remotes[cluster] = []
for server in response.json()["serverAddressByClientCIDRs"]:
# getting remote addresses for cluster
address = server["serverAddress"]
remotes[cluster].append(address)
api_client = k8s_api.api_client(cluster, "CustomObjectsApi")
try:
clusters = api_client.list_cluster_custom_object(
group="clusterregistry.k8s.io",
version="v1alpha1",
plural="clusters")["items"]
# listing all remote clusters accessible from the current cluster
for item in clusters:
uid = item['metadata']['uid']
cluster_objects[uid] = item
remote_addresses = []
for ep in item['spec']['kubernetesApiEndpoints'][
'serverEndpoints']:
remote_addresses.append(ep['serverAddress'])
remotes_by_uids[uid] = remote_addresses
except ApiException:
pass
clusters = {} # { local cluster name : cluster object }
# matching clusters to remote using addresses
for uid in remotes_by_uids: # iterating through remote clusters
for cluster in remotes: # iterating through local clusters' remote addresses
local = [locals[cluster].replace('https://', '')]
if sorted(remotes[cluster]) == sorted(
remotes_by_uids[uid]
): # when remote address is contained in cluster object
clusters[cluster] = cluster_objects[uid]
elif local == remotes_by_uids[
uid]: # when local host and server from the remote object matches
clusters[cluster] = cluster_objects[uid]
return clusters
