def get_pods(node_name):
"""
Get the pods of a specific node.
This method get the pods of a node
"""
logger = get_logger("get_pods")
api_instance = client.CoreV1Api()
try:
api_response = api_instance.list_pod_for_all_namespaces(
field_selector="spec.nodeName={}".format(node_name))
return api_response
except ApiException as e:
logger.debug("Error: " + e)
print("CoreV1Api->list_pod_for_all_namespaces: %s\n" % e)
def uncordon_node(name):
"""
Uncordon a node from the cluster.
This method set a node as schedulable
"""
logger = get_logger("uncordon_node")
core_v1 = client.CoreV1Api()
body = {"spec": {"unschedulable": False}}
try:
core_v1.patch_node(name=name, body=body)
return True
except Exception as e:
logger.debug("Error: " + e)
return False
def drain_node(node_name):
"""
Drain a node from all the possible pods.
This method will drain a node
"""
logger = get_logger("drain_node")
logger.debug("Starting to drain: " + node_name)
# We get all the pods from the node
ret = get_pods(node_name)
# Now we will try to remove as much pods as we can
to_evict = calc_evict_list(ret.items)
if not to_evict:
logger.debug("No pods to remove from node.")
return True
logger.debug("Found " + str(len(to_evict)) + "pods to evict")
for pod in to_evict:
evict_pod(name=pod.metadata.name, namespace=pod.metadata.namespace)
pods = to_evict[:]
started = time.time()
timeout = 180
api_instance = client.CoreV1Api()
while True:
logger.debug("Waiting for " + str(len(pods)) + "pods to go")
if time.time() - started > timeout:
remaining_pods = "\n".join([p.metadata.name for p in pods])
raise Exception("Draining nodes did not completed within " +
timeout + "s."
"Remaining pods are: " + remaining_pods)
pending_pods = pods[:]
for pod in pods:
try:
p = api_instance.read_namespaced_pod(pod.metadata.name,
pod.metadata.namespace)
if p.metadata.uid != pod.metadata.uid:
pending_pods.remove(pod)
continue
logger.debug("Pod " + p.metadata.name + " still around "
"in phase: " + p.status.phase)
except ApiException as x:
if x.status == 404:
pending_pods.remove(pod)
pods = pending_pods[:]
if not pods:
logger.debug("Evicted all pods")
break
time.sleep(10)
return True
def calc_evict_list(pod_list):
"""
Calculate eviction list.
This method calculates the pods to remove
"""
logger = get_logger("calc_evict_list")
delete_pods_with_local_storage = False
to_evict = []
for pod in pod_list:
name = pod.metadata.name
phase = pod.status.phase
volumes = pod.spec.volumes
annotations = pod.metadata.annotations
logger.debug("Checking POD: " + name)
if annotations and "kubernetes.io/config.mirror" in annotations:
logger.debug("Not deleting, mirror pod: " + name)
continue
if any(filter(lambda v: v.empty_dir is not None, volumes)):
logger.debug("Pod " + name +
" has a volume made of a local storage")
if not delete_pods_with_local_storage:
logger.debug("Not evicting a pod with local storage")
continue
to_evict.append(pod)
continue
if phase in ["Succeeded", "Failed"]:
to_evict.append(pod)
continue
for owner in pod.metadata.owner_references:
if owner.controller and owner.kind != "DaemonSet":
to_evict.append(pod)
break
elif owner.kind == "DaemonSet":
logger.debug("Pod " + name + "is owned by a DaemonSet")
break
else:
raise Exception("Pod " + name + " is unmanaged, "
"can not drain. Delete it manually")
return to_evict
def get_worker_nodes():
"""
Get all the workers from the cluster.
This method get the worker nodes
"""
logger = get_logger("get_worker_nodes")
api_instance = client.CoreV1Api()
try:
names = []
resp = api_instance.list_node(
pretty='true', label_selector='node-role.kubernetes.io/worker')
for node in resp.items:
names.append(node.metadata.name)
return names
except ApiException as e:
logger.debug("Error: " + e)
print("CoreV1Api->list_node: %s\n" % e)
def evict_pod(name, namespace):
"""
Evict a pod from a node.
This method evicts a single pod from a node
"""
logger = get_logger("evict_pod")
api_instance = client.CoreV1Api()
ev = client.V1beta1Eviction()
ev.metadata = client.V1ObjectMeta()
ev.metadata.name = name
ev.metadata.namespace = namespace
ev.delete_options = client.V1DeleteOptions()
try:
api_instance.create_namespaced_pod_eviction(name=name,
namespace=namespace,
body=ev)
except Exception as e:
logger.debug(e)
raise Exception("Failed to evict pod " + name + ": " + e)