def delete_detached_pvcs(
api: CoreV1Api,
namespace: str,
claim_prefix: str,
pvc_cleanup_delay: timedelta,
detached_pvc_cache: Dict[str, PvcCacheEntry],
):
"""
Delete persistent volume claims that are not attached to any pods.
If a persistent volume claim is deleted while attached to a pod, then the
underlying persistent volume will remain bound until the delete is
finalized, and the delete will not be finalized until the pod is also
deleted.
If a stateful set immediately recreates a pod (e.g. via `kubectl rollout
restart`) that was attached to a persistent volume claim that was deleted,
then the stateful set may still try to reuse the persistent volume claim
after the delete is finalized. Delete the pod again to cause the stateful
set to recreate the persistent volume claim when it next recreates the pod.
"""
attached_pvcs = {
volume.persistent_volume_claim.claim_name
for pod in api.list_namespaced_pod(namespace).items
if not _unschedulable_due_to_pvc(pod) and pod.spec and pod.spec.volumes
for volume in pod.spec.volumes
if volume.persistent_volume_claim
}
for pvc in api.list_namespaced_persistent_volume_claim(namespace).items:
if (
pvc.metadata.name.startswith(claim_prefix)
and pvc.metadata.name not in attached_pvcs
and not pvc.metadata.deletion_timestamp
):
name, pv, now = pvc.metadata.name, pvc.spec.volume_name, datetime.utcnow()
if name not in detached_pvc_cache or detached_pvc_cache[name].pv != pv:
logger.info(f"found newly detached pvc: {pvc.metadata.name}")
detached_pvc_cache[name] = PvcCacheEntry(pv, now)
if (now - detached_pvc_cache[name].time) < pvc_cleanup_delay:
# wait for pvc to remain detached for pvc_cleanup_delay before deleting
continue
logger.info(f"deleting detached pvc: {pvc.metadata.name}")
try:
api.delete_namespaced_persistent_volume_claim(
name=pvc.metadata.name,
namespace=namespace,
body=V1DeleteOptions(
grace_period_seconds=0,
propagation_policy="Background",
preconditions=V1Preconditions(
resource_version=pvc.metadata.resource_version,
uid=pvc.metadata.uid,
),
),
)
except ApiException as e:
if e.reason not in (CONFLICT, NOT_FOUND):
raise
logger.info(f"pvc already deleted or updated: {pvc.metadata.name}")
else:
# pvc is not detached, drop from cache if present
detached_pvc_cache.pop(pvc.metadata.name, None)
def get_first_pod_name(v1: CoreV1Api, namespace) -> str:
"""
Return 1st pod_name in a list of pods in a namespace.
:param v1: CoreV1Api
:param namespace:
:return: str
"""
resp = v1.list_namespaced_pod(namespace)
return resp.items[0].metadata.name
def are_all_pods_in_ready_state(v1: CoreV1Api, namespace) -> bool:
"""
Check if all the pods have Ready condition.
:param v1: CoreV1Api
:param namespace: namespace
:return: bool
"""
pods = v1.list_namespaced_pod(namespace)
if not pods.items:
return False
pod_ready_amount = 0
for pod in pods.items:
if pod.status.conditions is None:
return False
for condition in pod.status.conditions:
# wait for 'Ready' state instead of 'ContainersReady' for backwards compatibility with k8s 1.10
if condition.type == 'Ready' and condition.status == 'True':
pod_ready_amount = pod_ready_amount + 1
break
return pod_ready_amount == len(pods.items)
def are_all_pods_in_ready_state(v1: CoreV1Api, namespace) -> bool:
"""
Check if all the pods have ContainersReady condition.
:param v1: CoreV1Api
:param namespace: namespace
:return: bool
"""
pods = v1.list_namespaced_pod(namespace)
if not pods.items:
return False
pod_ready_amount = 0
for pod in pods.items:
if pod.status.conditions is None:
return False
for condition in pod.status.conditions:
# wait for 'Ready' state instead of 'ContainersReady' for backwards compatibility with k8s 1.10
if condition.type == 'ContainersReady' and condition.status == 'True':
pod_ready_amount = pod_ready_amount + 1
break
return pod_ready_amount == len(pods.items)
def _snapshot_status(self, core: kubeclient.CoreV1Api, etcd_app_name: str, tries: int):
for t in range(tries):
r = core.list_namespaced_pod("backup", label_selector="etcd=%s" % etcd_app_name)
for p in r.items:
ip = p.status.host_ip
if p.status.phase != "Succeeded":
display("%d/%d pod %s status.phase: %s" % (t, tries, p.metadata.name, p.status.phase))
continue
try:
stdout = subprocess.check_output([
"ssh", "-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-o", "ConnectTimeout=1",
"-i", self.ssh_private_key,
"-lcore", ip,
'sudo /opt/bin/etcdctl3 snapshot status /var/lib/backup/etcd3/%s.snap -w json' % etcd_app_name
])
return json.loads(stdout.decode())
except Exception as e:
display(e)
time.sleep(self.testing_sleep_seconds)
def find_pods(core: CoreV1Api, namespace: str, name_regex: str):
for pod in core.list_namespaced_pod(namespace).items:
if re.match(name_regex, pod.metadata.name):
yield pod.metadata.name
def _select_pods(v1: client.CoreV1Api = None,
label_selector: str = None,
name_pattern: str = None,
all: bool = False,
rand: bool = False,
mode: str = "fixed",
qty: int = 1,
ns: str = "default",
order: str = "alphabetic") -> List[V1Pod]:
# Fail if CoreV1Api is not instanciated
if v1 is None:
raise ActivityFailed("Cannot select pods. Client API is None")
# Fail when quantity is less than 0
if qty < 0:
raise ActivityFailed(
"Cannot select pods. Quantity '{q}' is negative.".format(q=qty))
# Fail when mode is not `fixed` or `percentage`
if mode not in ['fixed', 'percentage']:
raise ActivityFailed(
"Cannot select pods. Mode '{m}' is invalid.".format(m=mode))
# Fail when order not `alphabetic` or `oldest`
if order not in ['alphabetic', 'oldest']:
raise ActivityFailed(
"Cannot select pods. Order '{o}' is invalid.".format(o=order))
if label_selector:
ret = v1.list_namespaced_pod(ns, label_selector=label_selector)
logger.debug("Found {d} pods labelled '{s}' in ns {n}".format(
d=len(ret.items), s=label_selector, n=ns))
else:
ret = v1.list_namespaced_pod(ns)
logger.debug("Found {d} pods in ns '{n}'".format(d=len(ret.items),
n=ns))
pods = []
if name_pattern:
pattern = re.compile(name_pattern)
for p in ret.items:
if pattern.search(p.metadata.name):
pods.append(p)
logger.debug(
"Pod '{p}' match pattern".format(p=p.metadata.name))
else:
pods = ret.items
if order == 'oldest':
pods.sort(key=_sort_by_pod_creation_timestamp)
if not all:
if mode == 'percentage':
qty = math.ceil((qty * len(pods)) / 100)
# If quantity is greater than number of pods present, cap the
# quantity to maximum number of pods
qty = min(qty, len(pods))
if rand:
pods = random.sample(pods, qty)
else:
pods = pods[:qty]
return pods
def list_pods(api: CoreV1Api):
return api.list_namespaced_pod("default").items
