def delete_unschedulable_pods(api: CoreV1Api, namespace: str):
"""
Delete pods that are unschedulable due to a missing persistent volume claim.
A stateful set may create a pod attached to a missing persistent volume
claim if the pod is recreated while the persistent volume claim is pending
delete.
When this happens, delete the pod so that the stateful set will create a
new persistent volume claim when it next creates the pod.
"""
for pod in api.list_namespaced_pod(namespace).items:
if _unschedulable_due_to_pvc(pod):
logger.info(f"deleting unschedulable pod: {pod.metadata.name}")
try:
api.delete_namespaced_pod(
name=pod.metadata.name,
namespace=namespace,
body=V1DeleteOptions(
grace_period_seconds=0,
propagation_policy="Background",
preconditions=V1Preconditions(
resource_version=pod.metadata.resource_version,
uid=pod.metadata.uid,
),
),
)
except ApiException as e:
if e.reason not in (CONFLICT, NOT_FOUND):
raise
logger.info(f"pod already deleted or updated: {pod.metadata.name}")
def delete_complete_jobs(api: CoreV1Api, batch_api: BatchV1Api, namespace: str):
"""Delete complete jobs."""
for job in batch_api.list_namespaced_job(namespace).items:
if (
job.status.conditions
and job.status.conditions[0].type == "Complete"
and not job.metadata.deletion_timestamp
and _is_flush_job(job)
):
logger.info(f"deleting complete job: {job.metadata.name}")
# configure persistent volume claims to be deleted with the job
pv_name = _pv_name_from_job(job)
logger.info(f"including pv in pvc delete: {pv_name}")
api.patch_persistent_volume(
name=pv_name,
body=V1PersistentVolume(
spec=V1PersistentVolumeSpec(
persistent_volume_reclaim_policy="Delete",
)
),
)
logger.info(f"including pvc in job delete: {job.metadata.name}")
api.patch_namespaced_persistent_volume_claim(
name=job.metadata.name,
namespace=namespace,
body=V1PersistentVolumeClaim(
metadata=V1ObjectMeta(
owner_references=[
V1OwnerReference(
api_version="batch/v1",
kind="Job",
name=job.metadata.name,
uid=job.metadata.uid,
block_owner_deletion=True,
)
]
)
),
)
try:
batch_api.delete_namespaced_job(
name=job.metadata.name,
namespace=namespace,
body=V1DeleteOptions(
grace_period_seconds=0,
propagation_policy="Foreground",
preconditions=V1Preconditions(
resource_version=job.metadata.resource_version,
uid=job.metadata.uid,
),
),
)
except ApiException as e:
if e.reason not in (CONFLICT, NOT_FOUND):
raise
logger.info(f"job already deleted or updated: {job.metadata.name}")
def delete_pvcs(api: CoreV1Api):
for pvc in list_pvcs(api):
api.delete_namespaced_persistent_volume_claim(
pvc.metadata.name,
pvc.metadata.namespace,
body=V1DeleteOptions(
grace_period_seconds=0,
propagation_policy="Background",
preconditions=V1Preconditions(
resource_version=pvc.metadata.resource_version,
uid=pvc.metadata.uid,
),
),
)
def restart_web_pods(api: CoreV1Api):
for pod in api.list_namespaced_pod("default").items:
if pod.metadata.labels.get("app") != "web":
continue
api.delete_namespaced_pod(
pod.metadata.name,
pod.metadata.namespace,
body=V1DeleteOptions(
grace_period_seconds=0,
propagation_policy="Background",
preconditions=V1Preconditions(
resource_version=pod.metadata.resource_version,
uid=pod.metadata.uid,
),
),
)
def delete_detached_pvcs(api: CoreV1Api, namespace: str, claim_prefix: str):
"""
Delete persistent volume claims that are not attached to any pods.
If a persistent volume claim is deleted while attached to a pod, then the
underlying persistent volume will remain bound until the delete is
finalized, and the delete will not be finalized until the pod is also
deleted.
If a stateful set immediately recreates a pod (e.g. via `kubectl rollout
restart`) that was attached to a persistent volume claim that was deleted,
then the stateful set may still try to reuse the persistent volume claim
after the delete is finalized. Delete the pod again to cause the stateful
set to recreate the persistent volume claim when it next recreates the pod.
"""
attached_pvcs = {
volume.persistent_volume_claim.claim_name
for pod in api.list_namespaced_pod(namespace).items
if not _unschedulable_due_to_pvc(pod) and pod.spec and pod.spec.volumes
for volume in pod.spec.volumes if volume.persistent_volume_claim
}
for pvc in api.list_namespaced_persistent_volume_claim(namespace).items:
if (pvc.metadata.name.startswith(claim_prefix)
and pvc.metadata.name not in attached_pvcs
and not pvc.metadata.deletion_timestamp):
logger.info(f"deleting detached pvc: {pvc.metadata.name}")
try:
api.delete_namespaced_persistent_volume_claim(
name=pvc.metadata.name,
namespace=namespace,
body=V1DeleteOptions(
grace_period_seconds=0,
propagation_policy="Background",
preconditions=V1Preconditions(
resource_version=pvc.metadata.resource_version,
uid=pvc.metadata.uid,
),
),
)
except ApiException as e:
if e.reason not in (CONFLICT, NOT_FOUND):
raise
logger.info(
f"pvc already deleted or updated: {pvc.metadata.name}")
