def test_pvc_delete_and_verify_size_is_returned_to_backend_pool(
self, pause_and_resume_cluster_load, pvc_factory, pod_factory
):
"""
Test case to verify after delete pvc size returned to backend pools
"""
cbp_name = helpers.default_ceph_block_pool()
# TODO: Get exact value of replica size
replica_size = 3
pvc_obj = pvc_factory(
interface=constants.CEPHBLOCKPOOL, size=10, status=constants.STATUS_BOUND
)
pod_obj = pod_factory(
interface=constants.CEPHBLOCKPOOL,
pvc=pvc_obj,
status=constants.STATUS_RUNNING,
)
pvc_obj.reload()
used_before_io = helpers.fetch_used_size(cbp_name)
logger.info(f"Used before IO {used_before_io}")
# Write 6Gb
pod.run_io_and_verify_mount_point(pod_obj, bs="10M", count="600")
exp_size = used_before_io + (6 * replica_size)
used_after_io = helpers.fetch_used_size(cbp_name, exp_size)
logger.info(f"Used space after IO {used_after_io}")
rbd_image_id = pvc_obj.image_uuid
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name)
verify_pv_not_exists(pvc_obj, cbp_name, rbd_image_id)
used_after_deleting_pvc = helpers.fetch_used_size(cbp_name, used_before_io)
logger.info(f"Used after deleting PVC {used_after_deleting_pvc}")
def test_delete_provisioner_pod_while_thick_provisioning(
self,
pvc_factory,
pod_factory,
):
"""
Test to delete RBD provisioner leader pod while creating a PVC using thick provision enabled storage class
"""
pvc_size = 20
pool_name = default_ceph_block_pool()
executor = ThreadPoolExecutor(max_workers=1)
DISRUPTION_OPS.set_resource(resource="rbdplugin_provisioner",
leader_type="provisioner")
# Start creation of PVC
pvc_create = executor.submit(
pvc_factory,
interface=constants.CEPHBLOCKPOOL,
project=self.proj_obj,
storageclass=default_thick_storage_class(),
size=pvc_size,
access_mode=constants.ACCESS_MODE_RWO,
status="",
)
# Ensure that the PVC is being created before deleting the rbd provisioner pod
ret = helpers.wait_for_resource_count_change(get_all_pvcs, 0,
self.proj_obj.namespace,
"increase")
assert ret, "Wait timeout: PVC is not being created."
logger.info("PVC creation has started.")
DISRUPTION_OPS.delete_resource()
logger.info("Deleted RBD provisioner leader pod.")
pvc_obj = pvc_create.result()
# Confirm that the PVC is Bound
helpers.wait_for_resource_state(resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=600)
pvc_obj.reload()
logger.info(f"Verified: PVC {pvc_obj.name} reached Bound state.")
image_name = pvc_obj.get_rbd_image_name
pv_obj = pvc_obj.backed_pv_obj
# Verify thick provision by checking the image used size
assert check_rbd_image_used_size(
pvc_objs=[pvc_obj],
usage_to_compare=f"{pvc_size}GiB",
rbd_pool=pool_name,
expect_match=True,
), f"PVC {pvc_obj.name} is not thick provisioned.\n PV describe :\n {pv_obj.describe()}"
logger.info("Verified: The PVC is thick provisioned")
# Create pod and run IO
pod_obj = pod_factory(
interface=constants.CEPHBLOCKPOOL,
pvc=pvc_obj,
status=constants.STATUS_RUNNING,
)
pod_obj.run_io(
storage_type="fs",
size=f"{pvc_size-1}G",
fio_filename=f"{pod_obj.name}_io",
end_fsync=1,
)
# Get IO result
get_fio_rw_iops(pod_obj)
logger.info(f"Deleting pod {pod_obj.name}")
pod_obj.delete()
pod_obj.ocp.wait_for_delete(pod_obj.name,
180), f"Pod {pod_obj.name} is not deleted"
# Fetch image id for verification
image_uid = pvc_obj.image_uuid
logger.info(f"Deleting PVC {pvc_obj.name}")
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(
pvc_obj.name), f"PVC {pvc_obj.name} is not deleted"
logger.info(f"Verified: PVC {pvc_obj.name} is deleted.")
pv_obj.ocp.wait_for_delete(
pv_obj.name), f"PV {pv_obj.name} is not deleted"
logger.info(f"Verified: PV {pv_obj.name} is deleted.")
# Verify the rbd image is deleted
logger.info(f"Wait for the RBD image {image_name} to get deleted")
assert verify_volume_deleted_in_backend(
interface=constants.CEPHBLOCKPOOL,
image_uuid=image_uid,
pool_name=pool_name,
timeout=300,
), f"Wait timeout - RBD image {image_name} is not deleted"
logger.info(f"Verified: RBD image {image_name} is deleted")
def test_delete_rbd_pvc_while_thick_provisioning(
self,
resource_to_delete,
pvc_factory,
pod_factory,
):
"""
Test to delete RBD PVC while thick provisioning is progressing and verify that no stale image is present.
Based on the value of "resource_to_delete", provisioner pod also will be deleted.
"""
pvc_size = 15
executor = ThreadPoolExecutor(max_workers=1)
if resource_to_delete:
DISRUPTION_OPS.set_resource(resource=resource_to_delete,
leader_type="provisioner")
ct_pod = get_ceph_tools_pod()
# Collect the list of RBD images
image_list_out_initial = ct_pod.exec_ceph_cmd(
ceph_cmd=f"rbd ls -p {constants.DEFAULT_BLOCKPOOL}", format="")
image_list_initial = image_list_out_initial.strip().split()
log.info(
f"List of RBD images before creating the PVC {image_list_initial}")
# Start creation of PVC
pvc_obj = pvc_factory(
interface=constants.CEPHBLOCKPOOL,
project=self.proj_obj,
storageclass=default_thick_storage_class(),
size=pvc_size,
access_mode=constants.ACCESS_MODE_RWO,
status="",
)
# Ensure that the PVC is being created
ret = wait_for_resource_count_change(get_all_pvcs, 0,
self.proj_obj.namespace,
"increase")
assert ret, "Wait timeout: PVC is not being created."
log.info("PVC creation has started.")
if resource_to_delete:
log.info(f"Deleting {resource_to_delete} pod.")
delete_provisioner = executor.submit(
DISRUPTION_OPS.delete_resource)
# Delete PVC
log.info(f"Deleting PVC {pvc_obj.name}")
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(pvc_obj.name)
log.info(f"Verified: PVC {pvc_obj.name} is deleted.")
if resource_to_delete:
delete_provisioner.result()
# Collect the list of RBD images
image_list_out_final = ct_pod.exec_ceph_cmd(
ceph_cmd=f"rbd ls -p {default_ceph_block_pool()}", format="")
image_list_final = image_list_out_final.strip().split()
log.info(
f"List of RBD images after deleting the PVC {image_list_final}")
stale_images = [
image for image in image_list_final
if image not in image_list_initial
]
# Check whether more than one new image is present
if len(stale_images) > 1:
raise UnexpectedBehaviour(
f"Could not verify the test result. Found more than one new rbd image - {stale_images}."
)
if stale_images:
stale_image = stale_images[0].strip()
# Wait for the image to get deleted
image_deleted = verify_volume_deleted_in_backend(
constants.CEPHBLOCKPOOL,
image_uuid=stale_image.split("csi-vol-")[1],
pool_name=default_ceph_block_pool(),
timeout=300,
)
if not image_deleted:
du_out = ct_pod.exec_ceph_cmd(
ceph_cmd=
f"rbd du -p {default_ceph_block_pool()} {stale_image}",
format="",
)
assert image_deleted, (
f"Wait timeout: RBD image {stale_image} is not deleted. Check the logs to ensure that"
f" this is the stale image of the deleted PVC. rbd du output of the image : {du_out}"
)
log.info(
f"Image {stale_image} deleted within the wait time period")
else:
log.info("No stale image found")
def test_disruptive_during_pod_pvc_deletion_and_io(
self, interface, resource_to_delete, setup_base
):
"""
Delete ceph/rook pod while PVCs deletion, pods deletion and IO are
progressing
"""
pvc_objs, pod_objs, rwx_pod_objs = setup_base
namespace = pvc_objs[0].project.namespace
num_of_pods_to_delete = 3
num_of_io_pods = 1
# Select pods to be deleted
pods_to_delete = pod_objs[:num_of_pods_to_delete]
pods_to_delete.extend(
[
pod
for pod in rwx_pod_objs
for pod_obj in pods_to_delete
if (pod_obj.pvc == pod.pvc)
]
)
# Select pods to run IO
io_pods = pod_objs[
num_of_pods_to_delete : num_of_pods_to_delete + num_of_io_pods
]
io_pods.extend(
[
pod
for pod in rwx_pod_objs
for pod_obj in io_pods
if (pod_obj.pvc == pod.pvc)
]
)
# Select pods which are having PVCs to delete
pods_for_pvc = pod_objs[num_of_pods_to_delete + num_of_io_pods :]
pvcs_to_delete = [pod_obj.pvc for pod_obj in pods_for_pvc]
pods_for_pvc.extend(
[
pod
for pod in rwx_pod_objs
for pod_obj in pods_for_pvc
if (pod_obj.pvc == pod.pvc)
]
)
log.info(
f"{len(pods_to_delete)} pods selected for deletion in which "
f"{len(pods_to_delete) - num_of_pods_to_delete} pairs of pod "
f"share same RWX PVC"
)
log.info(
f"{len(io_pods)} pods selected for running IO in which "
f"{len(io_pods) - num_of_io_pods} pairs of pod share same "
f"RWX PVC"
)
no_of_rwx_pvcs_delete = len(pods_for_pvc) - len(pvcs_to_delete)
log.info(
f"{len(pvcs_to_delete)} PVCs selected for deletion. "
f"RWO PVCs: {len(pvcs_to_delete) - no_of_rwx_pvcs_delete}, "
f"RWX PVCs: {no_of_rwx_pvcs_delete}"
)
pod_functions = {
"mds": partial(get_mds_pods),
"mon": partial(get_mon_pods),
"mgr": partial(get_mgr_pods),
"osd": partial(get_osd_pods),
"rbdplugin": partial(get_plugin_pods, interface=interface),
"cephfsplugin": partial(get_plugin_pods, interface=interface),
"cephfsplugin_provisioner": partial(get_cephfsplugin_provisioner_pods),
"rbdplugin_provisioner": partial(get_rbdfsplugin_provisioner_pods),
"operator": partial(get_operator_pods),
}
disruption = disruption_helpers.Disruptions()
disruption.set_resource(resource=resource_to_delete)
executor = ThreadPoolExecutor(max_workers=len(pod_objs) + len(rwx_pod_objs))
# Get number of pods of type 'resource_to_delete'
num_of_resource_to_delete = len(pod_functions[resource_to_delete]())
# Fetch the number of Pods and PVCs
initial_num_of_pods = len(get_all_pods(namespace=namespace))
initial_num_of_pvc = len(get_all_pvcs(namespace=namespace)["items"])
# Fetch PV names to verify after deletion
pv_objs = []
for pvc_obj in pvcs_to_delete:
pvc_obj.reload()
pv_objs.append(pvc_obj.backed_pv_obj)
# Fetch volume details from pods for the purpose of verification
node_pv_dict = {}
for pod_obj in pods_to_delete:
pod_info = pod_obj.get()
node = pod_info["spec"]["nodeName"]
pvc = pod_info["spec"]["volumes"][0]["persistentVolumeClaim"]["claimName"]
for pvc_obj in pvc_objs:
if pvc_obj.name == pvc:
pvc_obj.reload()
pv = pvc_obj.backed_pv
break
if node in node_pv_dict:
node_pv_dict[node].append(pv)
else:
node_pv_dict[node] = [pv]
# Fetch image uuid associated with PVCs to be deleted
pvc_uuid_map = {}
for pvc_obj in pvcs_to_delete:
pvc_uuid_map[pvc_obj.name] = pvc_obj.image_uuid
log.info("Fetched image uuid associated with each PVC")
# Do setup on pods for running IO
log.info("Setting up pods for running IO.")
for pod_obj in pod_objs + rwx_pod_objs:
pvc_info = pod_obj.pvc.get()
if pvc_info["spec"]["volumeMode"] == "Block":
storage_type = "block"
else:
storage_type = "fs"
executor.submit(pod_obj.workload_setup, storage_type=storage_type)
# Wait for setup on pods to complete
for pod_obj in pod_objs + rwx_pod_objs:
log.info(f"Waiting for IO setup to complete on pod {pod_obj.name}")
for sample in TimeoutSampler(180, 2, getattr, pod_obj, "wl_setup_done"):
if sample:
log.info(
f"Setup for running IO is completed on pod " f"{pod_obj.name}."
)
break
log.info("Setup for running IO is completed on all pods.")
# Start IO on pods having PVCs to delete to load data
log.info("Starting IO on pods having PVCs to delete.")
self.run_io_on_pods(pods_for_pvc)
log.info("IO started on pods having PVCs to delete.")
log.info("Fetching IO results from the pods having PVCs to delete.")
for pod_obj in pods_for_pvc:
get_fio_rw_iops(pod_obj)
log.info("Verified IO result on pods having PVCs to delete.")
# Delete pods having PVCs to delete.
delete_pods(pods_for_pvc)
for pod_obj in pods_for_pvc:
pod_obj.ocp.wait_for_delete(pod_obj.name)
log.info("Verified: Deleted pods which are having PVCs to delete.")
# Start IO on pods to be deleted
log.info("Starting IO on pods to be deleted.")
self.run_io_on_pods(pods_to_delete)
log.info("IO started on pods to be deleted.")
# Start deleting PVCs
pvc_bulk_delete = executor.submit(delete_pvcs, pvcs_to_delete)
log.info("Started deleting PVCs")
# Start deleting pods
pod_bulk_delete = executor.submit(delete_pods, pods_to_delete, wait=False)
log.info("Started deleting pods")
# Start IO on IO pods
self.run_io_on_pods(io_pods)
log.info("Started IO on IO pods")
# Verify pvc deletion has started
pvc_deleting = executor.submit(
wait_for_resource_count_change,
func_to_use=get_all_pvcs,
previous_num=initial_num_of_pvc,
namespace=namespace,
change_type="decrease",
min_difference=1,
timeout=30,
interval=0.01,
)
# Verify pod deletion has started
pod_deleting = executor.submit(
wait_for_resource_count_change,
func_to_use=get_all_pods,
previous_num=initial_num_of_pods,
namespace=namespace,
change_type="decrease",
min_difference=1,
timeout=30,
interval=0.01,
)
assert pvc_deleting.result(), "Wait timeout: PVCs are not being deleted."
log.info("PVCs deletion has started.")
assert pod_deleting.result(), "Wait timeout: Pods are not being deleted."
log.info("Pods deletion has started.")
# Delete pod of type 'resource_to_delete'
disruption.delete_resource()
pod_bulk_delete.result()
# Verify pods are deleted
for pod_obj in pods_to_delete:
pod_obj.ocp.wait_for_delete(pod_obj.name, 300)
log.info("Verified: Pods are deleted.")
# Verify that the mount point is removed from nodes after deleting pod
node_pv_mounted = verify_pv_mounted_on_node(node_pv_dict)
for node, pvs in node_pv_mounted.items():
assert not pvs, (
f"PVs {pvs} is still present on node {node} after "
f"deleting the pods."
)
log.info(
"Verified: mount points are removed from nodes after deleting " "the pods"
)
pvcs_deleted = pvc_bulk_delete.result()
assert pvcs_deleted, "Deletion of PVCs failed."
# Verify PVCs are deleted
for pvc_obj in pvcs_to_delete:
pvc_obj.ocp.wait_for_delete(pvc_obj.name)
log.info("Verified: PVCs are deleted.")
# Verify PVs are deleted
for pv_obj in pv_objs:
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name, timeout=300)
log.info("Verified: PVs are deleted.")
# Verify PV using ceph toolbox. Image/Subvolume should be deleted.
pool_name = default_ceph_block_pool()
for pvc_name, uuid in pvc_uuid_map.items():
if interface == constants.CEPHBLOCKPOOL:
ret = verify_volume_deleted_in_backend(
interface=interface, image_uuid=uuid, pool_name=pool_name
)
if interface == constants.CEPHFILESYSTEM:
ret = verify_volume_deleted_in_backend(
interface=interface, image_uuid=uuid
)
assert ret, (
f"Volume associated with PVC {pvc_name} still exists " f"in backend"
)
log.info("Fetching IO results from the pods.")
for pod_obj in io_pods:
fio_result = pod_obj.get_fio_results()
err_count = fio_result.get("jobs")[0].get("error")
assert (
err_count == 0
), f"FIO error on pod {pod_obj.name}. FIO result: {fio_result}"
log.info("Verified IO result on pods.")
# Verify number of pods of type 'resource_to_delete'
final_num_resource_to_delete = len(pod_functions[resource_to_delete]())
assert final_num_resource_to_delete == num_of_resource_to_delete, (
f"Total number of {resource_to_delete} pods is not matching with "
f"initial value. Total number of pods before deleting a pod: "
f"{num_of_resource_to_delete}. Total number of pods present now: "
f"{final_num_resource_to_delete}"
)
# Check ceph status
ceph_health_check(namespace=config.ENV_DATA["cluster_namespace"])
log.info("Ceph cluster health is OK")
def test_daemon_kill_during_pvc_pod_creation_deletion_and_io(
self, setup_base, multi_pvc_factory, pod_factory
):
"""
Kill ceph daemons while PVCs creation, PVCs deletion, pods creation, pods deletion
and IO are progressing
"""
daemons_to_kill = [
"mgr",
"mon",
"osd",
"mds",
]
(
pvc_objs,
pod_objs,
rwx_pod_objs,
cephfs_pvc_for_pods,
rbd_pvc_for_pods,
) = setup_base
num_of_pods_to_delete = 3
num_of_io_pods = 1
num_pvc_create_during_disruption = len(
self.access_modes_cephfs + self.access_modes_rbd
)
# Select pods to be deleted
pods_to_delete = pod_objs[:num_of_pods_to_delete]
pods_to_delete.extend(
[
pod
for pod in rwx_pod_objs
for pod_obj in pods_to_delete
if (pod_obj.pvc == pod.pvc)
]
)
# Select pods to run IO
io_pods = pod_objs[
num_of_pods_to_delete : num_of_pods_to_delete + num_of_io_pods
]
io_pods.extend(
[
pod
for pod in rwx_pod_objs
for pod_obj in io_pods
if (pod_obj.pvc == pod.pvc)
]
)
# Select pods which are having PVCs to delete
pods_for_pvc = pod_objs[num_of_pods_to_delete + num_of_io_pods :]
pvcs_to_delete = [pod_obj.pvc for pod_obj in pods_for_pvc]
pods_for_pvc.extend(
[
pod
for pod in rwx_pod_objs
for pod_obj in pods_for_pvc
if (pod_obj.pvc == pod.pvc)
]
)
io_pods = [
pod_obj
for pod_obj in io_pods
if pod_obj.pvc in select_unique_pvcs([pod_obj.pvc for pod_obj in io_pods])
]
log.info(
f"{len(pods_to_delete)} pods selected for deletion in which "
f"{len(pods_to_delete) - num_of_pods_to_delete} pairs of pod "
f"share same RWX PVC"
)
log.info(
f"{len(io_pods)} pods selected for running IO in which one "
f"pair of pod share same RWX PVC"
)
no_of_rwx_pvcs_delete = len(pods_for_pvc) - len(pvcs_to_delete)
log.info(
f"{len(pvcs_to_delete)} PVCs selected for deletion. "
f"RWO PVCs: {len(pvcs_to_delete) - no_of_rwx_pvcs_delete}, "
f"RWX PVCs: {no_of_rwx_pvcs_delete}"
)
pod_functions = {
"mds": partial(get_mds_pods),
"mon": partial(get_mon_pods),
"mgr": partial(get_mgr_pods),
"osd": partial(get_osd_pods),
}
# Disruption object for each daemon type
disruption_ops = [disruption_helpers.Disruptions() for _ in daemons_to_kill]
# Select the resource of each type
for disruption, pod_type in zip(disruption_ops, daemons_to_kill):
disruption.set_resource(resource=pod_type)
executor = ThreadPoolExecutor(
max_workers=len(pod_objs)
+ len(rwx_pod_objs)
+ len(rbd_pvc_for_pods)
+ len(cephfs_pvc_for_pods)
+ len(daemons_to_kill)
+ num_pvc_create_during_disruption
)
# Get number of pods of the type given in daemons_to_kill list
num_of_resource_pods = [
len(pod_functions[resource_name]()) for resource_name in daemons_to_kill
]
# Fetch PV names to verify after deletion
pv_objs = []
for pvc_obj in pvcs_to_delete:
pv_objs.append(pvc_obj.backed_pv_obj)
# Fetch volume details from pods for the purpose of verification
node_pv_dict = {}
for pod_obj in pods_to_delete:
pod_info = pod_obj.get()
node = pod_info["spec"]["nodeName"]
pvc = pod_info["spec"]["volumes"][0]["persistentVolumeClaim"]["claimName"]
for pvc_obj in pvc_objs:
if pvc_obj.name == pvc:
pv = pvc_obj.backed_pv
break
if node in node_pv_dict:
node_pv_dict[node].append(pv)
else:
node_pv_dict[node] = [pv]
# Fetch image uuid associated with PVCs to be deleted
pvc_uuid_map = {}
for pvc_obj in pvcs_to_delete:
pvc_uuid_map[pvc_obj] = pvc_obj.image_uuid
log.info("Fetched image uuid associated with each PVC")
# Do setup on pods for running IO
log.info("Setting up pods for running IO.")
for pod_obj in pod_objs + rwx_pod_objs:
if pod_obj.pvc.get_pvc_vol_mode == "Block":
storage_type = "block"
else:
storage_type = "fs"
executor.submit(pod_obj.workload_setup, storage_type=storage_type)
# Wait for setup on pods to complete
for pod_obj in pod_objs + rwx_pod_objs:
log.info(f"Waiting for IO setup to complete on pod {pod_obj.name}")
for sample in TimeoutSampler(360, 2, getattr, pod_obj, "wl_setup_done"):
if sample:
log.info(
f"Setup for running IO is completed on pod " f"{pod_obj.name}."
)
break
log.info("Setup for running IO is completed on all pods.")
# Start IO on pods having PVCs to delete to load data
pods_for_pvc_io = [
pod_obj
for pod_obj in pods_for_pvc
if pod_obj.pvc
in select_unique_pvcs([pod_obj.pvc for pod_obj in pods_for_pvc])
]
log.info("Starting IO on pods having PVCs to delete.")
self.run_io_on_pods(pods_for_pvc_io)
log.info("IO started on pods having PVCs to delete.")
log.info("Fetching IO results from the pods having PVCs to delete.")
for pod_obj in pods_for_pvc_io:
get_fio_rw_iops(pod_obj)
log.info("Verified IO result on pods having PVCs to delete.")
# Delete pods having PVCs to delete.
assert self.delete_pods(
pods_for_pvc
), "Couldn't delete pods which are having PVCs to delete."
for pod_obj in pods_for_pvc:
pod_obj.ocp.wait_for_delete(pod_obj.name)
log.info("Verified: Deleted pods which are having PVCs to delete.")
# Select daemon of each type of resource and identify the daemons running on each node
nodes_and_pids = {}
for disruption in disruption_ops:
disruption.select_daemon()
node_name = disruption.resource_obj[0].pod_data.get("spec").get("nodeName")
# Create node-daemons dict. Value as string for passing in the 'kill' command
nodes_and_pids[
node_name
] = f"{nodes_and_pids.get(node_name, '')} {disruption.daemon_pid}"
# Start IO on pods to be deleted
pods_to_delete_io = [
pod_obj
for pod_obj in pods_to_delete
if pod_obj.pvc
in select_unique_pvcs([pod_obj.pvc for pod_obj in pods_to_delete])
]
log.info("Starting IO on selected pods to be deleted.")
self.run_io_on_pods(pods_to_delete_io)
log.info("IO started on selected pods to be deleted.")
# Start creating new pods
log.info("Start creating new pods.")
pod_create_rbd = executor.submit(
helpers.create_pods,
rbd_pvc_for_pods,
pod_factory,
constants.CEPHBLOCKPOOL,
2,
)
pod_create_cephfs = executor.submit(
helpers.create_pods,
cephfs_pvc_for_pods,
pod_factory,
constants.CEPHFILESYSTEM,
2,
)
# Start creation of new CephFS PVCs.
log.info("Start creating new CephFS PVCs.")
pvc_create_cephfs = executor.submit(
multi_pvc_factory,
interface=constants.CEPHFILESYSTEM,
project=self.project,
storageclass=None,
size=self.pvc_size,
access_modes=self.access_modes_cephfs,
access_modes_selection="distribute_random",
status="",
num_of_pvc=len(self.access_modes_cephfs),
wait_each=False,
)
# Start creation of new RBD PVCs
log.info("Start creating new RBD PVCs.")
pvc_create_rbd = executor.submit(
multi_pvc_factory,
interface=constants.CEPHBLOCKPOOL,
project=self.project,
storageclass=None,
size=self.pvc_size,
access_modes=self.access_modes_rbd,
access_modes_selection="distribute_random",
status="",
num_of_pvc=len(self.access_modes_rbd),
wait_each=False,
)
# Start deleting PVCs
pvc_bulk_delete = executor.submit(delete_pvcs, pvcs_to_delete)
log.info("Started deleting PVCs")
# Start deleting pods
pod_bulk_delete = executor.submit(self.delete_pods, pods_to_delete)
log.info("Started deleting pods")
# Start IO on IO pods
self.run_io_on_pods(io_pods)
log.info("Started IO on IO pods")
# Wait for 1 second before killing daemons. This is to wait for the create/delete operations to start
sleep(1)
# Kill daemons
node_and_kill_proc = {}
log.info(f"Killing daemons of {daemons_to_kill}")
for node_name, pids in nodes_and_pids.items():
# Command to kill the daemon
kill_cmd = f"oc debug node/{node_name} -- chroot /host kill -9 {pids}"
# Create node-kill process map for verifying the result
node_and_kill_proc[node_name] = executor.submit(run_cmd, kill_cmd)
# Verify daemon kill process
for node_name, daemon_kill_proc in node_and_kill_proc.items():
# Get the type of daemons killed on the particular node
resources = [
disruption.resource
for disruption in disruption_ops
if disruption.daemon_pid in nodes_and_pids[node_name]
]
# 'daemon_kill_proc' result will be an empty string if command is success
cmd_out = daemon_kill_proc.result()
assert isinstance(cmd_out, str) and (not cmd_out), (
f"Failed to kill {resources } daemons in the node {node_name}. "
f"Daemon kill command output - {cmd_out}"
)
# Wait for new daemon to come up
[disruption.check_new_pid() for disruption in disruption_ops]
log.info("Verified daemons kill")
pods_deleted = pod_bulk_delete.result()
assert pods_deleted, "Deletion of pods failed."
# Verify pods are deleted
for pod_obj in pods_to_delete:
pod_obj.ocp.wait_for_delete(pod_obj.name, 300)
log.info("Verified: Pods are deleted.")
# Verify that the mount point is removed from nodes after deleting pod
node_pv_mounted = verify_pv_mounted_on_node(node_pv_dict)
for node, pvs in node_pv_mounted.items():
assert not pvs, (
f"PVs {pvs} is still present on node {node} after "
f"deleting the pods."
)
log.info(
"Verified: mount points are removed from nodes after deleting " "the pods"
)
pvcs_deleted = pvc_bulk_delete.result()
assert pvcs_deleted, "Deletion of PVCs failed."
# Verify PVCs are deleted
for pvc_obj in pvcs_to_delete:
pvc_obj.ocp.wait_for_delete(pvc_obj.name)
log.info("Verified: PVCs are deleted.")
# Getting result of PVC creation as list of PVC objects
log.info("Getting the result of CephFS PVC creation process")
pvc_objs_cephfs_new = pvc_create_cephfs.result()
log.info("Getting the result of RBD PVC creation process")
pvc_objs_rbd_new = pvc_create_rbd.result()
# Set interface argument for reference
for pvc_obj in pvc_objs_cephfs_new:
pvc_obj.interface = constants.CEPHFILESYSTEM
# Set interface argument for reference
for pvc_obj in pvc_objs_rbd_new:
pvc_obj.interface = constants.CEPHBLOCKPOOL
# Confirm PVCs are Bound
log.info("Verifying the new CephFS and RBD PVCs are Bound")
for pvc_obj in pvc_objs_cephfs_new + pvc_objs_rbd_new:
helpers.wait_for_resource_state(
resource=pvc_obj, state=constants.STATUS_BOUND, timeout=180
)
pvc_obj.reload()
log.info("Verified: New CephFS and RBD PVCs are Bound.")
# Getting result of pods creation as list of Pod objects
log.info("Getting the result of pods creation process")
pod_objs_rbd_new = pod_create_rbd.result()
pod_objs_cephfs_new = pod_create_cephfs.result()
# Verify new pods are Running
log.info("Verifying the new pods are Running")
for pod_obj in pod_objs_rbd_new + pod_objs_cephfs_new:
helpers.wait_for_resource_state(
resource=pod_obj, state=constants.STATUS_RUNNING, timeout=90
)
pod_obj.reload()
log.info("Verified: All new pods are Running.")
# Verify PVs are deleted
for pv_obj in pv_objs:
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name, timeout=300)
log.info("Verified: PVs are deleted.")
# Verify PV using ceph toolbox. Image/Subvolume should be deleted.
pool_name = default_ceph_block_pool()
for pvc_obj, uuid in pvc_uuid_map.items():
if pvc_obj.interface == constants.CEPHBLOCKPOOL:
ret = verify_volume_deleted_in_backend(
interface=constants.CEPHBLOCKPOOL,
image_uuid=uuid,
pool_name=pool_name,
)
if pvc_obj.interface == constants.CEPHFILESYSTEM:
ret = verify_volume_deleted_in_backend(
interface=constants.CEPHFILESYSTEM, image_uuid=uuid
)
assert (
ret
), f"Volume associated with PVC {pvc_obj.name} still exists in the backend"
log.info("Fetching IO results from the pods.")
for pod_obj in io_pods:
fio_result = pod_obj.get_fio_results()
err_count = fio_result.get("jobs")[0].get("error")
assert (
err_count == 0
), f"FIO error on pod {pod_obj.name}. FIO result: {fio_result}"
log.info("Verified IO result on pods.")
# Verify that the new PVCs are usable by creating new pods
log.info("Verify that the new PVCs are usable by creating new pods")
pod_objs_rbd_re = helpers.create_pods(
pvc_objs_rbd_new, pod_factory, constants.CEPHBLOCKPOOL, 2
)
pod_objs_cephfs_re = helpers.create_pods(
pvc_objs_cephfs_new, pod_factory, constants.CEPHFILESYSTEM, 2
)
# Verify pods are Running
log.info("Verifying the pods are Running")
for pod_obj in pod_objs_rbd_re + pod_objs_cephfs_re:
helpers.wait_for_resource_state(
resource=pod_obj, state=constants.STATUS_RUNNING, timeout=90
)
pod_obj.reload()
log.info(
"Successfully created and verified the status of the pods using the new CephFS and RBD PVCs."
)
new_pods = (
pod_objs_rbd_new
+ pod_objs_cephfs_new
+ pod_objs_rbd_re
+ pod_objs_cephfs_re
)
# Do setup on the new pods for running IO
log.info("Setting up the new pods for running IO.")
for pod_obj in new_pods:
if pod_obj.pvc.get_pvc_vol_mode == "Block":
storage_type = "block"
else:
storage_type = "fs"
executor.submit(pod_obj.workload_setup, storage_type=storage_type)
# Wait for setup on the new pods to complete
for pod_obj in new_pods:
log.info(f"Waiting for IO setup to complete on pod {pod_obj.name}")
for sample in TimeoutSampler(360, 2, getattr, pod_obj, "wl_setup_done"):
if sample:
log.info(
f"Setup for running IO is completed on pod " f"{pod_obj.name}."
)
break
log.info("Setup for running IO is completed on the new pods.")
# Start IO on the new pods
log.info("Start IO on the new pods")
self.run_io_on_pods(new_pods)
log.info("Started IO on the new pods")
log.info("Fetching IO results from the new pods.")
for pod_obj in new_pods:
get_fio_rw_iops(pod_obj)
log.info("Verified IO result on the new pods.")
# Verify number of pods of each daemon type
final_num_resource_name = [
len(pod_functions[resource_name]()) for resource_name in daemons_to_kill
]
assert final_num_resource_name == num_of_resource_pods, (
f"Total number of pods of each type is not matching with "
f"initial value. Total number of pods of each type before daemon kill: "
f"{num_of_resource_pods}. Total number of pods of each type present now: "
f"{final_num_resource_name}"
)
# Check ceph status
ceph_health_check(namespace=config.ENV_DATA["cluster_namespace"])
log.info("Ceph cluster health is OK")
def test_verify_rbd_thick_pvc_utilization(
self,
pvc_factory,
pod_factory,
):
"""
Test to verify the storage utilization of RBD thick provisioned PVC
"""
pvc_size = 15
replica_size = 3
file1 = "fio_file1"
file2 = "fio_file2"
rbd_pool = default_ceph_block_pool()
size_before_pvc = fetch_used_size(rbd_pool)
log.info(
f"Storage pool used size before creating the PVC is {size_before_pvc}"
)
# Create RBD thick PVC
pvc_obj = pvc_factory(
interface=constants.CEPHBLOCKPOOL,
project=self.proj_obj,
storageclass=default_thick_storage_class(),
size=pvc_size,
access_mode=constants.ACCESS_MODE_RWO,
status=constants.STATUS_BOUND,
)
size_after_pvc = fetch_used_size(
rbd_pool, size_before_pvc + (pvc_size * replica_size))
log.info(
f"Verified: Storage pool used size after creating the PVC is {size_after_pvc}"
)
pod_obj = pod_factory(
interface=constants.CEPHBLOCKPOOL,
pvc=pvc_obj,
status=constants.STATUS_RUNNING,
)
# Create 5GB file
pod_obj.run_io(
storage_type="fs",
size="5G",
runtime=60,
fio_filename=file1,
end_fsync=1,
)
pod_obj.get_fio_results()
# Verify the used size after IO
fetch_used_size(rbd_pool, size_before_pvc + (pvc_size * replica_size))
# Create another 5GB file
pod_obj.run_io(
storage_type="fs",
size="5G",
runtime=60,
fio_filename=file2,
end_fsync=1,
)
pod_obj.get_fio_results()
# Verify the used size after IO
fetch_used_size(rbd_pool, size_before_pvc + (pvc_size * replica_size))
# Delete the files created by fio
mount_point = pod_obj.get_storage_path()
rm_cmd = f"rm {path.join(mount_point, file1)} {path.join(mount_point, file2)}"
pod_obj.exec_cmd_on_pod(command=rm_cmd, out_yaml_format=False)
# Verify the used size after deleting the files
fetch_used_size(rbd_pool, size_before_pvc + (pvc_size * replica_size))
# Delete the pod
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
# Delete the PVC
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name)
# Verify used size after deleting the PVC
size_after_pvc_delete = fetch_used_size(rbd_pool, size_before_pvc)
log.info(
f"Verified: Storage pool used size after deleting the PVC is {size_after_pvc_delete}"
)
def disruptive_base(self, interface, operation_to_disrupt,
resource_to_delete):
"""
Base function for disruptive tests.
Deletion of 'resource_to_delete' will be introduced while
'operation_to_disrupt' is progressing.
"""
pod_functions = {
"mds": partial(get_mds_pods),
"mon": partial(get_mon_pods),
"mgr": partial(get_mgr_pods),
"osd": partial(get_osd_pods),
"rbdplugin": partial(get_plugin_pods, interface=interface),
"cephfsplugin": partial(get_plugin_pods, interface=interface),
"cephfsplugin_provisioner":
partial(get_cephfsplugin_provisioner_pods),
"rbdplugin_provisioner": partial(get_rbdfsplugin_provisioner_pods),
"operator": partial(get_operator_pods),
}
disruption = disruption_helpers.Disruptions()
disruption.set_resource(resource=resource_to_delete)
executor = ThreadPoolExecutor(max_workers=1)
# Get number of pods of type 'resource_to_delete'
num_of_resource_to_delete = len(pod_functions[resource_to_delete]())
# Fetch the number of Pods and PVCs
initial_num_of_pods = len(get_all_pods(namespace=self.namespace))
initial_num_of_pvc = len(
get_all_pvcs(namespace=self.namespace)["items"])
# Fetch PV names
pv_objs = []
for pvc_obj in self.pvc_objs:
pvc_obj.reload()
pv_objs.append(pvc_obj.backed_pv_obj)
# Fetch volume details from pods for the purpose of verification
node_pv_dict = {}
for pod_obj in self.pod_objs:
pod_info = pod_obj.get()
node = pod_info["spec"]["nodeName"]
pvc = pod_info["spec"]["volumes"][0]["persistentVolumeClaim"][
"claimName"]
for pvc_obj in self.pvc_objs:
if pvc_obj.name == pvc:
pvc_obj.reload()
pv = pvc_obj.backed_pv
break
if node in node_pv_dict:
node_pv_dict[node].append(pv)
else:
node_pv_dict[node] = [pv]
# Do setup for running IO on pods
log.info("Setting up pods for running IO")
for pod_obj in self.pod_objs:
pvc_info = pod_obj.pvc.get()
if pvc_info["spec"]["volumeMode"] == "Block":
pod_obj.pvc.storage_type = "block"
else:
pod_obj.pvc.storage_type = "fs"
pod_obj.workload_setup(storage_type=pod_obj.pvc.storage_type)
log.info("Setup for running IO is completed on pods")
# Start IO on each pod. RWX PVC will be used on two pods. So split the
# size accordingly
log.info("Starting IO on pods")
for pod_obj in self.pod_objs:
if pod_obj.pvc.access_mode == constants.ACCESS_MODE_RWX:
io_size = int((self.pvc_size - 1) / 2)
else:
io_size = self.pvc_size - 1
pod_obj.run_io(
storage_type=pod_obj.pvc.storage_type,
size=f"{io_size}G",
fio_filename=f"{pod_obj.name}_io",
end_fsync=1,
)
log.info("IO started on all pods.")
# Start deleting pods
pod_bulk_delete = executor.submit(delete_pods,
self.pod_objs,
wait=False)
if operation_to_disrupt == "delete_pods":
ret = wait_for_resource_count_change(
get_all_pods,
initial_num_of_pods,
self.namespace,
"decrease",
timeout=50,
)
assert ret, "Wait timeout: Pods are not being deleted."
log.info("Pods deletion has started.")
disruption.delete_resource()
pod_bulk_delete.result()
# Verify pods are deleted
for pod_obj in self.pod_objs:
assert pod_obj.ocp.wait_for_delete(
pod_obj.name, 180), f"Pod {pod_obj.name} is not deleted"
log.info("Verified: Pods are deleted.")
# Verify that the mount point is removed from nodes after deleting pod
for node, pvs in node_pv_dict.items():
cmd = f"oc debug nodes/{node} -- df"
df_on_node = run_cmd(cmd)
for pv in pvs:
assert pv not in df_on_node, (
f"{pv} is still present on node {node} after "
f"deleting the pods.")
log.info(
"Verified: mount points are removed from nodes after deleting "
"the pods")
# Fetch image uuid associated with PVCs
pvc_uuid_map = {}
for pvc_obj in self.pvc_objs:
pvc_uuid_map[pvc_obj.name] = pvc_obj.image_uuid
log.info("Fetched image uuid associated with each PVC")
# Start deleting PVCs
pvc_bulk_delete = executor.submit(delete_pvcs, self.pvc_objs)
if operation_to_disrupt == "delete_pvcs":
ret = wait_for_resource_count_change(get_all_pvcs,
initial_num_of_pvc,
self.namespace,
"decrease",
timeout=50)
assert ret, "Wait timeout: PVCs are not being deleted."
log.info("PVCs deletion has started.")
disruption.delete_resource()
pvcs_deleted = pvc_bulk_delete.result()
assert pvcs_deleted, "Deletion of PVCs failed."
# Verify PVCs are deleted
for pvc_obj in self.pvc_objs:
assert pvc_obj.ocp.wait_for_delete(
pvc_obj.name), f"PVC {pvc_obj.name} is not deleted"
log.info("Verified: PVCs are deleted.")
# Verify PVs are deleted
for pv_obj in pv_objs:
assert pv_obj.ocp.wait_for_delete(
pv_obj.name, 120), f"PV {pv_obj.name} is not deleted"
log.info("Verified: PVs are deleted.")
# Verify PV using ceph toolbox. Image/Subvolume should be deleted.
pool_name = default_ceph_block_pool()
for pvc_name, uuid in pvc_uuid_map.items():
if interface == constants.CEPHBLOCKPOOL:
ret = verify_volume_deleted_in_backend(interface=interface,
image_uuid=uuid,
pool_name=pool_name)
if interface == constants.CEPHFILESYSTEM:
ret = verify_volume_deleted_in_backend(interface=interface,
image_uuid=uuid)
assert ret, (f"Volume associated with PVC {pvc_name} still exists "
f"in backend")
# Verify number of pods of type 'resource_to_delete'
final_num_resource_to_delete = len(pod_functions[resource_to_delete]())
assert final_num_resource_to_delete == num_of_resource_to_delete, (
f"Total number of {resource_to_delete} pods is not matching with "
f"initial value. Total number of pods before deleting a pod: "
f"{num_of_resource_to_delete}. Total number of pods present now: "
f"{final_num_resource_to_delete}")
# Check ceph status
ceph_health_check(namespace=config.ENV_DATA["cluster_namespace"])
log.info("Ceph cluster health is OK")
def test_multiple_pvc_concurrent_creation_deletion(self, interface,
multi_pvc_factory):
"""
To exercise resource creation and deletion
"""
proj_obj = self.pvc_objs[0].project
executor = ThreadPoolExecutor(max_workers=1)
# Get PVs
pv_objs = []
for pvc in self.pvc_objs:
pv_objs.append(pvc.backed_pv_obj)
# Fetch image uuid associated with PVCs
pvc_uuid_map = {}
for pvc_obj in self.pvc_objs:
pvc_uuid_map[pvc_obj.name] = pvc_obj.image_uuid
log.info("Fetched image uuid associated with each PVC")
# Start deleting 100 PVCs
log.info("Start deleting PVCs.")
pvc_delete = executor.submit(delete_pvcs, self.pvc_objs)
# Create 100 PVCs
log.info("Start creating new PVCs")
self.new_pvc_objs = multi_pvc_factory(
interface=interface,
project=proj_obj,
size=self.pvc_size,
access_modes=self.access_modes,
status="",
num_of_pvc=self.num_of_pvcs,
wait_each=False,
)
for pvc_obj in self.new_pvc_objs:
wait_for_resource_state(pvc_obj, constants.STATUS_BOUND)
pvc_obj.reload()
log.info(f"Newly created {self.num_of_pvcs} PVCs are in Bound state.")
# Verify PVCs are deleted
res = pvc_delete.result()
assert res, "Deletion of PVCs failed"
log.info("PVC deletion was successful.")
for pvc in self.pvc_objs:
pvc.ocp.wait_for_delete(resource_name=pvc.name)
log.info(f"Successfully deleted initial {self.num_of_pvcs} PVCs")
# Verify PVs are deleted
for pv_obj in pv_objs:
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name, timeout=180)
log.info(f"Successfully deleted initial {self.num_of_pvcs} PVs")
# Verify PV using ceph toolbox. Image/Subvolume should be deleted.
for pvc_name, uuid in pvc_uuid_map.items():
pool_name = None
if interface == constants.CEPHBLOCKPOOL:
pool_name = default_ceph_block_pool()
ret = verify_volume_deleted_in_backend(interface=interface,
image_uuid=uuid,
pool_name=pool_name)
assert ret, (f"Volume associated with PVC {pvc_name} still exists "
f"in backend")
# Verify status of nodes
for node in get_node_objs():
node_status = node.ocp.get_resource_status(node.name)
assert (node_status == constants.NODE_READY
), f"Node {node.name} is in {node_status} state."
def uninstall_cluster_logging():
"""
Function to uninstall cluster-logging from the cluster
Deletes the project "openshift-logging" and "openshift-operators-redhat"
"""
# Validating the pods before deleting the instance
pod_list = get_all_pods(namespace=constants.OPENSHIFT_LOGGING_NAMESPACE)
for pod in pod_list:
logger.info(
f"Pods running in the openshift-logging namespace {pod.name}")
# Excluding cluster-logging-operator from pod_list and getting pod names
pod_names_list = [
pod.name for pod in pod_list
if not pod.name.startswith("cluster-logging-operator")
]
pvc_objs = get_all_pvc_objs(
namespace=constants.OPENSHIFT_LOGGING_NAMESPACE)
# Fetch image uuid associated with PVCs to be deleted
pvc_uuid_map = {}
for pvc_obj in pvc_objs:
pvc_uuid_map[pvc_obj.name] = pvc_obj.image_uuid
# Checking for used space
cbp_name = default_ceph_block_pool()
used_space_before_deletion = fetch_used_size(cbp_name)
logger.info(
f"Used space before deletion of cluster logging {used_space_before_deletion}"
)
# Deleting the clusterlogging instance
clusterlogging_obj = ocp.OCP(
kind=constants.CLUSTER_LOGGING,
namespace=constants.OPENSHIFT_LOGGING_NAMESPACE)
assert clusterlogging_obj.delete(resource_name="instance")
check_pod_vanished(pod_names_list)
for pvc_obj in pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
assert delete_pvcs(pvc_objs=pvc_objs), "PVCs deletion failed"
for pvc_obj in pvc_objs:
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name, timeout=300)
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name, timeout=300)
logger.info("Verified: PVCs are deleted.")
logger.info("Verified: PV are deleted")
for pvc_name, uuid in pvc_uuid_map.items():
rbd = verify_volume_deleted_in_backend(
interface=constants.CEPHBLOCKPOOL,
image_uuid=uuid,
pool_name=cbp_name)
assert rbd, f"Volume associated with PVC {pvc_name} still exists " f"in backend"
# Checking for used space after PVC deletion
used_space_after_deletion = fetch_used_size(cbp_name, exp_val=30)
logger.info(
f"Used space after deletion of cluster logging {used_space_after_deletion}"
)
if used_space_after_deletion < used_space_before_deletion:
logger.info("Expected !!! Space has reclaimed")
else:
logger.warning(
"Unexpected !! No space reclaimed after deletion of PVC")
# Deleting the RBAC permission set
rbac_role = ocp.OCP(
kind=constants.ROLE,
namespace=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE)
rbac_role.delete(yaml_file=constants.EO_RBAC_YAML)
openshift_logging_namespace = ocp.OCP(
kind=constants.NAMESPACES,
namespace=constants.OPENSHIFT_LOGGING_NAMESPACE)
openshift_operators_redhat_namespace = ocp.OCP(
kind=constants.NAMESPACES,
namespace=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE,
)
if openshift_operators_redhat_namespace.get():
assert openshift_operators_redhat_namespace.delete(
resource_name=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE)
logger.info(
"The project openshift-opertors-redhat got deleted successfully")
if openshift_logging_namespace.get():
assert openshift_logging_namespace.delete(
resource_name=constants.OPENSHIFT_LOGGING_NAMESPACE)
logger.info("The namespace openshift-logging got deleted successfully")
def test_change_reclaim_policy_of_pv(self, interface, reclaim_policy,
pod_factory):
"""
This test case tests update of reclaim policy of PV
"""
reclaim_policy_to = "Delete" if reclaim_policy == "Retain" else (
"Retain")
# Fetch name of PVs
pvs = [pvc_obj.backed_pv_obj for pvc_obj in self.pvc_objs]
# Fetch image uuid associated with PVCs
pvc_uuid_map = {}
for pvc_obj in self.pvc_objs:
pvc_uuid_map[pvc_obj.name] = pvc_obj.image_uuid
log.info("Fetched image uuid associated with each PVC")
# Select PVs to change reclaim policy
changed_pvs = pvs[:5]
# Run IO on pods
self.run_and_verify_io(self.pod_objs)
log.info("Verified IO result on pods.")
# Change relaimPolicy to 'reclaim_policy_to'
for pv_obj in changed_pvs:
pv_name = pv_obj.name
patch_param = (f'{{"spec":{{"persistentVolumeReclaimPolicy":'
f'"{reclaim_policy_to}"}}}}')
assert pv_obj.ocp.patch(
resource_name=pv_name,
params=patch_param,
format_type="strategic"), (
f"Failed to change persistentVolumeReclaimPolicy of pv "
f"{pv_name} to {reclaim_policy_to}")
log.info(f"Changed persistentVolumeReclaimPolicy of pv {pv_name} "
f"to {reclaim_policy_to}")
retain_pvs = []
delete_pvs = []
# Verify reclaim policy of all PVs
for pv_obj in pvs:
policy = pv_obj.get().get("spec").get(
"persistentVolumeReclaimPolicy")
retain_pvs.append(pv_obj) if policy == "Retain" else (
delete_pvs.append(pv_obj))
if pv_obj in changed_pvs:
assert policy == reclaim_policy_to, (
f"Reclaim policy of {pv_obj.name} is {policy}. "
f"It has not changed to {reclaim_policy_to}")
else:
assert policy == reclaim_policy, (
f"Reclaim policy of {pv_obj.name} is {policy} instead "
f"of {reclaim_policy}.")
log.info("Verified reclaim policy of all PVs")
# Run IO on pods
self.run_and_verify_io(self.pod_objs, do_setup=False)
log.info("Ran and verified IO on pods after changing reclaim policy.")
# Delete all pods
log.info("Deleting all pods")
for pod_obj in self.pod_objs:
pod_obj.delete()
# Verify pods are deleted
for pod_obj in self.pod_objs:
pod_obj.ocp.wait_for_delete(pod_obj.name, 300)
log.info("Verified: Pods are deleted.")
# Create new pods mounting one volume on each pod
log.info("Creating new pods.")
new_pod_objs = []
for pvc_obj in self.pvc_objs:
new_pod_objs.append(
pod_factory(interface=interface, pvc=pvc_obj, status=None))
for pod in new_pod_objs:
wait_for_resource_state(pod, constants.STATUS_RUNNING)
pod.reload()
# Run IO on new pods
self.run_and_verify_io(new_pod_objs)
log.info("Ran and verified IO on new pods.")
# Delete all pods
log.info("Deleting all new pods.")
for pod_obj in new_pod_objs:
pod_obj.delete()
# Verify pods are deleted
for pod_obj in new_pod_objs:
pod_obj.ocp.wait_for_delete(pod_obj.name, 300)
log.info("Verified: All new pods are deleted.")
# Delete PVCs
log.info("Deleting all PVCs.")
for pvc_obj in self.pvc_objs:
pvc_obj.delete()
# Verify PVCs are deleted
for pvc_obj in self.pvc_objs:
pvc_obj.ocp.wait_for_delete(pvc_obj.name, 300)
log.info("Verified: All PVCs are deleted")
# PVs having reclaim policy 'Delete' will be deleted
for pv_obj in delete_pvs:
pv_obj.ocp.wait_for_delete(pv_obj.name, 300)
log.info(
"Verified: All PVs having reclaim policy 'Delete' are deleted.")
# PVs having reclaim policy 'Retain' will be in Released state
for pv_obj in retain_pvs:
wait_for_resource_state(resource=pv_obj,
state=constants.STATUS_RELEASED)
log.info("Verified: All PVs having reclaim policy 'Retain' are "
"in 'Released' state.")
# Change relaimPolicy to Delete
for pv_obj in retain_pvs:
pv_name = pv_obj.name
patch_param = '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}'
assert pv_obj.ocp.patch(
resource_name=pv_name,
params=patch_param,
format_type="strategic"), (
f"Failed to change persistentVolumeReclaimPolicy "
f"for pv {pv_name} to Delete")
log.info("Changed reclaim policy of all remaining PVs to Delete")
# Verify PVs deleted. PVs will be deleted immediately after setting
# reclaim policy to Delete
for pv_obj in retain_pvs:
pv_obj.ocp.wait_for_delete(pv_obj.name, 300)
log.info(
"Verified: All remaining PVs are deleted after changing reclaim "
"policy to Delete.")
# Verify PV using ceph toolbox. Wait for Image/Subvolume to be deleted.
pool_name = (default_ceph_block_pool()
if interface == constants.CEPHBLOCKPOOL else None)
for pvc_name, uuid in pvc_uuid_map.items():
assert verify_volume_deleted_in_backend(
interface=interface, image_uuid=uuid, pool_name=pool_name
), f"Volume associated with PVC {pvc_name} still exists in backend"
log.info("Verified: Image/Subvolume removed from backend.")
