def test_rbd_space_reclaim_no_space(self):
"""
Test to verify RBD space reclamation
Steps:
1. Create and attach RBD PVC of size 25 GiB to an app pod.
2. Get the used size of the RBD pool
3. Create a file of size 10GiB
4. Verify the used size of the RBD pool
5. Create ReclaimSpaceJob
6. Verify the size of the RBD pool, no changes should be seen.
"""
pvc_obj = self.pvc[0]
pod_obj = self.pod[0]
fio_filename1 = "fio_file1"
fio_filename2 = "fio_file2"
# Fetch the used size of pool
cbp_name = self.sc_obj.get().get("parameters").get("pool")
used_size_before_io = fetch_used_size(cbp_name)
log.info(f"Used size before IO is {used_size_before_io}")
# Create a 10 GB file
for filename in [fio_filename1, fio_filename2]:
pod_obj.run_io(
storage_type="fs",
size="10G",
runtime=100,
fio_filename=filename,
end_fsync=1,
)
pod_obj.get_fio_results()
# Verify used size after IO
exp_used_size_after_io = used_size_before_io + (20 * self.pool_replica)
used_size_after_io = fetch_used_size(cbp_name, exp_used_size_after_io)
log.info(f"Used size after IO is {used_size_after_io}")
# Create ReclaimSpaceJob
reclaim_space_job = pvc_obj.create_reclaim_space_job()
# Verify Succeeded result of ReclaimSpaceJob
self.reclaim_space_job(reclaim_space_job)
# Verify space is reclaimed by checking the used size of the RBD pool
used_after_reclaiming_space = fetch_used_size(cbp_name,
used_size_after_io)
log.info(
f"Memory remains intact. Used size after io is {used_after_reclaiming_space}."
)
def test_pvc_delete_and_verify_size_is_returned_to_backend_pool(
self, pause_and_resume_cluster_load, pvc_factory, pod_factory
):
"""
Test case to verify after delete pvc size returned to backend pools
"""
cbp_name = helpers.default_ceph_block_pool()
# TODO: Get exact value of replica size
replica_size = 3
pvc_obj = pvc_factory(
interface=constants.CEPHBLOCKPOOL, size=10, status=constants.STATUS_BOUND
)
pod_obj = pod_factory(
interface=constants.CEPHBLOCKPOOL,
pvc=pvc_obj,
status=constants.STATUS_RUNNING,
)
pvc_obj.reload()
used_before_io = helpers.fetch_used_size(cbp_name)
logger.info(f"Used before IO {used_before_io}")
# Write 6Gb
pod.run_io_and_verify_mount_point(pod_obj, bs="10M", count="600")
exp_size = used_before_io + (6 * replica_size)
used_after_io = helpers.fetch_used_size(cbp_name, exp_size)
logger.info(f"Used space after IO {used_after_io}")
rbd_image_id = pvc_obj.image_uuid
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name)
verify_pv_not_exists(pvc_obj, cbp_name, rbd_image_id)
used_after_deleting_pvc = helpers.fetch_used_size(cbp_name, used_before_io)
logger.info(f"Used after deleting PVC {used_after_deleting_pvc}")
def test_verify_rbd_thick_pvc_utilization(
self,
pvc_factory,
pod_factory,
):
"""
Test to verify the storage utilization of RBD thick provisioned PVC
"""
pvc_size = 15
replica_size = 3
file1 = "fio_file1"
file2 = "fio_file2"
rbd_pool = default_ceph_block_pool()
size_before_pvc = fetch_used_size(rbd_pool)
log.info(
f"Storage pool used size before creating the PVC is {size_before_pvc}"
)
# Create RBD thick PVC
pvc_obj = pvc_factory(
interface=constants.CEPHBLOCKPOOL,
project=self.proj_obj,
storageclass=default_thick_storage_class(),
size=pvc_size,
access_mode=constants.ACCESS_MODE_RWO,
status=constants.STATUS_BOUND,
)
size_after_pvc = fetch_used_size(
rbd_pool, size_before_pvc + (pvc_size * replica_size))
log.info(
f"Verified: Storage pool used size after creating the PVC is {size_after_pvc}"
)
pod_obj = pod_factory(
interface=constants.CEPHBLOCKPOOL,
pvc=pvc_obj,
status=constants.STATUS_RUNNING,
)
# Create 5GB file
pod_obj.run_io(
storage_type="fs",
size="5G",
runtime=60,
fio_filename=file1,
end_fsync=1,
)
pod_obj.get_fio_results()
# Verify the used size after IO
fetch_used_size(rbd_pool, size_before_pvc + (pvc_size * replica_size))
# Create another 5GB file
pod_obj.run_io(
storage_type="fs",
size="5G",
runtime=60,
fio_filename=file2,
end_fsync=1,
)
pod_obj.get_fio_results()
# Verify the used size after IO
fetch_used_size(rbd_pool, size_before_pvc + (pvc_size * replica_size))
# Delete the files created by fio
mount_point = pod_obj.get_storage_path()
rm_cmd = f"rm {path.join(mount_point, file1)} {path.join(mount_point, file2)}"
pod_obj.exec_cmd_on_pod(command=rm_cmd, out_yaml_format=False)
# Verify the used size after deleting the files
fetch_used_size(rbd_pool, size_before_pvc + (pvc_size * replica_size))
# Delete the pod
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
# Delete the PVC
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name)
# Verify used size after deleting the PVC
size_after_pvc_delete = fetch_used_size(rbd_pool, size_before_pvc)
log.info(
f"Verified: Storage pool used size after deleting the PVC is {size_after_pvc_delete}"
)
def test_rbd_space_reclaim_cronjob(self, pause_and_resume_cluster_load):
"""
Test to verify RBD space reclamation
Steps:
1. Create and attach RBD PVC of size 25 GiB to an app pod.
2. Get the used size of the RBD pool
3. Create four files of size 4GiB
4. Verify the increased used size of the RBD pool
5. Delete three file
6. Create ReclaimSpaceJob
7. Verify the decreased used size of the RBD pool
8. Verify the presence of other files in the folder
"""
pvc_obj = self.pvc[0]
pod_obj = self.pod[0]
fio_filename1 = "fio_file1"
fio_filename2 = "fio_file2"
fio_filename3 = "fio_file3"
fio_filename4 = "fio_file4"
schedule = ["hourly", "midnight", "weekly"]
# Fetch the used size of pool
cbp_name = self.sc_obj.get().get("parameters").get("pool")
log.info(f"Cephblock pool name {cbp_name}")
used_size_before_io = fetch_used_size(cbp_name)
log.info(f"Used size before IO is {used_size_before_io}")
# Create four 4 GB file
for filename in [
fio_filename1, fio_filename2, fio_filename3, fio_filename4
]:
pod_obj.run_io(
storage_type="fs",
size="4G",
runtime=100,
fio_filename=filename,
end_fsync=1,
)
pod_obj.get_fio_results()
# Verify used size after IO
exp_used_size_after_io = used_size_before_io + (16 * self.pool_replica)
used_size_after_io = fetch_used_size(cbp_name, exp_used_size_after_io)
log.info(f"Used size after IO is {used_size_after_io}")
# Delete the file and validate the reclaimspace cronjob
for filename in [fio_filename1, fio_filename2, fio_filename3]:
file_path = get_file_path(pod_obj, filename)
pod_obj.exec_cmd_on_pod(command=f"rm -f {file_path}",
out_yaml_format=False,
timeout=100)
# Verify file is deleted
try:
check_file_existence(pod_obj=pod_obj, file_path=file_path)
except CommandFailed as cmdfail:
if "No such file or directory" not in str(cmdfail):
raise
log.info(f"Verified: File {file_path} deleted.")
# Create ReclaimSpaceCronJob
for type in schedule:
reclaim_space_job = pvc_obj.create_reclaim_space_cronjob(
schedule=type)
# Wait for the Succeeded result of ReclaimSpaceJob
try:
for reclaim_space_job_yaml in TimeoutSampler(
timeout=120, sleep=5, func=reclaim_space_job.get):
result = reclaim_space_job_yaml["spec"]["schedule"]
if result == "@" + type:
log.info(
f"ReclaimSpaceJob {reclaim_space_job.name} succeeded"
)
break
else:
log.info(
f"Waiting for the Succeeded result of the ReclaimSpaceCronJob {reclaim_space_job.name}. "
f"Present value of result is {result}")
except TimeoutExpiredError:
raise UnexpectedBehaviour(
f"ReclaimSpaceJob {reclaim_space_job.name} is not successful. Yaml output:{reclaim_space_job.get()}"
)
# Verify the presence of another file in the directory
log.info("Verifying the existence of remaining file in the directory")
file_path = get_file_path(pod_obj, fio_filename4)
log.info(check_file_existence(pod_obj=pod_obj, file_path=file_path))
if check_file_existence(pod_obj=pod_obj, file_path=file_path):
log.info(f"{fio_filename4} is intact")
def test_rbd_space_reclaim(self):
"""
Test to verify RBD space reclamation
Steps:
1. Create and attach RBD PVC of size 25 GiB to an app pod.
2. Get the used size of the RBD pool
3. Create two files of size 10GiB
4. Verify the increased used size of the RBD pool
5. Delete one file
6. Create ReclaimSpaceJob
7. Verify the decreased used size of the RBD pool
"""
pvc_obj = self.pvc[0]
pod_obj = self.pod[0]
fio_filename1 = "fio_file1"
fio_filename2 = "fio_file2"
# Fetch the used size of pool
cbp_name = self.sc_obj.get().get("parameters").get("pool")
used_size_before_io = fetch_used_size(cbp_name)
log.info(f"Used size before IO is {used_size_before_io}")
# Create two 10 GB file
for filename in fio_filename1, fio_filename2:
pod_obj.run_io(
storage_type="fs",
size="10G",
runtime=120,
fio_filename=filename,
end_fsync=1,
)
pod_obj.get_fio_results()
# Verify used size after IO
exp_used_size_after_io = used_size_before_io + (20 * self.pool_replica)
used_size_after_io = fetch_used_size(cbp_name, exp_used_size_after_io)
log.info(f"Used size after IO is {used_size_after_io}")
# Delete one file
file_path = get_file_path(pod_obj, fio_filename2)
pod_obj.exec_cmd_on_pod(command=f"rm -f {file_path}",
out_yaml_format=False)
# Verify file is deleted
try:
check_file_existence(pod_obj=pod_obj, file_path=file_path)
except CommandFailed as cmdfail:
if "No such file or directory" not in str(cmdfail):
raise
log.info(f"Verified: File {file_path} deleted.")
# Create ReclaimSpaceJob
reclaim_space_job = pvc_obj.create_reclaim_space_job()
# Wait for the Succeeded result of ReclaimSpaceJob
try:
for reclaim_space_job_yaml in TimeoutSampler(
timeout=120, sleep=5, func=reclaim_space_job.get):
result = reclaim_space_job_yaml.get("status", {}).get("result")
if result == "Succeeded":
log.info(
f"ReclaimSpaceJob {reclaim_space_job.name} succeeded")
break
else:
log.info(
f"Waiting for the Succeeded result of the ReclaimSpaceJob {reclaim_space_job.name}. "
f"Present value of result is {result}")
except TimeoutExpiredError:
raise UnexpectedBehaviour(
f"ReclaimSpaceJob {reclaim_space_job.name} is not successful. Yaml output:{reclaim_space_job.get()}"
)
# Verify space is reclaimed by checking the used size of the RBD pool
used_after_reclaiming_space = fetch_used_size(
cbp_name, used_size_after_io - (10 * self.pool_replica))
log.info(
f"Space reclamation verified. Used size after reclaiming space is {used_after_reclaiming_space}."
)
def uninstall_cluster_logging():
"""
Function to uninstall cluster-logging from the cluster
Deletes the project "openshift-logging" and "openshift-operators-redhat"
"""
# Validating the pods before deleting the instance
pod_list = get_all_pods(namespace=constants.OPENSHIFT_LOGGING_NAMESPACE)
for pod in pod_list:
logger.info(
f"Pods running in the openshift-logging namespace {pod.name}")
# Excluding cluster-logging-operator from pod_list and getting pod names
pod_names_list = [
pod.name for pod in pod_list
if not pod.name.startswith("cluster-logging-operator")
]
pvc_objs = get_all_pvc_objs(
namespace=constants.OPENSHIFT_LOGGING_NAMESPACE)
# Fetch image uuid associated with PVCs to be deleted
pvc_uuid_map = {}
for pvc_obj in pvc_objs:
pvc_uuid_map[pvc_obj.name] = pvc_obj.image_uuid
# Checking for used space
cbp_name = default_ceph_block_pool()
used_space_before_deletion = fetch_used_size(cbp_name)
logger.info(
f"Used space before deletion of cluster logging {used_space_before_deletion}"
)
# Deleting the clusterlogging instance
clusterlogging_obj = ocp.OCP(
kind=constants.CLUSTER_LOGGING,
namespace=constants.OPENSHIFT_LOGGING_NAMESPACE)
assert clusterlogging_obj.delete(resource_name="instance")
check_pod_vanished(pod_names_list)
for pvc_obj in pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
assert delete_pvcs(pvc_objs=pvc_objs), "PVCs deletion failed"
for pvc_obj in pvc_objs:
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name, timeout=300)
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name, timeout=300)
logger.info("Verified: PVCs are deleted.")
logger.info("Verified: PV are deleted")
for pvc_name, uuid in pvc_uuid_map.items():
rbd = verify_volume_deleted_in_backend(
interface=constants.CEPHBLOCKPOOL,
image_uuid=uuid,
pool_name=cbp_name)
assert rbd, f"Volume associated with PVC {pvc_name} still exists " f"in backend"
# Checking for used space after PVC deletion
used_space_after_deletion = fetch_used_size(cbp_name, exp_val=30)
logger.info(
f"Used space after deletion of cluster logging {used_space_after_deletion}"
)
if used_space_after_deletion < used_space_before_deletion:
logger.info("Expected !!! Space has reclaimed")
else:
logger.warning(
"Unexpected !! No space reclaimed after deletion of PVC")
# Deleting the RBAC permission set
rbac_role = ocp.OCP(
kind=constants.ROLE,
namespace=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE)
rbac_role.delete(yaml_file=constants.EO_RBAC_YAML)
openshift_logging_namespace = ocp.OCP(
kind=constants.NAMESPACES,
namespace=constants.OPENSHIFT_LOGGING_NAMESPACE)
openshift_operators_redhat_namespace = ocp.OCP(
kind=constants.NAMESPACES,
namespace=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE,
)
if openshift_operators_redhat_namespace.get():
assert openshift_operators_redhat_namespace.delete(
resource_name=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE)
logger.info(
"The project openshift-opertors-redhat got deleted successfully")
if openshift_logging_namespace.get():
assert openshift_logging_namespace.delete(
resource_name=constants.OPENSHIFT_LOGGING_NAMESPACE)
logger.info("The namespace openshift-logging got deleted successfully")
def test_rbd_space_reclaim(self):
"""
Test to verify RBD space reclamation
Steps:
1. Create and attach RBD PVC of size 25 GiB to an app pod.
2. Get the used size of the RBD pool
3. Create two files of size 10GiB
4. Verify the increased used size of the RBD pool
5. Delete one file
6. Create ReclaimSpaceJob
7. Verify the decreased used size of the RBD pool
"""
pvc_obj = self.pvc[0]
pod_obj = self.pod[0]
fio_filename1 = "fio_file1"
fio_filename2 = "fio_file2"
# Fetch the used size of pool
cbp_name = self.sc_obj.get().get("parameters").get("pool")
used_size_before_io = fetch_used_size(cbp_name)
log.info(f"Used size before IO is {used_size_before_io}")
# Create two 10 GB file
for filename in fio_filename1, fio_filename2:
pod_obj.run_io(
storage_type="fs",
size="10G",
runtime=120,
fio_filename=filename,
end_fsync=1,
)
pod_obj.get_fio_results()
# Verify used size after IO
exp_used_size_after_io = used_size_before_io + (20 * self.pool_replica)
used_size_after_io = fetch_used_size(cbp_name, exp_used_size_after_io)
log.info(f"Used size after IO is {used_size_after_io}")
# Delete one file
file_path = get_file_path(pod_obj, fio_filename1)
pod_obj.exec_cmd_on_pod(command=f"rm -f {file_path}",
out_yaml_format=False)
# Verify whether file is deleted
try:
check_file_existence(pod_obj=pod_obj, file_path=file_path)
except CommandFailed as cmdfail:
if "No such file or directory" not in str(cmdfail):
raise
log.info(f"Verified: File {file_path} deleted.")
# Wait for 15 seconds after deleting the file
time.sleep(15)
# Create ReclaimSpaceJob
reclaim_space_job = pvc_obj.create_reclaim_space_job()
# Verify Succeeded result of ReclaimSpaceJob
self.reclaim_space_job(reclaim_space_job)
time.sleep(120)
# Verify space is reclaimed by checking the used size of the RBD pool
used_after_reclaiming_space = fetch_used_size(
cbp_name, used_size_after_io - (10 * self.pool_replica))
log.info(
f"Space has been reclaimed. Used size after io is {used_after_reclaiming_space}."
)
# Verify the presence of another file in the directory
log.info("Verifying the existence of remaining file in the pod")
file_path = get_file_path(pod_obj, fio_filename2)
log.info(check_file_existence(pod_obj=pod_obj, file_path=file_path))
if check_file_existence(pod_obj=pod_obj, file_path=file_path):
log.info(f"{fio_filename2} is intact")
def test_no_volume_mounted(self):
"""
Test reclaimspace job with no volume mounted
Steps:
1. Create and attach RBD PVC of size 25 GiB to an app pod.
2. Get the used size of the RBD pool
3. Create a file of size 10GiB
4. Delete the file
5. Delete the pod
6. Create ReclaimSpaceJob
7. No errors should be seen in reclaim space job
"""
pvc_obj = self.pvc[0]
pod_obj = self.pod[0]
fio_filename1 = "fio_file1"
# Fetch the used size of pool
cbp_name = self.sc_obj.get().get("parameters").get("pool")
used_size_before_io = fetch_used_size(cbp_name)
log.info(f"Used size before IO is {used_size_before_io}")
# Create a 10 GB file
pod_obj.run_io(
storage_type="fs",
size="10G",
runtime=120,
fio_filename=fio_filename1,
end_fsync=1,
)
pod_obj.get_fio_results()
# Verify used size after IO
exp_used_size_after_io = used_size_before_io + (10 * self.pool_replica)
used_size_after_io = fetch_used_size(cbp_name, exp_used_size_after_io)
log.info(f"Used size after IO is {used_size_after_io}")
# Delete the file
file_path = get_file_path(pod_obj, fio_filename1)
pod_obj.exec_cmd_on_pod(command=f"rm -f {file_path}",
out_yaml_format=False)
# Verify whether file is deleted
try:
check_file_existence(pod_obj=pod_obj, file_path=file_path)
except CommandFailed as cmdfail:
if "No such file or directory" not in str(cmdfail):
raise
log.info(f"Verified: File {file_path} deleted.")
# Delete the pod
log.info(f"Deleting the pod {pod_obj}")
delete_pods([pod_obj])
# Create ReclaimSpaceJob
reclaim_space_job = pvc_obj.create_reclaim_space_job()
# Verify Succeeded result of ReclaimSpaceJob
self.reclaim_space_job(reclaim_space_job)