def write_fio_on_pod(pod_obj, file_size):
"""
Writes IO of file_size size to a pod
Args:
pod_obj: pod object to write IO
file_size: the size of the IO to be written opn pod
"""
file_name = pod_obj.name
logger.info(f"Starting IO on the POD {pod_obj.name}")
now = datetime.now()
# Going to run only write IO to write to PVC file size data before creating a clone
pod_obj.fillup_fs(size=file_size, fio_filename=file_name)
# Wait for fio to finish
fio_result = pod_obj.get_fio_results(timeout=3600)
err_count = fio_result.get("jobs")[0].get("error")
assert err_count == 0, f"IO error on pod {pod_obj.name}. FIO result: {fio_result}."
logger.info("IO on the PVC Finished")
later = datetime.now()
diff = int((later - now).total_seconds() / 60)
logger.info(f"Writing of {file_size} took {diff} mins")
# Verify presence of the file on pvc
file_path = pod.get_file_path(pod_obj, file_name)
logger.info(f"Actual file path on the pod is {file_path}.")
assert pod.check_file_existence(
pod_obj, file_path), f"File {file_name} does not exist"
logger.info(f"File {file_name} exists in {pod_obj.name}.")
def get_postgres_used_file_space(self, pod_obj_list):
"""
Get the used file space on a mount point
Args:
pod_obj_list (POD): List of pod objects
Returns:
list: List of pod object
"""
# Get the used file space on a mount point
for pod_obj in pod_obj_list:
filepath = get_file_path(pod_obj, "pgdata")
filespace = pod_obj.exec_cmd_on_pod(command=f"du -sh {filepath}",
out_yaml_format=False)
filespace = filespace.split()[0]
pod_obj.filespace = filespace
return pod_obj_list
def run_fio_on_pod(self, file_size):
"""
Args:
file_size (str): Size of file to write in MB, e.g. 200M or 50000M
"""
file_name = self.pod_obj.name
log.info(f"Starting IO on the POD {self.pod_obj.name}")
# Going to run only write IO to fill the PVC for the before creating a clone
self.pod_obj.fillup_fs(size=file_size, fio_filename=file_name)
# Wait for fio to finish - non default timeout is needed for big pvc/clones
fio_result = self.pod_obj.get_fio_results(timeout=18000)
err_count = fio_result.get("jobs")[0].get("error")
assert err_count == 0, (f"IO error on pod {self.pod_obj.name}. "
f"FIO result: {fio_result}.")
log.info("IO on the PVC Finished")
# Verify presence of the file on pvc
file_path = pod.get_file_path(self.pod_obj, file_name)
log.info(f"Actual file path on the pod is {file_path}.")
assert pod.check_file_existence(
self.pod_obj, file_path), f"File {file_name} does not exist"
log.info(f"File {file_name} exists in {self.pod_obj.name}.")
def test_pvc_to_pvc_clone(self, interface_type, teardown_factory):
"""
Create a clone from an existing pvc,
verify data is preserved in the cloning.
"""
logger.info(f"Running IO on pod {self.pod_obj.name}")
file_name = self.pod_obj.name
logger.info(f"File created during IO {file_name}")
self.pod_obj.run_io(storage_type="fs",
size="500M",
fio_filename=file_name)
# Wait for fio to finish
self.pod_obj.get_fio_results()
logger.info(f"Io completed on pod {self.pod_obj.name}.")
# Verify presence of the file
file_path = pod.get_file_path(self.pod_obj, file_name)
logger.info(f"Actual file path on the pod {file_path}")
assert pod.check_file_existence(
self.pod_obj, file_path), f"File {file_name} does not exist"
logger.info(f"File {file_name} exists in {self.pod_obj.name}")
# Calculate md5sum of the file.
orig_md5_sum = pod.cal_md5sum(self.pod_obj, file_name)
# Create a clone of the existing pvc.
sc_name = self.pvc_obj.backed_sc
parent_pvc = self.pvc_obj.name
clone_yaml = constants.CSI_RBD_PVC_CLONE_YAML
if interface_type == constants.CEPHFILESYSTEM:
clone_yaml = constants.CSI_CEPHFS_PVC_CLONE_YAML
cloned_pvc_obj = pvc.create_pvc_clone(sc_name, parent_pvc, clone_yaml)
teardown_factory(cloned_pvc_obj)
helpers.wait_for_resource_state(cloned_pvc_obj, constants.STATUS_BOUND)
cloned_pvc_obj.reload()
# Create and attach pod to the pvc
clone_pod_obj = helpers.create_pod(
interface_type=interface_type,
pvc_name=cloned_pvc_obj.name,
namespace=cloned_pvc_obj.namespace,
pod_dict_path=constants.NGINX_POD_YAML,
)
# Confirm that the pod is running
helpers.wait_for_resource_state(resource=clone_pod_obj,
state=constants.STATUS_RUNNING)
clone_pod_obj.reload()
teardown_factory(clone_pod_obj)
# Verify file's presence on the new pod
logger.info(f"Checking the existence of {file_name} on cloned pod "
f"{clone_pod_obj.name}")
assert pod.check_file_existence(
clone_pod_obj, file_path), f"File {file_path} does not exist"
logger.info(f"File {file_name} exists in {clone_pod_obj.name}")
# Verify Contents of a file in the cloned pvc
# by validating if md5sum matches.
logger.info(f"Verifying that md5sum of {file_name} "
f"on pod {self.pod_obj.name} matches with md5sum "
f"of the same file on restore pod {clone_pod_obj.name}")
assert pod.verify_data_integrity(
clone_pod_obj, file_name,
orig_md5_sum), "Data integrity check failed"
logger.info("Data integrity check passed, md5sum are same")
logger.info("Run IO on new pod")
clone_pod_obj.run_io(storage_type="fs", size="100M", runtime=10)
# Wait for IO to finish on the new pod
clone_pod_obj.get_fio_results()
logger.info(f"IO completed on pod {clone_pod_obj.name}")
def test_rbd_space_reclaim_cronjob(self, pause_and_resume_cluster_load):
"""
Test to verify RBD space reclamation
Steps:
1. Create and attach RBD PVC of size 25 GiB to an app pod.
2. Get the used size of the RBD pool
3. Create four files of size 4GiB
4. Verify the increased used size of the RBD pool
5. Delete three file
6. Create ReclaimSpaceJob
7. Verify the decreased used size of the RBD pool
8. Verify the presence of other files in the folder
"""
pvc_obj = self.pvc[0]
pod_obj = self.pod[0]
fio_filename1 = "fio_file1"
fio_filename2 = "fio_file2"
fio_filename3 = "fio_file3"
fio_filename4 = "fio_file4"
schedule = ["hourly", "midnight", "weekly"]
# Fetch the used size of pool
cbp_name = self.sc_obj.get().get("parameters").get("pool")
log.info(f"Cephblock pool name {cbp_name}")
used_size_before_io = fetch_used_size(cbp_name)
log.info(f"Used size before IO is {used_size_before_io}")
# Create four 4 GB file
for filename in [
fio_filename1, fio_filename2, fio_filename3, fio_filename4
]:
pod_obj.run_io(
storage_type="fs",
size="4G",
runtime=100,
fio_filename=filename,
end_fsync=1,
)
pod_obj.get_fio_results()
# Verify used size after IO
exp_used_size_after_io = used_size_before_io + (16 * self.pool_replica)
used_size_after_io = fetch_used_size(cbp_name, exp_used_size_after_io)
log.info(f"Used size after IO is {used_size_after_io}")
# Delete the file and validate the reclaimspace cronjob
for filename in [fio_filename1, fio_filename2, fio_filename3]:
file_path = get_file_path(pod_obj, filename)
pod_obj.exec_cmd_on_pod(command=f"rm -f {file_path}",
out_yaml_format=False,
timeout=100)
# Verify file is deleted
try:
check_file_existence(pod_obj=pod_obj, file_path=file_path)
except CommandFailed as cmdfail:
if "No such file or directory" not in str(cmdfail):
raise
log.info(f"Verified: File {file_path} deleted.")
# Create ReclaimSpaceCronJob
for type in schedule:
reclaim_space_job = pvc_obj.create_reclaim_space_cronjob(
schedule=type)
# Wait for the Succeeded result of ReclaimSpaceJob
try:
for reclaim_space_job_yaml in TimeoutSampler(
timeout=120, sleep=5, func=reclaim_space_job.get):
result = reclaim_space_job_yaml["spec"]["schedule"]
if result == "@" + type:
log.info(
f"ReclaimSpaceJob {reclaim_space_job.name} succeeded"
)
break
else:
log.info(
f"Waiting for the Succeeded result of the ReclaimSpaceCronJob {reclaim_space_job.name}. "
f"Present value of result is {result}")
except TimeoutExpiredError:
raise UnexpectedBehaviour(
f"ReclaimSpaceJob {reclaim_space_job.name} is not successful. Yaml output:{reclaim_space_job.get()}"
)
# Verify the presence of another file in the directory
log.info("Verifying the existence of remaining file in the directory")
file_path = get_file_path(pod_obj, fio_filename4)
log.info(check_file_existence(pod_obj=pod_obj, file_path=file_path))
if check_file_existence(pod_obj=pod_obj, file_path=file_path):
log.info(f"{fio_filename4} is intact")
def test_snapshot_at_different_usage_level(
self, snapshot_factory, snapshot_restore_factory, pod_factory
):
"""
Test to take multiple snapshots of same PVC when the PVC usage is at
0%, 20%, 40%, 60%, and 80%, then delete the parent PVC and restore the
snapshots to create new PVCs. Delete snapshots and attach the restored
PVCs to pods to verify the data.
"""
snapshots = []
usage_percent = [0, 20, 40, 60, 80]
for usage in usage_percent:
if usage != 0:
for pod_obj in self.pods:
log.info(f"Running IO on pod {pod_obj.name} to utilize {usage}%")
pod_obj.pvc.filename = f"{pod_obj.name}_{usage}"
pod_obj.run_io(
storage_type="fs",
size=f"{int(self.pvc_size/len(usage_percent))}G",
runtime=20,
fio_filename=pod_obj.pvc.filename,
)
log.info(f"IO started on all pods to utilize {usage}%")
for pod_obj in self.pods:
# Wait for fio to finish
pod_obj.get_fio_results()
log.info(
f"IO to utilize {usage}% finished on pod " f"{pod_obj.name}"
)
# Calculate md5sum
md5_sum = pod.cal_md5sum(pod_obj, pod_obj.pvc.filename)
if not getattr(pod_obj.pvc, "md5_sum", None):
setattr(pod_obj.pvc, "md5_sum", {})
pod_obj.pvc.md5_sum[pod_obj.pvc.filename] = md5_sum
# Take snapshot of all PVCs
log.info(f"Creating snapshot of all PVCs at {usage}%")
for pvc_obj in self.pvcs:
log.info(f"Creating snapshot of PVC {pvc_obj.name} at {usage}%")
snap_obj = snapshot_factory(pvc_obj, wait=False)
# Set a dict containing filename:md5sum for later verification
setattr(snap_obj, "md5_sum", deepcopy(getattr(pvc_obj, "md5_sum", {})))
snap_obj.usage_on_mount = get_used_space_on_mount_point(
pvc_obj.get_attached_pods()[0]
)
snapshots.append(snap_obj)
log.info(f"Created snapshot of PVC {pvc_obj.name} at {usage}%")
log.info(f"Created snapshot of all PVCs at {usage}%")
log.info("Snapshots creation completed.")
# Verify snapshots are ready
log.info("Verify snapshots are ready")
for snapshot in snapshots:
snapshot.ocp.wait_for_resource(
condition="true",
resource_name=snapshot.name,
column=constants.STATUS_READYTOUSE,
timeout=90,
)
# Delete pods
log.info("Deleting the pods")
for pod_obj in self.pods:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
log.info("Deleted all the pods")
# Delete parent PVCs
log.info("Deleting parent PVCs")
for pvc_obj in self.pvcs:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name)
log.info(
f"Deleted PVC {pvc_obj.name}. Verifying whether PV "
f"{pv_obj.name} is deleted."
)
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
log.info(
"Deleted parent PVCs before restoring snapshot. " "PVs are also deleted."
)
restore_pvc_objs = []
# Create PVCs out of the snapshots
log.info("Creating new PVCs from snapshots")
for snapshot in snapshots:
log.info(f"Creating a PVC from snapshot {snapshot.name}")
restore_pvc_obj = snapshot_restore_factory(
snapshot_obj=snapshot,
size=f"{self.pvc_size}Gi",
volume_mode=snapshot.parent_volume_mode,
access_mode=snapshot.parent_access_mode,
status="",
)
log.info(
f"Created PVC {restore_pvc_obj.name} from snapshot " f"{snapshot.name}"
)
restore_pvc_objs.append(restore_pvc_obj)
log.info("Created new PVCs from all the snapshots")
# Confirm that the restored PVCs are Bound
# Increased wait time to 600 seconds as a workaround for BZ 1899968
# TODO: Revert wait time to 200 seconds once BZ 1899968 is fixed
log.info("Verify the restored PVCs are Bound")
for pvc_obj in restore_pvc_objs:
wait_for_resource_state(
resource=pvc_obj, state=constants.STATUS_BOUND, timeout=600
)
pvc_obj.reload()
log.info("Verified: Restored PVCs are Bound.")
snapcontent_objs = []
# Get VolumeSnapshotContent form VolumeSnapshots and delete
# VolumeSnapshots
log.info("Deleting snapshots")
for snapshot in snapshots:
snapcontent_objs.append(get_snapshot_content_obj(snap_obj=snapshot))
snapshot.delete()
# Verify volume snapshots are deleted
log.info("Verify snapshots are deleted")
for snapshot in snapshots:
snapshot.ocp.wait_for_delete(resource_name=snapshot.name)
log.info("Verified: Snapshots are deleted")
# Verify VolumeSnapshotContents are deleted
for snapcontent_obj in snapcontent_objs:
snapcontent_obj.ocp.wait_for_delete(
resource_name=snapcontent_obj.name, timeout=180
)
# Attach the restored PVCs to pods
log.info("Attach the restored PVCs to pods")
restore_pod_objs = []
for restore_pvc_obj in restore_pvc_objs:
interface = (
constants.CEPHFILESYSTEM
if (constants.CEPHFS_INTERFACE in restore_pvc_obj.snapshot.parent_sc)
else constants.CEPHBLOCKPOOL
)
restore_pod_obj = pod_factory(
interface=interface, pvc=restore_pvc_obj, status=""
)
log.info(
f"Attached the PVC {restore_pvc_obj.name} to pod "
f"{restore_pod_obj.name}"
)
restore_pod_objs.append(restore_pod_obj)
# Verify the new pods are running
log.info("Verify the new pods are running")
for pod_obj in restore_pod_objs:
timeout = (
300
if config.ENV_DATA["platform"] == constants.IBMCLOUD_PLATFORM
else 60
)
wait_for_resource_state(pod_obj, constants.STATUS_RUNNING, timeout)
log.info("Verified: New pods are running")
# Verify md5sum of files
log.info("Verifying md5sum of files on all the pods")
for restore_pod_obj in restore_pod_objs:
log.info(
f"Verifying md5sum of these files on pod "
f"{restore_pod_obj.name}:"
f"{restore_pod_obj.pvc.snapshot.md5_sum}"
)
for (
file_name,
actual_md5_sum,
) in restore_pod_obj.pvc.snapshot.md5_sum.items():
file_path = pod.get_file_path(restore_pod_obj, file_name)
log.info(
f"Checking the existence of file {file_name} on pod "
f"{restore_pod_obj.name}"
)
assert pod.check_file_existence(restore_pod_obj, file_path), (
f"File {file_name} does not exist on pod " f"{restore_pod_obj.name}"
)
log.info(f"File {file_name} exists on pod {restore_pod_obj.name}")
# Verify that the md5sum matches
log.info(
f"Verifying md5sum of file {file_name} on pod "
f"{restore_pod_obj.name}"
)
pod.verify_data_integrity(restore_pod_obj, file_name, actual_md5_sum)
log.info(
f"Verified md5sum of file {file_name} on pod "
f"{restore_pod_obj.name}"
)
log.info(
f"Verified md5sum of these files on pod "
f"{restore_pod_obj.name}:"
f"{restore_pod_obj.pvc.snapshot.md5_sum}"
)
log.info("md5sum verified")
# Verify usage on mount point
log.info("Verify usage on new pods")
for pod_obj in restore_pod_objs:
usage_on_pod = get_used_space_on_mount_point(pod_obj)
assert usage_on_pod == pod_obj.pvc.snapshot.usage_on_mount, (
f"Usage on mount point is not the expected value on pod "
f"{pod_obj.name}. Usage in percentage {usage_on_pod}. "
f"Expected usage in percentage "
f"{pod_obj.pvc.snapshot.usage_on_mount}"
)
log.info(
f"Verified usage on new pod {pod_obj.name}. Usage in "
f"percentage {usage_on_pod}. Expected usage in percentage "
f"{pod_obj.pvc.snapshot.usage_on_mount}"
)
log.info("Verified usage on new pods")
def test_snapshot_at_different_usage_level(self, snapshot_factory,
snapshot_restore_factory,
pod_factory):
"""
Test to take multiple snapshots of same PVC when the PVC usage is at
0%, 20%, 40%, 60%, and 80%, then delete the parent PVC and restore the
snapshots to create new PVCs. Delete snapshots and attach the restored
PVCs to pods to verify the data.
"""
snapshots = []
usage_percent = [0, 20, 40, 60, 80]
for usage in usage_percent:
if usage != 0:
for pod_obj in self.pods:
log.info(
f"Running IO on pod {pod_obj.name} to utilize {usage}%"
)
pod_obj.pvc.filename = f'{pod_obj.name}_{usage}'
pod_obj.run_io(
storage_type='fs',
size=f'{int(self.pvc_size/len(usage_percent))}G',
runtime=20,
fio_filename=pod_obj.pvc.filename)
log.info(f"IO started on all pods to utilize {usage}%")
for pod_obj in self.pods:
# Wait for fio to finish
pod_obj.get_fio_results()
log.info(f"IO to utilize {usage}% finished on pod "
f"{pod_obj.name}")
# Calculate md5sum
md5_sum = pod.cal_md5sum(pod_obj, pod_obj.pvc.filename)
if not getattr(pod_obj.pvc, 'md5_sum', None):
setattr(pod_obj.pvc, 'md5_sum', {})
pod_obj.pvc.md5_sum[pod_obj.pvc.filename] = md5_sum
# Take snapshot of all PVCs
log.info(f"Creating snapshot of all PVCs at {usage}%")
for pvc_obj in self.pvcs:
log.info(
f"Creating snapshot of PVC {pvc_obj.name} at {usage}%")
snap_obj = snapshot_factory(pvc_obj, wait=False)
# Set a dict containing filename:md5sum for later verification
setattr(snap_obj, 'md5_sum',
deepcopy(getattr(pvc_obj, 'md5_sum', {})))
snapshots.append(snap_obj)
log.info(f"Created snapshot of PVC {pvc_obj.name} at {usage}%")
log.info(f"Created snapshot of all PVCs at {usage}%")
log.info("Snapshots creation completed.")
# Verify snapshots are ready
log.info("Verify snapshots are ready")
for snapshot in snapshots:
snapshot.ocp.wait_for_resource(condition='true',
resource_name=snapshot.name,
column=constants.STATUS_READYTOUSE,
timeout=90)
# Delete pods
log.info("Deleting the pods")
for pod_obj in self.pods:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
log.info("Deleted all the pods")
# Delete parent PVCs
log.info("Deleting parent PVCs")
for pvc_obj in self.pvcs:
# TODO: Unblock parent PVC deletion for cephfs PVC when the bug 1854501 is fixed
if constants.RBD_INTERFACE in pvc_obj.backed_sc:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name)
log.info(f"Deleted PVC {pvc_obj.name}. Verifying whether PV "
f"{pv_obj.name} is deleted.")
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
log.info("Deleted parent PVCs before restoring snapshot. "
"PVs are also deleted.")
restore_pvc_objs = []
# Create PVCs out of the snapshots
log.info("Creating new PVCs from snapshots")
for snapshot in snapshots:
log.info(f"Creating a PVC from snapshot {snapshot.name}")
restore_pvc_obj = snapshot_restore_factory(
snapshot_obj=snapshot,
size=f'{self.pvc_size}Gi',
volume_mode=snapshot.parent_volume_mode,
access_mode=snapshot.parent_access_mode,
status='')
log.info(f"Created PVC {restore_pvc_obj.name} from snapshot "
f"{snapshot.name}")
restore_pvc_objs.append(restore_pvc_obj)
log.info("Created new PVCs from all the snapshots")
# Confirm that the restored PVCs are Bound
log.info("Verify the restored PVCs are Bound")
for pvc_obj in restore_pvc_objs:
wait_for_resource_state(resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=90)
pvc_obj.reload()
log.info("Verified: Restored PVCs are Bound.")
# Delete volume snapshots
log.info("Deleting snapshots")
for snapshot in snapshots:
snapshot.delete()
# Verify volume snapshots are deleted
log.info("Verify snapshots are deleted")
for snapshot in snapshots:
snapshot.ocp.wait_for_delete(resource_name=snapshot.name)
log.info("Verified: Snapshots are deleted")
# Attach the restored PVCs to pods
log.info("Attach the restored PVCs to pods")
restore_pod_objs = []
for restore_pvc_obj in restore_pvc_objs:
interface = constants.CEPHFILESYSTEM if (
constants.CEPHFS_INTERFACE
in restore_pvc_obj.snapshot.parent_sc
) else constants.CEPHBLOCKPOOL
restore_pod_obj = pod_factory(interface=interface,
pvc=restore_pvc_obj,
status='')
log.info(f"Attached the PVC {restore_pvc_obj.name} to pod "
f"{restore_pod_obj.name}")
restore_pod_objs.append(restore_pod_obj)
# Verify the new pods are running
log.info("Verify the new pods are running")
for pod_obj in restore_pod_objs:
wait_for_resource_state(pod_obj, constants.STATUS_RUNNING)
log.info("Verified: New pods are running")
# Verify md5sum of files
log.info("Verifying md5sum of files on all the pods")
for restore_pod_obj in restore_pod_objs:
log.info(f"Verifying md5sum of these files on pod "
f"{restore_pod_obj.name}:"
f"{restore_pod_obj.pvc.snapshot.md5_sum}")
for file_name, actual_md5_sum in (
restore_pod_obj.pvc.snapshot.md5_sum.items()):
file_path = pod.get_file_path(restore_pod_obj, file_name)
log.info(f"Checking the existence of file {file_name} on pod "
f"{restore_pod_obj.name}")
assert pod.check_file_existence(
restore_pod_obj,
file_path), (f"File {file_name} does not exist on pod "
f"{restore_pod_obj.name}")
log.info(
f"File {file_name} exists on pod {restore_pod_obj.name}")
# Verify that the md5sum matches
log.info(f"Verifying md5sum of file {file_name} on pod "
f"{restore_pod_obj.name}")
pod.verify_data_integrity(restore_pod_obj, file_name,
actual_md5_sum)
log.info(f"Verified md5sum of file {file_name} on pod "
f"{restore_pod_obj.name}")
log.info(f"Verified md5sum of these files on pod "
f"{restore_pod_obj.name}:"
f"{restore_pod_obj.pvc.snapshot.md5_sum}")
log.info("md5sum verified")
def test_pvc_snapshot_performance(self, pvc_size):
"""
1. Run I/O on a pod file
2. Calculate md5sum of the file
3. Take a snapshot of the PVC
4. Measure the total snapshot creation time and the CSI snapshot creation time
4. Restore From the snapshot and measure the time
5. Attach a new pod to it
6. Verify that the file is present on the new pod also
7. Verify that the md5sum of the file on the new pod matches
with the md5sum of the file on the original pod
This scenario run 3 times and report all the average results of the 3 runs
and will send them to the ES
Args:
pvc_size: the size of the PVC to be tested - parametrize
"""
# Getting the total Storage capacity
ceph_cluster = CephCluster()
ceph_capacity = ceph_cluster.get_ceph_capacity()
log.info(f"Total capacity size is : {ceph_capacity}")
log.info(f"PVC Size is : {pvc_size}")
log.info(f"Needed capacity is {int(int(pvc_size) * 5)}")
if int(ceph_capacity) < int(pvc_size) * 5:
log.error(
f"PVC size is {pvc_size}GiB and it is too large for this system"
f" which have only {ceph_capacity}GiB")
return
# Calculating the file size as 25% of the PVC size
# in the end the PVC will be 75% full
filesize = self.pvc_obj.size * 0.25
# Change the file size to MB and from int to str
file_size = f"{int(filesize * 1024)}M"
all_results = []
self.results_path = get_full_test_logs_path(cname=self)
log.info(f"Logs file path name is : {self.full_log_path}")
# Produce ES report
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
self.full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"pvc_snapshot_perf",
))
self.full_results.add_key("pvc_size", pvc_size + " GiB")
self.full_results.add_key("interface", self.sc)
self.full_results.all_results["creation_time"] = []
self.full_results.all_results["csi_creation_time"] = []
self.full_results.all_results["creation_speed"] = []
self.full_results.all_results["restore_time"] = []
self.full_results.all_results["restore_speed"] = []
self.full_results.all_results["restore_csi_time"] = []
for test_num in range(self.tests_numbers):
test_results = {
"test_num": test_num + 1,
"dataset": (test_num + 1) * filesize * 1024, # size in MiB
"create": {
"time": None,
"csi_time": None,
"speed": None
},
"restore": {
"time": None,
"speed": None
},
}
log.info(f"Starting test phase number {test_num}")
# Step 1. Run I/O on a pod file.
file_name = f"{self.pod_object.name}-{test_num}"
log.info(f"Starting IO on the POD {self.pod_object.name}")
# Going to run only write IO to fill the PVC for the snapshot
self.pod_object.fillup_fs(size=file_size, fio_filename=file_name)
# Wait for fio to finish
fio_result = self.pod_object.get_fio_results()
err_count = fio_result.get("jobs")[0].get("error")
assert (
err_count == 0
), f"IO error on pod {self.pod_object.name}. FIO result: {fio_result}"
log.info("IO on the PVC Finished")
# Verify presence of the file
file_path = pod.get_file_path(self.pod_object, file_name)
log.info(f"Actual file path on the pod {file_path}")
assert pod.check_file_existence(
self.pod_object, file_path), f"File {file_name} doesn't exist"
log.info(f"File {file_name} exists in {self.pod_object.name}")
# Step 2. Calculate md5sum of the file.
orig_md5_sum = pod.cal_md5sum(self.pod_object, file_name)
# Step 3. Take a snapshot of the PVC and measure the time of creation.
snap_name = self.pvc_obj.name.replace("pvc-test",
f"snapshot-test{test_num}")
log.info(f"Taking snapshot of the PVC {snap_name}")
start_time = datetime.datetime.utcnow().strftime(
"%Y-%m-%dT%H:%M:%SZ")
test_results["create"]["time"] = self.measure_create_snapshot_time(
pvc_name=self.pvc_obj.name,
snap_name=snap_name,
namespace=self.pod_object.namespace,
interface=self.interface,
start_time=start_time,
)
test_results["create"][
"csi_time"] = performance_lib.measure_csi_snapshot_creation_time(
interface=self.interface,
snapshot_id=self.snap_uid,
start_time=start_time,
)
test_results["create"]["speed"] = int(
test_results["dataset"] / test_results["create"]["time"])
log.info(
f' Test {test_num} dataset is {test_results["dataset"]} MiB')
log.info(
f"Snapshot name {snap_name} and id {self.snap_uid} creation time is"
f' : {test_results["create"]["time"]} sec.')
log.info(
f"Snapshot name {snap_name} and id {self.snap_uid} csi creation time is"
f' : {test_results["create"]["csi_time"]} sec.')
log.info(
f'Snapshot speed is : {test_results["create"]["speed"]} MB/sec'
)
# Step 4. Restore the PVC from the snapshot and measure the time
# Same Storage class of the original PVC
sc_name = self.pvc_obj.backed_sc
# Size should be same as of the original PVC
pvc_size = str(self.pvc_obj.size) + "Gi"
# Create pvc out of the snapshot
# Both, the snapshot and the restore PVC should be in same namespace
log.info("Restoring from the Snapshot")
restore_pvc_name = self.pvc_obj.name.replace(
"pvc-test", f"restore-pvc{test_num}")
restore_pvc_yaml = constants.CSI_RBD_PVC_RESTORE_YAML
if self.interface == constants.CEPHFILESYSTEM:
restore_pvc_yaml = constants.CSI_CEPHFS_PVC_RESTORE_YAML
csi_start_time = self.get_time("csi")
log.info("Restoring the PVC from Snapshot")
restore_pvc_obj = pvc.create_restore_pvc(
sc_name=sc_name,
snap_name=self.snap_obj.name,
namespace=self.snap_obj.namespace,
size=pvc_size,
pvc_name=restore_pvc_name,
restore_pvc_yaml=restore_pvc_yaml,
)
helpers.wait_for_resource_state(
restore_pvc_obj,
constants.STATUS_BOUND,
timeout=3600 # setting this to 60 Min.
# since it can be take long time to restore, and we want it to finished.
)
restore_pvc_obj.reload()
log.info("PVC was restored from the snapshot")
test_results["restore"][
"time"] = helpers.measure_pvc_creation_time(
self.interface, restore_pvc_obj.name)
test_results["restore"]["speed"] = int(
test_results["dataset"] / test_results["restore"]["time"])
log.info(
f'Snapshot restore time is : {test_results["restore"]["time"]}'
)
log.info(
f'restore speed is : {test_results["restore"]["speed"]} MB/sec'
)
test_results["restore"][
"csi_time"] = performance_lib.csi_pvc_time_measure(
self.interface, restore_pvc_obj, "create", csi_start_time)
log.info(
f'Snapshot csi restore time is : {test_results["restore"]["csi_time"]}'
)
# Step 5. Attach a new pod to the restored PVC
restore_pod_object = helpers.create_pod(
interface_type=self.interface,
pvc_name=restore_pvc_obj.name,
namespace=self.snap_obj.namespace,
)
# Confirm that the pod is running
helpers.wait_for_resource_state(resource=restore_pod_object,
state=constants.STATUS_RUNNING)
restore_pod_object.reload()
# Step 6. Verify that the file is present on the new pod also.
log.info(f"Checking the existence of {file_name} "
f"on restore pod {restore_pod_object.name}")
assert pod.check_file_existence(
restore_pod_object,
file_path), f"File {file_name} doesn't exist"
log.info(f"File {file_name} exists in {restore_pod_object.name}")
# Step 7. Verify that the md5sum matches
log.info(
f"Verifying that md5sum of {file_name} "
f"on pod {self.pod_object.name} matches with md5sum "
f"of the same file on restore pod {restore_pod_object.name}")
assert pod.verify_data_integrity(
restore_pod_object, file_name,
orig_md5_sum), "Data integrity check failed"
log.info("Data integrity check passed, md5sum are same")
restore_pod_object.delete()
restore_pvc_obj.delete()
all_results.append(test_results)
# clean the enviroment
self.pod_object.delete()
self.pvc_obj.delete()
self.delete_test_project()
# logging the test summary, all info in one place for easy log reading
c_speed, c_runtime, c_csi_runtime, r_speed, r_runtime, r_csi_runtime = (
0 for i in range(6))
log.info("Test summary :")
for tst in all_results:
c_speed += tst["create"]["speed"]
c_runtime += tst["create"]["time"]
c_csi_runtime += tst["create"]["csi_time"]
r_speed += tst["restore"]["speed"]
r_runtime += tst["restore"]["time"]
r_csi_runtime += tst["restore"]["csi_time"]
self.full_results.all_results["creation_time"].append(
tst["create"]["time"])
self.full_results.all_results["csi_creation_time"].append(
tst["create"]["csi_time"])
self.full_results.all_results["creation_speed"].append(
tst["create"]["speed"])
self.full_results.all_results["restore_time"].append(
tst["restore"]["time"])
self.full_results.all_results["restore_speed"].append(
tst["restore"]["speed"])
self.full_results.all_results["restore_csi_time"].append(
tst["restore"]["csi_time"])
self.full_results.all_results["dataset_inMiB"] = tst["dataset"]
log.info(
f"Test {tst['test_num']} results : dataset is {tst['dataset']} MiB. "
f"Take snapshot time is {tst['create']['time']} "
f"at {tst['create']['speed']} MiB/Sec "
f"Restore from snapshot time is {tst['restore']['time']} "
f"at {tst['restore']['speed']} MiB/Sec ")
avg_snap_c_time = c_runtime / self.tests_numbers
avg_snap_csi_c_time = c_csi_runtime / self.tests_numbers
avg_snap_c_speed = c_speed / self.tests_numbers
avg_snap_r_time = r_runtime / self.tests_numbers
avg_snap_r_speed = r_speed / self.tests_numbers
avg_snap_r_csi_time = r_csi_runtime / self.tests_numbers
log.info(f" Average snapshot creation time is {avg_snap_c_time} sec.")
log.info(
f" Average csi snapshot creation time is {avg_snap_csi_c_time} sec."
)
log.info(
f" Average snapshot creation speed is {avg_snap_c_speed} MiB/sec")
log.info(f" Average snapshot restore time is {avg_snap_r_time} sec.")
log.info(
f" Average snapshot restore speed is {avg_snap_r_speed} MiB/sec")
log.info(
f" Average snapshot restore csi time is {avg_snap_r_csi_time} sec."
)
self.full_results.add_key("avg_snap_creation_time_insecs",
avg_snap_c_time)
self.full_results.add_key("avg_snap_csi_creation_time_insecs",
avg_snap_csi_c_time)
self.full_results.add_key("avg_snap_creation_speed", avg_snap_c_speed)
self.full_results.add_key("avg_snap_restore_time_insecs",
avg_snap_r_time)
self.full_results.add_key("avg_snap_restore_speed", avg_snap_r_speed)
self.full_results.add_key("avg_snap_restore_csi_time_insecs",
avg_snap_r_csi_time)
# Write the test results into the ES server
log.info("writing results to elastic search server")
if self.full_results.es_write():
res_link = self.full_results.results_link()
# write the ES link to the test results in the test log.
log.info(f"The result can be found at : {res_link}")
self.write_result_to_file(res_link)
def test_rbd_space_reclaim(self):
"""
Test to verify RBD space reclamation
Steps:
1. Create and attach RBD PVC of size 25 GiB to an app pod.
2. Get the used size of the RBD pool
3. Create two files of size 10GiB
4. Verify the increased used size of the RBD pool
5. Delete one file
6. Create ReclaimSpaceJob
7. Verify the decreased used size of the RBD pool
"""
pvc_obj = self.pvc[0]
pod_obj = self.pod[0]
fio_filename1 = "fio_file1"
fio_filename2 = "fio_file2"
# Fetch the used size of pool
cbp_name = self.sc_obj.get().get("parameters").get("pool")
used_size_before_io = fetch_used_size(cbp_name)
log.info(f"Used size before IO is {used_size_before_io}")
# Create two 10 GB file
for filename in fio_filename1, fio_filename2:
pod_obj.run_io(
storage_type="fs",
size="10G",
runtime=120,
fio_filename=filename,
end_fsync=1,
)
pod_obj.get_fio_results()
# Verify used size after IO
exp_used_size_after_io = used_size_before_io + (20 * self.pool_replica)
used_size_after_io = fetch_used_size(cbp_name, exp_used_size_after_io)
log.info(f"Used size after IO is {used_size_after_io}")
# Delete one file
file_path = get_file_path(pod_obj, fio_filename1)
pod_obj.exec_cmd_on_pod(command=f"rm -f {file_path}",
out_yaml_format=False)
# Verify whether file is deleted
try:
check_file_existence(pod_obj=pod_obj, file_path=file_path)
except CommandFailed as cmdfail:
if "No such file or directory" not in str(cmdfail):
raise
log.info(f"Verified: File {file_path} deleted.")
# Wait for 15 seconds after deleting the file
time.sleep(15)
# Create ReclaimSpaceJob
reclaim_space_job = pvc_obj.create_reclaim_space_job()
# Verify Succeeded result of ReclaimSpaceJob
self.reclaim_space_job(reclaim_space_job)
time.sleep(120)
# Verify space is reclaimed by checking the used size of the RBD pool
used_after_reclaiming_space = fetch_used_size(
cbp_name, used_size_after_io - (10 * self.pool_replica))
log.info(
f"Space has been reclaimed. Used size after io is {used_after_reclaiming_space}."
)
# Verify the presence of another file in the directory
log.info("Verifying the existence of remaining file in the pod")
file_path = get_file_path(pod_obj, fio_filename2)
log.info(check_file_existence(pod_obj=pod_obj, file_path=file_path))
if check_file_existence(pod_obj=pod_obj, file_path=file_path):
log.info(f"{fio_filename2} is intact")
def expand_verify_pvcs(pvc_objs, pod_objs, pvc_size_new, file_name, fio_size):
"""
Expands size of each PVC in the provided list of PVCs,
Verifies data integrity by checking the existence and md5sum of file in the expanded PVC
and
Runs FIO on expanded PVCs and verifies results.
Args:
pvc_objs (list) : List of PVC objects which are to be expanded.
pod_objs (list) : List of POD objects attached to the PVCs.
pvc_size_new (int) : Size of the expanded PVC in GB.
file_name (str) : Name of the file on which FIO is performed.
fio_size (int) : Size in MB of FIO.
"""
# Expand original PVCs
log.info("Started expansion of the PVCs.")
for pvc_obj in pvc_objs:
log.info(f"Expanding size of PVC {pvc_obj.name} to {pvc_size_new}G")
pvc_obj.resize_pvc(pvc_size_new, True)
log.info("Successfully expanded the PVCs.")
# Verify that the fio exists and md5sum matches
for pod_no in range(len(pod_objs)):
pod_obj = pod_objs[pod_no]
if pod_obj.pvc.get_pvc_vol_mode == constants.VOLUME_MODE_BLOCK:
pod.verify_data_integrity_after_expansion_for_block_pvc(
pod_obj, pvc_objs[pod_no], fio_size)
else:
pod.verify_data_integrity(pod_obj, file_name,
pvc_objs[pod_no].md5sum)
# Run IO to utilize 50% of volume
log.info(
"Run IO on all pods to utilise 50% of the expanded PVC used space")
expanded_file_name = "fio_50"
for pod_obj in pod_objs:
log.info(f"Running IO on pod {pod_obj.name}")
log.info(f"File created during IO {expanded_file_name}")
fio_size = int(0.50 * pvc_size_new * 1000)
storage_type = ("block" if pod_obj.pvc.get_pvc_vol_mode
== constants.VOLUME_MODE_BLOCK else "fs")
pod_obj.wl_setup_done = True
pod_obj.wl_obj = workload.WorkLoad(
"test_workload_fio",
pod_obj.get_storage_path(storage_type),
"fio",
storage_type,
pod_obj,
1,
)
pod_obj.run_io(
storage_type=storage_type,
size=f"{fio_size}M",
runtime=20,
fio_filename=expanded_file_name,
end_fsync=1,
)
log.info("Started IO on all pods to utilise 50% of PVCs")
for pod_obj in pod_objs:
# Wait for IO to finish
pod_obj.get_fio_results(3600)
log.info(f"IO finished on pod {pod_obj.name}")
is_block = (True if pod_obj.pvc.get_pvc_vol_mode
== constants.VOLUME_MODE_BLOCK else False)
expanded_file_name_pod = (expanded_file_name if not is_block else
pod_obj.get_storage_path(
storage_type="block"))
# Verify presence of the file
expanded_file_path = (expanded_file_name_pod
if is_block else pod.get_file_path(
pod_obj, expanded_file_name_pod))
log.info(f"Actual file path on the pod {expanded_file_path}")
assert pod.check_file_existence(
pod_obj, expanded_file_path
), f"File {expanded_file_name_pod} does not exist"
log.info(f"File {expanded_file_name_pod} exists in {pod_obj.name}")
def test_rwx_dynamic_pvc(self, interface_type, reclaim_policy, setup,
pvc_factory, pod_factory):
"""
RWX Dynamic PVC creation tests with Reclaim policy set to Retain/Delete
"""
access_mode = constants.ACCESS_MODE_RWX
storage_type = "fs"
sc_obj, worker_nodes_list = setup
logger.info("CephFS RWX test")
logger.info(f"Creating PVC with {access_mode} access mode")
pvc_obj = pvc_factory(
interface=interface_type,
storageclass=sc_obj,
size=self.pvc_size,
access_mode=access_mode,
status=constants.STATUS_BOUND,
)
logger.info(f"Creating first pod on node: {worker_nodes_list[0]} "
f"with pvc {pvc_obj.name}")
pod_obj1 = pod_factory(
interface=interface_type,
pvc=pvc_obj,
status=constants.STATUS_RUNNING,
node_name=worker_nodes_list[0],
pod_dict_path=constants.NGINX_POD_YAML,
)
logger.info(f"Creating second pod on node: {worker_nodes_list[1]} "
f"with pvc {pvc_obj.name}")
pod_obj2 = pod_factory(
interface=interface_type,
pvc=pvc_obj,
status=constants.STATUS_RUNNING,
node_name=worker_nodes_list[1],
pod_dict_path=constants.NGINX_POD_YAML,
)
node_pod1 = pod_obj1.get().get("spec").get("nodeName")
node_pod2 = pod_obj2.get().get("spec").get("nodeName")
assert node_pod1 != node_pod2, "Both pods are on the same node"
# Run IO on both the pods
logger.info(f"Running IO on pod {pod_obj1.name}")
file_name1 = pod_obj1.name
logger.info(file_name1)
pod_obj1.run_io(storage_type=storage_type,
size="1G",
fio_filename=file_name1)
logger.info(f"Running IO on pod {pod_obj2.name}")
file_name2 = pod_obj2.name
pod_obj2.run_io(storage_type=storage_type,
size="1G",
fio_filename=file_name2)
# Check IO and calculate md5sum of files
pod.get_fio_rw_iops(pod_obj1)
md5sum_pod1_data = pod.cal_md5sum(pod_obj=pod_obj1,
file_name=file_name1)
pod.get_fio_rw_iops(pod_obj2)
md5sum_pod2_data = pod.cal_md5sum(pod_obj=pod_obj2,
file_name=file_name2)
logger.info("verify data from alternate pods")
pod.verify_data_integrity(pod_obj=pod_obj2,
file_name=file_name1,
original_md5sum=md5sum_pod1_data)
pod.verify_data_integrity(pod_obj=pod_obj1,
file_name=file_name2,
original_md5sum=md5sum_pod2_data)
# Verify that data is mutable from any pod
logger.info("Perform modification of files from alternate pod")
# Access and rename file written by pod-2 from pod-1
file_path2 = pod.get_file_path(pod_obj2, file_name2)
logger.info(file_path2)
pod_obj1.exec_cmd_on_pod(
command=f'bash -c "mv {file_path2} {file_path2}-renamed"',
out_yaml_format=False,
)
# Access and rename file written by pod-1 from pod-2
file_path1 = pod.get_file_path(pod_obj1, file_name1)
logger.info(file_path1)
pod_obj2.exec_cmd_on_pod(
command=f'bash -c "mv {file_path1} {file_path1}-renamed"',
out_yaml_format=False,
)
logger.info("Verify presence of renamed files from both pods")
file_names = [f"{file_path1}-renamed", f"{file_path2}-renamed"]
for file in file_names:
assert pod.check_file_existence(pod_obj1,
file), f"File {file} doesn't exist"
logger.info(f"File {file} exists in {pod_obj1.name} ")
assert pod.check_file_existence(pod_obj2,
file), f"File {file} doesn't exist"
logger.info(f"File {file} exists in {pod_obj2.name}")
def test_pvc_snapshot(self, interface, teardown_factory):
"""
1. Run I/O on a pod file.
2. Calculate md5sum of the file.
3. Take a snapshot of the PVC.
4. Create a new PVC out of that snapshot.
5. Attach a new pod to it.
6. Verify that the file is present on the new pod also.
7. Verify that the md5sum of the file on the new pod matches
with the md5sum of the file on the original pod.
Args:
interface(str): The type of the interface
(e.g. CephBlockPool, CephFileSystem)
pvc_factory: A fixture to create new pvc
teardown_factory: A fixture to destroy objects
"""
log.info(f"Running IO on pod {self.pod_obj.name}")
file_name = self.pod_obj.name
log.info(f"File created during IO {file_name}")
self.pod_obj.run_io(storage_type="fs",
size="1G",
fio_filename=file_name)
# Wait for fio to finish
fio_result = self.pod_obj.get_fio_results()
err_count = fio_result.get("jobs")[0].get("error")
assert err_count == 0, (f"IO error on pod {self.pod_obj.name}. "
f"FIO result: {fio_result}")
log.info(f"Verified IO on pod {self.pod_obj.name}.")
# Verify presence of the file
file_path = pod.get_file_path(self.pod_obj, file_name)
log.info(f"Actual file path on the pod {file_path}")
assert pod.check_file_existence(
self.pod_obj, file_path), f"File {file_name} doesn't exist"
log.info(f"File {file_name} exists in {self.pod_obj.name}")
# Calculate md5sum
orig_md5_sum = pod.cal_md5sum(self.pod_obj, file_name)
# Take a snapshot
snap_yaml = constants.CSI_RBD_SNAPSHOT_YAML
if interface == constants.CEPHFILESYSTEM:
snap_yaml = constants.CSI_CEPHFS_SNAPSHOT_YAML
snap_name = helpers.create_unique_resource_name("test", "snapshot")
snap_obj = pvc.create_pvc_snapshot(
self.pvc_obj.name,
snap_yaml,
snap_name,
self.pvc_obj.namespace,
helpers.default_volumesnapshotclass(interface).name,
)
snap_obj.ocp.wait_for_resource(
condition="true",
resource_name=snap_obj.name,
column=constants.STATUS_READYTOUSE,
timeout=60,
)
teardown_factory(snap_obj)
# Same Storage class of the original PVC
sc_name = self.pvc_obj.backed_sc
# Size should be same as of the original PVC
pvc_size = str(self.pvc_obj.size) + "Gi"
# Create pvc out of the snapshot
# Both, the snapshot and the restore PVC should be in same namespace
restore_pvc_name = helpers.create_unique_resource_name(
"test", "restore-pvc")
restore_pvc_yaml = constants.CSI_RBD_PVC_RESTORE_YAML
if interface == constants.CEPHFILESYSTEM:
restore_pvc_yaml = constants.CSI_CEPHFS_PVC_RESTORE_YAML
restore_pvc_obj = pvc.create_restore_pvc(
sc_name=sc_name,
snap_name=snap_obj.name,
namespace=snap_obj.namespace,
size=pvc_size,
pvc_name=restore_pvc_name,
restore_pvc_yaml=restore_pvc_yaml,
)
helpers.wait_for_resource_state(restore_pvc_obj,
constants.STATUS_BOUND)
restore_pvc_obj.reload()
teardown_factory(restore_pvc_obj)
# Create and attach pod to the pvc
restore_pod_obj = helpers.create_pod(
interface_type=interface,
pvc_name=restore_pvc_obj.name,
namespace=snap_obj.namespace,
pod_dict_path=constants.NGINX_POD_YAML,
)
# Confirm that the pod is running
helpers.wait_for_resource_state(resource=restore_pod_obj,
state=constants.STATUS_RUNNING)
restore_pod_obj.reload()
teardown_factory(restore_pod_obj)
# Verify that the file is present on the new pod
log.info(f"Checking the existence of {file_name} "
f"on restore pod {restore_pod_obj.name}")
assert pod.check_file_existence(
restore_pod_obj, file_path), f"File {file_name} doesn't exist"
log.info(f"File {file_name} exists in {restore_pod_obj.name}")
# Verify that the md5sum matches
log.info(f"Verifying that md5sum of {file_name} "
f"on pod {self.pod_obj.name} matches with md5sum "
f"of the same file on restore pod {restore_pod_obj.name}")
assert pod.verify_data_integrity(
restore_pod_obj, file_name,
orig_md5_sum), "Data integrity check failed"
log.info("Data integrity check passed, md5sum are same")
log.info("Running IO on new pod")
# Run IO on new pod
restore_pod_obj.run_io(storage_type="fs", size="1G", runtime=20)
# Wait for fio to finish
restore_pod_obj.get_fio_results()
log.info("IO finished o new pod")
def test_pvc_snapshot_performance(self, teardown_factory, pvc_size):
"""
1. Run I/O on a pod file.
2. Calculate md5sum of the file.
3. Take a snapshot of the PVC and measure the time of creation.
4. Restore From the snapshot and measure the time
5. Attach a new pod to it.
6. Verify that the file is present on the new pod also.
7. Verify that the md5sum of the file on the new pod matches
with the md5sum of the file on the original pod.
This scenario run 3 times and report all results
Args:
teardown_factory: A fixture to destroy objects
pvc_size: the size of the PVC to be tested - parametrize
"""
# Getting the total Storage capacity
ceph_cluster = CephCluster()
ceph_capacity = ceph_cluster.get_ceph_capacity()
log.info(f"Total capacity size is : {ceph_capacity}")
log.info(f"PVC Size is : {pvc_size}")
log.info(f"Needed capacity is {int(int(pvc_size) * 5)}")
if int(ceph_capacity) < int(pvc_size) * 5:
log.error(
f"PVC size is {pvc_size}GiB and it is too large for this system"
f" which have only {ceph_capacity}GiB")
return
# Calculating the file size as 25% of the PVC size
# in the end the PVC will be 75% full
filesize = self.pvc_obj.size * 0.25
# Change the file size to MB and from int to str
file_size = f"{int(filesize * 1024)}M"
all_results = []
for test_num in range(self.tests_numbers):
test_results = {
"test_num": test_num + 1,
"dataset": (test_num + 1) * filesize * 1024, # size in MiB
"create": {
"time": None,
"speed": None
},
"restore": {
"time": None,
"speed": None
},
}
log.info(f"Starting test phase number {test_num}")
# Step 1. Run I/O on a pod file.
file_name = f"{self.pod_obj.name}-{test_num}"
log.info(f"Starting IO on the POD {self.pod_obj.name}")
# Going to run only write IO to fill the PVC for the snapshot
self.pod_obj.fillup_fs(size=file_size, fio_filename=file_name)
# Wait for fio to finish
fio_result = self.pod_obj.get_fio_results()
err_count = fio_result.get("jobs")[0].get("error")
assert (
err_count == 0
), f"IO error on pod {self.pod_obj.name}. FIO result: {fio_result}"
log.info("IO on the PVC Finished")
# Verify presence of the file
file_path = pod.get_file_path(self.pod_obj, file_name)
log.info(f"Actual file path on the pod {file_path}")
assert pod.check_file_existence(
self.pod_obj, file_path), f"File {file_name} doesn't exist"
log.info(f"File {file_name} exists in {self.pod_obj.name}")
# Step 2. Calculate md5sum of the file.
orig_md5_sum = pod.cal_md5sum(self.pod_obj, file_name)
# Step 3. Take a snapshot of the PVC and measure the time of creation.
snap_name = self.pvc_obj.name.replace("pvc-test",
f"snapshot-test{test_num}")
log.info(f"Taking snapshot of the PVC {snap_name}")
test_results["create"]["time"] = self.measure_create_snapshot_time(
pvc_name=self.pvc_obj.name,
snap_name=snap_name,
interface=self.interface,
)
test_results["create"]["speed"] = int(
test_results["dataset"] / test_results["create"]["time"])
log.info(
f' Test {test_num} dataset is {test_results["dataset"]} MiB')
log.info(
f'Snapshot creation time is : {test_results["create"]["time"]} sec.'
)
log.info(
f'Snapshot speed is : {test_results["create"]["speed"]} MB/sec'
)
# Step 4. Restore the PVC from the snapshot and measure the time
# Same Storage class of the original PVC
sc_name = self.pvc_obj.backed_sc
# Size should be same as of the original PVC
pvc_size = str(self.pvc_obj.size) + "Gi"
# Create pvc out of the snapshot
# Both, the snapshot and the restore PVC should be in same namespace
log.info("Restoring from the Snapshot")
restore_pvc_name = self.pvc_obj.name.replace(
"pvc-test", f"restore-pvc{test_num}")
restore_pvc_yaml = constants.CSI_RBD_PVC_RESTORE_YAML
if self.interface == constants.CEPHFILESYSTEM:
restore_pvc_yaml = constants.CSI_CEPHFS_PVC_RESTORE_YAML
log.info("Resorting the PVC from Snapshot")
restore_pvc_obj = pvc.create_restore_pvc(
sc_name=sc_name,
snap_name=self.snap_obj.name,
namespace=self.snap_obj.namespace,
size=pvc_size,
pvc_name=restore_pvc_name,
restore_pvc_yaml=restore_pvc_yaml,
)
helpers.wait_for_resource_state(
restore_pvc_obj,
constants.STATUS_BOUND,
timeout=3600 # setting this to 60 Min.
# since it can be take long time to restore, and we want it to finished.
)
teardown_factory(restore_pvc_obj)
restore_pvc_obj.reload()
log.info("PVC was restored from the snapshot")
test_results["restore"][
"time"] = helpers.measure_pvc_creation_time(
self.interface, restore_pvc_obj.name)
test_results["restore"]["speed"] = int(
test_results["dataset"] / test_results["restore"]["time"])
log.info(
f'Snapshot restore time is : {test_results["restore"]["time"]}'
)
log.info(
f'restore sped is : {test_results["restore"]["speed"]} MB/sec')
# Step 5. Attach a new pod to the restored PVC
restore_pod_obj = helpers.create_pod(
interface_type=self.interface,
pvc_name=restore_pvc_obj.name,
namespace=self.snap_obj.namespace,
pod_dict_path=constants.NGINX_POD_YAML,
)
# Confirm that the pod is running
helpers.wait_for_resource_state(resource=restore_pod_obj,
state=constants.STATUS_RUNNING)
teardown_factory(restore_pod_obj)
restore_pod_obj.reload()
# Step 6. Verify that the file is present on the new pod also.
log.info(f"Checking the existence of {file_name} "
f"on restore pod {restore_pod_obj.name}")
assert pod.check_file_existence(
restore_pod_obj, file_path), f"File {file_name} doesn't exist"
log.info(f"File {file_name} exists in {restore_pod_obj.name}")
# Step 7. Verify that the md5sum matches
log.info(f"Verifying that md5sum of {file_name} "
f"on pod {self.pod_obj.name} matches with md5sum "
f"of the same file on restore pod {restore_pod_obj.name}")
assert pod.verify_data_integrity(
restore_pod_obj, file_name,
orig_md5_sum), "Data integrity check failed"
log.info("Data integrity check passed, md5sum are same")
all_results.append(test_results)
# logging the test summery, all info in one place for easy log reading
c_speed, c_runtime, r_speed, r_runtime = (0 for i in range(4))
log.info("Test summery :")
for tst in all_results:
c_speed += tst["create"]["speed"]
c_runtime += tst["create"]["time"]
r_speed += tst["restore"]["speed"]
r_runtime += tst["restore"]["time"]
log.info(
f"Test {tst['test_num']} results : dataset is {tst['dataset']} MiB. "
f"Take snapshot time is {tst['create']['time']} "
f"at {tst['create']['speed']} MiB/Sec "
f"Restore from snapshot time is {tst['restore']['time']} "
f"at {tst['restore']['speed']} MiB/Sec ")
log.info(
f" Average snapshot creation time is {c_runtime / self.tests_numbers} sec."
)
log.info(
f" Average snapshot creation speed is {c_speed / self.tests_numbers} MiB/sec"
)
log.info(
f" Average snapshot restore time is {r_runtime / self.tests_numbers} sec."
)
log.info(
f" Average snapshot restore speed is {r_speed / self.tests_numbers} MiB/sec"
)
def test_rbd_space_reclaim(self):
"""
Test to verify RBD space reclamation
Steps:
1. Create and attach RBD PVC of size 25 GiB to an app pod.
2. Get the used size of the RBD pool
3. Create two files of size 10GiB
4. Verify the increased used size of the RBD pool
5. Delete one file
6. Create ReclaimSpaceJob
7. Verify the decreased used size of the RBD pool
"""
pvc_obj = self.pvc[0]
pod_obj = self.pod[0]
fio_filename1 = "fio_file1"
fio_filename2 = "fio_file2"
# Fetch the used size of pool
cbp_name = self.sc_obj.get().get("parameters").get("pool")
used_size_before_io = fetch_used_size(cbp_name)
log.info(f"Used size before IO is {used_size_before_io}")
# Create two 10 GB file
for filename in fio_filename1, fio_filename2:
pod_obj.run_io(
storage_type="fs",
size="10G",
runtime=120,
fio_filename=filename,
end_fsync=1,
)
pod_obj.get_fio_results()
# Verify used size after IO
exp_used_size_after_io = used_size_before_io + (20 * self.pool_replica)
used_size_after_io = fetch_used_size(cbp_name, exp_used_size_after_io)
log.info(f"Used size after IO is {used_size_after_io}")
# Delete one file
file_path = get_file_path(pod_obj, fio_filename2)
pod_obj.exec_cmd_on_pod(command=f"rm -f {file_path}",
out_yaml_format=False)
# Verify file is deleted
try:
check_file_existence(pod_obj=pod_obj, file_path=file_path)
except CommandFailed as cmdfail:
if "No such file or directory" not in str(cmdfail):
raise
log.info(f"Verified: File {file_path} deleted.")
# Create ReclaimSpaceJob
reclaim_space_job = pvc_obj.create_reclaim_space_job()
# Wait for the Succeeded result of ReclaimSpaceJob
try:
for reclaim_space_job_yaml in TimeoutSampler(
timeout=120, sleep=5, func=reclaim_space_job.get):
result = reclaim_space_job_yaml.get("status", {}).get("result")
if result == "Succeeded":
log.info(
f"ReclaimSpaceJob {reclaim_space_job.name} succeeded")
break
else:
log.info(
f"Waiting for the Succeeded result of the ReclaimSpaceJob {reclaim_space_job.name}. "
f"Present value of result is {result}")
except TimeoutExpiredError:
raise UnexpectedBehaviour(
f"ReclaimSpaceJob {reclaim_space_job.name} is not successful. Yaml output:{reclaim_space_job.get()}"
)
# Verify space is reclaimed by checking the used size of the RBD pool
used_after_reclaiming_space = fetch_used_size(
cbp_name, used_size_after_io - (10 * self.pool_replica))
log.info(
f"Space reclamation verified. Used size after reclaiming space is {used_after_reclaiming_space}."
)
def test_pvc_rwx_writeable_after_pod_deletions(
self, pvc_factory, teardown_factory
):
"""
Test assign nodeName to a pod using RWX pvc
1. Create a new project.
2. Create a RWX CEPHFS based PVC
3. Attach the same PVC to multiple PODs and start IO on all the PODs
4. Delete all but one pod.
5. Verify mount point is still write-able.
- Start IO again on the Running pod.
6. Also, access the data written by deleted pods from the Running pod
"""
worker_nodes_list = helpers.get_worker_nodes()
# Create a RWX PVC
pvc_obj = pvc_factory(
interface=constants.CEPHFILESYSTEM, access_mode=constants.ACCESS_MODE_RWX,
size=10, status=constants.STATUS_BOUND
)
logger.info(
f"Creating pods on all worker nodes backed"
f"with same pvc {pvc_obj.name}"
)
pod_list = []
for each_node in worker_nodes_list:
pod_obj = helpers.create_pod(
interface_type=constants.CEPHFILESYSTEM, pvc_name=pvc_obj.name,
namespace=pvc_obj.namespace, node_name=each_node,
pod_dict_path=constants.NGINX_POD_YAML
)
pod_list.append(pod_obj)
teardown_factory(pod_obj)
# Confirm pods are created and are running on designated nodes
node_count = 0
for pod_obj in pod_list:
helpers.wait_for_resource_state(
resource=pod_obj, state=constants.STATUS_RUNNING,
timeout=120
)
pod_obj.reload()
assert pod.verify_node_name(pod_obj, worker_nodes_list[node_count]), (
f'Pod {pod_obj.name} is running on a different node '
f'than the selected node'
)
node_count = node_count + 1
# Run IOs on all pods. FIO Filename is kept same as pod name
with ThreadPoolExecutor() as p:
for pod_obj in pod_list:
logger.info(f"Running IO on pod {pod_obj.name}")
p.submit(
pod_obj.run_io, storage_type='fs', size='512M',
runtime=30, fio_filename=pod_obj.name
)
# Check IO from all pods
for pod_obj in pod_list:
pod.get_fio_rw_iops(pod_obj)
# Calculate md5sum of each file
md5sum_pod_data = []
for pod_obj in pod_list:
md5sum_pod_data.append(pod.cal_md5sum(
pod_obj=pod_obj, file_name=pod_obj.name
))
# Delete all but the last app pod.
for index in range(node_count - 1):
pod_list[index].delete()
pod_list[index].ocp.wait_for_delete(
resource_name=pod_list[index].name
)
# Verify presence of files written by each pod
logger.info(
f"Verify existence of each file from app pod "
f"{pod_list[-1].name} "
)
for pod_obj in pod_list:
file_path = pod.get_file_path(pod_list[-1], pod_obj.name)
assert pod.check_file_existence(pod_list[-1], file_path), (
f"File {pod_obj.name} doesnt exist"
)
logger.info(
f"File {pod_obj.name} exists in {pod_list[-1].name}"
)
# From surviving pod, verify data integrity of files
# written by deleted pods
logger.info(f"verify all data from {pod_list[-1].name}")
for index, pod_obj in enumerate(pod_list):
assert pod.verify_data_integrity(
pod_obj=pod_list[-1], file_name=pod_obj.name,
original_md5sum=md5sum_pod_data[index]
)
# From surviving pod, confirm mount point is still write-able
logger.info(f"Re-running IO on pod {pod_list[-1].name}")
fio_new_file = f"{pod_list[-1].name}-new-file"
pod_list[-1].run_io(
storage_type='fs', size='512M', runtime=30,
fio_filename=fio_new_file
)
pod.get_fio_rw_iops(pod_list[-1])
file_path = pod.get_file_path(pod_list[-1], fio_new_file)
assert pod.check_file_existence(pod_list[-1], file_path), (
f"File {fio_new_file} doesnt exist"
)
logger.info(
f"File {fio_new_file} exists in {pod_list[-1].name} "
)
def factory(
num_of_pvcs=100,
pvc_size=2,
bulk=False,
project=None,
measure=True,
delete=True,
file_name=None,
fio_percentage=25,
verify_fio=False,
expand=False,
):
"""
Args:
num_of_pvcs (int) : Number of PVCs / PODs we want to create.
pvc_size (int) : Size of each PVC in GB.
bulk (bool) : True for bulk operations, False otherwise.
project (obj) : Project obj inside which the PODs/PVCs are created.
measure (bool) : True if we want to measure the PVC creation/deletion time and POD to PVC attach time,
False otherwise.
delete (bool) : True if we want to delete PVCs and PODs, False otherwise
file_name (str) : Name of the file on which FIO is performed.
fio_percentage (float) : Percentage of PVC space we want to be utilized for FIO.
verify_fio (bool) : True if we want to verify FIO, False otherwise.
expand (bool) : True if we want to verify_fio for expansion of PVCs operation, False otherwise.
"""
if not project:
project = project_factory("longevity")
pvc_objs = list()
executor = ThreadPoolExecutor(max_workers=1)
start_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
for interface in (constants.CEPHFILESYSTEM, constants.CEPHBLOCKPOOL):
if interface == constants.CEPHFILESYSTEM:
access_modes = [
constants.ACCESS_MODE_RWO, constants.ACCESS_MODE_RWX
]
num_of_pvc = num_of_pvcs // 2
else:
access_modes = [
constants.ACCESS_MODE_RWO,
constants.ACCESS_MODE_RWO + "-" +
constants.VOLUME_MODE_BLOCK,
constants.ACCESS_MODE_RWX + "-" +
constants.VOLUME_MODE_BLOCK,
]
num_of_pvc = num_of_pvcs - num_of_pvcs // 2
# Create PVCs
if num_of_pvc > 0:
pvc_objs_tmp = multi_pvc_factory(
interface=interface,
size=pvc_size,
project=project,
access_modes=access_modes,
status=constants.STATUS_BOUND,
num_of_pvc=num_of_pvc,
wait_each=not bulk,
)
log.info("PVC creation was successful.")
pvc_objs.extend(pvc_objs_tmp)
if measure:
# Measure PVC Creation Time
measure_pvc_creation_time(interface, pvc_objs_tmp,
start_time)
else:
log.info(
f"Num of PVCs of interface - {interface} = {num_of_pvc}. So no PVCs created."
)
# PVC and PV Teardown
for pvc_obj in pvc_objs:
teardown_factory(pvc_obj)
teardown_factory(pvc_obj.backed_pv_obj)
# Create PODs
pod_objs = list()
for pvc_obj in pvc_objs:
if pvc_obj.get_pvc_vol_mode == constants.VOLUME_MODE_BLOCK:
if not bulk:
pod_objs.append(
pod_factory(
pvc=pvc_obj,
raw_block_pv=True,
status=constants.STATUS_RUNNING,
pod_dict_path=constants.PERF_BLOCK_POD_YAML,
))
else:
pod_objs.append(
pod_factory(
pvc=pvc_obj,
raw_block_pv=True,
pod_dict_path=constants.PERF_BLOCK_POD_YAML,
))
else:
if not bulk:
pod_objs.append(
pod_factory(
pvc=pvc_obj,
status=constants.STATUS_RUNNING,
pod_dict_path=constants.PERF_POD_YAML,
))
else:
pod_objs.append(
pod_factory(pvc=pvc_obj,
pod_dict_path=constants.PERF_POD_YAML))
log.info(f"POD {pod_objs[-1].name} creation was successful.")
log.info("All PODs are created.")
if bulk:
for pod_obj in pod_objs:
executor.submit(
helpers.wait_for_resource_state,
pod_obj,
constants.STATUS_RUNNING,
timeout=300,
)
log.info(f"POD {pod_obj.name} reached Running State.")
log.info("All PODs reached Running State.")
if measure:
# Measure POD to PVC attach time
measure_pod_to_pvc_attach_time(pod_objs)
# POD Teardown
for pod_obj in pod_objs:
teardown_factory(pod_obj)
# Run FIO on PODs
fio_size = int((fio_percentage / 100) * pvc_size * 1000)
for pod_obj in pod_objs:
storage_type = ("block" if pod_obj.pvc.get_pvc_vol_mode
== constants.VOLUME_MODE_BLOCK else "fs")
pod_obj.wl_setup_done = True
pod_obj.wl_obj = workload.WorkLoad(
"test_workload_fio",
pod_obj.get_storage_path(storage_type),
"fio",
storage_type,
pod_obj,
1,
)
if not file_name:
pod_obj.run_io(storage_type, f"{fio_size}M")
else:
pod_obj.run_io(
storage_type=storage_type,
size=f"{fio_size}M",
runtime=20,
fio_filename=file_name,
end_fsync=1,
)
if verify_fio:
log.info(
"Waiting for IO to complete on all pods to utilise 25% of PVC used space"
)
for pod_obj in pod_objs:
# Wait for IO to finish
pod_obj.get_fio_results(3600)
log.info(f"IO finished on pod {pod_obj.name}")
is_block = (True if pod_obj.pvc.get_pvc_vol_mode
== constants.VOLUME_MODE_BLOCK else False)
file_name_pod = (file_name
if not is_block else pod_obj.get_storage_path(
storage_type="block"))
# Verify presence of the file
file_path = (file_name_pod if is_block else pod.get_file_path(
pod_obj, file_name_pod))
log.info(f"Actual file path on the pod {file_path}")
assert pod.check_file_existence(
pod_obj, file_path), f"File {file_name_pod} does not exist"
log.info(f"File {file_name_pod} exists in {pod_obj.name}")
if expand and is_block:
# Read IO from block PVCs using dd and calculate md5sum.
# This dd command reads the data from the device, writes it to
# stdout, and reads md5sum from stdin.
pod_obj.pvc.md5sum = pod_obj.exec_sh_cmd_on_pod(
command=(f"dd iflag=direct if={file_path} bs=10M "
f"count={fio_size // 10} | md5sum"))
log.info(
f"md5sum of {file_name_pod}: {pod_obj.pvc.md5sum}")
else:
# Calculate md5sum of the file
pod_obj.pvc.md5sum = pod.cal_md5sum(pod_obj, file_name_pod)
log.info("POD FIO was successful.")
if delete:
# Delete PODs
pod_delete = executor.submit(delete_pods, pod_objs, wait=not bulk)
pod_delete.result()
log.info("Verified: Pods are deleted.")
# Delete PVCs
pvc_delete = executor.submit(delete_pvcs,
pvc_objs,
concurrent=bulk)
res = pvc_delete.result()
if not res:
raise ex.UnexpectedBehaviour("Deletion of PVCs failed")
log.info("PVC deletion was successful.")
# Validate PV Deletion
for pvc_obj in pvc_objs:
helpers.validate_pv_delete(pvc_obj.backed_pv)
log.info("PV deletion was successful.")
if measure:
# Measure PVC Deletion Time
for interface in (constants.CEPHFILESYSTEM,
constants.CEPHBLOCKPOOL):
if interface == constants.CEPHFILESYSTEM:
measure_pvc_deletion_time(
interface,
pvc_objs[:num_of_pvcs // 2],
)
else:
measure_pvc_deletion_time(
interface,
pvc_objs[num_of_pvcs // 2:],
)
log.info(f"Successfully deleted {num_of_pvcs} PVCs")
else:
return pvc_objs, pod_objs
def test_rwx_dynamic_pvc(self, setup_base):
"""
RWX Dynamic PVC creation tests with Reclaim policy set to Delete/Retain
"""
logger.info(f"CephFS RWX test")
logger.info(
f"Creating second pod on node: {self.worker_nodes_list[1]} "
f"with pvc {self.pvc_obj.name}")
pod_obj2 = helpers.create_pod(interface_type=self.interface_type,
pvc_name=self.pvc_obj.name,
namespace=self.namespace,
node_name=self.worker_nodes_list[1],
pod_dict_path=constants.NGINX_POD_YAML)
helpers.wait_for_resource_state(pod_obj2, constants.STATUS_RUNNING)
pod_obj2.reload()
node_pod1 = self.pod_obj1.get().get('spec').get('nodeName')
node_pod2 = pod_obj2.get().get('spec').get('nodeName')
assert node_pod1 != node_pod2, 'Both pods are on the same node'
# Run IO on both the pods
logger.info(f"Running IO on pod {self.pod_obj1.name}")
file_name1 = self.pod_obj1.name
logger.info(file_name1)
self.pod_obj1.run_io(storage_type=self.storage_type,
size=self.io_size,
runtime=30,
fio_filename=file_name1)
logger.info(f"Running IO on pod {pod_obj2.name}")
file_name2 = pod_obj2.name
pod_obj2.run_io(storage_type=self.storage_type,
size=self.io_size,
runtime=30,
fio_filename=file_name2)
# Check IO and calculate md5sum of files
pod.get_fio_rw_iops(self.pod_obj1)
md5sum_pod1_data = pod.cal_md5sum(pod_obj=self.pod_obj1,
file_name=file_name1)
pod.get_fio_rw_iops(pod_obj2)
md5sum_pod2_data = pod.cal_md5sum(pod_obj=pod_obj2,
file_name=file_name2)
logger.info(f"verify data from alternate pods")
assert pod.verify_data_integrity(pod_obj=pod_obj2,
file_name=file_name1,
original_md5sum=md5sum_pod1_data)
assert pod.verify_data_integrity(pod_obj=self.pod_obj1,
file_name=file_name2,
original_md5sum=md5sum_pod2_data)
# Verify that data is mutable from any pod
logger.info(f"Perform modification of files from alternate pod")
# Access and rename file written by pod-2 from pod-1
file_path2 = pod.get_file_path(pod_obj2, file_name2)
logger.info(file_path2)
self.pod_obj1.exec_cmd_on_pod(
command=f"bash -c \"mv {file_path2} {file_path2}-renamed\"",
out_yaml_format=False)
# Access and rename file written by pod-1 from pod-2
file_path1 = pod.get_file_path(self.pod_obj1, file_name1)
logger.info(file_path1)
pod_obj2.exec_cmd_on_pod(
command=f"bash -c \"mv {file_path1} {file_path1}-renamed\"",
out_yaml_format=False)
logger.info(f"Verify presence of renamed files from both pods")
file_names = [f"{file_path1}-renamed", f"{file_path2}-renamed"]
for file in file_names:
assert pod.check_file_existence(
self.pod_obj1, file), (f"File {file} doesn't exist")
logger.info(f"File {file} exists in {self.pod_obj1.name} ")
assert pod.check_file_existence(
pod_obj2, file), (f"File {file} doesn't exist")
logger.info(f"File {file} exists in {pod_obj2.name}")
def test_no_volume_mounted(self):
"""
Test reclaimspace job with no volume mounted
Steps:
1. Create and attach RBD PVC of size 25 GiB to an app pod.
2. Get the used size of the RBD pool
3. Create a file of size 10GiB
4. Delete the file
5. Delete the pod
6. Create ReclaimSpaceJob
7. No errors should be seen in reclaim space job
"""
pvc_obj = self.pvc[0]
pod_obj = self.pod[0]
fio_filename1 = "fio_file1"
# Fetch the used size of pool
cbp_name = self.sc_obj.get().get("parameters").get("pool")
used_size_before_io = fetch_used_size(cbp_name)
log.info(f"Used size before IO is {used_size_before_io}")
# Create a 10 GB file
pod_obj.run_io(
storage_type="fs",
size="10G",
runtime=120,
fio_filename=fio_filename1,
end_fsync=1,
)
pod_obj.get_fio_results()
# Verify used size after IO
exp_used_size_after_io = used_size_before_io + (10 * self.pool_replica)
used_size_after_io = fetch_used_size(cbp_name, exp_used_size_after_io)
log.info(f"Used size after IO is {used_size_after_io}")
# Delete the file
file_path = get_file_path(pod_obj, fio_filename1)
pod_obj.exec_cmd_on_pod(command=f"rm -f {file_path}",
out_yaml_format=False)
# Verify whether file is deleted
try:
check_file_existence(pod_obj=pod_obj, file_path=file_path)
except CommandFailed as cmdfail:
if "No such file or directory" not in str(cmdfail):
raise
log.info(f"Verified: File {file_path} deleted.")
# Delete the pod
log.info(f"Deleting the pod {pod_obj}")
delete_pods([pod_obj])
# Create ReclaimSpaceJob
reclaim_space_job = pvc_obj.create_reclaim_space_job()
# Verify Succeeded result of ReclaimSpaceJob
self.reclaim_space_job(reclaim_space_job)
