def setup(self, multi_pvc_factory, pod_factory):
"""
Create resources for the test
"""
access_modes_cephfs = [
constants.ACCESS_MODE_RWO, constants.ACCESS_MODE_RWX
]
access_modes_rbd = [
constants.ACCESS_MODE_RWO,
f"{constants.ACCESS_MODE_RWO}-Block",
f"{constants.ACCESS_MODE_RWX}-Block",
]
self.pvcs_cephfs = multi_pvc_factory(
interface=constants.CEPHFILESYSTEM,
size=10,
access_modes=access_modes_cephfs,
status=constants.STATUS_BOUND,
num_of_pvc=2,
timeout=90,
)
self.pvcs_rbd = multi_pvc_factory(
interface=constants.CEPHBLOCKPOOL,
project=self.pvcs_cephfs[0].project,
size=10,
access_modes=access_modes_rbd,
status=constants.STATUS_BOUND,
num_of_pvc=3,
timeout=90,
)
pods_cephfs = helpers.create_pods(
self.pvcs_cephfs,
pod_factory,
constants.CEPHFILESYSTEM,
2,
constants.STATUS_RUNNING,
)
pods_rbd = helpers.create_pods(
self.pvcs_rbd,
pod_factory,
constants.CEPHBLOCKPOOL,
2,
constants.STATUS_RUNNING,
)
self.pods = pods_cephfs + pods_rbd
# Set volume mode on PVC objects
for pvc_obj in self.pvcs_cephfs + self.pvcs_rbd:
pvc_info = pvc_obj.get()
setattr(pvc_obj, "volume_mode", pvc_info["spec"]["volumeMode"])
def setup(self, project_factory, snapshot_restore_factory,
multi_pvc_factory, pod_factory):
"""
Create PVCs and pods
"""
self.pvc_size = 5
self.pvc_objs = multi_pvc_factory(
interface=constants.CEPHBLOCKPOOL,
size=self.pvc_size,
access_modes=[
f"{constants.ACCESS_MODE_RWX}-Block",
f"{constants.ACCESS_MODE_RWO}-Block",
],
status=constants.STATUS_BOUND,
num_of_pvc=2,
wait_each=False,
)
self.pod_objs = create_pods(
self.pvc_objs,
pod_factory,
constants.CEPHBLOCKPOOL,
pods_for_rwx=1,
status=constants.STATUS_RUNNING,
)
def test_pvc_disruptive(
self,
interface,
operation_to_disrupt,
resource_to_delete,
multi_pvc_factory,
pod_factory,
):
"""
Base function for PVC disruptive tests.
Deletion of 'resource_to_delete' will be introduced while
'operation_to_disrupt' is progressing.
"""
pod_functions = {
"mds":
partial(pod.get_mds_pods),
"mon":
partial(pod.get_mon_pods),
"mgr":
partial(pod.get_mgr_pods),
"osd":
partial(pod.get_osd_pods),
"rbdplugin":
partial(pod.get_plugin_pods, interface=interface),
"cephfsplugin":
partial(pod.get_plugin_pods, interface=interface),
"cephfsplugin_provisioner":
partial(pod.get_cephfsplugin_provisioner_pods),
"rbdplugin_provisioner":
partial(pod.get_rbdfsplugin_provisioner_pods),
"operator":
partial(pod.get_operator_pods),
}
# Get number of pods of type 'resource_to_delete'
num_of_resource_to_delete = len(pod_functions[resource_to_delete]())
num_of_pvc = 12
namespace = self.proj_obj.namespace
# Fetch the number of Pods and PVCs
initial_num_of_pods = len(pod.get_all_pods(namespace=namespace))
initial_num_of_pvc = len(get_all_pvcs(namespace=namespace)["items"])
executor = ThreadPoolExecutor(max_workers=(2 * num_of_pvc))
DISRUPTION_OPS.set_resource(resource=resource_to_delete)
access_modes = [constants.ACCESS_MODE_RWO]
if interface == constants.CEPHFILESYSTEM:
access_modes.append(constants.ACCESS_MODE_RWX)
# Modify access_modes list to create rbd `block` type volume with
# RWX access mode. RWX is not supported in non-block type rbd
if interface == constants.CEPHBLOCKPOOL:
access_modes.extend([
f"{constants.ACCESS_MODE_RWO}-Block",
f"{constants.ACCESS_MODE_RWX}-Block",
])
# Start creation of PVCs
bulk_pvc_create = executor.submit(
multi_pvc_factory,
interface=interface,
project=self.proj_obj,
size=5,
access_modes=access_modes,
access_modes_selection="distribute_random",
status=constants.STATUS_BOUND,
num_of_pvc=num_of_pvc,
wait_each=False,
timeout=90,
)
if operation_to_disrupt == "create_pvc":
# Ensure PVCs are being created before deleting the resource
ret = helpers.wait_for_resource_count_change(
get_all_pvcs, initial_num_of_pvc, namespace, "increase")
assert ret, "Wait timeout: PVCs are not being created."
logger.info("PVCs creation has started.")
DISRUPTION_OPS.delete_resource()
pvc_objs = bulk_pvc_create.result()
# Confirm that PVCs are Bound
for pvc_obj in pvc_objs:
helpers.wait_for_resource_state(resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=120)
pvc_obj.reload()
logger.info("Verified: PVCs are Bound.")
# Start creating pods
bulk_pod_create = executor.submit(helpers.create_pods, pvc_objs,
pod_factory, interface, 2)
if operation_to_disrupt == "create_pod":
# Ensure that pods are being created before deleting the resource
ret = helpers.wait_for_resource_count_change(
pod.get_all_pods, initial_num_of_pods, namespace, "increase")
assert ret, "Wait timeout: Pods are not being created."
logger.info("Pods creation has started.")
DISRUPTION_OPS.delete_resource()
pod_objs = bulk_pod_create.result()
# Verify pods are Running
for pod_obj in pod_objs:
helpers.wait_for_resource_state(resource=pod_obj,
state=constants.STATUS_RUNNING)
pod_obj.reload()
logger.info("Verified: All pods are Running.")
# Do setup on pods for running IO
logger.info("Setting up pods for running IO.")
for pod_obj in pod_objs:
pvc_info = pod_obj.pvc.get()
if pvc_info["spec"]["volumeMode"] == "Block":
storage_type = "block"
else:
storage_type = "fs"
executor.submit(pod_obj.workload_setup, storage_type=storage_type)
# Wait for setup on pods to complete
for pod_obj in pod_objs:
logger.info(
f"Waiting for IO setup to complete on pod {pod_obj.name}")
for sample in TimeoutSampler(180, 2, getattr, pod_obj,
"wl_setup_done"):
if sample:
logger.info(f"Setup for running IO is completed on pod "
f"{pod_obj.name}.")
break
logger.info("Setup for running IO is completed on all pods.")
# Start IO on each pod
for pod_obj in pod_objs:
pvc_info = pod_obj.pvc.get()
if pvc_info["spec"]["volumeMode"] == "Block":
storage_type = "block"
else:
storage_type = "fs"
pod_obj.run_io(
storage_type=storage_type,
size="1G",
runtime=10,
fio_filename=f"{pod_obj.name}_io_file1",
)
logger.info("FIO started on all pods.")
if operation_to_disrupt == "run_io":
DISRUPTION_OPS.delete_resource()
logger.info("Fetching FIO results.")
for pod_obj in pod_objs:
fio_result = pod_obj.get_fio_results()
err_count = fio_result.get("jobs")[0].get("error")
assert (
err_count == 0
), f"FIO error on pod {pod_obj.name}. FIO result: {fio_result}"
logger.info("Verified FIO result on pods.")
# Delete pods
for pod_obj in pod_objs:
pod_obj.delete(wait=True)
for pod_obj in pod_objs:
pod_obj.ocp.wait_for_delete(pod_obj.name)
# Verify that PVCs are reusable by creating new pods
pod_objs = helpers.create_pods(pvc_objs, pod_factory, interface, 2)
# Verify new pods are Running
for pod_obj in pod_objs:
helpers.wait_for_resource_state(resource=pod_obj,
state=constants.STATUS_RUNNING)
pod_obj.reload()
logging.info("Verified: All new pods are Running.")
# Run IO on each of the new pods
for pod_obj in pod_objs:
pvc_info = pod_obj.pvc.get()
if pvc_info["spec"]["volumeMode"] == "Block":
storage_type = "block"
else:
storage_type = "fs"
pod_obj.run_io(
storage_type=storage_type,
size="1G",
runtime=10,
fio_filename=f"{pod_obj.name}_io_file2",
)
logger.info("Fetching FIO results from new pods")
for pod_obj in pod_objs:
fio_result = pod_obj.get_fio_results()
err_count = fio_result.get("jobs")[0].get("error")
assert (
err_count == 0
), f"FIO error on pod {pod_obj.name}. FIO result: {fio_result}"
logger.info("Verified FIO result on new pods.")
# Verify number of pods of type 'resource_to_delete'
final_num_resource_to_delete = len(pod_functions[resource_to_delete]())
assert final_num_resource_to_delete == num_of_resource_to_delete, (
f"Total number of {resource_to_delete} pods is not matching with "
f"initial value. Total number of pods before deleting a pod: "
f"{num_of_resource_to_delete}. Total number of pods present now: "
f"{final_num_resource_to_delete}")
# Check ceph status
ceph_health_check(namespace=config.ENV_DATA["cluster_namespace"])
logger.info("Ceph cluster health is OK")
def test_create_delete_pvcs(multi_pvc_factory, pod_factory, project=None):
# create the pods for deleting
# Create rbd pvcs for pods
pvc_objs_rbd = create_pvcs(multi_pvc_factory,
"CephBlockPool",
project=project)
storageclass_rbd = pvc_objs_rbd[0].storageclass
# Create cephfs pvcs for pods
pvc_objs_cephfs = create_pvcs(multi_pvc_factory,
"CephFileSystem",
project=project)
storageclass_cephfs = pvc_objs_cephfs[0].storageclass
all_pvc_for_pods = pvc_objs_rbd + pvc_objs_cephfs
# Check pvc status
for pvc_obj in all_pvc_for_pods:
helpers.wait_for_resource_state(
resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=1200, # Timeout given 20 minutes
)
pvc_info = pvc_obj.get()
setattr(pvc_obj, "volume_mode", pvc_info["spec"]["volumeMode"])
# Create pods
rbd_pods_to_delete = helpers.create_pods(pvc_objs_rbd, pod_factory,
constants.RBD_INTERFACE)
cephfs_pods_to_delete = helpers.create_pods(pvc_objs_cephfs, pod_factory,
constants.CEPHFS_INTERFACE)
pods_to_delete = rbd_pods_to_delete + cephfs_pods_to_delete
for pod_obj in pods_to_delete:
helpers.wait_for_resource_state(
resource=pod_obj,
state=constants.STATUS_RUNNING,
timeout=300, # Timeout given 5 minutes
)
log.info(
f"#### Created the pods for deletion later...pods = {pods_to_delete}")
# Create PVCs for deleting
# Create rbd pvcs for deleting
pvc_objs_rbd = create_pvcs(
multi_pvc_factory=multi_pvc_factory,
interface="CephBlockPool",
project=project,
status="",
storageclass=storageclass_rbd,
)
# Create cephfs pvcs for deleting
pvc_objs_cephfs = create_pvcs(
multi_pvc_factory=multi_pvc_factory,
interface="CephFileSystem",
project=project,
status="",
storageclass=storageclass_cephfs,
)
all_pvc_to_delete = pvc_objs_rbd + pvc_objs_cephfs
# Check pvc status
for pvc_obj in all_pvc_to_delete:
helpers.wait_for_resource_state(
resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=300, # Timeout given 5 minutes
)
log.info(
f"#### Created the PVCs for deletion later...PVCs={all_pvc_to_delete}")
# Create PVCs for new pods
pvc_objs_rbd = create_pvcs(
multi_pvc_factory=multi_pvc_factory,
interface="CephBlockPool",
project=project,
status="",
storageclass=storageclass_rbd,
)
# Create cephfs pvcs for new pods # for deleting
pvc_objs_cephfs = create_pvcs(
multi_pvc_factory=multi_pvc_factory,
interface="CephFileSystem",
project=project,
status="",
storageclass=storageclass_cephfs,
)
all_pvc_for_new_pods = pvc_objs_rbd + pvc_objs_cephfs
# Check pvc status
for pvc_obj in all_pvc_for_new_pods:
helpers.wait_for_resource_state(
resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=300, # Timeout given 5 minutes
)
pvc_info = pvc_obj.get()
setattr(pvc_obj, "volume_mode", pvc_info["spec"]["volumeMode"])
log.info(
f"#### Created the PVCs required for creating New Pods...{all_pvc_for_new_pods}"
)
executor = ThreadPoolExecutor(max_workers=10)
# Start creating new PVCs
# Start creating rbd PVCs
rbd_pvc_exeuter = executor.submit(
create_pvcs,
multi_pvc_factory=multi_pvc_factory,
interface="CephBlockPool",
project=project,
status="",
storageclass=storageclass_rbd,
)
log.info("#### Started creating new RBD PVCs in thread...")
# Start creating cephfs pvc
cephfs_pvc_exeuter = executor.submit(
create_pvcs,
multi_pvc_factory=multi_pvc_factory,
interface="CephFileSystem",
project=project,
status="",
storageclass=storageclass_cephfs,
)
log.info("#### Started creating new cephfs PVCs in thread...")
# Start creating pods
rbd_pods_create_executer = executor.submit(helpers.create_pods,
pvc_objs_rbd, pod_factory,
constants.RBD_INTERFACE)
cephfs_pods_create_executer = executor.submit(helpers.create_pods,
pvc_objs_cephfs, pod_factory,
constants.CEPHFS_INTERFACE)
# Start deleting pods
pods_delete_executer = executor.submit(delete_pods, pods_to_delete)
log.info(f"### Started deleting the pods_to_delete = {pods_to_delete}")
# Start deleting PVC
pvc_delete_executer = executor.submit(delete_pvcs, all_pvc_to_delete)
log.info(
f"### Started deleting the all_pvc_to_delete = {all_pvc_to_delete}")
log.info(
"These process are started: Bulk delete PVC, Pods. Bulk create PVC, "
"Pods. Waiting for its completion")
while not (rbd_pvc_exeuter.done() and cephfs_pvc_exeuter.done()
and rbd_pods_create_executer.done()
and cephfs_pods_create_executer.done()
and pods_delete_executer.done() and pvc_delete_executer.done()):
sleep(10)
log.info(
"#### create_delete_pvcs....Waiting for threads to complete...")
new_rbd_pvcs = rbd_pvc_exeuter.result()
new_cephfs_pvcs = cephfs_pvc_exeuter.result()
new_pods = cephfs_pods_create_executer.result(
) + rbd_pods_create_executer.result()
# Check pvc status
for pvc_obj in new_rbd_pvcs + new_cephfs_pvcs:
helpers.wait_for_resource_state(
resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=300, # Timeout given 5 minutes
)
log.info("All new PVCs are bound")
# Check pods status
for pod_obj in new_pods:
helpers.wait_for_resource_state(
resource=pod_obj,
state=constants.STATUS_RUNNING,
timeout=300, # Timeout given 5 minutes
)
log.info("All new pods are running")
# Check pods are deleted
for pod_obj in pods_to_delete:
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
log.info("All pods are deleted as expected.")
# Check PVCs are deleted
for pvc_obj in all_pvc_to_delete:
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name)
log.info("All PVCs are deleted as expected")
def test_rbd_block_pvc_snapshot(self, snapshot_factory,
snapshot_restore_factory, pod_factory):
"""
Test to take snapshots of RBD Block VolumeMode PVCs
"""
# Run IO
log.info("Find initial md5sum value and run IO on all pods")
for pod_obj in self.pod_objs:
# Find initial md5sum
pod_obj.md5sum_before_io = cal_md5sum(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
block=True,
)
pod_obj.run_io(
storage_type="block",
size=f"{self.pvc_size - 1}G",
io_direction="write",
runtime=60,
)
log.info("IO started on all pods")
# Wait for IO completion
for pod_obj in self.pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on all pods")
snap_objs = []
# Verify md5sum has changed after IO. Create snapshot
log.info("Verify md5sum has changed after IO and create snapshot from "
"all PVCs")
for pod_obj in self.pod_objs:
md5sum_after_io = cal_md5sum(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
block=True,
)
assert (pod_obj.md5sum_before_io != md5sum_after_io
), f"md5sum has not changed after IO on pod {pod_obj.name}"
log.info(f"Creating snapshot of PVC {pod_obj.pvc.name}")
snap_obj = snapshot_factory(pod_obj.pvc, wait=False)
snap_obj.md5sum = md5sum_after_io
snap_objs.append(snap_obj)
log.info("Snapshots created")
# Verify snapshots are ready
log.info("Verify snapshots are ready")
for snap_obj in snap_objs:
snap_obj.ocp.wait_for_resource(
condition="true",
resource_name=snap_obj.name,
column=constants.STATUS_READYTOUSE,
timeout=180,
)
# Delete pods
log.info("Deleting the pods")
for pod_obj in self.pod_objs:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
log.info("Deleted all the pods")
# Delete parent PVCs to verify snapshot is independent
log.info("Deleting parent PVCs")
for pvc_obj in self.pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name)
log.info(f"Deleted PVC {pvc_obj.name}. Verifying whether PV "
f"{pv_obj.name} is deleted.")
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
log.info("Deleted parent PVCs before restoring snapshot. "
"PVs are also deleted.")
restore_pvc_objs = []
# Create PVCs out of the snapshots
log.info("Creating new PVCs from snapshots")
for snap_obj in snap_objs:
log.info(f"Creating a PVC from snapshot {snap_obj.name}")
restore_pvc_obj = snapshot_restore_factory(
snapshot_obj=snap_obj,
size=f"{self.pvc_size}Gi",
volume_mode=snap_obj.parent_volume_mode,
access_mode=snap_obj.parent_access_mode,
status="",
)
log.info(f"Created PVC {restore_pvc_obj.name} from snapshot "
f"{snap_obj.name}")
restore_pvc_obj.md5sum = snap_obj.md5sum
restore_pvc_objs.append(restore_pvc_obj)
log.info("Created new PVCs from all the snapshots")
# Confirm that the restored PVCs are Bound
log.info("Verify the restored PVCs are Bound")
for pvc_obj in restore_pvc_objs:
wait_for_resource_state(resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=180)
pvc_obj.reload()
log.info("Verified: Restored PVCs are Bound.")
# Attach the restored PVCs to pods. Attach RWX PVC on two pods
log.info("Attach the restored PVCs to pods")
restore_pod_objs = create_pods(
restore_pvc_objs,
pod_factory,
constants.CEPHBLOCKPOOL,
pods_for_rwx=2,
status="",
)
# Verify the new pods are running
log.info("Verify the new pods are running")
for pod_obj in restore_pod_objs:
wait_for_resource_state(pod_obj, constants.STATUS_RUNNING)
log.info("Verified: New pods are running")
log.info("Verifying md5sum on new pods")
for pod_obj in restore_pod_objs:
log.info(f"Verifying md5sum on pod {pod_obj.name}")
verify_data_integrity(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
original_md5sum=pod_obj.pvc.md5sum,
block=True,
)
log.info(f"Verified md5sum on pod {pod_obj.name}")
log.info("Verified md5sum on all pods")
# Run IO on new pods
log.info("Starting IO on new pods")
for pod_obj in restore_pod_objs:
pod_obj.run_io(storage_type="block", size="500M", runtime=15)
# Wait for IO completion on new pods
log.info("Waiting for IO completion on new pods")
for pod_obj in restore_pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on new pods.")
def test_rbd_pv_encryption_vaulttenantsa(
self,
project_factory,
storageclass_factory,
multi_pvc_factory,
pod_factory,
kv_version,
):
"""
Test to verify creation and deletion of encrypted RBD PVC using vaulttenantsa method
"""
# Create a project
proj_obj = project_factory()
# Create an encryption enabled storageclass for RBD
sc_obj = storageclass_factory(
interface=constants.CEPHBLOCKPOOL,
encrypted=True,
encryption_kms_id=self.kms.kmsid,
)
# Create serviceaccount in the tenant namespace
self.kms.create_tenant_sa(namespace=proj_obj.namespace)
# Create role in Vault
self.kms.create_vault_kube_auth_role(namespace=proj_obj.namespace)
# Create RBD PVCs with volume mode Block
pvc_size = 5
pvc_objs = multi_pvc_factory(
interface=constants.CEPHBLOCKPOOL,
project=proj_obj,
storageclass=sc_obj,
size=pvc_size,
access_modes=[
f"{constants.ACCESS_MODE_RWX}-Block",
f"{constants.ACCESS_MODE_RWO}-Block",
],
status=constants.STATUS_BOUND,
num_of_pvc=3,
wait_each=False,
)
# Create pods
pod_objs = create_pods(
pvc_objs,
pod_factory,
constants.CEPHBLOCKPOOL,
pods_for_rwx=1,
status=constants.STATUS_RUNNING,
)
# Verify if the key is created in Vault
vol_handles = []
for pvc_obj in pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
vol_handle = pv_obj.get().get("spec").get("csi").get(
"volumeHandle")
vol_handles.append(vol_handle)
# Check if encryption key is created in Vault
if kms.is_key_present_in_path(key=vol_handle,
path=self.kms.vault_backend_path):
log.info(f"Vault: Found key for {pvc_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for {pvc_obj.name}")
# Verify whether encrypted device is present inside the pod and run IO
for vol_handle, pod_obj in zip(vol_handles, pod_objs):
if pod_obj.exec_sh_cmd_on_pod(
command=f"lsblk | grep {vol_handle} | grep crypt"):
log.info(f"Encrypted device found in {pod_obj.name}")
else:
raise ResourceNotFoundError(
f"Encrypted device not found in {pod_obj.name}")
pod_obj.run_io(
storage_type="block",
size=f"{pvc_size - 1}G",
io_direction="write",
runtime=60,
)
log.info("IO started on all pods")
# Wait for IO completion
for pod_obj in pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on all pods")
# Delete the pod
for pod_obj in pod_objs:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
# Delete the PVC
for pvc_obj in pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
# Verify whether the key is deleted in Vault
for vol_handle in vol_handles:
if not kms.is_key_present_in_path(
key=vol_handle, path=self.kms.vault_backend_path):
log.info(f"Vault: Key deleted for {vol_handle}")
else:
raise KMSResourceCleaneupError(
f"Vault: Key deletion failed for {vol_handle}")
def test_daemon_kill_during_pvc_pod_creation_deletion_and_io(
self, setup_base, multi_pvc_factory, pod_factory
):
"""
Kill ceph daemons while PVCs creation, PVCs deletion, pods creation, pods deletion
and IO are progressing
"""
daemons_to_kill = [
"mgr",
"mon",
"osd",
"mds",
]
(
pvc_objs,
pod_objs,
rwx_pod_objs,
cephfs_pvc_for_pods,
rbd_pvc_for_pods,
) = setup_base
num_of_pods_to_delete = 3
num_of_io_pods = 1
num_pvc_create_during_disruption = len(
self.access_modes_cephfs + self.access_modes_rbd
)
# Select pods to be deleted
pods_to_delete = pod_objs[:num_of_pods_to_delete]
pods_to_delete.extend(
[
pod
for pod in rwx_pod_objs
for pod_obj in pods_to_delete
if (pod_obj.pvc == pod.pvc)
]
)
# Select pods to run IO
io_pods = pod_objs[
num_of_pods_to_delete : num_of_pods_to_delete + num_of_io_pods
]
io_pods.extend(
[
pod
for pod in rwx_pod_objs
for pod_obj in io_pods
if (pod_obj.pvc == pod.pvc)
]
)
# Select pods which are having PVCs to delete
pods_for_pvc = pod_objs[num_of_pods_to_delete + num_of_io_pods :]
pvcs_to_delete = [pod_obj.pvc for pod_obj in pods_for_pvc]
pods_for_pvc.extend(
[
pod
for pod in rwx_pod_objs
for pod_obj in pods_for_pvc
if (pod_obj.pvc == pod.pvc)
]
)
io_pods = [
pod_obj
for pod_obj in io_pods
if pod_obj.pvc in select_unique_pvcs([pod_obj.pvc for pod_obj in io_pods])
]
log.info(
f"{len(pods_to_delete)} pods selected for deletion in which "
f"{len(pods_to_delete) - num_of_pods_to_delete} pairs of pod "
f"share same RWX PVC"
)
log.info(
f"{len(io_pods)} pods selected for running IO in which one "
f"pair of pod share same RWX PVC"
)
no_of_rwx_pvcs_delete = len(pods_for_pvc) - len(pvcs_to_delete)
log.info(
f"{len(pvcs_to_delete)} PVCs selected for deletion. "
f"RWO PVCs: {len(pvcs_to_delete) - no_of_rwx_pvcs_delete}, "
f"RWX PVCs: {no_of_rwx_pvcs_delete}"
)
pod_functions = {
"mds": partial(get_mds_pods),
"mon": partial(get_mon_pods),
"mgr": partial(get_mgr_pods),
"osd": partial(get_osd_pods),
}
# Disruption object for each daemon type
disruption_ops = [disruption_helpers.Disruptions() for _ in daemons_to_kill]
# Select the resource of each type
for disruption, pod_type in zip(disruption_ops, daemons_to_kill):
disruption.set_resource(resource=pod_type)
executor = ThreadPoolExecutor(
max_workers=len(pod_objs)
+ len(rwx_pod_objs)
+ len(rbd_pvc_for_pods)
+ len(cephfs_pvc_for_pods)
+ len(daemons_to_kill)
+ num_pvc_create_during_disruption
)
# Get number of pods of the type given in daemons_to_kill list
num_of_resource_pods = [
len(pod_functions[resource_name]()) for resource_name in daemons_to_kill
]
# Fetch PV names to verify after deletion
pv_objs = []
for pvc_obj in pvcs_to_delete:
pv_objs.append(pvc_obj.backed_pv_obj)
# Fetch volume details from pods for the purpose of verification
node_pv_dict = {}
for pod_obj in pods_to_delete:
pod_info = pod_obj.get()
node = pod_info["spec"]["nodeName"]
pvc = pod_info["spec"]["volumes"][0]["persistentVolumeClaim"]["claimName"]
for pvc_obj in pvc_objs:
if pvc_obj.name == pvc:
pv = pvc_obj.backed_pv
break
if node in node_pv_dict:
node_pv_dict[node].append(pv)
else:
node_pv_dict[node] = [pv]
# Fetch image uuid associated with PVCs to be deleted
pvc_uuid_map = {}
for pvc_obj in pvcs_to_delete:
pvc_uuid_map[pvc_obj] = pvc_obj.image_uuid
log.info("Fetched image uuid associated with each PVC")
# Do setup on pods for running IO
log.info("Setting up pods for running IO.")
for pod_obj in pod_objs + rwx_pod_objs:
if pod_obj.pvc.get_pvc_vol_mode == "Block":
storage_type = "block"
else:
storage_type = "fs"
executor.submit(pod_obj.workload_setup, storage_type=storage_type)
# Wait for setup on pods to complete
for pod_obj in pod_objs + rwx_pod_objs:
log.info(f"Waiting for IO setup to complete on pod {pod_obj.name}")
for sample in TimeoutSampler(360, 2, getattr, pod_obj, "wl_setup_done"):
if sample:
log.info(
f"Setup for running IO is completed on pod " f"{pod_obj.name}."
)
break
log.info("Setup for running IO is completed on all pods.")
# Start IO on pods having PVCs to delete to load data
pods_for_pvc_io = [
pod_obj
for pod_obj in pods_for_pvc
if pod_obj.pvc
in select_unique_pvcs([pod_obj.pvc for pod_obj in pods_for_pvc])
]
log.info("Starting IO on pods having PVCs to delete.")
self.run_io_on_pods(pods_for_pvc_io)
log.info("IO started on pods having PVCs to delete.")
log.info("Fetching IO results from the pods having PVCs to delete.")
for pod_obj in pods_for_pvc_io:
get_fio_rw_iops(pod_obj)
log.info("Verified IO result on pods having PVCs to delete.")
# Delete pods having PVCs to delete.
assert self.delete_pods(
pods_for_pvc
), "Couldn't delete pods which are having PVCs to delete."
for pod_obj in pods_for_pvc:
pod_obj.ocp.wait_for_delete(pod_obj.name)
log.info("Verified: Deleted pods which are having PVCs to delete.")
# Select daemon of each type of resource and identify the daemons running on each node
nodes_and_pids = {}
for disruption in disruption_ops:
disruption.select_daemon()
node_name = disruption.resource_obj[0].pod_data.get("spec").get("nodeName")
# Create node-daemons dict. Value as string for passing in the 'kill' command
nodes_and_pids[
node_name
] = f"{nodes_and_pids.get(node_name, '')} {disruption.daemon_pid}"
# Start IO on pods to be deleted
pods_to_delete_io = [
pod_obj
for pod_obj in pods_to_delete
if pod_obj.pvc
in select_unique_pvcs([pod_obj.pvc for pod_obj in pods_to_delete])
]
log.info("Starting IO on selected pods to be deleted.")
self.run_io_on_pods(pods_to_delete_io)
log.info("IO started on selected pods to be deleted.")
# Start creating new pods
log.info("Start creating new pods.")
pod_create_rbd = executor.submit(
helpers.create_pods,
rbd_pvc_for_pods,
pod_factory,
constants.CEPHBLOCKPOOL,
2,
)
pod_create_cephfs = executor.submit(
helpers.create_pods,
cephfs_pvc_for_pods,
pod_factory,
constants.CEPHFILESYSTEM,
2,
)
# Start creation of new CephFS PVCs.
log.info("Start creating new CephFS PVCs.")
pvc_create_cephfs = executor.submit(
multi_pvc_factory,
interface=constants.CEPHFILESYSTEM,
project=self.project,
storageclass=None,
size=self.pvc_size,
access_modes=self.access_modes_cephfs,
access_modes_selection="distribute_random",
status="",
num_of_pvc=len(self.access_modes_cephfs),
wait_each=False,
)
# Start creation of new RBD PVCs
log.info("Start creating new RBD PVCs.")
pvc_create_rbd = executor.submit(
multi_pvc_factory,
interface=constants.CEPHBLOCKPOOL,
project=self.project,
storageclass=None,
size=self.pvc_size,
access_modes=self.access_modes_rbd,
access_modes_selection="distribute_random",
status="",
num_of_pvc=len(self.access_modes_rbd),
wait_each=False,
)
# Start deleting PVCs
pvc_bulk_delete = executor.submit(delete_pvcs, pvcs_to_delete)
log.info("Started deleting PVCs")
# Start deleting pods
pod_bulk_delete = executor.submit(self.delete_pods, pods_to_delete)
log.info("Started deleting pods")
# Start IO on IO pods
self.run_io_on_pods(io_pods)
log.info("Started IO on IO pods")
# Wait for 1 second before killing daemons. This is to wait for the create/delete operations to start
sleep(1)
# Kill daemons
node_and_kill_proc = {}
log.info(f"Killing daemons of {daemons_to_kill}")
for node_name, pids in nodes_and_pids.items():
# Command to kill the daemon
kill_cmd = f"oc debug node/{node_name} -- chroot /host kill -9 {pids}"
# Create node-kill process map for verifying the result
node_and_kill_proc[node_name] = executor.submit(run_cmd, kill_cmd)
# Verify daemon kill process
for node_name, daemon_kill_proc in node_and_kill_proc.items():
# Get the type of daemons killed on the particular node
resources = [
disruption.resource
for disruption in disruption_ops
if disruption.daemon_pid in nodes_and_pids[node_name]
]
# 'daemon_kill_proc' result will be an empty string if command is success
cmd_out = daemon_kill_proc.result()
assert isinstance(cmd_out, str) and (not cmd_out), (
f"Failed to kill {resources } daemons in the node {node_name}. "
f"Daemon kill command output - {cmd_out}"
)
# Wait for new daemon to come up
[disruption.check_new_pid() for disruption in disruption_ops]
log.info("Verified daemons kill")
pods_deleted = pod_bulk_delete.result()
assert pods_deleted, "Deletion of pods failed."
# Verify pods are deleted
for pod_obj in pods_to_delete:
pod_obj.ocp.wait_for_delete(pod_obj.name, 300)
log.info("Verified: Pods are deleted.")
# Verify that the mount point is removed from nodes after deleting pod
node_pv_mounted = verify_pv_mounted_on_node(node_pv_dict)
for node, pvs in node_pv_mounted.items():
assert not pvs, (
f"PVs {pvs} is still present on node {node} after "
f"deleting the pods."
)
log.info(
"Verified: mount points are removed from nodes after deleting " "the pods"
)
pvcs_deleted = pvc_bulk_delete.result()
assert pvcs_deleted, "Deletion of PVCs failed."
# Verify PVCs are deleted
for pvc_obj in pvcs_to_delete:
pvc_obj.ocp.wait_for_delete(pvc_obj.name)
log.info("Verified: PVCs are deleted.")
# Getting result of PVC creation as list of PVC objects
log.info("Getting the result of CephFS PVC creation process")
pvc_objs_cephfs_new = pvc_create_cephfs.result()
log.info("Getting the result of RBD PVC creation process")
pvc_objs_rbd_new = pvc_create_rbd.result()
# Set interface argument for reference
for pvc_obj in pvc_objs_cephfs_new:
pvc_obj.interface = constants.CEPHFILESYSTEM
# Set interface argument for reference
for pvc_obj in pvc_objs_rbd_new:
pvc_obj.interface = constants.CEPHBLOCKPOOL
# Confirm PVCs are Bound
log.info("Verifying the new CephFS and RBD PVCs are Bound")
for pvc_obj in pvc_objs_cephfs_new + pvc_objs_rbd_new:
helpers.wait_for_resource_state(
resource=pvc_obj, state=constants.STATUS_BOUND, timeout=180
)
pvc_obj.reload()
log.info("Verified: New CephFS and RBD PVCs are Bound.")
# Getting result of pods creation as list of Pod objects
log.info("Getting the result of pods creation process")
pod_objs_rbd_new = pod_create_rbd.result()
pod_objs_cephfs_new = pod_create_cephfs.result()
# Verify new pods are Running
log.info("Verifying the new pods are Running")
for pod_obj in pod_objs_rbd_new + pod_objs_cephfs_new:
helpers.wait_for_resource_state(
resource=pod_obj, state=constants.STATUS_RUNNING, timeout=90
)
pod_obj.reload()
log.info("Verified: All new pods are Running.")
# Verify PVs are deleted
for pv_obj in pv_objs:
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name, timeout=300)
log.info("Verified: PVs are deleted.")
# Verify PV using ceph toolbox. Image/Subvolume should be deleted.
pool_name = default_ceph_block_pool()
for pvc_obj, uuid in pvc_uuid_map.items():
if pvc_obj.interface == constants.CEPHBLOCKPOOL:
ret = verify_volume_deleted_in_backend(
interface=constants.CEPHBLOCKPOOL,
image_uuid=uuid,
pool_name=pool_name,
)
if pvc_obj.interface == constants.CEPHFILESYSTEM:
ret = verify_volume_deleted_in_backend(
interface=constants.CEPHFILESYSTEM, image_uuid=uuid
)
assert (
ret
), f"Volume associated with PVC {pvc_obj.name} still exists in the backend"
log.info("Fetching IO results from the pods.")
for pod_obj in io_pods:
fio_result = pod_obj.get_fio_results()
err_count = fio_result.get("jobs")[0].get("error")
assert (
err_count == 0
), f"FIO error on pod {pod_obj.name}. FIO result: {fio_result}"
log.info("Verified IO result on pods.")
# Verify that the new PVCs are usable by creating new pods
log.info("Verify that the new PVCs are usable by creating new pods")
pod_objs_rbd_re = helpers.create_pods(
pvc_objs_rbd_new, pod_factory, constants.CEPHBLOCKPOOL, 2
)
pod_objs_cephfs_re = helpers.create_pods(
pvc_objs_cephfs_new, pod_factory, constants.CEPHFILESYSTEM, 2
)
# Verify pods are Running
log.info("Verifying the pods are Running")
for pod_obj in pod_objs_rbd_re + pod_objs_cephfs_re:
helpers.wait_for_resource_state(
resource=pod_obj, state=constants.STATUS_RUNNING, timeout=90
)
pod_obj.reload()
log.info(
"Successfully created and verified the status of the pods using the new CephFS and RBD PVCs."
)
new_pods = (
pod_objs_rbd_new
+ pod_objs_cephfs_new
+ pod_objs_rbd_re
+ pod_objs_cephfs_re
)
# Do setup on the new pods for running IO
log.info("Setting up the new pods for running IO.")
for pod_obj in new_pods:
if pod_obj.pvc.get_pvc_vol_mode == "Block":
storage_type = "block"
else:
storage_type = "fs"
executor.submit(pod_obj.workload_setup, storage_type=storage_type)
# Wait for setup on the new pods to complete
for pod_obj in new_pods:
log.info(f"Waiting for IO setup to complete on pod {pod_obj.name}")
for sample in TimeoutSampler(360, 2, getattr, pod_obj, "wl_setup_done"):
if sample:
log.info(
f"Setup for running IO is completed on pod " f"{pod_obj.name}."
)
break
log.info("Setup for running IO is completed on the new pods.")
# Start IO on the new pods
log.info("Start IO on the new pods")
self.run_io_on_pods(new_pods)
log.info("Started IO on the new pods")
log.info("Fetching IO results from the new pods.")
for pod_obj in new_pods:
get_fio_rw_iops(pod_obj)
log.info("Verified IO result on the new pods.")
# Verify number of pods of each daemon type
final_num_resource_name = [
len(pod_functions[resource_name]()) for resource_name in daemons_to_kill
]
assert final_num_resource_name == num_of_resource_pods, (
f"Total number of pods of each type is not matching with "
f"initial value. Total number of pods of each type before daemon kill: "
f"{num_of_resource_pods}. Total number of pods of each type present now: "
f"{final_num_resource_name}"
)
# Check ceph status
ceph_health_check(namespace=config.ENV_DATA["cluster_namespace"])
log.info("Ceph cluster health is OK")
def setup(self, interface, multi_pvc_factory, pod_factory):
"""
Create PVCs and pods
"""
access_modes = [constants.ACCESS_MODE_RWO]
if interface == constants.CEPHFILESYSTEM:
access_modes.append(constants.ACCESS_MODE_RWX)
# Modify access_modes list to create rbd `block` type volume with
# RWX access mode. RWX is not supported in filesystem type rbd
if interface == constants.CEPHBLOCKPOOL:
access_modes.extend(
[
f"{constants.ACCESS_MODE_RWO}-Block",
f"{constants.ACCESS_MODE_RWX}-Block",
]
)
pvc_objs = multi_pvc_factory(
interface=interface,
project=None,
storageclass=None,
size=self.pvc_size,
access_modes=access_modes,
status=constants.STATUS_BOUND,
num_of_pvc=self.num_of_pvcs,
wait_each=False,
)
# Set volume mode on PVC objects
for pvc_obj in pvc_objs:
pvc_info = pvc_obj.get()
setattr(pvc_obj, "volume_mode", pvc_info["spec"]["volumeMode"])
rwo_pvcs = [
pvc_obj
for pvc_obj in pvc_objs
if (pvc_obj.access_mode == constants.ACCESS_MODE_RWO)
]
rwx_pvcs = [
pvc_obj
for pvc_obj in pvc_objs
if (pvc_obj.access_mode == constants.ACCESS_MODE_RWX)
]
num_of_rwo_pvc = len(rwo_pvcs)
num_of_rwx_pvc = len(rwx_pvcs)
block_rwo_pvcs = []
for pvc_obj in rwo_pvcs[:]:
if pvc_obj.volume_mode == "Block":
block_rwo_pvcs.append(pvc_obj)
rwo_pvcs.remove(pvc_obj)
log.info(
f"Created {num_of_rwo_pvc} RWO PVCs in which "
f"{len(block_rwo_pvcs)} are rbd block type."
)
log.info(f"Created {num_of_rwx_pvc} RWX PVCs.")
# Select 6 PVCs for IO pods
if block_rwo_pvcs:
pvc_objs_for_io_pods = rwo_pvcs[0:2] + rwx_pvcs[0:2] + block_rwo_pvcs[0:2]
pvc_objs_new_pods = rwo_pvcs[2:] + rwx_pvcs[2:] + block_rwo_pvcs[2:]
else:
pvc_objs_for_io_pods = rwo_pvcs[0:3] + rwx_pvcs[0:3]
pvc_objs_new_pods = rwo_pvcs[3:] + rwx_pvcs[3:]
# Create one pod using each RWO PVC and two pods using each RWX PVC
# for running IO
io_pods = helpers.create_pods(pvc_objs_for_io_pods, pod_factory, interface, 2)
# Wait for pods to be in Running state
for pod_obj in io_pods:
helpers.wait_for_resource_state(
resource=pod_obj, state=constants.STATUS_RUNNING
)
pod_obj.reload()
log.info(f"Created {len(io_pods)} pods for running IO.")
return pvc_objs, io_pods, pvc_objs_new_pods, access_modes
def test_daemon_kill_during_pvc_pod_creation_and_io(
self, interface, resource_name, setup, multi_pvc_factory, pod_factory
):
"""
Kill 'resource_name' daemon while PVCs creation, pods
creation and IO operation are progressing.
"""
num_of_new_pvcs = 5
pvc_objs, io_pods, pvc_objs_new_pods, access_modes = setup
proj_obj = pvc_objs[0].project
storageclass = pvc_objs[0].storageclass
pod_functions = {
"mds": partial(get_mds_pods),
"mon": partial(get_mon_pods),
"mgr": partial(get_mgr_pods),
"osd": partial(get_osd_pods),
"rbdplugin": partial(get_plugin_pods, interface=interface),
"cephfsplugin": partial(get_plugin_pods, interface=interface),
"cephfsplugin_provisioner": partial(get_cephfsplugin_provisioner_pods),
"rbdplugin_provisioner": partial(get_rbdfsplugin_provisioner_pods),
"operator": partial(get_operator_pods),
}
executor = ThreadPoolExecutor(max_workers=len(io_pods))
disruption = disruption_helpers.Disruptions()
disruption.set_resource(resource=resource_name)
# Get number of pods of type 'resource_name'
resource_pods_num = len(pod_functions[resource_name]())
# Do setup for running IO on pods
log.info("Setting up pods for running IO")
for pod_obj in io_pods:
if pod_obj.pvc.volume_mode == "Block":
storage_type = "block"
else:
storage_type = "fs"
executor.submit(pod_obj.workload_setup, storage_type=storage_type)
# Wait for setup on pods to complete
for pod_obj in io_pods:
log.info(f"Waiting for IO setup to complete on pod {pod_obj.name}")
for sample in TimeoutSampler(180, 2, getattr, pod_obj, "wl_setup_done"):
if sample:
log.info(
f"Setup for running IO is completed on pod " f"{pod_obj.name}."
)
break
log.info("Setup for running IO is completed on pods")
# Set daemon to be killed
disruption.select_daemon()
# Start creating new pods
log.info("Start creating new pods.")
bulk_pod_create = executor.submit(
helpers.create_pods, pvc_objs_new_pods, pod_factory, interface, 2
)
# Start creation of new PVCs
log.info("Start creating new PVCs.")
bulk_pvc_create = executor.submit(
multi_pvc_factory,
interface=interface,
project=proj_obj,
storageclass=storageclass,
size=self.pvc_size,
access_modes=access_modes,
access_modes_selection="distribute_random",
status="",
num_of_pvc=num_of_new_pvcs,
wait_each=False,
)
# Start IO on each pod
log.info("Start IO on pods")
for pod_obj in io_pods:
if pod_obj.pvc.volume_mode == "Block":
storage_type = "block"
else:
storage_type = "fs"
pod_obj.run_io(
storage_type=storage_type,
size="1G",
runtime=10,
fio_filename=f"{pod_obj.name}_io_file1",
)
log.info("IO started on all pods.")
# Kill daemon
disruption.kill_daemon()
# Getting result of PVC creation as list of PVC objects
pvc_objs_new = bulk_pvc_create.result()
# Confirm PVCs are Bound
for pvc_obj in pvc_objs_new:
helpers.wait_for_resource_state(
resource=pvc_obj, state=constants.STATUS_BOUND, timeout=180
)
pvc_obj.reload()
log.info("Verified: New PVCs are Bound.")
# Getting result of pods creation as list of Pod objects
pod_objs_new = bulk_pod_create.result()
# Verify new pods are Running
for pod_obj in pod_objs_new:
helpers.wait_for_resource_state(
resource=pod_obj, state=constants.STATUS_RUNNING
)
pod_obj.reload()
log.info("Verified: All new pods are Running.")
# Verify IO
log.info("Fetching IO results from IO pods.")
for pod_obj in io_pods:
fio_result = pod_obj.get_fio_results()
err_count = fio_result.get("jobs")[0].get("error")
assert (
err_count == 0
), f"FIO error on pod {pod_obj.name}. FIO result: {fio_result}"
log.info(f"IOPs after FIO on pod {pod_obj.name}:")
log.info(f"Read: {fio_result.get('jobs')[0].get('read').get('iops')}")
log.info(f"Write: {fio_result.get('jobs')[0].get('write').get('iops')}")
log.info("Verified IO result on IO pods.")
all_pod_objs = io_pods + pod_objs_new
# Fetch volume details from pods for the purpose of verification
node_pv_dict = {}
for pod in all_pod_objs:
pod_info = pod.get()
node = pod_info["spec"]["nodeName"]
pvc = pod_info["spec"]["volumes"][0]["persistentVolumeClaim"]["claimName"]
for pvc_obj in pvc_objs:
if pvc_obj.name == pvc:
pvc_obj.reload()
pv = pvc_obj.backed_pv
break
if node in node_pv_dict:
node_pv_dict[node].append(pv)
else:
node_pv_dict[node] = [pv]
# Delete pods
for pod_obj in all_pod_objs:
pod_obj.delete(wait=False)
# Verify pods are deleted
for pod_obj in all_pod_objs:
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
# Verify number of 'resource_name' type pods
final_resource_pods_num = len(pod_functions[resource_name]())
assert final_resource_pods_num == resource_pods_num, (
f"Total number of {resource_name} pods is not matching with "
f"initial value. Total number of pods before daemon kill: "
f"{resource_pods_num}. Total number of pods present now: "
f"{final_resource_pods_num}"
)
# Verify volumes are unmapped from nodes after deleting the pods
node_pv_mounted = helpers.verify_pv_mounted_on_node(node_pv_dict)
for node, pvs in node_pv_mounted.items():
assert not pvs, (
f"PVs {pvs} is still present on node {node} after "
f"deleting the pods."
)
log.info(
"Verified: mount points are removed from nodes after deleting " "the pods"
)
# Set volume mode on PVC objects
for pvc_obj in pvc_objs_new:
pvc_info = pvc_obj.get()
setattr(pvc_obj, "volume_mode", pvc_info["spec"]["volumeMode"])
# Verify that PVCs are reusable by creating new pods
all_pvc_objs = pvc_objs + pvc_objs_new
pod_objs_re = helpers.create_pods(all_pvc_objs, pod_factory, interface, 2)
# Verify pods are Running
for pod_obj in pod_objs_re:
helpers.wait_for_resource_state(
resource=pod_obj, state=constants.STATUS_RUNNING
)
pod_obj.reload()
log.info("Successfully created new pods using all PVCs.")
# Run IO on each of the newly created pods
for pod_obj in pod_objs_re:
if pod_obj.pvc.volume_mode == "Block":
storage_type = "block"
else:
storage_type = "fs"
pod_obj.run_io(
storage_type=storage_type,
size="1G",
runtime=10,
fio_filename=f"{pod_obj.name}_io_file2",
)
log.info("Fetching IO results from newly created pods")
for pod_obj in pod_objs_re:
fio_result = pod_obj.get_fio_results()
err_count = fio_result.get("jobs")[0].get("error")
assert (
err_count == 0
), f"FIO error on pod {pod_obj.name}. FIO result: {fio_result}"
log.info(f"IOPs after FIO on pod {pod_obj.name}:")
log.info(f"Read: {fio_result.get('jobs')[0].get('read').get('iops')}")
log.info(f"Write: {fio_result.get('jobs')[0].get('write').get('iops')}")
log.info("Verified IO result on newly created pods.")
def setup(
self,
kv_version,
kms_provider,
use_vault_namespace,
pv_encryption_kms_setup_factory,
project_factory,
multi_pvc_factory,
pod_factory,
storageclass_factory,
):
"""
Setup csi-kms-connection-details configmap and create resources for the test
"""
log.info("Setting up csi-kms-connection-details configmap")
self.kms = pv_encryption_kms_setup_factory(kv_version,
use_vault_namespace)
log.info("csi-kms-connection-details setup successful")
# Create a project
self.proj_obj = project_factory()
# Create an encryption enabled storageclass for RBD
self.sc_obj = storageclass_factory(
interface=constants.CEPHBLOCKPOOL,
encrypted=True,
encryption_kms_id=self.kms.kmsid,
)
if kms_provider == constants.VAULT_KMS_PROVIDER:
# Create ceph-csi-kms-token in the tenant namespace
self.kms.vault_path_token = self.kms.generate_vault_token()
self.kms.create_vault_csi_kms_token(
namespace=self.proj_obj.namespace)
# Create PVC and Pods
self.pvc_size = 1
self.pvc_objs = multi_pvc_factory(
interface=constants.CEPHBLOCKPOOL,
project=self.proj_obj,
storageclass=self.sc_obj,
size=self.pvc_size,
access_modes=[
f"{constants.ACCESS_MODE_RWX}-Block",
f"{constants.ACCESS_MODE_RWO}-Block",
],
status=constants.STATUS_BOUND,
num_of_pvc=2,
wait_each=False,
)
self.pod_objs = helpers.create_pods(
self.pvc_objs,
pod_factory,
constants.CEPHBLOCKPOOL,
pods_for_rwx=1,
status=constants.STATUS_RUNNING,
)
# Verify if the key is created in Vault
self.vol_handles = []
for pvc_obj in self.pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
vol_handle = pv_obj.get().get("spec").get("csi").get(
"volumeHandle")
self.vol_handles.append(vol_handle)
if kms_provider == constants.VAULT_KMS_PROVIDER:
if kms.is_key_present_in_path(
key=vol_handle, path=self.kms.vault_backend_path):
log.info(f"Vault: Found key for {pvc_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for {pvc_obj.name}")
def test_pvc_to_pvc_clone(self, kv_version, kms_provider, pod_factory):
"""
Test to create a clone from an existing encrypted RBD PVC.
Verify that the cloned PVC is encrypted and all the data is preserved.
"""
log.info("Checking for encrypted device and running IO on all pods")
for vol_handle, pod_obj in zip(self.vol_handles, self.pod_objs):
if pod_obj.exec_sh_cmd_on_pod(
command=f"lsblk | grep {vol_handle} | grep crypt"):
log.info(f"Encrypted device found in {pod_obj.name}")
else:
raise ResourceNotFoundError(
f"Encrypted device not found in {pod_obj.name}")
log.info(f"File created during IO {pod_obj.name}")
pod_obj.run_io(
storage_type="block",
size="500M",
io_direction="write",
runtime=60,
end_fsync=1,
direct=1,
)
log.info("IO started on all pods")
# Wait for IO completion
for pod_obj in self.pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on all pods")
cloned_pvc_objs, cloned_vol_handles = ([] for i in range(2))
# Calculate the md5sum value and create clones of exisiting PVCs
log.info("Calculate the md5sum after IO and create clone of all PVCs")
for pod_obj in self.pod_objs:
pod_obj.md5sum_after_io = pod.cal_md5sum(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
block=True,
)
cloned_pvc_obj = pvc.create_pvc_clone(
self.sc_obj.name,
pod_obj.pvc.name,
constants.CSI_RBD_PVC_CLONE_YAML,
self.proj_obj.namespace,
volume_mode=constants.VOLUME_MODE_BLOCK,
access_mode=pod_obj.pvc.access_mode,
)
helpers.wait_for_resource_state(cloned_pvc_obj,
constants.STATUS_BOUND)
cloned_pvc_obj.reload()
cloned_pvc_obj.md5sum = pod_obj.md5sum_after_io
cloned_pvc_objs.append(cloned_pvc_obj)
log.info("Clone of all PVCs created")
# Create and attach pod to the pvc
cloned_pod_objs = helpers.create_pods(
cloned_pvc_objs,
pod_factory,
constants.CEPHBLOCKPOOL,
pods_for_rwx=1,
status="",
)
# Verify the new pods are running
log.info("Verify the new pods are running")
for pod_obj in cloned_pod_objs:
helpers.wait_for_resource_state(pod_obj, constants.STATUS_RUNNING)
pod_obj.reload()
log.info("Verified: New pods are running")
# Verify encryption keys are created for cloned PVCs in Vault
for pvc_obj in cloned_pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
vol_handle = pv_obj.get().get("spec").get("csi").get(
"volumeHandle")
cloned_vol_handles.append(vol_handle)
if kms_provider == constants.VAULT_KMS_PROVIDER:
if kms.is_key_present_in_path(
key=vol_handle, path=self.kms.vault_backend_path):
log.info(
f"Vault: Found key for restore PVC {pvc_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for restored PVC {pvc_obj.name}"
)
# Verify encrypted device is present and md5sum on all pods
for vol_handle, pod_obj in zip(cloned_vol_handles, cloned_pod_objs):
if pod_obj.exec_sh_cmd_on_pod(
command=f"lsblk | grep {vol_handle} | grep crypt"):
log.info(f"Encrypted device found in {pod_obj.name}")
else:
raise ResourceNotFoundError(
f"Encrypted device not found in {pod_obj.name}")
log.info(f"Verifying md5sum on pod {pod_obj.name}")
pod.verify_data_integrity(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
original_md5sum=pod_obj.pvc.md5sum,
block=True,
)
log.info(f"Verified md5sum on pod {pod_obj.name}")
# Run IO on new pods
log.info("Starting IO on new pods")
for pod_obj in cloned_pod_objs:
pod_obj.run_io(storage_type="block", size="100M", runtime=10)
# Wait for IO completion on new pods
log.info("Waiting for IO completion on new pods")
for pod_obj in cloned_pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on new pods.")
# Delete the restored pods, PVC and snapshots
log.info("Deleting all pods")
for pod_obj in cloned_pod_objs + self.pod_objs:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
log.info("Deleting all PVCs")
for pvc_obj in cloned_pvc_objs + self.pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
if kms_provider == constants.VAULT_KMS_PROVIDER:
# Verify if the keys for parent and cloned PVCs are deleted from Vault
if kv_version == "v1" or Version.coerce(
config.ENV_DATA["ocs_version"]) >= Version.coerce("4.9"):
log.info(
"Verify whether the keys for cloned PVCs are deleted from vault"
)
for key in cloned_vol_handles + self.vol_handles:
if not kms.is_key_present_in_path(
key=key, path=self.kms.vault_backend_path):
log.info(f"Vault: Key deleted for {key}")
else:
raise KMSResourceCleaneupError(
f"Vault: Key deletion failed for {key}")
log.info("All keys from vault were deleted")
def test_rbd_thick_provisioning(self, multi_pvc_factory, pod_factory):
"""
Test to verify RBD thick provisioning enabled storage class creation, PVC creation and consumption using
the storage class.
"""
# Dict represents the PVC size and size of the file used to consume the volume
pvc_and_file_sizes = {1: "900M", 5: "4G"}
access_modes_rbd = [
constants.ACCESS_MODE_RWO,
f"{constants.ACCESS_MODE_RWO}-Block",
f"{constants.ACCESS_MODE_RWX}-Block",
]
pvcs = []
# Create PVCs
for pvc_size in pvc_and_file_sizes.keys():
pvc_objs = multi_pvc_factory(
interface=constants.CEPHBLOCKPOOL,
storageclass=self.sc_obj,
size=pvc_size,
access_modes=access_modes_rbd,
status=constants.STATUS_BOUND,
num_of_pvc=3,
timeout=300,
)
for pvc_obj in pvc_objs:
pvc_obj.io_file_size = pvc_and_file_sizes[pvc_size]
pvc_obj.storage_type = (constants.WORKLOAD_STORAGE_TYPE_BLOCK
if pvc_obj.get()["spec"]["volumeMode"]
== constants.VOLUME_MODE_BLOCK else
constants.WORKLOAD_STORAGE_TYPE_FS)
pvcs.extend(pvc_objs)
# Create pods
pods = helpers.create_pods(
pvc_objs=pvcs,
pod_factory=pod_factory,
interface=constants.CEPHBLOCKPOOL,
status=constants.STATUS_RUNNING,
)
executor = ThreadPoolExecutor(max_workers=len(pods))
# Do setup for running IO on pods
log.info("Setting up pods to running IO")
for pod_obj in pods:
executor.submit(pod_obj.workload_setup,
storage_type=pod_obj.pvc.storage_type)
# Wait for setup on pods to complete
for pod_obj in pods:
log.info(f"Waiting for IO setup to complete on pod {pod_obj.name}")
for sample in TimeoutSampler(360, 2, getattr, pod_obj,
"wl_setup_done"):
if sample:
log.info(f"Setup for running IO is completed on pod "
f"{pod_obj.name}.")
break
log.info("Setup for running IO is completed on pods")
# Start IO
for pod_obj in pods:
log.info(f"Starting IO on pod {pod_obj.name}.")
pod_obj.run_io(
storage_type=pod_obj.pvc.storage_type,
size=pod_obj.pvc.io_file_size,
runtime=30,
)
log.info(f"IO started on pod {pod_obj.name}.")
log.info("IO started on pods.")
log.info("Verifying IO on pods.")
for pod_obj in pods:
pod_obj.get_fio_results()
log.info(f"IO completed on pod {pod_obj.name}.")
log.info("IO finished on all pods.")
def setup(self, request, interface, multi_pvc_factory, pod_factory):
"""
Create PVCs and pods
"""
if config.ENV_DATA["platform"].lower(
) in constants.MANAGED_SERVICE_PLATFORMS:
# Get the index of current consumer cluster
self.consumer_cluster_index = config.cur_index
def teardown():
# Switching to provider cluster context will be done during the test case.
# Switch back to consumer cluster context after the test case.
config.switch_to_consumer(self.consumer_cluster_index)
request.addfinalizer(teardown)
access_modes = [constants.ACCESS_MODE_RWO]
if interface == constants.CEPHFILESYSTEM:
access_modes.append(constants.ACCESS_MODE_RWX)
# Modify access_modes list to create rbd `block` type volume with
# RWX access mode. RWX is not supported in filesystem type rbd
if interface == constants.CEPHBLOCKPOOL:
access_modes.extend([
f"{constants.ACCESS_MODE_RWO}-Block",
f"{constants.ACCESS_MODE_RWX}-Block",
])
pvc_objs = multi_pvc_factory(
interface=interface,
project=None,
storageclass=None,
size=self.pvc_size,
access_modes=access_modes,
status=constants.STATUS_BOUND,
num_of_pvc=self.num_of_pvcs,
wait_each=False,
)
# Set volume mode on PVC objects
for pvc_obj in pvc_objs:
pvc_info = pvc_obj.get()
setattr(pvc_obj, "volume_mode", pvc_info["spec"]["volumeMode"])
rwo_pvcs = [
pvc_obj for pvc_obj in pvc_objs
if (pvc_obj.access_mode == constants.ACCESS_MODE_RWO)
]
rwx_pvcs = [
pvc_obj for pvc_obj in pvc_objs
if (pvc_obj.access_mode == constants.ACCESS_MODE_RWX)
]
num_of_rwo_pvc = len(rwo_pvcs)
num_of_rwx_pvc = len(rwx_pvcs)
block_rwo_pvcs = []
for pvc_obj in rwo_pvcs[:]:
if pvc_obj.volume_mode == "Block":
block_rwo_pvcs.append(pvc_obj)
rwo_pvcs.remove(pvc_obj)
log.info(f"Created {num_of_rwo_pvc} RWO PVCs in which "
f"{len(block_rwo_pvcs)} are rbd block type.")
log.info(f"Created {num_of_rwx_pvc} RWX PVCs.")
# Select 6 PVCs for IO pods
if block_rwo_pvcs:
pvc_objs_for_io_pods = rwo_pvcs[0:2] + rwx_pvcs[
0:2] + block_rwo_pvcs[0:2]
pvc_objs_new_pods = rwo_pvcs[2:] + rwx_pvcs[2:] + block_rwo_pvcs[2:]
else:
pvc_objs_for_io_pods = rwo_pvcs[0:3] + rwx_pvcs[0:3]
pvc_objs_new_pods = rwo_pvcs[3:] + rwx_pvcs[3:]
# Create one pod using each RWO PVC and two pods using each RWX PVC
# for running IO
io_pods = helpers.create_pods(pvc_objs_for_io_pods, pod_factory,
interface, 2)
# Wait for pods to be in Running state
for pod_obj in io_pods:
helpers.wait_for_resource_state(resource=pod_obj,
state=constants.STATUS_RUNNING)
pod_obj.reload()
log.info(f"Created {len(io_pods)} pods for running IO.")
return pvc_objs, io_pods, pvc_objs_new_pods, access_modes
def test_encrypted_rbd_block_pvc_snapshot(
self,
kms_provider,
snapshot_factory,
snapshot_restore_factory,
pod_factory,
kv_version,
):
"""
Test to take snapshots of encrypted RBD Block VolumeMode PVCs
"""
log.info(
"Check for encrypted device, find initial md5sum value and run IO on all pods"
)
for vol_handle, pod_obj in zip(self.vol_handles, self.pod_objs):
# Verify whether encrypted device is present inside the pod
if pod_obj.exec_sh_cmd_on_pod(
command=f"lsblk | grep {vol_handle} | grep crypt"):
log.info(f"Encrypted device found in {pod_obj.name}")
else:
raise ResourceNotFoundError(
f"Encrypted device not found in {pod_obj.name}")
# Find initial md5sum
pod_obj.md5sum_before_io = cal_md5sum(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
block=True,
)
pod_obj.run_io(
storage_type="block",
size=f"{self.pvc_size - 1}G",
io_direction="write",
runtime=60,
)
log.info("IO started on all pods")
# Wait for IO completion
for pod_obj in self.pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on all pods")
snap_objs, snap_handles = ([] for i in range(2))
# Verify md5sum has changed after IO. Create snapshot
log.info(
"Verify md5sum has changed after IO and create snapshot from all PVCs"
)
for pod_obj in self.pod_objs:
md5sum_after_io = cal_md5sum(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
block=True,
)
assert (pod_obj.md5sum_before_io != md5sum_after_io
), f"md5sum has not changed after IO on pod {pod_obj.name}"
log.info(f"Creating snapshot of PVC {pod_obj.pvc.name}")
snap_obj = snapshot_factory(pod_obj.pvc, wait=False)
snap_obj.md5sum = md5sum_after_io
snap_objs.append(snap_obj)
log.info("Snapshots created")
# Verify snapshots are ready and verify if encryption key is created in vault
log.info("Verify snapshots are ready")
for snap_obj in snap_objs:
snap_obj.ocp.wait_for_resource(
condition="true",
resource_name=snap_obj.name,
column=constants.STATUS_READYTOUSE,
timeout=180,
)
snapshot_content = get_snapshot_content_obj(snap_obj=snap_obj)
snap_handle = snapshot_content.get().get("status").get(
"snapshotHandle")
if kms_provider == constants.VAULT_KMS_PROVIDER:
if kms.is_key_present_in_path(
key=snap_handle, path=self.kms.vault_backend_path):
log.info(f"Vault: Found key for snapshot {snap_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for snapshot {snap_obj.name}")
snap_handles.append(snap_handle)
# Delete pods
log.info("Deleting the pods")
for pod_obj in self.pod_objs:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
log.info("Deleted all the pods")
# Delete parent PVCs to verify snapshot is independent
log.info("Deleting parent PVCs")
for pvc_obj in self.pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name)
log.info(f"Deleted PVC {pvc_obj.name}. Verifying whether PV "
f"{pv_obj.name} is deleted.")
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
log.info(
"All parent PVCs and PVs are deleted before restoring snapshot.")
restore_pvc_objs, restore_vol_handles = ([] for i in range(2))
# Create PVCs out of the snapshots
log.info("Creating new PVCs from snapshots")
for snap_obj in snap_objs:
log.info(f"Creating a PVC from snapshot {snap_obj.name}")
restore_pvc_obj = snapshot_restore_factory(
snapshot_obj=snap_obj,
storageclass=self.sc_obj.name,
size=f"{self.pvc_size}Gi",
volume_mode=snap_obj.parent_volume_mode,
access_mode=snap_obj.parent_access_mode,
status="",
)
log.info(f"Created PVC {restore_pvc_obj.name} from snapshot "
f"{snap_obj.name}")
restore_pvc_obj.md5sum = snap_obj.md5sum
restore_pvc_objs.append(restore_pvc_obj)
log.info("Created new PVCs from all the snapshots")
# Confirm that the restored PVCs are Bound
log.info("Verify the restored PVCs are Bound")
for pvc_obj in restore_pvc_objs:
wait_for_resource_state(resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=180)
pvc_obj.reload()
log.info("Verified: Restored PVCs are Bound.")
# Attach the restored PVCs to pods. Attach RWX PVC on two pods
log.info("Attach the restored PVCs to pods")
restore_pod_objs = create_pods(
restore_pvc_objs,
pod_factory,
constants.CEPHBLOCKPOOL,
pods_for_rwx=1,
status="",
)
# Verify the new pods are running
log.info("Verify the new pods are running")
for pod_obj in restore_pod_objs:
timeout = (300 if config.ENV_DATA["platform"]
== constants.IBMCLOUD_PLATFORM else 60)
wait_for_resource_state(pod_obj, constants.STATUS_RUNNING, timeout)
log.info("Verified: New pods are running")
# Verify encryption keys are created for restored PVCs in Vault
for pvc_obj in restore_pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
vol_handle = pv_obj.get().get("spec").get("csi").get(
"volumeHandle")
restore_vol_handles.append(vol_handle)
if kms_provider == constants.VAULT_KMS_PROVIDER:
if kms.is_key_present_in_path(
key=vol_handle, path=self.kms.vault_backend_path):
log.info(
f"Vault: Found key for restore PVC {pvc_obj.name}")
else:
raise ResourceNotFoundError(
f"Vault: Key not found for restored PVC {pvc_obj.name}"
)
# Verify encrypted device is present and md5sum on all pods
for vol_handle, pod_obj in zip(restore_vol_handles, restore_pod_objs):
if pod_obj.exec_sh_cmd_on_pod(
command=f"lsblk | grep {vol_handle} | grep crypt"):
log.info(f"Encrypted device found in {pod_obj.name}")
else:
raise ResourceNotFoundError(
f"Encrypted device not found in {pod_obj.name}")
log.info(f"Verifying md5sum on pod {pod_obj.name}")
verify_data_integrity(
pod_obj=pod_obj,
file_name=pod_obj.get_storage_path(storage_type="block"),
original_md5sum=pod_obj.pvc.md5sum,
block=True,
)
log.info(f"Verified md5sum on pod {pod_obj.name}")
# Run IO on new pods
log.info("Starting IO on new pods")
for pod_obj in restore_pod_objs:
pod_obj.run_io(storage_type="block", size="500M", runtime=15)
# Wait for IO completion on new pods
log.info("Waiting for IO completion on new pods")
for pod_obj in restore_pod_objs:
pod_obj.get_fio_results()
log.info("IO completed on new pods.")
# Delete the restored pods, PVC and snapshots
log.info("Deleting pods using restored PVCs")
for pod_obj in restore_pod_objs:
pod_obj.delete()
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
log.info("Deleting restored PVCs")
for pvc_obj in restore_pvc_objs:
pv_obj = pvc_obj.backed_pv_obj
pvc_obj.delete()
pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name)
log.info("Deleting the snapshots")
for snap_obj in snap_objs:
snapcontent_obj = get_snapshot_content_obj(snap_obj=snap_obj)
snap_obj.delete()
snapcontent_obj.ocp.wait_for_delete(
resource_name=snapcontent_obj.name)
if kms_provider == constants.VAULT_KMS_PROVIDER:
# Verify if keys for PVCs and snapshots are deleted from Vault
if kv_version == "v1" or Version.coerce(
config.ENV_DATA["ocs_version"]) >= Version.coerce("4.9"):
log.info(
"Verify whether the keys for PVCs and snapshots are deleted in vault"
)
for key in self.vol_handles + snap_handles + restore_vol_handles:
if not kms.is_key_present_in_path(
key=key, path=self.kms.vault_backend_path):
log.info(f"Vault: Key deleted for {key}")
else:
raise KMSResourceCleaneupError(
f"Vault: Key deletion failed for {key}")
log.info("All keys from vault were deleted")
