def setup(self, multi_pvc_factory, pod_factory):
"""
Create resources for the test
"""
access_modes_cephfs = [
constants.ACCESS_MODE_RWO, constants.ACCESS_MODE_RWX
]
access_modes_rbd = [
constants.ACCESS_MODE_RWO, f'{constants.ACCESS_MODE_RWO}-Block',
f'{constants.ACCESS_MODE_RWX}-Block'
]
self.pvcs_cephfs = multi_pvc_factory(
interface=constants.CEPHFILESYSTEM,
size=10,
access_modes=access_modes_cephfs,
status=constants.STATUS_BOUND,
num_of_pvc=2,
timeout=90)
self.pvcs_rbd = multi_pvc_factory(interface=constants.CEPHBLOCKPOOL,
project=self.pvcs_cephfs[0].project,
size=10,
access_modes=access_modes_rbd,
status=constants.STATUS_BOUND,
num_of_pvc=3,
timeout=90)
pods_cephfs = helpers.create_pods(self.pvcs_cephfs, pod_factory,
constants.CEPHFILESYSTEM, 2,
constants.STATUS_RUNNING)
pods_rbd = helpers.create_pods(self.pvcs_rbd, pod_factory,
constants.CEPHBLOCKPOOL, 2,
constants.STATUS_RUNNING)
self.pods = pods_cephfs + pods_rbd
# Set volume mode on PVC objects
for pvc_obj in self.pvcs_cephfs + self.pvcs_rbd:
pvc_info = pvc_obj.get()
setattr(pvc_obj, 'volume_mode', pvc_info['spec']['volumeMode'])
def test_pvc_disruptive(self, interface, operation_to_disrupt,
resource_to_delete, multi_pvc_factory,
pod_factory):
"""
Base function for PVC disruptive tests.
Deletion of 'resource_to_delete' will be introduced while
'operation_to_disrupt' is progressing.
"""
pod_functions = {
'mds':
partial(pod.get_mds_pods),
'mon':
partial(pod.get_mon_pods),
'mgr':
partial(pod.get_mgr_pods),
'osd':
partial(pod.get_osd_pods),
'rbdplugin':
partial(pod.get_plugin_pods, interface=interface),
'cephfsplugin':
partial(pod.get_plugin_pods, interface=interface),
'cephfsplugin_provisioner':
partial(pod.get_cephfsplugin_provisioner_pods),
'rbdplugin_provisioner':
partial(pod.get_rbdfsplugin_provisioner_pods),
'operator':
partial(pod.get_operator_pods)
}
# Get number of pods of type 'resource_to_delete'
num_of_resource_to_delete = len(pod_functions[resource_to_delete]())
num_of_pvc = 12
namespace = self.proj_obj.namespace
# Fetch the number of Pods and PVCs
initial_num_of_pods = len(pod.get_all_pods(namespace=namespace))
initial_num_of_pvc = len(get_all_pvcs(namespace=namespace)['items'])
executor = ThreadPoolExecutor(max_workers=(2 * num_of_pvc))
DISRUPTION_OPS.set_resource(resource=resource_to_delete)
access_modes = [constants.ACCESS_MODE_RWO]
if interface == constants.CEPHFILESYSTEM:
access_modes.append(constants.ACCESS_MODE_RWX)
# Modify access_modes list to create rbd `block` type volume with
# RWX access mode. RWX is not supported in non-block type rbd
if interface == constants.CEPHBLOCKPOOL:
access_modes.extend([
f'{constants.ACCESS_MODE_RWO}-Block',
f'{constants.ACCESS_MODE_RWX}-Block'
])
# Start creation of PVCs
bulk_pvc_create = executor.submit(
multi_pvc_factory,
interface=interface,
project=self.proj_obj,
size=5,
access_modes=access_modes,
access_modes_selection='distribute_random',
status=constants.STATUS_BOUND,
num_of_pvc=num_of_pvc,
wait_each=False)
if operation_to_disrupt == 'create_pvc':
# Ensure PVCs are being created before deleting the resource
ret = helpers.wait_for_resource_count_change(
get_all_pvcs, initial_num_of_pvc, namespace, 'increase')
assert ret, "Wait timeout: PVCs are not being created."
logger.info(f"PVCs creation has started.")
DISRUPTION_OPS.delete_resource()
pvc_objs = bulk_pvc_create.result()
# Confirm that PVCs are Bound
for pvc_obj in pvc_objs:
helpers.wait_for_resource_state(resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=120)
pvc_obj.reload()
logger.info("Verified: PVCs are Bound.")
# Start creating pods
bulk_pod_create = executor.submit(helpers.create_pods, pvc_objs,
pod_factory, interface, 2)
if operation_to_disrupt == 'create_pod':
# Ensure that pods are being created before deleting the resource
ret = helpers.wait_for_resource_count_change(
pod.get_all_pods, initial_num_of_pods, namespace, 'increase')
assert ret, "Wait timeout: Pods are not being created."
logger.info(f"Pods creation has started.")
DISRUPTION_OPS.delete_resource()
pod_objs = bulk_pod_create.result()
# Verify pods are Running
for pod_obj in pod_objs:
helpers.wait_for_resource_state(resource=pod_obj,
state=constants.STATUS_RUNNING)
pod_obj.reload()
logger.info("Verified: All pods are Running.")
# Do setup on pods for running IO
logger.info("Setting up pods for running IO.")
for pod_obj in pod_objs:
pvc_info = pod_obj.pvc.get()
if pvc_info['spec']['volumeMode'] == 'Block':
storage_type = 'block'
else:
storage_type = 'fs'
executor.submit(pod_obj.workload_setup, storage_type=storage_type)
# Wait for setup on pods to complete
for pod_obj in pod_objs:
for sample in TimeoutSampler(180, 2, getattr, pod_obj,
'wl_setup_done'):
if sample:
logger.info(f"Setup for running IO is completed on pod "
f"{pod_obj.name}.")
break
logger.info("Setup for running IO is completed on all pods.")
# Start IO on each pod
for pod_obj in pod_objs:
pvc_info = pod_obj.pvc.get()
if pvc_info['spec']['volumeMode'] == 'Block':
storage_type = 'block'
else:
storage_type = 'fs'
pod_obj.run_io(storage_type=storage_type,
size='1G',
runtime=10,
fio_filename=f'{pod_obj.name}_io_file1')
logger.info("FIO started on all pods.")
if operation_to_disrupt == 'run_io':
DISRUPTION_OPS.delete_resource()
logger.info("Fetching FIO results.")
for pod_obj in pod_objs:
fio_result = pod_obj.get_fio_results()
err_count = fio_result.get('jobs')[0].get('error')
assert err_count == 0, (
f"FIO error on pod {pod_obj.name}. FIO result: {fio_result}")
logger.info("Verified FIO result on pods.")
# Delete pods
for pod_obj in pod_objs:
pod_obj.delete(wait=True)
for pod_obj in pod_objs:
pod_obj.ocp.wait_for_delete(pod_obj.name)
# Verify that PVCs are reusable by creating new pods
pod_objs = helpers.create_pods(pvc_objs, pod_factory, interface, 2)
# Verify new pods are Running
for pod_obj in pod_objs:
helpers.wait_for_resource_state(resource=pod_obj,
state=constants.STATUS_RUNNING)
pod_obj.reload()
logging.info("Verified: All new pods are Running.")
# Run IO on each of the new pods
for pod_obj in pod_objs:
pvc_info = pod_obj.pvc.get()
if pvc_info['spec']['volumeMode'] == 'Block':
storage_type = 'block'
else:
storage_type = 'fs'
pod_obj.run_io(storage_type=storage_type,
size='1G',
runtime=10,
fio_filename=f'{pod_obj.name}_io_file2')
logger.info("Fetching FIO results from new pods")
for pod_obj in pod_objs:
fio_result = pod_obj.get_fio_results()
err_count = fio_result.get('jobs')[0].get('error')
assert err_count == 0, (
f"FIO error on pod {pod_obj.name}. FIO result: {fio_result}")
logger.info("Verified FIO result on new pods.")
# Verify number of pods of type 'resource_to_delete'
final_num_resource_to_delete = len(pod_functions[resource_to_delete]())
assert final_num_resource_to_delete == num_of_resource_to_delete, (
f"Total number of {resource_to_delete} pods is not matching with "
f"initial value. Total number of pods before deleting a pod: "
f"{num_of_resource_to_delete}. Total number of pods present now: "
f"{final_num_resource_to_delete}")
# Check ceph status
ceph_health_check(namespace=config.ENV_DATA['cluster_namespace'])
logger.info("Ceph cluster health is OK")
def setup(self, interface, multi_pvc_factory, pod_factory):
"""
Create PVCs and pods
"""
access_modes = [constants.ACCESS_MODE_RWO]
if interface == constants.CEPHFILESYSTEM:
access_modes.append(constants.ACCESS_MODE_RWX)
# Modify access_modes list to create rbd `block` type volume with
# RWX access mode. RWX is not supported in filesystem type rbd
if interface == constants.CEPHBLOCKPOOL:
access_modes.extend(
[
f'{constants.ACCESS_MODE_RWO}-Block',
f'{constants.ACCESS_MODE_RWX}-Block'
]
)
pvc_objs = multi_pvc_factory(
interface=interface,
project=None,
storageclass=None,
size=self.pvc_size,
access_modes=access_modes,
status=constants.STATUS_BOUND,
num_of_pvc=self.num_of_pvcs,
wait_each=False
)
# Set volume mode on PVC objects
for pvc_obj in pvc_objs:
pvc_info = pvc_obj.get()
setattr(pvc_obj, 'volume_mode', pvc_info['spec']['volumeMode'])
rwo_pvcs = [pvc_obj for pvc_obj in pvc_objs if (
pvc_obj.access_mode == constants.ACCESS_MODE_RWO
)]
rwx_pvcs = [pvc_obj for pvc_obj in pvc_objs if (
pvc_obj.access_mode == constants.ACCESS_MODE_RWX
)]
num_of_rwo_pvc = len(rwo_pvcs)
num_of_rwx_pvc = len(rwx_pvcs)
block_rwo_pvcs = []
for pvc_obj in rwo_pvcs[:]:
if pvc_obj.volume_mode == 'Block':
block_rwo_pvcs.append(pvc_obj)
rwo_pvcs.remove(pvc_obj)
log.info(
f"Created {num_of_rwo_pvc} RWO PVCs in which "
f"{len(block_rwo_pvcs)} are rbd block type."
)
log.info(f"Created {num_of_rwx_pvc} RWX PVCs.")
# Select 6 PVCs for IO pods
if block_rwo_pvcs:
pvc_objs_for_io_pods = rwo_pvcs[0:2] + rwx_pvcs[0:2] + block_rwo_pvcs[0:2]
pvc_objs_new_pods = rwo_pvcs[2:] + rwx_pvcs[2:] + block_rwo_pvcs[2:]
else:
pvc_objs_for_io_pods = rwo_pvcs[0:3] + rwx_pvcs[0:3]
pvc_objs_new_pods = rwo_pvcs[3:] + rwx_pvcs[3:]
# Create one pod using each RWO PVC and two pods using each RWX PVC
# for running IO
io_pods = helpers.create_pods(
pvc_objs_for_io_pods, pod_factory, interface, 2
)
# Wait for pods to be in Running state
for pod_obj in io_pods:
helpers.wait_for_resource_state(
resource=pod_obj, state=constants.STATUS_RUNNING
)
pod_obj.reload()
log.info(f"Created {len(io_pods)} pods for running IO.")
return pvc_objs, io_pods, pvc_objs_new_pods, access_modes
def test_daemon_kill_during_pvc_pod_creation_and_io(
self, interface, resource_name, setup, multi_pvc_factory,
pod_factory
):
"""
Kill 'resource_name' daemon while PVCs creation, pods
creation and IO operation are progressing.
"""
num_of_new_pvcs = 5
pvc_objs, io_pods, pvc_objs_new_pods, access_modes = setup
proj_obj = pvc_objs[0].project
storageclass = pvc_objs[0].storageclass
pod_functions = {
'mds': partial(get_mds_pods), 'mon': partial(get_mon_pods),
'mgr': partial(get_mgr_pods), 'osd': partial(get_osd_pods),
'rbdplugin': partial(get_plugin_pods, interface=interface),
'cephfsplugin': partial(get_plugin_pods, interface=interface),
'cephfsplugin_provisioner': partial(get_cephfsplugin_provisioner_pods),
'rbdplugin_provisioner': partial(get_rbdfsplugin_provisioner_pods),
'operator': partial(get_operator_pods)
}
executor = ThreadPoolExecutor(max_workers=len(io_pods))
disruption = disruption_helpers.Disruptions()
disruption.set_resource(resource=resource_name)
# Get number of pods of type 'resource_name'
resource_pods_num = len(pod_functions[resource_name]())
# Do setup for running IO on pods
log.info("Setting up pods for running IO")
for pod_obj in io_pods:
if pod_obj.pvc.volume_mode == 'Block':
storage_type = 'block'
else:
storage_type = 'fs'
executor.submit(pod_obj.workload_setup, storage_type=storage_type)
# Wait for setup on pods to complete
for pod_obj in io_pods:
log.info(f"Waiting for IO setup to complete on pod {pod_obj.name}")
for sample in TimeoutSampler(
180, 2, getattr, pod_obj, 'wl_setup_done'
):
if sample:
log.info(
f"Setup for running IO is completed on pod "
f"{pod_obj.name}."
)
break
log.info("Setup for running IO is completed on pods")
# Set daemon to be killed
disruption.select_daemon()
# Start creating new pods
log.info("Start creating new pods.")
bulk_pod_create = executor.submit(
helpers.create_pods, pvc_objs_new_pods, pod_factory, interface, 2
)
# Start creation of new PVCs
log.info("Start creating new PVCs.")
bulk_pvc_create = executor.submit(
multi_pvc_factory, interface=interface,
project=proj_obj, storageclass=storageclass, size=self.pvc_size,
access_modes=access_modes,
access_modes_selection='distribute_random',
status="", num_of_pvc=num_of_new_pvcs, wait_each=False
)
# Start IO on each pod
log.info("Start IO on pods")
for pod_obj in io_pods:
if pod_obj.pvc.volume_mode == 'Block':
storage_type = 'block'
else:
storage_type = 'fs'
pod_obj.run_io(
storage_type=storage_type, size='1G', runtime=10,
fio_filename=f'{pod_obj.name}_io_file1'
)
log.info("IO started on all pods.")
# Kill daemon
disruption.kill_daemon()
# Getting result of PVC creation as list of PVC objects
pvc_objs_new = bulk_pvc_create.result()
# Confirm PVCs are Bound
for pvc_obj in pvc_objs_new:
helpers.wait_for_resource_state(
resource=pvc_obj, state=constants.STATUS_BOUND, timeout=180
)
pvc_obj.reload()
log.info("Verified: New PVCs are Bound.")
# Getting result of pods creation as list of Pod objects
pod_objs_new = bulk_pod_create.result()
# Verify new pods are Running
for pod_obj in pod_objs_new:
helpers.wait_for_resource_state(
resource=pod_obj, state=constants.STATUS_RUNNING
)
pod_obj.reload()
log.info("Verified: All new pods are Running.")
# Verify IO
log.info("Fetching IO results from IO pods.")
for pod_obj in io_pods:
fio_result = pod_obj.get_fio_results()
err_count = fio_result.get('jobs')[0].get('error')
assert err_count == 0, (
f"FIO error on pod {pod_obj.name}. FIO result: {fio_result}"
)
log.info(f"IOPs after FIO on pod {pod_obj.name}:")
log.info(
f"Read: {fio_result.get('jobs')[0].get('read').get('iops')}"
)
log.info(
f"Write: {fio_result.get('jobs')[0].get('write').get('iops')}"
)
log.info("Verified IO result on IO pods.")
all_pod_objs = io_pods + pod_objs_new
# Fetch volume details from pods for the purpose of verification
node_pv_dict = {}
for pod in all_pod_objs:
pod_info = pod.get()
node = pod_info['spec']['nodeName']
pvc = pod_info['spec']['volumes'][0]['persistentVolumeClaim']['claimName']
for pvc_obj in pvc_objs:
if pvc_obj.name == pvc:
pvc_obj.reload()
pv = pvc_obj.backed_pv
break
if node in node_pv_dict:
node_pv_dict[node].append(pv)
else:
node_pv_dict[node] = [pv]
# Delete pods
for pod_obj in all_pod_objs:
pod_obj.delete(wait=False)
# Verify pods are deleted
for pod_obj in all_pod_objs:
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
# Verify number of 'resource_name' type pods
final_resource_pods_num = len(pod_functions[resource_name]())
assert final_resource_pods_num == resource_pods_num, (
f"Total number of {resource_name} pods is not matching with "
f"initial value. Total number of pods before daemon kill: "
f"{resource_pods_num}. Total number of pods present now: "
f"{final_resource_pods_num}"
)
# Verify volumes are unmapped from nodes after deleting the pods
node_pv_mounted = helpers.verify_pv_mounted_on_node(node_pv_dict)
for node, pvs in node_pv_mounted.items():
assert not pvs, (
f"PVs {pvs} is still present on node {node} after "
f"deleting the pods."
)
log.info(
"Verified: mount points are removed from nodes after deleting "
"the pods"
)
# Set volume mode on PVC objects
for pvc_obj in pvc_objs_new:
pvc_info = pvc_obj.get()
setattr(pvc_obj, 'volume_mode', pvc_info['spec']['volumeMode'])
# Verify that PVCs are reusable by creating new pods
all_pvc_objs = pvc_objs + pvc_objs_new
pod_objs_re = helpers.create_pods(
all_pvc_objs, pod_factory, interface, 2
)
# Verify pods are Running
for pod_obj in pod_objs_re:
helpers.wait_for_resource_state(
resource=pod_obj, state=constants.STATUS_RUNNING
)
pod_obj.reload()
log.info("Successfully created new pods using all PVCs.")
# Run IO on each of the newly created pods
for pod_obj in pod_objs_re:
if pod_obj.pvc.volume_mode == 'Block':
storage_type = 'block'
else:
storage_type = 'fs'
pod_obj.run_io(
storage_type=storage_type, size='1G', runtime=10,
fio_filename=f'{pod_obj.name}_io_file2'
)
log.info("Fetching IO results from newly created pods")
for pod_obj in pod_objs_re:
fio_result = pod_obj.get_fio_results()
err_count = fio_result.get('jobs')[0].get('error')
assert err_count == 0, (
f"FIO error on pod {pod_obj.name}. FIO result: {fio_result}"
)
log.info(f"IOPs after FIO on pod {pod_obj.name}:")
log.info(
f"Read: {fio_result.get('jobs')[0].get('read').get('iops')}"
)
log.info(
f"Write: {fio_result.get('jobs')[0].get('write').get('iops')}"
)
log.info("Verified IO result on newly created pods.")
def operations_base(self, resource_to_delete):
"""
Delete resource 'resource_to_delete' while PVCs creation, Pods
creation and IO operation are progressing.
Verifies PVCs can be re-used by creating new pods.
Steps:
1. Create pods for running IO and verify they are Running.
2. Start creating more pods.
3. Start creating new PVCs.
4. Start IO on pods created in Step 1.
5. Delete the resource 'resource_to_delete'.
6. Verify that PVCs created in Step 3 are in Bound state.
7. Verify that pods created in Step 2 are Running.
8. Verify IO results.
9. Delete pods created in Steps 1 and 2.
10. Verify the total number of 'resource_to_delete' pods.
11. Verify volumes are unmapped from nodes after deleting pods.
12. Use all PVCs to create new pods. One PVC for one pod.
13. Start IO on all pods created in Step 10.
14. Verify IO results.
"""
# Separate the available PVCs
pvc_objs_for_io_pods = self.pvc_objs[0:self.pvc_num_for_io_pods]
pvc_objs_new_pods = self.pvc_objs[self.pvc_num_for_io_pods:]
pod_functions = {
'mds': get_mds_pods,
'mon': get_mon_pods,
'mgr': get_mgr_pods,
'osd': get_osd_pods
}
executor = ThreadPoolExecutor(max_workers=2)
disruption = disruption_helpers.Disruptions()
disruption.set_resource(resource=resource_to_delete)
# Get number of pods
initial_pods_num = len(pod_functions[resource_to_delete]())
# Create pods for running IO
io_pods = helpers.create_pods(pvc_objs_list=pvc_objs_for_io_pods,
interface_type=self.interface,
desired_status=constants.STATUS_RUNNING,
wait=True,
namespace=self.namespace)
# Updating self.pod_objs for the purpose of teardown
self.pod_objs.extend(io_pods)
# Do setup for running IO on pods
log.info("Setting up pods for running IO")
for pod_obj in io_pods:
pod_obj.workload_setup(storage_type='fs')
log.info("Setup for running IO is completed on pods")
# Start creating new pods
log.info("Start creating new pods.")
bulk_pod_create = executor.submit(helpers.create_pods,
pvc_objs_list=pvc_objs_new_pods,
interface_type=self.interface,
wait=False,
namespace=self.namespace)
# Start creation of new PVCs
log.info("Start creating new PVCs.")
bulk_pvc_create = executor.submit(helpers.create_multiple_pvcs,
sc_name=self.sc_obj.name,
namespace=self.namespace,
number_of_pvc=self.num_of_new_pvcs,
size=self.pvc_size,
wait=False)
# Start IO on each pod
log.info("Start IO on pods")
for pod_obj in io_pods:
pod_obj.run_io(storage_type='fs', size=f'{self.pvc_size_int - 1}G')
log.info("IO started on all pods.")
# Delete the resource
disruption.delete_resource()
# Getting result of PVC creation as list of PVC objects
pvc_objs_new = bulk_pvc_create.result()
# Updating self.pvc_objs_new for the purpose of teardown
self.pvc_objs_new.extend(pvc_objs_new)
# Verify PVCs are Bound
for pvc_obj in pvc_objs_new:
assert pvc_obj.ocp.wait_for_resource(
condition=constants.STATUS_BOUND,
resource_name=pvc_obj.name,
timeout=240,
sleep=10
), (f"Wait timeout: PVC {pvc_obj.name} is not in 'Bound' status")
log.info("Verified: New PVCs are Bound.")
# Getting result of pods creation as list of Pod objects
pod_objs_new = bulk_pod_create.result()
# Updating self.pod_objs for the purpose of teardown
self.pod_objs.extend(pod_objs_new)
# Verify new pods are Running
for pod_obj in pod_objs_new:
assert pod_obj.ocp.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_name=pod_obj.name,
timeout=240,
sleep=10), (
f"Wait timeout: Pod {pod_obj.name} is not in 'Running' "
f"state even after 120 seconds.")
log.info("Verified: All pods are Running.")
# Verify IO
log.info("Fetching IO results.")
for pod_obj in io_pods:
get_fio_rw_iops(pod_obj)
log.info("Verified IO result on pods.")
all_pod_objs = io_pods + pod_objs_new
# Fetch volume details from pods for the purpose of verification
node_pv_dict = {}
for pod in all_pod_objs:
pod_info = pod.get()
node = pod_info['spec']['nodeName']
pvc = pod_info['spec']['volumes'][0]['persistentVolumeClaim'][
'claimName']
for pvc_obj in self.pvc_objs:
if pvc_obj.name == pvc:
pvc_obj.reload()
pv = pvc_obj.backed_pv
break
if node in node_pv_dict:
node_pv_dict[node].append(pv)
else:
node_pv_dict[node] = [pv]
# Delete pods
for pod_obj in all_pod_objs:
pod_obj.delete(wait=False)
# Verify pods are deleted
for pod_obj in all_pod_objs:
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
# Updating self.pod_objs for the purpose of teardown
self.pod_objs.clear()
# Verify number of 'resource_to_delete' type pods
final_pods_num = len(pod_functions[resource_to_delete]())
assert final_pods_num == initial_pods_num, (
f"Total number of {resource_to_delete} pods is not matching with "
f"initial value. Total number of pods before deleting a pod: "
f"{initial_pods_num}. Total number of pods present now: "
f"{final_pods_num}")
# Verify volumes are unmapped from nodes after deleting the pods
for node, pvs in node_pv_dict.items():
cmd = f'oc debug nodes/{node} -- df'
df_on_node = run_cmd(cmd)
for pv in pvs:
assert pv not in df_on_node, (
f"{pv} is still present on node {node} after "
f"deleting the pods.")
# Verify that PVCs are reusable by creating new pods
all_pvc_objs = self.pvc_objs + pvc_objs_new
pod_objs_re = helpers.create_pods(
pvc_objs_list=all_pvc_objs,
interface_type=self.interface,
desired_status=constants.STATUS_RUNNING,
wait=True,
namespace=self.namespace)
log.info("Successfully created new pods using all PVCs.")
# Updating self.pod_objs for the purpose of teardown
self.pod_objs.extend(pod_objs_re)
# Run IO on each of the newly created pods
for pod_obj in pod_objs_re:
pod_obj.run_io(storage_type='fs',
size='100M',
runtime=10,
fio_filename='fio-file-retest')
log.info("Fetching IO results from newly created pods")
for pod_obj in pod_objs_re:
get_fio_rw_iops(pod_obj)
log.info("Verified IO result on newly created pods.")
def test_create_delete_pvcs(multi_pvc_factory, pod_factory, project=None):
# create the pods for deleting
# Create rbd pvcs for pods
pvc_objs_rbd = create_pvcs(multi_pvc_factory,
'CephBlockPool',
project=project)
storageclass_rbd = pvc_objs_rbd[0].storageclass
# Create cephfs pvcs for pods
pvc_objs_cephfs = create_pvcs(multi_pvc_factory,
'CephFileSystem',
project=project)
storageclass_cephfs = pvc_objs_cephfs[0].storageclass
all_pvc_for_pods = pvc_objs_rbd + pvc_objs_cephfs
# Check pvc status
for pvc_obj in all_pvc_for_pods:
helpers.wait_for_resource_state(
resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=1200 # Timeout given 5 minutes
)
pvc_info = pvc_obj.get()
setattr(pvc_obj, 'volume_mode', pvc_info['spec']['volumeMode'])
# Create pods
rbd_pods_to_delete = helpers.create_pods(pvc_objs_rbd, pod_factory,
constants.RBD_INTERFACE)
cephfs_pods_to_delete = helpers.create_pods(pvc_objs_cephfs, pod_factory,
constants.CEPHFS_INTERFACE)
pods_to_delete = rbd_pods_to_delete + cephfs_pods_to_delete
for pod_obj in pods_to_delete:
helpers.wait_for_resource_state(
resource=pod_obj,
state=constants.STATUS_RUNNING,
timeout=300 # Timeout given 5 minutes
)
logging.info(
f"#### Created the pods for deletion later...pods = {pods_to_delete}")
# Create PVCs for deleting
# Create rbd pvcs for deleting
pvc_objs_rbd = create_pvcs(multi_pvc_factory=multi_pvc_factory,
interface='CephBlockPool',
project=project,
status="",
storageclass=storageclass_rbd)
# Create cephfs pvcs for deleting
pvc_objs_cephfs = create_pvcs(multi_pvc_factory=multi_pvc_factory,
interface='CephFileSystem',
project=project,
status="",
storageclass=storageclass_cephfs)
all_pvc_to_delete = pvc_objs_rbd + pvc_objs_cephfs
# Check pvc status
for pvc_obj in all_pvc_to_delete:
helpers.wait_for_resource_state(
resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=300 # Timeout given 5 minutes
)
logging.info(
f"#### Created the PVCs for deletion later...PVCs={all_pvc_to_delete}")
# Create PVCs for new pods
pvc_objs_rbd = create_pvcs(multi_pvc_factory=multi_pvc_factory,
interface='CephBlockPool',
project=project,
status="",
storageclass=storageclass_rbd)
# Create cephfs pvcs for new pods # for deleting
pvc_objs_cephfs = create_pvcs(multi_pvc_factory=multi_pvc_factory,
interface='CephFileSystem',
project=project,
status="",
storageclass=storageclass_cephfs)
all_pvc_for_new_pods = pvc_objs_rbd + pvc_objs_cephfs
# Check pvc status
for pvc_obj in all_pvc_for_new_pods:
helpers.wait_for_resource_state(
resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=300 # Timeout given 5 minutes
)
pvc_info = pvc_obj.get()
setattr(pvc_obj, 'volume_mode', pvc_info['spec']['volumeMode'])
logging.info(
f"#### Created the PVCs required for creating New Pods...{all_pvc_for_new_pods}"
)
executor = ThreadPoolExecutor(max_workers=10)
# Start creating new PVCs
# Start creating rbd PVCs
rbd_pvc_exeuter = executor.submit(create_pvcs,
multi_pvc_factory=multi_pvc_factory,
interface='CephBlockPool',
project=project,
status="",
storageclass=storageclass_rbd)
logging.info("#### Started creating new RBD PVCs in thread...")
# Start creating cephfs pvc
cephfs_pvc_exeuter = executor.submit(create_pvcs,
multi_pvc_factory=multi_pvc_factory,
interface='CephFileSystem',
project=project,
status="",
storageclass=storageclass_cephfs)
logging.info("#### Started creating new cephfs PVCs in thread...")
# Start creating pods
rbd_pods_create_executer = executor.submit(helpers.create_pods,
pvc_objs_rbd, pod_factory,
constants.RBD_INTERFACE)
cephfs_pods_create_executer = executor.submit(helpers.create_pods,
pvc_objs_cephfs, pod_factory,
constants.CEPHFS_INTERFACE)
# Start deleting pods
pods_delete_executer = executor.submit(delete_pods, pods_to_delete)
logging.info(f"### Started deleting the pods_to_delete = {pods_to_delete}")
# Start deleting PVC
pvc_delete_executer = executor.submit(delete_pvcs, all_pvc_to_delete)
logging.info(
f"### Started deleting the all_pvc_to_delete = {all_pvc_to_delete}")
log.info(
"These process are started: Bulk delete PVC, Pods. Bulk create PVC, "
"Pods. Waiting for its completion")
while not (rbd_pvc_exeuter.done() and cephfs_pvc_exeuter.done()
and rbd_pods_create_executer.done()
and cephfs_pods_create_executer.done()
and pods_delete_executer.done() and pvc_delete_executer.done()):
sleep(10)
logging.info(
"#### create_delete_pvcs....Waiting for threads to complete...")
new_rbd_pvcs = rbd_pvc_exeuter.result()
new_cephfs_pvcs = cephfs_pvc_exeuter.result()
new_pods = cephfs_pods_create_executer.result(
) + rbd_pods_create_executer.result()
# Check pvc status
for pvc_obj in new_rbd_pvcs + new_cephfs_pvcs:
helpers.wait_for_resource_state(
resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=300 # Timeout given 5 minutes
)
log.info("All new PVCs are bound")
# Check pods status
for pod_obj in new_pods:
helpers.wait_for_resource_state(
resource=pod_obj,
state=constants.STATUS_RUNNING,
timeout=300 # Timeout given 5 minutes
)
log.info("All new pods are running")
# Check pods are deleted
for pod_obj in pods_to_delete:
pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name)
log.info("All pods are deleted as expected.")
# Check PVCs are deleted
for pvc_obj in all_pvc_to_delete:
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name)
log.info("All PVCs are deleted as expected")
