def test_fio_with_block_storage(self):
name = 'test_workload'
spec = self.pod_obj.data.get('spec')
path = (
spec.get('containers')[0].get('volumeMounts')[0].get('mountPath'))
work_load = 'fio'
storage_type = 'fs'
# few io parameters for Fio
runtime = 10
size = '200M'
wl = workload.WorkLoad(name, path, work_load, storage_type,
self.pod_obj)
assert wl.setup()
io_params = templating.load_yaml(constants.FIO_IO_PARAMS_YAML)
io_params['runtime'] = runtime
io_params['size'] = size
future_result = wl.run(**io_params)
timeout = 1200
sample = TimeoutSampler(timeout=timeout,
sleep=3,
func=future_result.done)
assert sample.wait_for_func_status(result=True)
try:
logger.info(future_result.result())
except exceptions.CommandFailed:
logger.exception(f"FIO failed")
raise
except Exception:
logger.exception(f"Found Exception")
raise
def test_fio_with_block_storage(self):
name = "test_workload"
spec = self.pod_obj.data.get("spec")
path = spec.get("containers")[0].get("volumeMounts")[0].get(
"mountPath")
work_load = "fio"
storage_type = "fs"
# few io parameters for Fio
runtime = 10
size = "200M"
wl = workload.WorkLoad(name, path, work_load, storage_type,
self.pod_obj)
assert wl.setup()
io_params = templating.load_yaml(constants.FIO_IO_PARAMS_YAML)
io_params["runtime"] = runtime
io_params["size"] = size
future_result = wl.run(**io_params)
timeout = 1200
sample = TimeoutSampler(timeout=timeout,
sleep=3,
func=future_result.done)
assert sample.wait_for_func_status(result=True)
try:
logger.info(future_result.result())
except exceptions.CommandFailed:
logger.exception("FIO failed")
raise
except Exception:
logger.exception("Found Exception")
raise
def run_git_clone(self):
"""
Execute git clone on a pod to simulate a Jenkins user
"""
name = 'test_workload'
work_load = 'jenkins'
wl = workload.WorkLoad(name=name, work_load=work_load, pod=self)
assert wl.setup(), "Setup up for git failed"
wl.run()
def run_io(self,
storage_type,
size,
io_direction='rw',
rw_ratio=75,
jobs=1,
runtime=60,
depth=4,
fio_filename=None):
"""
Execute FIO on a pod
This operation will run in background and will store the results in
'self.thread.result()'.
In order to wait for the output and not continue with the test until
FIO is done, call self.thread.result() right after calling run_io.
See tests/manage/test_pvc_deletion_during_io.py::test_run_io
for usage of FIO
Args:
storage_type (str): 'fs' or 'block'
size (str): Size in MB, e.g. '200M'
io_direction (str): Determines the operation:
'ro', 'wo', 'rw' (default: 'rw')
rw_ratio (int): Determines the reads and writes using a
<rw_ratio>%/100-<rw_ratio>%
(e.g. the default is 75 which means it is 75%/25% which
equivalent to 3 reads are performed for every 1 write)
jobs (int): Number of jobs to execute FIO
runtime (int): Number of seconds IO should run for
depth (int): IO depth
fio_filename(str): Name of fio file created on app pod's mount point
"""
name = 'test_workload'
spec = self.pod_data.get('spec')
path = (
spec.get('containers')[0].get('volumeMounts')[0].get('mountPath'))
work_load = 'fio'
# few io parameters for Fio
wl = workload.WorkLoad(name, path, work_load, storage_type, self, jobs)
assert wl.setup(), "Setup up for FIO failed"
if io_direction == 'rw':
io_params = templating.load_yaml_to_dict(
constants.FIO_IO_RW_PARAMS_YAML)
io_params['rwmixread'] = rw_ratio
else:
io_params = templating.load_yaml_to_dict(
constants.FIO_IO_PARAMS_YAML)
io_params['runtime'] = runtime
io_params['size'] = size
if fio_filename:
io_params['filename'] = fio_filename
io_params['iodepth'] = depth
self.fio_thread = wl.run(**io_params)
def workload_setup(self, storage_type, jobs=1):
"""
Do setup on pod for running FIO
Args:
storage_type (str): 'fs' or 'block'
jobs (int): Number of jobs to execute FIO
"""
work_load = 'fio'
name = f'test_workload_{work_load}'
path = self.get_storage_path(storage_type)
# few io parameters for Fio
self.wl_obj = workload.WorkLoad(name, path, work_load, storage_type,
self, jobs)
assert self.wl_obj.setup(), f"Setup for FIO failed on pod {self.name}"
self.wl_setup_done = True
def factory(
num_of_pvcs=100,
pvc_size=2,
bulk=False,
project=None,
measure=True,
delete=True,
file_name=None,
fio_percentage=25,
verify_fio=False,
expand=False,
):
"""
Args:
num_of_pvcs (int) : Number of PVCs / PODs we want to create.
pvc_size (int) : Size of each PVC in GB.
bulk (bool) : True for bulk operations, False otherwise.
project (obj) : Project obj inside which the PODs/PVCs are created.
measure (bool) : True if we want to measure the PVC creation/deletion time and POD to PVC attach time,
False otherwise.
delete (bool) : True if we want to delete PVCs and PODs, False otherwise
file_name (str) : Name of the file on which FIO is performed.
fio_percentage (float) : Percentage of PVC space we want to be utilized for FIO.
verify_fio (bool) : True if we want to verify FIO, False otherwise.
expand (bool) : True if we want to verify_fio for expansion of PVCs operation, False otherwise.
"""
if not project:
project = project_factory("longevity")
pvc_objs = list()
executor = ThreadPoolExecutor(max_workers=1)
start_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
for interface in (constants.CEPHFILESYSTEM, constants.CEPHBLOCKPOOL):
if interface == constants.CEPHFILESYSTEM:
access_modes = [
constants.ACCESS_MODE_RWO, constants.ACCESS_MODE_RWX
]
num_of_pvc = num_of_pvcs // 2
else:
access_modes = [
constants.ACCESS_MODE_RWO,
constants.ACCESS_MODE_RWO + "-" +
constants.VOLUME_MODE_BLOCK,
constants.ACCESS_MODE_RWX + "-" +
constants.VOLUME_MODE_BLOCK,
]
num_of_pvc = num_of_pvcs - num_of_pvcs // 2
# Create PVCs
if num_of_pvc > 0:
pvc_objs_tmp = multi_pvc_factory(
interface=interface,
size=pvc_size,
project=project,
access_modes=access_modes,
status=constants.STATUS_BOUND,
num_of_pvc=num_of_pvc,
wait_each=not bulk,
)
log.info("PVC creation was successful.")
pvc_objs.extend(pvc_objs_tmp)
if measure:
# Measure PVC Creation Time
measure_pvc_creation_time(interface, pvc_objs_tmp,
start_time)
else:
log.info(
f"Num of PVCs of interface - {interface} = {num_of_pvc}. So no PVCs created."
)
# PVC and PV Teardown
for pvc_obj in pvc_objs:
teardown_factory(pvc_obj)
teardown_factory(pvc_obj.backed_pv_obj)
# Create PODs
pod_objs = list()
for pvc_obj in pvc_objs:
if pvc_obj.get_pvc_vol_mode == constants.VOLUME_MODE_BLOCK:
if not bulk:
pod_objs.append(
pod_factory(
pvc=pvc_obj,
raw_block_pv=True,
status=constants.STATUS_RUNNING,
pod_dict_path=constants.PERF_BLOCK_POD_YAML,
))
else:
pod_objs.append(
pod_factory(
pvc=pvc_obj,
raw_block_pv=True,
pod_dict_path=constants.PERF_BLOCK_POD_YAML,
))
else:
if not bulk:
pod_objs.append(
pod_factory(
pvc=pvc_obj,
status=constants.STATUS_RUNNING,
pod_dict_path=constants.PERF_POD_YAML,
))
else:
pod_objs.append(
pod_factory(pvc=pvc_obj,
pod_dict_path=constants.PERF_POD_YAML))
log.info(f"POD {pod_objs[-1].name} creation was successful.")
log.info("All PODs are created.")
if bulk:
for pod_obj in pod_objs:
executor.submit(
helpers.wait_for_resource_state,
pod_obj,
constants.STATUS_RUNNING,
timeout=300,
)
log.info(f"POD {pod_obj.name} reached Running State.")
log.info("All PODs reached Running State.")
if measure:
# Measure POD to PVC attach time
measure_pod_to_pvc_attach_time(pod_objs)
# POD Teardown
for pod_obj in pod_objs:
teardown_factory(pod_obj)
# Run FIO on PODs
fio_size = int((fio_percentage / 100) * pvc_size * 1000)
for pod_obj in pod_objs:
storage_type = ("block" if pod_obj.pvc.get_pvc_vol_mode
== constants.VOLUME_MODE_BLOCK else "fs")
pod_obj.wl_setup_done = True
pod_obj.wl_obj = workload.WorkLoad(
"test_workload_fio",
pod_obj.get_storage_path(storage_type),
"fio",
storage_type,
pod_obj,
1,
)
if not file_name:
pod_obj.run_io(storage_type, f"{fio_size}M")
else:
pod_obj.run_io(
storage_type=storage_type,
size=f"{fio_size}M",
runtime=20,
fio_filename=file_name,
end_fsync=1,
)
if verify_fio:
log.info(
"Waiting for IO to complete on all pods to utilise 25% of PVC used space"
)
for pod_obj in pod_objs:
# Wait for IO to finish
pod_obj.get_fio_results(3600)
log.info(f"IO finished on pod {pod_obj.name}")
is_block = (True if pod_obj.pvc.get_pvc_vol_mode
== constants.VOLUME_MODE_BLOCK else False)
file_name_pod = (file_name
if not is_block else pod_obj.get_storage_path(
storage_type="block"))
# Verify presence of the file
file_path = (file_name_pod if is_block else pod.get_file_path(
pod_obj, file_name_pod))
log.info(f"Actual file path on the pod {file_path}")
assert pod.check_file_existence(
pod_obj, file_path), f"File {file_name_pod} does not exist"
log.info(f"File {file_name_pod} exists in {pod_obj.name}")
if expand and is_block:
# Read IO from block PVCs using dd and calculate md5sum.
# This dd command reads the data from the device, writes it to
# stdout, and reads md5sum from stdin.
pod_obj.pvc.md5sum = pod_obj.exec_sh_cmd_on_pod(
command=(f"dd iflag=direct if={file_path} bs=10M "
f"count={fio_size // 10} | md5sum"))
log.info(
f"md5sum of {file_name_pod}: {pod_obj.pvc.md5sum}")
else:
# Calculate md5sum of the file
pod_obj.pvc.md5sum = pod.cal_md5sum(pod_obj, file_name_pod)
log.info("POD FIO was successful.")
if delete:
# Delete PODs
pod_delete = executor.submit(delete_pods, pod_objs, wait=not bulk)
pod_delete.result()
log.info("Verified: Pods are deleted.")
# Delete PVCs
pvc_delete = executor.submit(delete_pvcs,
pvc_objs,
concurrent=bulk)
res = pvc_delete.result()
if not res:
raise ex.UnexpectedBehaviour("Deletion of PVCs failed")
log.info("PVC deletion was successful.")
# Validate PV Deletion
for pvc_obj in pvc_objs:
helpers.validate_pv_delete(pvc_obj.backed_pv)
log.info("PV deletion was successful.")
if measure:
# Measure PVC Deletion Time
for interface in (constants.CEPHFILESYSTEM,
constants.CEPHBLOCKPOOL):
if interface == constants.CEPHFILESYSTEM:
measure_pvc_deletion_time(
interface,
pvc_objs[:num_of_pvcs // 2],
)
else:
measure_pvc_deletion_time(
interface,
pvc_objs[num_of_pvcs // 2:],
)
log.info(f"Successfully deleted {num_of_pvcs} PVCs")
else:
return pvc_objs, pod_objs
def expand_verify_pvcs(pvc_objs, pod_objs, pvc_size_new, file_name, fio_size):
"""
Expands size of each PVC in the provided list of PVCs,
Verifies data integrity by checking the existence and md5sum of file in the expanded PVC
and
Runs FIO on expanded PVCs and verifies results.
Args:
pvc_objs (list) : List of PVC objects which are to be expanded.
pod_objs (list) : List of POD objects attached to the PVCs.
pvc_size_new (int) : Size of the expanded PVC in GB.
file_name (str) : Name of the file on which FIO is performed.
fio_size (int) : Size in MB of FIO.
"""
# Expand original PVCs
log.info("Started expansion of the PVCs.")
for pvc_obj in pvc_objs:
log.info(f"Expanding size of PVC {pvc_obj.name} to {pvc_size_new}G")
pvc_obj.resize_pvc(pvc_size_new, True)
log.info("Successfully expanded the PVCs.")
# Verify that the fio exists and md5sum matches
for pod_no in range(len(pod_objs)):
pod_obj = pod_objs[pod_no]
if pod_obj.pvc.get_pvc_vol_mode == constants.VOLUME_MODE_BLOCK:
pod.verify_data_integrity_after_expansion_for_block_pvc(
pod_obj, pvc_objs[pod_no], fio_size)
else:
pod.verify_data_integrity(pod_obj, file_name,
pvc_objs[pod_no].md5sum)
# Run IO to utilize 50% of volume
log.info(
"Run IO on all pods to utilise 50% of the expanded PVC used space")
expanded_file_name = "fio_50"
for pod_obj in pod_objs:
log.info(f"Running IO on pod {pod_obj.name}")
log.info(f"File created during IO {expanded_file_name}")
fio_size = int(0.50 * pvc_size_new * 1000)
storage_type = ("block" if pod_obj.pvc.get_pvc_vol_mode
== constants.VOLUME_MODE_BLOCK else "fs")
pod_obj.wl_setup_done = True
pod_obj.wl_obj = workload.WorkLoad(
"test_workload_fio",
pod_obj.get_storage_path(storage_type),
"fio",
storage_type,
pod_obj,
1,
)
pod_obj.run_io(
storage_type=storage_type,
size=f"{fio_size}M",
runtime=20,
fio_filename=expanded_file_name,
end_fsync=1,
)
log.info("Started IO on all pods to utilise 50% of PVCs")
for pod_obj in pod_objs:
# Wait for IO to finish
pod_obj.get_fio_results(3600)
log.info(f"IO finished on pod {pod_obj.name}")
is_block = (True if pod_obj.pvc.get_pvc_vol_mode
== constants.VOLUME_MODE_BLOCK else False)
expanded_file_name_pod = (expanded_file_name if not is_block else
pod_obj.get_storage_path(
storage_type="block"))
# Verify presence of the file
expanded_file_path = (expanded_file_name_pod
if is_block else pod.get_file_path(
pod_obj, expanded_file_name_pod))
log.info(f"Actual file path on the pod {expanded_file_path}")
assert pod.check_file_existence(
pod_obj, expanded_file_path
), f"File {expanded_file_name_pod} does not exist"
log.info(f"File {expanded_file_name_pod} exists in {pod_obj.name}")
