def measure_pvc_deletion_time(interface, pvc_objs):
"""
Measures and Logs PVC Deletion Time of all PVCs.
Args:
interface (str) : an interface (RBD or CephFS) to run on.
pvc_objs (list) : List of PVC objects for which we have to measure the time.
Logs:
PVC Deletion Time of all the PVCs.
"""
accepted_deletion_time = 30
num_of_pvcs = len(pvc_objs)
pv_name_list = list()
pv_to_pvc = dict()
for pvc_no in range(num_of_pvcs):
pv_name = pvc_objs[pvc_no].backed_pv
pv_name_list.append(pv_name)
pv_to_pvc[pv_name] = pvc_objs[pvc_no].name
pvc_deletion_time = helpers.measure_pv_deletion_time_bulk(
interface=interface, pv_name_list=pv_name_list)
for pv_name, deletion_time in pvc_deletion_time.items():
if deletion_time <= accepted_deletion_time:
log.info(
f"PVC {pv_to_pvc[pv_name]} was deleted in {deletion_time} seconds."
)
else:
log.error(
f"PVC {pv_to_pvc[pv_name]} deletion time is {deletion_time} and is greater than "
f"{accepted_deletion_time} seconds.")
def test_multiple_pvc_deletion_measurement_performance(
self, teardown_factory):
"""
Measuring PVC deletion time of 120 PVCs in 180 seconds
Args:
teardown_factory: A fixture used when we want a new resource that was created during the tests
to be removed in the teardown phase.
Returns:
"""
number_of_pvcs = 120
pvc_size = "1Gi"
msg_prefix = f"Interface: {self.interface}, PVC size: {pvc_size}."
log.info(f"{msg_prefix} Start creating new 120 PVCs")
pvc_objs = helpers.create_multiple_pvcs(
sc_name=self.sc_obj.name,
namespace=defaults.ROOK_CLUSTER_NAMESPACE,
number_of_pvc=number_of_pvcs,
size=pvc_size,
burst=True,
)
for pvc_obj in pvc_objs:
pvc_obj.reload()
teardown_factory(pvc_obj)
with ThreadPoolExecutor(max_workers=5) as executor:
for pvc_obj in pvc_objs:
executor.submit(helpers.wait_for_resource_state, pvc_obj,
constants.STATUS_BOUND)
executor.submit(pvc_obj.reload)
pod_objs = []
for pvc_obj in pvc_objs:
pod_obj = self.write_file_on_pvc(pvc_obj, 0.3)
pod_objs.append(pod_obj)
# Get pvc_name, require pvc_name to fetch deletion time data from log
threads = list()
for pvc_obj in pvc_objs:
process = threading.Thread(target=pvc_obj.reload)
process.start()
threads.append(process)
for process in threads:
process.join()
pvc_name_list, pv_name_list = ([] for i in range(2))
threads = list()
for pvc_obj in pvc_objs:
process1 = threading.Thread(
target=pvc_name_list.append(pvc_obj.name))
process2 = threading.Thread(
target=pv_name_list.append(pvc_obj.backed_pv))
process1.start()
process2.start()
threads.append(process1)
threads.append(process2)
for process in threads:
process.join()
log.info(f"{msg_prefix} Preparing to delete 120 PVC")
# Delete PVC
for pvc_obj, pod_obj in zip(pvc_objs, pod_objs):
pod_obj.delete(wait=True)
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(pvc_obj.name)
# Get PVC deletion time
pvc_deletion_time = helpers.measure_pv_deletion_time_bulk(
interface=self.interface, pv_name_list=pv_name_list)
log.info(
f"{msg_prefix} {number_of_pvcs} bulk deletion time is {pvc_deletion_time}"
)
# accepted deletion time is 2 secs for each PVC
accepted_pvc_deletion_time = number_of_pvcs * 2
for del_time in pvc_deletion_time.values():
if del_time > accepted_pvc_deletion_time:
raise ex.PerformanceException(
f"{msg_prefix} {number_of_pvcs} PVCs deletion time is {pvc_deletion_time.values()} and is "
f"greater than {accepted_pvc_deletion_time} seconds")
logging.info(f"{msg_prefix} {number_of_pvcs} PVCs deletion times are:")
for name, a_time in pvc_deletion_time.items():
logging.info(f"{name} deletion time is: {a_time} seconds")
def test_bulk_pvc_creation_deletion_measurement_performance(
self, teardown_factory, bulk_size):
"""
Measuring PVC creation and deletion time of bulk_size PVCs
and sends results to the Elastic Search DB
Args:
teardown_factory: A fixture used when we want a new resource that was created during the tests
to be removed in the teardown phase.
bulk_size: Size of the bulk to be tested
Returns:
"""
bulk_creation_time_limit = bulk_size / 2
log.info(f"Start creating new {bulk_size} PVCs")
pvc_objs, yaml_creation_dir = helpers.create_multiple_pvcs(
sc_name=self.sc_obj.name,
namespace=self.namespace,
number_of_pvc=bulk_size,
size=self.pvc_size,
burst=True,
)
logging.info(f"PVC creation dir is {yaml_creation_dir}")
for pvc_obj in pvc_objs:
pvc_obj.reload()
teardown_factory(pvc_obj)
with ThreadPoolExecutor(max_workers=5) as executor:
for pvc_obj in pvc_objs:
executor.submit(helpers.wait_for_resource_state, pvc_obj,
constants.STATUS_BOUND)
executor.submit(pvc_obj.reload)
start_time = helpers.get_provision_time(self.interface,
pvc_objs,
status="start")
end_time = helpers.get_provision_time(self.interface,
pvc_objs,
status="end")
total_time = (end_time - start_time).total_seconds()
logging.info(
f"{bulk_size} Bulk PVCs creation time is {total_time} seconds.")
if total_time > bulk_creation_time_limit:
raise ex.PerformanceException(
f"{bulk_size} Bulk PVCs creation time is {total_time} and "
f"greater than {bulk_creation_time_limit} seconds")
pv_names_list = []
for pvc_obj in pvc_objs:
pv_names_list.append(pvc_obj.backed_pv)
logging.info(f"Starting to delete bulk of {bulk_size} PVCs")
helpers.delete_bulk_pvcs(yaml_creation_dir,
pv_names_list,
namespace=self.namespace)
logging.info(
f"Deletion of bulk of {bulk_size} PVCs successfully completed")
log_deletion_times = helpers.measure_pv_deletion_time_bulk(
self.interface, pv_names_list, return_log_times=True)
all_start_times = [
a_tuple[0] for a_tuple in log_deletion_times.values()
]
bulk_start_time = sorted(all_start_times)[0] # the eariles start time
start_deletion_time = datetime.datetime.strptime(
bulk_start_time, helpers.DATE_TIME_FORMAT)
all_end_times = [a_tuple[1] for a_tuple in log_deletion_times.values()]
bulk_deletion_time = sorted(all_end_times)[-1] # the latest end time
end_deletion_time = datetime.datetime.strptime(
bulk_deletion_time, helpers.DATE_TIME_FORMAT)
total_deletion_time = (end_deletion_time -
start_deletion_time).total_seconds()
logging.info(
f"{bulk_size} Bulk PVCs deletion time is {total_deletion_time} seconds."
)
# Produce ES report
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"bulk_creation_deletion_measurement",
))
full_results.add_key("interface", self.interface)
full_results.add_key("bulk_size", bulk_size)
full_results.add_key("pvc_size", self.pvc_size)
full_results.add_key("bulk_pvc_creation_time", total_time)
full_results.add_key("bulk_pvc_deletion_time", total_deletion_time)
# Write the test results into the ES server
full_results.es_write()
def test_multiple_pvc_deletion_measurement_performance(
self, teardown_factory):
"""
Measuring PVC deletion time of 120 PVCs in 180 seconds
Args:
teardown_factory: A fixture used when we want a new resource that was created during the tests
to be removed in the teardown phase.
Returns:
"""
number_of_pvcs = 120
pvc_size = "1Gi"
msg_prefix = f"Interface: {self.interface}, PVC size: {pvc_size}."
log.info(f"{msg_prefix} Start creating new {number_of_pvcs} PVCs")
pvc_objs, _ = helpers.create_multiple_pvcs(
sc_name=self.sc_obj.name,
namespace=self.namespace,
number_of_pvc=number_of_pvcs,
size=pvc_size,
burst=True,
)
for pvc_obj in pvc_objs:
pvc_obj.reload()
teardown_factory(pvc_obj)
timeout = 600 if self.interface == constants.CEPHBLOCKPOOL_THICK else 60
with ThreadPoolExecutor(max_workers=5) as executor:
for pvc_obj in pvc_objs:
executor.submit(
helpers.wait_for_resource_state,
pvc_obj,
constants.STATUS_BOUND,
timeout=timeout,
)
executor.submit(pvc_obj.reload)
pod_objs = []
for pvc_obj in pvc_objs:
pod_obj = self.write_file_on_pvc(pvc_obj, 0.3)
pod_objs.append(pod_obj)
# Get pvc_name, require pvc_name to fetch deletion time data from log
threads = list()
for pvc_obj in pvc_objs:
process = threading.Thread(target=pvc_obj.reload)
process.start()
threads.append(process)
for process in threads:
process.join()
pvc_name_list, pv_name_list = ([] for i in range(2))
threads = list()
for pvc_obj in pvc_objs:
process1 = threading.Thread(
target=pvc_name_list.append(pvc_obj.name))
process2 = threading.Thread(
target=pv_name_list.append(pvc_obj.backed_pv))
process1.start()
process2.start()
threads.append(process1)
threads.append(process2)
for process in threads:
process.join()
log.info(f"{msg_prefix} Preparing to delete 120 PVC")
# Delete PVC
for pvc_obj, pod_obj in zip(pvc_objs, pod_objs):
pod_obj.delete(wait=True)
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(pvc_obj.name)
# Get PVC deletion time
pvc_deletion_time = helpers.measure_pv_deletion_time_bulk(
interface=self.interface, pv_name_list=pv_name_list)
log.info(
f"{msg_prefix} {number_of_pvcs} bulk deletion time is {pvc_deletion_time}"
)
# accepted deletion time is 2 secs for each PVC
accepted_pvc_deletion_time = number_of_pvcs * 2
for del_time in pvc_deletion_time.values():
if del_time > accepted_pvc_deletion_time:
raise ex.PerformanceException(
f"{msg_prefix} {number_of_pvcs} PVCs deletion time is {pvc_deletion_time.values()} and is "
f"greater than {accepted_pvc_deletion_time} seconds")
logging.info(f"{msg_prefix} {number_of_pvcs} PVCs deletion times are:")
for name, a_time in pvc_deletion_time.items():
logging.info(f"{name} deletion time is: {a_time} seconds")
if self.interface == constants.CEPHBLOCKPOOL:
self.sc = "RBD"
elif self.interface == constants.CEPHFILESYSTEM:
self.sc = "CephFS"
elif self.interface == constants.CEPHBLOCKPOOL_THICK:
self.sc = "RBD-Thick"
full_log_path = get_full_test_logs_path(
cname=self) + f"-{self.sc}-{pvc_size}"
self.results_path = get_full_test_logs_path(cname=self)
log.info(f"Logs file path name is : {full_log_path}")
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
full_log_path,
"pvc_bulk_deletion_fullres",
))
full_results.add_key("interface", self.interface)
full_results.add_key("bulk_size", number_of_pvcs)
full_results.add_key("pvc_size", pvc_size)
full_results.all_results["bulk_deletion_time"] = pvc_deletion_time
if full_results.es_write():
res_link = full_results.results_link()
log.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtest (3 - according to the parameters)
self.write_result_to_file(res_link)
def test_multiple_pvc_creation_deletion_scale(self, namespace, tmp_path,
access_mode, interface):
"""
Measuring PVC creation time while scaling PVC
Measure PVC deletion time after creation test
"""
scale_pvc_count = scale_lib.get_max_pvc_count()
log.info(
f"Start creating {access_mode}-{interface} {scale_pvc_count} PVC")
if interface == constants.CEPHBLOCKPOOL:
sc_name = constants.DEFAULT_STORAGECLASS_RBD
elif interface == constants.CEPHFS_INTERFACE:
sc_name = constants.DEFAULT_STORAGECLASS_CEPHFS
# Get pvc_dict_list, append all the pvc.yaml dict to pvc_dict_list
pvc_dict_list1 = scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job(
no_of_pvc=int(scale_pvc_count / 2),
access_mode=access_mode,
sc_name=sc_name)
pvc_dict_list2 = scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job(
no_of_pvc=int(scale_pvc_count / 2),
access_mode=access_mode,
sc_name=sc_name)
# There is 2 kube_job to reduce the load, observed time_out problems
# during delete process of single kube_job and heavy load.
job_file1 = ObjectConfFile(
name="job_profile_1",
obj_dict_list=pvc_dict_list1,
project=self.namespace,
tmp_path=tmp_path,
)
job_file2 = ObjectConfFile(
name="job_profile_2",
obj_dict_list=pvc_dict_list2,
project=self.namespace,
tmp_path=tmp_path,
)
# Create kube_job
job_file1.create(namespace=self.namespace)
job_file2.create(namespace=self.namespace)
# Check all the PVC reached Bound state
pvc_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_file1,
namespace=self.namespace,
no_of_pvc=int(scale_pvc_count / 2),
)
pvc_bound_list.extend(
scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_file2,
namespace=self.namespace,
no_of_pvc=int(scale_pvc_count / 2),
))
log.info(f"Number of PVCs in Bound state {len(pvc_bound_list)}")
# Get PVC creation time
pvc_create_time = helpers.measure_pvc_creation_time_bulk(
interface=interface,
pvc_name_list=pvc_bound_list,
wait_time=300,
)
# TODO: Update below code with google API, to record value in spreadsheet
# TODO: For now observing Google API limit to write more than 100 writes
log_path = f"{ocsci_log_path()}/{interface}-{access_mode}"
with open(f"{log_path}-creation-time.csv", "w") as fd:
csv_obj = csv.writer(fd)
for k, v in pvc_create_time.items():
csv_obj.writerow([k, v])
log.info(f"Create data present in {log_path}-creation-time.csv file")
# Get pv_name, require pv_name to fetch deletion time data from log
pv_name_list = list()
get_kube_job_1 = job_file1.get(namespace=self.namespace)
for i in range(int(scale_pvc_count / 2)):
pv_name_list.append(
get_kube_job_1["items"][i]["spec"]["volumeName"])
get_kube_job_2 = job_file2.get(namespace=self.namespace)
for i in range(int(scale_pvc_count / 2)):
pv_name_list.append(
get_kube_job_2["items"][i]["spec"]["volumeName"])
# Delete kube_job
job_file1.delete(namespace=self.namespace)
job_file2.delete(namespace=self.namespace)
# Adding 1min wait time for PVC deletion logs to be updated
# Observed failure when we immediately check the logs for pvc delete time
# https://github.com/red-hat-storage/ocs-ci/issues/3371
time.sleep(60)
# Get PVC deletion time
pvc_deletion_time = helpers.measure_pv_deletion_time_bulk(
interface=interface, pv_name_list=pv_name_list)
# Update result to csv file.
# TODO: Update below code with google API, to record value in spreadsheet
# TODO: For now observing Google API limit to write more than 100 writes
with open(f"{log_path}-deletion-time.csv", "w") as fd:
csv_obj = csv.writer(fd)
for k, v in pvc_deletion_time.items():
csv_obj.writerow([k, v])
log.info(f"Delete data present in {log_path}-deletion-time.csv file")
end_time = default_timer()
log.info(f"Elapsed time -- {end_time - self.start_time} seconds")
def test_all_4_type_pvc_creation_deletion_scale(self, namespace, tmp_path):
"""
Measuring PVC creation time while scaling PVC of all 4 types,
A total of 500 times the number of worker nodes
will be created, i.e. 375 each pvc type
Measure PVC deletion time in scale env
"""
scale_pvc_count = scale_lib.get_max_pvc_count()
log.info(f"Start creating {scale_pvc_count} PVC of all 4 types")
cephfs_sc_obj = constants.DEFAULT_STORAGECLASS_CEPHFS
rbd_sc_obj = constants.DEFAULT_STORAGECLASS_RBD
# Get pvc_dict_list, append all the pvc.yaml dict to pvc_dict_list
rbd_pvc_dict_list, cephfs_pvc_dict_list = ([] for i in range(2))
for mode in [constants.ACCESS_MODE_RWO, constants.ACCESS_MODE_RWX]:
rbd_pvc_dict_list.extend(
scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job(
no_of_pvc=int(scale_pvc_count / 4),
access_mode=mode,
sc_name=rbd_sc_obj,
))
cephfs_pvc_dict_list.extend(
scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job(
no_of_pvc=int(scale_pvc_count / 4),
access_mode=mode,
sc_name=cephfs_sc_obj,
))
# There is 2 kube_job for cephfs and rbd PVCs
job_file_rbd = ObjectConfFile(
name="rbd_pvc_job",
obj_dict_list=rbd_pvc_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
job_file_cephfs = ObjectConfFile(
name="cephfs_pvc_job",
obj_dict_list=cephfs_pvc_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
# Create kube_job
job_file_rbd.create(namespace=self.namespace)
job_file_cephfs.create(namespace=self.namespace)
# Check all the PVC reached Bound state
rbd_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_file_rbd,
namespace=self.namespace,
no_of_pvc=int(scale_pvc_count / 2),
)
fs_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_file_cephfs,
namespace=self.namespace,
no_of_pvc=int(scale_pvc_count / 2),
)
# Get pvc objs from namespace, which is used to identify backend pv
rbd_pvc_obj, cephfs_pvc_obj = ([] for i in range(2))
pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace)
for pvc_obj in pvc_objs:
if pvc_obj.backed_sc == constants.DEFAULT_STORAGECLASS_RBD:
rbd_pvc_obj.append(pvc_obj)
elif pvc_obj.backed_sc == constants.DEFAULT_STORAGECLASS_CEPHFS:
cephfs_pvc_obj.append(pvc_obj)
# Get PVC creation time
fs_pvc_create_time = helpers.measure_pvc_creation_time_bulk(
interface=constants.CEPHFS_INTERFACE, pvc_name_list=fs_pvc_name)
rbd_pvc_create_time = helpers.measure_pvc_creation_time_bulk(
interface=constants.CEPHBLOCKPOOL, pvc_name_list=rbd_pvc_name)
fs_pvc_create_time.update(rbd_pvc_create_time)
# TODO: Update below code with google API, to record value in spreadsheet
# TODO: For now observing Google API limit to write more than 100 writes
log_path = f"{ocsci_log_path()}/All-type-PVC"
with open(f"{log_path}-creation-time.csv", "w") as fd:
csv_obj = csv.writer(fd)
for k, v in fs_pvc_create_time.items():
csv_obj.writerow([k, v])
log.info(f"Create data present in {log_path}-creation-time.csv file")
# Get pv_name, require pv_name to fetch deletion time data from log
rbd_pv_list, fs_pv_list = ([] for i in range(2))
get_rbd_kube_job = job_file_rbd.get(namespace=self.namespace)
for i in range(int(scale_pvc_count / 2)):
rbd_pv_list.append(
get_rbd_kube_job["items"][i]["spec"]["volumeName"])
get_fs_kube_job = job_file_cephfs.get(namespace=self.namespace)
for i in range(int(scale_pvc_count / 2)):
fs_pv_list.append(
get_fs_kube_job["items"][i]["spec"]["volumeName"])
# Delete kube_job
job_file_rbd.delete(namespace=self.namespace)
job_file_cephfs.delete(namespace=self.namespace)
# Adding 1min wait time for PVC deletion logs to be updated
# Observed failure when we immediately check the logs for pvc delete time
# https://github.com/red-hat-storage/ocs-ci/issues/3371
time.sleep(60)
# Get PV deletion time
fs_pvc_deletion_time = helpers.measure_pv_deletion_time_bulk(
interface=constants.CEPHFS_INTERFACE, pv_name_list=fs_pv_list)
rbd_pvc_deletion_time = helpers.measure_pv_deletion_time_bulk(
interface=constants.CEPHBLOCKPOOL, pv_name_list=rbd_pv_list)
fs_pvc_deletion_time.update(rbd_pvc_deletion_time)
# TODO: Update below code with google API, to record value in spreadsheet
# TODO: For now observing Google API limit to write more than 100 writes
with open(f"{log_path}-deletion-time.csv", "w") as fd:
csv_obj = csv.writer(fd)
for k, v in fs_pvc_deletion_time.items():
csv_obj.writerow([k, v])
log.info(f"Delete data present in {log_path}-deletion-time.csv file")
end_time = default_timer()
log.info(f"Elapsed time -- {end_time - self.start_time} seconds")
def test_bulk_pvc_creation_deletion_measurement_performance(
self, storageclass_factory, interface_type, bulk_size
):
"""
Measuring PVC creation and deletion time of bulk_size PVCs
and sends results to the Elastic Search DB
Args:
bulk_size: Size of the bulk to be tested
Returns:
"""
self.interface = interface_type
self.sc_obj = storageclass_factory(self.interface)
bulk_creation_time_limit = bulk_size / 2
log.info(f"Start creating new {bulk_size} PVCs")
# Getting the start time of the test.
self.test_start_time = self.get_time()
# Run the Bulk Creation test
csi_bulk_start_time = self.get_time(time_format="csi")
self.pvc_bulk_create_and_wait_for_bound(bulk_size)
log.info(f"PVC creation dir is {self.yaml_creation_dir}")
total_time = self.get_bulk_creation_time()
log.info(f"{bulk_size} Bulk PVCs creation time is {total_time} seconds.")
csi_creation_times = performance_lib.csi_bulk_pvc_time_measure(
self.interface, self.pvc_objs, "create", csi_bulk_start_time
)
if total_time > bulk_creation_time_limit:
raise ex.PerformanceException(
f"{bulk_size} Bulk PVCs creation time is {total_time} and "
f"greater than {bulk_creation_time_limit} seconds"
)
# Run the Bulk Deletion test
pv_names_list = []
for pvc_obj in self.pvc_objs:
pv_names_list.append(pvc_obj.backed_pv)
log.info(f"Starting to delete bulk of {bulk_size} PVCs")
helpers.delete_bulk_pvcs(
self.yaml_creation_dir, pv_names_list, namespace=self.namespace
)
log.info(f"Deletion of bulk of {bulk_size} PVCs successfully completed")
log_deletion_times = helpers.measure_pv_deletion_time_bulk(
self.interface, pv_names_list, return_log_times=True
)
all_start_times = [a_tuple[0] for a_tuple in log_deletion_times.values()]
bulk_start_time = sorted(all_start_times)[0] # the eariles start time
start_deletion_time = datetime.datetime.strptime(
bulk_start_time, helpers.DATE_TIME_FORMAT
)
all_end_times = [a_tuple[1] for a_tuple in log_deletion_times.values()]
bulk_deletion_time = sorted(all_end_times)[-1] # the latest end time
end_deletion_time = datetime.datetime.strptime(
bulk_deletion_time, helpers.DATE_TIME_FORMAT
)
total_deletion_time = (end_deletion_time - start_deletion_time).total_seconds()
log.info(
f"{bulk_size} Bulk PVCs deletion time is {total_deletion_time} seconds."
)
csi_deletion_times = performance_lib.csi_bulk_pvc_time_measure(
self.interface, self.pvc_objs, "delete", csi_bulk_start_time
)
# Getting the end time of the test
self.test_end_time = self.get_time()
# reset the list oc PVCs since thay was deleted, and do not need to be deleted
# in the teardown phase.
self.pvc_objs = []
# Produce ES report
self.results_path = os.path.join(
"/",
*self.results_path,
"test_bulk_pvc_creation_deletion_measurement_performance",
)
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"bulk_creation_deletion_measurement",
)
)
# Add the test time to the ES report
full_results.add_key(
"test_time", {"start": self.test_start_time, "end": self.test_end_time}
)
full_results.add_key("bulk_size", bulk_size)
full_results.add_key("bulk_pvc_creation_time", total_time)
full_results.add_key("bulk_pvc_csi_creation_time", csi_creation_times)
full_results.add_key("bulk_pvc_deletion_time", total_deletion_time)
full_results.add_key("bulk_pvc_csi_deletion_time", csi_deletion_times)
# Write the test results into the ES server
if full_results.es_write():
res_link = full_results.results_link()
log.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtest (4 - according to the parameters)
self.write_result_to_file(res_link)
def test_multiple_pvc_deletion_measurement_performance(self, teardown_factory):
"""
Measuring PVC deletion time of 120 PVCs in 180 seconds
Args:
teardown_factory: A fixture used when we want a new resource that was created during the tests
to be removed in the teardown phase.
Returns:
"""
number_of_pvcs = 120
pvc_size = "1Gi"
log.info("Start creating new 120 PVCs")
pvc_objs = helpers.create_multiple_pvcs(
sc_name=self.sc_obj.name,
namespace=defaults.ROOK_CLUSTER_NAMESPACE,
number_of_pvc=number_of_pvcs,
size=pvc_size,
)
for pvc_obj in pvc_objs:
pvc_obj.reload()
teardown_factory(pvc_obj)
with ThreadPoolExecutor(max_workers=5) as executor:
for pvc_obj in pvc_objs:
executor.submit(
helpers.wait_for_resource_state, pvc_obj, constants.STATUS_BOUND
)
executor.submit(pvc_obj.reload)
# Get pvc_name, require pvc_name to fetch deletion time data from log
threads = list()
for pvc_obj in pvc_objs:
process = threading.Thread(target=pvc_obj.reload)
process.start()
threads.append(process)
for process in threads:
process.join()
pvc_name_list, pv_name_list = ([] for i in range(2))
threads = list()
for pvc_obj in pvc_objs:
process1 = threading.Thread(target=pvc_name_list.append(pvc_obj.name))
process2 = threading.Thread(target=pv_name_list.append(pvc_obj.backed_pv))
process1.start()
process2.start()
threads.append(process1)
threads.append(process2)
for process in threads:
process.join()
log.info("Preparing to delete 120 PVC")
# Delete PVC
for obj in pvc_objs:
obj.delete()
for obj in pvc_objs:
obj.ocp.wait_for_delete(obj.name)
# Get PVC deletion time
pvc_deletion_time = helpers.measure_pv_deletion_time_bulk(
interface=self.interface, pv_name_list=pv_name_list
)
logging.info(f"{number_of_pvcs} PVCs deletion time took {pvc_deletion_time}")
def test_bulk_pvc_creation_deletion_measurement_performance(
self, teardown_factory, bulk_size):
"""
Measuring PVC creation and deletion time of bulk_size PVCs
Args:
teardown_factory: A fixture used when we want a new resource that was created during the tests
to be removed in the teardown phase.
bulk_size: Size of the bulk to be tested
Returns:
"""
bulk_creation_time_limit = bulk_size / 2
log.info(f"Start creating new {bulk_size} PVCs")
pvc_objs, yaml_creation_dir = helpers.create_multiple_pvcs(
sc_name=self.sc_obj.name,
namespace=defaults.ROOK_CLUSTER_NAMESPACE,
number_of_pvc=bulk_size,
size=self.pvc_size,
burst=True,
)
logging.info(f"PVC creation dir is {yaml_creation_dir}")
for pvc_obj in pvc_objs:
pvc_obj.reload()
teardown_factory(pvc_obj)
with ThreadPoolExecutor(max_workers=5) as executor:
for pvc_obj in pvc_objs:
executor.submit(helpers.wait_for_resource_state, pvc_obj,
constants.STATUS_BOUND)
executor.submit(pvc_obj.reload)
start_time = helpers.get_provision_time(self.interface,
pvc_objs,
status="start")
end_time = helpers.get_provision_time(self.interface,
pvc_objs,
status="end")
total_time = (end_time - start_time).total_seconds()
logging.info(
f"{bulk_size} Bulk PVCs creation time is {total_time} seconds.")
if total_time > bulk_creation_time_limit:
raise ex.PerformanceException(
f"{bulk_size} Bulk PVCs creation time is {total_time} and "
f"greater than {bulk_creation_time_limit} seconds")
pv_names_list = []
for pvc_obj in pvc_objs:
pv_names_list.append(pvc_obj.backed_pv)
logging.info(f"Starting to delete bulk of {bulk_size} PVCs")
helpers.delete_bulk_pvcs(yaml_creation_dir, pv_names_list)
logging.info(
f"Deletion of bulk of {bulk_size} PVCs successfully completed")
log_deletion_times = helpers.measure_pv_deletion_time_bulk(
self.interface, pv_names_list, return_log_times=True)
all_start_times = [
a_tuple[0] for a_tuple in log_deletion_times.values()
]
bulk_start_time = sorted(all_start_times)[0] # the eariles start time
start_deletion_time = datetime.datetime.strptime(
bulk_start_time, helpers.DATE_TIME_FORMAT)
all_end_times = [a_tuple[1] for a_tuple in log_deletion_times.values()]
bulk_deletion_time = sorted(all_end_times)[-1] # the latest end time
end_deletion_time = datetime.datetime.strptime(
bulk_deletion_time, helpers.DATE_TIME_FORMAT)
total_deletion_time = (end_deletion_time -
start_deletion_time).total_seconds()
logging.info(
f"{bulk_size} Bulk PVCs deletion time is {total_deletion_time} seconds."
)