def run_fio_on_pvcs(self, pvc_dict_list, pod_factory):
total_files_size = 0
searched_pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace)
logging.info(f"Found {len(searched_pvc_objs)} PVCs")
for pvc_yaml in pvc_dict_list:
pvc_name = pvc_yaml["metadata"]["name"]
pvc_size = pvc_yaml["spec"]["resources"]["requests"]["storage"]
logging.info(f"Size of pvc {pvc_name} is {pvc_size}")
pvc_size_int = int(pvc_size[:-2]) # without "Gi"
file_size_mb = int(pvc_size_int * 0.6) * constants.GB2MB
total_files_size += file_size_mb
file_size_mb_str = str(file_size_mb) + "M"
logging.info(f"Writing file of size {file_size_mb_str}")
# now find pvc_obj by name and create pod_obj to write to
pvc_obj = None
for obj in searched_pvc_objs:
if obj.name == pvc_name:
pvc_obj = obj
searched_pvc_objs.remove(obj)
break
assert pvc_obj is not None, f"Cannot find PVC with name {pvc_name}"
logging.info(f"PVC with name {pvc_name} found")
pod_obj = pod_factory(interface=self.interface,
pvc=pvc_obj,
status=constants.STATUS_RUNNING)
performance_lib.write_fio_on_pod(pod_obj, file_size_mb_str)
return total_files_size
def run_fio_on_pvcs(self, pvc_size):
searched_pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace)
pod_objs = pod.get_all_pods(namespace=self.namespace)
log.info(f"Found {len(searched_pvc_objs)} PVCs")
pvc_size_int = int(pvc_size[:-2]) # without "Gi"
file_size_mb = int(pvc_size_int * 0.6) * constants.GB2MB
total_files_size = file_size_mb * len(searched_pvc_objs)
file_size_mb_str = str(file_size_mb) + "M"
log.info(f"Writing file of size {file_size_mb_str} in each PVC")
for objs in pod_objs:
performance_lib.write_fio_on_pod(objs, file_size_mb_str)
return total_files_size
def test_pvc_multiple_clone_performance(
self,
interface_iterate,
teardown_factory,
storageclass_factory,
pvc_factory,
pod_factory,
):
"""
1. Creating PVC
PVC size is calculated in the test and depends on the storage capacity, but not less then 1 GiB
it will use ~75% capacity of the Storage, Min storage capacity 1 TiB
2. Fill the PVC with 70% of data
3. Take a clone of the PVC and measure time and speed of creation by reading start creation and end creation
times from relevant logs
4. Repeat the previous step number of times (maximal num_of_clones is 512)
5. Print all measured statistics for all the clones.
Raises:
StorageNotSufficientException: in case of not enough capacity on the cluster
"""
num_of_clones = 512
# Getting the total Storage capacity
ceph_cluster = CephCluster()
ceph_capacity = int(ceph_cluster.get_ceph_capacity())
# Use 70% of the storage capacity in the test
capacity_to_use = int(ceph_capacity * 0.7)
# since we do not want to use more then 65%, we add 35% to the needed
# capacity, and minimum PVC size is 1 GiB
need_capacity = int((num_of_clones + 2) * 1.35)
# Test will run only on system with enough capacity
if capacity_to_use < need_capacity:
err_msg = (f"The system have only {ceph_capacity} GiB, "
f"we want to use only {capacity_to_use} GiB, "
f"and we need {need_capacity} GiB to run the test")
log.error(err_msg)
raise exceptions.StorageNotSufficientException(err_msg)
# Calculating the PVC size in GiB
pvc_size = int(capacity_to_use / (num_of_clones + 2))
self.interface = interface_iterate
self.sc_obj = storageclass_factory(self.interface)
self.pvc_obj = pvc_factory(interface=self.interface,
size=pvc_size,
status=constants.STATUS_BOUND)
self.pod_obj = pod_factory(interface=self.interface,
pvc=self.pvc_obj,
status=constants.STATUS_RUNNING)
# Calculating the file size as 70% of the PVC size
filesize = self.pvc_obj.size * 0.70
# Change the file size to MB for the FIO function
file_size = f"{int(filesize * constants.GB2MB)}M"
file_name = self.pod_obj.name
log.info(f"Total capacity size is : {ceph_capacity} GiB, "
f"Going to use {need_capacity} GiB, "
f"With {num_of_clones} clones to {pvc_size} GiB PVC. "
f"File size to be written is : {file_size} "
f"with the name of {file_name}")
self.params = {}
self.params["clonenum"] = f"{num_of_clones}"
self.params["filesize"] = file_size
self.params["ERRMSG"] = "Error in command"
clone_yaml = self.build_params()
performance_lib.write_fio_on_pod(self.pod_obj, file_size)
# Running the test
results = []
for test_num in range(1, int(self.params["clonenum"]) + 1):
log.info(f"Starting test number {test_num}")
ct = self.create_clone(test_num, clone_yaml)
speed = self.params["datasize"] / ct
results.append({"Clone Num": test_num, "time": ct, "speed": speed})
log.info(
f"Results for clone number {test_num} are : "
f"Creation time is {ct} secs, Creation speed {speed} MB/sec")
for r in results:
log.info(
f"Clone number {r['Clone Num']} creation time is {r['time']} secs."
)
log.info(
f"Clone number {r['Clone Num']} creation speed is {r['speed']} MB/sec."
)
def test_pvc_multiple_clone_performance(
self,
interface_iterate,
teardown_factory,
storageclass_factory,
pvc_factory,
pod_factory,
):
"""
1. Creating PVC
PVC size is calculated in the test and depends on the storage capacity, but not less then 1 GiB
it will use ~75% capacity of the Storage, Min storage capacity 1 TiB
2. Fill the PVC with 70% of data
3. Take a clone of the PVC and measure time and speed of creation by reading start creation and end creation
times from relevant logs
4. Repeat the previous step number of times (maximal num_of_clones is 512)
5. Print all measured statistics for all the clones.
Raises:
StorageNotSufficientException: in case of not enough capacity on the cluster
"""
num_of_clones = 512
# Getting the total Storage capacity
ceph_cluster = CephCluster()
ceph_capacity = int(ceph_cluster.get_ceph_capacity())
# Use 70% of the storage capacity in the test
capacity_to_use = int(ceph_capacity * 0.7)
# since we do not want to use more then 65%, we add 35% to the needed
# capacity, and minimum PVC size is 1 GiB
need_capacity = int((num_of_clones + 2) * 1.35)
# Test will run only on system with enough capacity
if capacity_to_use < need_capacity:
err_msg = (f"The system have only {ceph_capacity} GiB, "
f"we want to use only {capacity_to_use} GiB, "
f"and we need {need_capacity} GiB to run the test")
log.error(err_msg)
raise exceptions.StorageNotSufficientException(err_msg)
# Calculating the PVC size in GiB
pvc_size = int(capacity_to_use / (num_of_clones + 2))
self.interface = interface_iterate
self.sc_obj = storageclass_factory(self.interface)
if self.interface == constants.CEPHFILESYSTEM:
sc = "CephFS"
if self.interface == constants.CEPHBLOCKPOOL:
sc = "RBD"
self.full_log_path = get_full_test_logs_path(cname=self)
self.full_log_path += f"-{sc}"
self.pvc_obj = pvc_factory(interface=self.interface,
size=pvc_size,
status=constants.STATUS_BOUND)
self.pod_obj = pod_factory(interface=self.interface,
pvc=self.pvc_obj,
status=constants.STATUS_RUNNING)
# Calculating the file size as 70% of the PVC size
filesize = self.pvc_obj.size * 0.70
# Change the file size to MB for the FIO function
file_size = f"{int(filesize * constants.GB2MB)}M"
file_name = self.pod_obj.name
log.info(f"Total capacity size is : {ceph_capacity} GiB, "
f"Going to use {need_capacity} GiB, "
f"With {num_of_clones} clones to {pvc_size} GiB PVC. "
f"File size to be written is : {file_size} "
f"with the name of {file_name}")
self.params = {}
self.params["clonenum"] = f"{num_of_clones}"
self.params["filesize"] = file_size
self.params["ERRMSG"] = "Error in command"
clone_yaml = self.build_params()
performance_lib.write_fio_on_pod(self.pod_obj, file_size)
# Running the test
results = []
for test_num in range(1, int(self.params["clonenum"]) + 1):
log.info(f"Starting test number {test_num}")
ct = self.create_clone(test_num, clone_yaml)
speed = self.params["datasize"] / ct
results.append({"Clone Num": test_num, "time": ct, "speed": speed})
log.info(
f"Results for clone number {test_num} are : "
f"Creation time is {ct} secs, Creation speed {speed} MB/sec")
for r in results:
log.info(
f"Clone number {r['Clone Num']} creation time is {r['time']} secs."
)
log.info(
f"Clone number {r['Clone Num']} creation speed is {r['speed']} MB/sec."
)
creation_time_list = [r["time"] for r in results]
average_creation_time = statistics.mean(creation_time_list)
log.info(f"Average creation time is {average_creation_time} secs.")
creation_speed_list = [r["speed"] for r in results]
average_creation_speed = statistics.mean(creation_speed_list)
log.info(f"Average creation speed is {average_creation_time} MB/sec.")
self.results_path = get_full_test_logs_path(cname=self)
# Produce ES report
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"pvc_multiple_clone_measurement",
))
full_results.add_key("interface", self.interface)
full_results.add_key("clones_num", num_of_clones)
full_results.add_key("clone_size", pvc_size)
full_results.add_key("multi_clone_creation_time", creation_time_list)
full_results.add_key("multi_clone_creation_time_average",
average_creation_time)
full_results.add_key("multi_clone_creation_speed", creation_speed_list)
full_results.add_key("multi_clone_creation_speed_average",
average_creation_speed)
# Write the test results into the ES server
if full_results.es_write():
res_link = full_results.results_link()
log.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtest (4 - according to the parameters)
self.write_result_to_file(res_link)
def test_clone_create_delete_performance(self, interface_type, pvc_size,
file_size, teardown_factory):
"""
Write data (60% of PVC capacity) to the PVC created in setup
Create single clone for an existing pvc,
Measure clone creation time and speed
Delete the created clone
Measure clone deletion time and speed
Note: by increasing max_num_of_clones value you increase number of the clones to be created/deleted
"""
file_size_for_io = file_size[:-1]
performance_lib.write_fio_on_pod(self.pod_obj, file_size_for_io)
max_num_of_clones = 1
clone_creation_measures = []
clones_list = []
timeout = 18000
sc_name = self.pvc_obj.backed_sc
parent_pvc = self.pvc_obj.name
clone_yaml = constants.CSI_RBD_PVC_CLONE_YAML
namespace = self.pvc_obj.namespace
if interface_type == constants.CEPHFILESYSTEM:
clone_yaml = constants.CSI_CEPHFS_PVC_CLONE_YAML
file_size_mb = convert_device_size(file_size, "MB")
# creating single clone ( or many one by one if max_mum_of_clones > 1)
logger.info(
f"Start creating {max_num_of_clones} clones on {interface_type} PVC of size {pvc_size} GB."
)
for i in range(max_num_of_clones):
logger.info(f"Start creation of clone number {i + 1}.")
cloned_pvc_obj = pvc.create_pvc_clone(sc_name,
parent_pvc,
clone_yaml,
namespace,
storage_size=pvc_size + "Gi")
teardown_factory(cloned_pvc_obj)
helpers.wait_for_resource_state(cloned_pvc_obj,
constants.STATUS_BOUND, timeout)
cloned_pvc_obj.reload()
logger.info(
f"Clone with name {cloned_pvc_obj.name} for {pvc_size} pvc {parent_pvc} was created."
)
clones_list.append(cloned_pvc_obj)
create_time = helpers.measure_pvc_creation_time(
interface_type, cloned_pvc_obj.name)
creation_speed = int(file_size_mb / create_time)
logger.info(
f"Clone number {i+1} creation time is {create_time} secs for {pvc_size} GB pvc."
)
logger.info(
f"Clone number {i+1} creation speed is {creation_speed} MB/sec for {pvc_size} GB pvc."
)
creation_measures = {
"clone_num": i + 1,
"time": create_time,
"speed": creation_speed,
}
clone_creation_measures.append(creation_measures)
# deleting one by one and measuring deletion times and speed for each one of the clones create above
# in case of single clone will run one time
clone_deletion_measures = []
logger.info(
f"Start deleting {max_num_of_clones} clones on {interface_type} PVC of size {pvc_size} GB."
)
for i in range(max_num_of_clones):
cloned_pvc_obj = clones_list[i]
pvc_reclaim_policy = cloned_pvc_obj.reclaim_policy
cloned_pvc_obj.delete()
logger.info(
f"Deletion of clone number {i + 1} , the clone name is {cloned_pvc_obj.name}."
)
cloned_pvc_obj.ocp.wait_for_delete(cloned_pvc_obj.name, timeout)
if pvc_reclaim_policy == constants.RECLAIM_POLICY_DELETE:
helpers.validate_pv_delete(cloned_pvc_obj.backed_pv)
delete_time = helpers.measure_pvc_deletion_time(
interface_type, cloned_pvc_obj.backed_pv)
logger.info(
f"Clone number {i + 1} deletion time is {delete_time} secs for {pvc_size} GB pvc."
)
deletion_speed = int(file_size_mb / delete_time)
logger.info(
f"Clone number {i+1} deletion speed is {deletion_speed} MB/sec for {pvc_size} GB pvc."
)
deletion_measures = {
"clone_num": i + 1,
"time": delete_time,
"speed": deletion_speed,
}
clone_deletion_measures.append(deletion_measures)
logger.info(
f"Printing clone creation time and speed for {max_num_of_clones} clones "
f"on {interface_type} PVC of size {pvc_size} GB:")
for c in clone_creation_measures:
logger.info(
f"Clone number {c['clone_num']} creation time is {c['time']} secs for {pvc_size} GB pvc ."
)
logger.info(
f"Clone number {c['clone_num']} creation speed is {c['speed']} MB/sec for {pvc_size} GB pvc."
)
logger.info(
f"Clone deletion time and speed for {interface_type} PVC of size {pvc_size} GB are:"
)
for d in clone_deletion_measures:
logger.info(
f"Clone number {d['clone_num']} deletion time is {d['time']} secs for {pvc_size} GB pvc."
)
logger.info(
f"Clone number {d['clone_num']} deletion speed is {d['speed']} MB/sec for {pvc_size} GB pvc."
)
logger.info("test_clones_creation_performance finished successfully.")
def test_clone_create_delete_performance(self, interface_type, pvc_size,
file_size, teardown_factory):
"""
Write data (60% of PVC capacity) to the PVC created in setup
Create clones for an existing pvc,
Measure clones average creation time and speed
Delete the created clone
Measure clone average deletion time and speed
Note: by increasing max_num_of_clones value you increase number of the clones to be created/deleted
"""
file_size_for_io = file_size[:-1]
performance_lib.write_fio_on_pod(self.pod_object, file_size_for_io)
max_num_of_clones = 10
clone_creation_measures = []
csi_clone_creation_measures = []
clones_list = []
timeout = 18000
sc_name = self.pvc_obj.backed_sc
parent_pvc = self.pvc_obj.name
clone_yaml = constants.CSI_RBD_PVC_CLONE_YAML
namespace = self.pvc_obj.namespace
if interface_type == constants.CEPHFILESYSTEM:
clone_yaml = constants.CSI_CEPHFS_PVC_CLONE_YAML
file_size_mb = convert_device_size(file_size, "MB")
logger.info(
f"Start creating {max_num_of_clones} clones on {interface_type} PVC of size {pvc_size} GB."
)
# taking the time, so parsing the provision log will be faster.
start_time = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
for i in range(max_num_of_clones):
logger.info(f"Start creation of clone number {i + 1}.")
cloned_pvc_obj = pvc.create_pvc_clone(sc_name,
parent_pvc,
clone_yaml,
namespace,
storage_size=pvc_size + "Gi")
teardown_factory(cloned_pvc_obj)
helpers.wait_for_resource_state(cloned_pvc_obj,
constants.STATUS_BOUND, timeout)
cloned_pvc_obj.reload()
logger.info(
f"Clone with name {cloned_pvc_obj.name} for {pvc_size} pvc {parent_pvc} was created."
)
clones_list.append(cloned_pvc_obj)
create_time = helpers.measure_pvc_creation_time(
interface_type, cloned_pvc_obj.name)
creation_speed = int(file_size_mb / create_time)
logger.info(
f"Clone number {i+1} creation time is {create_time} secs for {pvc_size} GB pvc."
)
logger.info(
f"Clone number {i+1} creation speed is {creation_speed} MB/sec for {pvc_size} GB pvc."
)
creation_measures = {
"clone_num": i + 1,
"time": create_time,
"speed": creation_speed,
}
clone_creation_measures.append(creation_measures)
csi_clone_creation_measures.append(
performance_lib.csi_pvc_time_measure(self.interface,
cloned_pvc_obj, "create",
start_time))
# deleting one by one and measuring deletion times and speed for each one of the clones create above
# in case of single clone will run one time
clone_deletion_measures = []
csi_clone_deletion_measures = []
logger.info(
f"Start deleting {max_num_of_clones} clones on {interface_type} PVC of size {pvc_size} GB."
)
for i in range(max_num_of_clones):
cloned_pvc_obj = clones_list[i]
pvc_reclaim_policy = cloned_pvc_obj.reclaim_policy
cloned_pvc_obj.delete()
logger.info(
f"Deletion of clone number {i + 1} , the clone name is {cloned_pvc_obj.name}."
)
cloned_pvc_obj.ocp.wait_for_delete(cloned_pvc_obj.name, timeout)
if pvc_reclaim_policy == constants.RECLAIM_POLICY_DELETE:
helpers.validate_pv_delete(cloned_pvc_obj.backed_pv)
delete_time = helpers.measure_pvc_deletion_time(
interface_type, cloned_pvc_obj.backed_pv)
logger.info(
f"Clone number {i + 1} deletion time is {delete_time} secs for {pvc_size} GB pvc."
)
deletion_speed = int(file_size_mb / delete_time)
logger.info(
f"Clone number {i+1} deletion speed is {deletion_speed} MB/sec for {pvc_size} GB pvc."
)
deletion_measures = {
"clone_num": i + 1,
"time": delete_time,
"speed": deletion_speed,
}
clone_deletion_measures.append(deletion_measures)
csi_clone_deletion_measures.append(
performance_lib.csi_pvc_time_measure(self.interface,
cloned_pvc_obj, "delete",
start_time))
logger.info(
f"Printing clone creation time and speed for {max_num_of_clones} clones "
f"on {interface_type} PVC of size {pvc_size} GB:")
for c in clone_creation_measures:
logger.info(
f"Clone number {c['clone_num']} creation time is {c['time']} secs for {pvc_size} GB pvc ."
)
logger.info(
f"Clone number {c['clone_num']} creation speed is {c['speed']} MB/sec for {pvc_size} GB pvc."
)
logger.info(
f"Clone deletion time and speed for {interface_type} PVC of size {pvc_size} GB are:"
)
creation_time_list = [r["time"] for r in clone_creation_measures]
creation_speed_list = [r["speed"] for r in clone_creation_measures]
average_creation_time = statistics.mean(creation_time_list)
average_csi_creation_time = statistics.mean(
csi_clone_creation_measures)
average_creation_speed = statistics.mean(creation_speed_list)
logger.info(f"Average creation time is {average_creation_time} secs.")
logger.info(
f"Average creation speed is {average_creation_speed} Mb/sec.")
for d in clone_deletion_measures:
logger.info(
f"Clone number {d['clone_num']} deletion time is {d['time']} secs for {pvc_size} GB pvc."
)
logger.info(
f"Clone number {d['clone_num']} deletion speed is {d['speed']} MB/sec for {pvc_size} GB pvc."
)
deletion_time_list = [r["time"] for r in clone_deletion_measures]
deletion_speed_list = [r["speed"] for r in clone_deletion_measures]
average_deletion_time = statistics.mean(deletion_time_list)
average_csi_deletion_time = statistics.mean(
csi_clone_deletion_measures)
average_deletion_speed = statistics.mean(deletion_speed_list)
logger.info(f"Average deletion time is {average_deletion_time} secs.")
logger.info(
f"Average deletion speed is {average_deletion_speed} Mb/sec.")
logger.info("test_clones_creation_performance finished successfully.")
self.results_path = get_full_test_logs_path(cname=self)
# Produce ES report
# Collecting environment information
self.get_env_info()
self.full_log_path = get_full_test_logs_path(cname=self)
self.results_path = get_full_test_logs_path(cname=self)
self.full_log_path += f"-{self.interface}-{pvc_size}-{file_size}"
logger.info(f"Logs file path name is : {self.full_log_path}")
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"pvc_clone_performance",
))
full_results.add_key("interface", self.interface)
full_results.add_key("total_clone_number", max_num_of_clones)
full_results.add_key("pvc_size", self.pvc_size)
full_results.add_key("average_clone_creation_time",
average_creation_time)
full_results.add_key("average_csi_clone_creation_time",
average_csi_creation_time)
full_results.add_key("average_clone_deletion_time",
average_deletion_time)
full_results.add_key("average_csi_clone_deletion_time",
average_csi_deletion_time)
full_results.add_key("average_clone_creation_speed",
average_creation_speed)
full_results.add_key("average_clone_deletion_speed",
average_deletion_speed)
full_results.all_results = {
"clone_creation_time": creation_time_list,
"csi_clone_creation_time": csi_clone_creation_measures,
"clone_deletion_time": deletion_time_list,
"csi_clone_deletion_time": csi_clone_deletion_measures,
"clone_creation_speed": creation_speed_list,
"clone_deletion_speed": deletion_speed_list,
}
# Write the test results into the ES server
if full_results.es_write():
res_link = full_results.results_link()
logger.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtest (8 - according to the parameters)
self.write_result_to_file(res_link)