def test_pvc_multiple_clone_performance(
self,
interface_iterate,
teardown_factory,
storageclass_factory,
pvc_factory,
pod_factory,
):
"""
1. Creating PVC
PVC size is calculated in the test and depends on the storage capacity, but not less then 1 GiB
it will use ~75% capacity of the Storage, Min storage capacity 1 TiB
2. Fill the PVC with 70% of data
3. Take a clone of the PVC and measure time and speed of creation by reading start creation and end creation
times from relevant logs
4. Repeat the previous step number of times (maximal num_of_clones is 512)
5. Print all measured statistics for all the clones.
Raises:
StorageNotSufficientException: in case of not enough capacity on the cluster
"""
num_of_clones = 512
# Getting the total Storage capacity
ceph_cluster = CephCluster()
ceph_capacity = int(ceph_cluster.get_ceph_capacity())
# Use 70% of the storage capacity in the test
capacity_to_use = int(ceph_capacity * 0.7)
# since we do not want to use more then 65%, we add 35% to the needed
# capacity, and minimum PVC size is 1 GiB
need_capacity = int((num_of_clones + 2) * 1.35)
# Test will run only on system with enough capacity
if capacity_to_use < need_capacity:
err_msg = (f"The system have only {ceph_capacity} GiB, "
f"we want to use only {capacity_to_use} GiB, "
f"and we need {need_capacity} GiB to run the test")
log.error(err_msg)
raise exceptions.StorageNotSufficientException(err_msg)
# Calculating the PVC size in GiB
pvc_size = int(capacity_to_use / (num_of_clones + 2))
self.interface = interface_iterate
self.sc_obj = storageclass_factory(self.interface)
if self.interface == constants.CEPHFILESYSTEM:
sc = "CephFS"
if self.interface == constants.CEPHBLOCKPOOL:
sc = "RBD"
self.full_log_path = get_full_test_logs_path(cname=self)
self.full_log_path += f"-{sc}"
self.pvc_obj = pvc_factory(interface=self.interface,
size=pvc_size,
status=constants.STATUS_BOUND)
self.pod_obj = pod_factory(interface=self.interface,
pvc=self.pvc_obj,
status=constants.STATUS_RUNNING)
# Calculating the file size as 70% of the PVC size
filesize = self.pvc_obj.size * 0.70
# Change the file size to MB for the FIO function
file_size = f"{int(filesize * constants.GB2MB)}M"
file_name = self.pod_obj.name
log.info(f"Total capacity size is : {ceph_capacity} GiB, "
f"Going to use {need_capacity} GiB, "
f"With {num_of_clones} clones to {pvc_size} GiB PVC. "
f"File size to be written is : {file_size} "
f"with the name of {file_name}")
self.params = {}
self.params["clonenum"] = f"{num_of_clones}"
self.params["filesize"] = file_size
self.params["ERRMSG"] = "Error in command"
clone_yaml = self.build_params()
performance_lib.write_fio_on_pod(self.pod_obj, file_size)
# Running the test
results = []
for test_num in range(1, int(self.params["clonenum"]) + 1):
log.info(f"Starting test number {test_num}")
ct = self.create_clone(test_num, clone_yaml)
speed = self.params["datasize"] / ct
results.append({"Clone Num": test_num, "time": ct, "speed": speed})
log.info(
f"Results for clone number {test_num} are : "
f"Creation time is {ct} secs, Creation speed {speed} MB/sec")
for r in results:
log.info(
f"Clone number {r['Clone Num']} creation time is {r['time']} secs."
)
log.info(
f"Clone number {r['Clone Num']} creation speed is {r['speed']} MB/sec."
)
creation_time_list = [r["time"] for r in results]
average_creation_time = statistics.mean(creation_time_list)
log.info(f"Average creation time is {average_creation_time} secs.")
creation_speed_list = [r["speed"] for r in results]
average_creation_speed = statistics.mean(creation_speed_list)
log.info(f"Average creation speed is {average_creation_time} MB/sec.")
self.results_path = get_full_test_logs_path(cname=self)
# Produce ES report
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"pvc_multiple_clone_measurement",
))
full_results.add_key("interface", self.interface)
full_results.add_key("clones_num", num_of_clones)
full_results.add_key("clone_size", pvc_size)
full_results.add_key("multi_clone_creation_time", creation_time_list)
full_results.add_key("multi_clone_creation_time_average",
average_creation_time)
full_results.add_key("multi_clone_creation_speed", creation_speed_list)
full_results.add_key("multi_clone_creation_speed_average",
average_creation_speed)
# Write the test results into the ES server
if full_results.es_write():
res_link = full_results.results_link()
log.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtest (4 - according to the parameters)
self.write_result_to_file(res_link)
def test_pvc_snapshot_performance(self, pvc_size):
"""
1. Run I/O on a pod file
2. Calculate md5sum of the file
3. Take a snapshot of the PVC
4. Measure the total snapshot creation time and the CSI snapshot creation time
4. Restore From the snapshot and measure the time
5. Attach a new pod to it
6. Verify that the file is present on the new pod also
7. Verify that the md5sum of the file on the new pod matches
with the md5sum of the file on the original pod
This scenario run 3 times and report all the average results of the 3 runs
and will send them to the ES
Args:
pvc_size: the size of the PVC to be tested - parametrize
"""
# Getting the total Storage capacity
ceph_cluster = CephCluster()
ceph_capacity = ceph_cluster.get_ceph_capacity()
log.info(f"Total capacity size is : {ceph_capacity}")
log.info(f"PVC Size is : {pvc_size}")
log.info(f"Needed capacity is {int(int(pvc_size) * 5)}")
if int(ceph_capacity) < int(pvc_size) * 5:
log.error(
f"PVC size is {pvc_size}GiB and it is too large for this system"
f" which have only {ceph_capacity}GiB")
return
# Calculating the file size as 25% of the PVC size
# in the end the PVC will be 75% full
filesize = self.pvc_obj.size * 0.25
# Change the file size to MB and from int to str
file_size = f"{int(filesize * 1024)}M"
all_results = []
self.results_path = get_full_test_logs_path(cname=self)
log.info(f"Logs file path name is : {self.full_log_path}")
# Produce ES report
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
self.full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"pvc_snapshot_perf",
))
self.full_results.add_key("pvc_size", pvc_size + " GiB")
self.full_results.add_key("interface", self.sc)
self.full_results.all_results["creation_time"] = []
self.full_results.all_results["csi_creation_time"] = []
self.full_results.all_results["creation_speed"] = []
self.full_results.all_results["restore_time"] = []
self.full_results.all_results["restore_speed"] = []
self.full_results.all_results["restore_csi_time"] = []
for test_num in range(self.tests_numbers):
test_results = {
"test_num": test_num + 1,
"dataset": (test_num + 1) * filesize * 1024, # size in MiB
"create": {
"time": None,
"csi_time": None,
"speed": None
},
"restore": {
"time": None,
"speed": None
},
}
log.info(f"Starting test phase number {test_num}")
# Step 1. Run I/O on a pod file.
file_name = f"{self.pod_object.name}-{test_num}"
log.info(f"Starting IO on the POD {self.pod_object.name}")
# Going to run only write IO to fill the PVC for the snapshot
self.pod_object.fillup_fs(size=file_size, fio_filename=file_name)
# Wait for fio to finish
fio_result = self.pod_object.get_fio_results()
err_count = fio_result.get("jobs")[0].get("error")
assert (
err_count == 0
), f"IO error on pod {self.pod_object.name}. FIO result: {fio_result}"
log.info("IO on the PVC Finished")
# Verify presence of the file
file_path = pod.get_file_path(self.pod_object, file_name)
log.info(f"Actual file path on the pod {file_path}")
assert pod.check_file_existence(
self.pod_object, file_path), f"File {file_name} doesn't exist"
log.info(f"File {file_name} exists in {self.pod_object.name}")
# Step 2. Calculate md5sum of the file.
orig_md5_sum = pod.cal_md5sum(self.pod_object, file_name)
# Step 3. Take a snapshot of the PVC and measure the time of creation.
snap_name = self.pvc_obj.name.replace("pvc-test",
f"snapshot-test{test_num}")
log.info(f"Taking snapshot of the PVC {snap_name}")
start_time = datetime.datetime.utcnow().strftime(
"%Y-%m-%dT%H:%M:%SZ")
test_results["create"]["time"] = self.measure_create_snapshot_time(
pvc_name=self.pvc_obj.name,
snap_name=snap_name,
namespace=self.pod_object.namespace,
interface=self.interface,
start_time=start_time,
)
test_results["create"][
"csi_time"] = performance_lib.measure_csi_snapshot_creation_time(
interface=self.interface,
snapshot_id=self.snap_uid,
start_time=start_time,
)
test_results["create"]["speed"] = int(
test_results["dataset"] / test_results["create"]["time"])
log.info(
f' Test {test_num} dataset is {test_results["dataset"]} MiB')
log.info(
f"Snapshot name {snap_name} and id {self.snap_uid} creation time is"
f' : {test_results["create"]["time"]} sec.')
log.info(
f"Snapshot name {snap_name} and id {self.snap_uid} csi creation time is"
f' : {test_results["create"]["csi_time"]} sec.')
log.info(
f'Snapshot speed is : {test_results["create"]["speed"]} MB/sec'
)
# Step 4. Restore the PVC from the snapshot and measure the time
# Same Storage class of the original PVC
sc_name = self.pvc_obj.backed_sc
# Size should be same as of the original PVC
pvc_size = str(self.pvc_obj.size) + "Gi"
# Create pvc out of the snapshot
# Both, the snapshot and the restore PVC should be in same namespace
log.info("Restoring from the Snapshot")
restore_pvc_name = self.pvc_obj.name.replace(
"pvc-test", f"restore-pvc{test_num}")
restore_pvc_yaml = constants.CSI_RBD_PVC_RESTORE_YAML
if self.interface == constants.CEPHFILESYSTEM:
restore_pvc_yaml = constants.CSI_CEPHFS_PVC_RESTORE_YAML
csi_start_time = self.get_time("csi")
log.info("Restoring the PVC from Snapshot")
restore_pvc_obj = pvc.create_restore_pvc(
sc_name=sc_name,
snap_name=self.snap_obj.name,
namespace=self.snap_obj.namespace,
size=pvc_size,
pvc_name=restore_pvc_name,
restore_pvc_yaml=restore_pvc_yaml,
)
helpers.wait_for_resource_state(
restore_pvc_obj,
constants.STATUS_BOUND,
timeout=3600 # setting this to 60 Min.
# since it can be take long time to restore, and we want it to finished.
)
restore_pvc_obj.reload()
log.info("PVC was restored from the snapshot")
test_results["restore"][
"time"] = helpers.measure_pvc_creation_time(
self.interface, restore_pvc_obj.name)
test_results["restore"]["speed"] = int(
test_results["dataset"] / test_results["restore"]["time"])
log.info(
f'Snapshot restore time is : {test_results["restore"]["time"]}'
)
log.info(
f'restore speed is : {test_results["restore"]["speed"]} MB/sec'
)
test_results["restore"][
"csi_time"] = performance_lib.csi_pvc_time_measure(
self.interface, restore_pvc_obj, "create", csi_start_time)
log.info(
f'Snapshot csi restore time is : {test_results["restore"]["csi_time"]}'
)
# Step 5. Attach a new pod to the restored PVC
restore_pod_object = helpers.create_pod(
interface_type=self.interface,
pvc_name=restore_pvc_obj.name,
namespace=self.snap_obj.namespace,
)
# Confirm that the pod is running
helpers.wait_for_resource_state(resource=restore_pod_object,
state=constants.STATUS_RUNNING)
restore_pod_object.reload()
# Step 6. Verify that the file is present on the new pod also.
log.info(f"Checking the existence of {file_name} "
f"on restore pod {restore_pod_object.name}")
assert pod.check_file_existence(
restore_pod_object,
file_path), f"File {file_name} doesn't exist"
log.info(f"File {file_name} exists in {restore_pod_object.name}")
# Step 7. Verify that the md5sum matches
log.info(
f"Verifying that md5sum of {file_name} "
f"on pod {self.pod_object.name} matches with md5sum "
f"of the same file on restore pod {restore_pod_object.name}")
assert pod.verify_data_integrity(
restore_pod_object, file_name,
orig_md5_sum), "Data integrity check failed"
log.info("Data integrity check passed, md5sum are same")
restore_pod_object.delete()
restore_pvc_obj.delete()
all_results.append(test_results)
# clean the enviroment
self.pod_object.delete()
self.pvc_obj.delete()
self.delete_test_project()
# logging the test summary, all info in one place for easy log reading
c_speed, c_runtime, c_csi_runtime, r_speed, r_runtime, r_csi_runtime = (
0 for i in range(6))
log.info("Test summary :")
for tst in all_results:
c_speed += tst["create"]["speed"]
c_runtime += tst["create"]["time"]
c_csi_runtime += tst["create"]["csi_time"]
r_speed += tst["restore"]["speed"]
r_runtime += tst["restore"]["time"]
r_csi_runtime += tst["restore"]["csi_time"]
self.full_results.all_results["creation_time"].append(
tst["create"]["time"])
self.full_results.all_results["csi_creation_time"].append(
tst["create"]["csi_time"])
self.full_results.all_results["creation_speed"].append(
tst["create"]["speed"])
self.full_results.all_results["restore_time"].append(
tst["restore"]["time"])
self.full_results.all_results["restore_speed"].append(
tst["restore"]["speed"])
self.full_results.all_results["restore_csi_time"].append(
tst["restore"]["csi_time"])
self.full_results.all_results["dataset_inMiB"] = tst["dataset"]
log.info(
f"Test {tst['test_num']} results : dataset is {tst['dataset']} MiB. "
f"Take snapshot time is {tst['create']['time']} "
f"at {tst['create']['speed']} MiB/Sec "
f"Restore from snapshot time is {tst['restore']['time']} "
f"at {tst['restore']['speed']} MiB/Sec ")
avg_snap_c_time = c_runtime / self.tests_numbers
avg_snap_csi_c_time = c_csi_runtime / self.tests_numbers
avg_snap_c_speed = c_speed / self.tests_numbers
avg_snap_r_time = r_runtime / self.tests_numbers
avg_snap_r_speed = r_speed / self.tests_numbers
avg_snap_r_csi_time = r_csi_runtime / self.tests_numbers
log.info(f" Average snapshot creation time is {avg_snap_c_time} sec.")
log.info(
f" Average csi snapshot creation time is {avg_snap_csi_c_time} sec."
)
log.info(
f" Average snapshot creation speed is {avg_snap_c_speed} MiB/sec")
log.info(f" Average snapshot restore time is {avg_snap_r_time} sec.")
log.info(
f" Average snapshot restore speed is {avg_snap_r_speed} MiB/sec")
log.info(
f" Average snapshot restore csi time is {avg_snap_r_csi_time} sec."
)
self.full_results.add_key("avg_snap_creation_time_insecs",
avg_snap_c_time)
self.full_results.add_key("avg_snap_csi_creation_time_insecs",
avg_snap_csi_c_time)
self.full_results.add_key("avg_snap_creation_speed", avg_snap_c_speed)
self.full_results.add_key("avg_snap_restore_time_insecs",
avg_snap_r_time)
self.full_results.add_key("avg_snap_restore_speed", avg_snap_r_speed)
self.full_results.add_key("avg_snap_restore_csi_time_insecs",
avg_snap_r_csi_time)
# Write the test results into the ES server
log.info("writing results to elastic search server")
if self.full_results.es_write():
res_link = self.full_results.results_link()
# write the ES link to the test results in the test log.
log.info(f"The result can be found at : {res_link}")
self.write_result_to_file(res_link)
def test_pvc_snapshot_performance_multiple_files(self, file_size, files,
threads, interface):
"""
Run SmallFile Workload and the take snapshot.
test will run with 1M of file on the volume - total data set
is the same for all tests, ~30GiB, and then take snapshot and measure
the time it takes.
the test will run 3 time to check consistency.
Args:
file_size (int): the size of the file to be create - in KiB
files (int): number of files each thread will create
threads (int): number of threads will be used in the workload
interface (str): the volume interface that will be used
CephBlockPool / CephFileSystem
Raises:
TimeoutError : in case of creation files take too long time
more then 2 Hours
"""
# Deploying elastic-search server in the cluster for use by the
# SmallFiles workload, since it is mandatory for the workload.
# This is deployed once for all test iterations and will be deleted
# in the end of the test.
self.es = ElasticSearch()
# Loading the main template yaml file for the benchmark and update some
# fields with new values
sf_data = templating.load_yaml(constants.SMALLFILE_BENCHMARK_YAML)
if interface == constants.CEPHBLOCKPOOL:
storageclass = constants.DEFAULT_STORAGECLASS_RBD
else:
storageclass = constants.DEFAULT_STORAGECLASS_CEPHFS
log.info(f"Using {storageclass} Storageclass")
# Setting up the parameters for this test
sf_data["spec"]["workload"]["args"]["samples"] = 1
sf_data["spec"]["workload"]["args"]["operation"] = ["create"]
sf_data["spec"]["workload"]["args"]["file_size"] = file_size
sf_data["spec"]["workload"]["args"]["files"] = files
sf_data["spec"]["workload"]["args"]["threads"] = threads
sf_data["spec"]["workload"]["args"]["storageclass"] = storageclass
sf_data["spec"]["elasticsearch"] = {
"url": f"http://{self.es.get_ip()}:{self.es.get_port()}"
}
"""
Calculating the size of the volume that need to be test, it should
be at least twice in the size then the size of the files, and at
least 100Gi.
Since the file_size is in Kb and the vol_size need to be in Gb, more
calculation is needed.
"""
total_files = int(files * threads)
total_data = int(files * threads * file_size / constants.GB2KB)
data_set = int(total_data * 3) # calculate data with replica
vol_size = data_set if data_set >= 100 else 100
sf_data["spec"]["workload"]["args"]["storagesize"] = f"{vol_size}Gi"
environment = get_environment_info()
if not environment["user"] == "":
sf_data["spec"]["test_user"] = environment["user"]
else:
# since full results object need this parameter, initialize it from CR file
environment["user"] = sf_data["spec"]["test_user"]
sf_data["spec"]["clustername"] = environment["clustername"]
log.debug(f"The smallfile yaml file is {sf_data}")
# Deploy the benchmark-operator, so we can use the SmallFiles workload
# to fill up the volume with files, and switch to the benchmark-operator namespace.
log.info("Deploy the benchmark-operator")
self.deploy_benchmark_operator()
switch_to_project(BMO_NAME)
all_results = []
self.results_path = get_full_test_logs_path(cname=self)
log.info(f"Logs file path name is : {self.full_log_path}")
# Produce ES report
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
self.full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"pvc_snapshot_perf_multiple_files",
))
self.full_results.add_key("file_size_inKB", file_size)
self.full_results.add_key("threads", threads)
self.full_results.add_key("interface", interface)
for test_num in range(self.tests_numbers):
test_results = {"creation_time": None, "csi_creation_time": None}
# deploy the smallfile workload
log.info("Running SmallFile bench")
sf_obj = OCS(**sf_data)
sf_obj.create()
# wait for benchmark pods to get created - takes a while
for bench_pod in TimeoutSampler(
240,
10,
get_pod_name_by_pattern,
"smallfile-client",
BMO_NAME,
):
try:
if bench_pod[0] is not None:
small_file_client_pod = bench_pod[0]
break
except IndexError:
log.info("Bench pod not ready yet")
bench_pod = OCP(kind="pod", namespace=BMO_NAME)
log.info("Waiting for SmallFile benchmark to Run")
assert bench_pod.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_name=small_file_client_pod,
sleep=30,
timeout=600,
)
# Initialize the pvc_name variable so it will not be in loop scope only.
pvc_name = ""
for item in bench_pod.get()["items"]:
if item.get("metadata").get("name") == small_file_client_pod:
for volume in item.get("spec").get("volumes"):
if "persistentVolumeClaim" in volume:
pvc_name = volume["persistentVolumeClaim"][
"claimName"]
break
log.info(f"Benchmark PVC name is : {pvc_name}")
# Creation of 1M files on CephFS can take a lot of time
timeout = 7200
while timeout >= 0:
logs = bench_pod.get_logs(name=small_file_client_pod)
if "RUN STATUS DONE" in logs:
break
timeout -= 30
if timeout == 0:
raise TimeoutError(
"Timed out waiting for benchmark to complete")
time.sleep(30)
log.info(f"Smallfile test ({test_num + 1}) finished.")
# Taking snapshot of the PVC (which contain files)
snap_name = pvc_name.replace("claim", "snapshot-")
log.info(f"Taking snapshot of the PVC {pvc_name}")
log.info(f"Snapshot name : {snap_name}")
start_time = datetime.datetime.utcnow().strftime(
"%Y-%m-%dT%H:%M:%SZ")
test_results["creation_time"] = self.measure_create_snapshot_time(
pvc_name=pvc_name,
snap_name=snap_name,
namespace=BMO_NAME,
interface=interface,
start_time=start_time,
)
log.info(
f"Snapshot with name {snap_name} and id {self.snap_uid} creation time is"
f' {test_results["creation_time"]} seconds')
test_results[
"csi_creation_time"] = performance_lib.measure_csi_snapshot_creation_time(
interface=interface,
snapshot_id=self.snap_uid,
start_time=start_time)
log.info(
f"Snapshot with name {snap_name} and id {self.snap_uid} csi creation time is"
f' {test_results["csi_creation_time"]} seconds')
all_results.append(test_results)
# Delete the smallfile workload - which will delete also the PVC
log.info("Deleting the smallfile workload")
if sf_obj.delete(wait=True):
log.info("The smallfile workload was deleted successfully")
# Delete VolumeSnapshots
log.info("Deleting the snapshots")
if self.snap_obj.delete(wait=True):
log.info("The snapshot deleted successfully")
log.info("Verify (and wait if needed) that ceph health is OK")
ceph_health_check(tries=45, delay=60)
# Sleep for 1 Min. between test samples
time.sleep(60)
# Cleanup the elasticsearch instance.
log.info("Deleting the elastic-search instance")
self.es.cleanup()
creation_times = [t["creation_time"] for t in all_results]
avg_c_time = statistics.mean(creation_times)
csi_creation_times = [t["csi_creation_time"] for t in all_results]
avg_csi_c_time = statistics.mean(csi_creation_times)
t_dateset = int(data_set / 3)
log.info(f"Full test report for {interface}:")
log.info(f"Test ran {self.tests_numbers} times, "
f"All snapshot creation results are {creation_times} seconds")
log.info(
f"The average snapshot creation time is : {avg_c_time} seconds")
log.info(f"Test ran {self.tests_numbers} times, "
f"All snapshot csi creation results are {csi_creation_times}")
log.info(
f"The average csi snapshot creation time is : {avg_csi_c_time}")
log.info(f"Number of Files on the volume : {total_files:,}, "
f"Total dataset : {t_dateset} GiB")
self.full_results.add_key("avg_snapshot_creation_time_insecs",
avg_c_time)
self.full_results.all_results["total_files"] = total_files
self.full_results.all_results["total_dataset"] = t_dateset
self.full_results.all_results["creation_time"] = creation_times
self.full_results.all_results["csi_creation_time"] = csi_creation_times
# Write the test results into the ES server
log.info("writing results to elastic search server")
if self.full_results.es_write():
res_link = self.full_results.results_link()
# write the ES link to the test results in the test log.
log.info(f"The result can be found at : {res_link}")
# Create text file with results of all subtest
self.write_result_to_file(res_link)
def test_clone_create_delete_performance(self, interface_type, pvc_size,
file_size, teardown_factory):
"""
Write data (60% of PVC capacity) to the PVC created in setup
Create clones for an existing pvc,
Measure clones average creation time and speed
Delete the created clone
Measure clone average deletion time and speed
Note: by increasing max_num_of_clones value you increase number of the clones to be created/deleted
"""
file_size_for_io = file_size[:-1]
performance_lib.write_fio_on_pod(self.pod_object, file_size_for_io)
max_num_of_clones = 10
clone_creation_measures = []
csi_clone_creation_measures = []
clones_list = []
timeout = 18000
sc_name = self.pvc_obj.backed_sc
parent_pvc = self.pvc_obj.name
clone_yaml = constants.CSI_RBD_PVC_CLONE_YAML
namespace = self.pvc_obj.namespace
if interface_type == constants.CEPHFILESYSTEM:
clone_yaml = constants.CSI_CEPHFS_PVC_CLONE_YAML
file_size_mb = convert_device_size(file_size, "MB")
logger.info(
f"Start creating {max_num_of_clones} clones on {interface_type} PVC of size {pvc_size} GB."
)
# taking the time, so parsing the provision log will be faster.
start_time = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
for i in range(max_num_of_clones):
logger.info(f"Start creation of clone number {i + 1}.")
cloned_pvc_obj = pvc.create_pvc_clone(sc_name,
parent_pvc,
clone_yaml,
namespace,
storage_size=pvc_size + "Gi")
teardown_factory(cloned_pvc_obj)
helpers.wait_for_resource_state(cloned_pvc_obj,
constants.STATUS_BOUND, timeout)
cloned_pvc_obj.reload()
logger.info(
f"Clone with name {cloned_pvc_obj.name} for {pvc_size} pvc {parent_pvc} was created."
)
clones_list.append(cloned_pvc_obj)
create_time = helpers.measure_pvc_creation_time(
interface_type, cloned_pvc_obj.name)
creation_speed = int(file_size_mb / create_time)
logger.info(
f"Clone number {i+1} creation time is {create_time} secs for {pvc_size} GB pvc."
)
logger.info(
f"Clone number {i+1} creation speed is {creation_speed} MB/sec for {pvc_size} GB pvc."
)
creation_measures = {
"clone_num": i + 1,
"time": create_time,
"speed": creation_speed,
}
clone_creation_measures.append(creation_measures)
csi_clone_creation_measures.append(
performance_lib.csi_pvc_time_measure(self.interface,
cloned_pvc_obj, "create",
start_time))
# deleting one by one and measuring deletion times and speed for each one of the clones create above
# in case of single clone will run one time
clone_deletion_measures = []
csi_clone_deletion_measures = []
logger.info(
f"Start deleting {max_num_of_clones} clones on {interface_type} PVC of size {pvc_size} GB."
)
for i in range(max_num_of_clones):
cloned_pvc_obj = clones_list[i]
pvc_reclaim_policy = cloned_pvc_obj.reclaim_policy
cloned_pvc_obj.delete()
logger.info(
f"Deletion of clone number {i + 1} , the clone name is {cloned_pvc_obj.name}."
)
cloned_pvc_obj.ocp.wait_for_delete(cloned_pvc_obj.name, timeout)
if pvc_reclaim_policy == constants.RECLAIM_POLICY_DELETE:
helpers.validate_pv_delete(cloned_pvc_obj.backed_pv)
delete_time = helpers.measure_pvc_deletion_time(
interface_type, cloned_pvc_obj.backed_pv)
logger.info(
f"Clone number {i + 1} deletion time is {delete_time} secs for {pvc_size} GB pvc."
)
deletion_speed = int(file_size_mb / delete_time)
logger.info(
f"Clone number {i+1} deletion speed is {deletion_speed} MB/sec for {pvc_size} GB pvc."
)
deletion_measures = {
"clone_num": i + 1,
"time": delete_time,
"speed": deletion_speed,
}
clone_deletion_measures.append(deletion_measures)
csi_clone_deletion_measures.append(
performance_lib.csi_pvc_time_measure(self.interface,
cloned_pvc_obj, "delete",
start_time))
logger.info(
f"Printing clone creation time and speed for {max_num_of_clones} clones "
f"on {interface_type} PVC of size {pvc_size} GB:")
for c in clone_creation_measures:
logger.info(
f"Clone number {c['clone_num']} creation time is {c['time']} secs for {pvc_size} GB pvc ."
)
logger.info(
f"Clone number {c['clone_num']} creation speed is {c['speed']} MB/sec for {pvc_size} GB pvc."
)
logger.info(
f"Clone deletion time and speed for {interface_type} PVC of size {pvc_size} GB are:"
)
creation_time_list = [r["time"] for r in clone_creation_measures]
creation_speed_list = [r["speed"] for r in clone_creation_measures]
average_creation_time = statistics.mean(creation_time_list)
average_csi_creation_time = statistics.mean(
csi_clone_creation_measures)
average_creation_speed = statistics.mean(creation_speed_list)
logger.info(f"Average creation time is {average_creation_time} secs.")
logger.info(
f"Average creation speed is {average_creation_speed} Mb/sec.")
for d in clone_deletion_measures:
logger.info(
f"Clone number {d['clone_num']} deletion time is {d['time']} secs for {pvc_size} GB pvc."
)
logger.info(
f"Clone number {d['clone_num']} deletion speed is {d['speed']} MB/sec for {pvc_size} GB pvc."
)
deletion_time_list = [r["time"] for r in clone_deletion_measures]
deletion_speed_list = [r["speed"] for r in clone_deletion_measures]
average_deletion_time = statistics.mean(deletion_time_list)
average_csi_deletion_time = statistics.mean(
csi_clone_deletion_measures)
average_deletion_speed = statistics.mean(deletion_speed_list)
logger.info(f"Average deletion time is {average_deletion_time} secs.")
logger.info(
f"Average deletion speed is {average_deletion_speed} Mb/sec.")
logger.info("test_clones_creation_performance finished successfully.")
self.results_path = get_full_test_logs_path(cname=self)
# Produce ES report
# Collecting environment information
self.get_env_info()
self.full_log_path = get_full_test_logs_path(cname=self)
self.results_path = get_full_test_logs_path(cname=self)
self.full_log_path += f"-{self.interface}-{pvc_size}-{file_size}"
logger.info(f"Logs file path name is : {self.full_log_path}")
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"pvc_clone_performance",
))
full_results.add_key("interface", self.interface)
full_results.add_key("total_clone_number", max_num_of_clones)
full_results.add_key("pvc_size", self.pvc_size)
full_results.add_key("average_clone_creation_time",
average_creation_time)
full_results.add_key("average_csi_clone_creation_time",
average_csi_creation_time)
full_results.add_key("average_clone_deletion_time",
average_deletion_time)
full_results.add_key("average_csi_clone_deletion_time",
average_csi_deletion_time)
full_results.add_key("average_clone_creation_speed",
average_creation_speed)
full_results.add_key("average_clone_deletion_speed",
average_deletion_speed)
full_results.all_results = {
"clone_creation_time": creation_time_list,
"csi_clone_creation_time": csi_clone_creation_measures,
"clone_deletion_time": deletion_time_list,
"csi_clone_deletion_time": csi_clone_deletion_measures,
"clone_creation_speed": creation_speed_list,
"clone_deletion_speed": deletion_speed_list,
}
# Write the test results into the ES server
if full_results.es_write():
res_link = full_results.results_link()
logger.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtest (8 - according to the parameters)
self.write_result_to_file(res_link)
def test_bulk_pvc_creation_after_deletion_performance(
self, interface_iterate, storageclass_factory
):
"""
Measuring PVC creation time of bulk of 75% of initial PVC bulk (120) in the same
rate after deleting ( serial deletion) 75% of the initial PVCs
and sends results to the Elastic Search DB
"""
self.interface = interface_iterate
self.sc_obj = storageclass_factory(self.interface)
initial_number_of_pvcs = 120
number_of_pvcs = math.ceil(initial_number_of_pvcs * 0.75)
# Getting the test start time
self.test_start_time = self.get_time()
log.info(f"Start creating new {initial_number_of_pvcs} PVCs in a bulk")
self.pvc_bulk_create_and_wait_for_bound(initial_number_of_pvcs)
log.info(f"Deleting 75% of the PVCs - {number_of_pvcs} PVCs")
assert pvc.delete_pvcs(
self.pvc_objs[:number_of_pvcs], True
), "Deletion of 75% of PVCs failed"
# save the list of pvcs which not deleted, for the teardown phase
original_pvcs = self.pvc_objs[number_of_pvcs:]
log.info(f"Re-creating the {number_of_pvcs} PVCs")
csi_bulk_start_time = self.get_time(time_format="csi")
self.pvc_bulk_create_and_wait_for_bound(number_of_pvcs)
# Get the bulk recraation time - total time.
total_time = self.get_bulk_creation_time()
log.info(
f"Creation after deletion time of {number_of_pvcs} is {total_time} seconds."
)
if total_time > 50:
raise ex.PerformanceException(
f"{number_of_pvcs} PVCs creation (after initial deletion of "
f"75% of PVCs) time is {total_time} and greater than 50 seconds."
)
log.info(f"{number_of_pvcs} PVCs creation time took less than a 50 seconds")
csi_creation_times = performance_lib.csi_bulk_pvc_time_measure(
self.interface, self.pvc_objs, "create", csi_bulk_start_time
)
# Getting the end time of the test
self.test_end_time = self.get_time()
# update the list of pvcs for the teardown process
self.pvc_objs += original_pvcs
# Produce ES report
self.results_path = os.path.join(
"/",
*self.results_path,
"test_bulk_pvc_creation_after_deletion_performance",
)
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"bulk_pvc_creation_after_deletion_measurement",
)
)
# Add the test time to the ES report
full_results.add_key(
"test_time", {"start": self.test_start_time, "end": self.test_end_time}
)
full_results.add_key("number_of_pvcs", number_of_pvcs)
full_results.add_key("creation_after_deletion_time", total_time)
full_results.add_key("creation_after_deletion_csi_time", csi_creation_times)
# Write the test results into the ES server
if full_results.es_write():
res_link = full_results.results_link()
log.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtest (2 - according to the parameters)
self.write_result_to_file(res_link)
def test_multiple_pvc_deletion_measurement_performance(
self, teardown_factory):
"""
Measuring PVC deletion time of 120 PVCs in 180 seconds
Args:
teardown_factory: A fixture used when we want a new resource that was created during the tests
to be removed in the teardown phase.
Returns:
"""
number_of_pvcs = 120
pvc_size = "1Gi"
msg_prefix = f"Interface: {self.interface}, PVC size: {pvc_size}."
log.info(f"{msg_prefix} Start creating new {number_of_pvcs} PVCs")
pvc_objs, _ = helpers.create_multiple_pvcs(
sc_name=self.sc_obj.name,
namespace=self.namespace,
number_of_pvc=number_of_pvcs,
size=pvc_size,
burst=True,
)
for pvc_obj in pvc_objs:
pvc_obj.reload()
teardown_factory(pvc_obj)
timeout = 600 if self.interface == constants.CEPHBLOCKPOOL_THICK else 60
with ThreadPoolExecutor(max_workers=5) as executor:
for pvc_obj in pvc_objs:
executor.submit(
helpers.wait_for_resource_state,
pvc_obj,
constants.STATUS_BOUND,
timeout=timeout,
)
executor.submit(pvc_obj.reload)
pod_objs = []
for pvc_obj in pvc_objs:
pod_obj = self.write_file_on_pvc(pvc_obj, 0.3)
pod_objs.append(pod_obj)
# Get pvc_name, require pvc_name to fetch deletion time data from log
threads = list()
for pvc_obj in pvc_objs:
process = threading.Thread(target=pvc_obj.reload)
process.start()
threads.append(process)
for process in threads:
process.join()
pvc_name_list, pv_name_list = ([] for i in range(2))
threads = list()
for pvc_obj in pvc_objs:
process1 = threading.Thread(
target=pvc_name_list.append(pvc_obj.name))
process2 = threading.Thread(
target=pv_name_list.append(pvc_obj.backed_pv))
process1.start()
process2.start()
threads.append(process1)
threads.append(process2)
for process in threads:
process.join()
log.info(f"{msg_prefix} Preparing to delete 120 PVC")
# Delete PVC
for pvc_obj, pod_obj in zip(pvc_objs, pod_objs):
pod_obj.delete(wait=True)
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(pvc_obj.name)
# Get PVC deletion time
pvc_deletion_time = helpers.measure_pv_deletion_time_bulk(
interface=self.interface, pv_name_list=pv_name_list)
log.info(
f"{msg_prefix} {number_of_pvcs} bulk deletion time is {pvc_deletion_time}"
)
# accepted deletion time is 2 secs for each PVC
accepted_pvc_deletion_time = number_of_pvcs * 2
for del_time in pvc_deletion_time.values():
if del_time > accepted_pvc_deletion_time:
raise ex.PerformanceException(
f"{msg_prefix} {number_of_pvcs} PVCs deletion time is {pvc_deletion_time.values()} and is "
f"greater than {accepted_pvc_deletion_time} seconds")
logging.info(f"{msg_prefix} {number_of_pvcs} PVCs deletion times are:")
for name, a_time in pvc_deletion_time.items():
logging.info(f"{name} deletion time is: {a_time} seconds")
if self.interface == constants.CEPHBLOCKPOOL:
self.sc = "RBD"
elif self.interface == constants.CEPHFILESYSTEM:
self.sc = "CephFS"
elif self.interface == constants.CEPHBLOCKPOOL_THICK:
self.sc = "RBD-Thick"
full_log_path = get_full_test_logs_path(
cname=self) + f"-{self.sc}-{pvc_size}"
self.results_path = get_full_test_logs_path(cname=self)
log.info(f"Logs file path name is : {full_log_path}")
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
full_log_path,
"pvc_bulk_deletion_fullres",
))
full_results.add_key("interface", self.interface)
full_results.add_key("bulk_size", number_of_pvcs)
full_results.add_key("pvc_size", pvc_size)
full_results.all_results["bulk_deletion_time"] = pvc_deletion_time
if full_results.es_write():
res_link = full_results.results_link()
log.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtest (3 - according to the parameters)
self.write_result_to_file(res_link)
def test_bulk_pvc_creation_after_deletion_performance(
self, teardown_factory):
"""
Measuring PVC creation time of bulk of 75% of initial PVC bulk (120) in the same
rate after deleting ( serial deletion) 75% of the initial PVCs
and sends results to the Elastic Search DB
Args:
teardown_factory: A fixture used when we want a new resource that was created during the tests
to be removed in the teardown phase.
Returns:
"""
initial_number_of_pvcs = 120
number_of_pvcs = math.ceil(initial_number_of_pvcs * 0.75)
log.info(f"Start creating new {initial_number_of_pvcs} PVCs in a bulk")
pvc_objs, _ = helpers.create_multiple_pvcs(
sc_name=self.sc_obj.name,
namespace=self.namespace,
number_of_pvc=initial_number_of_pvcs,
size=self.pvc_size,
burst=True,
)
for pvc_obj in pvc_objs:
teardown_factory(pvc_obj)
with ThreadPoolExecutor() as executor:
for pvc_obj in pvc_objs:
executor.submit(helpers.wait_for_resource_state, pvc_obj,
constants.STATUS_BOUND)
executor.submit(pvc_obj.reload)
log.info("Deleting 75% of the PVCs - 90 PVCs")
assert pvc.delete_pvcs(pvc_objs[:number_of_pvcs],
True), "Deletion of 75% of PVCs failed"
log.info("Re-creating the 90 PVCs")
pvc_objs, _ = helpers.create_multiple_pvcs(
sc_name=self.sc_obj.name,
namespace=self.namespace,
number_of_pvc=number_of_pvcs,
size=self.pvc_size,
burst=True,
)
start_time = helpers.get_provision_time(self.interface,
pvc_objs,
status="start")
end_time = helpers.get_provision_time(self.interface,
pvc_objs,
status="end")
total = end_time - start_time
total_time = total.total_seconds()
logging.info(
f"Creation after deletion time of {number_of_pvcs} is {total_time} seconds."
)
for pvc_obj in pvc_objs:
teardown_factory(pvc_obj)
with ThreadPoolExecutor() as executor:
for pvc_obj in pvc_objs:
executor.submit(helpers.wait_for_resource_state, pvc_obj,
constants.STATUS_BOUND)
executor.submit(pvc_obj.reload)
if total_time > 50:
raise ex.PerformanceException(
f"{number_of_pvcs} PVCs creation (after initial deletion of "
f"75% of PVCs) time is {total_time} and greater than 50 seconds."
)
logging.info(
f"{number_of_pvcs} PVCs creation time took less than a 50 seconds")
# Produce ES report
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"bulk_pvc_creation_after_deletion_measurement",
))
full_results.add_key("interface", self.interface)
full_results.add_key("number_of_pvcs", number_of_pvcs)
full_results.add_key("pvc_size", self.pvc_size)
full_results.add_key("creation_after_deletion_time", total_time)
# Write the test results into the ES server
full_results.es_write()
def test_bulk_clone_performance(self, tmp_path, interface_iterate):
"""
Creates number of PVCs in a bulk using kube job
Write 60% of PVC capacity to each one of the created PVCs
Creates 1 clone per each PVC altogether in a bulk
Measuring total and csi creation times for bulk of clones
"""
self.interface = interface_iterate
job_pod_file, job_pvc_file, job_clone_file = [None, None, None]
log.info(f"Start creating {self.interface} {self.pvc_count} PVC")
try:
pvc_dict_list = scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job(
no_of_pvc=self.pvc_count,
access_mode=Interfaces_info[self.interface]["accessmode"],
sc_name=Interfaces_info[self.interface]["sc_name"],
pvc_size=self.vol_size,
)
job_pvc_file = ObjectConfFile(
name="job_profile_pvc",
obj_dict_list=pvc_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
# Create kube_job
job_pvc_file.create(namespace=self.namespace)
# Check all the PVC reached Bound state
performance_lib.wait_for_resource_bulk_status(
resource="pvc",
resource_count=self.pvc_count,
namespace=self.namespace,
status=constants.STATUS_BOUND,
timeout=120,
sleep_time=5,
)
log.info(
f"All the PVCs ({self.pvc_count}) was created and are in Bound state"
)
# Getting the list of the PVC names
pvc_bound_list = [
p.name for p in pvc.get_all_pvc_objs(namespace=self.namespace)
]
# Kube_job to Create pod
log.info(
"Attaching PODs to the PVCs and filling them with data (60%)")
pod_dict_list = self.attach_pvcs_to_pod_dict(pvc_bound_list)
job_pod_file = ObjectConfFile(
name="job_profile_pod",
obj_dict_list=pod_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
job_pod_file.create(namespace=self.namespace)
# Check all PODs are in Completed state
performance_lib.wait_for_resource_bulk_status(
resource="pod",
resource_count=self.pvc_count,
namespace=self.namespace,
status=constants.STATUS_COMPLETED,
timeout=1200,
sleep_time=30,
)
log.info("All the PODs completed writing data to the PVC's")
clone_dict_list = scale_lib.construct_pvc_clone_yaml_bulk_for_kube_job(
pvc_dict_list,
Interfaces_info[self.interface]["clone_yaml"],
Interfaces_info[self.interface]["sc_name"],
)
log.info("Created clone dict list")
csi_bulk_start_time = self.get_time(time_format="csi")
job_clone_file = ObjectConfFile(
name="job_profile_clone",
obj_dict_list=clone_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
# Create kube_job that creates clones
job_clone_file.create(namespace=self.namespace)
log.info("Going to check bound status for clones")
# Check all the clones reached Bound state
try:
performance_lib.wait_for_resource_bulk_status(
resource="pvc",
resource_count=self.pvc_count * 2,
namespace=self.namespace,
status=constants.STATUS_BOUND,
timeout=1200,
sleep_time=30,
)
except Exception as ex:
log.error("Failed to cvreate clones for PVCs")
raise ex
log.info(
f"All the Clones ({self.pvc_count}) was created and are in Bound state"
)
all_pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace)
clone_objs = [
cl for cl in all_pvc_objs if re.match("clone", cl.name)
]
for clone_yaml in clone_dict_list:
name = clone_yaml["metadata"]["name"]
size = clone_yaml["spec"]["resources"]["requests"]["storage"]
log.info(f"Clone {name} of size {size} created")
start_time = get_provision_time(self.interface,
clone_objs,
status="start")
end_time = get_provision_time(self.interface,
clone_objs,
status="end")
total_time = (end_time - start_time).total_seconds()
speed = round(self.total_files_size / total_time, 2)
csi_creation_time = performance_lib.csi_bulk_pvc_time_measure(
self.interface, clone_objs, "create", csi_bulk_start_time)
log.info(
f"Total creation time = {total_time} secs, csi creation time = {csi_creation_time},"
f" data size = {self.total_files_size} MB, speed = {speed} MB/sec "
f"for {self.interface} clone in bulk of {self.pvc_count} clones."
)
# Produce ES report
# Collecting environment information
self.get_env_info()
# Initialize the results' doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"bulk_clone_perf_fullres",
))
full_results.add_key("interface", self.interface)
full_results.add_key("bulk_size", self.pvc_count)
full_results.add_key("clone_size", self.vol_size)
full_results.add_key("bulk_creation_time", total_time)
full_results.add_key("bulk_csi_creation_time", csi_creation_time)
full_results.add_key("data_size(MB)", self.total_files_size)
full_results.add_key("speed", speed)
full_results.add_key("es_results_link",
full_results.results_link())
# Write the test results into the ES server
full_results.es_write()
self.results_path = get_full_test_logs_path(cname=self)
res_link = full_results.results_link()
# write the ES link to the test results in the test log.
log.info(f"The result can be found at : {res_link}")
# Create text file with results of all subtest (3 - according to the parameters)
self.write_result_to_file(res_link)
# Finally, is used to clean up the resources created
# Irrespective of try block pass/fail finally will be executed.
finally:
# Cleanup activities
log.info(
"Cleanup of all the resources created during test execution")
for object_file in [job_pod_file, job_clone_file, job_pvc_file]:
if object_file:
object_file.delete(namespace=self.namespace)
try:
object_file.wait_for_delete(
resource_name=object_file.name,
namespace=self.namespace)
except Exception:
log.error(f"{object_file['name']} didnt deleted !")
# Check ceph health status
utils.ceph_health_check(tries=20)
def test_pvc_creation_deletion_measurement_performance(
self, interface_type, pvc_size):
"""
Measuring PVC creation and deletion times for pvc samples.
filling up each PVC with 70% of data.
Verifying that those times are within the required limits
Args:
interface_type (str): the interface type to run against -
CephBlockPool or CephFileSystem
pvc_size (str): the size of the pvc to create
"""
# Initializing test variables
self.interface = interface_type
num_of_samples = 5
if self.dev_mode:
num_of_samples = 2
accepted_creation_time = 1
accepted_deletion_time = Interface_Info[self.interface]["delete_time"]
accepted_creation_deviation_percent = 50
accepted_deletion_deviation_percent = 50
all_mesuring_times = {
"create": [],
"delete": [],
"csi_create": [],
"csi_delete": [],
}
msg_prefix = f"Interface: {self.interface}, PVC size: {pvc_size}."
self.set_results_path_and_file(
"test_pvc_creation_deletion_measurement_performance")
self.start_time = self.get_time()
self.get_env_info()
# Initialize the results doc file.
self.full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"pvc_create_delete_fullres",
))
self.full_results.add_key("pvc_size", pvc_size)
self.full_results.add_key("samples", num_of_samples)
self.create_fio_pod_yaml(pvc_size=int(pvc_size.replace("Gi", "")))
# Creating PVC(s) for creation time mesurment
start_time = self.create_pvcs_and_wait_for_bound(msg_prefix,
num_of_samples,
pvc_size,
burst=False)
# Fillup the PVC with data (70% of the total PVC size)
self.run_io()
# Deleting PVC(s) for deletion time mesurment
log.info("Try to delete all created PVCs")
for pvc_obj in self.pvc_objs:
pvc_obj.delete()
log.info("Wait for all PVC(s) to be deleted")
performance_lib.wait_for_resource_bulk_status("pvc", 0, self.namespace,
constants.STATUS_BOUND,
num_of_samples * 2, 5)
log.info("All PVC(s) was deleted")
mesure_data = "create"
rec_policy = performance_lib.run_oc_command(
f'get sc {Interface_Info[self.interface]["sc"]} -o jsonpath="' +
'{.reclaimPolicy}"')
if rec_policy[0].strip('"') == constants.RECLAIM_POLICY_DELETE:
log.info("Wait for all PVC(s) backed PV(s) to be deleted")
# Timeout for each PV to be deleted is 20 sec.
performance_lib.wait_for_resource_bulk_status(
"pv", 0, self.namespace, self.namespace, num_of_samples * 20,
5)
log.info("All backed PV(s) was deleted")
mesure_data = "all"
# Mesuring the time it took to create and delete the PVC(s)
log.info("Reading Creation/Deletion time from provisioner logs")
self.results_times = performance_lib.get_pvc_provision_times(
interface=self.interface,
pvc_name=self.pvc_objs,
start_time=start_time,
time_type="all",
op=mesure_data,
)
# Analaysing the test results
for i, pvc_res in enumerate(self.results_times):
data = self.results_times[pvc_res]
msg = f"{msg_prefix} PVC number {i + 1} was"
for op in Operations_Mesurment:
log.info(f"{msg} {op}d in {data[op]['time']} seconds.")
if data["create"]["time"] > accepted_creation_time:
raise ex.PerformanceException(
f"{msg_prefix} PVC creation time is {data['create']['time']} and is greater than "
f"{accepted_creation_time} seconds.")
if rec_policy == constants.RECLAIM_POLICY_DELETE:
if data["delete"]["time"] > accepted_deletion_time:
raise ex.PerformanceException(
f"{msg_prefix} PVC deletion time is {data['delete']['time']} and is greater than "
f"{accepted_deletion_time} seconds.")
all_mesuring_times["delete"].append(data["delete"]["time"])
all_mesuring_times["csi_delete"].append(
data["csi_delete"]["time"])
all_mesuring_times["create"].append(data["create"]["time"])
all_mesuring_times["csi_create"].append(data["csi_create"]["time"])
for op in Operations_Mesurment:
if rec_policy == constants.RECLAIM_POLICY_DELETE and "del" in op:
self.process_time_measurements(
op,
all_mesuring_times[op],
accepted_deletion_deviation_percent,
msg_prefix,
)
if "create" in op:
self.process_time_measurements(
op,
all_mesuring_times[op],
accepted_creation_deviation_percent,
msg_prefix,
)
self.full_results.all_results = self.results_times
self.end_time = self.get_time()
self.full_results.add_key("test_time", {
"start": self.start_time,
"end": self.end_time
})
if self.full_results.es_write():
res_link = self.full_results.results_link()
log.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtest (6 - according to the parameters)
self.write_result_to_file(res_link)
def test_mcg_cosbench_performance(self, cosbench):
"""
This test to perform reads and write objects to a bucket with multiple of samples and sizes.
The operation will be defined with number of % read and write.
After running main workload, performance numbers will be collected and saved to a spreadsheet for
performance analysing.
"""
bucket_prefix = "bucket-"
buckets = 1
objects = 10000
timeout = 3600
run_samples = 3
throughput_list = []
bandwidth_list = []
# Sizes in KB
self.sizes = [4, 16, 32, 128]
# Operations to perform and its ratio(%)
operations = {"read": 50, "write": 50}
# Deployment of cosbench
cosbench.setup_cosbench()
# Getting the start time of the test
self.test_start_time = self.get_time()
for size in self.sizes:
for i in range(run_samples):
# Create initial containers and objects
cosbench.run_init_workload(
prefix=bucket_prefix,
containers=buckets,
objects=objects,
validate=True,
size=size,
timeout=timeout,
)
# Run main workload
throughput, bandwidth = cosbench.run_main_workload(
operation_type=operations,
prefix=bucket_prefix,
containers=buckets,
objects=objects,
validate=True,
result=True,
size=size,
timeout=timeout,
)
throughput_list.append(throughput)
bandwidth_list.append(bandwidth)
# Dispose containers and objects
cosbench.run_cleanup_workload(
prefix=bucket_prefix,
containers=buckets,
objects=objects,
validate=True,
timeout=timeout,
)
# Getting the end time of the test
self.test_end_time = self.get_time()
# Collecting environment information
self.get_env_info()
# Initialize the results doc file
full_results = self.init_full_results(
ResultsAnalyse(self.uuid, self.crd_data, self.full_log_path,
"mcg_cosbench"))
# Add the result to ES report
full_results.add_key("test_time", {
"start": self.test_start_time,
"end": self.test_end_time
})
full_results.add_key("number_of_bucket", buckets)
full_results.add_key("number_of_objects", objects)
full_results.add_key("size_of_file", self.sizes)
full_results.add_key("throughput", throughput_list)
full_results.add_key("bandwidth", bandwidth_list)
self.results_path = get_full_test_logs_path(cname=self)
self.full_log_path = get_full_test_logs_path(cname=self)
self.full_log_path += f"-{self.sizes}"
# Write test results to ES server
if full_results.es_write():
res_link = full_results.results_link()
log.info(f"Results can be found at: {res_link}")
# Create text file with results
self.write_result_to_file(res_link)
def test_pvc_snapshot_performance_multiple_files(self, file_size, files,
threads, interface):
"""
Run SmallFile Workload and the take snapshot.
test will run with 1M of file on the volume - total data set
is the same for all tests, ~30GiB, and then take snapshot and measure
the time it takes.
the test will run 3 time to check consistency.
Args:
file_size (int): the size of the file to be create - in KiB
files (int): number of files each thread will create
threads (int): number of threads will be used in the workload
interface (str): the volume interface that will be used
CephBlockPool / CephFileSystem
Raises:
TimeoutError : in case of creation files take too long time
more then 2 Hours
"""
# Loading the main template yaml file for the benchmark and update some
# fields with new values
sf_data = templating.load_yaml(constants.SMALLFILE_BENCHMARK_YAML)
# Deploying elastic-search server in the cluster for use by the
# SmallFiles workload, since it is mandatory for the workload.
# This is deployed once for all test iterations and will be deleted
# in the end of the test.
if config.PERF.get("deploy_internal_es"):
self.es = ElasticSearch()
sf_data["spec"]["elasticsearch"] = {
"url": f"http://{self.es.get_ip()}:{self.es.get_port()}"
}
else:
if config.PERF.get("internal_es_server") == "":
self.es = None
return
else:
self.es = {
"server":
config.PERF.get("internal_es_server"),
"port":
config.PERF.get("internal_es_port"),
"url":
f"http://{config.PERF.get('internal_es_server')}:{config.PERF.get('internal_es_port')}",
}
# verify that the connection to the elasticsearch server is OK
if not super(TestPvcSnapshotPerformance, self).es_connect():
self.es = None
log.error(
"ElasticSearch doesn't exist ! The test cannot run")
return
sf_data["spec"]["elasticsearch"] = {"url": self.es["url"]}
if interface == constants.CEPHBLOCKPOOL:
storageclass = constants.DEFAULT_STORAGECLASS_RBD
else:
storageclass = constants.DEFAULT_STORAGECLASS_CEPHFS
log.info(f"Using {storageclass} Storageclass")
# Setting up the parameters for this test
sf_data["spec"]["workload"]["args"]["samples"] = 1
sf_data["spec"]["workload"]["args"]["operation"] = ["create"]
sf_data["spec"]["workload"]["args"]["file_size"] = file_size
sf_data["spec"]["workload"]["args"]["files"] = files
sf_data["spec"]["workload"]["args"]["threads"] = threads
sf_data["spec"]["workload"]["args"]["storageclass"] = storageclass
"""
Calculating the size of the volume that need to be test, it should
be at least twice in the size then the size of the files, and at
least 100Gi.
Since the file_size is in Kb and the vol_size need to be in Gb, more
calculation is needed.
"""
total_files = int(files * threads)
total_data = int(files * threads * file_size / constants.GB2KB)
data_set = int(total_data * 3) # calculate data with replica
vol_size = data_set if data_set >= 100 else 100
sf_data["spec"]["workload"]["args"]["storagesize"] = f"{vol_size}Gi"
environment = get_environment_info()
if not environment["user"] == "":
sf_data["spec"]["test_user"] = environment["user"]
else:
# since full results object need this parameter, initialize it from CR file
environment["user"] = sf_data["spec"]["test_user"]
sf_data["spec"]["clustername"] = environment["clustername"]
log.debug(f"The smallfile yaml file is {sf_data}")
# Deploy the benchmark-operator, so we can use the SmallFiles workload
# to fill up the volume with files, and switch to the benchmark-operator namespace.
log.info("Deploy the benchmark-operator")
self.deploy_benchmark_operator()
switch_to_project(BMO_NAME)
all_results = []
# Produce ES report
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
self.full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"pvc_snapshot_perf_multiple_files",
))
self.full_results.add_key("file_size_inKB", file_size)
self.full_results.add_key("threads", threads)
self.full_results.add_key("interface", interface)
for test_num in range(self.tests_numbers):
test_results = {"creation_time": None, "csi_creation_time": None}
# deploy the smallfile workload
self.crd_data = sf_data
self.client_pod_name = "smallfile-client"
self.deploy_and_wait_for_wl_to_start(timeout=240)
# Initialize the pvc_name variable so it will not be in loop scope only.
pvc_name = (OCP(kind="pvc", namespace=BMO_NAME).get().get("items")
[0].get("metadata").get("name"))
log.info(f"Benchmark PVC name is : {pvc_name}")
self.wait_for_wl_to_finish(sleep=30)
# Taking snapshot of the PVC (which contain files)
snap_name = pvc_name.replace("claim", "snapshot-")
log.info(f"Taking snapshot of the PVC {pvc_name}")
log.info(f"Snapshot name : {snap_name}")
start_time = self.get_time("csi")
test_results["creation_time"] = self.measure_create_snapshot_time(
pvc_name=pvc_name,
snap_name=snap_name,
namespace=BMO_NAME,
interface=interface,
start_time=start_time,
)
log.info(
f"Snapshot with name {snap_name} and id {self.snap_uid} creation time is"
f' {test_results["creation_time"]} seconds')
test_results[
"csi_creation_time"] = performance_lib.measure_csi_snapshot_creation_time(
interface=interface,
snapshot_id=self.snap_uid,
start_time=start_time)
log.info(
f"Snapshot with name {snap_name} and id {self.snap_uid} csi creation time is"
f' {test_results["csi_creation_time"]} seconds')
all_results.append(test_results)
# Delete the smallfile workload - which will delete also the PVC
log.info("Deleting the smallfile workload")
if self.benchmark_obj.delete(wait=True):
log.info("The smallfile workload was deleted successfully")
# Delete VolumeSnapshots
log.info("Deleting the snapshots")
if self.snap_obj.delete(wait=True):
log.info("The snapshot deleted successfully")
log.info("Verify (and wait if needed) that ceph health is OK")
ceph_health_check(tries=45, delay=60)
# Sleep for 1 Min. between test samples
time.sleep(60)
# Cleanup the elasticsearch instance, if needed.
if isinstance(self.es, ElasticSearch):
log.info("Deleting the elastic-search instance")
self.es.cleanup()
creation_times = [t["creation_time"] for t in all_results]
avg_c_time = statistics.mean(creation_times)
csi_creation_times = [t["csi_creation_time"] for t in all_results]
avg_csi_c_time = statistics.mean(csi_creation_times)
t_dateset = int(data_set / 3)
log.info(f"Full test report for {interface}:")
log.info(f"Test ran {self.tests_numbers} times, "
f"All snapshot creation results are {creation_times} seconds")
log.info(
f"The average snapshot creation time is : {avg_c_time} seconds")
log.info(f"Test ran {self.tests_numbers} times, "
f"All snapshot csi creation results are {csi_creation_times}")
log.info(
f"The average csi snapshot creation time is : {avg_csi_c_time}")
log.info(f"Number of Files on the volume : {total_files:,}, "
f"Total dataset : {t_dateset} GiB")
self.full_results.add_key("avg_snapshot_creation_time_insecs",
avg_c_time)
self.full_results.all_results["total_files"] = total_files
self.full_results.all_results["total_dataset"] = t_dateset
self.full_results.all_results["creation_time"] = creation_times
self.full_results.all_results["csi_creation_time"] = csi_creation_times
# Write the test results into the ES server
log.info("writing results to elastic search server")
self.results_path = helpers.get_full_test_logs_path(cname=self)
if self.full_results.es_write():
res_link = self.full_results.results_link()
# write the ES link to the test results in the test log.
log.info(f"The result can be found at : {res_link}")
# Create text file with results of all subtest
self.write_result_to_file(res_link)
def test_bulk_pod_attach_performance(self, interface_type, bulk_size):
"""
Measures pods attachment time in bulk_size bulk
Args:
interface_type (str): The interface type to be tested - CephBlockPool / CephFileSystem.
bulk_size (int): Size of the bulk to be tested
Returns:
"""
self.interface = interface_type
if self.dev_mode:
bulk_size = 3
# Initialize some variables
timeout = bulk_size * 5
pvc_names_list = list()
pod_data_list = list()
# Getting the test start time
test_start_time = self.get_time()
csi_start_time = self.get_time("csi")
log.info(f"Start creating bulk of new {bulk_size} PVCs")
self.pvc_objs, _ = helpers.create_multiple_pvcs(
sc_name=Interfaces_info[self.interface]["sc"],
namespace=self.namespace,
number_of_pvc=bulk_size,
size=self.pvc_size,
burst=True,
do_reload=False,
)
log.info("Wait for all of the PVCs to be in Bound state")
performance_lib.wait_for_resource_bulk_status("pvc", bulk_size,
self.namespace,
constants.STATUS_BOUND,
timeout, 10)
# in case of creation faliure, the wait_for_resource_bulk_status function
# will raise an exception. so in this point the creation succeed
log.info("All PVCs was created and in Bound state.")
# Reload all PVC(s) information
for pvc_obj in self.pvc_objs:
pvc_obj.reload()
pvc_names_list.append(pvc_obj.name)
log.debug(f"The PVCs names are : {pvc_names_list}")
# Create kube_job for pod creation
pod_data_list.extend(
scale_lib.attach_multiple_pvc_to_pod_dict(
pvc_list=pvc_names_list,
namespace=self.namespace,
pvcs_per_pod=1,
))
self.pods_obj = ObjectConfFile(
name="pod_kube_obj",
obj_dict_list=pod_data_list,
project=self.namespace,
tmp_path=pathlib.Path(ocsci_log_path()),
)
log.debug(f"PODs data list is : {json.dumps(pod_data_list, indent=3)}")
log.info(f"{self.interface} : Before pod attach")
bulk_start_time = time.time()
self.pods_obj.create(namespace=self.namespace)
# Check all the PODs reached Running state
log.info("Checking that pods are running")
performance_lib.wait_for_resource_bulk_status("pod", bulk_size,
self.namespace,
constants.STATUS_RUNNING,
timeout, 2)
log.info("All POD(s) are in Running State.")
bulk_end_time = time.time()
bulk_total_time = bulk_end_time - bulk_start_time
log.info(
f"Bulk attach time of {bulk_size} pods is {bulk_total_time} seconds"
)
csi_bulk_total_time = performance_lib.pod_bulk_attach_csi_time(
self.interface, self.pvc_objs, csi_start_time, self.namespace)
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(self.uuid, self.crd_data, self.full_log_path,
"pod_bulk_attachtime"))
full_results.add_key("storageclass",
Interfaces_info[self.interface]["name"])
full_results.add_key("pod_bulk_attach_time", bulk_total_time)
full_results.add_key("pod_csi_bulk_attach_time", csi_bulk_total_time)
full_results.add_key("pvc_size", self.pvc_size)
full_results.add_key("bulk_size", bulk_size)
# Getting the test end time
test_end_time = self.get_time()
# Add the test time to the ES report
full_results.add_key("test_time", {
"start": test_start_time,
"end": test_end_time
})
# Write the test results into the ES server
self.results_path = helpers.get_full_test_logs_path(cname=self)
if full_results.es_write():
res_link = full_results.results_link()
# write the ES link to the test results in the test log.
log.info(f"The result can be found at : {res_link}")
# Create text file with results of all subtests (4 - according to the parameters)
self.write_result_to_file(res_link)
def test_bulk_pod_attach_performance(self, teardown_factory, bulk_size):
"""
Measures pods attachment time in bulk_size bulk
Args:
teardown_factory: A fixture used when we want a new resource that was created during the tests
to be removed in the teardown phase.
bulk_size: Size of the bulk to be tested
Returns:
"""
# Getting the test start time
test_start_time = PASTest.get_time()
log.info(f"Start creating bulk of new {bulk_size} PVCs")
pvc_objs, _ = helpers.create_multiple_pvcs(
sc_name=self.sc_obj.name,
namespace=self.namespace,
number_of_pvc=bulk_size,
size=self.pvc_size,
burst=True,
)
for pvc_obj in pvc_objs:
pvc_obj.reload()
teardown_factory(pvc_obj)
with ThreadPoolExecutor(max_workers=5) as executor:
for pvc_obj in pvc_objs:
executor.submit(helpers.wait_for_resource_state, pvc_obj,
constants.STATUS_BOUND)
executor.submit(pvc_obj.reload)
start_time = helpers.get_provision_time(self.interface,
pvc_objs,
status="start")
end_time = helpers.get_provision_time(self.interface,
pvc_objs,
status="end")
total_time = (end_time - start_time).total_seconds()
log.info(
f"{self.interface}: Bulk of {bulk_size} PVCs creation time is {total_time} seconds."
)
pvc_names_list = []
for pvc_obj in pvc_objs:
pvc_names_list.append(pvc_obj.name)
log.info(f"{self.interface} : Before pod attach")
bulk_start_time = time.time()
pod_data_list = list()
pod_data_list.extend(
scale_lib.attach_multiple_pvc_to_pod_dict(
pvc_list=pvc_names_list,
namespace=self.namespace,
pvcs_per_pod=1,
))
lcl = locals()
tmp_path = pathlib.Path(ocsci_log_path())
obj_name = "obj1"
# Create kube_job for pod creation
lcl[f"pod_kube_{obj_name}"] = ObjectConfFile(
name=f"pod_kube_{obj_name}",
obj_dict_list=pod_data_list,
project=defaults.ROOK_CLUSTER_NAMESPACE,
tmp_path=tmp_path,
)
lcl[f"pod_kube_{obj_name}"].create(namespace=self.namespace)
log.info("Checking that pods are running")
# Check all the PODs reached Running state
pod_running_list = scale_lib.check_all_pod_reached_running_state_in_kube_job(
kube_job_obj=lcl[f"pod_kube_{obj_name}"],
namespace=self.namespace,
no_of_pod=len(pod_data_list),
timeout=180,
)
for pod_name in pod_running_list:
pod_obj = get_pod_obj(pod_name, self.namespace)
teardown_factory(pod_obj)
bulk_end_time = time.time()
bulk_total_time = bulk_end_time - bulk_start_time
log.info(
f"Bulk attach time of {len(pod_running_list)} pods is {bulk_total_time} seconds"
)
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_log_path = get_full_test_logs_path(cname=self)
self.results_path = get_full_test_logs_path(cname=self)
full_log_path += f"-{self.sc}"
full_results = self.init_full_results(
ResultsAnalyse(self.uuid, self.crd_data, full_log_path,
"pod_bulk_attachtime"))
full_results.add_key("storageclass", self.sc)
full_results.add_key("pod_bulk_attach_time", bulk_total_time)
full_results.add_key("pvc_size", self.pvc_size)
full_results.add_key("bulk_size", bulk_size)
# Getting the test end time
test_end_time = PASTest.get_time()
# Add the test time to the ES report
full_results.add_key("test_time", {
"start": test_start_time,
"end": test_end_time
})
# Write the test results into the ES server
if full_results.es_write():
res_link = full_results.results_link()
# write the ES link to the test results in the test log.
log.info(f"The result can be found at : {res_link}")
# Create text file with results of all subtest (4 - according to the parameters)
self.write_result_to_file(res_link)
def test_pvc_clone_performance_multiple_files(
self,
pvc_factory,
interface,
copies,
timeout,
):
"""
Test assign nodeName to a pod using RWX pvc
Each kernel (unzipped) is 892M and 61694 files
The test creates a pvc and a pods, writes kernel files multiplied by number of copies
The test creates number of clones samples, calculates creation and deletion times for each one the clones
and calculates the average creation and average deletion times
"""
kernel_url = "https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.19.5.tar.gz"
download_path = "tmp"
test_start_time = self.get_time()
helpers.pull_images(constants.PERF_IMAGE)
# Download a linux Kernel
dir_path = os.path.join(os.getcwd(), download_path)
file_path = os.path.join(dir_path, "file.gz")
if not os.path.exists(dir_path):
os.makedirs(dir_path)
urllib.request.urlretrieve(kernel_url, file_path)
# Create a PVC
accessmode = constants.ACCESS_MODE_RWX
if interface == constants.CEPHBLOCKPOOL:
accessmode = constants.ACCESS_MODE_RWO
pvc_size = "100"
try:
pvc_obj = pvc_factory(
interface=interface,
access_mode=accessmode,
status=constants.STATUS_BOUND,
size=pvc_size,
)
except Exception as e:
logger.error(f"The PVC sample was not created, exception {str(e)}")
raise PVCNotCreated("PVC did not reach BOUND state.")
# Create a pod on one node
logger.info(f"Creating Pod with pvc {pvc_obj.name}")
try:
pod_obj = helpers.create_pod(
interface_type=interface,
pvc_name=pvc_obj.name,
namespace=pvc_obj.namespace,
pod_dict_path=constants.PERF_POD_YAML,
)
except Exception as e:
logger.error(
f"Pod on PVC {pvc_obj.name} was not created, exception {str(e)}"
)
raise PodNotCreated("Pod on PVC was not created.")
# Confirm that pod is running on the selected_nodes
logger.info("Checking whether pods are running on the selected nodes")
helpers.wait_for_resource_state(resource=pod_obj,
state=constants.STATUS_RUNNING,
timeout=timeout)
pod_name = pod_obj.name
pod_path = "/mnt"
_ocp = OCP(namespace=pvc_obj.namespace)
rsh_cmd = f"rsync {dir_path} {pod_name}:{pod_path}"
_ocp.exec_oc_cmd(rsh_cmd)
rsh_cmd = f"exec {pod_name} -- tar xvf {pod_path}/tmp/file.gz -C {pod_path}/tmp"
_ocp.exec_oc_cmd(rsh_cmd)
for x in range(copies):
rsh_cmd = f"exec {pod_name} -- mkdir -p {pod_path}/folder{x}"
_ocp.exec_oc_cmd(rsh_cmd)
rsh_cmd = f"exec {pod_name} -- cp -r {pod_path}/tmp {pod_path}/folder{x}"
_ocp.exec_oc_cmd(rsh_cmd)
rsh_cmd = f"exec {pod_name} -- sync"
_ocp.exec_oc_cmd(rsh_cmd)
logger.info("Getting the amount of data written to the PVC")
rsh_cmd = f"exec {pod_name} -- df -h {pod_path}"
data_written = _ocp.exec_oc_cmd(rsh_cmd).split()[-4]
logger.info(f"The amount of written data is {data_written}")
rsh_cmd = f"exec {pod_name} -- find {pod_path} -type f"
files_written = len(_ocp.exec_oc_cmd(rsh_cmd).split())
logger.info(
f"For {interface} - The number of files written to the pod is {files_written}"
)
# delete the pod
pod_obj.delete(wait=False)
logger.info("Wait for the pod to be deleted")
performance_lib.wait_for_resource_bulk_status(
"pod", 0, pvc_obj.namespace, constants.STATUS_COMPLETED, timeout,
5)
logger.info("The pod was deleted")
num_of_clones = 11
# increasing the timeout since clone creation time is longer than pod attach time
timeout = 18000
clone_yaml = constants.CSI_RBD_PVC_CLONE_YAML
if interface == constants.CEPHFILESYSTEM:
clone_yaml = constants.CSI_CEPHFS_PVC_CLONE_YAML
clone_creation_measures = []
csi_clone_creation_measures = []
clone_deletion_measures = []
csi_clone_deletion_measures = []
# taking the time, so parsing the provision log will be faster.
start_time = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
for i in range(num_of_clones):
logger.info(f"Start creation of clone number {i + 1}.")
cloned_pvc_obj = pvc.create_pvc_clone(
pvc_obj.backed_sc,
pvc_obj.name,
clone_yaml,
pvc_obj.namespace,
storage_size=pvc_size + "Gi",
)
helpers.wait_for_resource_state(cloned_pvc_obj,
constants.STATUS_BOUND, timeout)
cloned_pvc_obj.reload()
logger.info(
f"Clone with name {cloned_pvc_obj.name} for {pvc_size} pvc {pvc_obj.name} was created."
)
create_time = helpers.measure_pvc_creation_time(
interface, cloned_pvc_obj.name)
logger.info(
f"Clone number {i+1} creation time is {create_time} secs for {pvc_size} GB pvc."
)
clone_creation_measures.append(create_time)
csi_clone_creation_measures.append(
performance_lib.csi_pvc_time_measure(interface, cloned_pvc_obj,
"create", start_time))
pvc_reclaim_policy = cloned_pvc_obj.reclaim_policy
cloned_pvc_obj.delete()
logger.info(
f"Deletion of clone number {i + 1} , the clone name is {cloned_pvc_obj.name}."
)
cloned_pvc_obj.ocp.wait_for_delete(cloned_pvc_obj.name, timeout)
if pvc_reclaim_policy == constants.RECLAIM_POLICY_DELETE:
helpers.validate_pv_delete(cloned_pvc_obj.backed_pv)
delete_time = helpers.measure_pvc_deletion_time(
interface, cloned_pvc_obj.backed_pv)
logger.info(
f"Clone number {i + 1} deletion time is {delete_time} secs for {pvc_size} GB pvc."
)
clone_deletion_measures.append(delete_time)
csi_clone_deletion_measures.append(
performance_lib.csi_pvc_time_measure(interface, cloned_pvc_obj,
"delete", start_time))
os.remove(file_path)
os.rmdir(dir_path)
pvc_obj.delete()
average_creation_time = statistics.mean(clone_creation_measures)
logger.info(f"Average creation time is {average_creation_time} secs.")
average_csi_creation_time = statistics.mean(
csi_clone_creation_measures)
logger.info(
f"Average csi creation time is {average_csi_creation_time} secs.")
average_deletion_time = statistics.mean(clone_deletion_measures)
logger.info(f"Average deletion time is {average_deletion_time} secs.")
average_csi_deletion_time = statistics.mean(
csi_clone_deletion_measures)
logger.info(
f"Average csi deletion time is {average_csi_deletion_time} secs.")
# Produce ES report
# Collecting environment information
self.get_env_info()
self.results_path = get_full_test_logs_path(cname=self)
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"test_pvc_clone_performance_multiple_files_fullres",
))
full_results.add_key("files_number", files_written)
test_end_time = self.get_time()
full_results.add_key("test_time", {
"start": test_start_time,
"end": test_end_time
})
full_results.add_key("interface", interface)
full_results.add_key("clones_number", num_of_clones)
full_results.add_key("pvc_size", pvc_size)
full_results.add_key("average_clone_creation_time",
average_creation_time)
full_results.add_key("average_csi_clone_creation_time",
average_csi_creation_time)
full_results.add_key("average_clone_deletion_time",
average_deletion_time)
full_results.add_key("average_csi_clone_deletion_time",
average_csi_deletion_time)
full_results.all_results = {
"clone_creation_time": clone_creation_measures,
"csi_clone_creation_time": csi_clone_creation_measures,
"clone_deletion_time": clone_deletion_measures,
"csi_clone_deletion_time": csi_clone_deletion_measures,
}
# Write the test results into the ES server
if full_results.es_write():
res_link = full_results.results_link()
logger.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtest (4 - according to the parameters)
self.results_path = get_full_test_logs_path(
cname=self, fname="test_pvc_clone_performance_multiple_files")
self.write_result_to_file(res_link)
def test_bulk_clone_performance(self, namespace, tmp_path):
"""
Creates number of PVCs in a bulk using kube job
Write 60% of PVC capacity to each one of the created PVCs
Creates 1 clone per each PVC altogether in a bulk
Measuring total and csi creation times for bulk of clones
"""
pvc_count = 50
vol_size = "5Gi"
job_pod_file, job_pvc_file, job_clone_file = [None, None, None]
log.info(f"Start creating {self.interface} {pvc_count} PVC")
if self.interface == constants.CEPHBLOCKPOOL:
sc_name = constants.DEFAULT_STORAGECLASS_RBD
clone_yaml = constants.CSI_RBD_PVC_CLONE_YAML
elif self.interface == constants.CEPHFILESYSTEM:
sc_name = constants.DEFAULT_STORAGECLASS_CEPHFS
clone_yaml = constants.CSI_CEPHFS_PVC_CLONE_YAML
try:
pvc_dict_list = scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job(
no_of_pvc=pvc_count,
access_mode=constants.ACCESS_MODE_RWO,
sc_name=sc_name,
pvc_size=vol_size,
)
job_pvc_file = ObjectConfFile(
name="job_profile_pvc",
obj_dict_list=pvc_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
# Create kube_job
job_pvc_file.create(namespace=self.namespace)
# Check all the PVC reached Bound state
pvc_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_pvc_file,
namespace=self.namespace,
no_of_pvc=pvc_count,
)
log.info(f"Number of PVCs in Bound state {len(pvc_bound_list)}")
# Kube_job to Create pod
pod_dict_list = scale_lib.attach_multiple_pvc_to_pod_dict(
pvc_list=pvc_bound_list,
namespace=self.namespace,
pvcs_per_pod=1,
start_io=False,
pod_yaml=constants.NGINX_POD_YAML,
)
job_pod_file = ObjectConfFile(
name="job_profile_pod",
obj_dict_list=pod_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
job_pod_file.create(namespace=self.namespace)
# Check all PODs in Running state
scale_lib.check_all_pod_reached_running_state_in_kube_job(
kube_job_obj=job_pod_file,
namespace=self.namespace,
no_of_pod=len(pod_dict_list),
timeout=90,
)
log.info(f"Number of PODs in Running state {len(pod_dict_list)}")
total_files_size = self.run_fio_on_pvcs(vol_size)
clone_dict_list = scale_lib.construct_pvc_clone_yaml_bulk_for_kube_job(
pvc_dict_list, clone_yaml, sc_name)
log.info("Created clone dict list")
csi_bulk_start_time = self.get_time(time_format="csi")
job_clone_file = ObjectConfFile(
name="job_profile_clone",
obj_dict_list=clone_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
# Create kube_job that creates clones
job_clone_file.create(namespace=self.namespace)
log.info("Going to check bound status for clones")
# Check all the clones reached Bound state
clone_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_clone_file,
namespace=self.namespace,
no_of_pvc=pvc_count,
timeout=180,
)
log.info(
f"Number of clones in Bound state {len(clone_bound_list)}")
clone_objs = []
all_pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace)
for clone_yaml in clone_dict_list:
name = clone_yaml["metadata"]["name"]
size = clone_yaml["spec"]["resources"]["requests"]["storage"]
log.info(f"Clone {name} of size {size} created")
for pvc_obj in all_pvc_objs:
if pvc_obj.name == name:
clone_objs.append(pvc_obj)
assert len(clone_bound_list) == len(
clone_objs
), "Not all clones reached BOUND state, cannot measure time"
start_time = helpers.get_provision_time(self.interface,
clone_objs,
status="start")
end_time = helpers.get_provision_time(self.interface,
clone_objs,
status="end")
total_time = (end_time - start_time).total_seconds()
speed = round(total_files_size / total_time, 2)
csi_creation_time = performance_lib.csi_bulk_pvc_time_measure(
self.interface, clone_objs, "create", csi_bulk_start_time)
log.info(
f"Total creation time = {total_time} secs, csi creation time = {csi_creation_time},"
f" data size = {total_files_size} MB, speed = {speed} MB/sec "
f"for {self.interface} clone in bulk of {pvc_count} clones.")
# Produce ES report
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"bulk_clone_perf_fullres",
))
full_results.add_key("interface", self.interface)
full_results.add_key("bulk_size", pvc_count)
full_results.add_key("clone_size", vol_size)
full_results.add_key("bulk_creation_time", total_time)
full_results.add_key("bulk_csi_creation_time", csi_creation_time)
full_results.add_key("data_size(MB)", total_files_size)
full_results.add_key("speed", speed)
full_results.add_key("es_results_link",
full_results.results_link())
# Write the test results into the ES server
full_results.es_write()
self.results_path = get_full_test_logs_path(cname=self)
res_link = full_results.results_link()
# write the ES link to the test results in the test log.
log.info(f"The result can be found at : {res_link}")
# Create text file with results of all subtest (3 - according to the parameters)
self.write_result_to_file(res_link)
# Finally is used to clean-up the resources created
# Irrespective of try block pass/fail finally will be executed.
finally:
# Cleanup activities
log.info(
"Cleanup of all the resources created during test execution")
if job_pod_file:
job_pod_file.delete(namespace=self.namespace)
job_pod_file.wait_for_delete(resource_name=job_pod_file.name,
namespace=self.namespace)
if job_clone_file:
job_clone_file.delete(namespace=self.namespace)
job_clone_file.wait_for_delete(
resource_name=job_clone_file.name,
namespace=self.namespace)
if job_pvc_file:
job_pvc_file.delete(namespace=self.namespace)
job_pvc_file.wait_for_delete(resource_name=job_pvc_file.name,
namespace=self.namespace)
# Check ceph health status
utils.ceph_health_check(tries=20)
def test_multiple_pvc_deletion_measurement_performance(
self, interface_type):
"""
Measuring PVC deletion time of 120 PVCs in 180 seconds
Args:
interface_type: the inteface type which the test run with - RBD / CephFS.
"""
# Initialize the test variables
self.interface = interface_type
number_of_pvcs = 120
if self.dev_mode:
number_of_pvcs = 5
pvc_size = "1Gi"
# accepted deletion time is 2 secs for each PVC
accepted_pvc_deletion_time = number_of_pvcs * 2
msg_prefix = f"Interface: {self.interface}, PVC size: {pvc_size}."
self.set_results_path_and_file(
"test_multiple_pvc_deletion_measurement_performance")
bulk_data = {
"create": {
"start": [],
"end": []
},
"csi_create": {
"start": [],
"end": []
},
"delete": {
"start": [],
"end": []
},
"csi_delete": {
"start": [],
"end": []
},
}
bulk_times = {
"create": None,
"delete": None,
"csi_create": None,
"csi_delete": None,
}
self.start_time = self.get_time()
self.get_env_info()
# Initialize the results doc file.
self.full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"pvc_bulk_deletion_fullres",
))
self.full_results.add_key("bulk_size", number_of_pvcs)
self.full_results.add_key("pvc_size", pvc_size)
self.create_fio_pod_yaml(pvc_size=int(pvc_size.replace("Gi", "")))
# Creating PVC(s) for creation time mesurment and wait for bound state
start_time = self.create_pvcs_and_wait_for_bound(msg_prefix,
number_of_pvcs,
pvc_size,
burst=True)
# Fillup the PVC with data (70% of the total PVC size)
self.run_io()
# Deleting PVC(s) for deletion time mesurment
log.info("Try to delete all created PVCs")
for pvc_obj in self.pvc_objs:
pvc_obj.delete(wait=False)
performance_lib.wait_for_resource_bulk_status("pvc", 0, self.namespace,
constants.STATUS_BOUND,
number_of_pvcs * 2, 5)
log.info("All PVC(s) was deleted")
log.info("Wait for all PVC(s) backed PV(s) to be deleted")
# Timeout for each PV to be deleted is 20 sec.
performance_lib.wait_for_resource_bulk_status("pv", 0, self.namespace,
self.namespace,
number_of_pvcs * 20, 5)
log.info("All backed PV(s) was deleted")
# Mesuring the time it took to delete the PVC(s)
log.info("Reading Creation/Deletion time from provisioner logs")
self.results_times = performance_lib.get_pvc_provision_times(
interface=self.interface,
pvc_name=self.pvc_objs,
start_time=start_time,
time_type="all",
op="all",
)
for i, pvc_res in enumerate(self.results_times):
data = self.results_times[pvc_res]
msg = f"{msg_prefix} PVC number {i + 1} was"
for op in Operations_Mesurment:
log.info(f"{msg} {op}d in {data[op]['time']} seconds.")
bulk_data[op]["start"].append(data[op]["start"])
bulk_data[op]["end"].append(data[op]["end"])
if data["delete"]["time"] > accepted_pvc_deletion_time:
raise ex.PerformanceException(
f"{msg_prefix} {number_of_pvcs} PVCs deletion time is {data['delete']['time']} "
f"and is greater than {accepted_pvc_deletion_time} seconds"
)
for op in Operations_Mesurment:
bulk_times[op] = {
"start": sorted(bulk_data[op]["start"])[0],
"end": sorted(bulk_data[op]["end"])[-1],
"time": None,
}
bulk_times[op]["time"] = performance_lib.calculate_operation_time(
f"bulk_{op}", bulk_times[op])
log.info(
f"Bulk {op}ion Time is : { bulk_times[op]['time']} seconds")
self.full_results.add_key(f"multi_{op}", bulk_times[op]["time"])
self.full_results.all_results = self.results_times
self.end_time = self.get_time()
self.full_results.add_key("test_time", {
"start": self.start_time,
"end": self.end_time
})
if self.full_results.es_write():
res_link = self.full_results.results_link()
log.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtest (3 - according to the parameters)
self.write_result_to_file(res_link)
def test_pvc_creation_deletion_measurement_performance(
self, teardown_factory, pvc_size):
"""
Measuring PVC creation and deletion times for pvc samples
Verifying that those times are within the required limits
"""
# Getting the full path for the test logs
self.full_log_path = get_full_test_logs_path(cname=self)
self.results_path = get_full_test_logs_path(cname=self)
if self.interface == constants.CEPHBLOCKPOOL:
self.sc = "RBD"
elif self.interface == constants.CEPHFILESYSTEM:
self.sc = "CephFS"
elif self.interface == constants.CEPHBLOCKPOOL_THICK:
self.sc = "RBD-Thick"
self.full_log_path += f"-{self.sc}-{pvc_size}"
log.info(f"Logs file path name is : {self.full_log_path}")
self.start_time = time.strftime("%Y-%m-%dT%H:%M:%SGMT", time.gmtime())
self.get_env_info()
# Initialize the results doc file.
self.full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"pvc_create_delete_fullres",
))
self.full_results.add_key("pvc_size", pvc_size)
num_of_samples = 5
accepted_creation_time = (600 if self.interface
== constants.CEPHBLOCKPOOL_THICK else 1)
# accepted deletion time for RBD is 1 sec, for CephFS is 2 secs and for RBD Thick is 5 secs
if self.interface == constants.CEPHFILESYSTEM:
accepted_deletion_time = 2
elif self.interface == constants.CEPHBLOCKPOOL:
accepted_deletion_time = 1
else:
accepted_deletion_time = 5
self.full_results.add_key("samples", num_of_samples)
accepted_creation_deviation_percent = 50
accepted_deletion_deviation_percent = 50
creation_time_measures = []
deletion_time_measures = []
msg_prefix = f"Interface: {self.interface}, PVC size: {pvc_size}."
for i in range(num_of_samples):
logging.info(f"{msg_prefix} Start creating PVC number {i + 1}.")
start_time = datetime.datetime.utcnow().strftime(
"%Y-%m-%dT%H:%M:%SZ")
pvc_obj = helpers.create_pvc(sc_name=self.sc_obj.name,
size=pvc_size)
timeout = 600 if self.interface == constants.CEPHBLOCKPOOL_THICK else 60
helpers.wait_for_resource_state(pvc_obj,
constants.STATUS_BOUND,
timeout=timeout)
pvc_obj.reload()
creation_time = performance_lib.measure_pvc_creation_time(
self.interface, pvc_obj.name, start_time)
logging.info(
f"{msg_prefix} PVC number {i + 1} was created in {creation_time} seconds."
)
if creation_time > accepted_creation_time:
raise ex.PerformanceException(
f"{msg_prefix} PVC creation time is {creation_time} and is greater than "
f"{accepted_creation_time} seconds.")
creation_time_measures.append(creation_time)
pv_name = pvc_obj.backed_pv
pvc_reclaim_policy = pvc_obj.reclaim_policy
pod_obj = self.write_file_on_pvc(pvc_obj)
pod_obj.delete(wait=True)
teardown_factory(pvc_obj)
logging.info(f"{msg_prefix} Start deleting PVC number {i + 1}")
if pvc_reclaim_policy == constants.RECLAIM_POLICY_DELETE:
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(pvc_obj.name)
helpers.validate_pv_delete(pvc_obj.backed_pv)
deletion_time = helpers.measure_pvc_deletion_time(
self.interface, pv_name)
logging.info(
f"{msg_prefix} PVC number {i + 1} was deleted in {deletion_time} seconds."
)
if deletion_time > accepted_deletion_time:
raise ex.PerformanceException(
f"{msg_prefix} PVC deletion time is {deletion_time} and is greater than "
f"{accepted_deletion_time} seconds.")
deletion_time_measures.append(deletion_time)
else:
logging.info(
f"Reclaim policy of the PVC {pvc_obj.name} is not Delete;"
f" therefore not measuring deletion time for this PVC.")
creation_average = self.process_time_measurements(
"creation",
creation_time_measures,
accepted_creation_deviation_percent,
msg_prefix,
)
self.full_results.add_key("creation-time", creation_average)
deletion_average = self.process_time_measurements(
"deletion",
deletion_time_measures,
accepted_deletion_deviation_percent,
msg_prefix,
)
self.full_results.add_key("deletion-time", deletion_average)
self.full_results.all_results["creation"] = creation_time_measures
self.full_results.all_results["deletion"] = deletion_time_measures
self.end_time = time.strftime("%Y-%m-%dT%H:%M:%SGMT", time.gmtime())
self.full_results.add_key("test_time", {
"start": self.start_time,
"end": self.end_time
})
if self.full_results.es_write():
res_link = self.full_results.results_link()
log.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtest (4 - according to the parameters)
self.write_result_to_file(res_link)
def test_pvc_multiple_snapshot_performance(
self,
pvc_factory,
pod_factory,
secret_factory,
interface_type,
snap_number,
):
"""
1. Creating PVC
size is depend on storage capacity, but not less then 1 GiB
it will use ~75% capacity of the Storage, Min storage capacity 1 TiB
2. Fill the PVC with 80% of data
3. Take a snapshot of the PVC and measure the total and CSI times of creation.
4. re-write the data on the PVC
5. Take a snapshot of the PVC and measure the total and the CSI times of creation.
6. repeat steps 4-5 the numbers of snapshot we want to take : 512
this will be run by outside script for low memory consumption
7. print all information.
Raises:
StorageNotSufficientException: in case of not enough capacity
"""
# Getting the full path for the test logs
self.results_path = get_full_test_logs_path(cname=self)
self.full_log_path = f"{self.results_path}-{interface_type}-{snap_number}"
log.info(f"Logs file path name is : {self.full_log_path}")
log.info(f"Reslut path is : {self.results_path}")
self.full_teardown = True
self.num_of_snaps = snap_number
if self.dev_mode:
self.num_of_snaps = 2
log.info(
f"Going to create {self.num_of_snaps} {interface_type} snapshots")
# since we do not want to use more then 65%, we add 35% to the needed
# capacity, and minimum PVC size is 1 GiB
self.need_capacity = int((self.num_of_snaps + 2) * 1.35)
# Test will run only on system with enough capacity
if self.capacity_to_use < self.need_capacity:
err_msg = (f"The system have only {self.ceph_capacity} GiB, "
f"we want to use only {self.capacity_to_use} GiB, "
f"and we need {self.need_capacity} GiB to run the test")
log.error(err_msg)
raise exceptions.StorageNotSufficientException(err_msg)
# Calculating the PVC size in GiB
self.pvc_size = int(self.capacity_to_use / (self.num_of_snaps + 2))
if self.dev_mode:
self.pvc_size = 5
self.interface = interface_type
self.sc_name = "pas-testing-rbd"
pool_name = self.sc_name
if self.interface == constants.CEPHFILESYSTEM:
self.sc_name = "pas-testing-cephfs"
pool_name = f"{self.sc_name}-data0"
# Creating new storage pool
self.create_new_pool(self.sc_name)
# Creating new StorageClass (pool) for the test.
secret = secret_factory(interface=self.interface)
self.sc_obj = helpers.create_storage_class(
interface_type=self.interface,
interface_name=pool_name,
secret_name=secret.name,
sc_name=self.sc_name,
fs_name=self.sc_name,
)
log.info(f"The new SC is : {self.sc_obj.name}")
log.debug(f"All SC data is {json.dumps(self.sc_obj.data, indent=3)}")
# Create new VolumeSnapshotClass
self.snap_class = self.create_snapshotclass(self.interface)
# Create new PVC
log.info(f"Creating {self.pvc_size} GiB PVC of {interface_type}")
self.pvc_obj = pvc_factory(
interface=self.interface,
storageclass=self.sc_obj,
size=self.pvc_size,
status=constants.STATUS_BOUND,
project=self.proj,
)
# Create POD which will attache to the new PVC
log.info("Creating A POD")
self.pod_obj = pod_factory(
interface=self.interface,
pvc=self.pvc_obj,
status=constants.STATUS_RUNNING,
pod_dict_path=constants.PERF_POD_YAML,
)
# Calculating the file size as 80% of the PVC size
self.filesize = self.pvc_obj.size * 0.80
# Change the file size to MB for the FIO function
self.file_size = f"{int(self.filesize * constants.GB2MB)}M"
self.file_name = self.pod_obj.name
log.info(
f"Total capacity size is : {self.ceph_capacity} GiB, "
f"Going to use {self.need_capacity} GiB, "
f"With {self.num_of_snaps} Snapshots to {self.pvc_size} GiB PVC. "
f"File size to be written is : {self.file_size} "
f"with the name of {self.file_name}")
# Reading basic snapshot yaml file
self.snap_yaml = constants.CSI_CEPHFS_SNAPSHOT_YAML
self.sc = constants.DEFAULT_VOLUMESNAPSHOTCLASS_CEPHFS
self.fs_type = "cephfs"
if interface_type == constants.CEPHBLOCKPOOL:
self.snap_yaml = constants.CSI_RBD_SNAPSHOT_YAML
self.fs_type = "rbd"
self.sc = constants.DEFAULT_VOLUMESNAPSHOTCLASS_RBD
with open(self.snap_yaml, "r") as stream:
try:
self.snap_templ = yaml.safe_load(stream)
self.snap_templ["spec"]["volumeSnapshotClassName"] = self.sc
self.snap_templ["spec"]["source"][
"persistentVolumeClaimName"] = self.pvc_obj.name
except yaml.YAMLError as exc:
log.error(f"Can not read template yaml file {exc}")
log.debug(
f"Snapshot yaml file : {self.snap_yaml} "
f"Content of snapshot yaml file {json.dumps(self.snap_templ, indent=4)}"
)
self.build_fio_command()
self.start_time = self.get_time()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(self.uuid, self.crd_data, self.full_log_path,
"multiple_snapshots"))
full_results.all_results = self.run()
self.end_time = self.get_time()
full_results.add_key(
"avg_creation_time",
f"{float(self.total_creation_time / self.num_of_snaps):.2f}",
)
full_results.add_key(
"avg_csi_creation_time",
f"{float(self.total_csi_creation_time / self.num_of_snaps):.2f}",
)
full_results.add_key(
"avg_creation_speed",
f"{float(self.total_creation_speed / self.num_of_snaps):.2f}",
)
full_results.add_key("test_time", {
"start": self.start_time,
"end": self.end_time
})
# Writing the analyzed test results to the Elastic-Search server
if full_results.es_write():
res_link = full_results.results_link()
log.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtests (2 - according to the parameters)
self.write_result_to_file(res_link)
def test_bulk_pvc_creation_deletion_measurement_performance(
self, teardown_factory, bulk_size):
"""
Measuring PVC creation and deletion time of bulk_size PVCs
and sends results to the Elastic Search DB
Args:
teardown_factory: A fixture used when we want a new resource that was created during the tests
to be removed in the teardown phase.
bulk_size: Size of the bulk to be tested
Returns:
"""
bulk_creation_time_limit = bulk_size / 2
log.info(f"Start creating new {bulk_size} PVCs")
pvc_objs, yaml_creation_dir = helpers.create_multiple_pvcs(
sc_name=self.sc_obj.name,
namespace=self.namespace,
number_of_pvc=bulk_size,
size=self.pvc_size,
burst=True,
)
logging.info(f"PVC creation dir is {yaml_creation_dir}")
for pvc_obj in pvc_objs:
pvc_obj.reload()
teardown_factory(pvc_obj)
with ThreadPoolExecutor(max_workers=5) as executor:
for pvc_obj in pvc_objs:
executor.submit(helpers.wait_for_resource_state, pvc_obj,
constants.STATUS_BOUND)
executor.submit(pvc_obj.reload)
start_time = helpers.get_provision_time(self.interface,
pvc_objs,
status="start")
end_time = helpers.get_provision_time(self.interface,
pvc_objs,
status="end")
total_time = (end_time - start_time).total_seconds()
logging.info(
f"{bulk_size} Bulk PVCs creation time is {total_time} seconds.")
if total_time > bulk_creation_time_limit:
raise ex.PerformanceException(
f"{bulk_size} Bulk PVCs creation time is {total_time} and "
f"greater than {bulk_creation_time_limit} seconds")
pv_names_list = []
for pvc_obj in pvc_objs:
pv_names_list.append(pvc_obj.backed_pv)
logging.info(f"Starting to delete bulk of {bulk_size} PVCs")
helpers.delete_bulk_pvcs(yaml_creation_dir,
pv_names_list,
namespace=self.namespace)
logging.info(
f"Deletion of bulk of {bulk_size} PVCs successfully completed")
log_deletion_times = helpers.measure_pv_deletion_time_bulk(
self.interface, pv_names_list, return_log_times=True)
all_start_times = [
a_tuple[0] for a_tuple in log_deletion_times.values()
]
bulk_start_time = sorted(all_start_times)[0] # the eariles start time
start_deletion_time = datetime.datetime.strptime(
bulk_start_time, helpers.DATE_TIME_FORMAT)
all_end_times = [a_tuple[1] for a_tuple in log_deletion_times.values()]
bulk_deletion_time = sorted(all_end_times)[-1] # the latest end time
end_deletion_time = datetime.datetime.strptime(
bulk_deletion_time, helpers.DATE_TIME_FORMAT)
total_deletion_time = (end_deletion_time -
start_deletion_time).total_seconds()
logging.info(
f"{bulk_size} Bulk PVCs deletion time is {total_deletion_time} seconds."
)
# Produce ES report
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"bulk_creation_deletion_measurement",
))
full_results.add_key("interface", self.interface)
full_results.add_key("bulk_size", bulk_size)
full_results.add_key("pvc_size", self.pvc_size)
full_results.add_key("bulk_pvc_creation_time", total_time)
full_results.add_key("bulk_pvc_deletion_time", total_deletion_time)
# Write the test results into the ES server
full_results.es_write()
def test_bulk_pvc_creation_deletion_measurement_performance(
self, storageclass_factory, interface_type, bulk_size
):
"""
Measuring PVC creation and deletion time of bulk_size PVCs
and sends results to the Elastic Search DB
Args:
bulk_size: Size of the bulk to be tested
Returns:
"""
self.interface = interface_type
self.sc_obj = storageclass_factory(self.interface)
bulk_creation_time_limit = bulk_size / 2
log.info(f"Start creating new {bulk_size} PVCs")
# Getting the start time of the test.
self.test_start_time = self.get_time()
# Run the Bulk Creation test
csi_bulk_start_time = self.get_time(time_format="csi")
self.pvc_bulk_create_and_wait_for_bound(bulk_size)
log.info(f"PVC creation dir is {self.yaml_creation_dir}")
total_time = self.get_bulk_creation_time()
log.info(f"{bulk_size} Bulk PVCs creation time is {total_time} seconds.")
csi_creation_times = performance_lib.csi_bulk_pvc_time_measure(
self.interface, self.pvc_objs, "create", csi_bulk_start_time
)
if total_time > bulk_creation_time_limit:
raise ex.PerformanceException(
f"{bulk_size} Bulk PVCs creation time is {total_time} and "
f"greater than {bulk_creation_time_limit} seconds"
)
# Run the Bulk Deletion test
pv_names_list = []
for pvc_obj in self.pvc_objs:
pv_names_list.append(pvc_obj.backed_pv)
log.info(f"Starting to delete bulk of {bulk_size} PVCs")
helpers.delete_bulk_pvcs(
self.yaml_creation_dir, pv_names_list, namespace=self.namespace
)
log.info(f"Deletion of bulk of {bulk_size} PVCs successfully completed")
log_deletion_times = helpers.measure_pv_deletion_time_bulk(
self.interface, pv_names_list, return_log_times=True
)
all_start_times = [a_tuple[0] for a_tuple in log_deletion_times.values()]
bulk_start_time = sorted(all_start_times)[0] # the eariles start time
start_deletion_time = datetime.datetime.strptime(
bulk_start_time, helpers.DATE_TIME_FORMAT
)
all_end_times = [a_tuple[1] for a_tuple in log_deletion_times.values()]
bulk_deletion_time = sorted(all_end_times)[-1] # the latest end time
end_deletion_time = datetime.datetime.strptime(
bulk_deletion_time, helpers.DATE_TIME_FORMAT
)
total_deletion_time = (end_deletion_time - start_deletion_time).total_seconds()
log.info(
f"{bulk_size} Bulk PVCs deletion time is {total_deletion_time} seconds."
)
csi_deletion_times = performance_lib.csi_bulk_pvc_time_measure(
self.interface, self.pvc_objs, "delete", csi_bulk_start_time
)
# Getting the end time of the test
self.test_end_time = self.get_time()
# reset the list oc PVCs since thay was deleted, and do not need to be deleted
# in the teardown phase.
self.pvc_objs = []
# Produce ES report
self.results_path = os.path.join(
"/",
*self.results_path,
"test_bulk_pvc_creation_deletion_measurement_performance",
)
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"bulk_creation_deletion_measurement",
)
)
# Add the test time to the ES report
full_results.add_key(
"test_time", {"start": self.test_start_time, "end": self.test_end_time}
)
full_results.add_key("bulk_size", bulk_size)
full_results.add_key("bulk_pvc_creation_time", total_time)
full_results.add_key("bulk_pvc_csi_creation_time", csi_creation_times)
full_results.add_key("bulk_pvc_deletion_time", total_deletion_time)
full_results.add_key("bulk_pvc_csi_deletion_time", csi_deletion_times)
# Write the test results into the ES server
if full_results.es_write():
res_link = full_results.results_link()
log.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtest (4 - according to the parameters)
self.write_result_to_file(res_link)
def test_pod_reattach_time_performance(
self, storageclass_factory, copies, timeout, total_time_limit
):
"""
Test assign nodeName to a pod using RWX pvc
Each kernel (unzipped) is 892M and 61694 files
The test creates samples_num pvcs and pods, writes kernel files multiplied by number of copies
and calculates average total and csi reattach times and standard deviation
"""
kernel_url = "https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.19.5.tar.gz"
download_path = "tmp"
samples_num = 7
if self.dev_mode:
samples_num = 3
test_start_time = PASTest.get_time()
helpers.pull_images(constants.PERF_IMAGE)
# Download a linux Kernel
dir_path = os.path.join(os.getcwd(), download_path)
file_path = os.path.join(dir_path, "file.gz")
if not os.path.exists(dir_path):
os.makedirs(dir_path)
urllib.request.urlretrieve(kernel_url, file_path)
worker_nodes_list = node.get_worker_nodes()
assert len(worker_nodes_list) > 1
node_one = worker_nodes_list[0]
node_two = worker_nodes_list[1]
time_measures, csi_time_measures, files_written_list, data_written_list = (
[],
[],
[],
[],
)
self.sc_obj = storageclass_factory(self.interface)
for sample_index in range(1, samples_num + 1):
csi_start_time = self.get_time("csi")
logger.info(f"Start creating PVC number {sample_index}.")
pvc_obj = helpers.create_pvc(
sc_name=self.sc_obj.name, size="100Gi", namespace=self.namespace
)
helpers.wait_for_resource_state(pvc_obj, constants.STATUS_BOUND)
# Create a pod on one node
logger.info(f"Creating Pod with pvc {pvc_obj.name} on node {node_one}")
pvc_obj.reload()
self.pvc_list.append(pvc_obj)
try:
pod_obj1 = helpers.create_pod(
interface_type=self.interface,
pvc_name=pvc_obj.name,
namespace=pvc_obj.namespace,
node_name=node_one,
pod_dict_path=constants.PERF_POD_YAML,
)
except Exception as e:
logger.error(
f"Pod on PVC {pvc_obj.name} was not created, exception {str(e)}"
)
raise PodNotCreated("Pod on PVC was not created.")
# Confirm that pod is running on the selected_nodes
logger.info("Checking whether pods are running on the selected nodes")
helpers.wait_for_resource_state(
resource=pod_obj1, state=constants.STATUS_RUNNING, timeout=timeout
)
pod_name = pod_obj1.name
pod_path = "/mnt"
_ocp = OCP(namespace=pvc_obj.namespace)
rsh_cmd = f"rsync {dir_path} {pod_name}:{pod_path}"
_ocp.exec_oc_cmd(rsh_cmd)
rsh_cmd = (
f"exec {pod_name} -- tar xvf {pod_path}/tmp/file.gz -C {pod_path}/tmp"
)
_ocp.exec_oc_cmd(rsh_cmd)
for x in range(copies):
rsh_cmd = f"exec {pod_name} -- mkdir -p {pod_path}/folder{x}"
_ocp.exec_oc_cmd(rsh_cmd)
rsh_cmd = (
f"exec {pod_name} -- cp -r {pod_path}/tmp {pod_path}/folder{x}"
)
_ocp.exec_oc_cmd(rsh_cmd)
rsh_cmd = f"exec {pod_name} -- sync"
_ocp.exec_oc_cmd(rsh_cmd)
logger.info("Getting the amount of data written to the PVC")
rsh_cmd = f"exec {pod_name} -- df -h {pod_path}"
data_written_str = _ocp.exec_oc_cmd(rsh_cmd).split()[-4]
logger.info(f"The amount of written data is {data_written_str}")
data_written = float(data_written_str[:-1])
rsh_cmd = f"exec {pod_name} -- find {pod_path} -type f"
files_written = len(_ocp.exec_oc_cmd(rsh_cmd).split())
logger.info(
f"For {self.interface} - The number of files written to the pod is {files_written}"
)
files_written_list.append(files_written)
data_written_list.append(data_written)
logger.info("Deleting the pod")
rsh_cmd = f"delete pod {pod_name}"
_ocp.exec_oc_cmd(rsh_cmd)
logger.info(f"Creating Pod with pvc {pvc_obj.name} on node {node_two}")
try:
pod_obj2 = helpers.create_pod(
interface_type=self.interface,
pvc_name=pvc_obj.name,
namespace=pvc_obj.namespace,
node_name=node_two,
pod_dict_path=constants.PERF_POD_YAML,
)
except Exception as e:
logger.error(
f"Pod on PVC {pvc_obj.name} was not created, exception {str(e)}"
)
raise PodNotCreated("Pod on PVC was not created.")
start_time = time.time()
pod_name = pod_obj2.name
helpers.wait_for_resource_state(
resource=pod_obj2, state=constants.STATUS_RUNNING, timeout=timeout
)
end_time = time.time()
total_time = end_time - start_time
if total_time > total_time_limit:
logger.error(
f"Pod creation time is {total_time} and greater than {total_time_limit} seconds"
)
raise ex.PerformanceException(
f"Pod creation time is {total_time} and greater than {total_time_limit} seconds"
)
csi_time = performance_lib.pod_attach_csi_time(
self.interface, pvc_obj.backed_pv, csi_start_time, pvc_obj.namespace
)[0]
csi_time_measures.append(csi_time)
logger.info(
f"PVC #{pvc_obj.name} pod {pod_name} creation time took {total_time} seconds, "
f"csi time is {csi_time} seconds"
)
time_measures.append(total_time)
logger.info("Deleting the pod")
rsh_cmd = f"delete pod {pod_name}"
_ocp.exec_oc_cmd(rsh_cmd)
# teardown_factory(pod_obj2)
average = statistics.mean(time_measures)
logger.info(
f"The average time of {self.interface} pod creation on {samples_num} PVCs is {average} seconds"
)
st_deviation = statistics.stdev(time_measures)
logger.info(
f"The standard deviation of {self.interface} pod creation time on {samples_num} PVCs is {st_deviation}"
)
csi_average = statistics.mean(csi_time_measures)
logger.info(
f"The average csi time of {self.interface} pod creation on {samples_num} PVCs is {csi_average} seconds"
)
csi_st_deviation = statistics.stdev(csi_time_measures)
logger.info(
f"The standard deviation of {self.interface} csi pod creation time on {samples_num} PVCs "
f"is {csi_st_deviation}"
)
files_written_average = statistics.mean(files_written_list)
data_written_average = statistics.mean(data_written_list)
os.remove(file_path)
os.rmdir(dir_path)
# Produce ES report
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"pod_reattach_time_fullres",
)
)
full_results.add_key("storageclass", self.sc)
full_results.add_key("pod_reattach_time", time_measures)
full_results.add_key("copies_number", copies)
full_results.add_key("files_number_average", files_written_average)
full_results.add_key("data_average", data_written_average)
full_results.add_key("pod_reattach_time_average", average)
full_results.add_key("pod_reattach_standard_deviation", st_deviation)
full_results.add_key("pod_csi_reattach_time_average", csi_average)
full_results.add_key("pod_csi_reattach_standard_deviation", csi_st_deviation)
test_end_time = PASTest.get_time()
# Add the test time to the ES report
full_results.add_key(
"test_time", {"start": test_start_time, "end": test_end_time}
)
# Write the test results into the ES server
if full_results.es_write():
res_link = full_results.results_link()
logger.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtest (4 - according to the parameters)
self.results_path = get_full_test_logs_path(
cname=self, fname="test_pod_reattach_time_performance"
)
self.write_result_to_file(res_link)
def test_pvc_multiple_clone_performance(
self,
interface_iterate,
secret_factory,
):
"""
1. Creating PVC
PVC size is calculated in the test and depends on the storage capacity, but not less then 1 GiB
it will use ~75% capacity of the Storage, Min storage capacity 1 TiB
2. Fill the PVC with 70% of data
3. Take a clone of the PVC and measure Total time and speed of creation of each clone
by reading start creation and end creation times from relevant logs
4. Measure CSI time for creation of each clone
5. Repeat the previous steps number of times (maximal num_of_clones is 512)
6. Print and push to the ES all the measured statistics for all the clones.
Raises:
StorageNotSufficientException: in case of not enough capacity on the cluster
"""
log.info(
f"Total capacity size is : {self.ceph_capacity} GiB, "
f"Going to use {self.need_capacity} GiB, "
f"With {self.num_of_clones} clones to {self.pvc_size} GiB PVC. "
f"File size to be written is : {self.file_size} ")
self.interface = interface_iterate
# Create new pool and sc only for RBD, for CepgFS use the++ default
if self.interface == constants.CEPHBLOCKPOOL:
# Creating new pool to run the test on it
self.create_new_pool_and_sc(secret_factory)
else:
# use the default ceph filesystem pool
self.sc_obj = ocs.OCS(
kind="StorageCluster",
metadata={
"namespace": self.namespace,
"name": Interfaces_info[self.interface]["sc"],
},
)
# Create a PVC
self.create_testing_pvc_and_wait_for_bound()
# Create a POD
self.create_testing_pod_and_wait_for_completion(
filesize=self.file_size)
# Running the test
creation_time_list, creation_speed_list, csi_creation_time_list = ([],
[],
[])
self.cloned_obj_list = []
for test_num in range(1, self.num_of_clones + 1):
log.info(f"Starting test number {test_num}")
try:
cloned_obj, ct, csi_ct = self.create_clone(test_num)
except Exception as e:
log.error(f"Failed to create clone number {test_num} : [{e}]")
break
self.cloned_obj_list.append(cloned_obj)
speed = self.filesize / ct
creation_time_list.append(ct)
creation_speed_list.append(speed)
csi_creation_time_list.append(csi_ct)
# Analyse the results and log the results
for i, val in enumerate(self.cloned_obj_list):
log.info(f"The Results for clone number {i+1} ({val}) :")
log.info(
f" Creation time is : {creation_time_list[i]:,.3f} secs.")
log.info(
f" Csi Creation time is : {csi_creation_time_list[i]:,.3f} secs."
)
log.info(
f" Creation speed is : {creation_speed_list[i]:,.3f} MB/sec."
)
average_creation_time = statistics.mean(creation_time_list)
average_creation_speed = statistics.mean(creation_speed_list)
average_csi_creation_time = statistics.mean(csi_creation_time_list)
log.info("The Average results are :")
log.info(
f" Average creation time is : {average_creation_time:,.3f} secs."
)
log.info(
f" Average csi creation time is : {average_csi_creation_time:,.3f} secs."
)
log.info(
f" Average creation speed is : {average_creation_speed:,.3f} MB/sec."
)
if len(self.cloned_obj_list) != self.num_of_clones:
log.error("Not all clones created.")
raise exceptions.BenchmarkTestFailed("Not all clones created.")
self.results_path = get_full_test_logs_path(cname=self)
# Produce ES report
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"pvc_multiple_clone_measurement",
))
full_results.add_key("multi_clone_creation_time", creation_time_list)
full_results.add_key("multi_clone_creation_time_average",
average_creation_time)
full_results.add_key("multi_clone_creation_speed", creation_speed_list)
full_results.add_key("multi_clone_creation_speed_average",
average_creation_speed)
full_results.add_key("multi_clone_csi_creation_time",
csi_creation_time_list)
full_results.add_key("multi_clone_csi_creation_time_average",
average_csi_creation_time)
# Write the test results into the ES server
if full_results.es_write():
res_link = full_results.results_link()
log.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtest (4 - according to the parameters)
self.write_result_to_file(res_link)
