def test_scale_pvcs_pods_pre_upgrade():
"""
Function to scale PVCs and PODs
"""
# Scale 1500+ PVCs and PODs in the cluster
fioscale = FioPodScale(kind=constants.DEPLOYMENTCONFIG,
node_selector=constants.SCALE_NODE_SELECTOR)
kube_pod_obj_list, kube_pvc_obj_list = fioscale.create_scale_pods(
scale_count=SCALE_COUNT, pvc_per_pod_count=PVCS_PER_POD)
namespace = fioscale.namespace
scale_round_up_count = SCALE_COUNT + 20
# Get PVCs and PODs count and list
pod_running_list, pvc_bound_list = ([], [])
for pod_objs in kube_pod_obj_list:
pod_running_list.extend(
scale_lib.check_all_pod_reached_running_state_in_kube_job(
kube_job_obj=pod_objs,
namespace=namespace,
no_of_pod=int(scale_round_up_count / 40),
))
for pvc_objs in kube_pvc_obj_list:
pvc_bound_list.extend(
scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=pvc_objs,
namespace=namespace,
no_of_pvc=int(scale_round_up_count / 4),
))
logging.info(f"Running PODs count {len(pod_running_list)} & "
f"Bound PVCs count {len(pvc_bound_list)} "
f"in namespace {fioscale.namespace}")
# Write namespace, PVC and POD data in a SCALE_DATA_FILE which
# will be used during post_upgrade validation tests
with open(SCALE_DATA_FILE, "a+") as w_obj:
w_obj.write(str("# Scale Data File\n"))
w_obj.write(str(f"NAMESPACE: {namespace}\n"))
w_obj.write(str(f"POD_SCALE_LIST: {pod_running_list}\n"))
w_obj.write(str(f"PVC_SCALE_LIST: {pvc_bound_list}\n"))
# Check ceph health status
utils.ceph_health_check(tries=30)
def test_bulk_clone_performance(self, namespace, tmp_path, pod_factory):
"""
Creates number of PVCs in a bulk using kube job
Write 60% of PVC capacity to each one of the created PVCs
Creates 1 clone per each PVC altogether in a bulk
Measuring time for bulk of clones creation
"""
pvc_count = 50
log.info(f"Start creating {self.interface} {pvc_count} PVC")
if self.interface == constants.CEPHBLOCKPOOL:
sc_name = constants.DEFAULT_STORAGECLASS_RBD
clone_yaml = constants.CSI_RBD_PVC_CLONE_YAML
elif self.interface == constants.CEPHFILESYSTEM:
sc_name = constants.DEFAULT_STORAGECLASS_CEPHFS
clone_yaml = constants.CSI_CEPHFS_PVC_CLONE_YAML
pvc_dict_list = scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job(
no_of_pvc=pvc_count,
access_mode=constants.ACCESS_MODE_RWO,
sc_name=sc_name,
pvc_size="5Gi",
)
job_pvc_file = ObjectConfFile(
name="job_profile_pvc",
obj_dict_list=pvc_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
# Create kube_job
job_pvc_file.create(namespace=self.namespace)
# Check all the PVC reached Bound state
pvc_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_pvc_file,
namespace=self.namespace,
no_of_pvc=pvc_count,
)
logging.info(f"Number of PVCs in Bound state {len(pvc_bound_list)}")
total_files_size = self.run_fio_on_pvcs(pvc_dict_list, pod_factory)
clone_dict_list = scale_lib.construct_pvc_clone_yaml_bulk_for_kube_job(
pvc_dict_list, clone_yaml, sc_name)
logging.info("Created clone dict list")
job_clone_file = ObjectConfFile(
name="job_profile_clone",
obj_dict_list=clone_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
# Create kube_job that creates clones
job_clone_file.create(namespace=self.namespace)
logging.info("Going to check bound status for clones")
# Check all the clones reached Bound state
clone_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_clone_file,
namespace=self.namespace,
no_of_pvc=pvc_count,
timeout=200,
)
logging.info(
f"Number of clones in Bound state {len(clone_bound_list)}")
clone_objs = []
all_pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace)
for clone_yaml in clone_dict_list:
name = clone_yaml["metadata"]["name"]
size = clone_yaml["spec"]["resources"]["requests"]["storage"]
logging.info(f"Clone {name} of size {size} created")
for pvc_obj in all_pvc_objs:
if pvc_obj.name == name:
clone_objs.append(pvc_obj)
assert len(clone_bound_list) == len(
clone_objs
), "Not all clones reached BOUND state, cannot measure time"
start_time = helpers.get_provision_time(self.interface,
clone_objs,
status="start")
end_time = helpers.get_provision_time(self.interface,
clone_objs,
status="end")
total_time = (end_time - start_time).total_seconds()
speed = round(total_files_size / total_time, 2)
logging.info(
f"Total creation time = {total_time} secs, data size = {total_files_size} MB, speed = {speed} MB/sec "
f"for {self.interface} clone in bulk of {pvc_count} clones.")
def test_bulk_clone_performance(self, namespace, tmp_path):
"""
Creates number of PVCs in a bulk using kube job
Write 60% of PVC capacity to each one of the created PVCs
Creates 1 clone per each PVC altogether in a bulk
Measuring total and csi creation times for bulk of clones
"""
pvc_count = 50
vol_size = "5Gi"
job_pod_file, job_pvc_file, job_clone_file = [None, None, None]
log.info(f"Start creating {self.interface} {pvc_count} PVC")
if self.interface == constants.CEPHBLOCKPOOL:
sc_name = constants.DEFAULT_STORAGECLASS_RBD
clone_yaml = constants.CSI_RBD_PVC_CLONE_YAML
elif self.interface == constants.CEPHFILESYSTEM:
sc_name = constants.DEFAULT_STORAGECLASS_CEPHFS
clone_yaml = constants.CSI_CEPHFS_PVC_CLONE_YAML
try:
pvc_dict_list = scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job(
no_of_pvc=pvc_count,
access_mode=constants.ACCESS_MODE_RWO,
sc_name=sc_name,
pvc_size=vol_size,
)
job_pvc_file = ObjectConfFile(
name="job_profile_pvc",
obj_dict_list=pvc_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
# Create kube_job
job_pvc_file.create(namespace=self.namespace)
# Check all the PVC reached Bound state
pvc_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_pvc_file,
namespace=self.namespace,
no_of_pvc=pvc_count,
)
log.info(f"Number of PVCs in Bound state {len(pvc_bound_list)}")
# Kube_job to Create pod
pod_dict_list = scale_lib.attach_multiple_pvc_to_pod_dict(
pvc_list=pvc_bound_list,
namespace=self.namespace,
pvcs_per_pod=1,
start_io=False,
pod_yaml=constants.NGINX_POD_YAML,
)
job_pod_file = ObjectConfFile(
name="job_profile_pod",
obj_dict_list=pod_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
job_pod_file.create(namespace=self.namespace)
# Check all PODs in Running state
scale_lib.check_all_pod_reached_running_state_in_kube_job(
kube_job_obj=job_pod_file,
namespace=self.namespace,
no_of_pod=len(pod_dict_list),
timeout=90,
)
log.info(f"Number of PODs in Running state {len(pod_dict_list)}")
total_files_size = self.run_fio_on_pvcs(vol_size)
clone_dict_list = scale_lib.construct_pvc_clone_yaml_bulk_for_kube_job(
pvc_dict_list, clone_yaml, sc_name)
log.info("Created clone dict list")
csi_bulk_start_time = self.get_time(time_format="csi")
job_clone_file = ObjectConfFile(
name="job_profile_clone",
obj_dict_list=clone_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
# Create kube_job that creates clones
job_clone_file.create(namespace=self.namespace)
log.info("Going to check bound status for clones")
# Check all the clones reached Bound state
clone_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_clone_file,
namespace=self.namespace,
no_of_pvc=pvc_count,
timeout=180,
)
log.info(
f"Number of clones in Bound state {len(clone_bound_list)}")
clone_objs = []
all_pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace)
for clone_yaml in clone_dict_list:
name = clone_yaml["metadata"]["name"]
size = clone_yaml["spec"]["resources"]["requests"]["storage"]
log.info(f"Clone {name} of size {size} created")
for pvc_obj in all_pvc_objs:
if pvc_obj.name == name:
clone_objs.append(pvc_obj)
assert len(clone_bound_list) == len(
clone_objs
), "Not all clones reached BOUND state, cannot measure time"
start_time = helpers.get_provision_time(self.interface,
clone_objs,
status="start")
end_time = helpers.get_provision_time(self.interface,
clone_objs,
status="end")
total_time = (end_time - start_time).total_seconds()
speed = round(total_files_size / total_time, 2)
csi_creation_time = performance_lib.csi_bulk_pvc_time_measure(
self.interface, clone_objs, "create", csi_bulk_start_time)
log.info(
f"Total creation time = {total_time} secs, csi creation time = {csi_creation_time},"
f" data size = {total_files_size} MB, speed = {speed} MB/sec "
f"for {self.interface} clone in bulk of {pvc_count} clones.")
# Produce ES report
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"bulk_clone_perf_fullres",
))
full_results.add_key("interface", self.interface)
full_results.add_key("bulk_size", pvc_count)
full_results.add_key("clone_size", vol_size)
full_results.add_key("bulk_creation_time", total_time)
full_results.add_key("bulk_csi_creation_time", csi_creation_time)
full_results.add_key("data_size(MB)", total_files_size)
full_results.add_key("speed", speed)
full_results.add_key("es_results_link",
full_results.results_link())
# Write the test results into the ES server
full_results.es_write()
self.results_path = get_full_test_logs_path(cname=self)
res_link = full_results.results_link()
# write the ES link to the test results in the test log.
log.info(f"The result can be found at : {res_link}")
# Create text file with results of all subtest (3 - according to the parameters)
self.write_result_to_file(res_link)
# Finally is used to clean-up the resources created
# Irrespective of try block pass/fail finally will be executed.
finally:
# Cleanup activities
log.info(
"Cleanup of all the resources created during test execution")
if job_pod_file:
job_pod_file.delete(namespace=self.namespace)
job_pod_file.wait_for_delete(resource_name=job_pod_file.name,
namespace=self.namespace)
if job_clone_file:
job_clone_file.delete(namespace=self.namespace)
job_clone_file.wait_for_delete(
resource_name=job_clone_file.name,
namespace=self.namespace)
if job_pvc_file:
job_pvc_file.delete(namespace=self.namespace)
job_pvc_file.wait_for_delete(resource_name=job_pvc_file.name,
namespace=self.namespace)
# Check ceph health status
utils.ceph_health_check(tries=20)
def test_multiple_pvc_creation_deletion_scale(self, namespace, tmp_path,
access_mode, interface):
"""
Measuring PVC creation time while scaling PVC
Measure PVC deletion time after creation test
"""
scale_pvc_count = scale_lib.get_max_pvc_count()
log.info(
f"Start creating {access_mode}-{interface} {scale_pvc_count} PVC")
if interface == constants.CEPHBLOCKPOOL:
sc_name = constants.DEFAULT_STORAGECLASS_RBD
elif interface == constants.CEPHFS_INTERFACE:
sc_name = constants.DEFAULT_STORAGECLASS_CEPHFS
# Get pvc_dict_list, append all the pvc.yaml dict to pvc_dict_list
pvc_dict_list1 = scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job(
no_of_pvc=int(scale_pvc_count / 2),
access_mode=access_mode,
sc_name=sc_name)
pvc_dict_list2 = scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job(
no_of_pvc=int(scale_pvc_count / 2),
access_mode=access_mode,
sc_name=sc_name)
# There is 2 kube_job to reduce the load, observed time_out problems
# during delete process of single kube_job and heavy load.
job_file1 = ObjectConfFile(
name="job_profile_1",
obj_dict_list=pvc_dict_list1,
project=self.namespace,
tmp_path=tmp_path,
)
job_file2 = ObjectConfFile(
name="job_profile_2",
obj_dict_list=pvc_dict_list2,
project=self.namespace,
tmp_path=tmp_path,
)
# Create kube_job
job_file1.create(namespace=self.namespace)
job_file2.create(namespace=self.namespace)
# Check all the PVC reached Bound state
pvc_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_file1,
namespace=self.namespace,
no_of_pvc=int(scale_pvc_count / 2),
)
pvc_bound_list.extend(
scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_file2,
namespace=self.namespace,
no_of_pvc=int(scale_pvc_count / 2),
))
log.info(f"Number of PVCs in Bound state {len(pvc_bound_list)}")
# Get PVC creation time
pvc_create_time = helpers.measure_pvc_creation_time_bulk(
interface=interface,
pvc_name_list=pvc_bound_list,
wait_time=300,
)
# TODO: Update below code with google API, to record value in spreadsheet
# TODO: For now observing Google API limit to write more than 100 writes
log_path = f"{ocsci_log_path()}/{interface}-{access_mode}"
with open(f"{log_path}-creation-time.csv", "w") as fd:
csv_obj = csv.writer(fd)
for k, v in pvc_create_time.items():
csv_obj.writerow([k, v])
log.info(f"Create data present in {log_path}-creation-time.csv file")
# Get pv_name, require pv_name to fetch deletion time data from log
pv_name_list = list()
get_kube_job_1 = job_file1.get(namespace=self.namespace)
for i in range(int(scale_pvc_count / 2)):
pv_name_list.append(
get_kube_job_1["items"][i]["spec"]["volumeName"])
get_kube_job_2 = job_file2.get(namespace=self.namespace)
for i in range(int(scale_pvc_count / 2)):
pv_name_list.append(
get_kube_job_2["items"][i]["spec"]["volumeName"])
# Delete kube_job
job_file1.delete(namespace=self.namespace)
job_file2.delete(namespace=self.namespace)
# Adding 1min wait time for PVC deletion logs to be updated
# Observed failure when we immediately check the logs for pvc delete time
# https://github.com/red-hat-storage/ocs-ci/issues/3371
time.sleep(60)
# Get PVC deletion time
pvc_deletion_time = helpers.measure_pv_deletion_time_bulk(
interface=interface, pv_name_list=pv_name_list)
# Update result to csv file.
# TODO: Update below code with google API, to record value in spreadsheet
# TODO: For now observing Google API limit to write more than 100 writes
with open(f"{log_path}-deletion-time.csv", "w") as fd:
csv_obj = csv.writer(fd)
for k, v in pvc_deletion_time.items():
csv_obj.writerow([k, v])
log.info(f"Delete data present in {log_path}-deletion-time.csv file")
end_time = default_timer()
log.info(f"Elapsed time -- {end_time - self.start_time} seconds")
def test_all_4_type_pvc_creation_deletion_scale(self, namespace, tmp_path):
"""
Measuring PVC creation time while scaling PVC of all 4 types,
A total of 500 times the number of worker nodes
will be created, i.e. 375 each pvc type
Measure PVC deletion time in scale env
"""
scale_pvc_count = scale_lib.get_max_pvc_count()
log.info(f"Start creating {scale_pvc_count} PVC of all 4 types")
cephfs_sc_obj = constants.DEFAULT_STORAGECLASS_CEPHFS
rbd_sc_obj = constants.DEFAULT_STORAGECLASS_RBD
# Get pvc_dict_list, append all the pvc.yaml dict to pvc_dict_list
rbd_pvc_dict_list, cephfs_pvc_dict_list = ([] for i in range(2))
for mode in [constants.ACCESS_MODE_RWO, constants.ACCESS_MODE_RWX]:
rbd_pvc_dict_list.extend(
scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job(
no_of_pvc=int(scale_pvc_count / 4),
access_mode=mode,
sc_name=rbd_sc_obj,
))
cephfs_pvc_dict_list.extend(
scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job(
no_of_pvc=int(scale_pvc_count / 4),
access_mode=mode,
sc_name=cephfs_sc_obj,
))
# There is 2 kube_job for cephfs and rbd PVCs
job_file_rbd = ObjectConfFile(
name="rbd_pvc_job",
obj_dict_list=rbd_pvc_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
job_file_cephfs = ObjectConfFile(
name="cephfs_pvc_job",
obj_dict_list=cephfs_pvc_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
# Create kube_job
job_file_rbd.create(namespace=self.namespace)
job_file_cephfs.create(namespace=self.namespace)
# Check all the PVC reached Bound state
rbd_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_file_rbd,
namespace=self.namespace,
no_of_pvc=int(scale_pvc_count / 2),
)
fs_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_file_cephfs,
namespace=self.namespace,
no_of_pvc=int(scale_pvc_count / 2),
)
# Get pvc objs from namespace, which is used to identify backend pv
rbd_pvc_obj, cephfs_pvc_obj = ([] for i in range(2))
pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace)
for pvc_obj in pvc_objs:
if pvc_obj.backed_sc == constants.DEFAULT_STORAGECLASS_RBD:
rbd_pvc_obj.append(pvc_obj)
elif pvc_obj.backed_sc == constants.DEFAULT_STORAGECLASS_CEPHFS:
cephfs_pvc_obj.append(pvc_obj)
# Get PVC creation time
fs_pvc_create_time = helpers.measure_pvc_creation_time_bulk(
interface=constants.CEPHFS_INTERFACE, pvc_name_list=fs_pvc_name)
rbd_pvc_create_time = helpers.measure_pvc_creation_time_bulk(
interface=constants.CEPHBLOCKPOOL, pvc_name_list=rbd_pvc_name)
fs_pvc_create_time.update(rbd_pvc_create_time)
# TODO: Update below code with google API, to record value in spreadsheet
# TODO: For now observing Google API limit to write more than 100 writes
log_path = f"{ocsci_log_path()}/All-type-PVC"
with open(f"{log_path}-creation-time.csv", "w") as fd:
csv_obj = csv.writer(fd)
for k, v in fs_pvc_create_time.items():
csv_obj.writerow([k, v])
log.info(f"Create data present in {log_path}-creation-time.csv file")
# Get pv_name, require pv_name to fetch deletion time data from log
rbd_pv_list, fs_pv_list = ([] for i in range(2))
get_rbd_kube_job = job_file_rbd.get(namespace=self.namespace)
for i in range(int(scale_pvc_count / 2)):
rbd_pv_list.append(
get_rbd_kube_job["items"][i]["spec"]["volumeName"])
get_fs_kube_job = job_file_cephfs.get(namespace=self.namespace)
for i in range(int(scale_pvc_count / 2)):
fs_pv_list.append(
get_fs_kube_job["items"][i]["spec"]["volumeName"])
# Delete kube_job
job_file_rbd.delete(namespace=self.namespace)
job_file_cephfs.delete(namespace=self.namespace)
# Adding 1min wait time for PVC deletion logs to be updated
# Observed failure when we immediately check the logs for pvc delete time
# https://github.com/red-hat-storage/ocs-ci/issues/3371
time.sleep(60)
# Get PV deletion time
fs_pvc_deletion_time = helpers.measure_pv_deletion_time_bulk(
interface=constants.CEPHFS_INTERFACE, pv_name_list=fs_pv_list)
rbd_pvc_deletion_time = helpers.measure_pv_deletion_time_bulk(
interface=constants.CEPHBLOCKPOOL, pv_name_list=rbd_pv_list)
fs_pvc_deletion_time.update(rbd_pvc_deletion_time)
# TODO: Update below code with google API, to record value in spreadsheet
# TODO: For now observing Google API limit to write more than 100 writes
with open(f"{log_path}-deletion-time.csv", "w") as fd:
csv_obj = csv.writer(fd)
for k, v in fs_pvc_deletion_time.items():
csv_obj.writerow([k, v])
log.info(f"Delete data present in {log_path}-deletion-time.csv file")
end_time = default_timer()
log.info(f"Elapsed time -- {end_time - self.start_time} seconds")
def test_scale_pvcs_pods(self):
"""
Scale 6000 PVCs and PODs in cluster with 12 worker nodes
"""
scale_count = 6000
pvcs_per_pod = 20
try:
# Scale
fioscale = FioPodScale(
kind=constants.DEPLOYMENTCONFIG,
node_selector=constants.SCALE_NODE_SELECTOR,
)
kube_pod_obj_list, kube_pvc_obj_list = fioscale.create_scale_pods(
scale_count=scale_count, pvc_per_pod_count=pvcs_per_pod)
namespace = fioscale.namespace
scale_round_up_count = scale_count + 80
# Get PVCs and PODs count and list
pod_running_list, pvc_bound_list = ([], [])
for pod_objs in kube_pod_obj_list:
pod_running_list.extend(
scale_lib.check_all_pod_reached_running_state_in_kube_job(
kube_job_obj=pod_objs,
namespace=namespace,
no_of_pod=int(scale_round_up_count / 160),
))
for pvc_objs in kube_pvc_obj_list:
pvc_bound_list.extend(
scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=pvc_objs,
namespace=namespace,
no_of_pvc=int(scale_round_up_count / 16),
))
logging.info(f"Running PODs count {len(pod_running_list)} & "
f"Bound PVCs count {len(pvc_bound_list)} "
f"in namespace {fioscale.namespace}")
# Get kube obj files in the list to update in scale_data_file
pod_obj_file_list, pvc_obj_file_list = ([], [])
files = os.listdir(ocsci_log_path())
for f in files:
if "pod" in f:
pod_obj_file_list.append(f)
elif "pvc" in f:
pvc_obj_file_list.append(f)
# Write namespace, PVC and POD data in a SCALE_DATA_FILE which
# will be used during post_upgrade validation tests
with open(SCALE_DATA_FILE, "a+") as w_obj:
w_obj.write(str("# Scale Data File\n"))
w_obj.write(str(f"NAMESPACE: {namespace}\n"))
w_obj.write(str(f"POD_SCALE_LIST: {pod_running_list}\n"))
w_obj.write(str(f"PVC_SCALE_LIST: {pvc_bound_list}\n"))
w_obj.write(str(f"POD_OBJ_FILE_LIST: {pod_obj_file_list}\n"))
w_obj.write(str(f"PVC_OBJ_FILE_LIST: {pvc_obj_file_list}\n"))
# Check ceph health status
utils.ceph_health_check(tries=30)
except UnexpectedBehaviour:
TestAddNode.skip_all = True
logging.info(
"Cluster is not in expected state, unexpected behaviour")
raise