def create_mutiple_pvcs_statistics(self, num_of_samples, teardown_factory,
pvc_size):
"""
Creates number (samples_num) of PVCs, measures creation time for each PVC and returns list of creation times.
Args:
num_of_samples: Number of the sampled created PVCs.
teardown_factory: A fixture used when we want a new resource that was created during the tests.
pvc_size: Size of the created PVCs.
Returns:
List of the creation times of all the created PVCs.
"""
time_measures = []
for i in range(num_of_samples):
log.info(f"Start creation of PVC number {i + 1}.")
pvc_obj = helpers.create_pvc(sc_name=self.sc_obj.name,
size=pvc_size)
helpers.wait_for_resource_state(pvc_obj, constants.STATUS_BOUND)
pvc_obj.reload()
teardown_factory(pvc_obj)
create_time = helpers.measure_pvc_creation_time(
self.interface, pvc_obj.name)
logging.info(f"PVC created in {create_time} seconds")
time_measures.append(create_time)
return time_measures
def test_delete_create_pvc_same_name(self, interface, pvc_factory,
teardown_factory):
"""
Delete PVC and create a new PVC with same name
"""
# Create a PVC
pvc_obj1 = pvc_factory(
interface=interface,
access_mode=constants.ACCESS_MODE_RWO,
status=constants.STATUS_BOUND,
)
# Delete the PVC
logger.info(f"Deleting PVC {pvc_obj1.name}")
pvc_obj1.delete()
pvc_obj1.ocp.wait_for_delete(pvc_obj1.name)
logger.info(f"Deleted PVC {pvc_obj1.name}")
# Create a new PVC with same name
logger.info(f"Creating new PVC with same name {pvc_obj1.name}")
pvc_obj2 = helpers.create_pvc(
sc_name=pvc_obj1.storageclass.name,
pvc_name=pvc_obj1.name,
namespace=pvc_obj1.project.namespace,
do_reload=False,
)
teardown_factory(pvc_obj2)
# Check the new PVC and PV are Bound
helpers.wait_for_resource_state(resource=pvc_obj2,
state=constants.STATUS_BOUND)
pv_obj2 = pvc_obj2.backed_pv_obj
helpers.wait_for_resource_state(resource=pv_obj2,
state=constants.STATUS_BOUND)
def create_resource_hsbench(self):
"""
Create resource for hsbench mark test:
Create service account
Create PVC
Create golang pod
"""
# Create service account
self.sa_name = helpers.create_serviceaccount(self.namespace)
self.sa_name = self.sa_name.name
helpers.add_scc_policy(sa_name=self.sa_name, namespace=self.namespace)
# Create test pvc+pod
log.info(
f"Create Golang pod to generate S3 workload... {self.namespace}")
pvc_size = "50Gi"
self.pod_name = "hsbench-pod"
self.pvc_obj = helpers.create_pvc(
sc_name=constants.DEFAULT_STORAGECLASS_RBD,
namespace=self.namespace,
size=pvc_size,
)
self.pod_obj = helpers.create_pod(
constants.CEPHBLOCKPOOL,
namespace=self.namespace,
pod_name=self.pod_name,
pvc_name=self.pvc_obj.name,
sa_name=self.sa_name,
pod_dict_path=self.pod_dic_path,
dc_deployment=True,
deploy_pod_status=constants.STATUS_COMPLETED,
)
def change_registry_backend_to_ocs():
"""
Function to deploy registry with OCS backend.
Raises:
AssertionError: When failure in change of registry backend to OCS
"""
sc = helpers.default_storage_class(interface_type=constants.CEPHFILESYSTEM)
pv_obj = helpers.create_pvc(
sc_name=sc.name,
pvc_name="registry-cephfs-rwx-pvc",
namespace=constants.OPENSHIFT_IMAGE_REGISTRY_NAMESPACE,
size="100Gi",
access_mode=constants.ACCESS_MODE_RWX,
)
helpers.wait_for_resource_state(pv_obj, "Bound")
param_cmd = f'[{{"op": "add", "path": "/spec/storage", "value": {{"pvc": {{"claim": "{pv_obj.name}"}}}}}}]'
run_cmd(f"oc patch {constants.IMAGE_REGISTRY_CONFIG} -p "
f"'{param_cmd}' --type json")
# Validate registry pod status
retry((CommandFailed, UnexpectedBehaviour), tries=3,
delay=15)(validate_registry_pod_status)()
# Validate pvc mount in the registry pod
retry((CommandFailed, UnexpectedBehaviour, AssertionError),
tries=3,
delay=15)(validate_pvc_mount_on_registry_pod)()
def __init__(self):
with open(constants.CSI_CEPHFS_POD_YAML, "r") as pod_fd:
pod_info = yaml.safe_load(pod_fd)
pvc_name = pod_info["spec"]["volumes"][0]["persistentVolumeClaim"][
"claimName"]
self.pod_name = pod_info["metadata"]["name"]
config.RUN["cli_params"]["teardown"] = True
self.cephfs_pvc = helpers.create_pvc(
sc_name=constants.DEFAULT_STORAGECLASS_CEPHFS,
namespace=config.ENV_DATA["cluster_namespace"],
pvc_name=pvc_name,
size=SIZE,
)
helpers.wait_for_resource_state(self.cephfs_pvc,
constants.STATUS_BOUND,
timeout=300)
self.cephfs_pod = helpers.create_pod(
interface_type=constants.CEPHFILESYSTEM,
namespace=config.ENV_DATA["cluster_namespace"],
pvc_name=pvc_name,
node_name="compute-0",
pod_name=self.pod_name,
)
helpers.wait_for_resource_state(self.cephfs_pod,
constants.STATUS_RUNNING,
timeout=300)
logging.info("pvc and cephfs pod created")
self.ocp_obj = ocp.OCP(
kind=constants.POD,
namespace=config.ENV_DATA["cluster_namespace"],
)
self.test_file_list = add_million_files(self.pod_name, self.ocp_obj)
logging.info("cephfs test files created")
def test_pvc_deletion_measurement_performance(self, teardown_factory, pvc_size):
"""
Measuring PVC deletion time is within supported limits
"""
logging.info("Start creating new PVC")
pvc_obj = helpers.create_pvc(sc_name=self.sc_obj.name, size=pvc_size)
helpers.wait_for_resource_state(pvc_obj, constants.STATUS_BOUND)
pvc_obj.reload()
pv_name = pvc_obj.backed_pv
pvc_reclaim_policy = pvc_obj.reclaim_policy
teardown_factory(pvc_obj)
pvc_obj.delete()
logging.info("Start deletion of PVC")
pvc_obj.ocp.wait_for_delete(pvc_obj.name)
if pvc_reclaim_policy == constants.RECLAIM_POLICY_DELETE:
helpers.validate_pv_delete(pvc_obj.backed_pv)
delete_time = helpers.measure_pvc_deletion_time(self.interface, pv_name)
# Deletion time for CephFS PVC is a little over 3 seconds
deletion_time = 4 if self.interface == constants.CEPHFILESYSTEM else 3
logging.info(f"PVC deleted in {delete_time} seconds")
if delete_time > deletion_time:
raise ex.PerformanceException(
f"PVC deletion time is {delete_time} and greater than {deletion_time} second"
)
push_to_pvc_time_dashboard(self.interface, "1-pvc-deletion", delete_time)
def test_ocs_347(self, resources):
pod, pvc, storageclass = resources
log.info("Creating RBD StorageClass")
storageclass.append(
helpers.create_storage_class(
interface_type=constants.CEPHBLOCKPOOL,
interface_name=self.cbp_obj.name,
secret_name=self.rbd_secret_obj.name,
)
)
log.info("Creating a PVC")
pvc.append(helpers.create_pvc(sc_name=storageclass[0].name))
for pvc_obj in pvc:
helpers.wait_for_resource_state(pvc_obj, constants.STATUS_BOUND)
pvc_obj.reload()
log.info(f"Creating a pod on with pvc {pvc[0].name}")
pod_obj = helpers.create_pod(
interface_type=constants.CEPHBLOCKPOOL,
pvc_name=pvc[0].name,
pod_dict_path=constants.NGINX_POD_YAML,
)
pod.append(pod_obj)
helpers.wait_for_resource_state(pod_obj, constants.STATUS_RUNNING)
pod_obj.reload()
def test_create_storage_class_with_wrong_provisioner(self, interface):
"""
Test function which creates Storage Class with
wrong provisioner and verifies PVC status
"""
log.info(f"Creating a {interface} storage class")
if interface == "RBD":
interface_type = constants.CEPHBLOCKPOOL
secret = self.rbd_secret_obj.name
interface_name = self.cbp_obj.name
else:
interface_type = constants.CEPHFILESYSTEM
secret = self.cephfs_secret_obj.name
interface_name = helpers.get_cephfs_data_pool_name()
sc_obj = helpers.create_storage_class(
interface_type=interface_type,
interface_name=interface_name,
secret_name=secret,
provisioner=constants.AWS_EFS_PROVISIONER,
)
log.info(
f"{interface}Storage class: {sc_obj.name} created successfully")
# Create PVC
pvc_obj = helpers.create_pvc(sc_name=sc_obj.name, do_reload=False)
# Check PVC status
pvc_output = pvc_obj.get()
pvc_status = pvc_output["status"]["phase"]
log.info(f"Status of PVC {pvc_obj.name} after creation: {pvc_status}")
log.info(f"Waiting for status '{constants.STATUS_PENDING}' "
f"for 20 seconds (it shouldn't change)")
pvc_obj.ocp.wait_for_resource(
resource_name=pvc_obj.name,
condition=constants.STATUS_PENDING,
timeout=20,
sleep=5,
)
# Check PVC status again after 20 seconds
pvc_output = pvc_obj.get()
pvc_status = pvc_output["status"]["phase"]
assert_msg = (
f"PVC {pvc_obj.name} is not in {constants.STATUS_PENDING} "
f"status")
assert pvc_status == constants.STATUS_PENDING, assert_msg
log.info(f"Status of {pvc_obj.name} after 20 seconds: {pvc_status}")
# Delete PVC
log.info(f"Deleting PVC: {pvc_obj.name}")
assert pvc_obj.delete()
log.info(f"PVC {pvc_obj.name} delete successfully")
# Delete Storage Class
log.info(f"Deleting Storageclass: {sc_obj.name}")
assert sc_obj.delete()
log.info(f"Storage Class: {sc_obj.name} deleted successfully")
def pod_obj_list(
self,
interface,
storageclass_factory,
pod_factory,
pvc_factory,
teardown_factory,
samples_num,
pvc_size,
):
"""
Prepare sample pods for the test
Returns:
pod obj: List of pod instances
"""
self.interface = interface
pod_result_list = []
self.msg_prefix = f"Interface: {self.interface}, PVC size: {pvc_size}."
self.samples_num = samples_num
self.pvc_size = pvc_size
if self.interface == constants.CEPHBLOCKPOOL_THICK:
self.sc_obj = storageclass_factory(
interface=constants.CEPHBLOCKPOOL,
new_rbd_pool=True,
rbd_thick_provision=True,
)
else:
self.sc_obj = storageclass_factory(self.interface)
for i in range(samples_num):
logging.info(
f"{self.msg_prefix} Start creating PVC number {i + 1}.")
pvc_obj = helpers.create_pvc(sc_name=self.sc_obj.name,
size=pvc_size)
teardown_factory(pvc_obj)
timeout = 600 if self.interface == constants.CEPHBLOCKPOOL_THICK else 60
helpers.wait_for_resource_state(pvc_obj,
constants.STATUS_BOUND,
timeout=timeout)
pvc_obj.reload()
logging.info(
f"{self.msg_prefix} PVC number {i + 1} was successfully created ."
)
pod_obj = pod_factory(interface=self.interface,
pvc=pvc_obj,
status=constants.STATUS_RUNNING)
teardown_factory(pod_obj)
pod_result_list.append(pod_obj)
return pod_result_list
def _deploy_es(self):
"""
Deploying the Elasticsearch server
"""
# Creating PVC for the elasticsearch server and wait until it bound
log.info("Creating 10 GiB PVC for the ElasticSearch cluster on")
try:
self.pvc_obj = create_pvc(
sc_name=self.args.get("sc") or constants.CEPHBLOCKPOOL_SC,
namespace=self.namespace,
pvc_name="elasticsearch-data-quickstart-es-default-0",
access_mode=constants.ACCESS_MODE_RWO,
size="10Gi",
)
# Make sure the PVC bound, or delete it and raise exception
wait_for_resource_state(self.pvc_obj, constants.STATUS_BOUND)
except ResourceWrongStatusException:
log.error("The PVC couldn't created")
return False
self.pvc_obj.reload()
log.info("Deploy the ElasticSearch cluster")
self.ocp.apply(self.crd)
sample = TimeoutSampler(
timeout=300,
sleep=10,
func=self._pod_is_found,
pattern="quickstart-es-default",
)
if not sample.wait_for_func_status(True):
log.error("The ElasticSearch pod deployment Failed")
return False
self.espod = get_pod_name_by_pattern("quickstart-es-default",
self.namespace)[0]
log.info(f"The ElasticSearch pod {self.espod} Started")
es_pod = OCP(kind="pod", namespace=self.namespace)
log.info("Waiting for ElasticSearch to Run")
if not es_pod.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_name=self.espod,
sleep=30,
timeout=600,
):
log.error("TThe ElasticSearch pod is not running !")
return False
else:
log.info("Elastic Search is ready !!!")
return True
def create_pvc(request):
"""
Create a persistent Volume Claim
"""
class_instance = request.node.cls
class_instance.pvc_obj = helpers.create_pvc(
sc_name=class_instance.sc_obj.name, namespace=class_instance.namespace)
helpers.wait_for_resource_state(class_instance.pvc_obj,
constants.STATUS_BOUND)
class_instance.pvc_obj.reload()
def test_capacity_breakdown_ui(
self, setup_ui_class, project_name, pod_name, sc_type, teardown_project_factory
):
"""
Test Capacity Breakdown UI
project_name (str): the project name
pod_name (str): pod name
sc_type (str): storage class [fs, block]
"""
project_obj = helpers.create_project(project_name=project_name)
teardown_project_factory(project_obj)
logger.info(
f"Created new pvc sc_name={sc_type} namespace={project_name}, "
f"size=6Gi, access_mode={constants.ACCESS_MODE_RWO}"
)
pvc_obj = helpers.create_pvc(
sc_name=sc_type,
namespace=project_name,
size="6Gi",
do_reload=False,
access_mode=constants.ACCESS_MODE_RWO,
)
logger.info(
f"Create new pod. Pod name={pod_name},"
f"interface_type={constants.CEPHBLOCKPOOL}"
)
pod_obj = helpers.create_pod(
pvc_name=pvc_obj.name,
namespace=project_obj.namespace,
interface_type=constants.CEPHBLOCKPOOL,
pod_name=pod_name,
)
logger.info(f"Wait for pod {pod_name} move to Running state")
helpers.wait_for_resource_state(
pod_obj, state=constants.STATUS_RUNNING, timeout=300
)
logger.info("Run fio workload")
pod_obj.run_io(
storage_type=constants.WORKLOAD_STORAGE_TYPE_FS,
size="4GB",
)
fio_result = pod_obj.get_fio_results()
logger.info("IOPs after FIO:")
reads = fio_result.get("jobs")[0].get("read").get("iops")
writes = fio_result.get("jobs")[0].get("write").get("iops")
logger.info(f"Read: {reads}")
logger.info(f"Write: {writes}")
validation_ui_obj = ValidationUI(setup_ui_class)
assert validation_ui_obj.check_capacity_breakdown(
project_name=project_name, pod_name=pod_name
), "The Project/Pod not created on Capacity Breakdown"
def base_setup(
self,
interface_iterate,
storageclass_factory,
pvc_size,
):
"""
A setup phase for the test - creating resources
Args:
interface_iterate: A fixture to iterate over ceph interfaces
storageclass_factory: A fixture to create everything needed for a
storageclass
pvc_size: The size of the PVC in Gi
"""
self.interface = interface_iterate
self.sc_obj = storageclass_factory(self.interface)
if self.interface == constants.CEPHBLOCKPOOL:
self.sc = "RBD"
elif self.interface == constants.CEPHFILESYSTEM:
self.sc = "CephFS"
elif self.interface == constants.CEPHBLOCKPOOL_THICK:
self.sc = "RBD-Thick"
self.create_test_project()
self.pvc_obj = helpers.create_pvc(sc_name=self.sc_obj.name,
size=pvc_size + "Gi",
namespace=self.namespace)
helpers.wait_for_resource_state(self.pvc_obj, constants.STATUS_BOUND)
self.pvc_obj.reload()
# Create a POD and attach it the the PVC
try:
self.pod_object = helpers.create_pod(
interface_type=self.interface,
pvc_name=self.pvc_obj.name,
namespace=self.namespace,
pod_dict_path=constants.PERF_POD_YAML,
)
helpers.wait_for_resource_state(self.pod_object,
constants.STATUS_RUNNING)
self.pod_object.reload()
self.pod_object.workload_setup("fs", jobs=1, fio_installed=True)
except Exception as e:
log.error(
f"Pod on PVC {self.pvc_obj.name} was not created, exception {str(e)}"
)
raise ex.PodNotCreated("Pod on PVC was not created.")
def test_basics_rbd(self, test_fixture_rbd):
"""
Testing basics: secret creation,
storage class creation,pvc and pod with rbd
"""
global RBD_PVC_OBJ, RBD_POD_OBJ
log.info("creating pvc for RBD ")
pvc_name = helpers.create_unique_resource_name("test-rbd", "pvc")
RBD_PVC_OBJ = helpers.create_pvc(sc_name=RBD_SC_OBJ.name,
pvc_name=pvc_name)
helpers.wait_for_resource_state(RBD_PVC_OBJ, constants.STATUS_BOUND)
RBD_PVC_OBJ.reload()
if RBD_PVC_OBJ.backed_pv is None:
RBD_PVC_OBJ.reload()
RBD_POD_OBJ = helpers.create_pod(
interface_type=constants.CEPHBLOCKPOOL, pvc_name=RBD_PVC_OBJ.name)
helpers.wait_for_resource_state(RBD_POD_OBJ, constants.STATUS_RUNNING)
RBD_POD_OBJ.reload()
def create_jenkins_pvc(self):
"""
create jenkins pvc
Returns:
List: pvc_objs
"""
pvc_objs = []
for project in self.projects:
log.info(f"create jenkins pvc on project {project}")
pvc_obj = create_pvc(
pvc_name="dependencies",
size="10Gi",
sc_name=constants.DEFAULT_STORAGECLASS_RBD,
namespace=project,
)
pvc_objs.append(pvc_obj)
return pvc_objs
def create_testing_pvc_and_wait_for_bound(self):
log.info("Creating PVC for the test")
try:
self.pvc_obj = helpers.create_pvc(
sc_name=self.sc_obj.name,
pvc_name="pvc-pas-test",
size=f"{self.pvc_size}Gi",
namespace=self.namespace,
# access_mode=Interfaces_info[self.interface]["accessmode"],
)
except Exception as e:
log.exception(f"The PVC was not created, exception [{str(e)}]")
raise PVCNotCreated("PVC did not reach BOUND state.")
# Wait for the PVC to be Bound
performance_lib.wait_for_resource_bulk_status(
"pvc", 1, self.namespace, constants.STATUS_BOUND, 120, 5
)
log.info(f"The PVC {self.pvc_obj.name} was created and in Bound state.")
def run(self):
"""
Running the test
"""
for i in range(self.samples_num):
# Creating PVC to attache POD to it
csi_start_time = self.get_time("csi")
log.info(f"{self.msg_prefix} Start creating PVC number {i + 1}.")
pvc_obj = helpers.create_pvc(sc_name=self.sc_obj.name,
size=self.pvc_size,
namespace=self.namespace)
helpers.wait_for_resource_state(pvc_obj, constants.STATUS_BOUND)
pvc_obj.reload()
self.pvc_list.append(pvc_obj)
log.info(
f"{self.msg_prefix} PVC number {i + 1} was successfully created ."
)
# Create a POD and attache it the the PVC
try:
pod_obj = helpers.create_pod(
interface_type=self.interface,
pvc_name=pvc_obj.name,
namespace=self.namespace,
pod_dict_path=constants.PERF_POD_YAML,
)
helpers.wait_for_resource_state(pod_obj,
constants.STATUS_RUNNING)
pod_obj.reload()
except Exception as e:
log.error(
f"Pod on PVC {pvc_obj.name} was not created, exception {str(e)}"
)
raise ex.PodNotCreated("Pod on PVC was not created.")
self.pod_result_list.append(pod_obj)
# Get the POD start time including the attache time
self.start_time_dict_list.append(helpers.pod_start_time(pod_obj))
self.csi_time_dict_list.append(
performance_lib.pod_attach_csi_time(self.interface,
pvc_obj.backed_pv,
csi_start_time,
self.namespace)[0])
def test_basics_cephfs(self, test_fixture_cephfs):
"""
Testing basics: secret creation,
storage class creation, pvc and pod with cephfs
"""
global CEPHFS_PVC_OBJ, CEPHFS_POD_OBJ
log.info("creating pvc for CephFS ")
pvc_name = helpers.create_unique_resource_name("test-cephfs", "pvc")
CEPHFS_PVC_OBJ = helpers.create_pvc(sc_name=CEPHFS_SC_OBJ.name,
pvc_name=pvc_name)
helpers.wait_for_resource_state(CEPHFS_PVC_OBJ, constants.STATUS_BOUND)
CEPHFS_PVC_OBJ.reload()
log.info("creating cephfs pod")
CEPHFS_POD_OBJ = helpers.create_pod(
interface_type=constants.CEPHFILESYSTEM,
pvc_name=CEPHFS_PVC_OBJ.name)
helpers.wait_for_resource_state(CEPHFS_POD_OBJ,
constants.STATUS_RUNNING)
CEPHFS_POD_OBJ.reload()
def create_resource_hsbench(self):
"""
Create resource for hsbench mark test:
Create service account
Create PVC
Create golang pod
"""
# Check for existing rgw pods on cluster
self.rgw_pod = pod.get_rgw_pods()
if self.rgw_pod:
# Create service account
self.sa_name = helpers.create_serviceaccount(self.namespace)
self.sa_name = self.sa_name.name
helpers.add_scc_policy(sa_name=self.sa_name,
namespace=self.namespace)
# Create test pvc+pod
log.info(
f"Create Golang pod to generate S3 workload... {self.namespace}"
)
pvc_size = "50Gi"
node_name = "compute-0"
self.pod_name = "hsbench-pod"
self.pvc_obj = helpers.create_pvc(
sc_name=constants.DEFAULT_STORAGECLASS_RBD,
namespace=self.namespace,
size=pvc_size,
)
self.pod_obj = helpers.create_pod(
constants.CEPHBLOCKPOOL,
namespace=self.namespace,
pod_name=self.pod_name,
pvc_name=self.pvc_obj.name,
node_name=node_name,
sa_name=self.sa_name,
pod_dict_path=self.pod_dic_path,
dc_deployment=True,
deploy_pod_status=constants.STATUS_COMPLETED,
)
else:
raise UnexpectedBehaviour(
"This cluster doesn't have RGW pod(s) to perform hsbench")
def _deploy_es(self):
"""
Deploying the Elasticsearch server
"""
# Creating PVC for the elasticsearch server and wait until it bound
log.info("Creating 10 GiB PVC for the ElasticSearch cluster on")
self.pvc_obj = create_pvc(
sc_name=constants.CEPHBLOCKPOOL_SC,
namespace=self.namespace,
pvc_name="elasticsearch-data-quickstart-es-default-0",
access_mode=constants.ACCESS_MODE_RWO,
size="10Gi",
)
wait_for_resource_state(self.pvc_obj, constants.STATUS_BOUND)
self.pvc_obj.reload()
log.info("Deploy the ElasticSearch cluster")
self.ocp.apply(self.crd)
sample = TimeoutSampler(
timeout=300,
sleep=10,
func=self._pod_is_found,
pattern="quickstart-es-default",
)
if not sample.wait_for_func_status(True):
self.cleanup()
raise Exception("The ElasticSearch pod deployment Failed")
self.espod = get_pod_name_by_pattern("quickstart-es-default",
self.namespace)[0]
log.info(f"The ElasticSearch pod {self.espod} Started")
es_pod = OCP(kind="pod", namespace=self.namespace)
log.info("Waiting for ElasticSearch to Run")
assert es_pod.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_name=self.espod,
sleep=30,
timeout=600,
)
log.info("Elastic Search is ready !!!")
def create_jenkins_pvc(self):
"""
create jenkins pvc
Returns:
List: pvc_objs
"""
pvc_objs = []
sc_name = (constants.DEFAULT_EXTERNAL_MODE_STORAGECLASS_RBD
if storagecluster_independent_check() else
constants.DEFAULT_STORAGECLASS_RBD)
for project in self.projects:
log.info(f"create jenkins pvc on project {project}")
pvc_obj = create_pvc(
pvc_name="dependencies",
size="10Gi",
sc_name=sc_name,
namespace=project,
)
pvc_objs.append(pvc_obj)
return pvc_objs
def test_create_multiple_sc_with_same_pool_name(self, interface_type,
resources):
"""
This test function does below,
*. Creates multiple Storage Classes with same pool name
*. Creates PVCs using each Storage Class
*. Mount each PVC to an app pod
*. Run IO on each app pod
"""
# Unpack resources
pods, pvcs, storageclasses = resources
# Create 3 Storage Classes with same pool name
if interface_type == constants.CEPHBLOCKPOOL:
secret = self.rbd_secret_obj.name
interface_name = self.cbp_obj.name
else:
interface_type = constants.CEPHFILESYSTEM
secret = self.cephfs_secret_obj.name
interface_name = helpers.get_cephfs_data_pool_name()
for i in range(3):
log.info(f"Creating a {interface_type} storage class")
storageclasses.append(
helpers.create_storage_class(
interface_type=interface_type,
interface_name=interface_name,
secret_name=secret,
))
log.info(f"{interface_type}StorageClass: {storageclasses[i].name} "
f"created successfully")
# Create PVCs using each SC
for i in range(3):
log.info(f"Creating a PVC using {storageclasses[i].name}")
pvcs.append(helpers.create_pvc(storageclasses[i].name))
for pvc in pvcs:
helpers.wait_for_resource_state(pvc, constants.STATUS_BOUND)
pvc.reload()
# Create app pod and mount each PVC
for i in range(3):
log.info(f"Creating an app pod and mount {pvcs[i].name}")
pods.append(
helpers.create_pod(
interface_type=interface_type,
pvc_name=pvcs[i].name,
namespace=defaults.ROOK_CLUSTER_NAMESPACE,
))
for pod in pods:
helpers.wait_for_resource_state(pod, constants.STATUS_RUNNING)
pod.reload()
log.info(f"{pods[i].name} created successfully and "
f"mounted {pvcs[i].name}")
# Run IO on each app pod for sometime
for pod in pods:
log.info(f"Running FIO on {pod.name}")
pod.run_io("fs", size="2G")
for pod in pods:
get_fio_rw_iops(pod)
def test_pvc_creation_deletion_measurement_performance(
self, teardown_factory, pvc_size):
"""
Measuring PVC creation and deletion times for pvc samples
Verifying that those times are within required limits
"""
# Getting the full path for the test logs
self.full_log_path = get_full_test_logs_path(cname=self)
if self.interface == constants.CEPHBLOCKPOOL:
self.sc = "RBD"
if self.interface == constants.CEPHFILESYSTEM:
self.sc = "CephFS"
self.full_log_path += f"-{self.sc}-{pvc_size}"
log.info(f"Logs file path name is : {self.full_log_path}")
self.start_time = time.strftime("%Y-%m-%dT%H:%M:%SGMT", time.gmtime())
self.get_env_info()
# Initialize the results doc file.
self.full_results = self.init_full_results(
ResultsAnalyse(self.uuid, self.crd_data, self.full_log_path))
self.full_results.add_key("pvc_size", pvc_size)
num_of_samples = 5
accepted_creation_time = 1
accepted_deletion_time = 2 if self.interface == constants.CEPHFILESYSTEM else 1
self.full_results.add_key("samples", num_of_samples)
accepted_creation_deviation_percent = 50
accepted_deletion_deviation_percent = 50
creation_time_measures = []
deletion_time_measures = []
msg_prefix = f"Interface: {self.interface}, PVC size: {pvc_size}."
for i in range(num_of_samples):
logging.info(f"{msg_prefix} Start creating PVC number {i + 1}.")
start_time = datetime.datetime.utcnow().strftime(
"%Y-%m-%dT%H:%M:%SZ")
pvc_obj = helpers.create_pvc(sc_name=self.sc_obj.name,
size=pvc_size)
helpers.wait_for_resource_state(pvc_obj, constants.STATUS_BOUND)
pvc_obj.reload()
creation_time = performance_lib.measure_pvc_creation_time(
self.interface, pvc_obj.name, start_time)
logging.info(
f"{msg_prefix} PVC number {i + 1} was created in {creation_time} seconds."
)
if creation_time > accepted_creation_time:
raise ex.PerformanceException(
f"{msg_prefix} PVC creation time is {creation_time} and is greater than "
f"{accepted_creation_time} seconds.")
creation_time_measures.append(creation_time)
pv_name = pvc_obj.backed_pv
pvc_reclaim_policy = pvc_obj.reclaim_policy
pod_obj = self.write_file_on_pvc(pvc_obj)
pod_obj.delete(wait=True)
teardown_factory(pvc_obj)
logging.info(f"{msg_prefix} Start deleting PVC number {i + 1}")
if pvc_reclaim_policy == constants.RECLAIM_POLICY_DELETE:
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(pvc_obj.name)
helpers.validate_pv_delete(pvc_obj.backed_pv)
deletion_time = helpers.measure_pvc_deletion_time(
self.interface, pv_name)
logging.info(
f"{msg_prefix} PVC number {i + 1} was deleted in {deletion_time} seconds."
)
if deletion_time > accepted_deletion_time:
raise ex.PerformanceException(
f"{msg_prefix} PVC deletion time is {deletion_time} and is greater than "
f"{accepted_deletion_time} seconds.")
deletion_time_measures.append(deletion_time)
else:
logging.info(
f"Reclaim policy of the PVC {pvc_obj.name} is not Delete;"
f" therefore not measuring deletion time for this PVC.")
creation_average = self.process_time_measurements(
"creation",
creation_time_measures,
accepted_creation_deviation_percent,
msg_prefix,
)
self.full_results.add_key("creation-time", creation_average)
deletion_average = self.process_time_measurements(
"deletion",
deletion_time_measures,
accepted_deletion_deviation_percent,
msg_prefix,
)
self.full_results.add_key("deletion-time", deletion_average)
self.full_results.all_results["creation"] = creation_time_measures
self.full_results.all_results["deletion"] = deletion_time_measures
self.end_time = time.strftime("%Y-%m-%dT%H:%M:%SGMT", time.gmtime())
self.full_results.add_key("test_time", {
"start": self.start_time,
"end": self.end_time
})
self.full_results.es_write()
log.info(
f"The Result can be found at : {self.full_results.results_link()}")
if not self.dev_mode:
# all the results are OK, the test passes, push the results to the codespeed
push_to_pvc_time_dashboard(self.interface, "1-pvc-creation",
creation_average)
push_to_pvc_time_dashboard(self.interface, "1-pvc-deletion",
deletion_average)
def raw_block_pv(self):
"""
Testing basic creation of app pod with RBD RWX raw block pv support
"""
worker_nodes = node.get_worker_nodes()
pvcs = list()
size_mb = "500Mi"
size_gb = "10Gi"
if config.ENV_DATA["platform"].lower(
) in constants.MANAGED_SERVICE_PLATFORMS:
size_tb = str(convert_device_size("50Gi", "TB")) + "Ti"
else:
size_tb = "1Ti"
for size in [size_mb, size_gb, size_tb]:
pvcs.append(
helpers.create_pvc(
sc_name=self.sc_obj.name,
size=size,
access_mode=constants.ACCESS_MODE_RWX,
namespace=self.namespace,
volume_mode="Block",
))
pvc_mb, pvc_gb, pvc_tb = pvcs[0], pvcs[1], pvcs[2]
for pvc in pvcs:
helpers.wait_for_resource_state(resource=pvc,
state=constants.STATUS_BOUND,
timeout=120)
pvs = [pvc.backed_pv_obj for pvc in pvcs]
pods = list()
pod_dict = constants.CSI_RBD_RAW_BLOCK_POD_YAML
for pvc in pvc_mb, pvc_gb, pvc_tb:
for _ in range(3):
pods.append(
helpers.create_pod(
interface_type=constants.CEPHBLOCKPOOL,
pvc_name=pvc.name,
namespace=self.namespace,
raw_block_pv=True,
pod_dict_path=pod_dict,
node_name=random.choice(worker_nodes),
))
pvc_mb_pods, pvc_gb_pods, pvc_tb_pods = pods[0:3], pods[3:6], pods[6:9]
for pod in pods:
helpers.wait_for_resource_state(resource=pod,
state=constants.STATUS_RUNNING,
timeout=120)
storage_type = "block"
with ThreadPoolExecutor() as p:
for pod in pvc_mb_pods:
log.info(f"running io on pod {pod.name}")
p.submit(
pod.run_io,
storage_type=storage_type,
size=f"{random.randint(10,200)}M",
invalidate=0,
)
for pod in pvc_gb_pods:
log.info(f"running io on pod {pod.name}")
p.submit(
pod.run_io,
storage_type=storage_type,
size=f"{random.randint(1,5)}G",
invalidate=0,
)
for pod in pvc_tb_pods:
log.info(f"running io on pod {pod.name}")
p.submit(
pod.run_io,
storage_type=storage_type,
size=f"{random.randint(10,15)}G",
invalidate=0,
)
for pod in pods:
get_fio_rw_iops(pod)
return pods, pvcs, pvs
def test_pod_reattach_time_performance(
self, storageclass_factory, copies, timeout, total_time_limit
):
"""
Test assign nodeName to a pod using RWX pvc
Each kernel (unzipped) is 892M and 61694 files
The test creates samples_num pvcs and pods, writes kernel files multiplied by number of copies
and calculates average total and csi reattach times and standard deviation
"""
kernel_url = "https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.19.5.tar.gz"
download_path = "tmp"
samples_num = 7
if self.dev_mode:
samples_num = 3
test_start_time = PASTest.get_time()
helpers.pull_images(constants.PERF_IMAGE)
# Download a linux Kernel
dir_path = os.path.join(os.getcwd(), download_path)
file_path = os.path.join(dir_path, "file.gz")
if not os.path.exists(dir_path):
os.makedirs(dir_path)
urllib.request.urlretrieve(kernel_url, file_path)
worker_nodes_list = node.get_worker_nodes()
assert len(worker_nodes_list) > 1
node_one = worker_nodes_list[0]
node_two = worker_nodes_list[1]
time_measures, csi_time_measures, files_written_list, data_written_list = (
[],
[],
[],
[],
)
self.sc_obj = storageclass_factory(self.interface)
for sample_index in range(1, samples_num + 1):
csi_start_time = self.get_time("csi")
logger.info(f"Start creating PVC number {sample_index}.")
pvc_obj = helpers.create_pvc(
sc_name=self.sc_obj.name, size="100Gi", namespace=self.namespace
)
helpers.wait_for_resource_state(pvc_obj, constants.STATUS_BOUND)
# Create a pod on one node
logger.info(f"Creating Pod with pvc {pvc_obj.name} on node {node_one}")
pvc_obj.reload()
self.pvc_list.append(pvc_obj)
try:
pod_obj1 = helpers.create_pod(
interface_type=self.interface,
pvc_name=pvc_obj.name,
namespace=pvc_obj.namespace,
node_name=node_one,
pod_dict_path=constants.PERF_POD_YAML,
)
except Exception as e:
logger.error(
f"Pod on PVC {pvc_obj.name} was not created, exception {str(e)}"
)
raise PodNotCreated("Pod on PVC was not created.")
# Confirm that pod is running on the selected_nodes
logger.info("Checking whether pods are running on the selected nodes")
helpers.wait_for_resource_state(
resource=pod_obj1, state=constants.STATUS_RUNNING, timeout=timeout
)
pod_name = pod_obj1.name
pod_path = "/mnt"
_ocp = OCP(namespace=pvc_obj.namespace)
rsh_cmd = f"rsync {dir_path} {pod_name}:{pod_path}"
_ocp.exec_oc_cmd(rsh_cmd)
rsh_cmd = (
f"exec {pod_name} -- tar xvf {pod_path}/tmp/file.gz -C {pod_path}/tmp"
)
_ocp.exec_oc_cmd(rsh_cmd)
for x in range(copies):
rsh_cmd = f"exec {pod_name} -- mkdir -p {pod_path}/folder{x}"
_ocp.exec_oc_cmd(rsh_cmd)
rsh_cmd = (
f"exec {pod_name} -- cp -r {pod_path}/tmp {pod_path}/folder{x}"
)
_ocp.exec_oc_cmd(rsh_cmd)
rsh_cmd = f"exec {pod_name} -- sync"
_ocp.exec_oc_cmd(rsh_cmd)
logger.info("Getting the amount of data written to the PVC")
rsh_cmd = f"exec {pod_name} -- df -h {pod_path}"
data_written_str = _ocp.exec_oc_cmd(rsh_cmd).split()[-4]
logger.info(f"The amount of written data is {data_written_str}")
data_written = float(data_written_str[:-1])
rsh_cmd = f"exec {pod_name} -- find {pod_path} -type f"
files_written = len(_ocp.exec_oc_cmd(rsh_cmd).split())
logger.info(
f"For {self.interface} - The number of files written to the pod is {files_written}"
)
files_written_list.append(files_written)
data_written_list.append(data_written)
logger.info("Deleting the pod")
rsh_cmd = f"delete pod {pod_name}"
_ocp.exec_oc_cmd(rsh_cmd)
logger.info(f"Creating Pod with pvc {pvc_obj.name} on node {node_two}")
try:
pod_obj2 = helpers.create_pod(
interface_type=self.interface,
pvc_name=pvc_obj.name,
namespace=pvc_obj.namespace,
node_name=node_two,
pod_dict_path=constants.PERF_POD_YAML,
)
except Exception as e:
logger.error(
f"Pod on PVC {pvc_obj.name} was not created, exception {str(e)}"
)
raise PodNotCreated("Pod on PVC was not created.")
start_time = time.time()
pod_name = pod_obj2.name
helpers.wait_for_resource_state(
resource=pod_obj2, state=constants.STATUS_RUNNING, timeout=timeout
)
end_time = time.time()
total_time = end_time - start_time
if total_time > total_time_limit:
logger.error(
f"Pod creation time is {total_time} and greater than {total_time_limit} seconds"
)
raise ex.PerformanceException(
f"Pod creation time is {total_time} and greater than {total_time_limit} seconds"
)
csi_time = performance_lib.pod_attach_csi_time(
self.interface, pvc_obj.backed_pv, csi_start_time, pvc_obj.namespace
)[0]
csi_time_measures.append(csi_time)
logger.info(
f"PVC #{pvc_obj.name} pod {pod_name} creation time took {total_time} seconds, "
f"csi time is {csi_time} seconds"
)
time_measures.append(total_time)
logger.info("Deleting the pod")
rsh_cmd = f"delete pod {pod_name}"
_ocp.exec_oc_cmd(rsh_cmd)
# teardown_factory(pod_obj2)
average = statistics.mean(time_measures)
logger.info(
f"The average time of {self.interface} pod creation on {samples_num} PVCs is {average} seconds"
)
st_deviation = statistics.stdev(time_measures)
logger.info(
f"The standard deviation of {self.interface} pod creation time on {samples_num} PVCs is {st_deviation}"
)
csi_average = statistics.mean(csi_time_measures)
logger.info(
f"The average csi time of {self.interface} pod creation on {samples_num} PVCs is {csi_average} seconds"
)
csi_st_deviation = statistics.stdev(csi_time_measures)
logger.info(
f"The standard deviation of {self.interface} csi pod creation time on {samples_num} PVCs "
f"is {csi_st_deviation}"
)
files_written_average = statistics.mean(files_written_list)
data_written_average = statistics.mean(data_written_list)
os.remove(file_path)
os.rmdir(dir_path)
# Produce ES report
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"pod_reattach_time_fullres",
)
)
full_results.add_key("storageclass", self.sc)
full_results.add_key("pod_reattach_time", time_measures)
full_results.add_key("copies_number", copies)
full_results.add_key("files_number_average", files_written_average)
full_results.add_key("data_average", data_written_average)
full_results.add_key("pod_reattach_time_average", average)
full_results.add_key("pod_reattach_standard_deviation", st_deviation)
full_results.add_key("pod_csi_reattach_time_average", csi_average)
full_results.add_key("pod_csi_reattach_standard_deviation", csi_st_deviation)
test_end_time = PASTest.get_time()
# Add the test time to the ES report
full_results.add_key(
"test_time", {"start": test_start_time, "end": test_end_time}
)
# Write the test results into the ES server
if full_results.es_write():
res_link = full_results.results_link()
logger.info(f"The Result can be found at : {res_link}")
# Create text file with results of all subtest (4 - according to the parameters)
self.results_path = get_full_test_logs_path(
cname=self, fname="test_pod_reattach_time_performance"
)
self.write_result_to_file(res_link)
def test_pvc_creation_deletion_measurement_performance(
self, teardown_factory, pvc_size
):
"""
Measuring PVC creation and deletion times for pvc samples
Verifying that those times are within required limits
"""
num_of_samples = 5
accepted_creation_time = 1
accepted_deletion_time = 2 if self.interface == constants.CEPHFILESYSTEM else 1
accepted_creation_deviation_percent = 50
accepted_deletion_deviation_percent = 50
creation_time_measures = []
deletion_time_measures = []
msg_prefix = f"Interface: {self.interface}, PVC size: {pvc_size}."
for i in range(num_of_samples):
logging.info(f"{msg_prefix} Start creating PVC number {i + 1}.")
start_time = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
pvc_obj = helpers.create_pvc(sc_name=self.sc_obj.name, size=pvc_size)
helpers.wait_for_resource_state(pvc_obj, constants.STATUS_BOUND)
pvc_obj.reload()
creation_time = performance_lib.measure_pvc_creation_time(
self.interface, pvc_obj.name, start_time
)
logging.info(
f"{msg_prefix} PVC number {i + 1} was created in {creation_time} seconds."
)
if creation_time > accepted_creation_time:
raise ex.PerformanceException(
f"{msg_prefix} PVC creation time is {creation_time} and is greater than "
f"{accepted_creation_time} seconds."
)
creation_time_measures.append(creation_time)
pv_name = pvc_obj.backed_pv
pvc_reclaim_policy = pvc_obj.reclaim_policy
pod_obj = self.write_file_on_pvc(pvc_obj)
pod_obj.delete(wait=True)
teardown_factory(pvc_obj)
logging.info(f"{msg_prefix} Start deleting PVC number {i + 1}")
if pvc_reclaim_policy == constants.RECLAIM_POLICY_DELETE:
pvc_obj.delete()
pvc_obj.ocp.wait_for_delete(pvc_obj.name)
helpers.validate_pv_delete(pvc_obj.backed_pv)
deletion_time = helpers.measure_pvc_deletion_time(
self.interface, pv_name
)
logging.info(
f"{msg_prefix} PVC number {i + 1} was deleted in {deletion_time} seconds."
)
if deletion_time > accepted_deletion_time:
raise ex.PerformanceException(
f"{msg_prefix} PVC deletion time is {deletion_time} and is greater than "
f"{accepted_deletion_time} seconds."
)
deletion_time_measures.append(deletion_time)
else:
logging.info(
f"Reclaim policy of the PVC {pvc_obj.name} is not Delete;"
f" therefore not measuring deletion time for this PVC."
)
creation_average = self.process_time_measurements(
"creation",
creation_time_measures,
accepted_creation_deviation_percent,
msg_prefix,
)
deletion_average = self.process_time_measurements(
"deletion",
deletion_time_measures,
accepted_deletion_deviation_percent,
msg_prefix,
)
# all the results are OK, the test passes, push the results to the codespeed
push_to_pvc_time_dashboard(self.interface, "1-pvc-creation", creation_average)
push_to_pvc_time_dashboard(self.interface, "1-pvc-deletion", deletion_average)
def test_create_multiple_sc_with_different_pool_name(
self, teardown_factory):
"""
This test function does below,
*. Creates multiple Storage Classes with different pool name
*. Creates PVCs using each Storage Class
*. Mount each PVC to an app pod
*. Run IO on each app pod
"""
# Create 2 storageclasses, each with different pool name
cbp_list = []
sc_list = []
for i in range(2):
log.info("Creating cephblockpool")
cbp_obj = helpers.create_ceph_block_pool()
log.info(f"{cbp_obj.name} created successfully")
log.info(f"Creating a RBD storage class using {cbp_obj.name}")
cbp_list.append(cbp_obj)
sc_obj = helpers.create_storage_class(
interface_type=constants.CEPHBLOCKPOOL,
interface_name=cbp_obj.name,
secret_name=self.rbd_secret_obj.name,
)
log.info(f"StorageClass: {sc_obj.name} "
f"created successfully using {cbp_obj.name}")
sc_list.append(sc_obj)
teardown_factory(cbp_obj)
teardown_factory(sc_obj)
# Create PVCs using each SC
pvc_list = []
for i in range(2):
log.info(f"Creating a PVC using {sc_list[i].name}")
pvc_obj = helpers.create_pvc(sc_list[i].name)
log.info(f"PVC: {pvc_obj.name} created successfully using "
f"{sc_list[i].name}")
pvc_list.append(pvc_obj)
teardown_factory(pvc_obj)
helpers.wait_for_resource_state(pvc_obj, constants.STATUS_BOUND)
pvc_obj.reload()
# Create app pod and mount each PVC
pod_list = []
for i in range(2):
log.info(f"Creating an app pod and mount {pvc_list[i].name}")
pod_obj = helpers.create_pod(
interface_type=constants.CEPHBLOCKPOOL,
pvc_name=pvc_list[i].name,
)
log.info(f"{pod_obj.name} created successfully and "
f"mounted {pvc_list[i].name}")
pod_list.append(pod_obj)
teardown_factory(pod_obj)
helpers.wait_for_resource_state(pod_obj, constants.STATUS_RUNNING)
pod_obj.reload()
# Run IO on each app pod for sometime
for pod in pod_list:
log.info(f"Running FIO on {pod.name}")
pod.run_io("fs", size="2G")
for pod in pod_list:
get_fio_rw_iops(pod)
