def test_respin_mcg_pod_and_check_data_integrity(self, mcg_obj, cld_mgr,
awscli_pod,
ns_resource_factory,
bucket_factory, mcg_pod):
"""
Test Write to ns bucket using MCG RPC and read directly from AWS.
Respin one of mcg pods when data are uploaded.
"""
logger.info("Create the namespace resource and verify health")
resource = ns_resource_factory()
target_bucket_name = resource[0]
ns_resource_name = resource[1]
s3_creds = {
"access_key_id": cld_mgr.aws_client.access_key,
"access_key": cld_mgr.aws_client.secret_key,
"endpoint": constants.MCG_NS_AWS_ENDPOINT,
"region": self.DEFAULT_REGION,
}
logger.info(
"Create the namespace bucket on top of the namespace resource")
rand_ns_bucket = bucket_factory(
amount=1,
interface="mcg-namespace",
write_ns_resource=ns_resource_name,
read_ns_resources=[ns_resource_name],
)[0].name
logger.info("Upload files to NS bucket")
self.write_files_to_pod_and_upload(mcg_obj,
awscli_pod,
bucket_to_write=rand_ns_bucket,
amount=3)
logger.info(f"Respin mcg resource {mcg_pod}")
noobaa_pods = pod.get_noobaa_pods()
pod_obj = [pod for pod in noobaa_pods
if pod.name.startswith(mcg_pod)][0]
pod_obj.delete(force=True)
logger.info("Wait for noobaa pods to come up")
assert pod_obj.ocp.wait_for_resource(
condition="Running",
selector="app=noobaa",
resource_count=len(noobaa_pods),
timeout=1000,
)
logger.info("Wait for noobaa health to be OK")
ceph_cluster_obj = CephCluster()
ceph_cluster_obj.wait_for_noobaa_health_ok()
logger.info("Read files directly from AWS")
self.download_files(mcg_obj,
awscli_pod,
bucket_to_read=target_bucket_name,
s3_creds=s3_creds)
logger.info("Compare between uploaded files and downloaded files")
assert self.compare_dirs(awscli_pod, amount=3)
class Sanity:
"""
Class for cluster health and functional validations
"""
def __init__(self):
"""
Initializer for Sanity class - Init CephCluster() in order to
set the cluster status before starting the tests
"""
self.pvc_objs = list()
self.pod_objs = list()
self.obc_objs = list()
self.obj_data = ""
self.ceph_cluster = CephCluster()
def health_check(self, cluster_check=True, tries=20):
"""
Perform Ceph and cluster health checks
"""
wait_for_cluster_connectivity(tries=400)
logger.info("Checking cluster and Ceph health")
node.wait_for_nodes_status(timeout=300)
ceph_health_check(namespace=config.ENV_DATA["cluster_namespace"],
tries=tries)
if cluster_check:
self.ceph_cluster.cluster_health_check(timeout=60)
def create_resources(self,
pvc_factory,
pod_factory,
bucket_factory,
rgw_bucket_factory,
run_io=True):
"""
Sanity validation: Create resources - pods, OBCs (RGW and MCG), PVCs (FS and RBD) and run IO
Args:
pvc_factory (function): A call to pvc_factory function
pod_factory (function): A call to pod_factory function
bucket_factory (function): A call to bucket_factory function
rgw_bucket_factory (function): A call to rgw_bucket_factory function
run_io (bool): True for run IO, False otherwise
"""
logger.info(
"Creating resources and running IO as a sanity functional validation"
)
for interface in [constants.CEPHBLOCKPOOL, constants.CEPHFILESYSTEM]:
pvc_obj = pvc_factory(interface)
self.pvc_objs.append(pvc_obj)
self.pod_objs.append(pod_factory(pvc=pvc_obj, interface=interface))
if run_io:
for pod in self.pod_objs:
pod.run_io("fs", "1G", runtime=30)
for pod in self.pod_objs:
get_fio_rw_iops(pod)
if rgw_bucket_factory:
self.obc_objs.extend(rgw_bucket_factory(1, "rgw-oc"))
if bucket_factory:
self.obc_objs.extend(bucket_factory(amount=1, interface="OC"))
self.ceph_cluster.wait_for_noobaa_health_ok()
def delete_resources(self):
"""
Sanity validation - Delete resources (pods, PVCs and OBCs)
"""
logger.info("Deleting resources as a sanity functional validation")
for pod_obj in self.pod_objs:
pod_obj.delete()
for pod_obj in self.pod_objs:
pod_obj.ocp.wait_for_delete(pod_obj.name)
for pvc_obj in self.pvc_objs:
pvc_obj.delete()
for pvc_obj in self.pvc_objs:
pvc_obj.ocp.wait_for_delete(pvc_obj.name)
for obc_obj in self.obc_objs:
obc_obj.delete(), f"OBC {obc_obj.name} still exists"
@ignore_leftovers
def create_pvc_delete(self, multi_pvc_factory, project=None):
"""
Creates and deletes all types of PVCs
"""
# Create rbd pvcs
pvc_objs_rbd = create_pvcs(
multi_pvc_factory=multi_pvc_factory,
interface="CephBlockPool",
project=project,
status="",
storageclass=None,
)
# Create cephfs pvcs
pvc_objs_cephfs = create_pvcs(
multi_pvc_factory=multi_pvc_factory,
interface="CephFileSystem",
project=project,
status="",
storageclass=None,
)
all_pvc_to_delete = pvc_objs_rbd + pvc_objs_cephfs
# Check pvc status
for pvc_obj in all_pvc_to_delete:
helpers.wait_for_resource_state(resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=300)
# Start deleting PVC
delete_pvcs(all_pvc_to_delete)
# Check PVCs are deleted
for pvc_obj in all_pvc_to_delete:
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name)
logger.info("All PVCs are deleted as expected")
def obc_put_obj_create_delete(self, mcg_obj, bucket_factory):
"""
Creates bucket then writes, reads and deletes objects
"""
bucket_name = bucket_factory(amount=1, interface="OC")[0].name
self.obj_data = "A string data"
for i in range(0, 30):
key = "Object-key-" + f"{i}"
logger.info(f"Write, read and delete object with key: {key}")
assert s3_put_object(mcg_obj, bucket_name, key,
self.obj_data), f"Failed: Put object, {key}"
assert s3_get_object(mcg_obj, bucket_name,
key), f"Failed: Get object, {key}"
assert s3_delete_object(mcg_obj, bucket_name,
key), f"Failed: Delete object, {key}"
class Sanity:
"""
Class for cluster health and functional validations
"""
def __init__(self):
"""
Initializer for Sanity class - Init CephCluster() in order to
set the cluster status before starting the tests
"""
self.pvc_objs = list()
self.pod_objs = list()
self.obj_data = ""
self.ceph_cluster = CephCluster()
def health_check(self, cluster_check=True, tries=20):
"""
Perform Ceph and cluster health checks
"""
wait_for_cluster_connectivity(tries=400)
logger.info("Checking cluster and Ceph health")
node.wait_for_nodes_status(timeout=300)
ceph_health_check(namespace=config.ENV_DATA["cluster_namespace"],
tries=tries)
if cluster_check:
self.ceph_cluster.cluster_health_check(timeout=60)
def create_resources(self, pvc_factory, pod_factory, run_io=True):
"""
Sanity validation - Create resources (FS and RBD) and run IO
Args:
pvc_factory (function): A call to pvc_factory function
pod_factory (function): A call to pod_factory function
run_io (bool): True for run IO, False otherwise
"""
logger.info(
"Creating resources and running IO as a sanity functional validation"
)
for interface in [constants.CEPHBLOCKPOOL, constants.CEPHFILESYSTEM]:
pvc_obj = pvc_factory(interface)
self.pvc_objs.append(pvc_obj)
self.pod_objs.append(pod_factory(pvc=pvc_obj, interface=interface))
if run_io:
for pod in self.pod_objs:
pod.run_io("fs", "1G", runtime=30)
for pod in self.pod_objs:
get_fio_rw_iops(pod)
self.create_obc()
self.verify_obc()
def create_obc(self):
"""
OBC creation for RGW and Nooba
"""
if config.ENV_DATA["platform"] in constants.ON_PREM_PLATFORMS:
obc_rgw = templating.load_yaml(constants.RGW_OBC_YAML)
obc_rgw_data_yaml = tempfile.NamedTemporaryFile(
mode="w+", prefix="obc_rgw_data", delete=False)
templating.dump_data_to_temp_yaml(obc_rgw, obc_rgw_data_yaml.name)
logger.info("Creating OBC for rgw")
run_cmd(f"oc create -f {obc_rgw_data_yaml.name}", timeout=2400)
self.obc_rgw = obc_rgw["metadata"]["name"]
obc_nooba = templating.load_yaml(constants.MCG_OBC_YAML)
obc_mcg_data_yaml = tempfile.NamedTemporaryFile(mode="w+",
prefix="obc_mcg_data",
delete=False)
templating.dump_data_to_temp_yaml(obc_nooba, obc_mcg_data_yaml.name)
logger.info("create OBC for mcg")
run_cmd(f"oc create -f {obc_mcg_data_yaml.name}", timeout=2400)
self.obc_mcg = obc_nooba["metadata"]["name"]
def delete_obc(self):
"""
Clenaup OBC resources created above
"""
if config.ENV_DATA["platform"] in constants.ON_PREM_PLATFORMS:
logger.info(f"Deleting rgw obc {self.obc_rgw}")
obcrgw = OCP(kind="ObjectBucketClaim",
resource_name=f"{self.obc_rgw}")
run_cmd(f"oc delete obc/{self.obc_rgw}")
obcrgw.wait_for_delete(resource_name=f"{self.obc_rgw}",
timeout=300)
logger.info(f"Deleting mcg obc {self.obc_mcg}")
obcmcg = OCP(kind="ObjectBucketClaim", resource_name=f"{self.obc_mcg}")
run_cmd(f"oc delete obc/{self.obc_mcg} -n "
f"{defaults.ROOK_CLUSTER_NAMESPACE}")
obcmcg.wait_for_delete(resource_name=f"{self.obc_mcg}", timeout=300)
def verify_obc(self):
"""
OBC verification from external cluster perspective,
we will check 2 OBCs
"""
self.ceph_cluster.wait_for_noobaa_health_ok()
def delete_resources(self):
"""
Sanity validation - Delete resources (FS and RBD)
"""
logger.info("Deleting resources as a sanity functional validation")
self.delete_obc()
for pod_obj in self.pod_objs:
pod_obj.delete()
for pod_obj in self.pod_objs:
pod_obj.ocp.wait_for_delete(pod_obj.name)
for pvc_obj in self.pvc_objs:
pvc_obj.delete()
for pvc_obj in self.pvc_objs:
pvc_obj.ocp.wait_for_delete(pvc_obj.name)
@ignore_leftovers
def create_pvc_delete(self, multi_pvc_factory, project=None):
"""
Creates and deletes all types of PVCs
"""
# Create rbd pvcs
pvc_objs_rbd = create_pvcs(
multi_pvc_factory=multi_pvc_factory,
interface="CephBlockPool",
project=project,
status="",
storageclass=None,
)
# Create cephfs pvcs
pvc_objs_cephfs = create_pvcs(
multi_pvc_factory=multi_pvc_factory,
interface="CephFileSystem",
project=project,
status="",
storageclass=None,
)
all_pvc_to_delete = pvc_objs_rbd + pvc_objs_cephfs
# Check pvc status
for pvc_obj in all_pvc_to_delete:
helpers.wait_for_resource_state(resource=pvc_obj,
state=constants.STATUS_BOUND,
timeout=300)
# Start deleting PVC
delete_pvcs(all_pvc_to_delete)
# Check PVCs are deleted
for pvc_obj in all_pvc_to_delete:
pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name)
logger.info("All PVCs are deleted as expected")
def obc_put_obj_create_delete(self, mcg_obj, bucket_factory):
"""
Creates bucket then writes, reads and deletes objects
"""
bucket_name = bucket_factory(amount=1, interface="OC")[0].name
self.obj_data = "A string data"
for i in range(0, 30):
key = "Object-key-" + f"{i}"
logger.info(f"Write, read and delete object with key: {key}")
assert s3_put_object(mcg_obj, bucket_name, key,
self.obj_data), f"Failed: Put object, {key}"
assert s3_get_object(mcg_obj, bucket_name,
key), f"Failed: Get object, {key}"
assert s3_delete_object(mcg_obj, bucket_name,
key), f"Failed: Delete object, {key}"
def test_respin_mcg_pod_and_check_data_integrity_crd(
self,
mcg_obj,
cld_mgr,
awscli_pod,
namespace_store_factory,
bucket_factory,
mcg_pod,
):
"""
Test Write to ns bucket using CRDs and read directly from AWS.
Respin one of mcg pods when data are uploaded.
"""
logger.info("Create the namespace resources and verify health")
nss_tup = ("oc", {"aws": [(1, self.DEFAULT_REGION)]})
ns_store = namespace_store_factory(*nss_tup)[0]
logger.info(
"Create the namespace bucket on top of the namespace stores")
bucketclass_dict = {
"interface": "OC",
"namespace_policy_dict": {
"type": "Single",
"namespacestores": [ns_store],
},
}
logger.info(
"Create the namespace bucket on top of the namespace resource")
ns_bucket = bucket_factory(
amount=1,
interface=bucketclass_dict["interface"],
bucketclass=bucketclass_dict,
)[0].name
s3_creds = {
"access_key_id": cld_mgr.aws_client.access_key,
"access_key": cld_mgr.aws_client.secret_key,
"endpoint": constants.MCG_NS_AWS_ENDPOINT,
"region": self.DEFAULT_REGION,
}
logger.info("Upload files to NS bucket")
self.write_files_to_pod_and_upload(mcg_obj,
awscli_pod,
bucket_to_write=ns_bucket,
amount=3)
logger.info(f"Respin mcg resource {mcg_pod}")
noobaa_pods = pod.get_noobaa_pods()
pod_obj = [pod for pod in noobaa_pods
if pod.name.startswith(mcg_pod)][0]
pod_obj.delete(force=True)
logger.info("Wait for noobaa pods to come up")
assert pod_obj.ocp.wait_for_resource(
condition="Running",
selector="app=noobaa",
resource_count=len(noobaa_pods),
timeout=1000,
)
logger.info("Wait for noobaa health to be OK")
ceph_cluster_obj = CephCluster()
ceph_cluster_obj.wait_for_noobaa_health_ok()
logger.info("Read files directly from AWS")
self.download_files(mcg_obj,
awscli_pod,
bucket_to_read=ns_store.uls_name,
s3_creds=s3_creds)
logger.info("Compare between uploaded files and downloaded files")
assert self.compare_dirs(awscli_pod, amount=3)
