def test_drain_mcg_pod_node(self, node_drain_teardown, reduce_and_resume_cluster_load, pod_to_drain): """ Test drianage of nodes which contain NB resources """ # Retrieve the relevant pod object pod_obj = pod.Pod(**pod.get_pods_having_label( label=self.labels_map[pod_to_drain], namespace=defaults.ROOK_CLUSTER_NAMESPACE, )[0]) # Retrieve the node name on which the pod resides node_name = pod_obj.get()["spec"]["nodeName"] # Drain the node drain_nodes([node_name]) # Verify the node was drained properly wait_for_nodes_status([node_name], status=constants.NODE_READY_SCHEDULING_DISABLED) # Retrieve the new pod that should've been created post-drainage pod_obj = pod.Pod(**pod.get_pods_having_label( label=self.labels_map[pod_to_drain], namespace=defaults.ROOK_CLUSTER_NAMESPACE, )[0]) # Verify that the new pod has reached a 'RUNNNING' status again and recovered successfully wait_for_resource_state(pod_obj, constants.STATUS_RUNNING, timeout=120) # Check the NB status to verify the system is healthy self.cl_obj.wait_for_noobaa_health_ok()
def get_registry_pod_obj(): """ Function to get registry pod obj Returns: pod_obj (list): List of Registry pod objs Raises: UnexpectedBehaviour: When image-registry pod is not present. """ # Sometimes when there is a update in config crd, there will be 2 registry pods # i.e. old pod will be terminated and new pod will be up based on new crd # so below loop waits till old pod terminates wait_time = 30 for iteration in range(10): pod_data = pod.get_pods_having_label( label='docker-registry=default', namespace=constants.OPENSHIFT_IMAGE_REGISTRY_NAMESPACE) pod_obj = [pod.Pod(**data) for data in pod_data] if len(pod_obj) == 1: break elif len(pod_obj) == 0: raise UnexpectedBehaviour("Image-registry pod not present") elif iteration > 5: raise UnexpectedBehaviour( "Waited for 3 mins Image-registry pod is not in Running state") else: logger.info( f"Waiting for 30 sec's for registry pod to be up iteration {iteration}" ) time.sleep(wait_time) return pod_obj
def create_pod(desired_status=constants.STATUS_RUNNING, wait=True, **kwargs): """ Create a pod Args: desired_status (str): The status of the pod to wait for wait (bool): True for waiting for the pod to reach the desired status, False otherwise **kwargs: The pod data yaml converted to dict Returns: Pod: A Pod instance Raises: AssertionError: In case of any failure """ pod_obj = pod.Pod(**kwargs) pod_name = kwargs.get('metadata').get('name') created_resource = pod_obj.create() assert created_resource, ( f"Failed to create resource {pod_name}" ) if wait: assert pod_obj.ocp.wait_for_resource( condition=desired_status, resource_name=pod_name ), f"{pod_obj.kind} {pod_name} failed to reach" f"status {desired_status}" return pod_obj
def test_db_scc(self, teardown): """ Test noobaa db is assigned with scc(anyuid) after changing the default noobaa SCC """ scc_name = constants.NOOBAA_DB_SERVICE_ACCOUNT_NAME service_account = constants.NOOBAA_DB_SERVICE_ACCOUNT pod_obj = pod.Pod(**pod.get_pods_having_label( label=self.labels_map["noobaa_db"], namespace=defaults.ROOK_CLUSTER_NAMESPACE, )[0]) ocp_scc = ocp.OCP(kind=constants.SCC, namespace=defaults.ROOK_CLUSTER_NAMESPACE) pod_data = pod_obj.get() log.info(f"Verifying current SCC is {scc_name} in db pod") assert (pod_data.get("metadata").get("annotations").get( "openshift.io/scc") == scc_name), "Invalid default scc" log.info("Deleting the user array from the Noobaa scc") ocp_scc.patch( resource_name=scc_name, params='[{"op": "remove", "path": "/users/0", ' f'"value":{service_account}}}]', format_type="json", ) assert not helpers.validate_scc_policy( sa_name=scc_name, namespace=defaults.ROOK_CLUSTER_NAMESPACE, scc_name=scc_name, ), "SA name is present in noobaa scc" log.info("Adding the noobaa system sa user to anyuid scc") ocp_scc.patch( resource_name=constants.ANYUID, params='[{"op": "add", "path": "/users/0", ' f'"value":{service_account}}}]', format_type="json", ) assert helpers.validate_scc_policy( sa_name=scc_name, namespace=defaults.ROOK_CLUSTER_NAMESPACE, scc_name=constants.ANYUID, ), "SA name is not present in anyuid scc" pod_obj.delete(force=True) # Verify that the new pod has reached a 'RUNNNING' status assert pod_obj.ocp.wait_for_resource( condition=constants.STATUS_RUNNING, selector=self.labels_map["noobaa_db"], resource_count=1, timeout=300, ), "Noobaa pod did not reach running state" pod_data = pod_obj.get() log.info("Verifying SCC is now anyuid in the db pod") assert (pod_data.get("metadata").get("annotations").get( "openshift.io/scc") == constants.ANYUID), "Invalid scc" # Check the NB status to verify the system is healthy self.cl_obj.wait_for_noobaa_health_ok()
def create_pod(interface_type=None, pvc_name=None, desired_status=constants.STATUS_RUNNING, wait=True, namespace=defaults.ROOK_CLUSTER_NAMESPACE, node_name=None, pod_dict_path=None): """ Create a pod Args: interface_type (str): The interface type (CephFS, RBD, etc.) pvc_name (str): The PVC that should be attached to the newly created pod desired_status (str): The status of the pod to wait for wait (bool): True for waiting for the pod to reach the desired status, False otherwise namespace (str): The namespace for the new resource creation node_name (str): The name of specific node to schedule the pod pod_dict_path (str): YAML path for the pod Returns: Pod: A Pod instance Raises: AssertionError: In case of any failure """ if interface_type == constants.CEPHBLOCKPOOL: pod_dict = pod_dict_path if pod_dict_path else constants.CSI_RBD_POD_YAML interface = constants.RBD_INTERFACE else: pod_dict = pod_dict_path if pod_dict_path else constants.CSI_CEPHFS_POD_YAML interface = constants.CEPHFS_INTERFACE pod_data = templating.load_yaml_to_dict(pod_dict) pod_data['metadata']['name'] = create_unique_resource_name( f'test-{interface}', 'pod') pod_data['metadata']['namespace'] = namespace if pvc_name: pod_data['spec']['volumes'][0]['persistentVolumeClaim'][ 'claimName'] = pvc_name if node_name: pod_data['spec']['nodeName'] = node_name else: if 'nodeName' in pod_data.get('spec'): del pod_data['spec']['nodeName'] pod_obj = pod.Pod(**pod_data) pod_name = pod_data.get('metadata').get('name') created_resource = pod_obj.create(do_reload=wait) assert created_resource, (f"Failed to create resource {pod_name}") if wait: assert wait_for_resource_state(resource=pod_obj, state=desired_status, timeout=120) return pod_obj
def test_delete_noobaa_resources(self, resource_to_delete): """ Test Noobaa resources delete and check Noobaa health """ pod_obj = pod.Pod(**pod.get_pods_having_label( label=self.labels_map[resource_to_delete], namespace=defaults.ROOK_CLUSTER_NAMESPACE)[0]) pod_obj.delete(force=True) assert pod_obj.ocp.wait_for_resource( condition=constants.STATUS_RUNNING, selector=self.labels_map[resource_to_delete], resource_count=1, timeout=300) self.cl_obj.wait_for_noobaa_health_ok()
def finalizer(): scc_name = constants.NOOBAA_DB_SERVICE_ACCOUNT_NAME service_account = constants.NOOBAA_DB_SERVICE_ACCOUNT pod_obj = pod.Pod(**pod.get_pods_having_label( label=self.labels_map["noobaa_db"], namespace=defaults.ROOK_CLUSTER_NAMESPACE, )[0]) pod_data_list = pod_obj.get() ocp_scc = ocp.OCP(kind=constants.SCC, namespace=defaults.ROOK_CLUSTER_NAMESPACE) if helpers.validate_scc_policy( sa_name=scc_name, namespace=defaults.ROOK_CLUSTER_NAMESPACE, scc_name=constants.ANYUID, ): ocp_scc.patch( resource_name=constants.ANYUID, params='[{"op": "remove", "path": "/users/0", ' f'"value":{service_account}}}]', format_type="json", ) if not helpers.validate_scc_policy( sa_name=scc_name, namespace=defaults.ROOK_CLUSTER_NAMESPACE, scc_name=scc_name, ): ocp_scc.patch( resource_name=scc_name, params='[{"op": "add", "path": "/users/0", ' f'"value":{service_account}}}]', format_type="json", ) if (pod_data_list.get("metadata").get("annotations").get( "openshift.io/scc") == constants.ANYUID): pod_obj.delete(force=True) assert pod_obj.ocp.wait_for_resource( condition=constants.STATUS_RUNNING, selector=self.labels_map["noobaa_db"], resource_count=1, timeout=300, ), "Noobaa pod did not reach running state" pod_data_list = pod_obj.get() assert (pod_data_list.get("metadata").get("annotations").get( "openshift.io/scc") == scc_name), "Invalid scc"
def test_restart_noobaa_resources(self, resource_to_delete): """ Test Noobaa resources restart and check Noobaa health """ labels_map = { 'noobaa_core': constants.NOOBAA_CORE_POD_LABEL, 'noobaa_db': constants.NOOBAA_DB_LABEL } pod_obj = self.resource_obj = pod.Pod(**pod.get_pods_having_label( label=labels_map[resource_to_delete], namespace=defaults.ROOK_CLUSTER_NAMESPACE)[0]) pod_obj.delete(force=True) assert pod_obj.ocp.wait_for_resource( condition=constants.STATUS_RUNNING, selector=labels_map[resource_to_delete], resource_count=1, timeout=300) self.cl_obj.wait_for_noobaa_health_ok()
def get_registry_pod_obj(): """ Function to get registry pod obj Returns: pod_obj (list): List of Registry pod objs Raises: UnexpectedBehaviour: When image-registry pod is not present. """ # Sometimes when there is a update in config crd, there will be 2 registry pods # i.e. old pod will be terminated and new pod will be up based on new crd # so below loop waits till old pod terminates registry_deployment = ocp.OCP( kind="deployment", namespace=constants.OPENSHIFT_IMAGE_REGISTRY_NAMESPACE, resource_name=constants.OPENSHIFT_IMAGE_REGISTRY_DEPLOYMENT, ) replicas = registry_deployment.data["spec"].get("replicas", 1) registry_pods = ocp.OCP( kind="pod", namespace=constants.OPENSHIFT_IMAGE_REGISTRY_NAMESPACE, selector=constants.OPENSHIFT_IMAGE_SELECTOR, ) registry_pods.wait_for_resource( condition=constants.STATUS_RUNNING, timeout=400, resource_count=replicas, dont_allow_other_resources=True, ) pod_objs = [pod.Pod(**data) for data in registry_pods.data["items"]] pod_objs_len = len(pod_objs) if pod_objs_len == 0: raise UnexpectedBehaviour("No image-registry pod is present!") elif pod_objs_len != replicas: raise UnexpectedBehaviour( f"Expected {replicas} image-registry pod(s), but {pod_objs_len} " f"found!") return pod_objs
def test_mcg_namespace_disruptions_crd( self, mcg_obj, cld_mgr, awscli_pod, bucketclass_dict, bucket_factory, node_drain_teardown, ): """ Test MCG namespace disruption flow 1. Create NS resources with CRDs 2. Create NS bucket with CRDs 3. Upload to NS bucket 4. Delete noobaa related pods and verify integrity of objects 5. Create public access policy on NS bucket and verify Get op 6. Drain nodes containing noobaa pods and verify integrity of objects 7. Perform put operation to validate public access denial 7. Edit/verify and remove objects on NS bucket """ data = "Sample string content to write to a S3 object" object_key = "ObjKey-" + str(uuid.uuid4().hex) awscli_node_name = awscli_pod.get()["spec"]["nodeName"] aws_s3_creds = { "access_key_id": cld_mgr.aws_client.access_key, "access_key": cld_mgr.aws_client.secret_key, "endpoint": constants.MCG_NS_AWS_ENDPOINT, "region": config.ENV_DATA["region"], } # S3 account details user_name = "nb-user" + str(uuid.uuid4().hex) email = user_name + "@mail.com" logger.info("Setting up test files for upload, to the bucket/resources") setup_base_objects(awscli_pod, MCG_NS_ORIGINAL_DIR, MCG_NS_RESULT_DIR, amount=3) # Create the namespace resource and verify health ns_buc = bucket_factory( amount=1, interface=bucketclass_dict["interface"], bucketclass=bucketclass_dict, )[0] ns_bucket = ns_buc.name aws_target_bucket = ns_buc.bucketclass.namespacestores[0].uls_name logger.info(f"Namespace bucket: {ns_bucket} created") logger.info(f"Uploading objects to ns bucket: {ns_bucket}") sync_object_directory( awscli_pod, src=MCG_NS_ORIGINAL_DIR, target=f"s3://{ns_bucket}", s3_obj=mcg_obj, ) for pod_to_respin in self.labels_map: logger.info(f"Re-spinning mcg resource: {self.labels_map[pod_to_respin]}") pod_obj = pod.Pod( **pod.get_pods_having_label( label=self.labels_map[pod_to_respin], namespace=defaults.ROOK_CLUSTER_NAMESPACE, )[0] ) pod_obj.delete(force=True) assert pod_obj.ocp.wait_for_resource( condition=constants.STATUS_RUNNING, selector=self.labels_map[pod_to_respin], resource_count=1, timeout=300, ) logger.info( f"Downloading objects from ns bucket: {ns_bucket} " f"after re-spinning: {self.labels_map[pod_to_respin]}" ) sync_object_directory( awscli_pod, src=f"s3://{ns_bucket}", target=MCG_NS_RESULT_DIR, s3_obj=mcg_obj, ) logger.info( f"Verifying integrity of objects " f"after re-spinning: {self.labels_map[pod_to_respin]}" ) compare_directory( awscli_pod, MCG_NS_ORIGINAL_DIR, MCG_NS_RESULT_DIR, amount=3 ) # S3 account user = NoobaaAccount(mcg_obj, name=user_name, email=email, buckets=[ns_bucket]) logger.info(f"Noobaa account: {user.email_id} with S3 access created") # Admin sets Public access policy(*) bucket_policy_generated = gen_bucket_policy( user_list=["*"], actions_list=["GetObject"], resources_list=[f'{ns_bucket}/{"*"}'], ) bucket_policy = json.dumps(bucket_policy_generated) logger.info( f"Creating bucket policy on bucket: {ns_bucket} with wildcard (*) Principal" ) put_policy = put_bucket_policy(mcg_obj, ns_bucket, bucket_policy) logger.info(f"Put bucket policy response from Admin: {put_policy}") logger.info(f"Getting bucket policy on bucket: {ns_bucket}") get_policy = get_bucket_policy(mcg_obj, ns_bucket) logger.info(f"Got bucket policy: {get_policy['Policy']}") # MCG admin writes an object to bucket logger.info(f"Writing object on bucket: {ns_bucket} by admin") assert s3_put_object(mcg_obj, ns_bucket, object_key, data), "Failed: PutObject" # Verifying whether Get operation is allowed to any S3 user logger.info( f"Get object action on namespace bucket: {ns_bucket} " f"with user: {user.email_id}" ) assert s3_get_object(user, ns_bucket, object_key), "Failed: GetObject" # Upload files to NS target logger.info( f"Uploading objects directly to ns resource target: {aws_target_bucket}" ) sync_object_directory( awscli_pod, src=MCG_NS_ORIGINAL_DIR, target=f"s3://{aws_target_bucket}", signed_request_creds=aws_s3_creds, ) for pod_to_drain in self.labels_map: pod_obj = pod.Pod( **pod.get_pods_having_label( label=self.labels_map[pod_to_drain], namespace=defaults.ROOK_CLUSTER_NAMESPACE, )[0] ) # Retrieve the node name on which the pod resides node_name = pod_obj.get()["spec"]["nodeName"] if awscli_node_name == node_name: logger.info( f"Skipping node drain since aws cli pod node: " f"{awscli_node_name} is same as {pod_to_drain} " f"pod node: {node_name}" ) continue # Drain the node drain_nodes([node_name]) wait_for_nodes_status( [node_name], status=constants.NODE_READY_SCHEDULING_DISABLED ) schedule_nodes([node_name]) wait_for_nodes_status(timeout=300) # Retrieve the new pod pod_obj = pod.Pod( **pod.get_pods_having_label( label=self.labels_map[pod_to_drain], namespace=defaults.ROOK_CLUSTER_NAMESPACE, )[0] ) wait_for_resource_state(pod_obj, constants.STATUS_RUNNING, timeout=120) # Verify all storage pods are running wait_for_storage_pods() logger.info( f"Downloading objects from ns bucket: {ns_bucket} " f"after draining node: {node_name} with pod {pod_to_drain}" ) sync_object_directory( awscli_pod, src=f"s3://{ns_bucket}", target=MCG_NS_RESULT_DIR, s3_obj=mcg_obj, ) logger.info( f"Verifying integrity of objects " f"after draining node with pod: {pod_to_drain}" ) compare_directory( awscli_pod, MCG_NS_ORIGINAL_DIR, MCG_NS_RESULT_DIR, amount=3 ) logger.info(f"Editing the namespace resource bucket: {ns_bucket}") namespace_bucket_update( mcg_obj, bucket_name=ns_bucket, read_resource=[aws_target_bucket], write_resource=aws_target_bucket, ) logger.info(f"Verifying object download after edit on ns bucket: {ns_bucket}") sync_object_directory( awscli_pod, src=f"s3://{ns_bucket}", target=MCG_NS_RESULT_DIR, s3_obj=mcg_obj, ) # Verifying whether Put object action is denied logger.info( f"Verifying whether user: {user.email_id} has only public read access" ) logger.info(f"Removing objects from ns bucket: {ns_bucket}") rm_object_recursive(awscli_pod, target=ns_bucket, mcg_obj=mcg_obj)
def create_pod(interface_type=None, pvc_name=None, do_reload=True, namespace=defaults.ROOK_CLUSTER_NAMESPACE, node_name=None, pod_dict_path=None, sa_name=None, dc_deployment=False, raw_block_pv=False, raw_block_device=constants.RAW_BLOCK_DEVICE, replica_count=1, pod_name=None): """ Create a pod Args: interface_type (str): The interface type (CephFS, RBD, etc.) pvc_name (str): The PVC that should be attached to the newly created pod do_reload (bool): True for reloading the object after creation, False otherwise namespace (str): The namespace for the new resource creation node_name (str): The name of specific node to schedule the pod pod_dict_path (str): YAML path for the pod sa_name (str): Serviceaccount name dc_deployment (bool): True if creating pod as deploymentconfig raw_block_pv (bool): True for creating raw block pv based pod, False otherwise raw_block_device (str): raw block device for the pod replica_count (int): Replica count for deployment config pod_name (str): Name of the pod to create Returns: Pod: A Pod instance Raises: AssertionError: In case of any failure """ if interface_type == constants.CEPHBLOCKPOOL: pod_dict = pod_dict_path if pod_dict_path else constants.CSI_RBD_POD_YAML interface = constants.RBD_INTERFACE else: pod_dict = pod_dict_path if pod_dict_path else constants.CSI_CEPHFS_POD_YAML interface = constants.CEPHFS_INTERFACE if dc_deployment: pod_dict = pod_dict_path if pod_dict_path else constants.FEDORA_DC_YAML pod_data = templating.load_yaml(pod_dict) if not pod_name: pod_name = create_unique_resource_name(f'test-{interface}', 'pod') pod_data['metadata']['name'] = pod_name pod_data['metadata']['namespace'] = namespace if dc_deployment: pod_data['metadata']['labels']['app'] = pod_name pod_data['spec']['template']['metadata']['labels']['name'] = pod_name pod_data['spec']['replicas'] = replica_count if pvc_name: if dc_deployment: pod_data['spec']['template']['spec']['volumes'][0][ 'persistentVolumeClaim']['claimName'] = pvc_name else: pod_data['spec']['volumes'][0]['persistentVolumeClaim'][ 'claimName'] = pvc_name if interface_type == constants.CEPHBLOCKPOOL and raw_block_pv: pod_data['spec']['containers'][0]['volumeDevices'][0][ 'devicePath'] = raw_block_device pod_data['spec']['containers'][0]['volumeDevices'][0][ 'name'] = pod_data.get('spec').get('volumes')[0].get('name') if node_name: pod_data['spec']['nodeName'] = node_name else: if 'nodeName' in pod_data.get('spec'): del pod_data['spec']['nodeName'] if sa_name and dc_deployment: pod_data['spec']['template']['spec']['serviceAccountName'] = sa_name if dc_deployment: ocs_obj = create_resource(**pod_data) logger.info(ocs_obj.name) assert (ocp.OCP(kind='pod', namespace=namespace)).wait_for_resource( condition=constants.STATUS_COMPLETED, resource_name=pod_name + '-1-deploy', resource_count=0, timeout=180, sleep=3) dpod_list = pod.get_all_pods(namespace=namespace) for dpod in dpod_list: if '-1-deploy' not in dpod.name: if pod_name in dpod.name: return dpod else: pod_obj = pod.Pod(**pod_data) pod_name = pod_data.get('metadata').get('name') logger.info(f'Creating new Pod {pod_name} for test') created_resource = pod_obj.create(do_reload=do_reload) assert created_resource, (f"Failed to create Pod {pod_name}") return pod_obj
def test_quay_with_failures(self, quay_operator): """ Test quay operations with Noobaa core failure 1. Creates quay operator and registry on ODF. 2. Initializes Quay super user to access the API's. 3. Gets the super user token. 4. Creates a new repo 5. Creates a new org 6. Pushes the image to the new repo 7. Pulls the image locally from the quay repo 8. Re-spins noobaa core 9. Pulls the image again 10. Deletes the repo """ # Deploy quay operator quay_operator.setup_quay_operator() # Create quay registry quay_operator.create_quay_registry() log.info("Waiting for quay endpoint to start serving") sleep(90) endpoint = quay_operator.get_quay_endpoint() log.info("Pulling test image") exec_cmd(f"podman pull {constants.COSBENCH_IMAGE}") log.info("Getting the Super user token") token = get_super_user_token(endpoint) log.info("Validating super_user using token") check_super_user(endpoint, token) podman_url = endpoint.replace("https://", "") log.info(f"Logging into quay endpoint: {podman_url}") quay_super_user_login(podman_url) repo_name = "test_repo" test_image = f"{constants.QUAY_SUPERUSER}/{repo_name}:latest" create_quay_repository( endpoint, token, org_name=constants.QUAY_SUPERUSER, repo_name=repo_name ) org_name = "test_org" log.info(f"Creating a new organization name: {org_name}") create_quay_org(endpoint, token, org_name) log.info("Tagging a test image") exec_cmd(f"podman tag {constants.COSBENCH_IMAGE} {podman_url}/{test_image}") log.info(f"Pushing the test image to quay repo: {repo_name}") exec_cmd(f"podman push {podman_url}/{test_image} --tls-verify=false") log.info(f"Validating whether the image can be pull from quay: {repo_name}") exec_cmd(f"podman pull {podman_url}/{test_image} --tls-verify=false") # TODO: Trigger build pod_obj = pod.Pod( **pod.get_pods_having_label( label=constants.NOOBAA_CORE_POD_LABEL, namespace=defaults.ROOK_CLUSTER_NAMESPACE, )[0] ) pod_obj.delete(force=True) assert pod_obj.ocp.wait_for_resource( condition=constants.STATUS_RUNNING, selector=constants.NOOBAA_CORE_POD_LABEL, resource_count=1, timeout=800, sleep=60, ) log.info("Pulling the image again from quay, post noobaa core failure") exec_cmd(f"podman pull {podman_url}/{test_image} --tls-verify=false") log.info(f"Deleting the repository: {repo_name}") delete_quay_repository( endpoint, token, org_name=constants.QUAY_SUPERUSER, repo_name=repo_name )