def setUp(self): """Deploys, Verifies and adds resources required for testcases in cleanup method """ self.oc_node = self.ocp_master_node[0] self.gluster_pod = get_ocp_gluster_pod_names(self.oc_node)[0] self.gluster_pod_obj = podcmd.Pod(self.oc_node, self.gluster_pod) # prefix used to create resources, generating using glusto_test_id # which uses time and date of test case self.prefix = "autotest-%s" % (self.glustotest_run_id.replace("_", "")) _storage_class = self.storage_classes.get( 'storage_class2', self.storage_classes.get('block_storage_class')) self.provisioner = _storage_class["provisioner"] self.restsecretnamespace = _storage_class["restsecretnamespace"] self.restuser = _storage_class["restuser"] self.resturl = _storage_class["resturl"] # using pvc size count as 1 by default self.pvcsize = 1 # using pvc count as 10 by default self.pvccount = 10 # create gluster block storage class, PVC and user app pod self.sc_name, self.pvc_name, self.dc_name, self.secret_name = ( self.deploy_resouces()) # verify storage class oc_get_yaml(self.oc_node, "sc", self.sc_name) # verify pod creation, it's state and get the pod name self.pod_name = get_pod_name_from_dc(self.oc_node, self.dc_name, timeout=180, wait_step=3) wait_for_pod_be_ready(self.oc_node, self.pod_name, timeout=180, wait_step=3) verify_pvc_status_is_bound(self.oc_node, self.pvc_name) # create pvc's to test self.pvc_list = [] for pvc in range(self.pvccount): test_pvc_name = oc_create_pvc(self.oc_node, self.sc_name, pvc_name_prefix=self.prefix, pvc_size=self.pvcsize) self.pvc_list.append(test_pvc_name) self.addCleanup(wait_for_resource_absence, self.oc_node, "pvc", test_pvc_name, timeout=600, interval=10) for pvc_name in self.pvc_list: self.addCleanup(oc_delete, self.oc_node, "pvc", pvc_name)
def test_usage_of_default_storage_class(self): """Validate PVs creation for SC with default custom volname prefix""" # Unset 'default' option from all the existing Storage Classes unset_sc_annotation_cmd = ( r"""oc annotate sc %s """ r""""storageclass%s.kubernetes.io/is-default-class"-""") set_sc_annotation_cmd = ( r"""oc patch storageclass %s -p'{"metadata": {"annotations": """ r"""{"storageclass%s.kubernetes.io/is-default-class": "%s"}}}'""") get_sc_cmd = ( r'oc get sc --no-headers ' r'-o=custom-columns=:.metadata.name,' r':".metadata.annotations.storageclass\.' r'kubernetes\.io\/is-default-class",:".metadata.annotations.' r'storageclass\.beta\.kubernetes\.io\/is-default-class"') sc_list = self.cmd_run(get_sc_cmd) for sc in sc_list.split("\n"): sc = sc.split() if len(sc) != 3: self.skipTest( "Unexpected output for list of storage classes. " "Following is expected to contain 3 keys:: %s" % sc) for value, api_type in ((sc[1], ''), (sc[2], '.beta')): if value == '<none>': continue self.cmd_run(unset_sc_annotation_cmd % (sc[0], api_type)) self.addCleanup( self.cmd_run, set_sc_annotation_cmd % (sc[0], api_type, value)) # Create new SC prefix = "autotests-default-sc" self.create_storage_class(sc_name_prefix=prefix) # Make new SC be the default one and sleep for 1 sec to avoid races self.cmd_run(set_sc_annotation_cmd % (self.sc_name, '', 'true')) self.cmd_run(set_sc_annotation_cmd % (self.sc_name, '.beta', 'true')) time.sleep(1) # Create PVC without specification of SC pvc_name = oc_create_pvc( self.node, sc_name=None, pvc_name_prefix=prefix) self.addCleanup( wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup(oc_delete, self.node, 'pvc', pvc_name) # Wait for successful creation of PVC and check its SC verify_pvc_status_is_bound(self.node, pvc_name) get_sc_of_pvc_cmd = ( "oc get pvc %s --no-headers " "-o=custom-columns=:.spec.storageClassName" % pvc_name) out = self.cmd_run(get_sc_of_pvc_cmd) self.assertEqual(out, self.sc_name)
def deploy_resouces(self): """Deploys required resources storage class, pvc and user app with continous I/O runnig Returns: sc_name (str): deployed storage class name pvc_name (str): deployed persistent volume claim name dc_name (str): deployed deployment config name secretname (str): created secret file name """ secretname = oc_create_secret(self.oc_node, namespace=self.restsecretnamespace, data_key=self.heketi_cli_key, secret_type=self.provisioner) self.addCleanup(oc_delete, self.oc_node, 'secret', secretname) sc_name = oc_create_sc(self.oc_node, sc_name_prefix=self.prefix, provisioner=self.provisioner, resturl=self.resturl, restuser=self.restuser, restsecretnamespace=self.restsecretnamespace, restsecretname=secretname, volumenameprefix=self.prefix) self.addCleanup(oc_delete, self.oc_node, "sc", sc_name) pvc_name = oc_create_pvc(self.oc_node, sc_name, pvc_name_prefix=self.prefix, pvc_size=self.pvcsize) self.addCleanup(wait_for_resource_absence, self.oc_node, "pvc", pvc_name, timeout=120, interval=5) self.addCleanup(oc_delete, self.oc_node, "pvc", pvc_name) dc_name = oc_create_app_dc_with_io(self.oc_node, pvc_name, dc_name_prefix=self.prefix) self.addCleanup(oc_delete, self.oc_node, "dc", dc_name) self.addCleanup(scale_dc_pod_amount_and_wait, self.oc_node, dc_name, 0) return sc_name, pvc_name, dc_name, secretname
def create_sc_with_parameter(self, vol_type, success=False, parameter={}): """creates storage class, pvc and validates event Args: vol_type (str): storage type either gluster file or block success (bool): if True check for successfull else failure for pvc creation event parameter (dict): dictionary with storage class parameters """ if vol_type == "glusterfile": sc = self.storage_classes.get( 'storage_class1', self.storage_classes.get('file_storage_class')) # Create secret file for usage in storage class self.secret_name = oc_create_secret( self.ocp_master_node[0], namespace=sc.get('secretnamespace', 'default'), data_key=self.heketi_cli_key, secret_type=sc.get('provisioner', 'kubernetes.io/glusterfs')) self.addCleanup(oc_delete, self.ocp_master_node[0], 'secret', self.secret_name) sc_parameter = { "secretnamespace": sc['secretnamespace'], "secretname": self.secret_name, "volumetype": "replicate:3" } elif vol_type == "glusterblock": sc = self.storage_classes.get( 'storage_class2', self.storage_classes.get('block_storage_class')) # Create secret file for usage in storage class self.secret_name = oc_create_secret( self.ocp_master_node[0], namespace=sc.get('restsecretnamespace', 'default'), data_key=self.heketi_cli_key, secret_type=sc.get('provisioner', 'gluster.org/glusterblock')) self.addCleanup(oc_delete, self.ocp_master_node[0], 'secret', self.secret_name) sc_parameter = { "provisioner": "gluster.org/glusterblock", "restsecretnamespace": sc['restsecretnamespace'], "restsecretname": self.secret_name, "hacount": sc['hacount'] } else: err_msg = "invalid vol_type %s" % vol_type g.log.error(err_msg) raise AssertionError(err_msg) sc_parameter['resturl'] = sc['resturl'] sc_parameter['restuser'] = sc['restuser'] sc_parameter.update(parameter) # Create storage class self.sc_name = oc_create_sc(self.ocp_master_node[0], **sc_parameter) self.addCleanup(oc_delete, self.ocp_master_node[0], 'sc', self.sc_name) # Create PVC self.pvc_name = oc_create_pvc(self.ocp_client[0], self.sc_name) self.addCleanup(wait_for_resource_absence, self.ocp_master_node[0], 'pvc', self.pvc_name) self.addCleanup(oc_delete, self.ocp_master_node[0], 'pvc', self.pvc_name) # Wait for event with error event_reason = 'ProvisioningFailed' if success: event_reason = 'ProvisioningSucceeded' wait_for_events(self.ocp_master_node[0], obj_name=self.pvc_name, obj_type='PersistentVolumeClaim', event_reason=event_reason)
def test_dynamic_provisioning_glusterfile_heketipod_failure(self): """Validate dynamic provisioning for gluster file when heketi pod down """ mount_path = "/mnt" datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id()) # Create secret and storage class sc_name = self.create_storage_class() # Create PVC app_1_pvc_name = self.create_and_wait_for_pvc( pvc_name_prefix="autotest-file", sc_name=sc_name ) # Create app POD with attached volume app_1_pod_name = oc_create_tiny_pod_with_volume( self.node, app_1_pvc_name, "test-pvc-mount-on-app-pod", mount_path=mount_path) self.addCleanup( wait_for_resource_absence, self.node, 'pod', app_1_pod_name) self.addCleanup(oc_delete, self.node, 'pod', app_1_pod_name) # Wait for app POD be up and running wait_for_pod_be_ready( self.node, app_1_pod_name, timeout=60, wait_step=2) # Write data to the app POD write_data_cmd = ( "dd if=/dev/urandom of=%s bs=1K count=100" % datafile_path) ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd) self.assertEqual( ret, 0, "Failed to execute command %s on %s" % (write_data_cmd, self.node)) # Remove Heketi pod heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % ( self.heketi_dc_name, self.storage_project_name) heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % ( self.heketi_dc_name, self.storage_project_name) self.addCleanup(self.cmd_run, heketi_up_cmd) heketi_pod_name = get_pod_name_from_dc( self.node, self.heketi_dc_name, timeout=10, wait_step=3) self.cmd_run(heketi_down_cmd) wait_for_resource_absence(self.node, 'pod', heketi_pod_name) app_2_pvc_name = oc_create_pvc( self.node, pvc_name_prefix="autotest-file2", sc_name=sc_name ) self.addCleanup( wait_for_resource_absence, self.node, 'pvc', app_2_pvc_name) self.addCleanup( oc_delete, self.node, 'pvc', app_2_pvc_name, raise_on_absence=False ) # Create second app POD app_2_pod_name = oc_create_tiny_pod_with_volume( self.node, app_2_pvc_name, "test-pvc-mount-on-app-pod", mount_path=mount_path) self.addCleanup( wait_for_resource_absence, self.node, 'pod', app_2_pod_name) self.addCleanup(oc_delete, self.node, 'pod', app_2_pod_name) # Bring Heketi POD back self.cmd_run(heketi_up_cmd) # Wait for Heketi POD be up and running new_heketi_pod_name = get_pod_name_from_dc( self.node, self.heketi_dc_name, timeout=10, wait_step=2) wait_for_pod_be_ready( self.node, new_heketi_pod_name, wait_step=5, timeout=120) # Wait for second PVC and app POD be ready verify_pvc_status_is_bound(self.node, app_2_pvc_name) wait_for_pod_be_ready( self.node, app_2_pod_name, timeout=60, wait_step=2) # Verify that we are able to write data ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd) self.assertEqual( ret, 0, "Failed to execute command %s on %s" % (write_data_cmd, self.node))
def create_and_wait_for_pvcs(self, pvc_size=1, pvc_name_prefix="autotests-pvc", pvc_amount=1, sc_name=None, timeout=120, wait_step=3): node = self.ocp_client[0] # Create storage class if not specified if not sc_name: if getattr(self, "sc_name", ""): sc_name = self.sc_name else: sc_name = self.create_storage_class() # Create PVCs pvc_names = [] for i in range(pvc_amount): pvc_name = oc_create_pvc(node, sc_name, pvc_name_prefix=pvc_name_prefix, pvc_size=pvc_size) pvc_names.append(pvc_name) self.addCleanup(wait_for_resource_absence, node, 'pvc', pvc_name) # Wait for PVCs to be in bound state try: for pvc_name in pvc_names: verify_pvc_status_is_bound(node, pvc_name, timeout, wait_step) finally: reclaim_policy = oc_get_custom_resource(node, 'sc', ':.reclaimPolicy', sc_name)[0] for pvc_name in pvc_names: if reclaim_policy == 'Retain': pv_name = get_pv_name_from_pvc(node, pvc_name) self.addCleanup(oc_delete, node, 'pv', pv_name, raise_on_absence=False) custom = (r':.metadata.annotations."gluster\.kubernetes' r'\.io\/heketi\-volume\-id"') vol_id = oc_get_custom_resource(node, 'pv', custom, pv_name)[0] if self.sc.get('provisioner') == "kubernetes.io/glusterfs": self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_id, raise_on_error=False) else: self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, vol_id, raise_on_error=False) self.addCleanup(oc_delete, node, 'pvc', pvc_name, raise_on_absence=False) return pvc_names
def test_create_delete_pvcs_to_make_gluster_reuse_released_space(self): """Validate reuse of volume space after deletion of PVCs""" min_storage_gb = 10 # Set arbiter:disabled tags to the first 2 nodes data_nodes = [] biggest_disks = [] self.assertGreater(len(self.node_id_list), 2) for node_id in self.node_id_list[0:2]: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) biggest_disk_free_space = 0 for device in node_info['devices']: disk_free_space = int(device['storage']['free']) if disk_free_space < (min_storage_gb * 1024**2): self.skipTest("Devices are expected to have more than " "%sGb of free space" % min_storage_gb) if disk_free_space > biggest_disk_free_space: biggest_disk_free_space = disk_free_space self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], 'disabled', revert_to=device.get('tags', {}).get('arbiter')) biggest_disks.append(biggest_disk_free_space) self._set_arbiter_tag_with_further_revert(self.heketi_client_node, self.heketi_server_url, 'node', node_id, 'disabled', revert_to=node_info.get( 'tags', {}).get('arbiter')) data_nodes.append(node_info) # Set arbiter:required tag to all other nodes and their devices arbiter_nodes = [] for node_id in self.node_id_list[2:]: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) for device in node_info['devices']: self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], 'required', revert_to=device.get('tags', {}).get('arbiter')) self._set_arbiter_tag_with_further_revert(self.heketi_client_node, self.heketi_server_url, 'node', node_id, 'required', revert_to=node_info.get( 'tags', {}).get('arbiter')) arbiter_nodes.append(node_info) # Calculate size and amount of volumes to be created pvc_size = int(min(biggest_disks) / 1024**2) pvc_amount = max([len(n['devices']) for n in data_nodes]) + 1 # Create sc with gluster arbiter info self.create_storage_class(is_arbiter_vol=True) # Create and delete 3 small volumes concurrently pvc_names = [] for i in range(3): pvc_name = oc_create_pvc(self.node, self.sc_name, pvc_name_prefix='arbiter-pvc', pvc_size=int(pvc_size / 3)) pvc_names.append(pvc_name) exception_exists = False for pvc_name in pvc_names: try: verify_pvc_status_is_bound(self.node, pvc_name) except Exception: for pvc_name in pvc_names: self.addCleanup(wait_for_resource_absence, self.node, 'pvc', pvc_name) for pvc_name in pvc_names: self.addCleanup(oc_delete, self.node, 'pvc', pvc_name) exception_exists = True if exception_exists: raise for pvc_name in pvc_names: oc_delete(self.node, 'pvc', pvc_name) for pvc_name in pvc_names: wait_for_resource_absence(self.node, 'pvc', pvc_name) # Create and delete big volumes in a loop for i in range(pvc_amount): pvc_name = oc_create_pvc(self.node, self.sc_name, pvc_name_prefix='arbiter-pvc', pvc_size=pvc_size) try: verify_pvc_status_is_bound(self.node, pvc_name) except Exception: self.addCleanup(wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup(oc_delete, self.node, 'pvc', pvc_name) raise oc_delete(self.node, 'pvc', pvc_name) wait_for_resource_absence(self.node, 'pvc', pvc_name)