def test_validate_logging_pods_and_pvc(self): """Validate logging pods and PVC""" # Wait for kibana pod to be ready kibana_pod = openshift_ops.get_pod_name_from_dc( self._master, self._logging_kibana_dc) openshift_ops.wait_for_pod_be_ready(self._master, kibana_pod) # Wait for fluentd pods to be ready fluentd_custom = [ ":.status.desiredNumberScheduled", ":.spec.template.metadata.labels" ] count_and_selector = openshift_ops.oc_get_custom_resource( self._master, "ds", fluentd_custom, self._logging_fluentd_ds) selector = count_and_selector[1][4:].replace(":", "=") openshift_ops.wait_for_pods_be_ready(self._master, int(count_and_selector[0]), selector) # Wait for PVC to be bound and elasticsearch pod to be ready es_pod = openshift_ops.get_pod_name_from_dc(self._master, self._logging_es_dc) pvc_custom = ":.spec.volumes[*].persistentVolumeClaim.claimName" pvc_name = openshift_ops.oc_get_custom_resource( self._master, "pod", pvc_custom, es_pod)[0] openshift_ops.verify_pvc_status_is_bound(self._master, pvc_name) openshift_ops.wait_for_pod_be_ready(self._master, es_pod) # Validate iscsi and multipath self.verify_iscsi_sessions_and_multipath( pvc_name, self._logging_es_dc, heketi_server_url=self._registry_heketi_server_url, is_registry_gluster=True)
def validate_volumes_and_blocks(self): """Validates PVC and block volumes generated through heketi and OCS """ # verify pvc status is in "Bound" for all the pvc for pvc in self.pvc_list: verify_pvc_status_is_bound(self.oc_node, pvc, timeout=300, wait_step=10) # validate pvcs and pvs created on OCS match_pvc_and_pv(self.oc_node, self.prefix) # get list of block volumes using heketi heketi_block_volume_ids, heketi_block_volume_names = ( self.get_heketi_block_volumes()) # validate block volumes listed by heketi and pvs match_pv_and_heketi_block_volumes(self.oc_node, heketi_block_volume_ids, self.prefix) # validate block volumes listed by heketi and gluster match_heketi_and_gluster_block_volumes_by_prefix( heketi_block_volume_names, "%s_" % self.prefix)
def test_validate_metrics_pods_and_pvc(self): """Validate metrics pods and PVC""" # Get cassandra pod name and PVC name hawkular_cassandra = get_pod_name_from_rc( self.master, self.metrics_rc_hawkular_cassandra) custom = ":.spec.volumes[*].persistentVolumeClaim.claimName" pvc_name = oc_get_custom_resource(self.master, "pod", custom, hawkular_cassandra)[0] # Wait for pods to get ready and PVC to be bound verify_pvc_status_is_bound(self.master, pvc_name) wait_for_pod_be_ready(self.master, hawkular_cassandra) hawkular_metrics = get_pod_name_from_rc( self.master, self.metrics_rc_hawkular_metrics) wait_for_pod_be_ready(self.master, hawkular_metrics) heapster = get_pod_name_from_rc(self.master, self.metrics_rc_heapster) wait_for_pod_be_ready(self.master, heapster) # Validate iscsi and multipath self.verify_iscsi_sessions_and_multipath( pvc_name, self.metrics_rc_hawkular_cassandra, rtype='rc', heketi_server_url=self.registry_heketi_server_url, is_registry_gluster=True)
def validate_volumes_and_blocks(self): """Validates PVC and block volumes generated through heketi and OCS """ # verify pvc status is in "Bound" for all the pvc for pvc in self.pvc_list: verify_pvc_status_is_bound( self.oc_node, pvc, timeout=300, wait_step=10 ) # validate pvcs and pvs created on OCS match_pvc_and_pv(self.oc_node, self.prefix) # get list of block volumes using heketi heketi_block_volume_ids, heketi_block_volume_names = ( self.get_heketi_block_volumes() ) # validate block volumes listed by heketi and pvs match_pv_and_heketi_block_volumes( self.oc_node, heketi_block_volume_ids, self.prefix ) # validate block volumes listed by heketi and gluster match_heketi_and_gluster_block_volumes_by_prefix( heketi_block_volume_names, "%s_" % self.prefix)
def test_sc_glusterfile_missing_parameter(self, parameter): """Validate glusterfile storage with missing parameters""" node, sc = self.ocp_master_node[0], self.sc secret_name = self.create_secret() parameters = {'resturl': sc['resturl'], 'restuser': sc['restuser']} if parameter == 'secretName': parameters['secretName'] = secret_name elif parameter == 'secretNamespace': parameters['secretNamespace'] = sc['secretnamespace'] sc_name = oc_create_sc(node, **parameters) self.addCleanup(oc_delete, node, 'sc', sc_name) # Create PVC pvc_name = oc_create_pvc(node, sc_name) self.addCleanup(wait_for_resource_absence, node, 'pvc', pvc_name) self.addCleanup(oc_delete, node, 'pvc', pvc_name) # Wait for event with error wait_for_events(node, obj_name=pvc_name, obj_type='PersistentVolumeClaim', event_reason='ProvisioningFailed') # Verify PVC did not get bound with self.assertRaises(ExecutionError): verify_pvc_status_is_bound(node, pvc_name, timeout=1)
def test_usage_of_default_storage_class(self): """Validate PVs creation for SC with default custom volname prefix""" if get_openshift_version() < "3.9": self.skipTest( "'volumenameprefix' option for Heketi is not supported" " in OCP older than 3.9") # Unset 'default' option from all the existing Storage Classes unset_sc_annotation_cmd = ( r"""oc annotate sc %s """ r""""storageclass%s.kubernetes.io/is-default-class"-""") set_sc_annotation_cmd = ( r"""oc patch storageclass %s -p'{"metadata": {"annotations": """ r"""{"storageclass%s.kubernetes.io/is-default-class": "%s"}}}'""") get_sc_cmd = ( r'oc get sc --no-headers ' r'-o=custom-columns=:.metadata.name,' r':".metadata.annotations.storageclass\.' r'kubernetes\.io\/is-default-class",:".metadata.annotations.' r'storageclass\.beta\.kubernetes\.io\/is-default-class"') sc_list = self.cmd_run(get_sc_cmd) for sc in sc_list.split("\n"): sc = sc.split() if len(sc) != 3: self.skipTest("Unexpected output for list of storage classes. " "Following is expected to contain 3 keys:: %s" % sc) for value, api_type in ((sc[1], ''), (sc[2], '.beta')): if value == '<none>': continue self.cmd_run(unset_sc_annotation_cmd % (sc[0], api_type)) self.addCleanup( self.cmd_run, set_sc_annotation_cmd % (sc[0], api_type, value)) # Create new SC prefix = "autotests-default-sc" self.create_storage_class(sc_name_prefix=prefix) # Make new SC be the default one and sleep for 1 sec to avoid races self.cmd_run(set_sc_annotation_cmd % (self.sc_name, '', 'true')) self.cmd_run(set_sc_annotation_cmd % (self.sc_name, '.beta', 'true')) time.sleep(1) # Create PVC without specification of SC pvc_name = oc_create_pvc(self.node, sc_name=None, pvc_name_prefix=prefix) self.addCleanup(wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup(oc_delete, self.node, 'pvc', pvc_name) # Wait for successful creation of PVC and check its SC verify_pvc_status_is_bound(self.node, pvc_name) get_sc_of_pvc_cmd = ("oc get pvc %s --no-headers " "-o=custom-columns=:.spec.storageClassName" % pvc_name) out = self.cmd_run(get_sc_of_pvc_cmd) self.assertEqual(out, self.sc_name)
def test_usage_of_default_storage_class(self): """Validate PVs creation for SC with default custom volname prefix""" if get_openshift_version() < "3.9": self.skipTest( "'volumenameprefix' option for Heketi is not supported" " in OCP older than 3.9") # Unset 'default' option from all the existing Storage Classes unset_sc_annotation_cmd = ( r"""oc annotate sc %s """ r""""storageclass%s.kubernetes.io/is-default-class"-""") set_sc_annotation_cmd = ( r"""oc patch storageclass %s -p'{"metadata": {"annotations": """ r"""{"storageclass%s.kubernetes.io/is-default-class": "%s"}}}'""") get_sc_cmd = ( r'oc get sc --no-headers ' r'-o=custom-columns=:.metadata.name,' r':".metadata.annotations.storageclass\.' r'kubernetes\.io\/is-default-class",:".metadata.annotations.' r'storageclass\.beta\.kubernetes\.io\/is-default-class"') sc_list = self.cmd_run(get_sc_cmd) for sc in sc_list.split("\n"): sc = sc.split() if len(sc) != 3: self.skipTest( "Unexpected output for list of storage classes. " "Following is expected to contain 3 keys:: %s" % sc) for value, api_type in ((sc[1], ''), (sc[2], '.beta')): if value == '<none>': continue self.cmd_run(unset_sc_annotation_cmd % (sc[0], api_type)) self.addCleanup( self.cmd_run, set_sc_annotation_cmd % (sc[0], api_type, value)) # Create new SC prefix = "autotests-default-sc" self.create_storage_class(sc_name_prefix=prefix) # Make new SC be the default one and sleep for 1 sec to avoid races self.cmd_run(set_sc_annotation_cmd % (self.sc_name, '', 'true')) self.cmd_run(set_sc_annotation_cmd % (self.sc_name, '.beta', 'true')) time.sleep(1) # Create PVC without specification of SC pvc_name = oc_create_pvc( self.node, sc_name=None, pvc_name_prefix=prefix) self.addCleanup( wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup(oc_delete, self.node, 'pvc', pvc_name) # Wait for successful creation of PVC and check its SC verify_pvc_status_is_bound(self.node, pvc_name) get_sc_of_pvc_cmd = ( "oc get pvc %s --no-headers " "-o=custom-columns=:.spec.storageClassName" % pvc_name) out = self.cmd_run(get_sc_of_pvc_cmd) self.assertEqual(out, self.sc_name)
def create_and_wait_for_pvcs(self, pvc_size=1, pvc_name_prefix="autotests-pvc", pvc_amount=1, sc_name=None, timeout=120, wait_step=3): node = self.ocp_client[0] # Create storage class if not specified if not sc_name: if getattr(self, "sc_name", ""): sc_name = self.sc_name else: sc_name = self.create_storage_class() # Create PVCs pvc_names = [] for i in range(pvc_amount): pvc_name = oc_create_pvc( node, sc_name, pvc_name_prefix=pvc_name_prefix, pvc_size=pvc_size) pvc_names.append(pvc_name) self.addCleanup( wait_for_resource_absence, node, 'pvc', pvc_name) # Wait for PVCs to be in bound state try: for pvc_name in pvc_names: verify_pvc_status_is_bound(node, pvc_name, timeout, wait_step) finally: if get_openshift_version() < "3.9": reclaim_policy = "Delete" else: reclaim_policy = oc_get_custom_resource( node, 'sc', ':.reclaimPolicy', sc_name)[0] for pvc_name in pvc_names: if reclaim_policy == 'Retain': pv_name = get_pv_name_from_pvc(node, pvc_name) self.addCleanup(oc_delete, node, 'pv', pv_name, raise_on_absence=False) custom = (r':.metadata.annotations."gluster\.kubernetes' r'\.io\/heketi\-volume\-id"') vol_id = oc_get_custom_resource( node, 'pv', custom, pv_name)[0] if self.sc.get('provisioner') == "kubernetes.io/glusterfs": self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_id, raise_on_error=False) else: self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, vol_id, raise_on_error=False) self.addCleanup(oc_delete, node, 'pvc', pvc_name, raise_on_absence=False) return pvc_names
def test_creation_of_block_vol_greater_than_the_default_size_of_BHV_neg( self): """Verify that block volume creation fails when we create block volume of size greater than the default size of BHV. Verify that block volume creation succeed when we create BHV of size greater than the default size of BHV. """ default_bhv_size = get_default_block_hosting_volume_size( self.node, self.heketi_dc_name) reserve_size = default_bhv_size * 0.02 reserve_size = int(math.ceil(reserve_size)) self.verify_free_space(default_bhv_size + reserve_size + 2) with self.assertRaises(ExecutionError): # create a block vol greater than default BHV size bvol_info = heketi_blockvolume_create( self.heketi_client_node, self.heketi_server_url, (default_bhv_size + 1), json=True) self.addCleanup( heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, bvol_info['id']) sc_name = self.create_storage_class() # create a block pvc greater than default BHV size pvc_name = oc_create_pvc( self.node, sc_name, pvc_size=(default_bhv_size + 1)) self.addCleanup( wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup( oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False) wait_for_events( self.node, pvc_name, obj_type='PersistentVolumeClaim', event_type='Warning', event_reason='ProvisioningFailed') # create block hosting volume greater than default BHV size vol_info = heketi_volume_create( self.heketi_client_node, self.heketi_server_url, (default_bhv_size + reserve_size + 2), block=True, json=True) self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) # Cleanup PVC before block hosting volume to avoid failures self.addCleanup( wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup( oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False) verify_pvc_status_is_bound(self.node, pvc_name)
def test_creation_of_block_vol_greater_than_the_default_size_of_BHV_neg( self): """Verify that block volume creation fails when we create block volume of size greater than the default size of BHV. Verify that block volume creation succeed when we create BHV of size greater than the default size of BHV. """ default_bhv_size = get_default_block_hosting_volume_size( self.node, self.heketi_dc_name) reserve_size = default_bhv_size * 0.02 reserve_size = int(math.ceil(reserve_size)) self.verify_free_space(default_bhv_size + reserve_size + 2) with self.assertRaises(AssertionError): # create a block vol greater than default BHV size bvol_info = heketi_blockvolume_create( self.heketi_client_node, self.heketi_server_url, (default_bhv_size + 1), json=True) self.addCleanup( heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, bvol_info['id']) sc_name = self.create_storage_class() # create a block pvc greater than default BHV size pvc_name = oc_create_pvc( self.node, sc_name, pvc_size=(default_bhv_size + 1)) self.addCleanup( wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup( oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False) wait_for_events( self.node, pvc_name, obj_type='PersistentVolumeClaim', event_type='Warning', event_reason='ProvisioningFailed') # create block hosting volume greater than default BHV size vol_info = heketi_volume_create( self.heketi_client_node, self.heketi_server_url, (default_bhv_size + reserve_size + 2), block=True, json=True) self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_info['id']) # Cleanup PVC before block hosting volume to avoid failures self.addCleanup( wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup( oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False) verify_pvc_status_is_bound(self.node, pvc_name)
def setUp(self): """Deploys, Verifies and adds resources required for testcases in cleanup method """ self.oc_node = self.ocp_master_node[0] self.prefix = "autotest-%s" % utils.get_random_str() _storage_class = self.storage_classes.get( 'storage_class2', self.storage_classes.get('block_storage_class')) self.provisioner = _storage_class["provisioner"] self.restsecretnamespace = _storage_class["restsecretnamespace"] self.restuser = _storage_class["restuser"] self.resturl = _storage_class["resturl"] # using pvc size count as 1 by default self.pvcsize = 1 # using pvc count as 10 by default self.pvccount = 10 # create gluster block storage class, PVC and user app pod self.sc_name, self.pvc_name, self.dc_name, self.secret_name = ( self.deploy_resouces()) # verify storage class oc_get_yaml(self.oc_node, "sc", self.sc_name) # verify pod creation, it's state and get the pod name self.pod_name = get_pod_name_from_dc(self.oc_node, self.dc_name, timeout=180, wait_step=3) wait_for_pod_be_ready(self.oc_node, self.pod_name, timeout=180, wait_step=3) verify_pvc_status_is_bound(self.oc_node, self.pvc_name) # create pvc's to test self.pvc_list = [] for pvc in range(self.pvccount): test_pvc_name = oc_create_pvc(self.oc_node, self.sc_name, pvc_name_prefix=self.prefix, pvc_size=self.pvcsize) self.pvc_list.append(test_pvc_name) self.addCleanup(wait_for_resource_absence, self.oc_node, "pvc", test_pvc_name, timeout=600, interval=10) for pvc_name in self.pvc_list: self.addCleanup(oc_delete, self.oc_node, "pvc", pvc_name)
def setUp(self): """Deploys, Verifies and adds resources required for testcases in cleanup method """ self.oc_node = self.ocp_master_node[0] self.prefix = "autotest-%s" % utils.get_random_str() _storage_class = self.storage_classes.get( 'storage_class2', self.storage_classes.get('block_storage_class')) self.provisioner = _storage_class["provisioner"] self.restsecretnamespace = _storage_class["restsecretnamespace"] self.restuser = _storage_class["restuser"] self.resturl = _storage_class["resturl"] # using pvc size count as 1 by default self.pvcsize = 1 # using pvc count as 10 by default self.pvccount = 10 # create gluster block storage class, PVC and user app pod self.sc_name, self.pvc_name, self.dc_name, self.secret_name = ( self.deploy_resouces() ) # verify storage class oc_get_yaml(self.oc_node, "sc", self.sc_name) # verify pod creation, it's state and get the pod name self.pod_name = get_pod_name_from_dc( self.oc_node, self.dc_name, timeout=180, wait_step=3 ) wait_for_pod_be_ready( self.oc_node, self.pod_name, timeout=180, wait_step=3 ) verify_pvc_status_is_bound(self.oc_node, self.pvc_name) # create pvc's to test self.pvc_list = [] for pvc in range(self.pvccount): test_pvc_name = oc_create_pvc( self.oc_node, self.sc_name, pvc_name_prefix=self.prefix, pvc_size=self.pvcsize ) self.pvc_list.append(test_pvc_name) self.addCleanup( wait_for_resource_absence, self.oc_node, "pvc", test_pvc_name, timeout=600, interval=10 ) for pvc_name in self.pvc_list: self.addCleanup(oc_delete, self.oc_node, "pvc", pvc_name)
def validate_volumes_and_blocks(self): """Validates PVC and block volumes generated through heketi and OCS """ heketi_operations = heketi_server_operations_list( self.heketi_client_node, self.heketi_server_url, secret=self.heketi_cli_key, user=self.heketi_cli_user) for heketi_operation in heketi_operations: if heketi_operation["status"] == "failed": heketi_server_operation_cleanup(self.heketi_client_node, self.heketi_server_url, heketi_operation["id"], secret=self.heketi_cli_key, user=self.heketi_cli_user) # verify pvc status is in "Bound" for all the pvc for pvc in self.pvc_list: verify_pvc_status_is_bound(self.oc_node, pvc, timeout=300, wait_step=10) # validate pvcs and pvs created on OCS match_pvc_and_pv(self.oc_node, self.prefix) # get list of block volumes using heketi h_blockvol_list = heketi_blockvolume_list_by_name_prefix( self.heketi_client_node, self.heketi_server_url, self.prefix) # validate block volumes listed by heketi and pvs heketi_blockvolume_ids = sorted([bv[0] for bv in h_blockvol_list]) match_pv_and_heketi_block_volumes(self.oc_node, heketi_blockvolume_ids, self.prefix) # validate block volumes listed by heketi and gluster heketi_blockvolume_names = sorted( [bv[1].replace("%s_" % self.prefix, "") for bv in h_blockvol_list]) match_heketi_and_gluster_block_volumes_by_prefix( heketi_blockvolume_names, "%s_" % self.prefix)
def _dynamic_provisioning_block_with_bhv_cleanup( self, sc_name, pvc_size, bhv_list): """Dynamic provisioning for glusterblock with BHV Cleanup""" h_node, h_url = self.heketi_client_node, self.heketi_server_url pvc_name = oc_create_pvc(self.node, sc_name, pvc_size=pvc_size) try: verify_pvc_status_is_bound(self.node, pvc_name) pv_name = get_pv_name_from_pvc(self.node, pvc_name) custom = [r':.metadata.annotations."gluster\.org\/volume\-id"'] bvol_id = oc_get_custom_resource( self.node, 'pv', custom, pv_name)[0] bhv_id = heketi_blockvolume_info( h_node, h_url, bvol_id, json=True)['blockhostingvolume'] if bhv_id not in bhv_list: self.addCleanup( heketi_volume_delete, h_node, h_url, bhv_id) finally: self.addCleanup( wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup( oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=True)
def test_create_delete_pvcs_to_make_gluster_reuse_released_space(self): """Validate reuse of volume space after deletion of PVCs""" min_storage_gb = 10 # Set arbiter:disabled tags to the first 2 nodes data_nodes = [] biggest_disks = [] self.assertGreater(len(self.node_id_list), 2) for node_id in self.node_id_list[0:2]: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) biggest_disk_free_space = 0 for device in node_info['devices']: disk_free_space = int(device['storage']['free']) if disk_free_space < (min_storage_gb * 1024**2): self.skipTest("Devices are expected to have more than " "%sGb of free space" % min_storage_gb) if disk_free_space > biggest_disk_free_space: biggest_disk_free_space = disk_free_space self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], 'disabled', revert_to=device.get('tags', {}).get('arbiter')) biggest_disks.append(biggest_disk_free_space) self._set_arbiter_tag_with_further_revert(self.heketi_client_node, self.heketi_server_url, 'node', node_id, 'disabled', revert_to=node_info.get( 'tags', {}).get('arbiter')) data_nodes.append(node_info) # Set arbiter:required tag to all other nodes and their devices arbiter_nodes = [] for node_id in self.node_id_list[2:]: node_info = heketi_ops.heketi_node_info(self.heketi_client_node, self.heketi_server_url, node_id, json=True) for device in node_info['devices']: self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], 'required', revert_to=device.get('tags', {}).get('arbiter')) self._set_arbiter_tag_with_further_revert(self.heketi_client_node, self.heketi_server_url, 'node', node_id, 'required', revert_to=node_info.get( 'tags', {}).get('arbiter')) arbiter_nodes.append(node_info) # Calculate size and amount of volumes to be created pvc_size = int(min(biggest_disks) / 1024**2) pvc_amount = max([len(n['devices']) for n in data_nodes]) + 1 # Create sc with gluster arbiter info self.create_storage_class(is_arbiter_vol=True) # Create and delete 3 small volumes concurrently pvc_names = self.create_and_wait_for_pvcs( pvc_size=int(pvc_size / 3), pvc_name_prefix='arbiter-pvc', pvc_amount=3, sc_name=self.sc_name) for pvc_name in pvc_names: openshift_ops.oc_delete(self.node, 'pvc', pvc_name) for pvc_name in pvc_names: openshift_ops.wait_for_resource_absence(self.node, 'pvc', pvc_name) # Create and delete big volumes in a loop for i in range(pvc_amount): pvc_name = openshift_ops.oc_create_pvc( self.node, self.sc_name, pvc_name_prefix='arbiter-pvc', pvc_size=pvc_size) try: openshift_ops.verify_pvc_status_is_bound( self.node, pvc_name, 300, 10) except Exception: self.addCleanup(openshift_ops.wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup(openshift_ops.oc_delete, self.node, 'pvc', pvc_name) raise openshift_ops.oc_delete(self.node, 'pvc', pvc_name) openshift_ops.wait_for_resource_absence(self.node, 'pvc', pvc_name)
def test_dynamic_provisioning_glusterblock_heketipod_failure(self): """Validate PVC with glusterblock creation when heketi pod is down""" datafile_path = '/mnt/fake_file_for_%s' % self.id() # Create DC with attached PVC sc_name = self.create_storage_class() app_1_pvc_name = self.create_and_wait_for_pvc( pvc_name_prefix='autotest-block', sc_name=sc_name) app_1_dc_name, app_1_pod_name = self.create_dc_with_pvc(app_1_pvc_name) # Write test data write_data_cmd = ( "dd if=/dev/urandom of=%s bs=1K count=100" % datafile_path) ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd) self.assertEqual( ret, 0, "Failed to execute command %s on %s" % (write_data_cmd, self.node)) # Remove Heketi pod heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % ( self.heketi_dc_name, self.storage_project_name) heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % ( self.heketi_dc_name, self.storage_project_name) self.addCleanup(self.cmd_run, heketi_up_cmd) heketi_pod_name = get_pod_name_from_dc( self.node, self.heketi_dc_name, timeout=10, wait_step=3) self.cmd_run(heketi_down_cmd) wait_for_resource_absence(self.node, 'pod', heketi_pod_name) # Create second PVC app_2_pvc_name = oc_create_pvc( self.node, pvc_name_prefix='autotest-block2', sc_name=sc_name ) self.addCleanup( wait_for_resource_absence, self.node, 'pvc', app_2_pvc_name) self.addCleanup( oc_delete, self.node, 'pvc', app_2_pvc_name ) # Create second app POD app_2_dc_name = oc_create_app_dc_with_io(self.node, app_2_pvc_name) self.addCleanup(oc_delete, self.node, 'dc', app_2_dc_name) self.addCleanup( scale_dc_pod_amount_and_wait, self.node, app_2_dc_name, 0) app_2_pod_name = get_pod_name_from_dc(self.node, app_2_dc_name) # Bring Heketi pod back self.cmd_run(heketi_up_cmd) # Wait for Heketi POD be up and running new_heketi_pod_name = get_pod_name_from_dc( self.node, self.heketi_dc_name, timeout=10, wait_step=2) wait_for_pod_be_ready( self.node, new_heketi_pod_name, wait_step=5, timeout=120) # Wait for second PVC and app POD be ready verify_pvc_status_is_bound(self.node, app_2_pvc_name) wait_for_pod_be_ready( self.node, app_2_pod_name, timeout=150, wait_step=3) # Verify that we are able to write data ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd) self.assertEqual( ret, 0, "Failed to execute command %s on %s" % (write_data_cmd, self.node))
def test_dynamic_provisioning_glusterfile_heketipod_failure(self): """Validate dynamic provisioning for gluster file when heketi pod down """ mount_path = "/mnt" datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id()) # Create secret and storage class sc_name = self.create_storage_class() # Create PVC app_1_pvc_name = self.create_and_wait_for_pvc( pvc_name_prefix="autotest-file", sc_name=sc_name) # Create app POD with attached volume app_1_pod_name = oc_create_tiny_pod_with_volume( self.node, app_1_pvc_name, "test-pvc-mount-on-app-pod", mount_path=mount_path, image=self.io_container_image_cirros) self.addCleanup(wait_for_resource_absence, self.node, 'pod', app_1_pod_name) self.addCleanup(oc_delete, self.node, 'pod', app_1_pod_name) # Wait for app POD be up and running wait_for_pod_be_ready(self.node, app_1_pod_name, timeout=60, wait_step=2) # Write data to the app POD write_data_cmd = ("dd if=/dev/urandom of=%s bs=1K count=100" % datafile_path) ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd) self.assertEqual( ret, 0, "Failed to execute command %s on %s" % (write_data_cmd, self.node)) # Remove Heketi pod heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % ( self.heketi_dc_name, self.storage_project_name) heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % ( self.heketi_dc_name, self.storage_project_name) self.addCleanup(self.cmd_run, heketi_up_cmd) heketi_pod_name = get_pod_name_from_dc(self.node, self.heketi_dc_name, timeout=10, wait_step=3) self.cmd_run(heketi_down_cmd) wait_for_resource_absence(self.node, 'pod', heketi_pod_name) app_2_pvc_name = oc_create_pvc(self.node, pvc_name_prefix="autotest-file2", sc_name=sc_name) self.addCleanup(wait_for_resource_absence, self.node, 'pvc', app_2_pvc_name) self.addCleanup(oc_delete, self.node, 'pvc', app_2_pvc_name, raise_on_absence=False) # Create second app POD app_2_pod_name = oc_create_tiny_pod_with_volume( self.node, app_2_pvc_name, "test-pvc-mount-on-app-pod", mount_path=mount_path, image=self.io_container_image_cirros) self.addCleanup(wait_for_resource_absence, self.node, 'pod', app_2_pod_name) self.addCleanup(oc_delete, self.node, 'pod', app_2_pod_name) # Bring Heketi POD back self.cmd_run(heketi_up_cmd) # Wait for Heketi POD be up and running new_heketi_pod_name = get_pod_name_from_dc(self.node, self.heketi_dc_name, timeout=10, wait_step=2) wait_for_pod_be_ready(self.node, new_heketi_pod_name, wait_step=5, timeout=120) # Wait for second PVC and app POD be ready verify_pvc_status_is_bound(self.node, app_2_pvc_name) wait_for_pod_be_ready(self.node, app_2_pod_name, timeout=60, wait_step=2) # Verify that we are able to write data ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd) self.assertEqual( ret, 0, "Failed to execute command %s on %s" % (write_data_cmd, self.node))
def test_dynamic_provisioning_glusterblock_heketipod_failure(self): """Validate PVC with glusterblock creation when heketi pod is down""" datafile_path = '/mnt/fake_file_for_%s' % self.id() # Create DC with attached PVC sc_name = self.create_storage_class() app_1_pvc_name = self.create_and_wait_for_pvc( pvc_name_prefix='autotest-block', sc_name=sc_name) app_1_dc_name, app_1_pod_name = self.create_dc_with_pvc(app_1_pvc_name) # Write test data write_data_cmd = ("dd if=/dev/urandom of=%s bs=1K count=100" % datafile_path) ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd) self.assertEqual( ret, 0, "Failed to execute command %s on %s" % (write_data_cmd, self.node)) # Remove Heketi pod heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % ( self.heketi_dc_name, self.storage_project_name) heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % ( self.heketi_dc_name, self.storage_project_name) self.addCleanup(self.cmd_run, heketi_up_cmd) heketi_pod_name = get_pod_name_from_dc(self.node, self.heketi_dc_name, timeout=10, wait_step=3) self.cmd_run(heketi_down_cmd) wait_for_resource_absence(self.node, 'pod', heketi_pod_name) # Create second PVC app_2_pvc_name = oc_create_pvc(self.node, pvc_name_prefix='autotest-block2', sc_name=sc_name) self.addCleanup(wait_for_resource_absence, self.node, 'pvc', app_2_pvc_name) self.addCleanup(oc_delete, self.node, 'pvc', app_2_pvc_name) # Create second app POD app_2_dc_name = oc_create_app_dc_with_io(self.node, app_2_pvc_name) self.addCleanup(oc_delete, self.node, 'dc', app_2_dc_name) self.addCleanup(scale_dc_pod_amount_and_wait, self.node, app_2_dc_name, 0) app_2_pod_name = get_pod_name_from_dc(self.node, app_2_dc_name) # Bring Heketi pod back self.cmd_run(heketi_up_cmd) # Wait for Heketi POD be up and running new_heketi_pod_name = get_pod_name_from_dc(self.node, self.heketi_dc_name, timeout=10, wait_step=2) wait_for_pod_be_ready(self.node, new_heketi_pod_name, wait_step=5, timeout=120) # Wait for second PVC and app POD be ready verify_pvc_status_is_bound(self.node, app_2_pvc_name) wait_for_pod_be_ready(self.node, app_2_pod_name, timeout=150, wait_step=3) # Verify that we are able to write data ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd) self.assertEqual( ret, 0, "Failed to execute command %s on %s" % (write_data_cmd, self.node))
def test_create_delete_pvcs_to_make_gluster_reuse_released_space(self): """Validate reuse of volume space after deletion of PVCs""" min_storage_gb = 10 # Set arbiter:disabled tags to the first 2 nodes data_nodes = [] biggest_disks = [] self.assertGreater(len(self.node_id_list), 2) for node_id in self.node_id_list[0:2]: node_info = heketi_ops.heketi_node_info( self.heketi_client_node, self.heketi_server_url, node_id, json=True) biggest_disk_free_space = 0 for device in node_info['devices']: disk_free_space = int(device['storage']['free']) if disk_free_space < (min_storage_gb * 1024**2): self.skipTest( "Devices are expected to have more than " "%sGb of free space" % min_storage_gb) if disk_free_space > biggest_disk_free_space: biggest_disk_free_space = disk_free_space self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], 'disabled', revert_to=device.get('tags', {}).get('arbiter')) biggest_disks.append(biggest_disk_free_space) self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'node', node_id, 'disabled', revert_to=node_info.get('tags', {}).get('arbiter')) data_nodes.append(node_info) # Set arbiter:required tag to all other nodes and their devices arbiter_nodes = [] for node_id in self.node_id_list[2:]: node_info = heketi_ops.heketi_node_info( self.heketi_client_node, self.heketi_server_url, node_id, json=True) for device in node_info['devices']: self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'device', device['id'], 'required', revert_to=device.get('tags', {}).get('arbiter')) self._set_arbiter_tag_with_further_revert( self.heketi_client_node, self.heketi_server_url, 'node', node_id, 'required', revert_to=node_info.get('tags', {}).get('arbiter')) arbiter_nodes.append(node_info) # Calculate size and amount of volumes to be created pvc_size = int(min(biggest_disks) / 1024**2) pvc_amount = max([len(n['devices']) for n in data_nodes]) + 1 # Create sc with gluster arbiter info self.create_storage_class(is_arbiter_vol=True) # Create and delete 3 small volumes concurrently pvc_names = [] for i in range(3): pvc_name = oc_create_pvc( self.node, self.sc_name, pvc_name_prefix='arbiter-pvc', pvc_size=int(pvc_size / 3)) pvc_names.append(pvc_name) exception_exists = False for pvc_name in pvc_names: try: verify_pvc_status_is_bound(self.node, pvc_name) except Exception: for pvc_name in pvc_names: self.addCleanup( wait_for_resource_absence, self.node, 'pvc', pvc_name) for pvc_name in pvc_names: self.addCleanup(oc_delete, self.node, 'pvc', pvc_name) exception_exists = True if exception_exists: raise for pvc_name in pvc_names: oc_delete(self.node, 'pvc', pvc_name) for pvc_name in pvc_names: wait_for_resource_absence(self.node, 'pvc', pvc_name) # Create and delete big volumes in a loop for i in range(pvc_amount): pvc_name = oc_create_pvc( self.node, self.sc_name, pvc_name_prefix='arbiter-pvc', pvc_size=pvc_size) try: verify_pvc_status_is_bound(self.node, pvc_name) except Exception: self.addCleanup( wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup(oc_delete, self.node, 'pvc', pvc_name) raise oc_delete(self.node, 'pvc', pvc_name) wait_for_resource_absence(self.node, 'pvc', pvc_name)
def test_dynamic_provisioning_glusterfile_heketipod_failure(self): """Validate dynamic provisioning for gluster file when heketi pod down """ mount_path = "/mnt" datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id()) # Create secret and storage class sc_name = self.create_storage_class() # Create PVC app_1_pvc_name = self.create_and_wait_for_pvc( pvc_name_prefix="autotest-file", sc_name=sc_name ) # Create app POD with attached volume app_1_pod_name = oc_create_tiny_pod_with_volume( self.node, app_1_pvc_name, "test-pvc-mount-on-app-pod", mount_path=mount_path) self.addCleanup( wait_for_resource_absence, self.node, 'pod', app_1_pod_name) self.addCleanup(oc_delete, self.node, 'pod', app_1_pod_name) # Wait for app POD be up and running wait_for_pod_be_ready( self.node, app_1_pod_name, timeout=60, wait_step=2) # Write data to the app POD write_data_cmd = ( "dd if=/dev/urandom of=%s bs=1K count=100" % datafile_path) ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd) self.assertEqual( ret, 0, "Failed to execute command %s on %s" % (write_data_cmd, self.node)) # Remove Heketi pod heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % ( self.heketi_dc_name, self.storage_project_name) heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % ( self.heketi_dc_name, self.storage_project_name) self.addCleanup(self.cmd_run, heketi_up_cmd) heketi_pod_name = get_pod_name_from_dc( self.node, self.heketi_dc_name, timeout=10, wait_step=3) self.cmd_run(heketi_down_cmd) wait_for_resource_absence(self.node, 'pod', heketi_pod_name) app_2_pvc_name = oc_create_pvc( self.node, pvc_name_prefix="autotest-file2", sc_name=sc_name ) self.addCleanup( wait_for_resource_absence, self.node, 'pvc', app_2_pvc_name) self.addCleanup( oc_delete, self.node, 'pvc', app_2_pvc_name, raise_on_absence=False ) # Create second app POD app_2_pod_name = oc_create_tiny_pod_with_volume( self.node, app_2_pvc_name, "test-pvc-mount-on-app-pod", mount_path=mount_path) self.addCleanup( wait_for_resource_absence, self.node, 'pod', app_2_pod_name) self.addCleanup(oc_delete, self.node, 'pod', app_2_pod_name) # Bring Heketi POD back self.cmd_run(heketi_up_cmd) # Wait for Heketi POD be up and running new_heketi_pod_name = get_pod_name_from_dc( self.node, self.heketi_dc_name, timeout=10, wait_step=2) wait_for_pod_be_ready( self.node, new_heketi_pod_name, wait_step=5, timeout=120) # Wait for second PVC and app POD be ready verify_pvc_status_is_bound(self.node, app_2_pvc_name) wait_for_pod_be_ready( self.node, app_2_pod_name, timeout=60, wait_step=2) # Verify that we are able to write data ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd) self.assertEqual( ret, 0, "Failed to execute command %s on %s" % (write_data_cmd, self.node))
def create_and_wait_for_pvcs(self, pvc_size=1, pvc_name_prefix="autotests-pvc", pvc_amount=1, sc_name=None, timeout=120, wait_step=3): node = self.ocp_client[0] # Create storage class if not specified if not sc_name: if getattr(self, "sc_name", ""): sc_name = self.sc_name else: sc_name = self.create_storage_class() # Create PVCs pvc_names = [] for i in range(pvc_amount): pvc_name = oc_create_pvc(node, sc_name, pvc_name_prefix=pvc_name_prefix, pvc_size=pvc_size) pvc_names.append(pvc_name) self.addCleanup(wait_for_resource_absence, node, 'pvc', pvc_name) # Wait for PVCs to be in bound state try: for pvc_name in pvc_names: verify_pvc_status_is_bound(node, pvc_name, timeout, wait_step) finally: if get_openshift_version() < "3.9": reclaim_policy = "Delete" else: reclaim_policy = oc_get_custom_resource( node, 'sc', ':.reclaimPolicy', sc_name)[0] for pvc_name in pvc_names: if reclaim_policy == 'Retain': pv_name = get_pv_name_from_pvc(node, pvc_name) self.addCleanup(oc_delete, node, 'pv', pv_name, raise_on_absence=False) custom = (r':.metadata.annotations."gluster\.kubernetes' r'\.io\/heketi\-volume\-id"') vol_id = oc_get_custom_resource(node, 'pv', custom, pv_name)[0] if self.sc.get('provisioner') == "kubernetes.io/glusterfs": self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_id, raise_on_error=False) else: self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, vol_id, raise_on_error=False) self.addCleanup(oc_delete, node, 'pvc', pvc_name, raise_on_absence=False) return pvc_names