def test_heketi_metrics_validating_vol_count_on_vol_deletion(self): """Validate heketi metrics VolumeCount after volume deletion""" vol_list = [] for i in range(3): # Create volume vol = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(vol) self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol['id'], raise_on_error=False) volume_list = heketi_volume_list(self.heketi_client_node, self.heketi_server_url) self.assertIn(vol['id'], volume_list) vol_list.append(vol) for vol in vol_list: # delete volume heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, vol['id']) volume_list = heketi_volume_list(self.heketi_client_node, self.heketi_server_url) self.assertNotIn(vol['id'], volume_list) self.verify_volume_count()
def test_delete_heketidb_volume(self): """Method to test heketidb volume deletion via heketi-cli.""" for i in range(0, 2): volume_info = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 10, json=True) self.addCleanup(heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_info["id"]) volume_list_info = heketi_ops.heketi_volume_list( self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(volume_list_info["volumes"], "Heketi volume list empty.") for volume_id in volume_list_info["volumes"]: volume_info = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) if volume_info["name"] == "heketidbstorage": self.assertRaises(AssertionError, heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_id) return raise ExecutionError( "Warning: heketidbstorage doesn't exist in list of volumes")
def test_blockvolume_create_no_free_space(self): """Validate error is returned when free capacity is exhausted""" # Create first small blockvolume blockvol1 = heketi_blockvolume_create( self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(blockvol1, "Failed to create block volume.") self.addCleanup( heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, blockvol1['id']) # Get info about block hosting volumes file_volumes = heketi_volume_list( self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(file_volumes) self.assertIn("volumes", file_volumes) self.assertTrue(file_volumes["volumes"]) max_block_hosting_vol_size, file_volumes_debug_info = 0, [] for vol_id in file_volumes["volumes"]: vol = heketi_volume_info( self.heketi_client_node, self.heketi_server_url, vol_id, json=True) current_block_hosting_vol_size = vol.get('size', 0) if current_block_hosting_vol_size > max_block_hosting_vol_size: max_block_hosting_vol_size = current_block_hosting_vol_size if current_block_hosting_vol_size: file_volumes_debug_info.append(six.text_type({ 'id': vol.get('id', '?'), 'name': vol.get('name', '?'), 'size': current_block_hosting_vol_size, 'blockinfo': vol.get('blockinfo', '?'), })) self.assertGreater(max_block_hosting_vol_size, 0) # Try to create blockvolume with size bigger than available too_big_vol_size = max_block_hosting_vol_size + 1 try: blockvol2 = heketi_blockvolume_create( self.heketi_client_node, self.heketi_server_url, too_big_vol_size, json=True) except ExecutionError: return if blockvol2 and blockvol2.get('id'): self.addCleanup( heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, blockvol2['id']) block_hosting_vol = heketi_volume_info( self.heketi_client_node, self.heketi_server_url, blockvol2.get('blockhostingvolume'), json=True) self.assertGreater( block_hosting_vol.get('size', -2), blockvol2.get('size', -1), ("Block volume unexpectedly was created. " "Calculated 'max free size' is '%s'.\nBlock volume info is: %s \n" "File volume info, which hosts block volume: \n%s," "Block hosting volumes which were considered: \n%s" % ( max_block_hosting_vol_size, blockvol2, block_hosting_vol, '\n'.join(file_volumes_debug_info))))
def test_arbiter_volume_delete_using_pvc(self): """Test Arbiter volume delete using pvc when volume is not mounted on app pod """ prefix = "autotest-%s" % utils.get_random_str() # Create sc with gluster arbiter info sc_name = self.create_storage_class(vol_name_prefix=prefix, is_arbiter_vol=True) # Create PVC and wait for it to be in 'Bound' state pvc_name = self.create_and_wait_for_pvc(pvc_name_prefix=prefix, sc_name=sc_name) # Get vol info gluster_vol_info = openshift_ops.get_gluster_vol_info_by_pvc_name( self.node, pvc_name) # Verify arbiter volume properties self.verify_amount_and_proportion_of_arbiter_and_data_bricks( gluster_vol_info) # Get volume ID gluster_vol_id = gluster_vol_info["gluster_vol_id"] # Delete the pvc openshift_ops.oc_delete(self.node, 'pvc', pvc_name) openshift_ops.wait_for_resource_absence(self.node, 'pvc', pvc_name) # Check the heketi volume list if pvc is deleted g.log.info("List heketi volumes") heketi_volumes = heketi_ops.heketi_volume_list(self.heketi_client_node, self.heketi_server_url) err_msg = "Failed to delete heketi volume by prefix %s" % prefix self.assertNotIn(prefix, heketi_volumes, err_msg) # Check presence for the gluster volume get_gluster_vol_info = volume_ops.get_volume_info( "auto_get_gluster_endpoint", gluster_vol_id) err_msg = "Failed to delete gluster volume %s" % gluster_vol_id self.assertFalse(get_gluster_vol_info, err_msg) # Check presence of bricks and lvs for brick in gluster_vol_info['bricks']['brick']: gluster_node_ip, brick_name = brick["name"].split(":") with self.assertRaises(exceptions.ExecutionError): cmd = "df %s" % brick_name openshift_ops.cmd_run_on_gluster_pod_or_node( self.node, cmd, gluster_node_ip) with self.assertRaises(exceptions.ExecutionError): lv_match = re.search(BRICK_REGEX, brick["name"]) if lv_match: cmd = "lvs %s" % lv_match.group(2).strip() openshift_ops.cmd_run_on_gluster_pod_or_node( self.node, cmd, gluster_node_ip)
def test_blockvolume_create_no_free_space(self): """Validate error is returned when free capacity is exhausted""" # Create first small blockvolume blockvol1 = heketi_blockvolume_create( self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(blockvol1, "Failed to create block volume.") self.addCleanup( heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, blockvol1['id']) # Get info about block hosting volumes file_volumes = heketi_volume_list( self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(file_volumes) self.assertIn("volumes", file_volumes) self.assertTrue(file_volumes["volumes"]) max_block_hosting_vol_size, file_volumes_debug_info = 0, [] for vol_id in file_volumes["volumes"]: vol = heketi_volume_info( self.heketi_client_node, self.heketi_server_url, vol_id, json=True) current_block_hosting_vol_size = vol.get('size', 0) if current_block_hosting_vol_size > max_block_hosting_vol_size: max_block_hosting_vol_size = current_block_hosting_vol_size if current_block_hosting_vol_size: file_volumes_debug_info.append(six.text_type({ 'id': vol.get('id', '?'), 'name': vol.get('name', '?'), 'size': current_block_hosting_vol_size, 'blockinfo': vol.get('blockinfo', '?'), })) self.assertGreater(max_block_hosting_vol_size, 0) # Try to create blockvolume with size bigger than available too_big_vol_size = max_block_hosting_vol_size + 1 try: blockvol2 = heketi_blockvolume_create( self.heketi_client_node, self.heketi_server_url, too_big_vol_size, json=True) except AssertionError: return if blockvol2 and blockvol2.get('id'): self.addCleanup( heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, blockvol2['id']) block_hosting_vol = heketi_volume_info( self.heketi_client_node, self.heketi_server_url, blockvol2.get('blockhostingvolume'), json=True) self.assertGreater( block_hosting_vol.get('size', -2), blockvol2.get('size', -1), ("Block volume unexpectedly was created. " "Calculated 'max free size' is '%s'.\nBlock volume info is: %s \n" "File volume info, which hosts block volume: \n%s," "Block hosting volumes which were considered: \n%s" % ( max_block_hosting_vol_size, blockvol2, block_hosting_vol, '\n'.join(file_volumes_debug_info))))
def test_volume_create_and_list_volume(self): """Validate heketi and gluster volume list""" g.log.info("List gluster volumes before Heketi volume creation") existing_g_vol_list = get_volume_list('auto_get_gluster_endpoint') self.assertTrue(existing_g_vol_list, ("Unable to get volumes list")) g.log.info("List heketi volumes before volume creation") existing_h_vol_list = heketi_volume_list(self.heketi_client_node, self.heketi_server_url, json=True)["volumes"] g.log.info("Heketi volumes successfully listed") g.log.info("Create a heketi volume") out = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, self.volume_size, json=True) g.log.info("Heketi volume successfully created" % out) volume_id = out["bricks"][0]["volume"] self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_id) g.log.info("List heketi volumes after volume creation") h_vol_list = heketi_volume_list(self.heketi_client_node, self.heketi_server_url, json=True)["volumes"] g.log.info("Heketi volumes successfully listed") g.log.info("List gluster volumes after Heketi volume creation") g_vol_list = get_volume_list('auto_get_gluster_endpoint') self.assertTrue(g_vol_list, ("Unable to get volumes list")) g.log.info("Successfully got the volumes list") # Perform checks self.assertEqual( len(existing_g_vol_list) + 1, len(g_vol_list), "Expected creation of only one volume in Gluster creating " "Heketi volume. Here is lists before and after volume creation: " "%s \n%s" % (existing_g_vol_list, g_vol_list)) self.assertEqual( len(existing_h_vol_list) + 1, len(h_vol_list), "Expected creation of only one volume in Heketi. Here is lists " "of Heketi volumes before and after volume creation: %s\n%s" % (existing_h_vol_list, h_vol_list))
def test_volume_create_and_list_volume(self): """Validate heketi and gluster volume list""" g.log.info("List gluster volumes before Heketi volume creation") existing_g_vol_list = get_volume_list('auto_get_gluster_endpoint') self.assertTrue(existing_g_vol_list, ("Unable to get volumes list")) g.log.info("List heketi volumes before volume creation") existing_h_vol_list = heketi_volume_list( self.heketi_client_node, self.heketi_server_url, json=True)["volumes"] g.log.info("Heketi volumes successfully listed") g.log.info("Create a heketi volume") out = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, self.volume_size, json=True) g.log.info("Heketi volume successfully created" % out) volume_id = out["bricks"][0]["volume"] self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_id) g.log.info("List heketi volumes after volume creation") h_vol_list = heketi_volume_list( self.heketi_client_node, self.heketi_server_url, json=True)["volumes"] g.log.info("Heketi volumes successfully listed") g.log.info("List gluster volumes after Heketi volume creation") g_vol_list = get_volume_list('auto_get_gluster_endpoint') self.assertTrue(g_vol_list, ("Unable to get volumes list")) g.log.info("Successfully got the volumes list") # Perform checks self.assertEqual( len(existing_g_vol_list) + 1, len(g_vol_list), "Expected creation of only one volume in Gluster creating " "Heketi volume. Here is lists before and after volume creation: " "%s \n%s" % (existing_g_vol_list, g_vol_list)) self.assertEqual( len(existing_h_vol_list) + 1, len(h_vol_list), "Expected creation of only one volume in Heketi. Here is lists " "of Heketi volumes before and after volume creation: %s\n%s" % ( existing_h_vol_list, h_vol_list))
def validate_file_volumes_count(self, h_node, h_server, node_ip): # check volume count from heketi and gluster are same heketi_topology_info(h_node, h_server, json=True) h_volume_list = heketi_volume_list(h_node, h_server, json=True) vol_list = get_volume_list(node_ip) self.assertIsNotNone( vol_list, "Failed to get volumes list") self.assertEqual( len(h_volume_list['volumes']), len(vol_list), "Failed to verify volume count Expected:'{}', Actual:'{}'".format( len(h_volume_list['volumes']), len(vol_list)))
def test_dynamic_provisioning_glusterfile_reclaim_policy_retain(self): """Validate retain policy for glusterfs after deletion of pvc""" if get_openshift_version() < "3.9": self.skipTest( "'Reclaim' feature is not supported in OCP older than 3.9") self.create_storage_class(reclaim_policy='Retain') self.create_and_wait_for_pvc() # get the name of the volume pv_name = get_pv_name_from_pvc(self.node, self.pvc_name) custom = [ r':.metadata.annotations.' r'"gluster\.kubernetes\.io\/heketi\-volume\-id"', r':.spec.persistentVolumeReclaimPolicy' ] vol_id, reclaim_policy = oc_get_custom_resource( self.node, 'pv', custom, pv_name) self.assertEqual(reclaim_policy, 'Retain') # Create DC with POD and attached PVC to it. try: dc_name = oc_create_app_dc_with_io( self.node, self.pvc_name, image=self.io_container_image_cirros) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) finally: scale_dc_pod_amount_and_wait(self.node, dc_name, 0) oc_delete(self.node, 'dc', dc_name) wait_for_resource_absence(self.node, 'pod', pod_name) oc_delete(self.node, 'pvc', self.pvc_name) with self.assertRaises(ExecutionError): wait_for_resource_absence(self.node, 'pvc', self.pvc_name, interval=3, timeout=30) heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, vol_id) vol_list = heketi_volume_list(self.heketi_client_node, self.heketi_server_url) self.assertNotIn(vol_id, vol_list) oc_delete(self.node, 'pv', pv_name) wait_for_resource_absence(self.node, 'pv', pv_name)
def _cleanup_heketi_volumes(self, existing_volumes): h_node, h_url = self.heketi_client_node, self.heketi_server_url volumes = heketi_volume_list(h_node, h_url, json=True).get("volumes") new_volumes = list(set(volumes) - set(existing_volumes)) for volume in new_volumes: h_vol_info = heketi_volume_info(h_node, h_url, volume, json=True) if h_vol_info.get("block"): for block_vol in ( h_vol_info.get("blockinfo").get("blockvolume")): heketi_blockvolume_delete(h_node, h_url, block_vol) heketi_volume_delete(h_node, h_url, volume, raise_on_error=False)
def test_heketi_authentication_with_user_credentials(self): """Heketi command authentication with invalid and valid credentials""" h_client, h_server = self.heketi_client_node, self.heketi_server_url err_msg = "Error: Invalid JWT token: Token missing iss claim" # Run heketi commands with invalid credentials for each_cmd in ("volume list", "topology info"): cmd = "timeout 120 heketi-cli -s {} {}".format( self.heketi_server_url, each_cmd) ret, _, err = g.run(h_client, cmd) self.assertTrue(ret, "Command execution with invalid credentials" " should not succeed") self.assertEqual( err_msg, err.strip(), "Error is different from the command" " execution {}".format(err.strip())) # Run heketi commands with valid credentials kwar = {'json_arg': True, 'secret': self.heketi_cli_key, 'user': self.heketi_cli_user} heketi_ops.heketi_volume_list(h_client, h_server, **kwar) heketi_ops.heketi_topology_info(h_client, h_server, **kwar)
def test_to_check_deletion_of_cluster(self): """Validate deletion of cluster with volumes""" # List heketi volumes g.log.info("List heketi volumes") volumes = heketi_volume_list(self.heketi_client_node, self.heketi_server_url, json=True) if (len(volumes["volumes"]) == 0): g.log.info("Creating heketi volume") out = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, self.volume_size, json=True) self.assertTrue(out, ("Failed to create heketi " "volume of size %s" % self.volume_size)) g.log.info("Heketi volume successfully created" % out) volume_id = out["bricks"][0]["volume"] self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_id) # List heketi cluster's g.log.info("Listing heketi cluster list") out = heketi_cluster_list(self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(out, ("Failed to list heketi cluster")) g.log.info("All heketi cluster successfully listed") cluster_id = out["clusters"][0] # Deleting a heketi cluster g.log.info("Trying to delete a heketi cluster" " which contains volumes and/or nodes:" " Expected to fail") self.assertRaises( AssertionError, heketi_cluster_delete, self.heketi_client_node, self.heketi_server_url, cluster_id, ) g.log.info("Expected result: Unable to delete cluster %s" " because it contains volumes " " and/or nodes" % cluster_id) # To confirm deletion failed, check heketi cluster list g.log.info("Listing heketi cluster list") out = heketi_cluster_list(self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(out, ("Failed to list heketi cluster")) g.log.info("All heketi cluster successfully listed")
def test_heketi_metrics_validating_vol_count_on_vol_deletion(self): """Validate heketi metrics VolumeCount after volume deletion""" vol_list = [] for i in range(3): # Create volume vol = heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(vol) self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol['id'], raise_on_error=False) volume_list = heketi_volume_list( self.heketi_client_node, self.heketi_server_url) self.assertIn(vol['id'], volume_list) vol_list.append(vol) for vol in vol_list: # delete volume heketi_volume_delete( self.heketi_client_node, self.heketi_server_url, vol['id']) volume_list = heketi_volume_list( self.heketi_client_node, self.heketi_server_url) self.assertNotIn(vol['id'], volume_list) self.verify_volume_count()
def test_to_check_deletion_of_cluster(self): """Validate deletion of cluster with volumes""" # List heketi volumes g.log.info("List heketi volumes") volumes = heketi_volume_list(self.heketi_client_node, self.heketi_server_url, json=True) if (len(volumes["volumes"]) == 0): g.log.info("Creating heketi volume") out = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, self.volume_size, json=True) self.assertTrue(out, ("Failed to create heketi " "volume of size %s" % self.volume_size)) g.log.info("Heketi volume successfully created" % out) volume_id = out["bricks"][0]["volume"] self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_id) # List heketi cluster's g.log.info("Listing heketi cluster list") out = heketi_cluster_list(self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(out, ("Failed to list heketi cluster")) g.log.info("All heketi cluster successfully listed") cluster_id = out["clusters"][0] # Deleting a heketi cluster g.log.info("Trying to delete a heketi cluster" " which contains volumes and/or nodes:" " Expected to fail") self.assertRaises( ExecutionError, heketi_cluster_delete, self.heketi_client_node, self.heketi_server_url, cluster_id, ) g.log.info("Expected result: Unable to delete cluster %s" " because it contains volumes " " and/or nodes" % cluster_id) # To confirm deletion failed, check heketi cluster list g.log.info("Listing heketi cluster list") out = heketi_cluster_list(self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(out, ("Failed to list heketi cluster")) g.log.info("All heketi cluster successfully listed")
def test_dynamic_provisioning_glusterfile_reclaim_policy_retain(self): """Validate retain policy for glusterfs after deletion of pvc""" if get_openshift_version() < "3.9": self.skipTest( "'Reclaim' feature is not supported in OCP older than 3.9") self.create_storage_class(reclaim_policy='Retain') self.create_and_wait_for_pvc() # get the name of the volume pv_name = get_pv_name_from_pvc(self.node, self.pvc_name) custom = [r':.metadata.annotations.' r'"gluster\.kubernetes\.io\/heketi\-volume\-id"', r':.spec.persistentVolumeReclaimPolicy'] vol_id, reclaim_policy = oc_get_custom_resource( self.node, 'pv', custom, pv_name) self.assertEqual(reclaim_policy, 'Retain') # Create DC with POD and attached PVC to it. try: dc_name = oc_create_app_dc_with_io(self.node, self.pvc_name) pod_name = get_pod_name_from_dc(self.node, dc_name) wait_for_pod_be_ready(self.node, pod_name) finally: scale_dc_pod_amount_and_wait(self.node, dc_name, 0) oc_delete(self.node, 'dc', dc_name) wait_for_resource_absence(self.node, 'pod', pod_name) oc_delete(self.node, 'pvc', self.pvc_name) with self.assertRaises(ExecutionError): wait_for_resource_absence( self.node, 'pvc', self.pvc_name, interval=3, timeout=30) heketi_volume_delete(self.heketi_client_node, self.heketi_server_url, vol_id) vol_list = heketi_volume_list(self.heketi_client_node, self.heketi_server_url) self.assertNotIn(vol_id, vol_list) oc_delete(self.node, 'pv', pv_name) wait_for_resource_absence(self.node, 'pv', pv_name)
def test_heketi_metrics_heketipod_failure(self): """Validate heketi metrics after heketi pod failure""" scale_dc_pod_amount_and_wait(self.ocp_master_node[0], self.heketi_dc_name, pod_amount=0) self.addCleanup(scale_dc_pod_amount_and_wait, self.ocp_master_node[0], self.heketi_dc_name, pod_amount=1) # verify that metrics is not accessable when heketi pod is down with self.assertRaises(AssertionError): get_heketi_metrics(self.heketi_client_node, self.heketi_server_url, prometheus_format=True) scale_dc_pod_amount_and_wait(self.ocp_master_node[0], self.heketi_dc_name, pod_amount=1) pod_name = get_pod_name_from_dc(self.ocp_master_node[0], self.heketi_dc_name, self.heketi_dc_name) wait_for_pod_be_ready(self.ocp_master_node[0], pod_name, wait_step=5) for i in range(3): vol = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(vol) self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol['id'], raise_on_error=False) vol_list = heketi_volume_list(self.heketi_client_node, self.heketi_server_url) self.assertIn(vol['id'], vol_list) self.verify_heketi_metrics_with_topology_info()
def _heketi_vols(ocp_node, url): # Unfortunately, getting json from heketi-cli only gets the ids # To get a mapping of ids & volume names without a lot of # back and forth between the test and the ocp_node we end up having # to scrape the output of 'volume list' # TODO: This probably should be made into a utility function out = heketi_volume_list(ocp_node, url, json=False) res = [] for line in out.splitlines(): if not line.startswith('Id:'): continue row = {} for section in line.split(): if ':' in section: key, value = section.split(':', 1) row[key.lower()] = value.strip() res.append(row) return res
def create_heketi_volume_with_name_and_wait(self, name, size, raise_on_cleanup_error=True, timeout=600, wait_step=10, **kwargs): json = kwargs.get("json", False) try: h_volume_info = heketi_volume_create(self.heketi_client_node, self.heketi_server_url, size, name=name, **kwargs) except Exception as e: if ('more required' in six.text_type(e) or ('Failed to allocate new volume' in six.text_type(e))): raise for w in Waiter(timeout, wait_step): h_volumes = heketi_volume_list(self.heketi_client_node, self.heketi_server_url) h_volume_match = re.search(HEKETI_VOLUME_REGEX % name, h_volumes) if h_volume_match: h_volume_info = heketi_volume_info(self.heketi_client_node, self.heketi_server_url, h_volume_match.group(1), json=json) break if w.expired: g.log.info( "Heketi volume with name %s not created in 600 sec" % name) raise self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, h_volume_info["id"], raise_on_error=raise_on_cleanup_error) return h_volume_info
def test_delete_heketidb_volume(self): """ Method to test heketidb volume deletion via heketi-cli """ heketidbexists = False msg = "Error: Cannot delete volume containing the Heketi database" for i in range(0, 2): volume_info = heketi_ops.heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 10, json=True) self.addCleanup( heketi_ops.heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, volume_info["id"]) volume_list_info = heketi_ops.heketi_volume_list( self.heketi_client_node, self.heketi_server_url, json=True) if volume_list_info["volumes"] == []: raise ExecutionError("Heketi volume list empty") for volume_id in volume_list_info["volumes"]: volume_info = heketi_ops.heketi_volume_info( self.heketi_client_node, self.heketi_server_url, volume_id, json=True) if volume_info["name"] == "heketidbstorage": heketidbexists = True delete_ret, delete_output, delete_error = ( heketi_ops.heketi_volume_delete( self.heketi_client_node, self.heketi_server_url, volume_id, raw_cli_output=True)) self.assertNotEqual(delete_ret, 0, "Return code not 0") self.assertEqual( delete_error.strip(), msg, "Invalid reason for heketidb deletion failure") if not heketidbexists: raise ExecutionError( "Warning: heketidbstorage doesn't exist in list of volumes")
def test_heketi_metrics_heketipod_failure(self): """Validate heketi metrics after heketi pod failure""" scale_dc_pod_amount_and_wait( self.ocp_master_node[0], self.heketi_dc_name, pod_amount=0) self.addCleanup( scale_dc_pod_amount_and_wait, self.ocp_master_node[0], self.heketi_dc_name, pod_amount=1) # verify that metrics is not accessable when heketi pod is down with self.assertRaises(exceptions.ExecutionError): get_heketi_metrics( self.heketi_client_node, self.heketi_server_url, prometheus_format=True) scale_dc_pod_amount_and_wait( self.ocp_master_node[0], self.heketi_dc_name, pod_amount=1) pod_name = get_pod_name_from_dc( self.ocp_master_node[0], self.heketi_dc_name, self.heketi_dc_name) wait_for_pod_be_ready(self.ocp_master_node[0], pod_name, wait_step=5) for i in range(3): vol = heketi_volume_create( self.heketi_client_node, self.heketi_server_url, 1, json=True) self.assertTrue(vol) self.addCleanup( heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol['id'], raise_on_error=False) vol_list = heketi_volume_list( self.heketi_client_node, self.heketi_server_url) self.assertIn(vol['id'], vol_list) self.verify_heketi_metrics_with_topology_info()
def _cleanup_heketi_volumes(self, existing_volumes): """Cleanup created BHV and BV""" volumes = heketi_ops.heketi_volume_list(self.h_node, self.h_server, json=True).get("volumes") new_volumes = list(set(volumes) - set(existing_volumes)) for volume in new_volumes: h_vol_info = heketi_ops.heketi_volume_info(self.h_node, self.h_server, volume, json=True) if h_vol_info.get("block"): for block_vol in ( h_vol_info.get("blockinfo").get("blockvolume")): heketi_ops.heketi_blockvolume_delete(self.h_node, self.h_server, block_vol, raise_on_error=False) heketi_ops.heketi_volume_delete(self.h_node, self.h_server, volume, raise_on_error=False)
def _create_distributed_replica_vol(self, validate_cleanup, block=False): # Create distributed vol vol_size_gb = self._get_vol_size() heketi_url = self.heketi_server_url h_volume_name = "autotests-heketi-volume-%s" % utils.get_random_str() try: heketi_vol = self.create_heketi_volume_with_name_and_wait( h_volume_name, vol_size_gb, json=True, raise_on_cleanup_error=False, block=block) except AssertionError as e: # NOTE: rare situation when we need to decrease size of a volume. # and we expect this vol to be distributed. g.log.info("Failed to create distributed '%s'Gb volume. " "Trying to create another one, smaller for 1Gb.") if not ('more required' in str(e) and ('Insufficient suitable allocatable extents for ' 'logical volume' in str(e))): raise vol_size_gb -= 1 heketi_vol = self.create_heketi_volume_with_name_and_wait( h_volume_name, vol_size_gb, json=True, raise_on_cleanup_error=False, block=block) g.log.info("Successfully created distributed volume.") vol_name = heketi_vol['name'] vol_id = heketi_vol["bricks"][0]["volume"] # Get gluster volume info g.log.info("Get gluster volume '%s' info" % vol_name) gluster_vol = get_volume_info('auto_get_gluster_endpoint', volname=vol_name) self.assertTrue(gluster_vol, "Failed to get volume '%s' info" % vol_name) g.log.info("Successfully got volume '%s' info" % vol_name) gluster_vol = gluster_vol[vol_name] self.assertEqual( gluster_vol["typeStr"], "Distributed-Replicate", "'%s' gluster vol isn't a Distributed-Replicate volume" % vol_name) # Check amount of bricks brick_amount = len(gluster_vol['bricks']['brick']) self.assertEqual( brick_amount % 3, 0, "Brick amount is expected to be divisible by 3. " "Actual amount is '%s'" % brick_amount) self.assertGreater( brick_amount, 3, "Brick amount is expected to be bigger than 3. " "Actual amount is '%s'." % brick_amount) # Run unique actions to Validate whether deleting a dist-rep # volume is handled by heketi else return if not validate_cleanup: return vol_id # Get the free space after creating heketi volume free_space_after_creating_vol = self._get_free_space() # Delete heketi volume g.log.info("Deleting heketi volume '%s'" % vol_id) volume_deleted = heketi_volume_delete(self.heketi_client_node, heketi_url, vol_id) self.assertTrue(volume_deleted, "Failed to delete heketi volume '%s'" % vol_id) g.log.info("Heketi volume '%s' has successfully been deleted" % vol_id) # Check the heketi volume list g.log.info("List heketi volumes") heketi_volumes = heketi_volume_list(self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(heketi_volumes, "Failed to list heketi volumes") g.log.info("Heketi volumes have successfully been listed") heketi_volumes = heketi_volumes.get('volumes', heketi_volumes) self.assertNotIn(vol_id, heketi_volumes) self.assertNotIn(vol_name, heketi_volumes) # Check the gluster volume list g.log.info("Get the gluster volume list") gluster_volumes = get_volume_list('auto_get_gluster_endpoint') self.assertTrue(gluster_volumes, "Unable to get Gluster volume list") g.log.info("Successfully got Gluster volume list" % gluster_volumes) self.assertNotIn(vol_id, gluster_volumes) self.assertNotIn(vol_name, gluster_volumes) # Get the used space after deleting heketi volume free_space_after_deleting_vol = self._get_free_space() # Compare the free space before and after deleting the volume g.log.info("Comparing the free space before and after deleting volume") self.assertLessEqual(free_space_after_creating_vol + (3 * vol_size_gb), free_space_after_deleting_vol) g.log.info( "Volume successfully deleted and space is reallocated. " "Free space after creating volume %s. " "Free space after deleting volume %s." % (free_space_after_creating_vol, free_space_after_deleting_vol))
def test_heketi_manual_cleanup_operation_in_bhv(self): """Validate heketi db cleanup will resolve the mismatch in the free size of the block hosting volume with failed block device create operations. """ bhv_size_before, bhv_size_after, vol_count = [], [], 5 ocp_node, g_node = self.ocp_master_node[0], self.gluster_servers[0] h_node, h_url = self.heketi_client_node, self.heketi_server_url # Get existing heketi volume list existing_volumes = heketi_volume_list(h_node, h_url, json=True) # Add function to clean stale volumes created during test self.addCleanup(self._cleanup_heketi_volumes, existing_volumes.get("volumes")) # Get nodes id list node_id_list = heketi_node_list(h_node, h_url) # Disable 4th and other nodes for node_id in node_id_list[3:]: heketi_node_disable(h_node, h_url, node_id) self.addCleanup(heketi_node_enable, h_node, h_url, node_id) # Calculate heketi volume size free_space, nodenum = get_total_free_space(h_node, h_url) free_space_available = int(free_space / nodenum) if free_space_available > vol_count: h_volume_size = int(free_space_available / vol_count) if h_volume_size > 50: h_volume_size = 50 else: h_volume_size, vol_count = 1, free_space_available # Create BHV in case blockvolume size is greater than default BHV size default_bhv_size = get_default_block_hosting_volume_size( h_node, self.heketi_dc_name) if default_bhv_size < h_volume_size: h_volume_name = "autotest-{}".format(utils.get_random_str()) bhv_info = self.create_heketi_volume_with_name_and_wait( h_volume_name, free_space_available, raise_on_cleanup_error=False, block=True, json=True) free_space_available -= ( int(bhv_info.get("blockinfo").get("reservedsize")) + 1) h_volume_size = int(free_space_available / vol_count) # Get BHV list h_bhv_list = get_block_hosting_volume_list(h_node, h_url).keys() self.assertTrue(h_bhv_list, "Failed to get the BHV list") # Get BHV size for bhv in h_bhv_list: vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bhv_vol_size_before = vol_info.get("freesize") bhv_size_before.append(bhv_vol_size_before) # Kill Tcmu-runner service services = ("tcmu-runner", "gluster-block-target", "gluster-blockd") kill_service_on_gluster_pod_or_node(ocp_node, "tcmu-runner", g_node) # Restart the services for service in services: state = ('exited' if service == 'gluster-block-target' else 'running') self.addCleanup(wait_for_service_status_on_gluster_pod_or_node, ocp_node, service, 'active', state, g_node) self.addCleanup(restart_service_on_gluster_pod_or_node, ocp_node, service, g_node) def run_async(cmd, hostname, raise_on_error=True): return g.run_async(host=hostname, command=cmd) # Create stale block volumes in async for count in range(vol_count): with mock.patch.object(json, 'loads', side_effect=(lambda j: j)): with mock.patch.object(command, 'cmd_run', side_effect=run_async): heketi_blockvolume_create(h_node, h_url, h_volume_size, json=True) # Wait for pending operation to get generated self._check_for_pending_operations(h_node, h_url) # Restart the services for service in services: state = ('exited' if service == 'gluster-block-target' else 'running') restart_service_on_gluster_pod_or_node(ocp_node, service, g_node) wait_for_service_status_on_gluster_pod_or_node( ocp_node, service, 'active', state, g_node) # Cleanup pending operation heketi_server_operation_cleanup(h_node, h_url) # wait for pending operation to get cleaned up for w in waiter.Waiter(timeout=120, interval=10): # Get BHV size for bhv in h_bhv_list: vol_info = heketi_volume_info(h_node, h_url, bhv, json=True) bhv_vol_size_after = vol_info.get("freesize") bhv_size_after.append(bhv_vol_size_after) if (set(bhv_size_before) == set(bhv_size_after)): break if w.expired: raise exceptions.ExecutionError( "Failed to Validate volume size Actual:{}," " Expected:{}".format(set(bhv_size_before), set(bhv_size_after)))
def test_verify_create_heketi_volumes_pending_entries_in_db( self, vol_type): """Verify pending entries of file/block volumes in db during volumes creation from heketi side """ # Create large volumes to observe the pending operations vol_count, h_vol_creation_async_op = 3, [] h_node, h_url = self.heketi_client_node, self.heketi_server_url # Verify file/block volumes pending operation before creation, h_db_check_before = heketi_db_check(h_node, h_url) h_db_check_vol_before = (h_db_check_before.get( "{}volumes".format(vol_type))) # Delete heketi pod to clean db operations if (h_db_check_vol_before.get("pending") or h_db_check_before.get("bricks").get("pending")): self._respin_heketi_pod() # Calculate heketi volume size free_space, nodenum = get_total_free_space(h_node, h_url) free_space_available = int(free_space / nodenum) if free_space_available > vol_count: h_volume_size = int(free_space_available / vol_count) if h_volume_size > 30: h_volume_size = 30 else: h_volume_size, vol_count = 1, free_space_available # Get existing heketi volume list existing_volumes = heketi_volume_list(h_node, h_url, json=True) # Add cleanup function to clean stale volumes created during test self.addCleanup(self._cleanup_heketi_volumes, existing_volumes.get("volumes")) # Create BHV in case blockvolume size is greater than default BHV size if vol_type: default_bhv_size = get_default_block_hosting_volume_size( h_node, self.heketi_dc_name) if default_bhv_size < h_volume_size: h_volume_name = "autotest-{}".format(utils.get_random_str()) bhv_info = self.create_heketi_volume_with_name_and_wait( h_volume_name, free_space_available, raise_on_cleanup_error=False, block=True, json=True) free_space_available -= ( int(bhv_info.get("blockinfo").get("reservedsize")) + 1) h_volume_size = int(free_space_available / vol_count) # Temporary replace g.run with g.async_run in heketi_blockvolume_create # func to be able to run it in background.Also, avoid parsing the # output as it won't be json at that moment. Parse it after reading # the async operation results. def run_async(cmd, hostname, raise_on_error=True): return g.run_async(host=hostname, command=cmd) for count in range(vol_count): with mock.patch.object(json, 'loads', side_effect=(lambda j: j)): with mock.patch.object(command, 'cmd_run', side_effect=run_async): h_vol_creation_async_op.append( eval("heketi_{}volume_create".format(vol_type))( h_node, h_url, h_volume_size, json=True)) # Check for pending operations for w in waiter.Waiter(timeout=120, interval=10): h_db_check = heketi_db_check(h_node, h_url) h_db_check_vol = h_db_check.get("{}volumes".format(vol_type)) if h_db_check_vol.get("pending"): h_db_check_bricks = h_db_check.get("bricks") break if w.expired: raise exceptions.ExecutionError( "No any pending operations found during {}volumes creation " "{}".format(vol_type, h_db_check_vol.get("pending"))) # Verify bricks pending operation during creation if not vol_type: self.assertTrue(h_db_check_bricks.get("pending"), "Expecting at least one bricks pending count") self.assertFalse( h_db_check_bricks.get("pending") % 3, "Expecting bricks pending count to be multiple of 3 but " "found {}".format(h_db_check_bricks.get("pending"))) # Wait for all counts of pending operations to be zero for w in waiter.Waiter(timeout=300, interval=10): h_db_check = heketi_db_check(h_node, h_url) h_db_check_vol = h_db_check.get("{}volumes".format(vol_type)) if not h_db_check_vol.get("pending"): break if w.expired: raise exceptions.ExecutionError( "Expecting no pending operations after 300 sec but " "found {} operation".format(h_db_check_vol.get("pending"))) # Get heketi server DB details h_db_check_after = heketi_db_check(h_node, h_url) h_db_check_vol_after = (h_db_check_after.get( "{}volumes".format(vol_type))) h_db_check_bricks_after = h_db_check_after.get("bricks") # Verify if initial and final file/block volumes are same act_vol_count = h_db_check_vol_after.get("total") exp_vol_count = h_db_check_vol_before.get("total") + vol_count err_msg = ( "Actual {} and expected {} {}volume counts are not matched".format( act_vol_count, exp_vol_count, vol_type)) self.assertEqual(act_vol_count, exp_vol_count, err_msg) # Verify if initial and final bricks are same for file volume volumes = heketi_volume_list(h_node, h_url, json=True).get("volumes") new_volumes = list(set(volumes) - set(existing_volumes)) exp_brick_count = 0 for volume in new_volumes: vol_info = heketi_volume_info(h_node, h_url, volume, json=True) exp_brick_count += len(vol_info.get("bricks")) err_msg = "Actual {} and expected {} bricks counts are not matched" act_brick_count = h_db_check_bricks_after.get("total") self.assertEqual(act_brick_count, exp_brick_count, err_msg.format(act_brick_count, exp_brick_count))
def test_verify_delete_heketi_volumes_pending_entries_in_db( self, vol_type): """Verify pending entries of blockvolumes/volumes and bricks in heketi db during blockvolume/volume delete operation. """ # Create a large volumes to observe the pending operation vol_count, volume_ids, async_obj = 10, [], [] h_node, h_url = self.heketi_client_node, self.heketi_server_url # Verify file/block volumes pending operation before creation, h_db_check_before = heketi_db_check(h_node, h_url) h_db_check_bricks_before = h_db_check_before.get("bricks") h_db_check_vol_before = (h_db_check_before.get( "{}volumes".format(vol_type))) # Get existing heketi volume list existing_volumes = heketi_volume_list(h_node, h_url, json=True) # Add cleanup function to clean stale volumes created during test self.addCleanup(self._cleanup_heketi_volumes, existing_volumes.get("volumes")) # Delete heketi pod to clean db operations if (h_db_check_bricks_before.get("pending") or h_db_check_vol_before.get("pending")): self._respin_heketi_pod() # Calculate heketi volume size free_space, nodenum = get_total_free_space(h_node, h_url) free_space_available = int(free_space / nodenum) if free_space_available > vol_count: h_volume_size = int(free_space_available / vol_count) if h_volume_size > 50: h_volume_size = 50 else: h_volume_size, vol_count = 1, free_space_available # Create BHV in case blockvolume size is greater than default BHV size if vol_type: default_bhv_size = get_default_block_hosting_volume_size( h_node, self.heketi_dc_name) if default_bhv_size < h_volume_size: h_volume_name = "autotest-{}".format(utils.get_random_str()) bhv_info = self.create_heketi_volume_with_name_and_wait( h_volume_name, free_space_available, raise_on_cleanup_error=False, block=True, json=True) free_space_available -= ( int(bhv_info.get("blockinfo").get("reservedsize")) + 1) h_volume_size = int(free_space_available / vol_count) # Create file/block volumes for _ in range(vol_count): vol_id = eval("heketi_{}volume_create".format(vol_type))( h_node, h_url, h_volume_size, json=True).get("id") volume_ids.append(vol_id) self.addCleanup(eval("heketi_{}volume_delete".format(vol_type)), h_node, h_url, vol_id, raise_on_error=False) def run_async(cmd, hostname, raise_on_error=True): async_op = g.run_async(host=hostname, command=cmd) async_obj.append(async_op) return async_op bhv_list = [] for vol_id in volume_ids: # Get BHV ids to delete in case of block volumes if vol_type: vol_info = (heketi_blockvolume_info(h_node, h_url, vol_id, json=True)) if not vol_info.get("blockhostingvolume") in bhv_list: bhv_list.append(vol_info.get("blockhostingvolume")) # Temporary replace g.run with g.async_run in heketi_volume_delete # and heketi_blockvolume_delete func to be able to run it in # background. with mock.patch.object(command, 'cmd_run', side_effect=run_async): eval("heketi_{}volume_delete".format(vol_type))(h_node, h_url, vol_id) # Wait for pending operations to get generate for w in waiter.Waiter(timeout=30, interval=3): h_db_check = heketi_db_check(h_node, h_url) h_db_check_vol = h_db_check.get("{}volumes".format(vol_type)) if h_db_check_vol.get("pending"): h_db_check_bricks = h_db_check.get("bricks") break if w.expired: raise exceptions.ExecutionError( "No any pending operations found during {}volumes deletion " "{}".format(vol_type, h_db_check_vol.get("pending"))) # Verify bricks pending operation during creation if not vol_type: self.assertTrue(h_db_check_bricks.get("pending"), "Expecting at least one bricks pending count") self.assertFalse( h_db_check_bricks.get("pending") % 3, "Expecting bricks pending count to be multiple of 3 but " "found {}".format(h_db_check_bricks.get("pending"))) # Verify file/block volume pending operation during delete for w in waiter.Waiter(timeout=120, interval=10): h_db_check = heketi_db_check(h_node, h_url) h_db_check_vol = h_db_check.get("{}volumes".format(vol_type)) h_db_check_bricks = h_db_check.get("bricks") if ((not h_db_check_bricks.get("pending")) and (not h_db_check_vol.get("pending"))): break if w.expired: raise AssertionError( "Failed to delete {}volumes after 120 secs".format(vol_type)) # Check that all background processes got exited for obj in async_obj: ret, out, err = obj.async_communicate() self.assertFalse( ret, "Failed to delete {}volume due to error: {}".format( vol_type, err)) # Delete BHV created during block volume creation if vol_type: for bhv_id in bhv_list: heketi_volume_delete(h_node, h_url, bhv_id) # Verify bricks and volume pending operations h_db_check_after = heketi_db_check(h_node, h_url) h_db_check_bricks_after = h_db_check_after.get("bricks") h_db_check_vol_after = (h_db_check_after.get( "{}volumes".format(vol_type))) act_brick_count = h_db_check_bricks_after.get("pending") act_vol_count = h_db_check_vol_after.get("pending") # Verify bricks pending operation after delete err_msg = "{} operations are pending for {} after {}volume deletion" if not vol_type: self.assertFalse( act_brick_count, err_msg.format(act_brick_count, "brick", vol_type)) # Verify file/bock volumes pending operation after delete self.assertFalse(act_vol_count, err_msg.format(act_vol_count, "volume", vol_type)) act_brick_count = h_db_check_bricks_after.get("total") act_vol_count = h_db_check_vol_after.get("total") exp_brick_count = h_db_check_bricks_before.get("total") exp_vol_count = h_db_check_vol_before.get("total") err_msg = "Actual {} and expected {} {} counts are not matched" # Verify if initial and final file/block volumes are same self.assertEqual( act_vol_count, exp_vol_count, err_msg.format(act_vol_count, exp_vol_count, "volume")) # Verify if initial and final bricks are same self.assertEqual( act_brick_count, exp_brick_count, err_msg.format(act_brick_count, exp_brick_count, "brick"))
def test_heketi_device_replacement_in_node(self): """Validate device replacement operation on single node""" h_client, h_server = self.heketi_client_node, self.heketi_server_url try: gluster_server_0 = list(g.config["gluster_servers"].values())[0] manage_hostname = gluster_server_0["manage"] add_device_name = gluster_server_0["additional_devices"][0] except (KeyError, IndexError): self.skipTest( "Additional disk is not specified for node with following " "hostnames and IP addresses: {}, {}".format( gluster_server_0.get('manage', '?'), gluster_server_0.get('storage', '?'))) # Get existing heketi volume list existing_volumes = heketi_volume_list(h_client, h_server, json=True) # Add cleanup function to clean stale volumes created during test self.addCleanup(self._cleanup_heketi_volumes, existing_volumes.get("volumes")) # Get nodes info node_id_list = heketi_node_list(h_client, h_server) # Disable 4th and other nodes if len(node_id_list) > 3: for node_id in node_id_list[3:]: heketi_node_disable(h_client, h_server, node_id) self.addCleanup(heketi_node_enable, h_client, h_server, node_id) # Create volume when 3 nodes are online vol_size, vol_count = 2, 4 for _ in range(vol_count): vol_info = heketi_blockvolume_create(h_client, h_server, vol_size, json=True) self.addCleanup(heketi_blockvolume_delete, h_client, h_server, vol_info['id']) # Get node ID of the Gluster hostname topology_info = heketi_topology_info(h_client, h_server, json=True) self.assertIsNotNone(topology_info, "Failed to get topology info") self.assertIn("clusters", topology_info.keys(), "Failed to get cluster " "details from topology info") node_list = topology_info["clusters"][0]["nodes"] self.assertTrue(node_list, "Cluster info command returned empty list of nodes") node_id = None for node in node_list: if manage_hostname == node['hostnames']["manage"][0]: node_id = node["id"] break self.assertTrue( node_id, "Failed to get node info for node id '{}'".format( manage_hostname)) # Add extra device, then remember it's ID and size device_id_new, device_size_new = self._add_heketi_device( add_device_name, node_id) # Remove one of the existing devices on node except new device device_name, device_id = None, None node_info_after_addition = heketi_node_info(h_client, h_server, node_id, json=True) for device in node_info_after_addition["devices"]: if (device["name"] != add_device_name and device["storage"]["total"] == device_size_new): device_name = device["name"] device_id = device["id"] break self.assertIsNotNone(device_name, "Failed to get device name") self.assertIsNotNone(device_id, "Failed to get device id") self.addCleanup(heketi_device_enable, h_client, h_server, device_id, raise_on_error=False) self.addCleanup(heketi_device_add, h_client, h_server, device_name, node_id, raise_on_error=False) heketi_device_disable(h_client, h_server, device_id) heketi_device_remove(h_client, h_server, device_id) heketi_device_delete(h_client, h_server, device_id)
def test_heketi_node_add_with_valid_cluster(self): """Test heketi node add operation with valid cluster id""" if (openshift_storage_version.get_openshift_storage_version() < "3.11.4"): self.skipTest( "This test case is not supported for < OCS 3.11.4 builds due " "to bug BZ-1732831") h_client, h_server = self.heketi_client_node, self.heketi_server_url ocp_node = self.ocp_master_node[0] # Get heketi endpoints before adding node h_volume_ids = heketi_ops.heketi_volume_list( h_client, h_server, json=True) h_endpoints_before_new_node = heketi_ops.heketi_volume_endpoint_patch( h_client, h_server, h_volume_ids["volumes"][0]) cluster_info = heketi_ops.heketi_cluster_list( h_client, h_server, json=True) storage_hostname, storage_ip = self.add_heketi_node_to_cluster( cluster_info["clusters"][0]) # Get heketi nodes and validate for newly added node h_node_ids = heketi_ops.heketi_node_list(h_client, h_server, json=True) for h_node_id in h_node_ids: node_hostname = heketi_ops.heketi_node_info( h_client, h_server, h_node_id, json=True) if node_hostname["hostnames"]["manage"][0] == storage_hostname: break node_hostname = None err_msg = ("Newly added heketi node %s not found in heketi node " "list %s" % (storage_hostname, h_node_ids)) self.assertTrue(node_hostname, err_msg) # Check gluster peer status for newly added node if self.is_containerized_gluster(): gluster_pods = openshift_ops.get_ocp_gluster_pod_details(ocp_node) gluster_pod = [ gluster_pod["pod_name"] for gluster_pod in gluster_pods if gluster_pod["pod_hostname"] == storage_hostname][0] gluster_peer_status = peer_ops.get_peer_status( podcmd.Pod(ocp_node, gluster_pod)) else: gluster_peer_status = peer_ops.get_peer_status( storage_hostname) self.assertEqual( len(gluster_peer_status), len(self.gluster_servers)) err_msg = "Expected peer status is 1 and actual is %s" for peer in gluster_peer_status: peer_status = int(peer["connected"]) self.assertEqual(peer_status, 1, err_msg % peer_status) # Get heketi endpoints after adding node h_endpoints_after_new_node = heketi_ops.heketi_volume_endpoint_patch( h_client, h_server, h_volume_ids["volumes"][0]) # Get openshift openshift endpoints and patch with heketi endpoints heketi_db_endpoint = openshift_ops.oc_get_custom_resource( ocp_node, "dc", name=self.heketi_dc_name, custom=".:spec.template.spec.volumes[*].glusterfs.endpoints")[0] openshift_ops.oc_patch( ocp_node, "ep", heketi_db_endpoint, h_endpoints_after_new_node) self.addCleanup( openshift_ops.oc_patch, ocp_node, "ep", heketi_db_endpoint, h_endpoints_before_new_node) ep_addresses = openshift_ops.oc_get_custom_resource( ocp_node, "ep", name=heketi_db_endpoint, custom=".:subsets[*].addresses[*].ip")[0].split(",") err_msg = "Hostname %s not present in endpoints %s" % ( storage_ip, ep_addresses) self.assertIn(storage_ip, ep_addresses, err_msg)