def test_heketi_node_states_enable_disable(self): """Test node enable and disable functionality """ h_client, h_server = self.heketi_client_node, self.heketi_server_url node_list = heketi_ops.heketi_node_list(h_client, h_server) online_hosts = [] for node_id in node_list: node_info = heketi_ops.heketi_node_info(h_client, h_server, node_id, json=True) if node_info["state"] == "online": online_hosts.append(node_info) if len(online_hosts) < 3: raise self.skipTest( "This test can run only if online hosts are more than 2") # Disable n-3 nodes, in case we have n nodes for node_info in online_hosts[3:]: node_id = node_info["id"] heketi_ops.heketi_node_disable(h_client, h_server, node_id) self.addCleanup(heketi_ops.heketi_node_enable, h_client, h_server, node_id) # Create volume when 3 nodes are online vol_size = 1 vol_info = heketi_ops.heketi_volume_create(h_client, h_server, vol_size, json=True) self.addCleanup(heketi_ops.heketi_volume_delete, h_client, h_server, vol_info['id']) node_id = online_hosts[0]['id'] try: heketi_ops.heketi_node_disable(h_client, h_server, node_id) # Try to create a volume, volume creation should fail with self.assertRaises(AssertionError): heketi_volume = heketi_ops.heketi_volume_create( h_client, h_server, vol_size) self.addCleanup(heketi_ops.heketi_volume_delete, h_client, h_server, heketi_volume["id"]) finally: # Enable heketi node heketi_ops.heketi_node_enable(h_client, h_server, node_id) # Create volume when heketi node is enabled vol_info = heketi_ops.heketi_volume_create(h_client, h_server, vol_size, json=True) heketi_ops.heketi_volume_delete(h_client, h_server, vol_info['id'])
def enable_node(self, node_id): """ Enable node through heketi-cli. :param node_id: str node ID """ out = heketi_node_enable(self.heketi_client_node, self.heketi_server_url, node_id) self.assertNotEqual(out, False, "Failed to enable node of" " id %s" % node_id)
def enable_node(self, node_id): """ Enable node through heketi-cli. :param node_id: str node ID """ out = heketi_node_enable(self.heketi_client_node, self.heketi_server_url, node_id) self.assertNotEqual(out, False, "Failed to enable node of" " id %s" % node_id)
def test_dev_path_mapping_heketi_node_delete(self): """Validate dev path mapping for heketi node deletion lifecycle""" h_client, h_url = self.heketi_client_node, self.heketi_server_url node_ids = heketi_ops.heketi_node_list(h_client, h_url) self.assertTrue(node_ids, "Failed to get heketi node list") # Fetch #4th node for the operations h_disable_node = node_ids[3] # Fetch bricks on the devices before volume create h_node_details_before, h_node = self._get_bricks_and_device_details() # Bricks count on the node before pvc creation brick_count_before = [count[1] for count in h_node_details_before] # Create file volume with app pod and verify IO's # and compare path, UUID, vg_name pod_name, dc_name, use_percent = self._create_app_pod_and_verify_pvs() # Check if IO's are running use_percent_after = self._get_space_use_percent_in_app_pod(pod_name) self.assertNotEqual( use_percent, use_percent_after, "Failed to execute IO's in the app pod {} after respin".format( pod_name)) # Fetch bricks on the devices after volume create h_node_details_after, h_node = self._get_bricks_and_device_details() # Bricks count on the node after pvc creation brick_count_after = [count[1] for count in h_node_details_after] self.assertGreater( sum(brick_count_after), sum(brick_count_before), "Failed to add bricks on the node {}".format(h_node)) self.addCleanup(heketi_ops.heketi_node_disable, h_client, h_url, h_disable_node) # Enable the #4th node heketi_ops.heketi_node_enable(h_client, h_url, h_disable_node) node_info = heketi_ops.heketi_node_info(h_client, h_url, h_disable_node, json=True) h_node_id = node_info['id'] self.assertEqual(node_info['state'], "online", "Failed to enable node {}".format(h_disable_node)) # Disable the node and check for brick migrations self.addCleanup(heketi_ops.heketi_node_enable, h_client, h_url, h_node, raise_on_error=False) heketi_ops.heketi_node_disable(h_client, h_url, h_node) node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node, json=True) self.assertEqual(node_info['state'], "offline", "Failed to disable node {}".format(h_node)) # Before bricks migration h_node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node, json=True) # Bricks before migration on the node i.e to be deleted bricks_counts_before = 0 for device in h_node_info['devices']: bricks_counts_before += (len(device['bricks'])) # Remove the node heketi_ops.heketi_node_remove(h_client, h_url, h_node) # After bricks migration h_node_info_after = heketi_ops.heketi_node_info(h_client, h_url, h_node, json=True) # Bricks after migration on the node i.e to be delete bricks_counts = 0 for device in h_node_info_after['devices']: bricks_counts += (len(device['bricks'])) self.assertFalse( bricks_counts, "Failed to remove all the bricks from node {}".format(h_node)) # Old node which is to deleted, new node were bricks resides old_node, new_node = h_node, h_node_id # Node info for the new node were brick reside after migration h_node_info_new = heketi_ops.heketi_node_info(h_client, h_url, new_node, json=True) bricks_counts_after = 0 for device in h_node_info_new['devices']: bricks_counts_after += (len(device['bricks'])) self.assertEqual( bricks_counts_before, bricks_counts_after, "Failed to migrated bricks from {} node to {}".format( old_node, new_node)) # Fetch device list i.e to be deleted h_node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node, json=True) devices_list = [[device['id'], device['name']] for device in h_node_info['devices']] for device in devices_list: device_id = device[0] device_name = device[1] self.addCleanup(heketi_ops.heketi_device_add, h_client, h_url, device_name, h_node, raise_on_error=False) # Device deletion from heketi node device_delete = heketi_ops.heketi_device_delete( h_client, h_url, device_id) self.assertTrue(device_delete, "Failed to delete the device {}".format(device_id)) node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node, json=True) cluster_id = node_info['cluster'] zone = node_info['zone'] storage_hostname = node_info['hostnames']['manage'][0] storage_ip = node_info['hostnames']['storage'][0] # Delete the node self.addCleanup(heketi_ops.heketi_node_add, h_client, h_url, zone, cluster_id, storage_hostname, storage_ip, raise_on_error=False) heketi_ops.heketi_node_delete(h_client, h_url, h_node) # Verify if the node is deleted node_ids = heketi_ops.heketi_node_list(h_client, h_url) self.assertNotIn(old_node, node_ids, "Failed to delete the node {}".format(old_node)) # Check if IO's are running use_percent_after = self._get_space_use_percent_in_app_pod(pod_name) self.assertNotEqual( use_percent, use_percent_after, "Failed to execute IO's in the app pod {} after respin".format( pod_name)) # Adding node back h_node_info = heketi_ops.heketi_node_add(h_client, h_url, zone, cluster_id, storage_hostname, storage_ip, json=True) self.assertTrue( h_node_info, "Failed to add the node in the cluster {}".format(cluster_id)) h_node_id = h_node_info["id"] # Adding devices to the new node for device in devices_list: storage_device = device[1] # Add device to the new heketi node heketi_ops.heketi_device_add(h_client, h_url, storage_device, h_node_id) heketi_node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node_id, json=True) device_id = None for device in heketi_node_info["devices"]: if device["name"] == storage_device: device_id = device["id"] break self.assertTrue( device_id, "Failed to add device {} on node {}".format( storage_device, h_node_id)) # Create n pvc in order to verfiy if the bricks reside on the new node pvc_amount, pvc_size = 5, 1 # Fetch bricks on the devices before volume create h_node_details_before, h_node = self._get_bricks_and_device_details() # Bricks count on the node before pvc creation brick_count_before = [count[1] for count in h_node_details_before] # Create file volumes pvc_name = self.create_and_wait_for_pvcs(pvc_size=pvc_size, pvc_amount=pvc_amount) self.assertEqual(len(pvc_name), pvc_amount, "Failed to create {} pvc".format(pvc_amount)) # Fetch bricks on the devices before volume create h_node_details_after, h_node = self._get_bricks_and_device_details() # Bricks count on the node after pvc creation brick_count_after = [count[1] for count in h_node_details_after] self.assertGreater( sum(brick_count_after), sum(brick_count_before), "Failed to add bricks on the new node {}".format(new_node)) # Check if IO's are running after new node is added use_percent_after = self._get_space_use_percent_in_app_pod(pod_name) self.assertNotEqual( use_percent, use_percent_after, "Failed to execute IO's in the app pod {} after respin".format( pod_name))
def test_dev_path_mapping_heketi_device_delete(self): """Validate dev path mapping for heketi device delete lifecycle""" h_client, h_url = self.heketi_client_node, self.heketi_server_url node_ids = heketi_ops.heketi_node_list(h_client, h_url) self.assertTrue(node_ids, "Failed to get heketi node list") # Fetch #4th node for the operations h_disable_node = node_ids[3] # Fetch bricks on the devices before volume create h_node_details_before, h_node = self._get_bricks_and_device_details() # Bricks count on the node before pvc creation brick_count_before = [count[1] for count in h_node_details_before] # Create file volume with app pod and verify IO's # and compare path, UUID, vg_name pod_name, dc_name, use_percent = self._create_app_pod_and_verify_pvs() # Check if IO's are running use_percent_after = self._get_space_use_percent_in_app_pod(pod_name) self.assertNotEqual( use_percent, use_percent_after, "Failed to execute IO's in the app pod {} after respin".format( pod_name)) # Fetch bricks on the devices after volume create h_node_details_after, h_node = self._get_bricks_and_device_details() # Bricks count on the node after pvc creation brick_count_after = [count[1] for count in h_node_details_after] self.assertGreater( sum(brick_count_after), sum(brick_count_before), "Failed to add bricks on the node {}".format(h_node)) # Enable the #4th node heketi_ops.heketi_node_enable(h_client, h_url, h_disable_node) node_info = heketi_ops.heketi_node_info(h_client, h_url, h_disable_node, json=True) h_node_id = node_info['id'] self.assertEqual(node_info['state'], "online", "Failed to enable node {}".format(h_disable_node)) # Fetch device list i.e to be deleted h_node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node, json=True) devices_list = [[device['id'], device['name']] for device in h_node_info['devices']] # Device deletion operation for device in devices_list: device_id, device_name = device[0], device[1] self.addCleanup(heketi_ops.heketi_device_enable, h_client, h_url, device_id, raise_on_error=False) # Disable device from heketi device_disable = heketi_ops.heketi_device_disable( h_client, h_url, device_id) self.assertTrue( device_disable, "Device {} could not be disabled".format(device_id)) device_info = heketi_ops.heketi_device_info(h_client, h_url, device_id, json=True) self.assertEqual(device_info['state'], "offline", "Failed to disable device {}".format(device_id)) # Remove device from heketi device_remove = heketi_ops.heketi_device_remove( h_client, h_url, device_id) self.assertTrue(device_remove, "Device {} could not be removed".format(device_id)) # Bricks after device removal device_info = heketi_ops.heketi_device_info(h_client, h_url, device_id, json=True) bricks_count_after = len(device_info['bricks']) self.assertFalse( bricks_count_after, "Failed to remove the bricks from the device {}".format( device_id)) # Delete device from heketi self.addCleanup(heketi_ops.heketi_device_add, h_client, h_url, device_name, h_node, raise_on_error=False) device_delete = heketi_ops.heketi_device_delete( h_client, h_url, device_id) self.assertTrue(device_delete, "Device {} could not be deleted".format(device_id)) # Check if IO's are running after device is deleted use_percent_after = self._get_space_use_percent_in_app_pod(pod_name) self.assertNotEqual( use_percent, use_percent_after, "Failed to execute IO's in the app pod {} after respin".format( pod_name)) # Add device operations for device in devices_list: device_name = device[1] # Add device back to the node heketi_ops.heketi_device_add(h_client, h_url, device_name, h_node) # Fetch device info after device add node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node, json=True) device_id = None for device in node_info["devices"]: if device["name"] == device_name: device_id = device["id"] break self.assertTrue( device_id, "Failed to add device {} on node" " {}".format(device_name, h_node)) # Disable the #4th node heketi_ops.heketi_node_disable(h_client, h_url, h_node_id) node_info = heketi_ops.heketi_node_info(h_client, h_url, h_node_id, json=True) self.assertEqual(node_info['state'], "offline", "Failed to disable node {}".format(h_node_id)) pvc_amount, pvc_size = 5, 1 # Fetch bricks on the devices before volume create h_node_details_before, h_node = self._get_bricks_and_device_details() # Bricks count on the node before pvc creation brick_count_before = [count[1] for count in h_node_details_before] # Create file volumes pvc_name = self.create_and_wait_for_pvcs(pvc_size=pvc_size, pvc_amount=pvc_amount) self.assertEqual(len(pvc_name), pvc_amount, "Failed to create {} pvc".format(pvc_amount)) # Fetch bricks on the devices after volume create h_node_details_after, h_node = self._get_bricks_and_device_details() # Bricks count on the node after pvc creation brick_count_after = [count[1] for count in h_node_details_after] self.assertGreater( sum(brick_count_after), sum(brick_count_before), "Failed to add bricks on the node {}".format(h_node)) # Check if IO's are running after new device is added use_percent_after = self._get_space_use_percent_in_app_pod(pod_name) self.assertNotEqual( use_percent, use_percent_after, "Failed to execute IO's in the app pod {} after respin".format( pod_name))
def test_check_node_disable_based_on_heketi_zone( self, zone_count, is_disable_on_different_zone, is_set_env=False): """Validate node disable in different heketi zones""" expected_node_count, heketi_zone_checking, sc_name = 4, "strict", None # Check amount of available online nodes online_node_count = len(self._get_online_nodes()) if online_node_count < expected_node_count: self.skipTest('Available node count {} is less than expected node ' 'count {}'.format(online_node_count, expected_node_count)) # Check amount of available online heketi zones self._check_for_available_zones(zone_count) # Get the online devices and nodes w.r.t. to zone zone_devices_nodes = self._get_online_devices_and_nodes_with_zone() # Create sc or else directly set env to "strict" inside dc is_create_sc = not is_set_env if is_create_sc: sc_name = self.create_storage_class( sc_name_prefix=self.prefix, vol_name_prefix=self.prefix, heketi_zone_checking=heketi_zone_checking) if is_set_env: self._set_zone_check_env_in_heketi_dc(heketi_zone_checking) # Choose a zone and node_id to disable the device for zone, nodes_and_devices in zone_devices_nodes.items(): if zone_count == 3: # Select a node with a zone having multiple nodes in same # zone to cover the test cases disable node in same zone if len(nodes_and_devices['nodes']) > 1: zone_with_disabled_node = zone disabled_node = nodes_and_devices['nodes'][0] break else: # Select node from any of the zones zone_with_disabled_node = zone disabled_node = nodes_and_devices['nodes'][0] break # Disable the selected node heketi_ops.heketi_node_disable(self.h_client, self.h_server, disabled_node) self.addCleanup(heketi_ops.heketi_node_enable, self.h_client, self.h_server, disabled_node) # Create some DCs with PVCs and check brick placement in heketi zones pod_names = self._create_dcs_and_check_brick_placement( self.prefix, sc_name, heketi_zone_checking, zone_count) # Enable disabled node heketi_ops.heketi_node_enable(self.h_client, self.h_server, disabled_node) if is_disable_on_different_zone: # Select the new node in a different zone for zone, nodes_and_devices in zone_devices_nodes.items(): if zone != zone_with_disabled_node: new_node_to_disable = nodes_and_devices['nodes'][0] break else: # Select the new node in the same zone new_node_to_disable = zone_devices_nodes[zone_with_disabled_node][ 'nodes'][1] # Disable the newly selected node heketi_ops.heketi_node_disable(self.h_client, self.h_server, new_node_to_disable) self.addCleanup(heketi_ops.heketi_node_enable, self.h_client, self.h_server, new_node_to_disable) # Verify if pods are in ready state for pod_name in pod_names: openshift_ops.wait_for_pod_be_ready(self.node, pod_name, timeout=5, wait_step=2)