def test_heketi_node_states_enable_disable(self):
"""Test node enable and disable functionality
"""
h_client, h_server = self.heketi_client_node, self.heketi_server_url
node_list = heketi_ops.heketi_node_list(h_client, h_server)
online_hosts = []
for node_id in node_list:
node_info = heketi_ops.heketi_node_info(h_client,
h_server,
node_id,
json=True)
if node_info["state"] == "online":
online_hosts.append(node_info)
if len(online_hosts) < 3:
raise self.skipTest(
"This test can run only if online hosts are more than 2")
# Disable n-3 nodes, in case we have n nodes
for node_info in online_hosts[3:]:
node_id = node_info["id"]
heketi_ops.heketi_node_disable(h_client, h_server, node_id)
self.addCleanup(heketi_ops.heketi_node_enable, h_client, h_server,
node_id)
# Create volume when 3 nodes are online
vol_size = 1
vol_info = heketi_ops.heketi_volume_create(h_client,
h_server,
vol_size,
json=True)
self.addCleanup(heketi_ops.heketi_volume_delete, h_client, h_server,
vol_info['id'])
node_id = online_hosts[0]['id']
try:
heketi_ops.heketi_node_disable(h_client, h_server, node_id)
# Try to create a volume, volume creation should fail
with self.assertRaises(AssertionError):
heketi_volume = heketi_ops.heketi_volume_create(
h_client, h_server, vol_size)
self.addCleanup(heketi_ops.heketi_volume_delete, h_client,
h_server, heketi_volume["id"])
finally:
# Enable heketi node
heketi_ops.heketi_node_enable(h_client, h_server, node_id)
# Create volume when heketi node is enabled
vol_info = heketi_ops.heketi_volume_create(h_client,
h_server,
vol_size,
json=True)
heketi_ops.heketi_volume_delete(h_client, h_server, vol_info['id'])
def enable_node(self, node_id):
"""
Enable node through heketi-cli.
:param node_id: str node ID
"""
out = heketi_node_enable(self.heketi_client_node,
self.heketi_server_url, node_id)
self.assertNotEqual(out, False, "Failed to enable node of"
" id %s" % node_id)
def enable_node(self, node_id):
"""
Enable node through heketi-cli.
:param node_id: str node ID
"""
out = heketi_node_enable(self.heketi_client_node,
self.heketi_server_url,
node_id)
self.assertNotEqual(out, False,
"Failed to enable node of"
" id %s" % node_id)
def test_dev_path_mapping_heketi_node_delete(self):
"""Validate dev path mapping for heketi node deletion lifecycle"""
h_client, h_url = self.heketi_client_node, self.heketi_server_url
node_ids = heketi_ops.heketi_node_list(h_client, h_url)
self.assertTrue(node_ids, "Failed to get heketi node list")
# Fetch #4th node for the operations
h_disable_node = node_ids[3]
# Fetch bricks on the devices before volume create
h_node_details_before, h_node = self._get_bricks_and_device_details()
# Bricks count on the node before pvc creation
brick_count_before = [count[1] for count in h_node_details_before]
# Create file volume with app pod and verify IO's
# and compare path, UUID, vg_name
pod_name, dc_name, use_percent = self._create_app_pod_and_verify_pvs()
# Check if IO's are running
use_percent_after = self._get_space_use_percent_in_app_pod(pod_name)
self.assertNotEqual(
use_percent, use_percent_after,
"Failed to execute IO's in the app pod {} after respin".format(
pod_name))
# Fetch bricks on the devices after volume create
h_node_details_after, h_node = self._get_bricks_and_device_details()
# Bricks count on the node after pvc creation
brick_count_after = [count[1] for count in h_node_details_after]
self.assertGreater(
sum(brick_count_after), sum(brick_count_before),
"Failed to add bricks on the node {}".format(h_node))
self.addCleanup(heketi_ops.heketi_node_disable, h_client, h_url,
h_disable_node)
# Enable the #4th node
heketi_ops.heketi_node_enable(h_client, h_url, h_disable_node)
node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_disable_node,
json=True)
h_node_id = node_info['id']
self.assertEqual(node_info['state'], "online",
"Failed to enable node {}".format(h_disable_node))
# Disable the node and check for brick migrations
self.addCleanup(heketi_ops.heketi_node_enable,
h_client,
h_url,
h_node,
raise_on_error=False)
heketi_ops.heketi_node_disable(h_client, h_url, h_node)
node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node,
json=True)
self.assertEqual(node_info['state'], "offline",
"Failed to disable node {}".format(h_node))
# Before bricks migration
h_node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node,
json=True)
# Bricks before migration on the node i.e to be deleted
bricks_counts_before = 0
for device in h_node_info['devices']:
bricks_counts_before += (len(device['bricks']))
# Remove the node
heketi_ops.heketi_node_remove(h_client, h_url, h_node)
# After bricks migration
h_node_info_after = heketi_ops.heketi_node_info(h_client,
h_url,
h_node,
json=True)
# Bricks after migration on the node i.e to be delete
bricks_counts = 0
for device in h_node_info_after['devices']:
bricks_counts += (len(device['bricks']))
self.assertFalse(
bricks_counts,
"Failed to remove all the bricks from node {}".format(h_node))
# Old node which is to deleted, new node were bricks resides
old_node, new_node = h_node, h_node_id
# Node info for the new node were brick reside after migration
h_node_info_new = heketi_ops.heketi_node_info(h_client,
h_url,
new_node,
json=True)
bricks_counts_after = 0
for device in h_node_info_new['devices']:
bricks_counts_after += (len(device['bricks']))
self.assertEqual(
bricks_counts_before, bricks_counts_after,
"Failed to migrated bricks from {} node to {}".format(
old_node, new_node))
# Fetch device list i.e to be deleted
h_node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node,
json=True)
devices_list = [[device['id'], device['name']]
for device in h_node_info['devices']]
for device in devices_list:
device_id = device[0]
device_name = device[1]
self.addCleanup(heketi_ops.heketi_device_add,
h_client,
h_url,
device_name,
h_node,
raise_on_error=False)
# Device deletion from heketi node
device_delete = heketi_ops.heketi_device_delete(
h_client, h_url, device_id)
self.assertTrue(device_delete,
"Failed to delete the device {}".format(device_id))
node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node,
json=True)
cluster_id = node_info['cluster']
zone = node_info['zone']
storage_hostname = node_info['hostnames']['manage'][0]
storage_ip = node_info['hostnames']['storage'][0]
# Delete the node
self.addCleanup(heketi_ops.heketi_node_add,
h_client,
h_url,
zone,
cluster_id,
storage_hostname,
storage_ip,
raise_on_error=False)
heketi_ops.heketi_node_delete(h_client, h_url, h_node)
# Verify if the node is deleted
node_ids = heketi_ops.heketi_node_list(h_client, h_url)
self.assertNotIn(old_node, node_ids,
"Failed to delete the node {}".format(old_node))
# Check if IO's are running
use_percent_after = self._get_space_use_percent_in_app_pod(pod_name)
self.assertNotEqual(
use_percent, use_percent_after,
"Failed to execute IO's in the app pod {} after respin".format(
pod_name))
# Adding node back
h_node_info = heketi_ops.heketi_node_add(h_client,
h_url,
zone,
cluster_id,
storage_hostname,
storage_ip,
json=True)
self.assertTrue(
h_node_info,
"Failed to add the node in the cluster {}".format(cluster_id))
h_node_id = h_node_info["id"]
# Adding devices to the new node
for device in devices_list:
storage_device = device[1]
# Add device to the new heketi node
heketi_ops.heketi_device_add(h_client, h_url, storage_device,
h_node_id)
heketi_node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node_id,
json=True)
device_id = None
for device in heketi_node_info["devices"]:
if device["name"] == storage_device:
device_id = device["id"]
break
self.assertTrue(
device_id, "Failed to add device {} on node {}".format(
storage_device, h_node_id))
# Create n pvc in order to verfiy if the bricks reside on the new node
pvc_amount, pvc_size = 5, 1
# Fetch bricks on the devices before volume create
h_node_details_before, h_node = self._get_bricks_and_device_details()
# Bricks count on the node before pvc creation
brick_count_before = [count[1] for count in h_node_details_before]
# Create file volumes
pvc_name = self.create_and_wait_for_pvcs(pvc_size=pvc_size,
pvc_amount=pvc_amount)
self.assertEqual(len(pvc_name), pvc_amount,
"Failed to create {} pvc".format(pvc_amount))
# Fetch bricks on the devices before volume create
h_node_details_after, h_node = self._get_bricks_and_device_details()
# Bricks count on the node after pvc creation
brick_count_after = [count[1] for count in h_node_details_after]
self.assertGreater(
sum(brick_count_after), sum(brick_count_before),
"Failed to add bricks on the new node {}".format(new_node))
# Check if IO's are running after new node is added
use_percent_after = self._get_space_use_percent_in_app_pod(pod_name)
self.assertNotEqual(
use_percent, use_percent_after,
"Failed to execute IO's in the app pod {} after respin".format(
pod_name))
def test_dev_path_mapping_heketi_device_delete(self):
"""Validate dev path mapping for heketi device delete lifecycle"""
h_client, h_url = self.heketi_client_node, self.heketi_server_url
node_ids = heketi_ops.heketi_node_list(h_client, h_url)
self.assertTrue(node_ids, "Failed to get heketi node list")
# Fetch #4th node for the operations
h_disable_node = node_ids[3]
# Fetch bricks on the devices before volume create
h_node_details_before, h_node = self._get_bricks_and_device_details()
# Bricks count on the node before pvc creation
brick_count_before = [count[1] for count in h_node_details_before]
# Create file volume with app pod and verify IO's
# and compare path, UUID, vg_name
pod_name, dc_name, use_percent = self._create_app_pod_and_verify_pvs()
# Check if IO's are running
use_percent_after = self._get_space_use_percent_in_app_pod(pod_name)
self.assertNotEqual(
use_percent, use_percent_after,
"Failed to execute IO's in the app pod {} after respin".format(
pod_name))
# Fetch bricks on the devices after volume create
h_node_details_after, h_node = self._get_bricks_and_device_details()
# Bricks count on the node after pvc creation
brick_count_after = [count[1] for count in h_node_details_after]
self.assertGreater(
sum(brick_count_after), sum(brick_count_before),
"Failed to add bricks on the node {}".format(h_node))
# Enable the #4th node
heketi_ops.heketi_node_enable(h_client, h_url, h_disable_node)
node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_disable_node,
json=True)
h_node_id = node_info['id']
self.assertEqual(node_info['state'], "online",
"Failed to enable node {}".format(h_disable_node))
# Fetch device list i.e to be deleted
h_node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node,
json=True)
devices_list = [[device['id'], device['name']]
for device in h_node_info['devices']]
# Device deletion operation
for device in devices_list:
device_id, device_name = device[0], device[1]
self.addCleanup(heketi_ops.heketi_device_enable,
h_client,
h_url,
device_id,
raise_on_error=False)
# Disable device from heketi
device_disable = heketi_ops.heketi_device_disable(
h_client, h_url, device_id)
self.assertTrue(
device_disable,
"Device {} could not be disabled".format(device_id))
device_info = heketi_ops.heketi_device_info(h_client,
h_url,
device_id,
json=True)
self.assertEqual(device_info['state'], "offline",
"Failed to disable device {}".format(device_id))
# Remove device from heketi
device_remove = heketi_ops.heketi_device_remove(
h_client, h_url, device_id)
self.assertTrue(device_remove,
"Device {} could not be removed".format(device_id))
# Bricks after device removal
device_info = heketi_ops.heketi_device_info(h_client,
h_url,
device_id,
json=True)
bricks_count_after = len(device_info['bricks'])
self.assertFalse(
bricks_count_after,
"Failed to remove the bricks from the device {}".format(
device_id))
# Delete device from heketi
self.addCleanup(heketi_ops.heketi_device_add,
h_client,
h_url,
device_name,
h_node,
raise_on_error=False)
device_delete = heketi_ops.heketi_device_delete(
h_client, h_url, device_id)
self.assertTrue(device_delete,
"Device {} could not be deleted".format(device_id))
# Check if IO's are running after device is deleted
use_percent_after = self._get_space_use_percent_in_app_pod(pod_name)
self.assertNotEqual(
use_percent, use_percent_after,
"Failed to execute IO's in the app pod {} after respin".format(
pod_name))
# Add device operations
for device in devices_list:
device_name = device[1]
# Add device back to the node
heketi_ops.heketi_device_add(h_client, h_url, device_name, h_node)
# Fetch device info after device add
node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node,
json=True)
device_id = None
for device in node_info["devices"]:
if device["name"] == device_name:
device_id = device["id"]
break
self.assertTrue(
device_id, "Failed to add device {} on node"
" {}".format(device_name, h_node))
# Disable the #4th node
heketi_ops.heketi_node_disable(h_client, h_url, h_node_id)
node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node_id,
json=True)
self.assertEqual(node_info['state'], "offline",
"Failed to disable node {}".format(h_node_id))
pvc_amount, pvc_size = 5, 1
# Fetch bricks on the devices before volume create
h_node_details_before, h_node = self._get_bricks_and_device_details()
# Bricks count on the node before pvc creation
brick_count_before = [count[1] for count in h_node_details_before]
# Create file volumes
pvc_name = self.create_and_wait_for_pvcs(pvc_size=pvc_size,
pvc_amount=pvc_amount)
self.assertEqual(len(pvc_name), pvc_amount,
"Failed to create {} pvc".format(pvc_amount))
# Fetch bricks on the devices after volume create
h_node_details_after, h_node = self._get_bricks_and_device_details()
# Bricks count on the node after pvc creation
brick_count_after = [count[1] for count in h_node_details_after]
self.assertGreater(
sum(brick_count_after), sum(brick_count_before),
"Failed to add bricks on the node {}".format(h_node))
# Check if IO's are running after new device is added
use_percent_after = self._get_space_use_percent_in_app_pod(pod_name)
self.assertNotEqual(
use_percent, use_percent_after,
"Failed to execute IO's in the app pod {} after respin".format(
pod_name))
def test_check_node_disable_based_on_heketi_zone(
self, zone_count, is_disable_on_different_zone, is_set_env=False):
"""Validate node disable in different heketi zones"""
expected_node_count, heketi_zone_checking, sc_name = 4, "strict", None
# Check amount of available online nodes
online_node_count = len(self._get_online_nodes())
if online_node_count < expected_node_count:
self.skipTest('Available node count {} is less than expected node '
'count {}'.format(online_node_count,
expected_node_count))
# Check amount of available online heketi zones
self._check_for_available_zones(zone_count)
# Get the online devices and nodes w.r.t. to zone
zone_devices_nodes = self._get_online_devices_and_nodes_with_zone()
# Create sc or else directly set env to "strict" inside dc
is_create_sc = not is_set_env
if is_create_sc:
sc_name = self.create_storage_class(
sc_name_prefix=self.prefix,
vol_name_prefix=self.prefix,
heketi_zone_checking=heketi_zone_checking)
if is_set_env:
self._set_zone_check_env_in_heketi_dc(heketi_zone_checking)
# Choose a zone and node_id to disable the device
for zone, nodes_and_devices in zone_devices_nodes.items():
if zone_count == 3:
# Select a node with a zone having multiple nodes in same
# zone to cover the test cases disable node in same zone
if len(nodes_and_devices['nodes']) > 1:
zone_with_disabled_node = zone
disabled_node = nodes_and_devices['nodes'][0]
break
else:
# Select node from any of the zones
zone_with_disabled_node = zone
disabled_node = nodes_and_devices['nodes'][0]
break
# Disable the selected node
heketi_ops.heketi_node_disable(self.h_client, self.h_server,
disabled_node)
self.addCleanup(heketi_ops.heketi_node_enable, self.h_client,
self.h_server, disabled_node)
# Create some DCs with PVCs and check brick placement in heketi zones
pod_names = self._create_dcs_and_check_brick_placement(
self.prefix, sc_name, heketi_zone_checking, zone_count)
# Enable disabled node
heketi_ops.heketi_node_enable(self.h_client, self.h_server,
disabled_node)
if is_disable_on_different_zone:
# Select the new node in a different zone
for zone, nodes_and_devices in zone_devices_nodes.items():
if zone != zone_with_disabled_node:
new_node_to_disable = nodes_and_devices['nodes'][0]
break
else:
# Select the new node in the same zone
new_node_to_disable = zone_devices_nodes[zone_with_disabled_node][
'nodes'][1]
# Disable the newly selected node
heketi_ops.heketi_node_disable(self.h_client, self.h_server,
new_node_to_disable)
self.addCleanup(heketi_ops.heketi_node_enable, self.h_client,
self.h_server, new_node_to_disable)
# Verify if pods are in ready state
for pod_name in pod_names:
openshift_ops.wait_for_pod_be_ready(self.node,
pod_name,
timeout=5,
wait_step=2)