def tearDown(self):
# Starting glusterd on node where stopped.
ret = start_glusterd(self.servers[self.random_server])
if ret:
ExecutionError("Failed to start glusterd.")
g.log.info("Successfully started glusterd.")
ret = wait_for_glusterd_to_start(self.servers)
if not ret:
ExecutionError("glusterd is not running on %s" % self.servers)
g.log.info("Glusterd start on the nodes succeeded")
# Checking if peer is connected.
ret = wait_for_peers_to_connect(self.mnode, self.servers)
if not ret:
ExecutionError("Peer is not in connected state.")
g.log.info("Peers is in connected state.")
# Stopping and deleting volume.
ret = cleanup_volume(self.mnode, self.volname)
if not ret:
raise ExecutionError("Unable to delete volume % s" % self.volname)
g.log.info("Volume deleted successfully : %s", self.volname)
self.get_super_method(self, 'tearDown')()
def tearDown(self):
# start the volume, it should succeed
ret, _, _ = volume_start(self.mnode, self.volname)
self.assertEqual(ret, 0, "Volume stop failed")
# start glusterd on all servers
ret = start_glusterd(self.servers)
if not ret:
raise ExecutionError("Failed to start glusterd on all servers")
for server in self.servers:
ret = wait_for_peers_to_connect(server, self.servers)
if not ret:
ret = peer_probe_servers(server, self.servers)
if not ret:
raise ExecutionError("Failed to peer probe all "
"the servers")
# clean up all volumes
vol_list = get_volume_list(self.mnode)
if vol_list is None:
raise ExecutionError("Failed to get the volume list")
for volume in vol_list:
ret = cleanup_volume(self.mnode, volume)
if not ret:
raise ExecutionError("Unable to delete volume % s" % volume)
g.log.info("Volume deleted successfully : %s", volume)
self.get_super_method(self, 'tearDown')()
def tearDown(self):
# Check and start if glusterd isn't running.
if not self.validate_peers_are_connected():
# Starting glusterd on node where stopped.
ret = start_glusterd(self.node_on_glusterd_to_stop)
if ret:
raise ExecutionError("Failed to start glusterd.")
g.log.info("Successfully started glusterd.")
# Checking if peer is connected.
ret = wait_for_peers_to_connect(self.mnode, self.servers)
self.assertTrue(
ret, "glusterd is not connected %s with peer %s" %
(self.mnode, self.servers))
g.log.info("Peers is in connected state.")
# stopping the volume and Cleaning up the volume
ret = self.cleanup_volume()
if not ret:
raise ExecutionError("Failed Cleanup the Volume %s" % self.volname)
g.log.info("Volume deleted successfully : %s", self.volname)
# Calling GlusterBaseClass tearDown
self.get_super_method(self, 'tearDown')()
def tearDown(self):
ret = is_glusterd_running(self.servers)
if ret:
ret = start_glusterd(self.servers)
if not ret:
raise ExecutionError("Failed to start glusterd on %s" %
self.servers)
g.log.info("Glusterd started successfully on %s", self.servers)
# Checking for peer status from every node
for server in self.servers:
ret = wait_for_peers_to_connect(server, self.servers)
if not ret:
raise ExecutionError("Servers are not in peer probed state")
ret = cleanup_volume(self.mnode, self.volname_2)
if not ret:
raise ExecutionError("Unable to delete volume % s" %
self.volname_2)
# Unmount and cleanup original volume
ret = self.unmount_volume_and_cleanup_volume(mounts=self.mounts)
if not ret:
raise ExecutionError("Failed to umount the vol & cleanup Volume")
# Calling GlusterBaseClass tearDown
self.get_super_method(self, 'tearDown')()
def tearDown(self):
"""
tearDown for every test
Clean up and unmount the volume
"""
# calling GlusterBaseClass tearDown
self.get_super_method(self, 'tearDown')()
# Delete the glusterfind sessions
ret, _, _ = gfind_delete(self.mnode, self.volname, self.session)
if ret:
raise ExecutionError("Failed to delete session %s" % self.session)
g.log.info("Successfully deleted session %s", self.session)
# Remove the outfiles created during 'glusterfind pre'
for out in self.outfiles:
ret = remove_file(self.mnode, out, force=True)
if not ret:
raise ExecutionError("Failed to remove the outfile %s" % out)
g.log.info("Successfully removed the outfiles")
# Wait for the peers to be connected.
ret = wait_for_peers_to_connect(self.mnode, self.servers, 100)
if not ret:
raise ExecutionError("Peers are not in connected state.")
# Cleanup the volume
ret = self.unmount_volume_and_cleanup_volume(mounts=self.mounts)
if not ret:
raise ExecutionError("Failed to Cleanup Volume")
g.log.info("Successful in Cleanup Volume")
def tearDown(self):
"""
tearDown for every test
"""
ret = is_glusterd_running(self.random_server)
if ret:
ret = start_glusterd(self.random_server)
if not ret:
raise ExecutionError("Failed to start glusterd on %s"
% self.random_server)
# checking for peer status from every node
ret = wait_for_peers_to_connect(self.mnode, self.servers)
self.assertTrue(ret, "glusterd is not connected %s with peer %s"
% (self.mnode, self.servers))
# stopping the volume and Cleaning up the volume
ret = self.cleanup_volume()
if not ret:
raise ExecutionError("Failed Cleanup the Volume %s"
% self.volname)
# Calling GlusterBaseClass tearDown
self.get_super_method(self, 'tearDown')()
def test_snap_info_glusterd_restart(self):
"""
Verify snapshot info before and after glusterd restart
* Create multiple snapshots
* Check snapshot info
- Without using snapname or volname
- Using snapname
- Using volname
* Restart glusterd on all servers
* Repeat the snapshot info step for all the three scenarios
mentioned above
"""
# pylint: disable=too-many-statements
# Create snapshots with description
for snap in self.snapshots:
ret, _, _ = snap_create(self.mnode,
self.volname,
snap,
description='$p3C!@l C#@R@cT#R$')
self.assertEqual(
ret, 0,
("Failed to create snapshot for volume %s" % self.volname))
g.log.info("Snapshot %s created successfully for volume %s", snap,
self.volname)
# Perform the snapshot info tests before glusterd restart
self.snapshot_info()
# Restart Glusterd on all servers
for server in self.servers:
ret = restart_glusterd(server)
self.assertTrue(ret,
("Failed to restart glusterd on node %s" % server))
g.log.info("Successfully restarted glusterd on node %s", server)
# Wait for glusterd to be online and validate glusterd running on all
# server nodes
self.assertTrue(
wait_for_glusterd_to_start(self.servers),
"Unexpected: glusterd not up on one or more of the nodes")
g.log.info("Glusterd is up and running on all nodes")
# Check if peers are connected
self.assertTrue(wait_for_peers_to_connect(self.mnode, self.servers),
"Unexpected: Peers are not in connected state")
g.log.info("Successful: All peers are in connected state")
# perform the snapshot info tests after glusterd restart
self.snapshot_info()
def tearDown(self):
""" tear Down Callback """
# Wait for peers to connect
ret = wait_for_peers_to_connect(self.mnode, self.servers, 50)
if not ret:
raise ExecutionError("Peers are not in connected state.")
# Cleanup the volume
ret = cleanup_volume(self.mnode, self.volname)
if not ret:
raise ExecutionError("Failed to cleanup volume")
g.log.info("Successfully cleaned up the volume")
# Calling GlusterBaseClass tearDown
self.get_super_method(self, 'tearDown')()
def test_daemons_after_reboot(self):
'''
Creating volume then performing FUSE mount
then enable quota to that volume, then set quota
limit to that volume then perform a reboot and check
the selfheal daemon and quota daemon running or not
after reboot
'''
# Enabling quota to volume
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(
ret, 0, "Failed to enable quota on volume : "
"%s" % self.volname)
g.log.info("quota enabled successfully on volume: %s", self.volname)
# Setting quota limit to volume
ret, _, _ = quota_limit_usage(self.mnode,
self.volname,
path='/',
limit='1GB',
soft_limit='')
self.assertEqual(
ret, 0, "Quota limit set failed "
"on volume : %s" % self.volname)
ret, _ = reboot_nodes_and_wait_to_come_online(self.servers[1])
self.assertTrue(ret, "Failed to reboot the node %s" % self.servers[1])
g.log.info("Node %s rebooted successfully", self.servers[1])
# Checking glusterd status and peer status afte reboot of server
self.assertTrue(wait_for_glusterd_to_start(self.servers[1]),
"Failed to start glusterd on %s" % self.servers[1])
self.assertTrue(wait_for_peers_to_connect(self.mnode, self.servers),
"some peers are not in connected state")
g.log.info("glusterd is running and all peers are in "
"connected state")
# Checks self heal daemon and quota daemon process running or not
ret = self.is_daemon_process_running()
self.assertTrue(
ret, "failed to run self-heal and quota daemon "
"processs on all hosts")
g.log.info("self-heal and quota daemons are running on all "
"hosts successfully")
def tearDown(self):
ret = is_glusterd_running(self.servers)
if ret:
ret = start_glusterd(self.servers)
if not ret:
raise ExecutionError("Failed to start glusterd on %s" %
self.servers)
g.log.info("Glusterd started successfully on %s", self.servers)
# checking for peer status from every node
ret = wait_for_peers_to_connect(self.mnode, self.servers)
if not ret:
raise ExecutionError("Peers are not in connected state")
# stopping the volume and Cleaning up the volume
ret = self.unmount_volume_and_cleanup_volume(self.mounts)
if not ret:
raise ExecutionError("Failed to Cleanup the Volume %s" %
self.volname)
g.log.info("Volume deleted successfully %s", self.volname)
def _wait_for_gluster_process_online_state(self):
"""
Function which waits for the glusterfs processes to come up
"""
# Wait for glusterd to be online and validate it's running.
self.assertTrue(wait_for_glusterd_to_start(self.servers),
"glusterd not up on the desired nodes.")
g.log.info("Glusterd is up and running on desired nodes.")
# Wait for peers to connect
ret = wait_for_peers_to_connect(self.mnode, self.servers, 50)
self.assertTrue(ret, "Peers not in connected state.")
g.log.info("Peers in connected state.")
# Wait for all volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode,
self.volname,
timeout=600)
self.assertTrue(ret, ("All volume processes not up."))
g.log.info("All volume processes are up.")
def tearDown(self):
"""tear Down Callback"""
# Wait for peers to connect.
ret = wait_for_peers_to_connect(self.mnode, self.servers, 50)
if not ret:
raise ExecutionError("Peers are not in connected state.")
# Unmount volume and cleanup.
ret = self.cleanup_volume()
if not ret:
raise ExecutionError("Failed to Unmount and Cleanup volume")
g.log.info("Successful in unmount and cleanup operations")
# Reset the cluster options.
ret = volume_reset(self.mnode, "all")
if not ret:
raise ExecutionError("Failed to Reset the cluster options.")
g.log.info("Successfully reset cluster options.")
# Calling GlusterBaseClass tearDown
self.get_super_method(self, 'tearDown')()
def tearDown(self):
# Starting glusterd on node where stopped.
ret = start_glusterd(self.servers[self.random_server])
if ret:
ExecutionError("Failed to start glusterd.")
g.log.info("Successfully started glusterd.")
# Checking if peer is connected
for server in self.servers:
ret = wait_for_peers_to_connect(self.mnode, server)
if not ret:
ExecutionError("Peers are not in connected state.")
g.log.info("Peers are in connected state.")
# Unmounting and cleaning volume.
ret = self.unmount_volume_and_cleanup_volume(self.mounts)
if not ret:
raise ExecutionError("Unable to delete volume % s" % self.volname)
g.log.info("Volume deleted successfully : %s", self.volname)
self.get_super_method(self, 'tearDown')()
def tearDown(self):
ret = is_glusterd_running(self.servers)
if ret:
ret = start_glusterd(self.servers)
if not ret:
raise ExecutionError("Failed to start glusterd on %s"
% self.servers)
g.log.info("Glusterd started successfully on %s", self.servers)
ret = wait_for_peers_to_connect(self.mnode, self.servers)
self.assertTrue(ret, "glusterd is not connected %s with peer %s"
% (self.servers, self.servers))
# stopping the volume and Cleaning up the volume
ret = self.cleanup_volume()
if not ret:
raise ExecutionError("Failed to Cleanup the Volume %s"
% self.volname)
g.log.info("Volume deleted successfully : %s", self.volname)
self.get_super_method(self, 'tearDown')()
def test_root_squash_enable(self):
"""
Tests to verify Nfs Ganesha rootsquash functionality when glusterd
service is restarted
Steps:
1. Create some files and dirs inside mount point
2. Set permission as 777 for mount point
3. Enable root-squash on volume
4. Create some more files and dirs
5. Restart glusterd on all the nodes
6. Try to edit file created in step 1
It should not allow to edit the file
7. Try to edit the file created in step 5
It should allow to edit the file
"""
# Start IO on mount point.
cmd = ("for i in {1..10}; do touch %s/file$i; done"
% self.mounts[0].mountpoint)
ret, _, err = g.run(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
self.assertEqual(ret, 0, err)
# Get stat of all the files/dirs created.
ret = get_mounts_stat(self.mounts)
self.assertTrue(ret, "Stat failed on some of the clients")
g.log.info("Successful in getting stats of files/dirs "
"from mount point")
# Set mount point permission to 777
ret = set_file_permissions(self.mounts[0].client_system,
self.mounts[0].mountpoint, 777)
self.assertTrue(ret, "Failed to set permission for directory")
g.log.info("Successfully set permissions for directory")
# Enable root-squash on volume
ret = set_root_squash(self.servers[0], self.volname)
self.assertTrue(ret, "Failed to enable root-squash on volume")
g.log.info("root-squash is enable on the volume")
# Start IO on mount point.
cmd = ("for i in {1..10}; do touch %s/Squashfile$i; done"
% self.mounts[0].mountpoint)
ret, _, err = g.run(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
self.assertEqual(ret, 0, err)
# Get stat of all the files/dirs created.
ret = get_mounts_stat(self.mounts)
self.assertTrue(ret, "Stat failed on some of the clients")
g.log.info("Successfull in getting stats of files/dirs "
"from mount point")
# Restart glusterd on all servers
ret = restart_glusterd(self.servers)
self.assertTrue(ret, ("Failed to restart glusterd on all servers %s",
self.servers))
g.log.info("Successfully restarted glusterd on all servers %s",
self.servers)
# Check if glusterd is running on all servers
ret = is_glusterd_running(self.servers)
self.assertEqual(ret, 0, ("Failed:Glusterd is not running on all "
"servers %s",
self.servers))
g.log.info("Glusterd is running on all the servers %s", self.servers)
# Checking if peer is connected.
ret = wait_for_peers_to_connect(self.mnode, self.servers)
self.assertTrue(ret, "Failed:Peer is not in connected state.")
g.log.info("Peers are in connected state.")
# Edit file created by root user
for mount_obj in self.mounts:
ret = append_string_to_file(mount_obj.client_system,
"%s/file10" % mount_obj.mountpoint,
'hello')
self.assertFalse(ret, "Unexpected:nfsnobody user editing file "
"created by root user should FAIL")
g.log.info("Successful:nfsnobody user failed to edit file "
"created by root user")
# Edit the file created by nfsnobody user
for mount_obj in self.mounts:
ret = append_string_to_file(mount_obj.client_system,
"%s/Squashfile5"
% mount_obj.mountpoint,
'hello')
self.assertTrue(ret, "Unexpected:nfsnobody user failed to edit "
"the file created by nfsnobody user")
g.log.info("Successful:nfsnobody user successfully edited the "
"file created by nfsnobody user")
def test_reserved_port_range_for_gluster(self):
"""
Test Case:
1) Set the max-port option in glusterd.vol file to 49200
2) Restart glusterd on one of the node
3) Create 50 volumes in a loop
4) Try to start the 50 volumes in a loop
5) Confirm that the 50th volume failed to start
6) Confirm the error message, due to which volume failed to start
7) Set the max-port option in glusterd.vol file back to default value
8) Restart glusterd on the same node
9) Starting the 50th volume should succeed now
"""
# Set max port number as 49200 in glusterd.vol file
cmd = "sed -i 's/60999/49200/' /etc/glusterfs/glusterd.vol"
ret, _, _ = g.run(self.mnode, cmd)
self.assertEqual(
ret, 0, "Failed to set the max-port to 49200 in"
" glusterd.vol file")
self.port_range_changed = True
# Restart glusterd
ret = restart_glusterd(self.mnode)
self.assertTrue(ret, "Failed to restart glusterd")
g.log.info("Successfully restarted glusterd on node: %s", self.mnode)
# Check node on which glusterd was restarted is back to 'Connected'
# state from any other peer
ret = wait_for_peers_to_connect(self.servers[1], self.servers)
self.assertTrue(ret, "All the peers are not in connected state")
# Fetch the available bricks dict
bricks_dict = get_servers_bricks_dict(self.servers,
self.all_servers_info)
self.assertIsNotNone(bricks_dict, "Failed to get the bricks dict")
# Create 50 volumes in a loop
for i in range(1, 51):
self.volname = "volume-%d" % i
bricks_list = []
j = 0
for key, value in bricks_dict.items():
j += 1
brick = choice(value)
brick = "{}:{}/{}_brick-{}".format(key, brick, self.volname, j)
bricks_list.append(brick)
ret, _, _ = volume_create(self.mnode, self.volname, bricks_list)
self.assertEqual(ret, 0,
"Failed to create volume: %s" % self.volname)
g.log.info("Successfully created volume: %s", self.volname)
# Try to start 50 volumes in loop
for i in range(1, 51):
self.volname = "volume-%d" % i
ret, _, err = volume_start(self.mnode, self.volname)
if ret:
break
g.log.info("Successfully started all the volumes until volume: %s",
self.volname)
# Confirm if the 50th volume failed to start
self.assertEqual(
i, 50, "Failed to start the volumes volume-1 to"
" volume-49 in a loop")
# Confirm the error message on volume start fail
err_msg = ("volume start: volume-50: failed: Commit failed on"
" localhost. Please check log file for details.")
self.assertEqual(
err.strip(), err_msg, "Volume start failed with"
" a different error message")
# Confirm the error message from the log file
cmd = ("cat /var/log/glusterfs/glusterd.log | %s" %
"grep -i 'All the ports in the range are exhausted' | wc -l")
ret, out, _ = g.run(self.mnode, cmd)
self.assertEqual(ret, 0, "Failed to 'grep' the glusterd.log file")
self.assertNotEqual(
out, "0", "Volume start didn't fail with expected"
" error message")
# Set max port number back to default value in glusterd.vol file
cmd = "sed -i 's/49200/60999/' /etc/glusterfs/glusterd.vol"
ret, _, _ = g.run(self.mnode, cmd)
self.assertEqual(
ret, 0, "Failed to set the max-port back to 60999 in"
" glusterd.vol file")
self.port_range_changed = False
# Restart glusterd on the same node
ret = restart_glusterd(self.mnode)
self.assertTrue(ret, "Failed to restart glusterd")
g.log.info("Successfully restarted glusterd on node: %s", self.mnode)
# Starting the 50th volume should succeed now
self.volname = "volume-%d" % i
ret, _, _ = volume_start(self.mnode, self.volname)
self.assertEqual(ret, 0, "Failed to start volume: %s" % self.volname)
def test_gluster_operation_after_removing_firewall(self):
"""
Test steps:
1. Add firewall services to the zones on 2 nodes
2. Create a cluster using the 2 nodes
3. Check peer status on both the nodes
4. Remove firewall services from both the nodes
5. Check peer status on both the nodes
6. Create a distribute volume using both the node bricks and start it
7. Mount the volume on different node, it should fail
8. Cleanup the volume, Detach the node and try to probe again
9. Check peer status
10. Remove firewall services permanently and reload firewall
11. Check peer status
12. Create a distribute volume using both the node bricks and start it
13. Mount the volume on different node, it should fail
"""
# pylint: disable=too-many-statements
# Add firewall services on first 2 nodes
ret = self._add_firewall_services(self.servers[:2])
self.assertTrue(ret, "Failed to add services to firewall")
self.firewall_added = True
# Peer probe second node
self._probe_peer(self.servers[1])
# Check peer status on both the nodes
ret = wait_for_peers_to_connect(self.mnode, self.servers[:2])
self.assertTrue(ret, "Peer is not connected")
# Remove firewall services
self._remove_firewall_services(self.servers[:2])
self.firewall_added = False
# Create a volume
self._create_distribute_volume("distribute_volume")
# Start the volume
self._start_the_volume(self.volname)
# Mount the volume on a different node, it should fail
self._try_mounting_volume()
# Cleanup volume before peer detach
ret = cleanup_volume(self.mnode, self.volname)
self.assertTrue(ret, "Failed to cleanup volume")
# Detach the probed node
ret, _, _ = peer_detach(self.mnode, self.servers[1])
self.assertEqual(ret, 0, "Failed to detach node: %s" % self.servers[1])
# Peer probe the node should fail
self._probe_peer(self.servers[1], True)
# Add firewall services permanently
ret = self._add_firewall_services(self.servers[:2])
self.assertTrue(ret, "Failed to add services to firewall")
self.firewall_added = True
# Reload firewall
ret = self._reload_firewall_service(self.servers[:2])
self.assertTrue(ret, "Failed to reload firewall service")
# Peer probe again
self._probe_peer(self.servers[1])
# Check peer status the probed node
ret = wait_for_peers_to_connect(self.mnode, self.servers[1])
self.assertTrue(ret, "Peer is not connected")
# Remove firewall services permanently
self._remove_firewall_services(self.servers[:2])
self.firewall_added = False
# Reload firewall
ret = self._reload_firewall_service(self.servers[:2])
self.assertTrue(ret, "Failed to reload firewall service")
# Check peer status
ret = is_peer_connected(self.mnode, self.servers[1])
self.assertTrue(ret, "Peer is not connected")
# Create a volume
self._create_distribute_volume("distribute_volume_2")
# Start the volume
self._start_the_volume(self.volname)
# Mount the volume on a different node, it should fail
self._try_mounting_volume()
def test_remove_brick_status(self):
'''
-> Create volume
-> Enable server quorum on volume
-> Stop glusterd on all nodes except first node
-> Verify brick status of nodes where glusterd is running with
default quorum ratio(51%)
-> Change the cluster.server-quorum-ratio from default to 95%
-> Start glusterd on all servers except last node
-> Verify the brick status again
'''
# Enabling server quorum
ret = set_volume_options(self.mnode, self.volname,
{'cluster.server-quorum-type': 'server'})
self.assertTrue(
ret, "Failed to set server quorum on volume %s" % self.volname)
g.log.info("Able to set server quorum on volume %s successfully",
self.volname)
# Getting brick list
brick_list = get_all_bricks(self.mnode, self.volname)
# Stopping glusterd on remaining servers except first node
ret = stop_glusterd(self.servers[1:])
self.assertTrue(
ret, "Failed to stop gluterd on some of the servers "
"%s" % self.servers[1:])
g.log.info("Glusterd stopped successfully on servers %s",
self.servers[1:])
# Checking brick status for glusterd running nodes with
# default quorum ratio(51%)
ret = are_bricks_offline(self.mnode, self.volname, brick_list[0:1])
self.assertTrue(
ret, "Bricks are online when quorum is in not "
"met condition for %s" % self.volname)
g.log.info(
"Bricks are offline when quorum is in not met "
"condition for %s", self.volname)
# Setting quorum ratio to 95%
ret = set_volume_options(self.mnode, 'all',
{'cluster.server-quorum-ratio': '95%'})
self.assertTrue(
ret, "Failed to set quorum ratio to 95 percentage on "
"servers %s" % self.servers)
g.log.info(
"Able to set server quorum ratio to 95 percentage "
"on servers %s", self.servers)
# Starting glusterd on remaining servers except last node
ret = start_glusterd(self.servers[1:5])
self.assertTrue(
ret, "Failed to start glusterd on some of the servers"
" %s" % self.servers[1:5])
g.log.info(
"Glusterd started successfully on all servers except "
"last node %s", self.servers[1:5])
self.assertTrue(
wait_for_peers_to_connect(self.mnode, self.servers[1:5]),
"Peers are not in connected state")
# Verfiying node count in volume status after glusterd
# started on servers, Its not possible to check the brick status
# immediately after glusterd start, that's why verifying that all
# glusterd started nodes available in gluster volume status or not
count = 0
while count < 200:
vol_status = get_volume_status(self.mnode, self.volname)
servers_count = len(vol_status[self.volname].keys())
if servers_count == 5:
break
sleep(2)
count += 1
# Checking brick status with quorum ratio(95%)
ret = are_bricks_offline(self.mnode, self.volname, brick_list[0:5])
self.assertTrue(
ret, "Bricks are online when quorum is in not "
"met condition for %s" % self.volname)
g.log.info(
"Bricks are offline when quorum is in not met "
"condition for %s", self.volname)
def test_brick_port(self):
# pylint: disable=too-many-statements, too-many-branches
"""
In this test case:
1. Trusted storage Pool of 2 nodes
2. Create a distributed volumes with 2 bricks
3. Start the volume
4. Stop glusterd on one node 2
5. Modify any of the volume option on node 1
6. Start glusterd on node 2
7. Check volume status, brick should get port
"""
my_server_info = {
self.servers[0]: self.all_servers_info[self.servers[0]]
}
my_servers = self.servers[0:2]
index = 1
ret, _, _ = peer_probe(self.servers[0], self.servers[index])
self.assertEqual(ret, 0, ("peer probe from %s to %s is failed",
self.servers[0], self.servers[index]))
g.log.info("peer probe is success from %s to "
"%s", self.servers[0], self.servers[index])
key = self.servers[index]
my_server_info[key] = self.all_servers_info[key]
self.volname = "testvol"
bricks_list = form_bricks_list(self.mnode, self.volname, 2,
my_servers,
my_server_info)
g.log.info("Creating a volume %s ", self.volname)
ret = volume_create(self.mnode, self.volname,
bricks_list, force=False)
self.assertEqual(ret[0], 0, ("Unable"
"to create volume %s" % self.volname))
g.log.info("Volume created successfully %s", self.volname)
ret, _, _ = volume_start(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to start the "
"volume %s", self.volname))
g.log.info("Get all the bricks of the volume")
bricks_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(bricks_list, "Failed to get the brick list")
g.log.info("Successfully got the list of bricks of volume")
vol_status = get_volume_status(self.mnode, self.volname)
self.assertIsNotNone(vol_status, "Failed to get volume "
"status for %s" % self.volname)
totport = 0
for _, value in vol_status.items():
for _, val in value.items():
for _, value1 in val.items():
if int(value1["port"]) > 0:
totport += 1
self.assertEqual(totport, 2, ("Volume %s is not started successfully"
"because no. of brick port is not equal"
" to 2", self.volname))
ret = stop_glusterd(self.servers[1])
self.assertTrue(ret, "Failed to stop glusterd on one of the node")
ret = wait_for_glusterd_to_start(self.servers[1])
self.assertFalse(ret, "glusterd is still running on %s"
% self.servers[1])
g.log.info("Glusterd stop on the nodes : %s "
"succeeded", self.servers[1])
option = {'performance.readdir-ahead': 'on'}
ret = set_volume_options(self.servers[0], self.volname, option)
self.assertTrue(ret, "gluster volume set %s performance.readdir-ahead"
"on is failed on server %s"
% (self.volname, self.servers[0]))
g.log.info("gluster volume set %s performance.readdir-ahead on"
"successfully on :%s", self.volname, self.servers[0])
ret = start_glusterd(self.servers[1])
self.assertTrue(ret, "Failed to start glusterd on one of the node")
g.log.info("Glusterd start on the nodes : %s "
"succeeded", self.servers[1])
ret = wait_for_glusterd_to_start(self.servers[1])
self.assertTrue(ret, "glusterd is not running on %s"
% self.servers[1])
g.log.info("Glusterd start on the nodes : %s "
"succeeded", self.servers[1])
ret = wait_for_peers_to_connect(self.servers[0], self.servers[1])
self.assertTrue(ret, "glusterd is not connected %s with peer %s"
% (self.servers[0], self.servers[1]))
vol_status = get_volume_status(self.mnode, self.volname)
self.assertIsNotNone(vol_status, "Failed to get volume "
"status for %s" % self.volname)
totport = 0
for _, value in vol_status.items():
for _, val in value.items():
for _, value1 in val.items():
if int(value1["port"]) > 0:
totport += 1
self.assertEqual(totport, 2, ("Volume %s is not started successfully"
"because no. of brick port is not equal"
" to 2", self.volname))
def test_profile_operations_with_one_node_down(self):
# pylint: disable=too-many-statements
"""
Test Case:
1) Create a volume and start it.
2) Mount volume on client and start IO.
3) Start profile info on the volume.
4) Stop glusterd on one node.
5) Run profile info with different parameters
and see if all bricks are present or not.
6) Stop profile on the volume.
"""
# Start IO on mount points.
g.log.info("Starting IO on all mounts...")
self.all_mounts_procs = []
counter = 1
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("/usr/bin/env python %s create_deep_dirs_with_files "
"--dir-depth 4 "
"--dirname-start-num %d "
"--dir-length 6 "
"--max-num-of-dirs 3 "
"--num-of-files 5 %s" % (
self.script_upload_path,
counter, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
counter += 1
# Start profile on volume.
ret, _, _ = profile_start(self.mnode, self.volname)
self.assertEqual(ret, 0, "Failed to start profile on volume: %s"
% self.volname)
g.log.info("Successfully started profile on volume: %s",
self.volname)
# Fetching a random server from list.
self.random_server = randint(1, len(self.servers)-1)
# Stopping glusterd on one node.
ret = stop_glusterd(self.servers[self.random_server])
self.assertTrue(ret, "Failed to stop glusterd on one node.")
g.log.info("Successfully stopped glusterd on one node.")
ret = wait_for_glusterd_to_start(self.servers[self.random_server])
self.assertFalse(ret, "glusterd is still running on %s"
% self.servers[self.random_server])
g.log.info("Glusterd stop on the nodes : %s "
"succeeded", self.servers[self.random_server])
# Getting and checking output of profile info.
ret, out, _ = profile_info(self.mnode, self.volname)
self.assertEqual(ret, 0, "Failed to run profile info on volume: %s"
% self.volname)
g.log.info("Successfully executed profile info on volume: %s",
self.volname)
# Checking if all bricks are present in profile info.
brick_list = get_online_bricks_list(self.mnode, self.volname)
for brick in brick_list:
self.assertTrue(brick in out,
"Brick %s not a part of profile info output."
% brick)
g.log.info("Brick %s showing in profile info output.",
brick)
# Running profile info with different profile options.
profile_options = ['peek', 'incremental', 'clear', 'incremental peek',
'cumulative']
for option in profile_options:
# Getting and checking output of profile info.
ret, out, _ = profile_info(self.mnode, self.volname,
options=option)
self.assertEqual(ret, 0,
"Failed to run profile info %s on volume: %s"
% (option, self.volname))
g.log.info("Successfully executed profile info %s on volume: %s",
option, self.volname)
# Checking if all bricks are present in profile info peek.
for brick in brick_list:
self.assertTrue(brick in out,
"Brick %s not a part of profile"
" info %s output."
% (brick, option))
g.log.info("Brick %s showing in profile info %s output.",
brick, option)
# Starting glusterd on node where stopped.
ret = start_glusterd(self.servers[self.random_server])
self.assertTrue(ret, "Failed to start glusterd.")
g.log.info("Successfully started glusterd.")
# Checking if peer is connected
ret = wait_for_peers_to_connect(self.mnode, self.servers)
self.assertTrue(ret, "Peers are not in connected state.")
g.log.info("Peers are in connected state.")
# Stop profile on volume.
ret, _, _ = profile_stop(self.mnode, self.volname)
self.assertEqual(ret, 0, "Failed to stop profile on volume: %s"
% self.volname)
g.log.info("Successfully stopped profile on volume: %s", self.volname)
# Validate IO
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients"
)
g.log.info("IO validation complete.")
def test_snap_glusterd_down(self):
# pylint: disable=too-many-statements
"""
Steps:
1. create a volume
2. mount volume
3. create snapshot of that volume
4. validate using snapshot info
5. Activate snapshot
6. List all snapshots present
7. validate using snapshot info
8. Stop glusterd on one node
9. Check glusterd status
10. deactivate created snapshot
11. Start glusterd on that node
12. Check glusterd status
13. validate using snapshot info
13. Check all peers are connected
"""
# Creating snapshot:
g.log.info("Starting to Create snapshot")
ret, _, _ = snap_create(self.mnode, self.volname, self.snap)
self.assertEqual(ret, 0,
("Failed to create snapshot %s for volume %s" %
(self.snap, self.volname)))
g.log.info("Snapshot %s created successfully "
"for volume %s", self.snap, self.volname)
# Check snapshot info
g.log.info("Checking snapshot info")
snap_info = get_snap_info_by_snapname(self.mnode, self.snap)
self.assertIsNotNone(
snap_info, "Failed to get snap information"
"for snapshot %s" % self.snap)
status = snap_info['snapVolume']['status']
self.assertNotEqual(status, 'Started', "snapshot %s "
"not started" % self.snap)
g.log.info("Successfully checked snapshot info")
# Activating snapshot
g.log.info("Starting to Activate Snapshot")
ret, _, _ = snap_activate(self.mnode, self.snap)
self.assertEqual(ret, 0,
("Failed to Activate snapshot %s" % self.snap))
g.log.info("Snapshot %s activated successfully", self.snap)
# snapshot list
g.log.info("Starting to validate list of snapshots")
snap_list1 = get_snap_list(self.mnode)
self.assertIsNotNone(snap_list1, "Failed to list all the snapshot")
self.assertEqual(len(snap_list1), 1, "Failed to validate snap list")
g.log.info("Snapshot list successfully validated")
# Check snapshot info
g.log.info("Checking snapshot info")
snap_info = get_snap_info_by_snapname(self.mnode, self.snap)
status = snap_info['snapVolume']['status']
self.assertEqual(status, 'Started', "Failed to" "start snapshot info")
g.log.info("Successfully checked snapshot info")
# Stop Glusterd on one node
g.log.info("Stopping Glusterd on one node")
ret = stop_glusterd(self.servers[1])
# Check Glusterd status
g.log.info("Check glusterd running or not")
self.assertFalse(wait_for_glusterd_to_start(self.servers[1]),
"glusterd is still running on %s" % self.servers[1])
g.log.info("Expected: Glusterd not running on node %s",
self.servers[1])
# de-activating snapshot
g.log.info("Starting to de-activate Snapshot")
ret, _, _ = snap_deactivate(self.mnode, self.snap)
self.assertEqual(ret, 0,
("Failed to deactivate snapshot %s" % self.snap))
g.log.info("Snapshot %s deactivated successfully", self.snap)
# validate snapshot info
g.log.info("Checking snapshot info")
snap_info = get_snap_info_by_snapname(self.mnode, self.snap)
status = snap_info['snapVolume']['status']
self.assertNotEqual(status, 'Started', "snapshot %s "
"not started" % self.snap)
g.log.info("Successfully validated snapshot info")
# Start Glusterd on node
g.log.info("Starting Glusterd on node %s", self.servers[1])
ret = start_glusterd(self.servers[1])
self.assertTrue(
ret, "Failed to start glusterd on %s node" % self.servers[1])
g.log.info("Successfully started glusterd on "
"%s node", self.servers[1])
# Check Glusterd status
g.log.info("Check glusterd running or not")
self.assertTrue(wait_for_glusterd_to_start(self.servers[1]),
"glusterd is still running on %s" % self.servers[1])
g.log.info("glusterd is running on %s node", self.servers[1])
# validate snapshot info
g.log.info("Checking snapshot info")
snap_info = get_snap_info_by_snapname(self.mnode, self.snap)
self.assertIsNotNone(
snap_info, "Failed to get snap info for"
" snapshot %s" % self.snap)
status = snap_info['snapVolume']['status']
self.assertNotEqual(
status, 'Started', "snapshot"
" %s failed to validate with snap info" % self.snap)
g.log.info("Successfully validated snapshot info")
# Check all the peers are in connected state
g.log.info("Validating all the peers are in connected state")
self.assertTrue(wait_for_peers_to_connect(self.mnode, self.servers),
"glusterd is still running on %s" % self.servers)
g.log.info("Successfully validated all the peers")