def test_glusterd_services(self):
"""Test restart, stop, start of glusterd
"""
# restart glusterd on all servers
g.log.info("Restart glusterd on all servers %s", self.servers)
ret = restart_glusterd(self.servers)
self.assertTrue(
ret,
("Failed to restart glusterd on all servers %s", self.servers))
g.log.info("Successfully restarted glusterd on all servers %s",
self.servers)
# Check if glusterd is running on all servers(expected: active)
g.log.info(
"Check if glusterd is running on all servers %s"
"(expected: active)", self.servers)
ret = is_glusterd_running(self.servers)
self.assertEqual(
ret, 0,
("Glusterd is not running on all servers %s", self.servers))
g.log.info("Glusterd is running on all the servers %s", self.servers)
# Stop glusterd on all servers
g.log.info("Stop glusterd on all servers %s", self.servers)
ret = stop_glusterd(self.servers)
self.assertTrue(
ret, ("Failed to stop glusterd on all servers %s", self.servers))
g.log.info("Successfully stopped glusterd on all servers %s",
self.servers)
# Check if glusterd is running on all servers(expected: not running)
g.log.info(
"Check if glusterd is running on all servers %s"
"(expected: not running)", self.servers)
ret = is_glusterd_running(self.servers)
self.assertNotEqual(ret, 0, ("Glusterd is still running on some "
"servers %s", self.servers))
g.log.info("Glusterd not running on any servers %s as expected.",
self.servers)
# Start glusterd on all servers
g.log.info("Start glusterd on all servers %s", self.servers)
ret = start_glusterd(self.servers)
self.assertTrue(
ret, ("Failed to start glusterd on all servers %s", self.servers))
g.log.info("Successfully started glusterd on all servers %s",
self.servers)
# Check if glusterd is running on all servers(expected: active)
g.log.info(
"Check if glusterd is running on all servers %s"
"(expected: active)", self.servers)
ret = is_glusterd_running(self.servers)
self.assertEqual(
ret, 0,
("Glusterd is not running on all servers %s", self.servers))
g.log.info("Glusterd is running on all the servers %s", self.servers)
def test_glusterd_restart_stop_start(self):
"""Tests glusterd stop, start, restart and validate if all
peers are in connected state after glusterd restarts.
"""
# restart glusterd on all servers
ret = restart_glusterd(self.servers)
self.assertTrue(
ret,
("Failed to restart glusterd on all servers %s", self.servers))
g.log.info("Successfully restarted glusterd on all servers %s",
self.servers)
# Check if glusterd is running on all servers(expected: active)
ret = is_glusterd_running(self.servers)
self.assertEqual(
ret, 0,
("Glusterd is not running on all servers %s", self.servers))
g.log.info("Glusterd is running on all the servers %s", self.servers)
# Stop glusterd on all servers
ret = stop_glusterd(self.servers)
self.assertTrue(
ret, ("Failed to stop glusterd on all servers %s", self.servers))
g.log.info("Successfully stopped glusterd on all servers %s",
self.servers)
# Check if glusterd is running on all servers(expected: not running)
ret = is_glusterd_running(self.servers)
self.assertNotEqual(ret, 0, ("Glusterd is still running on some "
"servers %s", self.servers))
g.log.info("Glusterd not running on any servers %s as expected.",
self.servers)
# Start glusterd on all servers
ret = start_glusterd(self.servers)
self.assertTrue(
ret, ("Failed to start glusterd on all servers %s", self.servers))
g.log.info("Successfully started glusterd on all servers %s",
self.servers)
# Check if glusterd is running on all servers(expected: active)
ret = is_glusterd_running(self.servers)
self.assertEqual(
ret, 0,
("Glusterd is not running on all servers %s", self.servers))
g.log.info("Glusterd is running on all the servers %s", self.servers)
# Wait for all the glusterd's to establish communication.
time.sleep(30)
# Validate all the peers are in connected state
ret = self.validate_peers_are_connected()
self.assertTrue(ret, "Validating Peers to be in Cluster Failed")
self.test_method_complete = True
def test_setting_volume_level_option_to_cluster(self):
# pylint: disable=too-many-statements
"""
Test Case:
1) Create a cluster.
2) Try to set volume level options to cluster level.
(These should fail!)
eg: gluster v set all transport.listen-backlog 128
gluster v set all performance.parallel-readdir on
3) Check if glusterd has crashed or not.(Should not crash!)
"""
# Set transport.listen-backlog to 128 for all volumes.(Should fail!)
ret = set_volume_options(self.mnode, 'all',
{'transport.listen-backlog': '128'})
self.assertFalse(
ret, "Error: Able to set transport.listen-backlog "
"to 128 for all volumes.")
g.log.info("EXPECTED: Failed to set transport.listen-backlog to 128"
" for all volumes.")
# Checking if glusterd is running on all the nodes.
ret = is_glusterd_running(self.servers)
self.assertEqual(ret, 0, "glusterd has crashed.")
g.log.info("glusterd is running on all servers.")
# Checking if all the peers are in connected state or not.
ret = is_peer_connected(self.mnode, self.servers)
self.assertTrue(ret, "All peers are not in connected state.")
g.log.info("All peers are in connected state.")
# Set performance.parallel-readdir to on for all volumes.(Should fail!)
ret = set_volume_options(self.mnode, 'all',
{'performance.parallel-readdir': 'on'})
self.assertFalse(
ret, "Error: Able to set performance.parallel"
"-readdir to ON for all volumes.")
g.log.info("EXPECTED: Failed to set parallel-readdir to"
" ON for all volumes.")
# Checking if glusterd is running on all the nodes
ret = is_glusterd_running(self.servers)
self.assertEqual(ret, 0, "glusterd has crashed.")
g.log.info("glusterd is running on all servers.")
# Checking if all the peers are in connected state or not.
ret = is_peer_connected(self.mnode, self.servers)
self.assertTrue(ret, "All peers are not in connected state.")
g.log.info("All peers are in connected state.")
def tearDown(self):
"""
tearDown for every test
"""
ret = is_glusterd_running(self.random_server)
if ret:
ret = start_glusterd(self.random_server)
if not ret:
raise ExecutionError("Failed to start glusterd on %s" %
self.random_server)
g.log.info("Glusterd started successfully on %s", self.random_server)
# checking for peer status from every node
count = 0
while count < 80:
ret = self.validate_peers_are_connected()
if ret:
break
sleep(2)
count += 1
if not ret:
raise ExecutionError("Servers are not in peer probed state")
g.log.info("All peers are in connected state")
# stopping the volume and Cleaning up the volume
ret = self.cleanup_volume()
if not ret:
raise ExecutionError("Failed Cleanup the Volume %s" % self.volname)
# Calling GlusterBaseClass tearDown
GlusterBaseClass.tearDown.im_func(self)
def tearDown(self):
ret = is_glusterd_running(self.servers)
if ret:
ret = start_glusterd(self.servers)
if not ret:
raise ExecutionError("Failed to start glusterd on servers")
# checking for peer status from every node
count = 0
while count < 80:
ret = self.validate_peers_are_connected()
if ret:
break
sleep(2)
count += 1
if not ret:
raise ExecutionError("Servers are not in connected state")
# Setting quorum ratio to 51%
ret = set_volume_options(self.mnode, 'all',
{'cluster.server-quorum-ratio': '51%'})
if not ret:
raise ExecutionError("Failed to set server quorum ratio on %s" %
self.volname)
# stopping the volume and Cleaning up the volume
ret = self.cleanup_volume()
if not ret:
raise ExecutionError("Failed to Cleanup the Volume %s" %
self.volname)
g.log.info("Volume deleted successfully : %s", self.volname)
GlusterBaseClass.tearDown.im_func(self)
def tearDown(self):
"""
tearDown for every test
"""
ret = is_glusterd_running(self.servers)
if ret:
ret = start_glusterd(self.servers)
if not ret:
raise ExecutionError("Failed to start glusterd on %s" %
self.servers)
# Takes 5 seconds to restart glusterd into peer connected state
sleep(5)
g.log.info("Glusterd started successfully on %s", self.servers)
# checking for peer status from every node
ret = is_peer_connected(self.mnode, self.servers)
if not ret:
ret = peer_probe_servers(self.mnode, self.servers)
if not ret:
raise ExecutionError("Failed to peer probe failed in "
"servers %s" % self.servers)
# stopping the volume and Cleaning up the volume
ret = self.cleanup_volume()
if not ret:
raise ExecutionError("Failed Cleanup the Volume %s" % self.volname)
g.log.info("Volume deleted successfully : %s", self.volname)
# Calling GlusterBaseClass tearDown
self.get_super_method(self, 'tearDown')()
def tearDown(self):
# stopping the volume and Cleaning up the volume
self.get_super_method(self, 'tearDown')()
ret = is_glusterd_running(self.servers)
if ret:
ret = start_glusterd(self.servers)
if not ret:
raise ExecutionError("Failed to start glusterd on %s" %
self.servers)
# Takes 5 seconds to restart glusterd into peer connected state
sleep(5)
g.log.info("Glusterd started successfully on %s", self.servers)
# checking for peer status from every node
ret = is_peer_connected(self.mnode, self.servers)
if not ret:
ret = peer_probe_servers(self.mnode, self.servers)
if not ret:
raise ExecutionError("Failed to peer probe failed in "
"servers %s" % self.servers)
g.log.info("All peers are in connected state")
vol_list = get_volume_list(self.mnode)
if vol_list is None:
raise ExecutionError("Failed to get the volume list")
for volume in vol_list:
ret = cleanup_volume(self.mnode, volume)
if not ret:
raise ExecutionError("Failed Cleanup the Volume")
g.log.info("Volume deleted successfully")
def tearDown(self):
"""
tearDown for every test
"""
ret = is_glusterd_running(self.random_server)
if ret:
ret = start_glusterd(self.random_server)
if not ret:
raise ExecutionError("Failed to start glusterd on %s"
% self.random_server)
# checking for peer status from every node
ret = wait_for_peers_to_connect(self.mnode, self.servers)
self.assertTrue(ret, "glusterd is not connected %s with peer %s"
% (self.mnode, self.servers))
# stopping the volume and Cleaning up the volume
ret = self.cleanup_volume()
if not ret:
raise ExecutionError("Failed Cleanup the Volume %s"
% self.volname)
# Calling GlusterBaseClass tearDown
self.get_super_method(self, 'tearDown')()
def tearDown(self):
ret = is_glusterd_running(self.servers)
if ret:
ret = start_glusterd(self.servers)
if not ret:
raise ExecutionError("Failed to start glusterd on %s" %
self.servers)
g.log.info("Glusterd started successfully on %s", self.servers)
# Checking for peer status from every node
for server in self.servers:
ret = wait_for_peers_to_connect(server, self.servers)
if not ret:
raise ExecutionError("Servers are not in peer probed state")
ret = cleanup_volume(self.mnode, self.volname_2)
if not ret:
raise ExecutionError("Unable to delete volume % s" %
self.volname_2)
# Unmount and cleanup original volume
ret = self.unmount_volume_and_cleanup_volume(mounts=self.mounts)
if not ret:
raise ExecutionError("Failed to umount the vol & cleanup Volume")
# Calling GlusterBaseClass tearDown
self.get_super_method(self, 'tearDown')()
def tearDown(self):
# restart glusterd on all servers
g.log.info("Restart glusterd on all servers %s", self.servers)
ret = restart_glusterd(self.servers)
if not ret:
raise ExecutionError("Failed to restart glusterd on all "
"servers %s" % self.servers)
g.log.info("Glusterd restart successful on all servers %s",
self.servers)
# Check if glusterd is running on all servers(expected: active)
g.log.info("Check if glusterd is running on all servers %s"
"(expected: active)", self.servers)
ret = is_glusterd_running(self.servers)
if ret != 0:
raise ExecutionError("Glusterd is not running on all servers "
"%s" % self.servers)
g.log.info("Glusterd is running on all the servers %s", self.servers)
# Validate all the peers are in connected state
count = 0
while count < 80:
ret = self.validate_peers_are_connected()
if ret:
g.log.info("All peers are in connected state")
break
sleep(2)
count += 1
if not ret:
raise ExecutionError("All peers are in connected state")
def tearDown(self):
ret = is_glusterd_running(self.servers)
if ret:
ret = start_glusterd(self.servers)
if not ret:
raise ExecutionError("Failed to start glusterd on servers")
# checking for peer status from every node
count = 0
while count < 80:
ret = self.validate_peers_are_connected()
if ret:
break
sleep(2)
count += 1
if not ret:
raise ExecutionError("Servers are not in connected state")
# stopping the volume and Cleaning up the volume
ret = self.cleanup_volume()
if not ret:
raise ExecutionError("Failed to Cleanup the Volume %s" %
self.volname)
g.log.info("Volume deleted successfully : %s", self.volname)
self.get_super_method(self, 'tearDown')()
def test_peer_probe_invalid_ip_nonexist_host_nonexist_ip(self):
'''
Test script to verify peer probe non existing ip,
non_exsting_host and invalid-ip, peer probe has to
be fail for invalid-ip, non-existing-ip and
non existing host, verify Glusterd services up and
running or not after invalid peer probe,
and core file should not get created
under "/", /var/log/core and /tmp directory
'''
ret, test_timestamp, _ = g.run_local('date +%s')
test_timestamp = test_timestamp.strip()
g.log.info("Running Test : %s", self.id())
# Assigning non existing ip to variable
self.non_exist_ip = '256.256.256.256'
# Assigning invalid ip to variable
self.invalid_ip = '10.11.a'
# Assigning non existing host to variable
self.non_exist_host = 'abc.lab.eng.blr.redhat.com'
# Peer probe checks for non existing host
g.log.info("peer probe checking for non existing host")
ret, _, _ = peer_probe(self.mnode, self.non_exist_host)
self.assertNotEqual(
ret, 0, "peer probe should fail for "
"non existhost: %s" % self.non_exist_host)
g.log.info("peer probe failed for non existing host")
# Peer probe checks for invalid ip
g.log.info("peer probe checking for invalid ip")
ret, _, _ = peer_probe(self.mnode, self.invalid_ip)
self.assertNotEqual(
ret, 0, "peer probe shouldfail for "
"invalid ip: %s" % self.invalid_ip)
g.log.info("peer probe failed for invalid_ip")
# peer probe checks for non existing ip
g.log.info("peer probe checking for non existing ip")
ret, _, _ = peer_probe(self.mnode, self.non_exist_ip)
self.assertNotEqual(
ret, 0, "peer probe should fail for non exist "
"ip :%s" % self.non_exist_ip)
g.log.info("peer probe failed for non existing ip")
# Checks Glusterd services running or not after peer probe
# to invalid host and non existing host
self.mnode_list = []
self.mnode_list.append(self.mnode)
ret = is_glusterd_running(self.mnode_list)
self.assertEqual(ret, 0, "Glusterd service should be running")
# Chekcing core file created or not in "/", "/tmp" and
# "/var/log/core" directory
ret = is_core_file_created(self.servers, test_timestamp)
self.assertTrue(ret, "core file found")
def tearDown(self):
"""
tearDown for every test
"""
if not self.glusterd_service:
ret = start_glusterd(self.servers[1])
if not ret:
raise ExecutionError("Failed to start glusterd services "
"for : %s" % self.servers[1])
# Checking glusterd service running or not
ret = is_glusterd_running(self.servers[1])
if ret == 0:
g.log.info("glusterd running on :%s", self.servers[1])
else:
raise ExecutionError("glusterd not running on :%s" %
self.servers[1])
# In this test case performing quorum operations,
# deleting volumes immediately after glusterd services start, volume
# deletions are failing with quorum not met,
# that's the reason verifying peers are connected or not before
# deleting volumes
peers_not_connected = True
count = 0
while count < 10:
ret = self.validate_peers_are_connected()
if ret:
peers_not_connected = False
break
count += 1
sleep(5)
if peers_not_connected:
raise ExecutionError("Servers are not in peer probed state")
# Reverting back the quorum ratio to 51%
self.quorum_perecent = {'cluster.server-quorum-ratio': '51%'}
ret = set_volume_options(self.mnode, 'all', self.quorum_perecent)
if not ret:
raise ExecutionError(
ret, "gluster volume set all cluster"
".server-quorum- ratio percentage Failed"
" :%s" % self.servers)
g.log.info(
"gluster volume set all cluster.server-quorum-ratio 51"
"percentage enabled successfully :%s", self.servers)
# stopping the volume and Cleaning up the volume
for volume in self.volume_list:
ret = cleanup_volume(self.mnode, volume)
if not ret:
raise ExecutionError("Failed to Cleanup the "
"Volume %s" % volume)
g.log.info("Volume deleted successfully : %s", volume)
# Calling GlusterBaseClass tearDown
self.get_super_method(self, 'tearDown')()
def test_volume_create_start_stop_start(self):
"""Tests volume create, start, status, stop, start.
Also Validates whether all the brick process are running after the
start of the volume.
"""
# Verify volume processes are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online" %
self.volname))
g.log.info("Successfully Verified volume %s processes are online",
self.volname)
# Stop Volume
ret, _, _ = volume_stop(self.mnode, self.volname, force=True)
self.assertEqual(ret, 0, "Failed to stop volume %s" % self.volname)
g.log.info("Successfully stopped volume %s", self.volname)
# Start Volume
ret, _, _ = volume_start(self.mnode, self.volname)
self.assertEqual(ret, 0, "Failed to start volume %s" % self.volname)
g.log.info("Successfully started volume %s", self.volname)
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
# Log Volume Info and Status
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to Log volume %s info and status",
self.volname))
g.log.info("Successfully logged Volume %s Info and Status",
self.volname)
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online" %
self.volname))
g.log.info("Successfully verified volume %s processes are online",
self.volname)
# Log Volume Info and Status
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to Log volume %s info and status",
self.volname))
g.log.info("Successfully logged Volume %s Info and Status",
self.volname)
# Check if glusterd is running on all servers(expected: active)
ret = is_glusterd_running(self.servers)
self.assertEqual(ret, 0, "Glusterd is not running on all servers")
g.log.info("Glusterd is running on all the servers")
def tearDown(self):
"""
tearDown for every test
"""
ret = is_glusterd_running(self.servers)
if ret:
ret = start_glusterd(self.servers)
if not ret:
raise ExecutionError("Glusterd not started on some of "
"the servers")
# checking for peer status from every node
count = 0
while count < 80:
ret = self.validate_peers_are_connected()
if ret:
break
sleep(2)
count += 1
if not ret:
raise ExecutionError("Servers are not in peer probed state")
# Setting Quorum ratio to 51%
ret = set_volume_options(self.mnode, 'all',
{'cluster.server-quorum-ratio': '51%'})
if not ret:
raise ExecutionError("Failed to set server quorum ratio on %s" %
self.servers)
g.log.info("Able to set server quorum ratio successfully on %s",
self.servers)
# stopping the volume and Cleaning up the volume
ret = self.cleanup_volume()
if not ret:
raise ExecutionError("Failed Cleanup the Volume %s" % self.volname)
g.log.info("Volume deleted successfully : %s", self.volname)
# Removing brick directories
if not self.replace_brick_failed:
for brick in self.brick_list:
node, brick_path = brick.split(r':')
cmd = "rm -rf " + brick_path
ret, _, _ = g.run(node, cmd)
if ret:
raise ExecutionError("Failed to delete the brick "
"dir's of deleted volume")
# Calling GlusterBaseClass tearDown
GlusterBaseClass.tearDown.im_func(self)
def tearDown(self):
# restart glusterd on the stopped server
g.log.info("Restart glusterd on %s", self.random_server)
ret = restart_glusterd(self.random_server)
if not ret:
raise ExecutionError("Failed to restart glusterd %s" %
self.random_server)
g.log.info("Successfully restarted glusterd on %s",
self.random_server)
# Check if glusterd is running on all servers(expected: active)
g.log.info("Check if glusterd is running on all servers %s"
"(expected: active)", self.servers)
ret = is_glusterd_running(self.servers)
if ret != 0:
raise ExecutionError("Glusterd is not running on all servers"
" %s" % self.servers)
g.log.info("Glusterd is running on all the servers "
"%s", self.servers)
# Check peer status from every node
count = 0
while count < 80:
ret = self.validate_peers_are_connected()
if ret:
break
sleep(2)
count += 1
if not ret:
raise ExecutionError("All peers are in connected state")
# Validate all the peers are in connected state
g.log.info("Validating all the peers are in Cluster and Connected")
ret = self.validate_peers_are_connected()
if not ret:
raise ExecutionError("Validating Peers to be in Cluster "
"Failed")
g.log.info("All peers are in connected state")
# Unmount and cleanup original volume
g.log.info("Starting to Unmount Volume and Cleanup Volume")
ret = self.unmount_volume_and_cleanup_volume(mounts=self.mounts)
if not ret:
raise ExecutionError("Failed to umount the vol & cleanup Volume")
g.log.info("Successful in umounting the volume and Cleanup")
# Calling GlusterBaseClass tearDown
self.get_super_method(self, 'tearDown')()
def tearDown(self):
"""
tearDown for every test
"""
if not self.glusterd_service:
ret = start_glusterd(self.servers[1])
if not ret:
raise ExecutionError("Failed to start glusterd services "
"for : %s" % self.servers[1])
# Checking glusterd service running or not
ret = is_glusterd_running(self.servers[1])
if ret == 0:
g.log.info("glusterd running on :%s", self.servers[1])
else:
raise ExecutionError("glusterd not running on :%s" %
self.servers[1])
# In this test case performing quorum operations,
# deleting volumes immediately after glusterd services start, volume
# deletions are failing with quorum not met,
# that's the reason verifying peers are connected or not before
# deleting volumes
peers_not_connected = True
count = 0
while count < 10:
ret = self.validate_peers_are_connected()
if ret:
peers_not_connected = False
break
count += 1
sleep(5)
if peers_not_connected:
raise ExecutionError("Servers are not in peer probed state")
# stopping the volume and Cleaning up the volume
for volume in self.volume_list:
ret = cleanup_volume(self.mnode, volume)
if not ret:
raise ExecutionError("Failed to Cleanup the "
"Volume %s" % volume)
g.log.info("Volume deleted successfully : %s", volume)
# Calling GlusterBaseClass tearDown
GlusterBaseClass.tearDown.im_func(self)
def setUp(self):
"""setUp required for tests
"""
# Calling GlusterBaseClass setUp
GlusterBaseClass.setUp.im_func(self)
# Check if glusterd is running on all servers(expected: active)
g.log.info("Check if glusterd is running on all servers %s"
"(expected: active)", self.servers)
ret = is_glusterd_running(self.servers)
if ret == 0:
g.log.info("Glusterd is running on all servers %s", self.servers)
elif ret == 1:
g.log.info("Glusterd is not running on all the servers %s",
self.servers)
elif ret == -1:
g.log.info("Glusterd is not running on all the servers %s. "
"PID is alive", self.servers)
def tearDown(self):
"""
tearDown for every test
"""
ret = is_glusterd_running(self.servers)
if ret:
ret = start_glusterd(self.servers)
if not ret:
raise ExecutionError("Glusterd not started on some of "
"the servers")
# checking for peer status from every node
count = 0
while count < 80:
ret = self.validate_peers_are_connected()
if ret:
break
sleep(2)
count += 1
if not ret:
raise ExecutionError("Servers are not in peer probed state")
# reset quorum ratio to default
g.log.info("resetting quorum ratio")
ret, _, _ = volume_reset(self.mnode, 'all')
self.assertEqual(ret, 0, "Failed to reset quorum ratio")
g.log.info("Successfully resetted quorum ratio")
# stopping the volume and Cleaning up the volume
ret = self.cleanup_volume()
if not ret:
raise ExecutionError("Failed Cleanup the Volume %s" % self.volname)
g.log.info("Volume deleted successfully : %s", self.volname)
# Removing brick directories
if not self.replace_brick_failed:
node, brick_path = self.random_brick.split(r':')
cmd = "rm -rf " + brick_path
ret, _, _ = g.run(node, cmd)
if ret:
raise ExecutionError("Failed to delete the brick dir's")
# Calling GlusterBaseClass tearDown
GlusterBaseClass.tearDown.im_func(self)
def test_glusterd_default_vol_behavior_and_quorum_options(self):
"""
Test default volume behavior and quorum options
1. Create a volume and start it.
2. Check that no quorum options are found in vol info.
3. Kill two glusterd processes.
4. There shouldn't be any effect to the running glusterfsd
processes.
"""
# Check the default quorum options are correct.
self._validate_vol_options('cluster.server-quorum-type', 'off')
self._validate_vol_options('cluster.server-quorum-ratio', '51', True)
# Get the count of number of glusterfsd processes running.
count_before_glusterd_kill = self._get_total_brick_processes_count()
# Kill two glusterd processes.
server_list = [self.servers[1], self.servers[2]]
ret = stop_glusterd(server_list)
self.assertTrue(ret, "Failed to stop glusterd on the specified nodes.")
ret = is_glusterd_running(server_list)
self.assertNotEqual(ret, 0, ("Glusterd is not stopped on the servers"
" where it was desired to be stopped."))
g.log.info("Glusterd processes stopped in the desired servers.")
# Get the count of number of glusterfsd processes running.
count_after_glusterd_kill = self._get_total_brick_processes_count()
# The count of glusterfsd processes should match
self.assertEqual(count_before_glusterd_kill, count_after_glusterd_kill,
("Glusterfsd processes are affected."))
g.log.info("Glusterd processes are not affected.")
# Start glusterd on all servers.
ret = start_glusterd(self.servers)
self.assertTrue(ret, "Failed to Start glusterd on the specified"
" nodes")
g.log.info("Started glusterd on all nodes.")
# Wait for glusterd to restart.
ret = wait_for_glusterd_to_start(self.servers)
self.assertTrue(ret, "Glusterd not up on all nodes.")
g.log.info("Glusterd is up and running on all nodes.")
def test_setting_vol_option_with_max_characters(self):
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, ("Failed to create "
"and start volume %s" % self.volname))
auth_list = []
for ip_addr in range(256):
auth_list.append('192.168.122.%d' % ip_addr)
for ip_addr in range(7):
auth_list.append('192.168.123.%d' % ip_addr)
ip_list = ','.join(auth_list)
# set auth.allow with <4096 characters and restart the glusterd
g.log.info("Setting auth.allow with string of length %d for %s",
len(ip_list), self.volname)
self.options = {"auth.allow": ip_list}
ret = set_volume_options(self.mnode, self.volname, self.options)
self.assertTrue(ret, ("Failed to set auth.allow with string of length"
" %d for %s" % (len(ip_list), self.volname)))
ret = restart_glusterd(self.mnode)
self.assertTrue(ret,
"Failed to restart the glusterd on %s" % self.mnode)
# set auth.allow with >4096 characters and restart the glusterd
ip_list = ip_list + ",192.168.123.7"
self.options = {"auth.allow": ip_list}
g.log.info("Setting auth.allow with string of length %d for %s",
len(ip_list), self.volname)
ret = set_volume_options(self.mnode, self.volname, self.options)
self.assertTrue(ret, ("Failed to set auth.allow with string of length"
" %d for %s" % (len(ip_list), self.volname)))
ret = restart_glusterd(self.mnode)
self.assertTrue(ret,
"Failed to restart the glusterd on %s" % self.mnode)
count = 0
while count < 60:
ret = is_glusterd_running(self.mnode)
if not ret:
break
sleep(2)
count += 1
self.assertEqual(ret, 0, "glusterd is not running on %s" % self.mnode)
def test_validate_authallow(self):
"""
-Set Authentication allow as empty string for volume
-Check if glusterd is running
"""
# pylint: disable=too-many-statements
# Set Authentication to blank string for volume
option = {"auth.allow": " "}
ret = set_volume_options(self.mnode, self.volname, option)
self.assertFalse(ret, ("Unexpected: Authentication set successfully "
"for Volume with option: %s" % option))
g.log.info(
"Expected: Failed to set authentication for Volume with "
"option: %s", option)
# Check if glusterd is running
ret = is_glusterd_running(self.servers)
self.assertEqual(ret, 0, "Glusterd service not running")
g.log.info("Expected : Glusterd service running")
def tearDown(self):
ret = is_glusterd_running(self.servers)
if ret:
ret = start_glusterd(self.servers)
if not ret:
raise ExecutionError("Failed to start glusterd on %s" %
self.servers)
g.log.info("Glusterd started successfully on %s", self.servers)
# checking for peer status from every node
ret = wait_for_peers_to_connect(self.mnode, self.servers)
if not ret:
raise ExecutionError("Peers are not in connected state")
# stopping the volume and Cleaning up the volume
ret = self.unmount_volume_and_cleanup_volume(self.mounts)
if not ret:
raise ExecutionError("Failed to Cleanup the Volume %s" %
self.volname)
g.log.info("Volume deleted successfully %s", self.volname)
def test_snap_info(self):
"""
1. Create volumes
2. create multiple snapshots
3. Check snapshot info for snapshots created
using snap name, using volume name and
without using snap name and volume name
4. restart glusterd
5. follow step 3
"""
# pylint: disable=too-many-statements
# Creating snapshot with description
g.log.info("Starting to Create snapshot")
for count in range(0, 2):
self.snap = "snap%s" % count
ret, _, _ = snap_create(self.mnode,
self.volname,
self.snap,
description='$p3C!@l C#@R@cT#R$')
self.assertEqual(
ret, 0,
("Failed to create snapshot for volume %s" % self.volname))
g.log.info("Snapshot %s created successfully"
" for volume %s", self.snap, self.volname)
self.snapshot_info()
# Restart Glusterd on all node
g.log.info("Restarting Glusterd on all node")
ret = restart_glusterd(self.servers)
self.assertTrue(ret, "Failed to stop glusterd")
g.log.info("Successfully stopped glusterd on all node")
# Check Glusterd status
g.log.info("Check glusterd running or not")
ret = is_glusterd_running(self.servers)
self.assertEqual(ret, 0, "glusterd running on node ")
g.log.info("glusterd is not running")
self.snapshot_info()
def tearDown(self):
"""
tearDown for every test
"""
ret = is_glusterd_running(self.servers)
if ret:
ret = start_glusterd(self.servers)
if not ret:
raise ExecutionError("Glusterd not started on some of "
"the servers")
# checking for peer status from every node
count = 0
while count < 80:
ret = self.validate_peers_are_connected()
if ret:
break
sleep(2)
count += 1
if not ret:
raise ExecutionError("Servers are not in peer probed state")
# Setting quorum ratio to 51%
ret = set_volume_options(self.mnode, 'all',
{'cluster.server-quorum-ratio': '51%'})
self.assertTrue(
ret, "Failed to set quorum ratio to 51 percentage on "
"servers %s" % self.servers)
g.log.info(
"Able to set server quorum ratio to 51 percentage "
"on servers %s", self.servers)
# stopping the volume and Cleaning up the volume
ret = self.cleanup_volume()
if not ret:
raise ExecutionError("Failed Cleanup the Volume %s" % self.volname)
g.log.info("Volume deleted successfully : %s", self.volname)
# Calling GlusterBaseClass tearDown
GlusterBaseClass.tearDown.im_func(self)
def test_volume_create_start_stop_start(self):
"""Tests volume create, start, status, stop, start.
Also Validates whether all the brick process are running after the
start of the volume.
"""
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
# Stop Volume
ret, _, _ = volume_stop(self.mnode, self.volname, force=True)
self.assertEqual(ret, 0, "Failed to stop volume %s" % self.volname)
# Start Volume
ret, _, _ = volume_start(self.mnode, self.volname)
self.assertEqual(ret, 0, "Unable to start volume %s" % self.volname)
time.sleep(15)
# Log Volume Info and Status
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(
ret, ("Logging volume %s info and status failed" % self.volname))
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
# Log Volume Info and Status
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(
ret, ("Logging volume %s info and status failed" % self.volname))
# Verify all glusterd's are running
ret = is_glusterd_running(self.servers)
self.assertEqual(
ret, 0, ("glusterd not running on all servers: %s" % self.servers))
def tearDown(self):
# restart glusterd on the stopped server
g.log.info("Restart glusterd on %s", self.random_server)
ret = restart_glusterd(self.random_server)
if not ret:
raise ExecutionError("Failed to restart glusterd %s" %
self.random_server)
g.log.info("Successfully restarted glusterd on %s",
self.random_server)
# Check if glusterd is running on all servers(expected: active)
g.log.info("Check if glusterd is running on all servers %s"
"(expected: active)", self.servers)
ret = is_glusterd_running(self.servers)
if ret != 0:
raise ExecutionError("Glusterd is not running on all servers"
" %s" % self.servers)
g.log.info("Glusterd is running on all the servers "
"%s", self.servers)
# Check peer status from every node
count = 0
while count < 80:
ret = self.validate_peers_are_connected()
if ret:
break
sleep(2)
count += 1
if not ret:
raise ExecutionError("All peers are in connected state")
# Validate all the peers are in connected state
g.log.info("Validating all the peers are in Cluster and Connected")
ret = self.validate_peers_are_connected()
if not ret:
raise ExecutionError("Validating Peers to be in Cluster "
"Failed")
g.log.info("All peers are in connected state")
def tearDown(self):
ret = is_glusterd_running(self.servers)
if ret:
ret = start_glusterd(self.servers)
if not ret:
raise ExecutionError("Failed to start glusterd on %s"
% self.servers)
g.log.info("Glusterd started successfully on %s", self.servers)
ret = wait_for_peers_to_connect(self.mnode, self.servers)
self.assertTrue(ret, "glusterd is not connected %s with peer %s"
% (self.servers, self.servers))
# stopping the volume and Cleaning up the volume
ret = self.cleanup_volume()
if not ret:
raise ExecutionError("Failed to Cleanup the Volume %s"
% self.volname)
g.log.info("Volume deleted successfully : %s", self.volname)
self.get_super_method(self, 'tearDown')()
def tearDown(self):
ret = is_glusterd_running(self.servers)
if ret:
ret = start_glusterd(self.servers)
if not ret:
raise ExecutionError("Failed to start glusterd on servers")
g.log.info("glusterd is running on all the nodes")
# checking for peer status from every node
count = 0
while count < 80:
ret = self.validate_peers_are_connected()
if ret:
break
sleep(2)
count += 1
if not ret:
raise ExecutionError("Servers are not in connected state")
g.log.info("Peers are in connected state")
# reset quorum ratio to default
g.log.info("resetting quorum ratio")
ret, _, _ = volume_reset(self.mnode, 'all')
self.assertEqual(ret, 0, "Failed to reset quorum ratio")
g.log.info("Successfully resetted quorum ratio")
# stopping the volume and Cleaning up the volume
ret = self.cleanup_volume()
if not ret:
raise ExecutionError("Failed to Cleanup the Volume %s" %
self.volname)
g.log.info("Volume deleted successfully : %s", self.volname)
GlusterBaseClass.tearDown.im_func(self)
def test_add_brick_when_quorum_not_met(self):
# pylint: disable=too-many-statements
# create and start a volume
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, ("Failed to create "
"and start volume %s" % self.volname))
g.log.info("Volume is created and started successfully")
# set cluster.server-quorum-type as server
ret = set_volume_options(self.mnode, self.volname,
{'cluster.server-quorum-type': 'server'})
self.assertTrue(ret, ("Failed to set the quorum type as a server"
" on volume %s", self.volname))
g.log.info("Able to set server quorum successfully on volume %s",
self.volname)
# Setting quorum ratio to 95%
ret = set_volume_options(self.mnode, 'all',
{'cluster.server-quorum-ratio': '95%'})
self.assertTrue(
ret, "Failed to set server quorum ratio on %s" % self.volname)
g.log.info("Able to set server quorum ratio successfully on %s",
self.servers)
# bring down glusterd of half nodes
num_of_servers = len(self.servers)
num_of_nodes_to_bring_down = num_of_servers / 2
for node in range(num_of_nodes_to_bring_down, num_of_servers):
ret = stop_glusterd(self.servers[node])
self.assertTrue(
ret, ("Failed to stop glusterd on %s" % self.servers[node]))
g.log.info("Glusterd stopped successfully on server %s",
self.servers[node])
for node in range(num_of_nodes_to_bring_down, num_of_servers):
count = 0
while count < 80:
ret = is_glusterd_running(self.servers[node])
if ret:
break
sleep(2)
count += 1
self.assertNotEqual(
ret, 0, "glusterd is still running on %s" % self.servers[node])
# Verifying node count in volume status after glusterd stopped
# on half of the servers, Its not possible to check the brick status
# immediately in volume status after glusterd stop
count = 0
while count < 100:
vol_status = get_volume_status(self.mnode, self.volname)
servers_count = len(vol_status[self.volname])
if servers_count == (num_of_servers - num_of_nodes_to_bring_down):
break
sleep(2)
count += 1
# confirm that quorum is not met, brick process should be down
bricks_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(bricks_list, "Failed to get the brick list")
bricks_to_check = bricks_list[0:num_of_nodes_to_bring_down]
ret = are_bricks_offline(self.mnode, self.volname, bricks_to_check)
self.assertTrue(
ret, "Unexpected: Server quorum is not met, "
"Bricks are up")
g.log.info("Server quorum is not met, bricks are down as expected")
# try add brick operation, which should fail
num_bricks_to_add = 1
brick = form_bricks_list(self.mnode, self.volname, num_bricks_to_add,
self.servers, self.all_servers_info)
ret, _, _ = add_brick(self.mnode, self.volname, brick)
self.assertNotEqual(ret, 0, ("Unexpected: add brick is success, "
"when quorum is not met"))
g.log.info("Add brick is failed as expected, when quorum is not met")
# confirm that, newly added brick is not part of volume
bricks_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(bricks_list, "Failed to get the brick list")
if brick in bricks_list:
ret = False
self.assertTrue(ret, ("Unexpected: add brick is success, "
"when quorum is not met"))
g.log.info("Add brick is failed as expected, when quorum is not met")
# set cluster.server-quorum-type as none
ret = set_volume_options(self.mnode, self.volname,
{'cluster.server-quorum-type': 'none'})
self.assertTrue(ret, ("Failed to set the quorum type as a server"
" on volume %s", self.volname))
g.log.info("Able to set server quorum successfully on volume %s",
self.volname)