def test_glustershd_on_all_volume_types(self):
"""
Test Script to verify the glustershd server vol file
has only entries for replicate volumes
* Create multiple volumes and start all volumes
* Check the glustershd processes - Only One glustershd should be listed
* Check the glustershd server vol file - should contain entries only
for replicated involved volumes
* Add bricks to the replicate volume - it should convert to
distributed-replicate
* Check the glustershd server vol file - newly added bricks
should present
* Check the glustershd processes - Only 1 glustershd should be listed
"""
# pylint: disable=too-many-statements
nodes = self.servers
# check the self-heal daemon process
g.log.info("Starting to get self-heal daemon process on "
"nodes %s", nodes)
ret, glustershd_pids = get_self_heal_daemon_pid(nodes)
self.assertTrue(ret, ("Either No self heal daemon process found or "
"more than One self heal daemon process "
"found : %s" % glustershd_pids))
g.log.info(
"Successful in getting Single self heal daemon process"
" on all nodes %s", nodes)
# For all the volumes, check whether bricks present in
# glustershd server vol file
volume_list = get_volume_list(self.mnode)
for volume in volume_list:
g.log.info("Volume Name: %s", volume)
volume_type_info = get_volume_type_info(self.mnode, volume)
volume_type = (volume_type_info['volume_type_info']['typeStr'])
# get the bricks for the volume
g.log.info("Fetching bricks for the volume : %s", volume)
bricks_list = get_all_bricks(self.mnode, volume)
g.log.info("Brick List : %s", bricks_list)
# validate the bricks present in volume info with
# glustershd server volume file
g.log.info("Start parsing file %s on "
"node %s", self.GLUSTERSHD, self.mnode)
ret = do_bricks_exist_in_shd_volfile(self.mnode, volume,
bricks_list)
if volume_type == 'Distribute':
self.assertFalse(ret,
("Bricks exist in glustershd server "
"volume file for %s Volume" % volume_type))
g.log.info(
"EXPECTED : Bricks doesn't exist in glustershd "
"server volume file for %s Volume", volume_type)
else:
self.assertTrue(ret, ("Brick List from volume info is "
"different from glustershd server "
"volume file. Please check log "
"file for details"))
g.log.info(
"Bricks exist in glustershd server volume file "
"for %s Volume", volume_type)
# expanding volume for Replicate
for volume in volume_list:
volume_type_info = get_volume_type_info(self.mnode, volume)
volume_type = (volume_type_info['volume_type_info']['typeStr'])
if volume_type == 'Replicate':
g.log.info("Start adding bricks to volume %s", volume)
ret = expand_volume(self.mnode, volume, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to add bricks to "
"volume %s " % volume))
g.log.info("Add brick successful")
# Log Volume Info and Status after expanding the volume
g.log.info("Logging volume info and Status after "
"expanding volume")
ret = log_volume_info_and_status(self.mnode, volume)
self.assertTrue(ret, ("Logging volume info and status failed "
"on volume %s", volume))
g.log.info(
"Successful in logging volume info and status "
"of volume %s", volume)
# Verify volume's all process are online for 60 sec
g.log.info("Verifying volume's all process are online")
ret = wait_for_volume_process_to_be_online(
self.mnode, volume, 60)
self.assertTrue(ret, ("Volume %s : All process are not "
"online", volume))
g.log.info(
"Successfully verified volume %s processes "
"are online", volume)
# check the type for the replicate volume
volume_type_info_for_replicate_after_adding_bricks = \
get_volume_type_info(self.mnode, volume)
volume_type_for_replicate_after_adding_bricks = \
(volume_type_info_for_replicate_after_adding_bricks
['volume_type_info']['typeStr'])
self.assertEqual(volume_type_for_replicate_after_adding_bricks,
'Distributed-Replicate',
("Replicate volume type is not converted to "
"Distributed-Replicate after adding bricks"))
g.log.info("Replicate Volume is successfully converted to"
" Distributed-Replicate after adding bricks")
# get the bricks for the volume after expanding
bricks_list_after_expanding = get_all_bricks(
self.mnode, volume)
g.log.info("Brick List after expanding "
"volume: %s", bricks_list_after_expanding)
# validate the bricks present in volume info
# with glustershd server volume file after adding bricks
g.log.info("Starting parsing file %s", self.GLUSTERSHD)
ret = do_bricks_exist_in_shd_volfile(
self.mnode, volume, bricks_list_after_expanding)
self.assertTrue(ret, ("Brick List from volume info is "
"different from glustershd server "
"volume file after expanding bricks. "
"Please check log file for details"))
g.log.info("Brick List from volume info is same as from "
"glustershd server volume file after "
"expanding bricks.")
# check the self-heal daemon process
g.log.info("Starting to get self-heal daemon process on "
"nodes %s", nodes)
ret, glustershd_pids_after_adding_bricks = \
get_self_heal_daemon_pid(nodes)
self.assertTrue(ret,
("Either No self heal daemon process found or "
"more than One self heal daemon process "
"found : %s" % glustershd_pids_after_adding_bricks))
g.log.info(
"Successful in getting Single self heal daemon process"
" on all nodes %s", nodes)
self.assertNotEqual(
glustershd_pids, glustershd_pids_after_adding_bricks,
"Self Daemon process is same before and"
" after adding bricks")
g.log.info("Self Heal Daemon Process is different before and "
"after adding bricks")
def test_glustershd_with_restarting_glusterd(self):
"""
Test Script to verify the self heal daemon process with restarting
glusterd and rebooting the server
* stop all volumes
* restart glusterd - should not run self heal daemon process
* start replicated involved volumes
* single self heal daemon process running
* restart glusterd
* self heal daemon pid will change
* bring down brick and restart glusterd
* self heal daemon pid will change and its different from previous
* brought up the brick
"""
# pylint: disable=too-many-statements
nodes = self.volume['servers']
# stop the volume
g.log.info("Stopping the volume %s", self.volname)
ret = volume_stop(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to stop volume %s" % self.volname))
g.log.info("Successfully stopped volume %s", self.volname)
# check the self heal daemon process after stopping the volume
g.log.info("Verifying the self heal daemon process for "
"volume %s", self.volname)
ret = are_all_self_heal_daemons_are_online(self.mnode, self.volname)
self.assertFalse(ret, ("Self Heal Daemon process is still running "
"even after stopping volume %s" % self.volname))
g.log.info("Self Heal Daemon is not running after stopping "
"volume %s", self.volname)
# restart glusterd service on all the servers
g.log.info("Restarting glusterd on all servers %s", nodes)
ret = restart_glusterd(nodes)
self.assertTrue(ret, ("Failed to restart glusterd on all nodes %s",
nodes))
g.log.info("Successfully restarted glusterd on all nodes %s",
nodes)
self.assertTrue(
wait_for_glusterd_to_start(self.servers),
"Failed to start glusterd on %s" % self.servers)
# check the self heal daemon process after restarting glusterd process
g.log.info("Starting to get self-heal daemon process on"
" nodes %s", nodes)
ret = are_all_self_heal_daemons_are_online(self.mnode, self.volname)
self.assertFalse(ret, ("Self Heal Daemon process is running after "
"glusterd restart with volume %s in "
"stop state" % self.volname))
g.log.info("Self Heal Daemon is not running after stopping "
"volume and restarting glusterd %s", self.volname)
# start the volume
g.log.info("Starting the volume %s", self.volname)
ret = volume_start(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to start volume %s" % self.volname))
g.log.info("Volume %s started successfully", self.volname)
# Verfiy glustershd process releases its parent process
g.log.info("Checking whether glustershd process is daemonized or not")
ret = is_shd_daemonized(nodes)
self.assertTrue(ret, ("Either No self heal daemon process found or "
"more than One self heal daemon process found"))
g.log.info("Single self heal daemon process on all nodes %s", nodes)
# get the self heal daemon pids after starting volume
g.log.info("Starting to get self-heal daemon process "
"on nodes %s", nodes)
ret, pids = get_self_heal_daemon_pid(nodes)
self.assertTrue(ret, ("Either No self heal daemon process found or "
"more than One self heal daemon process found"))
g.log.info("Successful in getting self heal daemon pids")
glustershd_pids = pids
# get the bricks for the volume
g.log.info("Fetching bricks for the volume : %s", self.volname)
bricks_list = get_all_bricks(self.mnode, self.volname)
g.log.info("Brick List : %s", bricks_list)
# validate the bricks present in volume info
# with glustershd server volume file
g.log.info("Starting parsing file %s on "
"node %s", self.glustershd, self.mnode)
ret = do_bricks_exist_in_shd_volfile(self.mnode, self.volname,
bricks_list)
self.assertTrue(ret, ("Brick List from volume info is different from "
"glustershd server volume file. "
"Please check log file for details."))
g.log.info("Successfully parsed %s file", self.glustershd)
# restart glusterd service on all the servers
g.log.info("Restarting glusterd on all servers %s", nodes)
ret = restart_glusterd(nodes)
self.assertTrue(ret, ("Failed to restart glusterd on all nodes %s",
nodes))
g.log.info("Successfully restarted glusterd on all nodes %s",
nodes)
# Verify volume's all process are online for 60 sec
g.log.info("Verifying volume's all process are online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname,
60)
self.assertTrue(ret, ("Volume %s : All process are not "
"online", self.volname))
g.log.info("Successfully Verified volume %s processes are online",
self.volname)
# Verfiy glustershd process releases its parent process
ret = is_shd_daemonized(nodes)
self.assertTrue(ret, ("Either No self heal daemon process found or "
"more than One self heal daemon process found"))
# check the self heal daemon process after starting volume and
# restarting glusterd process
g.log.info("Starting to get self-heal daemon process "
"on nodes %s", nodes)
ret, pids = get_self_heal_daemon_pid(nodes)
self.assertTrue(ret, ("Either No self heal daemon process found or "
"more than One self heal daemon process found"))
glustershd_pids_after_glusterd_restart = pids
self.assertNotEqual(glustershd_pids,
glustershd_pids_after_glusterd_restart,
("Self Heal Daemon pids are same after "
"restarting glusterd process"))
g.log.info("Self Heal Daemon process are different before and "
"after restarting glusterd process")
# select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = list(filter(None, (
bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks'])))
# bring bricks offline
g.log.info("Going to bring down the brick process "
"for %s", bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, ("Failed to bring down the bricks. Please "
"check the log file for more details."))
g.log.info("Brought down the brick process "
"for %s successfully", bricks_to_bring_offline)
# restart glusterd after brought down the brick
g.log.info("Restart glusterd on all servers %s", nodes)
ret = restart_glusterd(nodes)
self.assertTrue(ret, ("Failed to restart glusterd on all nodes %s",
nodes))
g.log.info("Successfully restarted glusterd on all nodes %s",
nodes)
# Verify volume's all process are online for 60 sec
g.log.info("Verifying volume's all process are online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname,
60)
self.assertTrue(ret, ("Volume %s : All process are not "
"online", self.volname))
g.log.info("Successfully Verified volume %s processes are online",
self.volname)
# Verfiy glustershd process releases its parent process
ret = is_shd_daemonized(nodes)
self.assertTrue(ret, ("Either No self heal daemon process found or "
"more than One self heal daemon process found"))
# check the self heal daemon process after killing brick and
# restarting glusterd process
g.log.info("Starting to get self-heal daemon process "
"on nodes %s", nodes)
ret, pids = get_self_heal_daemon_pid(nodes)
self.assertTrue(ret, ("Either No self heal daemon process found or "
"more than One self heal daemon process found"))
glustershd_pids_after_killing_brick = pids
self.assertNotEqual(glustershd_pids_after_glusterd_restart,
glustershd_pids_after_killing_brick,
("Self Heal Daemon process are same from before "
"killing the brick,restarting glusterd process"))
g.log.info("Self Heal Daemon process are different after killing the "
"brick, restarting the glusterd process")
# brought the brick online
g.log.info("bringing up the bricks : %s online",
bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, ("Failed to brought the bricks online"))
g.log.info("Successfully brought the bricks online")
# check all bricks are online
g.log.info("Verifying all bricka are online or not.....")
ret = are_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, ("Not all bricks are online"))
g.log.info("All bricks are online.")
def test_glustershd_with_add_remove_brick(self):
"""
Test script to verify glustershd process with adding and
removing bricks
* check glustershd process - only 1 glustershd process should
be running
* bricks must be present in glustershd-server.vol file for
the replicated involved volumes
* Add bricks
* check glustershd process - only 1 glustershd process should
be running and its should be different from previous one
* bricks which are added must present in glustershd-server.vol file
* remove bricks
* check glustershd process - only 1 glustershd process should
be running and its different from previous one
* bricks which are removed should not present
in glustershd-server.vol file
"""
# pylint: disable=too-many-statements
nodes = self.volume['servers']
bricks_list = []
glustershd_pids = {}
# check the self-heal daemon process
g.log.info("Starting to get self-heal daemon process on "
"nodes %s", nodes)
ret, pids = get_self_heal_daemon_pid(nodes)
self.assertTrue(ret, ("Either No self heal daemon process found or "
"more than One self heal daemon process "
"found : %s", pids))
g.log.info("Successful in getting Single self heal daemon process"
" on all nodes %s", nodes)
glustershd_pids = pids
# get the bricks for the volume
g.log.info("Fetching bricks for the volume : %s", self.volname)
bricks_list = get_all_bricks(self.mnode, self.volname)
g.log.info("Brick List : %s", bricks_list)
# validate the bricks present in volume info with
# glustershd server volume file
g.log.info("Starting parsing file %s on "
"node %s", self.glustershd, self.mnode)
ret = do_bricks_exist_in_shd_volfile(self.mnode, self.volname,
bricks_list)
self.assertTrue(ret, ("Brick List from volume info is different "
"from glustershd server volume file. "
"Please check log file for details"))
g.log.info("Successfully parsed %s file", self.glustershd)
# expanding volume
g.log.info("Start adding bricks to volume %s", self.volname)
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to add bricks to "
"volume %s " % self.volname))
g.log.info("Add brick successful")
# Log Volume Info and Status after expanding the volume
g.log.info("Logging volume info and Status after expanding volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed "
"on volume %s", self.volname))
g.log.info("Successful in logging volume info and status "
"of volume %s", self.volname)
# Verify volume's all process are online for 60 sec
g.log.info("Verifying volume's all process are online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname,
60)
self.assertTrue(ret, ("Volume %s : All process are not "
"online", self.volname))
g.log.info("Successfully Verified volume %s processes are online",
self.volname)
# Start Rebalance
g.log.info("Starting Rebalance on the volume")
ret, _, err = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to start rebalance on "
"the volume %s with error %s" %
(self.volname, err)))
g.log.info("Successfully started rebalance on the "
"volume %s", self.volname)
# Log Rebalance status
g.log.info("Log Rebalance status")
_, _, _ = rebalance_status(self.mnode, self.volname)
# Wait for rebalance to complete
g.log.info("Waiting for rebalance to complete")
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(ret, ("Rebalance is not yet complete "
"on the volume %s", self.volname))
g.log.info("Rebalance is successfully complete on "
"the volume %s", self.volname)
# Check Rebalance status after rebalance is complete
g.log.info("Checking Rebalance status")
ret, _, _ = rebalance_status(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to get rebalance status for "
"the volume %s", self.volname))
g.log.info("Successfully got rebalance status of the "
"volume %s", self.volname)
# Check the self-heal daemon process after adding bricks
g.log.info("Starting to get self-heal daemon process on "
"nodes %s", nodes)
glustershd_pids_after_expanding = {}
ret, pids = get_self_heal_daemon_pid(nodes)
self.assertTrue(ret, ("Either No self heal daemon process found or "
"more than One self heal daemon process found"))
g.log.info("Successful in getting self-heal daemon process "
"on nodes %s", nodes)
glustershd_pids_after_expanding = pids
g.log.info("Self Heal Daemon Process ID's after expanding "
"volume: %s", glustershd_pids_after_expanding)
self.assertNotEqual(glustershd_pids,
glustershd_pids_after_expanding,
"Self Daemon process is same before and"
" after adding bricks")
g.log.info("Self Heal Daemon Process is different before and "
"after adding bricks")
# get the bricks for the volume after expanding
bricks_list_after_expanding = get_all_bricks(self.mnode, self.volname)
g.log.info("Brick List after expanding "
"volume: %s", bricks_list_after_expanding)
# validate the bricks present in volume info
# with glustershd server volume file after adding bricks
g.log.info("Starting parsing file %s", self.glustershd)
ret = do_bricks_exist_in_shd_volfile(self.mnode, self.volname,
bricks_list_after_expanding)
self.assertTrue(ret, ("Brick List from volume info is different "
"from glustershd server volume file after "
"expanding bricks. Please check log file "
"for details"))
g.log.info("Successfully parsed %s file", self.glustershd)
# shrink the volume
g.log.info("Starting volume shrink")
ret = shrink_volume(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to shrink the volume on "
"volume %s", self.volname))
g.log.info("Shrinking volume is successful on "
"volume %s", self.volname)
# Log Volume Info and Status after shrinking the volume
g.log.info("Logging volume info and Status after shrinking volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status "
"of volume %s", self.volname)
# get the bricks after shrinking the volume
bricks_list_after_shrinking = get_all_bricks(self.mnode, self.volname)
g.log.info("Brick List after shrinking "
"volume: %s", bricks_list_after_shrinking)
self.assertEqual(len(bricks_list_after_shrinking), len(bricks_list),
"Brick Count is mismatched after "
"shrinking the volume %s" % self.volname)
g.log.info("Brick Count matched before before expanding "
"and after shrinking volume")
# Verfiy glustershd process releases its parent process
ret = is_shd_daemonized(nodes)
self.assertTrue(ret, ("Either No self heal daemon process found or "
"more than One self heal daemon process found"))
# check the self-heal daemon process after removing bricks
g.log.info("Starting to get self-heal daemon process "
"on nodes %s", nodes)
glustershd_pids_after_shrinking = {}
ret, pids = get_self_heal_daemon_pid(nodes)
glustershd_pids_after_shrinking = pids
self.assertNotEqual(glustershd_pids_after_expanding,
glustershd_pids_after_shrinking,
"Self Heal Daemon process is same "
"after adding bricks and shrinking volume")
g.log.info("Self Heal Daemon Process is different after adding bricks "
"and shrinking volume")
# validate bricks present in volume info
# with glustershd server volume file after removing bricks
g.log.info("Starting parsing file %s", self.glustershd)
ret = do_bricks_exist_in_shd_volfile(self.mnode, self.volname,
bricks_list_after_shrinking)
self.assertTrue(ret, ("Brick List from volume info is different "
"from glustershd server volume file after "
"removing bricks. Please check log file "
"for details"))
g.log.info("Successfully parsed %s file", self.glustershd)
def test_impact_of_replace_brick_for_glustershd(self):
# pylint: disable=too-many-statements,too-many-branches,too-many-locals
nodes = self.volume['servers']
replaced_bricks = []
# check the self-heal daemon process
g.log.info("Starting to get self-heal daemon process on "
"nodes %s", nodes)
ret, pids = get_self_heal_daemon_pid(nodes)
self.assertTrue(ret, ("Either No self heal daemon process found or "
"more than One self heal daemon process "
"found : %s" % pids))
g.log.info(
"Successful in getting Single self heal daemon process"
" on all nodes %s", nodes)
glustershd_pids = pids
# get the bricks for the volume
g.log.info("Fetching bricks for the volume : %s", self.volname)
bricks_list = get_all_bricks(self.mnode, self.volname)
g.log.info("Brick List : %s", bricks_list)
# validate the bricks present in volume info with
# glustershd server volume file
g.log.info("Starting parsing file %s on "
"node %s", self.glustershd, self.mnode)
ret = do_bricks_exist_in_shd_volfile(self.mnode, self.volname,
bricks_list)
self.assertTrue(ret, ("Brick List from volume info is different "
"from glustershd server volume file. "
"Please check log file for details"))
g.log.info("Successfully parsed %s file", self.glustershd)
# get the subvolumes
g.log.info("Starting to get sub-volumes for volume %s", self.volname)
subvols_dict = get_subvols(self.mnode, self.volname)
num_subvols = len(subvols_dict['volume_subvols'])
g.log.info("Number of subvolumes in volume %s:", num_subvols)
# replace brick from each sub-vol
for i in range(0, num_subvols):
subvol_brick_list = subvols_dict['volume_subvols'][i]
g.log.info("sub-volume %s brick list : %s", i, subvol_brick_list)
brick_to_replace = subvol_brick_list[-1]
new_brick = brick_to_replace + 'new'
g.log.info("Replacing the brick %s for the volume : %s",
brick_to_replace, self.volname)
ret, _, err = replace_brick(self.mnode, self.volname,
brick_to_replace, new_brick)
self.assertFalse(ret, err)
g.log.info('Replaced brick %s to %s successfully',
brick_to_replace, new_brick)
replaced_bricks.append(brick_to_replace)
# Verify volume's all process are online for 60 sec
g.log.info("Verifying volume's all process are online")
ret = wait_for_volume_process_to_be_online(self.mnode,
self.volname,
timeout=60)
self.assertTrue(ret, ("Volume %s : All process are not "
"online", self.volname))
g.log.info("Successfully Verified volume %s processes are online",
self.volname)
# Verify glustershd process releases its parent process
ret = is_shd_daemonized(nodes)
self.assertTrue(ret, ("Either No self heal daemon process found or "
"more than One self heal daemon process found"))
# check the self-heal daemon process
g.log.info("Starting to get self-heal daemon process on nodes "
"%s", nodes)
ret, pids = get_self_heal_daemon_pid(nodes)
self.assertTrue(ret, ("Either No self heal daemon process found or"
" more than One self heal daemon process"
" found : %s" % pids))
g.log.info(
"Successful in getting Single self heal daemon process"
" on all nodes %s", nodes)
glustershd_pids_after_replacement = pids
# Compare pids before and after replacing
self.assertNotEqual(
glustershd_pids, glustershd_pids_after_replacement,
"Self Daemon process is same before and"
" after replacing bricks")
g.log.info("Self Heal Daemon Process is different before and "
"after replacing bricks")
# get the bricks for the volume after replacing
bricks_list_after_replacing = get_all_bricks(self.mnode, self.volname)
g.log.info("Brick List after expanding "
"volume: %s", bricks_list_after_replacing)
# validate the bricks present in volume info
# with glustershd server volume file after replacing bricks
g.log.info("Starting parsing file %s", self.glustershd)
ret = do_bricks_exist_in_shd_volfile(self.mnode, self.volname,
bricks_list_after_replacing)
self.assertTrue(ret, ("Brick List from volume info is different "
"from glustershd server volume file after "
"replacing bricks. Please check log file "
"for details"))
g.log.info("Successfully parsed %s file", self.glustershd)
g.log.info("Starting to delete replaced brick dir's")
# Remove brick directories of the replaced bricks as this is not
# handled by tearDown class
for bricks in replaced_bricks:
node, brick_path = bricks.split(r':')
cmd = "rm -rf " + brick_path
ret, _, _ = g.run(node, cmd)
if ret:
raise ExecutionError("Failed to delete the brick dir's for"
" %s and brick %s" % (node, brick_path))
g.log.info("Successfully deleted brick dir's for replaced bricks")
def test_impact_of_replace_brick_on_glustershd(self):
"""
Test Script to verify the glustershd server vol file
has only entries for replicate volumes
1.Create multiple volumes and start all volumes
2.Check the glustershd processes - Only 1 glustershd should be listed
3.Do replace brick on the replicate volume
4.Confirm that the brick is replaced
5.Check the glustershd processes - Only 1 glustershd should be listed
and pid should be different
6.glustershd server vol should be updated with new bricks
"""
# Check the self-heal daemon process
ret, glustershd_pids = get_self_heal_daemon_pid(self.servers)
self.assertTrue(ret, ("Either no self heal daemon process found or "
"more than one self heal daemon process "
"found : %s" % glustershd_pids))
g.log.info(
"Successful in getting single self heal daemon process"
" on all nodes %s", self.servers)
volume_list = get_volume_list(self.mnode)
for volume in volume_list:
# Log Volume Info and Status before replacing brick
ret = log_volume_info_and_status(self.mnode, volume)
self.assertTrue(ret, ("Logging volume info and status "
"failed on volume %s", volume))
g.log.info(
"Successful in logging volume info and status "
"of volume %s", volume)
# Selecting a random source brick to replace
src_brick = choice(get_all_bricks(self.mnode, volume))
src_node, original_brick = src_brick.split(":")
# Creating a random destination brick in such a way
# that the brick is select from the same node but always
# picks a different from the original brick
list_of_bricks = [
brick for brick in get_servers_bricks_dict(
src_node, self.all_servers_info)[src_node]
if brick not in original_brick
]
dst_brick = ('{}:{}/{}_replaced'.format(
src_node, choice(list_of_bricks),
original_brick.split('/')[::-1][0]))
# Replace brick for the volume
ret, _, _ = replace_brick(self.mnode, volume, src_brick, dst_brick)
self.assertFalse(
ret, "Failed to replace brick "
"from the volume %s" % volume)
g.log.info(
"Successfully replaced faulty brick from "
"the volume %s", volume)
# Verify all volume process are online
ret = wait_for_volume_process_to_be_online(self.mnode, volume)
self.assertTrue(ret,
"Volume %s : All process are not online" % volume)
g.log.info("Volume %s : All process are online", volume)
# Check the self-heal daemon process after replacing brick
ret, pid_after_replace = get_self_heal_daemon_pid(self.servers)
self.assertTrue(
ret, "Either no self heal daemon process "
"found or more than one self heal "
"daemon process found : %s" % pid_after_replace)
g.log.info(
"Successful in getting Single self heal "
" daemon process on all nodes %s", self.servers)
# Compare the glustershd pids
self.assertNotEqual(
glustershd_pids, pid_after_replace,
"Self heal daemon process should be different "
"after replacing bricks in %s volume" % volume)
g.log.info("EXPECTED: Self heal daemon process should be different"
" after replacing bricks in replicate volume")
# Get the bricks for the volume
bricks_list = get_all_bricks(self.mnode, volume)
g.log.info("Brick List : %s", bricks_list)
# Validate the bricks present in volume info with
# glustershd server volume file
ret = do_bricks_exist_in_shd_volfile(self.mnode, volume,
bricks_list)
self.assertTrue(ret, ("Brick List from volume info is "
"different from glustershd server "
"volume file. Please check log file "
"for details"))
g.log.info(
"Bricks in volume %s exists in glustershd server "
"volume file", volume)
def test_impact_of_replace_brick_for_glustershd(self):
nodes = self.volume['servers']
# check the self-heal daemon process
g.log.info("Starting to get self-heal daemon process on "
"nodes %s" % nodes)
ret, pids = get_self_heal_daemon_pid(nodes)
self.assertTrue(ret, ("Either No self heal daemon process found or "
"more than One self heal daemon process "
"found : %s" % pids))
g.log.info("Successful in getting Single self heal daemon process"
" on all nodes %s", nodes)
glustershd_pids = pids
# get the bricks for the volume
g.log.info("Fetching bricks for the volume : %s" % self.volname)
bricks_list = get_all_bricks(self.mnode, self.volname)
g.log.info("Brick List : %s" % bricks_list)
# validate the bricks present in volume info with
# glustershd server volume file
g.log.info("Starting parsing file %s on "
"node %s" % (self.GLUSTERSHD, self.mnode))
ret = do_bricks_exist_in_shd_volfile(self.mnode, self.volname,
bricks_list)
self.assertTrue(ret, ("Brick List from volume info is different "
"from glustershd server volume file. "
"Please check log file for details"))
g.log.info("Successfully parsed %s file" % self.GLUSTERSHD)
# replace brick
brick_to_replace = bricks_list[-1]
new_brick = brick_to_replace + 'new'
g.log.info("Replacing the brick %s for the volume : %s"
% (brick_to_replace, self.volname))
ret, out, err = replace_brick(self.mnode, self.volname,
brick_to_replace, new_brick)
self.assertFalse(ret, err)
g.log.info('Replaced brick %s to %s successfully'
% (brick_to_replace, new_brick))
# check bricks
bricks_list = get_all_bricks(self.mnode, self.volname)
self.assertEqual(bricks_list[-1], new_brick, 'Replaced brick and '
'new brick are not equal')
# Verify volume's all process are online for 60 sec
g.log.info("Verifying volume's all process are online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname,
timeout=60)
self.assertTrue(ret, ("Volume %s : All process are not "
"online", self.volname))
g.log.info("Successfully Verified volume %s processes are online",
self.volname)
# Verify glustershd process releases its parent process
ret = is_shd_daemonized(nodes)
self.assertTrue(ret, ("Either No self heal daemon process found or "
"more than One self heal daemon process found"))
# check the self-heal daemon process
g.log.info("Starting to get self-heal daemon process on "
"nodes %s" % nodes)
ret, pids = get_self_heal_daemon_pid(nodes)
self.assertTrue(ret, ("Either No self heal daemon process found or "
"more than One self heal daemon process "
"found : %s" % pids))
g.log.info("Successful in getting Single self heal daemon process"
" on all nodes %s", nodes)
glustershd_pids_after_replacement = pids
# Compare pids before and after replacing
self.assertNotEqual(glustershd_pids,
glustershd_pids_after_replacement,
"Self Daemon process is same before and"
" after replacing bricks")
g.log.info("Self Heal Daemon Process is different before and "
"after replacing bricks")
# get the bricks for the volume after replacing
bricks_list_after_replacing = get_all_bricks(self.mnode, self.volname)
g.log.info("Brick List after expanding "
"volume: %s" % bricks_list_after_replacing)
# validate the bricks present in volume info
# with glustershd server volume file after replacing bricks
g.log.info("Starting parsing file %s" % self.GLUSTERSHD)
ret = do_bricks_exist_in_shd_volfile(self.mnode, self.volname,
bricks_list_after_replacing)
self.assertTrue(ret, ("Brick List from volume info is different "
"from glustershd server volume file after "
"replacing bricks. Please check log file "
"for details"))
g.log.info("Successfully parsed %s file" % self.GLUSTERSHD)