def test_replicated_to_arbiter_volume(self):
"""
Description:-
Reduce the replica count from replica 3 to arbiter
"""
# pylint: disable=too-many-statements
# Remove brick to reduce the replica count from replica 3
g.log.info("Removing bricks to form replica 2 volume")
ret = shrink_volume(self.mnode, self.volname, replica_num=0)
self.assertTrue(ret,
"Failed to remove brick on volume %s" % self.volname)
g.log.info("Successfully removed brick on volume %s", self.volname)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(
ret, "Volume %s process not online despite waiting "
"for 300 seconds" % self.volname)
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verifying all bricks online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, "Volume %s : All process are not online" % self.volname)
g.log.info("Volume %s : All process are online", self.volname)
# Adding the bricks to make arbiter brick
g.log.info("Adding bricks to convert to Arbiter Volume")
replica_arbiter = {'replica_count': 1, 'arbiter_count': 1}
ret = expand_volume(self.mnode,
self.volname,
self.servers,
self.all_servers_info,
add_to_hot_tier=False,
**replica_arbiter)
self.assertTrue(ret, "Failed to expand the volume %s" % self.volname)
g.log.info("Changing volume to arbiter volume is successful %s",
self.volname)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(
ret, "Failed to wait for volume %s processes "
"to be online" % self.volname)
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, "Volume %s : All process are not online" % self.volname)
g.log.info("Volume %s : All process are online", self.volname)
def test_volume_create_start_stop_start(self):
"""Tests volume create, start, status, stop, start.
Also Validates whether all the brick process are running after the
start of the volume.
"""
# Verify volume processes are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online" %
self.volname))
g.log.info("Successfully Verified volume %s processes are online",
self.volname)
# Stop Volume
ret, _, _ = volume_stop(self.mnode, self.volname, force=True)
self.assertEqual(ret, 0, "Failed to stop volume %s" % self.volname)
g.log.info("Successfully stopped volume %s", self.volname)
# Start Volume
ret, _, _ = volume_start(self.mnode, self.volname)
self.assertEqual(ret, 0, "Failed to start volume %s" % self.volname)
g.log.info("Successfully started volume %s", self.volname)
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
# Log Volume Info and Status
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to Log volume %s info and status",
self.volname))
g.log.info("Successfully logged Volume %s Info and Status",
self.volname)
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online" %
self.volname))
g.log.info("Successfully verified volume %s processes are online",
self.volname)
# Log Volume Info and Status
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to Log volume %s info and status",
self.volname))
g.log.info("Successfully logged Volume %s Info and Status",
self.volname)
# Check if glusterd is running on all servers(expected: active)
ret = is_glusterd_running(self.servers)
self.assertEqual(ret, 0, "Glusterd is not running on all servers")
g.log.info("Glusterd is running on all the servers")
def _bring_bricks_online_heal(self, mnode, volname, bricks_list):
"""
Bring bricks online and monitor heal completion
"""
# Bring bricks online
ret = bring_bricks_online(
mnode,
volname,
bricks_list,
bring_bricks_online_methods=['volume_start_force'])
self.assertTrue(ret, 'Failed to bring bricks online')
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(mnode, volname)
self.assertTrue(ret, ("Failed to wait for volume {} processes to "
"be online".format(volname)))
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(mnode, volname)
self.assertTrue(
ret, ("Volume {} : All process are not online".format(volname)))
g.log.info("Volume %s : All process are online", volname)
# Monitor heal completion
ret = monitor_heal_completion(mnode, volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check for split-brain
ret = is_volume_in_split_brain(mnode, volname)
self.assertFalse(ret, 'Volume is in split-brain state')
def test_shrinking_volume_when_io_in_progress(self):
"""Test shrinking volume (Decrease distribute count) using existing
servers bricks when IO is in progress.
Description:
- remove brick (start, status, commit)
- validate IO
"""
# Log Volume Info and Status before shrinking the volume.
g.log.info("Logging volume info and Status before shrinking volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status of volume %s",
self.volname)
# Shrinking volume by removing bricks from volume when IO in progress
g.log.info("Start removing bricks from volume when IO in progress")
ret = shrink_volume(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to shrink the volume when IO in "
"progress on volume %s", self.volname))
g.log.info("Shrinking volume when IO in progress is successful on "
"volume %s", self.volname)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info("Successful in waiting for volume %s processes to be "
"online", self.volname)
# Log Volume Info and Status after shrinking the volume
g.log.info("Logging volume info and Status after shrinking volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status of volume %s",
self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online after "
"shrinking volume")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online",
self.volname))
g.log.info("Volume %s : All process are online after shrinking volume",
self.volname)
# Validate IO
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.io_validation_complete = True
self.assertTrue(ret, "IO failed on some of the clients")
# List all files and dirs created
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
self.assertTrue(ret, "Failed to list all files and dirs")
g.log.info("Listing all files and directories is successful")
def test_replace_brick_when_io_in_progress(self):
"""Test replacing brick using existing servers bricks when IO is
in progress.
Description:
- replace_brick
- wait for heal to complete
- validate IO
"""
# Log Volume Info and Status before replacing brick from the volume.
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status of volume %s",
self.volname)
# Replace brick from a sub-volume
ret = replace_brick_from_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, "Failed to replace faulty brick from the volume")
g.log.info("Successfully replaced faulty brick from the volume")
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
# Log Volume Info and Status after replacing the brick
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status of volume %s",
self.volname)
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online", self.volname))
# Wait for self-heal to complete
ret = monitor_heal_completion(self.mnode,
self.volname,
timeout_period=1800)
self.assertTrue(
ret, "Self heal didn't complete even after waiting "
"for 30 minutes. 30 minutes is too much a time for "
"current test workload")
# Validate IO
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.io_validation_complete = True
self.assertTrue(ret, "IO failed on some of the clients")
# List all files and dirs created
ret = list_all_files_and_dirs_mounts(self.mounts)
self.assertTrue(ret, "Failed to list all files and dirs")
def test_volume_create_start_stop_start(self):
"""Tests volume create, start, status, stop, start.
Also Validates whether all the brick process are running after the
start of the volume.
"""
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
# Stop Volume
ret, _, _ = volume_stop(self.mnode, self.volname, force=True)
self.assertEqual(ret, 0, "Failed to stop volume %s" % self.volname)
# Start Volume
ret, _, _ = volume_start(self.mnode, self.volname)
self.assertEqual(ret, 0, "Unable to start volume %s" % self.volname)
time.sleep(15)
# Log Volume Info and Status
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(
ret, ("Logging volume %s info and status failed" % self.volname))
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
# Log Volume Info and Status
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(
ret, ("Logging volume %s info and status failed" % self.volname))
# Verify all glusterd's are running
ret = is_glusterd_running(self.servers)
self.assertEqual(
ret, 0, ("glusterd not running on all servers: %s" % self.servers))
def test_expanding_volume_when_io_in_progress(self):
# pylint: disable=too-many-statements
# Log Volume Info and Status before expanding the volume.
g.log.info("Logging volume info and Status before expanding volume")
log_volume_info_and_status(self.mnode, self.volname)
# Expanding volume by adding bricks to the volume when IO in progress
g.log.info("Start adding bricks to volume when IO in progress")
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to expand the volume while IO in "
"progress on volume %s", self.volname))
g.log.info(
"Expanding volume while IO in progress on "
"volume %s : Success", self.volname)
# Wait for gluster processes to come online
g.log.info("Wait for gluster processes to come online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info("Waiting for volume %s process to be online", self.volname)
# Log Volume Info and Status after expanding the volume
g.log.info("Logging volume info and Status after expanding volume")
log_volume_info_and_status(self.mnode, self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online", self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Start Rebalance
g.log.info("Starting Rebalance on the volume")
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to start rebalance on the volume "
"%s", self.volname))
g.log.info("Started rebalance on the volume %s: Success", self.volname)
# Wait for rebalance to complete
g.log.info("Waiting for rebalance to complete")
ret = wait_for_rebalance_to_complete(self.mnode,
self.volname,
timeout=1800)
self.assertTrue(ret, ("Rebalance is not yet complete on the volume "
"%s", self.volname))
g.log.info("Rebalance status on volume %s: Complete", self.volname)
# Check Rebalance status after rebalance is complete
g.log.info("Checking Rebalance status")
ret, _, _ = rebalance_status(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to get rebalance status for the "
"volume %s", self.volname))
g.log.info("Rebalance status on volume %s: Complete", self.volname)
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.io_validation_complete = True
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("IO on all mounts: Complete")
# List all files and dirs created
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
self.assertTrue(ret, "Failed to list all files and dirs")
g.log.info("List all files and directories: Success")
# DHT Layout validation
g.log.debug("Verifying hash layout values %s:%s", self.clients[0],
self.mounts[0].mountpoint)
ret = validate_files_in_dir(self.clients[0],
self.mounts[0].mountpoint,
test_type=LAYOUT_IS_COMPLETE,
file_type=FILETYPE_DIRS)
self.assertTrue(ret, "LAYOUT_IS_COMPLETE: FAILED")
g.log.info("LAYOUT_IS_COMPLETE: PASS")
# Checking if there are any migration failures
status = get_rebalance_status(self.mnode, self.volname)
for each_node in status['node']:
self.assertEqual(
0, int(each_node['failures']),
"Rebalance failed to migrate few files on %s" %
each_node['nodeName'])
g.log.info("No migration failures on %s", each_node['nodeName'])
def test_afr_dir_entry_creation_with_subvol_down(self):
"""
1. Create a distributed-replicated(3X3)/distributed-arbiter(3X(2+1))
and mount it on one client
2. Kill 3 bricks corresponding to the 1st subvol
3. Unmount and remount the volume on the same client
4. Create deep dir from mount point
mkdir -p dir1/subdir1/deepdir1
5. Create files under dir1/subdir1/deepdir1; touch <filename>
6. Now bring all sub-vols up by volume start force
7. Validate backend bricks for dir creation, the subvol which is
offline will have no dirs created, whereas other subvols will have
dirs created from step 4
8. Trigger heal from client by "#find . | xargs stat"
9. Verify that the directory entries are created on all back-end bricks
10. Create new dir (dir2) on location dir1/subdir1/deepdir1
11. Trigger rebalance and wait for the completion
12. Check backend bricks for all entries of dirs
13. Check if files are getting created on the subvol which was offline
"""
# Bring down first subvol of bricks offline
self.subvols = get_subvols(self.mnode, self.volname)['volume_subvols']
first_subvol = self.subvols[0]
ret = bring_bricks_offline(self.volname, first_subvol)
self.assertTrue(
ret, "Unable to bring {} bricks offline".format(first_subvol))
# Check bricks are offline or not
ret = are_bricks_offline(self.mnode, self.volname, first_subvol)
self.assertTrue(ret, "Bricks {} are still online".format(first_subvol))
# Unmount and remount the volume
ret, _, _ = umount_volume(self.mounts[0].client_system,
self.mounts[0].mountpoint)
self.assertFalse(ret, "Failed to unmount volume.")
ret, _, _ = mount_volume(self.volname, self.mount_type,
self.mounts[0].mountpoint, self.mnode,
self.mounts[0].client_system)
self.assertFalse(ret, "Failed to remount volume.")
g.log.info('Successfully umounted and remounted volume.')
# At this step, sleep is must otherwise file creation will fail
sleep(2)
# Create dir `dir1/subdir1/deepdir1` on mountpont
directory1 = "dir1/subdir1/deepdir1"
path = self.mounts[0].mountpoint + "/" + directory1
ret = mkdir(self.mounts[0].client_system, path, parents=True)
self.assertTrue(ret, "Directory {} creation failed".format(path))
# Create files on the 2nd and 3rd subvols which are online
brickobject = create_brickobjectlist(self.subvols, directory1)
self.assertIsNotNone(brickobject, "Failed to get brick object list")
self._create_number_of_files_on_the_subvol(brickobject[1],
directory1,
5,
mountpath=path)
self._create_number_of_files_on_the_subvol(brickobject[2],
directory1,
5,
mountpath=path)
# Bring bricks online using volume start force
ret, _, err = volume_start(self.mnode, self.volname, force=True)
self.assertEqual(ret, 0, err)
g.log.info("Volume: %s started successfully", self.volname)
# Check all bricks are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, "Few process after volume start are offline for "
"volume: {}".format(self.volname))
# Validate Directory is not created on the bricks of the subvol which
# is offline
for subvol in self.subvols:
self._check_file_exists(subvol,
"/" + directory1,
exists=(subvol != first_subvol))
# Trigger heal from the client
cmd = "cd {}; find . | xargs stat".format(self.mounts[0].mountpoint)
ret, _, err = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, err)
# Validate the directory1 is present on all the bricks
for subvol in self.subvols:
self._check_file_exists(subvol, "/" + directory1, exists=True)
# Create new dir (dir2) on location dir1/subdir1/deepdir1
directory2 = "/" + directory1 + '/dir2'
path = self.mounts[0].mountpoint + directory2
ret = mkdir(self.mounts[0].client_system, path, parents=True)
self.assertTrue(ret, "Directory {} creation failed".format(path))
# Trigger rebalance and validate the completion
ret, _, err = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, err)
g.log.info("Rebalance on volume %s started successfully", self.volname)
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(
ret, "Rebalance didn't complete on the volume: "
"{}".format(self.volname))
# Validate all dirs are present on all bricks in each subvols
for subvol in self.subvols:
for each_dir in ("/" + directory1, directory2):
self._check_file_exists(subvol, each_dir, exists=True)
# Validate if files are getting created on the subvol which was
# offline
self._create_number_of_files_on_the_subvol(brickobject[0],
directory1,
5,
mountpath=path)
def test_metadata_self_heal(self):
"""
Test MetaData Self-Heal (heal command)
Description:
- set the volume option
"metadata-self-heal": "off"
"entry-self-heal": "off"
"data-self-heal": "off"
- create IO
- set the volume option
"self-heal-daemon": "off"
- bring down all bricks processes from selected set
- Change the permissions, ownership and the group
of the files under "test_meta_data_self_heal" folder
- get arequal before getting bricks online
- bring bricks online
- set the volume option
"self-heal-daemon": "on"
- check daemons and start healing
- check is heal is completed
- check for split-brain
- get arequal after getting bricks online and compare with
arequal before getting bricks online
- check group and user are 'qa'
"""
# pylint: disable=too-many-locals,too-many-statements
# Setting options
g.log.info('Setting options...')
options = {"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Options "
"'metadata-self-heal', "
"'entry-self-heal', "
"'data-self-heal', "
"are set to 'off' successfully")
# Creating files on client side
all_mounts_procs = []
test_meta_data_self_heal_folder = 'test_meta_data_self_heal'
g.log.info("Generating data for %s:%s",
self.mounts[0].client_system, self.mounts[0].mountpoint)
# Create files
g.log.info('Creating files...')
command = ("cd %s/ ; "
"mkdir %s ;"
"cd %s/ ;"
"for i in `seq 1 50` ; "
"do dd if=/dev/urandom of=test.$i bs=10k count=1 ; "
"done ;"
% (self.mounts[0].mountpoint,
test_meta_data_self_heal_folder,
test_meta_data_self_heal_folder))
proc = g.run_async(self.mounts[0].client_system, command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# wait for io to complete
self.assertTrue(
wait_for_io_to_complete(all_mounts_procs, self.mounts),
"Io failed to complete on some of the clients")
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Option 'self-heal-daemon' is set to 'off' successfully")
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = list(filter(None, (
bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks'])))
# Bring brick offline
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Changing the permissions, ownership and the group
# of the files under "test_meta_data_self_heal" folder
g.log.info("Modifying data for %s:%s",
self.mounts[0].client_system, self.mounts[0].mountpoint)
# Change permissions to 444
g.log.info('Changing permissions...')
command = ("cd %s/%s/ ; "
"chmod -R 444 *"
% (self.mounts[0].mountpoint,
test_meta_data_self_heal_folder))
ret, out, err = g.run(self.mounts[0].client_system, command)
self.assertEqual(ret, 0, err)
g.log.info('Permissions are changed successfully')
# Change the ownership to qa
g.log.info('Changing the ownership...')
command = ("cd %s/%s/ ; "
"chown -R qa *"
% (self.mounts[0].mountpoint,
test_meta_data_self_heal_folder))
ret, out, err = g.run(self.mounts[0].client_system, command)
self.assertEqual(ret, 0, err)
g.log.info('Ownership is changed successfully')
# Change the group to qa
g.log.info('Changing the group...')
command = ("cd %s/%s/ ; "
"chgrp -R qa *"
% (self.mounts[0].mountpoint,
test_meta_data_self_heal_folder))
ret, out, err = g.run(self.mounts[0].client_system, command)
self.assertEqual(ret, 0, err)
g.log.info('Group is changed successfully')
# Get arequal before getting bricks online
g.log.info('Getting arequal before getting bricks online...')
ret, result_before_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks online '
'is successful')
# Bring brick online
g.log.info('Bringing bricks %s online...', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume process %s not online "
"despite waiting for 5 minutes", self.volname))
g.log.info("Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online"
% self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
g.log.info('Getting arequal after getting bricks online...')
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks online '
'is successful')
# Checking arequals before bringing bricks online
# and after bringing bricks online
self.assertItemsEqual(result_before_online, result_after_online,
'Checksums are not equal')
g.log.info('Checksums before bringing bricks online '
'and after bringing bricks online are equal')
# Adding servers and client in single dict to check permissions
nodes_to_check = {}
all_bricks = get_all_bricks(self.mnode, self.volname)
for brick in all_bricks:
node, brick_path = brick.split(':')
nodes_to_check[node] = brick_path
nodes_to_check[self.mounts[0].client_system] = \
self.mounts[0].mountpoint
# Checking for user and group
for node in nodes_to_check:
# Get file list
command = ("cd %s/%s/ ; "
"ls"
% (nodes_to_check[node],
test_meta_data_self_heal_folder))
ret, out, err = g.run(node, command)
file_list = out.split()
for file_name in file_list:
file_to_check = '%s/%s/%s' % (nodes_to_check[node],
test_meta_data_self_heal_folder,
file_name)
g.log.info('Checking for permissions, user and group for %s',
file_name)
# Check for permissions
cmd = ("stat -c '%a %n' {} | awk '{{print $1}}'"
.format(file_to_check))
ret, permissions, _ = g.run(node, cmd)
self.assertEqual(permissions.split('\n')[0], '444',
'Permissions %s is not equal to 444'
% permissions)
g.log.info("Permissions are '444' for %s", file_name)
# Check for user
cmd = ("ls -ld {} | awk '{{print $3}}'"
.format(file_to_check))
ret, username, _ = g.run(node, cmd)
self.assertEqual(username.split('\n')[0],
'qa', 'User %s is not equal qa'
% username)
g.log.info("User is 'qa' for %s", file_name)
# Check for group
cmd = ("ls -ld {} | awk '{{print $4}}'"
.format(file_to_check))
ret, groupname, _ = g.run(node, cmd)
self.assertEqual(groupname.split('\n')[0],
'qa', 'Group %s is not equal qa'
% groupname)
g.log.info("Group is 'qa' for %s", file_name)
def test_rebalance_with_force(self):
# Getting arequal checksum before rebalance
g.log.info("Getting arequal checksum before rebalance")
arequal_checksum_before_rebalance = collect_mounts_arequal(self.mounts)
# Log Volume Info and Status before expanding the volume.
g.log.info("Logging volume info and Status before expanding volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(
ret, "Logging volume info and status failed on "
"volume %s" % self.volname)
g.log.info(
"Successful in logging volume info and"
"status of volume %s", self.volname)
# Expanding volume by adding bricks to the volume
g.log.info("Start adding bricks to volume")
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Volume %s: Expand failed", self.volname))
g.log.info("Volume %s: Expand successful", self.volname)
# Wait for gluster processes to come online
g.log.info("Wait for gluster processes to come online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s: one or more volume process are "
"not up", self.volname))
g.log.info("All volume %s processes are online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online", self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Log Volume Info and Status after expanding the volume
g.log.info("Logging volume info and Status after expanding volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info(
"Successful in logging volume info and"
"status of volume %s", self.volname)
# Start Rebalance with force
g.log.info("Starting Rebalance on the volume")
ret, _, _ = rebalance_start(self.mnode, self.volname, force=True)
self.assertEqual(ret, 0, ("Volume %s: Failed to start rebalance with "
"force", self.volname))
g.log.info("Volume %s: Started rebalance with force option",
self.volname)
# Wait for rebalance to complete
g.log.info("Waiting for rebalance to complete")
ret = wait_for_rebalance_to_complete(self.mnode,
self.volname,
timeout=600)
self.assertTrue(
ret, ("Volume %s: Rebalance is still in-progress ", self.volname))
g.log.info("Volume %s: Rebalance completed", self.volname)
# Getting arequal checksum after rebalance
g.log.info("Getting arequal checksum after rebalance with force "
"option")
arequal_checksum_after_rebalance = collect_mounts_arequal(self.mounts)
# Comparing arequals checksum before and after rebalance with force
# option
g.log.info("Comparing arequals checksum before and after rebalance"
"with force option")
self.assertEqual(arequal_checksum_before_rebalance,
arequal_checksum_after_rebalance,
"arequal checksum is NOT MATCHNG")
g.log.info("arequal checksum is SAME")
# Checking if rebalance skipped any files
status = get_rebalance_status(self.mnode, self.volname)
for each_node in status['node']:
self.assertEqual(
int(each_node['skipped']), 0,
"Few files are skipped on node %s" % each_node['nodeName'])
g.log.info("No files are skipped on %s", each_node['nodeName'])
def test_snap_self_heal(self):
"""
Steps:
1. create a volume
2. mount volume
3. create snapshot of that volume
4. Activate snapshot
5. Clone snapshot and Mount
6. Perform I/O
7. Bring Down Few bricks from volume without
affecting the volume or cluster.
8. Perform I/O
9. Bring back down bricks to online
10. Validate heal is complete with areequal
"""
# pylint: disable=too-many-statements, too-many-locals
# Creating snapshot:
g.log.info("Starting to Create snapshot")
ret, _, _ = snap_create(self.mnode, self.volname, self.snap)
self.assertEqual(
ret, 0, ("Failed to create snapshot for volume %s" % self.volname))
g.log.info("Snapshot %s created successfully for volume %s", self.snap,
self.volname)
# Activating snapshot
g.log.info("Starting to Activate Snapshot")
ret, _, _ = snap_activate(self.mnode, self.snap)
self.assertEqual(ret, 0,
("Failed to Activate snapshot %s" % self.snap))
g.log.info("Snapshot %s activated successfully", self.snap)
# snapshot list
ret, _, _ = snap_list(self.mnode)
self.assertEqual(ret, 0, ("Failed to list all the snapshot"))
g.log.info("Snapshot list command was successful")
# Creating a Clone volume from snapshot:
g.log.info("Starting to Clone volume from Snapshot")
ret, _, _ = snap_clone(self.mnode, self.snap, self.clone)
self.assertEqual(ret, 0, ("Failed to clone %s from snapshot %s" %
(self.clone, self.snap)))
g.log.info("%s created successfully", self.clone)
# start clone volumes
g.log.info("start to created clone volumes")
ret, _, _ = volume_start(self.mnode, self.clone)
self.assertEqual(ret, 0, "Failed to start clone %s" % self.clone)
g.log.info("clone volume %s started successfully", self.clone)
# Mounting a clone volume
g.log.info("Mounting a clone volume")
ret, _, _ = mount_volume(self.clone, self.mount_type, self.mount1,
self.mnode, self.clients[0])
self.assertEqual(ret, 0,
"Failed to mount clone Volume %s" % self.clone)
g.log.info("Clone volume %s mounted Successfully", self.clone)
# Checking cloned volume mounted or not
ret = is_mounted(self.clone, self.mount1, self.mnode, self.clients[0],
self.mount_type)
self.assertTrue(
ret,
"Failed to mount clone volume on mount point: %s" % self.mount1)
g.log.info("clone Volume %s mounted on %s", self.clone, self.mount1)
# write files on all mounts
g.log.info("Starting IO on all mounts...")
g.log.info("mounts: %s", self.mount1)
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name file %s" %
(self.script_upload_path, self.mount1))
proc = g.run(self.clients[0], cmd)
all_mounts_procs.append(proc)
g.log.info("Successful in creating I/O on mounts")
# get the bricks from the volume
g.log.info("Fetching bricks for the volume : %s", self.clone)
bricks_list = get_all_bricks(self.mnode, self.clone)
g.log.info("Brick List : %s", bricks_list)
# Select bricks to bring offline
g.log.info("Starting to bring bricks to offline")
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = filter(
None, (bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks']))
g.log.info("Brick to bring offline: %s ", bricks_to_bring_offline)
ret = bring_bricks_offline(self.clone, bricks_to_bring_offline)
self.assertTrue(ret, "Failed to bring the bricks offline")
g.log.info("Successful in bringing bricks: %s offline",
bricks_to_bring_offline)
# Offline Bricks list
offline_bricks = get_offline_bricks_list(self.mnode, self.clone)
self.assertIsNotNone(
offline_bricks, "Failed to get offline bricklist"
"for volume %s" % self.clone)
for bricks in offline_bricks:
self.assertIn(bricks, bricks_to_bring_offline,
"Failed to validate "
"Bricks offline")
g.log.info("Bricks Offline: %s", offline_bricks)
# Online Bricks list
online_bricks = get_online_bricks_list(self.mnode, self.clone)
self.assertIsNotNone(
online_bricks, "Failed to get online bricks"
" for volume %s" % self.clone)
g.log.info("Bricks Online: %s", online_bricks)
# write files mountpoint
g.log.info("Starting IO on all mounts...")
g.log.info("mounts: %s", self.mount1)
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name file %s" %
(self.script_upload_path, self.mount1))
proc = g.run(self.clients[0], cmd)
all_mounts_procs.append(proc)
g.log.info("Successful in creating I/O on mounts")
# Bring all bricks online
g.log.info("bring all bricks online")
ret = bring_bricks_online(self.mnode, self.clone,
bricks_to_bring_offline)
self.assertTrue(ret, "Failed to bring bricks online")
g.log.info("Successful in bringing all bricks online")
# Validate Bricks are online
g.log.info("Validating all bricks are online")
ret = are_bricks_online(self.mnode, self.clone, bricks_list)
self.assertTrue(ret, "Failed to bring all the bricks online")
g.log.info("bricks online: %s", bricks_list)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.clone)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online" % self.clone))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.clone)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.clone)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.clone))
g.log.info("Volume %s : All process are online", self.clone)
# wait for the heal process to complete
g.log.info("waiting for heal process to complete")
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, "Failed to complete the heal process")
g.log.info("Successfully completed heal process")
# Check areequal
# get the subvolumes
g.log.info("Starting to get sub-volumes for volume %s", self.clone)
subvols = get_subvols(self.mnode, self.clone)
num_subvols = len(subvols['volume_subvols'])
g.log.info("Number of subvolumes in volume %s:", num_subvols)
# Get arequals and compare
g.log.info("Starting to Compare areequals")
for i in range(0, num_subvols):
# Get arequal for first brick
subvol_brick_list = subvols['volume_subvols'][i]
node, brick_path = subvol_brick_list[0].split(':')
command = ('arequal-checksum -p %s '
'-i .glusterfs -i .landfill -i .trashcan' % brick_path)
ret, arequal, _ = g.run(node, command)
first_brick_total = arequal.splitlines()[-1].split(':')[-1]
# Get arequal for every brick and compare with first brick
for brick in subvol_brick_list:
node, brick_path = brick.split(':')
command = ('arequal-checksum -p %s '
'-i .glusterfs -i .landfill -i .trashcan' % brick_path)
ret, brick_arequal, _ = g.run(node, command)
self.assertFalse(ret, 'Failed to get arequal on brick %s' % brick)
g.log.info('Getting arequal for %s is successful', brick)
brick_total = brick_arequal.splitlines()[-1].split(':')[-1]
self.assertEqual(
first_brick_total, brick_total,
'Arequals for subvol and %s are not equal' % brick)
g.log.info('Arequals for subvol and %s are equal', brick)
g.log.info('All arequals are equal for distributed-replicated')
def test_conservative_merge_of_files_heal_command(self):
"""
- set options:
"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off",
"self-heal-daemon": "off"
- Bring brick 0 offline
- Creating files on client side
- Bring brick 0 online
- Bring brick 1 offline
- Creating files on client side
- Bring brick 1 online
- Get arequal on bricks
- Setting option
"self-heal-daemon": "on"
- Start healing
- Get arequal on bricks and compare with arequals before healing
and mountpoint
"""
# pylint: disable=too-many-statements,too-many-locals
# set options
bricks_list = get_all_bricks(self.mnode, self.volname)
options = {
"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off",
"self-heal-daemon": "off"
}
g.log.info("setting options %s", options)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set volume option %s for"
"volume %s" % (options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# Bring brick 0 offline
g.log.info('Bringing bricks %s offline', bricks_list[0])
ret = bring_bricks_offline(self.volname, bricks_list[0])
self.assertTrue(ret,
'Failed to bring bricks %s offline' % bricks_list[0])
ret = are_bricks_offline(self.mnode, self.volname, [bricks_list[0]])
self.assertTrue(ret, 'Bricks %s are not offline' % bricks_list[0])
g.log.info('Bringing bricks %s offline is successful', bricks_list[0])
# Creating files on client side
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Create files
g.log.info('Creating files...')
command = ("python %s create_deep_dirs_with_files "
"-d 0 -l 5 -f 10 --dirname-start-num 1 %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# Bring brick 0 online
g.log.info('Bringing bricks %s online...', bricks_list[0])
ret = bring_bricks_online(self.mnode, self.volname, [bricks_list[0]])
self.assertTrue(ret,
'Failed to bring bricks %s online' % bricks_list[0])
g.log.info('Bringing bricks %s online is successful', bricks_list[0])
# Bring brick 1 offline
g.log.info('Bringing bricks %s offline', bricks_list[1])
ret = bring_bricks_offline(self.volname, bricks_list[1])
self.assertTrue(ret,
'Failed to bring bricks %s offline' % bricks_list[1])
ret = are_bricks_offline(self.mnode, self.volname, [bricks_list[1]])
self.assertTrue(ret, 'Bricks %s are not offline' % bricks_list[1])
g.log.info('Bringing bricks %s offline is successful', bricks_list[1])
# Creating files on client side
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Create files
g.log.info('Creating files...')
command = ("python %s create_deep_dirs_with_files "
"-d 0 -l 5 -f 10 --dirname-start-num 6 %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# Bring brick 1 online
g.log.info('Bringing bricks %s online...', bricks_list[1])
ret = bring_bricks_online(self.mnode, self.volname, [bricks_list[1]])
self.assertTrue(ret,
'Failed to bring bricks %s online' % bricks_list[1])
g.log.info('Bringing bricks %s online is successful', bricks_list[1])
# Get arequal on bricks
arequals_before_heal = {}
g.log.info('Getting arequal on bricks...')
for brick in bricks_list:
g.log.info('Getting arequal on bricks %s...', brick)
node, brick_path = brick.split(':')
command = ('arequal-checksum -p %s '
'-i .glusterfs -i .landfill -i .trashcan' % brick_path)
ret, arequal, _ = g.run(node, command)
self.assertFalse(ret, 'Failed to get arequal on brick %s' % brick)
g.log.info('Getting arequal for %s is successful', brick)
brick_total = arequal.splitlines()[-1].split(':')[-1]
arequals_before_heal[brick] = brick_total
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequals for mount
g.log.info('Getting arequal before getting bricks offline...')
ret, arequals = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after healing is successful')
mount_point_total = arequals[0].splitlines()[-1].split(':')[-1]
# Get arequal on bricks and compare with mount_point_total
# It should be the same
g.log.info('Getting arequal on bricks...')
arequals_after_heal = {}
for brick in bricks_list:
g.log.info('Getting arequal on bricks %s...', brick)
node, brick_path = brick.split(':')
command = ('arequal-checksum -p %s '
'-i .glusterfs -i .landfill -i .trashcan' % brick_path)
ret, arequal, _ = g.run(node, command)
self.assertFalse(ret, 'Failed to get arequal on brick %s' % brick)
g.log.info('Getting arequal for %s is successful', brick)
brick_total = arequal.splitlines()[-1].split(':')[-1]
arequals_after_heal[brick] = brick_total
self.assertEqual(
mount_point_total, brick_total,
'Arequals for mountpoint and %s are not equal' % brick)
g.log.info('Arequals for mountpoint and %s are equal', brick)
g.log.info('All arequals are equal for replicated')
self.assertNotEqual(
cmp(arequals_before_heal, arequals_after_heal), 0,
'Arequals are equal for bricks '
'before and after healing')
def test_fix_layout_start(self):
# pylint: disable=too-many-statements
# Get arequal checksum before starting fix-layout
g.log.info("Getting arequal checksum before fix-layout")
arequal_checksum_before_fix_layout = collect_mounts_arequal(
self.mounts)
# Log Volume Info and Status before expanding the volume.
g.log.info("Logging volume info and Status before expanding volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(
ret, "Logging volume info and status failed on "
"volume %s" % self.volname)
g.log.info(
"Successful in logging volume info and status of volume "
"%s", self.volname)
# Form brick list for expanding volume
add_brick_list = form_bricks_list_to_add_brick(self.mnode,
self.volname,
self.servers,
self.all_servers_info,
distribute_count=1)
self.assertIsNotNone(add_brick_list,
("Volume %s: Failed to form "
"bricks list to expand", self.volname))
g.log.info("Volume %s: Formed bricks list to expand", self.volname)
# Expanding volume by adding bricks to the volume
g.log.info("Volume %s: Expand start")
ret, _, _ = add_brick(self.mnode, self.volname, add_brick_list)
self.assertEqual(ret, 0, ("Volume %s: Expand failed", self.volname))
g.log.info("Volume %s: Expand successful", self.volname)
# Wait for gluster processes to come online
g.log.info("Wait for gluster processes to come online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s: one or more volume process are "
"not up", self.volname))
g.log.info("All volume %s processes are online", self.volname)
# Log Volume Info and Status after expanding the volume
g.log.info("Logging volume info and Status after expanding volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(
ret, "Logging volume info and status failed on "
"volume %s" % self.volname)
g.log.info(
"Successful in logging volume info and status of volume "
"%s", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online", self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Start Rebalance fix-layout
g.log.info("Starting fix-layout on the volume")
ret, _, _ = rebalance_start(self.mnode, self.volname, fix_layout=True)
self.assertEqual(ret, 0, ("Volume %s: fix-layout start failed"
"%s", self.volname))
g.log.info("Volume %s: fix-layout start success", self.volname)
# Wait for fix-layout to complete
g.log.info("Waiting for fix-layout to complete")
ret = wait_for_fix_layout_to_complete(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s: Fix-layout is still in-progress", self.volname))
g.log.info("Volume %s: Fix-layout completed successfully",
self.volname)
# Check Rebalance status after fix-layout is complete
g.log.info("Checking Rebalance status")
ret, _, _ = rebalance_status(self.mnode, self.volname)
self.assertEqual(
ret, 0,
("Volume %s: Failed to get rebalance status", self.volname))
g.log.info("Volume %s: Successfully got rebalance status",
self.volname)
# Get arequal checksum after fix-layout is complete
g.log.info("arequal after fix-layout is complete")
arequal_checksum_after_fix_layout = collect_mounts_arequal(self.mounts)
# Compare arequals checksum before and after fix-layout
g.log.info("Comparing checksum before and after fix-layout")
self.assertEqual(arequal_checksum_before_fix_layout,
arequal_checksum_after_fix_layout,
"arequal checksum is NOT MATCHNG")
g.log.info("arequal checksum is SAME")
# Check if there are any file migrations after fix-layout
status_info = get_rebalance_status(self.mnode, self.volname)
for node in range(len(status_info['node'])):
status_info = get_rebalance_status(self.mnode, self.volname)
file_migration_count = status_info['node'][node]['files']
self.assertEqual(
int(file_migration_count), 0,
("Server %s: Few files are migrated", self.servers[node]))
g.log.info("Server %s: No files are migrated")
# Check if new bricks contains any files
for brick in add_brick_list:
brick_node, brick_path = brick.split(":")
cmd = ('find %s -type f ! -perm 1000 | grep -ve .glusterfs' %
brick_path)
_, out, _ = g.run(brick_node, cmd)
self.assertEqual(
len(out), 0,
(("Files(excluded linkto files) are present on %s:%s"),
(brick_node, brick_path)))
g.log.info("No files (excluded linkto files) are present on %s:%s",
brick_node, brick_path)
def test_self_heal_when_io_in_progress(self):
"""Test self-heal is successful when IO is in progress.
Description:
- simulate brick down.
- bring bricks online
- wait for heal to complete
- validate IO
"""
# Log Volume Info and Status before simulating brick failure
g.log.info(
"Logging volume info and Status before bringing bricks "
"offlien from the volume %s", self.volname)
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status of volume %s",
self.volname)
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = filter(
None, (bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks']))
# Bring bricks offline
g.log.info("Bringing bricks: %s offline", bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret,
("Failed to bring bricks: %s offline", bricks_to_bring_offline))
g.log.info("Successful in bringing bricks: %s offline",
bricks_to_bring_offline)
# Wait for gluster processes to be offline
time.sleep(10)
# Log Volume Info and Status
g.log.info(
"Logging volume info and Status after bringing bricks "
"offline from the volume %s", self.volname)
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status of volume %s",
self.volname)
# Validate if bricks are offline
g.log.info("Validating if bricks: %s are offline",
bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, "Not all the bricks in list:%s are offline")
g.log.info("Successfully validated that bricks: %s are all offline")
# Add delay before bringing bricks online
time.sleep(40)
# Bring bricks online
g.log.info("Bring bricks: %s online", bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret,
("Failed to bring bricks: %s online", bricks_to_bring_offline))
g.log.info("Successfully brought all bricks:%s online",
bricks_to_bring_offline)
# Wait for gluster processes to be online
time.sleep(10)
# Log Volume Info and Status
g.log.info(
"Logging volume info and Status after bringing bricks "
"online from the volume %s", self.volname)
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status of volume %s",
self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online", self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal to complete
g.log.info("Wait for self-heal to complete")
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(
ret, "Self heal didn't complete even after waiting "
"for 20 minutes. 20 minutes is too much a time for "
"current test workload")
g.log.info("self-heal is successful after replace-brick operation")
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.io_validation_complete = True
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("IO is successful on all mounts")
# List all files and dirs created
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
self.assertTrue(ret, "Failed to list all files and dirs")
g.log.info("Listing all files and directories is successful")
def test_write_io_mount_point_resumed_quorum_restored_x3(self):
"""
- set cluster.quorum-type to auto
- start I/O from the mount point
- Do IO and check on subvols with two nodes to reboot
(do for each subvol)
- get files to delete/create for nodes to be offline
- delete files from mountpoint
- reboot nodes
- creating files on nodes while rebooting
- validate for rofs
- wait for volume processes to be online
- creating files on nodes after rebooting
- validate IO
- Do IO and check on subvols without nodes to reboot
(do for each subvol)
- get files to delete/create for nodes to be online
- delete files from mountpoint
- reboot nodes
- creating files on online nodes while rebooting other nodes
- validate IO
- Do IO and check and reboot two nodes on all subvols
- get files to delete/create for nodes to be offline
- delete files from mountpoint
- reboot nodes
- creating files on nodes while rebooting
- validate for rofs
- wait for volume processes to be online
- creating files on nodes after rebooting
- validate IO
"""
# pylint: disable=too-many-locals,too-many-statements,too-many-branches
# set cluster.quorum-type to auto
options = {"cluster.quorum-type": "auto"}
g.log.info("setting cluster.quorum-type to auto on volume %s",
self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set volume option %s for"
"volume %s" % (options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# Creating files on client side
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = "/usr/bin/env python %s create_files -f 30 %s" % (
self.script_upload_path, mount_obj.mountpoint)
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# Validate IO
self.io_validation_complete = False
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# Do IO and check on subvols with nodes to reboot
subvols_dict = get_subvols(self.mnode, self.volname)
for subvol in subvols_dict['volume_subvols']:
# define nodes to reboot
brick_list = subvol[0:2]
nodes_to_reboot = []
for brick in brick_list:
node, brick_path = brick.split(':')
nodes_to_reboot.append(node)
# get files to delete/create for nodes to be offline
node, brick_path = brick_list[0].split(':')
ret, brick_file_list, _ = g.run(node, 'ls %s' % brick_path)
self.assertFalse(ret, 'Failed to ls files on %s' % node)
file_list = brick_file_list.splitlines()
# delete files from mountpoint
for mount_obj in self.mounts:
g.log.info("Deleting data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ('cd %s/ ; rm -rf %s' %
(mount_obj.mountpoint, ' '.join(file_list)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(
ret, 'Failed to rm file on %s' % mount_obj.client_system)
g.log.info('Files %s are deleted', file_list)
# reboot nodes on subvol and wait while rebooting
g.log.info("Rebooting the nodes %s", nodes_to_reboot)
ret = reboot_nodes(nodes_to_reboot)
self.assertTrue(ret,
'Failed to reboot nodes %s ' % nodes_to_reboot)
# Creating files on nodes while rebooting
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = ("cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list)))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# Validate IO
self.io_validation_complete = False
g.log.info("Validating if IO failed with read-only filesystem")
ret = is_io_procs_fail_with_rofs(self, self.all_mounts_procs,
self.mounts)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on read-only filesystem"))
self.io_validation_complete = True
g.log.info("EXPECTED: "
"Read-only file system in IO while creating file")
# check if nodes are online
counter = 0
timeout = 300
_rc = False
while counter < timeout:
ret, reboot_results = are_nodes_online(nodes_to_reboot)
if not ret:
g.log.info("Nodes are offline, Retry after 5 seconds ... ")
time.sleep(5)
counter = counter + 5
else:
_rc = True
break
if not _rc:
for node in reboot_results:
if reboot_results[node]:
g.log.info("Node %s is online", node)
else:
g.log.error(
"Node %s is offline even after "
"%d minutes", node, timeout / 60.0)
else:
g.log.info("All nodes %s are up and running", nodes_to_reboot)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(
self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Creating files on nodes after rebooting
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = ("cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list)))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# Validate IO
self.io_validation_complete = False
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# Do IO and check on subvols without nodes to reboot
subvols_dict = get_subvols(self.mnode, self.volname)
for subvol in subvols_dict['volume_subvols']:
# define nodes to reboot
brick_list = subvol[0:2]
nodes_to_reboot = []
for brick in brick_list:
node, brick_path = brick.split(':')
nodes_to_reboot.append(node)
# get files to delete/create for nodes to be online
new_subvols_dict = get_subvols(self.mnode, self.volname)
subvol_to_operate = new_subvols_dict['volume_subvols']
subvol_to_operate.remove(subvol)
brick_list_subvol_online = subvol_to_operate[0]
node, brick_path_vol_online = \
brick_list_subvol_online[0].split(':')
ret, brick_file_list, _ = g.run(node,
'ls %s' % brick_path_vol_online)
self.assertFalse(ret, 'Failed to ls files on %s' % node)
file_list = brick_file_list.splitlines()
# delete files from mountpoint
for mount_obj in self.mounts:
g.log.info("Deleting data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ('cd %s/ ; rm -rf %s' %
(mount_obj.mountpoint, ' '.join(file_list)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(
ret, 'Failed to rm file on %s' % mount_obj.client_system)
g.log.info('Files %s are deleted', file_list)
# reboot nodes on subvol and wait while rebooting
g.log.info("Rebooting the nodes %s", nodes_to_reboot)
ret = reboot_nodes(nodes_to_reboot)
self.assertTrue(ret,
'Failed to reboot nodes %s ' % nodes_to_reboot)
# Creating files on nodes while rebooting
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = ("cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list)))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# Validate IO
self.io_validation_complete = False
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# check if nodes are online
counter = 0
timeout = 300
_rc = False
while counter < timeout:
ret, reboot_results = are_nodes_online(nodes_to_reboot)
if not ret:
g.log.info("Nodes are offline, Retry after 5 seconds ... ")
time.sleep(5)
counter = counter + 5
else:
_rc = True
break
if not _rc:
for node in reboot_results:
if reboot_results[node]:
g.log.info("Node %s is online", node)
else:
g.log.error(
"Node %s is offline even after "
"%d minutes", node, timeout / 60.0)
else:
g.log.info("All nodes %s are up and running", nodes_to_reboot)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(
self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Do IO and check and reboot nodes on all subvols
subvols_dict = get_subvols(self.mnode, self.volname)
nodes_to_reboot = []
file_list_for_all_subvols = []
for subvol in subvols_dict['volume_subvols']:
# define nodes to reboot
brick_list = subvol[0:2]
for brick in brick_list:
node, brick_path = brick.split(':')
nodes_to_reboot.append(node)
# get files to delete/create for nodes to be offline
node, brick_path = brick_list[0].split(':')
ret, brick_file_list, _ = g.run(node, 'ls %s' % brick_path)
self.assertFalse(ret, 'Failed to ls files on %s' % node)
file_list = brick_file_list.splitlines()
file_list_for_all_subvols.append(file_list)
# delete files from mountpoint
for mount_obj in self.mounts:
g.log.info("Deleting data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ('cd %s/ ; rm -rf %s' %
(mount_obj.mountpoint, ' '.join(file_list)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, 'Failed to rm file on %s' % node)
g.log.info('Files %s are deleted', file_list)
# reboot nodes on subvol and wait while rebooting
g.log.info("Rebooting the nodes %s", nodes_to_reboot)
ret = reboot_nodes(nodes_to_reboot)
self.assertTrue(ret, 'Failed to reboot nodes %s ' % nodes_to_reboot)
# Creating files on nodes while rebooting
all_mounts_procs, all_mounts_procs_1, all_mounts_procs_2 = [], [], []
# Create files for 1-st subvol and get all_mounts_procs_1
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = (
"cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[0])))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs_1.append(proc)
all_mounts_procs.append(all_mounts_procs_1)
# Create files for 2-st subvol and get all_mounts_procs_2
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = (
"cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[1])))
proc2 = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs_2.append(proc2)
all_mounts_procs.append(all_mounts_procs_2)
for mounts_procs in all_mounts_procs:
# Validate IO
self.io_validation_complete = False
g.log.info("Validating if IO failed with read-only filesystem")
ret = is_io_procs_fail_with_rofs(self, mounts_procs, self.mounts)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on read-only filesystem"))
self.io_validation_complete = True
g.log.info("EXPECTED: "
"Read-only file system in IO while creating file")
# check if nodes are online
counter = 0
timeout = 300
_rc = False
while counter < timeout:
ret, reboot_results = are_nodes_online(nodes_to_reboot)
if not ret:
g.log.info("Nodes are offline, Retry after 5 seconds ... ")
time.sleep(5)
counter = counter + 5
else:
_rc = True
break
if not _rc:
for node in reboot_results:
if reboot_results[node]:
g.log.info("Node %s is online", node)
else:
g.log.error("Node %s is offline even after "
"%d minutes", node, timeout / 60.0)
else:
g.log.info("All nodes %s are up and running", nodes_to_reboot)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Creating files on nodes after rebooting
all_mounts_procs, all_mounts_procs_1, all_mounts_procs_2 = [], [], []
# Create files for 1-st subvol and get all_mounts_procs_1
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = (
"cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[0])))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs_1.append(proc)
all_mounts_procs.append(all_mounts_procs_1)
# Create files for 2-st subvol and get all_mounts_procs_2
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = (
"cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[1])))
proc2 = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs_2.append(proc2)
all_mounts_procs.append(all_mounts_procs_2)
for mounts_procs in all_mounts_procs:
# Validate IO
self.io_validation_complete = False
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
def test_self_heal_differing_in_file_type(self):
"""
testing self heal of files with different file types
with default configuration
Description:
- create IO
- calculate arequal
- bring down all bricks processes from selected set
- calculate arequal and compare with arequal before
getting bricks offline
- modify the data
- arequal before getting bricks online
- bring bricks online
- check daemons and healing completion
- start healing
- calculate arequal and compare with arequal before bringing bricks
online and after bringing bricks online
"""
# pylint: disable=too-many-locals,too-many-statements
# Creating files on client side
all_mounts_procs = []
test_file_type_differs_self_heal_folder = \
'test_file_type_differs_self_heal'
g.log.info("Generating data for %s:%s",
self.mounts[0].client_system, self.mounts[0].mountpoint)
# Creating files
command = ("cd %s/ ; "
"mkdir %s ;"
"cd %s/ ;"
"for i in `seq 1 10` ; "
"do mkdir l1_dir.$i ; "
"for j in `seq 1 5` ; "
"do mkdir l1_dir.$i/l2_dir.$j ; "
"for k in `seq 1 10` ; "
"do dd if=/dev/urandom of=l1_dir.$i/l2_dir.$j/test.$k "
"bs=1k count=$k ; "
"done ; "
"done ; "
"done ; "
% (self.mounts[0].mountpoint,
test_file_type_differs_self_heal_folder,
test_file_type_differs_self_heal_folder))
proc = g.run_async(self.mounts[0].client_system, command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# wait for io to complete
self.assertTrue(
wait_for_io_to_complete(all_mounts_procs, self.mounts),
"Io failed to complete on some of the clients")
# Get arequal before getting bricks offline
g.log.info('Getting arequal before getting bricks offline...')
ret, result_before_offline = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks offline '
'is successful')
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = bricks_to_bring_offline_dict['volume_bricks']
# Bring brick offline
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Get arequal after getting bricks offline
g.log.info('Getting arequal after getting bricks offline...')
ret, result_after_offline = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks offline '
'is successful')
# Checking arequals before bringing bricks offline
# and after bringing bricks offline
self.assertEqual(sorted(result_before_offline),
sorted(result_after_offline),
'Checksums before and after bringing bricks'
' offline are not equal')
g.log.info('Checksums before and after '
'bringing bricks offline are equal')
# Modify the data
all_mounts_procs = []
g.log.info("Modifying data for %s:%s",
self.mounts[0].client_system, self.mounts[0].mountpoint)
command = ("cd %s/%s/ ; "
"for i in `seq 1 10` ; "
"do for j in `seq 1 5` ; "
"do for k in `seq 1 10` ; "
"do rm -f l1_dir.$i/l2_dir.$j/test.$k ; "
"mkdir l1_dir.$i/l2_dir.$j/test.$k ; "
"done ; "
"done ; "
"done ;"
% (self.mounts[0].mountpoint,
test_file_type_differs_self_heal_folder))
proc = g.run_async(self.mounts[0].client_system, command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients"
)
# Get arequal before getting bricks online
g.log.info('Getting arequal before getting bricks online...')
ret, result_before_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks online '
'is successful')
# Bring brick online
g.log.info('Bringing bricks %s online', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info("Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online"
% self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
g.log.info('Getting arequal after getting bricks online...')
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks online '
'is successful')
# Checking arequals before bringing bricks online
# and after bringing bricks online
self.assertEqual(sorted(result_before_online),
sorted(result_after_online),
'Checksums before and after bringing bricks'
' online are not equal')
g.log.info('Checksums before and after bringing bricks online '
'are equal')
def test_expanding_volume_when_io_in_progress(self):
"""Test expanding volume (Increase distribution) using existing
servers bricks when IO is in progress.
Description:
- add bricks
- starts rebalance
- wait for rebalance to complete
- validate IO
"""
# Log Volume Info and Status before expanding the volume.
g.log.info("Logging volume info and Status before expanding volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status of volume %s",
self.volname)
# Expanding volume by adding bricks to the volume when IO in progress
g.log.info("Start adding bricks to volume when IO in progress")
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to expand the volume when IO in "
"progress on volume %s", self.volname))
g.log.info(
"Expanding volume when IO in progress is successful on "
"volume %s", self.volname)
# Wait for gluster processes to come online
time.sleep(30)
# Log Volume Info and Status after expanding the volume
g.log.info("Logging volume info and Status after expanding volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status of volume %s",
self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online", self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Start Rebalance
g.log.info("Starting Rebalance on the volume")
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to start rebalance on the volume "
"%s", self.volname))
g.log.info("Successfully started rebalance on the volume %s",
self.volname)
# Log Rebalance status
g.log.info("Log Rebalance status")
_, _, _ = rebalance_status(self.mnode, self.volname)
# Wait for rebalance to complete
g.log.info("Waiting for rebalance to complete")
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(ret, ("Rebalance is not yet complete on the volume "
"%s", self.volname))
g.log.info("Rebalance is successfully complete on the volume %s",
self.volname)
# Check Rebalance status after rebalance is complete
g.log.info("Checking Rebalance status")
ret, _, _ = rebalance_status(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to get rebalance status for the "
"volume %s", self.volname))
g.log.info("Successfully got rebalance status of the volume %s",
self.volname)
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.io_validation_complete = True
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("IO is successful on all mounts")
# List all files and dirs created
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
self.assertTrue(ret, "Failed to list all files and dirs")
g.log.info("Listing all files and directories is successful")
def test_data_self_heal_daemon_off(self):
"""
Test Data-Self-Heal (heal command)
Description:
- set the volume option
"metadata-self-heal": "off"
"entry-self-heal": "off"
"data-self-heal": "off"
- create IO
- Get areequal before getting bricks offline
- set the volume option
"self-heal-daemon": "off"
- bring down all bricks processes from selected set
- Get areequal after getting bricks offline and compare with
areequal before getting bricks offline
- modify the data
- bring bricks online
- set the volume option
"self-heal-daemon": "on"
- check daemons and start healing
- check if heal is completed
- check for split-brain
- add bricks
- do rebalance
- create 5k files
- while creating files - kill bricks and bring bricks online one by one
in cycle
- validate IO
"""
# Setting options
g.log.info('Setting options...')
options = {"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off",
}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Successfully set %s for volume %s"
% (options, self.volname))
# Creating files on client side
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s"
% (mount_obj.client_system, mount_obj.mountpoint))
# Create files
g.log.info('Creating files...')
command = ("python %s create_files -f 100 --fixed-file-size 1k %s"
% (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
# Get areequal before getting bricks offline
g.log.info('Getting areequal before getting bricks offline...')
ret, result_before_offline = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting areequal before getting bricks offline '
'is successful')
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' is set to 'off' successfully")
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = filter(None, (
bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks']))
# Bring brick offline
g.log.info('Bringing bricks %s offline...' % bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful'
% bricks_to_bring_offline)
# Get areequal after getting bricks offline
g.log.info('Getting areequal after getting bricks offline...')
ret, result_after_offline = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting areequal after getting bricks offline '
'is successful')
# Checking areequals before bringing bricks offline
# and after bringing bricks offline
self.assertEqual(result_before_offline, result_after_offline,
'Checksums before and '
'after bringing bricks online are not equal')
g.log.info('Checksums before and after bringing bricks online '
'are equal')
# Modify the data
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Modifying data for %s:%s" %
(mount_obj.client_system, mount_obj.mountpoint))
# Create files
g.log.info('Creating files...')
command = ("python %s create_files -f 100 --fixed-file-size 10k %s"
% (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
# Bring brick online
g.log.info('Bringing bricks %s online...' % bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful'
% bricks_to_bring_offline)
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info("Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online"
% self.volname))
g.log.info("Volume %s : All process are online" % self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Add bricks
g.log.info("Start adding bricks to volume...")
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to expand the volume %s", self.volname))
g.log.info("Expanding volume is successful on "
"volume %s" % self.volname)
# Do rebalance
ret, out, err = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, 'Failed to start rebalance')
g.log.info('Rebalance is started')
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Rebalance is not completed')
g.log.info('Rebalance is completed successfully')
# Create 1k files
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Modifying data for %s:%s" %
(mount_obj.client_system, mount_obj.mountpoint))
# Create files
g.log.info('Creating files...')
command = ("python %s create_files -f 1000 %s"
% (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Kill all bricks in cycle
bricks_list = get_all_bricks(self.mnode, self.volname)
for brick in bricks_list:
# Bring brick offline
g.log.info('Bringing bricks %s offline' % brick)
ret = bring_bricks_offline(self.volname, [brick])
self.assertTrue(ret, 'Failed to bring bricks %s offline' % brick)
ret = are_bricks_offline(self.mnode, self.volname,
[brick])
self.assertTrue(ret, 'Bricks %s are not offline'
% brick)
g.log.info('Bringing bricks %s offline is successful'
% bricks_to_bring_offline)
# Bring brick online
g.log.info('Bringing bricks %s online...' % brick)
ret = bring_bricks_online(self.mnode, self.volname,
[brick])
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful'
% bricks_to_bring_offline)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info("Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online"
% self.volname))
g.log.info("Volume %s : All process are online" % self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found or"
"more than one self heal daemon process"
"found")
g.log.info("All self-heal-daemons are online")
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
def test_clone_delete_snap(self):
"""
clone from snap of one volume
* Create and Mount the volume
* Enable some volume options
* Creating 2 snapshots and activate
* reset the volume
* create a clone of snapshots created
* Mount both the clones
* Perform I/O on mount point
* Check volume options of cloned volumes
* Create snapshot of the cloned snapshot volume
* cleanup snapshots and volumes
"""
# pylint: disable=too-many-statements, too-many-locals
# Enabling Volume options on the volume and validating
g.log.info("Enabling volume options for volume %s ", self.volname)
options = {" features.uss": "enable"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(
ret, ("Failed to set volume options for volume %s" % self.volname))
g.log.info("Successfully set volume options"
"for volume %s", self.volname)
# Validate feature.uss enabled or not
g.log.info("Validating feature.uss is enabled")
option = "features.uss"
vol_option = get_volume_options(self.mnode, self.volname, option)
self.assertEqual(vol_option['features.uss'], 'enable', "Failed"
" to validate "
"volume options")
g.log.info("Successfully validated volume options"
"for volume %s", self.volname)
# Creating snapshot
g.log.info("Starting to Create snapshot")
for snap_count in range(0, 2):
ret, _, _ = snap_create(self.mnode, self.volname,
"snap%s" % snap_count)
self.assertEqual(
ret, 0,
("Failed to create snapshot for volume %s" % self.volname))
g.log.info("Snapshot snap%s created successfully"
"for volume %s", snap_count, self.volname)
# Activating snapshot
g.log.info("Starting to Activate Snapshot")
for snap_count in range(0, 2):
ret, _, _ = snap_activate(self.mnode, "snap%s" % snap_count)
self.assertEqual(
ret, 0, ("Failed to Activate snapshot snap%s" % snap_count))
g.log.info("Snapshot snap%s activated successfully", snap_count)
# Reset volume:
g.log.info("Starting to Reset Volume")
ret, _, _ = volume_reset(self.mnode, self.volname, force=False)
self.assertEqual(ret, 0, ("Failed to reset volume %s" % self.volname))
g.log.info("Reset Volume on volume %s is Successful", self.volname)
# Validate feature.uss enabled or not
g.log.info("Validating feature.uss is enabled")
option = "features.uss"
vol_option = get_volume_options(self.mnode, self.volname, option)
self.assertEqual(vol_option['features.uss'], 'off', "Failed"
" to validate "
"volume options")
g.log.info("Successfully validated volume options"
"for volume %s", self.volname)
# Verify volume's all process are online
g.log.info("Starting to Verify volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are"
"not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Creating and starting a Clone of snapshot
g.log.info("Starting to Clone Snapshot")
for clone_count in range(0, 2):
ret, _, _ = snap_clone(self.mnode, "snap%s" % clone_count,
"clone%s" % clone_count)
self.assertEqual(ret, 0,
("Failed to clone clone%s volume" % clone_count))
g.log.info("clone%s volume created successfully", clone_count)
# Start Cloned volume
g.log.info("starting to Validate clone volumes are started")
for clone_count in range(0, 2):
ret, _, _ = volume_start(self.mnode, "clone%s" % clone_count)
self.assertEqual(ret, 0, ("Failed to start clone%s" % clone_count))
g.log.info("clone%s started successfully", clone_count)
g.log.info("All the clone volumes are started Successfully")
# Validate Volume start of cloned volume
g.log.info("Starting to Validate Volume start")
for clone_count in range(0, 2):
vol_info = get_volume_info(self.mnode, "clone%s" % clone_count)
if vol_info["clone%s" % clone_count]['statusStr'] != 'Started':
raise ExecutionError("Failed to get volume info for clone%s" %
clone_count)
g.log.info("Volume clone%s is in Started state", clone_count)
# Validate feature.uss enabled or not
g.log.info("Validating feature.uss is enabled")
option = "features.uss"
for clone_count in range(0, 2):
vol_option = get_volume_options(self.mnode,
"clone%s" % clone_count, option)
self.assertEqual(vol_option['features.uss'], 'enable', "Failed"
" to validate"
"volume options")
g.log.info(
"Successfully validated volume options"
"for volume clone%s", clone_count)
# Mount both the cloned volumes
g.log.info("Mounting Cloned Volumes")
for mount_obj in range(0, 2):
self.mpoint = "/mnt/clone%s" % mount_obj
cmd = "mkdir -p %s" % self.mpoint
ret, _, _ = g.run(self.clients[0], cmd)
self.assertEqual(ret, 0, ("Creation of directory %s"
"for mounting"
"volume %s failed: Directory already"
"present" %
(self.mpoint, "clone%s" % mount_obj)))
g.log.info(
"Creation of directory %s for mounting volume %s "
"success", self.mpoint, ("clone%s" % mount_obj))
ret, _, _ = mount_volume("clone%s" % mount_obj, self.mount_type,
self.mpoint, self.mnode, self.clients[0])
self.assertEqual(ret, 0, ("clone%s is not mounted" % mount_obj))
g.log.info("clone%s is mounted Successfully", mount_obj)
# Perform I/O on mount
# Start I/O on all mounts
g.log.info("Starting to Perform I/O on Mountpoint")
all_mounts_procs = []
for mount_obj in range(0, 2):
cmd = ("cd /mnt/clone%s/; for i in {1..10};"
"do touch file$i; done; cd;") % mount_obj
proc = g.run(self.clients[0], cmd)
all_mounts_procs.append(proc)
g.log.info("I/O on mountpoint is successful")
# create snapshot
g.log.info("Starting to Create snapshot of clone volume")
ret0, _, _ = snap_create(self.mnode, "clone0", "snap2")
self.assertEqual(ret0, 0, "Failed to create the snapshot"
"snap2 from clone0")
g.log.info("Snapshots snap2 created successfully from clone0")
ret1, _, _ = snap_create(self.mnode, "clone1", "snap3")
self.assertEqual(ret1, 0, "Failed to create the snapshot snap3"
"from clone1")
g.log.info("Snapshots snap3 created successfully from clone1")
# Listing all Snapshots present
g.log.info("Starting to list all snapshots")
ret, _, _ = snap_list(self.mnode)
self.assertEqual(ret, 0, ("Failed to list snapshots present"))
g.log.info("Snapshots successfully listed")
def test_self_heal_50k_files_heal_command_by_add_brick(self):
"""
Test self-heal of 50k files (heal command)
Description:
- Set the volume option
"metadata-self-heal": "off"
"entry-self-heal": "off"
"data-self-heal": "off"
"self-heal-daemon": "off"
- Bring down all bricks processes from selected set
- Create IO (50k files)
- Get arequal before getting bricks online
- Bring bricks online
- Set the volume option
"self-heal-daemon": "on"
- Check for daemons
- Start healing
- Check if heal is completed
- Check for split-brain
- Get arequal after getting bricks online and compare with
arequal before getting bricks online
- Add bricks
- Do rebalance
- Get arequal after adding bricks and compare with
arequal after getting bricks online
"""
# pylint: disable=too-many-locals,too-many-statements
# Setting options
options = {
"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off",
"self-heal-daemon": "off"
}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Successfully set %s for volume %s", options, self.volname)
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = list(
filter(None, (bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks'])))
# Bring brick offline
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Creating files on client side
all_mounts_procs = []
# Create 50k files
g.log.info('Creating files...')
command = ("cd %s ; "
"for i in `seq 1 50000` ; "
"do dd if=/dev/urandom of=test.$i "
"bs=100k count=1 ; "
"done ;" % self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts[0]),
"IO failed on some of the clients")
# Get arequal before getting bricks online
ret, result_before_online = collect_mounts_arequal(self.mounts[0])
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks online '
'is successful')
# Bring brick online
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s online' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Setting options
ret = set_volume_options(self.mnode, self.volname,
{"self-heal-daemon": "on"})
self.assertTrue(ret, 'Failed to set option self-heal-daemon to ON.')
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode,
self.volname,
timeout_period=3600)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
ret, result_after_online = collect_mounts_arequal(self.mounts[0])
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks online '
'is successful')
# Checking arequals before bringing bricks online
# and after bringing bricks online
self.assertItemsEqual(
result_before_online, result_after_online, 'Checksums before and '
'after bringing bricks online are not equal')
g.log.info('Checksums before and after bringing bricks online '
'are equal')
# Add bricks
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to expand the volume when IO in "
"progress on volume %s", self.volname))
g.log.info("Expanding volume is successful on volume %s", self.volname)
# Do rebalance and wait for it to complete
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, 'Failed to start rebalance')
g.log.info('Rebalance is started')
ret = wait_for_rebalance_to_complete(self.mnode,
self.volname,
timeout=3600)
self.assertTrue(ret, 'Rebalance is not completed')
g.log.info('Rebalance is completed successfully')
# Get arequal after adding bricks
ret, result_after_adding_bricks = collect_mounts_arequal(
self.mounts[0])
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks ' 'is successful')
# Checking arequals after bringing bricks online
# and after adding bricks
self.assertItemsEqual(
result_after_online, result_after_adding_bricks,
'Checksums after bringing bricks online and '
'after adding bricks are not equal')
g.log.info('Checksums after bringing bricks online and '
'after adding bricks are equal')
def test_replace_brick_self_heal_io_in_progress(self):
"""
- Create directory on mount point and write files/dirs
- Create another set of files (1K files)
- While creation of files/dirs are in progress Kill one brick
- Remove the contents of the killed brick(simulating disk replacement)
- When the IO's are still in progress, restart glusterd on the nodes
where we simulated disk replacement to bring back bricks online
- Start volume heal
- Wait for IO's to complete
- Verify whether the files are self-healed
- Calculate arequals of the mount point and all the bricks
"""
# pylint: disable=too-many-locals,too-many-statements,too-many-branches
# Create dirs with files
g.log.info('Creating dirs with file...')
command = ("/usr/bin/env python %s create_deep_dirs_with_files "
"-d 2 -l 2 -n 2 -f 10 %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
ret, _, err = g.run(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
self.assertFalse(ret, err)
g.log.info("IO is successful")
# Creating another set of files (1K files)
self.all_mounts_procs = []
# Create dirs with files
g.log.info('Creating 1K files...')
command = ("/usr/bin/env python %s create_files "
"-f 1500 --fixed-file-size 10k %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
ret = validate_io_procs(self.all_mounts_procs, self.mounts[0])
self.assertTrue(ret, "IO failed on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = bricks_to_bring_offline_dict['volume_bricks']
# Bring brick offline
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Remove the content of the killed bricks
for brick in bricks_to_bring_offline:
brick_node, brick_path = brick.split(':')
# Removing files
command = ('cd %s ; rm -rf *' % brick_path)
ret, _, err = g.run(brick_node, command)
self.assertFalse(ret, err)
g.log.info('Files are deleted on brick %s', brick)
# Bring brick online
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s online' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal daemons are online")
# Start healing
ret = trigger_heal_full(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Check arequals for "replicated"
all_bricks = get_all_bricks(self.mnode, self.volname)
if self.volume_type == "replicated":
# Get arequal after bricks are online
ret, arequals = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after successfully bringing'
'bricks online.')
mount_point_total = arequals[0].splitlines()[-1].split(':')[-1]
# Get arequal on bricks and compare with mount_point_total
ret, arequals = collect_bricks_arequal(all_bricks)
self.assertTrue(ret, 'Failed to get arequal on bricks')
for arequal in arequals:
brick_total = arequal.splitlines()[-1].split(':')[-1]
self.assertEqual(
mount_point_total, brick_total,
'Arequals for mountpoint and brick '
'are not equal')
g.log.info('Arequals for mountpoint and brick are equal')
# Check arequals for "distributed-replicated"
if self.volume_type == "distributed-replicated":
# Get the subvolumes
subvols_dict = get_subvols(self.mnode, self.volname)
num_subvols = len(subvols_dict['volume_subvols'])
g.log.info("Number of subvolumes in volume %s:", num_subvols)
# Get arequals and compare
for i in range(0, num_subvols):
# Get arequal for first brick
subvol_brick_list = subvols_dict['volume_subvols'][i]
ret, arequal = collect_bricks_arequal(subvol_brick_list[0])
self.assertTrue(ret, 'Failed to get arequal on first brick')
first_brick_total = arequal[0].splitlines()[-1].split(':')[-1]
# Get arequal for every brick and compare with first brick
ret, arequals = collect_bricks_arequal(subvol_brick_list)
self.assertTrue(ret, 'Failed to get arequal on bricks')
for arequal in arequals:
brick_total = arequal.splitlines()[-1].split(':')[-1]
self.assertEqual(
first_brick_total, brick_total,
'Arequals for subvol and brick are '
'not equal')
g.log.info('Arequals for subvol and brick are equal')
def test_self_heal_50k_files(self):
"""
Description:
- Select bricks to bring offline
- Bring brick offline
- Create 50k files
- Validate IO
- Bring bricks online
- Monitor heal
- Check for split-brain
- Validate IO
"""
# pylint: disable=too-many-statements,too-many-locals
# Select bricks to bring offline
bricks_to_bring_offline_dict = select_bricks_to_bring_offline(
self.mnode, self.volname)
bricks_to_bring_offline = bricks_to_bring_offline_dict['volume_bricks']
# Bring brick offline
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
self.assertIsNotNone(bricks_to_bring_offline, "List is empty")
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Create 50k files
command = ("cd %s ; "
"for i in `seq 1 50000` ; "
"do dd if=/dev/urandom of=test.$i "
"bs=100k count=1 ; "
"done ;" % self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
# Validate IO
self.assertTrue(validate_io_procs([proc], self.mounts[0]),
"IO failed on some of the clients")
# Bring brick online
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s online' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Monitor heal completion
ret = monitor_heal_completion(self.mnode,
self.volname,
timeout_period=3000)
self.assertTrue(ret, 'Heal has not yet completed')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
def test_rebalance_with_hidden_files(self):
# pylint: disable=too-many-statements
# Start IO on mounts
g.log.info("Starting IO on all mounts...")
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("python %s create_files "
"--base-file-name . "
"-f 99 %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# validate IO
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# List all files and dirs created
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
self.assertTrue(ret, "Failed to list all files and dirs")
g.log.info("Listing all files and directories is successful")
# Verify DHT values across mount points
for mount_obj in self.mounts:
g.log.debug("Verifying hash layout values %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
ret = validate_files_in_dir(mount_obj.client_system,
mount_obj.mountpoint,
test_type=FILE_ON_HASHED_BRICKS,
file_type=FILETYPE_FILES)
self.assertTrue(
ret, "Expected - Files are created on only "
"sub-volume according to its hashed value")
g.log.info("Hash layout values are verified %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
# Getting areequal checksum before rebalance
g.log.info("Getting areequal checksum before rebalance")
arequal_checksum_before_rebalance = collect_mounts_arequal(self.mounts)
# Log Volume Info and Status before expanding the volume.
g.log.info("Logging volume info and Status before expanding volume")
log_volume_info_and_status(self.mnode, self.volname)
# Expanding volume by adding bricks to the volume
g.log.info("Start adding bricks to volume")
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to expand the volume %s", self.volname))
g.log.info("Expanding volume is successful on "
"volume %s", self.volname)
# Wait for gluster processes to come online
g.log.info("Wait for gluster processes to come online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online ", self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Log Volume Info and Status after expanding the volume
g.log.info("Logging volume info and Status after expanding volume")
log_volume_info_and_status(self.mnode, self.volname)
# Start Rebalance
g.log.info("Starting Rebalance on the volume")
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to start rebalance on the volume "
"%s", self.volname))
g.log.info("Successfully started rebalance on the volume %s",
self.volname)
# Wait for rebalance to complete
g.log.info("Waiting for rebalance to complete")
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(ret, ("Rebalance is not yet complete on the volume "
"%s", self.volname))
g.log.info("Rebalance is successfully complete on the volume %s",
self.volname)
# Checking if there are any migration failures
status = get_rebalance_status(self.mnode, self.volname)
for each_node in status['node']:
failed_files_count = int(each_node['failures'])
self.assertEqual(
failed_files_count, 0,
"Rebalance failed to migrate few files on %s" %
each_node['nodeName'])
g.log.info("There are no migration failures")
# Getting areequal checksum after rebalance
g.log.info("Getting areequal checksum after rebalance")
arequal_checksum_after_rebalance = collect_mounts_arequal(self.mounts)
# Comparing arequals checksum before and after rebalance
g.log.info("Comparing arequals checksum before and after rebalance")
self.assertEqual(arequal_checksum_before_rebalance,
arequal_checksum_after_rebalance,
"arequal checksum is NOT MATCHNG")
g.log.info("arequal checksum is SAME")
def test_self_heal_with_diff_algorithm(self):
"""
Test Steps:
1. Create a replicated/distributed-replicate volume and mount it
2. Set data/metadata/entry-self-heal to off and
data-self-heal-algorithm to diff
3. Create few files inside a directory with some data
4. Check arequal of the subvol and all the bricks in the subvol should
have same checksum
5. Bring down a brick from the subvol and validate it is offline
6. Modify the data of existing files under the directory
7. Bring back the brick online and wait for heal to complete
8. Check arequal of the subvol and all the brick in the same subvol
should have same checksum
"""
# Setting options
for key, value in (("data-self-heal",
"off"), ("metadata-self-heal",
"off"), ("entry-self-heal", "off"),
("data-self-heal-algorithm", "diff")):
ret = set_volume_options(self.mnode, self.volname, {key: value})
self.assertTrue(ret, 'Failed to set %s to %s.' % (key, value))
g.log.info("%s set to %s successfully", key, value)
# Create few files under a directory with data
mountpoint = self.mounts[0].mountpoint
client = self.mounts[0].client_system
cmd = ("mkdir %s/test_diff_self_heal ; cd %s/test_diff_self_heal ;"
"for i in `seq 1 100` ; do dd if=/dev/urandom of=file.$i "
" bs=1M count=1; done;" % (mountpoint, mountpoint))
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, "Failed to create file on mountpoint")
g.log.info("Successfully created files on mountpoint")
# Check arequal checksum of all the bricks is same
subvols = get_subvols(self.mnode, self.volname)['volume_subvols']
for subvol in subvols:
ret, arequal_from_the_bricks = collect_bricks_arequal(subvol)
self.assertTrue(
ret, "Arequal is collected successfully across "
"the bricks in the subvol {}".format(subvol))
cmd = len(set(arequal_from_the_bricks))
if (self.volume_type == "arbiter"
or self.volume_type == "distributed-arbiter"):
cmd = len(set(arequal_from_the_bricks[:2]))
self.assertEqual(
cmd, 1, "Arequal"
" is same on all the bricks in the subvol")
# List a brick in each subvol and bring them offline
brick_to_bring_offline = []
for subvol in subvols:
self.assertTrue(subvol, "List is empty")
brick_to_bring_offline.extend(sample(subvol, 1))
ret = bring_bricks_offline(self.volname, brick_to_bring_offline)
self.assertTrue(
ret,
"Unable to bring brick: {} offline".format(brick_to_bring_offline))
# Validate the brick is offline
ret = are_bricks_offline(self.mnode, self.volname,
brick_to_bring_offline)
self.assertTrue(
ret, "Brick:{} is still online".format(brick_to_bring_offline))
# Modify files under test_diff_self_heal directory
cmd = ("for i in `seq 1 100` ; do truncate -s 0 file.$i ; "
"truncate -s 2M file.$i ; done;")
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, "Failed to modify the files")
g.log.info("Successfully modified files")
# Start volume with force to bring all bricks online
ret, _, _ = volume_start(self.mnode, self.volname, force=True)
self.assertEqual(ret, 0, "Volume start with force failed")
g.log.info("Volume: %s started successfully", self.volname)
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online", self.volname))
# Monitor heal completion
self.assertTrue(
monitor_heal_completion(self.mnode,
self.volname,
interval_check=10),
"Heal failed after 20 mins")
# Check are there any files in split-brain
self.assertFalse(
is_volume_in_split_brain(self.mnode, self.volname),
"Some files are in split brain for "
"volume: {}".format(self.volname))
# Check arequal checksum of all the bricks is same
for subvol in subvols:
ret, arequal_from_the_bricks = collect_bricks_arequal(subvol)
self.assertTrue(
ret, "Arequal is collected successfully across "
"the bricks in the subvol {}".format(subvol))
cmd = len(set(arequal_from_the_bricks))
if (self.volume_type == "arbiter"
or self.volume_type == "distributed-arbiter"):
cmd = len(set(arequal_from_the_bricks[:2]))
self.assertEqual(
cmd, 1, "Arequal"
" is same on all the bricks in the subvol")
def test_replace_brick_when_io_in_progress(self):
"""Test replacing brick using existing servers bricks when IO is
in progress.
Description:
- replace_brick
- wait for heal to complete
- validate IO
"""
# Log Volume Info and Status before replacing brick from the volume.
g.log.info(
"Logging volume info and Status before replacing brick "
"from the volume %s", self.volname)
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status of volume %s",
self.volname)
# Replace brick from a sub-volume
g.log.info("Replace a faulty brick from the volume")
ret = replace_brick_from_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, "Failed to replace faulty brick from the volume")
g.log.info("Successfully replaced faulty brick from the volume")
# Wait for gluster processes to come online
time.sleep(30)
# Log Volume Info and Status after replacing the brick
g.log.info(
"Logging volume info and Status after replacing brick "
"from the volume %s", self.volname)
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status of volume %s",
self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online", self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal to complete
g.log.info("Wait for self-heal to complete")
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(
ret, "Self heal didn't complete even after waiting "
"for 20 minutes. 20 minutes is too much a time for "
"current test workload")
g.log.info("self-heal is successful after replace-brick operation")
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.io_validation_complete = True
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("IO is successful on all mounts")
# List all files and dirs created
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
self.assertTrue(ret, "Failed to list all files and dirs")
g.log.info("Listing all files and directories is successful")
def test_self_heal_algorithm_full_daemon_off(self):
"""""
Description:-
Checking healing when algorithm is set to "full" and self heal daemon
is "off".
"""""
# pylint: disable=too-many-statements
# Setting volume option of self heal & algorithm
options = {"metadata-self-heal": "disable",
"entry-self-heal": "disable",
"data-self-heal": "disable",
"data-self-heal-algorithm": "full",
"self-heal-daemon": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, "Failed to set the volume options %s" % options)
g.log.info(" Volume set options success")
# Select bricks to bring down
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = bricks_to_bring_offline_dict['volume_bricks']
g.log.info("Bringing bricks: %s offline", bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, "Failed to bring bricks: %s offline"
% bricks_to_bring_offline)
g.log.info("Successful in bringing bricks: %s offline",
bricks_to_bring_offline)
# Validate if bricks are offline
g.log.info("Validating if bricks: %s are offline",
bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, "Not all the bricks in list:%s are offline"
% bricks_to_bring_offline)
g.log.info("Successfully validated that bricks %s are all offline",
bricks_to_bring_offline)
# IO on the mount point
all_mounts_procs = []
g.log.info("Creating Files on %s:%s", self.mounts[0].client_system,
self.mounts[0].mountpoint)
cmd = ("cd %s ;for i in `seq 1 100` ;"
"do dd if=/dev/urandom of=file$i bs=1M "
"count=1;done"
% self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients"
)
# Collecting Arequal before bring the bricks up
g.log.info("Collecting Arequal before the bring of bricks down")
result_before = collect_mounts_arequal(self.mounts)
# Turning self heal daemon ON
optionstwo = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, optionstwo)
self.assertTrue(ret, "Failed to turn self-heal ON")
g.log.info("Volume set options %s: success", optionstwo)
# Bring bricks online
g.log.info("Bring bricks: %s online", bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, "Failed to bring bricks: %s online"
% bricks_to_bring_offline)
g.log.info("Successfully brought all bricks:%s online",
bricks_to_bring_offline)
# Waiting for bricks to come online
g.log.info("Waiting for brick process to come online")
ret = wait_for_bricks_to_be_online(self.mnode,
self.volname,
timeout=30)
self.assertTrue(ret, "bricks didn't come online after adding bricks")
g.log.info("Bricks are online")
# Verifying all bricks online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, "Volume %s : All process are not online"
% self.volname)
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self heal processes to come online
g.log.info("Wait for selfheal process to come online")
ret = wait_for_self_heal_daemons_to_be_online(self.mnode,
self.volname,
timeout=300)
self.assertTrue(ret, "Self-heal process are not online")
g.log.info("All self heal process are online")
# Wait for self-heal to complete
g.log.info("Wait for self-heal to complete")
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, "Self heal didn't complete even after waiting "
"for 20 minutes. 20 minutes is too much a time for "
"current test workload")
g.log.info("self-heal is successful after replace-brick operation")
# arequal after healing
g.log.info("Collecting Arequal before the bring of bricks down")
result_after = collect_mounts_arequal(self.mounts)
# Comparing the results
g.log.info("comparing both the results")
self.assertEqual(result_before, result_after, "Arequals are not equal")
def test_self_heal_daemon(self):
"""
Test Data-Self-Heal(heal command)
Description:
- Create directory test_hardlink_self_heal
- Create directory test_data_self_heal
- Creating files for hardlinks and data files
- Get arequal before getting bricks offline
- Select bricks to bring offline
- Bring brick offline
- Create hardlinks and append data to data files
- Bring brick online
- Wait for volume processes to be online
- Verify volume's all process are online
- Monitor heal completion
- Check for split-brain
- Get arequal after getting bricks online
- Select bricks to bring offline
- Bring brick offline
- Truncate data to data files and verify hardlinks
- Bring brick online
- Wait for volume processes to be online
- Verify volume's all process are online
- Monitor heal completion
- Check for split-brain
- Get arequal again
"""
# pylint: disable=too-many-branches,too-many-statements,too-many-locals
# Creating directory test_hardlink_self_heal
ret = mkdir(
self.mounts[0].client_system,
"{}/test_hardlink_self_heal".format(self.mounts[0].mountpoint))
self.assertTrue(ret, "Failed to create directory")
g.log.info(
"Directory 'test_hardlink_self_heal' on %s created "
"successfully", self.mounts[0])
# Creating directory test_data_self_heal
ret = mkdir(self.mounts[0].client_system,
"{}/test_data_self_heal".format(self.mounts[0].mountpoint))
self.assertTrue(ret, "Failed to create directory")
g.log.info(
"Directory test_hardlink_self_heal on %s created "
"successfully", self.mounts[0])
# Creating files for hardlinks and data files
cmd = ('cd %s/test_hardlink_self_heal;for i in `seq 1 5`;'
'do mkdir dir.$i ; for j in `seq 1 10` ; do dd if='
'/dev/urandom of=dir.$i/file.$j bs=1k count=$j;done; done;'
'cd ..' % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Failed to create file on mountpoint")
g.log.info("Successfully created files on mountpoint")
cmd = ('cd %s/test_data_self_heal;for i in `seq 1 100`;'
'do dd if=/dev/urandom of=file.$i bs=128K count=$i;done;'
'cd ..' % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Failed to create file on mountpoint")
g.log.info("Successfully created files on mountpoint")
# Get arequal before getting bricks offline
ret, result_before_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Arequal before getting bricks online-%s',
result_before_online)
# Select bricks to bring offline
bricks_to_bring_offline = select_volume_bricks_to_bring_offline(
self.mnode, self.volname)
self.assertIsNotNone(bricks_to_bring_offline, "List is empty")
# Bring brick offline
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks {} offline'.format(
bricks_to_bring_offline))
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Bricks {} are not offline'.format(bricks_to_bring_offline))
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Append data to data files and create hardlinks
cmd = ('cd %s/test_data_self_heal;for i in `seq 1 100`;'
'do dd if=/dev/urandom of=file.$i bs=512K count=$i ; done ;'
'cd .. ' % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Failed to modify data files.")
g.log.info("Successfully modified data files")
cmd = ('cd %s/test_hardlink_self_heal;for i in `seq 1 5` ;do '
'for j in `seq 1 10`;do ln dir.$i/file.$j dir.$i/link_file.$j;'
'done ; done ; cd .. ' % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Hardlinks creation failed")
g.log.info("Successfully created hardlinks of files")
# Bring bricks online
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret,
'Failed to bring bricks {} online'.format(bricks_to_bring_offline))
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume {} processes to "
"be online".format(self.volname)))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret,
("Volume {} : All process are not online".format(self.volname)))
g.log.info("Volume %s : All process are online", self.volname)
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Arequal after getting bricks online '
'is %s', result_after_online)
# Select bricks to bring offline
bricks_to_bring_offline = select_volume_bricks_to_bring_offline(
self.mnode, self.volname)
self.assertIsNotNone(bricks_to_bring_offline, "List is empty")
# Bring brick offline
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks {} offline'.format(
bricks_to_bring_offline))
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Bricks {} are not offline'.format(bricks_to_bring_offline))
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Truncate data to data files and verify hardlinks
cmd = ('cd %s/test_data_self_heal ; for i in `seq 1 100` ;'
'do truncate -s $(( $i * 128)) file.$i ; done ; cd ..' %
self.mounts[0].mountpoint)
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Failed to truncate files")
g.log.info("Successfully truncated files on mountpoint")
file_path = ('%s/test_hardlink_self_heal/dir{1..5}/file{1..10}' %
(self.mounts[0].mountpoint))
link_path = ('%s/test_hardlink_self_heal/dir{1..5}/link_file{1..10}' %
(self.mounts[0].mountpoint))
file_stat = get_file_stat(self.mounts[0], file_path)
link_stat = get_file_stat(self.mounts[0], link_path)
self.assertEqual(file_stat, link_stat, "Verification of hardlinks "
"failed")
g.log.info("Successfully verified hardlinks")
# Bring brick online
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret,
'Failed to bring bricks {} online'.format(bricks_to_bring_offline))
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume {} processes to "
"be online".format(self.volname)))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret,
("Volume {} : All process are not online".format(self.volname)))
g.log.info("Volume %s : All process are online", self.volname)
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
def test_gluster_clone_heal(self):
"""
Test gluster compilation on mount point(Heal command)
- Creating directory test_compilation
- Compile gluster on mountpoint
- Select bricks to bring offline
- Bring brick offline
- Validate IO
- Bring bricks online
- Wait for volume processes to be online
- Verify volume's all process are online
- Monitor heal completion
- Check for split-brain
- Get arequal after getting bricks online
- Compile gluster on mountpoint again
- Select bricks to bring offline
- Bring brick offline
- Validate IO
- Bring bricks online
- Wait for volume processes to be online
- Verify volume's all process are online
- Monitor heal completion
- Check for split-brain
- Get arequal after getting bricks online
"""
# pylint: disable=too-many-branches,too-many-statements,too-many-locals
# Creating directory test_compilation
ret = mkdir(self.mounts[0].client_system,
"{}/test_compilation".format(self.mounts[0].mountpoint))
self.assertTrue(ret, "Failed to create directory")
g.log.info(
"Directory 'test_compilation' on %s created "
"successfully", self.mounts[0])
# Compile gluster on mountpoint
cmd = ("cd %s/test_compilation ; rm -rf glusterfs; git clone"
" git://github.com/gluster/glusterfs.git ; cd glusterfs ;"
" ./autogen.sh ;./configure CFLAGS='-g3 -O0 -DDEBUG'; make ;"
" cd ../..;" % self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system, cmd)
# Select bricks to bring offline
bricks_to_bring_offline = select_volume_bricks_to_bring_offline(
self.mnode, self.volname)
self.assertIsNotNone(bricks_to_bring_offline, "List is empty")
# Bring brick offline
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks {} offline'.format(
bricks_to_bring_offline))
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Bricks {} are not offline'.format(bricks_to_bring_offline))
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Validate IO
self.assertTrue(validate_io_procs([proc], self.mounts[0]),
"IO failed on some of the clients")
# Bring bricks online
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret,
'Failed to bring bricks {} online'.format(bricks_to_bring_offline))
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume {} processes to "
"be online".format(self.volname)))
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret,
("Volume {} : All process are not online".format(self.volname)))
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info("Arequal of mountpoint %s", result_after_online)
# Compile gluster on mountpoint again
proc1 = g.run_async(self.mounts[0].client_system, cmd)
# Select bricks to bring offline
bricks_to_bring_offline = select_volume_bricks_to_bring_offline(
self.mnode, self.volname)
self.assertIsNotNone(bricks_to_bring_offline, "List is empty")
# Bring brick offline
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks {} offline'.format(
bricks_to_bring_offline))
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Bricks {} are not offline'.format(bricks_to_bring_offline))
# Validate IO
self.assertTrue(validate_io_procs([proc1], self.mounts[0]),
"IO failed on some of the clients")
# Bring bricks online
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret,
'Failed to bring bricks {} online'.format(bricks_to_bring_offline))
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume {} processes to "
"be online".format(self.volname)))
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret,
("Volume {} : All process are not online".format(self.volname)))
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
# Get arequal after getting bricks online
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info("Arequal of mountpoint %s", result_after_online)
def test_data_self_heal_algorithm_full_default(self):
"""
Test Volume Option - 'cluster.data-self-heal-algorithm' : 'full'
Description:
- set the volume option "data-self-heal-algorithm" to value "full"
- create IO
- bring down all bricks processes from selected set
- modify the data
- calculate arequal
- bring bricks online
- start healing
- calculate arequal and compare with arequal before bringing bricks
offline and after bringing bricks online
"""
# pylint: disable=too-many-locals,too-many-statements
# Setting options
g.log.info('Setting options "data-self-heal-algorithm": "full"...')
options = {"data-self-heal-algorithm": "full"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Option 'data-self-heal-algorithm' is set to 'full' "
"successfully")
# Creating files on client side
all_mounts_procs = []
g.log.info("Generating data for %s:%s", self.mounts[0].client_system,
self.mounts[0].mountpoint)
# Creating files
command = "/usr/bin/env python %s create_files -f 100 %s" % (
self.script_upload_path, self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = list(
filter(None, (bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks'])))
# Bring brick offline
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Modify the data
all_mounts_procs = []
g.log.info("Modifying data for %s:%s", self.mounts[0].client_system,
self.mounts[0].mountpoint)
command = ("/usr/bin/env python %s create_files -f 100 "
"--fixed-file-size 1M %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# Get arequal before getting bricks online
g.log.info('Getting arequal before getting bricks online...')
ret, result_before_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks online '
'is successful')
# Bring brick online
g.log.info('Bringing bricks %s online...', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s online' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
g.log.info('Getting arequal after getting bricks online...')
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks online '
'is successful')
# Checking arequals before bringing bricks online
# and after bringing bricks online
self.assertItemsEqual(result_before_online, result_after_online,
'Checksums are not equal')
g.log.info('Checksums before bringing bricks online '
'and after bringing bricks online are equal')
def test_validate_snaps_restore(self):
# pylint: disable=too-many-statements
# Start IO on all mounts.
all_mounts_procs = []
count = 1
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 10 "
"--max-num-of-dirs 5 "
"--num-of-files 5 %s" %
(self.script_upload_path, count, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
count = count + 10
# Validate IO
g.log.info("Validating IO's")
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Successfully validated all io's")
# Get stat of all the files/dirs created.
g.log.info("Get stat of all the files/dirs created.")
ret = get_mounts_stat(self.mounts)
self.assertTrue(ret, "Stat failed on some of the clients")
g.log.info("Successfully got stat of all files/dirs created")
# Setting some volume option related to snapshot
option_before_restore = {
'volumeConfig': [{
'softLimit': '100',
'effectiveHardLimit': '200',
'hardLimit': '256'
}],
'systemConfig': {
'softLimit': '90%',
'activateOnCreate': 'disable',
'hardLimit': '256',
'autoDelete': 'disable'
}
}
ret = set_snap_config(self.mnode, option_before_restore)
self.assertTrue(ret,
("Failed to set vol option on %s" % self.volname))
g.log.info("Volume options for%s is set successfully", self.volname)
# Get brick list before taking snap_restore
bricks_before_snap_restore = get_all_bricks(self.mnode, self.volname)
g.log.info("Brick List before snap restore "
"volume: %s", bricks_before_snap_restore)
# Creating snapshot
ret = snap_create(self.mnode, self.volname, "snap1")
self.assertTrue(ret,
("Failed to create snapshot for %s" % self.volname))
g.log.info("Snapshot snap1 created successfully for volume %s",
self.volname)
# Again start IO on all mounts.
all_mounts_procs = []
count = 1000
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 10 "
"--max-num-of-dirs 5 "
"--num-of-files 5 %s" %
(self.script_upload_path, count, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
count = count + 10
# Validate IO
g.log.info("Validating IO's")
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Successfully validated all io's")
# Get stat of all the files/dirs created.
g.log.info("Get stat of all the files/dirs created.")
ret = get_mounts_stat(self.mounts)
self.assertTrue(ret, "Stat failed on some of the clients")
g.log.info("Successfully got stat of all files/dirs created")
# Reset volume to make sure volume options will reset
ret = volume_reset(self.mnode, self.volname, force=False)
self.assertTrue(ret, ("Failed to reset %s" % self.volname))
g.log.info("Reset Volume %s is Successful", self.volname)
# Removing one brick
g.log.info("Starting volume shrink")
ret = shrink_volume(self.mnode, self.volname, force=True)
self.assertTrue(ret, ("Failed to shrink the volume on "
"volume %s", self.volname))
g.log.info("Shrinking volume is successful on "
"volume %s", self.volname)
# Restore snapshot
ret = snap_restore_complete(self.mnode, self.volname, "snap1")
self.assertTrue(ret, ("Failed to restore snap snap1 on the "
"volume %s", self.volname))
g.log.info(
"Restore of volume is successful from snap1 on "
"volume %s", self.volname)
# Validate volume is up and running
g.log.info("Verifying volume is up and process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online", self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Get volume options post restore
option_after_restore = get_snap_config(self.mnode)
# Compare volume options
self.assertNotEqual(option_before_restore, option_after_restore,
"Volume Options are not same after snap restore")
# Get brick list post restore
bricks_after_snap_restore = get_all_bricks(self.mnode, self.volname)
g.log.info("Brick List after snap restore "
"volume: %s", bricks_after_snap_restore)
# Compare brick_list
self.assertNotEqual(bricks_before_snap_restore,
bricks_after_snap_restore,
"Bricks are not same after snap restore")
# Creating snapshot
ret = snap_create(self.mnode, self.volname, "snap2")
self.assertTrue(ret,
("Failed to create snapshot for %s" % self.volname))
g.log.info("Snapshot snap2 created successfully for volume %s",
self.volname)
# Again start IO on all mounts after restore
all_mounts_procs = []
count = 1000
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 10 "
"--max-num-of-dirs 5 "
"--num-of-files 5 %s" %
(self.script_upload_path, count, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
count = count + 10
# Validate IO
g.log.info("Validating IO's")
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Successfully validated all io's")
# Get stat of all the files/dirs created.
g.log.info("Get stat of all the files/dirs created.")
ret = get_mounts_stat(self.mounts)
self.assertTrue(ret, "Stat failed on some of the clients")
g.log.info("Successfully got stat of all files/dirs created")
