def test_add_brick_rebalance_with_hardlinks(self):
"""
Test case:
1. Create a volume, start it and mount it using fuse.
2. Create 50 files on the mount point and create 50 hardlinks for the
files.
3. After the files and hard links creation is complete, add bricks to
the volume and trigger rebalance on the volume.
4. Wait for rebalance to complete and check if files are skipped
or not.
5. Trigger rebalance on the volume with force and repeat step 4.
"""
# Tuple of ops to be done
ops = (("dd if=/dev/urandom of=file_$i bs=1M count=1",
"Failed to create 50 files"),
("ln file_$i hardfile_$i",
"Failed to create hard links for files"))
# Create 50 files on the mount point and create 50 hard links
# for the files.
for operation, msg in ops:
self._run_command_50_times(operation, msg)
# Collect arequal checksum before add brick op
self.arequal_checksum_before = collect_mounts_arequal(self.mounts[0])
# After the file creation is complete, add bricks to the volume
self._add_bricks_to_volume()
# Trigger rebalance on the volume, wait for it to complete
self._trigger_rebalance_and_wait()
# Check if hardlinks are skipped or not
self._check_if_files_are_skipped_or_not()
# Trigger rebalance with force on the volume, wait for it to complete
self._trigger_rebalance_and_wait(rebal_force=True)
# Check if hardlinks are skipped or not
self._check_if_files_are_skipped_or_not()
# Compare arequals checksum before and after rebalance
self.arequal_checksum_after = collect_mounts_arequal(self.mounts[0])
self._check_arequal_checksum_is_equal_before_and_after()
def check_arequal_from_mount_point_and_bricks(self):
"""
Check if arequals of mount point and bricks are
are the same.
"""
# Check arequals for "replicated"
all_bricks = get_all_bricks(self.mnode, self.volname)
if self.volume_type == "replicated":
# Get arequal before getting bricks offline
ret, arequals = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks offline '
'is successful')
mount_point_total = arequals[0].splitlines()[-1].split(':')[-1]
# Get arequal on bricks and compare with mount_point_total
ret, arequals = collect_bricks_arequal(all_bricks)
self.assertTrue(ret, 'Failed to get arequal on bricks')
for arequal in arequals:
brick_total = arequal.splitlines()[-1].split(':')[-1]
self.assertEqual(mount_point_total, brick_total,
'Arequals for mountpoint and brick '
'are not equal')
g.log.info('Arequals for mountpoint and brick are equal')
g.log.info('All arequals are equal for replicated')
# Check arequals for "distributed-replicated"
if self.volume_type == "distributed-replicated":
# get the subvolumes
subvols_dict = get_subvols(self.mnode, self.volname)
num_subvols = len(subvols_dict['volume_subvols'])
g.log.info("Number of subvolumes in volume %s:", num_subvols)
# Get arequals and compare
for i in range(0, num_subvols):
# Get arequal for first brick
subvol_brick_list = subvols_dict['volume_subvols'][i]
ret, arequal = collect_bricks_arequal([subvol_brick_list[0]])
self.assertTrue(ret, 'Failed to get arequal on first')
# Get arequal for every brick and compare with first brick
first_brick_total = arequal[0].splitlines()[-1].split(':')[-1]
ret, arequals = collect_bricks_arequal(subvol_brick_list)
self.assertTrue(ret, 'Failed to get arequal on bricks')
for arequal in arequals:
brick_total = arequal.splitlines()[-1].split(':')[-1]
self.assertEqual(first_brick_total, brick_total,
'Arequals for subvol and brick are '
'not equal')
g.log.info('Arequals for subvol and brick are equal')
g.log.info('All arequals are equal for distributed-replicated')
def test_remove_brick_command_basic(self):
"""
Test case:
1. Create a volume, start it and mount it.
2. Create some data on the volume.
3. Run remove-brick start, status and finally commit.
4. Check if there is any data loss or not.
"""
# Create some data on the volume
self._run_io_on_mount_point()
# Collect arequal checksum before ops
arequal_checksum_before = collect_mounts_arequal(self.mounts[0])
# Run remove-brick start, status and finally commit
ret = shrink_volume(self.mnode, self.volname)
self.assertTrue(ret, "Failed to remove-brick from volume")
g.log.info("Remove-brick rebalance successful")
# Check for data loss by comparing arequal before and after ops
arequal_checksum_after = collect_mounts_arequal(self.mounts[0])
self.assertEqual(arequal_checksum_before, arequal_checksum_after,
"arequal checksum is NOT MATCHNG")
g.log.info("arequal checksum is SAME")
def _check_arequal_checksum_for_the_volume(self):
"""
Check if arequals of mount point and bricks are
are the same.
"""
if self.volume_type == "replicated":
# Check arequals for "replicated"
brick_list = get_all_bricks(self.mnode, self.volname)
dir_brick_list = self._add_dir_path_to_brick_list(brick_list)
# Get arequal before getting bricks offline
work_dir = '{}/test_link_self_heal/'.format(self.mountpoint)
ret, arequals = collect_mounts_arequal([self.mounts[0]],
path=work_dir)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks offline '
'is successful')
# Get arequal on bricks and compare with mount_point_total
self._check_arequal_on_bricks_with_a_specific_arequal(
arequals, dir_brick_list)
# Check arequals for "distributed-replicated"
if self.volume_type == "distributed-replicated":
# Get the subvolumes
subvols_dict = get_subvols(self.mnode, self.volname)
num_subvols = len(subvols_dict['volume_subvols'])
# Get arequals and compare
for i in range(0, num_subvols):
# Get arequal for first brick
brick_list = subvols_dict['volume_subvols'][i]
dir_brick_list = self._add_dir_path_to_brick_list(brick_list)
ret, arequals = collect_bricks_arequal([dir_brick_list[0]])
self.assertTrue(ret, 'Failed to get arequal on first brick')
# Get arequal for every brick and compare with first brick
self._check_arequal_on_bricks_with_a_specific_arequal(
arequals, dir_brick_list)
def test_accessing_file_when_dht_layout_is_stale(self):
'''
Description : Checks if a file can be opened and accessed if the dht
layout has become stale.
Steps:
1. Create, start and mount a volume consisting 2 subvols on 2 clients
2. Create a dir `dir` and file `dir/file` from client0
3. Take note of layouts of `brick1`/dir and `brick2`/dir of the volume
4. Validate for success lookup from only one brick path
5. Re-assign layouts ie., brick1/dir to brick2/dir and vice-versa
6. Remove `dir/file` from client0 and recreate same file from client0
and client1
7. Validate for success lookup from only one brick path (as layout is
changed file creation path will be changed)
8. Validate checksum is matched from both the clients
'''
# Will be used in _get_brick_node_and_path
self.dir_path = '/dir'
# Will be used in argument to _assert_file_lookup
file_name = '/file'
dir_path = self.mounts[0].mountpoint + self.dir_path
file_path = dir_path + file_name
client0, client1 = self.clients[0], self.clients[1]
fattr = 'trusted.glusterfs.dht'
io_cmd = ('cat /dev/urandom | tr -dc [:space:][:print:] | '
'head -c 1K > {}'.format(file_path))
# Create a dir from client0
ret = mkdir(self.clients[0], dir_path)
self.assertTrue(ret, 'Unable to create a directory from mount point')
# Touch a file with data from client0
ret, _, _ = g.run(client0, io_cmd)
self.assertEqual(ret, 0, 'Failed to create a file on mount')
# Yields `node` and `brick-path` from first brick of each subvol
gen = self._get_brick_node_and_path()
# Take note of newly created directory's layout from org_subvol1
node1, fqpath1 = next(gen)
layout1 = get_fattr(node1, fqpath1, fattr)
self.assertIsNotNone(layout1,
'{} is not present on {}'.format(fattr, fqpath1))
# Lookup on file from node1 should fail as `dir/file` will always get
# hashed to node2 in a 2-brick distribute volume by default
self._assert_file_lookup(node1,
fqpath1 + file_name,
when='before',
result=False)
# Take note of newly created directory's layout from org_subvol2
node2, fqpath2 = next(gen)
layout2 = get_fattr(node2, fqpath2, fattr)
self.assertIsNotNone(layout2,
'{} is not present on {}'.format(fattr, fqpath2))
# Lookup on file from node2 should pass
self._assert_file_lookup(node2,
fqpath2 + file_name,
when='before',
result=True)
# Set org_subvol2 directory layout to org_subvol1 and vice-versa
for node, fqpath, layout, vol in ((node1, fqpath1, layout2, (2, 1)),
(node2, fqpath2, layout1, (1, 2))):
ret = set_fattr(node, fqpath, fattr, layout)
self.assertTrue(
ret, 'Failed to set layout of org_subvol{} on '
'brick {} of org_subvol{}'.format(vol[0], fqpath, vol[1]))
# Remove file after layout change from client0
cmd = 'rm -f {}'.format(file_path)
ret, _, _ = g.run(client0, cmd)
self.assertEqual(ret, 0, 'Failed to delete file after layout change')
# Create file with same name as above after layout change from client0
# and client1
for client in (client0, client1):
ret, _, _ = g.run(client, io_cmd)
self.assertEqual(
ret, 0, 'Failed to create file from '
'{} after layout change'.format(client))
# After layout change lookup on file from node1 should pass
self._assert_file_lookup(node1,
fqpath1 + file_name,
when='after',
result=True)
# After layout change lookup on file from node2 should fail
self._assert_file_lookup(node2,
fqpath2 + file_name,
when='after',
result=False)
# Take note of checksum from client0 and client1
checksums = [None] * 2
for index, mount in enumerate(self.mounts):
ret, checksums[index] = collect_mounts_arequal(mount, dir_path)
self.assertTrue(
ret, 'Failed to get arequal on client {}'.format(
mount.client_system))
# Validate no checksum mismatch
self.assertEqual(checksums[0], checksums[1],
'Checksum mismatch between client0 and client1')
g.log.info('Pass: Test accessing file on stale layout is complete.')
def _testcase(self, number_of_expands=1):
"""
Test case:
1. Create a volume start it and mount on the client.
2. Set full permission on the mount point.
3. Add new user to the client.
4. As the new user create dirs/files.
5. Compute arequal checksum and check permission on / and subdir.
6. expand cluster according to number_of_expands and start rebalance.
7. After rebalance is completed:
7.1 check arequal checksum
7.2 verfiy no change in / and sub dir permissions.
7.3 As the new user create and delete file/dir.
"""
# Set full permissions on the mount point.
ret = set_file_permissions(self.clients[0], self.mountpoint, "-R 777")
self.assertTrue(ret, "Failed to set permissions on the mount point")
g.log.info("Set full permissions on the mount point")
# Create dirs/files as self.test_user
cmd = (r'su -l %s -c "cd %s;'
r'for i in {0..9}; do mkdir d\$i; done;'
r'for i in {0..99}; do let x=\$i%%10;'
r'dd if=/dev/urandom of=d\$x/f.\$i bs=1024 count=1; done"' %
(self.user, self.mountpoint))
ret, _, _ = g.run(self.client, cmd)
self.assertEqual(ret, 0, ("Failed to create files as %s", self.user))
g.log.info("IO as %s is successful", self.user)
# check permission on / and subdir
self._check_user_permission()
# get arequal checksum before expand
self.arequal_checksum_before = collect_mounts_arequal(self.mounts[0])
self._logged_vol_info()
# expand the volume
for i in range(number_of_expands):
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(
ret, ("Failed to expand iter %d volume %s", i, self.volname))
self._logged_vol_info()
# Start Rebalance and wait for completion
self._start_rebalance_and_wait()
# compare arequals checksum before and after rebalance
self._get_arequal_and_check_if_equal_to_before()
# permissions check on / and sub dir
self._check_user_permission()
# Create/Delete file as self.test_user
cmd = ('su -l %s -c '
'"cd %s; touch file.test;'
'find . -mindepth 1 -maxdepth 1 -type d | xargs rm -rf"' %
(self.user, self.mountpoint))
ret, _, _ = g.run(self.client, cmd)
self.assertEqual(ret, 0, ("User %s failed to create files", self.user))
g.log.info("IO as %s is successful", self.user)
def test_remove_brick_no_commit_followed_by_rebalance(self):
"""
Description: Tests to check that there is no data loss when
remove-brick operation is stopped and then new bricks
are added to the volume.
Steps :
1) Create a volume.
2) Mount the volume using FUSE.
3) Create files and dirs on the mount-point.
4) Calculate the arequal-checksum on the mount-point
5) Start remove-brick operation on the volume.
6) While migration is in progress, stop the remove-brick
operation.
7) Add-bricks to the volume and trigger rebalance.
8) Wait for rebalance to complete.
9) Calculate the arequal-checksum on the mount-point.
"""
# Start IO on mounts
m_point = self.mounts[0].mountpoint
cmd = ("/usr/bin/env python %s create_deep_dirs_with_files "
"--dir-length 10 --dir-depth 2 --max-num-of-dirs 1 "
"--num-of-files 50 --file-type empty-file %s" %
(self.script_upload_path, m_point))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
g.log.info("IO on %s:%s is started successfully",
self.mounts[0].client_system, m_point)
# Validate IO
self.assertTrue(validate_io_procs([proc], self.mounts[0]),
"IO failed on some of the clients")
# Calculate arequal-checksum before starting remove-brick
ret, arequal_before = collect_mounts_arequal(self.mounts[0])
self.assertTrue(ret, "Collecting arequal-checksum failed")
# Form bricks list for volume shrink
remove_brick_list = form_bricks_list_to_remove_brick(self.mnode,
self.volname,
subvol_name=1)
self.assertIsNotNone(remove_brick_list, ("Volume %s: Failed to "
"form bricks list for "
"shrink", self.volname))
g.log.info("Volume %s: Formed bricks list for shrink", self.volname)
# Shrink volume by removing bricks
ret, _, _ = remove_brick(self.mnode, self.volname, remove_brick_list,
"start")
self.assertEqual(ret, 0, ("Volume %s shrink failed ", self.volname))
g.log.info("Volume %s shrink started ", self.volname)
# Log remove-brick status
ret, out, _ = remove_brick(self.mnode, self.volname, remove_brick_list,
"status")
self.assertEqual(ret, 0,
("Remove-brick status failed on %s ", self.volname))
# Check if migration is in progress
if r'in progress' in out:
# Stop remove-brick process
g.log.info("Stop removing bricks from volume")
ret, out, _ = remove_brick(self.mnode, self.volname,
remove_brick_list, "stop")
self.assertEqual(ret, 0, "Failed to stop remove-brick process")
g.log.info("Stopped remove-brick process successfully")
else:
g.log.error("Migration for remove-brick is complete")
# Sleep for 30 secs so that any running remove-brick process stops
sleep(30)
# Add bricks to the volume
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Volume %s: Add-brick failed", self.volname))
g.log.info("Volume %s: Add-brick successful", self.volname)
# Tigger rebalance
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(
ret, 0, ("Volume %s: Failed to start rebalance", self.volname))
g.log.info("Volume %s: Rebalance started ", self.volname)
# Wait for rebalance to complete
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(ret, "Rebalance has not completed")
g.log.info("Rebalance has completed successfully")
# Calculate arequal-checksum on mount-point
ret, arequal_after = collect_mounts_arequal(self.mounts[0])
self.assertTrue(ret, "Collecting arequal-checksum failed")
# Check if there is any data loss
self.assertEqual(set(arequal_before), set(arequal_after),
("There is data loss"))
g.log.info("The checksum before and after rebalance is same."
" There is no data loss.")
def test_replace_brick_self_heal_io_in_progress(self):
"""
- Create directory on mount point and write files/dirs
- Create another set of files (1K files)
- While creation of files/dirs are in progress Kill one brick
- Remove the contents of the killed brick(simulating disk replacement)
- When the IO's are still in progress, restart glusterd on the nodes
where we simulated disk replacement to bring back bricks online
- Start volume heal
- Wait for IO's to complete
- Verify whether the files are self-healed
- Calculate arequals of the mount point and all the bricks
"""
# pylint: disable=too-many-locals,too-many-statements,too-many-branches
# Create dirs with files
g.log.info('Creating dirs with file...')
command = ("/usr/bin/env python %s create_deep_dirs_with_files "
"-d 2 -l 2 -n 2 -f 10 %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
ret, _, err = g.run(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
self.assertFalse(ret, err)
g.log.info("IO is successful")
# Creating another set of files (1K files)
self.all_mounts_procs = []
# Create dirs with files
g.log.info('Creating 1K files...')
command = ("/usr/bin/env python %s create_files "
"-f 1500 --fixed-file-size 10k %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
ret = validate_io_procs(self.all_mounts_procs, self.mounts[0])
self.assertTrue(ret, "IO failed on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = bricks_to_bring_offline_dict['volume_bricks']
# Bring brick offline
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Remove the content of the killed bricks
for brick in bricks_to_bring_offline:
brick_node, brick_path = brick.split(':')
# Removing files
command = ('cd %s ; rm -rf *' % brick_path)
ret, _, err = g.run(brick_node, command)
self.assertFalse(ret, err)
g.log.info('Files are deleted on brick %s', brick)
# Bring brick online
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s online' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal daemons are online")
# Start healing
ret = trigger_heal_full(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Check arequals for "replicated"
all_bricks = get_all_bricks(self.mnode, self.volname)
if self.volume_type == "replicated":
# Get arequal after bricks are online
ret, arequals = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after successfully bringing'
'bricks online.')
mount_point_total = arequals[0].splitlines()[-1].split(':')[-1]
# Get arequal on bricks and compare with mount_point_total
ret, arequals = collect_bricks_arequal(all_bricks)
self.assertTrue(ret, 'Failed to get arequal on bricks')
for arequal in arequals:
brick_total = arequal.splitlines()[-1].split(':')[-1]
self.assertEqual(
mount_point_total, brick_total,
'Arequals for mountpoint and brick '
'are not equal')
g.log.info('Arequals for mountpoint and brick are equal')
# Check arequals for "distributed-replicated"
if self.volume_type == "distributed-replicated":
# Get the subvolumes
subvols_dict = get_subvols(self.mnode, self.volname)
num_subvols = len(subvols_dict['volume_subvols'])
g.log.info("Number of subvolumes in volume %s:", num_subvols)
# Get arequals and compare
for i in range(0, num_subvols):
# Get arequal for first brick
subvol_brick_list = subvols_dict['volume_subvols'][i]
ret, arequal = collect_bricks_arequal(subvol_brick_list[0])
self.assertTrue(ret, 'Failed to get arequal on first brick')
first_brick_total = arequal[0].splitlines()[-1].split(':')[-1]
# Get arequal for every brick and compare with first brick
ret, arequals = collect_bricks_arequal(subvol_brick_list)
self.assertTrue(ret, 'Failed to get arequal on bricks')
for arequal in arequals:
brick_total = arequal.splitlines()[-1].split(':')[-1]
self.assertEqual(
first_brick_total, brick_total,
'Arequals for subvol and brick are '
'not equal')
g.log.info('Arequals for subvol and brick are equal')
def test_metadata_self_heal(self):
"""
Test MetaData Self-Heal (heal command)
Description:
- set the volume option
"metadata-self-heal": "off"
"entry-self-heal": "off"
"data-self-heal": "off"
- create IO
- set the volume option
"self-heal-daemon": "off"
- bring down all bricks processes from selected set
- Change the permissions, ownership and the group
of the files under "test_meta_data_self_heal" folder
- get arequal before getting bricks online
- bring bricks online
- set the volume option
"self-heal-daemon": "on"
- check daemons and start healing
- check is heal is completed
- check for split-brain
- get arequal after getting bricks online and compare with
arequal before getting bricks online
- check group and user are 'qa'
"""
# pylint: disable=too-many-locals,too-many-statements
# Setting options
g.log.info('Setting options...')
options = {"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Options "
"'metadata-self-heal', "
"'entry-self-heal', "
"'data-self-heal', "
"are set to 'off' successfully")
# Creating files on client side
all_mounts_procs = []
test_meta_data_self_heal_folder = 'test_meta_data_self_heal'
g.log.info("Generating data for %s:%s",
self.mounts[0].client_system, self.mounts[0].mountpoint)
# Create files
g.log.info('Creating files...')
command = ("cd %s/ ; "
"mkdir %s ;"
"cd %s/ ;"
"for i in `seq 1 50` ; "
"do dd if=/dev/urandom of=test.$i bs=10k count=1 ; "
"done ;"
% (self.mounts[0].mountpoint,
test_meta_data_self_heal_folder,
test_meta_data_self_heal_folder))
proc = g.run_async(self.mounts[0].client_system, command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# wait for io to complete
self.assertTrue(
wait_for_io_to_complete(all_mounts_procs, self.mounts),
"Io failed to complete on some of the clients")
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Option 'self-heal-daemon' is set to 'off' successfully")
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = list(filter(None, (
bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks'])))
# Bring brick offline
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Changing the permissions, ownership and the group
# of the files under "test_meta_data_self_heal" folder
g.log.info("Modifying data for %s:%s",
self.mounts[0].client_system, self.mounts[0].mountpoint)
# Change permissions to 444
g.log.info('Changing permissions...')
command = ("cd %s/%s/ ; "
"chmod -R 444 *"
% (self.mounts[0].mountpoint,
test_meta_data_self_heal_folder))
ret, out, err = g.run(self.mounts[0].client_system, command)
self.assertEqual(ret, 0, err)
g.log.info('Permissions are changed successfully')
# Change the ownership to qa
g.log.info('Changing the ownership...')
command = ("cd %s/%s/ ; "
"chown -R qa *"
% (self.mounts[0].mountpoint,
test_meta_data_self_heal_folder))
ret, out, err = g.run(self.mounts[0].client_system, command)
self.assertEqual(ret, 0, err)
g.log.info('Ownership is changed successfully')
# Change the group to qa
g.log.info('Changing the group...')
command = ("cd %s/%s/ ; "
"chgrp -R qa *"
% (self.mounts[0].mountpoint,
test_meta_data_self_heal_folder))
ret, out, err = g.run(self.mounts[0].client_system, command)
self.assertEqual(ret, 0, err)
g.log.info('Group is changed successfully')
# Get arequal before getting bricks online
g.log.info('Getting arequal before getting bricks online...')
ret, result_before_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks online '
'is successful')
# Bring brick online
g.log.info('Bringing bricks %s online...', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume process %s not online "
"despite waiting for 5 minutes", self.volname))
g.log.info("Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online"
% self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
g.log.info('Getting arequal after getting bricks online...')
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks online '
'is successful')
# Checking arequals before bringing bricks online
# and after bringing bricks online
self.assertItemsEqual(result_before_online, result_after_online,
'Checksums are not equal')
g.log.info('Checksums before bringing bricks online '
'and after bringing bricks online are equal')
# Adding servers and client in single dict to check permissions
nodes_to_check = {}
all_bricks = get_all_bricks(self.mnode, self.volname)
for brick in all_bricks:
node, brick_path = brick.split(':')
nodes_to_check[node] = brick_path
nodes_to_check[self.mounts[0].client_system] = \
self.mounts[0].mountpoint
# Checking for user and group
for node in nodes_to_check:
# Get file list
command = ("cd %s/%s/ ; "
"ls"
% (nodes_to_check[node],
test_meta_data_self_heal_folder))
ret, out, err = g.run(node, command)
file_list = out.split()
for file_name in file_list:
file_to_check = '%s/%s/%s' % (nodes_to_check[node],
test_meta_data_self_heal_folder,
file_name)
g.log.info('Checking for permissions, user and group for %s',
file_name)
# Check for permissions
cmd = ("stat -c '%a %n' {} | awk '{{print $1}}'"
.format(file_to_check))
ret, permissions, _ = g.run(node, cmd)
self.assertEqual(permissions.split('\n')[0], '444',
'Permissions %s is not equal to 444'
% permissions)
g.log.info("Permissions are '444' for %s", file_name)
# Check for user
cmd = ("ls -ld {} | awk '{{print $3}}'"
.format(file_to_check))
ret, username, _ = g.run(node, cmd)
self.assertEqual(username.split('\n')[0],
'qa', 'User %s is not equal qa'
% username)
g.log.info("User is 'qa' for %s", file_name)
# Check for group
cmd = ("ls -ld {} | awk '{{print $4}}'"
.format(file_to_check))
ret, groupname, _ = g.run(node, cmd)
self.assertEqual(groupname.split('\n')[0],
'qa', 'Group %s is not equal qa'
% groupname)
g.log.info("Group is 'qa' for %s", file_name)
def test_arb_to_repl_conversion_with_io(self):
"""
Description: To perform a volume conversion from Arbiter to Replicated
with background IOs
Steps:
- Create, start and mount an arbiter volume in two clients
- Create two dir's, fill IO in first dir and take note of arequal
- Start a continuous IO from second directory
- Convert arbiter to x2 replicated volume (remove brick)
- Convert x2 replicated to x3 replicated volume (add brick)
- Wait for ~5 min for vol file to be updated on all clients
- Enable client side heal options and issue volume heal
- Validate heal completes with no errors and arequal of first dir
matches against initial checksum
"""
client, m_point = (self.mounts[0].client_system,
self.mounts[0].mountpoint)
# Fill IO in first directory
cmd = ('/usr/bin/env python {} '
'create_deep_dirs_with_files --dir-depth 10 '
'--fixed-file-size 1M --num-of-files 100 '
'--dirname-start-num 1 {}'.format(self.script_path, m_point))
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, 'Not able to fill directory with IO')
# Take `arequal` checksum on first directory
ret, exp_arequal = collect_mounts_arequal(self.mounts[0],
m_point + '/user1')
self.assertTrue(ret, 'Failed to get arequal checksum on mount')
# Start continuous IO from second directory
client = self.mounts[1].client_system
cmd = ('/usr/bin/env python {} '
'create_deep_dirs_with_files --dir-depth 10 '
'--fixed-file-size 1M --num-of-files 250 '
'--dirname-start-num 2 {}'.format(self.script_path, m_point))
proc = g.run_async(client, cmd)
self.all_mounts_procs.append(proc)
# Wait for IO to fill before volume conversion
sleep(30)
# Remove arbiter bricks ( arbiter to x2 replicated )
kwargs = {'replica_count': 2}
ret, _, _ = remove_brick(self.mnode,
self.volname,
self._get_arbiter_bricks(),
option='force',
**kwargs)
self.assertEqual(ret, 0, 'Not able convert arbiter to x2 replicated '
'volume')
# Wait for IO to fill after volume conversion
sleep(30)
# Add bricks (x2 replicated to x3 replicated)
kwargs['replica_count'] = 3
vol_info = get_volume_info(self.mnode, volname=self.volname)
self.assertIsNotNone(vol_info, 'Not able to get volume info')
dist_count = vol_info[self.volname]['distCount']
bricks_list = form_bricks_list(
self.mnode,
self.volname,
number_of_bricks=int(dist_count) * 1,
servers=self.servers,
servers_info=self.all_servers_info,
)
self.assertTrue(bricks_list, 'Not able to get unused list of bricks')
ret, _, _ = add_brick(self.mnode,
self.volname,
bricks_list,
force='True',
**kwargs)
self.assertEqual(ret, 0, 'Not able to add-brick to '
'{}'.format(self.volname))
# Wait for IO post x3 replicated volume conversion
sleep(30)
# Validate volume info
vol_info = get_volume_info(self.mnode, volname=self.volname)
self.assertIsNotNone(vol_info, 'Not able to get volume info')
vol_info = vol_info[self.volname]
repl_count, brick_count = (vol_info['replicaCount'],
vol_info['brickCount'])
# Wait for the volfile to sync up on clients
cmd = ('grep -ir connected {}/.meta/graphs/active/{}-client-*/private '
'| wc -l')
wait_time = time() + 300
in_sync = False
while time() <= wait_time:
ret, rout, _ = g.run(client, cmd.format(m_point, self.volname))
self.assertEqual(ret, 0,
'Not able to grep for volfile sync from client')
if int(rout) == int(brick_count):
in_sync = True
break
sleep(30)
self.assertTrue(
in_sync, 'Volfiles from clients are not synced even '
'after polling for ~5 min')
self.assertEqual(
int(repl_count), kwargs['replica_count'], 'Not able '
'to validate x2 to x3 replicated volume conversion')
# Enable client side heal options, trigger and monitor heal
ret = set_volume_options(
self.mnode, self.volname, {
'data-self-heal': 'on',
'entry-self-heal': 'on',
'metadata-self-heal': 'on'
})
self.assertTrue(ret, 'Unable to set client side heal options')
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Unable to trigger heal on volume')
ret = monitor_heal_completion(self.mnode,
self.volname,
timeout_period=1800)
self.assertTrue(ret,
'Heal is not completed for {}'.format(self.volname))
# Validate IO
prev_time = datetime.now().replace(microsecond=0)
ret = validate_io_procs(self.all_mounts_procs, [self.mounts[1]])
curr_time = datetime.now().replace(microsecond=0)
self.assertTrue(ret, 'Not able to validate completion of IO on mount')
self.all_mounts_procs *= 0
# To ascertain IO was happening during brick operations
self.assertGreater(
curr_time - prev_time, timedelta(seconds=10), 'Unable '
'to validate IO was happening during brick operations')
# Take and validate `arequal` checksum on first directory
ret, act_areequal = collect_mounts_arequal(self.mounts[1],
m_point + '/user1')
self.assertTrue(ret, 'Failed to get arequal checksum from mount')
self.assertEqual(
exp_arequal, act_areequal, '`arequal` checksum did '
'not match post arbiter to x3 replicated volume conversion')
g.log.info('PASS: Arbiter to x3 replicated volume conversion complete')
def test_self_heal_daemon(self):
"""
Test Data-Self-Heal(heal command)
Description:
- Create directory test_hardlink_self_heal
- Create directory test_data_self_heal
- Creating files for hardlinks and data files
- Get arequal before getting bricks offline
- Select bricks to bring offline
- Bring brick offline
- Create hardlinks and append data to data files
- Bring brick online
- Wait for volume processes to be online
- Verify volume's all process are online
- Monitor heal completion
- Check for split-brain
- Get arequal after getting bricks online
- Select bricks to bring offline
- Bring brick offline
- Truncate data to data files and verify hardlinks
- Bring brick online
- Wait for volume processes to be online
- Verify volume's all process are online
- Monitor heal completion
- Check for split-brain
- Get arequal again
"""
# pylint: disable=too-many-branches,too-many-statements,too-many-locals
# Creating directory test_hardlink_self_heal
ret = mkdir(
self.mounts[0].client_system,
"{}/test_hardlink_self_heal".format(self.mounts[0].mountpoint))
self.assertTrue(ret, "Failed to create directory")
g.log.info(
"Directory 'test_hardlink_self_heal' on %s created "
"successfully", self.mounts[0])
# Creating directory test_data_self_heal
ret = mkdir(self.mounts[0].client_system,
"{}/test_data_self_heal".format(self.mounts[0].mountpoint))
self.assertTrue(ret, "Failed to create directory")
g.log.info(
"Directory test_hardlink_self_heal on %s created "
"successfully", self.mounts[0])
# Creating files for hardlinks and data files
cmd = ('cd %s/test_hardlink_self_heal;for i in `seq 1 5`;'
'do mkdir dir.$i ; for j in `seq 1 10` ; do dd if='
'/dev/urandom of=dir.$i/file.$j bs=1k count=$j;done; done;'
'cd ..' % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Failed to create file on mountpoint")
g.log.info("Successfully created files on mountpoint")
cmd = ('cd %s/test_data_self_heal;for i in `seq 1 100`;'
'do dd if=/dev/urandom of=file.$i bs=128K count=$i;done;'
'cd ..' % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Failed to create file on mountpoint")
g.log.info("Successfully created files on mountpoint")
# Get arequal before getting bricks offline
ret, result_before_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Arequal before getting bricks online-%s',
result_before_online)
# Select bricks to bring offline
bricks_to_bring_offline = select_volume_bricks_to_bring_offline(
self.mnode, self.volname)
self.assertIsNotNone(bricks_to_bring_offline, "List is empty")
# Bring brick offline
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks {} offline'.format(
bricks_to_bring_offline))
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Bricks {} are not offline'.format(bricks_to_bring_offline))
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Append data to data files and create hardlinks
cmd = ('cd %s/test_data_self_heal;for i in `seq 1 100`;'
'do dd if=/dev/urandom of=file.$i bs=512K count=$i ; done ;'
'cd .. ' % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Failed to modify data files.")
g.log.info("Successfully modified data files")
cmd = ('cd %s/test_hardlink_self_heal;for i in `seq 1 5` ;do '
'for j in `seq 1 10`;do ln dir.$i/file.$j dir.$i/link_file.$j;'
'done ; done ; cd .. ' % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Hardlinks creation failed")
g.log.info("Successfully created hardlinks of files")
# Bring bricks online
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret,
'Failed to bring bricks {} online'.format(bricks_to_bring_offline))
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume {} processes to "
"be online".format(self.volname)))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret,
("Volume {} : All process are not online".format(self.volname)))
g.log.info("Volume %s : All process are online", self.volname)
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Arequal after getting bricks online '
'is %s', result_after_online)
# Select bricks to bring offline
bricks_to_bring_offline = select_volume_bricks_to_bring_offline(
self.mnode, self.volname)
self.assertIsNotNone(bricks_to_bring_offline, "List is empty")
# Bring brick offline
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks {} offline'.format(
bricks_to_bring_offline))
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Bricks {} are not offline'.format(bricks_to_bring_offline))
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Truncate data to data files and verify hardlinks
cmd = ('cd %s/test_data_self_heal ; for i in `seq 1 100` ;'
'do truncate -s $(( $i * 128)) file.$i ; done ; cd ..' %
self.mounts[0].mountpoint)
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Failed to truncate files")
g.log.info("Successfully truncated files on mountpoint")
file_path = ('%s/test_hardlink_self_heal/dir{1..5}/file{1..10}' %
(self.mounts[0].mountpoint))
link_path = ('%s/test_hardlink_self_heal/dir{1..5}/link_file{1..10}' %
(self.mounts[0].mountpoint))
file_stat = get_file_stat(self.mounts[0], file_path)
link_stat = get_file_stat(self.mounts[0], link_path)
self.assertEqual(file_stat, link_stat, "Verification of hardlinks "
"failed")
g.log.info("Successfully verified hardlinks")
# Bring brick online
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret,
'Failed to bring bricks {} online'.format(bricks_to_bring_offline))
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume {} processes to "
"be online".format(self.volname)))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret,
("Volume {} : All process are not online".format(self.volname)))
g.log.info("Volume %s : All process are online", self.volname)
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
def test_data_self_heal_algorithm_diff_heal_command(self):
"""
Test Volume Option - 'cluster.data-self-heal-algorithm' : 'diff'
Description:
- set the volume option
"metadata-self-heal": "off"
"entry-self-heal": "off"
"data-self-heal": "off"
"data-self-heal-algorithm": "diff"
"self-heal-daemon": "off"
- create IO
- calculate arequal
- bring down all bricks processes from selected set
- modify the data
- get arequal before getting bricks online
- bring bricks online
- expand volume by adding bricks to the volume
- do rebalance
- set the volume option "self-heal-daemon": "on" and check for daemons
- start healing
- check if heal is completed
- check for split-brain
- calculate arequal and compare with arequal before bringing bricks
offline and after bringing bricks online
"""
# pylint: disable=too-many-branches,too-many-statements
# Setting options
g.log.info('Setting options...')
options = {
"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off",
"data-self-heal-algorithm": "diff"
}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Options "
"'metadata-self-heal', "
"'entry-self-heal', "
"'data-self-heal', "
"'self-heal-daemon' "
"are set to 'off',"
"'data-self-heal-algorithm' "
"is set to 'diff' successfully")
# Creating files on client side
all_mounts_procs = []
g.log.info("Generating data for %s:%s", self.mounts[0].client_system,
self.mounts[0].mountpoint)
# Creating files
command = "/usr/bin/env python %s create_files -f 100 %s" % (
self.script_upload_path, self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Option 'self-heal-daemon' is set to 'off' successfully")
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = list(
filter(None, (bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks'])))
# Bring brick offline
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Modify the data
all_mounts_procs = []
g.log.info("Modifying data for %s:%s", self.mounts[0].client_system,
self.mounts[0].mountpoint)
command = ("/usr/bin/env python %s create_files -f 100 "
"--fixed-file-size 1M %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# Get arequal before getting bricks online
g.log.info('Getting arequal before getting bricks online...')
ret, result_before_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks online '
'is successful')
# Bring brick online
g.log.info('Bringing bricks %s online...', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s online' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Expand volume by adding bricks to the volume
g.log.info("Start adding bricks to volume...")
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to expand the volume when IO in "
"progress on volume %s", self.volname))
g.log.info("Expanding volume is successful on volume %s", self.volname)
# Do rebalance
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, 'Failed to start rebalance')
g.log.info('Rebalance is started')
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Rebalance is not completed')
g.log.info('Rebalance is completed successfully')
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
g.log.info('Getting arequal after getting bricks online...')
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks online '
'is successful')
# Checking arequals before bringing bricks offline
# and after bringing bricks online
self.assertItemsEqual(result_before_online, result_after_online,
'Checksums are not equal')
g.log.info('Checksums are equal')
def test_handling_data_split_brain(self):
"""
- create IO
- calculate arequal from mountpoint
- set volume option 'self-heal-daemon' to value "off"
- kill data brick1
- calculate arequal checksum and compare it
- modify files and directories
- bring back all bricks processes online
- kill data brick3
- modify files and directories
- calculate arequal from mountpoint
- bring back all bricks processes online
- run the find command to trigger heal from mountpoint
- set volume option 'self-heal-daemon' to value "on"
- check if heal is completed
- check for split-brain
- read files
- calculate arequal checksum and compare it
"""
# pylint: disable=too-many-locals,too-many-statements
# Creating files on client side
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
# Create files
g.log.info('Creating files...')
command = ("cd %s ; "
"for i in `seq 1 10` ; "
"do mkdir dir.$i ; "
"for j in `seq 1 5` ; "
"do dd if=/dev/urandom of=dir.$i/file.$j "
"bs=1K count=1 ; "
"done ; "
"dd if=/dev/urandom of=file.$i bs=1k count=1 ; "
"done"
% mount_obj.mountpoint)
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
# Get arequal before getting bricks offline
g.log.info('Getting arequal before getting bricks offline...')
ret, result_before_offline = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks offline '
'is successful')
# Setting options
options = {"self-heal-daemon": "off"}
g.log.info('Setting options %s for volume %s',
options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' is set to 'off' successfully")
# get the bricks for the volume
g.log.info("Fetching bricks for the volume: %s", self.volname)
bricks_list = get_all_bricks(self.mnode, self.volname)
g.log.info("Brick list: %s", bricks_list)
# Bring brick 1 offline
bricks_to_bring_offline = [bricks_list[0]]
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Get arequal after getting bricks offline
g.log.info('Getting arequal after getting bricks offline...')
ret, result_after_offline = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks offline '
'is successful')
# Comparing arequals before getting bricks offline
# and after getting bricks offline
self.assertEqual(result_before_offline, result_after_offline,
'Arequals before getting bricks offline '
'and after getting bricks offline are not equal')
g.log.info('Arequals before getting bricks offline '
'and after getting bricks offline are equal')
# Modify the data
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Modifying data for %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
# Modify files
g.log.info('Modifying files...')
command = ("cd %s ; "
"for i in `seq 1 10` ; "
"do for j in `seq 1 5` ; "
"do dd if=/dev/urandom of=dir.$i/file.$j "
"bs=1M count=1 ; "
"done ; "
"dd if=/dev/urandom of=file.$i bs=1M count=1 ; "
"done"
% mount_obj.mountpoint)
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
# Bring 1-st brick online
g.log.info('Bringing bricks %s online...', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Bring brick 3rd offline
bricks_to_bring_offline = [bricks_list[-1]]
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Modify the data
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Modifying data for %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
# Create files
g.log.info('Modifying files...')
command = ("cd %s ; "
"for i in `seq 1 10` ; "
"do for j in `seq 1 5` ; "
"do dd if=/dev/urandom of=dir.$i/file.$j "
"bs=1M count=1 ; "
"done ; "
"dd if=/dev/urandom of=file.$i bs=1M count=1 ; "
"done"
% mount_obj.mountpoint)
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
# Get arequal before getting bricks online
g.log.info('Getting arequal before getting bricks online...')
ret, result_before_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks online '
'is successful')
# Bring 3rd brick online
g.log.info('Bringing bricks %s online...', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Mount and unmount mounts
ret = self.unmount_volume(self.mounts)
self.assertTrue(ret, 'Failed to unmount %s' % self.volname)
ret = self.mount_volume(self.mounts)
self.assertTrue(ret, 'Unable to mount %s' % self.volname)
# Start heal from mount point
g.log.info('Starting heal from mount point...')
for mount_obj in self.mounts:
g.log.info("Start heal for %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
command = "/usr/bin/env python %s read %s" % (
self.script_upload_path,
self.mounts[0].mountpoint)
ret, _, err = g.run(mount_obj.client_system, command)
self.assertFalse(ret, err)
g.log.info("Heal triggered for %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
g.log.info('Heal triggered for all mountpoints')
# Enable self-heal daemon
ret = enable_self_heal_daemon(self.mnode, self.volname)
self.assertTrue(ret, 'Successfully started self heal daemon')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Reading files
g.log.info('Reading files...')
for mount_obj in self.mounts:
g.log.info("Start reading files for %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
command = ('cd %s/ ; '
'for i in `seq 1 10` ; '
'do cat file.$i > /dev/null ; '
'for j in `seq 1 5` ; '
'do cat dir.$i/file.$j > /dev/null ; '
'done ; done'
% mount_obj.mountpoint)
ret, _, err = g.run(mount_obj.client_system, command)
self.assertFalse(ret, err)
g.log.info("Reading files successfully for %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
g.log.info('Reading files successfully for all mountpoints')
# Get arequal after getting bricks online
g.log.info('Getting arequal after getting bricks online...')
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks online '
'is successful')
# Comparing arequals before getting bricks online
# and after getting bricks online
self.assertEqual(result_before_online, result_after_online,
'Arequals before getting bricks online '
'and after getting bricks online are not equal')
g.log.info('Arequals before getting bricks online '
'and after getting bricks online are equal')
def test_add_brick_rebalance_with_self_heal_in_progress(self):
"""
Test case:
1. Create a volume, start it and mount it.
2. Start creating a few files on mount point.
3. While file creation is going on, kill one of the bricks
in the replica pair.
4. After file creattion is complete collect arequal checksum
on mount point.
5. Bring back the brick online by starting volume with force.
6. Check if all bricks are online and if heal is in progress.
7. Add bricks to the volume and start rebalance.
8. Wait for rebalance and heal to complete on volume.
9. Collect arequal checksum on mount point and compare
it with the one taken in step 4.
"""
# Start I/O from mount point and wait for it to complete
cmd = ("cd %s; for i in {1..1000} ; do "
"dd if=/dev/urandom of=file$i bs=10M count=1; done" %
self.mounts[0].mountpoint)
self.list_of_io_processes = [
g.run_async(self.mounts[0].client_system, cmd)
]
self.is_copy_running = True
# Get a list of all the bricks to kill brick
brick_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(brick_list, "Empty present brick list")
# Kill brick process of a brick which is being removed
brick = choice(brick_list)
node, _ = brick.split(":")
ret = kill_process(node, process_names="glusterfsd")
self.assertTrue(ret,
"Failed to kill brick process of brick %s" % brick)
# Validate if I/O was successful or not.
ret = validate_io_procs(self.list_of_io_processes, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
self.is_copy_running = False
# Collect arequal checksum before ops
arequal_checksum_before = collect_mounts_arequal(self.mounts[0])
# Bring back the brick online by starting volume with force
ret = bring_bricks_online(
self.mnode,
self.volname,
brick_list,
bring_bricks_online_methods=['volume_start_force'])
self.assertTrue(ret, "Error in bringing back brick online")
g.log.info('All bricks are online now')
# Add brick to volume
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, "Failed to add brick on volume %s" % self.volname)
# Trigger rebalance and wait for it to complete
ret, _, _ = rebalance_start(self.mnode, self.volname, force=True)
self.assertEqual(
ret, 0,
"Failed to start rebalance on the volume %s" % self.volname)
# Wait for rebalance to complete
ret = wait_for_rebalance_to_complete(self.mnode,
self.volname,
timeout=1200)
self.assertTrue(
ret, "Rebalance is not yet complete on the volume "
"%s" % self.volname)
g.log.info("Rebalance successfully completed")
# Wait for heal to complete
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, "heal has not yet completed")
g.log.info("Self heal completed")
# Check for data loss by comparing arequal before and after ops
arequal_checksum_after = collect_mounts_arequal(self.mounts[0])
self.assertEqual(arequal_checksum_before, arequal_checksum_after,
"arequal checksum is NOT MATCHNG")
g.log.info("arequal checksum is SAME")
def test_self_heal_algorithm_full_daemon_off(self):
"""""
Description:-
Checking healing when algorithm is set to "full" and self heal daemon
is "off".
"""""
# pylint: disable=too-many-statements
# Setting volume option of self heal & algorithm
options = {"metadata-self-heal": "disable",
"entry-self-heal": "disable",
"data-self-heal": "disable",
"data-self-heal-algorithm": "full",
"self-heal-daemon": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, "Failed to set the volume options %s" % options)
g.log.info(" Volume set options success")
# Select bricks to bring down
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = bricks_to_bring_offline_dict['volume_bricks']
g.log.info("Bringing bricks: %s offline", bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, "Failed to bring bricks: %s offline"
% bricks_to_bring_offline)
g.log.info("Successful in bringing bricks: %s offline",
bricks_to_bring_offline)
# Validate if bricks are offline
g.log.info("Validating if bricks: %s are offline",
bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, "Not all the bricks in list:%s are offline"
% bricks_to_bring_offline)
g.log.info("Successfully validated that bricks %s are all offline",
bricks_to_bring_offline)
# IO on the mount point
all_mounts_procs = []
g.log.info("Creating Files on %s:%s", self.mounts[0].client_system,
self.mounts[0].mountpoint)
cmd = ("cd %s ;for i in `seq 1 100` ;"
"do dd if=/dev/urandom of=file$i bs=1M "
"count=1;done"
% self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients"
)
# Collecting Arequal before bring the bricks up
g.log.info("Collecting Arequal before the bring of bricks down")
result_before = collect_mounts_arequal(self.mounts)
# Turning self heal daemon ON
optionstwo = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, optionstwo)
self.assertTrue(ret, "Failed to turn self-heal ON")
g.log.info("Volume set options %s: success", optionstwo)
# Bring bricks online
g.log.info("Bring bricks: %s online", bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, "Failed to bring bricks: %s online"
% bricks_to_bring_offline)
g.log.info("Successfully brought all bricks:%s online",
bricks_to_bring_offline)
# Waiting for bricks to come online
g.log.info("Waiting for brick process to come online")
ret = wait_for_bricks_to_be_online(self.mnode,
self.volname,
timeout=30)
self.assertTrue(ret, "bricks didn't come online after adding bricks")
g.log.info("Bricks are online")
# Verifying all bricks online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, "Volume %s : All process are not online"
% self.volname)
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self heal processes to come online
g.log.info("Wait for selfheal process to come online")
ret = wait_for_self_heal_daemons_to_be_online(self.mnode,
self.volname,
timeout=300)
self.assertTrue(ret, "Self-heal process are not online")
g.log.info("All self heal process are online")
# Wait for self-heal to complete
g.log.info("Wait for self-heal to complete")
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, "Self heal didn't complete even after waiting "
"for 20 minutes. 20 minutes is too much a time for "
"current test workload")
g.log.info("self-heal is successful after replace-brick operation")
# arequal after healing
g.log.info("Collecting Arequal before the bring of bricks down")
result_after = collect_mounts_arequal(self.mounts)
# Comparing the results
g.log.info("comparing both the results")
self.assertEqual(result_before, result_after, "Arequals are not equal")
def test_heal_full_after_deleting_files(self):
"""
- Create IO
- Calculate arequal from mount
- Delete data from backend from the EC volume
- Trigger heal full
- Check if heal is completed
- Check for split-brain
- Calculate arequal checksum and compare it
"""
# pylint: disable=too-many-locals,too-many-statements
# Creating files on client side
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Create dirs with file
g.log.info('Creating dirs with file...')
command = ("/usr/bin/env python %s create_deep_dirs_with_files "
"-d 2 -l 2 -n 2 -f 20 %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
# Get areequal before deleting the files from brick
g.log.info('Getting areequal before getting bricks offline...')
ret, result_before_killing_procs = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting areequal before getting bricks offline '
'is successful')
subvols = get_subvols(self.mnode, self.volname)['volume_subvols']
# Delete data from backend from the erasure node
for subvol in subvols:
erasure = subvol[-1]
g.log.info('Clearing ec brick %s', erasure)
node, brick_path = erasure.split(':')
ret, _, err = g.run(node, 'cd %s/ ; rm -rf *' % brick_path)
g.log.error('Clearing ec brick %s is unsuccessful', erasure)
self.assertFalse(ret, err)
g.log.info('Clearing data from brick is successful')
# Trigger heal full
ret = trigger_heal_full(self.mnode, self.volname)
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get areequal after healing
g.log.info('Getting areequal after getting bricks online...')
ret, result_after_healing = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting areequal after getting bricks online '
'is successful')
# Comparing areequals
self.assertEqual(
result_before_killing_procs, result_after_healing,
'Areequals areequals before before killing arbiter '
'processes and after healing are not equal')
g.log.info('Areequals areequals before before killing arbiter '
'processes and after healing equal')
def test_ec_replace_brick(self):
"""
- Start resource consumption tool
- Create directory dir1
- Create 5 directory and 5 files in dir of mountpoint
- Rename all files inside dir1 at mountpoint
- Create softlink and hardlink of files in dir1 of mountpoint
- Delete op for deleting all file in one of the dirs inside dir1
- Change chmod, chown, chgrp
- Create tiny, small, medium and large file
- Get arequal before replacing brick
- Replace brick
- Get arequal after replacing brick
- Compare Arequal's
- Create IO's
- Replace brick while IO's are going on
- Validating IO's and waiting for it to complete
"""
# pylint: disable=too-many-branches,too-many-statements,too-many-locals
# Starting resource consumption using top
log_file_mem_monitor = '/var/log/glusterfs/mem_usage.log'
cmd = ("for i in {1..20};do top -n 1 -b|egrep "
"'RES|gluster' & free -h 2>&1 >> %s ;"
"sleep 10;done" % (log_file_mem_monitor))
g.log.info(cmd)
cmd_list_procs = []
for server in self.servers:
proc = g.run_async(server, cmd)
cmd_list_procs.append(proc)
# Creating dir1
ret = mkdir(self.mounts[0].client_system,
"%s/dir1" % self.mounts[0].mountpoint)
self.assertTrue(ret, "Failed to create dir1")
g.log.info("Directory dir1 on %s created successfully", self.mounts[0])
# Create 5 dir and 5 files in each dir at mountpoint on dir1
start, end = 1, 5
for mount_obj in self.mounts:
# Number of dir and files to be created.
dir_range = ("%s..%s" % (str(start), str(end)))
file_range = ("%s..%s" % (str(start), str(end)))
# Create dir 1-5 at mountpoint.
ret = mkdir(mount_obj.client_system,
"%s/dir1/dir{%s}" % (mount_obj.mountpoint, dir_range))
self.assertTrue(ret, "Failed to create directory")
g.log.info("Directory created successfully")
# Create files inside each dir.
cmd = ('touch %s/dir1/dir{%s}/file{%s};' %
(mount_obj.mountpoint, dir_range, file_range))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "File creation failed")
g.log.info("File created successfull")
# Increment counter so that at next client dir and files are made
# with diff offset. Like at next client dir will be named
# dir6, dir7...dir10. Same with files.
start += 5
end += 5
# Rename all files inside dir1 at mountpoint on dir1
cmd = ('cd %s/dir1/dir1/; '
'for FILENAME in *;'
'do mv $FILENAME Unix_$FILENAME; cd ~;'
'done;' % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Failed to rename file on " "client")
g.log.info("Successfully renamed file on client")
# Truncate at any dir in mountpoint inside dir1
# start is an offset to be added to dirname to act on
# diff files at diff clients.
start = 1
for mount_obj in self.mounts:
cmd = ('cd %s/dir1/dir%s/; '
'for FILENAME in *;'
'do echo > $FILENAME; cd ~;'
'done;' % (mount_obj.mountpoint, str(start)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "Truncate failed")
g.log.info("Truncate of files successfull")
# Create softlink and hardlink of files in mountpoint. Start is an
# offset to be added to dirname to act on diff files at diff clients.
start = 1
for mount_obj in self.mounts:
cmd = ('cd %s/dir1/dir%s; '
'for FILENAME in *; '
'do ln -s $FILENAME softlink_$FILENAME; cd ~;'
'done;' % (mount_obj.mountpoint, str(start)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "Creating Softlinks have failed")
g.log.info("Softlink of files have been changed successfully")
cmd = ('cd %s/dir1/dir%s; '
'for FILENAME in *; '
'do ln $FILENAME hardlink_$FILENAME; cd ~;'
'done;' % (mount_obj.mountpoint, str(start + 1)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "Creating Hardlinks have failed")
g.log.info("Hardlink of files have been changed successfully")
start += 5
# chmod, chown, chgrp inside dir1
# start and end used as offset to access diff files
# at diff clients.
start, end = 2, 5
for mount_obj in self.mounts:
dir_file_range = '%s..%s' % (str(start), str(end))
cmd = ('chmod 777 %s/dir1/dir{%s}/file{%s}' %
(mount_obj.mountpoint, dir_file_range, dir_file_range))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "Changing mode of files has failed")
g.log.info("Mode of files have been changed successfully")
cmd = ('chown root %s/dir1/dir{%s}/file{%s}' %
(mount_obj.mountpoint, dir_file_range, dir_file_range))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "Changing owner of files has failed")
g.log.info("Owner of files have been changed successfully")
cmd = ('chgrp root %s/dir1/dir{%s}/file{%s}' %
(mount_obj.mountpoint, dir_file_range, dir_file_range))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "Changing group of files has failed")
g.log.info("Group of files have been changed successfully")
start += 5
end += 5
# Create tiny, small, medium and large file
# at mountpoint. Offset to differ filenames
# at diff clients.
offset = 1
for mount_obj in self.mounts:
cmd = 'fallocate -l 100 tiny_file%s.txt' % str(offset)
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "Fallocate for tiny files failed")
g.log.info("Fallocate for tiny files successfully")
cmd = 'fallocate -l 20M small_file%s.txt' % str(offset)
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "Fallocate for small files failed")
g.log.info("Fallocate for small files successfully")
cmd = 'fallocate -l 200M medium_file%s.txt' % str(offset)
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "Fallocate for medium files failed")
g.log.info("Fallocate for medium files successfully")
cmd = 'fallocate -l 1G large_file%s.txt' % str(offset)
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "Fallocate for large files failed")
g.log.info("Fallocate for large files successfully")
offset += 1
# Get arequal before replacing brick
ret, result_before_replacing_brick = (collect_mounts_arequal(
self.mounts[0]))
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before replacing of brick '
'is successful')
# Replacing a brick of random choice
ret = replace_brick_from_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, "Unexpected:Replace brick is not successful")
g.log.info("Expected : Replace brick is successful")
# Wait for brick to come online
ret = wait_for_bricks_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, "Unexpected:Bricks are not online")
g.log.info("Expected : Bricks are online")
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Unexpected:Heal has not yet completed')
g.log.info('Heal has completed successfully')
# Check if bricks are online
all_bricks = get_all_bricks(self.mnode, self.volname)
ret = are_bricks_online(self.mnode, self.volname, all_bricks)
self.assertTrue(ret, 'Unexpected:All bricks are not online')
g.log.info('All bricks are online')
# Get areequal after replacing brick
ret, result_after_replacing_brick = (collect_mounts_arequal(
self.mounts[0]))
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting areequal after replacing of brick '
'is successful')
# Comparing arequals
self.assertEqual(
result_before_replacing_brick, result_after_replacing_brick,
'Arequals are not equals before replacing '
'brick and after replacing brick')
g.log.info('Arequals are equals before replacing brick '
'and after replacing brick')
# Creating files on client side for dir1
# Write IO
all_mounts_procs, count = [], 1
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("/usr/bin/env python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 10 "
"--max-num-of-dirs 5 "
"--num-of-files 5 %s/dir1" %
(self.script_upload_path1, count, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
count += 10
# Replacing a brick while IO's are going on
ret = replace_brick_from_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, "Unexpected:Replace brick is not successful")
g.log.info("Expected : Replace brick is successful")
# Wait for brick to come online
ret = wait_for_bricks_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, "Unexpected:Bricks are not online")
g.log.info("Expected : Bricks are online")
# Validating IO's and waiting to complete
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Successfully validated all io's")
# Create 2 directories and start IO's which opens FD
ret = mkdir(self.mounts[0].client_system,
"%s/count{1..2}" % self.mounts[0].mountpoint)
self.assertTrue(ret, "Failed to create directories")
g.log.info("Directories created on %s successfully", self.mounts[0])
all_fd_procs, count = [], 1
for mount_obj in self.mounts:
cmd = ("cd %s ;/usr/bin/env python %s -n 10 -t 120 "
"-d 5 -c 16 --dir count%s" %
(mount_obj.mountpoint, self.script_upload_path2, count))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_fd_procs.append(proc)
count += 1
# Replacing a brick while open FD IO's are going on
ret = replace_brick_from_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, "Unexpected:Replace brick is not successful")
g.log.info("Expected : Replace brick is successful")
# Wait for brick to come online
ret = wait_for_bricks_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, "Unexpected:Bricks are not online")
g.log.info("Expected : Bricks are online")
# Validating IO's and waiting to complete
ret = validate_io_procs(all_fd_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Successfully validated all io's")
# Close connection and check file exist for memory log
ret = file_exists(self.mnode, '/var/log/glusterfs/mem_usage.log')
self.assertTrue(ret, "Unexpected:Memory log file does " "not exist")
g.log.info("Memory log file exists")
for proc in cmd_list_procs:
ret, _, _ = proc.async_communicate()
self.assertEqual(ret, 0, "Memory logging failed")
g.log.info("Memory logging is successful")
def test_data_self_heal_daemon_off(self):
"""
Test Data-Self-Heal (heal command)
Description:
- set the volume option
"metadata-self-heal": "off"
"entry-self-heal": "off"
"data-self-heal": "off"
- create IO
- Get areequal before getting bricks offline
- set the volume option
"self-heal-daemon": "off"
- bring down all bricks processes from selected set
- Get areequal after getting bricks offline and compare with
areequal before getting bricks offline
- modify the data
- bring bricks online
- set the volume option
"self-heal-daemon": "on"
- check daemons and start healing
- check if heal is completed
- check for split-brain
- add bricks
- do rebalance
- create 5k files
- while creating files - kill bricks and bring bricks online one by one
in cycle
- validate IO
"""
# Setting options
g.log.info('Setting options...')
options = {"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off",
}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Successfully set %s for volume %s"
% (options, self.volname))
# Creating files on client side
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s"
% (mount_obj.client_system, mount_obj.mountpoint))
# Create files
g.log.info('Creating files...')
command = ("python %s create_files -f 100 --fixed-file-size 1k %s"
% (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
# Get areequal before getting bricks offline
g.log.info('Getting areequal before getting bricks offline...')
ret, result_before_offline = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting areequal before getting bricks offline '
'is successful')
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' is set to 'off' successfully")
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = filter(None, (
bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks']))
# Bring brick offline
g.log.info('Bringing bricks %s offline...' % bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful'
% bricks_to_bring_offline)
# Get areequal after getting bricks offline
g.log.info('Getting areequal after getting bricks offline...')
ret, result_after_offline = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting areequal after getting bricks offline '
'is successful')
# Checking areequals before bringing bricks offline
# and after bringing bricks offline
self.assertEqual(result_before_offline, result_after_offline,
'Checksums before and '
'after bringing bricks online are not equal')
g.log.info('Checksums before and after bringing bricks online '
'are equal')
# Modify the data
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Modifying data for %s:%s" %
(mount_obj.client_system, mount_obj.mountpoint))
# Create files
g.log.info('Creating files...')
command = ("python %s create_files -f 100 --fixed-file-size 10k %s"
% (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
# Bring brick online
g.log.info('Bringing bricks %s online...' % bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful'
% bricks_to_bring_offline)
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info("Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online"
% self.volname))
g.log.info("Volume %s : All process are online" % self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Add bricks
g.log.info("Start adding bricks to volume...")
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to expand the volume %s", self.volname))
g.log.info("Expanding volume is successful on "
"volume %s" % self.volname)
# Do rebalance
ret, out, err = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, 'Failed to start rebalance')
g.log.info('Rebalance is started')
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Rebalance is not completed')
g.log.info('Rebalance is completed successfully')
# Create 1k files
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Modifying data for %s:%s" %
(mount_obj.client_system, mount_obj.mountpoint))
# Create files
g.log.info('Creating files...')
command = ("python %s create_files -f 1000 %s"
% (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Kill all bricks in cycle
bricks_list = get_all_bricks(self.mnode, self.volname)
for brick in bricks_list:
# Bring brick offline
g.log.info('Bringing bricks %s offline' % brick)
ret = bring_bricks_offline(self.volname, [brick])
self.assertTrue(ret, 'Failed to bring bricks %s offline' % brick)
ret = are_bricks_offline(self.mnode, self.volname,
[brick])
self.assertTrue(ret, 'Bricks %s are not offline'
% brick)
g.log.info('Bringing bricks %s offline is successful'
% bricks_to_bring_offline)
# Bring brick online
g.log.info('Bringing bricks %s online...' % brick)
ret = bring_bricks_online(self.mnode, self.volname,
[brick])
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful'
% bricks_to_bring_offline)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info("Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online"
% self.volname))
g.log.info("Volume %s : All process are online" % self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found or"
"more than one self heal daemon process"
"found")
g.log.info("All self-heal-daemons are online")
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
def test_rebalance_with_brick_down(self):
"""
Rebalance with brick down in replica
- Create a Replica volume.
- Bring down one of the brick down in the replica pair
- Do some IO and create files on the mount point
- Add a pair of bricks to the volume
- Initiate rebalance
- Bring back the brick which was down.
- After self heal happens, all the files should be present.
"""
# Log the volume info and status before brick is down.
log_volume_info_and_status(self.mnode, self.volname)
# Bring one fo the bricks offline
brick_list = get_all_bricks(self.mnode, self.volname)
ret = bring_bricks_offline(self.volname, choice(brick_list))
# Log the volume info and status after brick is down.
log_volume_info_and_status(self.mnode, self.volname)
# Create files at mountpoint.
cmd = (
"/usr/bin/env python %s create_files "
"-f 2000 --fixed-file-size 1k --base-file-name file %s"
% (self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(
self.mounts[0].client_system, cmd, user=self.mounts[0].user)
self.all_mounts_procs.append(proc)
# Wait for IO to complete.
self.assertTrue(wait_for_io_to_complete(self.all_mounts_procs,
self.mounts[0]),
"IO failed on some of the clients")
g.log.info("IO completed on the clients")
# Compute the arequal checksum before bringing all bricks online
arequal_before_all_bricks_online = collect_mounts_arequal(self.mounts)
# Log the volume info and status before expanding volume.
log_volume_info_and_status(self.mnode, self.volname)
# Expand the volume.
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to expand the volume %s", self.volname))
g.log.info("Expanding volume is successful on "
"volume %s", self.volname)
# Log the voluem info after expanding volume.
log_volume_info_and_status(self.mnode, self.volname)
# Start Rebalance.
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to start rebalance on the volume "
"%s", self.volname))
g.log.info("Successfully started rebalance on the volume %s",
self.volname)
# Wait for rebalance to complete
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(ret, ("Rebalance is not yet complete on the volume "
"%s", self.volname))
g.log.info("Rebalance is successfully complete on the volume %s",
self.volname)
# Log the voluem info and status before bringing all bricks online
log_volume_info_and_status(self.mnode, self.volname)
# Bring all bricks online.
ret, _, _ = volume_start(self.mnode, self.volname, force=True)
self.assertEqual(ret, 0, "Not able to start volume with force option")
g.log.info("Volume start with force option successful.")
# Log the volume info and status after bringing all beicks online
log_volume_info_and_status(self.mnode, self.volname)
# Monitor heal completion.
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, "heal has not yet completed")
g.log.info("Self heal completed")
# Compute the arequal checksum after all bricks online.
arequal_after_all_bricks_online = collect_mounts_arequal(self.mounts)
# Comparing arequal checksum before and after the operations.
self.assertEqual(arequal_before_all_bricks_online,
arequal_after_all_bricks_online,
"arequal checksum is NOT MATCHING")
g.log.info("arequal checksum is SAME")
def test_rebalance_with_quota_enabled(self):
"""
Test rebalance with quota enabled on root.
1. Create Volume of type distribute
2. Set Quota limit on the root directory
3. Do some IO to reach the Hard limit
4. After IO ends, compute arequal checksum
5. Add bricks to the volume.
6. Start rebalance
7. After rebalance is completed, check arequal checksum
"""
# Enable Quota
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(
ret, 0, ("Failed to enable quota on the volume %s", self.volname))
g.log.info("Successfully enabled quota on volume %s", self.volname)
# Set the Quota timeouts to 0 for strict accounting
ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, 0)
self.assertEqual(
ret, 0, ("Failed to set hard-timeout to 0 for %s", self.volname))
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, 0)
self.assertEqual(
ret, 0, ("Failed to set soft-timeout to 0 for %s", self.volname))
g.log.info("Quota soft and hard timeout has been set to 0 for %s",
self.volname)
# Set the quota limit of 1 GB on root dir of the volume
ret, _, _ = quota_limit_usage(self.mnode, self.volname, "/", "1GB")
self.assertEqual(ret, 0, "Failed to set Quota for dir root")
g.log.info("Successfully set quota limit for dir root")
# Do some IO until hard limit is reached.
cmd = ("/usr/bin/env python %s create_files "
"-f 1024 --fixed-file-size 1M --base-file-name file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
self.all_mounts_procs.append(proc)
# Wait for IO to complete and validate IO
self.assertTrue(
wait_for_io_to_complete(self.all_mounts_procs, self.mounts[0]),
"IO failed on some of the clients")
g.log.info("IO completed on the clients")
# Validate quota
ret = quota_validate(self.mnode,
self.volname,
path='/',
hard_limit=1073741824,
sl_exceeded=True,
hl_exceeded=True)
self.assertTrue(ret, "Quota validate Failed for '/'")
g.log.info("Quota Validated for path '/'")
# Compute arequal checksum.
arequal_checksum_before_rebalance = collect_mounts_arequal(self.mounts)
# Log Volume info and status before expanding volume.
log_volume_info_and_status(self.mnode, self.volname)
# Expand the volume.
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to expand the volume %s", self.volname))
g.log.info("Expanding volume is successful on "
"volume %s", self.volname)
# Log volume info and status after expanding volume.
log_volume_info_and_status(self.mnode, self.volname)
# Perform rebalance start operation.
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to start rebalance on the volume "
"%s", self.volname))
g.log.info("Rebalance started.")
# Check rebalance is in progress
rebalance_status = get_rebalance_status(self.mnode, self.volname)
ret = rebalance_status['aggregate']['statusStr']
self.assertEqual(ret, "in progress", ("Rebalance is not in "
"'in progress' state, either "
"rebalance is in completed state"
" or failed to get rebalance "
"status"))
g.log.info("Rebalance is 'in progress' state")
# Wait till rebalance ends.
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(ret, ("Rebalance is not yet complete on the volume "
"%s", self.volname))
g.log.info("Rebalance is successfully complete on the volume %s",
self.volname)
# Validate quota
ret = quota_validate(self.mnode,
self.volname,
path='/',
hard_limit=1073741824,
sl_exceeded=True,
hl_exceeded=True)
self.assertTrue(ret, "Quota validate Failed for '/'")
g.log.info("Quota Validated for path '/'")
# Compute arequal checksum.
arequal_checksum_after_rebalance = collect_mounts_arequal(self.mounts)
# Comparing arequals checksum before and after rebalance.
self.assertEqual(arequal_checksum_before_rebalance,
arequal_checksum_after_rebalance,
"arequal checksum is NOT MATCHING")
g.log.info("arequal checksum is SAME")
def test_gluster_clone_heal(self):
"""
Test gluster compilation on mount point(Heal command)
- Creating directory test_compilation
- Compile gluster on mountpoint
- Select bricks to bring offline
- Bring brick offline
- Validate IO
- Bring bricks online
- Wait for volume processes to be online
- Verify volume's all process are online
- Monitor heal completion
- Check for split-brain
- Get arequal after getting bricks online
- Compile gluster on mountpoint again
- Select bricks to bring offline
- Bring brick offline
- Validate IO
- Bring bricks online
- Wait for volume processes to be online
- Verify volume's all process are online
- Monitor heal completion
- Check for split-brain
- Get arequal after getting bricks online
"""
# pylint: disable=too-many-branches,too-many-statements,too-many-locals
# Creating directory test_compilation
ret = mkdir(self.mounts[0].client_system,
"{}/test_compilation".format(self.mounts[0].mountpoint))
self.assertTrue(ret, "Failed to create directory")
g.log.info(
"Directory 'test_compilation' on %s created "
"successfully", self.mounts[0])
# Compile gluster on mountpoint
cmd = ("cd %s/test_compilation ; rm -rf glusterfs; git clone"
" git://github.com/gluster/glusterfs.git ; cd glusterfs ;"
" ./autogen.sh ;./configure CFLAGS='-g3 -O0 -DDEBUG'; make ;"
" cd ../..;" % self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system, cmd)
# Select bricks to bring offline
bricks_to_bring_offline = select_volume_bricks_to_bring_offline(
self.mnode, self.volname)
self.assertIsNotNone(bricks_to_bring_offline, "List is empty")
# Bring brick offline
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks {} offline'.format(
bricks_to_bring_offline))
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Bricks {} are not offline'.format(bricks_to_bring_offline))
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Validate IO
self.assertTrue(validate_io_procs([proc], self.mounts[0]),
"IO failed on some of the clients")
# Bring bricks online
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret,
'Failed to bring bricks {} online'.format(bricks_to_bring_offline))
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume {} processes to "
"be online".format(self.volname)))
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret,
("Volume {} : All process are not online".format(self.volname)))
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info("Arequal of mountpoint %s", result_after_online)
# Compile gluster on mountpoint again
proc1 = g.run_async(self.mounts[0].client_system, cmd)
# Select bricks to bring offline
bricks_to_bring_offline = select_volume_bricks_to_bring_offline(
self.mnode, self.volname)
self.assertIsNotNone(bricks_to_bring_offline, "List is empty")
# Bring brick offline
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks {} offline'.format(
bricks_to_bring_offline))
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Bricks {} are not offline'.format(bricks_to_bring_offline))
# Validate IO
self.assertTrue(validate_io_procs([proc1], self.mounts[0]),
"IO failed on some of the clients")
# Bring bricks online
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret,
'Failed to bring bricks {} online'.format(bricks_to_bring_offline))
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume {} processes to "
"be online".format(self.volname)))
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret,
("Volume {} : All process are not online".format(self.volname)))
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
# Get arequal after getting bricks online
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info("Arequal of mountpoint %s", result_after_online)
def test_self_heal_50k_files_heal_command_by_add_brick(self):
"""
Test self-heal of 50k files (heal command)
Description:
- Set the volume option
"metadata-self-heal": "off"
"entry-self-heal": "off"
"data-self-heal": "off"
"self-heal-daemon": "off"
- Bring down all bricks processes from selected set
- Create IO (50k files)
- Get arequal before getting bricks online
- Bring bricks online
- Set the volume option
"self-heal-daemon": "on"
- Check for daemons
- Start healing
- Check if heal is completed
- Check for split-brain
- Get arequal after getting bricks online and compare with
arequal before getting bricks online
- Add bricks
- Do rebalance
- Get arequal after adding bricks and compare with
arequal after getting bricks online
"""
# pylint: disable=too-many-locals,too-many-statements
# Setting options
options = {
"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off",
"self-heal-daemon": "off"
}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Successfully set %s for volume %s", options, self.volname)
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = list(
filter(None, (bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks'])))
# Bring brick offline
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Creating files on client side
all_mounts_procs = []
# Create 50k files
g.log.info('Creating files...')
command = ("cd %s ; "
"for i in `seq 1 50000` ; "
"do dd if=/dev/urandom of=test.$i "
"bs=100k count=1 ; "
"done ;" % self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts[0]),
"IO failed on some of the clients")
# Get arequal before getting bricks online
ret, result_before_online = collect_mounts_arequal(self.mounts[0])
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks online '
'is successful')
# Bring brick online
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s online' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Setting options
ret = set_volume_options(self.mnode, self.volname,
{"self-heal-daemon": "on"})
self.assertTrue(ret, 'Failed to set option self-heal-daemon to ON.')
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode,
self.volname,
timeout_period=3600)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
ret, result_after_online = collect_mounts_arequal(self.mounts[0])
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks online '
'is successful')
# Checking arequals before bringing bricks online
# and after bringing bricks online
self.assertItemsEqual(
result_before_online, result_after_online, 'Checksums before and '
'after bringing bricks online are not equal')
g.log.info('Checksums before and after bringing bricks online '
'are equal')
# Add bricks
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to expand the volume when IO in "
"progress on volume %s", self.volname))
g.log.info("Expanding volume is successful on volume %s", self.volname)
# Do rebalance and wait for it to complete
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, 'Failed to start rebalance')
g.log.info('Rebalance is started')
ret = wait_for_rebalance_to_complete(self.mnode,
self.volname,
timeout=3600)
self.assertTrue(ret, 'Rebalance is not completed')
g.log.info('Rebalance is completed successfully')
# Get arequal after adding bricks
ret, result_after_adding_bricks = collect_mounts_arequal(
self.mounts[0])
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks ' 'is successful')
# Checking arequals after bringing bricks online
# and after adding bricks
self.assertItemsEqual(
result_after_online, result_after_adding_bricks,
'Checksums after bringing bricks online and '
'after adding bricks are not equal')
g.log.info('Checksums after bringing bricks online and '
'after adding bricks are equal')
def test_self_heal_differing_in_file_type(self):
"""
testing self heal of files with different file types
with default configuration
Description:
- create IO
- calculate arequal
- bring down all bricks processes from selected set
- calculate arequal and compare with arequal before
getting bricks offline
- modify the data
- arequal before getting bricks online
- bring bricks online
- check daemons and healing completion
- start healing
- calculate arequal and compare with arequal before bringing bricks
online and after bringing bricks online
"""
# pylint: disable=too-many-locals,too-many-statements
# Creating files on client side
all_mounts_procs = []
test_file_type_differs_self_heal_folder = \
'test_file_type_differs_self_heal'
g.log.info("Generating data for %s:%s",
self.mounts[0].client_system, self.mounts[0].mountpoint)
# Creating files
command = ("cd %s/ ; "
"mkdir %s ;"
"cd %s/ ;"
"for i in `seq 1 10` ; "
"do mkdir l1_dir.$i ; "
"for j in `seq 1 5` ; "
"do mkdir l1_dir.$i/l2_dir.$j ; "
"for k in `seq 1 10` ; "
"do dd if=/dev/urandom of=l1_dir.$i/l2_dir.$j/test.$k "
"bs=1k count=$k ; "
"done ; "
"done ; "
"done ; "
% (self.mounts[0].mountpoint,
test_file_type_differs_self_heal_folder,
test_file_type_differs_self_heal_folder))
proc = g.run_async(self.mounts[0].client_system, command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# wait for io to complete
self.assertTrue(
wait_for_io_to_complete(all_mounts_procs, self.mounts),
"Io failed to complete on some of the clients")
# Get arequal before getting bricks offline
g.log.info('Getting arequal before getting bricks offline...')
ret, result_before_offline = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks offline '
'is successful')
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = bricks_to_bring_offline_dict['volume_bricks']
# Bring brick offline
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Get arequal after getting bricks offline
g.log.info('Getting arequal after getting bricks offline...')
ret, result_after_offline = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks offline '
'is successful')
# Checking arequals before bringing bricks offline
# and after bringing bricks offline
self.assertEqual(sorted(result_before_offline),
sorted(result_after_offline),
'Checksums before and after bringing bricks'
' offline are not equal')
g.log.info('Checksums before and after '
'bringing bricks offline are equal')
# Modify the data
all_mounts_procs = []
g.log.info("Modifying data for %s:%s",
self.mounts[0].client_system, self.mounts[0].mountpoint)
command = ("cd %s/%s/ ; "
"for i in `seq 1 10` ; "
"do for j in `seq 1 5` ; "
"do for k in `seq 1 10` ; "
"do rm -f l1_dir.$i/l2_dir.$j/test.$k ; "
"mkdir l1_dir.$i/l2_dir.$j/test.$k ; "
"done ; "
"done ; "
"done ;"
% (self.mounts[0].mountpoint,
test_file_type_differs_self_heal_folder))
proc = g.run_async(self.mounts[0].client_system, command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients"
)
# Get arequal before getting bricks online
g.log.info('Getting arequal before getting bricks online...')
ret, result_before_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks online '
'is successful')
# Bring brick online
g.log.info('Bringing bricks %s online', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info("Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online"
% self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
g.log.info('Getting arequal after getting bricks online...')
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks online '
'is successful')
# Checking arequals before bringing bricks online
# and after bringing bricks online
self.assertEqual(sorted(result_before_online),
sorted(result_after_online),
'Checksums before and after bringing bricks'
' online are not equal')
g.log.info('Checksums before and after bringing bricks online '
'are equal')
def test_fix_layout_start(self):
# pylint: disable=too-many-statements
# Get arequal checksum before starting fix-layout
g.log.info("Getting arequal checksum before fix-layout")
arequal_checksum_before_fix_layout = collect_mounts_arequal(
self.mounts)
# Log Volume Info and Status before expanding the volume.
g.log.info("Logging volume info and Status before expanding volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(
ret, "Logging volume info and status failed on "
"volume %s" % self.volname)
g.log.info(
"Successful in logging volume info and status of volume "
"%s", self.volname)
# Form brick list for expanding volume
add_brick_list = form_bricks_list_to_add_brick(self.mnode,
self.volname,
self.servers,
self.all_servers_info,
distribute_count=1)
self.assertIsNotNone(add_brick_list,
("Volume %s: Failed to form "
"bricks list to expand", self.volname))
g.log.info("Volume %s: Formed bricks list to expand", self.volname)
# Expanding volume by adding bricks to the volume
g.log.info("Volume %s: Expand start")
ret, _, _ = add_brick(self.mnode, self.volname, add_brick_list)
self.assertEqual(ret, 0, ("Volume %s: Expand failed", self.volname))
g.log.info("Volume %s: Expand successful", self.volname)
# Wait for gluster processes to come online
g.log.info("Wait for gluster processes to come online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s: one or more volume process are "
"not up", self.volname))
g.log.info("All volume %s processes are online", self.volname)
# Log Volume Info and Status after expanding the volume
g.log.info("Logging volume info and Status after expanding volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(
ret, "Logging volume info and status failed on "
"volume %s" % self.volname)
g.log.info(
"Successful in logging volume info and status of volume "
"%s", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online", self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Start Rebalance fix-layout
g.log.info("Starting fix-layout on the volume")
ret, _, _ = rebalance_start(self.mnode, self.volname, fix_layout=True)
self.assertEqual(ret, 0, ("Volume %s: fix-layout start failed"
"%s", self.volname))
g.log.info("Volume %s: fix-layout start success", self.volname)
# Wait for fix-layout to complete
g.log.info("Waiting for fix-layout to complete")
ret = wait_for_fix_layout_to_complete(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s: Fix-layout is still in-progress", self.volname))
g.log.info("Volume %s: Fix-layout completed successfully",
self.volname)
# Check Rebalance status after fix-layout is complete
g.log.info("Checking Rebalance status")
ret, _, _ = rebalance_status(self.mnode, self.volname)
self.assertEqual(
ret, 0,
("Volume %s: Failed to get rebalance status", self.volname))
g.log.info("Volume %s: Successfully got rebalance status",
self.volname)
# Get arequal checksum after fix-layout is complete
g.log.info("arequal after fix-layout is complete")
arequal_checksum_after_fix_layout = collect_mounts_arequal(self.mounts)
# Compare arequals checksum before and after fix-layout
g.log.info("Comparing checksum before and after fix-layout")
self.assertEqual(arequal_checksum_before_fix_layout,
arequal_checksum_after_fix_layout,
"arequal checksum is NOT MATCHNG")
g.log.info("arequal checksum is SAME")
# Check if there are any file migrations after fix-layout
status_info = get_rebalance_status(self.mnode, self.volname)
for node in range(len(status_info['node'])):
status_info = get_rebalance_status(self.mnode, self.volname)
file_migration_count = status_info['node'][node]['files']
self.assertEqual(
int(file_migration_count), 0,
("Server %s: Few files are migrated", self.servers[node]))
g.log.info("Server %s: No files are migrated")
# Check if new bricks contains any files
for brick in add_brick_list:
brick_node, brick_path = brick.split(":")
cmd = ('find %s -type f ! -perm 1000 | grep -ve .glusterfs' %
brick_path)
_, out, _ = g.run(brick_node, cmd)
self.assertEqual(
len(out), 0,
(("Files(excluded linkto files) are present on %s:%s"),
(brick_node, brick_path)))
g.log.info("No files (excluded linkto files) are present on %s:%s",
brick_node, brick_path)
def test_rebalance_with_hidden_files(self):
# pylint: disable=too-many-statements
# Start IO on mounts
g.log.info("Starting IO on all mounts...")
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("python %s create_files "
"--base-file-name . "
"-f 99 %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# validate IO
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# List all files and dirs created
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
self.assertTrue(ret, "Failed to list all files and dirs")
g.log.info("Listing all files and directories is successful")
# Verify DHT values across mount points
for mount_obj in self.mounts:
g.log.debug("Verifying hash layout values %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
ret = validate_files_in_dir(mount_obj.client_system,
mount_obj.mountpoint,
test_type=FILE_ON_HASHED_BRICKS,
file_type=FILETYPE_FILES)
self.assertTrue(
ret, "Expected - Files are created on only "
"sub-volume according to its hashed value")
g.log.info("Hash layout values are verified %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
# Getting areequal checksum before rebalance
g.log.info("Getting areequal checksum before rebalance")
arequal_checksum_before_rebalance = collect_mounts_arequal(self.mounts)
# Log Volume Info and Status before expanding the volume.
g.log.info("Logging volume info and Status before expanding volume")
log_volume_info_and_status(self.mnode, self.volname)
# Expanding volume by adding bricks to the volume
g.log.info("Start adding bricks to volume")
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to expand the volume %s", self.volname))
g.log.info("Expanding volume is successful on "
"volume %s", self.volname)
# Wait for gluster processes to come online
g.log.info("Wait for gluster processes to come online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online ", self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Log Volume Info and Status after expanding the volume
g.log.info("Logging volume info and Status after expanding volume")
log_volume_info_and_status(self.mnode, self.volname)
# Start Rebalance
g.log.info("Starting Rebalance on the volume")
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to start rebalance on the volume "
"%s", self.volname))
g.log.info("Successfully started rebalance on the volume %s",
self.volname)
# Wait for rebalance to complete
g.log.info("Waiting for rebalance to complete")
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(ret, ("Rebalance is not yet complete on the volume "
"%s", self.volname))
g.log.info("Rebalance is successfully complete on the volume %s",
self.volname)
# Checking if there are any migration failures
status = get_rebalance_status(self.mnode, self.volname)
for each_node in status['node']:
failed_files_count = int(each_node['failures'])
self.assertEqual(
failed_files_count, 0,
"Rebalance failed to migrate few files on %s" %
each_node['nodeName'])
g.log.info("There are no migration failures")
# Getting areequal checksum after rebalance
g.log.info("Getting areequal checksum after rebalance")
arequal_checksum_after_rebalance = collect_mounts_arequal(self.mounts)
# Comparing arequals checksum before and after rebalance
g.log.info("Comparing arequals checksum before and after rebalance")
self.assertEqual(arequal_checksum_before_rebalance,
arequal_checksum_after_rebalance,
"arequal checksum is NOT MATCHNG")
g.log.info("arequal checksum is SAME")
def test_rebalance_with_force(self):
# Getting arequal checksum before rebalance
g.log.info("Getting arequal checksum before rebalance")
arequal_checksum_before_rebalance = collect_mounts_arequal(self.mounts)
# Log Volume Info and Status before expanding the volume.
g.log.info("Logging volume info and Status before expanding volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(
ret, "Logging volume info and status failed on "
"volume %s" % self.volname)
g.log.info(
"Successful in logging volume info and"
"status of volume %s", self.volname)
# Expanding volume by adding bricks to the volume
g.log.info("Start adding bricks to volume")
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Volume %s: Expand failed", self.volname))
g.log.info("Volume %s: Expand successful", self.volname)
# Wait for gluster processes to come online
g.log.info("Wait for gluster processes to come online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s: one or more volume process are "
"not up", self.volname))
g.log.info("All volume %s processes are online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online", self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Log Volume Info and Status after expanding the volume
g.log.info("Logging volume info and Status after expanding volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info(
"Successful in logging volume info and"
"status of volume %s", self.volname)
# Start Rebalance with force
g.log.info("Starting Rebalance on the volume")
ret, _, _ = rebalance_start(self.mnode, self.volname, force=True)
self.assertEqual(ret, 0, ("Volume %s: Failed to start rebalance with "
"force", self.volname))
g.log.info("Volume %s: Started rebalance with force option",
self.volname)
# Wait for rebalance to complete
g.log.info("Waiting for rebalance to complete")
ret = wait_for_rebalance_to_complete(self.mnode,
self.volname,
timeout=600)
self.assertTrue(
ret, ("Volume %s: Rebalance is still in-progress ", self.volname))
g.log.info("Volume %s: Rebalance completed", self.volname)
# Getting arequal checksum after rebalance
g.log.info("Getting arequal checksum after rebalance with force "
"option")
arequal_checksum_after_rebalance = collect_mounts_arequal(self.mounts)
# Comparing arequals checksum before and after rebalance with force
# option
g.log.info("Comparing arequals checksum before and after rebalance"
"with force option")
self.assertEqual(arequal_checksum_before_rebalance,
arequal_checksum_after_rebalance,
"arequal checksum is NOT MATCHNG")
g.log.info("arequal checksum is SAME")
# Checking if rebalance skipped any files
status = get_rebalance_status(self.mnode, self.volname)
for each_node in status['node']:
self.assertEqual(
int(each_node['skipped']), 0,
"Few files are skipped on node %s" % each_node['nodeName'])
g.log.info("No files are skipped on %s", each_node['nodeName'])
def test_manual_heal_should_trigger_heal(self):
"""
- create a single brick volume
- add some files and directories
- get arequal from mountpoint
- add-brick such that this brick makes the volume a replica vol 1x2
- start heal
- make sure heal is completed
- get arequals from all bricks and compare with arequal from mountpoint
"""
# pylint: disable=too-many-statements,too-many-locals
# Start IO on mounts
g.log.info("Starting IO on all mounts...")
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("python %s create_deep_dirs_with_files "
"--dir-length 1 "
"--dir-depth 1 "
"--max-num-of-dirs 1 "
"--num-of-files 10 %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
g.log.info("IO on %s:%s is started successfully",
mount_obj.client_system, mount_obj.mountpoint)
self.io_validation_complete = False
# Validate IO
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# Get arequal for mount before adding bricks
g.log.info('Getting arequal before adding bricks...')
ret, arequals = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after healing is successful')
mount_point_total = arequals[0].splitlines()[-1].split(':')[-1]
# Form brick list to add
g.log.info('Forming brick list to add...')
bricks_to_add = form_bricks_list(self.mnode, self.volname, 1,
self.servers, self.all_servers_info)
g.log.info('Brick list to add: %s', bricks_to_add)
# Add bricks
g.log.info("Start adding bricks to volume...")
ret, _, _ = add_brick(self.mnode,
self.volname,
bricks_to_add,
force=True,
replica_count=2)
self.assertFalse(ret, "Failed to add bricks %s" % bricks_to_add)
g.log.info("Adding bricks is successful on volume %s", self.volname)
# Make sure the newly added bricks are available in the volume
# get the bricks for the volume
g.log.info("Fetching bricks for the volume: %s", self.volname)
bricks_list = get_all_bricks(self.mnode, self.volname)
g.log.info("Brick list: %s", bricks_list)
for brick in bricks_to_add:
self.assertIn(brick, bricks_list,
'Brick %s is not in brick list' % brick)
g.log.info('New bricks are present in the volume')
# Make sure volume change from distribute to replicate volume
vol_info_dict = get_volume_type_info(self.mnode, self.volname)
vol_type = vol_info_dict['volume_type_info']['typeStr']
self.assertEqual(
'Replicate', vol_type, 'Volume type is not converted to Replicate '
'after adding bricks')
g.log.info('Volume type is successfully converted to Replicate '
'after adding bricks')
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal on bricks and compare with mount_point_total
# It should be the same
g.log.info('Getting arequal on bricks...')
arequals_after_heal = {}
for brick in bricks_list:
g.log.info('Getting arequal on bricks %s...', brick)
node, brick_path = brick.split(':')
command = ('arequal-checksum -p %s '
'-i .glusterfs -i .landfill -i .trashcan' % brick_path)
ret, arequal, _ = g.run(node, command)
self.assertFalse(ret, 'Failed to get arequal on brick %s' % brick)
g.log.info('Getting arequal for %s is successful', brick)
brick_total = arequal.splitlines()[-1].split(':')[-1]
arequals_after_heal[brick] = brick_total
self.assertEqual(
mount_point_total, brick_total,
'Arequals for mountpoint and %s are not equal' % brick)
g.log.info('Arequals for mountpoint and %s are equal', brick)
g.log.info('All arequals are equal for replicated')
def test_multiple_clients_dd_on_same_file_default(self):
"""
- Create 2GB file
- While creating file, start reading file
- Bring down brick1
- Bring back the brick brick1
- Start healing
- Bring down brick1
- Wait for IO to complete
- Wait for reading to complete
- Bring back the brick brick1
- Start healing
- Wait for heal to complete
- Check for split-brain
- Calculate arequals on all the bricks and compare with mountpoint
"""
# pylint: disable=too-many-statements,too-many-locals
bricks_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(bricks_list, 'Brick list is None')
# Creating files on client side
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
# Create files
g.log.info('Creating files...')
command = ("cd %s ; "
"dd if=/dev/urandom of=test_file bs=1M count=2020"
% mount_obj.mountpoint)
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Reading files on client side
all_mounts_procs_read = []
for mount_obj in self.mounts:
g.log.info("Reading data for %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
# Create files
g.log.info('Reading files...')
command = ("python %s read %s"
% (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
all_mounts_procs_read.append(proc)
# Bring brick1 offline
g.log.info('Bringing bricks %s offline...', bricks_list[1])
ret = bring_bricks_offline(self.volname, [bricks_list[1]])
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_list[1])
ret = are_bricks_offline(self.mnode, self.volname,
[bricks_list[1]])
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_list[1])
g.log.info('Bringing bricks %s offline is successful',
bricks_list[1])
# Bring brick1 online
g.log.info('Bringing bricks %s online...', bricks_list[1])
ret = bring_bricks_online(self.mnode, self.volname,
[bricks_list[1]])
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_list[1])
g.log.info('Bringing bricks %s online is successful',
bricks_list[1])
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Bring brick1 offline
g.log.info('Bringing bricks %s offline...', bricks_list[1])
ret = bring_bricks_offline(self.volname, [bricks_list[1]])
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_list[1])
ret = are_bricks_offline(self.mnode, self.volname,
[bricks_list[1]])
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_list[1])
g.log.info('Bringing bricks %s offline is successful',
bricks_list[1])
# Validate IO
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients"
)
# Validate reading
self.assertTrue(
validate_io_procs(all_mounts_procs_read, self.mounts),
"Reading failed on some of the clients"
)
self.io_validation_complete = True
# Bring brick1 online
g.log.info('Bringing bricks %s online...', bricks_list[1])
ret = bring_bricks_online(self.mnode, self.volname,
[bricks_list[1]])
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_list[1])
g.log.info('Bringing bricks %s online is successful',
bricks_list[1])
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal for mount
g.log.info('Getting arequal...')
ret, arequals = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after healing is successful')
mount_point_total = arequals[0].splitlines()[-1].split(':')[-1]
# Get arequal on bricks and compare with mount_point_total
# It should be the same
g.log.info('Getting arequal on bricks...')
arequals_after_heal = {}
for brick in bricks_list:
g.log.info('Getting arequal on bricks %s...', brick)
node, brick_path = brick.split(':')
command = ('arequal-checksum -p %s '
'-i .glusterfs -i .landfill -i .trashcan'
% brick_path)
ret, arequal, _ = g.run(node, command)
self.assertFalse(ret, 'Failed to get arequal on brick %s'
% brick)
g.log.info('Getting arequal for %s is successful', brick)
brick_total = arequal.splitlines()[-1].split(':')[-1]
arequals_after_heal[brick] = brick_total
self.assertEqual(mount_point_total, brick_total,
'Arequals for mountpoint and %s are not equal'
% brick)
g.log.info('Arequals for mountpoint and %s are equal', brick)
g.log.info('All arequals are equal')
def test_data_self_heal_algorithm_full_default(self):
"""
Test Volume Option - 'cluster.data-self-heal-algorithm' : 'full'
Description:
- set the volume option "data-self-heal-algorithm" to value "full"
- create IO
- bring down all bricks processes from selected set
- modify the data
- calculate arequal
- bring bricks online
- start healing
- calculate arequal and compare with arequal before bringing bricks
offline and after bringing bricks online
"""
# pylint: disable=too-many-locals,too-many-statements
# Setting options
g.log.info('Setting options "data-self-heal-algorithm": "full"...')
options = {"data-self-heal-algorithm": "full"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Option 'data-self-heal-algorithm' is set to 'full' "
"successfully")
# Creating files on client side
all_mounts_procs = []
g.log.info("Generating data for %s:%s", self.mounts[0].client_system,
self.mounts[0].mountpoint)
# Creating files
command = "/usr/bin/env python %s create_files -f 100 %s" % (
self.script_upload_path, self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = list(
filter(None, (bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks'])))
# Bring brick offline
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Modify the data
all_mounts_procs = []
g.log.info("Modifying data for %s:%s", self.mounts[0].client_system,
self.mounts[0].mountpoint)
command = ("/usr/bin/env python %s create_files -f 100 "
"--fixed-file-size 1M %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# Get arequal before getting bricks online
g.log.info('Getting arequal before getting bricks online...')
ret, result_before_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks online '
'is successful')
# Bring brick online
g.log.info('Bringing bricks %s online...', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s online' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
g.log.info('Getting arequal after getting bricks online...')
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks online '
'is successful')
# Checking arequals before bringing bricks online
# and after bringing bricks online
self.assertItemsEqual(result_before_online, result_after_online,
'Checksums are not equal')
g.log.info('Checksums before bringing bricks online '
'and after bringing bricks online are equal')
def test_entry_transaction_crash_consistency_rename(self):
"""
Test entry transaction crash consistency : rename
Description:
- Create IO of 50 files
- Rename 20 files
- Calculate arequal before creating snapshot
- Create snapshot
- Rename 20 files more
- Stop the volume
- Restore snapshot
- Start the volume
- Get arequal after restoring snapshot
- Compare arequals
"""
# Creating files on client side
count = 1
for mount_obj in self.mounts:
cmd = ("/usr/bin/env python %s create_files "
"--base-file-name %d -f 25 %s"
% (self.script_upload_path,
count, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
count = count + 10
# Wait for IO to complete
ret = wait_for_io_to_complete(self.all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed to complete on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
# Rename files
self.all_mounts_procs, self.io_validation_complete = [], False
cmd = ("/usr/bin/env python %s mv -s FirstRename %s"
% (self.script_upload_path,
self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
self.all_mounts_procs.append(proc)
# Wait for IO to complete
ret = wait_for_io_to_complete(self.all_mounts_procs, self.mounts[0])
self.assertTrue(ret, "IO failed to complete on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
# Get arequal before creating snapshot
ret, result_before_snapshot = collect_mounts_arequal(self.mounts[0])
self.assertTrue(ret, "Collecting arequal-checksum failed")
# Create snapshot
snapshot_name = ('entry_transaction_crash_consistency_rename-%s-%s'
% (self.volname, self.mount_type))
ret, _, err = snap_create(self.mnode, self.volname, snapshot_name)
self.assertEqual(ret, 0, err)
g.log.info("Snapshot %s created successfully", snapshot_name)
# Rename files
self.all_mounts_procs, self.io_validation_complete = [], False
cmd = ("/usr/bin/env python %s mv -s SecondRename %s"
% (self.script_upload_path,
self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
self.all_mounts_procs.append(proc)
# Wait for IO to complete
ret = wait_for_io_to_complete(self.all_mounts_procs, self.mounts[0])
self.assertTrue(ret, "IO failed to complete on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
# Restore snapshot
ret = snap_restore_complete(self.mnode, self.volname,
snapshot_name)
self.assertTrue(ret, 'Failed to restore snapshot %s'
% snapshot_name)
g.log.info("Snapshot %s restored successfully", snapshot_name)
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Wait for volume graph to get loaded.
sleep(10)
# Get arequal after restoring snapshot
ret, result_after_restoring = collect_mounts_arequal(self.mounts[0])
self.assertTrue(ret, "Collecting arequal-checksum failed")
# Checking arequal before creating snapshot
# and after restoring snapshot
self.assertEqual(result_before_snapshot, result_after_restoring,
'Checksums are not equal')
g.log.info('Checksums are equal')
