def test_replica_to_arbiter_volume_with_io(self):
"""
Description: Replica 3 to arbiter conversion with ongoing IO's
Steps :
1) Create a replica 3 volume and start volume.
2) Set client side self heal off.
3) Fuse mount the volume.
4) Create directory dir1 and write data.
Example: untar linux tar from the client into the dir1
5) When IO's is running, execute remove-brick command,
and convert replica 3 to replica 2 volume
6) Execute add-brick command and convert to arbiter volume,
provide the path of new arbiter brick.
7) Issue gluster volume heal.
8) Heal should be completed with no files in split-brain.
"""
# pylint: disable=too-many-statements
# Create a dir to start untar
self.linux_untar_dir = "{}/{}".format(self.mounts[0].mountpoint,
"linuxuntar")
ret = mkdir(self.clients[0], self.linux_untar_dir)
self.assertTrue(ret, "Failed to create dir linuxuntar for untar")
# Start linux untar on dir linuxuntar
self.io_process = run_linux_untar(self.clients[0],
self.mounts[0].mountpoint,
dirs=tuple(['linuxuntar']))
self.is_io_running = True
# Convert relicated to arbiter volume
self._convert_replicated_to_arbiter_volume()
# Wait for IO to complete.
ret = self._wait_for_untar_completion()
self.assertFalse(ret, "IO didn't complete or failed on client")
self.is_io_running = False
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode,
self.volname,
timeout_period=3600)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
def _test_driver(self, op_type, invoke_heal=False, initial_io=None):
'''Driver for all tests'''
self._perform_io_and_disable_self_heal(initial_io=initial_io)
self._perform_brick_ops_and_enable_self_heal(op_type=op_type)
if invoke_heal:
# Invoke `glfsheal`
self.assertTrue(trigger_heal(self.mnode, self.volname),
'Unable to trigger index heal on the volume')
self._validate_heal_completion_and_arequal(op_type=op_type)
def test_memory_leak_in_shd_with_cache_invalidation_on(self):
"""
Test case:
1. Create a volume, start it and mount it.
2. Set features.cache-invalidation to ON.
3. Start I/O from mount point.
4. Run gluster volume heal command in a loop
5. Check if there are any memory leaks and OOM killers on servers.
"""
# Start monitoring resource usage on servers and clients
monitor_proc_dict = self.start_memory_and_cpu_usage_logging(
self.test_id, count=10)
self.assertIsNotNone(
monitor_proc_dict, "Failed to start monitoring on servers and"
" clients")
# Set features.cache-invalidation to ON
ret = set_volume_options(self.mnode, self.volname,
{'features.cache-invalidation': 'on'})
self.assertTrue(ret, "Failed to set features.cache-invalidation to ON")
g.log.info("Successfully set features.cache-invalidation to ON")
# Start multiple I/O from mount points
self.list_of_io_processes = []
cmd = (
"cd {};for i in `seq 1 1000`;do echo 'abc' > myfile;done".format(
self.mounts[0].mountpoint))
ret = g.run_async(self.mounts[0].client_system, cmd)
self.list_of_io_processes = [ret]
self.is_io_running = True
# Run gluster volume heal command in a loop for 100 iterations
for iteration in range(0, 100):
g.log.info("Running gluster volume heal command for %d time",
iteration)
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, "Heal command triggered successfully")
g.log.info("Ran gluster volume heal command in a loop for "
"100 iterations.")
# Wait for I/O to complete and validate I/O on mount points
ret = validate_io_procs(self.list_of_io_processes, self.mounts[0])
self.assertTrue(ret, "I/O failed on mount point")
self.is_io_running = False
# Wait for monitoring processes to complete
ret = wait_for_logging_processes_to_stop(monitor_proc_dict,
cluster=True)
self.assertTrue(ret, "ERROR: Failed to stop monitoring processes")
# Check if there are any memory leaks and OOM killers
ret = self.check_for_memory_leaks_and_oom_kills_on_servers(
self.test_id)
self.assertFalse(ret,
"Memory leak and OOM kills check failed on servers")
def test_gfid_heal(self):
"""
- Create a 1x3 volume and fuse mount it.
- Create 1 directory with 1 file inside it directly on each brick.
- Access the directories from the mount.
- Launch heals and verify that the heals are over.
- Verify that the files and directories have gfid assigned.
"""
# pylint: disable=too-many-statements
# Create data on the bricks.
g.log.info("Creating directories and files on the backend.")
bricks_list = get_all_bricks(self.mnode, self.volname)
i = 0
for brick in bricks_list:
i += 1
brick_node, brick_path = brick.split(":")
ret, _, _ = g.run(brick_node, "mkdir %s/dir%d" % (brick_path, i))
self.assertEqual(ret, 0, "Dir creation failed on %s" % brick_path)
ret, _, _ = g.run(brick_node,
"touch %s/dir%d/file%d" % (brick_path, i, i))
self.assertEqual(ret, 0, "file creation failed on %s" % brick_path)
g.log.info("Created directories and files on the backend.")
# To circumvent is_fresh_file() check in glusterfs code.
time.sleep(2)
# Access files from mount
for i in range(1, 4):
cmd = ("ls %s/dir%d/file%d" % (self.mounts[0].mountpoint, i, i))
ret, _, _ = g.run(self.clients[0], cmd)
self.assertEqual(
ret, 0, "Failed to access dir%d/file%d on %s" %
(i, i, self.mounts[0].mountpoint))
# Trigger heal
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Starting heal failed')
g.log.info('Index heal launched')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Verify gfid and links at the backend.
self.verify_gfid_and_link_count("dir1", "file1")
self.verify_gfid_and_link_count("dir2", "file2")
self.verify_gfid_and_link_count("dir3", "file3")
def test_metadata_heal_from_heal_cmd(self):
'''Description: Verify files heal after triggering heal command when
metadata operations are performed while a brick was down
Steps:
1. Create, mount and run IO on volume
2. Set `self-heal-daemon` to `off`, cyclic brick down and perform
metadata operations
3. Set `self-heal-daemon` to `on`, invoke `gluster vol <vol> heal`
4. Validate areequal checksum on backend bricks
'''
op_type = 'metadata'
self._perform_io_and_disable_self_heal()
self._perform_brick_ops_and_enable_self_heal(op_type=op_type)
# Invoke `glfsheal`
self.assertTrue(trigger_heal(self.mnode, self.volname),
'Unable to trigger index heal on the volume')
self._validate_heal_completion_and_arequal(op_type=op_type)
g.log.info(
'Pass: Verification of metadata heal via `glfsheal` is complete')
def test_afr_gfid_heal(self):
"""
Description: This test case runs split-brain resolution
on a 5 files in split-brain on a 1x2 volume.
After resolving split-brain, it makes sure that
split brain resolution doesn't work on files
already in split brain.
"""
g.log.info("disabling the self heal daemon")
ret = disable_self_heal_daemon(self.mnode, self.volname)
self.assertTrue(ret, "unable to disable self heal daemon")
g.log.info("Successfully disabled the self heal daemon")
# getting list of all bricks
all_bricks = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(all_bricks, "failed to get list of bricks")
g.log.info("bringing down brick1")
ret = bring_bricks_offline(self.volname, all_bricks[0:1])
self.assertTrue(ret, "unable to bring brick1 offline")
g.log.info("Successfully brought the following brick offline "
": %s", str(all_bricks[0]))
g.log.info("verifying if brick1 is offline")
ret = are_bricks_offline(self.mnode, self.volname, all_bricks[0:1])
self.assertTrue(ret, "brick1 is still online")
g.log.info("verified: brick1 is offline")
g.log.info("creating 5 files from mount point")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s create_files "
"-f 5 --base-file-name test_file --fixed-file-size 1k %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate I/O
g.log.info("Wait for IO to complete and validate IO.....")
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("IO is successful on all mounts")
g.log.info("Successfully created a file from mount point")
g.log.info("bringing brick 1 back online")
ret = bring_bricks_online(self.mnode, self.volname, all_bricks[0:1])
self.assertIsNotNone(ret, "unable to bring brick 1 online")
g.log.info("Successfully brought the following brick online "
": %s", str(all_bricks[0]))
g.log.info("verifying if brick1 is online")
ret = are_bricks_online(self.mnode, self.volname, all_bricks[0:1])
self.assertTrue(ret, "brick1 is not online")
g.log.info("verified: brick1 is online")
g.log.info("bringing down brick2")
ret = bring_bricks_offline(self.volname, all_bricks[1:2])
self.assertTrue(ret, "unable to bring brick2 offline")
g.log.info("Successfully brought the following brick offline "
": %s", str(all_bricks[1]))
g.log.info("verifying if brick2 is offline")
ret = are_bricks_offline(self.mnode, self.volname, all_bricks[1:2])
self.assertTrue(ret, "brick2 is still online")
g.log.info("verified: brick2 is offline")
g.log.info("creating 5 new files of same name from mount point")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s create_files "
"-f 5 --base-file-name test_file --fixed-file-size 10k %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate I/O
g.log.info("Wait for IO to complete and validate IO.....")
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("IO is successful on all mounts")
g.log.info("Successfully created a new file of same name "
"from mount point")
g.log.info("bringing brick2 back online")
ret = bring_bricks_online(self.mnode, self.volname, all_bricks[1:2])
self.assertIsNotNone(ret, "unable to bring brick2 online")
g.log.info("Successfully brought the following brick online "
": %s", str(all_bricks[1]))
g.log.info("verifying if brick2 is online")
ret = are_bricks_online(self.mnode, self.volname, all_bricks[1:2])
self.assertTrue(ret, "brick2 is not online")
g.log.info("verified: brick2 is online")
g.log.info("enabling the self heal daemon")
ret = enable_self_heal_daemon(self.mnode, self.volname)
self.assertTrue(ret, "failed to enable self heal daemon")
g.log.info("Successfully enabled the self heal daemon")
g.log.info("checking if volume is in split-brain")
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertTrue(ret, "unable to create split-brain scenario")
g.log.info("Successfully created split brain scenario")
g.log.info("resolving split-brain by choosing first brick as "
"the source brick")
node, brick_path = all_bricks[0].split(':')
for fcount in range(5):
command = ("gluster v heal " + self.volname + " split-brain "
"source-brick " + all_bricks[0] + ' /test_file' +
str(fcount) + '.txt')
ret, _, _ = g.run(node, command)
self.assertEqual(ret, 0, "command execution not successful")
# triggering heal
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, "heal not triggered")
g.log.info("Successfully triggered heal")
# waiting for heal to complete
ret = monitor_heal_completion(self.mnode,
self.volname,
timeout_period=240)
self.assertTrue(ret, "heal not completed")
g.log.info("Heal completed successfully")
# checking if any file is in split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, "file still in split-brain")
g.log.info("Successfully resolved split brain situation using "
"CLI based resolution")
g.log.info("resolving split-brain on a file not in split-brain")
node, brick_path = all_bricks[0].split(':')
command = ("gluster v heal " + self.volname + " split-brain "
"source-brick " + all_bricks[1] + " /test_file0.txt")
ret, _, _ = g.run(node, command)
self.assertNotEqual(
ret, 0, "Unexpected: split-brain resolution "
"command is successful on a file which"
" is not in split-brain")
g.log.info("Expected: split-brian resolution command failed on "
"a file which is not in split-brain")
g.log.info("checking the split-brain status of each file")
for fcount in range(5):
fpath = (self.mounts[0].mountpoint + '/test_file' + str(fcount) +
'.txt')
status = get_fattr(self.mounts[0].client_system, fpath,
'replica.split-brain-status')
compare_string = ("The file is not under data or metadata "
"split-brain")
self.assertEqual(
status.rstrip('\x00'), compare_string,
"file test_file%s is under"
" split-brain" % str(fcount))
g.log.info("none of the files are under split-brain")
def test_gfid_split_brain_resolution(self):
"""
- create gfid split-brain of files and resolves them using source-brick
option of the CLI.
"""
# pylint: disable=too-many-statements
# pylint: disable=too-many-locals
# Disable all self-heals and client-quorum
options = {
"self-heal-daemon": "off",
"data-self-heal": "off",
"metadata-self-heal": "off",
"entry-self-heal": "off",
"cluster.quorum-type": "none"
}
g.log.info("setting volume options %s", options)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set volume option %s for "
"volume %s" % (options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# Create dir inside which I/O will be performed.
ret = mkdir(self.mounts[0].client_system,
"%s/test_gfid_split_brain" % self.mounts[0].mountpoint)
self.assertTrue(ret, "mkdir failed")
# get the subvolumes
g.log.info("Starting to get sub-volumes for volume %s", self.volname)
subvols_dict = get_subvols(self.mnode, self.volname)
num_subvols = len(subvols_dict['volume_subvols'])
g.log.info("Number of subvolumes in volume %s:", num_subvols)
# Toggle bricks and perform I/O
file_list = [
"file1.txt", "file2.txt", "file3.txt", "file4.txt", "file5.txt",
"file6.txt", "file7.txt", "file8.txt", "file9.txt", "file10.txt"
]
brick_index = 0
offline_bricks = []
for _ in range(0, 3):
for i in range(0, num_subvols):
subvol_brick_list = subvols_dict['volume_subvols'][i]
offline_bricks.append(subvol_brick_list[brick_index % 3])
offline_bricks.append(subvol_brick_list[(brick_index + 1) % 3])
self.toggle_bricks_and_perform_io(file_list, offline_bricks)
brick_index += 1
offline_bricks[:] = []
# Enable shd
g.log.info("enabling the self heal daemon")
ret = enable_self_heal_daemon(self.mnode, self.volname)
self.assertTrue(ret, "failed to enable self heal daemon")
g.log.info("Successfully enabled the self heal daemon")
# Wait for self heal processes to come online
g.log.info("Wait for selfheal process to come online")
timeout = 300
ret = wait_for_self_heal_daemons_to_be_online(self.mnode, self.volname,
timeout)
self.assertTrue(ret, "Self-heal process are not online")
g.log.info("All self heal process are online")
# Trigger heal
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Starting heal failed')
g.log.info('Index heal launched')
# checking if file is in split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertTrue(ret, "Files are not in split-brain as expected.")
g.log.info("Files are still in split-brain")
# First brick of each replica will be used as source-brick
first_brick_list = []
for i in range(0, num_subvols):
subvol_brick_list = subvols_dict['volume_subvols'][i]
brick = subvol_brick_list[0]
first_brick_list.append(brick)
# Find which dht subvols the 10 files are present in and trigger heal
for filename in file_list:
fpath = self.mounts[0].mountpoint + "/test_gfid_split_brain/" + \
filename
gfile = GlusterFile(self.clients[0], fpath)
for brick in first_brick_list:
_, brick_path = brick.split(':')
match = [
brick for item in gfile.hashed_bricks if brick_path in item
]
if match:
self.resolve_gfid_split_brain(
"/test_gfid_split_brain/" + filename, brick)
# Trigger heal to complete pending data/metadata heals
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Starting heal failed')
g.log.info('Index heal launched')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Get arequals and compare
for i in range(0, num_subvols):
# Get arequal for first brick
subvol_brick_list = subvols_dict['volume_subvols'][i]
node, brick_path = subvol_brick_list[0].split(':')
command = ('arequal-checksum -p %s '
'-i .glusterfs -i .landfill -i .trashcan' % brick_path)
ret, arequal, _ = g.run(node, command)
first_brick_total = arequal.splitlines()[-1].split(':')[-1]
# Get arequal for every brick and compare with first brick
for brick in subvol_brick_list[1:]:
node, brick_path = brick.split(':')
command = ('arequal-checksum -p %s '
'-i .glusterfs -i .landfill -i .trashcan' %
brick_path)
ret, brick_arequal, _ = g.run(node, command)
self.assertFalse(ret,
'Failed to get arequal on brick %s' % brick)
g.log.info('Getting arequal for %s is successful', brick)
brick_total = brick_arequal.splitlines()[-1].split(':')[-1]
self.assertEqual(
first_brick_total, brick_total,
'Arequals for subvol and %s are not equal' % brick)
g.log.info('Arequals for subvol and %s are equal', brick)
def test_manual_heal_should_trigger_heal(self):
"""
- create a single brick volume
- add some files and directories
- get arequal from mountpoint
- add-brick such that this brick makes the volume a replica vol 1x2
- start heal
- make sure heal is completed
- get arequals from all bricks and compare with arequal from mountpoint
"""
# pylint: disable=too-many-statements,too-many-locals
# Start IO on mounts
g.log.info("Starting IO on all mounts...")
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("python %s create_deep_dirs_with_files "
"--dir-length 1 "
"--dir-depth 1 "
"--max-num-of-dirs 1 "
"--num-of-files 10 %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
g.log.info("IO on %s:%s is started successfully",
mount_obj.client_system, mount_obj.mountpoint)
self.io_validation_complete = False
# Validate IO
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# Get arequal for mount before adding bricks
g.log.info('Getting arequal before adding bricks...')
ret, arequals = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after healing is successful')
mount_point_total = arequals[0].splitlines()[-1].split(':')[-1]
# Form brick list to add
g.log.info('Forming brick list to add...')
bricks_to_add = form_bricks_list(self.mnode, self.volname, 1,
self.servers, self.all_servers_info)
g.log.info('Brick list to add: %s', bricks_to_add)
# Add bricks
g.log.info("Start adding bricks to volume...")
ret, _, _ = add_brick(self.mnode,
self.volname,
bricks_to_add,
force=True,
replica_count=2)
self.assertFalse(ret, "Failed to add bricks %s" % bricks_to_add)
g.log.info("Adding bricks is successful on volume %s", self.volname)
# Make sure the newly added bricks are available in the volume
# get the bricks for the volume
g.log.info("Fetching bricks for the volume: %s", self.volname)
bricks_list = get_all_bricks(self.mnode, self.volname)
g.log.info("Brick list: %s", bricks_list)
for brick in bricks_to_add:
self.assertIn(brick, bricks_list,
'Brick %s is not in brick list' % brick)
g.log.info('New bricks are present in the volume')
# Make sure volume change from distribute to replicate volume
vol_info_dict = get_volume_type_info(self.mnode, self.volname)
vol_type = vol_info_dict['volume_type_info']['typeStr']
self.assertEqual(
'Replicate', vol_type, 'Volume type is not converted to Replicate '
'after adding bricks')
g.log.info('Volume type is successfully converted to Replicate '
'after adding bricks')
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal on bricks and compare with mount_point_total
# It should be the same
g.log.info('Getting arequal on bricks...')
arequals_after_heal = {}
for brick in bricks_list:
g.log.info('Getting arequal on bricks %s...', brick)
node, brick_path = brick.split(':')
command = ('arequal-checksum -p %s '
'-i .glusterfs -i .landfill -i .trashcan' % brick_path)
ret, arequal, _ = g.run(node, command)
self.assertFalse(ret, 'Failed to get arequal on brick %s' % brick)
g.log.info('Getting arequal for %s is successful', brick)
brick_total = arequal.splitlines()[-1].split(':')[-1]
arequals_after_heal[brick] = brick_total
self.assertEqual(
mount_point_total, brick_total,
'Arequals for mountpoint and %s are not equal' % brick)
g.log.info('Arequals for mountpoint and %s are equal', brick)
g.log.info('All arequals are equal for replicated')
def test_afr_heal_with_brickdown_hardlink(self):
"""
Steps:
1. Create 2 * 3 distribute replicate volume and disable all heals
2. Create a file and 3 hardlinks to it from fuse mount.
3. Kill brick4, rename HLINK1 to an appropriate name so that
it gets hashed to replicate-1
4. Likewise rename HLINK3 and HLINK7 as well, killing brick5 and brick6
respectively each time. i.e. a different brick of the 2nd
replica is down each time.
5. Now enable shd and let selfheals complete.
6. Heal should complete without split-brains.
"""
bricks_list = get_all_bricks(self.mnode, self.volname)
options = {
"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off",
"self-heal-daemon": "off"
}
g.log.info("setting options %s", options)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set volume option %s for"
"volume %s" % (options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
cmd = ("touch %s/FILE" % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.clients[0], cmd)
self.assertEqual(ret, 0, "file creation failed")
# Creating a hardlink for the file created
for i in range(1, 4):
ret = create_link_file(
self.clients[0], '{}/FILE'.format(self.mounts[0].mountpoint),
'{}/HLINK{}'.format(self.mounts[0].mountpoint, i))
self.assertTrue(ret, "Unable to create hard link file ")
# Bring brick3 offline,Rename file HLINK1,and bring back brick3 online
self._test_brick_down_with_file_rename("HLINK1", "NEW-HLINK1",
bricks_list[3])
# Bring brick4 offline,Rename file HLINK2,and bring back brick4 online
self._test_brick_down_with_file_rename("HLINK2", "NEW-HLINK2",
bricks_list[4])
# Bring brick5 offline,Rename file HLINK3,and bring back brick5 online
self._test_brick_down_with_file_rename("HLINK3", "NEW-HLINK3",
bricks_list[5])
# Setting options
options = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Check data on mount point
cmd = ("ls %s" % (self.mounts[0].mountpoint))
ret, _, _ = g.run(self.clients[0], cmd)
self.assertEqual(ret, 0, "failed to fetch data from mount point")
def test_gfid_split_brain_resolution(self):
"""
Description: Simulates gfid split brain on multiple files in a dir and
resolve them via `bigger-file`, `mtime` and `source-brick` methods
Steps:
- Create and mount a replicated volume, create a dir and ~10 data files
- Simulate gfid splits in 9 of the files
- Resolve each 3 set of files using `bigger-file`, `mtime` and
`source-bricks` split-brain resoultion methods
- Trigger and monitor for heal completion
- Validate all the files are healed and arequal matches for bricks in
subvols
"""
io_cmd = 'cat /dev/urandom | tr -dc [:space:][:print:] | head -c '
client, m_point = (self.mounts[0].client_system,
self.mounts[0].mountpoint)
arbiter = self.volume_type.find('arbiter') >= 0
# Disable self-heal daemon and set `quorum-type` option to `none`
ret = set_volume_options(self.mnode, self.volname, {
'self-heal-daemon': 'off',
'cluster.quorum-type': 'none'
})
self.assertTrue(
ret, 'Not able to disable `quorum-type` and '
'`self-heal` daemon volume options')
# Create required dir and files from the mount
split_dir = 'gfid_split_dir'
file_io = ('cd %s; for i in {1..10}; do ' + io_cmd +
' 1M > %s/file$i; done;')
ret = mkdir(client, '{}/{}'.format(m_point, split_dir))
self.assertTrue(ret, 'Unable to create a directory from mount point')
ret, _, _ = g.run(client, file_io % (m_point, split_dir))
# `file{4,5,6}` are re-created every time to be used in `bigger-file`
# resolution method
cmd = 'rm -rf {0}/file{1} && {2} {3}M > {0}/file{1}'
split_cmds = {
1:
';'.join(cmd.format(split_dir, i, io_cmd, 2) for i in range(1, 7)),
2:
';'.join(cmd.format(split_dir, i, io_cmd, 3) for i in range(4, 7)),
3: ';'.join(
cmd.format(split_dir, i, io_cmd, 1) for i in range(4, 10)),
4: ';'.join(
cmd.format(split_dir, i, io_cmd, 1) for i in range(7, 10)),
}
# Get subvols and simulate entry split brain
subvols = get_subvols(self.mnode, self.volname)['volume_subvols']
self.assertTrue(subvols, 'Not able to get list of subvols')
msg = ('Unable to bring files under {} dir to entry split brain while '
'{} are down')
for index, bricks in enumerate(self._get_two_bricks(subvols, arbiter),
1):
# Bring down two bricks from each subvol
ret = bring_bricks_offline(self.volname, list(bricks))
self.assertTrue(ret, 'Unable to bring {} offline'.format(bricks))
ret, _, _ = g.run(client,
'cd {}; {}'.format(m_point, split_cmds[index]))
self.assertEqual(ret, 0, msg.format(split_dir, bricks))
# Bricks will be brought down only two times in case of arbiter and
# bringing remaining files into split brain for `latest-mtime` heal
if arbiter and index == 2:
ret, _, _ = g.run(client,
'cd {}; {}'.format(m_point, split_cmds[4]))
self.assertEqual(ret, 0, msg.format(split_dir, bricks))
# Bring offline bricks online
ret = bring_bricks_online(
self.mnode,
self.volname,
bricks,
bring_bricks_online_methods='volume_start_force')
self.assertTrue(ret, 'Unable to bring {} online'.format(bricks))
# Enable self-heal daemon, trigger heal and assert volume is in split
# brain condition
ret = enable_self_heal_daemon(self.mnode, self.volname)
self.assertTrue(ret, 'Failed to enable self heal daemon')
ret = wait_for_self_heal_daemons_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, 'Not all self heal daemons are online')
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Unable to trigger index heal on the volume')
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertTrue(ret, 'Volume should be in split brain condition')
# Select source brick and take note of files in source brick
stop = len(subvols[0]) - 1 if arbiter else len(subvols[0])
source_bricks = [choice(subvol[0:stop]) for subvol in subvols]
files = [
self._get_files_in_brick(path, split_dir) for path in source_bricks
]
# Resolve `file1, file2, file3` gfid split files using `source-brick`
cmd = ('gluster volume heal ' + self.volname + ' split-brain '
'source-brick {} /' + split_dir + '/{}')
for index, source_brick in enumerate(source_bricks):
for each_file in files[index]:
run_cmd = cmd.format(source_brick, each_file)
self._run_cmd_and_assert(run_cmd)
# Resolve `file4, file5, file6` gfid split files using `bigger-file`
cmd = ('gluster volume heal ' + self.volname +
' split-brain bigger-file /' + split_dir + '/{}')
for each_file in ('file4', 'file5', 'file6'):
run_cmd = cmd.format(each_file)
self._run_cmd_and_assert(run_cmd)
# Resolve `file7, file8, file9` gfid split files using `latest-mtime`
cmd = ('gluster volume heal ' + self.volname +
' split-brain latest-mtime /' + split_dir + '/{}')
for each_file in ('file7', 'file8', 'file9'):
run_cmd = cmd.format(each_file)
self._run_cmd_and_assert(run_cmd)
# Unless `shd` is triggered manually/automatically files will still
# appear in `heal info`
ret = trigger_heal_full(self.mnode, self.volname)
self.assertTrue(ret, 'Unable to trigger full self heal')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(
ret, 'All files in volume should be healed after healing files via'
' `source-brick`, `bigger-file`, `latest-mtime` methods manually')
# Validate normal file `file10` and healed files don't differ in
# subvols via an `arequal`
for subvol in subvols:
# Disregard last brick if volume is of arbiter type
ret, arequal = collect_bricks_arequal(subvol[0:stop])
self.assertTrue(
ret, 'Unable to get `arequal` checksum on '
'{}'.format(subvol[0:stop]))
self.assertEqual(
len(set(arequal)), 1, 'Mismatch of `arequal` '
'checksum among {} is identified'.format(subvol[0:stop]))
g.log.info('Pass: Resolution of gfid split-brain via `source-brick`, '
'`bigger-file` and `latest-mtime` methods is complete')
def test_metadata_self_heal(self):
"""
Test MetaData Self-Heal (heal command)
Description:
- set the volume option
"metadata-self-heal": "off"
"entry-self-heal": "off"
"data-self-heal": "off"
- create IO
- set the volume option
"self-heal-daemon": "off"
- bring down all bricks processes from selected set
- Change the permissions, ownership and the group
of the files under "test_meta_data_self_heal" folder
- get arequal before getting bricks online
- bring bricks online
- set the volume option
"self-heal-daemon": "on"
- check daemons and start healing
- check is heal is completed
- check for split-brain
- get arequal after getting bricks online and compare with
arequal before getting bricks online
- check group and user are 'qa'
"""
# pylint: disable=too-many-locals,too-many-statements
# Setting options
g.log.info('Setting options...')
options = {"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Options "
"'metadata-self-heal', "
"'entry-self-heal', "
"'data-self-heal', "
"are set to 'off' successfully")
# Creating files on client side
all_mounts_procs = []
test_meta_data_self_heal_folder = 'test_meta_data_self_heal'
g.log.info("Generating data for %s:%s",
self.mounts[0].client_system, self.mounts[0].mountpoint)
# Create files
g.log.info('Creating files...')
command = ("cd %s/ ; "
"mkdir %s ;"
"cd %s/ ;"
"for i in `seq 1 50` ; "
"do dd if=/dev/urandom of=test.$i bs=10k count=1 ; "
"done ;"
% (self.mounts[0].mountpoint,
test_meta_data_self_heal_folder,
test_meta_data_self_heal_folder))
proc = g.run_async(self.mounts[0].client_system, command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# wait for io to complete
self.assertTrue(
wait_for_io_to_complete(all_mounts_procs, self.mounts),
"Io failed to complete on some of the clients")
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Option 'self-heal-daemon' is set to 'off' successfully")
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = list(filter(None, (
bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks'])))
# Bring brick offline
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Changing the permissions, ownership and the group
# of the files under "test_meta_data_self_heal" folder
g.log.info("Modifying data for %s:%s",
self.mounts[0].client_system, self.mounts[0].mountpoint)
# Change permissions to 444
g.log.info('Changing permissions...')
command = ("cd %s/%s/ ; "
"chmod -R 444 *"
% (self.mounts[0].mountpoint,
test_meta_data_self_heal_folder))
ret, out, err = g.run(self.mounts[0].client_system, command)
self.assertEqual(ret, 0, err)
g.log.info('Permissions are changed successfully')
# Change the ownership to qa
g.log.info('Changing the ownership...')
command = ("cd %s/%s/ ; "
"chown -R qa *"
% (self.mounts[0].mountpoint,
test_meta_data_self_heal_folder))
ret, out, err = g.run(self.mounts[0].client_system, command)
self.assertEqual(ret, 0, err)
g.log.info('Ownership is changed successfully')
# Change the group to qa
g.log.info('Changing the group...')
command = ("cd %s/%s/ ; "
"chgrp -R qa *"
% (self.mounts[0].mountpoint,
test_meta_data_self_heal_folder))
ret, out, err = g.run(self.mounts[0].client_system, command)
self.assertEqual(ret, 0, err)
g.log.info('Group is changed successfully')
# Get arequal before getting bricks online
g.log.info('Getting arequal before getting bricks online...')
ret, result_before_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks online '
'is successful')
# Bring brick online
g.log.info('Bringing bricks %s online...', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume process %s not online "
"despite waiting for 5 minutes", self.volname))
g.log.info("Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online"
% self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
g.log.info('Getting arequal after getting bricks online...')
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks online '
'is successful')
# Checking arequals before bringing bricks online
# and after bringing bricks online
self.assertItemsEqual(result_before_online, result_after_online,
'Checksums are not equal')
g.log.info('Checksums before bringing bricks online '
'and after bringing bricks online are equal')
# Adding servers and client in single dict to check permissions
nodes_to_check = {}
all_bricks = get_all_bricks(self.mnode, self.volname)
for brick in all_bricks:
node, brick_path = brick.split(':')
nodes_to_check[node] = brick_path
nodes_to_check[self.mounts[0].client_system] = \
self.mounts[0].mountpoint
# Checking for user and group
for node in nodes_to_check:
# Get file list
command = ("cd %s/%s/ ; "
"ls"
% (nodes_to_check[node],
test_meta_data_self_heal_folder))
ret, out, err = g.run(node, command)
file_list = out.split()
for file_name in file_list:
file_to_check = '%s/%s/%s' % (nodes_to_check[node],
test_meta_data_self_heal_folder,
file_name)
g.log.info('Checking for permissions, user and group for %s',
file_name)
# Check for permissions
cmd = ("stat -c '%a %n' {} | awk '{{print $1}}'"
.format(file_to_check))
ret, permissions, _ = g.run(node, cmd)
self.assertEqual(permissions.split('\n')[0], '444',
'Permissions %s is not equal to 444'
% permissions)
g.log.info("Permissions are '444' for %s", file_name)
# Check for user
cmd = ("ls -ld {} | awk '{{print $3}}'"
.format(file_to_check))
ret, username, _ = g.run(node, cmd)
self.assertEqual(username.split('\n')[0],
'qa', 'User %s is not equal qa'
% username)
g.log.info("User is 'qa' for %s", file_name)
# Check for group
cmd = ("ls -ld {} | awk '{{print $4}}'"
.format(file_to_check))
ret, groupname, _ = g.run(node, cmd)
self.assertEqual(groupname.split('\n')[0],
'qa', 'Group %s is not equal qa'
% groupname)
g.log.info("Group is 'qa' for %s", file_name)
def test_entry_heal_with_quota(self):
"""
- Create a 1x3 volume
- Set quota object limit
- Create files less than the limit
- Bring down a brick and create more files until limit is hit
- Delete one file so that we are below the limit, and create one more
file
- Bring the brick back up and launch heal
- Verify that after heal is complete, the deleted file does not
re-appear in any of the bricks.
"""
# pylint: disable=too-many-statements
# Enable Quota
g.log.info("Enabling quota on the volume %s", self.volname)
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(
ret, 0, ("Failed to enable quota on the volume %s", self.volname))
g.log.info("Successfully enabled quota on the volume %s", self.volname)
# Check if quota is enabled
g.log.info("Validate Quota is enabled on the volume %s", self.volname)
ret = is_quota_enabled(self.mnode, self.volname)
self.assertTrue(
ret, ("Quota is not enabled on the volume %s", self.volname))
g.log.info("Successfully Validated quota is enabled on volume %s",
self.volname)
# Set quota related options
options = {
"quota-deem-statfs": "on",
"soft-timeout": "0",
"hard-timeout": "0"
}
g.log.info("setting quota volume options %s", options)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set volume option %s for "
"volume %s" % (options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# Create directory on mount
ret = mkdir(self.mounts[0].client_system,
"%s/dir" % self.mounts[0].mountpoint)
self.assertTrue(ret, "mkdir failed")
# Set Quota limit on the directory
path = "/dir"
g.log.info(
"Setting Quota Limit object on the path %s of the "
"volume %s", path, self.volname)
ret, _, _ = quota_limit_objects(self.mnode,
self.volname,
path=path,
limit="10")
self.assertEqual(ret, 0,
("Failed to set quota limit object "
"on path %s of the volume %s", path, self.volname))
g.log.info(
"Successfully set the Quota limit object on %s of the "
"volume %s", path, self.volname)
cmd = ("touch %s/dir/file{1..5}" % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.clients[0], cmd)
self.assertEqual(ret, 0, "file creation failed")
# Bring brick3 offline
bricks_list = get_all_bricks(self.mnode, self.volname)
g.log.info('Bringing brick %s offline', bricks_list[2])
ret = bring_bricks_offline(self.volname, bricks_list[2])
self.assertTrue(ret,
'Failed to bring brick %s offline' % bricks_list[2])
ret = are_bricks_offline(self.mnode, self.volname, [bricks_list[2]])
self.assertTrue(ret, 'Brick %s is not offline' % bricks_list[2])
g.log.info('Bringing brick %s offline was successful', bricks_list[2])
# Create files until quota object limit
cmd = ("touch %s/dir/file{6..9}" % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.clients[0], cmd)
self.assertEqual(ret, 0, "file creation failed")
# The next create must fail
cmd = ("touch %s/dir/file10" % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.clients[0], cmd)
self.assertEqual(
ret, 1, ("Creation of %s/dir/file10 succeeded while "
"it was not supposed to." % self.mounts[0].mountpoint))
g.log.info(
"Creation of %s/dir/file10 failed as expected due to "
"quota object limit.", self.mounts[0].mountpoint)
# Delete one file and re-try the create to succeed.
cmd = ("rm %s/dir/file1" % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.clients[0], cmd)
self.assertEqual(ret, 0, "File deletion failed")
cmd = ("touch %s/dir/file10" % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.clients[0], cmd)
self.assertEqual(ret, 0, "File creation failed")
# Bring brick3 online and check status
g.log.info('Bringing brick %s online...', bricks_list[2])
ret = bring_bricks_online(self.mnode, self.volname, [bricks_list[2]])
self.assertTrue(ret,
'Failed to bring brick %s online' % bricks_list[2])
g.log.info('Bringing brick %s online is successful', bricks_list[2])
g.log.info("Verifying if brick3 is online....")
ret = are_bricks_online(self.mnode, self.volname, bricks_list)
self.assertTrue(ret, ("brick3 did not come up"))
g.log.info("brick3 has come online.")
# Trigger heal
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Starting heal failed')
g.log.info('Index heal launched')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Verify that file10 did not get recreated on the down brick by an
# accidental conservative merge.
for brick in bricks_list:
node, brick_path = brick.split(':')
ret, _, _ = g.run(node, 'stat %s/dir/file10' % brick_path)
self.assertFalse(ret, 'File present!')
def test_data_self_heal_daemon_off(self):
"""
Test Data-Self-Heal (heal command)
Description:
- set the volume option
"metadata-self-heal": "off"
"entry-self-heal": "off"
"data-self-heal": "off"
- create IO
- Get arequal before getting bricks offline
- set the volume option
"self-heal-daemon": "off"
- bring down all bricks processes from selected set
- Get areeual after getting bricks offline and compare with
arequal before getting bricks offline
- modify the data
- bring bricks online
- set the volume option
"self-heal-daemon": "on"
- check daemons and start healing
- check if heal is completed
- check for split-brain
- add bricks
- do rebalance
- create 1k files
- while creating files - kill bricks and bring bricks online one by one
in cycle
- validate IO
"""
# pylint: disable=too-many-statements,too-many-locals
# Setting options
g.log.info('Setting options...')
options = {
"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off"
}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Successfully set %s for volume %s", options, self.volname)
# Creating files on client side
g.log.info("Starting IO on %s:%s", self.mounts[0].client_system,
self.mounts[0].mountpoint)
cmd = ("/usr/bin/env python %s create_files -f 100"
" --fixed-file-size 1k %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
ret, _, err = g.run(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
self.assertFalse(
ret, 'Failed to create the data for %s: %s' %
(self.mounts[0].mountpoint, err))
g.log.info('Created IO for %s is successfully',
self.mounts[0].mountpoint)
# Get arequal before getting bricks offline
g.log.info('Getting arequal before getting bricks offline...')
ret, arequals = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
result_before_offline = arequals[0].splitlines()[-1].split(':')[-1]
g.log.info('Getting arequal before getting bricks offline '
'is successful')
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' is set to 'off' successfully")
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = list(
filter(None, (bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks'])))
# Bring brick offline
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Get arequal after getting bricks offline
g.log.info('Getting arequal after getting bricks offline...')
ret, arequals = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
result_after_offline = arequals[0].splitlines()[-1].split(':')[-1]
g.log.info('Getting arequal after getting bricks offline '
'is successful')
# Checking arequals before bringing bricks offline
# and after bringing bricks offline
self.assertEqual(
result_before_offline, result_after_offline,
'Checksums before and '
'after bringing bricks online are not equal')
g.log.info('Checksums before and after bringing bricks online '
'are equal')
# Modify the data
g.log.info("Modifying data for %s:%s", self.mounts[0].client_system,
self.mounts[0].mountpoint)
cmd = ("/usr/bin/env python %s create_files -f 100"
" --fixed-file-size 10k %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
ret, _, err = g.run(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
self.assertFalse(
ret, 'Failed to midify the data for %s: %s' %
(self.mounts[0].mountpoint, err))
g.log.info('Modified IO for %s is successfully',
self.mounts[0].mountpoint)
# Bring brick online
g.log.info('Bringing bricks %s online...', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s online' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Add bricks
g.log.info("Start adding bricks to volume...")
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to expand the volume %s", self.volname))
g.log.info("Expanding volume is successful on "
"volume %s", self.volname)
# Do rebalance
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, 'Failed to start rebalance')
g.log.info('Rebalance is started')
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Rebalance is not completed')
g.log.info('Rebalance is completed successfully')
# Create 1k files
all_mounts_procs = []
g.log.info("Modifying data for %s:%s", self.mounts[0].client_system,
self.mounts[0].mountpoint)
command = ("/usr/bin/env python %s create_files -f 1000"
" --base-file-name newfile %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Kill all bricks in cycle
bricks_list = get_all_bricks(self.mnode, self.volname)
for brick in bricks_list:
# Bring brick offline
g.log.info('Bringing bricks %s offline', brick)
ret = bring_bricks_offline(self.volname, [brick])
self.assertTrue(ret, 'Failed to bring bricks %s offline' % brick)
ret = are_bricks_offline(self.mnode, self.volname, [brick])
self.assertTrue(ret, 'Bricks %s are not offline' % brick)
g.log.info('Bringing bricks %s offline is successful', brick)
# Introducing 30 second sleep when brick is down
g.log.info(
"Waiting for 30 seconds, with ongoing IO while "
"brick %s is offline", brick)
sleep(30)
# Bring brick online
g.log.info('Bringing bricks %s online...', brick)
ret = bring_bricks_online(self.mnode, self.volname, [brick])
self.assertTrue(
ret,
'Failed to bring bricks %s online' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful', brick)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(
self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(
ret, "Either No self heal daemon process found or"
"more than one self heal daemon process"
"found")
g.log.info("All self-heal-daemons are online")
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
def test_afr_gfid_heal(self):
"""
Description: This test case runs split-brain resolution CLIs
on a file in gfid split-brain on 1x2 volume.
1. kill 1 brick
2. create a file at mount point
3. bring back the killed brick
4. kill the other brick
5. create same file at mount point
6. bring back the killed brick
7. try heal from CLI and check if it gets completed
"""
g.log.info("disabling the self heal daemon")
ret = disable_self_heal_daemon(self.mnode, self.volname)
self.assertTrue(ret, "unable to disable self heal daemon")
g.log.info("Successfully disabled the self heal daemon")
# getting list of all bricks
all_bricks = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(all_bricks, "unable to get list of bricks")
g.log.info("bringing down brick1")
ret = bring_bricks_offline(self.volname, all_bricks[0])
self.assertTrue(ret, "unable to bring %s offline" % all_bricks[0])
g.log.info("Successfully brought the following brick offline "
": %s", all_bricks[0])
g.log.info("creating a file from mount point")
all_mounts_procs = []
cmd = ("/usr/bin/env python %s create_files "
"-f 1 --base-file-name test_file --fixed-file-size 1k %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system, cmd)
all_mounts_procs.append(proc)
# Validate I/O
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
g.log.info("Successfully created a file from mount point")
g.log.info("bringing brick 1 back online")
ret = bring_bricks_online(self.mnode, self.volname, [all_bricks[0]])
self.assertIsNotNone(ret, "unable to bring %s online" % all_bricks[0])
g.log.info("Successfully brought the following brick online "
": %s", all_bricks[0])
g.log.info("bringing down brick2")
ret = bring_bricks_offline(self.volname, all_bricks[1])
self.assertTrue(ret, "unable to bring %s offline" % all_bricks[0])
g.log.info("Successfully brought the following brick offline "
": %s", all_bricks[1])
g.log.info("creating a new file of same name from mount point")
all_mounts_procs = []
cmd = ("/usr/bin/env python %s create_files "
"-f 1 --base-file-name test_file --fixed-file-size 1k %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system, cmd)
all_mounts_procs.append(proc)
# Validate I/O
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
g.log.info("Successfully created a new file of same name "
"from mount point")
g.log.info("bringing brick2 back online")
ret = bring_bricks_online(self.mnode, self.volname, [all_bricks[1]])
self.assertIsNotNone(ret, "unable to bring %s online" % all_bricks[0])
g.log.info("Successfully brought the following brick online "
": %s", all_bricks[1])
g.log.info("enabling the self heal daemon")
ret = enable_self_heal_daemon(self.mnode, self.volname)
self.assertTrue(ret, "failed to enable self heal daemon")
g.log.info("Successfully enabled the self heal daemon")
g.log.info("checking if file is in split-brain")
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertTrue(ret, "unable to create split-brain scenario")
g.log.info("Successfully created split brain scenario")
g.log.info("resolving split-brain by choosing second brick as "
"the source brick")
node, _ = all_bricks[0].split(':')
command = ("gluster volume heal %s split-brain source-brick %s "
"/test_file0.txt" % (self.volname, all_bricks[1]))
ret, _, _ = g.run(node, command)
self.assertEqual(ret, 0, "command execution not successful")
# triggering heal
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, "heal not triggered")
# waiting for heal to complete
ret = monitor_heal_completion(self.mnode,
self.volname,
timeout_period=120)
self.assertTrue(ret, "heal not completed")
# checking if file is in split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, "file still in split-brain")
g.log.info("Successfully resolved split brain situation using "
"CLI based resolution")
def test_self_heal_50k_files_heal_command_by_add_brick(self):
"""
Test self-heal of 50k files (heal command
Description:
- set the volume option
"metadata-self-heal": "off"
"entry-self-heal": "off"
"data-self-heal": "off"
"self-heal-daemon": "off"
- bring down all bricks processes from selected set
- create IO (50k files)
- Get arequal before getting bricks online
- bring bricks online
- set the volume option
"self-heal-daemon": "on"
- check for daemons
- start healing
- check if heal is completed
- check for split-brain
- get arequal after getting bricks online and compare with
arequal before getting bricks online
- add bricks
- do rebalance
- get arequal after adding bricks and compare with
arequal after getting bricks online
"""
# pylint: disable=too-many-locals,too-many-statements
# Setting options
g.log.info('Setting options...')
options = {"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off",
"self-heal-daemon": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Successfully set %s for volume %s", options, self.volname)
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = list(filter(None, (
bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks'])))
# Bring brick offline
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Creating files on client side
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
# Create 50k files
g.log.info('Creating files...')
command = ("/usr/bin/env python %s create_files -f 50000 %s" % (
self.script_upload_path,
mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients"
)
self.io_validation_complete = True
# Get arequal before getting bricks online
g.log.info('Getting arequal before getting bricks online...')
ret, result_before_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks online '
'is successful')
# Bring brick online
g.log.info('Bringing bricks %s online', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info("Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online"
% self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname,
timeout_period=3600)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
g.log.info('Getting arequal after getting bricks online...')
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks online '
'is successful')
# Checking arequals before bringing bricks online
# and after bringing bricks online
self.assertItemsEqual(result_before_online, result_after_online,
'Checksums before and '
'after bringing bricks online are not equal')
g.log.info('Checksums before and after bringing bricks online '
'are equal')
# Add bricks
g.log.info("Start adding bricks to volume...")
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to expand the volume when IO in "
"progress on volume %s", self.volname))
g.log.info("Expanding volume is successful on volume %s", self.volname)
# Do rebalance
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, 'Failed to start rebalance')
g.log.info('Rebalance is started')
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Rebalance is not completed')
g.log.info('Rebalance is completed successfully')
# Get arequal after adding bricks
g.log.info('Getting arequal after adding bricks...')
ret, result_after_adding_bricks = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks '
'is successful')
# Checking arequals after bringing bricks online
# and after adding bricks
self.assertItemsEqual(result_after_online, result_after_adding_bricks,
'Checksums after bringing bricks online and '
'after adding bricks are not equal')
g.log.info('Checksums after bringing bricks online and '
'after adding bricks are equal')
def test_self_heal_symbolic_links(self):
"""
Test Self-Heal of Symbolic Links (heal command)
Description:
- set the volume option
"metadata-self-heal": "off"
"entry-self-heal": "off"
"data-self-heal": "off"
"data-self-heal-algorithm": "diff"
"self-heal-daemon": "off"
- create IO
- calculate arequal
- bring down all bricks processes from selected set
- calculate arequals and compare with arequal
before bringing bricks offline
- modify the data and verify whether the links are properly created
- calculate arequal before getting bricks online
- bring bricks online
- set the volume option
"self-heal-daemon": "on"
- check daemons and start healing
- check is heal is complited
- check for split-brain
- calculate arequal after getting bricks online and compare with
arequal before getting bricks online
"""
# pylint: disable=too-many-locals,too-many-statements
# Setting options
g.log.info('Setting options...')
options = {"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off",
"self-heal-daemon": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Options "
"'metadata-self-heal', "
"'entry-self-heal', "
"'data-self-heal', "
"'self-heal-daemon' "
"are set to 'off' successfully")
# Creating files on client side
all_mounts_procs = []
test_sym_link_self_heal_folder = 'test_sym_link_self_heal'
g.log.info("Generating data for %s:%s",
self.mounts[0].client_system, self.mounts[0].mountpoint)
# Creating files
command = ("cd %s/ ; "
"mkdir %s ; "
"cd %s/ ;"
"for i in `seq 1 5` ; "
"do mkdir dir.$i ; "
"for j in `seq 1 10` ; "
"do dd if=/dev/urandom of=dir.$i/file.$j "
"bs=1k count=$j ; "
"done ; "
"done ;"
% (self.mounts[0].mountpoint,
test_sym_link_self_heal_folder,
test_sym_link_self_heal_folder))
proc = g.run_async(self.mounts[0].client_system, command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# wait for io to complete
self.assertTrue(
wait_for_io_to_complete(all_mounts_procs, self.mounts),
"Io failed to complete on some of the clients")
# Get arequal before getting bricks offline
g.log.info('Getting arequal before getting bricks offline...')
ret, result_before_offline = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks offline '
'is successful')
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = bricks_to_bring_offline_dict['volume_bricks']
# Bring brick offline
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Get arequal after getting bricks offline
g.log.info('Getting arequal after getting bricks offline...')
ret, result_after_offline = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks offline '
'is successful')
# Checking arequals before bringing bricks offline
# and after bringing bricks offline
self.assertEqual(sorted(result_before_offline),
sorted(result_after_offline),
'Checksums before and after bringing bricks '
'online are not equal')
g.log.info('Checksums before and after bringing bricks online '
'are equal')
# Modify the data
g.log.info("Modifying data for %s:%s",
self.mounts[0].client_system, self.mounts[0].mountpoint)
# Create symlinks
g.log.info('Creating symlinks...')
command = ("cd %s/%s/ ; "
"for i in `seq 1 5` ; "
"do ln -s dir.$i sym_link_dir.$i ; "
"done ;"
% (self.mounts[0].mountpoint,
test_sym_link_self_heal_folder))
ret, _, _ = g.run(self.mounts[0].client_system, command)
self.assertEqual(ret, 0, 'Failed to modify the data for %s...'
% self.mounts[0].mountpoint)
g.log.info('Modifying the data for %s is successful',
self.mounts[0].mountpoint)
# Verify whether the links are properly created
# Get symlink list
command = ("cd %s/%s/ ; "
"ls |grep 'sym'"
% (self.mounts[0].mountpoint,
test_sym_link_self_heal_folder))
_, out, _ = g.run(self.mounts[0].client_system, command)
symlink_list = out.strip().split('\n')
# Get folder list
command = ("cd %s/%s/ ; "
"ls |grep -v 'sym'"
% (self.mounts[0].mountpoint,
test_sym_link_self_heal_folder))
_, out, _ = g.run(self.mounts[0].client_system, command)
folder_list = out.strip().split('\n')
# Compare symlinks and folders
for symlink in symlink_list:
symlink_index = symlink_list.index(symlink)
command = ("cd %s/%s/ ; "
"readlink %s"
% (self.mounts[0].mountpoint,
test_sym_link_self_heal_folder,
symlink))
_, out, _ = g.run(self.mounts[0].client_system, command)
symlink_to_folder = out.strip()
self.assertEqual(symlink_to_folder, folder_list[symlink_index],
'Links are not properly created')
g.log.info('Links for %s are properly created',
self.mounts[0].mountpoint)
# Get arequal before getting bricks online
g.log.info('Getting arequal before getting bricks online...')
ret, result_before_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks online '
'is successful')
# Bring brick online
g.log.info('Bringing bricks %s online', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info("Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online"
% self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
g.log.info('Getting arequal after getting bricks online...')
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks online '
'is successful')
# Checking arequals before bringing bricks online
# and after bringing bricks online
self.assertEqual(sorted(result_before_online),
sorted(result_after_online),
'Checksums before and after bringing bricks '
'online are not equal')
g.log.info('Checksums before and after bringing bricks online '
'are equal')
def test_heal_when_dir_quota_exceeded_(self):
# Create a directory to set the quota_limit_usage
path = "/dir"
g.log.info("Creating a directory")
self.all_mounts_procs = []
for mount_object in self.mounts:
cmd = ("python %s create_deep_dir -d 0 -l 0 %s%s "
% (self.script_upload_path, mount_object.mountpoint,
path))
ret = g.run(mount_object.client_system, cmd)
self.assertTrue(ret, "Failed to create directory on mountpoint")
g.log.info("Directory created successfully on mountpoint")
# Enable Quota
g.log.info("Enabling quota on the volume %s", self.volname)
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to enable quota on the volume "
"%s", self.volname))
g.log.info("Successfully enabled quota on the volume %s",
self.volname)
# Set quota-soft-timeout to 0
g.log.info("Setting up soft timeout to 0")
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, "0")
self.assertEqual(ret, 0, ("Failed to set quota-soft-timeout"))
g.log.info("Successfully set the quota-soft-timeout")
# Set quota-hard-timeout to 0
g.log.info("Setting up hard timeout with 0")
ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, "0")
self.assertEqual(ret, 0, ("Failed to set quota-hard-timeout"))
g.log.info("successfully set the quota-hard-timeout")
# Set Quota limit on the newly created directory
g.log.info("Set Quota Limit on the path %s of the volume %s",
path, self.volname)
ret, _, _ = quota_limit_usage(self.mnode, self.volname,
path=path, limit="1GB")
self.assertEqual(ret, 0, ("Failed to set quota limit on path %s of "
" the volume %s", path, self.volname))
g.log.info("Successfully set the Quota limit on %s of the volume "
"%s", path, self.volname)
# Create 2 files of size 400MB inside the directory
for mount_object in self.mounts:
g.log.info("Creating Files on %s:%s", mount_object.client_system,
path)
cmd = ("cd %s%s && for i in `seq 1 2` ;"
"do dd if=/dev/urandom of=file$i bs=20M "
"count=20; done" % (mount_object.mountpoint, path))
ret, _, _ = g.run(mount_object.client_system, cmd)
self.assertEqual(ret, 0, ("Failed to create files on %s", path))
g.log.info("Files created successfully on mountpoint")
bricks_list = get_all_bricks(self.mnode, self.volname)
# Bring brick2 offline
g.log.info('Bringing bricks %s offline', bricks_list[2])
ret = bring_bricks_offline(self.volname, bricks_list[2])
self.assertTrue(ret, 'Failed to bring bricks %s offline'
% bricks_list[2])
ret = are_bricks_offline(self.mnode, self.volname,
[bricks_list[2]])
self.assertTrue(ret, 'Brick %s is not offline'
% bricks_list[2])
g.log.info('Bringing brick %s offline is successful',
bricks_list[2])
# Create a file of size 500MB inside the directory and it should fail
# as the quota limit exceeds
cmd = ("cd %s%s && dd if=/dev/urandom of=file3 bs=20M count=25"
% (mount_object.mountpoint, path))
ret, _, _ = g.run(mount_object.client_system, cmd)
self.assertEqual(ret, 1, ("Writing a file of 500MB succeeded while "
"it was not supposed to."))
g.log.info("Writing a file of size 500MB failed as expected "
"due to quota limit on the directory.")
# Bring brick2 online and check status
g.log.info('Bringing brick %s online...', bricks_list[2])
ret = bring_bricks_online(self.mnode, self.volname,
[bricks_list[2]])
self.assertTrue(ret, 'Failed to bring brick %s online' %
bricks_list[2])
g.log.info('Bringing brick %s online is successful', bricks_list[2])
g.log.info("Verifying if brick %s is online", bricks_list[2])
ret = are_bricks_online(self.mnode, self.volname, bricks_list)
self.assertTrue(ret, ("Brick %s did not come up", bricks_list[2]))
g.log.info("Brick %s has come online.", bricks_list[2])
# Trigger heal
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Starting heal failed')
g.log.info('Index heal launched')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
def test_heal_when_quota_object_limit_exceeded(self):
# Create a directory to set the quota_limit_objects
path = "/dir"
g.log.info("Creating a directory")
self.all_mounts_procs = []
for mount_object in self.mounts:
cmd = "/usr/bin/env python %s create_deep_dir -d 0 -l 0 %s%s" % (
self.script_upload_path, mount_object.mountpoint, path)
ret = g.run(mount_object.client_system, cmd)
self.assertTrue(ret, "Failed to create directory on mountpoint")
g.log.info("Directory created successfully on mountpoint")
# Enable Quota
g.log.info("Enabling quota on the volume %s", self.volname)
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to enable quota on the volume "
"%s", self.volname))
g.log.info("Successfully enabled quota on the volume %s", self.volname)
# Set quota-soft-timeout to 0
g.log.info("Setting up soft timeout to 0")
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, "0")
self.assertEqual(ret, 0, ("Failed to set quota-soft-timeout"))
g.log.info("Successfully set the quota-soft-timeout")
# Set quota-hard-timeout to 0
g.log.info("Setting up hard timeout with 0")
ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, "0")
self.assertEqual(ret, 0, ("Failed to set quota-hard-timeout"))
g.log.info("successfully set the quota-hard-timeout")
# Set Quota limit on the newly created directory
g.log.info("Set Quota Limit on the path %s of the volume %s", path,
self.volname)
ret, _, _ = quota_limit_objects(self.mnode,
self.volname,
path=path,
limit="5")
self.assertEqual(ret, 0, ("Failed to set quota limit on path %s of "
" the volume %s", path, self.volname))
g.log.info(
"Successfully set the quota limit on %s of the volume "
"%s", path, self.volname)
# Create 3 files inside the directory
for mount_object in self.mounts:
g.log.info("Creating Files on %s:%s", mount_object.client_system,
path)
cmd = ("/usr/bin/env python %s create_files -f 3 "
"--base-file-name file-0 %s%s" %
(self.script_upload_path, mount_object.mountpoint, path))
ret, _, _ = g.run(mount_object.client_system, cmd)
self.assertEqual(ret, 0, ("Failed to create files on %s", path))
g.log.info("Files created successfully on mountpoint")
bricks_list = get_all_bricks(self.mnode, self.volname)
# Bring brick3 offline
g.log.info('Bringing brick %s offline', bricks_list[2])
ret = bring_bricks_offline(self.volname, bricks_list[2])
self.assertTrue(ret,
'Failed to bring bricks %s offline' % bricks_list[2])
ret = are_bricks_offline(self.mnode, self.volname, [bricks_list[2]])
self.assertTrue(ret, 'Brick %s is not offline' % bricks_list[2])
g.log.info('Bringing brick %s offline is successful', bricks_list[2])
# Try creating 5 more files, which should fail as the quota limit
# exceeds
cmd = ("/usr/bin/env python %s create_files -f 5 --base-file-name "
"file-1 %s%s" %
(self.script_upload_path, mount_object.mountpoint, path))
ret, _, _ = g.run(mount_object.client_system, cmd)
self.assertNotEqual(ret, 0, ("Creating 5 files succeeded while it was"
"not supposed to."))
g.log.info("Creating 5 files failed as expected due to quota object"
"limit on the directory.")
# Bring brick3 online and check status
g.log.info('Bringing brick %s online', bricks_list[2])
ret = bring_bricks_online(self.mnode, self.volname, [bricks_list[2]])
self.assertTrue(ret,
'Failed to bring brick %s online' % bricks_list[2])
g.log.info('Bringing brick %s online is successful', bricks_list[2])
g.log.info("Verifying if brick %s is online", bricks_list[2])
ret = are_bricks_online(self.mnode, self.volname, bricks_list)
self.assertTrue(ret, ("Brick %s did not come up", bricks_list[2]))
g.log.info("Brick %s has come online.", bricks_list[2])
# Trigger heal
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Starting heal failed')
g.log.info('Index heal launched')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
def test_heal_info_no_hang(self):
"""
Testcase steps:
1. Start kernel untar on the mount
2. While untar is going on, kill a brick of the replica.
3. Wait for the untar to be over, resulting in pending heals.
4. Get the approx. number of pending heals and save it
5. Bring the brick back online.
6. Trigger heal
7. Run more I/Os with dd command
8. Run heal info command and check that it completes successfully under
a timeout that is based on the no. of heals in step 4.
"""
self.list_of_io_processes = []
self.linux_untar_dir = "{}/{}".format(self.mounts[0].mountpoint,
"linuxuntar")
ret = mkdir(self.clients[0], self.linux_untar_dir)
self.assertTrue(ret, "Failed to create dir linuxuntar for untar")
# Start linux untar on dir linuxuntar
ret = run_linux_untar(self.clients[0], self.mounts[0].mountpoint,
dirs=tuple(['linuxuntar']))
self.list_of_io_processes += ret
self.is_io_running = True
# Kill brick resulting in heal backlog.
brick_to_bring_offline = random.choice(self.bricks_list)
ret = bring_bricks_offline(self.volname, brick_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s offline'
% brick_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
[brick_to_bring_offline])
self.assertTrue(ret, 'Bricks %s are not offline'
% brick_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
brick_to_bring_offline)
ret = self._wait_for_untar_completion()
self.assertFalse(ret, "IO didn't complete or failed on client")
self.is_io_running = False
# Get approx. no. of entries to be healed.
cmd = ("gluster volume heal %s statistics heal-count | grep Number "
"| awk '{sum+=$4} END {print sum/2}'" % self.volname)
ret, self.num_entries, _ = g.run(self.mnode, cmd)
self.assertEqual(ret, 0, "Failed to get heal-count statistics")
# Restart the down bricks
ret = bring_bricks_online(self.mnode, self.volname,
[brick_to_bring_offline])
self.assertTrue(ret, 'Failed to bring brick %s online' %
brick_to_bring_offline)
g.log.info('Bringing brick %s online is successful',
brick_to_bring_offline)
# Trigger heal
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Starting heal failed')
g.log.info('Index heal launched')
# Run more I/O
cmd = ("for i in `seq 1 10`; do dd if=/dev/urandom of=%s/file_$i "
"bs=1M count=100; done" % self.mounts[0].mountpoint)
ret = g.run_async(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
# Get heal info
ret = self._does_heal_info_complete_within_timeout()
self.assertTrue(ret, 'Heal info timed out')
g.log.info('Heal info completed succesfully')
def test_conservative_merge_of_files_heal_command(self):
"""
- set options:
"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off",
"self-heal-daemon": "off"
- Bring brick 0 offline
- Creating files on client side
- Bring brick 0 online
- Bring brick 1 offline
- Creating files on client side
- Bring brick 1 online
- Get arequal on bricks
- Setting option
"self-heal-daemon": "on"
- Start healing
- Get arequal on bricks and compare with arequals before healing
and mountpoint
"""
# pylint: disable=too-many-statements,too-many-locals
# set options
bricks_list = get_all_bricks(self.mnode, self.volname)
options = {
"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off",
"self-heal-daemon": "off"
}
g.log.info("setting options %s", options)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set volume option %s for"
"volume %s" % (options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# Bring brick 0 offline
g.log.info('Bringing bricks %s offline', bricks_list[0])
ret = bring_bricks_offline(self.volname, bricks_list[0])
self.assertTrue(ret,
'Failed to bring bricks %s offline' % bricks_list[0])
ret = are_bricks_offline(self.mnode, self.volname, [bricks_list[0]])
self.assertTrue(ret, 'Bricks %s are not offline' % bricks_list[0])
g.log.info('Bringing bricks %s offline is successful', bricks_list[0])
# Creating files on client side
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Create files
g.log.info('Creating files...')
command = ("python %s create_deep_dirs_with_files "
"-d 0 -l 5 -f 10 --dirname-start-num 1 %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# Bring brick 0 online
g.log.info('Bringing bricks %s online...', bricks_list[0])
ret = bring_bricks_online(self.mnode, self.volname, [bricks_list[0]])
self.assertTrue(ret,
'Failed to bring bricks %s online' % bricks_list[0])
g.log.info('Bringing bricks %s online is successful', bricks_list[0])
# Bring brick 1 offline
g.log.info('Bringing bricks %s offline', bricks_list[1])
ret = bring_bricks_offline(self.volname, bricks_list[1])
self.assertTrue(ret,
'Failed to bring bricks %s offline' % bricks_list[1])
ret = are_bricks_offline(self.mnode, self.volname, [bricks_list[1]])
self.assertTrue(ret, 'Bricks %s are not offline' % bricks_list[1])
g.log.info('Bringing bricks %s offline is successful', bricks_list[1])
# Creating files on client side
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Create files
g.log.info('Creating files...')
command = ("python %s create_deep_dirs_with_files "
"-d 0 -l 5 -f 10 --dirname-start-num 6 %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# Bring brick 1 online
g.log.info('Bringing bricks %s online...', bricks_list[1])
ret = bring_bricks_online(self.mnode, self.volname, [bricks_list[1]])
self.assertTrue(ret,
'Failed to bring bricks %s online' % bricks_list[1])
g.log.info('Bringing bricks %s online is successful', bricks_list[1])
# Get arequal on bricks
arequals_before_heal = {}
g.log.info('Getting arequal on bricks...')
for brick in bricks_list:
g.log.info('Getting arequal on bricks %s...', brick)
node, brick_path = brick.split(':')
command = ('arequal-checksum -p %s '
'-i .glusterfs -i .landfill -i .trashcan' % brick_path)
ret, arequal, _ = g.run(node, command)
self.assertFalse(ret, 'Failed to get arequal on brick %s' % brick)
g.log.info('Getting arequal for %s is successful', brick)
brick_total = arequal.splitlines()[-1].split(':')[-1]
arequals_before_heal[brick] = brick_total
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequals for mount
g.log.info('Getting arequal before getting bricks offline...')
ret, arequals = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after healing is successful')
mount_point_total = arequals[0].splitlines()[-1].split(':')[-1]
# Get arequal on bricks and compare with mount_point_total
# It should be the same
g.log.info('Getting arequal on bricks...')
arequals_after_heal = {}
for brick in bricks_list:
g.log.info('Getting arequal on bricks %s...', brick)
node, brick_path = brick.split(':')
command = ('arequal-checksum -p %s '
'-i .glusterfs -i .landfill -i .trashcan' % brick_path)
ret, arequal, _ = g.run(node, command)
self.assertFalse(ret, 'Failed to get arequal on brick %s' % brick)
g.log.info('Getting arequal for %s is successful', brick)
brick_total = arequal.splitlines()[-1].split(':')[-1]
arequals_after_heal[brick] = brick_total
self.assertEqual(
mount_point_total, brick_total,
'Arequals for mountpoint and %s are not equal' % brick)
g.log.info('Arequals for mountpoint and %s are equal', brick)
g.log.info('All arequals are equal for replicated')
self.assertNotEqual(
cmp(arequals_before_heal, arequals_after_heal), 0,
'Arequals are equal for bricks '
'before and after healing')
def test_multiple_clients_dd_on_same_file_default(self):
"""
- Create 2GB file
- While creating file, start reading file
- Bring down brick1
- Bring back the brick brick1
- Start healing
- Bring down brick1
- Wait for IO to complete
- Wait for reading to complete
- Bring back the brick brick1
- Start healing
- Wait for heal to complete
- Check for split-brain
- Calculate arequals on all the bricks and compare with mountpoint
"""
# pylint: disable=too-many-statements,too-many-locals
bricks_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(bricks_list, 'Brick list is None')
# Creating files on client side
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
# Create files
g.log.info('Creating files...')
command = ("cd %s ; "
"dd if=/dev/urandom of=test_file bs=1M count=2020"
% mount_obj.mountpoint)
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Reading files on client side
all_mounts_procs_read = []
for mount_obj in self.mounts:
g.log.info("Reading data for %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
# Create files
g.log.info('Reading files...')
command = ("python %s read %s"
% (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
all_mounts_procs_read.append(proc)
# Bring brick1 offline
g.log.info('Bringing bricks %s offline...', bricks_list[1])
ret = bring_bricks_offline(self.volname, [bricks_list[1]])
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_list[1])
ret = are_bricks_offline(self.mnode, self.volname,
[bricks_list[1]])
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_list[1])
g.log.info('Bringing bricks %s offline is successful',
bricks_list[1])
# Bring brick1 online
g.log.info('Bringing bricks %s online...', bricks_list[1])
ret = bring_bricks_online(self.mnode, self.volname,
[bricks_list[1]])
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_list[1])
g.log.info('Bringing bricks %s online is successful',
bricks_list[1])
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Bring brick1 offline
g.log.info('Bringing bricks %s offline...', bricks_list[1])
ret = bring_bricks_offline(self.volname, [bricks_list[1]])
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_list[1])
ret = are_bricks_offline(self.mnode, self.volname,
[bricks_list[1]])
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_list[1])
g.log.info('Bringing bricks %s offline is successful',
bricks_list[1])
# Validate IO
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients"
)
# Validate reading
self.assertTrue(
validate_io_procs(all_mounts_procs_read, self.mounts),
"Reading failed on some of the clients"
)
self.io_validation_complete = True
# Bring brick1 online
g.log.info('Bringing bricks %s online...', bricks_list[1])
ret = bring_bricks_online(self.mnode, self.volname,
[bricks_list[1]])
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_list[1])
g.log.info('Bringing bricks %s online is successful',
bricks_list[1])
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal for mount
g.log.info('Getting arequal...')
ret, arequals = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after healing is successful')
mount_point_total = arequals[0].splitlines()[-1].split(':')[-1]
# Get arequal on bricks and compare with mount_point_total
# It should be the same
g.log.info('Getting arequal on bricks...')
arequals_after_heal = {}
for brick in bricks_list:
g.log.info('Getting arequal on bricks %s...', brick)
node, brick_path = brick.split(':')
command = ('arequal-checksum -p %s '
'-i .glusterfs -i .landfill -i .trashcan'
% brick_path)
ret, arequal, _ = g.run(node, command)
self.assertFalse(ret, 'Failed to get arequal on brick %s'
% brick)
g.log.info('Getting arequal for %s is successful', brick)
brick_total = arequal.splitlines()[-1].split(':')[-1]
arequals_after_heal[brick] = brick_total
self.assertEqual(mount_point_total, brick_total,
'Arequals for mountpoint and %s are not equal'
% brick)
g.log.info('Arequals for mountpoint and %s are equal', brick)
g.log.info('All arequals are equal')
def test_self_heal_50k_files_heal_command_by_add_brick(self):
"""
Test self-heal of 50k files (heal command)
Description:
- Set the volume option
"metadata-self-heal": "off"
"entry-self-heal": "off"
"data-self-heal": "off"
"self-heal-daemon": "off"
- Bring down all bricks processes from selected set
- Create IO (50k files)
- Get arequal before getting bricks online
- Bring bricks online
- Set the volume option
"self-heal-daemon": "on"
- Check for daemons
- Start healing
- Check if heal is completed
- Check for split-brain
- Get arequal after getting bricks online and compare with
arequal before getting bricks online
- Add bricks
- Do rebalance
- Get arequal after adding bricks and compare with
arequal after getting bricks online
"""
# pylint: disable=too-many-locals,too-many-statements
# Setting options
options = {
"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off",
"self-heal-daemon": "off"
}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Successfully set %s for volume %s", options, self.volname)
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = list(
filter(None, (bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks'])))
# Bring brick offline
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Creating files on client side
all_mounts_procs = []
# Create 50k files
g.log.info('Creating files...')
command = ("cd %s ; "
"for i in `seq 1 50000` ; "
"do dd if=/dev/urandom of=test.$i "
"bs=100k count=1 ; "
"done ;" % self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts[0]),
"IO failed on some of the clients")
# Get arequal before getting bricks online
ret, result_before_online = collect_mounts_arequal(self.mounts[0])
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks online '
'is successful')
# Bring brick online
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s online' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Setting options
ret = set_volume_options(self.mnode, self.volname,
{"self-heal-daemon": "on"})
self.assertTrue(ret, 'Failed to set option self-heal-daemon to ON.')
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode,
self.volname,
timeout_period=3600)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
ret, result_after_online = collect_mounts_arequal(self.mounts[0])
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks online '
'is successful')
# Checking arequals before bringing bricks online
# and after bringing bricks online
self.assertItemsEqual(
result_before_online, result_after_online, 'Checksums before and '
'after bringing bricks online are not equal')
g.log.info('Checksums before and after bringing bricks online '
'are equal')
# Add bricks
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to expand the volume when IO in "
"progress on volume %s", self.volname))
g.log.info("Expanding volume is successful on volume %s", self.volname)
# Do rebalance and wait for it to complete
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, 'Failed to start rebalance')
g.log.info('Rebalance is started')
ret = wait_for_rebalance_to_complete(self.mnode,
self.volname,
timeout=3600)
self.assertTrue(ret, 'Rebalance is not completed')
g.log.info('Rebalance is completed successfully')
# Get arequal after adding bricks
ret, result_after_adding_bricks = collect_mounts_arequal(
self.mounts[0])
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks ' 'is successful')
# Checking arequals after bringing bricks online
# and after adding bricks
self.assertItemsEqual(
result_after_online, result_after_adding_bricks,
'Checksums after bringing bricks online and '
'after adding bricks are not equal')
g.log.info('Checksums after bringing bricks online and '
'after adding bricks are equal')
def test_data_self_heal_algorithm_diff_heal_command(self):
"""
Test Volume Option - 'cluster.data-self-heal-algorithm' : 'diff'
Description:
- set the volume option
"metadata-self-heal": "off"
"entry-self-heal": "off"
"data-self-heal": "off"
"data-self-heal-algorithm": "diff"
"self-heal-daemon": "off"
- create IO
- calculate arequal
- bring down all bricks processes from selected set
- modify the data
- get arequal before getting bricks online
- bring bricks online
- expand volume by adding bricks to the volume
- do rebalance
- set the volume option "self-heal-daemon": "on" and check for daemons
- start healing
- check if heal is completed
- check for split-brain
- calculate arequal and compare with arequal before bringing bricks
offline and after bringing bricks online
"""
# pylint: disable=too-many-branches,too-many-statements
# Setting options
g.log.info('Setting options...')
options = {
"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off",
"data-self-heal-algorithm": "diff"
}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Options "
"'metadata-self-heal', "
"'entry-self-heal', "
"'data-self-heal', "
"'self-heal-daemon' "
"are set to 'off',"
"'data-self-heal-algorithm' "
"is set to 'diff' successfully")
# Creating files on client side
all_mounts_procs = []
g.log.info("Generating data for %s:%s", self.mounts[0].client_system,
self.mounts[0].mountpoint)
# Creating files
command = "/usr/bin/env python %s create_files -f 100 %s" % (
self.script_upload_path, self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Option 'self-heal-daemon' is set to 'off' successfully")
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = list(
filter(None, (bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks'])))
# Bring brick offline
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Modify the data
all_mounts_procs = []
g.log.info("Modifying data for %s:%s", self.mounts[0].client_system,
self.mounts[0].mountpoint)
command = ("/usr/bin/env python %s create_files -f 100 "
"--fixed-file-size 1M %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# Get arequal before getting bricks online
g.log.info('Getting arequal before getting bricks online...')
ret, result_before_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks online '
'is successful')
# Bring brick online
g.log.info('Bringing bricks %s online...', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s online' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Expand volume by adding bricks to the volume
g.log.info("Start adding bricks to volume...")
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to expand the volume when IO in "
"progress on volume %s", self.volname))
g.log.info("Expanding volume is successful on volume %s", self.volname)
# Do rebalance
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, 'Failed to start rebalance')
g.log.info('Rebalance is started')
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Rebalance is not completed')
g.log.info('Rebalance is completed successfully')
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
g.log.info('Getting arequal after getting bricks online...')
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks online '
'is successful')
# Checking arequals before bringing bricks offline
# and after bringing bricks online
self.assertItemsEqual(result_before_online, result_after_online,
'Checksums are not equal')
g.log.info('Checksums are equal')
def test_arb_to_repl_conversion_with_io(self):
"""
Description: To perform a volume conversion from Arbiter to Replicated
with background IOs
Steps:
- Create, start and mount an arbiter volume in two clients
- Create two dir's, fill IO in first dir and take note of arequal
- Start a continuous IO from second directory
- Convert arbiter to x2 replicated volume (remove brick)
- Convert x2 replicated to x3 replicated volume (add brick)
- Wait for ~5 min for vol file to be updated on all clients
- Enable client side heal options and issue volume heal
- Validate heal completes with no errors and arequal of first dir
matches against initial checksum
"""
client, m_point = (self.mounts[0].client_system,
self.mounts[0].mountpoint)
# Fill IO in first directory
cmd = ('/usr/bin/env python {} '
'create_deep_dirs_with_files --dir-depth 10 '
'--fixed-file-size 1M --num-of-files 100 '
'--dirname-start-num 1 {}'.format(self.script_path, m_point))
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, 'Not able to fill directory with IO')
# Take `arequal` checksum on first directory
ret, exp_arequal = collect_mounts_arequal(self.mounts[0],
m_point + '/user1')
self.assertTrue(ret, 'Failed to get arequal checksum on mount')
# Start continuous IO from second directory
client = self.mounts[1].client_system
cmd = ('/usr/bin/env python {} '
'create_deep_dirs_with_files --dir-depth 10 '
'--fixed-file-size 1M --num-of-files 250 '
'--dirname-start-num 2 {}'.format(self.script_path, m_point))
proc = g.run_async(client, cmd)
self.all_mounts_procs.append(proc)
# Wait for IO to fill before volume conversion
sleep(30)
# Remove arbiter bricks ( arbiter to x2 replicated )
kwargs = {'replica_count': 2}
ret, _, _ = remove_brick(self.mnode,
self.volname,
self._get_arbiter_bricks(),
option='force',
**kwargs)
self.assertEqual(ret, 0, 'Not able convert arbiter to x2 replicated '
'volume')
# Wait for IO to fill after volume conversion
sleep(30)
# Add bricks (x2 replicated to x3 replicated)
kwargs['replica_count'] = 3
vol_info = get_volume_info(self.mnode, volname=self.volname)
self.assertIsNotNone(vol_info, 'Not able to get volume info')
dist_count = vol_info[self.volname]['distCount']
bricks_list = form_bricks_list(
self.mnode,
self.volname,
number_of_bricks=int(dist_count) * 1,
servers=self.servers,
servers_info=self.all_servers_info,
)
self.assertTrue(bricks_list, 'Not able to get unused list of bricks')
ret, _, _ = add_brick(self.mnode,
self.volname,
bricks_list,
force='True',
**kwargs)
self.assertEqual(ret, 0, 'Not able to add-brick to '
'{}'.format(self.volname))
# Wait for IO post x3 replicated volume conversion
sleep(30)
# Validate volume info
vol_info = get_volume_info(self.mnode, volname=self.volname)
self.assertIsNotNone(vol_info, 'Not able to get volume info')
vol_info = vol_info[self.volname]
repl_count, brick_count = (vol_info['replicaCount'],
vol_info['brickCount'])
# Wait for the volfile to sync up on clients
cmd = ('grep -ir connected {}/.meta/graphs/active/{}-client-*/private '
'| wc -l')
wait_time = time() + 300
in_sync = False
while time() <= wait_time:
ret, rout, _ = g.run(client, cmd.format(m_point, self.volname))
self.assertEqual(ret, 0,
'Not able to grep for volfile sync from client')
if int(rout) == int(brick_count):
in_sync = True
break
sleep(30)
self.assertTrue(
in_sync, 'Volfiles from clients are not synced even '
'after polling for ~5 min')
self.assertEqual(
int(repl_count), kwargs['replica_count'], 'Not able '
'to validate x2 to x3 replicated volume conversion')
# Enable client side heal options, trigger and monitor heal
ret = set_volume_options(
self.mnode, self.volname, {
'data-self-heal': 'on',
'entry-self-heal': 'on',
'metadata-self-heal': 'on'
})
self.assertTrue(ret, 'Unable to set client side heal options')
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Unable to trigger heal on volume')
ret = monitor_heal_completion(self.mnode,
self.volname,
timeout_period=1800)
self.assertTrue(ret,
'Heal is not completed for {}'.format(self.volname))
# Validate IO
prev_time = datetime.now().replace(microsecond=0)
ret = validate_io_procs(self.all_mounts_procs, [self.mounts[1]])
curr_time = datetime.now().replace(microsecond=0)
self.assertTrue(ret, 'Not able to validate completion of IO on mount')
self.all_mounts_procs *= 0
# To ascertain IO was happening during brick operations
self.assertGreater(
curr_time - prev_time, timedelta(seconds=10), 'Unable '
'to validate IO was happening during brick operations')
# Take and validate `arequal` checksum on first directory
ret, act_areequal = collect_mounts_arequal(self.mounts[1],
m_point + '/user1')
self.assertTrue(ret, 'Failed to get arequal checksum from mount')
self.assertEqual(
exp_arequal, act_areequal, '`arequal` checksum did '
'not match post arbiter to x3 replicated volume conversion')
g.log.info('PASS: Arbiter to x3 replicated volume conversion complete')
def test_data_split_brain_resolution(self):
# Setting options
g.log.info('Setting options...')
options = {
"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off"
}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Successfully set %s for volume %s", options, self.volname)
# Creating files and directories on client side
g.log.info('Creating files and directories...')
cmd = ("for i in `seq 1 10`; do mkdir %s/dir.$i; for j in `seq 1 5`;"
"do dd if=/dev/urandom of=%s/dir.$i/file.$j bs=1K count=1;"
"done; dd if=/dev/urandom of=%s/file.$i bs=1K count=1; done" %
(self.mounts[0].mountpoint, self.mounts[0].mountpoint,
self.mounts[0].mountpoint))
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Creating files and directories failed")
g.log.info("Files & directories created successfully")
# Check arequals for all the bricks
g.log.info('Getting arequal before getting bricks offline...')
self.verify_brick_arequals()
g.log.info('Getting arequal before getting bricks offline '
'is successful')
# Set option self-heal-daemon to OFF
g.log.info('Setting option self-heal-daemon to off...')
options = {"self-heal-daemon": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' is set to 'off' successfully")
bricks_list = get_all_bricks(self.mnode, self.volname)
# Bring brick1 offline
g.log.info('Bringing brick %s offline', bricks_list[0])
ret = bring_bricks_offline(self.volname, bricks_list[0])
self.assertTrue(ret,
'Failed to bring bricks %s offline' % bricks_list[0])
ret = are_bricks_offline(self.mnode, self.volname, [bricks_list[0]])
self.assertTrue(ret, 'Brick %s is not offline' % bricks_list[0])
g.log.info('Bringing brick %s offline is successful', bricks_list[0])
# Modify the contents of the files
cmd = ("for i in `seq 1 10`; do for j in `seq 1 5`;"
"do dd if=/dev/urandom of=%s/dir.$i/file.$j bs=1M count=1;"
"done; dd if=/dev/urandom of=%s/file.$i bs=1K count=1; done" %
(self.mounts[0].mountpoint, self.mounts[0].mountpoint))
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Updating file contents failed")
g.log.info("File contents updated successfully")
# Bricng brick1 online and check the status
g.log.info('Bringing brick %s online', bricks_list[0])
ret = bring_bricks_online(self.mnode, self.volname, [bricks_list[0]])
self.assertTrue(ret,
'Failed to bring brick %s online' % bricks_list[0])
g.log.info('Bringing brick %s online is successful', bricks_list[0])
g.log.info("Verifying if brick %s is online", bricks_list[0])
ret = are_bricks_online(self.mnode, self.volname, bricks_list)
self.assertTrue(ret, ("Brick %s did not come up", bricks_list[0]))
g.log.info("Brick %s has come online.", bricks_list[0])
# Bring brick2 offline
g.log.info('Bringing brick %s offline', bricks_list[1])
ret = bring_bricks_offline(self.volname, bricks_list[1])
self.assertTrue(ret,
'Failed to bring bricks %s offline' % bricks_list[1])
ret = are_bricks_offline(self.mnode, self.volname, [bricks_list[1]])
self.assertTrue(ret, 'Brick %s is not offline' % bricks_list[1])
g.log.info('Bringing brick %s offline is successful', bricks_list[1])
# Modify the contents of the files
cmd = ("for i in `seq 1 10`; do for j in `seq 1 5`;"
"do dd if=/dev/urandom of=%s/dir.$i/file.$j bs=1M count=2;"
"done; dd if=/dev/urandom of=%s/file.$i bs=1K count=2; done" %
(self.mounts[0].mountpoint, self.mounts[0].mountpoint))
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Updating file contents failed")
g.log.info("File contents updated successfully")
# Bricng brick2 online and check the status
g.log.info('Bringing brick %s online', bricks_list[1])
ret = bring_bricks_online(self.mnode, self.volname, [bricks_list[1]])
self.assertTrue(ret,
'Failed to bring brick %s online' % bricks_list[1])
g.log.info('Bringing brick %s online is successful', bricks_list[1])
g.log.info("Verifying if brick %s is online", bricks_list[1])
ret = are_bricks_online(self.mnode, self.volname, bricks_list)
self.assertTrue(ret, ("Brick %s did not come up", bricks_list[1]))
g.log.info("Brick %s has come online.", bricks_list[1])
# Set option self-heal-daemon to ON
g.log.info('Setting option self-heal-daemon to on...')
options = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
g.log.info("Checking if files are in split-brain")
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertTrue(ret, "Unable to create split-brain scenario")
g.log.info("Successfully created split brain scenario")
g.log.info("Resolving split-brain by using the source-brick option "
"by choosing second brick as source for all the files")
node, _ = bricks_list[1].split(':')
command = ("gluster v heal " + self.volname + " split-brain "
"source-brick " + bricks_list[1])
ret, _, _ = g.run(node, command)
self.assertEqual(ret, 0, "Command execution not successful")
# triggering heal
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, "Heal not triggered")
# waiting for heal to complete
ret = monitor_heal_completion(self.mnode,
self.volname,
timeout_period=120)
self.assertTrue(ret, "Heal not completed")
# Try accessing the file content from the mount
cmd = ("for i in `seq 1 10`; do cat %s/file.$i > /dev/null;"
"for j in `seq 1 5` ; do cat %s/dir.$i/file.$j > /dev/null;"
"done ; done" %
(self.mounts[0].mountpoint, self.mounts[0].mountpoint))
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Unable to access the file contents")
g.log.info("File contents are accessible")
# checking if file is in split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, "File still in split-brain")
g.log.info("Successfully resolved split brain situation using "
"CLI based resolution")
# Check arequals for all the bricks
g.log.info('Getting arequal for all the bricks after heal...')
self.verify_brick_arequals()
g.log.info('Getting arequal after heal is successful')
def test_data_self_heal_command(self):
"""
Test Data-Self-Heal (heal command)
Description:
- get the client side healing volume options and check
if they have already been disabled by default
NOTE: Client side healing has been disabled by default
since GlusterFS 6.0
"metadata-self-heal": "off"
"entry-self-heal": "off"
"data-self-heal": "off"
- create IO
- Get arequal before getting bricks offline
- set the volume option
"self-heal-daemon": "off"
- bring down all bricks processes from selected set
- Get arequal after getting bricks offline and compare with
arequal before getting bricks offline
- modify the data
- bring bricks online
- set the volume option
"self-heal-daemon": "on"
- check daemons and start healing
- check if heal is completed
- check for split-brain
- create 5k files
- while creating files - kill bricks and bring bricks online one by one
in cycle
- validate IO
"""
# pylint: disable=too-many-statements
# Checking if Client side healing options are disabled by default
g.log.info('Checking Client side healing is disabled by default')
options = ('cluster.metadata-self-heal', 'cluster.data-self-heal',
'cluster.entry-self-heal')
for option in options:
ret = get_volume_options(self.mnode, self.volname, option)[option]
self.assertTrue(bool(ret == 'off' or ret == 'off (DEFAULT)'),
"{} option is not disabled by default"
.format(option))
g.log.info("Client side healing options are disabled by default")
# Creating files on client side
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
# Create files
g.log.info('Creating files...')
command = ("/usr/bin/env python %s create_files -f 100 "
"--fixed-file-size 1k %s" % (
self.script_upload_path,
mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients"
)
self.io_validation_complete = True
# Get arequal before getting bricks offline
g.log.info('Getting arequal before getting bricks offline...')
ret, result_before_offline = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks offline '
'is successful')
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = bricks_to_bring_offline_dict['volume_bricks']
# Bring brick offline
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Get arequal after getting bricks offline
g.log.info('Getting arequal after getting bricks offline...')
ret, result_after_offline = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks offline '
'is successful')
# Checking arequals before bringing bricks offline
# and after bringing bricks offline
self.assertEqual(result_before_offline, result_after_offline,
'Checksums before and '
'after bringing bricks online are not equal')
g.log.info('Checksums before and after bringing bricks online '
'are equal')
# Modify the data
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Modifying data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Create files
g.log.info('Creating files...')
command = ("/usr/bin/env python %s create_files -f 100 "
"--fixed-file-size 10k %s" % (
self.script_upload_path,
mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients"
)
self.io_validation_complete = True
# Bring brick online
g.log.info('Bringing bricks %s online...', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info("Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all processes are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online"
% self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Create 1k files
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Modifying data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Create files
g.log.info('Creating files...')
command = ("/usr/bin/env python %s create_files -f 1000 %s" % (
self.script_upload_path,
mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Kill all bricks in cycle
bricks_list = get_all_bricks(self.mnode, self.volname)
for brick in bricks_list:
# Bring brick offline
g.log.info('Bringing bricks %s offline', brick)
ret = bring_bricks_offline(self.volname, [brick])
self.assertTrue(ret, 'Failed to bring bricks %s offline' % brick)
ret = are_bricks_offline(self.mnode, self.volname,
[brick])
self.assertTrue(ret, 'Bricks %s are not offline'
% brick)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Bring brick online
g.log.info('Bringing bricks %s online...', brick)
ret = bring_bricks_online(self.mnode, self.volname,
[brick])
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info("Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online"
% self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found or"
"more than one self heal daemon process"
"found")
g.log.info("All self-heal-daemons are online")
# Validate IO
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients"
)
self.io_validation_complete = True
def test_heal_command_unsuccessful_as_bricks_down(self):
"""
- write 2 Gb file on mount
- while write is in progress, kill brick b0
- start heal on the volume (should fail and have error message)
- bring up the brick which was down (b0)
- bring down another brick (b1)
- start heal on the volume (should fail and have error message)
- bring bricks up
- wait for heal to complete
"""
# pylint: disable=too-many-statements
bricks_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(bricks_list, 'Brick list is None')
# Creating files on client side
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
# Create 2 Gb file
g.log.info('Creating files...')
command = ("cd %s ; dd if=/dev/zero of=file1 bs=10M count=200"
% mount_obj.mountpoint)
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Bring brick0 offline
g.log.info('Bringing bricks %s offline...', bricks_list[0])
ret = bring_bricks_offline(self.volname, [bricks_list[0]])
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_list[0])
ret = are_bricks_offline(self.mnode, self.volname,
[bricks_list[0]])
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_list[0])
g.log.info('Bringing bricks %s offline is successful',
bricks_list[0])
# Start healing
# Need to use 'gluster volume heal' command to check error message
# after g.run
cmd = "gluster volume heal %s" % self.volname
ret, _, err = g.run(self.mnode, cmd)
self.assertTrue(ret, 'Heal is started')
# Check for error message
self.assertIn("Launching heal operation to perform index self heal on "
"volume %s has been unsuccessful" % self.volname,
err,
"Error message is not present or not valid")
g.log.info('Expected: Healing is not started')
# Bring brick0 online
g.log.info("Bring bricks: %s online", bricks_list[0])
ret = bring_bricks_online(self.mnode, self.volname,
[bricks_list[0]])
self.assertTrue(ret, "Failed to bring bricks: %s online"
% bricks_list[0])
g.log.info("Successfully brought all bricks:%s online",
bricks_list[0])
# Bring brick1 offline
g.log.info('Bringing bricks %s offline...', bricks_list[1])
ret = bring_bricks_offline(self.volname, [bricks_list[1]])
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_list[1])
ret = are_bricks_offline(self.mnode, self.volname,
[bricks_list[1]])
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_list[1])
g.log.info('Bringing bricks %s offline is successful',
bricks_list[1])
# Start healing
# Need to use 'gluster volume heal' command to check error message
# after g.run
cmd = "gluster volume heal %s" % self.volname
ret, _, err = g.run(self.mnode, cmd)
self.assertTrue(ret, 'Heal is started')
# Check for error message
self.assertIn("Launching heal operation to perform index self heal on "
"volume %s has been unsuccessful" % self.volname,
err,
"Error message is not present or not valid")
g.log.info('Expected: Healing is not started')
# Bring brick 1 online
g.log.info("Bring bricks: %s online", bricks_list[1])
ret = bring_bricks_online(self.mnode, self.volname,
[bricks_list[1]])
self.assertTrue(ret, "Failed to bring bricks: %s online"
% bricks_list[1])
g.log.info("Successfully brought all bricks:%s online",
bricks_list[1])
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Validate IO
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients"
)
self.io_validation_complete = True
def test_resolving_meta_data(self):
"""
- Create a file test_file.txt
- Find out which brick the file resides on and kill arbiter brick
in the replica pair
- Modify the permissions of the file
- Bring back the killed brick
- Kill the other brick in the replica pair
- Modify the permissions of the file
- Bring back the killed brick
- Trigger heal
- Check if heal is completed
- Check for split-brain
"""
# pylint: disable=too-many-locals,too-many-statements
# Creating files on client side
file_to_create = 'test_file.txt'
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Create file
g.log.info('Creating file...')
command = ("cd %s ; "
"touch %s" % (mount_obj.mountpoint, file_to_create))
proc = g.run_async(mount_obj.client_system,
command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# get bricks with file
g.log.info('Getting bricks with file...')
subvols_dict = get_subvols(self.mnode, self.volname)
brick_list_with_file = []
for subvol in subvols_dict['volume_subvols']:
for brick in subvol:
node, brick_path = brick.split(':')
ret, brick_file_list, _ = g.run(node, 'ls %s' % brick_path)
if 'test_file.txt' in brick_file_list:
brick_list_with_file.append(brick)
g.log.info('Bricks with file: %s', brick_list_with_file)
# Bring arbiter brick offline
bricks_to_bring_offline = [brick_list_with_file[-1]]
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Modify the data
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Modifying data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Modify the permissions
g.log.info('Modifying the permissions of the file...')
command = ("cd %s ; "
"chmod 600 %s" % (mount_obj.mountpoint, file_to_create))
proc = g.run_async(mount_obj.client_system,
command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# Bring arbiter brick online
g.log.info('Bringing bricks %s online...', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s online' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Bring 1-st data brick offline
bricks_to_bring_offline = [brick_list_with_file[0]]
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Modify the data
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Modifying data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Modify the permissions
g.log.info('Modifying the permissions of the file...')
command = ("cd %s ; "
"chmod 644 %s" % (mount_obj.mountpoint, file_to_create))
proc = g.run_async(mount_obj.client_system,
command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# Bring 1-st data brick online
g.log.info('Bringing bricks %s online...', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s online' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
def test_data_self_heal_daemon_off(self):
"""
Test Data-Self-Heal (heal command)
Description:
- set the volume option
"metadata-self-heal": "off"
"entry-self-heal": "off"
"data-self-heal": "off"
- create IO
- Get areequal before getting bricks offline
- set the volume option
"self-heal-daemon": "off"
- bring down all bricks processes from selected set
- Get areequal after getting bricks offline and compare with
areequal before getting bricks offline
- modify the data
- bring bricks online
- set the volume option
"self-heal-daemon": "on"
- check daemons and start healing
- check if heal is completed
- check for split-brain
- add bricks
- do rebalance
- create 5k files
- while creating files - kill bricks and bring bricks online one by one
in cycle
- validate IO
"""
# Setting options
g.log.info('Setting options...')
options = {"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off",
}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Successfully set %s for volume %s"
% (options, self.volname))
# Creating files on client side
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s"
% (mount_obj.client_system, mount_obj.mountpoint))
# Create files
g.log.info('Creating files...')
command = ("python %s create_files -f 100 --fixed-file-size 1k %s"
% (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
# Get areequal before getting bricks offline
g.log.info('Getting areequal before getting bricks offline...')
ret, result_before_offline = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting areequal before getting bricks offline '
'is successful')
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' is set to 'off' successfully")
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = filter(None, (
bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks']))
# Bring brick offline
g.log.info('Bringing bricks %s offline...' % bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s offline' %
bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Bricks %s are not offline'
% bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful'
% bricks_to_bring_offline)
# Get areequal after getting bricks offline
g.log.info('Getting areequal after getting bricks offline...')
ret, result_after_offline = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting areequal after getting bricks offline '
'is successful')
# Checking areequals before bringing bricks offline
# and after bringing bricks offline
self.assertEqual(result_before_offline, result_after_offline,
'Checksums before and '
'after bringing bricks online are not equal')
g.log.info('Checksums before and after bringing bricks online '
'are equal')
# Modify the data
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Modifying data for %s:%s" %
(mount_obj.client_system, mount_obj.mountpoint))
# Create files
g.log.info('Creating files...')
command = ("python %s create_files -f 100 --fixed-file-size 10k %s"
% (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
# Bring brick online
g.log.info('Bringing bricks %s online...' % bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful'
% bricks_to_bring_offline)
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info("Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online"
% self.volname))
g.log.info("Volume %s : All process are online" % self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Add bricks
g.log.info("Start adding bricks to volume...")
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to expand the volume %s", self.volname))
g.log.info("Expanding volume is successful on "
"volume %s" % self.volname)
# Do rebalance
ret, out, err = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, 'Failed to start rebalance')
g.log.info('Rebalance is started')
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Rebalance is not completed')
g.log.info('Rebalance is completed successfully')
# Create 1k files
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Modifying data for %s:%s" %
(mount_obj.client_system, mount_obj.mountpoint))
# Create files
g.log.info('Creating files...')
command = ("python %s create_files -f 1000 %s"
% (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, command,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Kill all bricks in cycle
bricks_list = get_all_bricks(self.mnode, self.volname)
for brick in bricks_list:
# Bring brick offline
g.log.info('Bringing bricks %s offline' % brick)
ret = bring_bricks_offline(self.volname, [brick])
self.assertTrue(ret, 'Failed to bring bricks %s offline' % brick)
ret = are_bricks_offline(self.mnode, self.volname,
[brick])
self.assertTrue(ret, 'Bricks %s are not offline'
% brick)
g.log.info('Bringing bricks %s offline is successful'
% bricks_to_bring_offline)
# Bring brick online
g.log.info('Bringing bricks %s online...' % brick)
ret = bring_bricks_online(self.mnode, self.volname,
[brick])
self.assertTrue(ret, 'Failed to bring bricks %s online' %
bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful'
% bricks_to_bring_offline)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info("Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online"
% self.volname))
g.log.info("Volume %s : All process are online" % self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found or"
"more than one self heal daemon process"
"found")
g.log.info("All self-heal-daemons are online")
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
def test_entry_self_heal_heal_command(self):
"""
Test Entry-Self-Heal (heal command)
Description:
- set the volume option
"metadata-self-heal": "off"
"entry-self-heal": "off"
"data-self-heal": "off"
- create IO
- get arequal before getting bricks offline
- set the volume option
"self-heal-daemon": "off"
- bring down all bricks processes from selected set
- get arequal after getting bricks offline and compare with
arequal after bringing bricks offline
- modify the data
- get arequal before getting bricks online
- bring bricks online
- set the volume option
"self-heal-daemon": "on"
- check daemons and start healing
- check if heal is completed
- check for split-brain
- get arequal after getting bricks online and compare with
arequal before bringing bricks online
"""
# pylint: disable=too-many-statements
# Setting options
g.log.info('Setting options...')
options = {
"metadata-self-heal": "off",
"entry-self-heal": "off",
"data-self-heal": "off"
}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Options "
"'metadata-self-heal', "
"'entry-self-heal', "
"'data-self-heal', "
"are set to 'off'")
# Start IO on mounts
g.log.info("Starting IO on all mounts...")
g.log.info("Starting IO on %s:%s", self.mounts[0].client_system,
self.mounts[0].mountpoint)
cmd = ("python %s create_deep_dirs_with_files "
"--dir-length 2 "
"--dir-depth 2 "
"--max-num-of-dirs 2 "
"--num-of-files 20 %s/files" %
(self.script_upload_path, self.mounts[0].mountpoint))
ret, _, err = g.run(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
self.assertFalse(
ret, 'Failed to create the data for %s: %s' %
(self.mounts[0].mountpoint, err))
g.log.info('Created IO for %s is successfully',
self.mounts[0].mountpoint)
# Command list to do different operations with data -
# create, rename, copy and delete
cmd_list = [
"python %s create_files -f 20 %s/files",
"python %s mv %s/files",
# 'copy' command works incorrect. disable until fixed
# "python %s copy --dest-dir %s/new_dir %s/files",
"python %s delete %s"
]
for cmd in cmd_list:
# Get arequal before getting bricks offline
g.log.info('Getting arequal before getting bricks offline...')
ret, arequals = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
result_before_offline = arequals[0].splitlines()[-1].split(':')[-1]
g.log.info('Getting arequal before getting bricks offline '
'is successful')
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' "
"is set to 'off' successfully")
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = filter(
None, (bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks']))
# Bring brick offline
g.log.info('Bringing bricks %s offline...',
bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret,
'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Get arequal after getting bricks offline
g.log.info('Getting arequal after getting bricks offline...')
ret, arequals = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
result_after_offline = arequals[0].splitlines()[-1].split(':')[-1]
g.log.info('Getting arequal after getting bricks offline '
'is successful')
# Checking arequals before bringing bricks offline
# and after bringing bricks offline
self.assertEqual(result_before_offline, result_after_offline,
'Checksums are not equal')
g.log.info('Checksums before bringing bricks offline '
'and after bringing bricks offline are equal')
# Modify the data
g.log.info("Start modifying IO on all mounts...")
g.log.info("Modifying IO on %s:%s", self.mounts[0].client_system,
self.mounts[0].mountpoint)
if 'copy --dest-dir' in cmd:
parsed_cmd = cmd % (self.script_upload_path,
self.mounts[0].mountpoint,
self.mounts[0].mountpoint)
else:
parsed_cmd = cmd % (self.script_upload_path,
self.mounts[0].mountpoint)
ret, _, err = g.run(self.mounts[0].client_system,
parsed_cmd,
user=self.mounts[0].user)
self.assertFalse(
ret, 'Failed to modify the data for %s: %s' %
(self.mounts[0].mountpoint, err))
g.log.info('Modified IO for %s is successfully',
self.mounts[0].mountpoint)
# Get arequal before getting bricks online
g.log.info('Getting arequal before getting bricks online...')
ret, arequals = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
result_before_online = arequals[0].splitlines()[-1].split(':')[-1]
g.log.info('Getting arequal before getting bricks online '
'is successful')
# List all files and dirs created
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
if not ret:
raise ExecutionError("Failed to list all files and dirs")
g.log.info("Listing all files and directories is successful")
# Bring brick online
g.log.info('Bringing bricks %s online...', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret,
'Failed to bring bricks %s online' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Setting options
g.log.info('Setting options...')
options = {"self-heal-daemon": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Option 'self-heal-daemon' is set to 'on' successfully")
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(
self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Start healing
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not started')
g.log.info('Healing is started')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
g.log.info('Getting arequal after getting bricks online...')
ret, arequals = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
result_after_online = arequals[0].splitlines()[-1].split(':')[-1]
g.log.info('Getting arequal after getting bricks online '
'is successful')
# List all files and dirs created
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
if not ret:
raise ExecutionError("Failed to list all files and dirs")
g.log.info("Listing all files and directories is successful")
# Checking arequals before bringing bricks online
# and after bringing bricks online
self.assertEqual(result_before_online, result_after_online,
'Checksums are not equal')
g.log.info('Checksums before bringing bricks online '
'and after bringing bricks online are equal')