def _perform_quota_ops_after_brick_down(self):
"""
Refactor of common test steps across three test functions
"""
# Increase the quota limit on dir/dir1 and validate no errors on writes
self.alloc_size = self.free_disk_size - 50
self.fqpath = self.m_point + '/sparsefile'
self._fallocate_file()
ret, _, _ = quota_limit_usage(self.mnode,
self.volname,
path='/dir/dir1',
limit='40GB')
self.assertEqual(ret, 0, 'Not able to expand quota limit on /dir/dir1')
sleep(15)
self._insert_bp(self.client, self.logpath)
self.fqpath = self.m_point + '/dir/dir1'
proc = g.run_async(
self.client,
self.cmd.format(self.fqpath, self.rem_size * 3, self.bp_count))
self.assertTrue(
validate_io_procs([proc], self.mounts[0]),
'Fail: Not able to write data even after expanding quota limit')
sleep(10)
self._insert_bp(self.client, self.logpath)
self._validate_error_in_mount_log(pattern=self.quota_error,
exp_pre=False)
self._validate_error_in_mount_log(pattern=self.space_error,
exp_pre=False)
# Decrease the quota limit and validate error on reaching quota
self._insert_bp(self.client, self.logpath)
ret, _, _ = quota_limit_usage(self.mnode,
self.volname,
path='/dir/dir1',
limit='15GB')
self.assertEqual(ret, 0, 'Not able to expand quota limit on /dir/dir1')
sleep(10)
self.fqpath = self.m_point + '/dir/dir1'
self.rem_size = self._get_space_in_gb(self.client, self.fqpath)
proc = g.run_async(
self.client,
self.cmd.format(self.fqpath, self.rem_size * 3, self.bp_count))
self.assertFalse(
validate_io_procs([proc], self.mounts[0]),
'Fail: Process should not allow data more '
'than available space to be written')
sleep(10)
self._insert_bp(self.client, self.logpath)
self._validate_error_in_mount_log(pattern=self.quota_error)
self._validate_error_in_mount_log(pattern=self.space_error,
exp_pre=False)
def test_nfs_ganesha_remove_brick(self):
"""
Verify remove brick operation while IO is running
Steps:
1. Start IO on mount points
2. Perform remove brick operation
3. Validate IOs
"""
# pylint: disable=too-many-statements
# Start IO on all mount points
all_mounts_procs, count = [], 1
for mount_obj in self.mounts:
cmd = ("/usr/bin/env python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 10 "
"--max-num-of-dirs 5 "
"--num-of-files 5 %s" % (self.script_upload_path, count,
mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
count += 10
# Get stat of all the files/dirs created.
ret = get_mounts_stat(self.mounts)
self.assertTrue(ret, "Stat failed on some of the clients")
g.log.info("Successfully got stat of all files/dirs created")
# Perform remove brick operation
ret = shrink_volume(self.mnode, self.volname)
self.assertTrue(ret, ("Remove brick operation failed on "
"%s", self.volname))
g.log.info("Remove brick operation is successful on "
"volume %s", self.volname)
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("All volume %s processes failed to come up "
"online", self.volname))
g.log.info("All volume %s processes came up "
"online successfully after remove brick operation",
self.volname)
# Log volume info and status after performing remove brick
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status of volume %s",
self.volname)
# Validate IO
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Successfully validated all io's")
# Get stat of all the files/dirs created.
ret = get_mounts_stat(self.mounts)
self.assertTrue(ret, "Stat failed on some of the clients")
g.log.info("Successfully got stat of all files/dirs created")
def test_volume_set_when_io_in_progress(self):
"""Set Volume options which changes the client graphs while IO is
in progress.
Description:
- set volume option uss, shard to 'enable' and
validate it is successful
- validate IO to be successful
"""
# List of volume options to set
volume_options_list = ["features.uss", "features.shard"]
# enable and validate the volume options
g.log.info("Setting the volume options: %s", volume_options_list)
ret = enable_and_validate_volume_options(self.mnode, self.volname,
volume_options_list,
time_delay=30)
self.assertTrue(ret, ("Unable to enable the volume options: %s",
volume_options_list))
g.log.info("Successfully enabled all the volume options: %s",
volume_options_list)
# Validate IO
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.io_validation_complete = True
self.assertTrue(ret, "IO failed on some of the clients")
# List all files and dirs created
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
self.assertTrue(ret, "Failed to list all files and dirs")
g.log.info("Listing all files and directories is successful")
def start_and_wait_for_io_to_complete(self, mount_objs):
"""
This module start IO on clients and wait for io to complete.
Args:
mount_objs (lis): List of mounts
Returns:
True if IO is success, False otherwise.
"""
# Start IO on mounts
g.log.info("Starting IO on all mounts.")
self.all_mounts_procs = []
for mount_obj in mount_objs:
cmd = ("/usr/bin/env python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 15 "
"--max-num-of-dirs 5 "
"--num-of-files 10 %s" %
(self.script_upload_path, self.dir_start,
mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.dir_start += 10
# Adding a delay of 15 seconds giving some time to fill data
sleep(15)
# Validate IO
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
if not ret:
g.log.error("IO failed on some of the clients")
return False
return True
def test_new_mount_while_io_in_progress(self):
"""
Verify new mount will not cause any issues while IO is running
Steps:
1. Mount volume on one client
2. Start IO
3. Mount volume on new mountpoint
4. Start IO on new mountpoint
5. Validate IOs
"""
# Take 2 mounts if available
no_of_mount_objects = len(self.mounts)
if no_of_mount_objects == 1:
self.mount_obj1 = self.mounts[0]
self.mount_obj2 = deepcopy(self.mounts[0])
self.mount_obj2.mountpoint = '%s_new' % self.mount_obj2.mountpoint
else:
self.mount_obj1 = self.mounts[0]
self.mount_obj2 = self.mounts[1]
self.new_mounts = [self.mount_obj1, self.mount_obj2]
all_mounts_procs = []
dirname_start_num = 1
for mount_object in self.new_mounts:
# Mount volume
ret = self.mount_obj1.mount()
self.assertTrue(
ret, "Unable to mount volume %s on %s" %
(mount_object.volname, mount_object.client_system))
# Start IO
g.log.info("Starting IO on %s:%s", mount_object.client_system,
mount_object.mountpoint)
cmd = ("/usr/bin/env python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 10 "
"--max-num-of-dirs 5 "
"--num-of-files 5 %s" %
(self.script_upload_path, dirname_start_num,
mount_object.mountpoint))
proc = g.run_async(mount_object.client_system,
cmd,
user=mount_object.user)
all_mounts_procs.append(proc)
dirname_start_num += 10
# Validate IOs
g.log.info("Validating IO's")
ret = validate_io_procs(all_mounts_procs, self.new_mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Successfully validated all IOs")
# Get stat of all the files/dirs created.
g.log.info("Get stat of all the files/dirs created.")
ret = get_mounts_stat(self.new_mounts)
self.assertTrue(ret, "Stat failed on some of the clients")
g.log.info("Successfully got stat of all files/dirs created")
def test_file_dir_create_ops_on_volume(self):
"""Test File Directory Creation on the volume.
"""
# Start IO on all mounts.
all_mounts_procs = []
count = 1
for mount_obj in self.mounts:
cmd = ("python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 10 "
"--max-num-of-dirs 5 "
"--num-of-files 5 %s" %
(self.script_upload_path, count, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
count = count + 10
# Validate IO
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
# Get stat of all the files/dirs created.
ret = get_mounts_stat(self.mounts)
self.assertTrue(ret, "Stat failed on some of the clients")
def io_operation(name):
g.log.info("Starting to Perform I/O")
all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s",
mount_obj.client_system, mount_obj.mountpoint)
# Create files
g.log.info('Creating files...')
fname = "{}-{}".format(mount_obj.client_system, name)
command = ("/usr/bin/env python {} create_files -f 100 "
"--fixed-file-size 1k --base-file-name {}"
" {}".format(self.script_upload_path, fname,
mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
command,
user=mount_obj.user)
all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
return 0
def test_shrinking_volume_when_io_in_progress(self):
"""Test shrinking volume (Decrease distribute count) using existing
servers bricks when IO is in progress.
Description:
- remove brick (start, status, commit)
- validate IO
"""
# Log Volume Info and Status before shrinking the volume.
g.log.info("Logging volume info and Status before shrinking volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status of volume %s",
self.volname)
# Shrinking volume by removing bricks from volume when IO in progress
g.log.info("Start removing bricks from volume when IO in progress")
ret = shrink_volume(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to shrink the volume when IO in "
"progress on volume %s", self.volname))
g.log.info("Shrinking volume when IO in progress is successful on "
"volume %s", self.volname)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info("Successful in waiting for volume %s processes to be "
"online", self.volname)
# Log Volume Info and Status after shrinking the volume
g.log.info("Logging volume info and Status after shrinking volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status of volume %s",
self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online after "
"shrinking volume")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are not online",
self.volname))
g.log.info("Volume %s : All process are online after shrinking volume",
self.volname)
# Validate IO
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.io_validation_complete = True
self.assertTrue(ret, "IO failed on some of the clients")
# List all files and dirs created
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
self.assertTrue(ret, "Failed to list all files and dirs")
g.log.info("Listing all files and directories is successful")
def validate_io_on_clients(self):
"""
Validate I/O on client mount points.
"""
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
self.io_validation_complete = True
g.log.info("IO is successful on all mounts")
def test_nfs_ganesha_multiple_refresh_configs(self):
"""
Tests script to check nfs-ganehsa volume gets exported and IOs
are running after running multiple refresh configs.
"""
self.acl_check_flag = False
# Starting IO on the mounts
all_mounts_procs = []
count = 1
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("/usr/bin/env python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 10 "
"--max-num-of-dirs 5 "
"--num-of-files 5 %s" %
(self.script_upload_path, count, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
count = count + 10
for itr in range(1, 7):
# Enabling/Disabling ACLs to modify the export configuration
# before running refresh config
if itr % 2 == 0:
# Disable ACL
ret = set_acl(self.mnode, self.volname, False, False)
self.assertTrue(ret,
("Failed to disable acl on %s" % self.volname))
self.acl_check_flag = False
else:
# Enable ACL
ret = set_acl(self.mnode, self.volname, True, False)
self.assertTrue(ret,
("Failed to enable acl on %s" % self.volname))
self.acl_check_flag = True
ret = refresh_config(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to run refresh config"
"for volume %s" % self.volname))
g.log.info("Refresh-config completed. Iteration:%s", str(itr))
# Check nfs-ganesha cluster status
ret = is_nfs_ganesha_cluster_in_healthy_state(self.mnode)
self.assertTrue(
ret, "nfs-ganesha cluster is not healthy after "
"multiple refresh-config operations")
# Validate IO
g.log.info("Validating IO")
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Successfully validated all IO")
def test_basic_memory_leak(self):
"""
Test case:
1. Create a volume, start it and mount it.
2. Start I/O from mount point.
3. Check if there are any memory leaks and OOM killers.
"""
# Start monitoring resource usage on servers and clients
monitor_proc_dict = self.start_memory_and_cpu_usage_logging(
self.test_id, count=30)
self.assertIsNotNone(
monitor_proc_dict, "Failed to start monitoring on servers and "
"clients")
# Create a dir to start untar
self.linux_untar_dir = "{}/{}".format(self.mounts[1].mountpoint,
"linuxuntar")
ret = mkdir(self.mounts[1].client_system, self.linux_untar_dir)
self.assertTrue(ret, "Failed to create dir linuxuntar for untar")
# Start multiple I/O from mount points
self.list_of_io_processes = []
cmd = ("cd {};for i in `seq 1 100`; do mkdir dir.$i ;"
"for j in `seq 1 1000`; do dd if=/dev/random "
"of=dir.$i/testfile.$j bs=1k count=10;done;done".format(
self.mounts[0].mountpoint))
ret = g.run_async(self.mounts[0].client_system, cmd)
self.list_of_io_processes = [ret]
# Start linux untar on dir linuxuntar
ret = run_linux_untar(self.mounts[1].client_system,
self.mounts[1].mountpoint,
dirs=tuple(['linuxuntar']))
self.list_of_io_processes += ret
self.is_io_running = True
# Wait for I/O to complete and validate I/O on mount points
ret = validate_io_procs(self.list_of_io_processes, self.mounts)
self.assertTrue(ret, "I/O failed on mount point")
self.is_io_running = False
# Wait for monitoring processes to complete
ret = wait_for_logging_processes_to_stop(monitor_proc_dict,
cluster=True)
self.assertTrue(ret, "ERROR: Failed to stop monitoring processes")
# Check if there are any memory leaks and OOM killers
ret = self.check_for_memory_leaks_and_oom_kills_on_servers(
self.test_id)
self.assertFalse(ret,
"Memory leak and OOM kills check failed on servers")
ret = self.check_for_memory_leaks_and_oom_kills_on_clients(
self.test_id)
self.assertFalse(ret,
"Memory leak and OOM kills check failed on clients")
g.log.info("No memory leaks or OOM kills found on serves and clients")
def test_add_bricks_io_mount_point(self):
# Mount volume
ret = self.mount_volume(self.mounts)
self.assertTrue(ret, 'Mount volume: FAIL')
g.log.info('Mounted Volume %s: Success', self.volname)
# Upload io scripts for running IO on mounts
g.log.info(
"Upload io scripts to clients %s for running IO on "
"mounts", self.clients)
script_location = "/usr/share/glustolibs/io/scripts/file_dir_ops.py"
ret = upload_scripts(self.clients, script_location)
if not ret:
clients = ", ".join(self.clients)
g.log.error("Failed to upload IO scripts to clients %s", clients)
raise ExecutionError("Failed to upload IO scripts to clients %s" %
clients)
g.log.info("Successfully uploaded IO scripts to clients %s",
self.clients)
# Start IO on mounts
g.log.info("Starting IO on all mounts...")
for index, mount_obj in enumerate(self.mounts, start=1):
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("/usr/bin/env python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 2 "
"--max-num-of-dirs 2 "
"--num-of-files 10 %s" %
(script_location, index + 10, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
# Expand volume
g.log.debug("Expanding volume %s", self.volname)
ret = expand_volume(mnode=self.mnode,
volname=self.volname,
servers=self.servers,
all_servers_info=self.all_servers_info)
self.assertTrue(ret, "Expand volume %s: Fail" % self.volname)
g.log.info("Volume %s expanded: Success", self.volname)
# Validate IO on current mount point
g.log.debug('Validating IO on mount point %s:%s',
mount_obj.client_system, mount_obj.mountpoint)
self.assertTrue(
validate_io_procs([proc],
[mount_obj]), 'IO Failed on client %s:%s' %
(mount_obj.client_system, mount_obj.mountpoint))
g.log.debug("Unmounting mount points")
self.assertTrue(self.unmount_volume(self.mounts),
'Unmount end points: Fail')
g.log.info("Unmount mount points: Success")
g.log.info('Add brick during IO operations successfully')
def test_replace_brick_when_io_in_progress(self):
"""Test replacing brick using existing servers bricks when IO is
in progress.
Description:
- replace_brick
- wait for heal to complete
- validate IO
"""
# Log Volume Info and Status before replacing brick from the volume.
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status of volume %s",
self.volname)
# Replace brick from a sub-volume
ret = replace_brick_from_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, "Failed to replace faulty brick from the volume")
g.log.info("Successfully replaced faulty brick from the volume")
# Wait for volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
# Log Volume Info and Status after replacing the brick
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status of volume %s",
self.volname)
# Verify volume's all process are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online", self.volname))
# Wait for self-heal to complete
ret = monitor_heal_completion(self.mnode,
self.volname,
timeout_period=1800)
self.assertTrue(
ret, "Self heal didn't complete even after waiting "
"for 30 minutes. 30 minutes is too much a time for "
"current test workload")
# Validate IO
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.io_validation_complete = True
self.assertTrue(ret, "IO failed on some of the clients")
# List all files and dirs created
ret = list_all_files_and_dirs_mounts(self.mounts)
self.assertTrue(ret, "Failed to list all files and dirs")
def test_stat_prefetch(self):
# pylint: disable=ungrouped-imports
self.vips = (g.config['gluster']['cluster_config']['smb']['ctdb_vips'])
# Virtual Ip of first node to mount
self.vips_mnode = self.vips[0]['vip']
g.log.info("CTDB Virtual Ip %s", self.vips_mnode)
# run IOs
self.counter = 1
g.log.info("Starting IO on all mounts...")
self.all_mounts_procs = []
for mount_obj in self.mounts:
ret, _, _ = mount_volume(self.volname,
'cifs',
mount_obj.mountpoint,
self.vips_mnode,
mount_obj.client_system,
smbuser='******',
smbpasswd='foobar')
self.assertEqual(ret, 0, "Cifs Mount Failed")
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("python %s create_files -f 10000"
" --base-file-name ctdb-cifs "
" --fixed-file-size 10k %s/samba/" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Switch off and switch on stat-prefetch
options = {"stat-prefetch": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
if not ret:
raise ExecutionError("Failed to execute volume set"
"option command")
ret = get_volume_options(self.mnode, self.volname)
if ret['performance.stat-prefetch'] != "off":
raise ExecutionError("Failed to set stat-prefetch off")
options = {"stat-prefetch": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
if not ret:
raise ExecutionError("Failed to execute volume set"
"option command")
ret = get_volume_options(self.mnode, self.volname)
if ret['performance.stat-prefetch'] != "on":
raise ExecutionError("Failed to set stat-prefetch on")
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.io_validation_complete = True
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Creation of 10000 files Success")
g.log.info("test__samba_ctdb_cifs_io_rename PASSED")
def test_memory_leak_in_shd_with_cache_invalidation_on(self):
"""
Test case:
1. Create a volume, start it and mount it.
2. Set features.cache-invalidation to ON.
3. Start I/O from mount point.
4. Run gluster volume heal command in a loop
5. Check if there are any memory leaks and OOM killers on servers.
"""
# Start monitoring resource usage on servers and clients
monitor_proc_dict = self.start_memory_and_cpu_usage_logging(
self.test_id, count=10)
self.assertIsNotNone(
monitor_proc_dict, "Failed to start monitoring on servers and"
" clients")
# Set features.cache-invalidation to ON
ret = set_volume_options(self.mnode, self.volname,
{'features.cache-invalidation': 'on'})
self.assertTrue(ret, "Failed to set features.cache-invalidation to ON")
g.log.info("Successfully set features.cache-invalidation to ON")
# Start multiple I/O from mount points
self.list_of_io_processes = []
cmd = (
"cd {};for i in `seq 1 1000`;do echo 'abc' > myfile;done".format(
self.mounts[0].mountpoint))
ret = g.run_async(self.mounts[0].client_system, cmd)
self.list_of_io_processes = [ret]
self.is_io_running = True
# Run gluster volume heal command in a loop for 100 iterations
for iteration in range(0, 100):
g.log.info("Running gluster volume heal command for %d time",
iteration)
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, "Heal command triggered successfully")
g.log.info("Ran gluster volume heal command in a loop for "
"100 iterations.")
# Wait for I/O to complete and validate I/O on mount points
ret = validate_io_procs(self.list_of_io_processes, self.mounts[0])
self.assertTrue(ret, "I/O failed on mount point")
self.is_io_running = False
# Wait for monitoring processes to complete
ret = wait_for_logging_processes_to_stop(monitor_proc_dict,
cluster=True)
self.assertTrue(ret, "ERROR: Failed to stop monitoring processes")
# Check if there are any memory leaks and OOM killers
ret = self.check_for_memory_leaks_and_oom_kills_on_servers(
self.test_id)
self.assertFalse(ret,
"Memory leak and OOM kills check failed on servers")
def test_nfs_ganesha_HA_Basic_IO(self):
"""
Tests to create an HA cluster and run basic IO
"""
# Starting IO on the mounts
all_mounts_procs = []
count = 1
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("/usr/bin/env python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 10 "
"--max-num-of-dirs 5 "
"--num-of-files 5 %s" %
(self.script_upload_path, count, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
count = count + 10
# Validate IO
g.log.info("Validating IO's")
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Successfully validated all IO")
# Check nfs-ganesha status
g.log.info("Checking for Cluster Status after IO run")
ret = is_nfs_ganesha_cluster_in_healthy_state(self.mnode)
self.assertTrue(
ret, "Nfs Ganesha cluster is not healthy after "
"running IO")
# Running kernel untar now,single loop for the sanity test
g.log.info("Running kernel untars now")
for mount_obj in self.mounts:
cmd = ("cd %s ;mkdir $(hostname);cd $(hostname);"
"wget https://www.kernel.org/pub/linux/kernel/v2.6"
"/linux-2.6.1.tar.gz;"
"tar xvf linux-2.6.1.tar.gz" % mount_obj.mountpoint)
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertEqual(ret, 0, "Kernel untar failed!")
g.log.info("Kernel untar successful on %s" %
mount_obj.client_system)
# Check nfs-ganesha status
g.log.info("Checking for Cluster Status after kernel untar")
ret = is_nfs_ganesha_cluster_in_healthy_state(self.mnode)
self.assertTrue(
ret, "Nfs Ganesha cluster is not healthy after "
"kernel untar")
def test_copy_huge_file_with_remove_brick_in_progress(self):
"""
Test case:
1. Create a volume, start it and mount it.
2. Create files and dirs on the mount point.
3. Start remove-brick and copy huge file when remove-brick is
in progress.
4. Commit remove-brick and check checksum of orginal and copied file.
"""
# Create a directory with some files inside
cmd = ("cd %s; for i in {1..10}; do mkdir dir$i; for j in {1..5};"
" do dd if=/dev/urandom of=dir$i/file$j bs=1M count=1; done;"
" done" % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.first_client, cmd)
self.assertFalse(ret,
"Failed to create dirs and files.")
# Create a hug file under /mnt dir
ret, _, _ = g.run(self.first_client,
"fallocate -l 10G /mnt/huge_file.txt")
self.assertFalse(ret, "Failed to create hug file at /mnt")
# Copy a huge file when remove-brick is in progress
self.cp_running = False
cmd = ("sleep 60; cd %s;cp ../huge_file.txt ."
% self.mounts[0].mountpoint)
self.io_proc = [g.run_async(self.first_client, cmd)]
self.rename_running = True
# Start remove-brick on volume and wait for it to complete
ret = shrink_volume(self.mnode, self.volname, rebalance_timeout=1000)
self.assertTrue(ret, "Failed to remove-brick from volume")
g.log.info("Remove-brick rebalance successful")
# Validate if copy was successful or not
ret = validate_io_procs(self.io_proc, [self.mounts[0]])
self.assertTrue(ret, "dir rename failed on mount point")
self.cp_running = False
# Check checksum of orginal and copied file
original_file_checksum = get_md5sum(self.first_client,
"/mnt/huge_file.txt")
copied_file_checksum = get_md5sum(self.first_client,
"{}/huge_file.txt"
.format(self.mounts[0].mountpoint))
self.assertEqual(original_file_checksum.split(" ")[0],
copied_file_checksum.split(" ")[0],
"md5 checksum of original and copied file are"
" different")
g.log.info("md5 checksum of original and copied file are same.")
# Remove original huge file
ret, _, _ = g.run(self.first_client, "rm -rf /mnt/huge_file.txt")
self.assertFalse(ret, "Failed to remove huge_file from mount point")
def test_remove_brick_with_open_fd(self):
"""
Test case:
1. Create volume, start it and mount it.
2. Open file datafile on mount point and start copying /etc/passwd
line by line(Make sure that the copy is slow).
3. Start remove-brick of the subvol to which has datafile is hashed.
4. Once remove-brick is complete compare the checksum of /etc/passwd
and datafile.
"""
# Open file datafile on mount point and start copying /etc/passwd
# line by line
ret, out, _ = g.run(self.mounts[0].client_system,
"cat /etc/passwd | wc -l")
self.assertFalse(ret, "Failed to get number of lines of /etc/passwd")
cmd = ("cd {}; exec 30<> datafile ;for i in `seq 1 {}`; do "
"head -n $i /etc/passwd | tail -n 1 >> datafile; sleep 10; done"
.format(self.mounts[0].mountpoint, out.strip()))
self.list_of_io_processes = [
g.run_async(self.mounts[0].client_system, cmd)
]
self.is_copy_running = True
# Start remove-brick of the subvol to which has datafile is hashed
subvols = get_subvols(self.mnode, self.volname)['volume_subvols']
number = find_hashed_subvol(subvols, "/", 'datafile')[1]
ret = shrink_volume(self.mnode, self.volname, subvol_num=number)
self.assertTrue(ret, "Failed to remove-brick from volume")
g.log.info("Remove-brick rebalance successful")
# Validate if I/O was successful or not.
ret = validate_io_procs(self.list_of_io_processes, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
self.is_copy_running = False
# Compare md5checksum of /etc/passwd and datafile
md5_of_orginal_file = get_md5sum(self.mounts[0].client_system,
'/etc/passwd')
self.assertIsNotNone(md5_of_orginal_file,
'Unable to get md5 checksum of orignial file')
md5_of_copied_file = get_md5sum(
self.mounts[0].client_system,
'{}/datafile'.format(self.mounts[0].mountpoint))
self.assertIsNotNone(md5_of_copied_file,
'Unable to get md5 checksum of copied file')
self.assertEqual(
md5_of_orginal_file.split(" ")[0],
md5_of_copied_file.split(" ")[0],
"md5 checksum of original and copied file didn't"
" match")
g.log.info("md5 checksum of original and copied files are same")
def test_mem_leak_on_gluster_procs_with_brick_multiplex(self):
"""
Steps:
1) Enable cluster.brick-multiplex
2) Enable SSL on management layer
3) Start creating volumes
4) Mount a volume and starting I/O
5) Monitor the memory consumption by glusterd process
"""
# Enable cluster.brick-mulitplex
ret = enable_brick_mux(self.mnode)
self.assertTrue(ret, "Failed to enable brick-multiplex")
# Verify the operation
ret = is_brick_mux_enabled(self.mnode)
self.assertTrue(ret, "Brick mux enble op not successful")
# Create few volumes
self.volume['replica_count'] = 3
ret = bulk_volume_creation(self.mnode, 20, self.all_servers_info,
self.volume, is_force=True)
self.assertTrue(ret, "Failed to create bulk volume")
# Run IO
self._run_io()
# Start memory usage logging
monitor_proc_dict = self.start_memory_and_cpu_usage_logging(
self.test_id, count=60)
self.assertIsNotNone(monitor_proc_dict,
"Failed to start monitoring on servers and "
"clients")
ret = validate_io_procs(self.procs, self.mounts)
self.assertTrue(ret, "IO Failed")
# Wait for monitoring processes to complete
ret = wait_for_logging_processes_to_stop(monitor_proc_dict,
cluster=True)
self.assertTrue(ret, "ERROR: Failed to stop monitoring processes")
# Check if there are any memory leaks and OOM killers
self._verify_memory_leak()
g.log.info("No memory leaks/OOM kills found on serves and clients")
# Disable Brick multiplex
ret = disable_brick_mux(self.mnode)
self.assertTrue(ret, "Failed to brick multiplex")
def test_file_dir_create_ops_on_volume(self):
"""Test File Directory Creation on the volume.
"""
# Mount Volume
g.log.info("Starting to Mount Volume %s", self.volname)
ret = self.mount_volume(self.mounts)
self.assertTrue(ret, ("Failed to Mount Volume %s", self.volname))
g.log.info("Successful in Mounting Volume %s", self.volname)
# Start IO on all mounts.
all_mounts_procs = []
count = 1
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 10 "
"--max-num-of-dirs 5 "
"--num-of-files 5 %s" %
(self.script_upload_path, count, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
count = count + 10
# Validate IO
g.log.info("Validating IO's")
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Successfully validated all io's")
# Get stat of all the files/dirs created.
g.log.info("Get stat of all the files/dirs created.")
ret = get_mounts_stat(self.mounts)
self.assertTrue(ret, "Stat failed on some of the clients")
g.log.info("Successfully got stat of all files/dirs created")
# UnMount Volume
g.log.info("Starting to Unmount Volume %s", self.volname)
ret = self.unmount_volume(self.mounts)
self.assertTrue(ret, ("Failed to Unmount Volume %s" % self.volname))
g.log.info("Successfully Unmounted Volume %s", self.volname)
def start_and_wait_for_io_to_complete(self):
"""This module starts IO from clients and waits for io to complate.
Returns True, if io gets completed successfully. Otherwise, False
"""
# Start IO on mounts
g.log.info("Starting IO on all mounts...")
self.all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 15 "
"--max-num-of-dirs 5 "
"--num-of-files 10 %s" % (self.script_upload_path,
self.counter,
mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.counter = self.counter + 10
self.io_validation_complete = False
# Adding a delay of 15 seconds before test method starts. This
# is to ensure IO's are in progress and giving some time to fill data
time.sleep(15)
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.io_validation_complete = True
if not ret:
g.log.error("IO failed on some of the clients")
return False
g.log.info("IO is successful on all mounts")
# List all files and dirs created
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
if not ret:
g.log.error("Failed to list all files and dirs")
return False
g.log.info("Listing all files and directories is successful")
return True
def setUp(self):
"""setUp writes data from all mounts and selects subdirectory
required for the test and unmount the existing mounts.
"""
NfsGaneshaIOBaseClass.setUp.im_func(self)
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.io_validation_complete = True
if not ret:
raise ExecutionError("IO failed on some of the clients")
g.log.info("IO is successful on all mounts")
# List all files and dirs created
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
if not ret:
raise ExecutionError("Failed to list all files and dirs")
g.log.info("Listing all files and directories is successful")
mountpoint = self.mounts[0].mountpoint
client = self.mounts[0].client_system
# Select the subdirectory required for the test.
cmd = "find %s -type d -links 2 | grep -ve '.trashcan'" % mountpoint
ret, out, err = g.run(client, cmd)
if ret != 0:
raise ExecutionError("Failed to list the deep level directories")
self.subdir_path = out.split("\n")[0]
_rc = True
for mount_obj in self.mounts:
ret = mount_obj.unmount()
if not ret:
g.log.error("Unable to unmount volume '%s:%s' on '%s:%s'",
mount_obj.server_system, mount_obj.volname,
mount_obj.client_system, mount_obj.mountpoint)
_rc = False
if not _rc:
raise ExecutionError("Unmount of all mounts are not "
"successful")
def _perform_io_and_validate(self):
""" Performs IO on the mount points and validates it"""
all_mounts_procs, count = [], 1
for mount_obj in self.mounts:
cmd = ("/usr/bin/env python %s create_deep_dirs_with_files "
"--dirname-start-num %d --dir-depth 2 "
"--dir-length 3 --max-num-of-dirs 3 "
"--num-of-files 2 %s" %
(self.script_upload_path, count, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
count = count + 10
# Validating IO's on mount point and waiting to complete
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Successfully validated IO's")
def _create_src(self, m_point):
"""
Create the source directory and files under the
source directory.
"""
# Create source dir
ret = mkdir(self.mounts[0].client_system, "{}/src_dir".format(m_point))
self.assertTrue(ret, "mkdir of src_dir failed")
# Create files inside source dir
cmd = ("/usr/bin/env python %s create_files "
"-f 100 %s/src_dir/" % (self.script_upload_path, m_point))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
g.log.info("IO on %s:%s is started successfully",
self.mounts[0].client_system, m_point)
# Validate IO
self.assertTrue(validate_io_procs([proc], self.mounts[0]),
"IO failed on some of the clients")
def test_nfs_ganesha_multiple_refresh_configs(self):
"""
Tests script to check nfs-ganehsa volume gets exported and IOs
are running after running multiple refresh configs.
"""
self.acl_check_flag = False
for i in range(6):
# Enabling/Disabling ACLs to modify the export configuration
# before running refresh config
if i % 2 == 0:
ret = disable_acl(self.mnode, self. volname)
self.assertTrue(ret, ("Failed to disable acl on %s"
% self.volname))
self.acl_check_flag = False
else:
ret = enable_acl(self.mnode, self. volname)
self.assertTrue(ret, ("Failed to enable acl on %s"
% self.volname))
self.acl_check_flag = True
ret = run_refresh_config(self.mnode, self. volname)
self.assertTrue(ret, ("Failed to run refresh config"
"for volume %s" % self.volname))
time.sleep(2)
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.io_validation_complete = True
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("IO is successful on all mounts")
# List all files and dirs created
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
self.assertTrue(ret, "Failed to list all files and dirs")
g.log.info("Listing all files and directories is successful")
def test_mem_leak_on_gluster_procs_with_management_encrpytion(self):
"""
Steps:
1) Enable management encryption on the cluster.
2) Create a 2X3 volume.
3) Mount the volume using FUSE on a client node.
4) Start doing IO on the mount (ran IO till the volume is ~88% full)
5) Simultaneously start collecting the memory usage for
'glusterfsd' process.
6) Issue the command "# gluster v heal <volname> info" continuously
in a loop.
"""
# Run IO
self._run_io()
# Start monitoring resource usage on servers and clients
# default interval = 60 sec, count = 780 (60 *12) => for 12 hrs
monitor_proc_dict = self.start_memory_and_cpu_usage_logging(
self.test_id, count=780)
self.assertIsNotNone(monitor_proc_dict,
"Failed to start monitoring on servers and "
"clients")
ret = validate_io_procs(self.procs, self.mounts)
self.assertTrue(ret, "IO Failed")
self._perform_gluster_v_heal_for_12_hrs()
# Wait for monitoring processes to complete
ret = wait_for_logging_processes_to_stop(monitor_proc_dict,
cluster=True)
self.assertTrue(ret, "ERROR: Failed to stop monitoring processes")
# Check if there are any memory leaks and OOM killers
self._verify_memory_leak()
g.log.info("No memory leaks/OOM kills found on serves and clients")
def test_profile_operations_with_one_node_down(self):
# pylint: disable=too-many-statements
"""
Test Case:
1) Create a volume and start it.
2) Mount volume on client and start IO.
3) Start profile info on the volume.
4) Stop glusterd on one node.
5) Run profile info with different parameters
and see if all bricks are present or not.
6) Stop profile on the volume.
"""
# Start IO on mount points.
g.log.info("Starting IO on all mounts...")
self.all_mounts_procs = []
counter = 1
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("/usr/bin/env python %s create_deep_dirs_with_files "
"--dir-depth 4 "
"--dirname-start-num %d "
"--dir-length 6 "
"--max-num-of-dirs 3 "
"--num-of-files 5 %s" % (
self.script_upload_path,
counter, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
counter += 1
# Start profile on volume.
ret, _, _ = profile_start(self.mnode, self.volname)
self.assertEqual(ret, 0, "Failed to start profile on volume: %s"
% self.volname)
g.log.info("Successfully started profile on volume: %s",
self.volname)
# Fetching a random server from list.
self.random_server = randint(1, len(self.servers)-1)
# Stopping glusterd on one node.
ret = stop_glusterd(self.servers[self.random_server])
self.assertTrue(ret, "Failed to stop glusterd on one node.")
g.log.info("Successfully stopped glusterd on one node.")
ret = wait_for_glusterd_to_start(self.servers[self.random_server])
self.assertFalse(ret, "glusterd is still running on %s"
% self.servers[self.random_server])
g.log.info("Glusterd stop on the nodes : %s "
"succeeded", self.servers[self.random_server])
# Getting and checking output of profile info.
ret, out, _ = profile_info(self.mnode, self.volname)
self.assertEqual(ret, 0, "Failed to run profile info on volume: %s"
% self.volname)
g.log.info("Successfully executed profile info on volume: %s",
self.volname)
# Checking if all bricks are present in profile info.
brick_list = get_online_bricks_list(self.mnode, self.volname)
for brick in brick_list:
self.assertTrue(brick in out,
"Brick %s not a part of profile info output."
% brick)
g.log.info("Brick %s showing in profile info output.",
brick)
# Running profile info with different profile options.
profile_options = ['peek', 'incremental', 'clear', 'incremental peek',
'cumulative']
for option in profile_options:
# Getting and checking output of profile info.
ret, out, _ = profile_info(self.mnode, self.volname,
options=option)
self.assertEqual(ret, 0,
"Failed to run profile info %s on volume: %s"
% (option, self.volname))
g.log.info("Successfully executed profile info %s on volume: %s",
option, self.volname)
# Checking if all bricks are present in profile info peek.
for brick in brick_list:
self.assertTrue(brick in out,
"Brick %s not a part of profile"
" info %s output."
% (brick, option))
g.log.info("Brick %s showing in profile info %s output.",
brick, option)
# Starting glusterd on node where stopped.
ret = start_glusterd(self.servers[self.random_server])
self.assertTrue(ret, "Failed to start glusterd.")
g.log.info("Successfully started glusterd.")
# Checking if peer is connected
ret = wait_for_peers_to_connect(self.mnode, self.servers)
self.assertTrue(ret, "Peers are not in connected state.")
g.log.info("Peers are in connected state.")
# Stop profile on volume.
ret, _, _ = profile_stop(self.mnode, self.volname)
self.assertEqual(ret, 0, "Failed to stop profile on volume: %s"
% self.volname)
g.log.info("Successfully stopped profile on volume: %s", self.volname)
# Validate IO
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients"
)
g.log.info("IO validation complete.")
def test_volume_status_inode_while_io_in_progress(self):
'''
Create any type of volume then mount the volume, once
volume mounted successfully on client, start running IOs on
mount point then run the "gluster volume status volname inode"
command on all clusters randomly.
"gluster volume status volname inode" command should not get
hang while IOs in progress.
Then check that IOs completed successfully or not on mount point.
Check that files in mount point listing properly or not.
'''
# Mounting a volume
ret = self.mount_volume(self.mounts)
self.assertTrue(ret, "Volume mount failed for %s" % self.volname)
g.log.info("Volume mounted successfully : %s", self.volname)
# After Mounting immediately writing IO's are failing some times,
# that's why keeping sleep for 10 secs
sleep(10)
# run IOs
g.log.info("Starting IO on all mounts...")
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = (
"python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 15 "
"--max-num-of-dirs 5 "
"--num-of-files 25 %s" %
(self.script_upload_path, self.counter, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.counter = self.counter + 10
self.io_validation_complete = False
# performing "gluster volume status volname inode" command on
# all cluster servers randomly while io is in progress,
# this command should not get hang while io is in progress
# pylint: disable=unused-variable
for i in range(20):
ret, _, _ = g.run(
random.choice(self.servers),
"gluster --timeout=12000 volume status %s "
"inode" % self.volname)
self.assertEqual(ret, 0, ("Volume status 'inode' failed on "
"volume %s" % self.volname))
g.log.info(
"Successful in logging volume status"
"'inode' of volume %s", self.volname)
# Validate IO
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.io_validation_complete = True
self.assertTrue(ret, "IO failed on some of the clients")
# List all files and dirs created
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
self.assertTrue(ret, "Failed to list all files and dirs")
g.log.info("Listing all files and directories is successful")
def test_data_self_heal_algorithm_full_default(self):
"""
Test Volume Option - 'cluster.data-self-heal-algorithm' : 'full'
Description:
- set the volume option "data-self-heal-algorithm" to value "full"
- create IO
- bring down all bricks processes from selected set
- modify the data
- calculate arequal
- bring bricks online
- start healing
- calculate arequal and compare with arequal before bringing bricks
offline and after bringing bricks online
"""
# pylint: disable=too-many-locals,too-many-statements
# Setting options
g.log.info('Setting options "data-self-heal-algorithm": "full"...')
options = {"data-self-heal-algorithm": "full"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options')
g.log.info("Option 'data-self-heal-algorithm' is set to 'full' "
"successfully")
# Creating files on client side
all_mounts_procs = []
g.log.info("Generating data for %s:%s", self.mounts[0].client_system,
self.mounts[0].mountpoint)
# Creating files
command = "/usr/bin/env python %s create_files -f 100 %s" % (
self.script_upload_path, self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# Select bricks to bring offline
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = list(
filter(None, (bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks'])))
# Bring brick offline
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# Modify the data
all_mounts_procs = []
g.log.info("Modifying data for %s:%s", self.mounts[0].client_system,
self.mounts[0].mountpoint)
command = ("/usr/bin/env python %s create_files -f 100 "
"--fixed-file-size 1M %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# Get arequal before getting bricks online
g.log.info('Getting arequal before getting bricks online...')
ret, result_before_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks online '
'is successful')
# Bring brick online
g.log.info('Bringing bricks %s online...', bricks_to_bring_offline)
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s online' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s online is successful',
bricks_to_bring_offline)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Wait for self-heal-daemons to be online
g.log.info("Waiting for self-heal-daemons to be online")
ret = is_shd_daemonized(self.all_servers)
self.assertTrue(ret, "Either No self heal daemon process found")
g.log.info("All self-heal-daemons are online")
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Check for split-brain
ret = is_volume_in_split_brain(self.mnode, self.volname)
self.assertFalse(ret, 'Volume is in split-brain state')
g.log.info('Volume is not in split-brain state')
# Get arequal after getting bricks online
g.log.info('Getting arequal after getting bricks online...')
ret, result_after_online = collect_mounts_arequal(self.mounts)
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal after getting bricks online '
'is successful')
# Checking arequals before bringing bricks online
# and after bringing bricks online
self.assertItemsEqual(result_before_online, result_after_online,
'Checksums are not equal')
g.log.info('Checksums before bringing bricks online '
'and after bringing bricks online are equal')
def test_write_io_mount_point_resumed_quorum_restored_x3(self):
"""
- set cluster.quorum-type to auto
- start I/O from the mount point
- Do IO and check on subvols with two nodes to reboot
(do for each subvol)
- get files to delete/create for nodes to be offline
- delete files from mountpoint
- reboot nodes
- creating files on nodes while rebooting
- validate for rofs
- wait for volume processes to be online
- creating files on nodes after rebooting
- validate IO
- Do IO and check on subvols without nodes to reboot
(do for each subvol)
- get files to delete/create for nodes to be online
- delete files from mountpoint
- reboot nodes
- creating files on online nodes while rebooting other nodes
- validate IO
- Do IO and check and reboot two nodes on all subvols
- get files to delete/create for nodes to be offline
- delete files from mountpoint
- reboot nodes
- creating files on nodes while rebooting
- validate for rofs
- wait for volume processes to be online
- creating files on nodes after rebooting
- validate IO
"""
# pylint: disable=too-many-locals,too-many-statements,too-many-branches
# set cluster.quorum-type to auto
options = {"cluster.quorum-type": "auto"}
g.log.info("setting cluster.quorum-type to auto on volume %s",
self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set volume option %s for"
"volume %s" % (options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# Creating files on client side
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = "/usr/bin/env python %s create_files -f 30 %s" % (
self.script_upload_path, mount_obj.mountpoint)
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# Validate IO
self.io_validation_complete = False
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# Do IO and check on subvols with nodes to reboot
subvols_dict = get_subvols(self.mnode, self.volname)
for subvol in subvols_dict['volume_subvols']:
# define nodes to reboot
brick_list = subvol[0:2]
nodes_to_reboot = []
for brick in brick_list:
node, brick_path = brick.split(':')
nodes_to_reboot.append(node)
# get files to delete/create for nodes to be offline
node, brick_path = brick_list[0].split(':')
ret, brick_file_list, _ = g.run(node, 'ls %s' % brick_path)
self.assertFalse(ret, 'Failed to ls files on %s' % node)
file_list = brick_file_list.splitlines()
# delete files from mountpoint
for mount_obj in self.mounts:
g.log.info("Deleting data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ('cd %s/ ; rm -rf %s' %
(mount_obj.mountpoint, ' '.join(file_list)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(
ret, 'Failed to rm file on %s' % mount_obj.client_system)
g.log.info('Files %s are deleted', file_list)
# reboot nodes on subvol and wait while rebooting
g.log.info("Rebooting the nodes %s", nodes_to_reboot)
ret = reboot_nodes(nodes_to_reboot)
self.assertTrue(ret,
'Failed to reboot nodes %s ' % nodes_to_reboot)
# Creating files on nodes while rebooting
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = ("cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list)))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# Validate IO
self.io_validation_complete = False
g.log.info("Validating if IO failed with read-only filesystem")
ret = is_io_procs_fail_with_rofs(self, self.all_mounts_procs,
self.mounts)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on read-only filesystem"))
self.io_validation_complete = True
g.log.info("EXPECTED: "
"Read-only file system in IO while creating file")
# check if nodes are online
counter = 0
timeout = 300
_rc = False
while counter < timeout:
ret, reboot_results = are_nodes_online(nodes_to_reboot)
if not ret:
g.log.info("Nodes are offline, Retry after 5 seconds ... ")
time.sleep(5)
counter = counter + 5
else:
_rc = True
break
if not _rc:
for node in reboot_results:
if reboot_results[node]:
g.log.info("Node %s is online", node)
else:
g.log.error(
"Node %s is offline even after "
"%d minutes", node, timeout / 60.0)
else:
g.log.info("All nodes %s are up and running", nodes_to_reboot)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(
self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Creating files on nodes after rebooting
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = ("cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list)))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# Validate IO
self.io_validation_complete = False
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# Do IO and check on subvols without nodes to reboot
subvols_dict = get_subvols(self.mnode, self.volname)
for subvol in subvols_dict['volume_subvols']:
# define nodes to reboot
brick_list = subvol[0:2]
nodes_to_reboot = []
for brick in brick_list:
node, brick_path = brick.split(':')
nodes_to_reboot.append(node)
# get files to delete/create for nodes to be online
new_subvols_dict = get_subvols(self.mnode, self.volname)
subvol_to_operate = new_subvols_dict['volume_subvols']
subvol_to_operate.remove(subvol)
brick_list_subvol_online = subvol_to_operate[0]
node, brick_path_vol_online = \
brick_list_subvol_online[0].split(':')
ret, brick_file_list, _ = g.run(node,
'ls %s' % brick_path_vol_online)
self.assertFalse(ret, 'Failed to ls files on %s' % node)
file_list = brick_file_list.splitlines()
# delete files from mountpoint
for mount_obj in self.mounts:
g.log.info("Deleting data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ('cd %s/ ; rm -rf %s' %
(mount_obj.mountpoint, ' '.join(file_list)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(
ret, 'Failed to rm file on %s' % mount_obj.client_system)
g.log.info('Files %s are deleted', file_list)
# reboot nodes on subvol and wait while rebooting
g.log.info("Rebooting the nodes %s", nodes_to_reboot)
ret = reboot_nodes(nodes_to_reboot)
self.assertTrue(ret,
'Failed to reboot nodes %s ' % nodes_to_reboot)
# Creating files on nodes while rebooting
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = ("cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list)))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# Validate IO
self.io_validation_complete = False
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# check if nodes are online
counter = 0
timeout = 300
_rc = False
while counter < timeout:
ret, reboot_results = are_nodes_online(nodes_to_reboot)
if not ret:
g.log.info("Nodes are offline, Retry after 5 seconds ... ")
time.sleep(5)
counter = counter + 5
else:
_rc = True
break
if not _rc:
for node in reboot_results:
if reboot_results[node]:
g.log.info("Node %s is online", node)
else:
g.log.error(
"Node %s is offline even after "
"%d minutes", node, timeout / 60.0)
else:
g.log.info("All nodes %s are up and running", nodes_to_reboot)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(
self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Do IO and check and reboot nodes on all subvols
subvols_dict = get_subvols(self.mnode, self.volname)
nodes_to_reboot = []
file_list_for_all_subvols = []
for subvol in subvols_dict['volume_subvols']:
# define nodes to reboot
brick_list = subvol[0:2]
for brick in brick_list:
node, brick_path = brick.split(':')
nodes_to_reboot.append(node)
# get files to delete/create for nodes to be offline
node, brick_path = brick_list[0].split(':')
ret, brick_file_list, _ = g.run(node, 'ls %s' % brick_path)
self.assertFalse(ret, 'Failed to ls files on %s' % node)
file_list = brick_file_list.splitlines()
file_list_for_all_subvols.append(file_list)
# delete files from mountpoint
for mount_obj in self.mounts:
g.log.info("Deleting data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ('cd %s/ ; rm -rf %s' %
(mount_obj.mountpoint, ' '.join(file_list)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, 'Failed to rm file on %s' % node)
g.log.info('Files %s are deleted', file_list)
# reboot nodes on subvol and wait while rebooting
g.log.info("Rebooting the nodes %s", nodes_to_reboot)
ret = reboot_nodes(nodes_to_reboot)
self.assertTrue(ret, 'Failed to reboot nodes %s ' % nodes_to_reboot)
# Creating files on nodes while rebooting
all_mounts_procs, all_mounts_procs_1, all_mounts_procs_2 = [], [], []
# Create files for 1-st subvol and get all_mounts_procs_1
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = (
"cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[0])))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs_1.append(proc)
all_mounts_procs.append(all_mounts_procs_1)
# Create files for 2-st subvol and get all_mounts_procs_2
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = (
"cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[1])))
proc2 = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs_2.append(proc2)
all_mounts_procs.append(all_mounts_procs_2)
for mounts_procs in all_mounts_procs:
# Validate IO
self.io_validation_complete = False
g.log.info("Validating if IO failed with read-only filesystem")
ret = is_io_procs_fail_with_rofs(self, mounts_procs, self.mounts)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on read-only filesystem"))
self.io_validation_complete = True
g.log.info("EXPECTED: "
"Read-only file system in IO while creating file")
# check if nodes are online
counter = 0
timeout = 300
_rc = False
while counter < timeout:
ret, reboot_results = are_nodes_online(nodes_to_reboot)
if not ret:
g.log.info("Nodes are offline, Retry after 5 seconds ... ")
time.sleep(5)
counter = counter + 5
else:
_rc = True
break
if not _rc:
for node in reboot_results:
if reboot_results[node]:
g.log.info("Node %s is online", node)
else:
g.log.error("Node %s is offline even after "
"%d minutes", node, timeout / 60.0)
else:
g.log.info("All nodes %s are up and running", nodes_to_reboot)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Creating files on nodes after rebooting
all_mounts_procs, all_mounts_procs_1, all_mounts_procs_2 = [], [], []
# Create files for 1-st subvol and get all_mounts_procs_1
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = (
"cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[0])))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs_1.append(proc)
all_mounts_procs.append(all_mounts_procs_1)
# Create files for 2-st subvol and get all_mounts_procs_2
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = (
"cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[1])))
proc2 = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs_2.append(proc2)
all_mounts_procs.append(all_mounts_procs_2)
for mounts_procs in all_mounts_procs:
# Validate IO
self.io_validation_complete = False
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
