def tearDown(self):
# Disable brick multiplexing
g.log.info("Checking for brick multiplexing status...")
if is_brick_mux_enabled(self.mnode):
g.log.info("Disabling brick multiplexing...")
if not disable_brick_mux(self.mnode):
raise ExecutionError("Failed to disable brick multiplexing")
g.log.info("Disabled brick multiplexing successfully")
# Calling GlusterBaseClass teardown
self.get_super_method(self, 'tearDown')()
def test_mem_leak_on_gluster_procs_with_brick_multiplex(self):
"""
Steps:
1) Enable cluster.brick-multiplex
2) Enable SSL on management layer
3) Start creating volumes
4) Mount a volume and starting I/O
5) Monitor the memory consumption by glusterd process
"""
# Enable cluster.brick-mulitplex
ret = enable_brick_mux(self.mnode)
self.assertTrue(ret, "Failed to enable brick-multiplex")
# Verify the operation
ret = is_brick_mux_enabled(self.mnode)
self.assertTrue(ret, "Brick mux enble op not successful")
# Create few volumes
self.volume['replica_count'] = 3
ret = bulk_volume_creation(self.mnode, 20, self.all_servers_info,
self.volume, is_force=True)
self.assertTrue(ret, "Failed to create bulk volume")
# Run IO
self._run_io()
# Start memory usage logging
monitor_proc_dict = self.start_memory_and_cpu_usage_logging(
self.test_id, count=60)
self.assertIsNotNone(monitor_proc_dict,
"Failed to start monitoring on servers and "
"clients")
ret = validate_io_procs(self.procs, self.mounts)
self.assertTrue(ret, "IO Failed")
# Wait for monitoring processes to complete
ret = wait_for_logging_processes_to_stop(monitor_proc_dict,
cluster=True)
self.assertTrue(ret, "ERROR: Failed to stop monitoring processes")
# Check if there are any memory leaks and OOM killers
self._verify_memory_leak()
g.log.info("No memory leaks/OOM kills found on serves and clients")
# Disable Brick multiplex
ret = disable_brick_mux(self.mnode)
self.assertTrue(ret, "Failed to brick multiplex")
def tearDown(self):
# Disable brick_mux
if is_brick_mux_enabled(self.mnode):
ret = disable_brick_mux(self.mnode)
self.assertTrue(ret, "Failed to brick multiplex")
g.log.info("Disable brick multiplex")
# Unmount and cleanup original volume
ret = self.unmount_volume_and_cleanup_volume(mounts=[self.mounts[0]])
if not ret:
raise ExecutionError("Failed to umount the vol & cleanup Volume")
g.log.info("Successful in umounting the volume and Cleanup")
# Calling GlusterBaseClass tearDown
self.get_super_method(self, 'tearDown')()
def tearDown(self):
# Disable brick multiplexing
g.log.info("Checking for brick multiplexing status...")
if is_brick_mux_enabled(self.mnode):
g.log.info("Disabling brick multiplexing...")
if not disable_brick_mux(self.mnode):
raise ExecutionError("Failed to disable brick multiplexing")
g.log.info("Disabled brick multiplexing successfully")
ret, _, _ = reset_volume_option(self.mnode, "all", "all")
if ret:
raise ExecutionError("Unable to reset all volume options")
g.log.info("Successfully reset all the volume options.")
# Calling GlusterBaseClass teardown
self.get_super_method(self, 'tearDown')()
def tearDown(self):
# Disable brickmux on cluster
if not disable_brick_mux(self.mnode):
raise ExecutionError("Failed to disable brickmux on cluster")
# If I/O processes are running wait from them to complete
if self.is_io_running:
if not wait_for_io_to_complete(self.list_of_io_processes,
self.mounts):
raise ExecutionError("Failed to wait for I/O to complete")
# Unmounting and cleaning volume
ret = self.unmount_volume_and_cleanup_volume([self.mounts[0]])
if not ret:
raise ExecutionError("Unable to delete volume %s" % self.volname)
self.get_super_method(self, 'tearDown')()
def tearDown(self):
# Stopping all volumes
g.log.info("Starting to Cleanup all Volumes")
volume_list = get_volume_list(self.mnode)
for volume in volume_list:
ret = cleanup_volume(self.mnode, volume)
if not ret:
raise ExecutionError("Failed to cleanup Volume %s" % volume)
g.log.info("Volume: %s cleanup is done", volume)
g.log.info("Successfully Cleanedup all Volumes")
# Disable brick multiplexing
g.log.info("Checking for brick multiplexing status...")
if is_brick_mux_enabled(self.mnode):
g.log.info("Disabling brick multiplexing...")
if not disable_brick_mux(self.mnode):
raise ExecutionError("Failed to disable brick multiplexing")
g.log.info("Disabled brick multiplexing successfully")
# Calling GlusterBaseClass teardown
self.get_super_method(self, 'tearDown')()
def test_enabling_brick_mux(self):
"""
Test case:
- check if brick multiplex is disable by default
- check for warning message triggering by setting brick-multiplex and
choosing 'n' in y/n
- check if brick multiplex is disabled after triggering warning message
- check brick multiplex for all possible statuses
(positive and negative)
- check for brick multiplex status in /var/lib/glusterd/options file
"""
# Check if brickmux is disabled by default
g.log.info('Checking if brick multiplex operation is disabled...')
self.assertFalse(is_brick_mux_enabled(self.mnode),
"Brick multiplex is not disabled by default")
# Check for warning message while changing status
warning_message = ("Brick-multiplexing is supported only for "
"OCS converged or independent mode. Also it is "
"advised to make sure that either all volumes are "
"in stopped state or no bricks are running before "
"this option is modified."
"Do you still want to continue? (y/n)")
g.log.info('Triggering warning message...')
cmd = "gluster v set all cluster.brick-multiplex enable"
_, out, _ = g.run(self.mnode, cmd)
g.log.info('Checking for warning message in output...')
if "volume set: success" not in out:
self.assertIn(warning_message, out,
'There is no warning message in '
'output or message is incorrect.')
g.log.info('Warning message is correct.')
else:
g.log.info('Skipped warning message check.')
# If brick-mux is enabled then disabling it.
if is_brick_mux_enabled(self.mnode):
if not disable_brick_mux(self.mnode):
g.log.info("Disabling brick multiplexing as it"
" was enabled due to no warning message.")
# Check if brickmux is still disabled
g.log.info('Checking if brick multiplex is still disabled')
self.assertFalse(is_brick_mux_enabled(self.mnode),
"Brick multiplex operation is not disabled")
# Enable brick multiplex with all possible statuses
statuses = ['on', 'enable', '1', 'true',
'off', 'disable', '0', 'false']
for status in statuses:
g.log.info('Enabling brick multiplex with %s status...',
status)
cmd = ("yes | gluster v set all cluster.brick-multiplex %s"
% status)
_, out, _ = g.run(self.mnode, cmd)
self.assertIn('success', out,
'Failed on enabling brick multiplexing')
# Check if brick multiplex status is correct
g.log.info('Checking if brick multiplexing status is correct...')
gluster_status = get_brick_mux_status(self.mnode)
self.assertEqual(status, gluster_status,
"Brick multiplex status is not correct")
g.log.info('Brick multiplex status "%s" is correct',
status)
# Check for brick multiplexing status in file 'options'
g.log.info("Checking for brick multiplexing status '%s' in file "
"'/var/lib/glusterd/options'...", status)
search_pattern = 'cluster.brick-multiplex=%s' % status
self.assertTrue(search_pattern_in_file(self.mnode, search_pattern,
'/var/lib/glusterd/options',
'', ''))
g.log.info("Brick multiplexing status '%s' in file "
"'/var/lib/glusterd/options' is correct", status)
# Check brick multiplex with incorrect status
g.log.info('Checking brick multiplex with incorrect status...')
cmd = "yes | gluster v set all cluster.brick-multiplex incorrect"
ret, _, _ = g.run(self.mnode, cmd)
self.assertEqual(ret, 1, 'Incorrect status has passed')
def test_enable_brick_mux_with_max_bricks_per_process(self):
"""
Test Case:
- Create a gluster cluster
- With brick mux set to disable:
1.Set cluster.max-bricks-per-process to int and check
error message(Must fail)
2.Set cluster.max-bricks-per-process to string(Must fail)
- With brick mux set to enable:
1.Set cluster.max-bricks-per-process to string(Must fail)
2.Set cluster.max-bricks-per-process to 0
3.Set cluster.max-bricks-per-process to 1 and check
error message.(Must fail)
4.Set cluster.max-bricks-per-process to int value > 1.
"""
# Disabling cluster.brick-multiplex if not.
if is_brick_mux_enabled(self.mnode):
ret = disable_brick_mux(self.mnode)
self.assertTrue(ret, "Unable to disable brickmux")
g.log.info("Brick mux is disabled")
# Set cluster.max-bricks-per-process to int and check
# error message(Must fail)
cmd = "gluster v set all cluster.max-bricks-per-process 10"
ret, _, err = g.run(self.mnode, cmd)
self.assertEqual(
ret, 1, 'Able to set max-bricks-per-process'
'without enabling brick mux')
self.assertIn(
"volume set: failed: Brick-multiplexing is not enabled. "
"Please enable brick multiplexing before"
" trying to set this option.", err,
"Error message not proper on trying to "
"set max-bricks-per-process without brickmux")
# Set cluster.max-bricks-per-process to string(Must fail)
self.set_max_brick_process_to_string()
# Enable cluster.brick-multiplex.
ret = enable_brick_mux(self.mnode)
self.assertTrue(ret, "Unable to enable cluster.brick-multiplex")
g.log.info("Brick mux is enabled")
# Set cluster.max-bricks-per-process to string(Must fail)
self.set_max_brick_process_to_string()
# Set cluster.max-bricks-per-process to 0.
ret = set_volume_options(self.mnode, 'all',
{'cluster.max-bricks-per-process': '0'})
self.assertTrue(ret, "Unable to set "
"cluster.max-bricks-per-process to 0")
g.log.info("Successfully set cluster.max-bricks-per-process to 0")
# Set cluster.max-bricks-per-process to 1 and check
# error message.(Must fail)
cmd = "gluster v set all cluster.max-bricks-per-process 1"
ret, _, err = g.run(self.mnode, cmd)
self.assertEqual(
ret, 1, 'Able to set max-bricks-per-process'
'with enabling brick mux')
self.assertIn(
"volume set: failed: Brick-multiplexing is enabled."
" Please set this option to a value other than 1 to"
" make use of the brick-multiplexing feature.", err,
"Error message not proper on trying to set max-bricks-per-process"
" with brickmux")
# Set cluster.max-bricks-per-process to int value > 1
key = 'cluster.max-bricks-per-process'
temp_val = self.get_random_string(string.digits)
value = "{}".format(temp_val)
ret = set_volume_options(self.mnode, 'all', {key: value})
self.assertTrue(
ret, "Unexpected: Erroneous value {}, to option "
"{} should not result in failure".format(value, key))
g.log.info("Value %s, set to option %s", value, key)
