def test_check_bmux_enabled(self):
"""Check if the brickmultiplexing is enalbed"""
bmux_status = is_brick_mux_enabled('auto_get_gluster_endpoint')
# Validate the result
err_msg = ("Brick multiplex is not enabled")
self.assertTrue(bmux_status, err_msg)
def test_check_bmux_enabled(self):
"""Check if the brickmultiplexing is enalbed"""
bmux_status = is_brick_mux_enabled('auto_get_gluster_endpoint')
# Validate the result
err_msg = ("Brick multiplex is not enabled")
self.assertTrue(bmux_status, err_msg)
def tearDown(self):
# Disable brick multiplexing
g.log.info("Checking for brick multiplexing status...")
if is_brick_mux_enabled(self.mnode):
g.log.info("Disabling brick multiplexing...")
if not disable_brick_mux(self.mnode):
raise ExecutionError("Failed to disable brick multiplexing")
g.log.info("Disabled brick multiplexing successfully")
# Calling GlusterBaseClass teardown
self.get_super_method(self, 'tearDown')()
def setUp(self):
# Calling GlusterBaseClass setUp
self.get_super_method(self, 'setUp')()
# Enable brick multiplexing
g.log.info("Checking for brick multiplexing status...")
if not is_brick_mux_enabled(self.mnode):
g.log.info("Enabling brick multiplexing...")
if not enable_brick_mux(self.mnode):
raise ExecutionError("Failed to enable brick multiplexing")
g.log.info("Enabled brick multiplexing successfully")
def test_mem_leak_on_gluster_procs_with_brick_multiplex(self):
"""
Steps:
1) Enable cluster.brick-multiplex
2) Enable SSL on management layer
3) Start creating volumes
4) Mount a volume and starting I/O
5) Monitor the memory consumption by glusterd process
"""
# Enable cluster.brick-mulitplex
ret = enable_brick_mux(self.mnode)
self.assertTrue(ret, "Failed to enable brick-multiplex")
# Verify the operation
ret = is_brick_mux_enabled(self.mnode)
self.assertTrue(ret, "Brick mux enble op not successful")
# Create few volumes
self.volume['replica_count'] = 3
ret = bulk_volume_creation(self.mnode, 20, self.all_servers_info,
self.volume, is_force=True)
self.assertTrue(ret, "Failed to create bulk volume")
# Run IO
self._run_io()
# Start memory usage logging
monitor_proc_dict = self.start_memory_and_cpu_usage_logging(
self.test_id, count=60)
self.assertIsNotNone(monitor_proc_dict,
"Failed to start monitoring on servers and "
"clients")
ret = validate_io_procs(self.procs, self.mounts)
self.assertTrue(ret, "IO Failed")
# Wait for monitoring processes to complete
ret = wait_for_logging_processes_to_stop(monitor_proc_dict,
cluster=True)
self.assertTrue(ret, "ERROR: Failed to stop monitoring processes")
# Check if there are any memory leaks and OOM killers
self._verify_memory_leak()
g.log.info("No memory leaks/OOM kills found on serves and clients")
# Disable Brick multiplex
ret = disable_brick_mux(self.mnode)
self.assertTrue(ret, "Failed to brick multiplex")
def tearDown(self):
# Disable brick_mux
if is_brick_mux_enabled(self.mnode):
ret = disable_brick_mux(self.mnode)
self.assertTrue(ret, "Failed to brick multiplex")
g.log.info("Disable brick multiplex")
# Unmount and cleanup original volume
ret = self.unmount_volume_and_cleanup_volume(mounts=[self.mounts[0]])
if not ret:
raise ExecutionError("Failed to umount the vol & cleanup Volume")
g.log.info("Successful in umounting the volume and Cleanup")
# Calling GlusterBaseClass tearDown
self.get_super_method(self, 'tearDown')()
def tearDown(self):
# Disable brick multiplexing
g.log.info("Checking for brick multiplexing status...")
if is_brick_mux_enabled(self.mnode):
g.log.info("Disabling brick multiplexing...")
if not disable_brick_mux(self.mnode):
raise ExecutionError("Failed to disable brick multiplexing")
g.log.info("Disabled brick multiplexing successfully")
ret, _, _ = reset_volume_option(self.mnode, "all", "all")
if ret:
raise ExecutionError("Unable to reset all volume options")
g.log.info("Successfully reset all the volume options.")
# Calling GlusterBaseClass teardown
self.get_super_method(self, 'tearDown')()
def tearDown(self):
# Stopping all volumes
g.log.info("Starting to Cleanup all Volumes")
volume_list = get_volume_list(self.mnode)
for volume in volume_list:
ret = cleanup_volume(self.mnode, volume)
if not ret:
raise ExecutionError("Failed to cleanup Volume %s" % volume)
g.log.info("Volume: %s cleanup is done", volume)
g.log.info("Successfully Cleanedup all Volumes")
# Disable brick multiplexing
g.log.info("Checking for brick multiplexing status...")
if is_brick_mux_enabled(self.mnode):
g.log.info("Disabling brick multiplexing...")
if not disable_brick_mux(self.mnode):
raise ExecutionError("Failed to disable brick multiplexing")
g.log.info("Disabled brick multiplexing successfully")
# Calling GlusterBaseClass teardown
self.get_super_method(self, 'tearDown')()
def bring_bricks_online(mnode,
volname,
bricks_list,
bring_bricks_online_methods=None):
"""Bring the bricks specified in the bricks_list online.
Args:
mnode (str): Node on which commands will be executed.
volname (str): Name of the volume.
bricks_list (list): List of bricks to bring them online.
Kwargs:
bring_bricks_online_methods (list): List of methods using which bricks
will be brought online. The method to bring a brick online is
randomly selected from the bring_bricks_online_methods list.
By default all bricks will be brought online with
['glusterd_restart', 'volume_start_force'] methods.
If 'volume_start_force' command is randomly selected then all the
bricks would be started with the command execution. Hence we break
from bringing bricks online individually
Returns:
bool : True on successfully bringing all bricks online.
False otherwise
"""
if bring_bricks_online_methods is None:
bring_bricks_online_methods = [
'glusterd_restart', 'volume_start_force'
]
elif not isinstance(bring_bricks_online_methods, list):
bring_bricks_online_methods = [bring_bricks_online_methods]
g.log.info("Bringing bricks '%s' online with '%s'", bricks_list,
bring_bricks_online_methods)
_rc = True
failed_to_bring_online_list = []
for brick in bricks_list:
bring_brick_online_method = random.choice(bring_bricks_online_methods)
if is_brick_mux_enabled(mnode):
bring_bricks_online_command = ("gluster volume start %s force" %
volname)
ret, _, _ = g.run(mnode, bring_bricks_online_command)
if ret != 0:
g.log.error("Unable to start the volume %s with force option",
volname)
_rc = False
else:
g.log.info(
"Successfully restarted volume %s to bring all "
"the bricks '%s' online", volname, bricks_list)
elif bring_brick_online_method == 'glusterd_restart':
bring_brick_online_command = "service glusterd restart"
brick_node, _ = brick.split(":")
ret, _, _ = g.run(brick_node, bring_brick_online_command)
if ret != 0:
g.log.error("Unable to restart glusterd on node %s",
brick_node)
_rc = False
failed_to_bring_online_list.append(brick)
else:
g.log.info(
"Successfully restarted glusterd on node %s to "
"bring back brick %s online", brick_node, brick)
elif bring_brick_online_method == 'volume_start_force':
bring_brick_online_command = ("gluster volume start %s force" %
volname)
ret, _, _ = g.run(mnode, bring_brick_online_command)
if ret != 0:
g.log.error("Unable to start the volume %s with force option",
volname)
_rc = False
else:
g.log.info(
"Successfully restarted volume %s to bring all "
"the bricks '%s' online", volname, bricks_list)
break
else:
g.log.error("Invalid method '%s' to bring brick online",
bring_brick_online_method)
return False
g.log.info("Waiting for 30 seconds for all the bricks to be online")
time.sleep(30)
return _rc
def bring_bricks_offline(volname,
bricks_list,
bring_bricks_offline_methods=None):
"""Bring the bricks specified in the bricks_list offline.
Args:
volname (str): Name of the volume
bricks_list (list): List of bricks to bring them offline.
Kwargs:
bring_bricks_offline_methods (list): List of methods using which bricks
will be brought offline. The method to bring a brick offline is
randomly selected from the bring_bricks_offline_methods list.
By default all bricks will be brought offline with
'service_kill' method.
Returns:
bool : True on successfully bringing all bricks offline.
False otherwise
"""
if bring_bricks_offline_methods is None:
bring_bricks_offline_methods = ['service_kill']
elif not isinstance(bring_bricks_offline_methods, list):
bring_bricks_offline_methods = [bring_bricks_offline_methods]
if not isinstance(bricks_list, list):
bricks_list = [bricks_list]
node_list = []
for brick in bricks_list:
node, _ = brick.split(":")
node_list.append(node)
if is_brick_mux_enabled(node_list[0]):
_rc = True
failed_to_bring_offline_list = []
for brick in bricks_list:
brick_node, brick_path = brick.split(":")
cmd = ("pgrep glusterfsd")
_, out, _ = g.run(brick_node, cmd)
if len(out.split()) > 1:
cmd = ("ps -eaf | grep glusterfsd | "
" grep %s.%s | grep -o '/var/run/gluster/.*' | "
" awk '{ print $3 }' | grep -v 'awk' " %
(volname, brick_node))
else:
cmd = ("ps -eaf | grep glusterfsd | "
"grep -o '/var/run/gluster.*' | "
" awk '{ print $3 }' | grep -v 'awk'")
_, socket_path, _ = g.run(brick_node, cmd)
uds_path = socket_path.strip()
kill_cmd = ("gf_attach -d %s %s" % (uds_path, brick_path))
ret, _, _ = g.run(brick_node, kill_cmd)
if ret != 0:
g.log.error("Unable to kill the brick %s", brick)
failed_to_bring_offline_list.append(brick)
_rc = False
if not _rc:
g.log.error("Unable to bring some of the bricks %s offline",
failed_to_bring_offline_list)
return False
g.log.info("All the bricks : %s are brought offline", bricks_list)
return True
_rc = True
failed_to_bring_offline_list = []
for brick in bricks_list:
bring_brick_offline_method = (
random.choice(bring_bricks_offline_methods))
if bring_brick_offline_method == 'service_kill':
brick_node, brick_path = brick.split(":")
brick_path = brick_path.replace("/", "-")
kill_cmd = ("pid=`ps -ef | grep -ve 'grep' | "
"grep -e '%s%s.pid' | awk '{print $2}'` && "
"kill -15 $pid || kill -9 $pid" %
(brick_node, brick_path))
ret, _, _ = g.run(brick_node, kill_cmd)
if ret != 0:
g.log.error("Unable to kill the brick %s", brick)
failed_to_bring_offline_list.append(brick)
_rc = False
else:
g.log.error("Invalid method '%s' to bring brick offline",
bring_brick_offline_method)
return False
if not _rc:
g.log.error("Unable to bring some of the bricks %s offline",
failed_to_bring_offline_list)
return False
g.log.info("All the bricks : %s are brought offline", bricks_list)
return True
def test_enabling_brick_mux(self):
"""
Test case:
- check if brick multiplex is disable by default
- check for warning message triggering by setting brick-multiplex and
choosing 'n' in y/n
- check if brick multiplex is disabled after triggering warning message
- check brick multiplex for all possible statuses
(positive and negative)
- check for brick multiplex status in /var/lib/glusterd/options file
"""
# Check if brickmux is disabled by default
g.log.info('Checking if brick multiplex operation is disabled...')
self.assertFalse(is_brick_mux_enabled(self.mnode),
"Brick multiplex is not disabled by default")
# Check for warning message while changing status
warning_message = ("Brick-multiplexing is supported only for "
"OCS converged or independent mode. Also it is "
"advised to make sure that either all volumes are "
"in stopped state or no bricks are running before "
"this option is modified."
"Do you still want to continue? (y/n)")
g.log.info('Triggering warning message...')
cmd = "gluster v set all cluster.brick-multiplex enable"
_, out, _ = g.run(self.mnode, cmd)
g.log.info('Checking for warning message in output...')
if "volume set: success" not in out:
self.assertIn(warning_message, out,
'There is no warning message in '
'output or message is incorrect.')
g.log.info('Warning message is correct.')
else:
g.log.info('Skipped warning message check.')
# If brick-mux is enabled then disabling it.
if is_brick_mux_enabled(self.mnode):
if not disable_brick_mux(self.mnode):
g.log.info("Disabling brick multiplexing as it"
" was enabled due to no warning message.")
# Check if brickmux is still disabled
g.log.info('Checking if brick multiplex is still disabled')
self.assertFalse(is_brick_mux_enabled(self.mnode),
"Brick multiplex operation is not disabled")
# Enable brick multiplex with all possible statuses
statuses = ['on', 'enable', '1', 'true',
'off', 'disable', '0', 'false']
for status in statuses:
g.log.info('Enabling brick multiplex with %s status...',
status)
cmd = ("yes | gluster v set all cluster.brick-multiplex %s"
% status)
_, out, _ = g.run(self.mnode, cmd)
self.assertIn('success', out,
'Failed on enabling brick multiplexing')
# Check if brick multiplex status is correct
g.log.info('Checking if brick multiplexing status is correct...')
gluster_status = get_brick_mux_status(self.mnode)
self.assertEqual(status, gluster_status,
"Brick multiplex status is not correct")
g.log.info('Brick multiplex status "%s" is correct',
status)
# Check for brick multiplexing status in file 'options'
g.log.info("Checking for brick multiplexing status '%s' in file "
"'/var/lib/glusterd/options'...", status)
search_pattern = 'cluster.brick-multiplex=%s' % status
self.assertTrue(search_pattern_in_file(self.mnode, search_pattern,
'/var/lib/glusterd/options',
'', ''))
g.log.info("Brick multiplexing status '%s' in file "
"'/var/lib/glusterd/options' is correct", status)
# Check brick multiplex with incorrect status
g.log.info('Checking brick multiplex with incorrect status...')
cmd = "yes | gluster v set all cluster.brick-multiplex incorrect"
ret, _, _ = g.run(self.mnode, cmd)
self.assertEqual(ret, 1, 'Incorrect status has passed')
def test_enable_brick_mux_with_max_bricks_per_process(self):
"""
Test Case:
- Create a gluster cluster
- With brick mux set to disable:
1.Set cluster.max-bricks-per-process to int and check
error message(Must fail)
2.Set cluster.max-bricks-per-process to string(Must fail)
- With brick mux set to enable:
1.Set cluster.max-bricks-per-process to string(Must fail)
2.Set cluster.max-bricks-per-process to 0
3.Set cluster.max-bricks-per-process to 1 and check
error message.(Must fail)
4.Set cluster.max-bricks-per-process to int value > 1.
"""
# Disabling cluster.brick-multiplex if not.
if is_brick_mux_enabled(self.mnode):
ret = disable_brick_mux(self.mnode)
self.assertTrue(ret, "Unable to disable brickmux")
g.log.info("Brick mux is disabled")
# Set cluster.max-bricks-per-process to int and check
# error message(Must fail)
cmd = "gluster v set all cluster.max-bricks-per-process 10"
ret, _, err = g.run(self.mnode, cmd)
self.assertEqual(
ret, 1, 'Able to set max-bricks-per-process'
'without enabling brick mux')
self.assertIn(
"volume set: failed: Brick-multiplexing is not enabled. "
"Please enable brick multiplexing before"
" trying to set this option.", err,
"Error message not proper on trying to "
"set max-bricks-per-process without brickmux")
# Set cluster.max-bricks-per-process to string(Must fail)
self.set_max_brick_process_to_string()
# Enable cluster.brick-multiplex.
ret = enable_brick_mux(self.mnode)
self.assertTrue(ret, "Unable to enable cluster.brick-multiplex")
g.log.info("Brick mux is enabled")
# Set cluster.max-bricks-per-process to string(Must fail)
self.set_max_brick_process_to_string()
# Set cluster.max-bricks-per-process to 0.
ret = set_volume_options(self.mnode, 'all',
{'cluster.max-bricks-per-process': '0'})
self.assertTrue(ret, "Unable to set "
"cluster.max-bricks-per-process to 0")
g.log.info("Successfully set cluster.max-bricks-per-process to 0")
# Set cluster.max-bricks-per-process to 1 and check
# error message.(Must fail)
cmd = "gluster v set all cluster.max-bricks-per-process 1"
ret, _, err = g.run(self.mnode, cmd)
self.assertEqual(
ret, 1, 'Able to set max-bricks-per-process'
'with enabling brick mux')
self.assertIn(
"volume set: failed: Brick-multiplexing is enabled."
" Please set this option to a value other than 1 to"
" make use of the brick-multiplexing feature.", err,
"Error message not proper on trying to set max-bricks-per-process"
" with brickmux")
# Set cluster.max-bricks-per-process to int value > 1
key = 'cluster.max-bricks-per-process'
temp_val = self.get_random_string(string.digits)
value = "{}".format(temp_val)
ret = set_volume_options(self.mnode, 'all', {key: value})
self.assertTrue(
ret, "Unexpected: Erroneous value {}, to option "
"{} should not result in failure".format(value, key))
g.log.info("Value %s, set to option %s", value, key)
