def _validate_error_in_mount_log(self, pattern, exp_pre=True):
"""
Validate type of error from mount log on setting quota
"""
assert_method = self.assertTrue
assert_msg = ('Fail: Not able to validate presence of "{}" '
'in mount log'.format(pattern))
if not exp_pre:
assert_method = self.assertFalse
assert_msg = ('Fail: Not able to validate absence of "{}" '
'in mount log'.format(pattern))
ret = search_pattern_in_file(self.client, pattern, self.logpath,
self.bp_text + str(self.bp_count - 2),
self.bp_text + str(self.bp_count - 1))
assert_method(ret, assert_msg)
# Validate against `quota list` command
if 'quota' in pattern.lower():
dir_path = '/dir/dir1'
ret = quota_fetch_list(self.mnode, self.volname)
self.assertIsNotNone(
ret.get(dir_path),
'Not able to get quota list for the path {}'.format(dir_path))
ret = ret.get(dir_path)
verified = False
if ret['sl_exceeded'] is exp_pre and ret['hl_exceeded'] is exp_pre:
verified = True
self.assertTrue(
verified, 'Failed to validate Quota list command against '
'soft and hard limits')
def test_alert_time_out(self):
"""
Verifying directory quota functionality with respect to
alert-time, soft-timeout and hard-timeout.
* Enable quota
* Set limit on '/'
* Set default-soft-limit to 50%
* Set alert time to 1 sec
* Set soft-timeout to 0 sec
* Set hard timeout to 0 sec
* Check quota list
* Perform some IO such that soft limit is not exceeded
* Check for alert message in brick logfile (NO alert present)
* Perform IO and exceed the soft limit
* Check for alert message in brick logfile (Alert present)
* Remove some files so that usage falls below soft limit
* Create some files such that hard limit is exceeded
* Check for alert message in brick logfile
* Remove some files so that usage falls below soft limit
* Check for alert message in brick logfile (NO alert present)
"""
# pylint: disable=too-many-statements
# Enable Quota
g.log.info("Enabling quota on the volume %s", self.volname)
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertFalse(
ret, "Failed to enable quota on the volume %s" % self.volname)
g.log.info("Successfully enabled quota on the volume %s", self.volname)
# Path to set quota limit
path = "/"
# Create a directory from mount point
mount_obj = self.mounts[0]
mount_dir = mount_obj.mountpoint
client = mount_obj.client_system
g.log.info("Creating dir named 'foo' from client %s", client)
ret = mkdir(client, "%s/foo" % mount_dir)
self.assertTrue(
ret, "Failed to create dir under %s-%s" % (client, mount_dir))
g.log.info("Directory 'foo' created successfully")
# Set Quota limit on the root of the volume
g.log.info("Set Quota Limit on the path %s of the volume %s", path,
self.volname)
ret, _, _ = quota_limit_usage(self.mnode,
self.volname,
path=path,
limit="100MB")
self.assertEqual(ret, 0, ("Failed to set quota limit on path %s of "
" the volume %s", path, self.volname))
g.log.info("Successfully set the Quota limit on %s of the volume %s",
path, self.volname)
# Set default soft limit to 50%
g.log.info("Set default soft limit:")
ret, _, _ = quota_set_default_soft_limit(self.mnode, self.volname,
'50%')
self.assertEqual(ret, 0, ("Failed to set quota default soft limit"))
g.log.info("Quota default soft limit set successfully")
# Check quota list to validate limits
g.log.info("List all files and directories:")
ret = quota_validate(self.mnode,
self.volname,
path=path,
soft_limit_percent=50,
hard_limit=104857600)
self.assertTrue(ret, "Failed to validate Quota list")
g.log.info("Quota List successful")
# Set alert time to 1 second
g.log.info("Set quota alert timeout:")
ret, _, _ = quota_set_alert_time(self.mnode, self.volname, '1sec')
self.assertEqual(ret, 0, ("Failed to set alert timeout"))
g.log.info("Quota alert time set successful")
# Set soft timeout to 0 second
g.log.info("Set quota soft timeout:")
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, '0sec')
self.assertEqual(ret, 0, ("Failed to set soft timeout"))
g.log.info("Quota soft timeout set successful")
# Set hard timeout to 0 second
g.log.info("Set quota hard timeout:")
ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, '0sec')
self.assertEqual(ret, 0, ("Failed to set hard timeout"))
g.log.info("Quota hard timeout set successful")
# Get the brick log file path for a random node
bricks = get_all_bricks(self.mnode, self.volname)
selected_node, brick_path = random.choice(bricks[0:6]).split(':')
brickpath = str.replace(brick_path, '/', '-')
brickpathfinal = brickpath[1:]
brick_log = "/var/log/glusterfs/bricks/%s.log" % brickpathfinal
# Append unique string to the brick log
g.log.info("Appending string 'appended_string_1' to the log:")
append_string_to_file(selected_node, brick_log, "appended_string_1")
# Starting IO on the mounts without crossing the soft limit
# file creation should be normal
g.log.info("Creating Files on %s:%s", client, mount_dir)
cmd = ("cd %s/foo ; "
"for i in `seq 1 9` ; "
"do dd if=/dev/urandom of=file$i "
"bs=5M "
"count=1 ; "
"done" % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, ("Failed to create files on mountpoint"))
g.log.info("Files created succesfully on mountpoint")
# Append unique string to the brick log
g.log.info("Appending string 'appended_string_2' to the log:")
append_string_to_file(selected_node, brick_log, "appended_string_2")
# Soft limit not crossed
# Check if alert message is logged in the brick (shouldn't be logged)
g.log.info("Check for alert message in logfile:")
ret = search_pattern_in_file(selected_node, "120004", brick_log,
"appended_string_1", "appended_string_2")
self.assertFalse(ret, "Found unnecessary alert in logfile")
g.log.info("Alert message not seen before crossing soft limit")
# Continue IO on the mounts to exceed soft limit
g.log.info("Creating Files on %s:%s", client, mount_dir)
cmd = ("cd %s/foo ; "
"for i in `seq 1 200` ; "
"do dd if=/dev/urandom of=foo$i "
"bs=100K "
"count=1 ; "
"done" % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, ("Failed to create files on mountpoint"))
g.log.info("Files created succesfully on mountpoint and "
"exceed soft limit")
# Check if quota soft limit exceeded
g.log.info("Check soft limit:")
ret = quota_validate(self.mnode,
self.volname,
path=path,
sl_exceeded=True)
self.assertTrue(ret, "Failed: soft limit not exceeded")
g.log.info('Quota soft limit exceeded')
# Inserting sleep of 2 seconds so the alert message gets enough time
# to be logged
time.sleep(2)
# Append unique string to the brick log
g.log.info("Appending string 'appended_string_3' to the log:")
append_string_to_file(selected_node, brick_log, "appended_string_3")
# Check if alert message logged in the brick
g.log.info("Check for message:")
ret = search_pattern_in_file(selected_node, "120004", brick_log,
"appended_string_2", "appended_string_3")
self.assertTrue(ret, "Alert message not found")
g.log.info("Pattern Found: got alert message")
# Append unique string to the brick log
g.log.info("Appending string 'appended_string_4' to the log:")
append_string_to_file(selected_node, brick_log, "appended_string_4")
# Continue IO on the mounts by removing data
g.log.info("Removing Files on %s:%s", client, mount_dir)
cmd = ("rm -rfv %s/foo/foo* " % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, ("Failed to delete files on mountpoint"))
g.log.info("Files removed succesfully from mountpoint and reached "
"below soft limit")
# Check if quota soft limit exceeded
g.log.info("Check soft limit:")
ret = quota_validate(self.mnode,
self.volname,
path=path,
sl_exceeded=False)
self.assertTrue(ret, "Failed: soft limit exceeded")
g.log.info('Quota soft limit not exceeded')
# Inserting sleep of 2 seconds so the alert message gets enough time
# to be logged
time.sleep(2)
# Append unique string to the brick log
g.log.info("Appending string 'appended_string_5' to the log:")
append_string_to_file(selected_node, brick_log, "appended_string_5")
# Check if alert message is logged in the brick
g.log.info("Check for message:")
ret = search_pattern_in_file(selected_node, "120004", brick_log,
"appended_string_4", "appended_string_5")
self.assertFalse(ret, "Found unnecessary alert message in logfile")
g.log.info("Alert message not seen before crossing soft limit")
# Continue IO on the mounts to exceed hard limit
g.log.info("Creating Files on %s:%s", client, mount_dir)
cmd = ("cd %s/foo ; "
"for i in `seq 11 20` ; "
"do dd if=/dev/urandom of=file$i "
"bs=10M "
"count=1 ; "
"done" % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 1, ("Failed: Files created successfully inspite "
"of crossing hard-limit"))
g.log.info("Files creation stopped on mountpoint once exceeded "
"hard limit")
# Inserting sleep of 6 seconds so the alert message gets enough time
# to be logged
time.sleep(6)
# Append unique string to the brick log
g.log.info("Appending string 'appended_string_6' to the log:")
append_string_to_file(selected_node, brick_log, "appended_string_6")
# Check if alert message is logged in the brick
g.log.info("Check for message:")
ret = search_pattern_in_file(selected_node, "120004", brick_log,
"appended_string_5", "appended_string_6")
self.assertTrue(ret, "Alert message not seen in logfile")
g.log.info("Pattern Found: got alert message")
# Append unique string to the brick log
g.log.info("Appending string 'Done_with_alert_check_7' to the log:")
append_string_to_file(selected_node, brick_log,
"Done_with_alert_check_7")
# Continue IO on the mounts by removing data to come below hard limit
g.log.info("Removing Files on %s:%s", client, mount_dir)
cmd = ("rm -rfv %s/foo/file{11..20}" % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, ("Failed to delete files on mountpoint"))
g.log.info("Files removed succesfully on mountpoint and "
"reached below soft limit")
# Inserting sleep of 2 seconds so the alert message gets enough time
# to be logged
time.sleep(2)
# Append unique string to the brick log
g.log.info("Appending string 'Done_with_alert_check_8' to the log:")
append_string_to_file(selected_node, brick_log,
"Done_with_alert_check_8")
# Check if alert message is logged in the brick
g.log.info("Check for message:")
ret = search_pattern_in_file(selected_node, "120004", brick_log,
"Done_with_alert_check_7",
"Done_with_alert_check_8")
self.assertFalse(ret, "Found unnecessary alert in logfile")
g.log.info("EXPECTED: Alert message not seen before crossing "
"soft limit")
# have got below the soft and hard limit
# check quota list
g.log.info("List all files and directories:")
ret = quota_validate(self.mnode,
self.volname,
path=path,
sl_exceeded=False,
hl_exceeded=False)
self.assertTrue(
ret, "Failed to validate Quota list with "
"soft and hard limits")
g.log.info("Quota List validated successfully")
def test_enabling_brick_mux(self):
"""
Test case:
- check if brick multiplex is disable by default
- check for warning message triggering by setting brick-multiplex and
choosing 'n' in y/n
- check if brick multiplex is disabled after triggering warning message
- check brick multiplex for all possible statuses
(positive and negative)
- check for brick multiplex status in /var/lib/glusterd/options file
"""
# Check if brickmux is disabled by default
g.log.info('Checking if brick multiplex operation is disabled...')
self.assertFalse(is_brick_mux_enabled(self.mnode),
"Brick multiplex is not disabled by default")
# Check for warning message while changing status
warning_message = ("Brick-multiplexing is supported only for "
"OCS converged or independent mode. Also it is "
"advised to make sure that either all volumes are "
"in stopped state or no bricks are running before "
"this option is modified."
"Do you still want to continue? (y/n)")
g.log.info('Triggering warning message...')
cmd = "gluster v set all cluster.brick-multiplex enable"
_, out, _ = g.run(self.mnode, cmd)
g.log.info('Checking for warning message in output...')
if "volume set: success" not in out:
self.assertIn(warning_message, out,
'There is no warning message in '
'output or message is incorrect.')
g.log.info('Warning message is correct.')
else:
g.log.info('Skipped warning message check.')
# If brick-mux is enabled then disabling it.
if is_brick_mux_enabled(self.mnode):
if not disable_brick_mux(self.mnode):
g.log.info("Disabling brick multiplexing as it"
" was enabled due to no warning message.")
# Check if brickmux is still disabled
g.log.info('Checking if brick multiplex is still disabled')
self.assertFalse(is_brick_mux_enabled(self.mnode),
"Brick multiplex operation is not disabled")
# Enable brick multiplex with all possible statuses
statuses = ['on', 'enable', '1', 'true',
'off', 'disable', '0', 'false']
for status in statuses:
g.log.info('Enabling brick multiplex with %s status...',
status)
cmd = ("yes | gluster v set all cluster.brick-multiplex %s"
% status)
_, out, _ = g.run(self.mnode, cmd)
self.assertIn('success', out,
'Failed on enabling brick multiplexing')
# Check if brick multiplex status is correct
g.log.info('Checking if brick multiplexing status is correct...')
gluster_status = get_brick_mux_status(self.mnode)
self.assertEqual(status, gluster_status,
"Brick multiplex status is not correct")
g.log.info('Brick multiplex status "%s" is correct',
status)
# Check for brick multiplexing status in file 'options'
g.log.info("Checking for brick multiplexing status '%s' in file "
"'/var/lib/glusterd/options'...", status)
search_pattern = 'cluster.brick-multiplex=%s' % status
self.assertTrue(search_pattern_in_file(self.mnode, search_pattern,
'/var/lib/glusterd/options',
'', ''))
g.log.info("Brick multiplexing status '%s' in file "
"'/var/lib/glusterd/options' is correct", status)
# Check brick multiplex with incorrect status
g.log.info('Checking brick multiplex with incorrect status...')
cmd = "yes | gluster v set all cluster.brick-multiplex incorrect"
ret, _, _ = g.run(self.mnode, cmd)
self.assertEqual(ret, 1, 'Incorrect status has passed')