def _validate_error_in_mount_log(self, pattern, exp_pre=True): """ Validate type of error from mount log on setting quota """ assert_method = self.assertTrue assert_msg = ('Fail: Not able to validate presence of "{}" ' 'in mount log'.format(pattern)) if not exp_pre: assert_method = self.assertFalse assert_msg = ('Fail: Not able to validate absence of "{}" ' 'in mount log'.format(pattern)) ret = search_pattern_in_file(self.client, pattern, self.logpath, self.bp_text + str(self.bp_count - 2), self.bp_text + str(self.bp_count - 1)) assert_method(ret, assert_msg) # Validate against `quota list` command if 'quota' in pattern.lower(): dir_path = '/dir/dir1' ret = quota_fetch_list(self.mnode, self.volname) self.assertIsNotNone( ret.get(dir_path), 'Not able to get quota list for the path {}'.format(dir_path)) ret = ret.get(dir_path) verified = False if ret['sl_exceeded'] is exp_pre and ret['hl_exceeded'] is exp_pre: verified = True self.assertTrue( verified, 'Failed to validate Quota list command against ' 'soft and hard limits')
def test_alert_time_out(self): """ Verifying directory quota functionality with respect to alert-time, soft-timeout and hard-timeout. * Enable quota * Set limit on '/' * Set default-soft-limit to 50% * Set alert time to 1 sec * Set soft-timeout to 0 sec * Set hard timeout to 0 sec * Check quota list * Perform some IO such that soft limit is not exceeded * Check for alert message in brick logfile (NO alert present) * Perform IO and exceed the soft limit * Check for alert message in brick logfile (Alert present) * Remove some files so that usage falls below soft limit * Create some files such that hard limit is exceeded * Check for alert message in brick logfile * Remove some files so that usage falls below soft limit * Check for alert message in brick logfile (NO alert present) """ # pylint: disable=too-many-statements # Enable Quota g.log.info("Enabling quota on the volume %s", self.volname) ret, _, _ = quota_enable(self.mnode, self.volname) self.assertFalse( ret, "Failed to enable quota on the volume %s" % self.volname) g.log.info("Successfully enabled quota on the volume %s", self.volname) # Path to set quota limit path = "/" # Create a directory from mount point mount_obj = self.mounts[0] mount_dir = mount_obj.mountpoint client = mount_obj.client_system g.log.info("Creating dir named 'foo' from client %s", client) ret = mkdir(client, "%s/foo" % mount_dir) self.assertTrue( ret, "Failed to create dir under %s-%s" % (client, mount_dir)) g.log.info("Directory 'foo' created successfully") # Set Quota limit on the root of the volume g.log.info("Set Quota Limit on the path %s of the volume %s", path, self.volname) ret, _, _ = quota_limit_usage(self.mnode, self.volname, path=path, limit="100MB") self.assertEqual(ret, 0, ("Failed to set quota limit on path %s of " " the volume %s", path, self.volname)) g.log.info("Successfully set the Quota limit on %s of the volume %s", path, self.volname) # Set default soft limit to 50% g.log.info("Set default soft limit:") ret, _, _ = quota_set_default_soft_limit(self.mnode, self.volname, '50%') self.assertEqual(ret, 0, ("Failed to set quota default soft limit")) g.log.info("Quota default soft limit set successfully") # Check quota list to validate limits g.log.info("List all files and directories:") ret = quota_validate(self.mnode, self.volname, path=path, soft_limit_percent=50, hard_limit=104857600) self.assertTrue(ret, "Failed to validate Quota list") g.log.info("Quota List successful") # Set alert time to 1 second g.log.info("Set quota alert timeout:") ret, _, _ = quota_set_alert_time(self.mnode, self.volname, '1sec') self.assertEqual(ret, 0, ("Failed to set alert timeout")) g.log.info("Quota alert time set successful") # Set soft timeout to 0 second g.log.info("Set quota soft timeout:") ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, '0sec') self.assertEqual(ret, 0, ("Failed to set soft timeout")) g.log.info("Quota soft timeout set successful") # Set hard timeout to 0 second g.log.info("Set quota hard timeout:") ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, '0sec') self.assertEqual(ret, 0, ("Failed to set hard timeout")) g.log.info("Quota hard timeout set successful") # Get the brick log file path for a random node bricks = get_all_bricks(self.mnode, self.volname) selected_node, brick_path = random.choice(bricks[0:6]).split(':') brickpath = str.replace(brick_path, '/', '-') brickpathfinal = brickpath[1:] brick_log = "/var/log/glusterfs/bricks/%s.log" % brickpathfinal # Append unique string to the brick log g.log.info("Appending string 'appended_string_1' to the log:") append_string_to_file(selected_node, brick_log, "appended_string_1") # Starting IO on the mounts without crossing the soft limit # file creation should be normal g.log.info("Creating Files on %s:%s", client, mount_dir) cmd = ("cd %s/foo ; " "for i in `seq 1 9` ; " "do dd if=/dev/urandom of=file$i " "bs=5M " "count=1 ; " "done" % mount_dir) ret, _, _ = g.run(client, cmd) self.assertEqual(ret, 0, ("Failed to create files on mountpoint")) g.log.info("Files created succesfully on mountpoint") # Append unique string to the brick log g.log.info("Appending string 'appended_string_2' to the log:") append_string_to_file(selected_node, brick_log, "appended_string_2") # Soft limit not crossed # Check if alert message is logged in the brick (shouldn't be logged) g.log.info("Check for alert message in logfile:") ret = search_pattern_in_file(selected_node, "120004", brick_log, "appended_string_1", "appended_string_2") self.assertFalse(ret, "Found unnecessary alert in logfile") g.log.info("Alert message not seen before crossing soft limit") # Continue IO on the mounts to exceed soft limit g.log.info("Creating Files on %s:%s", client, mount_dir) cmd = ("cd %s/foo ; " "for i in `seq 1 200` ; " "do dd if=/dev/urandom of=foo$i " "bs=100K " "count=1 ; " "done" % mount_dir) ret, _, _ = g.run(client, cmd) self.assertEqual(ret, 0, ("Failed to create files on mountpoint")) g.log.info("Files created succesfully on mountpoint and " "exceed soft limit") # Check if quota soft limit exceeded g.log.info("Check soft limit:") ret = quota_validate(self.mnode, self.volname, path=path, sl_exceeded=True) self.assertTrue(ret, "Failed: soft limit not exceeded") g.log.info('Quota soft limit exceeded') # Inserting sleep of 2 seconds so the alert message gets enough time # to be logged time.sleep(2) # Append unique string to the brick log g.log.info("Appending string 'appended_string_3' to the log:") append_string_to_file(selected_node, brick_log, "appended_string_3") # Check if alert message logged in the brick g.log.info("Check for message:") ret = search_pattern_in_file(selected_node, "120004", brick_log, "appended_string_2", "appended_string_3") self.assertTrue(ret, "Alert message not found") g.log.info("Pattern Found: got alert message") # Append unique string to the brick log g.log.info("Appending string 'appended_string_4' to the log:") append_string_to_file(selected_node, brick_log, "appended_string_4") # Continue IO on the mounts by removing data g.log.info("Removing Files on %s:%s", client, mount_dir) cmd = ("rm -rfv %s/foo/foo* " % mount_dir) ret, _, _ = g.run(client, cmd) self.assertEqual(ret, 0, ("Failed to delete files on mountpoint")) g.log.info("Files removed succesfully from mountpoint and reached " "below soft limit") # Check if quota soft limit exceeded g.log.info("Check soft limit:") ret = quota_validate(self.mnode, self.volname, path=path, sl_exceeded=False) self.assertTrue(ret, "Failed: soft limit exceeded") g.log.info('Quota soft limit not exceeded') # Inserting sleep of 2 seconds so the alert message gets enough time # to be logged time.sleep(2) # Append unique string to the brick log g.log.info("Appending string 'appended_string_5' to the log:") append_string_to_file(selected_node, brick_log, "appended_string_5") # Check if alert message is logged in the brick g.log.info("Check for message:") ret = search_pattern_in_file(selected_node, "120004", brick_log, "appended_string_4", "appended_string_5") self.assertFalse(ret, "Found unnecessary alert message in logfile") g.log.info("Alert message not seen before crossing soft limit") # Continue IO on the mounts to exceed hard limit g.log.info("Creating Files on %s:%s", client, mount_dir) cmd = ("cd %s/foo ; " "for i in `seq 11 20` ; " "do dd if=/dev/urandom of=file$i " "bs=10M " "count=1 ; " "done" % mount_dir) ret, _, _ = g.run(client, cmd) self.assertEqual(ret, 1, ("Failed: Files created successfully inspite " "of crossing hard-limit")) g.log.info("Files creation stopped on mountpoint once exceeded " "hard limit") # Inserting sleep of 6 seconds so the alert message gets enough time # to be logged time.sleep(6) # Append unique string to the brick log g.log.info("Appending string 'appended_string_6' to the log:") append_string_to_file(selected_node, brick_log, "appended_string_6") # Check if alert message is logged in the brick g.log.info("Check for message:") ret = search_pattern_in_file(selected_node, "120004", brick_log, "appended_string_5", "appended_string_6") self.assertTrue(ret, "Alert message not seen in logfile") g.log.info("Pattern Found: got alert message") # Append unique string to the brick log g.log.info("Appending string 'Done_with_alert_check_7' to the log:") append_string_to_file(selected_node, brick_log, "Done_with_alert_check_7") # Continue IO on the mounts by removing data to come below hard limit g.log.info("Removing Files on %s:%s", client, mount_dir) cmd = ("rm -rfv %s/foo/file{11..20}" % mount_dir) ret, _, _ = g.run(client, cmd) self.assertEqual(ret, 0, ("Failed to delete files on mountpoint")) g.log.info("Files removed succesfully on mountpoint and " "reached below soft limit") # Inserting sleep of 2 seconds so the alert message gets enough time # to be logged time.sleep(2) # Append unique string to the brick log g.log.info("Appending string 'Done_with_alert_check_8' to the log:") append_string_to_file(selected_node, brick_log, "Done_with_alert_check_8") # Check if alert message is logged in the brick g.log.info("Check for message:") ret = search_pattern_in_file(selected_node, "120004", brick_log, "Done_with_alert_check_7", "Done_with_alert_check_8") self.assertFalse(ret, "Found unnecessary alert in logfile") g.log.info("EXPECTED: Alert message not seen before crossing " "soft limit") # have got below the soft and hard limit # check quota list g.log.info("List all files and directories:") ret = quota_validate(self.mnode, self.volname, path=path, sl_exceeded=False, hl_exceeded=False) self.assertTrue( ret, "Failed to validate Quota list with " "soft and hard limits") g.log.info("Quota List validated successfully")
def test_enabling_brick_mux(self): """ Test case: - check if brick multiplex is disable by default - check for warning message triggering by setting brick-multiplex and choosing 'n' in y/n - check if brick multiplex is disabled after triggering warning message - check brick multiplex for all possible statuses (positive and negative) - check for brick multiplex status in /var/lib/glusterd/options file """ # Check if brickmux is disabled by default g.log.info('Checking if brick multiplex operation is disabled...') self.assertFalse(is_brick_mux_enabled(self.mnode), "Brick multiplex is not disabled by default") # Check for warning message while changing status warning_message = ("Brick-multiplexing is supported only for " "OCS converged or independent mode. Also it is " "advised to make sure that either all volumes are " "in stopped state or no bricks are running before " "this option is modified." "Do you still want to continue? (y/n)") g.log.info('Triggering warning message...') cmd = "gluster v set all cluster.brick-multiplex enable" _, out, _ = g.run(self.mnode, cmd) g.log.info('Checking for warning message in output...') if "volume set: success" not in out: self.assertIn(warning_message, out, 'There is no warning message in ' 'output or message is incorrect.') g.log.info('Warning message is correct.') else: g.log.info('Skipped warning message check.') # If brick-mux is enabled then disabling it. if is_brick_mux_enabled(self.mnode): if not disable_brick_mux(self.mnode): g.log.info("Disabling brick multiplexing as it" " was enabled due to no warning message.") # Check if brickmux is still disabled g.log.info('Checking if brick multiplex is still disabled') self.assertFalse(is_brick_mux_enabled(self.mnode), "Brick multiplex operation is not disabled") # Enable brick multiplex with all possible statuses statuses = ['on', 'enable', '1', 'true', 'off', 'disable', '0', 'false'] for status in statuses: g.log.info('Enabling brick multiplex with %s status...', status) cmd = ("yes | gluster v set all cluster.brick-multiplex %s" % status) _, out, _ = g.run(self.mnode, cmd) self.assertIn('success', out, 'Failed on enabling brick multiplexing') # Check if brick multiplex status is correct g.log.info('Checking if brick multiplexing status is correct...') gluster_status = get_brick_mux_status(self.mnode) self.assertEqual(status, gluster_status, "Brick multiplex status is not correct") g.log.info('Brick multiplex status "%s" is correct', status) # Check for brick multiplexing status in file 'options' g.log.info("Checking for brick multiplexing status '%s' in file " "'/var/lib/glusterd/options'...", status) search_pattern = 'cluster.brick-multiplex=%s' % status self.assertTrue(search_pattern_in_file(self.mnode, search_pattern, '/var/lib/glusterd/options', '', '')) g.log.info("Brick multiplexing status '%s' in file " "'/var/lib/glusterd/options' is correct", status) # Check brick multiplex with incorrect status g.log.info('Checking brick multiplex with incorrect status...') cmd = "yes | gluster v set all cluster.brick-multiplex incorrect" ret, _, _ = g.run(self.mnode, cmd) self.assertEqual(ret, 1, 'Incorrect status has passed')