def test_negative_quota_timeouts(self):
""" This testcase try to enable soft/hard timeouts by giving
huge value , all cases has to return false
"""
ret, _, err = quota_enable(self.mnode, self.volname)
self.assertEqual(ret, 0,
"Error in enabling quota for %s" % (self.volname))
# now try to enable timeout with more time
time_in_secs = 100 * 60 * 60
g.log.info("Setting up soft timeout with %d secs", time_in_secs)
ret, _, err = quota_set_soft_timeout(self.mnode, self.volname,
str(time_in_secs))
errmsg = ("quota command failed : '%d' in "
"'option soft-timeout %d' is out "
"of range [0 - 1800]\n" % (time_in_secs, time_in_secs))
self.assertEqual(err, errmsg,
"expected %s but returned %s" % (errmsg, err))
# now try to enable hard timeout with more time
g.log.info("Setting up hard timeout with %d secs", time_in_secs)
ret, _, err = quota_set_hard_timeout(self.mnode, self.volname,
str(time_in_secs))
errmsg = ("quota command failed : '%d' in "
"'option hard-timeout %d' is "
"out of range [0 - 60]\n" % (time_in_secs, time_in_secs))
self.assertEqual(err, errmsg,
"expected %s but returned %s" % (errmsg, err))
def test_ec_quota(self):
"""
- Enable quota on the volume
- Set a limit of 4 GB on the root of the volume
- Set Quota soft-timeout to 0 seconds
- Set Quota hard-timeout to 0 second
- Create 10 directories from the mount point
- Create files of around 2.5 GB
- Reading files
- Decrease quota limit to 3 GB on the root of the volume
- Writing files of around 500 MB
- Reading files
"""
# pylint: disable=too-many-branches,too-many-statements,too-many-locals
# Enable quota on the volume
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(
ret, 0, ("Failed to enable quota on the volume %s", self.volname))
g.log.info("Successfully enabled quota on the volume %s", self.volname)
# Set a limit of 4 GB on the root of the volume
self.set_quota_limit(limit="4GB")
# Set Quota soft-timeout to 0 seconds
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, '0sec')
self.assertEqual(ret, 0, "Failed to set soft timeout")
g.log.info("Quota soft timeout set successful")
# Set Quota hard-timeout to 0 second
ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, '0sec')
self.assertEqual(ret, 0, "Failed to set hard timeout")
g.log.info("Quota hard timeout set successful")
# Create 10 directories from the mount point
mount_obj = self.mounts[0]
mount_dir = mount_obj.mountpoint
client = mount_obj.client_system
for i in range(1, 11):
ret = mkdir(client, "%s/dir%s" % (mount_dir, i))
self.assertTrue(
ret, ("Failed to create dir under %s-%s", client, mount_dir))
g.log.info("Directory 'dir%s' created successfully", i)
g.log.info("Successfully created directories on %s:%s", client,
mount_dir)
# Create files of around 2.5 GB and reading
self.read_write_files(files=100, mount_dir=mount_dir, client=client)
# Decrease quota limit to 3 GB on the root of the volume
self.set_quota_limit(limit="3GB")
# Writing files of around 500 MB and reading
self.read_write_files(files=10, mount_dir=mount_dir, client=client)
def test_rebalance_with_quota_enabled(self):
"""
Test rebalance with quota enabled on root.
1. Create Volume of type distribute
2. Set Quota limit on the root directory
3. Do some IO to reach the Hard limit
4. After IO ends, compute arequal checksum
5. Add bricks to the volume.
6. Start rebalance
7. After rebalance is completed, check arequal checksum
"""
# Enable Quota
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(
ret, 0, ("Failed to enable quota on the volume %s", self.volname))
g.log.info("Successfully enabled quota on volume %s", self.volname)
# Set the Quota timeouts to 0 for strict accounting
ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, 0)
self.assertEqual(
ret, 0, ("Failed to set hard-timeout to 0 for %s", self.volname))
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, 0)
self.assertEqual(
ret, 0, ("Failed to set soft-timeout to 0 for %s", self.volname))
g.log.info("Quota soft and hard timeout has been set to 0 for %s",
self.volname)
# Set the quota limit of 1 GB on root dir of the volume
ret, _, _ = quota_limit_usage(self.mnode, self.volname, "/", "1GB")
self.assertEqual(ret, 0, "Failed to set Quota for dir root")
g.log.info("Successfully set quota limit for dir root")
# Do some IO until hard limit is reached.
cmd = ("/usr/bin/env python %s create_files "
"-f 1024 --fixed-file-size 1M --base-file-name file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
self.all_mounts_procs.append(proc)
# Wait for IO to complete and validate IO
self.assertTrue(
wait_for_io_to_complete(self.all_mounts_procs, self.mounts[0]),
"IO failed on some of the clients")
g.log.info("IO completed on the clients")
# Validate quota
ret = quota_validate(self.mnode,
self.volname,
path='/',
hard_limit=1073741824,
sl_exceeded=True,
hl_exceeded=True)
self.assertTrue(ret, "Quota validate Failed for '/'")
g.log.info("Quota Validated for path '/'")
# Compute arequal checksum.
arequal_checksum_before_rebalance = collect_mounts_arequal(self.mounts)
# Log Volume info and status before expanding volume.
log_volume_info_and_status(self.mnode, self.volname)
# Expand the volume.
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Failed to expand the volume %s", self.volname))
g.log.info("Expanding volume is successful on "
"volume %s", self.volname)
# Log volume info and status after expanding volume.
log_volume_info_and_status(self.mnode, self.volname)
# Perform rebalance start operation.
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to start rebalance on the volume "
"%s", self.volname))
g.log.info("Rebalance started.")
# Check rebalance is in progress
rebalance_status = get_rebalance_status(self.mnode, self.volname)
ret = rebalance_status['aggregate']['statusStr']
self.assertEqual(ret, "in progress", ("Rebalance is not in "
"'in progress' state, either "
"rebalance is in completed state"
" or failed to get rebalance "
"status"))
g.log.info("Rebalance is 'in progress' state")
# Wait till rebalance ends.
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(ret, ("Rebalance is not yet complete on the volume "
"%s", self.volname))
g.log.info("Rebalance is successfully complete on the volume %s",
self.volname)
# Validate quota
ret = quota_validate(self.mnode,
self.volname,
path='/',
hard_limit=1073741824,
sl_exceeded=True,
hl_exceeded=True)
self.assertTrue(ret, "Quota validate Failed for '/'")
g.log.info("Quota Validated for path '/'")
# Compute arequal checksum.
arequal_checksum_after_rebalance = collect_mounts_arequal(self.mounts)
# Comparing arequals checksum before and after rebalance.
self.assertEqual(arequal_checksum_before_rebalance,
arequal_checksum_after_rebalance,
"arequal checksum is NOT MATCHING")
g.log.info("arequal checksum is SAME")
def test_brick_removal_with_quota(self):
"""
Test Brick removal with quota in place
1. Create Volume of type distribute
2. Set Quota limit on the directory
3. Do some IO to reach the Hard limit
4. After IO ends, remove bricks
5. Quota validation should succeed.
"""
# Enable Quota
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(
ret, 0, ("Failed to enable quota on the volume 5s", self.volname))
g.log.info("Successfully enabled quota on volume %s", self.volname)
# Set the Quota timeouts to 0 for strict accounting
ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, 0)
self.assertEqual(
ret, 0, ("Failed to set hard-timeout to 0 for %s", self.volname))
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, 0)
self.assertEqual(
ret, 0, ("Failed to set soft-timeout to 0 for %s", self.volname))
g.log.info("Quota soft and hard timeout has been set to 0 for %s",
self.volname)
# Set the quota limit of 100 MB on root dir of the volume
ret, _, _ = quota_limit_usage(self.mnode, self.volname, "/", "100MB")
self.assertEqual(ret, 0, "Failed to set Quota for dir root")
g.log.info("Successfully set quota limit for dir root")
# Do some IO until hard limit is reached.
cmd = ("/usr/bin/env python %s create_files "
"-f 100 --fixed-file-size 1M --base-file-name file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
self.all_mounts_procs.append(proc)
# Wait for IO to complete and validate IO
self.assertTrue(
wait_for_io_to_complete(self.all_mounts_procs, self.mounts[0]),
"IO failed on some of the clients")
g.log.info("IO completed on the clients")
# Validate quota
ret = quota_validate(self.mnode,
self.volname,
path='/',
hard_limit=104857600,
sl_exceeded=True,
hl_exceeded=True)
self.assertTrue(ret, "Quota validate Failed for '/'")
g.log.info("Quota Validated for path '/'")
# Log Volume info and status before shrinking volume.
log_volume_info_and_status(self.mnode, self.volname)
# Shrink the volume.
ret = shrink_volume(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to shrink volume on "
"volume %s", self.volname))
g.log.info("Shrinking volume is successful on "
"volume %s", self.volname)
# Log volume info and status after shrinking volume.
log_volume_info_and_status(self.mnode, self.volname)
# Perform rebalance start operation.
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to start rebalance on the volume "
"%s", self.volname))
g.log.info("Rebalance started.")
# Wait till rebalance ends.
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(ret, ("Rebalance is not yet complete on the volume "
"%s", self.volname))
g.log.info("Rebalance is successfully complete on the volume %s",
self.volname)
# Validate quota
ret = quota_validate(self.mnode,
self.volname,
path='/',
hard_limit=104857600,
sl_exceeded=True,
hl_exceeded=True)
self.assertTrue(ret, "Quota validate Failed for '/'")
g.log.info("Quota Validated for path '/'")
def test_quota_add_brick(self):
"""
Verifying quota functionality with respect to the
add-brick without rebalance
* Enable Quota
* Set limit of 1GB on /
* Mount the volume
* Create some random amount of data inside each directory until quota
is reached
* Perform a quota list operation
* Perform add-brick
* Trying add files and see if quota is honored.
"""
# Enable Quota
g.log.info("Enabling quota on the volume %s", self.volname)
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(
ret, 0, ("Failed to enable quota on the volume %s", self.volname))
g.log.info("Successfully enabled quota on the volume %s", self.volname)
# Path to set quota limit
path = "/"
# Set Quota limit on the root of the volume
g.log.info("Set Quota Limit on the path %s of the volume %s", path,
self.volname)
ret, _, _ = quota_limit_usage(self.mnode,
self.volname,
path=path,
limit="1GB")
self.assertEqual(ret, 0, ("Failed to set quota limit on path %s of "
"the volume %s", path, self.volname))
g.log.info("Successfully set the Quota limit on %s of the volume %s",
path, self.volname)
# Set soft timeout to 0 second
g.log.info("Set quota soft timeout:")
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, '0sec')
self.assertEqual(ret, 0, ("Failed to set soft timeout"))
g.log.info("Quota soft timeout set successful")
# Set hard timeout to 0 second
g.log.info("Set quota hard timeout:")
ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, '0sec')
self.assertEqual(ret, 0, ("Failed to set hard timeout"))
g.log.info("Quota hard timeout set successful")
mount_obj = self.mounts[0]
mount_dir = mount_obj.mountpoint
client = mount_obj.client_system
# Create data inside each directory from mount point
g.log.info("Creating Files on %s:%s", client, mount_dir)
cmd = ("cd %s/ ; "
"for i in `seq 100` ; "
"do dd if=/dev/zero of=testfile1$i "
"bs=10M "
"count=1 ; "
"done" % (mount_dir))
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, ("Failed to create files"))
g.log.info("Files created successfully")
# Quota validate
ret = quota_validate(self.mnode,
self.volname,
path=path,
hard_limit=1073741824,
sl_exceeded=True,
hl_exceeded=False)
self.assertTrue(ret, "Quota validate Failed for /")
# Add brick by forming the brick list
# Form bricks list for add-brick command based on the voltype
if 'replica_count' in self.volume['voltype']:
new_bricks_count = self.volume['voltype']['replica_count']
elif 'disperse_count' in self.volume['voltype']:
new_bricks_count = self.volume['voltype']['disperse_count']
else:
new_bricks_count = 3
bricks_list = form_bricks_list(self.mnode, self.volname,
new_bricks_count, self.servers,
self.all_servers_info)
g.log.info("new brick list: %s", bricks_list)
# Run add brick command
ret, _, _ = add_brick(self.mnode, self.volname, bricks_list, False)
self.assertEqual(ret, 0, "Failed to add the bricks to the volume")
g.log.info("Successfully added bricks to volume")
# Create data inside each directory from mount point
g.log.info("Creating Files on %s:%s", client, mount_dir)
cmd = ("cd %s/ ; "
"for i in `seq 50` ; "
"do dd if=/dev/zero of=testfile2$i "
"bs=1M "
"count=1 ; "
"done" % (mount_dir))
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 1, ("Failed: Files created successfully"))
g.log.info("Quota limit honored")
# Quota validate
ret = quota_validate(self.mnode,
self.volname,
path=path,
hard_limit=1073741824,
sl_exceeded=True,
hl_exceeded=True)
self.assertTrue(ret, "Quota validate Failed for /")
def test_alert_time_out(self):
"""
Verifying directory quota functionality with respect to
alert-time, soft-timeout and hard-timeout.
* Enable quota
* Set limit on '/'
* Set default-soft-limit to 50%
* Set alert time to 1 sec
* Set soft-timeout to 0 sec
* Set hard timeout to 0 sec
* Check quota list
* Perform some IO such that soft limit is not exceeded
* Check for alert message in brick logfile (NO alert present)
* Perform IO and exceed the soft limit
* Check for alert message in brick logfile (Alert present)
* Remove some files so that usage falls below soft limit
* Create some files such that hard limit is exceeded
* Check for alert message in brick logfile
* Remove some files so that usage falls below soft limit
* Check for alert message in brick logfile (NO alert present)
"""
# pylint: disable=too-many-statements
# Enable Quota
g.log.info("Enabling quota on the volume %s", self.volname)
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertFalse(
ret, "Failed to enable quota on the volume %s" % self.volname)
g.log.info("Successfully enabled quota on the volume %s", self.volname)
# Path to set quota limit
path = "/"
# Create a directory from mount point
mount_obj = self.mounts[0]
mount_dir = mount_obj.mountpoint
client = mount_obj.client_system
g.log.info("Creating dir named 'foo' from client %s", client)
ret = mkdir(client, "%s/foo" % mount_dir)
self.assertTrue(
ret, "Failed to create dir under %s-%s" % (client, mount_dir))
g.log.info("Directory 'foo' created successfully")
# Set Quota limit on the root of the volume
g.log.info("Set Quota Limit on the path %s of the volume %s", path,
self.volname)
ret, _, _ = quota_limit_usage(self.mnode,
self.volname,
path=path,
limit="100MB")
self.assertEqual(ret, 0, ("Failed to set quota limit on path %s of "
" the volume %s", path, self.volname))
g.log.info("Successfully set the Quota limit on %s of the volume %s",
path, self.volname)
# Set default soft limit to 50%
g.log.info("Set default soft limit:")
ret, _, _ = quota_set_default_soft_limit(self.mnode, self.volname,
'50%')
self.assertEqual(ret, 0, ("Failed to set quota default soft limit"))
g.log.info("Quota default soft limit set successfully")
# Check quota list to validate limits
g.log.info("List all files and directories:")
ret = quota_validate(self.mnode,
self.volname,
path=path,
soft_limit_percent=50,
hard_limit=104857600)
self.assertTrue(ret, "Failed to validate Quota list")
g.log.info("Quota List successful")
# Set alert time to 1 second
g.log.info("Set quota alert timeout:")
ret, _, _ = quota_set_alert_time(self.mnode, self.volname, '1sec')
self.assertEqual(ret, 0, ("Failed to set alert timeout"))
g.log.info("Quota alert time set successful")
# Set soft timeout to 0 second
g.log.info("Set quota soft timeout:")
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, '0sec')
self.assertEqual(ret, 0, ("Failed to set soft timeout"))
g.log.info("Quota soft timeout set successful")
# Set hard timeout to 0 second
g.log.info("Set quota hard timeout:")
ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, '0sec')
self.assertEqual(ret, 0, ("Failed to set hard timeout"))
g.log.info("Quota hard timeout set successful")
# Get the brick log file path for a random node
bricks = get_all_bricks(self.mnode, self.volname)
selected_node, brick_path = random.choice(bricks[0:6]).split(':')
brickpath = str.replace(brick_path, '/', '-')
brickpathfinal = brickpath[1:]
brick_log = "/var/log/glusterfs/bricks/%s.log" % brickpathfinal
# Append unique string to the brick log
g.log.info("Appending string 'appended_string_1' to the log:")
append_string_to_file(selected_node, brick_log, "appended_string_1")
# Starting IO on the mounts without crossing the soft limit
# file creation should be normal
g.log.info("Creating Files on %s:%s", client, mount_dir)
cmd = ("cd %s/foo ; "
"for i in `seq 1 9` ; "
"do dd if=/dev/urandom of=file$i "
"bs=5M "
"count=1 ; "
"done" % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, ("Failed to create files on mountpoint"))
g.log.info("Files created succesfully on mountpoint")
# Append unique string to the brick log
g.log.info("Appending string 'appended_string_2' to the log:")
append_string_to_file(selected_node, brick_log, "appended_string_2")
# Soft limit not crossed
# Check if alert message is logged in the brick (shouldn't be logged)
g.log.info("Check for alert message in logfile:")
ret = search_pattern_in_file(selected_node, "120004", brick_log,
"appended_string_1", "appended_string_2")
self.assertFalse(ret, "Found unnecessary alert in logfile")
g.log.info("Alert message not seen before crossing soft limit")
# Continue IO on the mounts to exceed soft limit
g.log.info("Creating Files on %s:%s", client, mount_dir)
cmd = ("cd %s/foo ; "
"for i in `seq 1 200` ; "
"do dd if=/dev/urandom of=foo$i "
"bs=100K "
"count=1 ; "
"done" % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, ("Failed to create files on mountpoint"))
g.log.info("Files created succesfully on mountpoint and "
"exceed soft limit")
# Check if quota soft limit exceeded
g.log.info("Check soft limit:")
ret = quota_validate(self.mnode,
self.volname,
path=path,
sl_exceeded=True)
self.assertTrue(ret, "Failed: soft limit not exceeded")
g.log.info('Quota soft limit exceeded')
# Inserting sleep of 2 seconds so the alert message gets enough time
# to be logged
time.sleep(2)
# Append unique string to the brick log
g.log.info("Appending string 'appended_string_3' to the log:")
append_string_to_file(selected_node, brick_log, "appended_string_3")
# Check if alert message logged in the brick
g.log.info("Check for message:")
ret = search_pattern_in_file(selected_node, "120004", brick_log,
"appended_string_2", "appended_string_3")
self.assertTrue(ret, "Alert message not found")
g.log.info("Pattern Found: got alert message")
# Append unique string to the brick log
g.log.info("Appending string 'appended_string_4' to the log:")
append_string_to_file(selected_node, brick_log, "appended_string_4")
# Continue IO on the mounts by removing data
g.log.info("Removing Files on %s:%s", client, mount_dir)
cmd = ("rm -rfv %s/foo/foo* " % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, ("Failed to delete files on mountpoint"))
g.log.info("Files removed succesfully from mountpoint and reached "
"below soft limit")
# Check if quota soft limit exceeded
g.log.info("Check soft limit:")
ret = quota_validate(self.mnode,
self.volname,
path=path,
sl_exceeded=False)
self.assertTrue(ret, "Failed: soft limit exceeded")
g.log.info('Quota soft limit not exceeded')
# Inserting sleep of 2 seconds so the alert message gets enough time
# to be logged
time.sleep(2)
# Append unique string to the brick log
g.log.info("Appending string 'appended_string_5' to the log:")
append_string_to_file(selected_node, brick_log, "appended_string_5")
# Check if alert message is logged in the brick
g.log.info("Check for message:")
ret = search_pattern_in_file(selected_node, "120004", brick_log,
"appended_string_4", "appended_string_5")
self.assertFalse(ret, "Found unnecessary alert message in logfile")
g.log.info("Alert message not seen before crossing soft limit")
# Continue IO on the mounts to exceed hard limit
g.log.info("Creating Files on %s:%s", client, mount_dir)
cmd = ("cd %s/foo ; "
"for i in `seq 11 20` ; "
"do dd if=/dev/urandom of=file$i "
"bs=10M "
"count=1 ; "
"done" % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 1, ("Failed: Files created successfully inspite "
"of crossing hard-limit"))
g.log.info("Files creation stopped on mountpoint once exceeded "
"hard limit")
# Inserting sleep of 6 seconds so the alert message gets enough time
# to be logged
time.sleep(6)
# Append unique string to the brick log
g.log.info("Appending string 'appended_string_6' to the log:")
append_string_to_file(selected_node, brick_log, "appended_string_6")
# Check if alert message is logged in the brick
g.log.info("Check for message:")
ret = search_pattern_in_file(selected_node, "120004", brick_log,
"appended_string_5", "appended_string_6")
self.assertTrue(ret, "Alert message not seen in logfile")
g.log.info("Pattern Found: got alert message")
# Append unique string to the brick log
g.log.info("Appending string 'Done_with_alert_check_7' to the log:")
append_string_to_file(selected_node, brick_log,
"Done_with_alert_check_7")
# Continue IO on the mounts by removing data to come below hard limit
g.log.info("Removing Files on %s:%s", client, mount_dir)
cmd = ("rm -rfv %s/foo/file{11..20}" % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, ("Failed to delete files on mountpoint"))
g.log.info("Files removed succesfully on mountpoint and "
"reached below soft limit")
# Inserting sleep of 2 seconds so the alert message gets enough time
# to be logged
time.sleep(2)
# Append unique string to the brick log
g.log.info("Appending string 'Done_with_alert_check_8' to the log:")
append_string_to_file(selected_node, brick_log,
"Done_with_alert_check_8")
# Check if alert message is logged in the brick
g.log.info("Check for message:")
ret = search_pattern_in_file(selected_node, "120004", brick_log,
"Done_with_alert_check_7",
"Done_with_alert_check_8")
self.assertFalse(ret, "Found unnecessary alert in logfile")
g.log.info("EXPECTED: Alert message not seen before crossing "
"soft limit")
# have got below the soft and hard limit
# check quota list
g.log.info("List all files and directories:")
ret = quota_validate(self.mnode,
self.volname,
path=path,
sl_exceeded=False,
hl_exceeded=False)
self.assertTrue(
ret, "Failed to validate Quota list with "
"soft and hard limits")
g.log.info("Quota List validated successfully")
def test_heal_when_quota_object_limit_exceeded(self):
# Create a directory to set the quota_limit_objects
path = "/dir"
g.log.info("Creating a directory")
self.all_mounts_procs = []
for mount_object in self.mounts:
cmd = "/usr/bin/env python %s create_deep_dir -d 0 -l 0 %s%s" % (
self.script_upload_path, mount_object.mountpoint, path)
ret = g.run(mount_object.client_system, cmd)
self.assertTrue(ret, "Failed to create directory on mountpoint")
g.log.info("Directory created successfully on mountpoint")
# Enable Quota
g.log.info("Enabling quota on the volume %s", self.volname)
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to enable quota on the volume "
"%s", self.volname))
g.log.info("Successfully enabled quota on the volume %s", self.volname)
# Set quota-soft-timeout to 0
g.log.info("Setting up soft timeout to 0")
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, "0")
self.assertEqual(ret, 0, ("Failed to set quota-soft-timeout"))
g.log.info("Successfully set the quota-soft-timeout")
# Set quota-hard-timeout to 0
g.log.info("Setting up hard timeout with 0")
ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, "0")
self.assertEqual(ret, 0, ("Failed to set quota-hard-timeout"))
g.log.info("successfully set the quota-hard-timeout")
# Set Quota limit on the newly created directory
g.log.info("Set Quota Limit on the path %s of the volume %s", path,
self.volname)
ret, _, _ = quota_limit_objects(self.mnode,
self.volname,
path=path,
limit="5")
self.assertEqual(ret, 0, ("Failed to set quota limit on path %s of "
" the volume %s", path, self.volname))
g.log.info(
"Successfully set the quota limit on %s of the volume "
"%s", path, self.volname)
# Create 3 files inside the directory
for mount_object in self.mounts:
g.log.info("Creating Files on %s:%s", mount_object.client_system,
path)
cmd = ("/usr/bin/env python %s create_files -f 3 "
"--base-file-name file-0 %s%s" %
(self.script_upload_path, mount_object.mountpoint, path))
ret, _, _ = g.run(mount_object.client_system, cmd)
self.assertEqual(ret, 0, ("Failed to create files on %s", path))
g.log.info("Files created successfully on mountpoint")
bricks_list = get_all_bricks(self.mnode, self.volname)
# Bring brick3 offline
g.log.info('Bringing brick %s offline', bricks_list[2])
ret = bring_bricks_offline(self.volname, bricks_list[2])
self.assertTrue(ret,
'Failed to bring bricks %s offline' % bricks_list[2])
ret = are_bricks_offline(self.mnode, self.volname, [bricks_list[2]])
self.assertTrue(ret, 'Brick %s is not offline' % bricks_list[2])
g.log.info('Bringing brick %s offline is successful', bricks_list[2])
# Try creating 5 more files, which should fail as the quota limit
# exceeds
cmd = ("/usr/bin/env python %s create_files -f 5 --base-file-name "
"file-1 %s%s" %
(self.script_upload_path, mount_object.mountpoint, path))
ret, _, _ = g.run(mount_object.client_system, cmd)
self.assertNotEqual(ret, 0, ("Creating 5 files succeeded while it was"
"not supposed to."))
g.log.info("Creating 5 files failed as expected due to quota object"
"limit on the directory.")
# Bring brick3 online and check status
g.log.info('Bringing brick %s online', bricks_list[2])
ret = bring_bricks_online(self.mnode, self.volname, [bricks_list[2]])
self.assertTrue(ret,
'Failed to bring brick %s online' % bricks_list[2])
g.log.info('Bringing brick %s online is successful', bricks_list[2])
g.log.info("Verifying if brick %s is online", bricks_list[2])
ret = are_bricks_online(self.mnode, self.volname, bricks_list)
self.assertTrue(ret, ("Brick %s did not come up", bricks_list[2]))
g.log.info("Brick %s has come online.", bricks_list[2])
# Trigger heal
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Starting heal failed')
g.log.info('Index heal launched')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
def test_quota_file_larger_than_limit(self):
# pylint: disable=too-many-statements
"""
Verifying directory Quota functionality with respect to the
limit-usage option.
If a limit is set and a file of size larger than limit is created
then the file creation will stop when it will reach the limit.
Quota list will show limit-set and size as same.
* Enable Quota
* Create a directory from mount point
* Set a limit of 10 MB on the directory
* Set Quota soft-timeout and hard-timeout to 0 seconds
* Create a file of size larger than the Quota limit
eg. 20 MB file
* Perform Quota list operation to check if all the fields are
appropriate such as hard_limit, available_space, sl_exceeded,
hl_execeeded, etc.
"""
# Enable Quota
g.log.info("Enabling Quota on the volume %s", self.volname)
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(
ret, 0, ("Failed to enable Quota on the volume %s", self.volname))
g.log.info("Successfully enabled Quota on the volume %s", self.volname)
# Path to set the Quota limit
path = '/foo'
# Create a directory 'foo' from the mount point
mount_obj = self.mounts[0]
mount_dir = mount_obj.mountpoint
client = mount_obj.client_system
g.log.info("Creating dir named 'foo' from client %s", client)
ret = mkdir(client, "%s/foo" % mount_dir)
self.assertTrue(
ret, "Failed to create dir under %s-%s" % (client, mount_dir))
g.log.info("Directory 'foo' created successfully")
# Set Quota limit of 10 MB on the directory 'foo' of the volume
g.log.info("Set Quota Limit on the path %s of the volume %s", path,
self.volname)
ret, _, _ = quota_limit_usage(self.mnode,
self.volname,
path=path,
limit="10MB")
self.assertEqual(ret, 0, ("Failed to set Quota limit on path %s of "
"the volume %s", path, self.volname))
g.log.info("Successfully set the Quota limit on %s of the volume %s",
path, self.volname)
# Set Quota soft-timeout to 0 seconds
g.log.info("Set Quota soft timeout:")
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, '0sec')
self.assertEqual(ret, 0, ("Failed to set soft timeout"))
g.log.info("Quota soft timeout set successful")
# Set Quota hard-timeout to 0 second
g.log.info("Set Quota hard timeout:")
ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, '0sec')
self.assertEqual(ret, 0, ("Failed to set hard timeout"))
g.log.info("Quota hard timeout set successful")
# Validate if the Quota limit set is appropriate
g.log.info(
"Validate if the Quota limit set is correct for the "
"directory %s of the volume %s", path, self.volname)
ret = quota_validate(self.mnode,
self.volname,
path=path,
hard_limit=10485760)
self.assertTrue(
ret, ("Quota Limit of 10 MB was not set properly on "
"the directory %s of the volume %s", path, self.volname))
g.log.info(
"Successfully Validated Quota Limit of 10 MB is set on the"
" directory %s of the volume %s", path, self.volname)
# Create a single file of size 20 MB
g.log.info("Creating Files on %s:%s", client, mount_dir)
cmd = ("cd %s/foo ; "
"dd if=/dev/zero of=20MBfile "
"bs=1M "
"count=20" % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(
ret, 1, "Unexpected: File creation succeeded even "
"after exceeding the hard-limit")
g.log.info("Expected: File creation failed after exceeding "
"hard-limit")
# List all files and dirs created
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
self.assertTrue(ret, "Failed to list all files and dirs")
g.log.info("Listing all files and directories is successful")
# Check if the file created above exists
g.log.info("Checking if the file created exists in the volume %s",
self.volname)
ret = file_exists(client, "%s/foo/20MBfile" % mount_dir)
self.assertTrue(ret,
("File does not exist in the volume %s", self.volname))
g.log.info(
"Successfully validated the presence of file in the "
"volume %s", self.volname)
# Validate if the Quota limit set is appropriate
g.log.info(
"Validate if the Quota list fields are appropriate for the "
"directory %s of the volume %s", path, self.volname)
ret = quota_validate(self.mnode,
self.volname,
path=path,
hard_limit=10485760,
avail_space=0,
sl_exceeded=True,
hl_exceeded=True)
self.assertTrue(ret, ("Failed to validate the Quota limits on "
"the volume %s", self.volname))
g.log.info(
"Successfully Validated Quota Limit of 100 MB is set on the"
" directory %s of the volume %s", path, self.volname)
def test_quota_rebalance(self):
"""
* Enable quota on the volume
* set hard and soft time out to zero.
* Create some files and directories from mount point
so that the limits are reached.
* Perform add-brick operation on the volume.
* Start rebalance on the volume.
* While rebalance is in progress, create some more files
and directories from the mount point until limit is hit
"""
# pylint: disable=too-many-statements
# Enable Quota on volume
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(
ret, 0, ("Failed to enable quota on the volume %s", self.volname))
g.log.info("Successfully enabled quota on the volume %s", self.volname)
# Set the Quota timeouts to 0 for strict accounting
ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, 0)
self.assertEqual(
ret, 0, ("Failed to set hard-timeout to 0 for %s", self.volname))
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, 0)
self.assertEqual(
ret, 0, ("Failed to set soft-timeout to 0 for %s", self.volname))
g.log.info("Quota soft and hard timeout has been set to 0 for %s",
self.volname)
# Set limit of 100 MB on root dir of the volume
ret, _, _ = quota_limit_usage(self.mnode, self.volname, "/", "100MB")
self.assertEqual(ret, 0, "Failed to set Quota for dir '/'")
g.log.info("Successfully set quota limit for dir '/'")
# Do some IO until hard limit is reached
cmd = ("/usr/bin/env python %s create_files "
"-f 100 --fixed-file-size 1M --base-file-name file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
self.all_mounts_procs.append(proc)
# Wait for IO to complete and validate IO
self.assertTrue(
wait_for_io_to_complete(self.all_mounts_procs, self.mounts[0]),
"IO failed on some of the clients")
g.log.info("IO is successful on all mounts")
# Add bricks to the volume
if "replica_count" in self.volume["voltype"]:
new_bricks_count = self.volume["voltype"]["replica_count"]
elif "disperse_count" in self.volume["voltype"]:
new_bricks_count = self.volume["voltype"]["disperse_count"]
else:
new_bricks_count = 3
bricks_list = form_bricks_list(self.mnode, self.volname,
new_bricks_count, self.servers,
self.all_servers_info)
g.log.info("new brick list: %s", bricks_list)
ret, _, _ = add_brick(self.mnode, self.volname, bricks_list, False)
self.assertEqual(ret, 0, "Failed to add the bricks to the volume")
g.log.info("Successfully added bricks to volume")
# Perform rebalance start operation
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, "Rebalance Start Failed")
# Wait for at least one file to be lookedup/scanned on the nodes
status_info = get_rebalance_status(self.mnode, self.volname)
count = 0
while count < 20:
lookups_start_count = 0
for node in range(len(status_info['node'])):
status_info = get_rebalance_status(self.mnode, self.volname)
lookups_file_count = status_info['node'][node]['lookups']
if int(lookups_file_count) > 0:
lookups_start_count += 1
sleep(2)
if lookups_start_count == len(self.servers):
g.log.info(
"Volume %s: At least one file is lookedup/scanned "
"on all nodes", self.volname)
break
count += 1
# Perform some more IO and check if hard limit is honoured
self.all_mounts_procs = []
cmd = ("/usr/bin/env python %s create_files "
"-f 100 --fixed-file-size 1M --base-file-name newfile %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
self.all_mounts_procs.append(proc)
# Wait for IO to complete and validate IO
# This should fail as the quotas were already reached
self.assertFalse(
validate_io_procs(self.all_mounts_procs, self.mounts[0]),
"Unexpected: IO passed on the client even after quota is reached")
g.log.info("Expected: IO failed as quota is reached")
# Wait for rebalance to finish
ret = wait_for_rebalance_to_complete(self.mnode,
self.volname,
timeout=180)
self.assertTrue(ret, "Unexpected: Rebalance did not complete")
g.log.info("Rebalance completed as expected")
def test_heal_when_dir_quota_exceeded_(self):
# Create a directory to set the quota_limit_usage
path = "/dir"
g.log.info("Creating a directory")
self.all_mounts_procs = []
for mount_object in self.mounts:
cmd = ("python %s create_deep_dir -d 0 -l 0 %s%s "
% (self.script_upload_path, mount_object.mountpoint,
path))
ret = g.run(mount_object.client_system, cmd)
self.assertTrue(ret, "Failed to create directory on mountpoint")
g.log.info("Directory created successfully on mountpoint")
# Enable Quota
g.log.info("Enabling quota on the volume %s", self.volname)
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to enable quota on the volume "
"%s", self.volname))
g.log.info("Successfully enabled quota on the volume %s",
self.volname)
# Set quota-soft-timeout to 0
g.log.info("Setting up soft timeout to 0")
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, "0")
self.assertEqual(ret, 0, ("Failed to set quota-soft-timeout"))
g.log.info("Successfully set the quota-soft-timeout")
# Set quota-hard-timeout to 0
g.log.info("Setting up hard timeout with 0")
ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, "0")
self.assertEqual(ret, 0, ("Failed to set quota-hard-timeout"))
g.log.info("successfully set the quota-hard-timeout")
# Set Quota limit on the newly created directory
g.log.info("Set Quota Limit on the path %s of the volume %s",
path, self.volname)
ret, _, _ = quota_limit_usage(self.mnode, self.volname,
path=path, limit="1GB")
self.assertEqual(ret, 0, ("Failed to set quota limit on path %s of "
" the volume %s", path, self.volname))
g.log.info("Successfully set the Quota limit on %s of the volume "
"%s", path, self.volname)
# Create 2 files of size 400MB inside the directory
for mount_object in self.mounts:
g.log.info("Creating Files on %s:%s", mount_object.client_system,
path)
cmd = ("cd %s%s && for i in `seq 1 2` ;"
"do dd if=/dev/urandom of=file$i bs=20M "
"count=20; done" % (mount_object.mountpoint, path))
ret, _, _ = g.run(mount_object.client_system, cmd)
self.assertEqual(ret, 0, ("Failed to create files on %s", path))
g.log.info("Files created successfully on mountpoint")
bricks_list = get_all_bricks(self.mnode, self.volname)
# Bring brick2 offline
g.log.info('Bringing bricks %s offline', bricks_list[2])
ret = bring_bricks_offline(self.volname, bricks_list[2])
self.assertTrue(ret, 'Failed to bring bricks %s offline'
% bricks_list[2])
ret = are_bricks_offline(self.mnode, self.volname,
[bricks_list[2]])
self.assertTrue(ret, 'Brick %s is not offline'
% bricks_list[2])
g.log.info('Bringing brick %s offline is successful',
bricks_list[2])
# Create a file of size 500MB inside the directory and it should fail
# as the quota limit exceeds
cmd = ("cd %s%s && dd if=/dev/urandom of=file3 bs=20M count=25"
% (mount_object.mountpoint, path))
ret, _, _ = g.run(mount_object.client_system, cmd)
self.assertEqual(ret, 1, ("Writing a file of 500MB succeeded while "
"it was not supposed to."))
g.log.info("Writing a file of size 500MB failed as expected "
"due to quota limit on the directory.")
# Bring brick2 online and check status
g.log.info('Bringing brick %s online...', bricks_list[2])
ret = bring_bricks_online(self.mnode, self.volname,
[bricks_list[2]])
self.assertTrue(ret, 'Failed to bring brick %s online' %
bricks_list[2])
g.log.info('Bringing brick %s online is successful', bricks_list[2])
g.log.info("Verifying if brick %s is online", bricks_list[2])
ret = are_bricks_online(self.mnode, self.volname, bricks_list)
self.assertTrue(ret, ("Brick %s did not come up", bricks_list[2]))
g.log.info("Brick %s has come online.", bricks_list[2])
# Trigger heal
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Starting heal failed')
g.log.info('Index heal launched')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
def test_subdir_with_quotaobject(self):
# pylint: disable=too-many-statements
"""
Mount the volume
Create 1 subdir on mountpoint "d1"
unmount volume
Auth allow - Client1(d1),Client2(full volume)
Mount the subdir "d1" on client1 and volume on client2
Enable quota on volume
Set quota object limit on subdir "d1" and volume
subdir "d1" quota limit- 50
Volume quota limit - 200
Start writing 49 files on both subdir "d1" and volume
Fetch quota limit object list
Write 1 more file on subdir.This should fail
Again reset quota object limit to 75 now on subdir "d1"
Create 24 directories on subdir and volume.This should pass
Fetch quota limit object list
Create 1 more directory on subdir.This should fail
Create 1 more directory on volume.This should pass
"""
# Create directory d1 on mount point
ret = mkdir(self.mounts[0].client_system,
"%s/d1" % self.mounts[0].mountpoint)
self.assertTrue(
ret, ("Failed to create directory 'd1' on "
"volume %s from client %s" %
(self.mounts[0].volname, self.mounts[0].client_system)))
# unmount volume
ret = self.unmount_volume(self.mounts)
self.assertTrue(ret, "Volumes Unmount failed")
g.log.info("Volumes Unmounted successfully")
# Set authentication on the subdirectoy "d1" to access by client1
# and volume to access by client2
g.log.info(
'Setting authentication on subdirectory d1 to access '
'by client %s and on volume to access by client %s',
self.clients[0], self.clients[1])
ret = set_auth_allow(self.volname, self.mnode, {
'/d1': [self.clients[0]],
'/': [self.clients[1]]
})
self.assertTrue(
ret, 'Failed to set Authentication on volume %s' % self.volume)
# Creating mount list for mounting subdir mount and volume
self.subdir_mounts = [
copy.deepcopy(self.mounts[0]),
copy.deepcopy(self.mounts[1])
]
self.subdir_mounts[0].volname = "%s/d1" % self.volname
# Mount Subdirectory d1 on client 1 and volume on client 2
for mount_obj in self.subdir_mounts:
ret = mount_obj.mount()
self.assertTrue(
ret, ("Failed to mount %s on client"
" %s" % (mount_obj.volname, mount_obj.client_system)))
g.log.info("Successfully mounted %s on client %s",
mount_obj.volname, mount_obj.client_system)
g.log.info("Successfully mounted sub directory and volume to "
"authenticated clients")
# Enable quota on volume
g.log.info("Enabling quota on the volume %s", self.volname)
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to enable quota on the volume "
"%s", self.volname))
g.log.info("Successfully enabled quota on the volume %s", self.volname)
# Check if quota is enabled
g.log.info("Validate Quota is enabled on the volume %s", self.volname)
ret = is_quota_enabled(self.mnode, self.volname)
self.assertTrue(
ret, ("Quota is not enabled on the volume %s", self.volname))
g.log.info("Successfully Validated quota is enabled on volume %s",
self.volname)
# Set quota-soft-timeout to 0
g.log.info("Setting up soft timeout to 0")
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, "0")
self.assertEqual(ret, 0, ("Failed to set quota-soft-timeout"))
g.log.info("Successfully set the quota-soft-timeout")
# Set quota-hard-timeout to 0
g.log.info("Setting up hard timeout with 0")
ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, "0")
self.assertEqual(ret, 0, ("Failed to set quota-hard-timeout"))
g.log.info("successfully set the quota-hard-timeout")
# Set Quota object limit on the subdir "d1" and on volume
for mount_obj in self.subdir_mounts:
if mount_obj.volname == "%s/d1" % self.volname:
path1 = "/d1"
limit = "50"
else:
path1 = "/"
limit = "200"
g.log.info("Set Quota Limit on the path %s of the volume %s",
path1, self.volname)
ret, _, _ = quota_limit_objects(self.mnode, self.volname, path1,
limit)
self.assertEqual(ret, 0,
("Failed to set quota limit on path "
"%s of the volume %s", path1, self.volname))
g.log.info(
"Successfully set the quota limit on %s of the volume "
"%s", path1, self.volname)
# Create near to 49 files on both subdir mount and volume mount
for mount_object in self.subdir_mounts:
g.log.info("Creating Files on %s:%s", mount_object.client_system,
mount_object.mountpoint)
cmd = ("cd %s ; for i in `seq 1 49` ;"
"do touch $i;done " % (mount_object.mountpoint))
ret, _, _ = g.run(mount_object.client_system, cmd)
self.assertEqual(ret, 0, "Failed to create files on mountpoint")
g.log.info("Files created successfully on mountpoint")
# Fetch Quota List object on the volume
g.log.info("Get Quota list on the volume %s", self.volname)
quota_list = quota_fetch_list_objects(self.mnode, self.volname)
self.assertIsNotNone(quota_list, ("Failed to get the quota list "
"of the volume %s", self.volname))
# Create 1 file on subdir to check if quota limit is
# adhere by subdir d1
g.log.info("Creating File on %s:%s", self.clients[0],
self.subdir_mounts[0].mountpoint)
cmd = ("cd %s ; touch test " % (self.subdir_mounts[0].mountpoint))
ret, _, _ = g.run(self.clients[0], cmd)
self.assertNotEqual(ret, 0, ("File creation was expected to Fail."
"But it got passed"))
g.log.info(
"File creation failed as expected on %s:%s as quota"
" limit reached already", self.clients[0],
self.subdir_mounts[0].mountpoint)
# Modify quota object limit for subdir from 50 to 75
path1 = "/d1"
g.log.info("Set Quota Limit on the path %s of the volume %s", path1,
self.volname)
ret, _, _ = quota_limit_objects(self.mnode, self.volname, path1, "75")
self.assertEqual(ret, 0, ("Failed to set quota limit on path %s of "
" the volume %s", path1, self.volname))
g.log.info(
"Successfully set the quota limit on %s of the volume "
"%s", path1, self.volname)
# Create near to 25 directories on both subdir mount "d1" and volume
for mount_object in self.subdir_mounts:
g.log.info("Creating directories on %s:%s",
mount_object.client_system, mount_object.mountpoint)
for i in range(0, 25):
ret = mkdir(mount_object.client_system,
"%s/dir%s" % (mount_object.mountpoint, i),
parents=True)
self.assertTrue(ret, "Failed to create directories"
"on mountpoint")
g.log.info("Directories created successfully on mountpoint")
# Get Quota List on the volume
g.log.info("Get Quota list on the volume %s", self.volname)
quota_list = quota_fetch_list_objects(self.mnode, self.volname)
self.assertIsNotNone(quota_list, ("Failed to get the quota list "
"of the volume %s", self.volname))
# Create 1 directory on subdir "d1" and volume to check if quota
# limit is adhere by subdir d1 and volume
for mount_object in self.subdir_mounts:
g.log.info("Creating directory on %s:%s",
mount_object.client_system, mount_object.mountpoint)
ret = mkdir(mount_object.client_system,
"%s/dirTest" % mount_object.mountpoint,
parents=True)
if mount_object.volname == "%s/d1" % self.volname:
self.assertFalse(
ret, "Directory creation was expected"
"to Fail.But it got passed")
g.log.info("Direction creation failed as expected on"
"subdir d1")
else:
self.assertTrue(ret, "Directory creation got failed"
"on volume")
g.log.info("Direction creation successful on volume")
def test_quota_unique_soft_limit(self):
"""
Validating directory quota functionality WRT soft-limit
* Enable quota
* Create 10 directories
* Set a hard-limit of 100 MB and a unique soft-limit on each directory
example : 5%, 15%, 25%, 35%, ...
* Create some data inside the directories such that the
soft-limit is exceeded
* Perform quota list operation to validate if the
soft-limit is exceeded
"""
# pylint: disable=too-many-statements
# Enable Quota on the volume
g.log.info("Enabling quota on the volume %s", self.volname)
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(
ret, 0, ("Failed to enable quota on the volume %s", self.volname))
g.log.info("Successfully enabled quota on the volume %s", self.volname)
# Set soft timeout to 0 second
g.log.info("Set quota soft timeout:")
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, '0sec')
self.assertEqual(ret, 0, ("Failed to set soft timeout"))
g.log.info("Quota soft timeout set successful")
# Create 10 directories from the mount point
mount_obj = self.mounts[0]
mount_dir = mount_obj.mountpoint
client = mount_obj.client_system
g.log.info("Creating directories on %s:%s", client, mount_dir)
for i in range(1, 11):
ret = mkdir(client, "%s/foo%s" % (mount_dir, i))
self.assertTrue(
ret, ("Failed to create dir under %s-%s", client, mount_dir))
g.log.info("Directory 'foo%s' created successfully", i)
g.log.info("Successfully created directories on %s:%s", client,
mount_dir)
# Set a hard-limit of 100 MB with a unique soft-limit
# having a difference of 10% on each directory
g.log.info(
"Setting a limit of 100 MB and unique soft-limit on all "
"the directories inside the volume %s", self.volname)
var1 = 1
for var2 in range(5, 100, 10):
dir_name = "/foo" + str(var1)
softlim = str(var2) + "%"
ret, _, _ = quota_limit_usage(self.mnode,
self.volname,
path=dir_name,
limit="100MB",
soft_limit=softlim)
self.assertEqual(ret, 0,
("Failed to set quota limits on path "
"%s with soft-limit %s for the "
"volume %s", dir_name, softlim, self.volname))
g.log.info(
"Successfully set the Quota limits on %s of "
"the volume %s", dir_name, self.volname)
var1 = var1 + 1
# Validate hard/soft limits set above , Create data inside the
# directories such that soft limit is exceeded and validate with
# quota list command
var3 = 1
for var4 in range(5, 100, 10):
dir_name = "/foo" + str(var3)
softlim = var4
data = var4 + 1
# Validate if the hard limit and soft limit is set properly
# on all directories
g.log.info(
"Validate quota limit usage on the directory %s of the "
"volume %s", dir_name, self.volname)
ret = quota_validate(self.mnode,
self.volname,
path=dir_name,
hard_limit=104857600,
soft_limit_percent=softlim)
self.assertTrue(
ret,
("Failed to validate quota limit usage on the"
" directory %s of the volume %s", dir_name, self.volname))
g.log.info(
"Successfully validated quota limit usage for the "
"directory %s of the volume %s", dir_name, self.volname)
# Perform IO on each Directory
g.log.info("Creating Files on %s:%s", client, mount_dir)
cmd = ("cd %s%s ; "
"dd if=/dev/zero of=foo%s "
"bs=%sM "
"count=1 " % (mount_dir, dir_name, var3, data))
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0,
("Failed to create files on %s", dir_name))
g.log.info("Files created successfully on %s", dir_name)
time.sleep(1)
# Validate quota list and check if soft-limit is exceeded for
# each directory
g.log.info(
"Validate quota limit usage and soft-limit on the "
"directory %s", dir_name)
ret = quota_validate(self.mnode,
self.volname,
path=dir_name,
hard_limit=104857600,
soft_limit_percent=softlim,
sl_exceeded=True)
self.assertTrue(
ret,
("Failed to validate quota limit usage on the"
" directory %s of the volume %s", dir_name, self.volname))
g.log.info(
"Successfully validated quota limit usage for the "
"directory %s of volume %s", dir_name, self.volname)
var3 = var3 + 1
def test_limit_usage_deep_dir(self):
# pylint: disable=too-many-statements
"""
Verifying directory quota functionality with respect to the
limit-usage option. Set limits on various directories [breadth]
and check for the quota list of all the directories.
* Enable Quota
* Create 10 directories one inside the other and set limit of 1GB
on each directory
* Perform a quota list operation
* Create some random amount of data inside each directory
* Perform a quota list operation
* Remove the quota limit and delete the data
"""
# Enable Quota
g.log.info("Enabling quota on the volume %s", self.volname)
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to enable quota on the volume %s",
self.volname))
g.log.info("Successfully enabled quota on the volume %s", self.volname)
# Create deep directories in the mount point
for mount_object in self.mounts:
g.log.info("Creating directories on %s:%s",
mount_object.client_system, mount_object.mountpoint)
ret = mkdir(mount_object.client_system,
"%s/dir1/dir2/dir3/dir4/dir5/dir6/dir7/dir8/dir9/dir10"
% (mount_object.mountpoint), parents=True)
self.assertTrue(ret, ("Failed to create dir under %s-%s",
mount_object.client_system,
mount_object.mountpoint))
g.log.info("Successfully created deep directories on %s:%s",
mount_object.client_system, mount_object.mountpoint)
# Validate IO
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients"
)
# Set soft timeout to 1 second
g.log.info("Set quota soft timeout:")
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, '1sec')
self.assertEqual(ret, 0, ("Failed to set soft timeout"))
g.log.info("Quota soft timeout set successful")
# Set hard timeout to 0 second
g.log.info("Set quota hard timeout:")
ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, '0sec')
self.assertEqual(ret, 0, ("Failed to set hard timeout"))
g.log.info("Quota hard timeout set successful")
# Get dir list
g.log.info('Getting dir list in %s', self.volname)
cmd = ("ls -R %s | grep ':' | tr -d :" % self.mounts[0].mountpoint)
ret, out, err = g.run(self.mounts[0].client_system, cmd)
g.log.info('mountpoint %s', self.mounts[0].mountpoint)
self.assertFalse(ret, err)
dir_list = out.split()
for dir_name in dir_list:
# Parsed to remove the mount point as quota doesn't work when
# passed with mountpoint.
tmp_name = dir_name.replace(self.mounts[0].mountpoint, "")
dir_list[dir_list.index(dir_name)] = '%s' % tmp_name
dir_list.pop(0)
# The first entry of ls -R is the current directory which is not
# necessary.
# Set limit of 1 GB on every directory created inside the mountpoint
g.log.info("Set Quota Limit on each directory of the volume %s",
self.volname)
for dir_name in dir_list:
ret, _, _ = quota_limit_usage(self.mnode, self.volname,
dir_name, '1GB')
self.assertFalse(ret, "Failed to set Quota for dir %s" %
dir_name)
g.log.info("Set quota for dir %s successfully", dir_name)
g.log.info("Successfully set the Quota limit on each path of the "
"volume %s", self.volname)
# Validate quota on every Directory of the Volume
g.log.info("Get Quota list for every directory on the volume %s",
self.volname)
for dir_name in dir_list:
ret = quota_validate(self.mnode, self.volname, path=dir_name,
hard_limit=1073741824)
self.assertTrue(ret, "Quota validate Failed for dir %s" %
dir_name)
# Create some data inside each directory and do a quota validate
self.all_mounts_procs = []
for mount_object in self.mounts:
g.log.info("Creating Files on %s:%s", mount_object.client_system,
mount_object.mountpoint)
# Data creation
# Creates one file of rand[0] size in each dir
rand = random.sample([1, 10, 512], 1)
cmd = ("/usr/bin/env python %s create_files "
"--fixed-file-size %sk %s/%s" % (
self.script_upload_path,
rand[0], mount_object.mountpoint, dir_list[0]))
ret, _, _ = g.run(mount_object.client_system, cmd)
self.assertFalse(ret, "Failed to create files")
# quota_validate for each dir
for dir_num, dir_name in enumerate(dir_list):
# To calculate the dir usage for quota
usage = (rand[0] * 1024) + \
((len(dir_list) - (dir_num + 1)) * rand[0] * 1024)
if usage >= 1073741824:
raise ExecutionError("usage crossed hardlimit")
ret = quota_validate(self.mnode, self.volname, path=dir_name,
hard_limit=1073741824, used_space=usage)
self.assertTrue(ret, "Quota validate Failed for dir %s" %
dir_name)
g.log.info("Quota list validate and file created successful "
"for %s", dir_name)
g.log.info("Files created and quota validated successfully")
# Deleting data and validating quota
self.all_mounts_procs = []
# Deleting deep directories in the mount point
for mount_object in self.mounts:
ret = rmdir(mount_object.client_system, "%s/dir1/dir2" %
(mount_object.mountpoint), force=True)
self.assertTrue(ret, ("Failed to delete dir under %s/dir1/dir2"
% (mount_object.mountpoint)))
g.log.info("Successfully deleted deep directories")
# Quota validate
# converting into bytes
usage = (rand[0] * 1024)
ret = quota_validate(self.mnode, self.volname,
path=dir_list[0],
used_space=usage)
self.assertTrue(ret, "Quota validate Failed for dir /dir1")
g.log.info("Quota list validate successful for /dir1")
# Remove Quota limit
g.log.info("Get Quota list for every directory on the volume %s",
self.volname)
ret = quota_remove(self.mnode, self.volname, path=dir_list[0])
self.assertTrue(ret, "Failed to remove Quota for dir %s" % dir_name)
g.log.info("Quota remove for dir %s successfully", dir_name)
def test_quota_multi_value_limits(self):
# pylint: disable=too-many-statements
"""
Verifying directory quota functionality with respect to
the limit-usage option. Set limits of various values being
big, small and decimal values. eg. 1GB, 10GB, 2.5GB, etc.
Set limits on various directories and check for the quota
list of all the directories.
* Enable Quota.
* Create some data on mount point.
* Set limit of 1 GB on path "/" , perform some I/O from mount
such that the hard limit is reached and validate with quota list.
* Set limit of 1.5 GB on path "/" , perform some I/O from mounts
such that the hard limit is reached and validate with quota list.
* Set limit of 10 GB on path "/" , perform some I/O from mounts
such that the hard limit is reached and validate with quota list.
"""
# Enable Quota
g.log.info("Enabling quota on the volume %s", self.volname)
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(
ret, 0, ("Failed to enable quota on the volume %s", self.volname))
g.log.info("Successfully enabled quota on the volume %s", self.volname)
# Perform some I/O on mounts
mount_obj = self.mounts[0]
mount_dir = mount_obj.mountpoint
client = mount_obj.client_system
g.log.info("Creating Files on %s:%s", client, mount_dir)
cmd = ("cd %s ; "
"for i in `seq 1 10` ; "
"do dd if=/dev/zero of=foo$i "
"bs=10M "
"count=1 ; "
"done" % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, "Failed to create files on mountpoint")
g.log.info("Files created successfully on mountpoint")
# Set soft timeout to 1 second
g.log.info("Set quota soft timeout:")
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, '1sec')
self.assertEqual(ret, 0, ("Failed to set soft timeout"))
g.log.info("Quota soft timeout set successfully")
# Set hard timeout to 0 second
g.log.info("Set quota hard timeout:")
ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, '0sec')
self.assertEqual(ret, 0, ("Failed to set hard timeout"))
g.log.info("Quota hard timeout set successfully")
# Path to set quota limit
path = "/"
# Set Quota limit of 1 GB on the root of the volume
g.log.info("Set Quota Limit on the path %s of the volume %s", path,
self.volname)
ret, _, _ = quota_limit_usage(self.mnode,
self.volname,
path=path,
limit="1GB")
self.assertEqual(ret, 0, ("Failed to set quota limit on path %s of "
"the volume %s", path, self.volname))
g.log.info("Successfully set the Quota limit on %s of the volume %s",
path, self.volname)
# Create data such that soft limit is crossed but not the hard limit
g.log.info("Creating Files on %s:%s", client, mount_dir)
cmd = ("cd %s ; "
"for i in `seq 11 102` ; "
"do dd if=/dev/zero of=foo$i "
"bs=10M "
"count=1 ; "
"done" % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, "Failed to create files on mountpoint")
g.log.info("Files created successfully on mountpoint")
# Create data such that hard limit is crossed
g.log.info("Creating Files on %s:%s", client, mount_dir)
cmd = ("cd %s ; "
"for i in `seq 1 10` ; "
"do dd if=/dev/zero of=small$i "
"bs=3M "
"count=1 ; "
"done" % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(
ret, 1, "Failed: Files created successfully in spite "
"of crossing hard-limit")
g.log.info("Files creation stopped on mountpoint once exceeded "
"hard limit")
# Validate if the Quota limit set is correct and hard limit is reached
# by checking the quota list
g.log.info(
"Validate if the Quota limit set is correct for the "
"volume %s", self.volname)
ret = quota_validate(self.mnode,
self.volname,
path=path,
hard_limit=1073741824,
avail_space=0,
sl_exceeded=True,
hl_exceeded=True)
self.assertTrue(ret, ("Quota Limit of 1 GB was not set properly on the"
" volume %s", self.volname))
g.log.info(
"Successfully Validated Quota Limit of 1 GB is set on "
"volume %s", self.volname)
# Set Quota limit as decimal value eg. 1.5 GB on the root of the volume
g.log.info("Set Quota Limit on the path %s of the volume %s", path,
self.volname)
ret, _, _ = quota_limit_usage(self.mnode,
self.volname,
path=path,
limit="1.5GB")
self.assertEqual(ret, 0, ("Failed to set quota limit on path %s of "
"the volume %s", path, self.volname))
g.log.info("Successfully set the Quota limit on %s of the volume %s",
path, self.volname)
# Create data such that soft limit is crossed but not the hard limit
g.log.info("Creating Files on %s:%s", client, mount_dir)
cmd = ("cd %s ; "
"for i in `seq 103 152` ; "
"do dd if=/dev/zero of=foo$i "
"bs=10M "
"count=1 ; "
"done" % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, "Failed to create files on mountpoint")
g.log.info("Files created successfully on mountpoint")
# Create data such that hard limit is crossed
g.log.info("Creating Files on %s:%s", client, mount_dir)
cmd = ("cd %s ; "
"for i in `seq 1 10` ; "
"do dd if=/dev/zero of=medium$i "
"bs=5M "
"count=1 ; "
"done" % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(
ret, 1, "Failed: Files created successfully in spite "
"of crossing hard-limit")
g.log.info("Files creation stopped on mountpoint once exceeded "
"hard limit")
# Validate if the Quota limit set is correct and hard limit is reached
# by checking the quota list
g.log.info(
"Validate if the Quota limit set is correct for the "
"volume %s", self.volname)
ret = quota_validate(self.mnode,
self.volname,
path=path,
hard_limit=1610612736,
avail_space=0,
sl_exceeded=True,
hl_exceeded=True)
self.assertTrue(ret, ("Quota Limit of 1.5GB was not set properly on "
"the volume %s", self.volname))
g.log.info(
"Successfully Validated Quota Limit of 1.5 GB is set on "
"volume %s", self.volname)
# Set Quota limit of 10 GB on the root of the volume
g.log.info("Set Quota Limit on the path %s of the volume %s", path,
self.volname)
ret, _, _ = quota_limit_usage(self.mnode,
self.volname,
path=path,
limit="10GB")
self.assertEqual(ret, 0, ("Failed to set quota limit on path %s of "
"the volume %s", path, self.volname))
g.log.info("Successfully set the Quota limit on %s of the volume %s",
path, self.volname)
# Create data such that soft limit is crossed but not the hard limit
g.log.info("Creating Files on %s:%s", client, mount_dir)
cmd = ("cd %s ; "
"for i in `seq 1 17` ; "
"do dd if=/dev/zero of=large$i "
"bs=500M "
"count=1 ; "
"done" % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, "Failed to create files on mountpoint")
g.log.info("Files created successfully on mountpoint")
# Create data such that hard limit is crossed
g.log.info("Creating Files on %s:%s", client, mount_dir)
cmd = ("cd %s ; "
"for i in `seq 1 30` ; "
"do dd if=/dev/zero of=largelimit$i "
"bs=10M "
"count=1 ; "
"done" % mount_dir)
ret, _, _ = g.run(client, cmd)
self.assertEqual(
ret, 1, "Failed: Files created successfully in spite "
"of crossing hard-limit")
g.log.info("Files creation stopped on mountpoint once exceeded "
"hard limit")
# Validate if the Quota limit set is correct and hard limit is reached
# by checking the quota list
g.log.info(
"Validate if the Quota limit set is correct for the "
"volume %s", self.volname)
ret = quota_validate(self.mnode,
self.volname,
path=path,
hard_limit=10737418240,
avail_space=0,
sl_exceeded=True,
hl_exceeded=True)
self.assertTrue(ret, ("Quota Limit of 10GB was not set properly on the"
" volume %s", self.volname))
g.log.info(
"Successfully Validated Quota Limit of 10 GB is set on "
"volume %s", self.volname)
# Remove Quota from the Volume
g.log.info("Remove Quota Limit set on path %s of the volume %s", path,
self.volname)
ret, _, _ = quota_remove(self.mnode, self.volname, path=path)
self.assertEqual(ret, 0, ("Failed to remove quota limit on path %s of "
"the volume %s", path, self.volname))
g.log.info(
"Successfully removed the Quota limit on path %s of the "
"volume %s", path, self.volname)
def test_quota_rebalance_heal(self):
"""
* Enable quota on the volume
* set hard and soft time out to zero.
* Create some files and directories from mount point
so that the limits are reached.
* Perform add-brick operation on the volume.
* Start rebalance on the volume.
* While rebalance is running, kill one of the bricks of the volume
and start after a while.
* While rebalance + self heal is in progress,
create some more files and
directories from the mount point until limit is hit
"""
# Enable Quota
g.log.info("Enabling quota on the volume %s", self.volname)
ret, _, _ = quota_enable(self.mnode, self.volname)
self.assertEqual(
ret, 0, ("Failed to enable quota on the volume %s", self.volname))
g.log.info("Successfully enabled quota on the volume %s", self.volname)
# Set the Quota timeouts to 0 for strict accounting
ret, _, _ = quota_set_hard_timeout(self.mnode, self.volname, 0)
self.assertEqual(
ret, 0, ("Failed to set softtimeout ot 0 for %s", self.volname))
ret, _, _ = quota_set_soft_timeout(self.mnode, self.volname, 0)
self.assertEqual(
ret, 0, ("Failed to set softtimeout ot 0 for %s", self.volname))
g.log.info("soft and hard timeout has been set to 0 for %s",
self.volname)
# Create Directories and files (write 4MB of data)
for mount_object in self.mounts:
g.log.info("Creating Directories on %s:%s",
mount_object.client_system, mount_object.mountpoint)
cmd = ("/usr/bin/env python %s create_deep_dirs_with_files -d 0 "
"-f 1024 -l 4 --fixed-file-size 1k %s" %
(self.script_upload_path, mount_object.mountpoint))
proc = g.run_async(mount_object.client_system,
cmd,
user=mount_object.user)
self.all_mounts_procs.append(proc)
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("IO is successful on all mounts")
# Set limit of 4 MB on root dir
g.log.info("Set Quota Limit on root directory of the volume %s",
self.volname)
ret, _, _ = quota_limit_usage(self.mnode, self.volname, '/', '4MB')
self.assertEqual(ret, 0, "Failed to set Quota for dir /.")
g.log.info("Set quota for dir / successfully.")
# Add bricks
replica_count_of_volume = self.volume['voltype']['replica_count']
bricks_list = form_bricks_list(self.mnode, self.volname,
replica_count_of_volume, self.servers,
self.all_servers_info)
g.log.info("new brick list: %s", str(bricks_list))
ret, _, _ = add_brick(self.mnode, self.volname, bricks_list, False)
self.assertEqual(ret, 0, "Failed to add the bricks to the volume")
g.log.info("Successfully added bricks to volume")
# Perform rebalance start operation
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, "Rebalance start is success")
# Sleep until rebalance has done some work
g.log.info("wait for rebalance to make progress")
sleep(3)
# Kill a brick and bring it up to trigerr self heal
all_bricks = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(all_bricks, "unable to get list of bricks")
g.log.info("bringing down brick: %s", all_bricks[0])
ret = bring_bricks_offline(self.volname, all_bricks[0])
self.assertTrue(ret, "unable to bring brick1 offline")
g.log.info("Successfully brought the following brick offline "
": %s", str(all_bricks[0]))
ret = bring_bricks_online(self.mnode, self.volname, [all_bricks[0]])
self.assertTrue(ret, "unable to bring %s online" % all_bricks[0])
g.log.info("Successfully brought the following brick online "
": %s", str(all_bricks[0]))
# Do some more IO and check if hard limit is honoured
all_mounts_procs = []
for mount_object in self.mounts:
cmd = ("/usr/bin/env python %s create_files "
"-f 100 --base-file-name file %s" %
(self.script_upload_path, mount_object.mountpoint))
proc = g.run_async(mount_object.client_system,
cmd,
user=mount_object.user)
all_mounts_procs.append(proc)
# Validate I/O
g.log.info("Wait for IO to complete and validate IO.....")
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertFalse(ret, "Writes allowed past quota limit")
g.log.info("Quota limits honored as expected")