def test_add_bricks_stopped_volume(self):
"""Add bricks to stopped volume
"""
ret, _, _ = volume_stop(self.mnode, self.volname)
self.assertEqual(ret, 0, "Can't stop volume")
g.log.info('Volume %s stopped successfully', self.volname)
vol_bricks = {
'before': get_all_bricks(self.mnode, self.volname),
'after': []
}
g.log.debug("Adding bricks to volume %s", self.volname)
ret = expand_volume(mnode=self.mnode,
volname=self.volname,
servers=self.servers,
all_servers_info=self.all_servers_info)
self.assertTrue(ret, 'Unable to expand volume %s' % self.volname)
g.log.info('Added bricks to stopped volume %s', self.volname)
vol_bricks['after'] = get_all_bricks(self.mnode, self.volname)
self.assertGreater(len(vol_bricks['after']), len(vol_bricks['before']),
"Expected new volume size to be greater than old")
g.log.info('Success in add bricks to stopped volume')
def test_rename_with_brick_min_free_limit_crossed(self):
"""
Test case:
1. Create a volume, start it and mount it.
2. Calculate the usable size and fill till it reachs min free limit
3. Rename the file
4. Try to perfrom I/O from mount point.(This should fail)
"""
bricks = get_all_bricks(self.mnode, self.volname)
# Calculate the usable size and fill till it reachs
# min free limit
usable_size = get_usable_size_per_disk(bricks[0])
ret, _, _ = g.run(
self.first_client,
"fallocate -l {}G {}/file".format(usable_size, self.mount_point))
self.assertFalse(ret, "Failed to fill disk to min free limit")
g.log.info("Disk filled up to min free limit")
# Rename the file
ret, _, _ = g.run(
self.first_client,
"mv {}/file {}/Renamedfile".format(self.mount_point,
self.mount_point))
self.assertFalse(ret, "Rename failed on file to Renamedfile")
g.log.info("File renamed successfully")
# Try to perfrom I/O from mount point(This should fail)
ret, _, _ = g.run(self.first_client,
"fallocate -l 5G {}/mfile".format(self.mount_point))
self.assertTrue(
ret, "Unexpected: Able to do I/O even when disks are "
"filled to min free limit")
g.log.info("Expected: Unable to perfrom I/O as min free disk is hit")
def _check_custom_xattr_trusted_foo(self, xattr_val, visible=True):
"""Check custom xttar from mount point and on bricks."""
# Check custom xattr from mount point
for fname in self.list_of_device_files:
ret = get_fattr(self.clients[0],
fname,
'trusted.foo',
encode='text')
if visible:
self.assertEqual(ret, xattr_val,
"Custom xattr not found from mount.")
else:
self.assertIsNone(
ret, "Custom attribute visible at mount "
"point even after deletion")
# Check custom xattr on bricks
for brick in get_all_bricks(self.mnode, self.volname):
node, brick_path = brick.split(':')
files_on_bricks = get_dir_contents(node, brick_path)
files = [
fname for fname in self.file_names if fname in files_on_bricks
]
for fname in files:
ret = get_fattr(node,
"{}/{}".format(brick_path, fname),
'trusted.foo',
encode='text')
if visible:
self.assertEqual(ret, xattr_val,
"Custom xattr not visible on bricks")
else:
self.assertIsNone(
ret, "Custom attribute visible on "
"brick even after deletion")
def _check_if_files_are_stored_only_on_expected_bricks(self):
"""Check if files are stored only on expected bricks"""
for fname in self.list_of_device_files:
# Fetch trusted.glusterfs.pathinfo and check if file is present on
# brick or not
ret = get_pathinfo(self.clients[0], fname)
self.assertIsNotNone(
ret, "Unable to get "
"trusted.glusterfs.pathinfo of file %s" % fname)
present_brick_list = []
for brick_path in ret['brickdir_paths']:
node, path = brick_path.split(":")
ret = file_exists(node, path)
self.assertTrue(
ret,
"Unable to find file {} on brick {}".format(fname, path))
brick_text = brick_path.split('/')[:-1]
if brick_text[0][0:2].isdigit():
brick_text[0] = gethostbyname(brick_text[0][:-1]) + ":"
present_brick_list.append('/'.join(brick_text))
# Check on other bricks where file doesn't exist
brick_list = get_all_bricks(self.mnode, self.volname)
other_bricks = [
brk for brk in brick_list if brk not in present_brick_list
]
for brick in other_bricks:
node, path = brick.split(':')
ret = file_exists(node, "{}/{}".format(path,
fname.split('/')[-1]))
self.assertFalse(
ret, "Unexpected: Able to find file {} on "
"brick {}".format(fname, path))
def _get_all_bricks(self):
"""Get all bricks where the shared storage is mounted"""
brick_list = get_all_bricks(self.mnode, "gluster_shared_storage")
self.assertIsNotNone(brick_list, "Unable to fetch brick list of shared"
" storage")
return brick_list
def test_character_and_block_device_file_removal_using_rm(self):
"""
Test case:
1. Create distributed volume with 5 sub-volumes, start and mount it.
2. Create character and block device files.
3. Check filetype of files from mount point.
4. Verify that the files are stored on only one bricks which is
mentioned in trusted.glusterfs.pathinfo xattr.
5. Delete the files.
6. Verify if the files are delete from all the bricks
"""
# Create Character and block device files
self._create_character_and_block_device_files()
# Check filetype of files from mount point
self._check_filetype_of_files_from_mountpoint()
# Verify that the files are stored on only the bricks which is
# mentioned in trusted.glusterfs.pathinfo xattr
self._check_if_files_are_stored_only_on_expected_bricks()
# Delete both the character and block device files
for fname in self.list_of_device_files:
ret, _, _ = g.run(self.clients[0], 'rm -rf {}'.format(fname))
self.assertEqual(ret, 0, 'Failed to remove {} file'.format(fname))
# Verify if the files are deleted from all bricks or not
for brick in get_all_bricks(self.mnode, self.volname):
node, path = brick.split(':')
for fname in self.file_names:
ret = file_exists(node, "{}/{}".format(path, fname))
self.assertFalse(
ret, "Unexpected: Able to find file {} on "
" brick {} even after deleting".format(fname, path))
def verify_all_process_of_volume_are_online(mnode, volname):
"""Verifies whether all the processes of volume are online
Args:
mnode (str): Node on which cmd has to be executed.
volname (str): volume name
Returns:
bool: Returns True if all the processes of volume are online.
False Otherwise.
"""
# Importing here to avoid cyclic imports
from glustolibs.gluster.brick_libs import are_bricks_online, get_all_bricks
# Verify all the brick process are online
bricks_list = get_all_bricks(mnode, volname)
if not bricks_list:
g.log.error("Failed to get the brick list "
"from the volume %s", volname)
return False
ret = are_bricks_online(mnode, volname, bricks_list)
if not ret:
g.log.error("All bricks are not online of "
"the volume %s", volname)
return False
# ToDO: Verify all self-heal-daemons are running for non-distribute volumes
return True
def setUp(self):
GlusterBaseClass.setUp.im_func(self)
for mount_object in self.mounts:
# Create user qa
g.log.info("Creating user 'qa'...")
command = "useradd qa"
ret, _, err = g.run(mount_object.client_system, command)
if 'already exists' in err:
g.log.warn("User 'qa' is already exists")
else:
g.log.info("User 'qa' is created successfully")
g.log.info("Starting to Setup Volume %s", self.volname)
# Setup Volume and Mount Volume
g.log.info("Starting to Setup Volume and Mount Volume")
ret = self.setup_volume_and_mount_volume(mounts=self.mounts)
if not ret:
raise ExecutionError("Failed to Setup_Volume and Mount_Volume")
g.log.info("Successful in Setup Volume and Mount Volume")
self.bricks_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(self.bricks_list, "unable to get list of bricks")
def test_offline_brick_status_when_quorum_not_met(self):
"""
Test Brick status when Quorum is not met after glusterd restart.
1. Create a volume and mount it.
2. Set the quorum type to 'server'.
3. Bring some nodes down such that quorum won't be met.
4. Brick status should be offline in the node which is up.
5. Restart glusterd in this node.
6. The brick status still should be offline as quorum isn't met.
"""
# Set the quorum type to server and validate it.
vol_option = {'cluster.server-quorum-type': 'server'}
ret = set_volume_options(self.mnode, self.volname, vol_option)
self.assertTrue(ret, "gluster volume option set of %s to %s failed"
% ('cluster.server-quorum-type', 'server'))
g.log.info("Cluster quorum set to type server.")
# Get the brick list.
brick_list = get_all_bricks(self.mnode, self.volname)
# Stop glusterd processes.
ret = stop_glusterd(self.servers[1:])
self.assertTrue(ret, "Failed to stop glusterd on specified nodes.")
g.log.info("Glusterd processes stopped in the desired servers.")
# Get the brick status in a node where glusterd is up.
ret = are_bricks_offline(self.mnode, self.volname, brick_list[0:1])
self.assertTrue(ret, "Bricks are online")
g.log.info("Bricks are offline as expected.")
# Restart one of the node which is up.
ret = restart_glusterd(self.servers[0])
self.assertTrue(ret, ("Failed to restart glusterd on desired node."))
g.log.info("Glusterd restarted on the desired node.")
# Wait for glusterd to be online and validate it's running.
self.assertTrue(wait_for_glusterd_to_start(self.servers[0]),
"Glusterd not up on the desired server.")
g.log.info("Glusterd is up in the desired server.")
# Get the brick status from the restarted node.
ret = are_bricks_offline(self.mnode, self.volname, brick_list[0:1])
self.assertTrue(ret, "Bricks are online")
g.log.info("Bricks are offline as expected.")
# Start glusterd on all servers.
ret = start_glusterd(self.servers)
self.assertTrue(ret, "Failed to start glusterd on the specified nodes")
g.log.info("Initiated start of glusterd on all nodes.")
# Wait for glusterd to start.
ret = wait_for_glusterd_to_start(self.servers)
self.assertTrue(ret, "Glusterd not up on all nodes.")
g.log.info("Glusterd is up and running on all nodes.")
# Wait for all volume processes to be online
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname,
timeout=600)
self.assertTrue(ret, ("All volume processes not up."))
g.log.info("All volume processes are up.")
def check_custom_xattr_visible(self, xattr_val):
"""
Check custom xttar from mount point and on bricks.
"""
# Check custom xattr from mount point
for mount_object in self.mounts:
for fname in self.files_and_soft_links:
attr_val = get_fattr(mount_object.client_system, fname,
'user.foo')
self.assertEqual(attr_val, xattr_val,
"Custom xattr not found from mount.")
g.log.info("Custom xattr found on mount point.")
# Check custom xattr on bricks
for brick in get_all_bricks(self.mnode, self.volname):
node, brick_path = brick.split(':')
files_on_bricks = get_dir_contents(node, brick_path)
files = [
fname.split('/')[3] for fname in self.list_of_files
if fname.split('/')[3] in files_on_bricks
]
for fname in files:
attr_val = get_fattr(node, "{}/{}".format(brick_path, fname),
'user.foo')
self.assertEqual(attr_val, xattr_val,
"Custom xattr not visible on bricks")
g.log.info("Custom xattr found on bricks.")
def check_mount_point_and_bricks_for_xattr(self, list_of_all_files):
"""
Check xattr on mount point and bricks.
"""
# Check if xattr is visable from mount point
for mount_object in self.mounts:
for fname in list_of_all_files:
ret = get_fattr(mount_object.client_system, fname, 'user.foo')
self.assertIsNone(
ret, "Custom attribute visible at mount "
"point even after deletion")
# Check if xattr is visable from bricks
for brick in get_all_bricks(self.mnode, self.volname):
node, brick_path = brick.split(':')
files_on_bricks = get_dir_contents(node, brick_path)
files = [
fname.split('/')[3] for fname in self.list_of_files
if fname.split('/')[3] in files_on_bricks
]
for fname in files:
ret = get_fattr(node, "{}/{}".format(brick_path, fname),
'user.foo')
self.assertIsNone(
ret, "Custom attribute visible on "
"brick even after deletion")
g.log.info("Custom xattr for file is not visible on "
"mount point and bricks")
def _check_arequal_checksum_for_the_volume(self):
"""
Check if arequals of mount point and bricks are
are the same.
"""
if self.volume_type == "replicated":
# Check arequals for "replicated"
brick_list = get_all_bricks(self.mnode, self.volname)
# Get arequal before getting bricks offline
ret, arequals = collect_mounts_arequal([self.mounts[0]])
self.assertTrue(ret, 'Failed to get arequal')
g.log.info('Getting arequal before getting bricks offline '
'is successful')
# Get arequal on bricks and compare with mount_point_total
self._check_arequal_on_bricks_with_a_specific_arequal(
arequals, brick_list)
# Check arequals for "distributed-replicated"
if self.volume_type == "distributed-replicated":
# Get the subvolumes
subvols_dict = get_subvols(self.mnode, self.volname)
num_subvols = len(subvols_dict['volume_subvols'])
# Get arequals and compare
for i in range(0, num_subvols):
# Get arequal for first brick
brick_list = subvols_dict['volume_subvols'][i]
ret, arequals = collect_bricks_arequal([brick_list[0]])
self.assertTrue(ret, 'Failed to get arequal on first brick')
# Get arequal for every brick and compare with first brick
self._check_arequal_on_bricks_with_a_specific_arequal(
arequals, brick_list)
def test_remove_brick_command_force(self):
"""
Test case:
1. Create a volume, start it and mount it.
2. Create some data on the volume.
3. Run remove-brick with force.
4. Check if bricks are still seen on volume or not
"""
# Create some data on the volume
self._run_io_on_mount_point()
# Remove-brick on the volume with force option
brick_list_to_remove = form_bricks_list_to_remove_brick(self.mnode,
self.volname)
self.assertIsNotNone(brick_list_to_remove, "Brick list is empty")
ret, _, _ = remove_brick(self.mnode, self.volname,
brick_list_to_remove, option="force")
self.assertFalse(ret, "Failed to run remove-brick with force")
g.log.info("Successfully run remove-brick with force")
# Get a list of all bricks
brick_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(brick_list, "Brick list is empty")
# Check if bricks removed brick are present or not in brick list
for brick in brick_list_to_remove:
self.assertNotIn(brick, brick_list,
"Brick still present in brick list even "
"after removing")
def monitor_heal_completion(mnode, volname, timeout_period=1200):
"""Monitors heal completion by looking into .glusterfs/indices/xattrop
directory of every brick for certain time. When there are no entries
in all the brick directories then heal is successful. Otherwise heal is
pending on the volume.
Args:
mnode : Node on which commands are executed
volname : Name of the volume
heal_monitor_timeout : time until which the heal monitoring to be done.
Default: 1200 i.e 20 minutes.
Return:
bool: True if heal is complete within timeout_period. False otherwise
"""
if timeout_period != 0:
heal_monitor_timeout = timeout_period
time_counter = heal_monitor_timeout
g.log.info("The heal monitoring timeout is : %d minutes" %
(heal_monitor_timeout / 60))
# Get all bricks
from glustolibs.gluster.brick_libs import get_all_bricks
bricks_list = get_all_bricks(mnode, volname)
if bricks_list is None:
g.log.error("Unable to get the bricks list. Hence unable to verify "
"whether self-heal-daemon process is running or not "
"on the volume %s" % volname)
return False
while time_counter > 0:
heal_complete = True
for brick in bricks_list:
brick_node, brick_path = brick.split(":")
cmd = ("ls -1 %s/.glusterfs/indices/xattrop/ | "
"grep -ve \"xattrop-\" | wc -l" % brick_path)
ret, out, err = g.run(brick_node, cmd)
if out.strip('\n') != "0":
heal_complete = False
if heal_complete:
break
else:
time.sleep(120)
time_counter = time_counter - 120
if heal_complete:
heal_completion_status = is_heal_complete(mnode, volname)
if heal_completion_status is True:
g.log.info("Heal has successfully completed on volume %s" %
volname)
return True
g.log.info("Heal has not yet completed on volume %s" % volname)
for brick in bricks_list:
brick_node, brick_path = brick.split(":")
cmd = ("ls -1 %s/.glusterfs/indices/xattrop/ " % brick_path)
g.run(brick_node, cmd)
return False
def test_volume_reduce_replica_count(self):
"""
Test case:
1) Create a 2x3 replica volume.
2) Remove bricks in the volume to make it a 2x2 replica volume.
3) Remove bricks in the volume to make it a distribute volume.
"""
# Create and start a volume
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, "Failed to create and start volume")
# Getting a list of all the bricks.
g.log.info("Get all the bricks of the volume")
self.brick_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(self.brick_list, "Failed to get the brick list")
g.log.info("Successfully got the list of bricks of volume")
# Converting 2x3 to 2x2 volume.
remove_brick_list = [self.brick_list[0], self.brick_list[3]]
ret, _, _ = remove_brick(self.mnode,
self.volname,
remove_brick_list,
'force',
replica_count=2)
self.assertEqual(ret, 0, "Failed to start remove brick operation")
g.log.info("Remove brick operation successfully")
# Checking if volume is 2x2 or not.
volume_info = get_volume_info(self.mnode, self.volname)
brick_count = int(volume_info[self.volname]['brickCount'])
self.assertEqual(brick_count, 4, "Failed to remove 2 bricks.")
g.log.info("Successfully removed 2 bricks.")
type_string = volume_info[self.volname]['typeStr']
self.assertEqual(type_string, 'Distributed-Replicate',
"Convertion to 2x2 failed.")
g.log.info("Convertion to 2x2 successful.")
# Converting 2x2 to distribute volume.
remove_brick_list = [self.brick_list[1], self.brick_list[4]]
ret, _, _ = remove_brick(self.mnode,
self.volname,
remove_brick_list,
'force',
replica_count=1)
self.assertEqual(ret, 0, "Failed to start remove brick operation")
g.log.info("Remove brick operation successfully")
# Checking if volume is pure distribute or not.
volume_info = get_volume_info(self.mnode, self.volname)
brick_count = int(volume_info[self.volname]['brickCount'])
self.assertEqual(brick_count, 2, "Failed to remove 2 bricks.")
g.log.info("Successfully removed 2 bricks.")
type_string = volume_info[self.volname]['typeStr']
self.assertEqual(type_string, 'Distribute',
"Convertion to distributed failed.")
g.log.info("Convertion to distributed successful.")
def test_gfid_assignment_on_lookup(self):
g.log.info("Creating directories on the backend.")
bricks_list = get_all_bricks(self.mnode, self.volname)
i = 0
for brick in bricks_list:
i += 1
brick_node, brick_path = brick.split(":")
ret, _, _ = g.run(brick_node, "mkdir %s/dir%d" % (brick_path, i))
self.assertEqual(ret, 0, "Dir creation failed on %s" % brick_path)
g.log.info("Created directories on the backend.")
# To circumvent is_fresh_file() check in glusterfs code.
time.sleep(2)
# Do named lookup on directories from mount
ret, _, err = g.run(self.clients[0],
"echo Hi > %s/dir1" % self.mounts[0].mountpoint)
errmsg = ("bash: %s/dir1: Is a directory\n" %
self.mounts[0].mountpoint)
msg = "expected %s, but returned %s" % (errmsg, err)
self.assertEqual(err, errmsg, msg)
g.log.info("Writing a file with same name as directory \"dir1\" failed"
" as expected on mount point.")
ret, _, _ = g.run(self.clients[0],
"touch %s/dir2" % self.mounts[0].mountpoint)
self.assertEqual(
ret, 0, "Touch of file with same name as directory "
"\"dir2\" failed.")
g.log.info("Touch of file with same name as directory \"dir2\" passed"
" but it will not create the file since a directory is "
"already present with the same name.")
ret, _, err = g.run(self.clients[0],
"mkdir %s/dir3" % self.mounts[0].mountpoint)
self.assertNotEqual(
ret, 0, "Creation of directory with same name as "
"directory \"dir3\" succeeded, which is not "
"supposed to.")
g.log.info("Creation of directory \"dir3\" failed as expected")
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Verify directories are present on the backend and gfids are assigned
self.verify_gfid("dir1")
self.verify_gfid("dir2")
self.verify_gfid("dir3")
# Check whether all the directories are listed on the mount
_, count, _ = g.run(self.clients[0],
"ls %s | wc -l" % self.mounts[0].mountpoint)
self.assertEqual(int(count), 3, "Not all the directories are listed on"
"the mount")
g.log.info("All the directories are listed on the mount.")
def test_rebalance_start_when_glusterd_down(self):
# Expanding volume by adding bricks to the volume
g.log.info("Start adding bricks to volume")
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, ("Volume %s: Expand failed", self.volname))
g.log.info("Volume %s: Expand success", self.volname)
# Get all servers IP addresses which are part of volume
ret = get_all_bricks(self.mnode, self.volname)
list_of_servers_used = []
for brick in ret:
list_of_servers_used.append(brick.split(":")[0])
self.assertTrue(ret, ("Failed to get server IP list for volume %s",
self.volname))
g.log.info("Succesfully got server IP list for volume %s",
self.volname)
# Form a new list of servers without mnode in it to prevent mnode
# from glusterd failure
for element in list_of_servers_used:
if element == self.mnode:
list_of_servers_used.remove(element)
# Stop glusterd on a server
self.random_server = choice(list_of_servers_used)
g.log.info("Stop glusterd on server %s", self.random_server)
ret = stop_glusterd(self.random_server)
self.assertTrue(ret, ("Server %s: Failed to stop glusterd",
self.random_server))
g.log.info("Server %s: Stopped glusterd", self.random_server)
# Start Rebalance
g.log.info("Starting rebalance on the volume")
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Volume %s: Failed to start rebalance",
self.volname))
g.log.info("Volume %s: Rebalance start success", self.volname)
# Wait for rebalance to complete
g.log.info("Waiting for rebalance to complete")
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertFalse(ret, ("Volume %s: Rebalance is completed",
self.volname))
g.log.info("Expected: Rebalance failed on one or more nodes."
" Check rebalance status for more details")
error_msg1 = "\"fix layout on / failed\""
error_msg2 = "\"Transport endpoint is not connected\""
ret, _, _ = g.run(self.mnode, "grep -w %s /var/log/glusterfs/"
"%s-rebalance.log| grep -w %s"
% (error_msg1, self.volname, error_msg2))
self.assertEqual(ret, 0, ("Unexpected : Rebalance failed on volume %s"
"not because of glusterd down on a node",
self.volname))
g.log.info("\n\nRebalance failed on volume %s due to glusterd down on"
"one of the nodes\n\n", self.volname)
def _replace_one_random_brick(self):
"""Replace one random brick from the volume"""
brick = choice(get_all_bricks(self.mnode, self.volname))
ret = replace_brick_from_volume(self.mnode,
self.volname,
self.servers,
self.all_servers_info,
src_brick=brick)
self.assertTrue(ret, "Failed to replace brick %s " % brick)
g.log.info("Successfully replaced brick %s", brick)
def setUp(self):
self.get_super_method(self, 'setUp')()
ret = self.setup_volume_and_mount_volume(mounts=self.mounts,
volume_create_force=False)
if not ret:
raise ExecutionError("Failed to Setup_Volume and Mount_Volume")
g.log.info("Successful in Setup Volume and Mount Volume")
self.bricks_list = get_all_bricks(self.mnode, self.volname)
def _check_permissions_of_dir(self):
"""Check permissions of dir created."""
for brick_path in get_all_bricks(self.mnode, self.volname):
node, path = brick_path.split(":")
ret = get_file_stat(node, "{}/dir".format(path))
self.assertEqual(
int(ret["access"]), 755,
"Unexpected:Permissions of dir is %s and not %d" %
(ret["access"], 755))
g.log.info("Permissions of dir directory is proper on all bricks")
def _get_total_brick_processes_count(self):
"""
Function to find the total number of brick processes in the cluster
"""
count = 0
self.brick_list = get_all_bricks(self.mnode, self.volname)
for brick in self.brick_list:
server = brick.split(":")[0]
count += get_brick_processes_count(server)
return count
def test_invalid_memory_read_after_freed(self):
"""
Test case:
1. Create a volume and start it.
2. Mount the volume using FUSE.
3. Create multiple level of dirs and files inside every dir.
4. Rename files such that linkto files are created.
5. From the mount point do an rm -rf * and check if all files
are delete or not from mount point as well as backend bricks.
"""
# Fetch timestamp to check for core files
ret, test_timestamp, _ = g.run(self.mnode, "date +%s")
self.assertEqual(ret, 0, "date command failed")
test_timestamp = test_timestamp.strip()
# Create multiple level of dirs and files inside every dir
cmd = ("cd %s; for i in {1..100}; do mkdir dir$i; cd dir$i; "
"for i in {1..200}; do dd if=/dev/urandom of=file$i bs=1K"
" count=1; done; done" % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.first_client, cmd)
self.assertFalse(ret, "Failed to create dirs and files")
# Rename files such that linkto files are created
cmd = ("cd %s; for i in {1..100}; do cd dir$i; for i in {1..200}; do "
"mv file$i ntfile$i; done; done" % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.first_client, cmd)
self.assertFalse(ret, "Failed to rename files")
g.log.info("Files created and renamed successfully")
# From the mount point do an rm -rf * and check if all files
# are delete or not from mount point as well as backend bricks.
ret, _, _ = g.run(self.first_client,
"rm -rf {}/*".format(self.mounts[0].mountpoint))
self.assertFalse(ret, "rn -rf * failed on mount point")
ret = get_dir_contents(self.first_client,
"{}/".format(self.mounts[0].mountpoint))
self.assertEqual(
ret, [], "Unexpected: Files and directories still "
"seen from mount point")
for brick in get_all_bricks(self.mnode, self.volname):
node, brick_path = brick.split(":")
ret = get_dir_contents(node, "{}/".format(brick_path))
self.assertEqual(
ret, [], "Unexpected: Files and dirs still seen "
"on brick %s on node %s" % (brick_path, node))
g.log.info("rm -rf * on mount point successful")
# Check for core file on servers and clients
servers = self.servers + [self.first_client]
ret = is_core_file_created(servers, test_timestamp)
self.assertTrue(ret, "Core files found on servers used for test")
g.log.info("No cores found on all participating servers")
def _check_trusted_glusterfs_dht_on_all_bricks(self):
"""Check trusted.glusterfs.dht xattr on the backend bricks"""
bricks = get_all_bricks(self.mnode, self.volname)
fattr_value = []
for brick_path in bricks:
node, path = brick_path.split(":")
ret = get_fattr(node, "{}".format(path), "trusted.glusterfs.dht")
fattr_value += [ret]
self.assertEqual(len(set(fattr_value)), 4,
"Value of trusted.glusterfs.dht is not as expected")
g.log.info("Successfully checked value of trusted.glusterfs.dht.")
def test_brick_full_add_brick_remove_brick(self):
"""
Test case:
1. Create a volume, start it and mount it.
2. Fill few bricks till min-free-limit is reached.
3. Add brick to the volume.(This should pass.)
4. Set cluster.min-free-disk to 30%.
5. Remove bricks from the volume.(This should pass.)
6. Check for data loss by comparing arequal before and after.
"""
# Fill few bricks till it is full
bricks = get_all_bricks(self.mnode, self.volname)
# Calculate the usable size and fill till it reaches
# min free limit
usable_size = get_usable_size_per_disk(bricks[0])
subvols = get_subvols(self.mnode, self.volname)['volume_subvols']
filename = "abc"
for _ in range(0, usable_size):
while (subvols[find_hashed_subvol(subvols, "/",
filename)[1]] == subvols[0]):
filename = self._get_random_string()
ret, _, _ = g.run(
self.mounts[0].client_system,
"fallocate -l 1G {}/{}".format(self.mounts[0].mountpoint,
filename))
self.assertFalse(ret, "Failed to fill disk to min free limit")
filename = self._get_random_string()
g.log.info("Disk filled up to min free limit")
# Collect arequal checksum before ops
arequal_checksum_before = collect_mounts_arequal(self.mounts[0])
# Add brick to volume
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, "Failed to add brick on volume %s" % self.volname)
# Set cluster.min-free-disk to 30%
ret = set_volume_options(self.mnode, self.volname,
{'cluster.min-free-disk': '30%'})
self.assertTrue(ret, "Failed to set cluster.min-free-disk to 30%")
# Remove bricks from the volume
ret = shrink_volume(self.mnode, self.volname, rebalance_timeout=1800)
self.assertTrue(ret, "Failed to remove-brick from volume")
g.log.info("Remove-brick rebalance successful")
# Check for data loss by comparing arequal before and after ops
arequal_checksum_after = collect_mounts_arequal(self.mounts[0])
self.assertEqual(arequal_checksum_before, arequal_checksum_after,
"arequal checksum is NOT MATCHNG")
g.log.info("arequal checksum is SAME")
def test_gfid_heal(self):
"""
- Create a 1x3 volume and fuse mount it.
- Create 1 directory with 1 file inside it directly on each brick.
- Access the directories from the mount.
- Launch heals and verify that the heals are over.
- Verify that the files and directories have gfid assigned.
"""
# pylint: disable=too-many-statements
# Create data on the bricks.
g.log.info("Creating directories and files on the backend.")
bricks_list = get_all_bricks(self.mnode, self.volname)
i = 0
for brick in bricks_list:
i += 1
brick_node, brick_path = brick.split(":")
ret, _, _ = g.run(brick_node, "mkdir %s/dir%d" % (brick_path, i))
self.assertEqual(ret, 0, "Dir creation failed on %s" % brick_path)
ret, _, _ = g.run(brick_node,
"touch %s/dir%d/file%d" % (brick_path, i, i))
self.assertEqual(ret, 0, "file creation failed on %s" % brick_path)
g.log.info("Created directories and files on the backend.")
# To circumvent is_fresh_file() check in glusterfs code.
time.sleep(2)
# Access files from mount
for i in range(1, 4):
cmd = ("ls %s/dir%d/file%d" % (self.mounts[0].mountpoint, i, i))
ret, _, _ = g.run(self.clients[0], cmd)
self.assertEqual(
ret, 0, "Failed to access dir%d/file%d on %s" %
(i, i, self.mounts[0].mountpoint))
# Trigger heal
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Starting heal failed')
g.log.info('Index heal launched')
# Monitor heal completion
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
# Check if heal is completed
ret = is_heal_complete(self.mnode, self.volname)
self.assertTrue(ret, 'Heal is not complete')
g.log.info('Heal is completed successfully')
# Verify gfid and links at the backend.
self.verify_gfid_and_link_count("dir1", "file1")
self.verify_gfid_and_link_count("dir2", "file2")
self.verify_gfid_and_link_count("dir3", "file3")
def test_gfid_assignment_on_all_subvols(self):
"""
- Create a dis-rep volume and mount it.
- Create a directory on mount and check whether all the bricks have
the same gfid.
- On the backend create a new directory on all the bricks.
- Do lookup from the mount.
- Check whether all the bricks have the same gfid assigned.
"""
# Enable client side healing
g.log.info("Enable client side healing options")
options = {
"metadata-self-heal": "on",
"entry-self-heal": "on",
"data-self-heal": "on"
}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Successfully set %s for volume %s", options, self.volname)
# Create a directory on the mount
g.log.info("Creating a directory")
cmd = "/usr/bin/env python %s create_deep_dir -d 0 -l 0 %s/dir1 " % (
self.script_upload_path, self.mounts[0].mountpoint)
ret, _, _ = g.run(self.clients[0], cmd)
self.assertEqual(ret, 0, "Failed to create directory on mountpoint")
g.log.info("Directory created successfully on mountpoint")
# Verify gfids are same on all the bricks
self.verify_gfid("dir1")
# Create a new directory on all the bricks directly
bricks_list = get_all_bricks(self.mnode, self.volname)
for brick in bricks_list:
brick_node, brick_path = brick.split(":")
ret, _, _ = g.run(brick_node, "mkdir %s/dir2" % (brick_path))
self.assertEqual(
ret, 0, "Failed to create directory on brick %s" % (brick))
# To circumvent is_fresh_file() check in glusterfs code.
time.sleep(2)
# Do a clinet side lookup on the new directory and verify the gfid
# All the bricks should have the same gfid assigned
ret, _, _ = g.run(self.clients[0],
"ls %s/dir2" % self.mounts[0].mountpoint)
self.assertEqual(ret, 0, "Lookup on directory \"dir2\" failed.")
g.log.info("Lookup on directory \"dir2\" successful")
# Verify gfid is assigned on all the bricks and are same
self.verify_gfid("dir2")
def test_volume_absent_bricks(self):
"""
Test Case:
1) Create Volume
2) Remove any one Brick directory
3) Start Volume and compare the failure message
4) Check the gluster volume status nad compare the status message
"""
# Fetching the brick list
brick_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(brick_list, "Failed to get the bricks in"
" the volume")
# Command for removing brick directory
random_brick = random.choice(brick_list)
node, brick_path = random_brick.split(r':')
cmd = 'rm -rf ' + brick_path
# Removing brick directory of one node
ret, _, _ = g.run(node, cmd)
self.assertEqual(ret, 0, "Failed to remove brick dir")
g.log.info("Brick directory removed successfully")
# Starting volume
ret, _, err = volume_start(self.mnode, self.volname)
self.assertNotEqual(
ret, 0, "Unexpected: Volume started successfully "
"even though brick directory is removed "
"for %s" % self.volname)
g.log.info("Expected: Failed to start volume %s", self.volname)
# Checking volume start failed message
msg = "Failed to find brick directory"
self.assertIn(
msg, err, "Expected message is %s but volume start "
"command failed with this "
"message %s" % (msg, err))
g.log.info("Volume start failed with correct error message %s", err)
# Checking Volume status
ret, _, err = volume_status(self.mnode, self.volname)
self.assertNotEqual(
ret, 0, "Success in getting volume status, volume "
"status should fail when volume is in "
"not started state ")
g.log.info("Failed to get volume status which is expected")
# Checking volume status message
msg = ' '.join(['Volume', self.volname, 'is not started'])
self.assertIn(msg, err, 'Incorrect error message for gluster vol '
'status')
g.log.info("Correct error message for volume status")
def setUp(self):
# Calling GlusterBaseClass setUp
self.get_super_method(self, 'setUp')()
self.all_mounts_procs, self.io_validation_complete = [], False
# Setup Volume and Mount Volume
ret = self.setup_volume_and_mount_volume(mounts=self.mounts)
if not ret:
raise ExecutionError("Failed to Setup_Volume and Mount_Volume")
g.log.info("Successful in Setup Volume and Mount Volume")
self.bricks_list = get_all_bricks(self.mnode, self.volname)
def setUp(self):
# Calling GlusterBaseClass setUp
GlusterBaseClass.setUp.im_func(self)
# Setup Volume and Mount Volume
g.log.info("Starting to Setup Volume and Mount Volume")
ret = self.setup_volume_and_mount_volume(mounts=self.mounts,
volume_create_force=False)
if not ret:
raise ExecutionError("Failed to Setup_Volume and Mount_Volume")
g.log.info("Successful in Setup Volume and Mount Volume")
self.bricks_list = get_all_bricks(self.mnode, self.volname)
def _get_min_brick(self):
# Returns the brick with min size
bricks_list = get_all_bricks(self.mnode, self.volname)
min_brick_size = -1
min_size_brick = None
for brick in bricks_list:
brick_node, brick_path = brick.split(":")
brick_size = get_size_of_mountpoint(brick_node, brick_path)
if ((brick_size is not None) and (min_brick_size == -1)
or (int(min_brick_size) > int(brick_size))):
min_brick_size = brick_size
min_size_brick = brick
return min_size_brick, min_brick_size
