def test_mount_remove_client_logs_dir_remount(self):
# pylint: disable=too-many-statements
"""
1. Create all types of volumes and start them.
2. Mount all volumes on clients.
3. Delete /var/log/glusterfs folder on client.
4. Run IO on all the mount points.
5. Unmount and remount all volumes.
6. Check if logs are regenerated or not.
"""
# Mounting the volume.
ret, _, _ = mount_volume(self.volname, mtype=self.mount_type,
mpoint=self.mounts[0].mountpoint,
mserver=self.mnode,
mclient=self.mounts[0].client_system)
self.assertEqual(ret, 0, ("Volume %s is not mounted.") % self.volname)
g.log.info("Volume mounted successfully : %s", self.volname)
# Removing dir /var/log/glusterfs on client.
cmd = 'mv /var/log/glusterfs /root/'
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Unable to remove /var/log/glusterfs dir.")
g.log.info("Successfully removed /var/log/glusterfs on client: %s",
self.mounts[0])
# Running IO on the mount point.
# Creating a dir on the mount point.
ret = mkdir(self.mounts[0].client_system,
self.mounts[0].mountpoint+"/dir")
self.assertTrue(ret, "Failed to create dir.")
g.log.info("dir created successfully for %s", self.mounts[0])
# Creating a file on the mount point.
cmd = ('touch %s/file' % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Failed to create file.")
g.log.info("file created successfully for %s", self.mounts[0])
# Unmounting and remounting volume.
ret, _, _ = umount_volume(mclient=self.mounts[0].client_system,
mpoint=self.mounts[0].mountpoint)
self.assertEqual(ret, 0,
("Volume %s is not unmounted.") % self.volname)
g.log.info("Volume unmounted successfully : %s", self.volname)
ret, _, _ = mount_volume(self.volname, mtype=self.mount_type,
mpoint=self.mounts[0].mountpoint,
mserver=self.mnode,
mclient=self.mounts[0].client_system)
self.assertEqual(ret, 0, ("Volume %s is not mounted.") % self.volname)
g.log.info("Volume mounted successfully : %s", self.volname)
# Checking if logs are regenerated or not.
ret = get_dir_contents(self.mounts[0].client_system,
'/var/log/glusterfs/')
self.assertIsNotNone(ret, 'Log files were not regenerated.')
g.log.info("Log files were properly regenearted.")
def _disable_io_encryption(self):
""" Disables IO encryption """
# UnMount Volume
g.log.info("Starting to Unmount Volume %s", self.volname)
ret, _, _ = umount_volume(self.mounts[0].client_system,
self.mounts[0].mountpoint,
mtype=self.mount_type)
self.assertEqual(ret, 0, "Failed to Unmount volume")
# Stop Volume
ret, _, _ = volume_stop(self.mnode, self.volname)
self.assertEqual(ret, 0, "Failed to Stop volume")
# Disable server and client SSL usage
options = {"server.ssl": "off",
"client.ssl": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, "Failed to set volume options")
# Start Volume
ret, _, _ = volume_start(self.mnode, self.volname)
self.assertEqual(ret, 0, "Failed to Start volume")
# Mount Volume
ret, _, _ = mount_volume(self.volname, mtype=self.mount_type,
mpoint=self.mounts[0].mountpoint,
mserver=self.mnode,
mclient=self.mounts[0].client_system)
self.assertEqual(ret, 0, "Failed to mount the volume back")
def mount_clone_and_io(clone, mpoint):
# define mounts
self.mount_points = []
self.mounts_dict_list = []
for client in self.all_clients_info:
mount = {
'protocol': self.mount_type,
'server': self.mnode,
'volname': clone,
'client': self.all_clients_info[client],
'mountpoint': (os.path.join("%s" % mpoint)),
'options': ''
}
self.mounts_dict_list.append(mount)
self.mounts1 = create_mount_objs(self.mounts_dict_list)
g.log.info("Successfully made entry in self.mounts")
# Mounting a volume
g.log.info("Starting to mount volume")
ret = mount_volume(clone, self.mount_type, mpoint, self.mnode,
self.clients[0])
self.assertTrue(ret, "Volume mount failed for clone1")
g.log.info("%s mounted Successfully", clone)
# Checking volume mounted or not
ret = is_mounted(clone, mpoint, self.mnode, self.clients[0],
self.mount_type)
self.assertTrue(ret,
"Volume not mounted on mount point: %s" % mpoint)
g.log.info("Volume %s mounted on %s", clone, mpoint)
return 0
def _try_mounting_volume(self):
""" Mount a volume """
ret, _, _ = mount_volume(self.volname,
mtype="glusterfs",
mpoint="/mnt/distribute-vol",
mserver=self.mnode,
mclient=self.servers[1])
self.assertNotEqual(ret, 0,
"Unexpected: Volume %s is mounted" % self.volname)
def test_stat_prefetch(self):
# pylint: disable=ungrouped-imports
self.vips = (g.config['gluster']['cluster_config']['smb']['ctdb_vips'])
# Virtual Ip of first node to mount
self.vips_mnode = self.vips[0]['vip']
g.log.info("CTDB Virtual Ip %s", self.vips_mnode)
# run IOs
self.counter = 1
g.log.info("Starting IO on all mounts...")
self.all_mounts_procs = []
for mount_obj in self.mounts:
ret, _, _ = mount_volume(self.volname,
'cifs',
mount_obj.mountpoint,
self.vips_mnode,
mount_obj.client_system,
smbuser='******',
smbpasswd='foobar')
self.assertEqual(ret, 0, "Cifs Mount Failed")
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("python %s create_files -f 10000"
" --base-file-name ctdb-cifs "
" --fixed-file-size 10k %s/samba/" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Switch off and switch on stat-prefetch
options = {"stat-prefetch": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
if not ret:
raise ExecutionError("Failed to execute volume set"
"option command")
ret = get_volume_options(self.mnode, self.volname)
if ret['performance.stat-prefetch'] != "off":
raise ExecutionError("Failed to set stat-prefetch off")
options = {"stat-prefetch": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
if not ret:
raise ExecutionError("Failed to execute volume set"
"option command")
ret = get_volume_options(self.mnode, self.volname)
if ret['performance.stat-prefetch'] != "on":
raise ExecutionError("Failed to set stat-prefetch on")
# Validate IO
g.log.info("Wait for IO to complete and validate IO ...")
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.io_validation_complete = True
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Creation of 10000 files Success")
g.log.info("test__samba_ctdb_cifs_io_rename PASSED")
def test_ec_read_from_hardlink(self):
"""
Test steps:
1. Enable metadata-cache(md-cache) options on the volume
2. Touch a file and create a hardlink for it
3. Read data from the hardlink.
4. Read data from the actual file.
"""
options = {'group': 'metadata-cache'}
# Set metadata-cache options as group
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, "Unable to set the volume options {}".
format(options))
g.log.info("Able to set the %s options", options)
# Mounting the volume on one client
ret, _, _ = mount_volume(self.volname, mtype=self.mount_type,
mpoint=self.mounts[0].mountpoint,
mserver=self.mnode,
mclient=self.mounts[0].client_system)
self.assertEqual(ret, 0, ("Volume {} is not mounted").
format(self.volname))
g.log.info("Volume mounted successfully : %s", self.volname)
file_name = self.mounts[0].mountpoint + "/test1"
content = "testfile"
hard_link = self.mounts[0].mountpoint + "/test1_hlink"
cmd = 'echo "{content}" > {file}'.format(file=file_name,
content=content)
# Creating a file with data
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Sucessful in creating a file with data")
g.log.info("file created successfully on %s",
self.mounts[0].mountpoint)
# Creating a hardlink for the file created
ret = create_link_file(self.mounts[0].client_system,
file_name, hard_link)
self.assertTrue(ret, "Link file creation failed")
g.log.info("Link file creation for %s is successful", file_name)
# Reading from the file as well as the hardlink
for each in (file_name, hard_link):
ret, out, _ = g.run(self.mounts[0].client_system,
"cat {}".format(each))
self.assertEqual(ret, 0, "Unable to read the {}".format(each))
self.assertEqual(content, out.strip('\n'), "The content {} and"
" data in file {} is not same".
format(content, each))
g.log.info("Read of %s file is successful", each)
def _set_option_and_mount_and_unmount_volumes(self,
option="",
is_allowed=True):
"""
Setting volume option and then mounting and unmounting the volume
"""
# Check if an option is passed
if option:
# Setting the option passed as an argument
ret = set_volume_options(self.mnode, self.volname,
{option: self.mounts[0].client_system})
self.assertTrue(
ret, "Failed to set %s option in volume: %s" %
(option, self.volname))
g.log.info("Successfully set %s option in volume: %s", option,
self.volname)
# Mounting a volume
ret, _, _ = mount_volume(self.volname,
mtype=self.mount_type,
mpoint=self.mounts[0].mountpoint,
mserver=self.mnode,
mclient=self.mounts[0].client_system)
# Checking if volume was successfully mounted or not
ret = is_mounted(self.volname,
mtype=self.mount_type,
mpoint=self.mounts[0].mountpoint,
mserver=self.mnode,
mclient=self.mounts[0].client_system)
if is_allowed:
self.assertTrue(ret,
"Failed to mount the volume: %s" % self.volname)
else:
self.assertFalse(
ret, "Unexpected: Mounting"
" the volume %s was successful" % self.volname)
# Unmount only if the volume is supposed to be mounted
if is_allowed:
ret, _, _ = umount_volume(self.mounts[0].client_system,
self.mounts[0].mountpoint,
mtype=self.mount_type)
self.assertEqual(ret, 0,
"Failed to unmount the volume: %s" % self.volname)
def get_pathinfo(mnode, filename, volname):
"""This module gets filepath of the given file in gluster server.
Example:
get_pathinfo(mnode, "file1", "testvol")
Args:
mnode (str): Node on which cmd has to be executed.
filename (str): relative path of file
volname (str): volume name
Returns:
NoneType: None if command execution fails, parse errors.
list: file path for the given file in gluster server
"""
mount_point = tempfile.mkdtemp()
# Performing glusterfs mount because only with glusterfs mount
# the file location in gluster server can be identified
ret, _, _ = mount_volume(volname,
mtype='glusterfs',
mpoint=mount_point,
mserver=mnode,
mclient=mnode)
if ret != 0:
g.log.error("Failed to do gluster mount on volume %s to fetch"
"pathinfo from server %s" % (volname, mnode))
return None
filename = mount_point + '/' + filename
attr_name = 'trusted.glusterfs.pathinfo'
output = get_extended_attributes_info(mnode, [filename],
attr_name=attr_name)
if output is None:
g.log.error("Failed to get path info for %s" % filename)
return None
pathinfo = output[filename][attr_name]
umount_volume(mnode, mount_point)
g.run(mnode, "rm -rf " + mount_point)
return re.findall(r".*?POSIX.*?:(\S+)\>", pathinfo)
def trigger_heal_from_mount_point(self):
"""
Trigger heal from mount point using read.
"""
# Unmouting and remounting volume to update the volume graph
# in client.
ret, _, _ = umount_volume(
self.mounts[0].client_system, self.mounts[0].mountpoint)
self.assertFalse(ret, "Failed to unmount volume.")
ret, _, _ = mount_volume(
self.volname, 'glusterfs', self.mounts[0].mountpoint,
self.mnode, self.mounts[0].client_system)
self.assertFalse(ret, "Failed to remount volume.")
g.log.info('Successfully umounted and remounted volume.')
# Trigger heal from client side
cmd = ("/usr/bin/env python {0} read {1}/{2}".format(
self.script_upload_path, self.mounts[0].mountpoint,
self.test_meta_data_self_heal_folder))
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertFalse(ret, 'Failed to trigger heal on %s'
% self.mounts[0].client_system)
g.log.info("Successfully triggered heal from mount point.")
def setUp(self):
self.get_super_method(self, 'setUp')()
# Setup Volume
if not self.setup_volume():
raise ExecutionError("Failed to Setup volume")
self.first_client = self.mounts[0].client_system
self.mount_point = self.mounts[0].mountpoint
# Mount volume with -o acl option
ret, _, _ = mount_volume(self.volname,
self.mount_type,
self.mount_point,
self.mnode,
self.first_client,
options='acl')
if ret:
raise ExecutionError("Failed to mount volume")
# Create a non-root user
if not add_user(self.first_client, 'joker'):
raise ExecutionError("Failed to create user joker")
def test_snap_clone_snapd(self):
"""
Steps:
1. create a volume
2. Create a snapshots and activate
3. Clone the snapshot and mount it
4. Check for snapd daemon
5. enable uss and validate snapd
5. stop cloned volume
6. Validate snapd
7. start cloned volume
8. validate snapd
9. Create 5 more snapshot
10. Validate total number of
snapshots created.
11. Activate 5 snapshots
12. Enable USS
13. Validate snapd
14. kill snapd on all nodes
15. validate snapd running
16. force start clone volume
17. validate snaps inside .snaps directory
"""
# pylint: disable=too-many-statements, too-many-locals
# Starting I/O
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("/usr/bin/env python %s create_files "
"-f 10 --base-file-name file %s" % (
self.script_upload_path,
mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate I/O
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("IO is successful on all mounts")
# Creating snapshot
ret, _, _ = snap_create(self.mnode, self.volname, self.snap)
self.assertEqual(ret, 0, ("Failed to create snapshot for volume %s"
% self.volname))
g.log.info("Snapshot %s created successfully for "
"volume %s", self.snap, self.volname)
# Activating created snapshots
ret, _, _ = snap_activate(self.mnode, self.snap)
self.assertEqual(ret, 0, ("Failed to activate snapshot %s"
% self.snap))
g.log.info("Snapshot snap%s activated successfully", self.snap)
# Snapshot list
self.assertIsNotNone(
get_snap_list(self.mnode), "Failed to list snapshot")
g.log.info("Snapshot list command Successful")
# Creating and starting a Clone of snapshot:
ret, _, _ = snap_clone(self.mnode, self.snap, self.clone_vol1)
self.assertEqual(ret, 0, "Failed to clone %s" % self.clone_vol1)
g.log.info("Clone volume %s created successfully", self.clone_vol1)
# Start the clone volumes
ret, _, _ = volume_start(self.mnode, self.clone_vol1)
self.assertEqual(ret, 0, "Failed to start %s" % self.clone_vol1)
g.log.info("%s started successfully", self.clone_vol1)
# Form server list
brick_list = get_all_bricks(self.mnode, self.clone_vol1)
for bricks in brick_list:
self.server_lists.append(bricks.split(":")[0])
self.server_list = list(set(self.server_lists))
# Get volume info
vol_info = get_volume_info(self.mnode, self.clone_vol1)
self.assertIsNotNone(vol_info, "Failed to get vol info")
g.log.info("Successfully in getting vol info")
# Redefining mounts for cloned volume
self.mount_points, self.mounts_dict_list = [], []
for client in self.all_clients_info:
mount = {
'protocol': self.mount_type,
'server': self.mnode,
'volname': self.volname,
'client': self.all_clients_info[client],
'mountpoint': (path.join(
"%s" % self.mpoint)),
'options': ''
}
self.mounts_dict_list.append(mount)
self.mount1 = create_mount_objs(self.mounts_dict_list)
self.mount_points.append(self.mpoint)
g.log.info("Successfully made entry in self.mount1")
# FUSE mount clone1 volume
for mount_obj in self.mounts:
ret, _, _ = mount_volume(self.clone_vol1, self.mount_type,
self.mpoint,
self.mnode, mount_obj.client_system)
self.assertEqual(ret, 0, "Volume mount failed for clone1")
g.log.info("%s mounted Successfully", self.clone_vol1)
# Validate clone volume is mounted or not
ret = is_mounted(self.clone_vol1, self.mpoint, self.mnode,
mount_obj.client_system, self.mount_type)
self.assertTrue(ret, "Volume not mounted on mount point: "
"%s" % self.mpoint)
g.log.info("Volume %s mounted on %s", self.clone_vol1, self.mpoint)
# Log Cloned Volume information
ret = log_volume_info_and_status(self.mnode, self.clone_vol1)
self.assertTrue("Failed to Log Info and Status of Volume "
"%s" % self.clone_vol1)
g.log.info("Successfully Logged Info and Status")
# Validate snapd running on all nodes
self.validate_snapd(check_condition=False)
# Enable USS
ret, _, _ = enable_uss(self.mnode, self.clone_vol1)
self.assertEqual(ret, 0, "Failed to enable USS on cloned volume")
g.log.info("Successfully enabled USS on Cloned volume")
# Validate USS running
self.validate_uss()
# Validate snapd running on all nodes
self.validate_snapd()
# Stop cloned volume
ret, _, _ = volume_stop(self.mnode, self.clone_vol1)
self.assertEqual(ret, 0, "Failed to stop cloned volume "
"%s" % self.clone_vol1)
g.log.info("Successfully Stopped Cloned volume %s", self.clone_vol1)
# Validate snapd running on all nodes
self.validate_snapd(check_condition=False)
# Start cloned volume
ret, _, _ = volume_start(self.mnode, self.clone_vol1)
self.assertEqual(ret, 0, "Failed to start cloned volume"
" %s" % self.clone_vol1)
g.log.info("Successfully started cloned volume"
" %s", self.clone_vol1)
# Validate snapd running on all nodes
self.validate_snapd()
# Create 5 snapshots
self.snaps_list = [('test_snap_clone_snapd-snap%s'
% i)for i in range(0, 5)]
for snapname in self.snaps_list:
ret, _, _ = snap_create(self.mnode, self.clone_vol1,
snapname)
self.assertEqual(ret, 0, ("Failed to create snapshot for volume"
" %s" % self.clone_vol1))
g.log.info("Snapshot %s created successfully for volume "
"%s", snapname, self.clone_vol1)
# Validate USS running
self.validate_uss()
# Check snapshot under .snaps directory
self.check_snaps()
# Activate Snapshots
for snapname in self.snaps_list:
ret, _, _ = snap_activate(self.mnode, snapname)
self.assertEqual(ret, 0, ("Failed to activate snapshot %s"
% snapname))
g.log.info("Snapshot %s activated "
"successfully", snapname)
# Validate USS running
self.validate_uss()
# Validate snapshots under .snaps folder
self.validate_snaps()
# Kill snapd on node and validate snapd except management node
for server in self.servers[1:]:
ret, _, _ = terminate_snapd_on_node(server)
self.assertEqual(ret, 0, "Failed to Kill snapd on node %s"
% server)
g.log.info("snapd Killed Successfully on node %s", server)
# Check snapd running
ret = is_snapd_running(server, self.clone_vol1)
self.assertTrue(ret, "Unexpected: Snapd running on node: "
"%s" % server)
g.log.info("Expected: Snapd is not running on node:%s", server)
# Check snapshots under .snaps folder
g.log.info("Validating snapshots under .snaps")
ret, _, _ = uss_list_snaps(self.clients[0], self.mpoint)
self.assertEqual(ret, 0, "Target endpoint not connected")
g.log.info("Successfully listed snapshots under .snaps")
# Kill snapd in management node
ret, _, _ = terminate_snapd_on_node(self.servers[0])
self.assertEqual(ret, 0, "Failed to Kill snapd on node %s"
% self.servers[0])
g.log.info("snapd Killed Successfully on node %s", self.servers[0])
# Validate snapd running on all nodes
self.validate_snapd(check_condition=False)
# Validating snapshots under .snaps
ret, _, _ = uss_list_snaps(self.clients[0], self.mpoint)
self.assertNotEqual(ret, 0, "Unexpected: Successfully listed "
"snapshots under .snaps")
g.log.info("Expected: Target endpoint not connected")
# Start the Cloned volume(force start)
ret, _, _ = volume_start(self.mnode, self.clone_vol1, force=True)
self.assertEqual(ret, 0, "Failed to start cloned volume "
"%s" % self.clone_vol1)
g.log.info("Successfully Started Cloned volume %s", self.clone_vol1)
# Validate snapd running on all nodes
self.validate_snapd()
# Validate snapshots under .snaps folder
self.validate_snaps()
def test_server_side_healing_happens_only_when_glustershd_running(self):
"""
Test Script which verifies that the server side healing must happen
only if the heal daemon is running on the node where source brick
resides.
* Create and start the Replicate volume
* Check the glustershd processes - Only 1 glustershd should be listed
* Bring down the bricks without affecting the cluster
* Create files on volume
* kill the glustershd on node where bricks is running
* bring the bricks up which was killed in previous steps
* check the heal info - heal info must show pending heal info, heal
shouldn't happen since glustershd is down on source node
* issue heal
* trigger client side heal
* heal should complete successfully
"""
# pylint: disable=too-many-locals,too-many-statements,too-many-lines
# Setting Volume options
options = {
"metadata-self-heal": "on",
"entry-self-heal": "on",
"data-self-heal": "on"
}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to set options %s' % options)
g.log.info("Successfully set %s for volume %s", options, self.volname)
# Check the self-heal daemon process
ret, pids = get_self_heal_daemon_pid(self.servers)
self.assertTrue(ret, ("Either No self heal daemon process found or "
"more than One self heal daemon process "
"found : %s" % pids))
g.log.info(
"Successful in verifying self heal daemon process"
" on all nodes %s", self.servers)
# Select the bricks to bring offline
bricks_to_bring_offline = (select_volume_bricks_to_bring_offline(
self.mnode, self.volname))
g.log.info("Brick List to bring offline : %s", bricks_to_bring_offline)
# Bring down the selected bricks
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, "Failed to bring down the bricks")
g.log.info("Brought down the brick process "
"for %s", bricks_to_bring_offline)
# Write files on all mounts
all_mounts_procs, num_files_to_write = [], 100
for mount_obj in self.mounts:
cmd = ("/usr/bin/env python %s create_files "
"-f %s --base-file-name file %s" %
(self.script_upload_path, num_files_to_write,
mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("IO is successful on all mounts")
# Get online bricks list
online_bricks = get_online_bricks_list(self.mnode, self.volname)
g.log.info("Online Bricks for volume %s : %s", self.volname,
online_bricks)
# Get the nodes where bricks are running
bring_offline_glustershd_nodes = []
for brick in online_bricks:
bring_offline_glustershd_nodes.append(brick.split(":")[0])
g.log.info("self heal deamon on nodes %s to be killed",
bring_offline_glustershd_nodes)
# Kill the self heal daemon process on nodes
ret = bring_self_heal_daemon_process_offline(
bring_offline_glustershd_nodes)
self.assertTrue(
ret, ("Unable to bring self heal daemon process"
" offline for nodes %s" % bring_offline_glustershd_nodes))
g.log.info(
"Sucessfully brought down self heal process for "
"nodes %s", bring_offline_glustershd_nodes)
# Check the heal info
heal_info = get_heal_info_summary(self.mnode, self.volname)
g.log.info("Successfully got heal info %s for the volume %s",
heal_info, self.volname)
# Bring bricks online
ret = bring_bricks_online(self.mnode, self.volname,
bricks_to_bring_offline, 'glusterd_restart')
self.assertTrue(
ret,
("Failed to bring bricks: %s online" % bricks_to_bring_offline))
# Issue heal
ret = trigger_heal_full(self.mnode, self.volname)
self.assertFalse(ret,
("Able to trigger heal on volume %s where "
"self heal daemon is not running" % self.volname))
g.log.info(
"Expected : Unable to trigger heal on volume %s where "
"self heal daemon is not running", self.volname)
# Wait for 130 sec to heal
ret = monitor_heal_completion(self.mnode, self.volname, 130)
self.assertFalse(ret, ("Heal Completed on volume %s" % self.volname))
g.log.info("Expected : Heal pending on volume %s", self.volname)
# Check the heal info
heal_info_after_triggering_heal = get_heal_info_summary(
self.mnode, self.volname)
g.log.info("Successfully got heal info for the volume %s",
self.volname)
# Compare with heal pending with the files wrote
for node in online_bricks:
self.assertGreaterEqual(
int(heal_info_after_triggering_heal[node]['numberOfEntries']),
num_files_to_write,
("Some of the files are healed from source bricks %s where "
"self heal daemon is not running" % node))
g.log.info("EXPECTED: No files are healed from source bricks where "
"self heal daemon is not running")
# Unmount and Mount volume again as volume options were set
# after mounting the volume
for mount_obj in self.mounts:
ret, _, _ = umount_volume(mount_obj.client_system,
mount_obj.mountpoint)
self.assertEqual(ret, 0,
"Failed to unmount %s" % mount_obj.client_system)
ret, _, _ = mount_volume(self.volname,
mtype='glusterfs',
mpoint=mount_obj.mountpoint,
mserver=self.mnode,
mclient=mount_obj.client_system)
self.assertEqual(ret, 0,
"Failed to mount %s" % mount_obj.client_system)
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("/usr/bin/env python %s read %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "Reads failed on some of the clients")
g.log.info("Reads successful on all mounts")
# Wait for heal to complete
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, "Unable to heal the pending entries")
g.log.info("Successfully healed the pending entries for volume %s",
self.volname)
def test_validate_authreject_vol(self):
"""
-Set Authentication Reject for client1
-Check if bricks are online
-Mounting the vol on client1
-Check if bricks are online
-Mounting the vol on client2
-Reset the Volume
-Check if bricks are online
-Mounting the vol on client1
"""
# pylint: disable=too-many-statements
# Obtain hostname of clients
ret, hostname_client1, _ = g.run(self.clients[0], "hostname")
self.assertEqual(
ret, 0,
("Failed to obtain hostname of client %s" % self.clients[0]))
g.log.info("Obtained hostname of client. IP- %s, hostname- %s",
self.clients[0], hostname_client1.strip())
# Set Authentication
option = {"auth.reject": hostname_client1.strip()}
ret = set_volume_options(self.mnode, self.volname, option)
self.assertTrue(
ret, ("Failed to set authentication with option: %s" % option))
g.log.info("Authentication Set successfully with option: %s", option)
# Fetching all the bricks
self.mountpoint = "/mnt/testvol"
bricks_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(bricks_list, "Brick list is empty")
g.log.info("Brick List : %s", bricks_list)
# Check are bricks online
ret = are_bricks_online(self.mnode, self.volname, bricks_list)
self.assertTrue(ret, "All bricks are not online")
# Using this way to check because of bug 1586036
# Mounting volume
ret, _, _ = mount_volume(self.volname, self.mount_type,
self.mountpoint, self.mnode, self.clients[0])
# Checking if volume is mounted
out = is_mounted(self.volname,
self.mountpoint,
self.mnode,
self.clients[0],
self.mount_type,
user='******')
if (ret == 0) & (not out):
g.log.error("Mount executed successfully due to bug 1586036")
elif (ret == 1) & (not out):
g.log.info("Expected:Mounting has failed successfully")
else:
raise ExecutionError(
"Unexpected Mounting of Volume %s successful" % self.volname)
# Checking client logs for authentication error
cmd = ("grep AUTH_FAILED /var/log/glusterfs/mnt-" "testvol.log")
ret, _, _ = g.run(self.clients[0], cmd)
self.assertEqual(
ret, 0, "Mounting has not failed due to"
"authentication error")
g.log.info("Mounting has failed due to authentication error")
# Mounting the vol on client2
# Check bricks are online
ret = are_bricks_online(self.mnode, self.volname, bricks_list)
self.assertTrue(ret, "All bricks are not online")
# Mounting Volume
ret, _, _ = mount_volume(self.volname, self.mount_type,
self.mountpoint, self.mnode, self.clients[1])
self.assertEqual(ret, 0, "Failed to mount volume")
g.log.info("Mounted Successfully")
# Checking if volume is mounted
out = is_mounted(self.volname,
self.mountpoint,
self.mnode,
self.clients[1],
self.mount_type,
user='******')
self.assertTrue(out, "Volume %s has failed to mount" % self.volname)
# Reset Volume
ret, _, _ = volume_reset(mnode=self.mnode, volname=self.volname)
self.assertEqual(ret, 0, "Failed to reset volume")
g.log.info("Volume %s reset operation is successful", self.volname)
# Checking if bricks are online
ret = are_bricks_online(self.mnode, self.volname, bricks_list)
self.assertTrue(ret, "All bricks are not online")
# Mounting Volume
ret, _, _ = mount_volume(self.volname, self.mount_type,
self.mountpoint, self.mnode, self.clients[0])
self.assertEqual(ret, 0, "Failed to mount volume")
g.log.info("Mounted Successfully")
# Checking if Volume is mounted
out = is_mounted(self.volname,
self.mountpoint,
self.servers[0],
self.clients[0],
self.mount_type,
user='******')
self.assertTrue(out, "Volume %s has failed to mount" % self.volname)
g.log.info("Volume is mounted successfully %s", self.volname)
def test_snap_rebalance(self):
# pylint: disable=too-many-statements, too-many-locals
"""
Snapshot rebalance contains tests which verifies snapshot clone,
creating snapshot and performing I/O on mountpoints
Steps:
1. Create snapshot of a volume
2. Activate snapshot
3. Clone snapshot and Activate
4. Mount Cloned volume
5. Perform I/O on mount point
6. Calculate areequal for bricks and mountpoints
7. Add-brick more brick to cloned volume
8. Initiate Re-balance
9. validate areequal of bricks and mountpoints
"""
# Creating snapshot:
g.log.info("Starting to Create snapshot")
ret, _, _ = snap_create(self.mnode, self.volname, self.snap)
self.assertEqual(
ret, 0, ("Failed to create snapshot for volume %s" % self.volname))
g.log.info("Snapshot %s created successfully for volume %s", self.snap,
self.volname)
# Activating snapshot
g.log.info("Starting to Activate Snapshot")
ret, _, _ = snap_activate(self.mnode, self.snap)
self.assertEqual(ret, 0,
("Failed to Activate snapshot %s" % self.snap))
g.log.info("Snapshot %s activated successfully", self.snap)
# Creating a Clone of snapshot:
g.log.info("creating Clone Snapshot")
ret, _, _ = snap_clone(self.mnode, self.snap, self.clone)
self.assertEqual(ret, 0, ("Failed to clone volume %s" % self.clone))
g.log.info("clone volume %s created successfully", self.clone)
# Starting clone volume
g.log.info("starting clone volume")
ret, _, _ = volume_start(self.mnode, self.clone)
self.assertEqual(ret, 0, "Failed to start %s" % self.clone)
g.log.info("clone volume %s started successfully", self.clone)
# Mounting a clone volume
g.log.info("Mounting created clone volume")
ret, _, _ = mount_volume(self.clone, self.mount_type, self.mount1,
self.mnode, self.clients[0])
self.assertEqual(ret, 0,
"clone Volume mount failed for %s" % self.clone)
g.log.info("cloned volume %s mounted Successfully", self.clone)
# Validate clone volume mounted or not
g.log.info("Validate clone volume mounted or not")
ret = is_mounted(self.clone, self.mount1, self.mnode, self.clients[0],
self.mount_type)
self.assertTrue(
ret, "Cloned Volume not mounted on mount point: %s" % self.mount1)
g.log.info("Cloned Volume %s mounted on %s", self.clone, self.mount1)
# write files to mountpoint
g.log.info("Starting IO on %s mountpoint...", self.mount1)
all_mounts_procs = []
cmd = ("/usr/bin/env python %s create_files "
"-f 10 --base-file-name file %s" %
(self.script_upload_path, self.mount1))
proc = g.run(self.clients[0], cmd)
all_mounts_procs.append(proc)
self.check_arequal()
# expanding volume
g.log.info("Starting to expand volume")
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, "Failed to expand volume %s" % self.clone)
g.log.info("Expand volume successful")
ret, _, _ = rebalance_start(self.mnode, self.clone)
self.assertEqual(ret, 0, "Failed to start rebalance")
g.log.info("Successfully started rebalance on the "
"volume %s", self.clone)
# Log Rebalance status
g.log.info("Log Rebalance status")
_, _, _ = rebalance_status(self.mnode, self.clone)
# Wait for rebalance to complete
g.log.info("Waiting for rebalance to complete")
ret = wait_for_rebalance_to_complete(self.mnode, self.clone)
self.assertTrue(ret, ("Rebalance is not yet complete "
"on the volume %s", self.clone))
g.log.info("Rebalance is successfully complete on "
"the volume %s", self.clone)
# Check Rebalance status after rebalance is complete
g.log.info("Checking Rebalance status")
ret, _, _ = rebalance_status(self.mnode, self.clone)
self.assertEqual(ret, 0, ("Failed to get rebalance status for "
"the volume %s", self.clone))
g.log.info("Successfully got rebalance status of the "
"volume %s", self.clone)
self.check_arequal()
def test_spurious_rebalance(self):
"""
In this test case:
1. Trusted storage Pool of 3 nodes
2. Create a distributed volumes with 3 bricks
3. Start the volume
4. Fuse mount the gluster volume on out of trusted nodes
5. Remove a brick from the volume
6. Check remove-brick status
7. Stop the remove brick process
8. Perform fix-layoyt on the volume
9. Get the rebalance fix-layout status
10. Create a directory from mount point
11. Check trusted.glusterfs.dht extended attribue for newly
created directory on the remove brick
"""
# pylint: disable=too-many-statements
my_servers = self.servers[0:3]
my_server_info = {}
for server in self.servers[0:3]:
my_server_info[server] = self.all_servers_info[server]
for index in range(1, 3):
ret, _, _ = peer_probe(self.servers[0], self.servers[index])
self.assertEqual(ret, 0, ("peer probe from %s to %s is failed",
self.servers[0], self.servers[index]))
g.log.info("peer probe is success from %s to "
"%s", self.servers[0], self.servers[index])
self.volname = "testvol"
bricks_list = form_bricks_list(self.mnode, self.volname, 3, my_servers,
my_server_info)
g.log.info("Creating a volume %s ", self.volname)
ret, _, _ = volume_create(self.mnode,
self.volname,
bricks_list,
force=False)
self.assertEqual(ret, 0, ("Unable"
"to create volume %s" % self.volname))
g.log.info("Volume created successfully %s", self.volname)
ret, _, _ = volume_start(self.mnode, self.volname, False)
self.assertEqual(ret, 0, ("Failed to start the "
"volume %s", self.volname))
g.log.info("Get all the bricks of the volume")
bricks_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(bricks_list, "Failed to get the brick list")
g.log.info("Successfully got the list of bricks of volume")
# Mounting a volume
ret, _, _ = mount_volume(self.volname,
mtype=self.mount_type,
mpoint=self.mounts[0].mountpoint,
mserver=self.mnode,
mclient=self.mounts[0].client_system)
self.assertEqual(ret, 0, ("Volume %s is not mounted") % self.volname)
g.log.info("Volume mounted successfully : %s", self.volname)
remove_brick_list = []
remove_brick_list.append(bricks_list[2])
ret, _, _ = remove_brick(self.mnode, self.volname, remove_brick_list,
'start')
self.assertEqual(ret, 0, "Failed to start remove brick operation")
g.log.info("Remove bricks operation started successfully")
ret, _, _ = remove_brick(self.mnode, self.volname, remove_brick_list,
'stop')
self.assertEqual(ret, 0, "Failed to stop remove brick operation")
g.log.info("Remove bricks operation stopped successfully")
g.log.info("Starting Fix-layoyt on the volume")
ret, _, _ = rebalance_start(self.mnode, self.volname, True)
self.assertEqual(ret, 0, ("Failed to start rebalance for fix-layout"
"on the volume %s", self.volname))
g.log.info("Successfully started fix-layout on the volume %s",
self.volname)
# Wait for fix-layout to complete
g.log.info("Waiting for fix-layout to complete")
ret = wait_for_fix_layout_to_complete(self.mnode, self.volname)
self.assertTrue(ret, ("Fix-layout is not yet complete on the volume "
"%s", self.volname))
g.log.info("Fix-layout is successfully complete on the volume %s",
self.volname)
ret = mkdir(self.mounts[0].client_system,
"%s/dir1" % self.mounts[0].mountpoint)
self.assertTrue(ret, ("Failed to create directory dir1"))
g.log.info("directory dir1 is created successfully")
brick_server, brick_dir = remove_brick_list[0].split(':')
folder_name = brick_dir + "/dir1"
g.log.info("Check trusted.glusterfs.dht on host %s for directory %s",
brick_server, folder_name)
ret = get_fattr(brick_server, folder_name, 'trusted.glusterfs.dht')
self.assertTrue(
ret, ("Failed to get trusted.glusterfs.dht for %s" % folder_name))
g.log.info("get trusted.glusterfs.dht xattr for %s successfully",
folder_name)
def test_rebalance_hang(self):
"""
In this test case:
1. Trusted storage Pool of 2 nodes
2. Create a distributed volumes with 2 bricks
3. Start the volume
4. Mount the volume
5. Add some data file on mount
6. Start rebalance with force
7. kill glusterd on 2nd node
8. Issue volume related command
"""
# pylint: disable=too-many-statements
my_server_info = {
self.servers[0]: self.all_servers_info[self.servers[0]]
}
my_servers = self.servers[0:2]
index = 1
ret, _, _ = peer_probe(self.servers[0], self.servers[index])
self.assertEqual(ret, 0, ("peer probe from %s to %s is failed",
self.servers[0], self.servers[index]))
g.log.info("peer probe is success from %s to "
"%s", self.servers[0], self.servers[index])
key = self.servers[index]
my_server_info[key] = self.all_servers_info[key]
self.volname = "testvol"
bricks_list = form_bricks_list(self.mnode, self.volname, 2, my_servers,
my_server_info)
g.log.info("Creating a volume %s ", self.volname)
ret, _, _ = volume_create(self.mnode,
self.volname,
bricks_list,
force=False)
self.assertEqual(ret, 0, ("Unable"
"to create volume %s" % self.volname))
g.log.info("Volume created successfully %s", self.volname)
ret, _, _ = volume_start(self.mnode, self.volname, False)
self.assertEqual(ret, 0, ("Failed to start the "
"volume %s", self.volname))
g.log.info("Get all the bricks of the volume")
bricks_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(bricks_list, "Failed to get the brick list")
g.log.info("Successfully got the list of bricks of volume")
# Mounting a volume
ret, _, _ = mount_volume(self.volname,
mtype=self.mount_type,
mpoint=self.mounts[0].mountpoint,
mserver=self.mnode,
mclient=self.mounts[0].client_system)
self.assertEqual(ret, 0, ("Volume %s is not mounted") % self.volname)
g.log.info("Volume mounted successfully : %s", self.volname)
self.all_mounts_procs = []
# Creating files
command = ("cd %s/ ; "
"for i in `seq 1 10` ; "
"do mkdir l1_dir.$i ; "
"for j in `seq 1 5` ; "
"do mkdir l1_dir.$i/l2_dir.$j ; "
"for k in `seq 1 10` ; "
"do dd if=/dev/urandom of=l1_dir.$i/l2_dir.$j/test.$k "
"bs=128k count=$k ; "
"done ; "
"done ; "
"done ; " % (self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.io_validation_complete = True
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Starting rebalance with force on the volume")
ret, _, _ = rebalance_start(self.mnode, self.volname, False, True)
self.assertEqual(
ret, 0, ("Failed to start rebalance for volume %s", self.volname))
g.log.info("Successfully rebalance on the volume %s", self.volname)
ret = stop_glusterd(self.servers[1])
self.assertTrue(ret, "Failed to stop glusterd on one of the node")
ret = is_glusterd_running(self.servers[1])
self.assertNotEqual(
ret, 0, ("Glusterd is not stopped on servers %s", self.servers[1]))
g.log.info("Glusterd stop on the nodes : %s succeeded",
self.servers[1])
# Wait for fix-layout to complete
g.log.info("Waiting for rebalance to complete")
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(ret, ("Rebalance is not yet complete on the volume "
"%s", self.volname))
g.log.info("Rebalance is successfully complete on the volume %s",
self.volname)
vol_status = get_volume_status(self.mnode, self.volname)
self.assertIsNotNone(
vol_status, "Failed to get volume "
"status for %s" % self.volname)
# Start glusterd on the node where it is stopped
ret = start_glusterd(self.servers[1])
self.assertTrue(ret, "glusterd start on the node failed")
count = 0
while count < 60:
ret = is_glusterd_running(self.servers[1])
if not ret:
break
sleep(2)
count += 1
self.assertEqual(ret, 0,
"glusterd is not running on %s" % self.servers[1])
g.log.info("Glusterd start on the nodes : %s "
"succeeded", self.servers[1])
def test_subdir_when_renamed(self):
# pylint: disable=too-many-statements
"""
Mount the volume
Create 1 subdir on mountpoint "d1"
Auth allow - Client1(d1),Client2(full volume)
Mount the subdir "d1" on client1 and volume on client2
Start IO's on all the mount points
Perform rename operation from client2.Rename the subdir
"d1" to "d1_renamed"
unmount volume and subdir from clients
Try mounting "d1" on client 1.This should fail.
Try mounting "d1_renamed" on client 1.This should fail.
Again set authentication.Auth allow -
Client1(d1_renamed),Client2(full volume)
Mount "d1_renamed" on client1 and volume on client2
"""
# Create directory d1 on mount point
ret = mkdir(self.mounts[0].client_system,
"%s/d1" % self.mounts[0].mountpoint)
self.assertTrue(
ret, ("Failed to create directory 'd1' on"
"volume %s from client %s" %
(self.mounts[0].volname, self.mounts[0].client_system)))
# unmount volume
ret = self.unmount_volume(self.mounts)
self.assertTrue(ret, "Volumes Unmount failed")
g.log.info("Volumes Unmounted successfully")
# Set authentication on the subdirectoy "d1" to access by client1
# and volume to access by client2
g.log.info(
'Setting authentication on subdirectory d1 to access'
'by client %s and on volume to access by client %s',
self.clients[0], self.clients[1])
ret = set_auth_allow(self.volname, self.mnode, {
'/d1': [self.clients[0]],
'/': [self.clients[1]]
})
self.assertTrue(
ret, 'Failed to set Authentication on volume %s' % self.volume)
# Creating mount list for mounting subdir mount and volume
self.subdir_mounts = [
copy.deepcopy(self.mounts[0]),
copy.deepcopy(self.mounts[1])
]
self.subdir_mounts[0].volname = "%s/d1" % self.volname
self.subdir_mounts[0].client_system = self.clients[0]
self.subdir_mounts[1].client_system = self.clients[1]
# Mount Subdirectory d1 on client 1 and volume on client 2
for mount_obj in self.subdir_mounts:
mountpoint = mount_obj.mountpoint
ret = mount_obj.mount()
self.assertTrue(
ret, ("Failed to mount %s on client"
" %s" % (mount_obj.volname, mount_obj.client_system)))
g.log.info("Successfully mounted %s on client %s",
mount_obj.volname, mount_obj.client_system)
g.log.info("Successfully mounted sub directory and volume to"
"authenticated clients")
# Start IO on all the mounts.
all_mounts_procs = []
count = 1
for mount_obj in self.subdir_mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 10 "
"--max-num-of-dirs 5 "
"--num-of-files 5 %s" %
(self.script_upload_path, count, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
count = count + 10
# Validate IO
g.log.info("Validating IO's")
ret = validate_io_procs(all_mounts_procs, self.subdir_mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Successfully validated all io's")
# Get stat of all the files/dirs created.
g.log.info("Get stat of all the files/dirs created.")
ret = get_mounts_stat(self.subdir_mounts)
self.assertTrue(ret, "Stat failed on some of the clients")
g.log.info("Successfully got stat of all files/dirs created")
# Rename the subdir "d1" to "d1_renamed" from client2
source_fpath = "%s/d1" % mountpoint
dest_fpath = "%s/d1_renamed" % mountpoint
ret = move_file(self.clients[1], source_fpath, dest_fpath)
self.assertTrue(ret, "Rename subdirectory failed")
g.log.info('Renamed directory %s to %s', source_fpath, dest_fpath)
# unmount volume and subdir from client
ret = self.unmount_volume(self.subdir_mounts)
self.assertTrue(ret, "Volumes UnMount failed")
g.log.info("Volumes Unmounted successfully")
# Try mounting subdir "d1" on client1
_, _, _ = mount_volume("%s/d1" % self.volname, self.mount_type,
mountpoint, self.mnode, self.clients[0])
ret = is_mounted(self.volname, mountpoint, self.mnode, self.clients[0],
self.mount_type)
self.assertEqual(
ret, 0, "d1 mount should have failed.But d1 is"
"successfully mounted on mount point: %s" % mountpoint)
g.log.info("subdir %s/d1 is not mounted as expected %s", self.volname,
mountpoint)
# Try mounting subdir "d1_renamed" on client1
_, _, _ = mount_volume("%s/d1_renamed" % self.volname, self.mount_type,
mountpoint, self.mnode, self.clients[0])
ret = is_mounted("%s/d1_renamed" % self.volname, mountpoint,
self.mnode, self.clients[0], self.mount_type)
self.assertEqual(
ret, 0, "d1_renamed mount should have failed.But"
"d1_renamed is successfully mounted on : %s" % mountpoint)
g.log.info("subdir %s/d1_renamed is not mounted as expected %s",
self.volname, mountpoint)
# Set authentication on the subdirectoy "d1_renamed" to access
# by client1 and volume to access by client2
g.log.info(
'Setting authentication on subdirectory d1_renamed to'
'access by client %s and on volume to access by client %s',
self.clients[0], self.clients[1])
ret = set_auth_allow(self.volname, self.mnode, {
'/d1_renamed': [self.clients[0]],
'/': [self.clients[1]]
})
self.assertTrue(
ret, 'Failed to set Authentication on volume %s' % self.volume)
# Overwriting the list of subdir mount, directory d1 to d1_renamed
self.subdir_mounts[0].volname = "%s/d1_renamed" % self.volname
# Mount Subdirectory d1_renamed on client 1 and volume on client 2
for mount_obj in self.subdir_mounts:
ret = mount_obj.mount()
self.assertTrue(
ret, ("Failed to mount %s on client"
" %s" % (mount_obj.volname, mount_obj.client_system)))
g.log.info("Successfully mounted %s on client %s",
mount_obj.volname, mount_obj.client_system)
g.log.info("Successfully mounted sub directory and volume to"
"authenticated clients")
# Get stat of all the files/dirs created from both clients.
g.log.info("Get stat of all the files/dirs created.")
ret = get_mounts_stat(self.subdir_mounts)
self.assertTrue(ret, "Stat failed on some of the clients")
g.log.info("Successfully got stat of all files/dirs created")
def test_create_vol_used_bricks(self):
'''
-> Create distributed-replica Volume
-> Add 6 bricks to the volume
-> Mount the volume
-> Perform some I/O's on mount point
-> unmount the volume
-> Stop and delete the volume
-> Create another volume using bricks of deleted volume
'''
# Create and start a volume
self.volume['name'] = "test_create_vol_with_fresh_bricks"
self.volname = "test_create_vol_with_fresh_bricks"
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, "Failed to create and start volume")
# Forming brick list
brick_list = form_bricks_list(self.mnode, self.volname, 6,
self.servers, self.all_servers_info)
# Adding bricks to the volume
ret, _, _ = add_brick(self.mnode, self.volname, brick_list)
self.assertEqual(
ret, 0, "Failed to add bricks to the volume %s" % self.volname)
g.log.info("Bricks added successfully to the volume %s", self.volname)
# Mounting the volume.
for mount_obj in self.mounts:
ret, _, _ = mount_volume(self.volname,
mtype=self.mount_type,
mpoint=mount_obj.mountpoint,
mserver=self.mnode,
mclient=mount_obj.client_system)
self.assertEqual(ret, 0,
("Volume %s is not mounted") % (self.volname))
g.log.info("Volume mounted successfully : %s", self.volname)
# run IOs
g.log.info("Starting IO on all mounts...")
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = (
"/usr/bin/env python %s create_deep_dirs_with_files "
"--dirname-start-num %d --dir-depth 2 "
"--dir-length 5 --max-num-of-dirs 3 "
"--num-of-files 10 %s" %
(self.script_upload_path, self.counter, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.counter = self.counter + 10
# Validate IO
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# Unmouting the volume.
for mount_obj in self.mounts:
ret, _, _ = umount_volume(mclient=mount_obj.client_system,
mpoint=mount_obj.mountpoint)
self.assertEqual(ret, 0,
"Volume %s is not unmounted" % (self.volname))
g.log.info("Volume unmounted successfully : %s", self.volname)
# Getting brick list
self.brick_list = get_all_bricks(self.mnode, self.volname)
if not self.brick_list:
raise ExecutionError("Failed to get the brick list of %s" %
self.volname)
# Stop volume
ret, _, _ = volume_stop(self.mnode, self.volname)
self.assertEqual(ret, 0, "Failed to stop the volume %s" % self.volname)
g.log.info("Volume %s stopped successfully", self.volname)
# Delete Volume
ret, _, _ = g.run(
self.mnode,
"gluster volume delete %s --mode=script" % self.volname)
self.assertEqual(ret, 0, "Failed to delete volume %s" % self.volname)
g.log.info("Volume deleted successfully %s", self.volname)
# Create another volume by using bricks of deleted volume
self.volname = "test_create_vol_used_bricks"
ret, _, err = volume_create(self.mnode,
self.volname,
brick_list[0:6],
replica_count=3)
self.assertNotEqual(
ret, 0, "Volume creation should fail with used "
"bricks but volume creation success")
g.log.info("Failed to create volume with used bricks")
# Checking failed message of volume creation
msg = ' '.join([
'volume create: test_create_vol_used_bricks: failed:',
brick_list[0].split(':')[1], 'is already part of a volume'
])
self.assertIn(
msg, err, "Incorrect error message for volume creation "
"with used bricks")
g.log.info("correct error message for volume creation with "
"used bricks")
def test_detach_node_used_to_mount(self):
# pylint: disable=too-many-statements
"""
Test case:
1.Create a 1X3 volume with only 3 nodes from the cluster.
2.Mount volume on client node using the ip of the fourth node.
3.Write IOs to the volume.
4.Detach node N4 from cluster.
5.Create a new directory on the mount point.
6.Create a few files using the same command used in step 3.
7.Add three more bricks to make the volume
2x3 using add-brick command.
8.Do a gluster volume rebalance on the volume.
9.Create more files from the client on the mount point.
10.Check for files on bricks from both replica sets.
11.Create a new directory from the client on the mount point.
12.Check for directory in both replica sets.
"""
# Create and start a volume
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, "Failed to create and start volume")
g.log.info("Volume %s created successfully", self.volname)
# Mounting the volume.
ret, _, _ = mount_volume(self.volname,
mtype=self.mount_type,
mpoint=self.mounts[0].mountpoint,
mserver=self.servers[4],
mclient=self.mounts[0].client_system)
self.assertEqual(ret, 0, ("Volume %s is not mounted") % self.volname)
g.log.info("Volume mounted successfully using %s", self.servers[4])
# Creating 100 files.
command = ('for number in `seq 1 100`;do touch ' +
self.mounts[0].mountpoint + '/file$number; done')
ret, _, _ = g.run(self.mounts[0].client_system, command)
self.assertEqual(ret, 0, "File creation failed.")
g.log.info("Files create on mount point.")
# Detach N4 from the list.
ret, _, _ = peer_detach(self.mnode, self.servers[4])
self.assertEqual(ret, 0, "Failed to detach %s" % self.servers[4])
g.log.info("Peer detach successful %s", self.servers[4])
# Creating a dir.
ret = mkdir(self.mounts[0].client_system,
self.mounts[0].mountpoint + "/dir1",
parents=True)
self.assertTrue(ret, ("Failed to create directory dir1."))
g.log.info("Directory dir1 created successfully.")
# Creating 100 files.
command = ('for number in `seq 101 200`;do touch ' +
self.mounts[0].mountpoint + '/file$number; done')
ret, _, _ = g.run(self.mounts[0].client_system, command)
self.assertEqual(ret, 0, "File creation failed.")
g.log.info("Files create on mount point.")
# Forming brick list
brick_list = form_bricks_list_to_add_brick(self.mnode, self.volname,
self.servers,
self.all_servers_info)
# Adding bricks
ret, _, _ = add_brick(self.mnode, self.volname, brick_list)
self.assertEqual(ret, 0,
"Failed to add brick to the volume %s" % self.volname)
g.log.info("Brick added successfully to the volume %s", self.volname)
# Start rebalance for volume.
g.log.info("Starting rebalance on the volume")
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to start rebalance "
"on the volume %s", self.volname))
g.log.info("Successfully started rebalance on the volume %s",
self.volname)
# Creating 100 files.
command = ('for number in `seq 201 300`;do touch ' +
self.mounts[0].mountpoint + '/file$number; done')
ret, _, _ = g.run(self.mounts[0].client_system, command)
self.assertEqual(ret, 0, "File creation failed.")
g.log.info("Files create on mount point.")
# Check for files on bricks.
attempts = 10
while attempts:
number = str(randint(1, 300))
for brick in brick_list:
brick_server, brick_dir = brick.split(':')
file_name = brick_dir + "/file" + number
if file_exists(brick_server, file_name):
g.log.info("Check xattr"
" on host %s for file %s", brick_server,
file_name)
ret = get_fattr_list(brick_server, file_name)
self.assertTrue(ret,
("Failed to get xattr for %s" % file_name))
g.log.info("Got xattr for %s successfully", file_name)
attempts -= 1
# Creating a dir.
ret = mkdir(self.mounts[0].client_system,
self.mounts[0].mountpoint + "/dir2")
if not ret:
attempts = 5
while attempts:
ret = mkdir(self.mounts[0].client_system,
self.mounts[0].mountpoint + "/dir2")
if ret:
break
attempts -= 1
self.assertTrue(ret, ("Failed to create directory dir2."))
g.log.info("Directory dir2 created successfully.")
# Check for directory in both replica sets.
for brick in brick_list:
brick_server, brick_dir = brick.split(':')
folder_name = brick_dir + "/dir2"
if file_exists(brick_server, folder_name):
g.log.info(
"Check trusted.glusterfs.dht"
" on host %s for directory %s", brick_server, folder_name)
ret = get_fattr(brick_server, folder_name,
'trusted.glusterfs.dht')
self.assertTrue(ret, ("Failed to get trusted.glusterfs.dht"
" xattr for %s" % folder_name))
g.log.info(
"Get trusted.glusterfs.dht xattr"
" for %s successfully", folder_name)
def test_induce_holes_then_lookup(self):
# pylint: disable=too-many-statements
m_point = self.mounts[0].mountpoint
command = 'mkdir -p ' + m_point + '/testdir'
ret, _, _ = g.run(self.clients[0], command)
self.assertEqual(ret, 0, "mkdir failed")
g.log.info("mkdir is successful")
# DHT Layout validation
g.log.debug("Verifying hash layout values %s:%s", self.clients[0],
self.mounts[0].mountpoint)
ret = validate_files_in_dir(self.clients[0],
self.mounts[0].mountpoint,
test_type=LAYOUT_IS_COMPLETE,
file_type=FILETYPE_DIRS)
self.assertTrue(ret, "LAYOUT_IS_COMPLETE: FAILED")
g.log.info("LAYOUT_IS_COMPLETE: PASS")
# Log Volume Info and Status before shrinking the volume.
g.log.info("Logging volume info and Status before shrinking volume")
log_volume_info_and_status(self.mnode, self.volname)
# Shrinking volume by removing bricks
g.log.info("Start removing bricks from volume")
ret, _, _ = remove_brick(self.mnode, self.volname,
self.remove_brick_list, "force")
self.assertFalse(ret, "Remove-brick with force: FAIL")
g.log.info("Remove-brick with force: PASS")
# Check the layout
dirpath = '/testdir'
ret = is_layout_complete(self.mnode, self.volname, dirpath)
self.assertFalse(ret, "Volume %s: Layout is complete")
g.log.info("Volume %s: Layout has some holes")
# Mount the volume on a new mount point
mount_point = tempfile.mkdtemp()
ret, _, _ = mount_volume(self.volname,
mtype='glusterfs',
mpoint=mount_point,
mserver=self.mnode,
mclient=self.mnode)
self.assertEqual(
ret, 0, ("Failed to do gluster mount on volume %s", self.volname))
g.log.info("Volume %s: mount success", self.mnode)
# Send a look up on the directory
cmd = 'ls %s%s' % (mount_point, dirpath)
ret, _, err = g.run(self.mnode, cmd)
self.assertEqual(ret, 0,
("Lookup failed on %s with error %s", (dirpath, err)))
g.log.info("Lookup sent successfully on %s", dirpath)
# DHT Layout validation
g.log.debug("Verifying hash layout values %s:%s", self.clients[0],
self.mounts[0].mountpoint)
ret = validate_files_in_dir(self.clients[0],
self.mounts[0].mountpoint,
test_type=LAYOUT_IS_COMPLETE,
file_type=FILETYPE_DIRS)
self.assertTrue(ret, "LAYOUT_IS_COMPLETE: FAILED")
g.log.info("LAYOUT_IS_COMPLETE: PASS")
def test_clone_delete_snap(self):
"""
clone from snap of one volume
* Create and Mount the volume
* Enable some volume options
* Creating 2 snapshots and activate
* reset the volume
* create a clone of snapshots created
* Mount both the clones
* Perform I/O on mount point
* Check volume options of cloned volumes
* Create snapshot of the cloned snapshot volume
* cleanup snapshots and volumes
"""
# pylint: disable=too-many-statements, too-many-locals
# Enabling Volume options on the volume and validating
g.log.info("Enabling volume options for volume %s ", self.volname)
options = {" features.uss": "enable"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(
ret, ("Failed to set volume options for volume %s" % self.volname))
g.log.info("Successfully set volume options"
"for volume %s", self.volname)
# Validate feature.uss enabled or not
g.log.info("Validating feature.uss is enabled")
option = "features.uss"
vol_option = get_volume_options(self.mnode, self.volname, option)
self.assertEqual(vol_option['features.uss'], 'enable', "Failed"
" to validate "
"volume options")
g.log.info("Successfully validated volume options"
"for volume %s", self.volname)
# Creating snapshot
g.log.info("Starting to Create snapshot")
for snap_count in range(0, 2):
ret, _, _ = snap_create(self.mnode, self.volname,
"snap%s" % snap_count)
self.assertEqual(
ret, 0,
("Failed to create snapshot for volume %s" % self.volname))
g.log.info("Snapshot snap%s created successfully"
"for volume %s", snap_count, self.volname)
# Activating snapshot
g.log.info("Starting to Activate Snapshot")
for snap_count in range(0, 2):
ret, _, _ = snap_activate(self.mnode, "snap%s" % snap_count)
self.assertEqual(
ret, 0, ("Failed to Activate snapshot snap%s" % snap_count))
g.log.info("Snapshot snap%s activated successfully", snap_count)
# Reset volume:
g.log.info("Starting to Reset Volume")
ret, _, _ = volume_reset(self.mnode, self.volname, force=False)
self.assertEqual(ret, 0, ("Failed to reset volume %s" % self.volname))
g.log.info("Reset Volume on volume %s is Successful", self.volname)
# Validate feature.uss enabled or not
g.log.info("Validating feature.uss is enabled")
option = "features.uss"
vol_option = get_volume_options(self.mnode, self.volname, option)
self.assertEqual(vol_option['features.uss'], 'off', "Failed"
" to validate "
"volume options")
g.log.info("Successfully validated volume options"
"for volume %s", self.volname)
# Verify volume's all process are online
g.log.info("Starting to Verify volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s : All process are"
"not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Creating and starting a Clone of snapshot
g.log.info("Starting to Clone Snapshot")
for clone_count in range(0, 2):
ret, _, _ = snap_clone(self.mnode, "snap%s" % clone_count,
"clone%s" % clone_count)
self.assertEqual(ret, 0,
("Failed to clone clone%s volume" % clone_count))
g.log.info("clone%s volume created successfully", clone_count)
# Start Cloned volume
g.log.info("starting to Validate clone volumes are started")
for clone_count in range(0, 2):
ret, _, _ = volume_start(self.mnode, "clone%s" % clone_count)
self.assertEqual(ret, 0, ("Failed to start clone%s" % clone_count))
g.log.info("clone%s started successfully", clone_count)
g.log.info("All the clone volumes are started Successfully")
# Validate Volume start of cloned volume
g.log.info("Starting to Validate Volume start")
for clone_count in range(0, 2):
vol_info = get_volume_info(self.mnode, "clone%s" % clone_count)
if vol_info["clone%s" % clone_count]['statusStr'] != 'Started':
raise ExecutionError("Failed to get volume info for clone%s" %
clone_count)
g.log.info("Volume clone%s is in Started state", clone_count)
# Validate feature.uss enabled or not
g.log.info("Validating feature.uss is enabled")
option = "features.uss"
for clone_count in range(0, 2):
vol_option = get_volume_options(self.mnode,
"clone%s" % clone_count, option)
self.assertEqual(vol_option['features.uss'], 'enable', "Failed"
" to validate"
"volume options")
g.log.info(
"Successfully validated volume options"
"for volume clone%s", clone_count)
# Mount both the cloned volumes
g.log.info("Mounting Cloned Volumes")
for mount_obj in range(0, 2):
self.mpoint = "/mnt/clone%s" % mount_obj
cmd = "mkdir -p %s" % self.mpoint
ret, _, _ = g.run(self.clients[0], cmd)
self.assertEqual(ret, 0, ("Creation of directory %s"
"for mounting"
"volume %s failed: Directory already"
"present" %
(self.mpoint, "clone%s" % mount_obj)))
g.log.info(
"Creation of directory %s for mounting volume %s "
"success", self.mpoint, ("clone%s" % mount_obj))
ret, _, _ = mount_volume("clone%s" % mount_obj, self.mount_type,
self.mpoint, self.mnode, self.clients[0])
self.assertEqual(ret, 0, ("clone%s is not mounted" % mount_obj))
g.log.info("clone%s is mounted Successfully", mount_obj)
# Perform I/O on mount
# Start I/O on all mounts
g.log.info("Starting to Perform I/O on Mountpoint")
all_mounts_procs = []
for mount_obj in range(0, 2):
cmd = ("cd /mnt/clone%s/; for i in {1..10};"
"do touch file$i; done; cd;") % mount_obj
proc = g.run(self.clients[0], cmd)
all_mounts_procs.append(proc)
g.log.info("I/O on mountpoint is successful")
# create snapshot
g.log.info("Starting to Create snapshot of clone volume")
ret0, _, _ = snap_create(self.mnode, "clone0", "snap2")
self.assertEqual(ret0, 0, "Failed to create the snapshot"
"snap2 from clone0")
g.log.info("Snapshots snap2 created successfully from clone0")
ret1, _, _ = snap_create(self.mnode, "clone1", "snap3")
self.assertEqual(ret1, 0, "Failed to create the snapshot snap3"
"from clone1")
g.log.info("Snapshots snap3 created successfully from clone1")
# Listing all Snapshots present
g.log.info("Starting to list all snapshots")
ret, _, _ = snap_list(self.mnode)
self.assertEqual(ret, 0, ("Failed to list snapshots present"))
g.log.info("Snapshots successfully listed")
def test_snap_self_heal(self):
"""
Steps:
1. create a volume
2. mount volume
3. create snapshot of that volume
4. Activate snapshot
5. Clone snapshot and Mount
6. Perform I/O
7. Bring Down Few bricks from volume without
affecting the volume or cluster.
8. Perform I/O
9. Bring back down bricks to online
10. Validate heal is complete with areequal
"""
# pylint: disable=too-many-statements, too-many-locals
# Creating snapshot:
g.log.info("Starting to Create snapshot")
ret, _, _ = snap_create(self.mnode, self.volname, self.snap)
self.assertEqual(
ret, 0, ("Failed to create snapshot for volume %s" % self.volname))
g.log.info("Snapshot %s created successfully for volume %s", self.snap,
self.volname)
# Activating snapshot
g.log.info("Starting to Activate Snapshot")
ret, _, _ = snap_activate(self.mnode, self.snap)
self.assertEqual(ret, 0,
("Failed to Activate snapshot %s" % self.snap))
g.log.info("Snapshot %s activated successfully", self.snap)
# snapshot list
ret, _, _ = snap_list(self.mnode)
self.assertEqual(ret, 0, ("Failed to list all the snapshot"))
g.log.info("Snapshot list command was successful")
# Creating a Clone volume from snapshot:
g.log.info("Starting to Clone volume from Snapshot")
ret, _, _ = snap_clone(self.mnode, self.snap, self.clone)
self.assertEqual(ret, 0, ("Failed to clone %s from snapshot %s" %
(self.clone, self.snap)))
g.log.info("%s created successfully", self.clone)
# start clone volumes
g.log.info("start to created clone volumes")
ret, _, _ = volume_start(self.mnode, self.clone)
self.assertEqual(ret, 0, "Failed to start clone %s" % self.clone)
g.log.info("clone volume %s started successfully", self.clone)
# Mounting a clone volume
g.log.info("Mounting a clone volume")
ret, _, _ = mount_volume(self.clone, self.mount_type, self.mount1,
self.mnode, self.clients[0])
self.assertEqual(ret, 0,
"Failed to mount clone Volume %s" % self.clone)
g.log.info("Clone volume %s mounted Successfully", self.clone)
# Checking cloned volume mounted or not
ret = is_mounted(self.clone, self.mount1, self.mnode, self.clients[0],
self.mount_type)
self.assertTrue(
ret,
"Failed to mount clone volume on mount point: %s" % self.mount1)
g.log.info("clone Volume %s mounted on %s", self.clone, self.mount1)
# write files on all mounts
g.log.info("Starting IO on all mounts...")
g.log.info("mounts: %s", self.mount1)
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name file %s" %
(self.script_upload_path, self.mount1))
proc = g.run(self.clients[0], cmd)
all_mounts_procs.append(proc)
g.log.info("Successful in creating I/O on mounts")
# get the bricks from the volume
g.log.info("Fetching bricks for the volume : %s", self.clone)
bricks_list = get_all_bricks(self.mnode, self.clone)
g.log.info("Brick List : %s", bricks_list)
# Select bricks to bring offline
g.log.info("Starting to bring bricks to offline")
bricks_to_bring_offline_dict = (select_bricks_to_bring_offline(
self.mnode, self.volname))
bricks_to_bring_offline = filter(
None, (bricks_to_bring_offline_dict['hot_tier_bricks'] +
bricks_to_bring_offline_dict['cold_tier_bricks'] +
bricks_to_bring_offline_dict['volume_bricks']))
g.log.info("Brick to bring offline: %s ", bricks_to_bring_offline)
ret = bring_bricks_offline(self.clone, bricks_to_bring_offline)
self.assertTrue(ret, "Failed to bring the bricks offline")
g.log.info("Successful in bringing bricks: %s offline",
bricks_to_bring_offline)
# Offline Bricks list
offline_bricks = get_offline_bricks_list(self.mnode, self.clone)
self.assertIsNotNone(
offline_bricks, "Failed to get offline bricklist"
"for volume %s" % self.clone)
for bricks in offline_bricks:
self.assertIn(bricks, bricks_to_bring_offline,
"Failed to validate "
"Bricks offline")
g.log.info("Bricks Offline: %s", offline_bricks)
# Online Bricks list
online_bricks = get_online_bricks_list(self.mnode, self.clone)
self.assertIsNotNone(
online_bricks, "Failed to get online bricks"
" for volume %s" % self.clone)
g.log.info("Bricks Online: %s", online_bricks)
# write files mountpoint
g.log.info("Starting IO on all mounts...")
g.log.info("mounts: %s", self.mount1)
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name file %s" %
(self.script_upload_path, self.mount1))
proc = g.run(self.clients[0], cmd)
all_mounts_procs.append(proc)
g.log.info("Successful in creating I/O on mounts")
# Bring all bricks online
g.log.info("bring all bricks online")
ret = bring_bricks_online(self.mnode, self.clone,
bricks_to_bring_offline)
self.assertTrue(ret, "Failed to bring bricks online")
g.log.info("Successful in bringing all bricks online")
# Validate Bricks are online
g.log.info("Validating all bricks are online")
ret = are_bricks_online(self.mnode, self.clone, bricks_list)
self.assertTrue(ret, "Failed to bring all the bricks online")
g.log.info("bricks online: %s", bricks_list)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.clone)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online" % self.clone))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.clone)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.clone)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.clone))
g.log.info("Volume %s : All process are online", self.clone)
# wait for the heal process to complete
g.log.info("waiting for heal process to complete")
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, "Failed to complete the heal process")
g.log.info("Successfully completed heal process")
# Check areequal
# get the subvolumes
g.log.info("Starting to get sub-volumes for volume %s", self.clone)
subvols = get_subvols(self.mnode, self.clone)
num_subvols = len(subvols['volume_subvols'])
g.log.info("Number of subvolumes in volume %s:", num_subvols)
# Get arequals and compare
g.log.info("Starting to Compare areequals")
for i in range(0, num_subvols):
# Get arequal for first brick
subvol_brick_list = subvols['volume_subvols'][i]
node, brick_path = subvol_brick_list[0].split(':')
command = ('arequal-checksum -p %s '
'-i .glusterfs -i .landfill -i .trashcan' % brick_path)
ret, arequal, _ = g.run(node, command)
first_brick_total = arequal.splitlines()[-1].split(':')[-1]
# Get arequal for every brick and compare with first brick
for brick in subvol_brick_list:
node, brick_path = brick.split(':')
command = ('arequal-checksum -p %s '
'-i .glusterfs -i .landfill -i .trashcan' % brick_path)
ret, brick_arequal, _ = g.run(node, command)
self.assertFalse(ret, 'Failed to get arequal on brick %s' % brick)
g.log.info('Getting arequal for %s is successful', brick)
brick_total = brick_arequal.splitlines()[-1].split(':')[-1]
self.assertEqual(
first_brick_total, brick_total,
'Arequals for subvol and %s are not equal' % brick)
g.log.info('Arequals for subvol and %s are equal', brick)
g.log.info('All arequals are equal for distributed-replicated')
def test_afr_dir_entry_creation_with_subvol_down(self):
"""
1. Create a distributed-replicated(3X3)/distributed-arbiter(3X(2+1))
and mount it on one client
2. Kill 3 bricks corresponding to the 1st subvol
3. Unmount and remount the volume on the same client
4. Create deep dir from mount point
mkdir -p dir1/subdir1/deepdir1
5. Create files under dir1/subdir1/deepdir1; touch <filename>
6. Now bring all sub-vols up by volume start force
7. Validate backend bricks for dir creation, the subvol which is
offline will have no dirs created, whereas other subvols will have
dirs created from step 4
8. Trigger heal from client by "#find . | xargs stat"
9. Verify that the directory entries are created on all back-end bricks
10. Create new dir (dir2) on location dir1/subdir1/deepdir1
11. Trigger rebalance and wait for the completion
12. Check backend bricks for all entries of dirs
13. Check if files are getting created on the subvol which was offline
"""
# Bring down first subvol of bricks offline
self.subvols = get_subvols(self.mnode, self.volname)['volume_subvols']
first_subvol = self.subvols[0]
ret = bring_bricks_offline(self.volname, first_subvol)
self.assertTrue(
ret, "Unable to bring {} bricks offline".format(first_subvol))
# Check bricks are offline or not
ret = are_bricks_offline(self.mnode, self.volname, first_subvol)
self.assertTrue(ret, "Bricks {} are still online".format(first_subvol))
# Unmount and remount the volume
ret, _, _ = umount_volume(self.mounts[0].client_system,
self.mounts[0].mountpoint)
self.assertFalse(ret, "Failed to unmount volume.")
ret, _, _ = mount_volume(self.volname, self.mount_type,
self.mounts[0].mountpoint, self.mnode,
self.mounts[0].client_system)
self.assertFalse(ret, "Failed to remount volume.")
g.log.info('Successfully umounted and remounted volume.')
# At this step, sleep is must otherwise file creation will fail
sleep(2)
# Create dir `dir1/subdir1/deepdir1` on mountpont
directory1 = "dir1/subdir1/deepdir1"
path = self.mounts[0].mountpoint + "/" + directory1
ret = mkdir(self.mounts[0].client_system, path, parents=True)
self.assertTrue(ret, "Directory {} creation failed".format(path))
# Create files on the 2nd and 3rd subvols which are online
brickobject = create_brickobjectlist(self.subvols, directory1)
self.assertIsNotNone(brickobject, "Failed to get brick object list")
self._create_number_of_files_on_the_subvol(brickobject[1],
directory1,
5,
mountpath=path)
self._create_number_of_files_on_the_subvol(brickobject[2],
directory1,
5,
mountpath=path)
# Bring bricks online using volume start force
ret, _, err = volume_start(self.mnode, self.volname, force=True)
self.assertEqual(ret, 0, err)
g.log.info("Volume: %s started successfully", self.volname)
# Check all bricks are online
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, "Few process after volume start are offline for "
"volume: {}".format(self.volname))
# Validate Directory is not created on the bricks of the subvol which
# is offline
for subvol in self.subvols:
self._check_file_exists(subvol,
"/" + directory1,
exists=(subvol != first_subvol))
# Trigger heal from the client
cmd = "cd {}; find . | xargs stat".format(self.mounts[0].mountpoint)
ret, _, err = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, err)
# Validate the directory1 is present on all the bricks
for subvol in self.subvols:
self._check_file_exists(subvol, "/" + directory1, exists=True)
# Create new dir (dir2) on location dir1/subdir1/deepdir1
directory2 = "/" + directory1 + '/dir2'
path = self.mounts[0].mountpoint + directory2
ret = mkdir(self.mounts[0].client_system, path, parents=True)
self.assertTrue(ret, "Directory {} creation failed".format(path))
# Trigger rebalance and validate the completion
ret, _, err = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, err)
g.log.info("Rebalance on volume %s started successfully", self.volname)
ret = wait_for_rebalance_to_complete(self.mnode, self.volname)
self.assertTrue(
ret, "Rebalance didn't complete on the volume: "
"{}".format(self.volname))
# Validate all dirs are present on all bricks in each subvols
for subvol in self.subvols:
for each_dir in ("/" + directory1, directory2):
self._check_file_exists(subvol, each_dir, exists=True)
# Validate if files are getting created on the subvol which was
# offline
self._create_number_of_files_on_the_subvol(brickobject[0],
directory1,
5,
mountpath=path)
def test_subdir_mounts(self):
"""
Mount the volume on client
Create 2 subdir on client subdir1,subdir2
Auth allow - Client1(subdir1),Client2(subdir2)
Mount subdir1 on client1.Try Mounting subdir1 on client2
Mount subdir2 on client2.Try Mounting subdir2 on client1
"""
# Create 2 subdirectories
cmd = ("mkdir %s/dir1 && mkdir %s/dir2" %
(self.mounts[0].mountpoint, self.mounts[0].mountpoint))
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Failed to create directories on mountpoint")
g.log.info("Directories created successfully on mountpoint")
# unmount volume
ret = self.unmount_volume(self.mounts)
self.assertTrue(ret, "Volumes UnMount failed")
g.log.info("Volumes UnMounted successfully")
# Set auth allow permission on subdirs
cmd = ("gluster volume set %s auth.allow '/dir1(%s),/dir2(%s)'" %
(self.volname, self.clients[0], self.clients[1]))
g.run(self.mnode, cmd)
# Sometimes the mount command is returning exit code as 0 in case of
# mount failures as well
# Hence not asserting while running mount command in test case.
# Instead asserting on basis on performing grep on mount point
# BZ 1590711
self.mpoint = "/mnt/Mount_Point1"
# Test Subdir2 mount on client 1
_, _, _ = mount_volume("%s/dir2" % self.volname, self.mount_type,
self.mpoint, self.mnode, self.clients[0])
cmd = ("mount | grep %s") % self.mpoint
ret, _, _ = g.run(self.clients[0], cmd)
if ret == 0:
raise ExecutionError("%s/dir2 mount should fail,"
"But dir2 mounted successfully on"
"unauthorized client" % self.volname)
g.log.info("%s/dir2 is not mounted on unauthorized client",
self.volname)
# Test Subdir1 mount on client 1
_, _, _ = mount_volume("%s/dir1" % self.volname, self.mount_type,
self.mpoint, self.mnode, self.clients[0])
cmd = ("mount | grep %s") % self.mpoint
ret, _, _ = g.run(self.clients[0], cmd)
self.assertEqual(ret, 0, ("%s/dir1 mount failed" % self.volname))
g.log.info("%s/dir1 is mounted Successfully", self.volname)
# Test Subdir1 mount on client 2
_, _, _ = mount_volume("%s/dir1" % self.volname, self.mount_type,
self.mpoint, self.mnode, self.clients[1])
cmd = ("mount | grep %s") % self.mpoint
ret, _, _ = g.run(self.clients[1], cmd)
if ret == 0:
raise ExecutionError("%s/dir1 mount should fail,"
"But dir1 mounted successfully on"
"unauthorized client" % self.volname)
g.log.info("%s/dir1 is not mounted on unauthorized client",
self.volname)
# Test Subdir2 mount on client 2
_, _, _ = mount_volume("%s/dir2" % self.volname, self.mount_type,
self.mpoint, self.mnode, self.clients[1])
cmd = ("mount | grep %s") % self.mpoint
ret, _, _ = g.run(self.clients[1], cmd)
self.assertEqual(ret, 0, ("%s/dir2 mount failed" % self.volname))
g.log.info("%s/dir2 is mounted Successfully", self.volname)
def test_remove_brick(self):
"""
In this test case:
1. Trusted storage Pool of 4 nodes
2. Create a distributed-replicated volumes with 4 bricks
3. Start the volume
4. Fuse mount the gluster volume on out of trusted nodes
5. Create some data file
6. Start remove-brick operation for one replica pair
7. Restart glusterd on all nodes
8. Try to commit the remove-brick operation while rebalance
is in progress, it should fail
"""
# pylint: disable=too-many-statements
my_servers = self.servers[0:4]
my_server_info = {}
for server in self.servers[0:4]:
my_server_info[server] = self.all_servers_info[server]
for index in range(1, 4):
ret, _, _ = peer_probe(self.servers[0], self.servers[index])
self.assertEqual(ret, 0, ("peer probe from %s to %s is failed",
self.servers[0], self.servers[index]))
g.log.info("peer probe is success from %s to "
"%s", self.servers[0], self.servers[index])
# Validating whether the peer are connected or not
# In jenkins This case is failing saying peers are not in connected
# state, that is reason adding a check whether peers are connected
# or not
count = 0
while count < 30:
ret = is_peer_connected(self.mnode, my_servers)
if ret:
g.log.info("Peers are in connected state")
break
sleep(3)
count = count + 1
self.assertTrue(ret, "Some peers are not in connected state")
self.volname = "testvol"
bricks_list = form_bricks_list(self.mnode, self.volname, 4, my_servers,
my_server_info)
g.log.info("Creating a volume %s ", self.volname)
kwargs = {}
kwargs['replica_count'] = 2
ret = volume_create(self.mnode,
self.volname,
bricks_list,
force=False,
**kwargs)
self.assertEqual(ret[0], 0, ("Unable"
"to create volume %s" % self.volname))
g.log.info("Volume created successfully %s", self.volname)
ret, _, _ = volume_start(self.mnode, self.volname, False)
self.assertEqual(ret, 0, ("Failed to start the "
"volume %s", self.volname))
g.log.info("Get all the bricks of the volume")
bricks_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(bricks_list, "Failed to get the brick list")
g.log.info("Successfully got the list of bricks of volume")
# Mounting a volume
ret, _, _ = mount_volume(self.volname,
mtype=self.mount_type,
mpoint=self.mounts[0].mountpoint,
mserver=self.mnode,
mclient=self.mounts[0].client_system)
self.assertEqual(ret, 0, ("Volume %s is not mounted") % self.volname)
g.log.info("Volume mounted successfully : %s", self.volname)
self.all_mounts_procs = []
# Creating files
command = ("cd %s/ ; "
"for i in `seq 1 10` ; "
"do mkdir l1_dir.$i ; "
"for j in `seq 1 5` ; "
"do mkdir l1_dir.$i/l2_dir.$j ; "
"for k in `seq 1 10` ; "
"do dd if=/dev/urandom of=l1_dir.$i/l2_dir.$j/test.$k "
"bs=128k count=$k ; "
"done ; "
"done ; "
"done ; " % (self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
command,
user=self.mounts[0].user)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
ret = validate_io_procs(self.all_mounts_procs, self.mounts)
self.io_validation_complete = True
self.assertTrue(ret, "IO failed on some of the clients")
remove_brick_list = bricks_list[2:4]
ret, _, _ = remove_brick(self.mnode, self.volname, remove_brick_list,
'start')
self.assertEqual(ret, 0, "Failed to start remove brick operation")
g.log.info("Remove bricks operation started successfully")
g.log.info("Restart glusterd on servers %s", self.servers)
ret = restart_glusterd(self.servers)
self.assertTrue(
ret, ("Failed to restart glusterd on servers %s", self.servers))
g.log.info("Successfully restarted glusterd on servers %s",
self.servers)
ret, _, _ = remove_brick(self.mnode, self.volname, remove_brick_list,
'commit')
self.assertNotEqual(ret, 0, "Remove brick commit ops should be fail")
g.log.info("Remove bricks commit operation failure is expected")
def test_subdir_with_removebrick(self):
# pylint: disable=too-many-statements
"""
Mount the volume
Create 2 subdir on client subdir1 and subdir2
Auth allow - Client1(subdir1,subdir2),Client2(subdir1,subdir2)
Mount the subdir to their respective clients
Start IO's on both subdirs
Perform remove-brick
Validate on client if subdir's are mounted post remove-brick
operation is performed
"""
# Create directories subdir1 and subdir2 on mount point
ret = mkdir(self.mounts[0].client_system,
"%s/subdir1" % self.mounts[0].mountpoint)
self.assertTrue(
ret, ("Failed to create directory 'subdir1' in"
"volume %s from client %s" %
(self.mounts[0].volname, self.mounts[0].client_system)))
ret = mkdir(self.mounts[0].client_system,
"%s/subdir2" % self.mounts[0].mountpoint)
self.assertTrue(
ret, ("Failed to create directory 'subdir2' in"
"volume %s from client %s" %
(self.mounts[0].volname, self.mounts[0].client_system)))
# unmount volume
ret = self.unmount_volume(self.mounts)
self.assertTrue(ret, "Volumes UnMount failed")
g.log.info("Volumes UnMounted successfully")
# Set authentication on the subdirectory subdir1
# and subdir2 to access by 2 clients
g.log.info(
'Setting authentication on subdir1 and subdir2'
'for client %s and %s', self.clients[0], self.clients[0])
ret = set_auth_allow(
self.volname, self.mnode, {
'/subdir1': [self.clients[0], self.clients[1]],
'/subdir2': [self.clients[0], self.clients[1]]
})
self.assertTrue(
ret, 'Failed to set Authentication on volume %s' % self.volume)
self.mpoint = "/mnt/Mount_Point1"
# Mount Subdir1 mount on client 1
_, _, _ = mount_volume("%s/subdir1" % self.volname, self.mount_type,
self.mpoint, self.mnode, self.clients[0])
# Checking subdir1 is mounted or not
ret = is_mounted("%s/subdir1" % self.volname, self.mpoint, self.mnode,
self.clients[0], self.mount_type)
self.assertTrue(ret,
"Volume not mounted on mount point: %s" % self.mpoint)
g.log.info("Volume %s mounted on %s/subdir1", self.volname,
self.mpoint)
# Mount Subdir2 mount on client 2
_, _, _ = mount_volume("%s/subdir2" % self.volname, self.mount_type,
self.mpoint, self.mnode, self.clients[1])
# Checking subdir2 is mounted or not
ret = is_mounted("%s/subdir2" % self.volname, self.mpoint, self.mnode,
self.clients[1], self.mount_type)
self.assertTrue(ret,
"Volume not mounted on mount point: %s" % self.mpoint)
g.log.info("Volume %s mounted on %s/subdir2", self.volname,
self.mpoint)
# Start IO on all the subdir mounts.
self.subdir_mounts = [
copy.deepcopy(self.mounts[0]),
copy.deepcopy(self.mounts[1])
]
self.subdir_mounts[0].volname = "%s/subdir1" % self.volname
self.subdir_mounts[1].volname = "%s/subdir2" % self.volname
all_mounts_procs = []
count = 1
for mount_obj in self.subdir_mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
self.mpoint)
cmd = ("/usr/bin/env python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 10 "
"--max-num-of-dirs 5 "
"--num-of-files 5 %s" %
(self.script_upload_path, count, self.mpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
count = count + 10
# Validate IO
g.log.info("Validating IO's")
ret = validate_io_procs(all_mounts_procs, self.subdir_mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Successfully validated all io's")
# Get stat of all the files/dirs created.
g.log.info("Get stat of all the files/dirs created.")
ret = get_mounts_stat(self.subdir_mounts)
self.assertTrue(ret, "Stat failed on some of the clients")
g.log.info("Successfully got stat of all files/dirs created")
# Perform remove brick operation when subdir is mounted on client
g.log.info("Start removing bricks from volume")
ret = shrink_volume(self.mnode, self.volname, rebalance_timeout=600)
self.assertTrue(ret, ("Remove brick operation failed on "
"%s", self.volname))
g.log.info("Remove brick operation is successful on "
"volume %s", self.volname)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("All volume %s processes failed to come up "
"online", self.volname))
g.log.info("All volume %s processes came up "
"online successfully", self.volname)
# Log Volume Info and Status after performing remove brick
g.log.info("Logging volume info and Status after shrinking volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed on "
"volume %s", self.volname))
g.log.info("Successful in logging volume info and status of volume %s",
self.volname)
# Again Checking subdir1 is mounted or not on Client 1
ret = is_mounted("%s/subdir1" % self.volname, self.mpoint, self.mnode,
self.clients[0], self.mount_type)
self.assertTrue(ret,
"Volume not mounted on mount point: %s" % self.mpoint)
g.log.info("Volume %s mounted on %s/subdir1", self.volname,
self.mpoint)
# Again Checking subdir2 is mounted or not on Client 2
ret = is_mounted("%s/subdir2" % self.volname, self.mpoint, self.mnode,
self.clients[1], self.mount_type)
self.assertTrue(ret,
"Volume not mounted on mount point: %s" % self.mpoint)
g.log.info("Volume %s mounted on %s/subdir2", self.volname,
self.mpoint)
def test_file_access(self):
"""
Test file access.
"""
# pylint: disable=protected-access
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
mount_obj = self.mounts[0]
mountpoint = mount_obj.mountpoint
# get subvol list
subvols = (get_subvols(self.mnode, self.volname))['volume_subvols']
self.assertIsNotNone(subvols, "failed to get subvols")
# create a file
srcfile = mountpoint + '/testfile'
ret, _, err = g.run(self.clients[0], ("touch %s" % srcfile))
self.assertEqual(ret, 0, ("File creation failed for %s err %s",
srcfile, err))
g.log.info("testfile creation successful")
# find hashed subvol
srchashed, scount = find_hashed_subvol(subvols, "/", "testfile")
self.assertIsNotNone(srchashed, "could not find srchashed")
g.log.info("hashed subvol for srcfile %s subvol count %s",
srchashed._host, str(scount))
# rename the file such that the new name hashes to a new subvol
tmp = find_new_hashed(subvols, "/", "testfile")
self.assertIsNotNone(tmp, "could not find new hashed for dstfile")
g.log.info("dst file name : %s dst hashed_subvol : %s "
"subvol count : %s", tmp.newname,
tmp.hashedbrickobject._host, str(tmp.subvol_count))
dstname = str(tmp.newname)
dstfile = mountpoint + "/" + dstname
dsthashed = tmp.hashedbrickobject
dcount = tmp.subvol_count
ret, _, err = g.run(self.clients[0], ("mv %s %s" %
(srcfile, dstfile)))
self.assertEqual(ret, 0, ("rename failed for %s err %s",
srcfile, err))
g.log.info("cmd: mv srcfile dstfile successful")
# check that on dsthash_subvol the file is a linkto file
filepath = dsthashed._fqpath + "/" + dstname
file_stat = get_file_stat(dsthashed._host, filepath)
self.assertEqual(file_stat['access'], "1000", ("Expected file "
"permission to be 1000"
" on subvol %s",
dsthashed._host))
g.log.info("dsthash_subvol has the expected linkto file")
# check on srchashed the file is a data file
filepath = srchashed._fqpath + "/" + dstname
file_stat = get_file_stat(srchashed._host, filepath)
self.assertNotEqual(file_stat['access'], "1000", ("Expected file "
"permission not to"
"be 1000 on subvol"
"%s",
srchashed._host))
# Bring down the hashed subvol of dstfile(linkto file)
ret = bring_bricks_offline(self.volname, subvols[dcount])
self.assertTrue(ret, ('Error in bringing down subvolume %s',
subvols[dcount]))
g.log.info('dst subvol %s is offline', subvols[dcount])
# Need to access the file through a fresh lookup through a new mount
# create a new dir(choosing server to do a mount)
ret, _, _ = g.run(self.mnode, ("mkdir -p /mnt"))
self.assertEqual(ret, 0, ('mkdir of mount dir failed'))
g.log.info("mkdir of mount dir succeeded")
# do a temp mount
ret = mount_volume(self.volname, self.mount_type, "/mnt",
self.mnode, self.mnode)
self.assertTrue(ret, ('temporary mount failed'))
g.log.info("temporary mount succeeded")
# check that file is accessible (stat)
ret, _, _ = g.run(self.mnode, ("stat /mnt/%s" % dstname))
self.assertEqual(ret, 0, ('stat error on for dst file %s', dstname))
g.log.info("stat on /mnt/%s successful", dstname)
# cleanup temporary mount
ret = umount_volume(self.mnode, "/mnt")
self.assertTrue(ret, ('temporary mount failed'))
g.log.info("umount successful")
# Bring up the hashed subvol
ret = bring_bricks_online(self.mnode, self.volname, subvols[dcount],
bring_bricks_online_methods=None)
self.assertTrue(ret, "Error in bringing back subvol online")
g.log.info('Subvol is back online')
# now bring down the cached subvol
ret = bring_bricks_offline(self.volname, subvols[scount])
self.assertTrue(ret, ('Error in bringing down subvolume %s',
subvols[scount]))
g.log.info('target subvol %s is offline', subvols[scount])
# file access should fail
ret, _, _ = g.run(self.clients[0], ("stat %s" % dstfile))
self.assertEqual(ret, 1, ('stat error on for file %s', dstfile))
g.log.info("dstfile access failed as expected")
def setUpClass(cls):
# Calling GlusterBaseClass setUpClass
GlusterBaseClass.setUpClass.im_func(cls)
# Upload io scripts for running IO on mounts
g.log.info("Upload io scripts to clients %s for running IO on mounts",
cls.clients)
script_local_path = ("/usr/share/glustolibs/io/scripts/"
"file_dir_ops.py")
cls.script_upload_path = ("/usr/share/glustolibs/io/scripts/"
"file_dir_ops.py")
ret = upload_scripts(cls.clients, [script_local_path])
if not ret:
raise ExecutionError("Failed to upload IO scripts to clients %s" %
cls.clients)
g.log.info("Successfully uploaded IO scripts to clients %s",
cls.clients)
cls.counter = 1
# int: Value of counter is used for dirname-start-num argument for
# file_dir_ops.py create_deep_dirs_with_files.
# The --dir-length argument value for file_dir_ops.py
# create_deep_dirs_with_files is set to 10 (refer to the cmd in setUp
# method). This means every mount will create
# 10 top level dirs. For every mountpoint/testcase to create new set of
# dirs, we are incrementing the counter by --dir-length value i.e 10 in
# this test suite.
# If we are changing the --dir-length to new value, ensure the counter
# is also incremented by same value to create new set of files/dirs.
# Setup Volumes
if cls.volume_type == "distributed-replicated":
cls.volume_configs = []
# Define two 2x2 distributed-replicated volumes
for i in range(1, 3):
cls.volume['voltype'] = {
'type': 'distributed-replicated',
'replica_count': 2,
'dist_count': 2,
'transport': 'tcp',
}
cls.volume_configs.append({
'name':
'testvol_%s_%d' % (cls.volume['voltype']['type'], i),
'servers':
cls.servers,
'voltype':
cls.volume['voltype']
})
# Define two 2x3 distributed-replicated volumes
for i in range(1, 3):
cls.volume['voltype'] = {
'type': 'distributed-replicated',
'replica_count': 3,
'dist_count': 2,
'transport': 'tcp',
}
cls.volume_configs.append({
'name':
'testvol_%s_%d' % (cls.volume['voltype']['type'], i + 2),
'servers':
cls.servers,
'voltype':
cls.volume['voltype']
})
# Define distributed volume
cls.volume['voltype'] = {
'type': 'distributed',
'dist_count': 3,
'transport': 'tcp',
}
cls.volume_configs.append({
'name':
'testvol_%s' % cls.volume['voltype']['type'],
'servers':
cls.servers,
'voltype':
cls.volume['voltype']
})
# Create and mount volumes
cls.mount_points = []
cls.mount_points_and_volnames = {}
cls.client = cls.clients[0]
for volume_config in cls.volume_configs:
# Setup volume
ret = setup_volume(mnode=cls.mnode,
all_servers_info=cls.all_servers_info,
volume_config=volume_config,
force=False)
if not ret:
raise ExecutionError("Failed to setup Volume"
" %s" % volume_config['name'])
g.log.info("Successful in setting volume %s",
volume_config['name'])
# Mount volume
mount_point = tempfile.mkdtemp()
cls.mount_points.append(mount_point)
cls.mount_points_and_volnames[volume_config['name']] = \
mount_point
ret, _, _ = mount_volume(volume_config['name'], cls.mount_type,
mount_point, cls.mnode, cls.client)
if ret:
raise ExecutionError(
"Failed to do gluster mount on volume %s " %
cls.volname)
g.log.info("Successfully mounted %s on client %s", cls.volname,
cls.client)
def setUpClass(cls):
# Calling GlusterBaseClass setUpClass
GlusterBaseClass.setUpClass.im_func(cls)
# Upload io scripts for running IO on mounts
g.log.info("Upload io scripts to clients %s for running IO on mounts",
cls.clients)
script_local_path = ("/usr/share/glustolibs/io/scripts/"
"file_dir_ops.py")
cls.script_upload_path = ("/usr/share/glustolibs/io/scripts/"
"file_dir_ops.py")
ret = upload_scripts(cls.clients, [script_local_path])
if not ret:
raise ExecutionError("Failed to upload IO scripts to clients %s" %
cls.clients)
g.log.info("Successfully uploaded IO scripts to clients %s",
cls.clients)
# Setup Volumes
if cls.volume_type == "replicated":
cls.volume_configs = []
cls.mounts_dict_list = []
# Define two replicated volumes
for i in range(1, 3):
cls.volume['voltype'] = {
'type': 'replicated',
'replica_count': 3,
'arbiter_count': 1,
'transport': 'tcp'
}
volume_config = {
'name':
'testvol_%s_%d' % (cls.volume['voltype']['type'], i),
'servers': cls.servers,
'voltype': cls.volume['voltype']
}
cls.volume_configs.append(volume_config)
# redefine mounts
for client in cls.all_clients_info.keys():
mount = {
'protocol':
cls.mount_type,
'server':
cls.mnode,
'volname':
volume_config['name'],
'client':
cls.all_clients_info[client],
'mountpoint': (os.path.join(
"/mnt",
'_'.join([volume_config['name'],
cls.mount_type]))),
'options':
''
}
cls.mounts_dict_list.append(mount)
cls.mounts = create_mount_objs(cls.mounts_dict_list)
# Create and mount volumes
cls.mount_points = []
cls.client = cls.clients[0]
for volume_config in cls.volume_configs:
# Setup volume
ret = setup_volume(mnode=cls.mnode,
all_servers_info=cls.all_servers_info,
volume_config=volume_config,
force=False)
if not ret:
raise ExecutionError("Failed to setup Volume %s" %
volume_config['name'])
g.log.info("Successful in setting volume %s",
volume_config['name'])
# Mount volume
mount_point = (os.path.join(
"/mnt", '_'.join([volume_config['name'], cls.mount_type])))
cls.mount_points.append(mount_point)
ret, _, _ = mount_volume(volume_config['name'], cls.mount_type,
mount_point, cls.mnode, cls.client)
if ret:
raise ExecutionError(
"Failed to do gluster mount on volume %s " %
cls.volname)
g.log.info("Successfully mounted %s on client %s", cls.volname,
cls.client)
def test_change_reserve_limit_to_lower_value(self):
# pylint: disable=too-many-statements
"""
Test Case:
1) Create a distributed-replicated volume and start it.
2) Enable storage.reserve option on the volume using below command:
gluster volume set <volname> storage.reserve <value>
3) Mount the volume on a client.
4) Write some data on the mount points.
5) Start remove-brick operation.
6) While remove-brick is in-progress change the reserve limit to a
lower value.
"""
# Create and start a volume
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, "Failed to create and start volume")
# Setting storage.reserve to 50
ret = set_volume_options(self.mnode, self.volname,
{'storage.reserve': '50'})
self.assertTrue(ret,
"Failed to set storage reserve on %s" % self.mnode)
g.log.info("Able to set storage reserve successfully on %s",
self.mnode)
# Mounting the volume.
ret, _, _ = mount_volume(self.volname,
mtype=self.mount_type,
mpoint=self.mounts[0].mountpoint,
mserver=self.mnode,
mclient=self.mounts[0].client_system)
self.assertEqual(ret, 0, ("Volume %s is not mounted") % self.volname)
g.log.info("Volume mounted successfully : %s", self.volname)
# Run IOs
g.log.info("Starting IO on all mounts...")
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = (
"/usr/bin/env python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 5 "
"--max-num-of-dirs 3 "
"--num-of-files 10 %s" %
(self.script_upload_path, self.counter, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.counter = self.counter + 10
# Validate IO
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# Getting a list of all the bricks.
g.log.info("Get all the bricks of the volume")
self.brick_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(self.brick_list, "Failed to get the brick list")
g.log.info("Successfully got the list of bricks of volume")
# Removing bricks from volume.
remove_brick_list = self.brick_list[3:6]
ret, _, _ = remove_brick(self.mnode, self.volname, remove_brick_list,
'start')
self.assertEqual(ret, 0, "Failed to start remove brick operation.")
g.log.info("Remove bricks operation started successfully.")
# Setting storage.reserve to 33
ret = set_volume_options(self.mnode, self.volname,
{'storage.reserve': '33'})
self.assertTrue(ret,
"Failed to set storage reserve on %s" % self.mnode)
g.log.info("Able to set storage reserve successfully on %s",
self.mnode)
# Stopping brick remove opeation.
ret, _, _ = remove_brick(self.mnode, self.volname, remove_brick_list,
'stop')
self.assertEqual(ret, 0, "Failed to stop remove brick operation")
g.log.info("Remove bricks operation stop successfully")
