def setUp(self):
"""
Setup and mount volume or raise ExecutionError
"""
# Calling GlusterBaseClass setUp
self.get_super_method(self, 'setUp')()
self.all_mounts_procs, self.io_validation_complete = [], False
# Setup Volume
ret = self.setup_volume_and_mount_volume([self.mounts[0]])
if not ret:
raise ExecutionError("Failed to Setup and Mount Volume")
# Upload io scripts for running IO on mounts
self.script_upload_path = ("/usr/share/glustolibs/io/scripts/"
"file_dir_ops.py")
ret = upload_scripts(self.clients[0], self.script_upload_path)
if not ret:
raise ExecutionError("Failed to upload IO scripts to clients")
# Form brick list for expanding volume
self.add_brick_list = form_bricks_list_to_add_brick(
self.mnode,
self.volname,
self.servers,
self.all_servers_info,
distribute_count=3)
if not self.add_brick_list:
raise ExecutionError("Volume %s: Failed to form bricks list for"
" expand" % self.volname)
g.log.info("Volume %s: Formed bricks list for expand", self.volname)
def setUp(self):
# Calling GlusterBaseClass setUp
GlusterBaseClass.setUp.im_func(self)
# Setup Volume and Mount Volume
g.log.info("Starting to Setup Volume and Mount Volume")
ret = self.setup_volume_and_mount_volume(mounts=self.mounts)
if not ret:
raise ExecutionError("Failed to Setup_Volume and Mount_Volume")
g.log.info("Successful in Setup Volume and Mount Volume")
# Form brick list for expanding volume
self.add_brick_list = form_bricks_list_to_add_brick(
self.mnode,
self.volname,
self.servers,
self.all_servers_info,
distribute_count=1)
if not self.add_brick_list:
g.log.error("Volume %s: Failed to form bricks list for expand",
self.volname)
raise ExecutionError("Volume %s: Failed to form bricks list for"
"expand" % self.volname)
g.log.info("Volume %s: Formed bricks list for expand", self.volname)
# Start IO on mounts
g.log.info("Starting IO on all mounts...")
all_mounts_procs = []
for index, mount_obj in enumerate(self.mounts, start=1):
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 2 "
"--max-num-of-dirs 2 "
"--num-of-files 10 %s" %
(self.script_upload_path, index + 10, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Wait for IO to complete
g.log.info("Wait for IO to complete as IO validation did not "
"succeed in test method")
ret = wait_for_io_to_complete(all_mounts_procs, self.mounts)
if not ret:
raise ExecutionError("IO failed on some of the clients")
g.log.info("IO is successful on all mounts")
# List all files and dirs created
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
if not ret:
raise ExecutionError("Failed to list all files and dirs")
g.log.info("Listing all files and directories is successful")
def test_add_brick_without_volname(self):
"""Test add-brick command without volume"""
# Form bricks list for add-brick
bricks_list = form_bricks_list_to_add_brick(self.mnode, self.volname,
self.servers,
self.all_servers_info)
cmd = ("gluster volume add-brick %s " % (' '.join(bricks_list)))
g.log.info("Adding bricks without specifying volume name")
ret, _, _ = g.run(self.mnode, cmd)
self.assertTrue(ret, "add-brick is successful")
g.log.info("Volume %s: add-brick failed", self.volname)
def test_add_brick_non_existent_volume(self):
"""Test add-bricks to an non existent volume"""
# Form bricks list for add-brick
bricks_list = form_bricks_list_to_add_brick(self.mnode, self.volname,
self.servers,
self.all_servers_info)
cmd = ("gluster volume add-brick novolume %s " %
(' '.join(bricks_list)))
g.log.info("Trying to add-bricks to a non-existent volume")
_, _, err = g.run(self.mnode, cmd)
self.assertIn("does not exist", err, "add-brick is successful")
g.log.info("Volume add-brick failed with error %s ", err)
def test_add_brick_replace_brick_fix_layout(self):
"""
Test case:
1. Create a volume, start it and mount it.
2. Create files and dirs on the mount point.
3. Add bricks to the volume.
4. Replace 2 old bricks to the volume.
5. Trigger rebalance fix layout and wait for it to complete.
6. Check layout on all the bricks through trusted.glusterfs.dht.
"""
# Create directories with some files on mount point
cmd = ("cd %s; for i in {1..10}; do mkdir dir$i; for j in {1..5};"
" do dd if=/dev/urandom of=dir$i/file$j bs=1M count=1; done;"
" done" % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.first_client, cmd)
self.assertFalse(ret, "Failed to create dirs and files.")
# Orginal brick list before add brick
brick_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(brick_list, "Empty present brick list")
# Add bricks to the volume
add_brick_list = form_bricks_list_to_add_brick(self.mnode,
self.volname,
self.servers,
self.all_servers_info)
self.assertIsNotNone(add_brick_list, "Empty add brick list")
ret, _, _ = add_brick(self.mnode, self.volname, add_brick_list)
self.assertFalse(ret, "Failed to add bricks to the volume")
g.log.info("Successfully added bricks to the volume")
# Replace 2 old bricks to the volume
for _ in range(0, 2):
brick = choice(brick_list)
self._replace_a_old_added_brick(brick)
brick_list.remove(brick)
# Start rebalance and wait for it to complete
ret, _, _ = rebalance_start(self.mnode, self.volname, fix_layout=True)
self.assertFalse(ret, "Failed to start rebalance on volume")
ret = wait_for_fix_layout_to_complete(self.mnode,
self.volname,
timeout=800)
self.assertTrue(ret, "Rebalance failed on volume")
# Check layout on all the bricks through trusted.glusterfs.dht
self._check_trusted_glusterfs_dht_on_all_bricks()
def setUp(self):
"""
Setup and mount volume or raise ExecutionError
"""
self.get_super_method(self, 'setUp')()
# Setup Volume
ret = self.setup_volume_and_mount_volume([self.mounts[0]])
if not ret:
raise ExecutionError("Failed to Setup and Mount Volume")
# Form brick list for add-brick operation
self.add_brick_list = form_bricks_list_to_add_brick(
self.mnode,
self.volname,
self.servers,
self.all_servers_info,
distribute_count=1)
if not self.add_brick_list:
raise ExecutionError("Volume %s: Failed to form bricks list for"
" add-brick" % self.volname)
g.log.info("Volume %s: Formed bricks list for add-brick operation",
(self.add_brick_list, self.volname))
def test_fix_layout_start(self):
# pylint: disable=too-many-statements
# Get arequal checksum before starting fix-layout
g.log.info("Getting arequal checksum before fix-layout")
arequal_checksum_before_fix_layout = collect_mounts_arequal(
self.mounts)
# Log Volume Info and Status before expanding the volume.
g.log.info("Logging volume info and Status before expanding volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(
ret, "Logging volume info and status failed on "
"volume %s" % self.volname)
g.log.info(
"Successful in logging volume info and status of volume "
"%s", self.volname)
# Form brick list for expanding volume
add_brick_list = form_bricks_list_to_add_brick(self.mnode,
self.volname,
self.servers,
self.all_servers_info,
distribute_count=1)
self.assertIsNotNone(add_brick_list,
("Volume %s: Failed to form "
"bricks list to expand", self.volname))
g.log.info("Volume %s: Formed bricks list to expand", self.volname)
# Expanding volume by adding bricks to the volume
g.log.info("Volume %s: Expand start")
ret, _, _ = add_brick(self.mnode, self.volname, add_brick_list)
self.assertEqual(ret, 0, ("Volume %s: Expand failed", self.volname))
g.log.info("Volume %s: Expand successful", self.volname)
# Wait for gluster processes to come online
g.log.info("Wait for gluster processes to come online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s: one or more volume process are "
"not up", self.volname))
g.log.info("All volume %s processes are online", self.volname)
# Log Volume Info and Status after expanding the volume
g.log.info("Logging volume info and Status after expanding volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(
ret, "Logging volume info and status failed on "
"volume %s" % self.volname)
g.log.info(
"Successful in logging volume info and status of volume "
"%s", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online", self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Start Rebalance fix-layout
g.log.info("Starting fix-layout on the volume")
ret, _, _ = rebalance_start(self.mnode, self.volname, fix_layout=True)
self.assertEqual(ret, 0, ("Volume %s: fix-layout start failed"
"%s", self.volname))
g.log.info("Volume %s: fix-layout start success", self.volname)
# Wait for fix-layout to complete
g.log.info("Waiting for fix-layout to complete")
ret = wait_for_fix_layout_to_complete(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s: Fix-layout is still in-progress", self.volname))
g.log.info("Volume %s: Fix-layout completed successfully",
self.volname)
# Check Rebalance status after fix-layout is complete
g.log.info("Checking Rebalance status")
ret, _, _ = rebalance_status(self.mnode, self.volname)
self.assertEqual(
ret, 0,
("Volume %s: Failed to get rebalance status", self.volname))
g.log.info("Volume %s: Successfully got rebalance status",
self.volname)
# Get arequal checksum after fix-layout is complete
g.log.info("arequal after fix-layout is complete")
arequal_checksum_after_fix_layout = collect_mounts_arequal(self.mounts)
# Compare arequals checksum before and after fix-layout
g.log.info("Comparing checksum before and after fix-layout")
self.assertEqual(arequal_checksum_before_fix_layout,
arequal_checksum_after_fix_layout,
"arequal checksum is NOT MATCHNG")
g.log.info("arequal checksum is SAME")
# Check if there are any file migrations after fix-layout
status_info = get_rebalance_status(self.mnode, self.volname)
for node in range(len(status_info['node'])):
status_info = get_rebalance_status(self.mnode, self.volname)
file_migration_count = status_info['node'][node]['files']
self.assertEqual(
int(file_migration_count), 0,
("Server %s: Few files are migrated", self.servers[node]))
g.log.info("Server %s: No files are migrated")
# Check if new bricks contains any files
for brick in add_brick_list:
brick_node, brick_path = brick.split(":")
cmd = ('find %s -type f ! -perm 1000 | grep -ve .glusterfs' %
brick_path)
_, out, _ = g.run(brick_node, cmd)
self.assertEqual(
len(out), 0,
(("Files(excluded linkto files) are present on %s:%s"),
(brick_node, brick_path)))
g.log.info("No files (excluded linkto files) are present on %s:%s",
brick_node, brick_path)
def test_client_io_threads_on_replicate_volumes(self):
"""
Test case 1:
1. Create distrubuted volume and start it.
2. Check the value of performance.client-io-threads it should be ON.
3. io-threads should be loaded in trusted-.tcp-fuse.vol.
4. Add brick to convert to replicate volume.
5. Check the value of performance.client-io-threads it should be OFF.
6. io-threads shouldn't be loaded in trusted-.tcp-fuse.vol.
7. Remove brick so thate volume type is back to distributed.
8. Check the value of performance.client-io-threads it should be ON.
9. performance.client-io-threads should be loaded in
trusted-.tcp-fuse.vol.
Test case 2:
1. Create a replicate volume and start it.
2. Set performance.client-io-threads to ON.
3. Check the value of performance.client-io-threads it should be ON.
4. io-threads should be loaded in trusted-.tcp-fuse.vol.
5. Add bricks to convert to make the volume 2x3.
6. Check the value of performance.client-io-threads it should be ON.
7. io-threads should be loaded in trusted-.tcp-fuse.vol.
8. Remove brick to make the volume 1x3 again.
9. Check the value of performance.client-io-threads it should be ON.
10. performance.client-io-threads should be loaded in
trusted-.tcp-fuse.vol.
"""
# If volume type is distributed then run test case 1.
if self.volume_type == "distributed":
# Check the value of performance.client-io-threads it should be ON
# and io-threads should be loaded in trusted-.tcp-fuse.vol
self._check_value_of_performance_client_io_threads()
# Add brick to convert to replicate volume
brick = form_bricks_list_to_add_brick(self.mnode, self.volname,
self.servers,
self.all_servers_info)
self.assertIsNotNone(brick,
"Failed to form brick list to add brick")
ret, _, _ = add_brick(self.mnode, self.volname, brick,
force=True, replica_count=2)
self.assertFalse(ret, "Failed to add brick on volume %s"
% self.volname)
g.log.info("Add-brick successful on volume")
# Check the value of performance.client-io-threads it should be ON
# and io-threads should be loaded in trusted-.tcp-fuse.vol
self._check_value_of_performance_client_io_threads(enabled=False)
# Remove brick so thate volume type is back to distributed
ret = shrink_volume(self.mnode, self.volname, replica_num=1)
self.assertTrue(ret, "Failed to remove-brick from volume")
g.log.info("Remove-brick successful on volume")
# Check the value of performance.client-io-threads it should be ON
# and io-threads should be loaded in trusted-.tcp-fuse.vol
self._check_value_of_performance_client_io_threads()
# If volume type is replicated then run test case 2.
else:
# Set performance.client-io-threads to ON
options = {"performance.client-io-threads": "on"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, "Unable to set volume option %s for"
"volume %s" % (options, self.volname))
g.log.info("Successfully set %s for volume %s",
options, self.volname)
# Check the value of performance.client-io-threads it should be ON
# and io-threads should be loaded in trusted-.tcp-fuse.vol
self._check_value_of_performance_client_io_threads()
# Add bricks to convert to make the volume 2x3
ret = expand_volume(self.mnode, self.volname, self.servers,
self.all_servers_info)
self.assertTrue(ret, "Failed to add brick on volume %s"
% self.volname)
g.log.info("Add-brick successful on volume")
# Check the value of performance.client-io-threads it should be ON
# and io-threads should be loaded in trusted-.tcp-fuse.vol
self._check_value_of_performance_client_io_threads()
# Remove brick to make the volume 1x3 again
ret = shrink_volume(self.mnode, self.volname)
self.assertTrue(ret, "Failed to remove-brick from volume")
g.log.info("Remove-brick successful on volume")
# Check the value of performance.client-io-threads it should be ON
# and io-threads should be loaded in trusted-.tcp-fuse.vol
self._check_value_of_performance_client_io_threads()
def test_rebalance_stop_with_large_file(self):
"""
Testcase Steps:
1. Create and start a volume.
2. Mount volume on client and create a large file.
3. Add bricks to the volume and check layout
4. Rename the file such that it hashs to different
subvol.
5. Start rebalance on volume.
6. Stop rebalance on volume.
"""
# Create file BIG1.
command = ("dd if=/dev/urandom of={}/BIG1 bs=1024K count=10000".format(
self.mounts[0].mountpoint))
ret, _, _ = g.run(self.mounts[0].client_system, command)
self.assertEqual(ret, 0, "Unable to create file I/O failed")
g.log.info('Successfully created file BIG1.')
# Checking if file created on correct subvol or not.
ret = validate_files_in_dir(
self.mounts[0].client_system,
self.mounts[0].mountpoint,
file_type=k.FILETYPE_FILES,
test_type=k.TEST_FILE_EXISTS_ON_HASHED_BRICKS)
self.assertTrue(ret, "Files not created on correct subvol.")
g.log.info("File BIG1 is on correct subvol according to "
"the hash value")
# Adding brick to volume
add_brick_list = form_bricks_list_to_add_brick(self.mnode,
self.volname,
self.servers,
self.all_servers_info)
ret, _, _ = add_brick(self.mnode, self.volname, add_brick_list)
self.assertEqual(ret, 0, "Unable to add bricks to volume")
g.log.info("Successfully added bricks to volume.")
# Check if brick is added successfully or not.
current_bricks = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(
current_bricks, "Unable to get "
"current active bricks of volume")
g.log.info("Successfully got active bricks of volume.")
for brick in add_brick_list:
self.assertIn(brick, current_bricks,
("Brick %s is not added to volume" % brick))
# Create directory testdir.
ret = mkdir(self.mounts[0].client_system,
self.mounts[0].mountpoint + '/testdir')
self.assertTrue(ret, "Failed to create testdir directory")
g.log.info("Successfuly created testdir directory.")
# Layout should be set on the new brick and should be
# continous and complete
ret = validate_files_in_dir(self.mounts[0].client_system,
self.mounts[0].mountpoint + '/testdir',
test_type=k.TEST_LAYOUT_IS_COMPLETE)
self.assertTrue(ret, "Layout not set for the new subvol")
g.log.info("New subvol has been added successfully")
# Rename file so that it gets hashed to different subvol
file_index = 0
path_info_dict = get_pathinfo(self.mounts[0].client_system,
self.mounts[0].mountpoint + '/BIG1')
initial_brick_set = path_info_dict['brickdir_paths']
while True:
# Calculate old_filename and new_filename and rename.
file_index += 1
old_filename = "{}/BIG{}".format(self.mounts[0].mountpoint,
file_index)
new_filename = "{}/BIG{}".format(self.mounts[0].mountpoint,
(file_index + 1))
ret, _, _ = g.run(self.mounts[0].client_system,
"mv {} {}".format(old_filename, new_filename))
self.assertEqual(ret, 0, "Rename not successful")
# Checking if it was moved to new subvol or not.
path_info_dict = get_pathinfo(
self.mounts[0].client_system,
self.mounts[0].mountpoint + '/BIG%d' % (file_index + 1))
if path_info_dict['brickdir_paths'] != initial_brick_set:
break
g.log.info("file renamed successfully")
# Start rebalance on volume
ret, _, _ = rebalance_start(self.mnode, self.volname, fix_layout=False)
self.assertEqual(ret, 0, "Rebalance did not start")
g.log.info("Rebalance started successfully on volume %s", self.volname)
# Stop rebelance on volume
ret, _, _ = rebalance_stop(self.mnode, self.volname)
self.assertEqual(ret, 0, "Rebalance stop command did not execute.")
g.log.info("Rebalance stopped successfully on volume %s", self.volname)
# Get rebalance status in xml
command = ("gluster volume rebalance {} status --xml".format(
self.volname))
ret, _, _ = g.run(self.mnode, command)
self.assertEqual(
ret, 1, "Unexpected: Rebalance still running "
"even after stop.")
g.log.info("Rebalance is not running after stop.")
def test_detach_node_used_to_mount(self):
# pylint: disable=too-many-statements
"""
Test case:
1.Create a 1X3 volume with only 3 nodes from the cluster.
2.Mount volume on client node using the ip of the fourth node.
3.Write IOs to the volume.
4.Detach node N4 from cluster.
5.Create a new directory on the mount point.
6.Create a few files using the same command used in step 3.
7.Add three more bricks to make the volume
2x3 using add-brick command.
8.Do a gluster volume rebalance on the volume.
9.Create more files from the client on the mount point.
10.Check for files on bricks from both replica sets.
11.Create a new directory from the client on the mount point.
12.Check for directory in both replica sets.
"""
# Create and start a volume
ret = setup_volume(self.mnode, self.all_servers_info, self.volume)
self.assertTrue(ret, "Failed to create and start volume")
g.log.info("Volume %s created successfully", self.volname)
# Mounting the volume.
ret, _, _ = mount_volume(self.volname,
mtype=self.mount_type,
mpoint=self.mounts[0].mountpoint,
mserver=self.servers[4],
mclient=self.mounts[0].client_system)
self.assertEqual(ret, 0, ("Volume %s is not mounted") % self.volname)
g.log.info("Volume mounted successfully using %s", self.servers[4])
# Creating 100 files.
command = ('for number in `seq 1 100`;do touch ' +
self.mounts[0].mountpoint + '/file$number; done')
ret, _, _ = g.run(self.mounts[0].client_system, command)
self.assertEqual(ret, 0, "File creation failed.")
g.log.info("Files create on mount point.")
# Detach N4 from the list.
ret, _, _ = peer_detach(self.mnode, self.servers[4])
self.assertEqual(ret, 0, "Failed to detach %s" % self.servers[4])
g.log.info("Peer detach successful %s", self.servers[4])
# Creating a dir.
ret = mkdir(self.mounts[0].client_system,
self.mounts[0].mountpoint + "/dir1",
parents=True)
self.assertTrue(ret, ("Failed to create directory dir1."))
g.log.info("Directory dir1 created successfully.")
# Creating 100 files.
command = ('for number in `seq 101 200`;do touch ' +
self.mounts[0].mountpoint + '/file$number; done')
ret, _, _ = g.run(self.mounts[0].client_system, command)
self.assertEqual(ret, 0, "File creation failed.")
g.log.info("Files create on mount point.")
# Forming brick list
brick_list = form_bricks_list_to_add_brick(self.mnode, self.volname,
self.servers,
self.all_servers_info)
# Adding bricks
ret, _, _ = add_brick(self.mnode, self.volname, brick_list)
self.assertEqual(ret, 0,
"Failed to add brick to the volume %s" % self.volname)
g.log.info("Brick added successfully to the volume %s", self.volname)
# Start rebalance for volume.
g.log.info("Starting rebalance on the volume")
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(ret, 0, ("Failed to start rebalance "
"on the volume %s", self.volname))
g.log.info("Successfully started rebalance on the volume %s",
self.volname)
# Creating 100 files.
command = ('for number in `seq 201 300`;do touch ' +
self.mounts[0].mountpoint + '/file$number; done')
ret, _, _ = g.run(self.mounts[0].client_system, command)
self.assertEqual(ret, 0, "File creation failed.")
g.log.info("Files create on mount point.")
# Check for files on bricks.
attempts = 10
while attempts:
number = str(randint(1, 300))
for brick in brick_list:
brick_server, brick_dir = brick.split(':')
file_name = brick_dir + "/file" + number
if file_exists(brick_server, file_name):
g.log.info("Check xattr"
" on host %s for file %s", brick_server,
file_name)
ret = get_fattr_list(brick_server, file_name)
self.assertTrue(ret,
("Failed to get xattr for %s" % file_name))
g.log.info("Got xattr for %s successfully", file_name)
attempts -= 1
# Creating a dir.
ret = mkdir(self.mounts[0].client_system,
self.mounts[0].mountpoint + "/dir2")
if not ret:
attempts = 5
while attempts:
ret = mkdir(self.mounts[0].client_system,
self.mounts[0].mountpoint + "/dir2")
if ret:
break
attempts -= 1
self.assertTrue(ret, ("Failed to create directory dir2."))
g.log.info("Directory dir2 created successfully.")
# Check for directory in both replica sets.
for brick in brick_list:
brick_server, brick_dir = brick.split(':')
folder_name = brick_dir + "/dir2"
if file_exists(brick_server, folder_name):
g.log.info(
"Check trusted.glusterfs.dht"
" on host %s for directory %s", brick_server, folder_name)
ret = get_fattr(brick_server, folder_name,
'trusted.glusterfs.dht')
self.assertTrue(ret, ("Failed to get trusted.glusterfs.dht"
" xattr for %s" % folder_name))
g.log.info(
"Get trusted.glusterfs.dht xattr"
" for %s successfully", folder_name)
def test_glusterd_rebalance(self):
'''
-> Create Volume
-> Fuse mount the volume
-> Perform I/O on fuse mount
-> Add bricks to the volume
-> Perform rebalance on the volume
-> While rebalance is in progress,
-> restart glusterd on all the nodes in the cluster
'''
# run IOs
g.log.info("Starting IO on all mounts...")
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = (
"python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 4 "
"--dir-length 6 "
"--max-num-of-dirs 3 "
"--num-of-files 25 %s" %
(self.script_upload_path, self.counter, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
self.counter = self.counter + 10
# Validate IO
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# Forming brick list
brick_list = form_bricks_list_to_add_brick(self.mnode, self.volname,
self.servers,
self.all_servers_info)
# Adding Bricks
ret, _, _ = add_brick(self.mnode, self.volname, brick_list)
self.assertEqual(ret, 0,
"Failed to add brick to the volume %s" % self.volname)
g.log.info("Brick added successfully to the volume %s", self.volname)
# Performing rebalance
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(
ret, 0, 'Failed to start rebalance on volume %s' % self.volname)
g.log.info("Rebalance started successfully on volume %s", self.volname)
# Checking Rebalance is in progress or not
rebalance_status = get_rebalance_status(self.mnode, self.volname)
if rebalance_status['aggregate']['statusStr'] != 'in progress':
raise ExecutionError("Rebalance is not in 'in progress' state, "
"either rebalance is in compeleted state or"
" failed to get rebalance status")
# Restart glusterd
ret = restart_glusterd(self.servers)
self.assertTrue(ret, "Failed to restart glusterd on servers")
g.log.info("Glusterd restarted successfully on %s", self.servers)
# Checking glusterd status
count = 0
while count < 60:
ret = is_glusterd_running(self.servers)
if not ret:
break
sleep(2)
count += 1
self.assertEqual(ret, 0, "Glusterd is not running on some of the "
"servers")
g.log.info("Glusterd is running on all servers %s", self.servers)
def test_rebalance_start_status_stop(self):
# pylint: disable=too-many-statements
# Form brick list for expanding volume
add_brick_list = form_bricks_list_to_add_brick(self.mnode,
self.volname,
self.servers,
self.all_servers_info,
distribute_count=1)
self.assertIsNotNone(add_brick_list,
("Volume %s: Failed to form "
"bricks list to expand", self.volname))
g.log.info("Volume %s: Formed bricks list to expand", self.volname)
# Expanding volume by adding bricks to the volume
g.log.info("Volume %s: Expand start")
ret, _, _ = add_brick(self.mnode, self.volname, add_brick_list)
self.assertEqual(ret, 0, ("Volume %s: Expand failed", self.volname))
g.log.info("Volume %s: Expand successful", self.volname)
# Wait for gluster processes to come online
g.log.info("Wait for gluster processes to come online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Volume %s: one or more volume process are "
"not up", self.volname))
g.log.info("All volume %s processes are online", self.volname)
# Log Volume Info and Status after expanding the volume
g.log.info("Logging volume info and Status after expanding volume")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(
ret, "Logging volume info and status failed on "
"volume %s" % self.volname)
g.log.info(
"Successful in logging volume info and status of volume "
"%s", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online", self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Getting arequal checksum before rebalance start
g.log.info("Getting arequal before rebalance start")
arequal_checksum_before_rebalance_start = collect_mounts_arequal(
self.mounts)
# Start Rebalance
g.log.info("Starting Rebalance on the volume")
ret, _, _ = rebalance_start(self.mnode, self.volname)
self.assertEqual(
ret, 0, ("Volume %s: Failed to start rebalance", self.volname))
g.log.info("Volume %s: Rebalance started ", self.volname)
# Stop on-going rebalance
g.log.info("Stop rebalance on the volume")
ret, _, _ = rebalance_stop(self.mnode, self.volname)
self.assertEqual(ret, 0,
("Volume %s: Failed to stop rebalance", self.volname))
g.log.info("Checking whether the migration is stopped or not")
# Wait till the on-going file migration completes on all servers
count = 0
while count < 80:
rebalance_count = 0
for server in self.servers:
ret, _, _ = g.run(server, "pgrep rebalance")
if ret != 0:
rebalance_count += 1
if rebalance_count == len(self.servers):
break
sleep(2)
count += 1
g.log.info("Volume %s: Rebalance process is not running on servers",
self.volname)
# List all files and dirs from mount point
g.log.info("List all files and directories:")
ret = list_all_files_and_dirs_mounts(self.mounts)
g.log.info("Listing all files and directories is successful")
# Getting arequal checksum after the rebalance is stopped
g.log.info("Getting arequal checksum after the rebalance is stopped")
arequal_checksum_after_rebalance_stop = collect_mounts_arequal(
self.mounts)
# Comparing arequals checksum before start of rebalance and
# after the rebalance is stopped
g.log.info("Comparing arequals checksum before start of rebalance and"
"after the rebalance is stopped")
self.assertEqual(arequal_checksum_before_rebalance_start,
arequal_checksum_after_rebalance_stop,
"arequal checksum is NOT MATCHNG")
g.log.info("arequal checksum is SAME")
