def test_client_side_quorum_with_fixed_validate_max_bricks(self):
"""
Test Script with Client Side Quorum with fixed should validate
maximum number of bricks to accept
* set cluster quorum to fixed
* set cluster.quorum-count to higher number which is greater than
number of replicas in a sub-voulme
* Above step should fail
"""
# set cluster.quorum-type to fixed
options = {"cluster.quorum-type": "fixed"}
g.log.info("setting %s for the volume %s" % (options, self.volname))
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s"
% (options, self.volname)))
g.log.info("Successfully set %s for volume %s"
% (options, self.volname))
# get the subvolumes
g.log.info("Starting to get sub-volumes for volume %s" % self.volname)
subvols_dict = get_subvols(self.mnode, self.volname)
num_subvols = len(subvols_dict['volume_subvols'])
g.log.info("Number of subvolumes in volume %s is %s"
% (self.volname, num_subvols))
# get the number of bricks in replica set
num_bricks_in_subvol = len(subvols_dict['volume_subvols'][0])
g.log.info("Number of bricks in each replica set : %s"
% num_bricks_in_subvol)
# set cluster.quorum-count to higher value than the number of bricks in
# repliac set
start_range = num_bricks_in_subvol + 1
end_range = num_bricks_in_subvol + 30
for i in range(start_range, end_range):
options = {"cluster.quorum-count": "%s" % start_range}
g.log.info("setting %s for the volume %s" %
(options, self.volname))
ret = set_volume_options(self.mnode, self.volname, options)
self.assertFalse(ret, ("Able to set %s for volume %s, quorum-count"
" should not be greater than number of"
" bricks in replica set"
% (options, self.volname)))
g.log.info("Expected: Unable to set %s for volume %s, "
"quorum-count should be less than number of bricks "
"in replica set" % (options, self.volname))
def _disable_io_encryption(self):
""" Disables IO encryption """
# UnMount Volume
g.log.info("Starting to Unmount Volume %s", self.volname)
ret, _, _ = umount_volume(self.mounts[0].client_system,
self.mounts[0].mountpoint,
mtype=self.mount_type)
self.assertEqual(ret, 0, "Failed to Unmount volume")
# Stop Volume
ret, _, _ = volume_stop(self.mnode, self.volname)
self.assertEqual(ret, 0, "Failed to Stop volume")
# Disable server and client SSL usage
options = {"server.ssl": "off",
"client.ssl": "off"}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, "Failed to set volume options")
# Start Volume
ret, _, _ = volume_start(self.mnode, self.volname)
self.assertEqual(ret, 0, "Failed to Start volume")
# Mount Volume
ret, _, _ = mount_volume(self.volname, mtype=self.mount_type,
mpoint=self.mounts[0].mountpoint,
mserver=self.mnode,
mclient=self.mounts[0].client_system)
self.assertEqual(ret, 0, "Failed to mount the volume back")
def test_lower_gluster_op_version(self):
"""
- Create volume
- Get the volume op-version
- Set the valid lower op-version
- Set the invalid op-version
"""
# Get the volume op-version
ret = get_volume_options(self.mnode, self.volname,
'cluster.op-version')
self.assertIsNotNone(ret, "Failed to get the op-version")
g.log.info("Successfully get the op-version")
# Lowest opversion is 30000
lowest_op_version = 30000
invalid_op_version = "abc"
lower_op_version_dict = {'cluster.op-version': lowest_op_version}
invalid_op_version_dict = {'cluster.op-version': invalid_op_version}
# Set the volume option with lower op-version
ret = set_volume_options(self.mnode, 'all', lower_op_version_dict)
self.assertFalse(
ret, "Expected: Should not be able to set lower "
"op-version \n Actual: Successfully set the lower"
" op-version")
g.log.info("Failed to set op-version %s as "
"expected", lowest_op_version)
# Setting invalid opversion
ret = set_volume_options(self.mnode, 'all', invalid_op_version_dict)
self.assertFalse(
ret, "Expected: Should not be able to set invalid "
"op-version \n Actual: Successfully set the invalid"
" op-version")
g.log.info("Failed to set op-version %s as "
"expected", invalid_op_version)
def tearDown(self):
"""
tearDown method for every test
"""
# Calling GlusterBaseClass tearDown
self.get_super_method(self, 'tearDown')()
ret = umount_volume(mclient=self.mounts[0].client_system,
mpoint=self.mountpoint)
if not ret:
raise ExecutionError("Unable to umount the volume")
g.log.info("Unmounting of the volume %s succeeded", self.volname)
# Resetting the volume option set in the setup
ret = set_volume_options(self.mnode, self.volname,
{'diagnostics.client-log-level': 'INFO'})
if not ret:
raise ExecutionError("Unable to set the client log level to INFO")
g.log.info("Volume option is set successfully.")
ret = self.cleanup_volume()
if not ret:
raise ExecutionError("Unable to perform volume clenaup")
g.log.info("Volume cleanup is successfull")
def test_write_io_mount_point_resumed_quorum_restored_x3(self):
"""
- set cluster.quorum-type to auto
- start I/O from the mount point
- Do IO and check on subvols with two nodes to reboot
(do for each subvol)
- get files to delete/create for nodes to be offline
- delete files from mountpoint
- reboot nodes
- creating files on nodes while rebooting
- validate for rofs
- wait for volume processes to be online
- creating files on nodes after rebooting
- validate IO
- Do IO and check on subvols without nodes to reboot
(do for each subvol)
- get files to delete/create for nodes to be online
- delete files from mountpoint
- reboot nodes
- creating files on online nodes while rebooting other nodes
- validate IO
- Do IO and check and reboot two nodes on all subvols
- get files to delete/create for nodes to be offline
- delete files from mountpoint
- reboot nodes
- creating files on nodes while rebooting
- validate for rofs
- wait for volume processes to be online
- creating files on nodes after rebooting
- validate IO
"""
# pylint: disable=too-many-locals,too-many-statements,too-many-branches
# set cluster.quorum-type to auto
options = {"cluster.quorum-type": "auto"}
g.log.info("setting cluster.quorum-type to auto on volume %s",
self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set volume option %s for"
"volume %s" % (options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# Creating files on client side
for mount_obj in self.mounts:
g.log.info("Generating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = "/usr/bin/env python %s create_files -f 30 %s" % (
self.script_upload_path, mount_obj.mountpoint)
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# Validate IO
self.io_validation_complete = False
self.assertTrue(validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# Do IO and check on subvols with nodes to reboot
subvols_dict = get_subvols(self.mnode, self.volname)
for subvol in subvols_dict['volume_subvols']:
# define nodes to reboot
brick_list = subvol[0:2]
nodes_to_reboot = []
for brick in brick_list:
node, brick_path = brick.split(':')
nodes_to_reboot.append(node)
# get files to delete/create for nodes to be offline
node, brick_path = brick_list[0].split(':')
ret, brick_file_list, _ = g.run(node, 'ls %s' % brick_path)
self.assertFalse(ret, 'Failed to ls files on %s' % node)
file_list = brick_file_list.splitlines()
# delete files from mountpoint
for mount_obj in self.mounts:
g.log.info("Deleting data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ('cd %s/ ; rm -rf %s' %
(mount_obj.mountpoint, ' '.join(file_list)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(
ret, 'Failed to rm file on %s' % mount_obj.client_system)
g.log.info('Files %s are deleted', file_list)
# reboot nodes on subvol and wait while rebooting
g.log.info("Rebooting the nodes %s", nodes_to_reboot)
ret = reboot_nodes(nodes_to_reboot)
self.assertTrue(ret,
'Failed to reboot nodes %s ' % nodes_to_reboot)
# Creating files on nodes while rebooting
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = ("cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list)))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# Validate IO
self.io_validation_complete = False
g.log.info("Validating if IO failed with read-only filesystem")
ret = is_io_procs_fail_with_rofs(self, self.all_mounts_procs,
self.mounts)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on read-only filesystem"))
self.io_validation_complete = True
g.log.info("EXPECTED: "
"Read-only file system in IO while creating file")
# check if nodes are online
counter = 0
timeout = 300
_rc = False
while counter < timeout:
ret, reboot_results = are_nodes_online(nodes_to_reboot)
if not ret:
g.log.info("Nodes are offline, Retry after 5 seconds ... ")
time.sleep(5)
counter = counter + 5
else:
_rc = True
break
if not _rc:
for node in reboot_results:
if reboot_results[node]:
g.log.info("Node %s is online", node)
else:
g.log.error(
"Node %s is offline even after "
"%d minutes", node, timeout / 60.0)
else:
g.log.info("All nodes %s are up and running", nodes_to_reboot)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(
self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Creating files on nodes after rebooting
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = ("cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list)))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# Validate IO
self.io_validation_complete = False
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# Do IO and check on subvols without nodes to reboot
subvols_dict = get_subvols(self.mnode, self.volname)
for subvol in subvols_dict['volume_subvols']:
# define nodes to reboot
brick_list = subvol[0:2]
nodes_to_reboot = []
for brick in brick_list:
node, brick_path = brick.split(':')
nodes_to_reboot.append(node)
# get files to delete/create for nodes to be online
new_subvols_dict = get_subvols(self.mnode, self.volname)
subvol_to_operate = new_subvols_dict['volume_subvols']
subvol_to_operate.remove(subvol)
brick_list_subvol_online = subvol_to_operate[0]
node, brick_path_vol_online = \
brick_list_subvol_online[0].split(':')
ret, brick_file_list, _ = g.run(node,
'ls %s' % brick_path_vol_online)
self.assertFalse(ret, 'Failed to ls files on %s' % node)
file_list = brick_file_list.splitlines()
# delete files from mountpoint
for mount_obj in self.mounts:
g.log.info("Deleting data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ('cd %s/ ; rm -rf %s' %
(mount_obj.mountpoint, ' '.join(file_list)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(
ret, 'Failed to rm file on %s' % mount_obj.client_system)
g.log.info('Files %s are deleted', file_list)
# reboot nodes on subvol and wait while rebooting
g.log.info("Rebooting the nodes %s", nodes_to_reboot)
ret = reboot_nodes(nodes_to_reboot)
self.assertTrue(ret,
'Failed to reboot nodes %s ' % nodes_to_reboot)
# Creating files on nodes while rebooting
self.all_mounts_procs = []
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = ("cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list)))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
self.all_mounts_procs.append(proc)
# Validate IO
self.io_validation_complete = False
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
# check if nodes are online
counter = 0
timeout = 300
_rc = False
while counter < timeout:
ret, reboot_results = are_nodes_online(nodes_to_reboot)
if not ret:
g.log.info("Nodes are offline, Retry after 5 seconds ... ")
time.sleep(5)
counter = counter + 5
else:
_rc = True
break
if not _rc:
for node in reboot_results:
if reboot_results[node]:
g.log.info("Node %s is online", node)
else:
g.log.error(
"Node %s is offline even after "
"%d minutes", node, timeout / 60.0)
else:
g.log.info("All nodes %s are up and running", nodes_to_reboot)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode,
self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(
self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Do IO and check and reboot nodes on all subvols
subvols_dict = get_subvols(self.mnode, self.volname)
nodes_to_reboot = []
file_list_for_all_subvols = []
for subvol in subvols_dict['volume_subvols']:
# define nodes to reboot
brick_list = subvol[0:2]
for brick in brick_list:
node, brick_path = brick.split(':')
nodes_to_reboot.append(node)
# get files to delete/create for nodes to be offline
node, brick_path = brick_list[0].split(':')
ret, brick_file_list, _ = g.run(node, 'ls %s' % brick_path)
self.assertFalse(ret, 'Failed to ls files on %s' % node)
file_list = brick_file_list.splitlines()
file_list_for_all_subvols.append(file_list)
# delete files from mountpoint
for mount_obj in self.mounts:
g.log.info("Deleting data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ('cd %s/ ; rm -rf %s' %
(mount_obj.mountpoint, ' '.join(file_list)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, 'Failed to rm file on %s' % node)
g.log.info('Files %s are deleted', file_list)
# reboot nodes on subvol and wait while rebooting
g.log.info("Rebooting the nodes %s", nodes_to_reboot)
ret = reboot_nodes(nodes_to_reboot)
self.assertTrue(ret, 'Failed to reboot nodes %s ' % nodes_to_reboot)
# Creating files on nodes while rebooting
all_mounts_procs, all_mounts_procs_1, all_mounts_procs_2 = [], [], []
# Create files for 1-st subvol and get all_mounts_procs_1
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = (
"cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[0])))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs_1.append(proc)
all_mounts_procs.append(all_mounts_procs_1)
# Create files for 2-st subvol and get all_mounts_procs_2
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = (
"cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[1])))
proc2 = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs_2.append(proc2)
all_mounts_procs.append(all_mounts_procs_2)
for mounts_procs in all_mounts_procs:
# Validate IO
self.io_validation_complete = False
g.log.info("Validating if IO failed with read-only filesystem")
ret = is_io_procs_fail_with_rofs(self, mounts_procs, self.mounts)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on read-only filesystem"))
self.io_validation_complete = True
g.log.info("EXPECTED: "
"Read-only file system in IO while creating file")
# check if nodes are online
counter = 0
timeout = 300
_rc = False
while counter < timeout:
ret, reboot_results = are_nodes_online(nodes_to_reboot)
if not ret:
g.log.info("Nodes are offline, Retry after 5 seconds ... ")
time.sleep(5)
counter = counter + 5
else:
_rc = True
break
if not _rc:
for node in reboot_results:
if reboot_results[node]:
g.log.info("Node %s is online", node)
else:
g.log.error("Node %s is offline even after "
"%d minutes", node, timeout / 60.0)
else:
g.log.info("All nodes %s are up and running", nodes_to_reboot)
# Wait for volume processes to be online
g.log.info("Wait for volume processes to be online")
ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
self.assertTrue(ret, ("Failed to wait for volume %s processes to "
"be online", self.volname))
g.log.info(
"Successful in waiting for volume %s processes to be "
"online", self.volname)
# Verify volume's all process are online
g.log.info("Verifying volume's all process are online")
ret = verify_all_process_of_volume_are_online(self.mnode, self.volname)
self.assertTrue(
ret, ("Volume %s : All process are not online" % self.volname))
g.log.info("Volume %s : All process are online", self.volname)
# Creating files on nodes after rebooting
all_mounts_procs, all_mounts_procs_1, all_mounts_procs_2 = [], [], []
# Create files for 1-st subvol and get all_mounts_procs_1
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = (
"cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[0])))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs_1.append(proc)
all_mounts_procs.append(all_mounts_procs_1)
# Create files for 2-st subvol and get all_mounts_procs_2
for mount_obj in self.mounts:
g.log.info("Creating data for %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
# Creating files
cmd = (
"cd %s/ ;"
"touch %s" %
(mount_obj.mountpoint, ' '.join(file_list_for_all_subvols[1])))
proc2 = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs_2.append(proc2)
all_mounts_procs.append(all_mounts_procs_2)
for mounts_procs in all_mounts_procs:
# Validate IO
self.io_validation_complete = False
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
def test_client_side_quorum_with_auto_option(self):
"""
Test Script to verify the Client Side Quorum with auto option
* set cluster.quorum-type to auto.
* start I/O from the mount point.
* kill 2 of the brick process from the each and every replica set
* perform ops
"""
# set cluster.quorum-type to auto
options = {"cluster.quorum-type": "auto"}
g.log.info("setting cluster.quorum-type to auto on "
"volume %s" % self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set volume option %s for"
"volume %s" % (options, self.volname)))
g.log.info("Sucessfully set %s for volume %s" %
(options, self.volname))
# write files on all mounts
g.log.info("Starting IO on all mounts...")
g.log.info("mounts: %s" % self.mounts)
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s create_files "
"-f 10 --base-file-name file %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating IO on mounts")
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("IO is successful on all mounts")
# get the subvolumes
g.log.info("Starting to get sub-volumes for volume %s" % self.volname)
subvols_dict = get_subvols(self.mnode, self.volname)
num_subvols = len(subvols_dict['volume_subvols'])
g.log.info("Number of subvolumes in volume %s:" % num_subvols)
# bring bricks offline( 2 bricks ) for all the subvolumes
for i in range(0, num_subvols):
subvol_brick_list = subvols_dict['volume_subvols'][i]
g.log.info("sub-volume %s brick list : %s" %
(i, subvol_brick_list))
# For volume type: 1 * 2, bring 1 brick offline
if len(subvol_brick_list) == 2:
bricks_to_bring_offline = subvol_brick_list[0:1]
else:
bricks_to_bring_offline = subvol_brick_list[0:2]
g.log.info("Going to bring down the brick process "
"for %s" % bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, ("Failed to bring down the bricks. Please "
"check the log file for more details."))
g.log.info("Brought down the brick process "
"for %s succesfully" % bricks_to_bring_offline)
# create 2 files named newfile0.txt and newfile1.txt
g.log.info("Start creating 2 files on all mounts...")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s create_files "
"-f 2 --base-file-name newfile %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with read-only filesystem")
ret = is_io_procs_fail_with_rofs(self, all_mounts_procs, self.mounts)
self.assertTrue(ret, ("Unexpected error and IO successfull"
" on read-only filesystem"))
g.log.info("EXPECTED: Read-only file system in IO while creating file")
# create directory user1
g.log.info("Start creating directory on all mounts...")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s create_deep_dir "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with read-only filesystem")
ret = is_io_procs_fail_with_rofs(self, all_mounts_procs, self.mounts)
self.assertTrue(ret, ("Unexpected error and IO successfull"
" on read-only filesystem"))
g.log.info("EXPECTED: Read-only file system in IO while"
" creating directory")
# create h/w link to file
g.log.info("Start creating hard link for file0.txt on all mounts")
for mount_obj in self.mounts:
cmd = "ln %s/file0.txt %s/file0.txt_hwlink" \
% (mount_obj.mountpoint, mount_obj.mountpoint)
ret, out, err = g.run(mount_obj.client_system, cmd)
self.assertTrue(ret, ("Unexpected error and creating hard link"
" successful on read-only filesystem"))
self.assertIn(
"Read-only file system", err,
"Read-only filesystem not found in "
"IO while truncating file")
g.log.info("EXPECTED: Read-only file system in IO")
# create s/w link
g.log.info("Start creating soft link for file1.txt on all mounts")
for mount_obj in self.mounts:
cmd = "ln -s %s/file1.txt %s/file1.txt_swlink" %\
(mount_obj.mountpoint, mount_obj.mountpoint)
ret, out, err = g.run(mount_obj.client_system, cmd)
self.assertTrue(ret, ("Unexpected error and creating soft link"
" successful on read-only filesystem"))
self.assertIn(
"Read-only file system", err,
"Read-only filesystem not found in "
"IO while truncating file")
g.log.info("EXPECTED: Read-only file system in IO")
# append to file
g.log.info("Appending to file1.txt on all mounts")
for mount_obj in self.mounts:
cmd = "cat %s/file0.txt >> %s/file1.txt" %\
(mount_obj.mountpoint, mount_obj.mountpoint)
ret, out, err = g.run(mount_obj.client_system, cmd)
self.assertTrue(ret, ("Unexpected error and append successful"
" on read-only filesystem"))
self.assertIn(
"Read-only file system", err,
"Read-only filesystem not found in "
"IO while truncating file")
g.log.info("EXPECTED: Read-only file system in IO")
# modify the file
g.log.info("Modifying file1.txt on all mounts")
for mount_obj in self.mounts:
cmd = "echo 'Modify Contents' > %s/file1.txt"\
% (mount_obj.mountpoint)
ret, out, err = g.run(mount_obj.client_system, cmd)
self.assertTrue(ret, ("Unexpected error and modifying successful"
" on read-only filesystem"))
self.assertIn(
"Read-only file system", err,
"Read-only filesystem not found in "
"IO while truncating file")
g.log.info("EXPECTED: Read-only file system in IO")
# truncate the file
g.log.info("Truncating file1.txt on all mounts")
for mount_obj in self.mounts:
cmd = "truncate -s 0 %s/file1.txt" % (mount_obj.mountpoint)
ret, out, err = g.run(mount_obj.client_system, cmd)
self.assertTrue(ret, ("Unexpected error and truncating file"
" successful on read-only filesystem"))
self.assertIn(
"Read-only file system", err,
"Read-only filesystem not found in "
"IO while truncating file")
g.log.info("EXPECTED: Read-only file system in IO")
# read the file
g.log.info("Starting reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("validating IO on all mounts")
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "Reads failed on some of the clients")
g.log.info("Reads successful on all mounts")
# stat on file
g.log.info("stat on file1.txt on all mounts")
for mount_obj in self.mounts:
cmd = "stat %s/file1.txt" % (mount_obj.mountpoint)
ret, out, err = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, ("Unexpected error and stat on file fails"
" on read-only filesystem"))
g.log.info("stat on file is successfull on read-only filesystem")
# stat on dir
g.log.info("stat on directory on all mounts")
for mount_obj in self.mounts:
cmd = ("python %s stat %s" %
(self.script_upload_path, mount_obj.mountpoint))
ret, out, err = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, ("Unexpected error and stat on directory"
" fails on read-only filesystem"))
g.log.info("stat on dir is successfull on read-only filesystem")
# ls on mount point
g.log.info("ls on mount point on all mounts")
for mount_obj in self.mounts:
cmd = ("python %s ls %s" %
(self.script_upload_path, mount_obj.mountpoint))
ret, out, err = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, ("Unexpected error and listing file fails"
" on read-only filesystem"))
g.log.info("listing files is successfull on read-only filesystem")
def test_heal_io_hang(self):
mountpoint = self.mounts[0].mountpoint
# disable client side heal
options = {"disperse.background-heals": 0}
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, 'Failed to disable client side heal')
g.log.info("Successfully disabled client side heal")
# Log Volume Info and Status after disabling client side heal
g.log.info("Logging volume info and status")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed "
"on volume %s", self.volname))
bricks_list = get_all_bricks(self.mnode, self.volname)
self.assertIsNotNone(bricks_list, "Failed to get the bricks list")
# Create files
cmd = ("cd %s; mkdir test; cd test; for i in `seq 1 100` ;"
"do touch file$i; done" % mountpoint)
ret, _, err = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, err)
g.log.info('Finished creating files while all the bricks are UP')
# Bring bricks offline
ret = bring_bricks_offline(self.volname, bricks_list[0:1])
self.assertTrue(ret, "Failed to bring down the bricks")
g.log.info("Successfully brought the bricks down")
# Start pumping IO from client
cmd = ("cd %s; mkdir test; cd test; for i in `seq 1 100` ;"
"do dd if=/dev/urandom of=file$i bs=1M "
"count=5;done" % mountpoint)
ret, _, err = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, err)
g.log.info('Finished writing on files while a brick is DOWN')
# Bring bricks online
ret = bring_bricks_online(self.mnode, self.volname, bricks_list[0:1])
self.assertTrue(ret, "Failed to bring up the bricks")
g.log.info("Successfully brought the bricks up")
# Verifying all bricks online
ret = are_bricks_online(self.mnode, self.volname, bricks_list)
self.assertTrue(ret, "All bricks are not online")
appendcmd = ("cd %s; mkdir test; cd test; for i in `seq 1 100` ;"
"do dd if=/dev/urandom of=file$i bs=1M "
"count=1 oflag=append conv=notrunc;done" % mountpoint)
ret, _, err = g.run(self.mounts[0].client_system, appendcmd)
self.assertEqual(ret, 0, err)
g.log.info('Finished append on files after bringing bricks online')
# check the heal info and completion
ec_check_heal_comp(self)
# Log Volume Info and Status after bringing the brick up
g.log.info("Logging volume info and status")
ret = log_volume_info_and_status(self.mnode, self.volname)
self.assertTrue(ret, ("Logging volume info and status failed "
"on volume %s", self.volname))
g.log.info(
"Successful in logging volume info and status "
"of volume %s", self.volname)
def test_client_side_quorum_with_auto_option_overwrite_fixed(self):
"""
Test Script to verify the Client Side Quorum with auto option
* check the default value of cluster.quorum-type
* try to set any junk value to cluster.quorum-type
other than {none,auto,fixed}
* check the default value of cluster.quorum-count
* set cluster.quorum-type to fixed and cluster.quorum-count to 1
* start I/O from the mount point
* kill 2 of the brick process from the each replica set.
* set cluster.quorum-type to auto
"""
# pylint: disable=too-many-locals,too-many-statements
# check the default value of cluster.quorum-type
option = "cluster.quorum-type"
g.log.info("Getting %s for the volume %s", option, self.volname)
option_dict = get_volume_options(self.mnode, self.volname, option)
self.assertIsNotNone(option_dict,
("Failed to get %s volume option"
" for volume %s" % (option, self.volname)))
self.assertEqual(option_dict['cluster.quorum-type'], 'auto',
("Default value for %s is not auto"
" for volume %s" % (option, self.volname)))
g.log.info("Succesfully verified default value of %s for volume %s",
option, self.volname)
# set the junk value to cluster.quorum-type
junk_values = ["123", "abcd", "fixxed", "Aauto"]
for each_junk_value in junk_values:
options = {"cluster.quorum-type": "%s" % each_junk_value}
g.log.info("setting %s for the volume "
"%s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertFalse(ret, ("Able to set junk value %s for "
"volume %s" % (options, self.volname)))
g.log.info(
"Expected: Unable to set junk value %s "
"for volume %s", options, self.volname)
# check the default value of cluster.quorum-count
option = "cluster.quorum-count"
g.log.info("Getting %s for the volume %s", option, self.volname)
option_dict = get_volume_options(self.mnode, self.volname, option)
self.assertIsNotNone(option_dict,
("Failed to get %s volume option"
" for volume %s" % (option, self.volname)))
self.assertEqual(option_dict['cluster.quorum-count'], '(null)',
("Default value for %s is not null"
" for volume %s" % (option, self.volname)))
g.log.info("Successful in getting %s for the volume %s", option,
self.volname)
# set cluster.quorum-type to fixed and cluster.quorum-count to 1
options = {"cluster.quorum-type": "fixed", "cluster.quorum-count": "1"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# create files
g.log.info("Starting IO on all mounts...")
g.log.info("mounts: %s", self.mounts)
cmd = ("/usr/bin/env python %s create_files "
"-f 10 --base-file-name file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
ret, _, err = g.run(self.mounts[0].client_system, cmd)
self.assertFalse(
ret,
"IO failed on %s with '%s'" % (self.mounts[0].client_system, err))
# get the subvolumes
g.log.info("starting to get subvolumes for volume %s", self.volname)
subvols_dict = get_subvols(self.mnode, self.volname)
num_subvols = len(subvols_dict['volume_subvols'])
g.log.info("Number of subvolumes in volume %s is %s", self.volname,
num_subvols)
# bring bricks offline( 2 bricks ) for all the subvolumes
for i in range(0, num_subvols):
subvol_brick_list = subvols_dict['volume_subvols'][i]
g.log.info("sub-volume %s brick list : %s", i, subvol_brick_list)
bricks_to_bring_offline = subvol_brick_list[0:2]
g.log.info("Going to bring down the brick process "
"for %s", bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, ("Failed to bring down the bricks. Please "
"check the log file for more details."))
g.log.info("Brought down the brick process "
"for %s successfully", bricks_to_bring_offline)
# create files
g.log.info("Starting IO on all mounts...")
g.log.info("mounts: %s", self.mounts)
cmd = ("/usr/bin/env python %s create_files "
"-f 10 --base-file-name second_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
ret, _, err = g.run(self.mounts[0].client_system, cmd)
self.assertFalse(
ret,
"IO failed on %s with '%s'" % (self.mounts[0].client_system, err))
# set cluster.quorum-type to auto
options = {"cluster.quorum-type": "auto"}
g.log.info("setting %s for volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set volume option %s for "
"volume %s" % (options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# create files
all_mounts_procs = []
g.log.info("Starting IO on mountpount...")
g.log.info("mounts: %s", self.mounts)
cmd = ("mkdir %s/newdir && touch %s/newdir/myfile{1..3}.txt" %
(self.mounts[0].mountpoint, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while creating files")
def test_client_side_quorum_with_fixed_for_cross2(self):
"""
Test Script to verify the Client Side Quorum with fixed
for cross 2 volume
* Disable self heal daemom
* set cluster.quorum-type to fixed.
* start I/O( write and read )from the mount point - must succeed
* Bring down brick1
* start I/0 ( write and read ) - must succeed
* set the cluster.quorum-count to 1
* start I/0 ( write and read ) - must succeed
* set the cluster.quorum-count to 2
* start I/0 ( write and read ) - read must pass, write will fail
* bring back the brick1 online
* start I/0 ( write and read ) - must succeed
* Bring down brick2
* start I/0 ( write and read ) - read must pass, write will fail
* set the cluster.quorum-count to 1
* start I/0 ( write and read ) - must succeed
* cluster.quorum-count back to 2 and cluster.quorum-type to auto
* start I/0 ( write and read ) - must succeed
* Bring back brick2 online
* Bring down brick1
* start I/0 ( write and read ) - read must pass, write will fail
* set the quorum-type to none
* start I/0 ( write and read ) - must succeed
"""
# pylint: disable=too-many-branches,too-many-statements
# Disable self heal daemon
options = {"cluster.self-heal-daemon": "off"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# set cluster.quorum-type to fixed
options = {"cluster.quorum-type": "fixed"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/O( write ) - must succeed
g.log.info("Starting IO on all mounts...")
g.log.info("mounts: %s", self.mounts)
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# get the subvolumes
g.log.info("Starting to get sub-volumes for volume %s", self.volname)
subvols_dict = get_subvols(self.mnode, self.volname)
num_subvols = len(subvols_dict['volume_subvols'])
g.log.info("Number of subvolumes in volume %s:", num_subvols)
# Bring down brick1 for all the subvolumes
subvolumes_first_brick_list = []
subvolumes_second_brick_list = []
for i in range(0, num_subvols):
subvol_brick_list = subvols_dict['volume_subvols'][i]
g.log.info("sub-volume %s brick list : %s", i, subvol_brick_list)
subvolumes_first_brick_list.append(subvol_brick_list[0])
subvolumes_second_brick_list.append(subvol_brick_list[1])
g.log.info("Going to bring down the brick process "
"for %s", subvolumes_first_brick_list)
ret = bring_bricks_offline(self.volname, subvolumes_first_brick_list)
self.assertTrue(ret, ("Failed to bring down the bricks. Please "
"check the log file for more details."))
g.log.info("Brought down the brick process "
"for %s successfully", subvolumes_first_brick_list)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on all mounts...")
g.log.info("mounts: %s", self.mounts)
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name second_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# set the cluster.quorum-count to 1
options = {"cluster.quorum-count": "1"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on mount.....")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name third_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# set the cluster.quorum-count to 2
options = {"cluster.quorum-count": "2"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/0 ( write and read ) - read must pass, write will fail
g.log.info("Starting IO on mount......")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name fourth_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with Read Only File System")
ret, _ = is_io_procs_fail_with_rofs(self, all_mounts_procs,
self.mounts)
self.assertTrue(ret, ("Unexpected Error and IO successful"
" on Read-Only File System"))
g.log.info("EXPECTED Read-only file system in IO while creating file")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# bring back the brick1 online for all subvolumes
g.log.info("bringing up the bricks : %s online",
subvolumes_first_brick_list)
ret = bring_bricks_online(self.mnode, self.volname,
subvolumes_first_brick_list)
self.assertTrue(ret, ("Failed to brought the bricks %s online" %
subvolumes_first_brick_list))
g.log.info("Successfully brought the bricks %s online",
subvolumes_first_brick_list)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on mount.....")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name fifth_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# Bring down brick2 for all the subvolumes
g.log.info("Going to bring down the brick process "
"for %s", subvolumes_second_brick_list)
ret = bring_bricks_offline(self.volname, subvolumes_second_brick_list)
self.assertTrue(ret, ("Failed to bring down the bricks. Please "
"check the log file for more details."))
g.log.info("Brought down the brick process "
"for %s successfully", subvolumes_second_brick_list)
# start I/0 ( write and read ) - read must pass, write will fail
g.log.info("Start creating files on mounts.....")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name sixth_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with Read Only File System")
ret, _ = is_io_procs_fail_with_rofs(self, all_mounts_procs,
self.mounts)
self.assertTrue(ret, ("Unexpected Error and IO successful"
" on Read-Only File System"))
g.log.info("EXPECTED Read-only file system in IO while creating file")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# set the cluster.quorum-count to 1
options = {"cluster.quorum-count": "1"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on mount.....")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name seventh_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# set cluster.quorum-type to auto and cluster.quorum-count back to 2
options = {"cluster.quorum-type": "auto", "cluster.quorum-count": "2"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on mount.....")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name eigth_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# Bring back brick2 online for all the subvolumes
g.log.info("bringing up the bricks : %s online",
subvolumes_second_brick_list)
ret = bring_bricks_online(self.mnode, self.volname,
subvolumes_second_brick_list)
self.assertTrue(ret, ("Failed to brought the brick %s online" %
subvolumes_second_brick_list))
g.log.info("Successfully brought the brick %s online",
subvolumes_second_brick_list)
# Bring down brick1 again for all the subvolumes
g.log.info("Going to bring down the brick process "
"for %s", subvolumes_first_brick_list)
ret = bring_bricks_offline(self.volname, subvolumes_first_brick_list)
self.assertTrue(ret, ("Failed to bring down the bricks. Please "
"check the log file for more details."))
g.log.info("Brought down the brick process "
"for %s successfully", subvolumes_first_brick_list)
# start I/0 ( write and read ) - read must pass, write will fail
g.log.info("Start creating files on mounts.....")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name ninth_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with Read Only File System")
ret, _ = is_io_procs_fail_with_rofs(self, all_mounts_procs,
self.mounts)
self.assertTrue(ret, ("Unexpected Error and IO successful"
" on Read-Only File System"))
g.log.info("EXPECTED Read-only file system in IO while creating file")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# set the quorum-type to none
options = {"cluster.quorum-type": "none"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on mount.....")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name tenth_file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# read the file
g.log.info("Start reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read "
"%s" % (self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# bring back the bricks online for all subvolumes
g.log.info("bringing up the brick : %s online",
subvolumes_first_brick_list)
ret = bring_bricks_online(self.mnode, self.volname,
subvolumes_first_brick_list)
self.assertTrue(ret, ("Failed to brought the brick %s online" %
subvolumes_first_brick_list))
g.log.info("Successfully brought the bricks")
def test_client_side_quorum_with_auto_option_cross2(self):
"""
Test Script to verify the Client Side Quorum with auto option
* set cluster.quorum-type to auto.
* start I/O from the mount point.
* kill 2-nd brick process from the each and every replica set
* perform ops
"""
# pylint: disable=too-many-branches,too-many-statements
# set cluster.quorum-type to auto
options = {"cluster.quorum-type": "auto"}
g.log.info("setting cluster.quorum-type to auto on "
"volume %s", self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set volume option %s for"
"volume %s" % (options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# Start IO on mounts
g.log.info("Starting IO .....")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# get the subvolumes
g.log.info("Starting to get sub-volumes for volume %s", self.volname)
subvols_dict = get_subvols(self.mnode, self.volname)
num_subvols = len(subvols_dict['volume_subvols'])
g.log.info("Number of subvolumes in volume %s:", num_subvols)
# bring 2-nd bricks offline for all the subvolumes
offline_bricks = []
for i in range(0, num_subvols):
subvol_brick_list = subvols_dict['volume_subvols'][i]
g.log.info("sub-volume %s brick list : %s", i, subvol_brick_list)
bricks_to_bring_offline = subvol_brick_list[1]
g.log.info("Going to bring down the brick process "
"for %s", bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, ("Failed to bring down the bricks. Please "
"check the log file for more details."))
g.log.info("Brought down the brick process "
"for %s successfully", bricks_to_bring_offline)
offline_bricks.append(bricks_to_bring_offline)
# create new file named newfile0.txt
g.log.info("Start creating new file on all mounts...")
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 1 --base-file-name newfile %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# create directory user1
g.log.info("Start creating directory on all mounts...")
all_mounts_procs = []
cmd = ("python %s create_deep_dir %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# create h/w link to file
g.log.info("Start creating hard link for file0.txt on mount")
cmd = ("ln %s/file0.txt %s/file0.txt_hwlink" %
(self.mounts[0].mountpoint, self.mounts[0].mountpoint))
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertFalse(
ret, 'Failed to create hard link '
'for file0.txt on %s' % self.mounts[0].mountpoint)
g.log.info("Hard link for file0.txt on %s is created successfully",
self.mounts[0].mountpoint)
# create s/w link
g.log.info("Start creating soft link for file1.txt on mount")
cmd = ("ln -s %s/file1.txt %s/file1.txt_swlink" %
(self.mounts[0].mountpoint, self.mounts[0].mountpoint))
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertFalse(
ret, 'Failed to create soft link '
'for file1.txt on %s' % self.mounts[0].mountpoint)
g.log.info("Soft link for file1.txt on %s is created successfully",
self.mounts[0].mountpoint)
# append to file
g.log.info("Appending to file1.txt on all mounts")
for mount_obj in self.mounts:
cmd = ("cat %s/file0.txt >> %s/file1.txt" %
(mount_obj.mountpoint, mount_obj.mountpoint))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(
ret, 'Failed to append file1.txt on %s' % mount_obj.mountpoint)
g.log.info("Appending for file1.txt on %s is successful",
mount_obj.mountpoint)
# modify the file
g.log.info("Modifying file1.txt on all mounts")
for mount_obj in self.mounts:
cmd = ("echo 'Modify Contents' > %s/file1.txt" %
mount_obj.mountpoint)
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(
ret, 'Failed to modify file1.txt on %s' % mount_obj.mountpoint)
g.log.info("Modifying for file1.txt on %s is successful",
mount_obj.mountpoint)
# truncate the file
g.log.info("Truncating file1.txt on all mounts")
for mount_obj in self.mounts:
cmd = "truncate -s 0 %s/file1.txt" % mount_obj.mountpoint
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(
ret,
'Failed to truncate file1.txt on %s' % mount_obj.mountpoint)
g.log.info("Truncating for file1.txt on %s is successful",
mount_obj.mountpoint)
# read the file
g.log.info("Starting reading files on all mounts")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s read %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# stat on file
g.log.info("stat on file1.txt on all mounts")
for mount_obj in self.mounts:
cmd = "stat %s/file1.txt" % mount_obj.mountpoint
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(
ret, 'Failed to stat file1.txt on %s' % mount_obj.mountpoint)
g.log.info("Stat for file1.txt on %s is successful",
mount_obj.mountpoint)
# stat on dir
g.log.info("stat on directory on all mounts")
for mount_obj in self.mounts:
cmd = ("python %s stat %s" %
(self.script_upload_path, mount_obj.mountpoint))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(
ret, 'Failed to stat directory on %s' % mount_obj.mountpoint)
g.log.info("Stat for directory on %s is successful",
mount_obj.mountpoint)
# ls on mount point
g.log.info("ls on mount point on all mounts")
for mount_obj in self.mounts:
cmd = ("python %s ls %s" %
(self.script_upload_path, mount_obj.mountpoint))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, 'Failed to ls on %s' % mount_obj.mountpoint)
g.log.info("ls for %s is successful", mount_obj.mountpoint)
# bring back the bricks online for all subvolumes
g.log.info("bringing up the brick : %s online", offline_bricks)
ret = bring_bricks_online(self.mnode, self.volname, offline_bricks)
self.assertTrue(
ret, ("Failed to brought the brick %s online" % offline_bricks))
g.log.info("Successfully brought the bricks")
def test_client_side_quorum_auto_local_to_volume_not_cluster(self):
"""
- create four volume as below
vol1->2x2
vol2->2x2
vol3->2x3
vol4->2x3
vol5->a pure distribute volume
- do IO to all vols
- set client side quorum to auto for vol1 and vol3
- get the client side quorum value for all vols and check for result
- bring down b0 on vol1 and b0 and b1 on vol3
- try to create files on all vols and check for result
"""
# pylint: disable=too-many-locals,too-many-statements
# Creating files for all volumes
for mount_point in self.mount_points:
self.all_mounts_procs = []
g.log.info('Creating files...')
command = ("python %s create_files -f 50 "
"--fixed-file-size 1k %s" %
(self.script_upload_path, mount_point))
proc = g.run_async(self.mounts[0].client_system, command)
self.all_mounts_procs.append(proc)
self.io_validation_complete = False
# Validate IO
self.assertTrue(
validate_io_procs(self.all_mounts_procs, self.mounts),
"IO failed on some of the clients")
self.io_validation_complete = True
volumes_to_change_options = ['1', '3']
# set cluster.quorum-type to auto
for vol_number in volumes_to_change_options:
vol_name = ('testvol_distributed-replicated_%s' % vol_number)
options = {"cluster.quorum-type": "auto"}
g.log.info(
"setting cluster.quorum-type to auto on "
"volume testvol_distributed-replicated_%s", vol_number)
ret = set_volume_options(self.mnode, vol_name, options)
self.assertTrue(ret, ("Unable to set volume option %s for "
"volume %s" % (options, vol_name)))
g.log.info("Successfully set %s for volume %s", options, vol_name)
# check is options are set correctly
volume_list = get_volume_list(self.mnode)
for volume in volume_list:
g.log.info('Checking for cluster.quorum-type option for %s',
volume)
volume_options_dict = get_volume_options(self.mnode, volume,
'cluster.quorum-type')
if (volume == 'testvol_distributed-replicated_1'
or volume == 'testvol_distributed-replicated_3'
or volume == 'testvol_distributed-replicated_4'):
self.assertEqual(
volume_options_dict['cluster.quorum-type'], 'auto',
'Option cluster.quorum-type '
'is not AUTO for %s' % volume)
g.log.info('Option cluster.quorum-type is AUTO for %s', volume)
else:
self.assertEqual(
volume_options_dict['cluster.quorum-type'], 'none',
'Option cluster.quorum-type '
'is not NONE for %s' % volume)
g.log.info('Option cluster.quorum-type is NONE for %s', volume)
# Get first brick server and brick path
# and get first file from filelist then delete it from volume
vols_file_list = {}
for volume in volume_list:
brick_list = get_all_bricks(self.mnode, volume)
brick_server, brick_path = brick_list[0].split(':')
ret, file_list, _ = g.run(brick_server, 'ls %s' % brick_path)
self.assertFalse(ret, 'Failed to ls files on %s' % brick_server)
file_from_vol = file_list.splitlines()[0]
ret, _, _ = g.run(brick_server,
'rm -rf %s/%s' % (brick_path, file_from_vol))
self.assertFalse(ret, 'Failed to rm file on %s' % brick_server)
vols_file_list[volume] = file_from_vol
# bring bricks offline
# bring first brick for testvol_distributed-replicated_1
volname = 'testvol_distributed-replicated_1'
brick_list = get_all_bricks(self.mnode, volname)
bricks_to_bring_offline = brick_list[0:1]
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, volname, bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# bring first two bricks for testvol_distributed-replicated_3
volname = 'testvol_distributed-replicated_3'
brick_list = get_all_bricks(self.mnode, volname)
bricks_to_bring_offline = brick_list[0:2]
g.log.info('Bringing bricks %s offline...', bricks_to_bring_offline)
ret = bring_bricks_offline(volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks %s offline' % bricks_to_bring_offline)
ret = are_bricks_offline(self.mnode, volname, bricks_to_bring_offline)
self.assertTrue(ret,
'Bricks %s are not offline' % bricks_to_bring_offline)
g.log.info('Bringing bricks %s offline is successful',
bricks_to_bring_offline)
# merge two dicts (volname: file_to_delete) and (volname: mountpoint)
temp_dict = [vols_file_list, self.mount_points_and_volnames]
file_to_delete_to_mountpoint_dict = {}
for k in vols_file_list.iterkeys():
file_to_delete_to_mountpoint_dict[k] = (tuple(
file_to_delete_to_mountpoint_dict[k]
for file_to_delete_to_mountpoint_dict in temp_dict))
# create files on all volumes and check for result
for volname, file_and_mountpoint in \
file_to_delete_to_mountpoint_dict.iteritems():
filename, mountpoint = file_and_mountpoint
# check for ROFS error for read-only file system for
# testvol_distributed-replicated_1 and
# testvol_distributed-replicated_3
if (volname == 'testvol_distributed-replicated_1'
or volname == 'testvol_distributed-replicated_3'):
# create new file taken from vols_file_list
g.log.info("Start creating new file on all mounts...")
all_mounts_procs = []
cmd = ("touch %s/%s" % (mountpoint, filename))
proc = g.run_async(self.client, cmd)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating if IO failed with read-only filesystem")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts,
self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on read-only filesystem"))
g.log.info("EXPECTED: "
"Read-only file system in IO while creating file")
# check for no errors for all the rest volumes
else:
# create new file taken from vols_file_list
g.log.info("Start creating new file on all mounts...")
all_mounts_procs = []
cmd = ("touch %s/%s" % (mountpoint, filename))
proc = g.run_async(self.client, cmd)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
def test_client_side_quorum_with_auto_option(self):
"""
Test Script to verify the Client Side Quorum with auto option
* set cluster.quorum-type to auto.
* start I/O from the mount point.
* kill 2 of the brick process from the each and every replica set
* perform ops
"""
# pylint: disable=too-many-branches,too-many-statements
# set cluster.quorum-type to auto
options = {"cluster.quorum-type": "auto"}
g.log.info("setting cluster.quorum-type to auto on "
"volume %s", self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set volume option %s for"
"volume %s" % (options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# write files on all mounts
g.log.info("Starting IO on all mounts...")
g.log.info("mounts: %s", self.mounts)
cmd = ("python %s create_files "
"-f 10 --base-file-name file %s" % (self.script_upload_path,
self.mounts[0].mountpoint))
ret, _, err = g.run(self.mounts[0].client_system, cmd)
self.assertFalse(ret, "IO failed on %s with %s"
% (self.mounts[0].client_system, err))
# get the subvolumes
g.log.info("Starting to get sub-volumes for volume %s", self.volname)
subvols_dict = get_subvols(self.mnode, self.volname)
num_subvols = len(subvols_dict['volume_subvols'])
g.log.info("Number of subvolumes in volume %s:", num_subvols)
# bring bricks offline( 2 bricks ) for all the subvolumes
for i in range(0, num_subvols):
subvol_brick_list = subvols_dict['volume_subvols'][i]
g.log.info("sub-volume %s brick list : %s", i, subvol_brick_list)
# For volume type: 1 * 2, bring 1 brick offline
if len(subvol_brick_list) == 2:
bricks_to_bring_offline = subvol_brick_list[0:1]
else:
bricks_to_bring_offline = subvol_brick_list[0:2]
g.log.info("Going to bring down the brick process "
"for %s", bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(ret, ("Failed to bring down the bricks. Please "
"check the log file for more details."))
g.log.info("Brought down the brick process "
"for %s successfully", bricks_to_bring_offline)
# create a file test_file
# cannot use python module here since we need the stderr output
all_mounts_procs = []
g.log.info("Start creating 2 files on mountpoint...")
cmd = ("dd if=/dev/urandom of=%s/test_file bs=1M count=1"
% self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while creating files")
# create directory user1
all_mounts_procs = []
g.log.info("Start creating directory...")
cmd = ("mkdir %s/user1 " % self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while creating folder")
# create h/w link to file
all_mounts_procs = []
g.log.info("Start creating hard link for file0.txt")
cmd = ("ln %s/file0.txt %s/file0.txt_hwlink"
% (self.mounts[0].mountpoint, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while creating hardlink")
# create s/w link
all_mounts_procs = []
g.log.info("Start creating soft link for file1.txt")
cmd = ("ln -s %s/file1.txt %s/file1.txt_swlink"
% (self.mounts[0].mountpoint, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while creating softlink")
# append to file
all_mounts_procs = []
g.log.info("Appending to file1.txt")
cmd = ("cat %s/file0.txt >> %s/file1.txt"
% (self.mounts[0].mountpoint, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while appending to file")
# modify the file
all_mounts_procs = []
g.log.info("Modifying file1.txt")
cmd = ("echo 'Modify Contents' > %s/file1.txt"
% self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while modifying file")
# truncate the file
all_mounts_procs = []
g.log.info("Truncating file1.txt")
cmd = "truncate -s 0 %s/file1.txt" % self.mounts[0].mountpoint
proc = g.run_async(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while truncating file")
# read the file
all_mounts_procs = []
g.log.info("Starting reading file")
cmd = ("cat %s/file1.txt" % self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while reading file")
# stat on file
all_mounts_procs = []
g.log.info("stat on file1.txt")
cmd = "stat %s/file1.txt" % self.mounts[0].mountpoint
proc = g.run_async(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while stat file")
# stat on dir
all_mounts_procs = []
g.log.info("stat on %s", self.mounts[0].mountpoint)
cmd = "stat %s" % self.mounts[0].mountpoint
proc = g.run_async(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while stat dir")
# ls on mount point
all_mounts_procs = []
g.log.info("ls on %s", self.mounts[0].mountpoint)
cmd = "ls %s" % self.mounts[0].mountpoint
proc = g.run_async(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while ls mountpoint")
def perform_test(self, ctime):
"""
Testcase steps:
1. Enable/disable features,ctime based on function argument.
2. Create a directory on the mount point.
3. Kill a brick and create a file inside the directory.
4. Bring the brick online.
5. Trigger heal and wait for its completion.
6. Verify that the atime, mtime and ctime of the directory are same on
all bricks of the replica.
"""
if ctime:
option = {'features.ctime': 'on'}
else:
option = {'features.ctime': 'off'}
ret = set_volume_options(self.mnode, self.volname, option)
self.assertTrue(
ret, 'failed to set option %s on %s' % (option, self.volume))
client, m_point = (self.mounts[0].client_system,
self.mounts[0].mountpoint)
dirpath = '{}/dir1'.format(m_point)
ret = mkdir(client, dirpath)
self.assertTrue(ret, 'Unable to create a directory from mount point')
bricks_to_bring_offline = select_volume_bricks_to_bring_offline(
self.mnode, self.volname)
self.assertIsNotNone(bricks_to_bring_offline, "List is empty")
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, 'Failed to bring bricks {} offline'.format(
bricks_to_bring_offline))
ret = are_bricks_offline(self.mnode, self.volname,
bricks_to_bring_offline)
self.assertTrue(
ret, 'Bricks {} are not offline'.format(bricks_to_bring_offline))
cmd = 'touch {}/file1'.format(dirpath)
ret, _, _ = g.run(client, cmd)
self.assertEqual(ret, 0, 'Unable to create file from mount point')
ret = bring_bricks_online(
self.mnode,
self.volname,
bricks_to_bring_offline,
bring_bricks_online_methods=['volume_start_force'])
self.assertTrue(
ret,
'Failed to bring bricks {} online'.format(bricks_to_bring_offline))
ret = trigger_heal(self.mnode, self.volname)
self.assertTrue(ret, 'Starting heal failed')
ret = monitor_heal_completion(self.mnode, self.volname)
self.assertTrue(ret, 'Heal has not yet completed')
if ctime:
ret = self.are_mdata_xattrs_equal()
self.assertTrue(ret,
"glusterfs.mdata mismatch for {}".format(dirpath))
else:
ret = self.are_stat_timestamps_equal()
self.assertTrue(ret, "stat mismatch for {}".format(dirpath))
ret = rmdir(client, dirpath, force=True)
self.assertTrue(ret, 'Unable to delete directory from mount point')
def test_client_side_quorum_with_fixed_for_cross3(self):
"""
Test Script to verify the Client Side Quorum with fixed
for cross 3 volume
* Disable self heal daemom
* set cluster.quorum-type to fixed.
* start I/O( write and read )from the mount point - must succeed
* Bring down brick1
* start I/0 ( write and read ) - must succeed
* Bring down brick2
* start I/0 ( write and read ) - must succeed
* set the cluster.quorum-count to 1
* start I/0 ( write and read ) - must succeed
* set the cluster.quorum-count to 2
* start I/0 ( write and read ) - read and write will fail
* bring back the brick1 online
* start I/0 ( write and read ) - must succeed
* Bring back brick2 online
* start I/0 ( write and read ) - must succeed
* set cluster.quorum-type to auto
* start I/0 ( write and read ) - must succeed
* Bring down brick1 and brick2
* start I/0 ( write and read ) - read and write will fail
* set the cluster.quorum-count to 1
* start I/0 ( write and read ) - read and write will fail
* set the cluster.quorum-count to 3
* start I/0 ( write and read ) - read and write will fail
* set the quorum-type to none
* start I/0 ( write and read ) - must succeed
"""
# pylint: disable=too-many-locals,too-many-statements,too-many-branches
# Disable self heal daemon
options = {"cluster.self-heal-daemon": "off"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# set cluster.quorum-type to fixed
options = {"cluster.quorum-type": "fixed"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/O( write ) - must succeed
all_mounts_procs = []
g.log.info("Starting IO on mountpoint %s", self.mounts[0].mountpoint)
cmd = ("python %s create_files "
"-f 10 --base-file-name file %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on mountpoint %s" % self.mounts[0].mountpoint)
# read the file
g.log.info("Start reading files on %s", self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("python %s read %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on some of the clients")
# get the subvolumes
g.log.info("Starting to get sub-volumes for volume %s", self.volname)
subvols_dict = get_subvols(self.mnode, self.volname)
num_subvols = len(subvols_dict['volume_subvols'])
g.log.info("Number of subvolumes in volume %s:", num_subvols)
# bring down brick1 for all the subvolumes
offline_brick1_from_replicasets = []
for i in range(0, num_subvols):
subvol_brick_list = subvols_dict['volume_subvols'][i]
g.log.info("sub-volume %s brick list : %s", i, subvol_brick_list)
brick_to_bring_offline1 = subvol_brick_list[0]
g.log.info("Going to bring down the brick process "
"for %s", brick_to_bring_offline1)
ret = bring_bricks_offline(self.volname, brick_to_bring_offline1)
self.assertTrue(ret, ("Failed to bring down the bricks. Please "
"check the log file for more details."))
g.log.info("Brought down the brick process "
"for %s successfully", brick_to_bring_offline1)
offline_brick1_from_replicasets.append(brick_to_bring_offline1)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on mountpoint %s", self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name testfile %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on mountpoint %s" % self.mounts[0].mountpoint)
# read the file
g.log.info("Start reading files on mountpoint %s",
self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("python %s read %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on mountpoint %s" % self.mounts[0].mountpoint)
# bring down brick2 for all the subvolumes
offline_brick2_from_replicasets = []
for i in range(0, num_subvols):
subvol_brick_list = subvols_dict['volume_subvols'][i]
g.log.info("sub-volume %s brick list : %s", i, subvol_brick_list)
brick_to_bring_offline2 = subvol_brick_list[1]
g.log.info("Going to bring down the brick process "
"for %s", brick_to_bring_offline2)
ret = bring_bricks_offline(self.volname, brick_to_bring_offline2)
self.assertTrue(ret, ("Failed to bring down the bricks. Please "
"check the log file for more details."))
g.log.info("Brought down the brick process "
"for %s successfully", brick_to_bring_offline2)
offline_brick2_from_replicasets.append(brick_to_bring_offline2)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on mountpoint %s", self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name newfile %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on mountpoint %s" % self.mounts[0].mountpoint)
# read the file
g.log.info("Start reading files on mountpoint %s",
self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("python %s read %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on mountpoint %s" % self.mounts[0].mountpoint)
# set the cluster.quorum-count to 1
options = {"cluster.quorum-count": "1"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(
ret, "Unable to set %s for volume %s" % (options, self.volname))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on mountpoint %s", self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name filename %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on mountpoint %s" % self.mounts[0].mountpoint)
# read the file
g.log.info("Start reading files on mountpoint %s",
self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("python %s read %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on mountpoint %s" % self.mounts[0].mountpoint)
# set the cluster.quorum-count to 2
options = {"cluster.quorum-count": "2"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/0 ( write and read ) - read and write will fail
g.log.info("Starting IO on mountpoint %s", self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("dd if=/dev/urandom of=%s/test_file bs=1M count=1" %
self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected Error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while creating file")
# read the file
g.log.info("Start reading files on mountpoint %s",
self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("cat %s/file1.txt" % self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while reading file")
# bring back the brick1 online for all subvolumes
g.log.info("bringing up the brick : %s online",
offline_brick1_from_replicasets)
ret = bring_bricks_online(
self.mnode,
self.volname,
offline_brick1_from_replicasets,
bring_bricks_online_methods='glusterd_restart')
self.assertTrue(ret, ("Failed to brought the brick %s online" %
offline_brick1_from_replicasets))
g.log.info("Successfully brought the brick %s online",
offline_brick1_from_replicasets)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on mountpoint %s", self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name newfilename %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on mountpoint %s" % self.mounts[0].mountpoint)
# read the file
g.log.info("Start reading files on mountpoint %s",
self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("python %s read %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on mountpoint %s" % self.mounts[0].mountpoint)
# Bring back brick2 online
g.log.info("bringing up the brick : %s online",
offline_brick2_from_replicasets)
ret = bring_bricks_online(
self.mnode,
self.volname,
offline_brick2_from_replicasets,
bring_bricks_online_methods='glusterd_restart')
self.assertTrue(ret, ("Failed to brought the brick %s online" %
offline_brick2_from_replicasets))
g.log.info("Successfully brought the brick %s online",
offline_brick2_from_replicasets)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on mountpoint %s", self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name textfile %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on mountpoint %s" % self.mounts[0].mountpoint)
# read the file
g.log.info("Start reading files on mountpoint %s",
self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("python %s read %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on mountpoint %s" % self.mounts[0].mountpoint)
# set cluster.quorum-type to auto
options = {"cluster.quorum-type": "auto"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(ret, ("Unable to set %s for volume %s" %
(options, self.volname)))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on mountpoint %s", self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name newtextfile %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on mountpoint %s" % self.mounts[0].mountpoint)
# read the file
g.log.info("Start reading files on mountpoint %s",
self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("python %s read %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on mountpoint %s" % self.mounts[0].mountpoint)
# bring down brick1 and brick2 for all the subvolumes
for i in range(0, num_subvols):
subvol_brick_list = subvols_dict['volume_subvols'][i]
g.log.info("sub-volume %s brick list : %s", i, subvol_brick_list)
bricks_to_bring_offline = subvol_brick_list[0:2]
g.log.info("Going to bring down the brick process for %s",
bricks_to_bring_offline)
ret = bring_bricks_offline(self.volname, bricks_to_bring_offline)
self.assertTrue(
ret, "Failed to bring down the bricks. Please "
"check the log file for more details.")
g.log.info("Brought down the brick process "
"for %s successfully", bricks_to_bring_offline)
# start I/0 ( write and read ) - read and write will fail
all_mounts_procs = []
g.log.info("Start creating file on mountpoint %s",
self.mounts[0].mountpoint)
cmd = ("dd if=/dev/urandom of=%s/new_test_file bs=1M count=1" %
self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while creating files")
# read the file
g.log.info("Start reading files on mountpoint %s",
self.mounts[0].mountpoint)
all_mounts_procs = []
g.log.info("Starting reading file")
cmd = ("cat %s/file1.txt" % self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while reading file")
# set the cluster.quorum-count to 1
options = {"cluster.quorum-count": "1"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(
ret, "Unable to set %s for volume %s" % (options, self.volname))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/0 ( write and read ) - read and write will fail
g.log.info("Start creating files on mountpoint %s",
self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("dd if=/dev/urandom of=%s/new_test_file bs=1M count=1" %
self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while creating files")
# read the file
g.log.info("Start reading files on mountpoint %s",
self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("cat %s/file1.txt" % self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while reading file")
# set the cluster.quorum-count to 3
options = {"cluster.quorum-count": "3"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(
ret, "Unable to set %s for volume %s" % (options, self.volname))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/0 ( write and read ) - read and write will fail
g.log.info("Start creating files on mountpoint %s",
self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("dd if=/dev/urandom of=%s/new_test_file bs=1M count=1" %
self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while creating files")
# read the file
g.log.info("Start reading files on mountpoint %s",
self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("cat %s/file1.txt" % self.mounts[0].mountpoint)
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
g.log.info("Validating whether IO failed with "
"Transport endpoint is not connected")
ret, _ = is_io_procs_fail_with_error(self, all_mounts_procs,
self.mounts, self.mount_type)
self.assertTrue(ret, ("Unexpected error and IO successful"
" on not connected transport endpoint"))
g.log.info("EXPECTED: Transport endpoint is not connected"
" while reading file")
# set the quorum-type to none
options = {"cluster.quorum-type": "none"}
g.log.info("setting %s for the volume %s", options, self.volname)
ret = set_volume_options(self.mnode, self.volname, options)
self.assertTrue(
ret, "Unable to set %s for volume %s" % (options, self.volname))
g.log.info("Successfully set %s for volume %s", options, self.volname)
# start I/0 ( write and read ) - must succeed
g.log.info("Starting IO on mountpoint %s", self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("python %s create_files "
"-f 10 --base-file-name lastfile %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on mountpoint %s" % self.mounts[0].mountpoint)
# read the file
g.log.info("Start reading files on mountpoint %s",
self.mounts[0].mountpoint)
all_mounts_procs = []
cmd = ("python %s read %s" %
(self.script_upload_path, self.mounts[0].mountpoint))
proc = g.run_async(self.mounts[0].client_system,
cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"Reads failed on mountpoint %s" % self.mounts[0].mountpoint)
