def mkdir_post_hashdown(self, subvols, parent_dir):
'''
case -1:
- bring down a subvol
- create a directory so that it does not hash to down subvol
- make sure stat is successful on the dir
'''
# pylint: disable=protected-access
# pylint: disable=pointless-string-statement
# Find a non hashed subvolume(or brick)
nonhashed_subvol, count = find_nonhashed_subvol(subvols, "/", "parent")
if nonhashed_subvol is None:
g.log.error('Error in finding nonhashed subvol for parent')
return False
# bring nonhashed_subbvol offline
ret = bring_bricks_offline(self.volname, subvols[count])
if ret == 0:
g.log.error('Error in bringing down subvolume %s', subvols[count])
return False
g.log.info('target subvol %s is offline', subvols[count])
# create parent dir
ret, _, err = g.run(self.clients[0], ("mkdir %s" % parent_dir))
if ret != 0:
g.log.error('mkdir failed for %s err: %s', parent_dir, err)
return False
g.log.info("mkdir of parent directory %s successful", parent_dir)
# this confirms both layout and stat of the directory
ret = validate_files_in_dir(self.clients[0],
self.mounts[0].mountpoint + '/parent_dir',
test_type=LAYOUT_IS_COMPLETE,
file_type=FILETYPE_DIRS)
self.assertTrue(ret, "Layout is not complete")
g.log.info('Layout is complete')
# bring up the subvol
ret = bring_bricks_online(self.mnode,
self.volname,
subvols[count],
bring_bricks_online_methods=None)
if ret == 0:
g.log.error("Error in bringing back subvol online")
return False
g.log.info('Subvol is back online')
# delete parent_dir
ret, _, err = g.run(self.clients[0], ("rmdir %s" % parent_dir))
if ret != 0:
g.log.error('rmdir failed for %s err: %s', parent_dir, err)
g.log.info("rmdir of directory %s successful", parent_dir)
return True
def test_rmdir_child_when_nonhash_vol_down(self):
"""
case -1:
- create parent
- bring down a non-hashed subvolume for directory child
- create parent/child
- rmdir /mnt/parent will fail with ENOTCONN
"""
# pylint: disable=protected-access
# pylint: disable=too-many-statements
# pylint: disable=unsubscriptable-object
# Find a non hashed subvolume(or brick)
# Create parent dir
parent_dir = self.mountpoint + '/parent'
child_dir = parent_dir + '/child'
ret = mkdir(self.clients[0], parent_dir)
self.assertTrue(ret, "mkdir failed")
g.log.info("mkdir of parent directory %s successful", parent_dir)
# Find a non hashed subvolume(or brick)
nonhashed_subvol, count = find_nonhashed_subvol(self.subvols,
"parent", "child")
self.assertIsNotNone(nonhashed_subvol,
"Error in finding nonhashed value")
g.log.info("nonhashed_subvol %s", nonhashed_subvol._host)
# Bring nonhashed_subbvol offline
ret = bring_bricks_offline(self.volname, self.subvols[count])
self.assertTrue(ret, ("Error in bringing down subvolume %s"
% self.subvols[count]))
g.log.info('target subvol %s is offline', self.subvols[count])
# Create child-dir
ret = mkdir(self.clients[0], child_dir)
self.assertTrue(ret, ('mkdir failed for %s ' % child_dir))
g.log.info("mkdir of child directory %s successful", child_dir)
# 'rmdir' on parent should fail with ENOTCONN
ret = rmdir(self.clients[0], parent_dir)
self.assertFalse(ret, ('Expected rmdir to fail for %s' % parent_dir))
g.log.info("rmdir of parent directory %s failed as expected",
parent_dir)
# Cleanup
# Bring up the subvol - restart volume
ret = volume_start(self.mnode, self.volname, force=True)
self.assertTrue(ret, "Error in force start the volume")
g.log.info('Volume restart success')
sleep(10)
# Delete parent_dir
ret = rmdir(self.clients[0], parent_dir, force=True)
self.assertTrue(ret, ('rmdir failed for %s ' % parent_dir))
g.log.info("rmdir of directory %s successful", parent_dir)
def test_rmdir_parent_pre_nonhash_vol_down(self):
"""
case -4:
- Bring down a non-hashed subvol for parent_dir
- mkdir parent
- rmdir parent should fails with ENOTCONN
"""
# pylint: disable=protected-access
# pylint: disable=too-many-statements
# pylint: disable=unsubscriptable-object
nonhashed_subvol, count = find_nonhashed_subvol(self.subvols,
"/", "parent")
self.assertIsNotNone(nonhashed_subvol,
'Error in finding nonhashed subvol')
g.log.info("nonhashed subvol %s", nonhashed_subvol._host)
# Bring nonhashed_subbvol offline
ret = bring_bricks_offline(self.volname, self.subvols[count])
self.assertTrue(ret, ('Error in bringing down subvolume %s'
% self.subvols[count]))
g.log.info('target subvol %s is offline', self.subvols[count])
parent_dir = self.mountpoint + '/parent'
ret = mkdir(self.clients[0], parent_dir)
self.assertTrue(ret, ('mkdir failed for %s ' % parent_dir))
g.log.info("mkdir of parent directory %s successful", parent_dir)
# 'rmdir' on parent should fail with ENOTCONN
ret = rmdir(self.clients[0], parent_dir)
self.assertFalse(ret, ('Expected rmdir to fail for %s' % parent_dir))
g.log.info("rmdir of parent directory %s failed as expected",
parent_dir)
# Cleanup
# Bring up the subvol - restart volume
ret = volume_start(self.mnode, self.volname, force=True)
self.assertTrue(ret, "Error in force start the volume")
g.log.info('Volume restart success.')
sleep(10)
# Delete parent_dir
ret = rmdir(self.clients[0], parent_dir, force=True)
self.assertTrue(ret, ('rmdir failed for %s ' % parent_dir))
g.log.info("rmdir of directory %s successful", parent_dir)
def test_rm_file_when_nonhash_vol_down(self):
"""
case -3:
- create parent
- mkdir parent/child
- touch parent/child/file
- bringdown a subvol where file is not present
- rm -rf parent
- Only file should be deleted
- rm -rf of parent should fail with ENOTCONN
"""
# pylint: disable=protected-access
# pylint: disable=too-many-statements
# pylint: disable=unsubscriptable-object
# Find a non hashed subvolume(or brick)
# Create parent dir
parent_dir = self.mountpoint + '/parent'
child_dir = parent_dir + '/child'
ret = mkdir(self.clients[0], parent_dir)
self.assertTrue(ret, ('mkdir failed for %s ' % parent_dir))
g.log.info("mkdir of parent directory %s successful", parent_dir)
# Create child dir
ret = mkdir(self.clients[0], child_dir)
self.assertTrue(ret, ('mkdir failed for %s ' % child_dir))
g.log.info("mkdir of child directory %s successful", child_dir)
# Create a file under child_dir
file_one = child_dir + '/file_one'
ret, _, err = g.run(self.clients[0], ("touch %s" % file_one))
self.assertFalse(ret, ('touch failed for %s err: %s' %
(file_one, err)))
# Find a non hashed subvolume(or brick)
nonhashed_subvol, count = find_nonhashed_subvol(self.subvols,
"parent/child",
"file_one")
self.assertIsNotNone(nonhashed_subvol,
"Error in finding nonhashed value")
g.log.info("nonhashed_subvol %s", nonhashed_subvol._host)
# Bring nonhashed_subbvol offline
ret = bring_bricks_offline(self.volname, self.subvols[count])
self.assertTrue(ret, ('Error in bringing down subvolume %s'
% self.subvols[count]))
g.log.info('target subvol %s is offline', self.subvols[count])
# 'rm -rf' on parent should fail with ENOTCONN
ret = rmdir(self.clients[0], parent_dir)
self.assertFalse(ret, ('Expected rmdir to fail for %s' % parent_dir))
g.log.info("rmdir of parent directory %s failed as expected"
" with err %s", parent_dir, err)
brickobject = create_brickobjectlist(self.subvols, "parent/child")
self.assertIsNotNone(brickobject,
"could not create brickobject list")
# Make sure file_one is deleted
for brickdir in brickobject:
dir_path = "%s/parent/child/file_one" % brickdir.path
brick_path = dir_path.split(":")
self.assertTrue((file_exists(brickdir._host, brick_path[1])) == 0,
('Expected file %s not to exist on servers'
% parent_dir))
g.log.info("file is deleted as expected")
# Cleanup
# Bring up the subvol - restart volume
ret = volume_start(self.mnode, self.volname, force=True)
self.assertTrue(ret, "Error in force start the volume")
g.log.info('Volume restart success.')
sleep(10)
# Delete parent_dir
ret = rmdir(self.clients[0], parent_dir, force=True)
self.assertTrue(ret, ('rmdir failed for %s ' % parent_dir))
g.log.info("rmdir of directory %s successful", parent_dir)
def test_rmdir_dir_when_hash_nonhash_vol_down(self):
"""
case -2:
- create dir1 and dir2
- bring down hashed subvol for dir1
- bring down a non-hashed subvol for dir2
- rmdir dir1 should fail with ENOTCONN
- rmdir dir2 should fail with ENOTCONN
"""
# pylint: disable=protected-access
# pylint: disable=too-many-statements
# pylint: disable=unsubscriptable-object
# Create dir1 and dir2
directory_list = []
for number in range(1, 3):
directory_list.append('{}/dir{}'.format(self.mountpoint, number))
ret = mkdir(self.clients[0], directory_list[-1])
self.assertTrue(ret, ('mkdir failed for %s '
% directory_list[-1]))
g.log.info("mkdir of directory %s successful",
directory_list[-1])
# Find a non hashed subvolume(or brick)
nonhashed_subvol, count = find_nonhashed_subvol(self.subvols, "/",
"dir1")
self.assertIsNotNone(nonhashed_subvol,
"Error in finding nonhashed value")
g.log.info("nonhashed_subvol %s", nonhashed_subvol._host)
# Bring nonhashed_subbvol offline
ret = bring_bricks_offline(self.volname, self.subvols[count])
self.assertTrue(ret, ('Error in bringing down subvolume %s'
% self.subvols[count]))
g.log.info('target subvol %s is offline', self.subvols[count])
# 'rmdir' on dir1 should fail with ENOTCONN
ret = rmdir(self.clients[0], directory_list[0])
self.assertFalse(ret, ('Expected rmdir to fail for %s'
% directory_list[0]))
g.log.info("rmdir of directory %s failed as expected",
directory_list[0])
# Bring up the subvol - restart volume
ret = volume_start(self.mnode, self.volname, force=True)
self.assertTrue(ret, "Error in force start the volume")
g.log.info('Volume restart success')
sleep(10)
# Unmounting and Mounting the volume back to Heal
ret, _, err = umount_volume(self.clients[1], self.mountpoint)
self.assertFalse(ret, "Error in creating temp mount %s" % err)
ret, _, err = mount_volume(self.volname,
mtype='glusterfs',
mpoint=self.mountpoint,
mserver=self.servers[0],
mclient=self.clients[1])
self.assertFalse(ret, "Error in creating temp mount")
ret, _, _ = g.run(self.clients[1], ("ls %s/dir1" % self.mountpoint))
self.assertEqual(ret, 0, "Error in lookup for dir1")
g.log.info("lookup successful for dir1")
# This confirms that healing is done on dir1
ret = validate_files_in_dir(self.clients[0],
directory_list[0],
test_type=LAYOUT_IS_COMPLETE,
file_type=FILETYPE_DIRS)
self.assertTrue(ret, "validate_files_in_dir for dir1 failed")
g.log.info("healing successful for dir1")
# Bring down the hashed subvol
# Find a hashed subvolume(or brick)
hashed_subvol, count = find_hashed_subvol(self.subvols, "/", "dir2")
self.assertIsNotNone(hashed_subvol,
"Error in finding nonhashed value")
g.log.info("hashed_subvol %s", hashed_subvol._host)
# Bring hashed_subbvol offline
ret = bring_bricks_offline(self.volname, self.subvols[count])
self.assertTrue(ret, ('Error in bringing down subvolume %s',
self.subvols[count]))
g.log.info('target subvol %s is offline', self.subvols[count])
# 'rmdir' on dir2 should fail with ENOTCONN
ret = rmdir(self.clients[0], directory_list[1])
self.assertFalse(ret, ('Expected rmdir to fail for %s'
% directory_list[1]))
g.log.info("rmdir of dir2 directory %s failed as expected",
directory_list[1])
# Cleanup
# Bring up the subvol - restart the volume
ret = volume_start(self.mnode, self.volname, force=True)
self.assertTrue(ret, "Error in force start the volume")
g.log.info('Volume restart success')
sleep(10)
# Delete dirs
for directory in directory_list:
ret = rmdir(self.clients[0], directory)
self.assertTrue(ret, ('rmdir failed for %s ' % directory))
g.log.info("rmdir of directory %s successful", directory)
def test_directory_heal(self):
'''
Test directory healing.
'''
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=protected-access
mount_obj = self.mounts[0]
mountpoint = mount_obj.mountpoint
# directory that needs to be created
parent_dir = mountpoint + '/parent'
target_dir = mountpoint + '/parent/child'
# create parent dir
ret, _, err = g.run(self.clients[0], ("mkdir %s" % parent_dir))
g.log.info("mkdir of parent directory %s successful", parent_dir)
self.assertEqual(ret, 0, ('mkdir failed for %s err: %s', parent_dir,
err))
g.log.info("mkdir of parent successful")
# find non-hashed subvol for child
subvols = (get_subvols(self.mnode, self.volname))['volume_subvols']
non_hashed, count = find_nonhashed_subvol(subvols, "parent", "child")
self.assertIsNotNone(non_hashed, "could not find non_hashed subvol")
g.log.info("non_hashed subvol %s", non_hashed._host)
# bring non_hashed offline
ret = bring_bricks_offline(self.volname, subvols[count])
self.assertTrue(ret, ('Error in bringing down subvolume %s',
subvols[count]))
g.log.info('target subvol %s is offline', subvols[count])
# create child directory
runc = ("mkdir %s" % target_dir)
ret, _, _ = g.run(self.clients[0], runc)
self.assertEqual(ret, 0, ('failed to create dir %s' % target_dir))
g.log.info('mkdir successful %s', target_dir)
# Check that the dir is not created on the down brick
brickpath = ("%s/child" % non_hashed._path)
ret, _, _ = g.run(non_hashed._host, ("stat %s" % brickpath))
self.assertEqual(ret, 1, ("Expected %s to be not present on %s" %
(brickpath, non_hashed._host)))
g.log.info("stat of %s failed as expected", brickpath)
# bring up the subvol
ret = bring_bricks_online(self.mnode, self.volname, subvols[count],
bring_bricks_online_methods=None)
self.assertTrue(ret, "Error in bringing back subvol online")
g.log.info('Subvol is back online')
runc = ("ls %s" % target_dir)
ret, _, _ = g.run(self.clients[0], runc)
self.assertEqual(ret, 0, ("lookup on %s failed", target_dir))
g.log.info("lookup is successful on %s", target_dir)
# check if the directory is created on non_hashed
absolutedirpath = ("%s/child" % non_hashed._path)
# check if directory is healed => i.e. layout is zeroed out
temp = BrickDir(absolutedirpath)
if temp is None:
self.assertIsNot(temp, None, 'temp is None')
ret = temp.has_zero_hashrange()
self.assertTrue(ret, ("hash range is not there %s", ret))
g.log.info("directory healing successful")
