def test_heketi_volume_snapshot_delete(self):
"""Test heketi volume snapshot delete operation"""
h_volume_size = 1
snap_name = 'snap_test_heketi_volume_snapshot_create_1'
h_node, h_url = self.heketi_client_node, self.heketi_server_url
h_volume_info = heketi_volume_create(h_node,
h_url,
h_volume_size,
json=True)
self.addCleanup(heketi_volume_delete, h_node, h_url,
h_volume_info["id"])
# Get the snapshot list before snap creation
snap_list_before = get_snap_list('auto_get_gluster_endpoint')
self.assertIsNotNone(
snap_list_before,
"Failed to get the snapshot list {}".format(snap_list_before))
# Create a snapshot
h_volume_name = h_volume_info["name"]
ret, _, err = snap_create('auto_get_gluster_endpoint',
h_volume_name,
snap_name,
timestamp=False)
self.addCleanup(podcmd.GlustoPod()(snap_delete),
"auto_get_gluster_endpoint", snap_name)
self.assertFalse(
ret, "Failed to create snapshot {} for heketi volume {} with"
" error {}".format(snap_name, h_volume_name, err))
snap_list = get_snap_list('auto_get_gluster_endpoint')
self.assertIsNotNone(
snap_list, "Failed to get the snapshot list {}".format(snap_list))
self.assertIn(
snap_name, snap_list,
"Heketi volume snapshot {} not found in {}".format(
snap_name, snap_list))
# Delete the snapshot
ret, _, err = snap_delete('auto_get_gluster_endpoint', snap_name)
self.assertFalse(
ret, "Failed to delete snapshot {} for heketi volume with err {}".
format(snap_name, err))
# Check for count after snapshot deletion
snap_list_after = get_snap_list('auto_get_gluster_endpoint')
self.assertIsNotNone(
snap_list_after,
"Failed to get the snapshot list {}".format(snap_list_after))
self.assertEqual(
snap_list_before, snap_list_after,
"Expecting Snapshot count before {} and after creation {} to be "
"same".format(snap_list_before, snap_list_after))
def test_snap_delete_snap_volume(self):
"""
Steps:
1. create a volume and mount it
2. create 9 snapshots of volume
3. delete snapshot with snapname
4. delete other snapshots with volume name
5. create one more snapshot
6. delete created snapshot with snap delete command
"""
# Creating snapshot
g.log.info("Starting to Create snapshot")
for snap_count in range(0, 9):
self.snap = "snap%s" % snap_count
ret, _, _ = snap_create(self.mnode, self.volname, self.snap)
self.assertEqual(ret, 0,
("Failed to create snapshot %s "
"for volume %s" % (self.snap, self.volname)))
g.log.info("Snapshot %s created successfully"
" for volume %s", self.snap, self.volname)
# deleting snapshot with snap name
g.log.info("Starting to delete snapshot with snap name")
for snap_count in range(0, 3):
self.snap = "snap%s" % snap_count
ret, _, _ = snap_delete(self.mnode, self.snap)
self.assertEqual(ret, 0,
("Failed to delete snapshot snap%s" % snap_count))
g.log.info("Snapshot snap%s deleted " "successfully", snap_count)
# delete all snapshot of volume
g.log.info("Starting to delete snapshot with volume name")
ret, _, _ = snap_delete_by_volumename(self.mnode, self.volname)
self.assertEqual(ret, 0,
("Failed to delete snapshot snap%s" % snap_count))
g.log.info("Snapshot deleted successfully for volume "
"%s", self.volname)
# create a new snapshot
g.log.info("Creating a new snapshot")
self.snap1 = "snapy"
ret, _, _ = snap_create(self.mnode, self.volname, self.snap1)
self.assertEqual(ret, 0, ("Failed to create snapshot %s for volume "
"%s" % (self.snap1, self.volname)))
g.log.info("Snapshot %s created successfully"
" for volume %s", self.snap1, self.volname)
# delete all snapshot created
g.log.info("Deleting all snapshots created")
ret, _, _ = snap_delete_all(self.mnode)
self.assertEqual(ret, 0, "Failed to delete snapshots")
g.log.info("All Snapshots deleted successfully")
def test_volume_create_snapshot_enabled(self):
"""Validate volume creation with snapshot enabled"""
factor = 1.5
vol_create_info = heketi_ops.heketi_volume_create(
self.heketi_client_node,
self.heketi_server_url,
1,
snapshot_factor=factor,
json=True)
self.addCleanup(heketi_ops.heketi_volume_delete,
self.heketi_client_node, self.heketi_server_url,
vol_create_info["id"])
snap_factor_count = vol_create_info["snapshot"]["factor"]
self.assertEqual(
snap_factor_count, factor, "snapshot factor %s is not same as %s" %
(snap_factor_count, factor))
vol_name, snap_name = vol_create_info["name"], "snap1"
try:
ret, out, err = snap_ops.snap_create('auto_get_gluster_endpoint',
vol_name,
snap_name,
timestamp=False)
self.assertEqual(ret, 0,
"Failed to create snapshot %s" % snap_name)
# Get gluster volume info
gluster_vol = volume_ops.get_volume_info(
'auto_get_gluster_endpoint', volname=vol_name)
self.assertTrue(gluster_vol,
"Failed to get volume '%s' info" % vol_name)
self.assertEqual(
gluster_vol[vol_name]['snapshotCount'], "1",
"Failed to get snapshot count for volume %s" % vol_name)
finally:
ret, out, err = snap_ops.snap_delete('auto_get_gluster_endpoint',
snap_name)
self.assertEqual(ret, 0,
"Failed to delete snapshot %s" % snap_name)
def test_snap_list_glusterd_restart(self):
"""
Verify snapshot list before and after glusterd restart
* Create 3 snapshots of the volume
* Delete one snapshot
* List all snapshots created
* Restart glusterd on all nodes
* List all snapshots
All snapshots must be listed except the one that was deleted
"""
# pylint: disable=too-many-statements
# Create snapshots
for snap in self.snapshots:
ret, _, _ = snap_create(self.mnode, self.volname, snap)
self.assertEqual(ret, 0, ("Failed to create snapshot %s for "
"volume %s" % (snap, self.volname)))
g.log.info("Snapshot %s created successfully "
"for volume %s", snap, self.volname)
# List the snapshots and validate with snapname
snap_list = get_snap_list(self.mnode)
self.assertIsNotNone(snap_list, "Failed to list all snapshots")
self.assertEqual(len(snap_list), 3, "Failed to validate snap list")
g.log.info("Successfully validated snap list")
for snap in self.snapshots:
self.assertIn(
snap, snap_list, "Failed to validate the snapshot "
"%s in the snapshot list" % snap)
g.log.info("Successfully validated the presence of snapshots using "
"snapname")
# Delete one snapshot
ret, _, _ = snap_delete(self.mnode, self.snapshots[0])
self.assertEqual(ret, 0,
("Failed to delete snapshot %s" % self.snapshots[0]))
g.log.info("Snapshots %s deleted Successfully", self.snapshots[0])
# List the snapshots and validate with snapname
snap_list = get_snap_list(self.mnode)
self.assertIsNotNone(snap_list, "Failed to list all snapshots")
self.assertEqual(len(snap_list), 2, "Failed to validate snap list")
g.log.info("Successfully validated snap list")
for snap in self.snapshots[1:]:
self.assertIn(
snap, snap_list, "Failed to validate the snapshot "
"%s in the snapshot list" % snap)
g.log.info("Successfully validated the presence of snapshots using "
"snapname")
# Restart glusterd on all the servers
ret = restart_glusterd(self.servers)
self.assertTrue(
ret, ("Failed to restart glusterd on nodes %s" % self.servers))
g.log.info("Successfully restarted glusterd on nodes %s", self.servers)
# Wait for glusterd to be online and validate glusterd running on all
# server nodes
self.assertTrue(
wait_for_glusterd_to_start(self.servers),
"Unexpected: glusterd not up on one or more of the nodes")
g.log.info("Glusterd is up and running on all nodes")
# Check if peers are connected
self.assertTrue(is_peer_connected(self.mnode, self.servers),
"Unexpected: Peers are not in connected state")
g.log.info("Successful: All peers are in connected state")
# List the snapshots after glusterd restart
# All snapshots must be listed except the one deleted
for server in self.servers:
snap_list = get_snap_list(server)
self.assertIsNotNone(
snap_list,
"Failed to get the list of snapshots in node %s" % server)
self.assertEqual(
len(snap_list), 2,
"Unexpected: Number of snapshots not consistent in the node %s"
% server)
g.log.info("Successfully validated snap list for node %s", server)
for snap in self.snapshots[1:]:
self.assertIn(
snap, snap_list, "Failed to validate the snapshot "
"%s in the snapshot list" % snap)
g.log.info(
"Successfully validated the presence of snapshots "
"using snapname for node %s", server)
def test_snap_delete_and_list_glusterd_down(self):
# pylint: disable=too-many-statements
"""
Steps:
1. create a volume
2. mount volume
3. create 3 snapshot of that volume
4. delete snapshot snap1
5. list all snapshots created
6. restart glusterd
7. list all snapshots created
except snap1
"""
# Creating snapshot:
g.log.info("Starting to Create snapshot")
for snap_count in range(0, 3):
self.snap = "snap%s" % snap_count
ret, _, _ = snap_create(self.mnode, self.volname, self.snap)
self.assertEqual(ret, 0, ("Failed to create snapshot for "
"volume %s" % self.volname))
g.log.info("Snapshot %s created successfully "
"for volume %s", self.snap, self.volname)
# delete snap1 snapshot
g.log.info("Starting to Delete snapshot snap1")
ret, _, _ = snap_delete(self.mnode, "snap1")
self.assertEqual(ret, 0, "Failed to delete" "snapshot snap1")
g.log.info("Snapshots snap1 deleted Successfully")
# snapshot list
g.log.info("Starting to list all snapshots")
out = get_snap_list(self.mnode)
self.assertIsNotNone(out, "Failed to list all snapshots")
self.assertEqual(len(out), 2, "Failed to validate snap list")
g.log.info("Successfully validated snap list")
# restart Glusterd
g.log.info("Restarting Glusterd on all nodes")
ret = restart_glusterd(self.servers)
self.assertTrue(
ret, "Failed to restart glusterd on nodes"
"%s" % self.servers)
g.log.info("Successfully restarted glusterd on nodes"
" %s", self.servers)
# check glusterd running
g.log.info("Checking glusterd is running or not")
count = 0
while count < 80:
ret = is_glusterd_running(self.servers)
if ret == 0:
break
time.sleep(2)
count += 1
self.assertEqual(
ret, 0, "Failed to validate glusterd "
"running on nodes %s" % self.servers)
g.log.info("glusterd is running on " "nodes %s", self.servers)
# snapshot list
g.log.info("Starting to list all snapshots")
for server in self.servers[0:]:
out = get_snap_list(server)
self.assertIsNotNone(out, "Failed to list snap in node"
"%s" % server)
self.assertEqual(
len(out), 2, "Failed to validate snap list"
"on node %s" % server)
g.log.info("Successfully validated snap list on node %s", server)
def test_ec_uss_snapshot(self):
"""
- Start resource consumption tool
- Create directory dir1
- Create 5 directory and 5 files in dir of mountpoint
- Rename all files inside dir1 at mountpoint
- Create softlink and hardlink of files in dir1 of mountpoint
- Delete op for deleting all file in one of the dirs inside dir1
- Create tiny, small, medium and large file
- Create IO's
- Enable USS
- Create a Snapshot
- Activate Snapshot
- List snapshot and the contents inside snapshot
- Delete Snapshot
- Create Snapshot with same name
- Activate Snapshot
- List snapshot and the contents inside snapshot
- Validating IO's and waiting for it to complete
- Close connection and check file exist for memory log
"""
# pylint: disable=too-many-branches,too-many-statements,too-many-locals
# Starting resource consumption using top
log_file_mem_monitor = '/var/log/glusterfs/mem_usage.log'
cmd = ("for i in {1..20};do top -n 1 -b|egrep "
"'RES|gluster' & free -h 2>&1 >> %s ;"
"sleep 10;done" % (log_file_mem_monitor))
g.log.info(cmd)
cmd_list_procs = []
for server in self.servers:
proc = g.run_async(server, cmd)
cmd_list_procs.append(proc)
# Creating dir1
ret = mkdir(self.mounts[0].client_system,
"%s/dir1" % self.mounts[0].mountpoint)
self.assertTrue(ret, "Failed to create dir1")
g.log.info("Directory dir1 on %s created successfully", self.mounts[0])
# Create 5 dir and 5 files in each dir at mountpoint on dir1
start, end = 1, 5
for mount_obj in self.mounts:
# Number of dir and files to be created.
dir_range = ("%s..%s" % (str(start), str(end)))
file_range = ("%s..%s" % (str(start), str(end)))
# Create dir 1-5 at mountpoint.
ret = mkdir(mount_obj.client_system,
"%s/dir1/dir{%s}" % (mount_obj.mountpoint, dir_range))
self.assertTrue(ret, "Failed to create directory")
g.log.info("Directory created successfully")
# Create files inside each dir.
cmd = ('touch %s/dir1/dir{%s}/file{%s};' %
(mount_obj.mountpoint, dir_range, file_range))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "File creation failed")
g.log.info("File created successfull")
# Increment counter so that at next client dir and files are made
# with diff offset. Like at next client dir will be named
# dir6, dir7...dir10. Same with files.
start += 5
end += 5
# Rename all files inside dir1 at mountpoint on dir1
cmd = ('cd %s/dir1/dir1/; '
'for FILENAME in *;'
'do mv $FILENAME Unix_$FILENAME;'
'done;' % self.mounts[0].mountpoint)
ret, _, _ = g.run(self.mounts[0].client_system, cmd)
self.assertEqual(ret, 0, "Failed to rename file on " "client")
g.log.info("Successfully renamed file on client")
# Truncate at any dir in mountpoint inside dir1
# start is an offset to be added to dirname to act on
# diff files at diff clients.
start = 1
for mount_obj in self.mounts:
cmd = ('cd %s/dir1/dir%s/; '
'for FILENAME in *;'
'do echo > $FILENAME;'
'done;' % (mount_obj.mountpoint, str(start)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "Truncate failed")
g.log.info("Truncate of files successfull")
# Create softlink and hardlink of files in mountpoint. Start is an
# offset to be added to dirname to act on diff files at diff clients.
start = 1
for mount_obj in self.mounts:
cmd = ('cd %s/dir1/dir%s; '
'for FILENAME in *; '
'do ln -s $FILENAME softlink_$FILENAME;'
'done;' % (mount_obj.mountpoint, str(start)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "Creating Softlinks have failed")
g.log.info("Softlink of files have been changed successfully")
cmd = ('cd %s/dir1/dir%s; '
'for FILENAME in *; '
'do ln $FILENAME hardlink_$FILENAME;'
'done;' % (mount_obj.mountpoint, str(start + 1)))
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "Creating Hardlinks have failed")
g.log.info("Hardlink of files have been changed successfully")
start += 5
# Create tiny, small, medium and large file
# at mountpoint. Offset to differ filenames
# at diff clients.
offset = 1
for mount_obj in self.mounts:
cmd = 'fallocate -l 100 tiny_file%s.txt' % str(offset)
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "Fallocate for tiny files failed")
g.log.info("Fallocate for tiny files successfully")
cmd = 'fallocate -l 20M small_file%s.txt' % str(offset)
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "Fallocate for small files failed")
g.log.info("Fallocate for small files successfully")
cmd = 'fallocate -l 200M medium_file%s.txt' % str(offset)
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "Fallocate for medium files failed")
g.log.info("Fallocate for medium files successfully")
cmd = 'fallocate -l 1G large_file%s.txt' % str(offset)
ret, _, _ = g.run(mount_obj.client_system, cmd)
self.assertFalse(ret, "Fallocate for large files failed")
g.log.info("Fallocate for large files successfully")
offset += 1
# Creating files on client side for dir1
# Write IO
all_mounts_procs, count = [], 1
for mount_obj in self.mounts:
g.log.info("Starting IO on %s:%s", mount_obj.client_system,
mount_obj.mountpoint)
cmd = ("/usr/bin/env python %s create_deep_dirs_with_files "
"--dirname-start-num %d "
"--dir-depth 2 "
"--dir-length 10 "
"--max-num-of-dirs 5 "
"--num-of-files 5 %s/dir1" %
(self.script_upload_path, count, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
count += 10
# Enable USS
ret, _, _ = enable_uss(self.mnode, self.volname)
self.assertEqual(ret, 0, "Failed to enable USS on volume")
g.log.info("Successfully enabled USS on volume")
# Create Snapshot
ret, _, _ = snap_create(self.mnode,
self.volname,
"ec_snap",
timestamp=False)
self.assertEqual(ret, 0, "Failed to create snapshot ec_snap")
g.log.info("Snapshot ec_snap of volume %s created"
"successfully.", self.volname)
# Activate snapshot
ret, _, _ = snap_activate(self.mnode, "ec_snap")
self.assertEqual(ret, 0, "Failed to activate snapshot ec_snap")
g.log.info("Snapshot activated successfully")
# List contents inside snaphot and wait before listing
sleep(5)
for mount_obj in self.mounts:
ret, out, _ = uss_list_snaps(mount_obj.client_system,
mount_obj.mountpoint)
self.assertEqual(
ret, 0, "Directory Listing Failed for"
" Activated Snapshot")
self.assertIn(
"ec_snap", out.split("\n"), "Failed to "
"validate ec_snap under .snaps directory")
g.log.info("Activated Snapshot listed Successfully")
# Delete Snapshot ec_snap
ret, _, _ = snap_delete(self.mnode, "ec_snap")
self.assertEqual(ret, 0, "Failed to delete snapshot")
g.log.info("Snapshot deleted Successfully")
# Creating snapshot with the same name
ret, _, _ = snap_create(self.mnode,
self.volname,
"ec_snap",
timestamp=False)
self.assertEqual(ret, 0, "Failed to create snapshot ec_snap")
g.log.info("Snapshot ec_snap of volume %s created"
"successfully.", self.volname)
# Activate snapshot ec_snap
ret, _, _ = snap_activate(self.mnode, "ec_snap")
self.assertEqual(ret, 0, "Failed to activate snapshot ec_snap")
g.log.info("Snapshot activated successfully")
# List contents inside ec_snap and wait before listing
sleep(5)
for mount_obj in self.mounts:
ret, out, _ = uss_list_snaps(mount_obj.client_system,
mount_obj.mountpoint)
self.assertEqual(
ret, 0, "Directory Listing Failed for"
" Activated Snapshot")
self.assertIn(
"ec_snap", out.split('\n'), "Failed to "
"validate ec_snap under .snaps directory")
g.log.info("Activated Snapshot listed Successfully")
# Validating IO's and waiting to complete
ret = validate_io_procs(all_mounts_procs, self.mounts)
self.assertTrue(ret, "IO failed on some of the clients")
g.log.info("Successfully validated all io's")
# Close connection and check file exist for memory log
ret = file_exists(self.mnode, '/var/log/glusterfs/mem_usage.log')
self.assertTrue(ret, "Unexpected:Memory log file does " "not exist")
g.log.info("Memory log file exists")
for proc in cmd_list_procs:
ret, _, _ = proc.async_communicate()
self.assertEqual(ret, 0, "Memory logging failed")
g.log.info("Memory logging is successful")
def test_snap_del_gd_down(self):
"""
Steps:
1. Create volumes
2. Create 5 snapshots
3. Bring one node down
4. Delete one snapshot
5. list snapshot and validate delete
6. Bring up the downed node
7. Validate number of snaps after handshake on the
brought down node.
"""
# Create 5 snapshot
g.log.info("Creating 5 snapshots for volume %s", self.volname)
for i in range(0, 5):
ret, _, _ = snap_create(self.mnode, self.volname, "snapy%s" % i)
self.assertEqual(
ret, 0, ("Failed to create snapshot for %s" % self.volname))
g.log.info("Snapshot %s created successfully for volume %s",
"snapy%s" % i, self.volname)
# Check for no of snaps using snap_list it should be 5 now
snap_list = get_snap_list(self.mnode)
self.assertEqual(
5, len(snap_list), "No of snaps not consistent "
"for volume %s" % self.volname)
g.log.info("Successfully validated number of snaps.")
# Stopping glusterd service on server[1]
ret = stop_glusterd(self.servers[1])
self.assertTrue(
ret,
"Failed to stop glusterd service on node : %s" % self.servers[1])
g.log.info("Stopped glusterd services successfully on: %s",
self.servers[1])
# Delete one snapshot snapy1
ret, _, _ = snap_delete(self.servers[0], "snapy1")
self.assertEqual(ret, 0, "Failed to delete snapshot snapy1")
g.log.info("Successfully deleted snapshot of snapy1")
# Check for no of snaps using snap_list it should be 4 now
snap_list = get_snap_list(self.mnode)
self.assertEqual(
4, len(snap_list), "No of snaps not consistent "
"for volume %s" % self.volname)
g.log.info("Successfully validated number of snaps.")
# Starting glusterd services on server[1]
ret = start_glusterd(self.servers[1])
self.assertTrue(
ret, "Failed to start glusterd on node "
": %s" % self.servers[1])
g.log.info("Started glusterd services successfully on: %s",
self.servers[1])
# Check for no of snaps using snap_list it should be 4 for server[1]
count = 0
# putting wait here for glusterd handshake
while count < 60:
snap_list = get_snap_list(self.servers[1])
if len(snap_list) == 4:
break
time.sleep(2)
count += 2
self.assertEqual(
4, len(snap_list), "No of snaps not consistent "
"for volume %s" % self.volname)
g.log.info("Successfully validated number of snaps.")