def test_snap_scheduler_behaviour(self):
"""
Steps:
1. Create volumes
2. Enable shared storage
3. Validate shared storage mounted
4. Validate shared storage is enabled
5. Initialise snapshot scheduler on all node
6. Enable snapshot scheduler
7. Validate snapshot scheduler status
8. Disable snapshot scheduler
9. Validate snapshot scheduler status
"""
# Enable shared storage
g.log.info("Enable shared storage")
ret = enable_shared_storage(self.mnode)
self.assertTrue(ret, "Failed to enable shared storage")
g.log.info("Successfully enabled shared storage")
# Validate shared storage mounted
g.log.info("Starting to validate shared storage mounted")
for server in self.servers:
ret = is_shared_volume_mounted(server)
self.assertTrue(ret, "Failed to mount shared volume")
g.log.info("Successfully mounted shared volume")
# Validate shared storage is enabled
g.log.info("Starting to validate shared storage volume")
self.shared = "gluster_shared_storage"
volinfo = get_volume_info(self.mnode, self.shared)
self.assertEqual(
volinfo['gluster_shared_storage']['options']
['cluster.enable-shared-storage'], 'enable',
"shared storage is disabled")
g.log.info("Shared storage enabled successfully")
# Initialise snap scheduler
g.log.info("Initialising snapshot scheduler on all nodes")
ret = scheduler_init(self.servers)
self.assertTrue(ret, "Failed to initialize scheduler on all nodes")
g.log.info("Successfully initialized scheduler on all nodes")
# Enable snap scheduler
g.log.info("Starting to enable snapshot scheduler on all nodes")
ret, _, _ = scheduler_enable(self.mnode)
self.assertEqual(ret, 0, "Failed to enable scheduler on all servers")
g.log.info("Successfully enabled scheduler on all nodes")
# Check snapshot scheduler status
g.log.info("checking status of snapshot scheduler")
for server in self.servers:
count = 0
while count < 40:
ret, status, _ = scheduler_status(server)
if status.strip().split(":")[2] == ' Enabled':
break
time.sleep(2)
count += 2
self.assertEqual(status.strip().split(":")[2], ' Enabled',
"Failed to check status of scheduler")
g.log.info("Successfully checked scheduler status")
# Disable snap scheduler
g.log.info("Starting to disable snapshot scheduler on all nodes")
ret, _, _ = scheduler_disable(self.mnode)
self.assertEqual(
ret, 0, "Failed to disable scheduler on node"
" %s" % self.mnode)
g.log.info("Successfully disabled scheduler on all nodes")
# Check snapshot scheduler status
g.log.info("checking status of snapshot scheduler")
for server in self.servers:
count = 0
while count < 40:
ret, status, _ = scheduler_status(server)
if status.strip().split(":")[2] == ' Disabled':
break
time.sleep(2)
count += 2
self.assertEqual(status.strip().split(":")[2], ' Disabled',
"Failed to check status of scheduler")
g.log.info("Successfully checked scheduler status")
def test_snap_delete_existing_scheduler(self):
# pylint: disable=too-many-statements
"""
Description:
Validating snapshot scheduler behavior when existing schedule
is deleted.
Steps:
* Enable shared volume
* Create a volume
* Initialise snap_scheduler on all nodes
* Enable snap_scheduler
* Validate snap_scheduler status
* Perform IO on mounts
* Schedule a job of creating snapshot every 30 mins
* Perform snap_scheduler list
* Delete scheduled job
* Validate IO is successful
* Perform snap_scheduler list
"""
# Initialise snap scheduler
g.log.info("Initialising snap_scheduler on all servers")
count = 0
while count < 80:
ret = scheduler_init(self.servers)
if ret:
break
sleep(2)
count += 1
self.assertTrue(ret, "Failed to initialise scheduler on all servers")
g.log.info("Successfully initialised scheduler on all servers")
# Enable snap scheduler
g.log.info("Enabling snap_scheduler")
ret, _, _ = scheduler_enable(self.mnode)
self.assertEqual(ret, 0, "Failed to enable scheduler on node %s"
% self.mnode)
g.log.info("Successfully enabled scheduler on node %s", self.mnode)
# Validate snapshot scheduler status
g.log.info("Validating status of snap_scheduler")
for server in self.servers:
count = 0
while count < 40:
ret, status, _ = scheduler_status(server)
if status.strip().split(":")[2] == ' Enabled':
break
sleep(2)
count += 2
self.assertEqual(status.strip().split(":")[2], ' Enabled',
"Failed to validate status of scheduler")
g.log.info("Successfully validated scheduler status")
# Write files on all mounts
g.log.info("Starting IO on all mounts...")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("/usr/bin/env python %s create_files "
"-f 10 --base-file-name file %s" % (
self.script_upload_path,
mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system, cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# Add a job to schedule snapshot every 30 mins
g.log.info("Starting to add new job")
self.scheduler = r"*/30 * * * *"
self.job_name = "Job1"
ret, _, _ = scheduler_add_jobs(self.mnode, self.job_name,
self.scheduler, self.volname)
self.assertEqual(ret, 0, "Failed to add job")
g.log.info("Successfully added Job on volume %s", self.volname)
# Perform snap_scheduler list
g.log.info("Starting to list all scheduler jobs")
ret, _, _ = scheduler_list(self.mnode)
self.assertEqual(ret, 0, "Failed to list scheduler jobs")
g.log.info("Successfully listed all jobs")
# Delete scheduled job
g.log.info("Starting to delete scheduled jobs")
ret, _, _ = scheduler_delete(self.mnode, self.job_name)
self.assertEqual(ret, 0, "Failed to delete scheduled job")
g.log.info("Successfully deleted scheduled job %s", self.job_name)
# Validate IO
self.assertTrue(
validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients"
)
# Perform snap_scheduler list (no active jobs should be present)
g.log.info("Starting to list all scheduler jobs")
ret, out, _ = scheduler_list(self.mnode)
self.assertEqual(ret, 0, "Failed to list scheduler jobs")
ret1 = out.strip().split(":")
self.assertEqual(ret1[1], " No snapshots scheduled", "Unexpected: "
"Jobs are getting listed even after being deleted")
g.log.info("Expected: No snapshots Jobs scheduled")
def test_snap_scheduler_status(self):
# pylint: disable=too-many-statements
"""
Steps:
1. create volumes
2. initialise snap scheduler without
enabling shared storage should fail
3. enable shared storage
4. initialise snap scheduler
5. check snapshot scheduler status
"""
# Validate shared storage is enabled
g.log.info("Starting to validate shared storage volume")
volinfo = get_volume_options(self.mnode,
self.volname,
option=("cluster.enable"
"-shared-storage"))
if volinfo["cluster.enable-shared-storage"] == "disable":
# Initialise snapshot scheduler
g.log.info("Initialising snapshot scheduler on all nodes")
ret = scheduler_init(self.servers)
self.assertFalse(
ret, "Unexpected: Successfully initialized "
"scheduler on all nodes")
g.log.info("As Expected, Failed to initialize scheduler on "
"all nodes")
self.assertEqual(volinfo["cluster.enable-shared-storage"], "disable",
"Unexpected: Shared storage "
"is enabled on cluster")
# Enable Shared storage
g.log.info("enabling shared storage")
ret = enable_shared_storage(self.mnode)
self.assertTrue(ret, "Failed to enable shared storage")
g.log.info("Successfully enabled shared storage")
# Validate shared storage mounted
g.log.info("validate shared storage mounted")
ret = is_shared_volume_mounted(self.mnode)
self.assertTrue(ret, "Failed to mount shared volume")
g.log.info("Successfully mounted shared volume")
# Validate shared storage volume is enabled
g.log.info("validate shared storage volume")
volinfo = get_volume_options(self.mnode,
self.volname,
option=("cluster.enable"
"-shared-storage"))
self.assertIsNotNone(volinfo, "Failed to validate volume option")
self.assertEqual(volinfo["cluster.enable-shared-storage"], "enable",
"Failed to enable shared storage volume")
g.log.info("Shared storage enabled successfully")
# Initialise snap scheduler
g.log.info("Initialising snapshot scheduler on all nodes")
count = 0
while count < 40:
ret = scheduler_init(self.servers)
if ret:
break
time.sleep(2)
count += 1
self.assertTrue(ret, "Failed to initialize scheduler on all nodes")
g.log.info("Successfully initialized scheduler on all nodes")
# Enable snap scheduler
g.log.info("Enabling snap scheduler")
ret, _, _ = scheduler_enable(self.mnode)
self.assertEqual(ret, 0,
"Failed to enable scheduler on %s node" % self.mnode)
g.log.info("Successfully enabled scheduler on %s node", self.mnode)
# Check snapshot scheduler status
g.log.info("checking status of snapshot scheduler")
for server in self.servers:
count = 0
while count < 40:
ret, status, _ = scheduler_status(server)
if ret == 0:
self.assertEqual(status.strip().split(":")[2], ' Enabled',
"Failed to check status of scheduler")
break
time.sleep(2)
count += 1
self.assertEqual(
ret, 0, "Failed to check status of scheduler"
" on nodes %s" % server)
g.log.info("Successfully checked scheduler status on %s nodes",
server)
# disable snap scheduler
g.log.info("disabling snap scheduler")
ret, _, _ = scheduler_disable(self.mnode)
self.assertEqual(ret, 0, "Unexpected: Failed to disable "
"snapshot scheduler")
g.log.info("Successfully disabled snapshot scheduler")
# disable shared storage
g.log.info("starting to disable shared storage")
ret = disable_shared_storage(self.mnode)
self.assertTrue(ret, "Failed to disable shared storage")
g.log.info("Successfully disabled shared storage")
# Validate shared volume unmounted
g.log.info("Validate shared volume unmounted")
ret = is_shared_volume_unmounted(self.mnode)
self.assertTrue(ret, "Failed to unmount shared storage")
g.log.info("Successfully unmounted shared storage")
def test_snap_delete_existing_scheduler(self):
# pylint: disable=too-many-statements
"""
Steps:
1. enable shared volume
2. create a volume
3. initialise snap scheduler on all nodes
4. enable snap scheduler
5. check snap scheduler status
6. perform io on mounts
7. schedule a job of creating snapshot
every 30 mins
8. list jobs created
9. delete scheduled job
10. validate io is successful
11. list job should not list
any existing snapshot jobs
"""
# Initialise snap scheduler
g.log.info("Initialising snap scheduler on all servers")
count = 0
while count < 80:
ret = scheduler_init(self.servers)
if ret:
break
time.sleep(2)
count += 1
self.assertTrue(ret, "Failed to initialise scheduler on all servers")
g.log.info("Successfully initialised scheduler on all servers")
# Enable snap scheduler
g.log.info("Enabling snap scheduler")
ret, _, _ = scheduler_enable(self.mnode)
self.assertEqual(ret, 0,
"Failed to enable scheduler on node %s" % self.mnode)
g.log.info("Successfully enabled scheduler on node %s", self.mnode)
# Check snapshot scheduler status
g.log.info("checking status of snapshot scheduler")
for server in self.servers:
count = 0
while count < 40:
ret, status, _ = scheduler_status(server)
if status.strip().split(":")[2] == ' Enabled':
break
time.sleep(2)
count += 2
self.assertEqual(status.strip().split(":")[2], ' Enabled',
"Failed to check status of scheduler")
g.log.info("Successfully checked scheduler status")
# write files on all mounts
g.log.info("Starting IO on all mounts...")
all_mounts_procs = []
for mount_obj in self.mounts:
cmd = ("python %s create_files "
"-f 10 --base-file-name file %s" %
(self.script_upload_path, mount_obj.mountpoint))
proc = g.run_async(mount_obj.client_system,
cmd,
user=mount_obj.user)
all_mounts_procs.append(proc)
# add a job to schedule snapshot every 30 mins
g.log.info("Starting to add new job")
self.scheduler = r"*/30 * * * *"
self.job_name = "Job1"
ret, _, _ = scheduler_add_jobs(self.mnode, self.job_name,
self.scheduler, self.volname)
self.assertEqual(ret, 0, "Failed to add job")
g.log.info("Successfully added Job on volume %s", self.volname)
# scheduler list
g.log.info("Starting to list all scheduler jobs")
ret, _, _ = scheduler_list(self.mnode)
self.assertEqual(ret, 0, "Failed to list scheduler jobs")
g.log.info("Successfully listed all jobs")
# delete scheduled job
g.log.info("Starting to delete scheduled jobs")
ret, _, _ = scheduler_delete(self.mnode, self.job_name)
self.assertEqual(ret, 0, "Failed to delete scheduled job")
g.log.info("Successfully deleted scheduled job %s", self.job_name)
# Validate IO
self.assertTrue(validate_io_procs(all_mounts_procs, self.mounts),
"IO failed on some of the clients")
# scheduler list (no active jobs should be there)
g.log.info("Starting to list all scheduler jobs")
ret, out, _ = scheduler_list(self.mnode)
self.assertEqual(ret, 0, "Failed to list scheduler jobs")
ret1 = out.strip().split(":")
self.assertEqual(
ret1[1], " No snapshots scheduled", "Unexpected:"
"Failed to delete scheduled job %s" % self.job_name)
g.log.info("Expected: No snapshots Jobs scheduled")
def test_snap_scheduler_status(self):
# pylint: disable=too-many-statements
"""
Validating the snapshot scheduler behavior when shared storage
volume is mounted/not mounted.
* Initialise snap_scheduler without enabling shared storage
* Enable shared storage
* Initialise snap_scheduler on all nodes
* Check snap_scheduler status
"""
# Validate shared storage is disabled
g.log.info("Validating shared storage is disabled")
volinfo = get_volume_options(self.mnode, self.volname,
option=("cluster.enable-shared-storage"))
if volinfo["cluster.enable-shared-storage"] == "disable":
# Initialise snapshot scheduler
g.log.info("Initialising snapshot scheduler on all nodes")
ret = scheduler_init(self.servers)
self.assertFalse(ret, "Unexpected: Successfully initialized "
"scheduler on all nodes")
g.log.info("Expected: Failed to initialize snap_scheduler on "
"all nodes")
self.assertEqual(volinfo["cluster.enable-shared-storage"],
"disable", "Unexpected: Shared storage "
"is enabled on cluster")
# Enable shared storage
g.log.info("enabling shared storage")
ret = enable_shared_storage(self.mnode)
self.assertTrue(ret, "Failed to enable shared storage")
g.log.info("Successfully enabled shared storage")
# Validate shared storage volume is mounted
g.log.info("Validating if shared storage volume is mounted")
count = 0
while count < 5:
ret = is_shared_volume_mounted(self.mnode)
if ret:
break
sleep(2)
count += 1
self.assertTrue(ret, "Failed to validate if shared volume is mounted")
g.log.info("Successfully validated shared volume is mounted")
# Validate shared storage volume is enabled
g.log.info("Validate shared storage is enabled")
volinfo = get_volume_options(self.mnode, self.volname,
option=("cluster.enable-shared-storage"))
self.assertIsNotNone(volinfo, "Failed to validate volume option")
self.assertEqual(volinfo["cluster.enable-shared-storage"], "enable",
"Failed to validate if shared storage is enabled")
g.log.info("Successfully validated shared storage is enabled")
# Initialise snap_scheduler on all nodes
g.log.info("Initialising snapshot scheduler on all nodes")
count = 0
sleep(2)
while count < 40:
ret = scheduler_init(self.servers)
if ret:
break
sleep(2)
count += 1
self.assertTrue(ret, "Failed to initialize scheduler on all nodes")
g.log.info("Successfully initialized scheduler on all nodes")
# Enable snap_scheduler
g.log.info("Enabling snap_scheduler")
ret, _, _ = scheduler_enable(self.mnode)
self.assertEqual(ret, 0, "Failed to enable scheduler on %s node" %
self.mnode)
g.log.info("Successfully enabled scheduler on %s node", self.mnode)
# Check snapshot scheduler status
g.log.info("checking status of snapshot scheduler")
for server in self.servers:
count = 0
while count < 40:
ret, status, _ = scheduler_status(server)
status = status.strip().split(":")[2]
if ret == 0 and status == ' Enabled':
break
sleep(2)
count += 1
self.assertEqual(ret, 0, "Failed to check status of scheduler"
" on node %s" % server)
g.log.info("Successfully checked scheduler status on %s nodes",
server)