def test_02_host_maintenance_mode_with_activities(self):
"""Test host maintenance mode with activities
"""
# Validate the following
# 1. Create Vms. Acquire IP. Create port forwarding & load balancing
# rules for Vms.
# 2. While activities are ongoing: Create snapshots, recurring
# snapshots, create templates, download volumes, Host 1: put to
# maintenance mode. All Vms should failover to Host 2 in cluster
# Vms should be in running state. All port forwarding rules and
# load balancing Rules should work.
# 3. After failover to Host 2 succeeds, deploy Vms. Deploy Vms on host
# 2 should succeed. All ongoing activities in step 3 should succeed
# 4. Host 1: cancel maintenance mode.
# 5. While activities are ongoing: Create snapshots, recurring
# snapshots, create templates, download volumes, Host 2: put to
# maintenance mode. All Vms should failover to Host 1 in cluster.
# 6. After failover to Host 1 succeeds, deploy VMs. Deploy Vms on
# host 1 should succeed. All ongoing activities in step 6 should
# succeed.
hosts = Host.list(self.apiclient,
zoneid=self.zone.id,
resourcestate='Enabled',
type='Routing')
self.assertEqual(isinstance(hosts, list), True,
"List hosts should return valid host response")
if len(hosts) < 2:
self.skipTest("There must be at least 2 hosts present in cluster")
self.debug("Checking HA with hosts: %s, %s" %
(hosts[0].name, hosts[1].name))
self.debug("Deploying VM in account: %s" % self.account.name)
# Spawn an instance in that network
virtual_machine = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id)
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.assertEqual(vm.state, "Running",
"Deployed VM should be in RUnning state")
networks = Network.list(self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True)
self.assertEqual(
isinstance(networks, list), True,
"List networks should return valid list for the account")
network = networks[0]
self.debug("Associating public IP for account: %s" % self.account.name)
public_ip = PublicIPAddress.create(self.apiclient,
accountid=self.account.name,
zoneid=self.zone.id,
domainid=self.account.domainid,
networkid=network.id)
self.debug("Associated %s with network %s" %
(public_ip.ipaddress.ipaddress, network.id))
self.debug("Creating PF rule for IP address: %s" %
public_ip.ipaddress.ipaddress)
NATRule.create(self.apiclient,
virtual_machine,
self.services["natrule"],
ipaddressid=public_ip.ipaddress.id)
self.debug("Creating LB rule on IP with NAT: %s" %
public_ip.ipaddress.ipaddress)
# Create Load Balancer rule on IP already having NAT rule
lb_rule = LoadBalancerRule.create(self.apiclient,
self.services["lbrule"],
ipaddressid=public_ip.ipaddress.id,
accountid=self.account.name)
self.debug("Created LB rule with ID: %s" % lb_rule.id)
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e))
# Get the Root disk of VM
volumes = list_volumes(self.apiclient,
virtualmachineid=virtual_machine.id,
type='ROOT',
listall=True)
volume = volumes[0]
self.debug("Root volume of VM(%s): %s" %
(virtual_machine.name, volume.name))
# Create a snapshot from the ROOTDISK
self.debug("Creating snapshot on ROOT volume: %s" % volume.name)
snapshot = Snapshot.create(self.apiclient, volumes[0].id)
self.debug("Snapshot created: ID - %s" % snapshot.id)
snapshots = list_snapshots(self.apiclient,
id=snapshot.id,
listall=True)
self.assertEqual(isinstance(snapshots, list), True,
"Check list response returns a valid list")
self.assertNotEqual(snapshots, None,
"Check if result exists in list snapshots call")
self.assertEqual(snapshots[0].id, snapshot.id,
"Check snapshot id in list resources call")
# Generate template from the snapshot
self.debug("Generating template from snapshot: %s" % snapshot.name)
template = Template.create_from_snapshot(self.apiclient, snapshot,
self.services["templates"])
self.debug("Created template from snapshot: %s" % template.id)
templates = list_templates(
self.apiclient,
templatefilter=self.services["templates"]["templatefilter"],
id=template.id)
self.assertEqual(
isinstance(templates, list), True,
"List template call should return the newly created template")
self.assertEqual(
templates[0].isready, True,
"The newly created template should be in ready state")
first_host = vm.hostid
self.debug("Enabling maintenance mode for host %s" % vm.hostid)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = first_host
self.apiclient.prepareHostForMaintenance(cmd)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
timeout = self.services["timeout"]
# Poll and check state of VM while it migrates from one host to another
while True:
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug("VM 1 state: %s" % vm.state)
if vm.state in [
"Stopping", "Stopped", "Running", "Starting", "Migrating"
]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail("VM migration from one-host-to-other failed\
while enabling maintenance")
second_host = vm.hostid
self.assertEqual(
vm.state, "Running",
"VM should be in Running state after enabling host maintenance")
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e))
self.debug("Deploying VM in account: %s" % self.account.name)
# Spawn an instance on other host
virtual_machine_2 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id)
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine_2.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.debug("VM 2 state: %s" % vm.state)
self.assertEqual(vm.state, "Running",
"Deployed VM should be in Running state")
self.debug("Canceling host maintenance for ID: %s" % first_host)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = first_host
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % first_host)
# Get the Root disk of VM
volumes = list_volumes(self.apiclient,
virtualmachineid=virtual_machine_2.id,
type='ROOT',
listall=True)
volume = volumes[0]
self.debug("Root volume of VM(%s): %s" %
(virtual_machine_2.name, volume.name))
# Create a snapshot from the ROOTDISK
self.debug("Creating snapshot on ROOT volume: %s" % volume.name)
snapshot = Snapshot.create(self.apiclient, volumes[0].id)
self.debug("Snapshot created: ID - %s" % snapshot.id)
snapshots = list_snapshots(self.apiclient,
id=snapshot.id,
listall=True)
self.assertEqual(isinstance(snapshots, list), True,
"Check list response returns a valid list")
self.assertNotEqual(snapshots, None,
"Check if result exists in list snapshots call")
self.assertEqual(snapshots[0].id, snapshot.id,
"Check snapshot id in list resources call")
# Generate template from the snapshot
self.debug("Generating template from snapshot: %s" % snapshot.name)
template = Template.create_from_snapshot(self.apiclient, snapshot,
self.services["templates"])
self.debug("Created template from snapshot: %s" % template.id)
templates = list_templates(
self.apiclient,
templatefilter=self.services["templates"]["templatefilter"],
id=template.id)
self.assertEqual(
isinstance(templates, list), True,
"List template call should return the newly created template")
self.assertEqual(
templates[0].isready, True,
"The newly created template should be in ready state")
self.debug("Enabling maintenance mode for host %s" % second_host)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = second_host
self.apiclient.prepareHostForMaintenance(cmd)
self.debug("Maintenance mode enabled for host: %s" % second_host)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
# Poll and check the status of VMs
timeout = self.services["timeout"]
while True:
vms = VirtualMachine.list(self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug(
"VM state after enabling maintenance on first host: %s" %
vm.state)
if vm.state in [
"Stopping", "Stopped", "Running", "Starting", "Migrating"
]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail("VM migration from one-host-to-other failed\
while enabling maintenance")
# Poll and check the status of VMs
timeout = self.services["timeout"]
while True:
vms = VirtualMachine.list(self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[1]
self.debug(
"VM state after enabling maintenance on first host: %s" %
vm.state)
if vm.state in [
"Stopping", "Stopped", "Running", "Starting", "Migrating"
]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail("VM migration from one-host-to-other failed\
while enabling maintenance")
for vm in vms:
self.debug(
"VM states after enabling maintenance mode on host: %s - %s" %
(first_host, vm.state))
self.assertEqual(vm.state, "Running",
"Deployed VM should be in Running state")
# Spawn an instance on other host
virtual_machine_3 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id)
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine_3.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.debug("VM 3 state: %s" % vm.state)
self.assertEqual(vm.state, "Running",
"Deployed VM should be in Running state")
self.debug("Canceling host maintenance for ID: %s" % second_host)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = second_host
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % second_host)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
return
def test_02_host_maintenance_mode_with_activities(self):
"""Test host maintenance mode with activities
"""
# Validate the following
# 1. Create Vms. Acquire IP. Create port forwarding & load balancing
# rules for Vms.
# 2. While activities are ongoing: Create snapshots, recurring
# snapshots, create templates, download volumes, Host 1: put to
# maintenance mode. All Vms should failover to Host 2 in cluster
# Vms should be in running state. All port forwarding rules and
# load balancing Rules should work.
# 3. After failover to Host 2 succeeds, deploy Vms. Deploy Vms on host
# 2 should succeed. All ongoing activities in step 3 should succeed
# 4. Host 1: cancel maintenance mode.
# 5. While activities are ongoing: Create snapshots, recurring
# snapshots, create templates, download volumes, Host 2: put to
# maintenance mode. All Vms should failover to Host 1 in cluster.
# 6. After failover to Host 1 succeeds, deploy VMs. Deploy Vms on
# host 1 should succeed. All ongoing activities in step 6 should
# succeed.
hosts = Host.list(
self.apiclient,
zoneid=self.zone.id,
resourcestate='Enabled',
type='Routing'
)
self.assertEqual(
isinstance(hosts, list),
True,
"List hosts should return valid host response"
)
if len(hosts) < 2:
self.skipTest("There must be at least 2 hosts present in cluster")
self.debug("Checking HA with hosts: %s, %s" % (
hosts[0].name,
hosts[1].name
))
self.debug("Deploying VM in account: %s" % self.account.name)
# Spawn an instance in that network
virtual_machine = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id
)
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.assertEqual(
vm.state,
"Running",
"Deployed VM should be in RUnning state"
)
networks = Network.list(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
self.assertEqual(
isinstance(networks, list),
True,
"List networks should return valid list for the account"
)
network = networks[0]
self.debug("Associating public IP for account: %s" %
self.account.name)
public_ip = PublicIPAddress.create(
self.apiclient,
accountid=self.account.name,
zoneid=self.zone.id,
domainid=self.account.domainid,
networkid=network.id
)
self.debug("Associated %s with network %s" % (
public_ip.ipaddress.ipaddress,
network.id
))
self.debug("Creating PF rule for IP address: %s" %
public_ip.ipaddress.ipaddress)
NATRule.create(
self.apiclient,
virtual_machine,
self.services["natrule"],
ipaddressid=public_ip.ipaddress.id
)
self.debug("Creating LB rule on IP with NAT: %s" %
public_ip.ipaddress.ipaddress)
# Create Load Balancer rule on IP already having NAT rule
lb_rule = LoadBalancerRule.create(
self.apiclient,
self.services["lbrule"],
ipaddressid=public_ip.ipaddress.id,
accountid=self.account.name
)
self.debug("Created LB rule with ID: %s" % lb_rule.id)
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e)
)
# Get the Root disk of VM
volumes = list_volumes(
self.apiclient,
virtualmachineid=virtual_machine.id,
type='ROOT',
listall=True
)
volume = volumes[0]
self.debug(
"Root volume of VM(%s): %s" % (
virtual_machine.name,
volume.name
))
# Create a snapshot from the ROOTDISK
self.debug("Creating snapshot on ROOT volume: %s" % volume.name)
snapshot = Snapshot.create(self.apiclient, volumes[0].id)
self.debug("Snapshot created: ID - %s" % snapshot.id)
snapshots = list_snapshots(
self.apiclient,
id=snapshot.id,
listall=True
)
self.assertEqual(
isinstance(snapshots, list),
True,
"Check list response returns a valid list"
)
self.assertNotEqual(
snapshots,
None,
"Check if result exists in list snapshots call"
)
self.assertEqual(
snapshots[0].id,
snapshot.id,
"Check snapshot id in list resources call"
)
# Generate template from the snapshot
self.debug("Generating template from snapshot: %s" % snapshot.name)
template = Template.create_from_snapshot(
self.apiclient,
snapshot,
self.services["templates"]
)
self.debug("Created template from snapshot: %s" % template.id)
templates = list_templates(
self.apiclient,
templatefilter=self.services["templates"]["templatefilter"],
id=template.id
)
self.assertEqual(
isinstance(templates, list),
True,
"List template call should return the newly created template"
)
self.assertEqual(
templates[0].isready,
True,
"The newly created template should be in ready state"
)
first_host = vm.hostid
self.debug("Enabling maintenance mode for host %s" % vm.hostid)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = first_host
self.apiclient.prepareHostForMaintenance(cmd)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
timeout = self.services["timeout"]
# Poll and check state of VM while it migrates from one host to another
while True:
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug("VM 1 state: %s" % vm.state)
if vm.state in ["Stopping",
"Stopped",
"Running",
"Starting",
"Migrating"]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail(
"VM migration from one-host-to-other failed\
while enabling maintenance"
)
second_host = vm.hostid
self.assertEqual(
vm.state,
"Running",
"VM should be in Running state after enabling host maintenance"
)
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e)
)
self.debug("Deploying VM in account: %s" % self.account.name)
# Spawn an instance on other host
virtual_machine_2 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id
)
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine_2.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.debug("VM 2 state: %s" % vm.state)
self.assertEqual(
vm.state,
"Running",
"Deployed VM should be in Running state"
)
self.debug("Canceling host maintenance for ID: %s" % first_host)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = first_host
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % first_host)
# Get the Root disk of VM
volumes = list_volumes(
self.apiclient,
virtualmachineid=virtual_machine_2.id,
type='ROOT',
listall=True
)
volume = volumes[0]
self.debug(
"Root volume of VM(%s): %s" % (
virtual_machine_2.name,
volume.name
))
# Create a snapshot from the ROOTDISK
self.debug("Creating snapshot on ROOT volume: %s" % volume.name)
snapshot = Snapshot.create(self.apiclient, volumes[0].id)
self.debug("Snapshot created: ID - %s" % snapshot.id)
snapshots = list_snapshots(
self.apiclient,
id=snapshot.id,
listall=True
)
self.assertEqual(
isinstance(snapshots, list),
True,
"Check list response returns a valid list"
)
self.assertNotEqual(
snapshots,
None,
"Check if result exists in list snapshots call"
)
self.assertEqual(
snapshots[0].id,
snapshot.id,
"Check snapshot id in list resources call"
)
# Generate template from the snapshot
self.debug("Generating template from snapshot: %s" % snapshot.name)
template = Template.create_from_snapshot(
self.apiclient,
snapshot,
self.services["templates"]
)
self.debug("Created template from snapshot: %s" % template.id)
templates = list_templates(
self.apiclient,
templatefilter=self.services["templates"]["templatefilter"],
id=template.id
)
self.assertEqual(
isinstance(templates, list),
True,
"List template call should return the newly created template"
)
self.assertEqual(
templates[0].isready,
True,
"The newly created template should be in ready state"
)
self.debug("Enabling maintenance mode for host %s" % second_host)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = second_host
self.apiclient.prepareHostForMaintenance(cmd)
self.debug("Maintenance mode enabled for host: %s" % second_host)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
# Poll and check the status of VMs
timeout = self.services["timeout"]
while True:
vms = VirtualMachine.list(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug(
"VM state after enabling maintenance on first host: %s" %
vm.state)
if vm.state in ["Stopping",
"Stopped",
"Running",
"Starting",
"Migrating"]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail(
"VM migration from one-host-to-other failed\
while enabling maintenance"
)
# Poll and check the status of VMs
timeout = self.services["timeout"]
while True:
vms = VirtualMachine.list(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[1]
self.debug(
"VM state after enabling maintenance on first host: %s" %
vm.state)
if vm.state in ["Stopping",
"Stopped",
"Running",
"Starting",
"Migrating"]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail(
"VM migration from one-host-to-other failed\
while enabling maintenance"
)
for vm in vms:
self.debug(
"VM states after enabling maintenance mode on host: %s - %s" %
(first_host, vm.state))
self.assertEqual(
vm.state,
"Running",
"Deployed VM should be in Running state"
)
# Spawn an instance on other host
virtual_machine_3 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id
)
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine_3.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.debug("VM 3 state: %s" % vm.state)
self.assertEqual(
vm.state,
"Running",
"Deployed VM should be in Running state"
)
self.debug("Canceling host maintenance for ID: %s" % second_host)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = second_host
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % second_host)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
return
def test_01_host_maintenance_mode(self):
"""Test host maintenance mode
"""
# Validate the following
# 1. Create Vms. Acquire IP. Create port forwarding & load balancing
# rules for Vms.
# 2. Host 1: put to maintenance mode. All Vms should failover to Host
# 2 in cluster. Vms should be in running state. All port forwarding
# rules and load balancing Rules should work.
# 3. After failover to Host 2 succeeds, deploy Vms. Deploy Vms on host
# 2 should succeed.
# 4. Host 1: cancel maintenance mode.
# 5. Host 2 : put to maintenance mode. All Vms should failover to
# Host 1 in cluster.
# 6. After failover to Host 1 succeeds, deploy VMs. Deploy Vms on
# host 1 should succeed.
hosts = Host.list(self.apiclient,
zoneid=self.zone.id,
resourcestate='Enabled',
type='Routing')
self.assertEqual(isinstance(hosts, list), True,
"List hosts should return valid host response")
if len(hosts) < 2:
self.skipTest("There must be at least 2 hosts present in cluster")
self.debug("Checking HA with hosts: %s, %s" %
(hosts[0].name, hosts[1].name))
self.debug("Deploying VM in account: %s" % self.account.name)
# Spawn an instance in that network
virtual_machine = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id)
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.assertEqual(vm.state, "Running",
"Deployed VM should be in RUnning state")
networks = Network.list(self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True)
self.assertEqual(
isinstance(networks, list), True,
"List networks should return valid list for the account")
network = networks[0]
self.debug("Associating public IP for account: %s" % self.account.name)
public_ip = PublicIPAddress.create(self.apiclient,
accountid=self.account.name,
zoneid=self.zone.id,
domainid=self.account.domainid,
networkid=network.id)
self.debug("Associated %s with network %s" %
(public_ip.ipaddress.ipaddress, network.id))
self.debug("Creating PF rule for IP address: %s" %
public_ip.ipaddress.ipaddress)
NATRule.create(self.apiclient,
virtual_machine,
self.services["natrule"],
ipaddressid=public_ip.ipaddress.id)
self.debug("Creating LB rule on IP with NAT: %s" %
public_ip.ipaddress.ipaddress)
# Create Load Balancer rule on IP already having NAT rule
lb_rule = LoadBalancerRule.create(self.apiclient,
self.services["lbrule"],
ipaddressid=public_ip.ipaddress.id,
accountid=self.account.name)
self.debug("Created LB rule with ID: %s" % lb_rule.id)
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e))
first_host = vm.hostid
self.debug("Enabling maintenance mode for host %s" % vm.hostid)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = first_host
self.apiclient.prepareHostForMaintenance(cmd)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
timeout = self.services["timeout"]
# Poll and check state of VM while it migrates from one host to another
while True:
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug("VM 1 state: %s" % vm.state)
if vm.state in [
"Stopping", "Stopped", "Running", "Starting", "Migrating"
]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail("VM migration from one-host-to-other\
failed while enabling maintenance")
second_host = vm.hostid
self.assertEqual(
vm.state, "Running",
"VM should be in Running state after enabling host maintenance")
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e))
self.debug("Deploying VM in account: %s" % self.account.name)
# Spawn an instance on other host
virtual_machine_2 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id)
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine_2.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.debug("VM 2 state: %s" % vm.state)
self.assertEqual(vm.state, "Running",
"Deployed VM should be in Running state")
self.debug("Canceling host maintenance for ID: %s" % first_host)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = first_host
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % first_host)
self.debug("Enabling maintenance mode for host %s" % second_host)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = second_host
self.apiclient.prepareHostForMaintenance(cmd)
self.debug("Maintenance mode enabled for host: %s" % second_host)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
# Poll and check the status of VMs
timeout = self.services["timeout"]
while True:
vms = VirtualMachine.list(self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug(
"VM state after enabling maintenance on first host: %s" %
vm.state)
if vm.state in [
"Stopping", "Stopped", "Running", "Starting", "Migrating"
]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail("VM migration from one-host-to-other failed\
while enabling maintenance")
# Poll and check the status of VMs
timeout = self.services["timeout"]
while True:
vms = VirtualMachine.list(self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[1]
self.debug(
"VM state after enabling maintenance on first host: %s" %
vm.state)
if vm.state in [
"Stopping", "Stopped", "Running", "Starting", "Migrating"
]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail("VM migration from one-host-to-other\
failed while enabling maintenance")
for vm in vms:
self.debug(
"VM states after enabling maintenance mode on host: %s - %s" %
(first_host, vm.state))
self.assertEqual(vm.state, "Running",
"Deployed VM should be in Running state")
# Spawn an instance on other host
virtual_machine_3 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id)
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine_3.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.debug("VM 3 state: %s" % vm.state)
self.assertEqual(vm.state, "Running",
"Deployed VM should be in Running state")
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e))
self.debug("Canceling host maintenance for ID: %s" % second_host)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = second_host
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % second_host)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
return
def test_01_host_maintenance_mode(self):
"""Test host maintenance mode
"""
# Validate the following
# 1. Create Vms. Acquire IP. Create port forwarding & load balancing
# rules for Vms.
# 2. Host 1: put to maintenance mode. All Vms should failover to Host
# 2 in cluster. Vms should be in running state. All port forwarding
# rules and load balancing Rules should work.
# 3. After failover to Host 2 succeeds, deploy Vms. Deploy Vms on host
# 2 should succeed.
# 4. Host 1: cancel maintenance mode.
# 5. Host 2 : put to maintenance mode. All Vms should failover to
# Host 1 in cluster.
# 6. After failover to Host 1 succeeds, deploy VMs. Deploy Vms on
# host 1 should succeed.
hosts = Host.list(
self.apiclient,
zoneid=self.zone.id,
resourcestate='Enabled',
type='Routing'
)
self.assertEqual(
isinstance(hosts, list),
True,
"List hosts should return valid host response"
)
if len(hosts) < 2:
self.skipTest("There must be at least 2 hosts present in cluster")
self.debug("Checking HA with hosts: %s, %s" % (
hosts[0].name,
hosts[1].name
))
self.debug("Deploying VM in account: %s" % self.account.name)
# Spawn an instance in that network
virtual_machine = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id
)
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.assertEqual(
vm.state,
"Running",
"Deployed VM should be in RUnning state"
)
networks = Network.list(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
self.assertEqual(
isinstance(networks, list),
True,
"List networks should return valid list for the account"
)
network = networks[0]
self.debug("Associating public IP for account: %s" %
self.account.name)
public_ip = PublicIPAddress.create(
self.apiclient,
accountid=self.account.name,
zoneid=self.zone.id,
domainid=self.account.domainid,
networkid=network.id
)
self.debug("Associated %s with network %s" % (
public_ip.ipaddress.ipaddress,
network.id
))
self.debug("Creating PF rule for IP address: %s" %
public_ip.ipaddress.ipaddress)
NATRule.create(
self.apiclient,
virtual_machine,
self.services["natrule"],
ipaddressid=public_ip.ipaddress.id
)
self.debug("Creating LB rule on IP with NAT: %s" %
public_ip.ipaddress.ipaddress)
# Create Load Balancer rule on IP already having NAT rule
lb_rule = LoadBalancerRule.create(
self.apiclient,
self.services["lbrule"],
ipaddressid=public_ip.ipaddress.id,
accountid=self.account.name
)
self.debug("Created LB rule with ID: %s" % lb_rule.id)
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e)
)
first_host = vm.hostid
self.debug("Enabling maintenance mode for host %s" % vm.hostid)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = first_host
self.apiclient.prepareHostForMaintenance(cmd)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
timeout = self.services["timeout"]
# Poll and check state of VM while it migrates from one host to another
while True:
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug("VM 1 state: %s" % vm.state)
if vm.state in ["Stopping",
"Stopped",
"Running",
"Starting",
"Migrating"]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail(
"VM migration from one-host-to-other\
failed while enabling maintenance"
)
second_host = vm.hostid
self.assertEqual(
vm.state,
"Running",
"VM should be in Running state after enabling host maintenance"
)
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e)
)
self.debug("Deploying VM in account: %s" % self.account.name)
# Spawn an instance on other host
virtual_machine_2 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id
)
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine_2.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.debug("VM 2 state: %s" % vm.state)
self.assertEqual(
vm.state,
"Running",
"Deployed VM should be in Running state"
)
self.debug("Canceling host maintenance for ID: %s" % first_host)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = first_host
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % first_host)
self.debug("Enabling maintenance mode for host %s" % second_host)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = second_host
self.apiclient.prepareHostForMaintenance(cmd)
self.debug("Maintenance mode enabled for host: %s" % second_host)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
# Poll and check the status of VMs
timeout = self.services["timeout"]
while True:
vms = VirtualMachine.list(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug(
"VM state after enabling maintenance on first host: %s" %
vm.state)
if vm.state in [
"Stopping",
"Stopped",
"Running",
"Starting",
"Migrating"
]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail(
"VM migration from one-host-to-other failed\
while enabling maintenance"
)
# Poll and check the status of VMs
timeout = self.services["timeout"]
while True:
vms = VirtualMachine.list(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[1]
self.debug(
"VM state after enabling maintenance on first host: %s" %
vm.state)
if vm.state in [
"Stopping",
"Stopped",
"Running",
"Starting",
"Migrating"
]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail(
"VM migration from one-host-to-other\
failed while enabling maintenance"
)
for vm in vms:
self.debug(
"VM states after enabling maintenance mode on host: %s - %s" %
(first_host, vm.state))
self.assertEqual(
vm.state,
"Running",
"Deployed VM should be in Running state"
)
# Spawn an instance on other host
virtual_machine_3 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id
)
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine_3.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.debug("VM 3 state: %s" % vm.state)
self.assertEqual(
vm.state,
"Running",
"Deployed VM should be in Running state"
)
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e)
)
self.debug("Canceling host maintenance for ID: %s" % second_host)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = second_host
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % second_host)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
return