def test_02_host_maintenance_mode_with_activities(self):
"""Test host maintenance mode with activities
"""
# Validate the following
# 1. Create Vms. Acquire IP. Create port forwarding & load balancing
# rules for Vms.
# 2. While activities are ongoing: Create snapshots, recurring
# snapshots, create templates, download volumes, Host 1: put to
# maintenance mode. All Vms should failover to Host 2 in cluster
# Vms should be in running state. All port forwarding rules and
# load balancing Rules should work.
# 3. After failover to Host 2 succeeds, deploy Vms. Deploy Vms on host
# 2 should succeed. All ongoing activities in step 3 should succeed
# 4. Host 1: cancel maintenance mode.
# 5. While activities are ongoing: Create snapshots, recurring
# snapshots, create templates, download volumes, Host 2: put to
# maintenance mode. All Vms should failover to Host 1 in cluster.
# 6. After failover to Host 1 succeeds, deploy VMs. Deploy Vms on
# host 1 should succeed. All ongoing activities in step 6 should
# succeed.
hosts = Host.list(self.apiclient,
zoneid=self.zone.id,
resourcestate='Enabled',
type='Routing')
self.assertEqual(isinstance(hosts, list), True,
"List hosts should return valid host response")
if len(hosts) < 2:
self.skipTest("There must be at least 2 hosts present in cluster")
self.debug("Checking HA with hosts: %s, %s" %
(hosts[0].name, hosts[1].name))
self.debug("Deploying VM in account: %s" % self.account.name)
# Spawn an instance in that network
virtual_machine = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id)
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.assertEqual(vm.state, "Running",
"Deployed VM should be in RUnning state")
networks = Network.list(self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True)
self.assertEqual(
isinstance(networks, list), True,
"List networks should return valid list for the account")
network = networks[0]
self.debug("Associating public IP for account: %s" % self.account.name)
public_ip = PublicIPAddress.create(self.apiclient,
accountid=self.account.name,
zoneid=self.zone.id,
domainid=self.account.domainid,
networkid=network.id)
self.debug("Associated %s with network %s" %
(public_ip.ipaddress.ipaddress, network.id))
self.debug("Creating PF rule for IP address: %s" %
public_ip.ipaddress.ipaddress)
NATRule.create(self.apiclient,
virtual_machine,
self.services["natrule"],
ipaddressid=public_ip.ipaddress.id)
self.debug("Creating LB rule on IP with NAT: %s" %
public_ip.ipaddress.ipaddress)
# Create Load Balancer rule on IP already having NAT rule
lb_rule = LoadBalancerRule.create(self.apiclient,
self.services["lbrule"],
ipaddressid=public_ip.ipaddress.id,
accountid=self.account.name)
self.debug("Created LB rule with ID: %s" % lb_rule.id)
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e))
# Get the Root disk of VM
volumes = list_volumes(self.apiclient,
virtualmachineid=virtual_machine.id,
type='ROOT',
listall=True)
volume = volumes[0]
self.debug("Root volume of VM(%s): %s" %
(virtual_machine.name, volume.name))
# Create a snapshot from the ROOTDISK
self.debug("Creating snapshot on ROOT volume: %s" % volume.name)
snapshot = Snapshot.create(self.apiclient, volumes[0].id)
self.debug("Snapshot created: ID - %s" % snapshot.id)
snapshots = list_snapshots(self.apiclient,
id=snapshot.id,
listall=True)
self.assertEqual(isinstance(snapshots, list), True,
"Check list response returns a valid list")
self.assertNotEqual(snapshots, None,
"Check if result exists in list snapshots call")
self.assertEqual(snapshots[0].id, snapshot.id,
"Check snapshot id in list resources call")
# Generate template from the snapshot
self.debug("Generating template from snapshot: %s" % snapshot.name)
template = Template.create_from_snapshot(self.apiclient, snapshot,
self.services["templates"])
self.debug("Created template from snapshot: %s" % template.id)
templates = list_templates(
self.apiclient,
templatefilter=self.services["templates"]["templatefilter"],
id=template.id)
self.assertEqual(
isinstance(templates, list), True,
"List template call should return the newly created template")
self.assertEqual(
templates[0].isready, True,
"The newly created template should be in ready state")
first_host = vm.hostid
self.debug("Enabling maintenance mode for host %s" % vm.hostid)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = first_host
self.apiclient.prepareHostForMaintenance(cmd)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
timeout = self.services["timeout"]
# Poll and check state of VM while it migrates from one host to another
while True:
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug("VM 1 state: %s" % vm.state)
if vm.state in [
"Stopping", "Stopped", "Running", "Starting", "Migrating"
]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail("VM migration from one-host-to-other failed\
while enabling maintenance")
second_host = vm.hostid
self.assertEqual(
vm.state, "Running",
"VM should be in Running state after enabling host maintenance")
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e))
self.debug("Deploying VM in account: %s" % self.account.name)
# Spawn an instance on other host
virtual_machine_2 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id)
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine_2.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.debug("VM 2 state: %s" % vm.state)
self.assertEqual(vm.state, "Running",
"Deployed VM should be in Running state")
self.debug("Canceling host maintenance for ID: %s" % first_host)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = first_host
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % first_host)
# Get the Root disk of VM
volumes = list_volumes(self.apiclient,
virtualmachineid=virtual_machine_2.id,
type='ROOT',
listall=True)
volume = volumes[0]
self.debug("Root volume of VM(%s): %s" %
(virtual_machine_2.name, volume.name))
# Create a snapshot from the ROOTDISK
self.debug("Creating snapshot on ROOT volume: %s" % volume.name)
snapshot = Snapshot.create(self.apiclient, volumes[0].id)
self.debug("Snapshot created: ID - %s" % snapshot.id)
snapshots = list_snapshots(self.apiclient,
id=snapshot.id,
listall=True)
self.assertEqual(isinstance(snapshots, list), True,
"Check list response returns a valid list")
self.assertNotEqual(snapshots, None,
"Check if result exists in list snapshots call")
self.assertEqual(snapshots[0].id, snapshot.id,
"Check snapshot id in list resources call")
# Generate template from the snapshot
self.debug("Generating template from snapshot: %s" % snapshot.name)
template = Template.create_from_snapshot(self.apiclient, snapshot,
self.services["templates"])
self.debug("Created template from snapshot: %s" % template.id)
templates = list_templates(
self.apiclient,
templatefilter=self.services["templates"]["templatefilter"],
id=template.id)
self.assertEqual(
isinstance(templates, list), True,
"List template call should return the newly created template")
self.assertEqual(
templates[0].isready, True,
"The newly created template should be in ready state")
self.debug("Enabling maintenance mode for host %s" % second_host)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = second_host
self.apiclient.prepareHostForMaintenance(cmd)
self.debug("Maintenance mode enabled for host: %s" % second_host)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
# Poll and check the status of VMs
timeout = self.services["timeout"]
while True:
vms = VirtualMachine.list(self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug(
"VM state after enabling maintenance on first host: %s" %
vm.state)
if vm.state in [
"Stopping", "Stopped", "Running", "Starting", "Migrating"
]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail("VM migration from one-host-to-other failed\
while enabling maintenance")
# Poll and check the status of VMs
timeout = self.services["timeout"]
while True:
vms = VirtualMachine.list(self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[1]
self.debug(
"VM state after enabling maintenance on first host: %s" %
vm.state)
if vm.state in [
"Stopping", "Stopped", "Running", "Starting", "Migrating"
]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail("VM migration from one-host-to-other failed\
while enabling maintenance")
for vm in vms:
self.debug(
"VM states after enabling maintenance mode on host: %s - %s" %
(first_host, vm.state))
self.assertEqual(vm.state, "Running",
"Deployed VM should be in Running state")
# Spawn an instance on other host
virtual_machine_3 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id)
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine_3.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.debug("VM 3 state: %s" % vm.state)
self.assertEqual(vm.state, "Running",
"Deployed VM should be in Running state")
self.debug("Canceling host maintenance for ID: %s" % second_host)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = second_host
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % second_host)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
return
def test_vmware_anti_affinity(self):
""" Test Set up anti-affinity rules
The test requires following pre-requisites
- VMWare cluster configured in fully automated mode
"""
# Validate the following
# 1. Deploy VMs on host 1 and 2
# 2. Enable maintenance mode for host 1
# 3. VM should be migrated to 3rd host
hosts = Host.list(self.apiclient,
zoneid=self.zone.id,
resourcestate='Enabled',
type='Routing')
self.assertEqual(isinstance(hosts, list), True,
"List hosts should return valid host response")
self.debug(len(hosts))
self.assertGreaterEqual(
len(hosts), 3,
"There must be at least 3 hosts present in a cluster")
aff_grp = self.create_aff_grp(
aff_grp=self.services["host_anti_affinity"],
acc=self.account.name,
domainid=self.domain.id)
vm_1 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.domain.id,
serviceofferingid=self.service_offering.id,
affinitygroupnames=[aff_grp.name])
vm_2 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.domain.id,
serviceofferingid=self.service_offering.id,
affinitygroupnames=[aff_grp.name])
host_1 = vm_1.hostid
host_2 = vm_2.hostid
vms = VirtualMachine.list(self.apiclient, id=vm_1.id, listall=True)
vm_list_validation_result = validateList(vms)
self.assertEqual(
vm_list_validation_result[0], PASS,
"vm list validation failed due to %s" %
vm_list_validation_result[1])
virtual_machine_1 = vm_list_validation_result[1]
self.debug("VM State: %s" % virtual_machine_1.state)
self.assertEqual(virtual_machine_1.state, "Running",
"Deployed VM should be in RUnning state")
vms = VirtualMachine.list(self.apiclient, id=vm_2.id, listall=True)
vm_list_validation_result = validateList(vms)
self.assertEqual(
vm_list_validation_result[0], PASS,
"vm list validation failed due to %s" %
vm_list_validation_result[1])
virtual_machine_2 = vm_list_validation_result[1]
self.debug("VM %s State: %s" %
(virtual_machine_2.name, virtual_machine_2.state))
self.assertEqual(virtual_machine_2.state, "Running",
"Deployed VM should be in RUnning state")
self.debug("Enabling maintenance mode on host_1: %s" % host_1)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = host_1
self.apiclient.prepareHostForMaintenance(cmd)
timeout = self.services["timeout"]
while True:
hosts = Host.list(self.apiclient,
zoneid=self.zone.id,
type='Routing',
id=host_1)
host_list_validation_result = validateList(hosts)
self.assertEqual(
host_list_validation_result[0], PASS,
"host list validation failed due to %s" %
host_list_validation_result[2])
host = host_list_validation_result[1]
if host.resourcestate == 'Maintenance':
break
elif timeout == 0:
self.fail("Failed to put host: %s in maintenance mode" %
host.name)
time.sleep(self.services["sleep"])
timeout = timeout - 1
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine_1.id,
listall=True)
vm_list_validation_result = validateList(vms)
self.assertEqual(
vm_list_validation_result[0], PASS,
"vm list validation failed due to %s" %
vm_list_validation_result[2])
vm = vm_list_validation_result[0]
self.assertEqual(vm.state, "Running",
"Deployed VM should be in RUnning state")
self.assertNotEqual(
vm.hostid, host_2,
"The host name should not match with second host name")
self.debug("Canceling host maintenance for ID: %s" % host_1.id)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = host_1.id
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % host_1.id)
return
def test_01_host_maintenance_mode(self):
"""Test host maintenance mode
"""
# Validate the following
# 1. Create Vms. Acquire IP. Create port forwarding & load balancing
# rules for Vms.
# 2. Host 1: put to maintenance mode. All Vms should failover to Host
# 2 in cluster. Vms should be in running state. All port forwarding
# rules and load balancing Rules should work.
# 3. After failover to Host 2 succeeds, deploy Vms. Deploy Vms on host
# 2 should succeed.
# 4. Host 1: cancel maintenance mode.
# 5. Host 2 : put to maintenance mode. All Vms should failover to
# Host 1 in cluster.
# 6. After failover to Host 1 succeeds, deploy VMs. Deploy Vms on
# host 1 should succeed.
hosts = Host.list(self.apiclient,
zoneid=self.zone.id,
resourcestate='Enabled',
type='Routing')
self.assertEqual(isinstance(hosts, list), True,
"List hosts should return valid host response")
if len(hosts) < 2:
self.skipTest("There must be at least 2 hosts present in cluster")
self.debug("Checking HA with hosts: %s, %s" %
(hosts[0].name, hosts[1].name))
self.debug("Deploying VM in account: %s" % self.account.name)
# Spawn an instance in that network
virtual_machine = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id)
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.assertEqual(vm.state, "Running",
"Deployed VM should be in RUnning state")
networks = Network.list(self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True)
self.assertEqual(
isinstance(networks, list), True,
"List networks should return valid list for the account")
network = networks[0]
self.debug("Associating public IP for account: %s" % self.account.name)
public_ip = PublicIPAddress.create(self.apiclient,
accountid=self.account.name,
zoneid=self.zone.id,
domainid=self.account.domainid,
networkid=network.id)
self.debug("Associated %s with network %s" %
(public_ip.ipaddress.ipaddress, network.id))
self.debug("Creating PF rule for IP address: %s" %
public_ip.ipaddress.ipaddress)
NATRule.create(self.apiclient,
virtual_machine,
self.services["natrule"],
ipaddressid=public_ip.ipaddress.id)
self.debug("Creating LB rule on IP with NAT: %s" %
public_ip.ipaddress.ipaddress)
# Create Load Balancer rule on IP already having NAT rule
lb_rule = LoadBalancerRule.create(self.apiclient,
self.services["lbrule"],
ipaddressid=public_ip.ipaddress.id,
accountid=self.account.name)
self.debug("Created LB rule with ID: %s" % lb_rule.id)
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e))
first_host = vm.hostid
self.debug("Enabling maintenance mode for host %s" % vm.hostid)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = first_host
self.apiclient.prepareHostForMaintenance(cmd)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
timeout = self.services["timeout"]
# Poll and check state of VM while it migrates from one host to another
while True:
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug("VM 1 state: %s" % vm.state)
if vm.state in [
"Stopping", "Stopped", "Running", "Starting", "Migrating"
]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail("VM migration from one-host-to-other\
failed while enabling maintenance")
second_host = vm.hostid
self.assertEqual(
vm.state, "Running",
"VM should be in Running state after enabling host maintenance")
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e))
self.debug("Deploying VM in account: %s" % self.account.name)
# Spawn an instance on other host
virtual_machine_2 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id)
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine_2.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.debug("VM 2 state: %s" % vm.state)
self.assertEqual(vm.state, "Running",
"Deployed VM should be in Running state")
self.debug("Canceling host maintenance for ID: %s" % first_host)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = first_host
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % first_host)
self.debug("Enabling maintenance mode for host %s" % second_host)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = second_host
self.apiclient.prepareHostForMaintenance(cmd)
self.debug("Maintenance mode enabled for host: %s" % second_host)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
# Poll and check the status of VMs
timeout = self.services["timeout"]
while True:
vms = VirtualMachine.list(self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug(
"VM state after enabling maintenance on first host: %s" %
vm.state)
if vm.state in [
"Stopping", "Stopped", "Running", "Starting", "Migrating"
]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail("VM migration from one-host-to-other failed\
while enabling maintenance")
# Poll and check the status of VMs
timeout = self.services["timeout"]
while True:
vms = VirtualMachine.list(self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[1]
self.debug(
"VM state after enabling maintenance on first host: %s" %
vm.state)
if vm.state in [
"Stopping", "Stopped", "Running", "Starting", "Migrating"
]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail("VM migration from one-host-to-other\
failed while enabling maintenance")
for vm in vms:
self.debug(
"VM states after enabling maintenance mode on host: %s - %s" %
(first_host, vm.state))
self.assertEqual(vm.state, "Running",
"Deployed VM should be in Running state")
# Spawn an instance on other host
virtual_machine_3 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id)
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine_3.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.debug("VM 3 state: %s" % vm.state)
self.assertEqual(vm.state, "Running",
"Deployed VM should be in Running state")
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e))
self.debug("Canceling host maintenance for ID: %s" % second_host)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = second_host
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % second_host)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
return
def test_vmware_anti_affinity(self):
""" Test Set up anti-affinity rules
The test requires following pre-requisites
- VMWare cluster configured in fully automated mode
"""
# Validate the following
# 1. Deploy VMs on host 1 and 2
# 2. Enable maintenance mode for host 1
# 3. VM should be migrated to 3rd host
hosts = Host.list(
self.apiclient,
zoneid=self.zone.id,
resourcestate='Enabled',
type='Routing'
)
self.assertEqual(
isinstance(hosts, list),
True,
"List hosts should return valid host response"
)
self.debug(len(hosts))
self.assertGreaterEqual(
len(hosts),
3,
"There must be at least 3 hosts present in a cluster"
)
aff_grp = self.create_aff_grp(aff_grp=self.services["host_anti_affinity"], acc=self.account.name, domainid=self.domain.id)
vm_1 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.domain.id,
serviceofferingid=self.service_offering.id,
affinitygroupnames=[aff_grp.name]
)
vm_2 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.domain.id,
serviceofferingid=self.service_offering.id,
affinitygroupnames=[aff_grp.name]
)
host_1 = vm_1.hostid
host_2 = vm_2.hostid
vms = VirtualMachine.list(
self.apiclient,
id=vm_1.id,
listall=True
)
vm_list_validation_result = validateList(vms)
self.assertEqual(vm_list_validation_result[0], PASS, "vm list validation failed due to %s" %
vm_list_validation_result[1])
virtual_machine_1 = vm_list_validation_result[1]
self.debug("VM State: %s" % virtual_machine_1.state)
self.assertEqual(
virtual_machine_1.state,
"Running",
"Deployed VM should be in RUnning state"
)
vms = VirtualMachine.list(
self.apiclient,
id=vm_2.id,
listall=True
)
vm_list_validation_result = validateList(vms)
self.assertEqual(vm_list_validation_result[0], PASS, "vm list validation failed due to %s" %
vm_list_validation_result[1])
virtual_machine_2 = vm_list_validation_result[1]
self.debug("VM %s State: %s" % (
virtual_machine_2.name,
virtual_machine_2.state
))
self.assertEqual(
virtual_machine_2.state,
"Running",
"Deployed VM should be in RUnning state"
)
self.debug("Enabling maintenance mode on host_1: %s" % host_1)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = host_1
self.apiclient.prepareHostForMaintenance(cmd)
timeout = self.services["timeout"]
while True:
hosts = Host.list(
self.apiclient,
zoneid=self.zone.id,
type='Routing',
id=host_1
)
host_list_validation_result = validateList(hosts)
self.assertEqual(host_list_validation_result[0], PASS, "host list validation failed due to %s"
% host_list_validation_result[2])
host = host_list_validation_result[1]
if host.resourcestate == 'Maintenance':
break
elif timeout == 0:
self.fail("Failed to put host: %s in maintenance mode" % host.name)
time.sleep(self.services["sleep"])
timeout = timeout - 1
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine_1.id,
listall=True
)
vm_list_validation_result = validateList(vms)
self.assertEqual(vm_list_validation_result[0], PASS, "vm list validation failed due to %s" %
vm_list_validation_result[2])
vm = vm_list_validation_result[0]
self.assertEqual(
vm.state,
"Running",
"Deployed VM should be in RUnning state"
)
self.assertNotEqual(
vm.hostid,
host_2,
"The host name should not match with second host name"
)
self.debug("Canceling host maintenance for ID: %s" % host_1.id)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = host_1.id
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % host_1.id)
return
def test_02_host_maintenance_mode_with_activities(self):
"""Test host maintenance mode with activities
"""
# Validate the following
# 1. Create Vms. Acquire IP. Create port forwarding & load balancing
# rules for Vms.
# 2. While activities are ongoing: Create snapshots, recurring
# snapshots, create templates, download volumes, Host 1: put to
# maintenance mode. All Vms should failover to Host 2 in cluster
# Vms should be in running state. All port forwarding rules and
# load balancing Rules should work.
# 3. After failover to Host 2 succeeds, deploy Vms. Deploy Vms on host
# 2 should succeed. All ongoing activities in step 3 should succeed
# 4. Host 1: cancel maintenance mode.
# 5. While activities are ongoing: Create snapshots, recurring
# snapshots, create templates, download volumes, Host 2: put to
# maintenance mode. All Vms should failover to Host 1 in cluster.
# 6. After failover to Host 1 succeeds, deploy VMs. Deploy Vms on
# host 1 should succeed. All ongoing activities in step 6 should
# succeed.
hosts = Host.list(
self.apiclient,
zoneid=self.zone.id,
resourcestate='Enabled',
type='Routing'
)
self.assertEqual(
isinstance(hosts, list),
True,
"List hosts should return valid host response"
)
if len(hosts) < 2:
self.skipTest("There must be at least 2 hosts present in cluster")
self.debug("Checking HA with hosts: %s, %s" % (
hosts[0].name,
hosts[1].name
))
self.debug("Deploying VM in account: %s" % self.account.name)
# Spawn an instance in that network
virtual_machine = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id
)
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.assertEqual(
vm.state,
"Running",
"Deployed VM should be in RUnning state"
)
networks = Network.list(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
self.assertEqual(
isinstance(networks, list),
True,
"List networks should return valid list for the account"
)
network = networks[0]
self.debug("Associating public IP for account: %s" %
self.account.name)
public_ip = PublicIPAddress.create(
self.apiclient,
accountid=self.account.name,
zoneid=self.zone.id,
domainid=self.account.domainid,
networkid=network.id
)
self.debug("Associated %s with network %s" % (
public_ip.ipaddress.ipaddress,
network.id
))
self.debug("Creating PF rule for IP address: %s" %
public_ip.ipaddress.ipaddress)
NATRule.create(
self.apiclient,
virtual_machine,
self.services["natrule"],
ipaddressid=public_ip.ipaddress.id
)
self.debug("Creating LB rule on IP with NAT: %s" %
public_ip.ipaddress.ipaddress)
# Create Load Balancer rule on IP already having NAT rule
lb_rule = LoadBalancerRule.create(
self.apiclient,
self.services["lbrule"],
ipaddressid=public_ip.ipaddress.id,
accountid=self.account.name
)
self.debug("Created LB rule with ID: %s" % lb_rule.id)
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e)
)
# Get the Root disk of VM
volumes = list_volumes(
self.apiclient,
virtualmachineid=virtual_machine.id,
type='ROOT',
listall=True
)
volume = volumes[0]
self.debug(
"Root volume of VM(%s): %s" % (
virtual_machine.name,
volume.name
))
# Create a snapshot from the ROOTDISK
self.debug("Creating snapshot on ROOT volume: %s" % volume.name)
snapshot = Snapshot.create(self.apiclient, volumes[0].id)
self.debug("Snapshot created: ID - %s" % snapshot.id)
snapshots = list_snapshots(
self.apiclient,
id=snapshot.id,
listall=True
)
self.assertEqual(
isinstance(snapshots, list),
True,
"Check list response returns a valid list"
)
self.assertNotEqual(
snapshots,
None,
"Check if result exists in list snapshots call"
)
self.assertEqual(
snapshots[0].id,
snapshot.id,
"Check snapshot id in list resources call"
)
# Generate template from the snapshot
self.debug("Generating template from snapshot: %s" % snapshot.name)
template = Template.create_from_snapshot(
self.apiclient,
snapshot,
self.services["templates"]
)
self.debug("Created template from snapshot: %s" % template.id)
templates = list_templates(
self.apiclient,
templatefilter=self.services["templates"]["templatefilter"],
id=template.id
)
self.assertEqual(
isinstance(templates, list),
True,
"List template call should return the newly created template"
)
self.assertEqual(
templates[0].isready,
True,
"The newly created template should be in ready state"
)
first_host = vm.hostid
self.debug("Enabling maintenance mode for host %s" % vm.hostid)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = first_host
self.apiclient.prepareHostForMaintenance(cmd)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
timeout = self.services["timeout"]
# Poll and check state of VM while it migrates from one host to another
while True:
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug("VM 1 state: %s" % vm.state)
if vm.state in ["Stopping",
"Stopped",
"Running",
"Starting",
"Migrating"]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail(
"VM migration from one-host-to-other failed\
while enabling maintenance"
)
second_host = vm.hostid
self.assertEqual(
vm.state,
"Running",
"VM should be in Running state after enabling host maintenance"
)
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e)
)
self.debug("Deploying VM in account: %s" % self.account.name)
# Spawn an instance on other host
virtual_machine_2 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id
)
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine_2.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.debug("VM 2 state: %s" % vm.state)
self.assertEqual(
vm.state,
"Running",
"Deployed VM should be in Running state"
)
self.debug("Canceling host maintenance for ID: %s" % first_host)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = first_host
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % first_host)
# Get the Root disk of VM
volumes = list_volumes(
self.apiclient,
virtualmachineid=virtual_machine_2.id,
type='ROOT',
listall=True
)
volume = volumes[0]
self.debug(
"Root volume of VM(%s): %s" % (
virtual_machine_2.name,
volume.name
))
# Create a snapshot from the ROOTDISK
self.debug("Creating snapshot on ROOT volume: %s" % volume.name)
snapshot = Snapshot.create(self.apiclient, volumes[0].id)
self.debug("Snapshot created: ID - %s" % snapshot.id)
snapshots = list_snapshots(
self.apiclient,
id=snapshot.id,
listall=True
)
self.assertEqual(
isinstance(snapshots, list),
True,
"Check list response returns a valid list"
)
self.assertNotEqual(
snapshots,
None,
"Check if result exists in list snapshots call"
)
self.assertEqual(
snapshots[0].id,
snapshot.id,
"Check snapshot id in list resources call"
)
# Generate template from the snapshot
self.debug("Generating template from snapshot: %s" % snapshot.name)
template = Template.create_from_snapshot(
self.apiclient,
snapshot,
self.services["templates"]
)
self.debug("Created template from snapshot: %s" % template.id)
templates = list_templates(
self.apiclient,
templatefilter=self.services["templates"]["templatefilter"],
id=template.id
)
self.assertEqual(
isinstance(templates, list),
True,
"List template call should return the newly created template"
)
self.assertEqual(
templates[0].isready,
True,
"The newly created template should be in ready state"
)
self.debug("Enabling maintenance mode for host %s" % second_host)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = second_host
self.apiclient.prepareHostForMaintenance(cmd)
self.debug("Maintenance mode enabled for host: %s" % second_host)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
# Poll and check the status of VMs
timeout = self.services["timeout"]
while True:
vms = VirtualMachine.list(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug(
"VM state after enabling maintenance on first host: %s" %
vm.state)
if vm.state in ["Stopping",
"Stopped",
"Running",
"Starting",
"Migrating"]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail(
"VM migration from one-host-to-other failed\
while enabling maintenance"
)
# Poll and check the status of VMs
timeout = self.services["timeout"]
while True:
vms = VirtualMachine.list(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[1]
self.debug(
"VM state after enabling maintenance on first host: %s" %
vm.state)
if vm.state in ["Stopping",
"Stopped",
"Running",
"Starting",
"Migrating"]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail(
"VM migration from one-host-to-other failed\
while enabling maintenance"
)
for vm in vms:
self.debug(
"VM states after enabling maintenance mode on host: %s - %s" %
(first_host, vm.state))
self.assertEqual(
vm.state,
"Running",
"Deployed VM should be in Running state"
)
# Spawn an instance on other host
virtual_machine_3 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id
)
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine_3.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.debug("VM 3 state: %s" % vm.state)
self.assertEqual(
vm.state,
"Running",
"Deployed VM should be in Running state"
)
self.debug("Canceling host maintenance for ID: %s" % second_host)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = second_host
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % second_host)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
return
def test_01_host_maintenance_mode(self):
"""Test host maintenance mode
"""
# Validate the following
# 1. Create Vms. Acquire IP. Create port forwarding & load balancing
# rules for Vms.
# 2. Host 1: put to maintenance mode. All Vms should failover to Host
# 2 in cluster. Vms should be in running state. All port forwarding
# rules and load balancing Rules should work.
# 3. After failover to Host 2 succeeds, deploy Vms. Deploy Vms on host
# 2 should succeed.
# 4. Host 1: cancel maintenance mode.
# 5. Host 2 : put to maintenance mode. All Vms should failover to
# Host 1 in cluster.
# 6. After failover to Host 1 succeeds, deploy VMs. Deploy Vms on
# host 1 should succeed.
hosts = Host.list(
self.apiclient,
zoneid=self.zone.id,
resourcestate='Enabled',
type='Routing'
)
self.assertEqual(
isinstance(hosts, list),
True,
"List hosts should return valid host response"
)
if len(hosts) < 2:
self.skipTest("There must be at least 2 hosts present in cluster")
self.debug("Checking HA with hosts: %s, %s" % (
hosts[0].name,
hosts[1].name
))
self.debug("Deploying VM in account: %s" % self.account.name)
# Spawn an instance in that network
virtual_machine = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id
)
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.assertEqual(
vm.state,
"Running",
"Deployed VM should be in RUnning state"
)
networks = Network.list(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
self.assertEqual(
isinstance(networks, list),
True,
"List networks should return valid list for the account"
)
network = networks[0]
self.debug("Associating public IP for account: %s" %
self.account.name)
public_ip = PublicIPAddress.create(
self.apiclient,
accountid=self.account.name,
zoneid=self.zone.id,
domainid=self.account.domainid,
networkid=network.id
)
self.debug("Associated %s with network %s" % (
public_ip.ipaddress.ipaddress,
network.id
))
self.debug("Creating PF rule for IP address: %s" %
public_ip.ipaddress.ipaddress)
NATRule.create(
self.apiclient,
virtual_machine,
self.services["natrule"],
ipaddressid=public_ip.ipaddress.id
)
self.debug("Creating LB rule on IP with NAT: %s" %
public_ip.ipaddress.ipaddress)
# Create Load Balancer rule on IP already having NAT rule
lb_rule = LoadBalancerRule.create(
self.apiclient,
self.services["lbrule"],
ipaddressid=public_ip.ipaddress.id,
accountid=self.account.name
)
self.debug("Created LB rule with ID: %s" % lb_rule.id)
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e)
)
first_host = vm.hostid
self.debug("Enabling maintenance mode for host %s" % vm.hostid)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = first_host
self.apiclient.prepareHostForMaintenance(cmd)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
timeout = self.services["timeout"]
# Poll and check state of VM while it migrates from one host to another
while True:
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug("VM 1 state: %s" % vm.state)
if vm.state in ["Stopping",
"Stopped",
"Running",
"Starting",
"Migrating"]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail(
"VM migration from one-host-to-other\
failed while enabling maintenance"
)
second_host = vm.hostid
self.assertEqual(
vm.state,
"Running",
"VM should be in Running state after enabling host maintenance"
)
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e)
)
self.debug("Deploying VM in account: %s" % self.account.name)
# Spawn an instance on other host
virtual_machine_2 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id
)
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine_2.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.debug("VM 2 state: %s" % vm.state)
self.assertEqual(
vm.state,
"Running",
"Deployed VM should be in Running state"
)
self.debug("Canceling host maintenance for ID: %s" % first_host)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = first_host
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % first_host)
self.debug("Enabling maintenance mode for host %s" % second_host)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = second_host
self.apiclient.prepareHostForMaintenance(cmd)
self.debug("Maintenance mode enabled for host: %s" % second_host)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
# Poll and check the status of VMs
timeout = self.services["timeout"]
while True:
vms = VirtualMachine.list(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug(
"VM state after enabling maintenance on first host: %s" %
vm.state)
if vm.state in [
"Stopping",
"Stopped",
"Running",
"Starting",
"Migrating"
]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail(
"VM migration from one-host-to-other failed\
while enabling maintenance"
)
# Poll and check the status of VMs
timeout = self.services["timeout"]
while True:
vms = VirtualMachine.list(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[1]
self.debug(
"VM state after enabling maintenance on first host: %s" %
vm.state)
if vm.state in [
"Stopping",
"Stopped",
"Running",
"Starting",
"Migrating"
]:
if vm.state == "Running":
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
else:
self.fail(
"VM migration from one-host-to-other\
failed while enabling maintenance"
)
for vm in vms:
self.debug(
"VM states after enabling maintenance mode on host: %s - %s" %
(first_host, vm.state))
self.assertEqual(
vm.state,
"Running",
"Deployed VM should be in Running state"
)
# Spawn an instance on other host
virtual_machine_3 = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id
)
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine_3.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm = vms[0]
self.debug("Deployed VM on host: %s" % vm.hostid)
self.debug("VM 3 state: %s" % vm.state)
self.assertEqual(
vm.state,
"Running",
"Deployed VM should be in Running state"
)
# Should be able to SSH VM
try:
self.debug("SSH into VM: %s" % virtual_machine.id)
virtual_machine.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress)
except Exception as e:
self.fail("SSH Access failed for %s: %s" %
(virtual_machine.ipaddress, e)
)
self.debug("Canceling host maintenance for ID: %s" % second_host)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = second_host
self.apiclient.cancelHostMaintenance(cmd)
self.debug("Maintenance mode canceled for host: %s" % second_host)
self.debug("Waiting for SSVMs to come up")
wait_for_ssvms(
self.apiclient,
zoneid=self.zone.id,
podid=self.pod.id,
)
return
def test_06_no_vm_without_ha_gets_migrated_to_ha_host_in_live_migration(
self):
""" Verify that none of the VMs without HA enabled migrate to an ha tagged host during live migration """
# Steps,
#1. Fresh install CS that supports this feature
#2. Create Basic zone, pod, cluster, add 3 hosts to cluster (host1, host2, host3), secondary & primary Storage
#3. When adding host3, assign the HA host tag.
#4. Create VMs with and without the Compute Service Offering with the HA tag.
#5. Note the VMs on host1 and whether any of the VMs have their 'HA enabled' flags enabled.
#6. Put host1 into maintenance mode.
# Validations,
#1. Make sure the VMs are created on either host1 or host2 and not on host3
#2. Putting host1 into maintenance mode should trigger a live migration. Make sure the VMs are not migrated to HA enabled host3.
# create and verify virtual machine with HA disabled service offering
self.hypervisor = self.testClient.getHypervisorInfo()
if self.hypervisor.lower() in ['lxc']:
self.skipTest("vm migrate is not supported in %s" %
self.hypervisor)
virtual_machine_without_ha = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering_without_ha.id)
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine_without_ha.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
vm_with_ha_disabled = vms[0]
#Verify the virtual machine got created on non HA host
list_hosts_response = list_hosts(self.apiclient,
id=vm_with_ha_disabled.hostid)
self.assertEqual(isinstance(list_hosts_response, list), True,
"Check list response returns a valid list")
self.assertNotEqual(len(list_hosts_response), 0,
"Check Host is available")
self.assertEqual(
list_hosts_response[0].hahost, False,
"The virtual machine is not ha enabled so check if VM is created on host which is also not ha enabled"
)
#put the Host in maintenance mode
self.debug("Enabling maintenance mode for host %s" %
vm_with_ha_disabled.hostid)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = vm_with_ha_disabled.hostid
self.apiclient.prepareHostForMaintenance(cmd)
timeout = self.services["timeout"]
#verify the VM live migration happened to another running host
self.debug("Waiting for VM to come up")
time.sleep(timeout)
vms = VirtualMachine.list(self.apiclient,
id=vm_with_ha_disabled.id,
listall=True)
self.assertEqual(
isinstance(vms, list), True,
"List VMs should return valid response for deployed VM")
self.assertNotEqual(
len(vms), 0,
"List VMs should return valid response for deployed VM")
list_hosts_response = list_hosts(self.apiclient, id=vms[0].hostid)
self.assertEqual(isinstance(list_hosts_response, list), True,
"Check list response returns a valid list")
self.assertNotEqual(len(list_hosts_response), 0,
"Check Host is available")
self.assertEqual(
list_hosts_response[0].hahost, False,
"The virtual machine is not ha enabled so check if VM is created on host which is also not ha enabled"
)
self.debug("Disabling the maintenance mode for host %s" %
vm_with_ha_disabled.hostid)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = vm_with_ha_disabled.hostid
self.apiclient.cancelHostMaintenance(cmd)
def test_06_no_vm_without_ha_gets_migrated_to_ha_host_in_live_migration(self):
""" Verify that none of the VMs without HA enabled migrate to an ha tagged host during live migration """
# Steps,
#1. Fresh install CS that supports this feature
#2. Create Basic zone, pod, cluster, add 3 hosts to cluster (host1, host2, host3), secondary & primary Storage
#3. When adding host3, assign the HA host tag.
#4. Create VMs with and without the Compute Service Offering with the HA tag.
#5. Note the VMs on host1 and whether any of the VMs have their 'HA enabled' flags enabled.
#6. Put host1 into maintenance mode.
# Validations,
#1. Make sure the VMs are created on either host1 or host2 and not on host3
#2. Putting host1 into maintenance mode should trigger a live migration. Make sure the VMs are not migrated to HA enabled host3.
# create and verify virtual machine with HA disabled service offering
self.hypervisor = self.testClient.getHypervisorInfo()
if self.hypervisor.lower() in ['lxc']:
self.skipTest("vm migrate is not supported in %s" % self.hypervisor)
virtual_machine_without_ha = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering_without_ha.id
)
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine_without_ha.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
vm_with_ha_disabled = vms[0]
#Verify the virtual machine got created on non HA host
list_hosts_response = list_hosts(
self.apiclient,
id=vm_with_ha_disabled.hostid
)
self.assertEqual(
isinstance(list_hosts_response, list),
True,
"Check list response returns a valid list"
)
self.assertNotEqual(
len(list_hosts_response),
0,
"Check Host is available"
)
self.assertEqual(
list_hosts_response[0].hahost,
False,
"The virtual machine is not ha enabled so check if VM is created on host which is also not ha enabled"
)
#put the Host in maintenance mode
self.debug("Enabling maintenance mode for host %s" % vm_with_ha_disabled.hostid)
cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
cmd.id = vm_with_ha_disabled.hostid
self.apiclient.prepareHostForMaintenance(cmd)
timeout = self.services["timeout"]
#verify the VM live migration happened to another running host
self.debug("Waiting for VM to come up")
time.sleep(timeout)
vms = VirtualMachine.list(
self.apiclient,
id=vm_with_ha_disabled.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List VMs should return valid response for deployed VM"
)
self.assertNotEqual(
len(vms),
0,
"List VMs should return valid response for deployed VM"
)
list_hosts_response = list_hosts(
self.apiclient,
id=vms[0].hostid
)
self.assertEqual(
isinstance(list_hosts_response, list),
True,
"Check list response returns a valid list"
)
self.assertNotEqual(
len(list_hosts_response),
0,
"Check Host is available"
)
self.assertEqual(
list_hosts_response[0].hahost,
False,
"The virtual machine is not ha enabled so check if VM is created on host which is also not ha enabled"
)
self.debug("Disabling the maintenance mode for host %s" % vm_with_ha_disabled.hostid)
cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
cmd.id = vm_with_ha_disabled.hostid
self.apiclient.cancelHostMaintenance(cmd)