Esempio n. 1
0
    def test_02_host_maintenance_mode_with_activities(self):
        """Test host maintenance mode with activities
        """

        # Validate the following
        # 1. Create Vms. Acquire IP. Create port forwarding & load balancing
        #    rules for Vms.
        # 2. While activities are ongoing: Create snapshots, recurring
        #    snapshots, create templates, download volumes, Host 1: put to
        #    maintenance mode. All Vms should failover to Host 2 in cluster
        #    Vms should be in running state. All port forwarding rules and
        #    load balancing Rules should work.
        # 3. After failover to Host 2 succeeds, deploy Vms. Deploy Vms on host
        #    2 should succeed. All ongoing activities in step 3 should succeed
        # 4. Host 1: cancel maintenance mode.
        # 5. While activities are ongoing: Create snapshots, recurring
        #    snapshots, create templates, download volumes, Host 2: put to
        #    maintenance mode. All Vms should failover to Host 1 in cluster.
        # 6. After failover to Host 1 succeeds, deploy VMs. Deploy Vms on
        #    host 1 should succeed. All ongoing activities in step 6 should
        #    succeed.

        hosts = Host.list(self.apiclient,
                          zoneid=self.zone.id,
                          resourcestate='Enabled',
                          type='Routing')
        self.assertEqual(isinstance(hosts, list), True,
                         "List hosts should return valid host response")
        if len(hosts) < 2:
            self.skipTest("There must be at least 2 hosts present in cluster")

        self.debug("Checking HA with hosts: %s, %s" %
                   (hosts[0].name, hosts[1].name))
        self.debug("Deploying VM in account: %s" % self.account.name)
        # Spawn an instance in that network
        virtual_machine = VirtualMachine.create(
            self.apiclient,
            self.services["virtual_machine"],
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering.id)
        vms = VirtualMachine.list(self.apiclient,
                                  id=virtual_machine.id,
                                  listall=True)
        self.assertEqual(
            isinstance(vms, list), True,
            "List VMs should return valid response for deployed VM")
        self.assertNotEqual(
            len(vms), 0,
            "List VMs should return valid response for deployed VM")
        vm = vms[0]
        self.debug("Deployed VM on host: %s" % vm.hostid)
        self.assertEqual(vm.state, "Running",
                         "Deployed VM should be in RUnning state")
        networks = Network.list(self.apiclient,
                                account=self.account.name,
                                domainid=self.account.domainid,
                                listall=True)
        self.assertEqual(
            isinstance(networks, list), True,
            "List networks should return valid list for the account")
        network = networks[0]

        self.debug("Associating public IP for account: %s" % self.account.name)
        public_ip = PublicIPAddress.create(self.apiclient,
                                           accountid=self.account.name,
                                           zoneid=self.zone.id,
                                           domainid=self.account.domainid,
                                           networkid=network.id)

        self.debug("Associated %s with network %s" %
                   (public_ip.ipaddress.ipaddress, network.id))
        self.debug("Creating PF rule for IP address: %s" %
                   public_ip.ipaddress.ipaddress)
        NATRule.create(self.apiclient,
                       virtual_machine,
                       self.services["natrule"],
                       ipaddressid=public_ip.ipaddress.id)

        self.debug("Creating LB rule on IP with NAT: %s" %
                   public_ip.ipaddress.ipaddress)

        # Create Load Balancer rule on IP already having NAT rule
        lb_rule = LoadBalancerRule.create(self.apiclient,
                                          self.services["lbrule"],
                                          ipaddressid=public_ip.ipaddress.id,
                                          accountid=self.account.name)
        self.debug("Created LB rule with ID: %s" % lb_rule.id)

        # Should be able to SSH VM
        try:
            self.debug("SSH into VM: %s" % virtual_machine.id)
            virtual_machine.get_ssh_client(
                ipaddress=public_ip.ipaddress.ipaddress)
        except Exception as e:
            self.fail("SSH Access failed for %s: %s" %
                      (virtual_machine.ipaddress, e))
        # Get the Root disk of VM
        volumes = list_volumes(self.apiclient,
                               virtualmachineid=virtual_machine.id,
                               type='ROOT',
                               listall=True)
        volume = volumes[0]
        self.debug("Root volume of VM(%s): %s" %
                   (virtual_machine.name, volume.name))
        # Create a snapshot from the ROOTDISK
        self.debug("Creating snapshot on ROOT volume: %s" % volume.name)
        snapshot = Snapshot.create(self.apiclient, volumes[0].id)
        self.debug("Snapshot created: ID - %s" % snapshot.id)

        snapshots = list_snapshots(self.apiclient,
                                   id=snapshot.id,
                                   listall=True)
        self.assertEqual(isinstance(snapshots, list), True,
                         "Check list response returns a valid list")
        self.assertNotEqual(snapshots, None,
                            "Check if result exists in list snapshots call")
        self.assertEqual(snapshots[0].id, snapshot.id,
                         "Check snapshot id in list resources call")

        # Generate template from the snapshot
        self.debug("Generating template from snapshot: %s" % snapshot.name)
        template = Template.create_from_snapshot(self.apiclient, snapshot,
                                                 self.services["templates"])
        self.debug("Created template from snapshot: %s" % template.id)

        templates = list_templates(
            self.apiclient,
            templatefilter=self.services["templates"]["templatefilter"],
            id=template.id)

        self.assertEqual(
            isinstance(templates, list), True,
            "List template call should return the newly created template")

        self.assertEqual(
            templates[0].isready, True,
            "The newly created template should be in ready state")

        first_host = vm.hostid
        self.debug("Enabling maintenance mode for host %s" % vm.hostid)
        cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
        cmd.id = first_host
        self.apiclient.prepareHostForMaintenance(cmd)

        self.debug("Waiting for SSVMs to come up")
        wait_for_ssvms(
            self.apiclient,
            zoneid=self.zone.id,
            podid=self.pod.id,
        )

        timeout = self.services["timeout"]
        # Poll and check state of VM while it migrates from one host to another
        while True:
            vms = VirtualMachine.list(self.apiclient,
                                      id=virtual_machine.id,
                                      listall=True)
            self.assertEqual(
                isinstance(vms, list), True,
                "List VMs should return valid response for deployed VM")
            self.assertNotEqual(
                len(vms), 0,
                "List VMs should return valid response for deployed VM")
            vm = vms[0]

            self.debug("VM 1 state: %s" % vm.state)
            if vm.state in [
                    "Stopping", "Stopped", "Running", "Starting", "Migrating"
            ]:
                if vm.state == "Running":
                    break
                else:
                    time.sleep(self.services["sleep"])
                    timeout = timeout - 1
            else:
                self.fail("VM migration from one-host-to-other failed\
                            while enabling maintenance")
        second_host = vm.hostid
        self.assertEqual(
            vm.state, "Running",
            "VM should be in Running state after enabling host maintenance")
        # Should be able to SSH VM
        try:
            self.debug("SSH into VM: %s" % virtual_machine.id)
            virtual_machine.get_ssh_client(
                ipaddress=public_ip.ipaddress.ipaddress)
        except Exception as e:
            self.fail("SSH Access failed for %s: %s" %
                      (virtual_machine.ipaddress, e))
        self.debug("Deploying VM in account: %s" % self.account.name)
        # Spawn an instance on other host
        virtual_machine_2 = VirtualMachine.create(
            self.apiclient,
            self.services["virtual_machine"],
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering.id)
        vms = VirtualMachine.list(self.apiclient,
                                  id=virtual_machine_2.id,
                                  listall=True)
        self.assertEqual(
            isinstance(vms, list), True,
            "List VMs should return valid response for deployed VM")
        self.assertNotEqual(
            len(vms), 0,
            "List VMs should return valid response for deployed VM")
        vm = vms[0]
        self.debug("Deployed VM on host: %s" % vm.hostid)
        self.debug("VM 2 state: %s" % vm.state)
        self.assertEqual(vm.state, "Running",
                         "Deployed VM should be in Running state")

        self.debug("Canceling host maintenance for ID: %s" % first_host)
        cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
        cmd.id = first_host
        self.apiclient.cancelHostMaintenance(cmd)
        self.debug("Maintenance mode canceled for host: %s" % first_host)

        # Get the Root disk of VM
        volumes = list_volumes(self.apiclient,
                               virtualmachineid=virtual_machine_2.id,
                               type='ROOT',
                               listall=True)
        volume = volumes[0]
        self.debug("Root volume of VM(%s): %s" %
                   (virtual_machine_2.name, volume.name))
        # Create a snapshot from the ROOTDISK
        self.debug("Creating snapshot on ROOT volume: %s" % volume.name)
        snapshot = Snapshot.create(self.apiclient, volumes[0].id)
        self.debug("Snapshot created: ID - %s" % snapshot.id)

        snapshots = list_snapshots(self.apiclient,
                                   id=snapshot.id,
                                   listall=True)
        self.assertEqual(isinstance(snapshots, list), True,
                         "Check list response returns a valid list")
        self.assertNotEqual(snapshots, None,
                            "Check if result exists in list snapshots call")
        self.assertEqual(snapshots[0].id, snapshot.id,
                         "Check snapshot id in list resources call")

        # Generate template from the snapshot
        self.debug("Generating template from snapshot: %s" % snapshot.name)
        template = Template.create_from_snapshot(self.apiclient, snapshot,
                                                 self.services["templates"])
        self.debug("Created template from snapshot: %s" % template.id)

        templates = list_templates(
            self.apiclient,
            templatefilter=self.services["templates"]["templatefilter"],
            id=template.id)

        self.assertEqual(
            isinstance(templates, list), True,
            "List template call should return the newly created template")

        self.assertEqual(
            templates[0].isready, True,
            "The newly created template should be in ready state")

        self.debug("Enabling maintenance mode for host %s" % second_host)
        cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
        cmd.id = second_host
        self.apiclient.prepareHostForMaintenance(cmd)
        self.debug("Maintenance mode enabled for host: %s" % second_host)

        self.debug("Waiting for SSVMs to come up")
        wait_for_ssvms(
            self.apiclient,
            zoneid=self.zone.id,
            podid=self.pod.id,
        )

        # Poll and check the status of VMs
        timeout = self.services["timeout"]
        while True:
            vms = VirtualMachine.list(self.apiclient,
                                      account=self.account.name,
                                      domainid=self.account.domainid,
                                      listall=True)
            self.assertEqual(
                isinstance(vms, list), True,
                "List VMs should return valid response for deployed VM")
            self.assertNotEqual(
                len(vms), 0,
                "List VMs should return valid response for deployed VM")
            vm = vms[0]
            self.debug(
                "VM state after enabling maintenance on first host: %s" %
                vm.state)
            if vm.state in [
                    "Stopping", "Stopped", "Running", "Starting", "Migrating"
            ]:
                if vm.state == "Running":
                    break
                else:
                    time.sleep(self.services["sleep"])
                    timeout = timeout - 1
            else:
                self.fail("VM migration from one-host-to-other failed\
                            while enabling maintenance")

        # Poll and check the status of VMs
        timeout = self.services["timeout"]
        while True:
            vms = VirtualMachine.list(self.apiclient,
                                      account=self.account.name,
                                      domainid=self.account.domainid,
                                      listall=True)
            self.assertEqual(
                isinstance(vms, list), True,
                "List VMs should return valid response for deployed VM")
            self.assertNotEqual(
                len(vms), 0,
                "List VMs should return valid response for deployed VM")
            vm = vms[1]
            self.debug(
                "VM state after enabling maintenance on first host: %s" %
                vm.state)
            if vm.state in [
                    "Stopping", "Stopped", "Running", "Starting", "Migrating"
            ]:
                if vm.state == "Running":
                    break
                else:
                    time.sleep(self.services["sleep"])
                    timeout = timeout - 1
            else:
                self.fail("VM migration from one-host-to-other failed\
                            while enabling maintenance")

        for vm in vms:
            self.debug(
                "VM states after enabling maintenance mode on host: %s - %s" %
                (first_host, vm.state))
            self.assertEqual(vm.state, "Running",
                             "Deployed VM should be in Running state")

        # Spawn an instance on other host
        virtual_machine_3 = VirtualMachine.create(
            self.apiclient,
            self.services["virtual_machine"],
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering.id)
        vms = VirtualMachine.list(self.apiclient,
                                  id=virtual_machine_3.id,
                                  listall=True)
        self.assertEqual(
            isinstance(vms, list), True,
            "List VMs should return valid response for deployed VM")
        self.assertNotEqual(
            len(vms), 0,
            "List VMs should return valid response for deployed VM")
        vm = vms[0]

        self.debug("Deployed VM on host: %s" % vm.hostid)
        self.debug("VM 3 state: %s" % vm.state)
        self.assertEqual(vm.state, "Running",
                         "Deployed VM should be in Running state")

        self.debug("Canceling host maintenance for ID: %s" % second_host)
        cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
        cmd.id = second_host
        self.apiclient.cancelHostMaintenance(cmd)
        self.debug("Maintenance mode canceled for host: %s" % second_host)

        self.debug("Waiting for SSVMs to come up")
        wait_for_ssvms(
            self.apiclient,
            zoneid=self.zone.id,
            podid=self.pod.id,
        )
        return
    def test_vmware_anti_affinity(self):
        """ Test Set up anti-affinity rules

            The test requires following pre-requisites
            - VMWare cluster configured in fully automated mode
        """

        # Validate the following
        # 1. Deploy VMs on host 1 and 2
        # 2. Enable maintenance mode for host 1
        # 3. VM should be migrated to 3rd host

        hosts = Host.list(self.apiclient,
                          zoneid=self.zone.id,
                          resourcestate='Enabled',
                          type='Routing')
        self.assertEqual(isinstance(hosts, list), True,
                         "List hosts should return valid host response")

        self.debug(len(hosts))

        self.assertGreaterEqual(
            len(hosts), 3,
            "There must be at least 3 hosts present in a cluster")

        aff_grp = self.create_aff_grp(
            aff_grp=self.services["host_anti_affinity"],
            acc=self.account.name,
            domainid=self.domain.id)

        vm_1 = VirtualMachine.create(
            self.apiclient,
            self.services["virtual_machine"],
            accountid=self.account.name,
            domainid=self.domain.id,
            serviceofferingid=self.service_offering.id,
            affinitygroupnames=[aff_grp.name])

        vm_2 = VirtualMachine.create(
            self.apiclient,
            self.services["virtual_machine"],
            accountid=self.account.name,
            domainid=self.domain.id,
            serviceofferingid=self.service_offering.id,
            affinitygroupnames=[aff_grp.name])

        host_1 = vm_1.hostid

        host_2 = vm_2.hostid

        vms = VirtualMachine.list(self.apiclient, id=vm_1.id, listall=True)
        vm_list_validation_result = validateList(vms)

        self.assertEqual(
            vm_list_validation_result[0], PASS,
            "vm list validation failed due to %s" %
            vm_list_validation_result[1])

        virtual_machine_1 = vm_list_validation_result[1]

        self.debug("VM State: %s" % virtual_machine_1.state)
        self.assertEqual(virtual_machine_1.state, "Running",
                         "Deployed VM should be in RUnning state")

        vms = VirtualMachine.list(self.apiclient, id=vm_2.id, listall=True)
        vm_list_validation_result = validateList(vms)

        self.assertEqual(
            vm_list_validation_result[0], PASS,
            "vm list validation failed due to %s" %
            vm_list_validation_result[1])

        virtual_machine_2 = vm_list_validation_result[1]

        self.debug("VM %s  State: %s" %
                   (virtual_machine_2.name, virtual_machine_2.state))
        self.assertEqual(virtual_machine_2.state, "Running",
                         "Deployed VM should be in RUnning state")
        self.debug("Enabling maintenance mode on host_1: %s" % host_1)

        cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
        cmd.id = host_1
        self.apiclient.prepareHostForMaintenance(cmd)

        timeout = self.services["timeout"]
        while True:
            hosts = Host.list(self.apiclient,
                              zoneid=self.zone.id,
                              type='Routing',
                              id=host_1)
            host_list_validation_result = validateList(hosts)

            self.assertEqual(
                host_list_validation_result[0], PASS,
                "host list validation failed due to %s" %
                host_list_validation_result[2])

            host = host_list_validation_result[1]

            if host.resourcestate == 'Maintenance':
                break
            elif timeout == 0:
                self.fail("Failed to put host: %s in maintenance mode" %
                          host.name)

            time.sleep(self.services["sleep"])
            timeout = timeout - 1

        vms = VirtualMachine.list(self.apiclient,
                                  id=virtual_machine_1.id,
                                  listall=True)
        vm_list_validation_result = validateList(vms)

        self.assertEqual(
            vm_list_validation_result[0], PASS,
            "vm list validation failed due to %s" %
            vm_list_validation_result[2])

        vm = vm_list_validation_result[0]

        self.assertEqual(vm.state, "Running",
                         "Deployed VM should be in RUnning state")
        self.assertNotEqual(
            vm.hostid, host_2,
            "The host name should not match with second host name")

        self.debug("Canceling host maintenance for ID: %s" % host_1.id)
        cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
        cmd.id = host_1.id
        self.apiclient.cancelHostMaintenance(cmd)
        self.debug("Maintenance mode canceled for host: %s" % host_1.id)

        return
Esempio n. 3
0
    def test_01_host_maintenance_mode(self):
        """Test host maintenance mode
        """

        # Validate the following
        # 1. Create Vms. Acquire IP. Create port forwarding & load balancing
        #    rules for Vms.
        # 2. Host 1: put to maintenance mode. All Vms should failover to Host
        #    2 in cluster. Vms should be in running state. All port forwarding
        #    rules and load balancing Rules should work.
        # 3. After failover to Host 2 succeeds, deploy Vms. Deploy Vms on host
        #    2 should succeed.
        # 4. Host 1: cancel maintenance mode.
        # 5. Host 2 : put to maintenance mode. All Vms should failover to
        #    Host 1 in cluster.
        # 6. After failover to Host 1 succeeds, deploy VMs. Deploy Vms on
        #    host 1 should succeed.

        hosts = Host.list(self.apiclient,
                          zoneid=self.zone.id,
                          resourcestate='Enabled',
                          type='Routing')
        self.assertEqual(isinstance(hosts, list), True,
                         "List hosts should return valid host response")
        if len(hosts) < 2:
            self.skipTest("There must be at least 2 hosts present in cluster")

        self.debug("Checking HA with hosts: %s, %s" %
                   (hosts[0].name, hosts[1].name))
        self.debug("Deploying VM in account: %s" % self.account.name)
        # Spawn an instance in that network
        virtual_machine = VirtualMachine.create(
            self.apiclient,
            self.services["virtual_machine"],
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering.id)
        vms = VirtualMachine.list(self.apiclient,
                                  id=virtual_machine.id,
                                  listall=True)
        self.assertEqual(
            isinstance(vms, list), True,
            "List VMs should return valid response for deployed VM")
        self.assertNotEqual(
            len(vms), 0,
            "List VMs should return valid response for deployed VM")
        vm = vms[0]
        self.debug("Deployed VM on host: %s" % vm.hostid)
        self.assertEqual(vm.state, "Running",
                         "Deployed VM should be in RUnning state")
        networks = Network.list(self.apiclient,
                                account=self.account.name,
                                domainid=self.account.domainid,
                                listall=True)
        self.assertEqual(
            isinstance(networks, list), True,
            "List networks should return valid list for the account")
        network = networks[0]

        self.debug("Associating public IP for account: %s" % self.account.name)
        public_ip = PublicIPAddress.create(self.apiclient,
                                           accountid=self.account.name,
                                           zoneid=self.zone.id,
                                           domainid=self.account.domainid,
                                           networkid=network.id)

        self.debug("Associated %s with network %s" %
                   (public_ip.ipaddress.ipaddress, network.id))
        self.debug("Creating PF rule for IP address: %s" %
                   public_ip.ipaddress.ipaddress)
        NATRule.create(self.apiclient,
                       virtual_machine,
                       self.services["natrule"],
                       ipaddressid=public_ip.ipaddress.id)

        self.debug("Creating LB rule on IP with NAT: %s" %
                   public_ip.ipaddress.ipaddress)

        # Create Load Balancer rule on IP already having NAT rule
        lb_rule = LoadBalancerRule.create(self.apiclient,
                                          self.services["lbrule"],
                                          ipaddressid=public_ip.ipaddress.id,
                                          accountid=self.account.name)
        self.debug("Created LB rule with ID: %s" % lb_rule.id)

        # Should be able to SSH VM
        try:
            self.debug("SSH into VM: %s" % virtual_machine.id)
            virtual_machine.get_ssh_client(
                ipaddress=public_ip.ipaddress.ipaddress)
        except Exception as e:
            self.fail("SSH Access failed for %s: %s" %
                      (virtual_machine.ipaddress, e))

        first_host = vm.hostid
        self.debug("Enabling maintenance mode for host %s" % vm.hostid)
        cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
        cmd.id = first_host
        self.apiclient.prepareHostForMaintenance(cmd)

        self.debug("Waiting for SSVMs to come up")
        wait_for_ssvms(
            self.apiclient,
            zoneid=self.zone.id,
            podid=self.pod.id,
        )

        timeout = self.services["timeout"]
        # Poll and check state of VM while it migrates from one host to another
        while True:
            vms = VirtualMachine.list(self.apiclient,
                                      id=virtual_machine.id,
                                      listall=True)
            self.assertEqual(
                isinstance(vms, list), True,
                "List VMs should return valid response for deployed VM")
            self.assertNotEqual(
                len(vms), 0,
                "List VMs should return valid response for deployed VM")
            vm = vms[0]

            self.debug("VM 1 state: %s" % vm.state)
            if vm.state in [
                    "Stopping", "Stopped", "Running", "Starting", "Migrating"
            ]:
                if vm.state == "Running":
                    break
                else:
                    time.sleep(self.services["sleep"])
                    timeout = timeout - 1
            else:
                self.fail("VM migration from one-host-to-other\
                            failed while enabling maintenance")
        second_host = vm.hostid
        self.assertEqual(
            vm.state, "Running",
            "VM should be in Running state after enabling host maintenance")
        # Should be able to SSH VM
        try:
            self.debug("SSH into VM: %s" % virtual_machine.id)
            virtual_machine.get_ssh_client(
                ipaddress=public_ip.ipaddress.ipaddress)
        except Exception as e:
            self.fail("SSH Access failed for %s: %s" %
                      (virtual_machine.ipaddress, e))
        self.debug("Deploying VM in account: %s" % self.account.name)
        # Spawn an instance on other host
        virtual_machine_2 = VirtualMachine.create(
            self.apiclient,
            self.services["virtual_machine"],
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering.id)
        vms = VirtualMachine.list(self.apiclient,
                                  id=virtual_machine_2.id,
                                  listall=True)
        self.assertEqual(
            isinstance(vms, list), True,
            "List VMs should return valid response for deployed VM")
        self.assertNotEqual(
            len(vms), 0,
            "List VMs should return valid response for deployed VM")
        vm = vms[0]
        self.debug("Deployed VM on host: %s" % vm.hostid)
        self.debug("VM 2 state: %s" % vm.state)
        self.assertEqual(vm.state, "Running",
                         "Deployed VM should be in Running state")

        self.debug("Canceling host maintenance for ID: %s" % first_host)
        cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
        cmd.id = first_host
        self.apiclient.cancelHostMaintenance(cmd)
        self.debug("Maintenance mode canceled for host: %s" % first_host)

        self.debug("Enabling maintenance mode for host %s" % second_host)
        cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
        cmd.id = second_host
        self.apiclient.prepareHostForMaintenance(cmd)
        self.debug("Maintenance mode enabled for host: %s" % second_host)

        self.debug("Waiting for SSVMs to come up")
        wait_for_ssvms(
            self.apiclient,
            zoneid=self.zone.id,
            podid=self.pod.id,
        )

        # Poll and check the status of VMs
        timeout = self.services["timeout"]
        while True:
            vms = VirtualMachine.list(self.apiclient,
                                      account=self.account.name,
                                      domainid=self.account.domainid,
                                      listall=True)
            self.assertEqual(
                isinstance(vms, list), True,
                "List VMs should return valid response for deployed VM")
            self.assertNotEqual(
                len(vms), 0,
                "List VMs should return valid response for deployed VM")
            vm = vms[0]
            self.debug(
                "VM state after enabling maintenance on first host: %s" %
                vm.state)
            if vm.state in [
                    "Stopping", "Stopped", "Running", "Starting", "Migrating"
            ]:
                if vm.state == "Running":
                    break
                else:
                    time.sleep(self.services["sleep"])
                    timeout = timeout - 1
            else:
                self.fail("VM migration from one-host-to-other failed\
                            while enabling maintenance")

                # Poll and check the status of VMs
        timeout = self.services["timeout"]
        while True:
            vms = VirtualMachine.list(self.apiclient,
                                      account=self.account.name,
                                      domainid=self.account.domainid,
                                      listall=True)
            self.assertEqual(
                isinstance(vms, list), True,
                "List VMs should return valid response for deployed VM")
            self.assertNotEqual(
                len(vms), 0,
                "List VMs should return valid response for deployed VM")
            vm = vms[1]
            self.debug(
                "VM state after enabling maintenance on first host: %s" %
                vm.state)
            if vm.state in [
                    "Stopping", "Stopped", "Running", "Starting", "Migrating"
            ]:
                if vm.state == "Running":
                    break
                else:
                    time.sleep(self.services["sleep"])
                    timeout = timeout - 1
            else:
                self.fail("VM migration from one-host-to-other\
                            failed while enabling maintenance")

        for vm in vms:
            self.debug(
                "VM states after enabling maintenance mode on host: %s - %s" %
                (first_host, vm.state))
            self.assertEqual(vm.state, "Running",
                             "Deployed VM should be in Running state")

        # Spawn an instance on other host
        virtual_machine_3 = VirtualMachine.create(
            self.apiclient,
            self.services["virtual_machine"],
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering.id)
        vms = VirtualMachine.list(self.apiclient,
                                  id=virtual_machine_3.id,
                                  listall=True)
        self.assertEqual(
            isinstance(vms, list), True,
            "List VMs should return valid response for deployed VM")
        self.assertNotEqual(
            len(vms), 0,
            "List VMs should return valid response for deployed VM")
        vm = vms[0]

        self.debug("Deployed VM on host: %s" % vm.hostid)
        self.debug("VM 3 state: %s" % vm.state)
        self.assertEqual(vm.state, "Running",
                         "Deployed VM should be in Running state")

        # Should be able to SSH VM
        try:
            self.debug("SSH into VM: %s" % virtual_machine.id)
            virtual_machine.get_ssh_client(
                ipaddress=public_ip.ipaddress.ipaddress)
        except Exception as e:
            self.fail("SSH Access failed for %s: %s" %
                      (virtual_machine.ipaddress, e))

        self.debug("Canceling host maintenance for ID: %s" % second_host)
        cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
        cmd.id = second_host
        self.apiclient.cancelHostMaintenance(cmd)
        self.debug("Maintenance mode canceled for host: %s" % second_host)

        self.debug("Waiting for SSVMs to come up")
        wait_for_ssvms(
            self.apiclient,
            zoneid=self.zone.id,
            podid=self.pod.id,
        )
        return
Esempio n. 4
0
    def test_vmware_anti_affinity(self):
        """ Test Set up anti-affinity rules

            The test requires following pre-requisites
            - VMWare cluster configured in fully automated mode
        """

        # Validate the following
        # 1. Deploy VMs on host 1 and 2
        # 2. Enable maintenance mode for host 1
        # 3. VM should be migrated to 3rd host

        hosts = Host.list(
                          self.apiclient,
                          zoneid=self.zone.id,
                          resourcestate='Enabled',
                          type='Routing'
                          )
        self.assertEqual(
                         isinstance(hosts, list),
                         True,
                         "List hosts should return valid host response"
                         )

        self.debug(len(hosts))

        self.assertGreaterEqual(
                         len(hosts),
                         3,
                         "There must be at least 3 hosts present in a cluster"
                        )

        aff_grp = self.create_aff_grp(aff_grp=self.services["host_anti_affinity"], acc=self.account.name, domainid=self.domain.id)

        vm_1 = VirtualMachine.create(
                              self.apiclient,
                              self.services["virtual_machine"],
                              accountid=self.account.name,
                              domainid=self.domain.id,
                              serviceofferingid=self.service_offering.id,
                              affinitygroupnames=[aff_grp.name]
                             )

        vm_2 = VirtualMachine.create(
                              self.apiclient,
                              self.services["virtual_machine"],
                              accountid=self.account.name,
                              domainid=self.domain.id,
                              serviceofferingid=self.service_offering.id,
                              affinitygroupnames=[aff_grp.name]
                             )

        host_1 = vm_1.hostid

        host_2 = vm_2.hostid

        vms = VirtualMachine.list(
                                  self.apiclient,
                                  id=vm_1.id,
                                  listall=True
                                  )
        vm_list_validation_result = validateList(vms)

        self.assertEqual(vm_list_validation_result[0], PASS, "vm list validation failed due to %s" %
                         vm_list_validation_result[1])

        virtual_machine_1 = vm_list_validation_result[1]

        self.debug("VM State: %s" % virtual_machine_1.state)
        self.assertEqual(
                         virtual_machine_1.state,
                         "Running",
                         "Deployed VM should be in RUnning state"
                         )

        vms = VirtualMachine.list(
                                  self.apiclient,
                                  id=vm_2.id,
                                  listall=True
                                  )
        vm_list_validation_result = validateList(vms)

        self.assertEqual(vm_list_validation_result[0], PASS, "vm list validation failed due to %s" %
                         vm_list_validation_result[1])

        virtual_machine_2 = vm_list_validation_result[1]

        self.debug("VM %s  State: %s" % (
                                         virtual_machine_2.name,
                                         virtual_machine_2.state
                                         ))
        self.assertEqual(
                         virtual_machine_2.state,
                         "Running",
                         "Deployed VM should be in RUnning state"
                         )
        self.debug("Enabling maintenance mode on host_1: %s" % host_1)

        cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
        cmd.id = host_1
        self.apiclient.prepareHostForMaintenance(cmd)

        timeout = self.services["timeout"]
        while True:
            hosts = Host.list(
                          self.apiclient,
                          zoneid=self.zone.id,
                          type='Routing',
                          id=host_1
                          )
            host_list_validation_result = validateList(hosts)

            self.assertEqual(host_list_validation_result[0], PASS, "host list validation failed due to %s"
                            % host_list_validation_result[2])

            host = host_list_validation_result[1]

            if host.resourcestate == 'Maintenance':
                break
            elif timeout == 0:
                self.fail("Failed to put host: %s in maintenance mode" % host.name)

            time.sleep(self.services["sleep"])
            timeout = timeout - 1

        vms = VirtualMachine.list(
                                  self.apiclient,
                                  id=virtual_machine_1.id,
                                  listall=True
                                  )
        vm_list_validation_result = validateList(vms)

        self.assertEqual(vm_list_validation_result[0], PASS, "vm list validation failed due to %s" %
                         vm_list_validation_result[2])

        vm = vm_list_validation_result[0]

        self.assertEqual(
                         vm.state,
                         "Running",
                         "Deployed VM should be in RUnning state"
                         )
        self.assertNotEqual(
                         vm.hostid,
                         host_2,
                         "The host name should not match with second host name"
                         )

        self.debug("Canceling host maintenance for ID: %s" % host_1.id)
        cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
        cmd.id = host_1.id
        self.apiclient.cancelHostMaintenance(cmd)
        self.debug("Maintenance mode canceled for host: %s" % host_1.id)

        return
    def test_02_host_maintenance_mode_with_activities(self):
        """Test host maintenance mode with activities
        """

        # Validate the following
        # 1. Create Vms. Acquire IP. Create port forwarding & load balancing
        #    rules for Vms.
        # 2. While activities are ongoing: Create snapshots, recurring
        #    snapshots, create templates, download volumes, Host 1: put to
        #    maintenance mode. All Vms should failover to Host 2 in cluster
        #    Vms should be in running state. All port forwarding rules and
        #    load balancing Rules should work.
        # 3. After failover to Host 2 succeeds, deploy Vms. Deploy Vms on host
        #    2 should succeed. All ongoing activities in step 3 should succeed
        # 4. Host 1: cancel maintenance mode.
        # 5. While activities are ongoing: Create snapshots, recurring
        #    snapshots, create templates, download volumes, Host 2: put to
        #    maintenance mode. All Vms should failover to Host 1 in cluster.
        # 6. After failover to Host 1 succeeds, deploy VMs. Deploy Vms on
        #    host 1 should succeed. All ongoing activities in step 6 should
        #    succeed.

        hosts = Host.list(
            self.apiclient,
            zoneid=self.zone.id,
            resourcestate='Enabled',
            type='Routing'
        )
        self.assertEqual(
            isinstance(hosts, list),
            True,
            "List hosts should return valid host response"
        )
        if len(hosts) < 2:
            self.skipTest("There must be at least 2 hosts present in cluster")

        self.debug("Checking HA with hosts: %s, %s" % (
            hosts[0].name,
            hosts[1].name
        ))
        self.debug("Deploying VM in account: %s" % self.account.name)
        # Spawn an instance in that network
        virtual_machine = VirtualMachine.create(
            self.apiclient,
            self.services["virtual_machine"],
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering.id
        )
        vms = VirtualMachine.list(
            self.apiclient,
            id=virtual_machine.id,
            listall=True
        )
        self.assertEqual(
            isinstance(vms, list),
            True,
            "List VMs should return valid response for deployed VM"
        )
        self.assertNotEqual(
            len(vms),
            0,
            "List VMs should return valid response for deployed VM"
        )
        vm = vms[0]
        self.debug("Deployed VM on host: %s" % vm.hostid)
        self.assertEqual(
            vm.state,
            "Running",
            "Deployed VM should be in RUnning state"
        )
        networks = Network.list(
            self.apiclient,
            account=self.account.name,
            domainid=self.account.domainid,
            listall=True
        )
        self.assertEqual(
            isinstance(networks, list),
            True,
            "List networks should return valid list for the account"
        )
        network = networks[0]

        self.debug("Associating public IP for account: %s" %
                   self.account.name)
        public_ip = PublicIPAddress.create(
            self.apiclient,
            accountid=self.account.name,
            zoneid=self.zone.id,
            domainid=self.account.domainid,
            networkid=network.id
        )

        self.debug("Associated %s with network %s" % (
            public_ip.ipaddress.ipaddress,
            network.id
        ))
        self.debug("Creating PF rule for IP address: %s" %
                   public_ip.ipaddress.ipaddress)
        NATRule.create(
            self.apiclient,
            virtual_machine,
            self.services["natrule"],
            ipaddressid=public_ip.ipaddress.id
        )

        self.debug("Creating LB rule on IP with NAT: %s" %
                   public_ip.ipaddress.ipaddress)

        # Create Load Balancer rule on IP already having NAT rule
        lb_rule = LoadBalancerRule.create(
            self.apiclient,
            self.services["lbrule"],
            ipaddressid=public_ip.ipaddress.id,
            accountid=self.account.name
        )
        self.debug("Created LB rule with ID: %s" % lb_rule.id)

        # Should be able to SSH VM
        try:
            self.debug("SSH into VM: %s" % virtual_machine.id)
            virtual_machine.get_ssh_client(
                ipaddress=public_ip.ipaddress.ipaddress)
        except Exception as e:
            self.fail("SSH Access failed for %s: %s" %
                      (virtual_machine.ipaddress, e)
                      )
        # Get the Root disk of VM
        volumes = list_volumes(
            self.apiclient,
            virtualmachineid=virtual_machine.id,
            type='ROOT',
            listall=True
        )
        volume = volumes[0]
        self.debug(
            "Root volume of VM(%s): %s" % (
                virtual_machine.name,
                volume.name
            ))
        # Create a snapshot from the ROOTDISK
        self.debug("Creating snapshot on ROOT volume: %s" % volume.name)
        snapshot = Snapshot.create(self.apiclient, volumes[0].id)
        self.debug("Snapshot created: ID - %s" % snapshot.id)

        snapshots = list_snapshots(
            self.apiclient,
            id=snapshot.id,
            listall=True
        )
        self.assertEqual(
            isinstance(snapshots, list),
            True,
            "Check list response returns a valid list"
        )
        self.assertNotEqual(
            snapshots,
            None,
            "Check if result exists in list snapshots call"
        )
        self.assertEqual(
            snapshots[0].id,
            snapshot.id,
            "Check snapshot id in list resources call"
        )

        # Generate template from the snapshot
        self.debug("Generating template from snapshot: %s" % snapshot.name)
        template = Template.create_from_snapshot(
            self.apiclient,
            snapshot,
            self.services["templates"]
        )
        self.debug("Created template from snapshot: %s" % template.id)

        templates = list_templates(
            self.apiclient,
            templatefilter=self.services["templates"]["templatefilter"],
            id=template.id
        )

        self.assertEqual(
            isinstance(templates, list),
            True,
            "List template call should return the newly created template"
        )

        self.assertEqual(
            templates[0].isready,
            True,
            "The newly created template should be in ready state"
        )

        first_host = vm.hostid
        self.debug("Enabling maintenance mode for host %s" % vm.hostid)
        cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
        cmd.id = first_host
        self.apiclient.prepareHostForMaintenance(cmd)

        self.debug("Waiting for SSVMs to come up")
        wait_for_ssvms(
            self.apiclient,
            zoneid=self.zone.id,
            podid=self.pod.id,
        )

        timeout = self.services["timeout"]
        # Poll and check state of VM while it migrates from one host to another
        while True:
            vms = VirtualMachine.list(
                self.apiclient,
                id=virtual_machine.id,
                listall=True
            )
            self.assertEqual(
                isinstance(vms, list),
                True,
                "List VMs should return valid response for deployed VM"
            )
            self.assertNotEqual(
                len(vms),
                0,
                "List VMs should return valid response for deployed VM"
            )
            vm = vms[0]

            self.debug("VM 1 state: %s" % vm.state)
            if vm.state in ["Stopping",
                            "Stopped",
                            "Running",
                            "Starting",
                            "Migrating"]:
                if vm.state == "Running":
                    break
                else:
                    time.sleep(self.services["sleep"])
                    timeout = timeout - 1
            else:
                self.fail(
                    "VM migration from one-host-to-other failed\
                            while enabling maintenance"
                )
        second_host = vm.hostid
        self.assertEqual(
            vm.state,
            "Running",
            "VM should be in Running state after enabling host maintenance"
        )
        # Should be able to SSH VM
        try:
            self.debug("SSH into VM: %s" % virtual_machine.id)
            virtual_machine.get_ssh_client(
                ipaddress=public_ip.ipaddress.ipaddress)
        except Exception as e:
            self.fail("SSH Access failed for %s: %s" %
                      (virtual_machine.ipaddress, e)
                      )
        self.debug("Deploying VM in account: %s" % self.account.name)
        # Spawn an instance on other host
        virtual_machine_2 = VirtualMachine.create(
            self.apiclient,
            self.services["virtual_machine"],
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering.id
        )
        vms = VirtualMachine.list(
            self.apiclient,
            id=virtual_machine_2.id,
            listall=True
        )
        self.assertEqual(
            isinstance(vms, list),
            True,
            "List VMs should return valid response for deployed VM"
        )
        self.assertNotEqual(
            len(vms),
            0,
            "List VMs should return valid response for deployed VM"
        )
        vm = vms[0]
        self.debug("Deployed VM on host: %s" % vm.hostid)
        self.debug("VM 2 state: %s" % vm.state)
        self.assertEqual(
            vm.state,
            "Running",
            "Deployed VM should be in Running state"
        )

        self.debug("Canceling host maintenance for ID: %s" % first_host)
        cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
        cmd.id = first_host
        self.apiclient.cancelHostMaintenance(cmd)
        self.debug("Maintenance mode canceled for host: %s" % first_host)

        # Get the Root disk of VM
        volumes = list_volumes(
            self.apiclient,
            virtualmachineid=virtual_machine_2.id,
            type='ROOT',
            listall=True
        )
        volume = volumes[0]
        self.debug(
            "Root volume of VM(%s): %s" % (
                virtual_machine_2.name,
                volume.name
            ))
        # Create a snapshot from the ROOTDISK
        self.debug("Creating snapshot on ROOT volume: %s" % volume.name)
        snapshot = Snapshot.create(self.apiclient, volumes[0].id)
        self.debug("Snapshot created: ID - %s" % snapshot.id)

        snapshots = list_snapshots(
            self.apiclient,
            id=snapshot.id,
            listall=True
        )
        self.assertEqual(
            isinstance(snapshots, list),
            True,
            "Check list response returns a valid list"
        )
        self.assertNotEqual(
            snapshots,
            None,
            "Check if result exists in list snapshots call"
        )
        self.assertEqual(
            snapshots[0].id,
            snapshot.id,
            "Check snapshot id in list resources call"
        )

        # Generate template from the snapshot
        self.debug("Generating template from snapshot: %s" % snapshot.name)
        template = Template.create_from_snapshot(
            self.apiclient,
            snapshot,
            self.services["templates"]
        )
        self.debug("Created template from snapshot: %s" % template.id)

        templates = list_templates(
            self.apiclient,
            templatefilter=self.services["templates"]["templatefilter"],
            id=template.id
        )

        self.assertEqual(
            isinstance(templates, list),
            True,
            "List template call should return the newly created template"
        )

        self.assertEqual(
            templates[0].isready,
            True,
            "The newly created template should be in ready state"
        )

        self.debug("Enabling maintenance mode for host %s" % second_host)
        cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
        cmd.id = second_host
        self.apiclient.prepareHostForMaintenance(cmd)
        self.debug("Maintenance mode enabled for host: %s" % second_host)

        self.debug("Waiting for SSVMs to come up")
        wait_for_ssvms(
            self.apiclient,
            zoneid=self.zone.id,
            podid=self.pod.id,
        )

        # Poll and check the status of VMs
        timeout = self.services["timeout"]
        while True:
            vms = VirtualMachine.list(
                self.apiclient,
                account=self.account.name,
                domainid=self.account.domainid,
                listall=True
            )
            self.assertEqual(
                isinstance(vms, list),
                True,
                "List VMs should return valid response for deployed VM"
            )
            self.assertNotEqual(
                len(vms),
                0,
                "List VMs should return valid response for deployed VM"
            )
            vm = vms[0]
            self.debug(
                "VM state after enabling maintenance on first host: %s" %
                vm.state)
            if vm.state in ["Stopping",
                            "Stopped",
                            "Running",
                            "Starting",
                            "Migrating"]:
                if vm.state == "Running":
                    break
                else:
                    time.sleep(self.services["sleep"])
                    timeout = timeout - 1
            else:
                self.fail(
                    "VM migration from one-host-to-other failed\
                            while enabling maintenance"
                )

        # Poll and check the status of VMs
        timeout = self.services["timeout"]
        while True:
            vms = VirtualMachine.list(
                self.apiclient,
                account=self.account.name,
                domainid=self.account.domainid,
                listall=True
            )
            self.assertEqual(
                isinstance(vms, list),
                True,
                "List VMs should return valid response for deployed VM"
            )
            self.assertNotEqual(
                len(vms),
                0,
                "List VMs should return valid response for deployed VM"
            )
            vm = vms[1]
            self.debug(
                "VM state after enabling maintenance on first host: %s" %
                vm.state)
            if vm.state in ["Stopping",
                            "Stopped",
                            "Running",
                            "Starting",
                            "Migrating"]:
                if vm.state == "Running":
                    break
                else:
                    time.sleep(self.services["sleep"])
                    timeout = timeout - 1
            else:
                self.fail(
                    "VM migration from one-host-to-other failed\
                            while enabling maintenance"
                )

        for vm in vms:
            self.debug(
                "VM states after enabling maintenance mode on host: %s - %s" %
                (first_host, vm.state))
            self.assertEqual(
                vm.state,
                "Running",
                "Deployed VM should be in Running state"
            )

        # Spawn an instance on other host
        virtual_machine_3 = VirtualMachine.create(
            self.apiclient,
            self.services["virtual_machine"],
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering.id
        )
        vms = VirtualMachine.list(
            self.apiclient,
            id=virtual_machine_3.id,
            listall=True
        )
        self.assertEqual(
            isinstance(vms, list),
            True,
            "List VMs should return valid response for deployed VM"
        )
        self.assertNotEqual(
            len(vms),
            0,
            "List VMs should return valid response for deployed VM"
        )
        vm = vms[0]

        self.debug("Deployed VM on host: %s" % vm.hostid)
        self.debug("VM 3 state: %s" % vm.state)
        self.assertEqual(
            vm.state,
            "Running",
            "Deployed VM should be in Running state"
        )

        self.debug("Canceling host maintenance for ID: %s" % second_host)
        cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
        cmd.id = second_host
        self.apiclient.cancelHostMaintenance(cmd)
        self.debug("Maintenance mode canceled for host: %s" % second_host)

        self.debug("Waiting for SSVMs to come up")
        wait_for_ssvms(
            self.apiclient,
            zoneid=self.zone.id,
            podid=self.pod.id,
        )
        return
    def test_01_host_maintenance_mode(self):
        """Test host maintenance mode
        """

        # Validate the following
        # 1. Create Vms. Acquire IP. Create port forwarding & load balancing
        #    rules for Vms.
        # 2. Host 1: put to maintenance mode. All Vms should failover to Host
        #    2 in cluster. Vms should be in running state. All port forwarding
        #    rules and load balancing Rules should work.
        # 3. After failover to Host 2 succeeds, deploy Vms. Deploy Vms on host
        #    2 should succeed.
        # 4. Host 1: cancel maintenance mode.
        # 5. Host 2 : put to maintenance mode. All Vms should failover to
        #    Host 1 in cluster.
        # 6. After failover to Host 1 succeeds, deploy VMs. Deploy Vms on
        #    host 1 should succeed.

        hosts = Host.list(
            self.apiclient,
            zoneid=self.zone.id,
            resourcestate='Enabled',
            type='Routing'
        )
        self.assertEqual(
            isinstance(hosts, list),
            True,
            "List hosts should return valid host response"
        )
        if len(hosts) < 2:
            self.skipTest("There must be at least 2 hosts present in cluster")

        self.debug("Checking HA with hosts: %s, %s" % (
            hosts[0].name,
            hosts[1].name
        ))
        self.debug("Deploying VM in account: %s" % self.account.name)
        # Spawn an instance in that network
        virtual_machine = VirtualMachine.create(
            self.apiclient,
            self.services["virtual_machine"],
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering.id
        )
        vms = VirtualMachine.list(
            self.apiclient,
            id=virtual_machine.id,
            listall=True
        )
        self.assertEqual(
            isinstance(vms, list),
            True,
            "List VMs should return valid response for deployed VM"
        )
        self.assertNotEqual(
            len(vms),
            0,
            "List VMs should return valid response for deployed VM"
        )
        vm = vms[0]
        self.debug("Deployed VM on host: %s" % vm.hostid)
        self.assertEqual(
            vm.state,
            "Running",
            "Deployed VM should be in RUnning state"
        )
        networks = Network.list(
            self.apiclient,
            account=self.account.name,
            domainid=self.account.domainid,
            listall=True
        )
        self.assertEqual(
            isinstance(networks, list),
            True,
            "List networks should return valid list for the account"
        )
        network = networks[0]

        self.debug("Associating public IP for account: %s" %
                   self.account.name)
        public_ip = PublicIPAddress.create(
            self.apiclient,
            accountid=self.account.name,
            zoneid=self.zone.id,
            domainid=self.account.domainid,
            networkid=network.id
        )

        self.debug("Associated %s with network %s" % (
            public_ip.ipaddress.ipaddress,
            network.id
        ))
        self.debug("Creating PF rule for IP address: %s" %
                   public_ip.ipaddress.ipaddress)
        NATRule.create(
            self.apiclient,
            virtual_machine,
            self.services["natrule"],
            ipaddressid=public_ip.ipaddress.id
        )

        self.debug("Creating LB rule on IP with NAT: %s" %
                   public_ip.ipaddress.ipaddress)

        # Create Load Balancer rule on IP already having NAT rule
        lb_rule = LoadBalancerRule.create(
            self.apiclient,
            self.services["lbrule"],
            ipaddressid=public_ip.ipaddress.id,
            accountid=self.account.name
        )
        self.debug("Created LB rule with ID: %s" % lb_rule.id)

        # Should be able to SSH VM
        try:
            self.debug("SSH into VM: %s" % virtual_machine.id)
            virtual_machine.get_ssh_client(
                ipaddress=public_ip.ipaddress.ipaddress)
        except Exception as e:
            self.fail("SSH Access failed for %s: %s" %
                      (virtual_machine.ipaddress, e)
                      )

        first_host = vm.hostid
        self.debug("Enabling maintenance mode for host %s" % vm.hostid)
        cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
        cmd.id = first_host
        self.apiclient.prepareHostForMaintenance(cmd)

        self.debug("Waiting for SSVMs to come up")
        wait_for_ssvms(
            self.apiclient,
            zoneid=self.zone.id,
            podid=self.pod.id,
        )

        timeout = self.services["timeout"]
        # Poll and check state of VM while it migrates from one host to another
        while True:
            vms = VirtualMachine.list(
                self.apiclient,
                id=virtual_machine.id,
                listall=True
            )
            self.assertEqual(
                isinstance(vms, list),
                True,
                "List VMs should return valid response for deployed VM"
            )
            self.assertNotEqual(
                len(vms),
                0,
                "List VMs should return valid response for deployed VM"
            )
            vm = vms[0]

            self.debug("VM 1 state: %s" % vm.state)
            if vm.state in ["Stopping",
                            "Stopped",
                            "Running",
                            "Starting",
                            "Migrating"]:
                if vm.state == "Running":
                    break
                else:
                    time.sleep(self.services["sleep"])
                    timeout = timeout - 1
            else:
                self.fail(
                    "VM migration from one-host-to-other\
                            failed while enabling maintenance"
                )
        second_host = vm.hostid
        self.assertEqual(
            vm.state,
            "Running",
            "VM should be in Running state after enabling host maintenance"
        )
        # Should be able to SSH VM
        try:
            self.debug("SSH into VM: %s" % virtual_machine.id)
            virtual_machine.get_ssh_client(
                ipaddress=public_ip.ipaddress.ipaddress)
        except Exception as e:
            self.fail("SSH Access failed for %s: %s" %
                      (virtual_machine.ipaddress, e)
                      )
        self.debug("Deploying VM in account: %s" % self.account.name)
        # Spawn an instance on other host
        virtual_machine_2 = VirtualMachine.create(
            self.apiclient,
            self.services["virtual_machine"],
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering.id
        )
        vms = VirtualMachine.list(
            self.apiclient,
            id=virtual_machine_2.id,
            listall=True
        )
        self.assertEqual(
            isinstance(vms, list),
            True,
            "List VMs should return valid response for deployed VM"
        )
        self.assertNotEqual(
            len(vms),
            0,
            "List VMs should return valid response for deployed VM"
        )
        vm = vms[0]
        self.debug("Deployed VM on host: %s" % vm.hostid)
        self.debug("VM 2 state: %s" % vm.state)
        self.assertEqual(
            vm.state,
            "Running",
            "Deployed VM should be in Running state"
        )

        self.debug("Canceling host maintenance for ID: %s" % first_host)
        cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
        cmd.id = first_host
        self.apiclient.cancelHostMaintenance(cmd)
        self.debug("Maintenance mode canceled for host: %s" % first_host)

        self.debug("Enabling maintenance mode for host %s" % second_host)
        cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
        cmd.id = second_host
        self.apiclient.prepareHostForMaintenance(cmd)
        self.debug("Maintenance mode enabled for host: %s" % second_host)

        self.debug("Waiting for SSVMs to come up")
        wait_for_ssvms(
            self.apiclient,
            zoneid=self.zone.id,
            podid=self.pod.id,
        )

        # Poll and check the status of VMs
        timeout = self.services["timeout"]
        while True:
            vms = VirtualMachine.list(
                self.apiclient,
                account=self.account.name,
                domainid=self.account.domainid,
                listall=True
            )
            self.assertEqual(
                isinstance(vms, list),
                True,
                "List VMs should return valid response for deployed VM"
            )
            self.assertNotEqual(
                len(vms),
                0,
                "List VMs should return valid response for deployed VM"
            )
            vm = vms[0]
            self.debug(
                "VM state after enabling maintenance on first host: %s" %
                vm.state)
            if vm.state in [
                "Stopping",
                "Stopped",
                "Running",
                "Starting",
                "Migrating"
            ]:
                if vm.state == "Running":
                    break
                else:
                    time.sleep(self.services["sleep"])
                    timeout = timeout - 1
            else:
                self.fail(
                    "VM migration from one-host-to-other failed\
                            while enabling maintenance"
                )

                # Poll and check the status of VMs
        timeout = self.services["timeout"]
        while True:
            vms = VirtualMachine.list(
                self.apiclient,
                account=self.account.name,
                domainid=self.account.domainid,
                listall=True
            )
            self.assertEqual(
                isinstance(vms, list),
                True,
                "List VMs should return valid response for deployed VM"
            )
            self.assertNotEqual(
                len(vms),
                0,
                "List VMs should return valid response for deployed VM"
            )
            vm = vms[1]
            self.debug(
                "VM state after enabling maintenance on first host: %s" %
                vm.state)
            if vm.state in [
                "Stopping",
                "Stopped",
                "Running",
                "Starting",
                "Migrating"
            ]:
                if vm.state == "Running":
                    break
                else:
                    time.sleep(self.services["sleep"])
                    timeout = timeout - 1
            else:
                self.fail(
                    "VM migration from one-host-to-other\
                            failed while enabling maintenance"
                )

        for vm in vms:
            self.debug(
                "VM states after enabling maintenance mode on host: %s - %s" %
                (first_host, vm.state))
            self.assertEqual(
                vm.state,
                "Running",
                "Deployed VM should be in Running state"
            )

        # Spawn an instance on other host
        virtual_machine_3 = VirtualMachine.create(
            self.apiclient,
            self.services["virtual_machine"],
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering.id
        )
        vms = VirtualMachine.list(
            self.apiclient,
            id=virtual_machine_3.id,
            listall=True
        )
        self.assertEqual(
            isinstance(vms, list),
            True,
            "List VMs should return valid response for deployed VM"
        )
        self.assertNotEqual(
            len(vms),
            0,
            "List VMs should return valid response for deployed VM"
        )
        vm = vms[0]

        self.debug("Deployed VM on host: %s" % vm.hostid)
        self.debug("VM 3 state: %s" % vm.state)
        self.assertEqual(
            vm.state,
            "Running",
            "Deployed VM should be in Running state"
        )

        # Should be able to SSH VM
        try:
            self.debug("SSH into VM: %s" % virtual_machine.id)
            virtual_machine.get_ssh_client(
                ipaddress=public_ip.ipaddress.ipaddress)
        except Exception as e:
            self.fail("SSH Access failed for %s: %s" %
                      (virtual_machine.ipaddress, e)
                      )

        self.debug("Canceling host maintenance for ID: %s" % second_host)
        cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
        cmd.id = second_host
        self.apiclient.cancelHostMaintenance(cmd)
        self.debug("Maintenance mode canceled for host: %s" % second_host)

        self.debug("Waiting for SSVMs to come up")
        wait_for_ssvms(
            self.apiclient,
            zoneid=self.zone.id,
            podid=self.pod.id,
        )
        return
    def test_06_no_vm_without_ha_gets_migrated_to_ha_host_in_live_migration(
            self):
        """ Verify that none of the VMs without HA enabled migrate to an ha tagged host during live migration """

        # Steps,
        #1. Fresh install CS that supports this feature
        #2. Create Basic zone, pod, cluster, add 3 hosts to cluster (host1, host2, host3), secondary & primary Storage
        #3. When adding host3, assign the HA host tag.
        #4. Create VMs with and without the Compute Service Offering with the HA tag.
        #5. Note the VMs on host1 and whether any of the VMs have their 'HA enabled' flags enabled.
        #6. Put host1 into maintenance mode.
        # Validations,
        #1. Make sure the VMs are created on either host1 or host2 and not on host3
        #2. Putting host1 into maintenance mode should trigger a live migration. Make sure the VMs are not migrated to HA enabled host3.

        # create and verify virtual machine with HA disabled service offering
        self.hypervisor = self.testClient.getHypervisorInfo()
        if self.hypervisor.lower() in ['lxc']:
            self.skipTest("vm migrate is not supported in %s" %
                          self.hypervisor)
        virtual_machine_without_ha = VirtualMachine.create(
            self.apiclient,
            self.services["virtual_machine"],
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering_without_ha.id)

        vms = VirtualMachine.list(self.apiclient,
                                  id=virtual_machine_without_ha.id,
                                  listall=True)

        self.assertEqual(
            isinstance(vms, list), True,
            "List VMs should return valid response for deployed VM")

        self.assertNotEqual(
            len(vms), 0,
            "List VMs should return valid response for deployed VM")

        vm_with_ha_disabled = vms[0]

        #Verify the virtual machine got created on non HA host
        list_hosts_response = list_hosts(self.apiclient,
                                         id=vm_with_ha_disabled.hostid)
        self.assertEqual(isinstance(list_hosts_response, list), True,
                         "Check list response returns a valid list")

        self.assertNotEqual(len(list_hosts_response), 0,
                            "Check Host is available")

        self.assertEqual(
            list_hosts_response[0].hahost, False,
            "The virtual machine is not ha enabled so check if VM is created on host which is also not ha enabled"
        )

        #put the Host in maintenance mode
        self.debug("Enabling maintenance mode for host %s" %
                   vm_with_ha_disabled.hostid)
        cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
        cmd.id = vm_with_ha_disabled.hostid
        self.apiclient.prepareHostForMaintenance(cmd)

        timeout = self.services["timeout"]

        #verify the VM live migration happened to another running host
        self.debug("Waiting for VM to come up")
        time.sleep(timeout)

        vms = VirtualMachine.list(self.apiclient,
                                  id=vm_with_ha_disabled.id,
                                  listall=True)

        self.assertEqual(
            isinstance(vms, list), True,
            "List VMs should return valid response for deployed VM")

        self.assertNotEqual(
            len(vms), 0,
            "List VMs should return valid response for deployed VM")

        list_hosts_response = list_hosts(self.apiclient, id=vms[0].hostid)
        self.assertEqual(isinstance(list_hosts_response, list), True,
                         "Check list response returns a valid list")

        self.assertNotEqual(len(list_hosts_response), 0,
                            "Check Host is available")

        self.assertEqual(
            list_hosts_response[0].hahost, False,
            "The virtual machine is not ha enabled so check if VM is created on host which is also not ha enabled"
        )

        self.debug("Disabling the maintenance mode for host %s" %
                   vm_with_ha_disabled.hostid)
        cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
        cmd.id = vm_with_ha_disabled.hostid
        self.apiclient.cancelHostMaintenance(cmd)
    def test_06_no_vm_without_ha_gets_migrated_to_ha_host_in_live_migration(self):
        """ Verify that none of the VMs without HA enabled migrate to an ha tagged host during live migration """

        # Steps,
        #1. Fresh install CS that supports this feature
        #2. Create Basic zone, pod, cluster, add 3 hosts to cluster (host1, host2, host3), secondary & primary Storage
        #3. When adding host3, assign the HA host tag.
        #4. Create VMs with and without the Compute Service Offering with the HA tag.
        #5. Note the VMs on host1 and whether any of the VMs have their 'HA enabled' flags enabled.
        #6. Put host1 into maintenance mode.
        # Validations,
        #1. Make sure the VMs are created on either host1 or host2 and not on host3
        #2. Putting host1 into maintenance mode should trigger a live migration. Make sure the VMs are not migrated to HA enabled host3.

        # create and verify virtual machine with HA disabled service offering
        self.hypervisor = self.testClient.getHypervisorInfo()
        if self.hypervisor.lower() in ['lxc']:
            self.skipTest("vm migrate is not supported in %s" % self.hypervisor)
        virtual_machine_without_ha = VirtualMachine.create(
            self.apiclient,
            self.services["virtual_machine"],
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering_without_ha.id
        )

        vms = VirtualMachine.list(
            self.apiclient,
            id=virtual_machine_without_ha.id,
            listall=True
        )

        self.assertEqual(
            isinstance(vms, list),
            True,
            "List VMs should return valid response for deployed VM"
        )

        self.assertNotEqual(
            len(vms),
            0,
            "List VMs should return valid response for deployed VM"
        )

        vm_with_ha_disabled = vms[0]

        #Verify the virtual machine got created on non HA host
        list_hosts_response = list_hosts(
            self.apiclient,
            id=vm_with_ha_disabled.hostid
        )
        self.assertEqual(
            isinstance(list_hosts_response, list),
            True,
            "Check list response returns a valid list"
        )

        self.assertNotEqual(
            len(list_hosts_response),
            0,
            "Check Host is available"
        )

        self.assertEqual(
            list_hosts_response[0].hahost,
            False,
            "The virtual machine is not ha enabled so check if VM is created on host which is also not ha enabled"
        )

        #put the Host in maintenance mode
        self.debug("Enabling maintenance mode for host %s" % vm_with_ha_disabled.hostid)
        cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
        cmd.id = vm_with_ha_disabled.hostid
        self.apiclient.prepareHostForMaintenance(cmd)

        timeout = self.services["timeout"]

        #verify the VM live migration happened to another running host
        self.debug("Waiting for VM to come up")
        time.sleep(timeout)

        vms = VirtualMachine.list(
            self.apiclient,
            id=vm_with_ha_disabled.id,
            listall=True
        )

        self.assertEqual(
            isinstance(vms, list),
            True,
            "List VMs should return valid response for deployed VM"
        )

        self.assertNotEqual(
            len(vms),
            0,
            "List VMs should return valid response for deployed VM"
        )

        list_hosts_response = list_hosts(
            self.apiclient,
            id=vms[0].hostid
        )
        self.assertEqual(
            isinstance(list_hosts_response, list),
            True,
            "Check list response returns a valid list"
        )

        self.assertNotEqual(
            len(list_hosts_response),
            0,
            "Check Host is available"
        )

        self.assertEqual(
            list_hosts_response[0].hahost,
            False,
            "The virtual machine is not ha enabled so check if VM is created on host which is also not ha enabled"
        )

        self.debug("Disabling the maintenance mode for host %s" % vm_with_ha_disabled.hostid)
        cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
        cmd.id = vm_with_ha_disabled.hostid
        self.apiclient.cancelHostMaintenance(cmd)