Example #1
0
    def deploy_neutron_vlan_ha(self):
        """Deploy cluster in HA mode with Neutron VLAN

        Scenario:
            1. Create cluster
            2. Add 3 nodes with controller role
            3. Add 2 nodes with compute role
            4. Deploy the cluster
            5. Run network verification
            6. Check Swift ring and rebalance it if needed
            7. Run OSTF

        Duration 80m
        Snapshot deploy_neutron_vlan_ha

        """
        self.env.revert_snapshot("ready_with_5_slaves")

        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=DEPLOYMENT_MODE,
            settings={
                "net_provider": 'neutron',
                "net_segment_type": NEUTRON_SEGMENT['vlan']
            }
        )
        self.fuel_web.update_nodes(
            cluster_id,
            {
                'slave-01': ['controller'],
                'slave-02': ['controller'],
                'slave-03': ['controller'],
                'slave-04': ['compute'],
                'slave-05': ['compute']
            }
        )
        self.fuel_web.update_internal_network(cluster_id, '192.168.196.0/22',
                                              '192.168.196.1')
        self.fuel_web.deploy_cluster_wait(cluster_id)

        cluster = self.fuel_web.client.get_cluster(cluster_id)
        assert_equal(str(cluster['net_provider']), 'neutron')
        os_conn = os_actions.OpenStackActions(
            self.fuel_web.get_public_vip(cluster_id))
        self.fuel_web.check_fixed_network_cidr(
            cluster_id, os_conn)

        self.fuel_web.verify_network(cluster_id)
        devops_node = self.fuel_web.get_nailgun_primary_node(
            self.env.d_env.nodes().slaves[0])
        logger.debug("devops node name is {0}".format(devops_node.name))
        ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip']
        Common.rebalance_swift_ring(ip)

        self.fuel_web.run_ostf(
            cluster_id=cluster_id, test_sets=['ha', 'smoke', 'sanity'])

        self.env.make_snapshot("deploy_neutron_vlan_ha")
Example #2
0
    def deploy_neutron_gre_ha(self):
        """Deploy cluster in HA mode with Neutron GRE (DEPRECATED)

        Scenario:
            1. Create cluster
            2. Add 3 nodes with controller role
            3. Add 2 nodes with compute role
            4. Deploy the cluster
            5. Run network verification
            6. Check Swift ring and rebalance it if needed
            7. Run OSTF

        Duration 80m
        Snapshot deploy_neutron_gre_ha

        """
        self.env.revert_snapshot("ready_with_5_slaves")

        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=DEPLOYMENT_MODE,
            settings={
                "net_provider": 'neutron',
                "net_segment_type": NEUTRON_SEGMENT['gre'],
                'tenant': 'haGre',
                'user': '******',
                'password': '******'
            }
        )
        self.fuel_web.update_nodes(
            cluster_id,
            {
                'slave-01': ['controller'],
                'slave-02': ['controller'],
                'slave-03': ['controller'],
                'slave-04': ['compute'],
                'slave-05': ['compute']
            }
        )
        self.fuel_web.deploy_cluster_wait(cluster_id)

        cluster = self.fuel_web.client.get_cluster(cluster_id)
        assert_equal(str(cluster['net_provider']), 'neutron')

        self.fuel_web.verify_network(cluster_id)
        devops_node = self.fuel_web.get_nailgun_primary_node(
            self.env.d_env.nodes().slaves[0])
        logger.debug("devops node name is {0}".format(devops_node.name))
        ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip']
        Common.rebalance_swift_ring(ip)

        self.fuel_web.run_ostf(
            cluster_id=cluster_id,
            test_sets=['ha', 'smoke', 'sanity'])

        self.env.make_snapshot("deploy_neutron_gre_ha")
Example #3
0
    def deploy_neutron_gre_ha(self):
        """Deploy cluster in HA mode with Neutron GRE (DEPRECATED)

        Scenario:
            1. Create cluster
            2. Add 3 nodes with controller role
            3. Add 2 nodes with compute role
            4. Deploy the cluster
            5. Run network verification
            6. Check Swift ring and rebalance it if needed
            7. Run OSTF

        Duration 80m
        Snapshot deploy_neutron_gre_ha

        """
        self.env.revert_snapshot("ready_with_5_slaves")

        cluster_id = self.fuel_web.create_cluster(name=self.__class__.__name__,
                                                  mode=DEPLOYMENT_MODE,
                                                  settings={
                                                      "net_provider":
                                                      'neutron',
                                                      "net_segment_type":
                                                      NEUTRON_SEGMENT['gre'],
                                                      'tenant':
                                                      'haGre',
                                                      'user':
                                                      '******',
                                                      'password':
                                                      '******'
                                                  })
        self.fuel_web.update_nodes(
            cluster_id, {
                'slave-01': ['controller'],
                'slave-02': ['controller'],
                'slave-03': ['controller'],
                'slave-04': ['compute'],
                'slave-05': ['compute']
            })
        self.fuel_web.deploy_cluster_wait(cluster_id)

        cluster = self.fuel_web.client.get_cluster(cluster_id)
        assert_equal(str(cluster['net_provider']), 'neutron')

        self.fuel_web.verify_network(cluster_id)
        devops_node = self.fuel_web.get_nailgun_primary_node(
            self.env.d_env.nodes().slaves[0])
        logger.debug("devops node name is {0}".format(devops_node.name))
        ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip']
        Common.rebalance_swift_ring(ip)

        self.fuel_web.run_ostf(cluster_id=cluster_id,
                               test_sets=['ha', 'smoke', 'sanity'])

        self.env.make_snapshot("deploy_neutron_gre_ha")
Example #4
0
    def deploy_neutron_vlan_ha(self):
        """Deploy cluster in HA mode with Neutron VLAN

        Scenario:
            1. Create cluster
            2. Add 3 nodes with controller role
            3. Add 2 nodes with compute role
            4. Deploy the cluster
            5. Run network verification
            6. Check Swift ring and rebalance it if needed
            7. Run OSTF

        Duration 80m
        Snapshot deploy_neutron_vlan_ha

        """
        self.env.revert_snapshot("ready_with_5_slaves")

        cluster_id = self.fuel_web.create_cluster(name=self.__class__.__name__,
                                                  mode=DEPLOYMENT_MODE,
                                                  settings={
                                                      "net_provider":
                                                      'neutron',
                                                      "net_segment_type":
                                                      NEUTRON_SEGMENT['vlan']
                                                  })
        self.fuel_web.update_nodes(
            cluster_id, {
                'slave-01': ['controller'],
                'slave-02': ['controller'],
                'slave-03': ['controller'],
                'slave-04': ['compute'],
                'slave-05': ['compute']
            })
        self.fuel_web.update_internal_network(cluster_id, '192.168.196.0/22',
                                              '192.168.196.1')
        self.fuel_web.deploy_cluster_wait(cluster_id)

        cluster = self.fuel_web.client.get_cluster(cluster_id)
        assert_equal(str(cluster['net_provider']), 'neutron')
        os_conn = os_actions.OpenStackActions(
            self.fuel_web.get_public_vip(cluster_id))
        self.fuel_web.check_fixed_network_cidr(cluster_id, os_conn)

        self.fuel_web.verify_network(cluster_id)
        devops_node = self.fuel_web.get_nailgun_primary_node(
            self.env.d_env.nodes().slaves[0])
        logger.debug("devops node name is {0}".format(devops_node.name))
        ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip']
        Common.rebalance_swift_ring(ip)

        self.fuel_web.run_ostf(cluster_id=cluster_id,
                               test_sets=['ha', 'smoke', 'sanity'])

        self.env.make_snapshot("deploy_neutron_vlan_ha")
    def deploy_neutron_tun_ha_base(self, snapshot_name):
        self.check_run(snapshot_name)
        self.env.revert_snapshot("ready_with_5_slaves")

        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=DEPLOYMENT_MODE,
            settings={
                "net_provider": 'neutron',
                "net_segment_type": NEUTRON_SEGMENT['tun'],
                'tenant': 'haTun',
                'user': '******',
                'password': '******'
            }
        )
        self.fuel_web.update_nodes(
            cluster_id,
            {
                'slave-01': ['controller'],
                'slave-02': ['controller'],
                'slave-03': ['controller'],
                'slave-04': ['compute'],
                'slave-05': ['compute']
            }
        )
        self.fuel_web.deploy_cluster_wait(cluster_id)

        cluster = self.fuel_web.client.get_cluster(cluster_id)
        assert_equal(str(cluster['net_provider']), 'neutron')

        self.fuel_web.verify_network(cluster_id)
        devops_node = self.fuel_web.get_nailgun_primary_node(
            self.env.d_env.nodes().slaves[0])
        logger.debug("devops node name is {0}".format(devops_node.name))
        ip = self.fuel_web.get_nailgun_node_by_name(devops_node
                                                    .name)['ip']
        Common.rebalance_swift_ring(ip)

        self.fuel_web.run_ostf(
            cluster_id=cluster_id,
            test_sets=['ha', 'smoke', 'sanity'])

        self.env.make_snapshot(snapshot_name, is_make=True)
Example #6
0
    def deploy_neutron_tun_ha_base(self, snapshot_name):
        self.check_run(snapshot_name)
        self.env.revert_snapshot("ready_with_5_slaves")

        cluster_id = self.fuel_web.create_cluster(name=self.__class__.__name__,
                                                  mode=DEPLOYMENT_MODE,
                                                  settings={
                                                      "net_provider":
                                                      'neutron',
                                                      "net_segment_type":
                                                      NEUTRON_SEGMENT['tun'],
                                                      'tenant':
                                                      'haTun',
                                                      'user':
                                                      '******',
                                                      'password':
                                                      '******'
                                                  })
        self.fuel_web.update_nodes(
            cluster_id, {
                'slave-01': ['controller'],
                'slave-02': ['controller'],
                'slave-03': ['controller'],
                'slave-04': ['compute'],
                'slave-05': ['compute']
            })
        self.fuel_web.deploy_cluster_wait(cluster_id)

        cluster = self.fuel_web.client.get_cluster(cluster_id)
        assert_equal(str(cluster['net_provider']), 'neutron')

        self.fuel_web.verify_network(cluster_id)
        devops_node = self.fuel_web.get_nailgun_primary_node(
            self.env.d_env.nodes().slaves[0])
        logger.debug("devops node name is {0}".format(devops_node.name))
        ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip']
        Common.rebalance_swift_ring(ip)

        self.fuel_web.run_ostf(cluster_id=cluster_id,
                               test_sets=['ha', 'smoke', 'sanity'])

        self.env.make_snapshot(snapshot_name, is_make=True)
Example #7
0
    def neutron_tun_scalability(self):
        """Check HA mode on scalability

        Scenario:
            1. Create cluster
            2. Add 1 controller node
            3. Deploy the cluster
            4. Check swift, and invoke swift-rings-rebalance.sh
               on primary controller if check failed
            5. Add 2 controller nodes
            6. Deploy changes
            7. Check swift, and invoke swift-rings-rebalance.sh
               on primary controller if check failed
            8. Run OSTF
            9. Add 2 controller 1 compute nodes
            10. Deploy changes
            11. Check swift, and invoke swift-rings-rebalance.sh
                on all the controllers
            12. Run OSTF
            13. Delete the primary and the last added controller.
            14. Deploy changes
            15. Check swift, and invoke swift-rings-rebalance.sh
                on all the controllers
            16. Run OSTF

        Duration 160m
        Snapshot neutron_tun_scalability

        """
        def _check_pacemaker(devops_nodes):
            for devops_node in devops_nodes:
                with QuietLogger():
                    self.fuel_web.assert_pacemaker(devops_node.name,
                                                   devops_nodes, [])
                ret = self.fuel_web.get_pacemaker_status(devops_node.name)
                assert_true(
                    re.search(
                        'vip__management\s+\(ocf::fuel:ns_IPaddr2\):'
                        '\s+Started node', ret), 'vip management started')
                assert_true(
                    re.search(
                        'vip__public\s+\(ocf::fuel:ns_IPaddr2\):'
                        '\s+Started node', ret), 'vip public started')

        self.env.revert_snapshot("ready_with_9_slaves")
        # Step 1  Create cluster with 1 controller
        logger.info("STEP1: Create new cluster {0}".format(
            self.__class__.__name__))
        cluster_id = self.fuel_web.create_cluster(name=self.__class__.__name__,
                                                  mode=DEPLOYMENT_MODE,
                                                  settings={
                                                      "net_provider":
                                                      'neutron',
                                                      "net_segment_type":
                                                      NEUTRON_SEGMENT['tun']
                                                  })

        nodes = {'slave-01': ['controller']}
        logger.info("Adding new node to the cluster: {0}".format(nodes))
        self.fuel_web.update_nodes(cluster_id, nodes)
        self.fuel_web.deploy_cluster_wait(cluster_id)

        logger.info("STEP3: Deploy 1 node cluster finishes")
        primary_node = self.env.d_env.get_node(name='slave-01')

        # Step 4. Check swift
        logger.info("STEP4: Check swift on primary controller {0}".format(
            primary_node))
        ip = self.fuel_web.get_nailgun_node_by_name(primary_node.name)['ip']
        Common.rebalance_swift_ring(ip)

        nodes = {'slave-02': ['controller'], 'slave-03': ['controller']}
        logger.info("STEP 4: Adding new nodes "
                    "to the cluster: {0}".format(nodes))
        self.fuel_web.update_nodes(cluster_id, nodes, True, False)
        self.fuel_web.deploy_cluster_wait(cluster_id)

        logger.info("STEP6: Deploy 3 ctrl node cluster has finished")
        controllers = ['slave-01', 'slave-02', 'slave-03']
        _check_pacemaker(self.env.d_env.get_nodes(name__in=controllers))

        primary_node_s3 = self.fuel_web.get_nailgun_primary_node(
            self.env.d_env.nodes().slaves[0])

        logger.info("Primary controller after STEP6 is {0}".format(
            primary_node_s3.name))
        logger.info("STEP7: Check swift on primary controller {0}".format(
            primary_node_s3))
        ip = self.fuel_web.get_nailgun_node_by_name(primary_node_s3.name)['ip']
        Common.rebalance_swift_ring(ip)

        # Run smoke tests only according to ha and
        # sanity executed in scope of deploy_cluster_wait()

        self.fuel_web.run_ostf(cluster_id=cluster_id, test_sets=['smoke'])

        nodes = {
            'slave-04': ['controller'],
            'slave-05': ['controller'],
            'slave-06': ['compute']
        }
        logger.info("Adding new nodes to the cluster: {0}".format(nodes))
        self.fuel_web.update_nodes(cluster_id, nodes, True, False)
        self.fuel_web.deploy_cluster_wait(cluster_id)

        logger.info("STEP10: Deploy 5 ctrl node cluster has finished")
        controllers = [
            'slave-01', 'slave-02', 'slave-03', 'slave-04', 'slave-05'
        ]
        _check_pacemaker(self.env.d_env.get_nodes(name__in=controllers))

        primary_node_s9 = self.fuel_web.get_nailgun_primary_node(
            self.env.d_env.nodes().slaves[0])

        logger.info("Primary controller after STEP10 is {0}".format(
            primary_node_s9.name))

        logger.info("STEP11: Check swift on primary controller {0}".format(
            primary_node_s9))

        ip = self.fuel_web.get_nailgun_node_by_name(primary_node_s9.name)['ip']
        Common.rebalance_swift_ring(ip)

        # Run smoke tests only according to ha and
        # sanity executed in scope of deploy_cluster_wait()

        # Step 12. Run OSTF
        self.fuel_web.run_ostf(cluster_id=cluster_id, test_sets=['smoke'])

        nodes = {
            primary_node_s9.name: ['controller'],
            'slave-05': ['controller']
        }
        logger.info(
            "STEP13: Deleting nodes from the cluster: {0}".format(nodes))
        self.fuel_web.update_nodes(cluster_id, nodes, False, True)

        # Step 14. Deploy changes
        self.fuel_web.deploy_cluster_wait(cluster_id)

        nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
            cluster_id, ['controller'])
        devops_nodes = [
            self.fuel_web.get_devops_node_by_nailgun_node(node)
            for node in nodes
        ]
        _check_pacemaker(devops_nodes)

        logger.info("STEP13-14: Scale down happened. "
                    "3 controller should be now")
        primary_node_s14 = self.fuel_web.get_nailgun_primary_node(
            self.env.d_env.get_node(name=devops_nodes[0].name))

        logger.info("Primary controller after STEP15 is {0}".format(
            primary_node_s14.name))

        logger.info("STEP15: Check swift on primary controller {0}".format(
            primary_node_s14))

        ip = self.fuel_web.get_nailgun_node_by_name(
            primary_node_s14.name)['ip']
        Common.rebalance_swift_ring(ip)

        # Step 16. Run OSTF
        self.fuel_web.run_ostf(cluster_id=cluster_id,
                               test_sets=['sanity', 'smoke', 'ha'])
        self.env.make_snapshot("neutron_vlan_ha_scalability")
Example #8
0
    def deploy_scale_controller_ironic(self):
        """Test cluster scaling with Controller and Ironic

        Scenario:
            1. Create cluster with 5 slave nodes
            2. Bootstrap 1 additional slave node
            3. Add 2 Controller nodes
            4. Add 1 Compute node
            5. Add 1 Controller+Ironic node
            6. Deploy the cluster
            7. Run OSTF tests
            8. Boot, check connectivity, delete Ironic VM
            9. Rebalance Swift rings
            10. Add 1 Controller node
            11. Add 1 Controller+Ironic node
            12. Redeploy the cluster
            13. Run OSTF tests
            14. Boot, check connectivity, delete Ironic VM
            15. Rebalance Swift rings
            16. Remove 1 Controller node
            17. Remove 1 Controller+Ironic node
            18. Redeploy the cluster
            19. Run OSTF tests
            20. Boot, check connectivity, delete Ironic VM

        Duration 90m
        Snapshot deploy_scale_controller_ironic
        """

        self.env.revert_snapshot("ready_with_5_slaves")
        # Deploy 1st part
        data = {'net_segment_type': NEUTRON_SEGMENT['vlan'], 'ironic': True}

        nodes = {
            'slave-01': ['controller'],
            'slave-02': ['controller'],
            'slave-03': ['controller', 'ironic'],
            'slave-04': ['compute']
        }

        self.show_step(1)
        self.show_step(2)
        self.env.bootstrap_nodes(self.env.d_env.nodes().slaves[5:6])
        self.show_step(3)
        self.show_step(4)
        self.show_step(5)
        self.show_step(6)
        self.show_step(7)
        cluster_id = self._deploy_ironic_cluster(settings=data, nodes=nodes)
        ironic_conn = ironic_actions.IronicActions(
            self.fuel_web.get_public_vip(cluster_id))
        self._create_os_resources(ironic_conn)
        self.show_step(8)
        self._boot_check_delete_vm(ironic_conn)

        # Rebalance swift rings, add nodes and redeploy
        self.show_step(9)
        primary_node = self.fuel_web.get_nailgun_primary_node(
            self.env.d_env.nodes().slaves[0])
        ip = self.fuel_web.get_nailgun_node_by_name(primary_node.name)['ip']
        Common.rebalance_swift_ring(ip)
        self.show_step(10)
        self.show_step(11)
        self.fuel_web.update_nodes(cluster_id, {
            'slave-05': ['controller'],
            'slave-06': ['controller', 'ironic']
        })
        self.show_step(12)
        self.fuel_web.deploy_cluster_wait(cluster_id)
        self.show_step(13)
        self.fuel_web.run_ostf(cluster_id=cluster_id)
        self.show_step(14)
        ironic_conn = ironic_actions.IronicActions(
            self.fuel_web.get_public_vip(cluster_id))
        self._boot_check_delete_vm(ironic_conn)

        # Rebalance swift rings, remove nodes and redeploy
        self.show_step(15)
        primary_node = self.fuel_web.get_nailgun_primary_node(
            self.env.d_env.nodes().slaves[0])
        ip = self.fuel_web.get_nailgun_node_by_name(primary_node.name)['ip']
        Common.rebalance_swift_ring(ip)
        self.show_step(16)
        self.show_step(17)
        self.fuel_web.update_nodes(cluster_id, {
            'slave-05': ['controller'],
            'slave-06': ['controller', 'ironic']
        },
                                   pending_addition=False,
                                   pending_deletion=True)
        self.show_step(18)
        self.fuel_web.deploy_cluster_wait(cluster_id)
        self.show_step(19)
        self.fuel_web.run_ostf(cluster_id=cluster_id)
        self.show_step(20)
        ironic_conn = ironic_actions.IronicActions(
            self.fuel_web.get_public_vip(cluster_id))
        self._boot_check_delete_vm(ironic_conn)

        self.env.make_snapshot("deploy_scale_controller_ironic")
    def deploy_scale_controller_ironic(self):
        """Test cluster scaling with Controller and Ironic

        Scenario:
            1. Create cluster with 5 slave nodes
            2. Bootstrap 1 additional slave node
            3. Add 2 Controller nodes
            4. Add 1 Compute node
            5. Add 1 Controller+Ironic node
            6. Deploy the cluster
            7. Run OSTF tests
            8. Boot, check connectivity, delete Ironic VM
            9. Rebalance Swift rings
            10. Add 1 Controller node
            11. Add 1 Controller+Ironic node
            12. Redeploy the cluster
            13. Run OSTF tests
            14. Boot, check connectivity, delete Ironic VM
            15. Rebalance Swift rings
            16. Remove 1 Controller node
            17. Remove 1 Controller+Ironic node
            18. Redeploy the cluster
            19. Run OSTF tests
            20. Boot, check connectivity, delete Ironic VM

        Duration 90m
        Snapshot deploy_scale_controller_ironic
        """

        self.env.revert_snapshot("ready_with_5_slaves")
        # Deploy 1st part
        data = {
            'net_segment_type': NEUTRON_SEGMENT['vlan'],
            'ironic': True}

        nodes = {
            'slave-01': ['controller'],
            'slave-02': ['controller'],
            'slave-03': ['controller', 'ironic'],
            'slave-04': ['compute']}

        self.show_step(1)
        self.show_step(2)
        self.env.bootstrap_nodes(self.env.d_env.nodes().slaves[5:6])
        self.show_step(3)
        self.show_step(4)
        self.show_step(5)
        self.show_step(6)
        self.show_step(7)
        cluster_id = self._deploy_ironic_cluster(settings=data, nodes=nodes)
        ironic_conn = ironic_actions.IronicActions(
            self.fuel_web.get_public_vip(cluster_id))
        self._create_os_resources(ironic_conn)
        self.show_step(8)
        self._boot_check_delete_vm(ironic_conn)

        # Rebalance swift rings, add nodes and redeploy
        self.show_step(9)
        primary_node = self.fuel_web.get_nailgun_primary_node(
            self.env.d_env.nodes().slaves[0])
        ip = self.fuel_web.get_nailgun_node_by_name(primary_node.name)['ip']
        Common.rebalance_swift_ring(ip)
        self.show_step(10)
        self.show_step(11)
        self.fuel_web.update_nodes(
            cluster_id,
            {
                'slave-05': ['controller'],
                'slave-06': ['controller', 'ironic']
            }
        )
        self.show_step(12)
        self.fuel_web.deploy_cluster_wait(cluster_id)
        self.show_step(13)
        self.fuel_web.run_ostf(cluster_id=cluster_id)
        self.show_step(14)
        ironic_conn = ironic_actions.IronicActions(
            self.fuel_web.get_public_vip(cluster_id))
        self._boot_check_delete_vm(ironic_conn)

        # Rebalance swift rings, remove nodes and redeploy
        self.show_step(15)
        primary_node = self.fuel_web.get_nailgun_primary_node(
            self.env.d_env.nodes().slaves[0])
        ip = self.fuel_web.get_nailgun_node_by_name(primary_node.name)['ip']
        Common.rebalance_swift_ring(ip)
        self.show_step(16)
        self.show_step(17)
        self.fuel_web.update_nodes(
            cluster_id,
            {
                'slave-05': ['controller'],
                'slave-06': ['controller', 'ironic']
            },
            pending_addition=False,
            pending_deletion=True
        )
        self.show_step(18)
        self.fuel_web.deploy_cluster_wait(cluster_id)
        self.show_step(19)
        self.fuel_web.run_ostf(cluster_id=cluster_id)
        self.show_step(20)
        ironic_conn = ironic_actions.IronicActions(
            self.fuel_web.get_public_vip(cluster_id))
        self._boot_check_delete_vm(ironic_conn)

        self.env.make_snapshot("deploy_scale_controller_ironic")
    def neutron_tun_scalability(self):
        """Check HA mode on scalability

        Scenario:
            1. Create cluster
            2. Add 1 controller node
            3. Deploy the cluster
            4. Check swift, and invoke swift-rings-rebalance.sh
               on primary controller if check failed
            5. Add 2 controller nodes
            6. Deploy changes
            7. Check swift, and invoke swift-rings-rebalance.sh
               on primary controller if check failed
            8. Run OSTF
            9. Add 2 controller 1 compute nodes
            10. Deploy changes
            11. Check swift, and invoke swift-rings-rebalance.sh
                on all the controllers
            12. Run OSTF
            13. Delete the primary and the last added controller.
            14. Deploy changes
            15. Check swift, and invoke swift-rings-rebalance.sh
                on all the controllers
            16. Run OSTF

        Duration 160m
        Snapshot neutron_tun_scalability

        """

        def _check_pacemaker(devops_nodes):
            for devops_node in devops_nodes:
                with QuietLogger():
                    self.fuel_web.assert_pacemaker(
                        devops_node.name,
                        devops_nodes, [])
                ret = self.fuel_web.get_pacemaker_status(devops_node.name)
                assert_true(
                    re.search('vip__management\s+\(ocf::fuel:ns_IPaddr2\):'
                              '\s+Started node', ret),
                    'vip management started')
                assert_true(
                    re.search('vip__public\s+\(ocf::fuel:ns_IPaddr2\):'
                              '\s+Started node', ret),
                    'vip public started')

        self.env.revert_snapshot("ready_with_9_slaves")
        # Step 1  Create cluster with 1 controller
        logger.info("STEP1: Create new cluster {0}".format(
            self.__class__.__name__))
        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=DEPLOYMENT_MODE,
            settings={
                "net_provider": 'neutron',
                "net_segment_type": NEUTRON_SEGMENT['tun']
            }
        )

        nodes = {'slave-01': ['controller']}
        logger.info("Adding new node to the cluster: {0}".format(nodes))
        self.fuel_web.update_nodes(
            cluster_id, nodes)
        self.fuel_web.deploy_cluster_wait(cluster_id)

        logger.info("STEP3: Deploy 1 node cluster finishes")
        primary_node = self.env.d_env.get_node(name='slave-01')

        # Step 4. Check swift
        logger.info("STEP4: Check swift on primary controller {0}".format(
            primary_node))
        ip = self.fuel_web.get_nailgun_node_by_name(primary_node.name)['ip']
        Common.rebalance_swift_ring(ip)

        nodes = {'slave-02': ['controller'],
                 'slave-03': ['controller']}
        logger.info("STEP 4: Adding new nodes "
                    "to the cluster: {0}".format(nodes))
        self.fuel_web.update_nodes(
            cluster_id, nodes,
            True, False
        )
        self.fuel_web.deploy_cluster_wait(cluster_id)

        logger.info("STEP6: Deploy 3 ctrl node cluster has finished")
        controllers = ['slave-01', 'slave-02', 'slave-03']
        _check_pacemaker(self.env.d_env.get_nodes(name__in=controllers))

        primary_node_s3 = self.fuel_web.get_nailgun_primary_node(
            self.env.d_env.nodes().slaves[0])

        logger.info("Primary controller after STEP6 is {0}".format(
            primary_node_s3.name))
        logger.info("STEP7: Check swift on primary controller {0}".format(
            primary_node_s3))
        ip = self.fuel_web.get_nailgun_node_by_name(primary_node_s3
                                                    .name)['ip']
        Common.rebalance_swift_ring(ip)

        # Run smoke tests only according to ha and
        # sanity executed in scope of deploy_cluster_wait()

        self.fuel_web.run_ostf(
            cluster_id=cluster_id,
            test_sets=['smoke'])

        nodes = {'slave-04': ['controller'],
                 'slave-05': ['controller'],
                 'slave-06': ['compute']}
        logger.info("Adding new nodes to the cluster: {0}".format(nodes))
        self.fuel_web.update_nodes(
            cluster_id, nodes,
            True, False
        )
        self.fuel_web.deploy_cluster_wait(cluster_id)

        logger.info("STEP10: Deploy 5 ctrl node cluster has finished")
        controllers = ['slave-01', 'slave-02', 'slave-03', 'slave-04',
                       'slave-05']
        _check_pacemaker(self.env.d_env.get_nodes(name__in=controllers))

        primary_node_s9 = self.fuel_web.get_nailgun_primary_node(
            self.env.d_env.nodes().slaves[0])

        logger.info("Primary controller after STEP10 is {0}".format(
            primary_node_s9.name))

        logger.info("STEP11: Check swift on primary controller {0}".format(
            primary_node_s9))

        ip = self.fuel_web.get_nailgun_node_by_name(primary_node_s9
                                                    .name)['ip']
        Common.rebalance_swift_ring(ip)

        # Run smoke tests only according to ha and
        # sanity executed in scope of deploy_cluster_wait()

        # Step 12. Run OSTF
        self.fuel_web.run_ostf(
            cluster_id=cluster_id,
            test_sets=['smoke'])

        nodes = {primary_node_s9.name: ['controller'],
                 'slave-05': ['controller']}
        logger.info("STEP13: Deleting nodes from the cluster: {0}".format(
            nodes))
        self.fuel_web.update_nodes(
            cluster_id, nodes,
            False, True
        )

        # Step 14. Deploy changes
        self.fuel_web.deploy_cluster_wait(cluster_id)

        nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
            cluster_id, ['controller'])
        devops_nodes = [self.fuel_web.get_devops_node_by_nailgun_node(node)
                        for node in nodes]
        _check_pacemaker(devops_nodes)

        logger.info("STEP13-14: Scale down happened. "
                    "3 controller should be now")
        primary_node_s14 = self.fuel_web.get_nailgun_primary_node(
            self.env.d_env.get_node(name=devops_nodes[0].name))

        logger.info("Primary controller after STEP15 is {0}".format(
            primary_node_s14.name))

        logger.info("STEP15: Check swift on primary controller {0}".format(
            primary_node_s14))

        ip = self.fuel_web.get_nailgun_node_by_name(primary_node_s14
                                                    .name)['ip']
        Common.rebalance_swift_ring(ip)

        # Step 16. Run OSTF
        self.fuel_web.run_ostf(
            cluster_id=cluster_id,
            test_sets=['sanity', 'smoke', 'ha'])
        self.env.make_snapshot("neutron_vlan_ha_scalability")