def manual_cic_maintenance_mode(self):
        """Check manual maintenance mode for controller

        Scenario:
            1. Revert snapshot
            2. Switch in maintenance mode
            3. Wait until controller is rebooting
            4. Exit maintenance mode
            5. Check the controller become available

        Duration 155m
        """
        self.env.revert_snapshot('cic_maintenance_mode')

        cluster_id = self.fuel_web.get_last_created_cluster()

        n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
            cluster_id, ['controller'])
        d_ctrls = self.fuel_web.get_devops_nodes_by_nailgun_nodes(n_ctrls)

        for devops_node in d_ctrls:
            _ip = self.fuel_web.get_nailgun_node_by_name(
                devops_node.name)['ip']
            logger.info('Maintenance mode for node {0}'.format(
                devops_node.name))
            asserts.assert_true('True' in checkers.check_available_mode(_ip),
                                "Maintenance mode is not available")
            self.ssh_manager.execute_on_remote(ip=_ip, cmd="umm on")

            logger.info('Wait a {0} node offline status after switching '
                        'maintenance mode '.format(devops_node.name))
            try:
                wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
                    devops_node)['online'],
                     timeout=60 * 10)
            except TimeoutError:
                asserts.assert_false(
                    self.fuel_web.get_nailgun_node_by_devops_node(devops_node)
                    ['online'], 'Node {0} has not become offline after'
                    'switching maintenance mode'.format(devops_node.name))

            logger.info('Check that {0} node in maintenance mode after '
                        'switching'.format(devops_node.name))

            _ip = self.fuel_web.get_nailgun_node_by_name(
                devops_node.name)['ip']
            asserts.assert_true(
                checkers.check_ping(self.env.get_admin_node_ip(),
                                    _ip,
                                    deadline=600),
                "Host {0} is not reachable by ping during 600 sec".format(_ip))

            asserts.assert_true('True' in checkers.check_auto_mode(_ip),
                                "Maintenance mode is not switch")

            self.ssh_manager.execute_on_remote(ip=_ip, cmd="umm off")

            logger.info('Wait a {0} node online status'.format(
                devops_node.name))
            try:
                wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(
                    devops_node)['online'],
                     timeout=60 * 10)
            except TimeoutError:
                asserts.assert_true(
                    self.fuel_web.get_nailgun_node_by_devops_node(devops_node)
                    ['online'], 'Node {0} has not become online after '
                    'exiting maintenance mode'.format(devops_node.name))

            # Wait until MySQL Galera is UP on some controller
            self.fuel_web.wait_mysql_galera_is_up([n.name for n in d_ctrls])

            # Wait until Cinder services UP on a controller
            self.fuel_web.wait_cinder_is_up([n.name for n in d_ctrls])

            _wait(lambda: self.fuel_web.run_single_ostf_test(
                cluster_id,
                test_sets=['sanity'],
                test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get(
                    'Check that required services are running')),
                  timeout=1500)
            logger.debug("Required services are running")

            _wait(lambda: self.fuel_web.run_ostf(cluster_id, test_sets=['ha']),
                  timeout=1500)
            logger.debug("HA tests are pass now")

            try:
                self.fuel_web.run_ostf(cluster_id,
                                       test_sets=['smoke', 'sanity'])
            except AssertionError:
                logger.debug("Test failed from first probe,"
                             " we sleep 600 second try one more time"
                             " and if it fails again - test will fails ")
                time.sleep(600)
                self.fuel_web.run_ostf(cluster_id,
                                       test_sets=['smoke', 'sanity'])
示例#2
0
    def manual_cic_maintenance_mode(self):
        """Check manual maintenance mode for controller

        Scenario:
            1. Revert snapshot
            2. Switch in maintenance mode
            3. Wait until controller is rebooting
            4. Exit maintenance mode
            5. Check the controller become available

        Duration 155m
        """
        self.env.revert_snapshot('cic_maintenance_mode')

        cluster_id = self.fuel_web.get_last_created_cluster()

        for nailgun_node in self.env.d_env.nodes().slaves[0:3]:
            with self.fuel_web.get_ssh_for_node(nailgun_node.name) as remote:
                assert_true('True' in check_available_mode(remote),
                            "Maintenance mode is not available")

                logger.info('Maintenance mode for node %s', nailgun_node.name)
                result = remote.execute('umm on')
                assert_equal(
                    result['exit_code'], 0,
                    'Failed to execute "{0}" on remote host: {1}'.format(
                        'umm on', result))
            logger.info(
                'Wait a %s node offline status after switching '
                'maintenance mode ', nailgun_node.name)
            try:
                wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
                    nailgun_node)['online'],
                     timeout=60 * 10)
            except TimeoutError:
                assert_false(
                    self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
                    ['online'], 'Node {0} has not become offline after'
                    'switching maintenance mode'.format(nailgun_node.name))

            logger.info(
                'Check that %s node in maintenance mode after '
                'switching', nailgun_node.name)

            with self.fuel_web.get_ssh_for_node(nailgun_node.name) as remote:
                assert_true('True' in check_auto_mode(remote),
                            "Maintenance mode is not switch")

                result = remote.execute('umm off')
                assert_equal(
                    result['exit_code'], 0,
                    'Failed to execute "{0}" on remote host: {1}'.format(
                        'umm off', result))

            logger.info('Wait a %s node online status', nailgun_node.name)
            try:
                wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(
                    nailgun_node)['online'],
                     timeout=60 * 10)
            except TimeoutError:
                assert_true(
                    self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
                    ['online'], 'Node {0} has not become online after '
                    'exiting maintenance mode'.format(nailgun_node.name))

            # Wait until MySQL Galera is UP on some controller
            self.fuel_web.wait_mysql_galera_is_up(
                [n.name for n in self.env.d_env.nodes().slaves[0:3]])

            # Wait until Cinder services UP on a controller
            self.fuel_web.wait_cinder_is_up(
                [n.name for n in self.env.d_env.nodes().slaves[0:3]])

            _wait(lambda: self.fuel_web.run_single_ostf_test(
                cluster_id,
                test_sets=['sanity'],
                test_name=map_ostf.OSTF_TEST_MAPPING.get(
                    'Check that required services are running')),
                  timeout=1500)
            logger.debug("Required services are running")

            _wait(lambda: self.fuel_web.run_ostf(cluster_id, test_sets=['ha']),
                  timeout=1500)
            logger.debug("HA tests are pass now")

            try:
                self.fuel_web.run_ostf(cluster_id,
                                       test_sets=['smoke', 'sanity'])
            except AssertionError:
                logger.debug("Test failed from first probe,"
                             " we sleep 600 second try one more time"
                             " and if it fails again - test will fails ")
                time.sleep(600)
                self.fuel_web.run_ostf(cluster_id,
                                       test_sets=['smoke', 'sanity'])
    def negative_auto_cic_maintenance_mode(self):
        """Check negative scenario for auto maintenance mode

        Scenario:
            1. Revert snapshot
            2. Disable UMM
            3. Change UMM.CONF
            4. Unexpected reboot
            5. Check the controller not switching in maintenance mode
            6. Check the controller become available

        Duration 85m
        """
        self.env.revert_snapshot('cic_maintenance_mode')

        cluster_id = self.fuel_web.get_last_created_cluster()

        n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
            cluster_id, ['controller'])
        d_ctrls = self.fuel_web.get_devops_nodes_by_nailgun_nodes(n_ctrls)

        for devops_node in d_ctrls:
            _ip = self.fuel_web.get_nailgun_node_by_name(
                devops_node.name)['ip']
            asserts.assert_true('True' in checkers.check_available_mode(_ip),
                                "Maintenance mode is not available")
            logger.info('Change UMM.CONF on node {0}'.format(devops_node.name))

            command1 = ("echo -e 'UMM=yes\nREBOOT_COUNT=0\n"
                        "COUNTER_RESET_TIME=10' > /etc/umm.conf")
            self.ssh_manager.execute_on_remote(ip=_ip, cmd=command1)

            self.ssh_manager.execute_on_remote(ip=_ip, cmd="umm disable")

            asserts.assert_false('True' in checkers.check_available_mode(_ip),
                                 "Maintenance mode should not be available")

            command2 = 'reboot --force >/dev/null & '

            logger.info('Unexpected reboot on node {0}'.format(
                devops_node.name))

            self.ssh_manager.execute_on_remote(ip=_ip, cmd=command2)

            wait(lambda: not checkers.check_ping(self.env.get_admin_node_ip(),
                                                 _ip),
                 timeout=60 * 10)

            # Node don't have enough time for set offline status
            # after reboot --force
            # Just waiting

            asserts.assert_true(
                checkers.check_ping(self.env.get_admin_node_ip(),
                                    _ip,
                                    deadline=600),
                "Host {0} is not reachable by ping during 600 sec".format(_ip))
            logger.info('Wait a {0} node online status after unexpected '
                        'reboot'.format(devops_node.name))

            self.fuel_web.wait_nodes_get_online_state([devops_node])

            logger.info('Check that {0} node not in maintenance mode after'
                        ' unexpected reboot'.format(devops_node.name))

            asserts.assert_false('True' in checkers.check_auto_mode(_ip),
                                 "Maintenance mode should not switched")

            # Wait until MySQL Galera is UP on some controller
            self.fuel_web.wait_mysql_galera_is_up([n.name for n in d_ctrls])

            # Wait until Cinder services UP on a controller
            self.fuel_web.wait_cinder_is_up([n.name for n in d_ctrls])

            _wait(lambda: self.fuel_web.run_single_ostf_test(
                cluster_id,
                test_sets=['sanity'],
                test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get(
                    'Check that required services are running')),
                  timeout=1500)
            logger.debug("Required services are running")

            _wait(lambda: self.fuel_web.run_ostf(cluster_id, test_sets=['ha']),
                  timeout=1500)
            logger.debug("HA tests are pass now")

            try:
                self.fuel_web.run_ostf(cluster_id,
                                       test_sets=['smoke', 'sanity'])
            except AssertionError:
                logger.debug("Test failed from first probe,"
                             " we sleep 600 second try one more time"
                             " and if it fails again - test will fails ")
                time.sleep(600)
                self.fuel_web.run_ostf(cluster_id,
                                       test_sets=['smoke', 'sanity'])
示例#4
0
    def auto_cic_maintenance_mode(self):
        """Check auto maintenance mode for controller

        Scenario:
            1. Revert snapshot
            2. Unexpected reboot
            3. Wait until controller is switching in maintenance mode
            4. Exit maintenance mode
            5. Check the controller become available

        Duration 155m
        """
        self.env.revert_snapshot('cic_maintenance_mode')

        cluster_id = self.fuel_web.get_last_created_cluster()

        for nailgun_node in self.env.d_env.nodes().slaves[0:3]:
            with self.fuel_web.get_ssh_for_node(nailgun_node.name) as remote:
                assert_true('True' in check_available_mode(remote),
                            "Maintenance mode is not available")

                logger.info('Change UMM.CONF on node %s', nailgun_node.name)
                command1 = ("echo -e 'UMM=yes\nREBOOT_COUNT=0\n"
                            "COUNTER_RESET_TIME=10' > /etc/umm.conf")

                result = remote.execute(command1)
                assert_equal(
                    result['exit_code'], 0,
                    'Failed to execute "{0}" on remote host: {1}'.format(
                        command1, result))

                logger.info('Unexpected reboot on node %s', nailgun_node.name)
                command2 = ('reboot --force >/dev/null & ')
                result = remote.execute(command2)
                assert_equal(
                    result['exit_code'], 0,
                    'Failed to execute "{0}" on remote host: {1}'.format(
                        command2, result))

            logger.info(
                'Wait a %s node offline status after unexpected '
                'reboot', nailgun_node.name)
            try:
                wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
                    nailgun_node)['online'],
                     timeout=60 * 10)
            except TimeoutError:
                assert_false(
                    self.fuel_web.get_nailgun_node_by_devops_node(
                        nailgun_node)['online'],
                    'Node {0} has not become offline after unexpected'
                    'reboot'.format(nailgun_node.name))

            logger.info(
                'Check that %s node in maintenance mode after'
                ' unexpected reboot', nailgun_node.name)

            with self.fuel_web.get_ssh_for_node(nailgun_node.name) as remote:
                assert_true('True' in check_auto_mode(remote),
                            "Maintenance mode is not switch")

                result = remote.execute('umm off')
                assert_equal(
                    result['exit_code'], 0,
                    'Failed to execute "{0}" on remote host: {1}'.format(
                        'umm off', result))
                # Wait umm stops
                time.sleep(30)
                command3 = ("echo -e 'UMM=yes\nREBOOT_COUNT=2\n"
                            "COUNTER_RESET_TIME=10' > /etc/umm.conf")
                result = remote.execute(command3)
                assert_equal(
                    result['exit_code'], 0,
                    'Failed to execute "{0}" on remote host: {1}'.format(
                        command3, result))

            logger.info('Wait a %s node online status', nailgun_node.name)
            try:
                wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(
                    nailgun_node)['online'],
                     timeout=90 * 10)
            except TimeoutError:
                assert_true(
                    self.fuel_web.get_nailgun_node_by_devops_node(
                        nailgun_node)['online'],
                    'Node {0} has not become online after umm off'.format(
                        nailgun_node.name))

            # Wait until MySQL Galera is UP on some controller
            self.fuel_web.wait_mysql_galera_is_up(
                [n.name for n in self.env.d_env.nodes().slaves[0:3]])

            # Wait until Cinder services UP on a controller
            self.fuel_web.wait_cinder_is_up(
                [n.name for n in self.env.d_env.nodes().slaves[0:3]])

            _wait(lambda: self.fuel_web.run_single_ostf_test(
                cluster_id,
                test_sets=['sanity'],
                test_name=map_ostf.OSTF_TEST_MAPPING.get(
                    'Check that required services are running')),
                  timeout=1500)
            logger.debug("Required services are running")

            _wait(lambda: self.fuel_web.run_ostf(cluster_id, test_sets=['ha']),
                  timeout=1500)
            logger.debug("HA tests are pass now")

            try:
                self.fuel_web.run_ostf(cluster_id,
                                       test_sets=['smoke', 'sanity'])
            except AssertionError:
                logger.debug("Test failed from first probe,"
                             " we sleep 600 second try one more time"
                             " and if it fails again - test will fails ")
                time.sleep(600)
                self.fuel_web.run_ostf(cluster_id,
                                       test_sets=['smoke', 'sanity'])
示例#5
0
    def negative_auto_cic_maintenance_mode(self):
        """Check negative scenario for auto maintenance mode

        Scenario:
            1. Revert snapshot
            2. Disable UMM
            3. Change UMM.CONF
            4. Unexpected reboot
            5. Check the controller not switching in maintenance mode
            6. Check the controller become available

        Duration 85m
        """
        self.env.revert_snapshot('cic_maintenance_mode')

        cluster_id = self.fuel_web.get_last_created_cluster()

        for nailgun_node in self.env.d_env.nodes().slaves[0:3]:
            with self.fuel_web.get_ssh_for_node(nailgun_node.name) as remote:
                assert_true('True' in check_available_mode(remote),
                            "Maintenance mode is not available")

                logger.info('Change UMM.CONF on node %s', nailgun_node.name)
                command1 = ("echo -e 'UMM=yes\nREBOOT_COUNT=0\n"
                            "COUNTER_RESET_TIME=10' > /etc/umm.conf")

                result = remote.execute(command1)
                assert_equal(
                    result['exit_code'], 0,
                    'Failed to execute "{0}" on remote host: {1}'.format(
                        command1, result))

                result = remote.execute('umm disable')
                assert_equal(
                    result['exit_code'], 0,
                    'Failed to execute "{0}" on remote host: {1}'.format(
                        'umm disable', result))

                assert_false('True' in check_available_mode(remote),
                             "Maintenance mode should not be available")

                logger.info('Unexpected reboot on node %s', nailgun_node.name)
                command2 = ('reboot --force >/dev/null & ')
                result = remote.execute(command2)
                assert_equal(
                    result['exit_code'], 0,
                    'Failed to execute "{0}" on remote host: {1}'.format(
                        command2, result))

            # Node don't have enough time for set offline status
            # after reboot --force
            # Just waiting

            _ip = self.fuel_web.get_nailgun_node_by_name(
                nailgun_node.name)['ip']
            _wait(lambda: _tcp_ping(_ip, 22), timeout=120)

            logger.info(
                'Wait a %s node online status after unexpected '
                'reboot', nailgun_node.name)
            self.fuel_web.wait_nodes_get_online_state([nailgun_node])

            logger.info(
                'Check that %s node not in maintenance mode after'
                ' unexpected reboot', nailgun_node.name)

            with self.fuel_web.get_ssh_for_node(nailgun_node.name) as remote:
                assert_false('True' in check_auto_mode(remote),
                             "Maintenance mode should not switched")

            # Wait until MySQL Galera is UP on some controller
            self.fuel_web.wait_mysql_galera_is_up(
                [n.name for n in self.env.d_env.nodes().slaves[0:3]])

            # Wait until Cinder services UP on a controller
            self.fuel_web.wait_cinder_is_up(
                [n.name for n in self.env.d_env.nodes().slaves[0:3]])

            _wait(lambda: self.fuel_web.run_single_ostf_test(
                cluster_id,
                test_sets=['sanity'],
                test_name=map_ostf.OSTF_TEST_MAPPING.get(
                    'Check that required services are running')),
                  timeout=1500)
            logger.debug("Required services are running")

            _wait(lambda: self.fuel_web.run_ostf(cluster_id, test_sets=['ha']),
                  timeout=1500)
            logger.debug("HA tests are pass now")

            try:
                self.fuel_web.run_ostf(cluster_id,
                                       test_sets=['smoke', 'sanity'])
            except AssertionError:
                logger.debug("Test failed from first probe,"
                             " we sleep 600 second try one more time"
                             " and if it fails again - test will fails ")
                time.sleep(600)
                self.fuel_web.run_ostf(cluster_id,
                                       test_sets=['smoke', 'sanity'])
    def manual_cic_maintenance_mode(self):
        """Check manual maintenance mode for controller

        Scenario:
            1. Revert snapshot
            2. Switch in maintenance mode
            3. Wait until controller is rebooting
            4. Exit maintenance mode
            5. Check the controller become available

        Duration 155m
        """
        self.env.revert_snapshot("cic_maintenance_mode")

        cluster_id = self.fuel_web.get_last_created_cluster()

        n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles(cluster_id, ["controller"])
        d_ctrls = self.fuel_web.get_devops_nodes_by_nailgun_nodes(n_ctrls)

        for devops_node in d_ctrls:
            with self.fuel_web.get_ssh_for_node(devops_node.name) as remote:
                assert_true("True" in check_available_mode(remote), "Maintenance mode is not available")

                logger.info("Maintenance mode for node %s", devops_node.name)
                result = remote.execute("umm on")
                assert_equal(
                    result["exit_code"], 0, 'Failed to execute "{0}" on remote host: {1}'.format("umm on", result)
                )
            logger.info("Wait a %s node offline status after switching " "maintenance mode ", devops_node.name)
            try:
                wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(devops_node)["online"], timeout=60 * 10)
            except TimeoutError:
                assert_false(
                    self.fuel_web.get_nailgun_node_by_devops_node(devops_node)["online"],
                    "Node {0} has not become offline after" "switching maintenance mode".format(devops_node.name),
                )

            logger.info("Check that %s node in maintenance mode after " "switching", devops_node.name)

            _ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)["ip"]
            wait(lambda: tcp_ping(_ip, 22), timeout=60 * 10)
            with self.fuel_web.get_ssh_for_node(devops_node.name) as remote:
                assert_true("True" in check_auto_mode(remote), "Maintenance mode is not switch")

                result = remote.execute("umm off")
                assert_equal(
                    result["exit_code"], 0, 'Failed to execute "{0}" on remote host: {1}'.format("umm off", result)
                )

            logger.info("Wait a %s node online status", devops_node.name)
            try:
                wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(devops_node)["online"], timeout=60 * 10)
            except TimeoutError:
                assert_true(
                    self.fuel_web.get_nailgun_node_by_devops_node(devops_node)["online"],
                    "Node {0} has not become online after " "exiting maintenance mode".format(devops_node.name),
                )

            # Wait until MySQL Galera is UP on some controller
            self.fuel_web.wait_mysql_galera_is_up([n.name for n in d_ctrls])

            # Wait until Cinder services UP on a controller
            self.fuel_web.wait_cinder_is_up([n.name for n in d_ctrls])

            _wait(
                lambda: self.fuel_web.run_single_ostf_test(
                    cluster_id,
                    test_sets=["sanity"],
                    test_name=map_ostf.OSTF_TEST_MAPPING.get("Check that required services are running"),
                ),
                timeout=1500,
            )
            logger.debug("Required services are running")

            _wait(lambda: self.fuel_web.run_ostf(cluster_id, test_sets=["ha"]), timeout=1500)
            logger.debug("HA tests are pass now")

            try:
                self.fuel_web.run_ostf(cluster_id, test_sets=["smoke", "sanity"])
            except AssertionError:
                logger.debug(
                    "Test failed from first probe,"
                    " we sleep 600 second try one more time"
                    " and if it fails again - test will fails "
                )
                time.sleep(600)
                self.fuel_web.run_ostf(cluster_id, test_sets=["smoke", "sanity"])
    def negative_auto_cic_maintenance_mode(self):
        """Check negative scenario for auto maintenance mode

        Scenario:
            1. Revert snapshot
            2. Disable UMM
            3. Change UMM.CONF
            4. Unexpected reboot
            5. Check the controller not switching in maintenance mode
            6. Check the controller become available

        Duration 85m
        """
        self.env.revert_snapshot("cic_maintenance_mode")

        cluster_id = self.fuel_web.get_last_created_cluster()

        n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles(cluster_id, ["controller"])
        d_ctrls = self.fuel_web.get_devops_nodes_by_nailgun_nodes(n_ctrls)

        for devops_node in d_ctrls:
            _ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)["ip"]
            with self.fuel_web.get_ssh_for_node(devops_node.name) as remote:
                assert_true("True" in check_available_mode(remote), "Maintenance mode is not available")

                logger.info("Change UMM.CONF on node %s", devops_node.name)
                command1 = "echo -e 'UMM=yes\nREBOOT_COUNT=0\n" "COUNTER_RESET_TIME=10' > /etc/umm.conf"

                result = remote.execute(command1)
                assert_equal(
                    result["exit_code"], 0, 'Failed to execute "{0}" on remote host: {1}'.format(command1, result)
                )

                result = remote.execute("umm disable")
                assert_equal(
                    result["exit_code"], 0, 'Failed to execute "{0}" on remote host: {1}'.format("umm disable", result)
                )

                assert_false("True" in check_available_mode(remote), "Maintenance mode should not be available")

                logger.info("Unexpected reboot on node %s", devops_node.name)
                command2 = "reboot --force >/dev/null & "
                result = remote.execute(command2)
                assert_equal(
                    result["exit_code"], 0, 'Failed to execute "{0}" on remote host: {1}'.format(command2, result)
                )
                wait(lambda: not tcp_ping(_ip, 22), timeout=60 * 10)

            # Node don't have enough time for set offline status
            # after reboot --force
            # Just waiting

            wait(lambda: tcp_ping(_ip, 22), timeout=60 * 10)

            logger.info("Wait a %s node online status after unexpected " "reboot", devops_node.name)
            self.fuel_web.wait_nodes_get_online_state([devops_node])

            logger.info("Check that %s node not in maintenance mode after" " unexpected reboot", devops_node.name)

            with self.fuel_web.get_ssh_for_node(devops_node.name) as remote:
                assert_false("True" in check_auto_mode(remote), "Maintenance mode should not switched")

            # Wait until MySQL Galera is UP on some controller
            self.fuel_web.wait_mysql_galera_is_up([n.name for n in d_ctrls])

            # Wait until Cinder services UP on a controller
            self.fuel_web.wait_cinder_is_up([n.name for n in d_ctrls])

            _wait(
                lambda: self.fuel_web.run_single_ostf_test(
                    cluster_id,
                    test_sets=["sanity"],
                    test_name=map_ostf.OSTF_TEST_MAPPING.get("Check that required services are running"),
                ),
                timeout=1500,
            )
            logger.debug("Required services are running")

            _wait(lambda: self.fuel_web.run_ostf(cluster_id, test_sets=["ha"]), timeout=1500)
            logger.debug("HA tests are pass now")

            try:
                self.fuel_web.run_ostf(cluster_id, test_sets=["smoke", "sanity"])
            except AssertionError:
                logger.debug(
                    "Test failed from first probe,"
                    " we sleep 600 second try one more time"
                    " and if it fails again - test will fails "
                )
                time.sleep(600)
                self.fuel_web.run_ostf(cluster_id, test_sets=["smoke", "sanity"])
    def manual_cic_maintenance_mode(self):
        """Check manual maintenance mode for controller

        Scenario:
            1. Revert snapshot
            2. Switch in maintenance mode
            3. Wait until controller is rebooting
            4. Exit maintenance mode
            5. Check the controller become available

        Duration 155m
        """
        self.env.revert_snapshot('cic_maintenance_mode')

        cluster_id = self.fuel_web.get_last_created_cluster()

        for nailgun_node in self.env.d_env.nodes().slaves[0:3]:
            with self.fuel_web.get_ssh_for_node(nailgun_node.name) as remote:
                assert_true('True' in check_available_mode(remote),
                            "Maintenance mode is not available")

                logger.info('Maintenance mode for node %s', nailgun_node.name)
                result = remote.execute('umm on')
                assert_equal(result['exit_code'], 0,
                             'Failed to execute "{0}" on remote host: {1}'.
                             format('umm on', result))
            logger.info('Wait a %s node offline status after switching '
                        'maintenance mode ', nailgun_node.name)
            try:
                wait(
                    lambda: not
                    self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
                    ['online'], timeout=60 * 10)
            except TimeoutError:
                assert_false(
                    self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
                    ['online'],
                    'Node {0} has not become offline after'
                    'switching maintenance mode'.format(nailgun_node.name))

            logger.info('Check that %s node in maintenance mode after '
                        'switching', nailgun_node.name)

            with self.fuel_web.get_ssh_for_node(nailgun_node.name) as remote:
                assert_true('True' in check_auto_mode(remote),
                            "Maintenance mode is not switch")

                result = remote.execute('umm off')
                assert_equal(result['exit_code'], 0,
                             'Failed to execute "{0}" on remote host: {1}'.
                             format('umm off', result))

            logger.info('Wait a %s node online status', nailgun_node.name)
            try:
                wait(
                    lambda:
                    self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
                    ['online'], timeout=60 * 10)
            except TimeoutError:
                assert_true(
                    self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
                    ['online'],
                    'Node {0} has not become online after '
                    'exiting maintenance mode'.format(nailgun_node.name))

            # Wait until MySQL Galera is UP on some controller
            self.fuel_web.wait_mysql_galera_is_up(
                [n.name for n in self.env.d_env.nodes().slaves[0:3]])

            # Wait until Cinder services UP on a controller
            self.fuel_web.wait_cinder_is_up(
                [n.name for n in self.env.d_env.nodes().slaves[0:3]])

            _wait(lambda:
                  self.fuel_web.run_single_ostf_test(
                      cluster_id, test_sets=['sanity'],
                      test_name=map_ostf.OSTF_TEST_MAPPING.get(
                          'Check that required services are running')),
                  timeout=1500)
            logger.debug("Required services are running")

            _wait(lambda:
                  self.fuel_web.run_ostf(cluster_id, test_sets=['ha']),
                  timeout=1500)
            logger.debug("HA tests are pass now")

            try:
                self.fuel_web.run_ostf(cluster_id,
                                       test_sets=['smoke', 'sanity'])
            except AssertionError:
                logger.debug("Test failed from first probe,"
                             " we sleep 600 second try one more time"
                             " and if it fails again - test will fails ")
                time.sleep(600)
                self.fuel_web.run_ostf(cluster_id,
                                       test_sets=['smoke', 'sanity'])
    def negative_auto_cic_maintenance_mode(self):
        """Check negative scenario for auto maintenance mode

        Scenario:
            1. Revert snapshot
            2. Disable UMM
            3. Change UMM.CONF
            4. Unexpected reboot
            5. Check the controller not switching in maintenance mode
            6. Check the controller become available

        Duration 85m
        """
        self.env.revert_snapshot('cic_maintenance_mode')

        cluster_id = self.fuel_web.get_last_created_cluster()

        for nailgun_node in self.env.d_env.nodes().slaves[0:3]:
            with self.fuel_web.get_ssh_for_node(nailgun_node.name) as remote:
                assert_true('True' in check_available_mode(remote),
                            "Maintenance mode is not available")

                logger.info('Change UMM.CONF on node %s', nailgun_node.name)
                command1 = ("echo -e 'UMM=yes\nREBOOT_COUNT=0\n"
                            "COUNTER_RESET_TIME=10' > /etc/umm.conf")

                result = remote.execute(command1)
                assert_equal(result['exit_code'], 0,
                             'Failed to execute "{0}" on remote host: {1}'.
                             format(command1, result))

                result = remote.execute('umm disable')
                assert_equal(result['exit_code'], 0,
                             'Failed to execute "{0}" on remote host: {1}'.
                             format('umm disable', result))

                assert_false('True' in check_available_mode(remote),
                             "Maintenance mode should not be available")

                logger.info('Unexpected reboot on node %s', nailgun_node.name)
                command2 = ('reboot --force >/dev/null & ')
                result = remote.execute(command2)
                assert_equal(result['exit_code'], 0,
                             'Failed to execute "{0}" on remote host: {1}'.
                             format(command2, result))

            # Node don't have enough time for set offline status
            # after reboot --force
            # Just waiting

            _ip = self.fuel_web.get_nailgun_node_by_name(
                nailgun_node.name)['ip']
            _wait(lambda: _tcp_ping(_ip, 22), timeout=120)

            logger.info('Wait a %s node online status after unexpected '
                        'reboot', nailgun_node.name)
            self.fuel_web.wait_nodes_get_online_state([nailgun_node])

            logger.info('Check that %s node not in maintenance mode after'
                        ' unexpected reboot', nailgun_node.name)

            with self.fuel_web.get_ssh_for_node(nailgun_node.name) as remote:
                assert_false('True' in check_auto_mode(remote),
                             "Maintenance mode should not switched")

            # Wait until MySQL Galera is UP on some controller
            self.fuel_web.wait_mysql_galera_is_up(
                [n.name for n in self.env.d_env.nodes().slaves[0:3]])

            # Wait until Cinder services UP on a controller
            self.fuel_web.wait_cinder_is_up(
                [n.name for n in self.env.d_env.nodes().slaves[0:3]])

            _wait(lambda:
                  self.fuel_web.run_single_ostf_test(
                      cluster_id, test_sets=['sanity'],
                      test_name=map_ostf.OSTF_TEST_MAPPING.get(
                          'Check that required services are running')),
                  timeout=1500)
            logger.debug("Required services are running")

            _wait(lambda:
                  self.fuel_web.run_ostf(cluster_id, test_sets=['ha']),
                  timeout=1500)
            logger.debug("HA tests are pass now")

            try:
                self.fuel_web.run_ostf(cluster_id,
                                       test_sets=['smoke', 'sanity'])
            except AssertionError:
                logger.debug("Test failed from first probe,"
                             " we sleep 600 second try one more time"
                             " and if it fails again - test will fails ")
                time.sleep(600)
                self.fuel_web.run_ostf(cluster_id,
                                       test_sets=['smoke', 'sanity'])
    def auto_cic_maintenance_mode(self):
        """Check auto maintenance mode for controller

        Scenario:
            1. Revert snapshot
            2. Unexpected reboot
            3. Wait until controller is switching in maintenance mode
            4. Exit maintenance mode
            5. Check the controller become available

        Duration 155m
        """
        self.env.revert_snapshot('cic_maintenance_mode')

        cluster_id = self.fuel_web.get_last_created_cluster()

        for nailgun_node in self.env.d_env.nodes().slaves[0:3]:
            with self.fuel_web.get_ssh_for_node(nailgun_node.name) as remote:
                assert_true('True' in check_available_mode(remote),
                            "Maintenance mode is not available")

                logger.info('Change UMM.CONF on node %s', nailgun_node.name)
                command1 = ("echo -e 'UMM=yes\nREBOOT_COUNT=0\n"
                            "COUNTER_RESET_TIME=10' > /etc/umm.conf")

                result = remote.execute(command1)
                assert_equal(result['exit_code'], 0,
                             'Failed to execute "{0}" on remote host: {1}'.
                             format(command1, result))

                logger.info('Unexpected reboot on node %s', nailgun_node.name)
                command2 = ('reboot --force >/dev/null & ')
                result = remote.execute(command2)
                assert_equal(result['exit_code'], 0,
                             'Failed to execute "{0}" on remote host: {1}'.
                             format(command2, result))

            logger.info('Wait a %s node offline status after unexpected '
                        'reboot', nailgun_node.name)
            try:
                wait(
                    lambda: not
                    self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
                    ['online'], timeout=60 * 10)
            except TimeoutError:
                assert_false(
                    self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
                    ['online'],
                    'Node {0} has not become offline after unexpected'
                    'reboot'.format(nailgun_node.name))

            logger.info('Check that %s node in maintenance mode after'
                        ' unexpected reboot', nailgun_node.name)

            with self.fuel_web.get_ssh_for_node(nailgun_node.name) as remote:
                assert_true('True' in check_auto_mode(remote),
                            "Maintenance mode is not switch")

                result = remote.execute('umm off')
                assert_equal(result['exit_code'], 0,
                             'Failed to execute "{0}" on remote host: {1}'.
                             format('umm off', result))
                # Wait umm stops
                time.sleep(30)
                command3 = ("echo -e 'UMM=yes\nREBOOT_COUNT=2\n"
                            "COUNTER_RESET_TIME=10' > /etc/umm.conf")
                result = remote.execute(command3)
                assert_equal(result['exit_code'], 0,
                             'Failed to execute "{0}" on remote host: {1}'.
                             format(command3, result))

            logger.info('Wait a %s node online status', nailgun_node.name)
            try:
                wait(
                    lambda:
                    self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
                    ['online'], timeout=90 * 10)
            except TimeoutError:
                assert_true(
                    self.fuel_web.get_nailgun_node_by_devops_node(nailgun_node)
                    ['online'],
                    'Node {0} has not become online after umm off'.format(
                        nailgun_node.name))

            # Wait until MySQL Galera is UP on some controller
            self.fuel_web.wait_mysql_galera_is_up(
                [n.name for n in self.env.d_env.nodes().slaves[0:3]])

            # Wait until Cinder services UP on a controller
            self.fuel_web.wait_cinder_is_up(
                [n.name for n in self.env.d_env.nodes().slaves[0:3]])

            _wait(lambda:
                  self.fuel_web.run_single_ostf_test(
                      cluster_id, test_sets=['sanity'],
                      test_name=map_ostf.OSTF_TEST_MAPPING.get(
                          'Check that required services are running')),
                  timeout=1500)
            logger.debug("Required services are running")

            _wait(lambda:
                  self.fuel_web.run_ostf(cluster_id, test_sets=['ha']),
                  timeout=1500)
            logger.debug("HA tests are pass now")

            try:
                self.fuel_web.run_ostf(cluster_id,
                                       test_sets=['smoke', 'sanity'])
            except AssertionError:
                logger.debug("Test failed from first probe,"
                             " we sleep 600 second try one more time"
                             " and if it fails again - test will fails ")
                time.sleep(600)
                self.fuel_web.run_ostf(cluster_id,
                                       test_sets=['smoke', 'sanity'])