def deploy_cluster_with_reboot_plugin(self):
        """Add pre-deployment reboot task to nailgun via plugin.

        Scenario:
        1. Revert snapshot with 5 nodes
        2. Download and install fuel-plugin-builder
        3. Create plugin with reboot task
        4. Build plugin and copy it in var directory
        5. Install plugin to fuel
        6. Create cluster and enable plugin
        7. Provision nodes
        8. Collect timestamps from nodes
        9. Deploy cluster
        10. Check if timestamps are changed

        Duration 40m
        """
        # define some plugin related variables
        plugin_name = 'reboot_plugin'
        source_plugin_path = os.path.join('/root/', plugin_name)
        plugin_path = '/var'
        tasks_path = os.path.dirname(os.path.abspath(__file__))
        tasks_file = 'reboot_tasks.yaml'
        self.show_step(1, initialize=True)
        self.env.revert_snapshot("ready_with_5_slaves")
        # let's get ssh client for the master node

        # initiate fuel plugin builder instance
        self.show_step(2)
        fpb = FuelPluginBuilder()
        # install fuel_plugin_builder on master node
        fpb.fpb_install()
        # create plugin template on the master node
        self.show_step(3)
        fpb.fpb_create_plugin(source_plugin_path)
        # replace plugin tasks with our file
        fpb.fpb_replace_plugin_content(
            os.path.join(tasks_path, tasks_file),
            os.path.join(source_plugin_path, 'tasks.yaml'))
        # build plugin
        self.show_step(4)
        packet_name = fpb.fpb_build_plugin(source_plugin_path)
        fpb.fpb_copy_plugin(
            os.path.join(source_plugin_path, packet_name), plugin_path)
        self.show_step(5)
        utils.install_plugin_check_code(
            ip=self.ssh_manager.admin_ip,
            plugin=os.path.join(plugin_path, packet_name))
        self.show_step(6)
        # create cluster
        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=DEPLOYMENT_MODE,
        )
        # get plugins from fuel and enable our one
        msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
        asserts.assert_true(
            self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
            msg)
        options = {'metadata/enabled': True}
        self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)

        logger.info('Cluster is {!s}'.format(cluster_id))

        self.fuel_web.update_nodes(
            cluster_id,
            {'slave-01': ['controller', 'ceph-osd'],
                'slave-02': ['compute', 'ceph-osd'],
                'slave-03': ['compute'],
                'slave-04': ['ceph-osd']}
        )
        # firstly, let's provision nodes
        self.show_step(7)
        self.fuel_web.provisioning_cluster_wait(cluster_id)
        # after provision is done, collect timestamps from nodes
        old_timestamps = {}

        nodes = {
            'slave-01': True,
            'slave-02': True,
            'slave-03': False,
            'slave-04': True
        }
        self.show_step(8)
        for node in nodes:
            logger.debug(
                "Get init object creation time from node {0}".format(node))
            cmd = 'stat --printf=\'%Y\' /proc/1'
            with self.fuel_web.get_ssh_for_node(node) as node_ssh:
                old_timestamps[node] = node_ssh.execute(cmd)['stdout'][0]

        # start deploying nodes
        # here nodes with controller and ceph roles should be rebooted
        self.show_step(9)
        self.fuel_web.deploy_cluster_wait_progress(cluster_id, 30)

        # collect new timestamps and check them
        self.show_step(10)
        for node in nodes:
            logger.debug(
                "Get init object creation time from node {0}".format(node))
            cmd = 'stat --printf=\'%Y\' /proc/1'
            with self.fuel_web.get_ssh_for_node(node) as node_ssh:
                new_timestamp = node_ssh.execute(cmd)['stdout'][0]
            # compute node without ceph role shouldn't reboot
            if not nodes[node]:
                asserts.assert_equal(
                    new_timestamp, old_timestamps[node],
                    'The new timestamp {0} is not equal to old one {1}, '
                    'but it shouldn\'t for {2} node'
                    .format(new_timestamp, old_timestamps[node], node)
                )
            else:
                # other nodes should be rebooted and have new timestamps
                # greater than old
                asserts.assert_true(
                    new_timestamp > old_timestamps[node],
                    'The new timestamp {0} is not greater than old one {1} '
                    'but it should for node {2}'
                    .format(new_timestamp, old_timestamps[node], node)
                )
    def vip_reservation_for_plugin(self):
        """Check vip reservation for fuel plugin

        Scenario:
        1. Revert snapshot with 3 nodes
        2. Download and install fuel-plugin-builder
        3. Create plugin with predefined network_roles.yaml
        4. Build and copy plugin from container nailgun
        5. Install plugin to fuel
        6. Create cluster and enable plugin
        7. Deploy cluster
        8. Check vip reservation

        Duration 40m
        """
        plugin_name = 'vip_reservation_plugin'
        plugin_path = '/var'
        dir_path = os.path.dirname(os.path.abspath(__file__))
        tasks_file = 'tasks.yaml'
        net_role_file = 'network_roles.yaml'
        metadata_file = 'metadata.yaml'
        namespace = 'haproxy'

        self.env.revert_snapshot("ready_with_3_slaves")
        with self.env.d_env.get_admin_remote() as admin_remote:
            # initiate fuel plugin builder instance
            fpb = FuelPluginBuilder(admin_remote)
            # install fuel_plugin_builder on master node
            fpb.fpb_install()
            # create plugin template on the master node
            fpb.fpb_create_plugin(plugin_name)
            # replace plugin tasks, metadata, network_roles
            fpb.fpb_replace_plugin_content(
                os.path.join(dir_path, net_role_file),
                os.path.join('/root/', plugin_name, net_role_file))
            fpb.fpb_replace_plugin_content(
                os.path.join(dir_path, tasks_file),
                os.path.join('/root/', plugin_name, tasks_file))
            fpb.fpb_replace_plugin_content(
                os.path.join(dir_path, metadata_file),
                os.path.join('/root/', plugin_name, metadata_file))
            # build plugin
            fpb.fpb_build_plugin(os.path.join('/root/', plugin_name))
            # copy plugin archive file from nailgun container
            # to the /var directory on the master node
            fpb.fpb_copy_plugin_from_container(plugin_name, plugin_path)
            # let's install plugin
            checkers.install_plugin_check_code(
                admin_remote,
                plugin=os.path.join(plugin_path, '{}.rpm'.format(plugin_name)))

        cluster_id = self.fuel_web.create_cluster(name=self.__class__.__name__,
                                                  mode=DEPLOYMENT_MODE,
                                                  settings={
                                                      "net_provider":
                                                      'neutron',
                                                      "net_segment_type":
                                                      NEUTRON_SEGMENT_TYPE
                                                  })
        # get plugins from fuel and enable our one
        msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
        asserts.assert_true(
            self.fuel_web.check_plugin_exists(cluster_id, plugin_name), msg)
        options = {'metadata/enabled': True}
        self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)

        logger.info('cluster is %s' % str(cluster_id))

        self.fuel_web.update_nodes(cluster_id, {
            'slave-01': ['controller'],
            'slave-02': ['compute']
        })
        self.fuel_web.deploy_cluster_wait(cluster_id)

        self.fuel_web.run_ostf(cluster_id=cluster_id)

        with self.fuel_web.get_ssh_for_node('slave-01') as remote:
            hiera_json_out = "ruby -rhiera -rjson -e \"h = Hiera.new(); " \
                             "Hiera.logger = 'noop'; puts JSON.dump " \
                             "(h.lookup('network_metadata', " \
                             "[], {}, nil, nil))\""
            for vip in ('reserved_pub', 'reserved_mng'):
                # get vips from hiera
                vip_hiera = json.loads(
                    remote.execute(hiera_json_out)['stdout']
                    [0])["vips"][vip]["ipaddr"]
                # get vips from database
                vip_db = self.env.postgres_actions.run_query(
                    db='nailgun',
                    query="select ip_addr from ip_addrs where "
                    "vip_type = '\"'\"'{0}'\"'\"';".format(vip))
                vip_array = [vip_hiera, vip_db]
                for ip in vip_array[1:]:
                    asserts.assert_equal(
                        vip_array[0], ip,
                        "Vip from hiera output {0} does not equal "
                        "to {1}".format(vip_array[0], ip))
                vip_pcs = remote.execute('pcs resource show {0}{1}'.format(
                    'vip__', vip))['exit_code']
                asserts.assert_not_equal(
                    0, vip_pcs, 'The vip__{0} was found in '
                    'pacemaker'.format(vip))
                vip_ns = remote.execute(
                    'ip netns exec {0} ip a | grep {1}{2}'.format(
                        namespace, 'b_', vip))['exit_code']
                asserts.assert_not_equal(
                    0, vip_ns, 'The {0} was found in {1} '
                    'namespace'.format(vip, namespace))
    def deploy_cluster_with_reboot_plugin(self):
        """Add pre-deployment reboot task to nailgun via plugin.

        Scenario:
        1. Revert snapshot with 5 nodes
        2. Download and install fuel-plugin-builder
        3. Create plugin with reboot task
        4. Build plugin and copy it in var directory
        5. Install plugin to fuel
        6. Create cluster and enable plugin
        7. Provision nodes
        8. Collect timestamps from nodes
        9. Deploy cluster
        10. Check if timestamps are changed

        Duration 40m
        """
        # define some plugin related variables
        plugin_name = 'reboot_plugin'
        source_plugin_path = os.path.join('/root/', plugin_name)
        plugin_path = '/var'
        tasks_path = os.path.dirname(os.path.abspath(__file__))
        tasks_file = 'reboot_tasks.yaml'
        self.show_step(1, initialize=True)
        self.env.revert_snapshot("ready_with_5_slaves")
        # let's get ssh client for the master node

        # initiate fuel plugin builder instance
        self.show_step(2)
        fpb = FuelPluginBuilder()
        # install fuel_plugin_builder on master node
        fpb.fpb_install()
        # create plugin template on the master node
        self.show_step(3)
        fpb.fpb_create_plugin(source_plugin_path)
        # replace plugin tasks with our file
        fpb.fpb_replace_plugin_content(
            os.path.join(tasks_path, tasks_file),
            os.path.join(source_plugin_path, 'tasks.yaml'))
        # build plugin
        self.show_step(4)
        packet_name = fpb.fpb_build_plugin(source_plugin_path)
        fpb.fpb_copy_plugin(os.path.join(source_plugin_path, packet_name),
                            plugin_path)
        self.show_step(5)
        utils.install_plugin_check_code(ip=self.ssh_manager.admin_ip,
                                        plugin=os.path.join(
                                            plugin_path, packet_name))
        self.show_step(6)
        # create cluster
        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=DEPLOYMENT_MODE,
        )
        # get plugins from fuel and enable our one
        msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
        asserts.assert_true(
            self.fuel_web.check_plugin_exists(cluster_id, plugin_name), msg)
        options = {'metadata/enabled': True}
        self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)

        logger.info('Cluster is {!s}'.format(cluster_id))

        self.fuel_web.update_nodes(
            cluster_id, {
                'slave-01': ['controller', 'ceph-osd'],
                'slave-02': ['compute', 'ceph-osd'],
                'slave-03': ['compute'],
                'slave-04': ['ceph-osd']
            })
        # firstly, let's provision nodes
        self.show_step(7)
        self.fuel_web.provisioning_cluster_wait(cluster_id)
        # after provision is done, collect timestamps from nodes
        old_timestamps = {}

        nodes = {
            'slave-01': True,
            'slave-02': True,
            'slave-03': False,
            'slave-04': True
        }
        self.show_step(8)
        for node in nodes:
            logger.debug(
                "Get init object creation time from node {0}".format(node))
            cmd = 'stat --printf=\'%Y\' /proc/1'
            with self.fuel_web.get_ssh_for_node(node) as node_ssh:
                old_timestamps[node] = node_ssh.execute(cmd)['stdout'][0]

        # start deploying nodes
        # here nodes with controller and ceph roles should be rebooted
        self.show_step(9)
        self.fuel_web.deploy_cluster_wait_progress(cluster_id, 30)

        # collect new timestamps and check them
        self.show_step(10)
        for node in nodes:
            logger.debug(
                "Get init object creation time from node {0}".format(node))
            cmd = 'stat --printf=\'%Y\' /proc/1'
            with self.fuel_web.get_ssh_for_node(node) as node_ssh:
                new_timestamp = node_ssh.execute(cmd)['stdout'][0]
            # compute node without ceph role shouldn't reboot
            if not nodes[node]:
                asserts.assert_equal(
                    new_timestamp, old_timestamps[node],
                    'The new timestamp {0} is not equal to old one {1}, '
                    'but it shouldn\'t for {2} node'.format(
                        new_timestamp, old_timestamps[node], node))
            else:
                # other nodes should be rebooted and have new timestamps
                # greater than old
                asserts.assert_true(
                    new_timestamp > old_timestamps[node],
                    'The new timestamp {0} is not greater than old one {1} '
                    'but it should for node {2}'.format(
                        new_timestamp, old_timestamps[node], node))
    def deploy_cluster_with_reboot_plugin_timeout(self):
        """Check deployment is failed by reboot task plugin.

        Scenario:
            1. Revert snapshot with 3 nodes
            2. Download and install fuel-plugin-builder
            3. Create plugin with reboot task,
               set timeout for reboot task as 1 second
            4. Build plugin
            5. Install plugin to fuel
            6. Create cluster and enable plugin
            7. Provision nodes
            8. Deploy cluster
            9. Check deployment was failed by reboot task
            10. Check error msg at the logs

        Duration 15m
        """
        # define some plugin related variables
        plugin_name = 'timeout_plugin'
        source_plugin_path = os.path.join('/root/', plugin_name)
        plugin_path = '/var'
        tasks_path = os.path.dirname(os.path.abspath(__file__))
        tasks_file = 'reboot_tasks.yaml'
        # start reverting snapshot
        self.show_step(1, initialize=True)
        self.env.revert_snapshot("ready_with_3_slaves")
        # let's get ssh client for the master node
        self.show_step(2)
        # initiate fuel plugin builder instance
        fpb = FuelPluginBuilder()
        # install fuel_plugin_builder on master node
        fpb.fpb_install()
        # change timeout to a new value '1'
        fpb.put_value_to_local_yaml(os.path.join(tasks_path, tasks_file),
                                    os.path.join('/tmp/', tasks_file),
                                    [1, 'parameters', 'timeout'],
                                    1)
        self.show_step(3)
        # create plugin template on the master node
        fpb.fpb_create_plugin(source_plugin_path)
        # replace plugin tasks with our file
        fpb.fpb_replace_plugin_content(
            os.path.join('/tmp/', tasks_file),
            os.path.join(source_plugin_path, 'tasks.yaml'))
        # build plugin
        self.show_step(4)
        packet_name = fpb.fpb_build_plugin(source_plugin_path)
        # copy plugin archive file
        # to the /var directory on the master node
        fpb.fpb_copy_plugin(
            os.path.join(source_plugin_path, packet_name),
            plugin_path)
        # let's install plugin
        self.show_step(5)
        utils.install_plugin_check_code(
            ip=self.ssh_manager.admin_ip,
            plugin=os.path.join(plugin_path, packet_name))
        # create cluster
        self.show_step(6)
        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=DEPLOYMENT_MODE,
        )
        # get plugins from fuel and enable it
        msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
        asserts.assert_true(
            self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
            msg)
        options = {'metadata/enabled': True}
        self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)

        logger.info('Cluster is {!s}'.format(cluster_id))

        self.fuel_web.update_nodes(
            cluster_id,
            {'slave-01': ['controller', 'ceph-osd']}
        )
        self.show_step(7)
        self.fuel_web.provisioning_cluster_wait(cluster_id)
        logger.info('Start cluster #%s deployment', cluster_id)
        self.show_step(8)
        task = self.fuel_web.client.deploy_nodes(cluster_id)
        self.show_step(9)
        self.fuel_web.assert_task_failed(task)

        msg = 'Time detection (1 sec) for node reboot has expired'
        cmd = 'grep "{0}" /var/log/astute/astute.log'.format(msg)
        self.show_step(10)
        with self.env.d_env.get_admin_remote() as admin_remote:
            result = admin_remote.execute(cmd)['stdout'][0]

        asserts.assert_true(
            msg in result,
            'Failed to find reboot plugin warning message in logs'
        )
Beispiel #5
0
    def vip_reservation_for_plugin_custom_ns(self):
        """Check vip reservation for custom ns plugin

        Scenario:
        1. Revert snapshot with 3 nodes
        2. Download and install fuel-plugin-builder
        3. Create plugin with predefined network_roles.yaml
        4. Build and copy plugin to /var
        5. Install plugin to fuel
        6. Create cluster and enable plugin
        7. Deploy cluster
        8. Check vip reservation

        Duration 40m
        """
        plugin_name = 'vip_reservation_plugin'
        source_plugin_path = os.path.join('/root/', plugin_name)
        plugin_path = '/var'
        task_path = os.path.dirname(os.path.abspath(__file__))
        tasks_file = 'tasks.yaml'
        net_role_file = 'network_roles.yaml'
        metadata_file = 'metadata.yaml'
        namespace = 'custom_ns'
        self.show_step(1, initialize=True)
        self.env.revert_snapshot("ready_with_3_slaves")

        self.show_step(2)
        # initiate fuel plugin builder instance
        fpb = FuelPluginBuilder()
        # install fuel_plugin_builder on master node
        fpb.fpb_install()
        # create plugin template on the master node
        self.show_step(3)
        fpb.fpb_create_plugin(source_plugin_path)
        # replace plugin tasks, metadata, network_roles
        fpb.fpb_replace_plugin_content(
            os.path.join(task_path, net_role_file),
            os.path.join(source_plugin_path, net_role_file))
        fpb.fpb_replace_plugin_content(
            os.path.join(task_path, tasks_file),
            os.path.join(source_plugin_path, tasks_file))
        fpb.fpb_replace_plugin_content(
            os.path.join(task_path, metadata_file),
            os.path.join(source_plugin_path, metadata_file))

        fpb.change_remote_yaml(os.path.join(source_plugin_path, net_role_file),
                               [0, 'properties', 'vip', 0, 'namespace'],
                               namespace)
        fpb.change_remote_yaml(os.path.join(source_plugin_path, net_role_file),
                               [1, 'properties', 'vip', 0, 'namespace'],
                               namespace)
        # build plugin
        self.show_step(4)
        packet_name = fpb.fpb_build_plugin(source_plugin_path)
        # copy plugin archive file
        # to the /var directory on the master node
        fpb.fpb_copy_plugin(os.path.join(source_plugin_path, packet_name),
                            plugin_path)
        self.show_step(5)
        # let's install plugin
        utils.install_plugin_check_code(ip=self.ssh_manager.admin_ip,
                                        plugin=os.path.join(
                                            plugin_path, packet_name))
        self.show_step(6)
        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=DEPLOYMENT_MODE,
        )
        # get plugins from fuel and enable our one
        msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
        asserts.assert_true(
            self.fuel_web.check_plugin_exists(cluster_id, plugin_name), msg)
        options = {'metadata/enabled': True}
        self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)

        logger.info('Cluster is {!s}'.format(cluster_id))

        self.fuel_web.update_nodes(cluster_id, {
            'slave-01': ['controller'],
            'slave-02': ['compute']
        })
        self.show_step(7)
        self.fuel_web.deploy_cluster_wait(cluster_id)

        self.fuel_web.run_ostf(cluster_id=cluster_id)

        self.show_step(8)
        with self.fuel_web.get_ssh_for_node('slave-01') as remote:
            hiera_json_out = "ruby -rhiera -rjson -e \"h = Hiera.new(); " \
                             "Hiera.logger = 'noop'; " \
                             "puts JSON.dump(h.lookup('network_metadata', " \
                             "[], {}, nil, nil))\""
            for vip in ('reserved_pub', 'reserved_mng'):
                # get vips from hiera
                vip_hiera = json.loads(
                    remote.execute(hiera_json_out)['stdout']
                    [0])["vips"][vip]["ipaddr"]
                # get vips from database
                vip_db = self.env.postgres_actions.run_query(
                    db='nailgun',
                    query="select ip_addr from ip_addrs where "
                    "vip_name = '\"'\"'{0}'\"'\"';".format(vip))
                # get vips from corosync
                vip_crm = remote.execute(
                    'crm_resource --resource {0}{1} --get-parameter=ip'.format(
                        'vip__', vip))['stdout'][0].rstrip()
                # get vips from namespace
                vip_ns = remote.execute(
                    'ip netns exec {0} ip -4 a show {1}{2}'.format(
                        namespace, 'b_',
                        vip))['stdout'][1].split(' ')[5].split('/')[0]
                vip_array = [vip_hiera, vip_db, vip_crm, vip_ns]
                for ip in vip_array[1:]:
                    asserts.assert_equal(
                        vip_array[0], ip,
                        "Vip from hiera output {0} does not equal "
                        "to {1}".format(vip_array[0], ip))
    def deploy_cluster_with_reboot_plugin_timeout(self):
        """Check deployment is failed by reboot task plugin.

        Scenario:
            1. Revert snapshot with 3 nodes
            2. Download and install fuel-plugin-builder
            3. Create plugin with reboot task,
               set timeout for reboot task as 1 second
            4. Build plugin
            5. Install plugin to fuel
            6. Create cluster and enable plugin
            7. Provision nodes
            8. Deploy cluster
            9. Check deployment was failed by reboot task
            10. Check error msg at the logs

        Duration 15m
        """
        # define some plugin related variables
        plugin_name = 'timeout_plugin'
        source_plugin_path = os.path.join('/root/', plugin_name)
        plugin_path = '/var'
        tasks_path = os.path.dirname(os.path.abspath(__file__))
        tasks_file = 'reboot_tasks.yaml'
        # start reverting snapshot
        self.show_step(1, initialize=True)
        self.env.revert_snapshot("ready_with_3_slaves")
        # let's get ssh client for the master node
        self.show_step(2)
        # initiate fuel plugin builder instance
        fpb = FuelPluginBuilder()
        # install fuel_plugin_builder on master node
        fpb.fpb_install()
        # change timeout to a new value '1'
        fpb.put_value_to_local_yaml(os.path.join(tasks_path, tasks_file),
                                    os.path.join('/tmp/', tasks_file),
                                    [1, 'parameters', 'timeout'], 1)
        self.show_step(3)
        # create plugin template on the master node
        fpb.fpb_create_plugin(source_plugin_path)
        # replace plugin tasks with our file
        fpb.fpb_replace_plugin_content(
            os.path.join('/tmp/', tasks_file),
            os.path.join(source_plugin_path, 'tasks.yaml'))
        # build plugin
        self.show_step(4)
        packet_name = fpb.fpb_build_plugin(source_plugin_path)
        # copy plugin archive file
        # to the /var directory on the master node
        fpb.fpb_copy_plugin(os.path.join(source_plugin_path, packet_name),
                            plugin_path)
        # let's install plugin
        self.show_step(5)
        utils.install_plugin_check_code(ip=self.ssh_manager.admin_ip,
                                        plugin=os.path.join(
                                            plugin_path, packet_name))
        # create cluster
        self.show_step(6)
        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=DEPLOYMENT_MODE,
        )
        # get plugins from fuel and enable it
        msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
        asserts.assert_true(
            self.fuel_web.check_plugin_exists(cluster_id, plugin_name), msg)
        options = {'metadata/enabled': True}
        self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)

        logger.info('Cluster is {!s}'.format(cluster_id))

        self.fuel_web.update_nodes(cluster_id,
                                   {'slave-01': ['controller', 'ceph-osd']})
        self.show_step(7)
        self.fuel_web.provisioning_cluster_wait(cluster_id)
        logger.info('Start cluster #%s deployment', cluster_id)
        self.show_step(8)
        task = self.fuel_web.client.deploy_nodes(cluster_id)
        self.show_step(9)
        self.fuel_web.assert_task_failed(task)

        msg = 'Time detection (1 sec) for node reboot has expired'
        cmd = 'grep "{0}" /var/log/astute/astute.log'.format(msg)
        self.show_step(10)
        with self.env.d_env.get_admin_remote() as admin_remote:
            result = admin_remote.execute(cmd)['stdout'][0]

        asserts.assert_true(
            msg in result,
            'Failed to find reboot plugin warning message in logs')
    def vip_reservation_for_plugin_custom_ns(self):
        """Check vip reservation for custom ns plugin

        Scenario:
        1. Revert snapshot with 3 nodes
        2. Download and install fuel-plugin-builder
        3. Create plugin with predefined network_roles.yaml
        4. Build and copy plugin from container nailgun
        5. Install plugin to fuel
        6. Create cluster and enable plugin
        7. Deploy cluster
        8. Check vip reservation

        Duration 40m
        """
        plugin_name = 'vip_reservation_plugin'
        container_plugin_path = os.path.join('/root/', plugin_name)
        plugin_path = '/var'
        task_path = os.path.dirname(os.path.abspath(__file__))
        tasks_file = 'tasks.yaml'
        net_role_file = 'network_roles.yaml'
        metadata_file = 'metadata.yaml'
        namespace = 'custom_ns'

        self.env.revert_snapshot("ready_with_3_slaves")

        with self.env.d_env.get_admin_remote() as admin_remote:
            # initiate fuel plugin builder instance
            fpb = FuelPluginBuilder(admin_remote)
            # install fuel_plugin_builder on master node
            fpb.fpb_install()
            # create plugin template on the master node
            fpb.fpb_create_plugin(container_plugin_path)
            # replace plugin tasks, metadata, network_roles
            fpb.fpb_replace_plugin_content(
                os.path.join(task_path, net_role_file),
                os.path.join(container_plugin_path, net_role_file))
            fpb.fpb_replace_plugin_content(
                os.path.join(task_path, tasks_file),
                os.path.join(container_plugin_path, tasks_file))
            fpb.fpb_replace_plugin_content(
                os.path.join(task_path, metadata_file),
                os.path.join(container_plugin_path, metadata_file))

            fpb.change_yaml_file_in_container(
                os.path.join(container_plugin_path, net_role_file),
                [0, 'properties', 'vip', 0, 'namespace'],
                namespace)
            fpb.change_yaml_file_in_container(
                os.path.join(container_plugin_path, net_role_file),
                [1, 'properties', 'vip', 0, 'namespace'],
                namespace)
            # build plugin
            fpb.fpb_build_plugin(container_plugin_path)
            # copy plugin archive file from nailgun container
            # to the /var directory on the master node
            fpb.fpb_copy_plugin_from_container(plugin_name, plugin_path)
            # let's install plugin
            checkers.install_plugin_check_code(
                admin_remote,
                plugin=os.path.join(plugin_path, '{}.rpm'.format(plugin_name)))

        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=DEPLOYMENT_MODE,
        )
        # get plugins from fuel and enable our one
        msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
        asserts.assert_true(
            self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
            msg)
        options = {'metadata/enabled': True}
        self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)

        logger.info('cluster is %s' % str(cluster_id))

        self.fuel_web.update_nodes(
            cluster_id,
            {
                'slave-01': ['controller'],
                'slave-02': ['compute']}
        )
        self.fuel_web.deploy_cluster_wait(cluster_id)

        self.fuel_web.run_ostf(cluster_id=cluster_id)

        with self.fuel_web.get_ssh_for_node('slave-01') as remote:
            hiera_json_out = "ruby -rhiera -rjson -e \"h = Hiera.new(); " \
                             "Hiera.logger = 'noop'; " \
                             "puts JSON.dump(h.lookup('network_metadata', " \
                             "[], {}, nil, nil))\""
            for vip in ('reserved_pub', 'reserved_mng'):
                # get vips from hiera
                vip_hiera = json.loads(
                    remote.execute(
                        hiera_json_out)['stdout'][0])["vips"][vip]["ipaddr"]
                # get vips from database
                vip_db = self.env.postgres_actions.run_query(
                    db='nailgun',
                    query="select ip_addr from ip_addrs where "
                          "vip_type = '\"'\"'{0}'\"'\"';".format(vip))
                # get vips from pacemaker
                vip_pcs = remote.execute(
                    'pcs resource show {0}{1}'.format(
                        'vip__', vip))['stdout'][1].split(' ')[6].split('=')[1]
                # get vips from namespace
                vip_ns = remote.execute(
                    'ip netns exec {0} ip -4 a show {1}{2}'.format(
                        namespace, 'b_',
                        vip))['stdout'][1].split(' ')[5].split('/')[0]
                vip_array = [vip_hiera, vip_db, vip_pcs, vip_ns]
                for ip in vip_array[1:]:
                    asserts.assert_equal(
                        vip_array[0], ip,
                        "Vip from hiera output {0} does not equal "
                        "to {1}".format(vip_array[0], ip))
    def vip_reservation_for_plugin(self):
        """Check vip reservation for fuel plugin

        Scenario:
        1. Revert snapshot with 3 nodes
        2. Download and install fuel-plugin-builder
        3. Create plugin with predefined network_roles.yaml
        4. Build and copy plugin to /var directory
        5. Install plugin to fuel
        6. Create cluster and enable plugin
        7. Deploy cluster
        8. Check vip reservation

        Duration 40m
        """
        plugin_name = 'vip_reservation_plugin'
        source_plugin_path = os.path.join('/root/', plugin_name)
        plugin_path = '/var'
        dir_path = os.path.dirname(os.path.abspath(__file__))
        tasks_file = 'tasks.yaml'
        net_role_file = 'network_roles.yaml'
        metadata_file = 'metadata.yaml'
        namespace = 'haproxy'

        self.show_step(1, initialize=True)
        self.env.revert_snapshot("ready_with_3_slaves")
        # initiate fuel plugin builder instance
        fpb = FuelPluginBuilder()
        # install fuel_plugin_builder on master node
        self.show_step(2)
        fpb.fpb_install()
        # create plugin template on the master node
        self.show_step(3)
        fpb.fpb_create_plugin(source_plugin_path)
        # replace plugin tasks, metadata, network_roles
        fpb.fpb_replace_plugin_content(
            os.path.join(dir_path, net_role_file),
            os.path.join(source_plugin_path, net_role_file))
        fpb.fpb_replace_plugin_content(
            os.path.join(dir_path, tasks_file),
            os.path.join(source_plugin_path, tasks_file))
        fpb.fpb_replace_plugin_content(
            os.path.join(dir_path, metadata_file),
            os.path.join(source_plugin_path, metadata_file))
        # build plugin
        self.show_step(4)
        packet_name = fpb.fpb_build_plugin(source_plugin_path)
        # copy plugin archive file from nailgun container
        # to the /var directory on the master node
        fpb.fpb_copy_plugin(os.path.join(source_plugin_path, packet_name),
                            plugin_path)
        # let's install plugin
        self.show_step(5)
        utils.install_plugin_check_code(
            ip=self.ssh_manager.admin_ip,
            plugin=os.path.join(plugin_path, packet_name))
        self.show_step(6)
        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=DEPLOYMENT_MODE,
        )
        # get plugins from fuel and enable our one
        msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
        asserts.assert_true(
            self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
            msg)
        options = {'metadata/enabled': True}
        self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)

        logger.info('Cluster is {!s}'.format(cluster_id))

        self.fuel_web.update_nodes(
            cluster_id,
            {
                'slave-01': ['controller'],
                'slave-02': ['compute']}
        )
        self.show_step(7)
        self.fuel_web.deploy_cluster_wait(cluster_id)

        self.fuel_web.run_ostf(cluster_id=cluster_id)

        self.show_step(8)
        with self.fuel_web.get_ssh_for_node('slave-01') as remote:
            hiera_json_out = "ruby -rhiera -rjson -e \"h = Hiera.new(); " \
                             "Hiera.logger = 'noop'; puts JSON.dump " \
                             "(h.lookup('network_metadata', " \
                             "[], {}, nil, nil))\""
            for vip in ('reserved_pub', 'reserved_mng'):
                # get vips from hiera
                vip_hiera = json.loads(
                    remote.execute(
                        hiera_json_out)['stdout'][0])["vips"][vip]["ipaddr"]
                # get vips from database
                vip_db = self.env.postgres_actions.run_query(
                    db='nailgun',
                    query="select ip_addr from ip_addrs where "
                          "vip_name = '\"'\"'{0}'\"'\"';".format(vip))
                vip_array = [vip_hiera, vip_db]
                for ip in vip_array[1:]:
                    asserts.assert_equal(
                        vip_array[0], ip,
                        "Vip from hiera output {0} does not equal "
                        "to {1}".format(vip_array[0], ip))
                vip_pcs = remote.execute(
                    'pcs resource show {0}{1}'.format(
                        'vip__', vip))['exit_code']
                asserts.assert_not_equal(0, vip_pcs,
                                         'The vip__{0} was found in '
                                         'pacemaker'.format(vip))
                vip_ns = remote.execute(
                    'ip netns exec {0} ip a | grep {1}{2}'.format(
                        namespace, 'b_', vip))['exit_code']
                asserts.assert_not_equal(0, vip_ns,
                                         'The {0} was found in {1} '
                                         'namespace'.format(vip, namespace))
    def vip_reservation_for_plugin_custom_ns(self):
        """Check vip reservation for custom ns plugin

        Scenario:
        1. Revert snapshot with 3 nodes
        2. Download and install fuel-plugin-builder
        3. Create plugin with predefined network_roles.yaml
        4. Build and copy plugin to /var
        5. Install plugin to fuel
        6. Create cluster and enable plugin
        7. Deploy cluster
        8. Check vip reservation

        Duration 40m
        """
        plugin_name = "vip_reservation_plugin"
        source_plugin_path = os.path.join("/root/", plugin_name)
        plugin_path = "/var"
        task_path = os.path.dirname(os.path.abspath(__file__))
        tasks_file = "tasks.yaml"
        net_role_file = "network_roles.yaml"
        metadata_file = "metadata.yaml"
        namespace = "custom_ns"
        self.show_step(1, initialize=True)
        self.env.revert_snapshot("ready_with_3_slaves")

        self.show_step(2)
        # initiate fuel plugin builder instance
        fpb = FuelPluginBuilder()
        # install fuel_plugin_builder on master node
        fpb.fpb_install()
        # create plugin template on the master node
        self.show_step(3)
        fpb.fpb_create_plugin(source_plugin_path)
        # replace plugin tasks, metadata, network_roles
        fpb.fpb_replace_plugin_content(
            os.path.join(task_path, net_role_file), os.path.join(source_plugin_path, net_role_file)
        )
        fpb.fpb_replace_plugin_content(
            os.path.join(task_path, tasks_file), os.path.join(source_plugin_path, tasks_file)
        )
        fpb.fpb_replace_plugin_content(
            os.path.join(task_path, metadata_file), os.path.join(source_plugin_path, metadata_file)
        )

        with YamlEditor(os.path.join(source_plugin_path, net_role_file), ip=fpb.admin_ip) as editor:
            editor.content[0]["properties"]["vip"][0]["namespace"] = namespace
            editor.content[1]["properties"]["vip"][0]["namespace"] = namespace
        # build plugin
        self.show_step(4)
        packet_name = fpb.fpb_build_plugin(source_plugin_path)
        # copy plugin archive file
        # to the /var directory on the master node
        fpb.fpb_copy_plugin(os.path.join(source_plugin_path, packet_name), plugin_path)
        self.show_step(5)
        # let's install plugin
        utils.install_plugin_check_code(ip=self.ssh_manager.admin_ip, plugin=os.path.join(plugin_path, packet_name))
        self.show_step(6)
        cluster_id = self.fuel_web.create_cluster(name=self.__class__.__name__, mode=DEPLOYMENT_MODE)
        # get plugins from fuel and enable our one
        msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
        asserts.assert_true(self.fuel_web.check_plugin_exists(cluster_id, plugin_name), msg)
        options = {"metadata/enabled": True}
        self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)

        logger.info("Cluster is {!s}".format(cluster_id))

        self.fuel_web.update_nodes(cluster_id, {"slave-01": ["controller"], "slave-02": ["compute"]})
        self.show_step(7)
        self.fuel_web.deploy_cluster_wait(cluster_id)

        self.fuel_web.run_ostf(cluster_id=cluster_id)

        self.show_step(8)
        with self.fuel_web.get_ssh_for_node("slave-01") as remote:
            hiera_json_out = (
                'ruby -rhiera -rjson -e "h = Hiera.new(); '
                "Hiera.logger = 'noop'; "
                "puts JSON.dump(h.lookup('network_metadata', "
                '[], {}, nil, nil))"'
            )
            for vip in ("reserved_pub", "reserved_mng"):
                # get vips from hiera
                vip_hiera = json.loads(remote.execute(hiera_json_out)["stdout"][0])["vips"][vip]["ipaddr"]
                # get vips from database
                vip_db = self.env.postgres_actions.run_query(
                    db="nailgun",
                    query="select ip_addr from ip_addrs where " "vip_name = '\"'\"'{0}'\"'\"';".format(vip),
                )
                # get vips from corosync
                vip_crm = remote.execute("crm_resource --resource {0}{1} --get-parameter=ip".format("vip__", vip))[
                    "stdout"
                ][0].rstrip()
                # get vips from namespace
                vip_ns = (
                    remote.execute("ip netns exec {0} ip -4 a show {1}{2}".format(namespace, "b_", vip))["stdout"][1]
                    .split(" ")[5]
                    .split("/")[0]
                )
                vip_array = [vip_hiera, vip_db, vip_crm, vip_ns]
                for ip in vip_array[1:]:
                    asserts.assert_equal(
                        vip_array[0], ip, "Vip from hiera output {0} does not equal " "to {1}".format(vip_array[0], ip)
                    )
    def etckeeper_plugin(self):
        """Check tracking /etc dir by etckeeper plugin

        Scenario:
        1. Revert snapshot with 1 node
        2. Download and install fuel-plugin-builder
        3. Clone plugin repo
        4. Build plugin
        5. Install plugin to fuel
        6. Create cluster and enable plugin
        7. Deploy cluster
        8. Check plugin

        Duration 50m
        """
        plugin_name = 'fuel-plugin-etckeeper'
        plugin_path = '/var'
        source_plugin_path = os.path.join(plugin_path, plugin_name)

        self.show_step(1)
        self.env.revert_snapshot("ready_with_1_slaves")

        self.show_step(2)
        fpb = FuelPluginBuilder()
        fpb.fpb_install()

        ip = self.ssh_manager.admin_ip
        self.ssh_manager.execute_on_remote(ip=ip,
                                           cmd='git clone {0} {1}'.format(
                                               ETCKEEPER_PLUGIN_REPO,
                                               source_plugin_path))

        self.show_step(4)
        packet_name = fpb.fpb_build_plugin(source_plugin_path)

        self.show_step(5)
        utils.install_plugin_check_code(ip=self.ssh_manager.admin_ip,
                                        plugin=os.path.join(
                                            source_plugin_path, packet_name))

        self.show_step(6)
        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=DEPLOYMENT_MODE,
            settings={'propagate_task_deploy': True})

        msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
        asserts.assert_true(
            self.fuel_web.check_plugin_exists(cluster_id, plugin_name), msg)
        options = {'metadata/enabled': True}

        self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)
        logger.info('Cluster is {!s}'.format(cluster_id))

        self.fuel_web.update_nodes(cluster_id, {'slave-01': ['controller']})

        self.show_step(7)
        self.fuel_web.deploy_cluster_wait(cluster_id)

        self.fuel_web.run_ostf(cluster_id=cluster_id)

        self.show_step(8)
        ip = self.fuel_web.get_nailgun_node_by_name("slave-01")['ip']
        etckeeper_status = self.ssh_manager.execute_on_remote(
            ip=ip, cmd='etckeeper vcs status')
        if 'branch master' not in etckeeper_status['stdout_str']:
            raise Exception("The etckeeper has wrong status {0}".format(
                etckeeper_status['stdout_str']))

        new_config = 'test_config'
        self.ssh_manager.execute_on_remote(
            ip=ip, cmd='>>{0}'.format(os.path.join('/etc', new_config)))

        etckeeper_status = self.ssh_manager.execute_on_remote(
            ip=ip, cmd='etckeeper vcs status')
        if new_config not in etckeeper_status['stdout_str']:
            raise Exception(
                "The etckeeper does not tracked adding the new config: {0}, "
                "actual status: {1}".format(new_config,
                                            etckeeper_status['stdout_str']))
    def deploy_cluster_with_reboot_plugin_timeout(self):
        """Check deployment is failed by reboot task plugin.

        Scenario:
            1. Revert snapshot with 3 nodes
            2. Download and install fuel-plugin-builder
            3. Create plugin with reboot task,
               set timeout for reboot task as 1 second
            4. Build and copy plugin from container nailgun
            5. Install plugin to fuel
            6. Create cluster and enable plugin
            7. Provision nodes
            8. Deploy cluster
            9. Check deployment was failed by reboot task
            10. Check error msg at the logs

        Duration 15m
        """
        # define some plugin related variables
        plugin_name = 'timeout_plugin'
        plugin_path = '/var'
        tasks_path = os.path.dirname(os.path.abspath(__file__))
        tasks_file = 'reboot_tasks.yaml'
        # start reverting snapshot
        self.env.revert_snapshot("ready_with_3_slaves")
        # let's get ssh client for the master node
        with self.env.d_env.get_admin_remote() as admin_remote:
            # initiate fuel plugin builder instance
            fpb = FuelPluginBuilder(admin_remote)
            # install fuel_plugin_builder on master node
            fpb.fpb_install()
            # change timeout to a new value '1'
            fpb.change_content_in_yaml(os.path.join(tasks_path, tasks_file),
                                       os.path.join('/tmp/', tasks_file),
                                       [1, 'parameters', 'timeout'],
                                       1)
            # create plugin template on the master node
            fpb.fpb_create_plugin(plugin_name)
            # replace plugin tasks with our file
            fpb.fpb_replace_plugin_content(
                os.path.join('/tmp/', tasks_file),
                os.path.join('/root/', plugin_name, 'tasks.yaml')
            )
            # change default supported version to 7.0
            fpb.change_yaml_file_in_container(
                '/root/{}/metadata.yaml'.format(plugin_name),
                ['fuel_version'], ['7.0'])
            for elem in range(2):
                fpb.change_yaml_file_in_container(
                    '/root/{}/metadata.yaml'.format(plugin_name),
                    ['releases', elem, 'version'], '-')
            # build plugin
            fpb.fpb_build_plugin(os.path.join('/root/', plugin_name))
            # copy plugin archive file from nailgun container
            # to the /var directory on the master node
            fpb.fpb_copy_plugin_from_container(plugin_name, plugin_path)
            # let's install plugin
            checkers.install_plugin_check_code(
                admin_remote,
                plugin=os.path.join(plugin_path, '{}.rpm'.format(plugin_name)))
        # create cluster
        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=DEPLOYMENT_MODE,
            settings={
                "net_provider": 'neutron',
                "net_segment_type": NEUTRON_SEGMENT_TYPE
            }
        )
        # get plugins from fuel and enable it
        msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
        asserts.assert_true(
            self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
            msg)
        options = {'metadata/enabled': True}
        self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)

        logger.info('cluster is %s' % str(cluster_id))

        self.fuel_web.update_nodes(
            cluster_id,
            {'slave-01': ['controller', 'ceph-osd']}
        )

        self.fuel_web.provisioning_cluster_wait(cluster_id)
        logger.info('Start cluster #%s deployment', cluster_id)
        task = self.fuel_web.client.deploy_nodes(cluster_id)
        self.fuel_web.assert_task_failed(task)

        msg = 'Time detection (1 sec) for node reboot has expired'
        cmd = 'grep "{0}" /var/log/docker-logs/astute/astute.log'.format(msg)
        result = admin_remote.execute(cmd)['stdout'][0]

        asserts.assert_true(
            msg in result,
            'Failed to find reboot plugin warning message in logs'
        )
    def vip_reservation_for_plugin(self):
        """Check vip reservation for fuel plugin.

        Scenario:
        1. Revert snapshot with 3 nodes
        2. Download and install fuel-plugin-builder
        3. Create plugin with predefined network_roles.yaml
        4. Build and copy plugin from container nailgun
        5. Install plugin to fuel
        6. Create cluster and enable plugin
        7. Deploy cluster
        8. Check vip reservation

        Duration 40m
        """
        plugin_name = 'vip_reservation_plugin'
        plugin_path = '/var'
        dir_path = os.path.dirname(os.path.abspath(__file__))
        tasks_file = 'tasks.yaml'
        net_role_file = 'network_roles.yaml'
        metadata_file = 'metadata.yaml'

        self.env.revert_snapshot("ready_with_3_slaves")

        admin_remote = self.env.d_env.get_admin_remote()
        # initiate fuel plugin builder instance
        fpb = FuelPluginBuilder(admin_remote)
        # install fuel_plugin_builder on master node
        fpb.fpb_install()
        # create plugin template on the master node
        fpb.fpb_create_plugin(plugin_name)
        # replace plugin tasks, metadata, network_roles
        fpb.fpb_replace_plugin_content(
            os.path.join(dir_path, net_role_file),
            os.path.join('/root/', plugin_name, net_role_file))
        fpb.fpb_replace_plugin_content(
            os.path.join(dir_path, tasks_file),
            os.path.join('/root/', plugin_name, tasks_file))
        fpb.fpb_replace_plugin_content(
            os.path.join(dir_path, metadata_file),
            os.path.join('/root/', plugin_name, metadata_file))
        # build plugin
        fpb.fpb_build_plugin(os.path.join('/root/', plugin_name))
        # copy plugin archive file from nailgun container
        # to the /var directory on the master node
        fpb.fpb_copy_plugin_from_container(plugin_name, plugin_path)
        # let's install plugin
        checkers.install_plugin_check_code(
            admin_remote,
            plugin=os.path.join(plugin_path, '{}.rpm'.format(plugin_name)))
        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=DEPLOYMENT_MODE,
            settings={
                "net_provider": 'neutron',
                "net_segment_type": NEUTRON_SEGMENT_TYPE
            }
        )
        # get plugins from fuel and enable our one
        msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
        asserts.assert_true(
            self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
            msg)
        options = {'metadata/enabled': True}
        self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)

        logger.info('cluster is %s' % str(cluster_id))

        self.fuel_web.update_nodes(
            cluster_id,
            {
                'slave-01': ['controller'],
                'slave-02': ['compute']}
        )
        self.fuel_web.deploy_cluster_wait(cluster_id)

        self.fuel_web.run_ostf(cluster_id=cluster_id)

        remote = self.fuel_web.get_ssh_for_node('slave-01')
        #get vips from hiera
        reserved_vip_pub = remote.execute(
            'hiera reserved_vip_pub')['stdout'][1].split('"')[3]
        reserved_vip_mng = remote.execute(
            'hiera reserved_vip_mng')['stdout'][1].split('"')[3]
        #get vips from database
        reserved_vip_pub_db = self.env.postgres_actions.run_query(
            db='nailgun', query="select ip_addr from ip_addrs where "
                                "vip_type = '\"'\"'reserved_vip_pub'\"'\"';")
        reserved_vip_mng_db = self.env.postgres_actions.run_query(
            db='nailgun', query="select ip_addr from ip_addrs where "
                                "vip_type = '\"'\"'reserved_vip_mng'\"'\"';")

        assert_equal(reserved_vip_pub, reserved_vip_pub_db,
                     "Vip public from hiera output {0} does not equal to "
                     "vip public from database {1}".format(
                         reserved_vip_pub, reserved_vip_pub_db))
        assert_equal(reserved_vip_mng, reserved_vip_mng_db,
                     "Vip management from hiera output {0} does not equal to "
                     "vip management from database {1}".format(
                         reserved_vip_mng, reserved_vip_mng_db))
    def etckeeper_plugin(self):
        """Check tracking /etc dir by etckeeper plugin

        Scenario:
        1. Revert snapshot with 1 node
        2. Download and install fuel-plugin-builder
        3. Clone plugin repo
        4. Build plugin
        5. Install plugin to fuel
        6. Create cluster and enable plugin
        7. Deploy cluster
        8. Check plugin

        Duration 50m
        """
        plugin_name = 'fuel-plugin-etckeeper'
        plugin_path = '/var'
        source_plugin_path = os.path.join(plugin_path, plugin_name)

        self.show_step(1)
        self.env.revert_snapshot("ready_with_1_slaves")

        self.show_step(2)
        fpb = FuelPluginBuilder()
        fpb.fpb_install()

        ip = self.ssh_manager.admin_ip
        self.ssh_manager.execute_on_remote(
            ip=ip,
            cmd='git clone {0} {1}'.format(
                ETCKEEPER_PLUGIN_REPO, source_plugin_path))

        self.show_step(4)
        packet_name = fpb.fpb_build_plugin(source_plugin_path)

        self.show_step(5)
        utils.install_plugin_check_code(
            ip=self.ssh_manager.admin_ip,
            plugin=os.path.join(source_plugin_path, packet_name))

        self.show_step(6)
        cluster_id = self.fuel_web.create_cluster(
            name=self.__class__.__name__,
            mode=DEPLOYMENT_MODE,
            settings={'propagate_task_deploy': True}
        )

        msg = "Plugin couldn't be enabled. Check plugin version. Test aborted"
        asserts.assert_true(
            self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
            msg)
        options = {'metadata/enabled': True}

        self.fuel_web.update_plugin_data(cluster_id, plugin_name, options)
        logger.info('Cluster is {!s}'.format(cluster_id))

        self.fuel_web.update_nodes(
            cluster_id,
            {
                'slave-01': ['controller']}
        )

        self.show_step(7)
        self.fuel_web.deploy_cluster_wait(cluster_id)

        self.fuel_web.run_ostf(cluster_id=cluster_id)

        self.show_step(8)
        ip = self.fuel_web.get_nailgun_node_by_name("slave-01")['ip']
        etckeeper_status = self.ssh_manager.execute_on_remote(
            ip=ip, cmd='etckeeper vcs status')
        if 'branch master' not in etckeeper_status['stdout_str']:
            raise Exception("The etckeeper has wrong status {0}".format(
                etckeeper_status['stdout_str']))

        new_config = 'test_config'
        self.ssh_manager.execute_on_remote(
            ip=ip,
            cmd='>>{0}'.format(os.path.join('/etc', new_config)))

        etckeeper_status = self.ssh_manager.execute_on_remote(
            ip=ip, cmd='etckeeper vcs status')
        if new_config not in etckeeper_status['stdout_str']:
            raise Exception(
                "The etckeeper does not tracked adding the new config: {0}, "
                "actual status: {1}".format(
                    new_config, etckeeper_status['stdout_str']))