Пример #1
0
class BaseNodeTestCase(BaseTestCase):

    def setUp(self):
        if CLEAN:
            self.ci().get_empty_state()
        self.client = NailgunClient(self.get_admin_node_ip())

    @logwrap
    def get_interface_description(self, ctrl_ssh, interface_short_name):
        return ''.join(
            ctrl_ssh.execute(
                '/sbin/ip addr show dev %s' % interface_short_name
            )['stdout']
        )

    def assertNetworkConfiguration(self, node):
        remote = SSHClient(node['ip'], username='******', password='******',
                           private_keys=self.get_private_keys())
        for interface in node['network_data']:
            if interface.get('vlan') is None:
                continue  # todo excess check fix interface json format
            interface_name = "%s.%s@%s" % (
                interface['dev'], interface['vlan'], interface['dev'])
            interface_short_name = "%s.%s" % (
                interface['dev'], interface['vlan'])
            interface_description = self.get_interface_description(
                remote, interface_short_name)
            self.assertIn(interface_name, interface_description)
            if interface.get('name') == 'floating':
                continue
            if interface.get('ip'):
                self.assertIn("inet %s" % interface.get('ip'),
                              interface_description)
            else:
                self.assertNotIn("inet ", interface_description)
            if interface.get('brd'):
                self.assertIn("brd %s" % interface['brd'],
                              interface_description)

    @logwrap
    def is_node_discovered(self, nailgun_node):
        return any(
            map(lambda node: node['mac'] == nailgun_node['mac']
                and node['status'] == 'discover', self.client.list_nodes()))

    @logwrap
    def get_target_devs(self, devops_nodes):
        return [
            interface.target_dev for interface in [
                val for var in map(lambda node: node.interfaces, devops_nodes)
                for val in var]]

    @logwrap
    def get_ebtables(self, cluster_id, devops_nodes):
        return Ebtables(
            self.get_target_devs(devops_nodes),
            self.client._get_cluster_vlans(cluster_id))

    @logwrap
    def _get_common_vlan(self, cluster_id):
        """Find vlan that must be at all two nodes.
        """
        return self.client.get_networks(
            cluster_id)['networks'][0]['vlan_start']

    @logwrap
    def _run_network_verify(self, cluster_id):
        return self.client.verify_networks(
            cluster_id, self.client.get_networks(cluster_id)['networks'])

    @logwrap
    def check_role_file(self, nodes_dict):
        for node, role in self.get_nailgun_node_roles(nodes_dict):
            remote = SSHClient(
                node['ip'], username='******', password='******',
                private_keys=self.get_private_keys())
            if role != "cinder":
                self.assertTrue(remote.isfile('/tmp/%s-file' % role))

    @logwrap
    def clean_clusters(self):
        self.client.clean_clusters()

    @logwrap
    def _basic_provisioning(self, cluster_id, nodes_dict, port=5514):
        self.client.add_syslog_server(
            cluster_id, self.ci().get_host_node_ip(), port)

        # update cluster deployment mode
        node_names = []
        for role in nodes_dict:
            node_names += nodes_dict[role]
        if len(node_names) > 1:
            controller_amount = len(nodes_dict.get('controller', []))
            if controller_amount == 1:
                self.client.update_cluster(
                    cluster_id,
                    {"mode": "multinode"})
            if controller_amount > 1:
                self.client.update_cluster(cluster_id, {"mode": "ha"})

        self.bootstrap_nodes(self.devops_nodes_by_names(node_names))

        # update nodes in cluster
        self.update_nodes(cluster_id, nodes_dict, True, False)

        task = self.deploy_cluster(cluster_id)
        self.assertTaskSuccess(task)
        self.check_role_file(nodes_dict)
        return cluster_id

    @logwrap
    def get_nailgun_node_roles(self, nodes_dict):
        nailgun_node_roles = []
        for role in nodes_dict:
            for node_name in nodes_dict[role]:
                slave = self.ci().environment().node_by_name(node_name)
                node = self.get_node_by_devops_node(slave)
                nailgun_node_roles.append((node, role))
        return nailgun_node_roles

    @logwrap
    def deploy_cluster(self, cluster_id):
        """Return hash with task description."""
        return self.client.deploy_cluster_changes(cluster_id)

    @logwrap
    def assertTaskSuccess(self, task, timeout=90 * 60):
        self.assertEquals('ready', self._task_wait(task, timeout)['status'])

    @logwrap
    def assertTaskFailed(self, task, timeout=70 * 60):
        self.assertEquals('error', self._task_wait(task, timeout)['status'])

    @logwrap
    def _task_wait(self, task, timeout):
        wait(
            lambda: self.client.get_task(
                task['id'])['status'] != 'running',
            timeout=timeout)
        return self.client.get_task(task['id'])

    @logwrap
    def _upload_sample_release(self):
        release_id = self.client.get_grizzly_release_id()
        if not release_id:
            raise Exception("Not implemented uploading of release")
        return release_id

    @logwrap
    def get_or_create_cluster(self, name, release_id):
        if not release_id:
            release_id = self._upload_sample_release()
        cluster_id = self.client.get_cluster_id(name)
        if not cluster_id:
            self.client.create_cluster(
                data={"name": name, "release": str(release_id)}
            )
            cluster_id = self.client.get_cluster_id(name)
        if not cluster_id:
            raise Exception("Could not get cluster '%s'" % name)
        return cluster_id

    @logwrap
    def create_cluster(self, name='default', release_id=None):
        """
        :param name:
        :param release_id:
        :return: cluster_id
        """
        return self.get_or_create_cluster(name, release_id)

    @logwrap
    def update_nodes(self, cluster_id, nodes_dict,
                     pending_addition=True, pending_deletion=False):
        # update nodes in cluster
        nodes_data = []
        for role in nodes_dict:
            for node_name in nodes_dict[role]:
                slave = self.ci().environment().node_by_name(node_name)
                node = self.get_node_by_devops_node(slave)
                node_data = {'cluster_id': cluster_id, 'id': node['id'],
                             'pending_addition': pending_addition,
                             'pending_deletion': pending_deletion,
                             'role': role}
                nodes_data.append(node_data)

        # assume nodes are going to be updated for one cluster only
        cluster_id = nodes_data[-1]['cluster_id']
        node_ids = [str(node_info['id']) for node_info in nodes_data]
        self.client.update_nodes(nodes_data)

        nailgun_nodes = self.client.list_cluster_nodes(cluster_id)
        cluster_node_ids = map(lambda node: str(node['id']), nailgun_nodes)
        self.assertTrue(
            all([node_id in cluster_node_ids for node_id in node_ids]))
        return nailgun_nodes

    @logwrap
    def get_node_by_devops_node(self, devops_node):
        """Returns dict with nailgun slave node description if node is
        registered. Otherwise return None.
        """
        mac_addresses = map(
            lambda interface: interface.mac_address.capitalize(),
            devops_node.interfaces)
        for nailgun_node in self.client.list_nodes():
            if nailgun_node['mac'].capitalize() in mac_addresses:
                nailgun_node['devops_name'] = devops_node.name
                return nailgun_node
        return None

    def nailgun_nodes(self, devops_nodes):
        return map(lambda node: self.get_node_by_devops_node(node),
                   devops_nodes)

    def devops_nodes_by_names(self, devops_node_names):
        return map(lambda name: self.ci().environment().node_by_name(name),
                   devops_node_names)

    @logwrap
    def bootstrap_nodes(self, devops_nodes, timeout=600):
        """Start vms and wait they are registered on nailgun.
        :rtype : List of registred nailgun nodes
        """
        for node in devops_nodes:
            node.start()
        wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout)
        return self.nailgun_nodes(devops_nodes)

    @logwrap
    def assert_service_list(self, remote, smiles_count):
        ret = remote.check_call('/usr/bin/nova-manage service list')
        self.assertEqual(
            smiles_count, ''.join(ret['stdout']).count(":-)"), "Smiles count")
        self.assertEqual(
            0, ''.join(ret['stdout']).count("XXX"), "Broken services count")

    @logwrap
    def assert_node_service_list(self, node_name, smiles_count):
        ip = self.get_node_by_devops_node(
            self.ci().environment().node_by_name(node_name))['ip']
        remote = SSHClient(ip, username='******', password='******',
                           private_keys=self.get_private_keys())
        return self.assert_service_list(remote, smiles_count)

    @logwrap
    def assert_glance_index(self, ctrl_ssh):
        ret = ctrl_ssh.check_call('. /root/openrc; glance index')
        self.assertEqual(1, ''.join(ret['stdout']).count("TestVM"))

    @logwrap
    def assert_network_list(self, networks_count, remote):
        ret = remote.check_call('/usr/bin/nova-manage network list')
        self.assertEqual(networks_count + 1, len(ret['stdout']))

    @logwrap
    def assertClusterReady(self, node_name, smiles_count,
                           networks_count=1, timeout=300):
        _wait(
            lambda: self.get_cluster_status(
                self.get_node_by_devops_node(
                    self.ci().environment().node_by_name(node_name))['ip'],
                smiles_count=smiles_count,
                networks_count=networks_count),
            timeout=timeout)

    @logwrap
    def get_cluster_status(self, ip, smiles_count, networks_count=1):
        remote = SSHClient(ip, username='******', password='******',
                           private_keys=self.get_private_keys())
        self.assert_service_list(remote, smiles_count)
        self.assert_glance_index(remote)
        self.assert_network_list(networks_count, remote)

    @logwrap
    def get_cluster_floating_list(self, ip):
        remote = SSHClient(ip, username='******', password='******',
                           private_keys=self.get_private_keys())
        ret = remote.check_call('/usr/bin/nova-manage floating list')
        ret_str = ''.join(ret['stdout'])
        return re.findall('(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', ret_str)

    @logwrap
    def assert_cluster_floating_list(self, node_name, expected_ips):
        ip = self.get_node_by_devops_node(
            self.ci().environment().node_by_name(node_name))['ip']
        current_ips = self.get_cluster_floating_list(ip)
        self.assertEqual(set(expected_ips), set(current_ips))

    @logwrap
    def get_private_keys(self):
        keys = []
        for key_string in ['/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa']:
            with self.remote().open(key_string) as f:
                keys.append(RSAKey.from_private_key(f))
        return keys

    @logwrap
    def update_vlan_network_fixed(
            self, cluster_id, amount=1, network_size=256):
        network_list = self.client.get_networks(cluster_id)['networks']
        for network in network_list:
            if network["name"] == 'fixed':
                network['amount'] = amount
                network['network_size'] = network_size

        self.client.update_network(
            cluster_id,
            networks=network_list,
            net_manager=NETWORK_MANAGERS['vlan'])
class BaseNodeTestCase(BaseTestCase):

    environment_states = {}

    def setUp(self):
        self.client = NailgunClient(self.get_admin_node_ip())

    @logwrap
    def get_interface_description(self, ctrl_ssh, interface_short_name):
        return ''.join(
            ctrl_ssh.execute(
                '/sbin/ip addr show dev %s' % interface_short_name
            )['stdout']
        )

    def assertNetworkConfiguration(self, node):
        remote = SSHClient(node['ip'], username='******', password='******',
                           private_keys=self.get_private_keys())
        for interface in node['network_data']:
            if interface.get('vlan') is None:
                continue  # todo excess check fix interface json format
            interface_name = "%s.%s@%s" % (
                interface['dev'], interface['vlan'], interface['dev'])
            interface_short_name = "%s.%s" % (
                interface['dev'], interface['vlan'])
            interface_description = self.get_interface_description(
                remote, interface_short_name)
            self.assertIn(interface_name, interface_description)
            if interface.get('name') == 'floating':
                continue
            if interface.get('ip'):
                self.assertIn("inet %s" % interface.get('ip'),
                              interface_description)
            else:
                self.assertNotIn("inet ", interface_description)
            if interface.get('brd'):
                self.assertIn("brd %s" % interface['brd'],
                              interface_description)

    @logwrap
    def is_node_discovered(self, nailgun_node):
        return any(
            map(lambda node: node['mac'] == nailgun_node['mac']
                and node['status'] == 'discover', self.client.list_nodes()))

    @logwrap
    def get_target_devs(self, devops_nodes):
        return [
            interface.target_dev for interface in [
                val for var in map(lambda node: node.interfaces, devops_nodes)
                for val in var]]

    @logwrap
    def get_ebtables(self, cluster_id, devops_nodes):
        return Ebtables(
            self.get_target_devs(devops_nodes),
            self.client._get_cluster_vlans(cluster_id))

    @logwrap
    def _get_common_vlan(self, cluster_id):
        """Find vlan that must be at all two nodes.
        """
        return self.client.get_networks(
            cluster_id)['networks'][0]['vlan_start']

    @logwrap
    def _run_network_verify(self, cluster_id):
        return self.client.verify_networks(
            cluster_id, self.client.get_networks(cluster_id)['networks'])

    @logwrap
    def check_role_file(self, nodes_dict):
        for node, roles in self.get_nailgun_node_roles(nodes_dict):
            remote = SSHClient(
                node['ip'], username='******', password='******',
                private_keys=self.get_private_keys())
            for role in roles:
                if role != "cinder":
                    self.assertTrue(remote.isfile('/tmp/%s-file' % role))

    @logwrap
    def clean_clusters(self):
        self.client.clean_clusters()

    @logwrap
    def configure_cluster(self, cluster_id, nodes_dict):
        self.update_nodes(cluster_id, nodes_dict, True, False)
        # TODO: update network configuration

    @logwrap
    def basic_provisioning(self, cluster_id, nodes_dict, port=5514):
        self.client.add_syslog_server(
            cluster_id, self.ci().get_host_node_ip(), port)

        self.bootstrap_nodes(self.devops_nodes_by_names(nodes_dict.keys()))
        self.configure_cluster(cluster_id, nodes_dict)

        task = self.deploy_cluster(cluster_id)
        self.assertTaskSuccess(task)
        self.check_role_file(nodes_dict)
        return cluster_id

    @logwrap
    def prepare_environment(self, name='cluster_name', mode="multinode",
                            settings=None):
        if not(self.ci().revert_to_state(settings)):
            self.get_ready_environment()
            if settings is None:
                return None

            if 'nodes' in settings:
                cluster_id = self.create_cluster(name=name, mode=mode)
                self.basic_provisioning(cluster_id, settings['nodes'])
            self.ci().snapshot_state(name, settings)

        # return id of last created cluster
        clusters = self.client.list_clusters()
        if len(clusters) > 0:
            return clusters.pop()['id']
        return None

    @logwrap
    def get_nailgun_node_roles(self, nodes_dict):
        nailgun_node_roles = []
        for node_name in nodes_dict:
            slave = self.ci().environment().node_by_name(node_name)
            node = self.get_node_by_devops_node(slave)
            nailgun_node_roles.append((node, nodes_dict[node_name]))
        return nailgun_node_roles

    @logwrap
    def deploy_cluster(self, cluster_id):
        """Return hash with task description."""
        return self.client.deploy_cluster_changes(cluster_id)

    @logwrap
    def assertTaskSuccess(self, task, timeout=90 * 60):
        self.assertEquals('ready', self._task_wait(task, timeout)['status'])

    @logwrap
    def assertTaskFailed(self, task, timeout=70 * 60):
        self.assertEquals('error', self._task_wait(task, timeout)['status'])

    @logwrap
    def assertOSTFRunSuccess(self, cluster_id, should_fail=0, should_pass=0,
                             timeout=10 * 60):
        set_result_list = self._ostf_test_wait(cluster_id, timeout)

        passed = 0
        failed = 0
        for set_result in set_result_list:
            passed += len(filter(lambda test: test['status'] == 'success',
                                 set_result['tests']))
            failed += len(
                filter(
                    lambda test: test['status'] == 'failure' or
                    test['status'] == 'error',
                    set_result['tests']
                )
            )
        self.assertEqual(passed, should_pass, 'Passed tests')
        self.assertEqual(failed, should_fail, 'Failed tests')

    @logwrap
    def run_OSTF(self, cluster_id, test_sets=None,
                 should_fail=0, should_pass=0):
        test_sets = test_sets \
            if test_sets is not None \
            else ['fuel_smoke', 'fuel_sanity']

        self.client.ostf_run_tests(cluster_id, test_sets)
        self.assertOSTFRunSuccess(cluster_id, should_fail=should_fail,
                                  should_pass=should_pass)

    @logwrap
    def _task_wait(self, task, timeout):
        wait(
            lambda: self.client.get_task(
                task['id'])['status'] != 'running',
            timeout=timeout)
        return self.client.get_task(task['id'])

    @logwrap
    def _ostf_test_wait(self, cluster_id, timeout):
        wait(
            lambda: all([run['status'] == 'finished'
                         for run in
                         self.client.get_ostf_test_run(cluster_id)]),
            timeout=timeout)
        return self.client.get_ostf_test_run(cluster_id)

    @logwrap
    def _tasks_wait(self, tasks, timeout):
        return [self._task_wait(task, timeout) for task in tasks]

    @logwrap
    def _upload_sample_release(self):
        release_id = self.client.get_release_id()
        if not release_id:
            raise Exception("Not implemented uploading of release")
        return release_id

    @logwrap
    def get_or_create_cluster(self, name, release_id, mode="multinode"):
        if not release_id:
            release_id = self._upload_sample_release()
        cluster_id = self.client.get_cluster_id(name)
        if not cluster_id:
            self.client.create_cluster(
                data={
                    "name": name,
                    "release": str(release_id),
                    "mode": mode
                }
            )
            cluster_id = self.client.get_cluster_id(name)
        if not cluster_id:
            raise Exception("Could not get cluster '%s'" % name)
        return cluster_id

    @logwrap
    def create_cluster(self, name='default', release_id=None,
                       mode="multinode"):
        """
        :param name:
        :param release_id:
        :param mode:
        :return: cluster_id
        """
        return self.get_or_create_cluster(name, release_id, mode)

    @logwrap
    def update_nodes(self, cluster_id, nodes_dict,
                     pending_addition=True, pending_deletion=False):
        # update nodes in cluster
        nodes_data = []
        for node_name in nodes_dict:
            devops_node = self.ci().environment().node_by_name(node_name)
            node = self.get_node_by_devops_node(devops_node)
            node_data = {'cluster_id': cluster_id, 'id': node['id'],
                         'pending_addition': pending_addition,
                         'pending_deletion': pending_deletion,
                         'pending_roles': nodes_dict[node_name]}
            nodes_data.append(node_data)

        # assume nodes are going to be updated for one cluster only
        cluster_id = nodes_data[-1]['cluster_id']
        node_ids = [str(node_info['id']) for node_info in nodes_data]
        self.client.update_nodes(nodes_data)

        nailgun_nodes = self.client.list_cluster_nodes(cluster_id)
        cluster_node_ids = map(lambda node: str(node['id']), nailgun_nodes)
        self.assertTrue(
            all([node_id in cluster_node_ids for node_id in node_ids]))
        return nailgun_nodes

    @logwrap
    def get_node_by_devops_node(self, devops_node):
        """Returns dict with nailgun slave node description if node is
        registered. Otherwise return None.
        """
        mac_addresses = map(
            lambda interface: interface.mac_address.capitalize(),
            devops_node.interfaces)
        for nailgun_node in self.client.list_nodes():
            if nailgun_node['mac'].capitalize() in mac_addresses:
                nailgun_node['devops_name'] = devops_node.name
                return nailgun_node
        return None

    def nailgun_nodes(self, devops_nodes):
        return map(lambda node: self.get_node_by_devops_node(node),
                   devops_nodes)

    def devops_nodes_by_names(self, devops_node_names):
        return map(lambda name: self.ci().environment().node_by_name(name),
                   devops_node_names)

    @logwrap
    def bootstrap_nodes(self, devops_nodes, timeout=600):
        """Start vms and wait they are registered on nailgun.
        :rtype : List of registred nailgun nodes
        """
        for node in devops_nodes:
            node.start()
        wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout)
        return self.nailgun_nodes(devops_nodes)

    @logwrap
    def assert_service_list(self, remote, smiles_count):
        ret = remote.check_call('/usr/bin/nova-manage service list')
        self.assertEqual(
            smiles_count, ''.join(ret['stdout']).count(":-)"), "Smiles count")
        self.assertEqual(
            0, ''.join(ret['stdout']).count("XXX"), "Broken services count")

    @logwrap
    def assert_node_service_list(self, node_name, smiles_count):
        ip = self.get_node_by_devops_node(
            self.ci().environment().node_by_name(node_name))['ip']
        remote = SSHClient(ip, username='******', password='******',
                           private_keys=self.get_private_keys())
        return self.assert_service_list(remote, smiles_count)

    @logwrap
    def assert_glance_index(self, ctrl_ssh):
        ret = ctrl_ssh.check_call('. /root/openrc; glance index')
        self.assertEqual(1, ''.join(ret['stdout']).count("TestVM"))

    @logwrap
    def assert_network_list(self, networks_count, remote):
        ret = remote.check_call('/usr/bin/nova-manage network list')
        self.assertEqual(networks_count + 1, len(ret['stdout']))

    @logwrap
    def assertClusterReady(self, node_name, smiles_count,
                           networks_count=1, timeout=300):
        _wait(
            lambda: self.get_cluster_status(
                self.get_node_by_devops_node(
                    self.ci().environment().node_by_name(node_name))['ip'],
                smiles_count=smiles_count,
                networks_count=networks_count),
            timeout=timeout)

    @logwrap
    def _get_remote(self, ip):
        return SSHClient(ip, username='******', password='******',
                         private_keys=self.get_private_keys())

    @logwrap
    def _get_remote_for_node(self, node_name):
        ip = self.get_node_by_devops_node(
            self.ci().environment().node_by_name(node_name))['ip']
        return self._get_remote(ip)

    @logwrap
    def get_cluster_status(self, ip, smiles_count, networks_count=1):
        remote = self._get_remote(ip)
        self.assert_service_list(remote, smiles_count)
        self.assert_glance_index(remote)
        self.assert_network_list(networks_count, remote)

    @logwrap
    def get_cluster_floating_list(self, node_name):
        remote = self._get_remote_for_node(node_name)
        ret = remote.check_call('/usr/bin/nova-manage floating list')
        ret_str = ''.join(ret['stdout'])
        return re.findall('(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', ret_str)

    @logwrap
    def get_cluster_block_devices(self, node_name):
        remote = self._get_remote_for_node(node_name)
        ret = remote.check_call('/bin/lsblk')
        return ''.join(ret['stdout'])

    @logwrap
    def assert_cluster_floating_list(self, node_name, expected_ips):
        current_ips = self.get_cluster_floating_list(node_name)
        self.assertEqual(set(expected_ips), set(current_ips))

    @logwrap
    def get_private_keys(self):
        keys = []
        for key_string in ['/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa']:
            with self.remote().open(key_string) as f:
                keys.append(RSAKey.from_private_key(f))
        return keys

    @logwrap
    def update_node_networks(self, node_id, interfaces_dict):
        interfaces = self.client.get_node_interfaces(node_id)
        for interface in interfaces:
            interface_name = interface['name']
            interface['assigned_networks'] = []
            for allowed_network in interface['allowed_networks']:
                key_exists = interface_name in interfaces_dict
                if key_exists and \
                        allowed_network['name'] \
                        in interfaces_dict[interface_name]:
                    interface['assigned_networks'].append(allowed_network)

        self.client.put_node_interfaces(
            [{'id': node_id, 'interfaces': interfaces}])

    @logwrap
    def update_vlan_network_fixed(
            self, cluster_id, amount=1, network_size=256):
        network_list = self.client.get_networks(cluster_id)['networks']
        for network in network_list:
            if network["name"] == 'fixed':
                network['amount'] = amount
                network['network_size'] = network_size

        self.client.update_network(
            cluster_id,
            networks=network_list,
            net_manager=NETWORK_MANAGERS['vlan'])

    @logwrap
    def get_ready_environment(self):
        if self.ci().get_state(READY_SNAPSHOT):
            self.environment().resume(verbose=False)
            return

        self.ci().get_empty_environment()
        if OPENSTACK_RELEASE == OPENSTACK_RELEASE_REDHAT:
            # update redhat credentials so that fuel may upload redhat
            # packages

            # download redhat repo from local place to boost the test
            # remote = self.nodes().admin.remote(
            #   'internal', 'root', 'r00tme')
            # remote.execute(
            #   'wget -q http://172.18.67.168/rhel6/rhel-rpms.tar.gz')
            # remote.execute('tar xzf rhel-rpms.tar.gz -C /')

            self.update_redhat_credentials()
            self.assert_release_state(OPENSTACK_RELEASE_REDHAT,
                                      state='available')

        self.environment().suspend(verbose=False)
        self.environment().snapshot(READY_SNAPSHOT)
        self.environment().resume(verbose=False)

    @logwrap
    def update_redhat_credentials(
            self, license_type=REDHAT_LICENSE_TYPE,
            username=REDHAT_USERNAME, password=REDHAT_PASSWORD,
            satellite_host=REDHAT_SATELLITE_HOST,
            activation_key=REDHAT_ACTIVATION_KEY):

        # release name is in environment variable OPENSTACK_RELEASE
        release_id = self.client.get_release_id('RHOS')
        self.client.update_redhat_setup({
            "release_id": release_id,
            "username": username,
            "license_type": license_type,
            "satellite": satellite_host,
            "password": password,
            "activation_key": activation_key})
        tasks = self.client.get_tasks()
        # wait for 'redhat_setup' task only. Front-end works same way
        for task in tasks:
            if task['name'] == 'redhat_setup' \
                    and task['result']['release_info']['release_id'] \
                            == release_id:
                return self._task_wait(task, 60 * 120)

    def assert_release_state(self, release_name, state='available'):
        for release in self.client.get_releases():
            if release["name"].find(release_name) != -1:
                self.assertEqual(release['state'], state)
                return release["id"]
Пример #3
0
class BaseNodeTestCase(BaseTestCase):
    def setUp(self):
        if CLEAN:
            self.ci().get_empty_state()
        self.client = NailgunClient(self.get_admin_node_ip())

    @logwrap
    def get_interface_description(self, ctrl_ssh, interface_short_name):
        return ''.join(
            ctrl_ssh.execute('/sbin/ip addr show dev %s' %
                             interface_short_name)['stdout'])

    def assertNetworkConfiguration(self, node):
        remote = SSHClient(node['ip'],
                           username='******',
                           password='******',
                           private_keys=self.get_private_keys())
        for interface in node['network_data']:
            if interface.get('vlan') is None:
                continue  # todo excess check fix interface json format
            interface_name = "%s.%s@%s" % (interface['dev'], interface['vlan'],
                                           interface['dev'])
            interface_short_name = "%s.%s" % (interface['dev'],
                                              interface['vlan'])
            interface_description = self.get_interface_description(
                remote, interface_short_name)
            self.assertIn(interface_name, interface_description)
            if interface.get('name') == 'floating':
                continue
            if interface.get('ip'):
                self.assertIn("inet %s" % interface.get('ip'),
                              interface_description)
            else:
                self.assertNotIn("inet ", interface_description)
            if interface.get('brd'):
                self.assertIn("brd %s" % interface['brd'],
                              interface_description)

    @logwrap
    def is_node_discovered(self, nailgun_node):
        return any(
            map(
                lambda node: node['mac'] == nailgun_node['mac'] and node[
                    'status'] == 'discover', self.client.list_nodes()))

    @logwrap
    def get_target_devs(self, devops_nodes):
        return [
            interface.target_dev for interface in [
                val for var in map(lambda node: node.interfaces, devops_nodes)
                for val in var
            ]
        ]

    @logwrap
    def get_ebtables(self, cluster_id, devops_nodes):
        return Ebtables(self.get_target_devs(devops_nodes),
                        self.client._get_cluster_vlans(cluster_id))

    @logwrap
    def _get_common_vlan(self, cluster_id):
        """Find vlan that must be at all two nodes.
        """
        return self.client.get_networks(
            cluster_id)['networks'][0]['vlan_start']

    @logwrap
    def _run_network_verify(self, cluster_id):
        return self.client.verify_networks(
            cluster_id,
            self.client.get_networks(cluster_id)['networks'])

    @logwrap
    def check_role_file(self, nodes_dict):
        for node, role in self.get_nailgun_node_roles(nodes_dict):
            remote = SSHClient(node['ip'],
                               username='******',
                               password='******',
                               private_keys=self.get_private_keys())
            if role != "cinder":
                self.assertTrue(remote.isfile('/tmp/%s-file' % role))

    @logwrap
    def clean_clusters(self):
        self.client.clean_clusters()

    @logwrap
    def _basic_provisioning(self, cluster_id, nodes_dict, port=5514):
        self.client.add_syslog_server(cluster_id,
                                      self.ci().get_host_node_ip(), port)

        # update cluster deployment mode
        node_names = []
        for role in nodes_dict:
            node_names += nodes_dict[role]
        if len(node_names) > 1:
            controller_amount = len(nodes_dict.get('controller', []))
            if controller_amount == 1:
                self.client.update_cluster(cluster_id, {"mode": "multinode"})
            if controller_amount > 1:
                self.client.update_cluster(cluster_id, {"mode": "ha"})

        self.bootstrap_nodes(self.devops_nodes_by_names(node_names))

        # update nodes in cluster
        self.update_nodes(cluster_id, nodes_dict, True, False)

        task = self.deploy_cluster(cluster_id)
        self.assertTaskSuccess(task)
        self.check_role_file(nodes_dict)
        return cluster_id

    @logwrap
    def get_nailgun_node_roles(self, nodes_dict):
        nailgun_node_roles = []
        for role in nodes_dict:
            for node_name in nodes_dict[role]:
                slave = self.ci().environment().node_by_name(node_name)
                node = self.get_node_by_devops_node(slave)
                nailgun_node_roles.append((node, role))
        return nailgun_node_roles

    @logwrap
    def deploy_cluster(self, cluster_id):
        """Return hash with task description."""
        return self.client.deploy_cluster_changes(cluster_id)

    @logwrap
    def assertTaskSuccess(self, task, timeout=90 * 60):
        self.assertEquals('ready', self._task_wait(task, timeout)['status'])

    @logwrap
    def assertTaskFailed(self, task, timeout=70 * 60):
        self.assertEquals('error', self._task_wait(task, timeout)['status'])

    @logwrap
    def _task_wait(self, task, timeout):
        wait(lambda: self.client.get_task(task['id'])['status'] != 'running',
             timeout=timeout)
        return self.client.get_task(task['id'])

    @logwrap
    def _upload_sample_release(self):
        release_id = self.client.get_grizzly_release_id()
        if not release_id:
            raise Exception("Not implemented uploading of release")
        return release_id

    @logwrap
    def get_or_create_cluster(self, name, release_id):
        if not release_id:
            release_id = self._upload_sample_release()
        cluster_id = self.client.get_cluster_id(name)
        if not cluster_id:
            self.client.create_cluster(data={
                "name": name,
                "release": str(release_id)
            })
            cluster_id = self.client.get_cluster_id(name)
        if not cluster_id:
            raise Exception("Could not get cluster '%s'" % name)
        return cluster_id

    @logwrap
    def create_cluster(self, name='default', release_id=None):
        """
        :param name:
        :param release_id:
        :return: cluster_id
        """
        return self.get_or_create_cluster(name, release_id)

    @logwrap
    def update_nodes(self,
                     cluster_id,
                     nodes_dict,
                     pending_addition=True,
                     pending_deletion=False):
        # update nodes in cluster
        nodes_data = []
        for role in nodes_dict:
            for node_name in nodes_dict[role]:
                slave = self.ci().environment().node_by_name(node_name)
                node = self.get_node_by_devops_node(slave)
                node_data = {
                    'cluster_id': cluster_id,
                    'id': node['id'],
                    'pending_addition': pending_addition,
                    'pending_deletion': pending_deletion,
                    'role': role
                }
                nodes_data.append(node_data)

        # assume nodes are going to be updated for one cluster only
        cluster_id = nodes_data[-1]['cluster_id']
        node_ids = [str(node_info['id']) for node_info in nodes_data]
        self.client.update_nodes(nodes_data)

        nailgun_nodes = self.client.list_cluster_nodes(cluster_id)
        cluster_node_ids = map(lambda node: str(node['id']), nailgun_nodes)
        self.assertTrue(
            all([node_id in cluster_node_ids for node_id in node_ids]))
        return nailgun_nodes

    @logwrap
    def get_node_by_devops_node(self, devops_node):
        """Returns dict with nailgun slave node description if node is
        registered. Otherwise return None.
        """
        mac_addresses = map(
            lambda interface: interface.mac_address.capitalize(),
            devops_node.interfaces)
        for nailgun_node in self.client.list_nodes():
            if nailgun_node['mac'].capitalize() in mac_addresses:
                nailgun_node['devops_name'] = devops_node.name
                return nailgun_node
        return None

    def nailgun_nodes(self, devops_nodes):
        return map(lambda node: self.get_node_by_devops_node(node),
                   devops_nodes)

    def devops_nodes_by_names(self, devops_node_names):
        return map(lambda name: self.ci().environment().node_by_name(name),
                   devops_node_names)

    @logwrap
    def bootstrap_nodes(self, devops_nodes, timeout=600):
        """Start vms and wait they are registered on nailgun.
        :rtype : List of registred nailgun nodes
        """
        for node in devops_nodes:
            node.start()
        wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout)
        return self.nailgun_nodes(devops_nodes)

    @logwrap
    def assert_service_list(self, remote, smiles_count):
        ret = remote.check_call('/usr/bin/nova-manage service list')
        self.assertEqual(smiles_count, ''.join(ret['stdout']).count(":-)"),
                         "Smiles count")
        self.assertEqual(0, ''.join(ret['stdout']).count("XXX"),
                         "Broken services count")

    @logwrap
    def assert_node_service_list(self, node_name, smiles_count):
        ip = self.get_node_by_devops_node(
            self.ci().environment().node_by_name(node_name))['ip']
        remote = SSHClient(ip,
                           username='******',
                           password='******',
                           private_keys=self.get_private_keys())
        return self.assert_service_list(remote, smiles_count)

    @logwrap
    def assert_glance_index(self, ctrl_ssh):
        ret = ctrl_ssh.check_call('. /root/openrc; glance index')
        self.assertEqual(1, ''.join(ret['stdout']).count("TestVM"))

    @logwrap
    def assert_network_list(self, networks_count, remote):
        ret = remote.check_call('/usr/bin/nova-manage network list')
        self.assertEqual(networks_count + 1, len(ret['stdout']))

    @logwrap
    def assertClusterReady(self,
                           node_name,
                           smiles_count,
                           networks_count=1,
                           timeout=300):
        _wait(lambda: self.get_cluster_status(self.get_node_by_devops_node(
            self.ci().environment().node_by_name(node_name))['ip'],
                                              smiles_count=smiles_count,
                                              networks_count=networks_count),
              timeout=timeout)

    @logwrap
    def get_cluster_status(self, ip, smiles_count, networks_count=1):
        remote = SSHClient(ip,
                           username='******',
                           password='******',
                           private_keys=self.get_private_keys())
        self.assert_service_list(remote, smiles_count)
        self.assert_glance_index(remote)
        self.assert_network_list(networks_count, remote)

    @logwrap
    def get_cluster_floating_list(self, ip):
        remote = SSHClient(ip,
                           username='******',
                           password='******',
                           private_keys=self.get_private_keys())
        ret = remote.check_call('/usr/bin/nova-manage floating list')
        ret_str = ''.join(ret['stdout'])
        return re.findall('(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', ret_str)

    @logwrap
    def assert_cluster_floating_list(self, node_name, expected_ips):
        ip = self.get_node_by_devops_node(
            self.ci().environment().node_by_name(node_name))['ip']
        current_ips = self.get_cluster_floating_list(ip)
        self.assertEqual(set(expected_ips), set(current_ips))

    @logwrap
    def get_private_keys(self):
        keys = []
        for key_string in ['/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa']:
            with self.remote().open(key_string) as f:
                keys.append(RSAKey.from_private_key(f))
        return keys

    @logwrap
    def update_vlan_network_fixed(self,
                                  cluster_id,
                                  amount=1,
                                  network_size=256):
        network_list = self.client.get_networks(cluster_id)['networks']
        for network in network_list:
            if network["name"] == 'fixed':
                network['amount'] = amount
                network['network_size'] = network_size

        self.client.update_network(cluster_id,
                                   networks=network_list,
                                   net_manager=NETWORK_MANAGERS['vlan'])
Пример #4
0
class BaseNodeTestCase(BaseTestCase):

    environment_states = {}

    def setUp(self):
        self.client = NailgunClient(self.get_admin_node_ip())

    @logwrap
    def get_interface_description(self, ctrl_ssh, interface_short_name):
        return ''.join(
            ctrl_ssh.execute('/sbin/ip addr show dev %s' %
                             interface_short_name)['stdout'])

    def assertNetworkConfiguration(self, node):
        remote = SSHClient(node['ip'],
                           username='******',
                           password='******',
                           private_keys=self.get_private_keys())
        for interface in node['network_data']:
            if interface.get('vlan') is None:
                continue  # todo excess check fix interface json format
            interface_name = "%s.%s@%s" % (interface['dev'], interface['vlan'],
                                           interface['dev'])
            interface_short_name = "%s.%s" % (interface['dev'],
                                              interface['vlan'])
            interface_description = self.get_interface_description(
                remote, interface_short_name)
            self.assertIn(interface_name, interface_description)
            if interface.get('name') == 'floating':
                continue
            if interface.get('ip'):
                self.assertIn("inet %s" % interface.get('ip'),
                              interface_description)
            else:
                self.assertNotIn("inet ", interface_description)
            if interface.get('brd'):
                self.assertIn("brd %s" % interface['brd'],
                              interface_description)

    @logwrap
    def is_node_discovered(self, nailgun_node):
        return any(
            map(
                lambda node: node['mac'] == nailgun_node['mac'] and node[
                    'status'] == 'discover', self.client.list_nodes()))

    @logwrap
    def get_target_devs(self, devops_nodes):
        return [
            interface.target_dev for interface in [
                val for var in map(lambda node: node.interfaces, devops_nodes)
                for val in var
            ]
        ]

    @logwrap
    def get_ebtables(self, cluster_id, devops_nodes):
        return Ebtables(self.get_target_devs(devops_nodes),
                        self.client._get_cluster_vlans(cluster_id))

    @logwrap
    def _get_common_vlan(self, cluster_id):
        """Find vlan that must be at all two nodes.
        """
        return self.client.get_networks(
            cluster_id)['networks'][0]['vlan_start']

    @logwrap
    def _run_network_verify(self, cluster_id):
        return self.client.verify_networks(
            cluster_id,
            self.client.get_networks(cluster_id)['networks'])

    @logwrap
    def check_role_file(self, nodes_dict):
        for node, roles in self.get_nailgun_node_roles(nodes_dict):
            remote = SSHClient(node['ip'],
                               username='******',
                               password='******',
                               private_keys=self.get_private_keys())
            for role in roles:
                if role != "cinder":
                    self.assertTrue(remote.isfile('/tmp/%s-file' % role))

    @logwrap
    def clean_clusters(self):
        self.client.clean_clusters()

    @logwrap
    def configure_cluster(self, cluster_id, nodes_dict):
        self.update_nodes(cluster_id, nodes_dict, True, False)
        # TODO: update network configuration

    @logwrap
    def basic_provisioning(self, cluster_id, nodes_dict, port=5514):
        self.client.add_syslog_server(cluster_id,
                                      self.ci().get_host_node_ip(), port)

        self.bootstrap_nodes(self.devops_nodes_by_names(nodes_dict.keys()))

        for node in self.devops_nodes_by_names(nodes_dict.keys()):
            self.sync_node_time(node.name)

        self.configure_cluster(cluster_id, nodes_dict)

        task = self.deploy_cluster(cluster_id)
        self.assertTaskSuccess(task)
        return cluster_id

    @logwrap
    def prepare_environment(self,
                            name='cluster_name',
                            mode="multinode",
                            settings=None,
                            save_state=True):
        if not (self.ci().revert_to_state(settings)):
            self.get_ready_environment()
            if settings is None:
                return None

            net_provider = None
            net_segment_type = None

            if 'net_provider' in settings:
                net_provider = settings['net_provider']

            if 'net_segment_type' in settings:
                net_segment_type = settings['net_segment_type']

            if 'nodes' in settings:
                cluster_id = self.create_cluster(
                    name=name,
                    mode=mode,
                    net_provider=net_provider,
                    net_segment_type=net_segment_type)
                self.other_cluster_settings(cluster_id, settings)
                self.basic_provisioning(cluster_id, settings['nodes'])

            if save_state:
                self.ci().snapshot_state(name, settings)

        # return id of last created cluster
        clusters = self.client.list_clusters()
        if len(clusters) > 0:
            return clusters.pop()['id']
        return None

    @logwrap
    def other_cluster_settings(self, cluster_id, settings):
        attributes = self.client.get_cluster_attributes(cluster_id)
        for option in settings:
            section = False
            if option in ('savanna', 'murano'):
                section = 'additional_components'
            if option in ('volumes_ceph', 'images_ceph'):
                section = 'storage'
            if section:
                attributes['editable'][section][option]['value'] = settings[
                    option]

        self.client.update_cluster_attributes(cluster_id, attributes)

    @logwrap
    def get_nailgun_node_roles(self, nodes_dict):
        nailgun_node_roles = []
        for node_name in nodes_dict:
            slave = self.ci().environment().node_by_name(node_name)
            node = self.get_node_by_devops_node(slave)
            nailgun_node_roles.append((node, nodes_dict[node_name]))
        return nailgun_node_roles

    @logwrap
    def deploy_cluster(self, cluster_id):
        """Return hash with task description."""
        return self.client.deploy_cluster_changes(cluster_id)

    @logwrap
    def assertTaskSuccess(self, task, timeout=90 * 60):
        self.assertEquals('ready', self._task_wait(task, timeout)['status'])

    @logwrap
    def assertTaskFailed(self, task, timeout=70 * 60):
        self.assertEquals('error', self._task_wait(task, timeout)['status'])

    @logwrap
    def assertOSTFRunSuccess(self,
                             cluster_id,
                             should_fail=0,
                             should_pass=0,
                             timeout=10 * 60):
        set_result_list = self._ostf_test_wait(cluster_id, timeout)

        passed = 0
        failed = 0
        for set_result in set_result_list:
            passed += len(
                filter(lambda test: test['status'] == 'success',
                       set_result['tests']))
            failed += len(
                filter(
                    lambda test: test['status'] == 'failure' or test['status']
                    == 'error', set_result['tests']))
        self.assertGreaterEqual(passed, should_pass,
                                'Passed tests, pass: {}'.format(passed))
        self.assertLessEqual(failed, should_fail,
                             'Failed tests,  fails: {}'.format(failed))

    @logwrap
    def run_OSTF(self,
                 cluster_id,
                 test_sets=None,
                 should_fail=0,
                 should_pass=0):
        test_sets = test_sets \
            if test_sets is not None \
            else ['smoke', 'sanity']

        self.client.ostf_run_tests(cluster_id, test_sets)
        self.assertOSTFRunSuccess(cluster_id,
                                  should_fail=should_fail,
                                  should_pass=should_pass)

    @logwrap
    def _task_wait(self, task, timeout):
        wait(lambda: self.client.get_task(task['id'])['status'] != 'running',
             timeout=timeout)
        return self.client.get_task(task['id'])

    @logwrap
    def _ostf_test_wait(self, cluster_id, timeout):
        wait(lambda: all([
            run['status'] == 'finished'
            for run in self.client.get_ostf_test_run(cluster_id)
        ]),
             timeout=timeout)
        return self.client.get_ostf_test_run(cluster_id)

    @logwrap
    def _tasks_wait(self, tasks, timeout):
        return [self._task_wait(task, timeout) for task in tasks]

    @logwrap
    def _upload_sample_release(self):
        release_id = self.client.get_release_id()
        if not release_id:
            raise Exception("Not implemented uploading of release")
        return release_id

    @logwrap
    def get_or_create_cluster(self,
                              name,
                              release_id,
                              mode="multinode",
                              net_provider=None,
                              net_segment_type=None):
        if not release_id:
            release_id = self._upload_sample_release()
        cluster_id = self.client.get_cluster_id(name)
        if not cluster_id:
            data = {"name": name, "release": str(release_id), "mode": mode}

            if net_provider:
                data.update({
                    'net_provider': net_provider,
                    'net_segment_type': net_segment_type
                })

            self.client.create_cluster(data=data)
            cluster_id = self.client.get_cluster_id(name)
        if not cluster_id:
            raise Exception("Could not get cluster '%s'" % name)
        return cluster_id

    @logwrap
    def create_cluster(self,
                       name='default',
                       release_id=None,
                       mode="multinode",
                       net_provider=None,
                       net_segment_type=None):
        """
        :param name:
        :param release_id:
        :param mode:
        :return: cluster_id
        """
        return self.get_or_create_cluster(name,
                                          release_id,
                                          mode,
                                          net_provider=net_provider,
                                          net_segment_type=net_segment_type)

    @logwrap
    def update_nodes(self,
                     cluster_id,
                     nodes_dict,
                     pending_addition=True,
                     pending_deletion=False):
        # update nodes in cluster
        nodes_data = []
        for node_name in nodes_dict:
            devops_node = self.ci().environment().node_by_name(node_name)
            node = self.get_node_by_devops_node(devops_node)
            node_data = {
                'cluster_id': cluster_id,
                'id': node['id'],
                'pending_addition': pending_addition,
                'pending_deletion': pending_deletion,
                'pending_roles': nodes_dict[node_name],
                'name':
                '%s_%s' % (self.ci().environment().name, devops_node.name)
            }
            nodes_data.append(node_data)

        # assume nodes are going to be updated for one cluster only
        cluster_id = nodes_data[-1]['cluster_id']
        node_ids = [str(node_info['id']) for node_info in nodes_data]
        self.client.update_nodes(nodes_data)

        nailgun_nodes = self.client.list_cluster_nodes(cluster_id)
        cluster_node_ids = map(lambda node: str(node['id']), nailgun_nodes)
        self.assertTrue(
            all([node_id in cluster_node_ids for node_id in node_ids]))
        return nailgun_nodes

    @logwrap
    def get_node_by_devops_node(self, devops_node):
        """Returns dict with nailgun slave node description if node is
        registered. Otherwise return None.
        """
        mac_addresses = map(
            lambda interface: interface.mac_address.capitalize(),
            devops_node.interfaces)
        for nailgun_node in self.client.list_nodes():
            if nailgun_node['mac'].capitalize() in mac_addresses:
                nailgun_node['devops_name'] = devops_node.name
                return nailgun_node
        return None

    def nailgun_nodes(self, devops_nodes):
        return map(lambda node: self.get_node_by_devops_node(node),
                   devops_nodes)

    def devops_nodes_by_names(self, devops_node_names):
        return map(lambda name: self.ci().environment().node_by_name(name),
                   devops_node_names)

    @logwrap
    def bootstrap_nodes(self, devops_nodes, timeout=600):
        """Start vms and wait they are registered on nailgun.
        :rtype : List of registred nailgun nodes
        """
        for node in devops_nodes:
            node.start()
        wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout)
        return self.nailgun_nodes(devops_nodes)

    @logwrap
    def assert_service_list(self, remote, smiles_count):
        ret = remote.check_call('/usr/bin/nova-manage service list')
        self.assertEqual(smiles_count, ''.join(ret['stdout']).count(":-)"),
                         "Smiles count")
        self.assertEqual(0, ''.join(ret['stdout']).count("XXX"),
                         "Broken services count")

    @logwrap
    def assert_node_service_list(self, node_name, smiles_count):
        ip = self.get_node_by_devops_node(
            self.ci().environment().node_by_name(node_name))['ip']
        remote = SSHClient(ip,
                           username='******',
                           password='******',
                           private_keys=self.get_private_keys())
        return self.assert_service_list(remote, smiles_count)

    @logwrap
    def assert_glance_index(self, ctrl_ssh):
        ret = ctrl_ssh.check_call('. /root/openrc; glance index')
        self.assertEqual(1, ''.join(ret['stdout']).count("TestVM"))

    @logwrap
    def assert_network_list(self, networks_count, remote):
        ret = remote.check_call('/usr/bin/nova-manage network list')
        self.assertEqual(networks_count + 1, len(ret['stdout']))

    @logwrap
    def assertClusterReady(self,
                           node_name,
                           smiles_count,
                           networks_count=1,
                           timeout=300):
        _wait(lambda: self.get_cluster_status(self.get_node_by_devops_node(
            self.ci().environment().node_by_name(node_name))['ip'],
                                              smiles_count=smiles_count,
                                              networks_count=networks_count),
              timeout=timeout)

    @logwrap
    def _get_remote(self, ip):
        return SSHClient(ip,
                         username='******',
                         password='******',
                         private_keys=self.get_private_keys())

    @logwrap
    def _get_remote_for_node(self, node_name):
        ip = self.get_node_by_devops_node(
            self.ci().environment().node_by_name(node_name))['ip']
        return self._get_remote(ip)

    @logwrap
    def _get_remote_for_role(self, nodes_dict, role):
        node_name = sorted(
            filter(lambda name: role in nodes_dict[name],
                   nodes_dict.keys()))[0]
        return self._get_remote_for_node(node_name)

    @logwrap
    def get_cluster_status(self, ip, smiles_count, networks_count=1):
        remote = self._get_remote(ip)
        self.assert_service_list(remote, smiles_count)
        self.assert_glance_index(remote)
        self.assert_network_list(networks_count, remote)

    @logwrap
    def get_cluster_floating_list(self, node_name):
        remote = self._get_remote_for_node(node_name)
        ret = remote.check_call('/usr/bin/nova-manage floating list')
        ret_str = ''.join(ret['stdout'])
        return re.findall('(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', ret_str)

    @logwrap
    def get_cluster_block_devices(self, node_name):
        remote = self._get_remote_for_node(node_name)
        ret = remote.check_call('/bin/lsblk')
        return ''.join(ret['stdout'])

    @logwrap
    def assert_cluster_floating_list(self, node_name, expected_ips):
        current_ips = self.get_cluster_floating_list(node_name)
        self.assertEqual(set(expected_ips), set(current_ips))

    @logwrap
    def get_private_keys(self):
        keys = []
        for key_string in ['/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa']:
            with self.remote().open(key_string) as f:
                keys.append(RSAKey.from_private_key(f))
        return keys

    @logwrap
    def sync_node_time(self, node_name):
        remote = self._get_remote_for_node(node_name)
        ret = remote.execute('hwclock --hctosys >/dev/null 2>&1 0<&1')
        ret = remote.execute('ntpd -g -u ntp:ntp >/dev/null 2>&1')
        ret = remote.execute('hwclock -w >/dev/null 2>&1')

    @logwrap
    def update_node_networks(self, node_id, interfaces_dict):
        interfaces = self.client.get_node_interfaces(node_id)
        for interface in interfaces:
            interface_name = interface['name']
            interface['assigned_networks'] = []
            for allowed_network in interface['allowed_networks']:
                key_exists = interface_name in interfaces_dict
                if key_exists and \
                        allowed_network['name'] \
                        in interfaces_dict[interface_name]:
                    interface['assigned_networks'].append(allowed_network)

        self.client.put_node_interfaces([{
            'id': node_id,
            'interfaces': interfaces
        }])

    @logwrap
    def update_vlan_network_fixed(self,
                                  cluster_id,
                                  amount=1,
                                  network_size=256):
        network_list = self.client.get_networks(cluster_id)['networks']
        for network in network_list:
            if network["name"] == 'fixed':
                network['amount'] = amount
                network['network_size'] = network_size

        self.client.update_network(cluster_id,
                                   networks=network_list,
                                   net_manager=NETWORK_MANAGERS['vlan'])

    @logwrap
    @upload_manifests
    def get_ready_environment(self):
        if self.ci().get_state(READY_SNAPSHOT):
            self.environment().resume(verbose=False)
            return

        self.ci().get_empty_environment()
        if OPENSTACK_RELEASE == OPENSTACK_RELEASE_REDHAT:
            # update redhat credentials so that fuel may upload redhat
            # packages

            # download redhat repo from local place to boost the test
            # remote = self.nodes().admin.remote(
            #   'internal', 'root', 'r00tme')
            # remote.execute(
            #   'wget -q http://172.18.67.168/rhel6/rhel-rpms.tar.gz')
            # remote.execute('tar xzf rhel-rpms.tar.gz -C /')

            self.update_redhat_credentials()
            self.assert_release_state(OPENSTACK_RELEASE_REDHAT,
                                      state='available')

        self.environment().suspend(verbose=False)
        self.environment().snapshot(READY_SNAPSHOT)
        self.environment().resume(verbose=False)

    @logwrap
    def update_redhat_credentials(self,
                                  license_type=REDHAT_LICENSE_TYPE,
                                  username=REDHAT_USERNAME,
                                  password=REDHAT_PASSWORD,
                                  satellite_host=REDHAT_SATELLITE_HOST,
                                  activation_key=REDHAT_ACTIVATION_KEY):

        # release name is in environment variable OPENSTACK_RELEASE
        release_id = self.client.get_release_id('RHOS')
        self.client.update_redhat_setup({
            "release_id": release_id,
            "username": username,
            "license_type": license_type,
            "satellite": satellite_host,
            "password": password,
            "activation_key": activation_key
        })
        tasks = self.client.get_tasks()
        # wait for 'redhat_setup' task only. Front-end works same way
        for task in tasks:
            if task['name'] == 'redhat_setup' \
                    and task['result']['release_info']['release_id'] \
                            == release_id:
                return self._task_wait(task, 60 * 120)

    def assert_release_state(self, release_name, state='available'):
        for release in self.client.get_releases():
            if release["name"].find(release_name) != -1:
                self.assertEqual(release['state'], state)
                return release["id"]

    @logwrap
    def assert_murano_service(self, node_name):
        ip = self.get_node_by_devops_node(
            self.ci().environment().node_by_name(node_name))['ip']
        remote = SSHClient(ip,
                           username='******',
                           password='******',
                           private_keys=self.get_private_keys())
        ps_output = remote.execute('ps ax')['stdout']

        murano_api = filter(lambda x: 'murano-api' in x, ps_output)
        self.assertEqual(len(murano_api), 1)

        muranoconductor = filter(lambda x: 'muranoconductor' in x, ps_output)
        self.assertEqual(len(muranoconductor), 1)

    @logwrap
    def assert_savanna_service(self, node_name):
        ip = self.get_node_by_devops_node(
            self.ci().environment().node_by_name(node_name))['ip']
        remote = SSHClient(ip,
                           username='******',
                           password='******',
                           private_keys=self.get_private_keys())
        ps_output = remote.execute('ps ax')['stdout']

        savanna_api = filter(lambda x: 'savanna-api' in x, ps_output)
        self.assertEquals(len(savanna_api), 1)