Exemple #1
0
 def __init__(self, base_image=None):
     """
     Constructor for create environment.
     """
     self.manager = Manager()
     self.base_image = base_image or BASE_IMAGE
     self.environment = self.create_env()
Exemple #2
0
 def __init__(self, os_image=None):
     self._virtual_environment = None
     self._keys = None
     self.manager = Manager()
     self.os_image = os_image
     self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self)
Exemple #3
0
        node = manager.node_create(name='test_node' + str(i),
                                   environment=environment)
        manager.interface_create(node=node, network=internal)
        manager.interface_create(node=node, network=external)
        manager.interface_create(node=node, network=private)
        volume = manager.volume_get_predefined(
            '/var/lib/libvirt/images/centos63-cobbler-base.qcow2')
        v3 = manager.volume_create_child('test_vp895' + str(i),
                                         backing_store=volume,
                                         environment=environment)
        v4 = manager.volume_create_child('test_vp896' + str(i),
                                         backing_store=volume,
                                         environment=environment)
        manager.node_attach_volume(node=node, volume=v3)
        manager.node_attach_volume(node, v4)
    environment.define()
    environment.start()
    remotes = []
    for node in environment.nodes:
        node. await ('internal')
        node.remote('internal', 'root', 'r00tme').check_stderr('ls -la',
                                                               verbose=True)
        remotes.append(node.remote('internal', 'root', 'r00tme'))
    SSHClient.execute_together(remotes, 'ls -la')


if __name__ == '__main__':
    from devops.manager import Manager

    one(Manager())
Exemple #4
0
class CiBase(object):
    def __init__(self):
        self._environment = None
        self.manager = Manager()
        # self.base_image = self.manager.volume_get_predefined(BASE_IMAGE)

    def get_or_create(self):
        try:
            return self.manager.environment_get(self.env_name())
        except:
            self._environment = self.describe_environment()
            self._environment.define()
            return self._environment

    def get_empty_state(self):
        if self.environment().has_snapshot(EMPTY_SNAPSHOT):
            self.environment().revert(EMPTY_SNAPSHOT)
        else:
            self.setup_environment()

    def environment(self):
        """
        :rtype : devops.models.Environment
        """
        self._environment = self._environment or self.get_or_create()
        return self._environment

    @abstractproperty
    def env_name(self):
        """
        :rtype : string
        """
        pass

    @abstractmethod
    def define(self):
        """
        :rtype : devops.models.Environment
        """
        pass

    @abstractmethod
    def describe_environment(self):
        """
        :rtype : devops.models.Environment
        """
        pass

    @abstractproperty
    def node_roles(self):
        """
        :rtype : NodeRoles
        """
        pass

    def add_empty_volume(self, node, name, capacity=20 * 1024 * 1024 * 1024, format="qcow2", device="disk", bus='virtio'):
        self.manager.node_attach_volume(
            node=node,
            device=device,
            bus=bus,
            volume=self.manager.volume_create(
                name=name, capacity=capacity, format=format,
                environment=self.environment()))

    def add_node(self, memory, name, boot=None):
        return self.manager.node_create(
            name=name,
            memory=memory,
            environment=self.environment())

    def describe_master_node(self, name, networks, memory=1024):
        node = self.add_node(memory, name, boot=['cdrom', 'hd'])
        for network in networks:
            self.manager.interface_create(network, node=node)
        self.add_empty_volume(node, name + '-system')
        self.add_empty_volume(node, name + '-iso', capacity=_get_file_size(ISO_IMAGE), format='raw', device='cdrom', bus='ide')
        return node

    def describe_empty_node(self, name, networks, memory=1024):
        node = self.add_node(memory, name)
        for network in networks:
            self.manager.interface_create(network, node=node)
        self.add_empty_volume(node, name + '-system')
        self.add_empty_volume(node, name + '-cinder')
        return node

    def nodes(self):
        return Nodes(self.environment(), self.node_roles())

    def add_nodes_to_hosts(self, remote, nodes):
        for node in nodes:
            add_to_hosts(remote,
                node.get_ip_address_by_network_name('internal'), node.name,
                node.name + '.localdomain')

    def setup_master_node(self, master_remote, nodes):
        setup_puppet_master(master_remote)
        add_nmap(master_remote)
        switch_off_ip_tables(master_remote)
        self.add_nodes_to_hosts(master_remote, nodes)

    def setup_agent_nodes(self, nodes):
        agent_config = load(
            root('fuel_test', 'config', 'puppet.agent.config'))
        for node in nodes:
            if node.name != 'master':
                remote = node.remote('public', login='******',
                    password='******')
                self.add_nodes_to_hosts(remote, self.environment().nodes)
                setup_puppet_client(remote)
                write_config(remote, '/etc/puppet/puppet.conf', agent_config)
                request_cerificate(remote)

    def rename_nodes(self, nodes):
        for node in nodes:
            remote = node.remote('public', login='******', password='******')
            change_host_name(remote, node.name,
                node.name + '.localdomain')
            logging.info("Renamed %s" % node.name)

    @abstractmethod
    def setup_environment(self):
        """
        :rtype : None
        """
        pass

    def internal_virtual_ip(self):
        return str(IPNetwork(
            self.environment().network_by_name('internal').ip_network)[-2])

    def floating_network(self):
        prefix = IPNetwork(self.environment().network_by_name('public').ip_network).prefixlen
        return str(
            IPNetwork(self.environment().network_by_name('public').ip_network).subnet(new_prefix=prefix + 2)[-1])

    def public_virtual_ip(self):
        prefix = IPNetwork(self.environment().network_by_name('public').ip_network).prefixlen
        return str(        
            IPNetwork(self.environment().network_by_name('public').ip_network).subnet(new_prefix=prefix + 2)[-2][
            -1])

    def public_router(self):
        return str(
            IPNetwork(
                self.environment().network_by_name('public').ip_network)[1])

    def internal_router(self):
        return str(
            IPNetwork(
                self.environment().network_by_name('internal').ip_network)[1])

    def fixed_network(self):
        return str(
            IPNetwork(self.environment().network_by_name('private').ip_network).subnet(
                new_prefix=27)[0])

    def internal_network(self):
        return str(IPNetwork(self.environment().network_by_name('internal').ip_network))

    def internal_net_mask(self):
        return str(IPNetwork(self.environment().network_by_name('internal').ip_network).netmask)

    def public_net_mask(self):
        return str(IPNetwork(self.environment().network_by_name('public').ip_network).netmask)

    def public_network(self):
        return str(IPNetwork(self.environment().network_by_name('public').ip_network))
Exemple #5
0
class Shell(object):
    def __init__(self):
        super(Shell, self).__init__()
        self.params = self.get_params()
        self.manager = Manager()

    def execute(self):
        self.commands.get(self.params.command)(self)

    def do_list(self):
        env_list = self.manager.environment_list().values('name')
        for env in env_list:
            print(env['name'])

        return env_list

    def node_dict(self, node):
        return {'name': node.name,
                'vnc': node.get_vnc_port()}

    def do_show(self):
        environment = self.manager.environment_get(self.params.name)

        print('%5s %25s' % ("VNC", "NODE-NAME"))
        for item in map(lambda x: self.node_dict(x), environment.nodes):
            print ('%5s %25s' % (item['vnc'], item['name']))

    def do_erase(self):
        self.manager.environment_get(self.params.name).erase()

    def do_start(self):
        self.manager.environment_get(self.params.name).start()

    def do_destroy(self):
        self.manager.environment_get(self.params.name).destroy(verbose=False)

    def do_suspend(self):
        self.manager.environment_get(self.params.name).suspend(verbose=False)

    def do_resume(self):
        self.manager.environment_get(self.params.name).resume(verbose=False)

    def do_revert(self):
        self.manager.environment_get(self.params.name).revert(
            self.params.snapshot_name)

    def do_snapshot(self):
        self.manager.environment_get(self.params.name).snapshot(
            self.params.snapshot_name)

    def do_synchronize(self):
        self.manager.synchronize_environments()

    def do_snapshot_list(self):
        environment = self.manager.environment_get(self.params.name)

        snap_nodes = {}
        max_len = 0
        for node in environment.nodes:
            snaps = sorted(node.get_snapshots())
            for snap in snaps:
                if len(snap) > max_len:
                    max_len = len(snap)
                if snap in snap_nodes:
                    snap_nodes[snap].append(node.name)
                else:
                    snap_nodes[snap] = [node.name, ]

        print("%*s     %50s" % (max_len, "SNAPSHOT", "NODES-NAME"))
        for snap in snap_nodes:
            print("%*s     %50s" % (max_len, snap,
                                    ', '.join(snap_nodes[snap])))

    def do_snapshot_delete(self):
        environment = self.manager.environment_get(self.params.name)
        for node in environment.nodes:
            snaps = sorted(node.get_snapshots())
            if self.params.snapshot_name in snaps:
                node.erase_snapshot(name=self.params.snapshot_name)

    def do_net_list(self):
        environment = self.manager.environment_get(self.params.name)
        networks = environment.networks
        print("%15s   %10s" % ("NETWORK NAME", "IP NET"))
        for network in networks:
            print("%15s  %10s" % (network.name, network.ip_network))

    def do_timesync(self):
        env = self.manager.environment_get(self.params.name)
        if not self.params.node_name:
            _nodes = {node.name: node.get_vnc_port() for node in env.nodes}
            for node_name in sorted(_nodes.keys()):
                if _nodes[node_name] != '-1':
                    sync_node_time(env, node_name)
        else:
            sync_node_time(env, self.params.node_name)

    def do_revert_resume(self):
        self.manager.environment_get(self.params.name).revert(
            self.params.snapshot_name)
        self.manager.environment_get(self.params.name).resume(verbose=False)
        if not self.params.no_timesync:
            print('time synchronization is starting')
            self.do_timesync()

    commands = {
        'list': do_list,
        'show': do_show,
        'erase': do_erase,
        'start': do_start,
        'destroy': do_destroy,
        'suspend': do_suspend,
        'resume': do_resume,
        'revert': do_revert,
        'snapshot': do_snapshot,
        'sync': do_synchronize,
        'snapshot-list': do_snapshot_list,
        'snapshot-delete': do_snapshot_delete,
        'net-list': do_net_list,
        'time-sync': do_timesync,
        'revert-resume': do_revert_resume
    }

    def get_params(self):
        name_parser = argparse.ArgumentParser(add_help=False)
        name_parser.add_argument('name', help='environment name',
                                 default=environ.get('ENV_NAME'),
                                 metavar='ENV_NAME')
        snapshot_name_parser = argparse.ArgumentParser(add_help=False)
        snapshot_name_parser.add_argument('--snapshot-name',
                                          help='snapshot name',
                                          default=environ.get('SNAPSHOT_NAME'))
        node_name_parser = argparse.ArgumentParser(add_help=False)
        node_name_parser.add_argument('--node-name',
                                      help='node name',
                                      default=None)
        no_timesync_parser = argparse.ArgumentParser(add_help=False)
        no_timesync_parser.add_argument('--no-timesync', dest='no_timesync',
                                        action='store_const', const=True,
                                        help='revert without timesync',
                                        default=False)
        parser = argparse.ArgumentParser(
            description="Manage virtual environments. "
                        "For addional help use command with -h/--help")
        subparsers = parser.add_subparsers(title="Operation commands",
                                           help='available commands',
                                           dest='command')
        subparsers.add_parser('list',
                              help="Show virtual environments",
                              description="Show virtual environments on host")
        subparsers.add_parser('show', parents=[name_parser],
                              help="Show VMs in environment",
                              description="Show VMs in environment")
        subparsers.add_parser('erase', parents=[name_parser],
                              help="Delete environment",
                              description="Delete environment and VMs on it")
        subparsers.add_parser('start', parents=[name_parser],
                              help="Start VMs",
                              description="Start VMs in selected environment")
        subparsers.add_parser('destroy', parents=[name_parser],
                              help="Destroy(stop) VMs",
                              description="Stop VMs in selected environment")
        subparsers.add_parser('suspend', parents=[name_parser],
                              help="Suspend VMs",
                              description="Suspend VMs in selected "
                              "environment")
        subparsers.add_parser('resume', parents=[name_parser],
                              help="Resume VMs",
                              description="Resume VMs in selected environment")
        subparsers.add_parser('revert',
                              parents=[name_parser, snapshot_name_parser],
                              help="Apply snapshot to environment",
                              description="Apply selected snapshot to "
                              "environment")
        subparsers.add_parser('snapshot',
                              parents=[name_parser, snapshot_name_parser],
                              help="Make environment snapshot",
                              description="Make environment snapshot")
        subparsers.add_parser('sync',
                              help="Synchronization environment and devops",
                              description="Synchronization environment "
                              "and devops"),
        subparsers.add_parser('snapshot-list',
                              parents=[name_parser],
                              help="Show snapshots in environment",
                              description="Show snapshots in selected "
                              "environment")
        subparsers.add_parser('snapshot-delete',
                              parents=[name_parser, snapshot_name_parser],
                              help="Delete snapshot from environment",
                              description="Delete snapshot from selected "
                              "environment")
        subparsers.add_parser('net-list',
                              parents=[name_parser],
                              help="Show networks in environment",
                              description="Display allocated networks for "
                              "environment")
        subparsers.add_parser('time-sync',
                              parents=[name_parser, node_name_parser],
                              help="Sync time on all env nodes",
                              description="Sync time on all active nodes "
                                          "of environment starting from "
                                          "admin")
        subparsers.add_parser('revert-resume',
                              parents=[name_parser, snapshot_name_parser,
                                       node_name_parser, no_timesync_parser],
                              help="Revert, resume, sync time on VMs",
                              description="Revert and resume VMs in selected"
                                          "environment, then"
                                          " sync time on VMs")
        return parser.parse_args()
Exemple #6
0
 def __init__(self):
     self.manager = Manager()
     self._environment = None
     self.saved_environment_states = {}
Exemple #7
0
 def __init__(self):
     self.manager = Manager()
     self.base_image = self.manager.volume_get_predefined(BASE_IMAGE)
     self._environment = None
class EnvironmentModel(object):
    hostname = 'nailgun'
    domain = 'test.domain.local'
    installation_timeout = 1800
    deployment_timeout = 1800
    puppet_timeout = 2000
    nat_interface = ''  # INTERFACES.get('admin')
    admin_net = 'admin'
    admin_net2 = 'admin2'
    multiple_cluster_networks = settings.MULTIPLE_NETWORKS

    def __init__(self, os_image=None):
        self._virtual_environment = None
        self._keys = None
        self.manager = Manager()
        self.os_image = os_image
        self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self)

    @property
    def nailgun_actions(self):
        return FuelActions.Nailgun(self.get_admin_remote())

    @property
    def postgres_actions(self):
        return FuelActions.Postgres(self.get_admin_remote())

    def _get_or_create(self):
        try:
            return self.manager.environment_get(self.env_name)
        except Exception:
            self._virtual_environment = self.describe_environment()
            self._virtual_environment.define()
            return self._virtual_environment

    def router(self, router_name=None):
        router_name = router_name or self.admin_net
        if router_name == self.admin_net2:
            return str(IPNetwork(self.get_virtual_environment().
                                 network_by_name(router_name).ip_network)[2])
        return str(
            IPNetwork(
                self.get_virtual_environment().network_by_name(router_name).
                ip_network)[1])

    @property
    def fuel_web(self):
        """FuelWebClient
        :rtype: FuelWebClient
        """
        return self._fuel_web

    @property
    def admin_node_ip(self):
        return self.fuel_web.admin_node_ip

    @property
    def node_roles(self):
        return NodeRoles(
            admin_names=['admin'],
            other_names=['slave-%02d' % x for x in range(1, int(
                settings.NODES_COUNT))]
        )

    @property
    def env_name(self):
        return settings.ENV_NAME

    def add_empty_volume(self, node, name,
                         capacity=settings.NODE_VOLUME_SIZE * 1024 * 1024
                         * 1024, device='disk', bus='virtio', format='qcow2'):
        self.manager.node_attach_volume(
            node=node,
            volume=self.manager.volume_create(
                name=name,
                capacity=capacity,
                environment=self.get_virtual_environment(),
                format=format),
            device=device,
            bus=bus)

    def add_node(self, memory, name, vcpu=1, boot=None):
        return self.manager.node_create(
            name=name,
            memory=memory,
            vcpu=vcpu,
            environment=self.get_virtual_environment(),
            boot=boot)

    @logwrap
    def add_syslog_server(self, cluster_id, port=5514):
        self.fuel_web.add_syslog_server(
            cluster_id, self.get_host_node_ip(), port)

    def bootstrap_nodes(self, devops_nodes, timeout=600):
        """Lists registered nailgun nodes
        Start vms and wait until they are registered on nailgun.
        :rtype : List of registered nailgun nodes
        """
        # self.dhcrelay_check()

        for node in devops_nodes:
            node.start()
            # TODO(aglarendil): LP#1317213 temporary sleep
            # remove after better fix is applied
            time.sleep(2)
        wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout)

        for node in self.nailgun_nodes(devops_nodes):
            self.sync_node_time(self.get_ssh_to_remote(node["ip"]))

        return self.nailgun_nodes(devops_nodes)

    def create_interfaces(self, networks, node,
                          model=settings.INTERFACE_MODEL):
        if settings.BONDING:
            for network in networks:
                self.manager.interface_create(
                    network, node=node, model=model,
                    interface_map=settings.BONDING_INTERFACES)
        else:
            for network in networks:
                self.manager.interface_create(network, node=node, model=model)

    def describe_environment(self):
        """Environment
        :rtype : Environment
        """
        environment = self.manager.environment_create(self.env_name)
        networks = []
        interfaces = settings.INTERFACE_ORDER
        if self.multiple_cluster_networks:
            logger.info('Multiple cluster networks feature is enabled!')
        if settings.BONDING:
            interfaces = settings.BONDING_INTERFACES.keys()

        for name in interfaces:
            networks.append(self.create_networks(name, environment))
        for name in self.node_roles.admin_names:
            self.describe_admin_node(name, networks)
        for name in self.node_roles.other_names:
            if self.multiple_cluster_networks:
                networks1 = [net for net in networks if net.name
                             in settings.NODEGROUPS[0]['pools']]
                networks2 = [net for net in networks if net.name
                             in settings.NODEGROUPS[1]['pools']]
                # If slave index is even number, then attach to
                # it virtual networks from the second network group.
                if int(name[-2:]) % 2 == 1:
                    self.describe_empty_node(name, networks1)
                elif int(name[-2:]) % 2 == 0:
                    self.describe_empty_node(name, networks2)
            else:
                self.describe_empty_node(name, networks)
        return environment

    def create_networks(self, name, environment):
        ip_networks = [
            IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',')]
        new_prefix = int(settings.POOLS.get(name)[1])
        pool = self.manager.create_network_pool(networks=ip_networks,
                                                prefix=int(new_prefix))
        return self.manager.network_create(
            name=name,
            environment=environment,
            pool=pool,
            forward=settings.FORWARDING.get(name),
            has_dhcp_server=settings.DHCP.get(name))

    def devops_nodes_by_names(self, devops_node_names):
        return map(
            lambda name:
            self.get_virtual_environment().node_by_name(name),
            devops_node_names)

    @logwrap
    def describe_admin_node(self, name, networks):
        node = self.add_node(
            memory=settings.HARDWARE.get("admin_node_memory", 1024),
            vcpu=settings.HARDWARE.get("admin_node_cpu", 1),
            name=name,
            boot=['hd', 'cdrom'])
        self.create_interfaces(networks, node)

        if self.os_image is None:
            self.add_empty_volume(node, name + '-system')
            self.add_empty_volume(
                node,
                name + '-iso',
                capacity=_get_file_size(settings.ISO_PATH),
                format='raw',
                device='cdrom',
                bus='ide')
        else:
            volume = self.manager.volume_get_predefined(self.os_image)
            vol_child = self.manager.volume_create_child(
                name=name + '-system',
                backing_store=volume,
                environment=self.get_virtual_environment()
            )
            self.manager.node_attach_volume(
                node=node,
                volume=vol_child
            )
        return node

    def describe_empty_node(self, name, networks):
        node = self.add_node(
            name=name,
            memory=settings.HARDWARE.get("slave_node_memory", 1024),
            vcpu=settings.HARDWARE.get("slave_node_cpu", 1))
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + '-system')

        if settings.USE_ALL_DISKS:
            self.add_empty_volume(node, name + '-cinder')
            self.add_empty_volume(node, name + '-swift')

        return node

    @logwrap
    def get_admin_remote(self, login=settings.SSH_CREDENTIALS['login'],
                         password=settings.SSH_CREDENTIALS['password']):
        """SSH to admin node
        :rtype : SSHClient
        """
        return self.nodes().admin.remote(self.admin_net,
                                         login=login,
                                         password=password)

    @logwrap
    def get_admin_node_ip(self):
        return str(
            self.nodes().admin.get_ip_address_by_network_name(self.admin_net))

    @logwrap
    def get_ebtables(self, cluster_id, devops_nodes):
        return Ebtables(self.get_target_devs(devops_nodes),
                        self.fuel_web.client.get_cluster_vlans(cluster_id))

    def get_host_node_ip(self):
        return self.router()

    def get_keys(self, node, custom=None, build_images=None):
        params = {
            'ip': node.get_ip_address_by_network_name(self.admin_net),
            'mask': self.get_net_mask(self.admin_net),
            'gw': self.router(),
            'hostname': '.'.join((self.hostname, self.domain)),
            'nat_interface': self.nat_interface,
            'dns1': settings.DNS,
            'showmenu': 'yes' if custom else 'no',
            'build_images': '1' if build_images else '0'

        }
        keys = (
            "<Wait>\n"
            "<Esc><Enter>\n"
            "<Wait>\n"
            "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n"
            " ip=%(ip)s\n"
            " netmask=%(mask)s\n"
            " gw=%(gw)s\n"
            " dns1=%(dns1)s\n"
            " hostname=%(hostname)s\n"
            " dhcp_interface=%(nat_interface)s\n"
            " showmenu=%(showmenu)s\n"
            " build_images=%(build_images)s\n"
            " <Enter>\n"
        ) % params
        return keys

    @logwrap
    def get_private_keys(self, force=False):
        if force or self._keys is None:
            self._keys = []
            for key_string in ['/root/.ssh/id_rsa',
                               '/root/.ssh/bootstrap.rsa']:
                with self.get_admin_remote().open(key_string) as f:
                    self._keys.append(RSAKey.from_private_key(f))
        return self._keys

    @logwrap
    def get_ssh_to_remote(self, ip):
        return SSHClient(ip,
                         username=settings.SSH_CREDENTIALS['login'],
                         password=settings.SSH_CREDENTIALS['password'],
                         private_keys=self.get_private_keys())

    @logwrap
    def get_ssh_to_remote_by_key(self, ip, keyfile):
        try:
            with open(keyfile) as f:
                keys = [RSAKey.from_private_key(f)]
                return SSHClient(ip, private_keys=keys)
        except IOError:
            logger.warning('Loading of SSH key from file failed. Trying to use'
                           ' SSH agent ...')
            keys = Agent().get_keys()
            return SSHClient(ip, private_keys=keys)

    @logwrap
    def get_ssh_to_remote_by_name(self, node_name):
        return self.get_ssh_to_remote(
            self.fuel_web.get_nailgun_node_by_devops_node(
                self.get_virtual_environment().node_by_name(node_name))['ip']
        )

    def get_target_devs(self, devops_nodes):
        return [
            interface.target_dev for interface in [
                val for var in map(lambda node: node.interfaces, devops_nodes)
                for val in var]]

    def get_virtual_environment(self):
        """Returns virtual environment
        :rtype : devops.models.Environment
        """
        if self._virtual_environment is None:
            self._virtual_environment = self._get_or_create()
        return self._virtual_environment

    def get_network(self, net_name):
        return str(
            IPNetwork(
                self.get_virtual_environment().network_by_name(net_name).
                ip_network))

    def get_net_mask(self, net_name):
        return str(
            IPNetwork(
                self.get_virtual_environment().network_by_name(
                    net_name).ip_network).netmask)

    def make_snapshot(self, snapshot_name, description="", is_make=False):
        if settings.MAKE_SNAPSHOT or is_make:
            self.get_virtual_environment().suspend(verbose=False)
            time.sleep(10)
            self.get_virtual_environment().snapshot(snapshot_name, force=True)
            revert_info(snapshot_name, description)
        if settings.FUEL_STATS_CHECK:
            self.get_virtual_environment().resume()
            try:
                self.nodes().admin.await(self.admin_net, timeout=60)
            except Exception:
                logger.error('Admin node is unavailable via SSH after '
                             'environment resume ')
                raise

    def nailgun_nodes(self, devops_nodes):
        return map(
            lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node),
            devops_nodes
        )

    def nodes(self):
        return Nodes(self.get_virtual_environment(), self.node_roles)

    def revert_snapshot(self, name):
        if self.get_virtual_environment().has_snapshot(name):
            logger.info('We have snapshot with such name %s' % name)

            self.get_virtual_environment().revert(name)
            logger.info('Starting snapshot reverting ....')

            self.get_virtual_environment().resume()
            logger.info('Starting snapshot resuming ...')

            admin = self.nodes().admin

            try:
                admin.await(
                    self.admin_net, timeout=10 * 60, by_port=8000)
            except Exception as e:
                logger.warning("From first time admin isn't reverted: "
                               "{0}".format(e))
                admin.destroy()
                logger.info('Admin node was destroyed. Wait 10 sec.')
                time.sleep(10)
                self.get_virtual_environment().start(self.nodes().admins)
                logger.info('Admin node started second time.')
                self.nodes().admin.await(
                    self.admin_net, timeout=10 * 60, by_port=8000)

            self.set_admin_ssh_password()
            try:
                _wait(self._fuel_web.client.get_releases,
                      expected=EnvironmentError, timeout=300)
            except exceptions.Unauthorized:
                self.set_admin_keystone_password()
                self._fuel_web.get_nailgun_version()

            self.sync_time_admin_node()

            for node in self.nodes().slaves:
                if not node.driver.node_active(node):
                    continue
                try:
                    logger.info("Sync time on revert for node %s" % node.name)
                    self.sync_node_time(
                        self.get_ssh_to_remote_by_name(node.name))
                except Exception as e:
                    logger.warning(
                        'Exception caught while trying to sync time on {0}:'
                        ' {1}'.format(node.name, e))
                self.run_nailgun_agent(
                    self.get_ssh_to_remote_by_name(node.name))
            return True
        return False

    def set_admin_ssh_password(self):
        try:
            remote = self.get_admin_remote(
                login=settings.SSH_CREDENTIALS['login'],
                password=settings.SSH_CREDENTIALS['password'])
            self.execute_remote_cmd(remote, 'date')
            logger.debug('Accessing admin node using SSH: SUCCESS')
        except Exception:
            logger.debug('Accessing admin node using SSH credentials:'
                         ' FAIL, trying to change password from default')
            remote = self.get_admin_remote(login='******', password='******')
            self.execute_remote_cmd(
                remote, 'echo -e "{1}\\n{1}" | passwd {0}'
                .format(settings.SSH_CREDENTIALS['login'],
                        settings.SSH_CREDENTIALS['password']))
            logger.debug("Admin node password has changed.")
        logger.info("Admin node login name: '{0}' , password: '******'".
                    format(settings.SSH_CREDENTIALS['login'],
                           settings.SSH_CREDENTIALS['password']))

    def set_admin_keystone_password(self):
        remote = self.get_admin_remote()
        try:
            self._fuel_web.client.get_releases()
        except exceptions.Unauthorized:
            self.execute_remote_cmd(
                remote, 'fuel user --newpass {0} --change-password'
                .format(settings.KEYSTONE_CREDS['password']))
            logger.info(
                'New Fuel UI (keystone) username: "******", password: "******"'
                .format(settings.KEYSTONE_CREDS['username'],
                        settings.KEYSTONE_CREDS['password']))

    def setup_environment(self, custom=settings.CUSTOM_ENV,
                          build_images=settings.BUILD_IMAGES):
        # start admin node
        admin = self.nodes().admin
        admin.disk_devices.get(device='cdrom').volume.upload(settings.ISO_PATH)
        self.get_virtual_environment().start(self.nodes().admins)
        logger.info("Waiting for admin node to start up")
        wait(lambda: admin.driver.node_active(admin), 60)
        logger.info("Proceed with installation")
        # update network parameters at boot screen
        admin.send_keys(self.get_keys(admin, custom=custom,
                        build_images=build_images))
        if custom:
            self.setup_customisation()
        # wait while installation complete
        admin.await(self.admin_net, timeout=10 * 60)
        self.set_admin_ssh_password()
        self.wait_bootstrap()
        time.sleep(10)
        self.set_admin_keystone_password()
        self.sync_time_admin_node()
        if settings.MULTIPLE_NETWORKS:
            self.describe_second_admin_interface()
            multiple_networks_hacks.configure_second_admin_cobbler(self)
            multiple_networks_hacks.configure_second_dhcrelay(self)
        self.nailgun_actions.set_collector_address(
            settings.FUEL_STATS_HOST,
            settings.FUEL_STATS_PORT,
            settings.FUEL_STATS_SSL)
        if settings.FUEL_STATS_ENABLED:
            self.fuel_web.client.send_fuel_stats(enabled=True)
            logger.info('Enabled sending of statistics to {0}:{1}'.format(
                settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT
            ))

    @upload_manifests
    def wait_for_provisioning(self):
        _wait(lambda: _tcp_ping(
            self.nodes().admin.get_ip_address_by_network_name
            (self.admin_net), 22), timeout=5 * 60)

    def setup_customisation(self):
        self.wait_for_provisioning()
        try:
            remote = self.get_admin_remote()
            pid = remote.execute("pgrep 'fuelmenu'")['stdout'][0]
            pid.rstrip('\n')
            remote.execute("kill -sigusr1 {0}".format(pid))
        except Exception:
            logger.error("Could not kill pid of fuelmenu")
            raise

    @retry(count=10, delay=60)
    @logwrap
    def sync_node_time(self, remote):
        self.execute_remote_cmd(remote, 'hwclock -s')
        self.execute_remote_cmd(remote, 'NTPD=$(find /etc/init.d/ -regex \''
                                        '/etc/init.d/\(ntp.?\|ntp-dev\)\');'
                                        '$NTPD stop && ntpd -dqg && $NTPD '
                                        'start')
        self.execute_remote_cmd(remote, 'hwclock -w')
        remote_date = remote.execute('date')['stdout']
        logger.info("Node time: %s" % remote_date)

    @retry(count=10, delay=60)
    @logwrap
    def sync_time_admin_node(self):
        logger.info("Sync time on revert for admin")
        remote = self.get_admin_remote()
        self.execute_remote_cmd(remote, 'hwclock -s')
        # Sync time using ntpd
        try:
            # If public NTP servers aren't accessible ntpdate will fail and
            # ntpd daemon shouldn't be restarted to avoid 'Server has gone
            # too long without sync' error while syncing time from slaves
            self.execute_remote_cmd(remote, "ntpdate -vu $(awk '/^server/ && "
                                            "$2 !~ /127.*/ {print $2}' "
                                            "/etc/ntp.conf)")
        except AssertionError as e:
            logger.warning('Error occurred while synchronizing time on master'
                           ': {0}'.format(e))
            raise
        else:
            self.execute_remote_cmd(remote, 'service ntpd stop && ntpd -dqg &&'
                                            ' service ntpd start')
            self.execute_remote_cmd(remote, 'hwclock -w')

        remote_date = remote.execute('date')['stdout']
        logger.info("Master node time: {0}".format(remote_date))

    def verify_network_configuration(self, node_name):
        checkers.verify_network_configuration(
            node=self.fuel_web.get_nailgun_node_by_name(node_name),
            remote=self.get_ssh_to_remote_by_name(node_name)
        )

    def wait_bootstrap(self):
        logger.info("Waiting while bootstrapping is in progress")
        log_path = "/var/log/puppet/bootstrap_admin_node.log"
        logger.info("Puppet timeout set in {0}".format(
            float(settings.PUPPET_TIMEOUT)))
        wait(
            lambda: not
            self.get_admin_remote().execute(
                "grep 'Fuel node deployment' '%s'" % log_path
            )['exit_code'],
            timeout=(float(settings.PUPPET_TIMEOUT))
        )
        result = self.get_admin_remote().execute("grep 'Fuel node deployment "
                                                 "complete' '%s'" % log_path
                                                 )['exit_code']
        if result != 0:
            raise Exception('Fuel node deployment failed.')

    def dhcrelay_check(self):
        admin_remote = self.get_admin_remote()
        out = admin_remote.execute("dhcpcheck discover "
                                   "--ifaces eth0 "
                                   "--repeat 3 "
                                   "--timeout 10")['stdout']

        assert_true(self.get_admin_node_ip() in "".join(out),
                    "dhcpcheck doesn't discover master ip")

    def run_nailgun_agent(self, remote):
        agent = remote.execute('/opt/nailgun/bin/agent')['exit_code']
        logger.info("Nailgun agent run with exit_code: %s" % agent)

    def get_fuel_settings(self, remote=None):
        if not remote:
            remote = self.get_admin_remote()
        cmd = 'cat {cfg_file}'.format(cfg_file=settings.FUEL_SETTINGS_YAML)
        result = remote.execute(cmd)
        if result['exit_code'] == 0:
            fuel_settings = yaml.load(''.join(result['stdout']))
        else:
            raise Exception('Can\'t output {cfg_file} file: {error}'.
                            format(cfg_file=settings.FUEL_SETTINGS_YAML,
                                   error=result['stderr']))
        return fuel_settings

    def admin_install_pkg(self, pkg_name):
        """Install a package <pkg_name> on the admin node"""
        admin_remote = self.get_admin_remote()
        remote_status = admin_remote.execute("rpm -q {0}'".format(pkg_name))
        if remote_status['exit_code'] == 0:
            logger.info("Package '{0}' already installed.".format(pkg_name))
        else:
            logger.info("Installing package '{0}' ...".format(pkg_name))
            remote_status = admin_remote.execute("yum -y install {0}"
                                                 .format(pkg_name))
            logger.info("Installation of the package '{0}' has been"
                        " completed with exit code {1}"
                        .format(pkg_name, remote_status['exit_code']))
        return remote_status['exit_code']

    def admin_run_service(self, service_name):
        """Start a service <service_name> on the admin node"""
        admin_remote = self.get_admin_remote()
        admin_remote.execute("service {0} start".format(service_name))
        remote_status = admin_remote.execute("service {0} status"
                                             .format(service_name))
        if any('running...' in status for status in remote_status['stdout']):
            logger.info("Service '{0}' is running".format(service_name))
        else:
            logger.info("Service '{0}' failed to start"
                        " with exit code {1} :\n{2}"
                        .format(service_name,
                                remote_status['exit_code'],
                                remote_status['stdout']))

    # Modifies a resolv.conf on the Fuel master node and returns
    # its original content.
    # * adds 'nameservers' at start of resolv.conf if merge=True
    # * replaces resolv.conf with 'nameservers' if merge=False
    def modify_resolv_conf(self, nameservers=[], merge=True):
        remote = self.get_admin_remote()
        resolv_conf = remote.execute('cat /etc/resolv.conf')
        assert_equal(0, resolv_conf['exit_code'], 'Executing "{0}" on the '
                     'admin node has failed with: {1}'
                     .format('cat /etc/resolv.conf', resolv_conf['stderr']))
        if merge:
            nameservers.extend(resolv_conf['stdout'])

        resolv_keys = ['search', 'domain', 'nameserver']
        resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers
                             if any(x in ns for x in resolv_keys))
        logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new))
        echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new)
        echo_result = remote.execute(echo_cmd)
        assert_equal(0, echo_result['exit_code'], 'Executing "{0}" on the '
                     'admin node has failed with: {1}'
                     .format(echo_cmd, echo_result['stderr']))
        return resolv_conf['stdout']

    @logwrap
    def execute_remote_cmd(self, remote, cmd, exit_code=0):
        result = remote.execute(cmd)
        assert_equal(result['exit_code'], exit_code,
                     'Failed to execute "{0}" on remote host: {1}'.
                     format(cmd, result))
        return result['stdout']

    @logwrap
    def describe_second_admin_interface(self):
        remote = self.get_admin_remote()
        second_admin_network = self.get_network(self.admin_net2).split('/')[0]
        second_admin_netmask = self.get_net_mask(self.admin_net2)
        second_admin_if = settings.INTERFACES.get(self.admin_net2)
        second_admin_ip = str(self.nodes().admin.
                              get_ip_address_by_network_name(self.admin_net2))
        logger.info(('Parameters for second admin interface configuration: '
                     'Network - {0}, Netmask - {1}, Interface - {2}, '
                     'IP Address - {3}').format(second_admin_network,
                                                second_admin_netmask,
                                                second_admin_if,
                                                second_admin_ip))
        add_second_admin_ip = ('DEVICE={0}\\n'
                               'ONBOOT=yes\\n'
                               'NM_CONTROLLED=no\\n'
                               'USERCTL=no\\n'
                               'PEERDNS=no\\n'
                               'BOOTPROTO=static\\n'
                               'IPADDR={1}\\n'
                               'NETMASK={2}\\n').format(second_admin_if,
                                                        second_admin_ip,
                                                        second_admin_netmask)
        cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};'
               'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format(
            add_second_admin_ip, second_admin_if, second_admin_ip)
        logger.debug('Trying to assign {0} IP to the {1} on master node...'.
                     format(second_admin_ip, second_admin_if))
        result = remote.execute(cmd)
        assert_equal(result['exit_code'], 0, ('Failed to assign second admin '
                     'IP address on master node: {0}').format(result))
        logger.debug('Done: {0}'.format(result['stdout']))
        multiple_networks_hacks.configure_second_admin_firewall(
            self,
            second_admin_network,
            second_admin_netmask)

    @logwrap
    def get_masternode_uuid(self):
        return self.postgres_actions.run_query(
            db='nailgun',
            query="select master_node_uid from master_node_settings limit 1;")
class CiBase(object):
    def __init__(self):
        self.manager = Manager()
        self._environment = None
        self.saved_environment_states = {}

    def _get_or_create(self):
        try:
            return self.manager.environment_get(self.env_name())
        except:
            self._environment = self.describe_environment()
            self._environment.define()
            return self._environment

    def get_state(self, name):
        if self.environment().has_snapshot(name):
            self.environment().revert(name)
            return True
        return False

    def environment(self):
        """
        :rtype : devops.models.Environment
        """
        self._environment = self._environment or self._get_or_create()
        return self._environment

    @abstractproperty
    def env_name(self):
        """
        :rtype : string
        """
        pass

    @abstractmethod
    def describe_environment(self):
        """
        :rtype : devops.models.Environment
        """
        pass

    @abstractproperty
    def node_roles(self):
        """
        :rtype : NodeRoles
        """
        pass

    def nodes(self):
        return Nodes(self.environment(), self.node_roles())

    # noinspection PyShadowingBuiltins
    def add_empty_volume(
        self, node, name, capacity=20 * 1024 * 1024 * 1024, device="disk", bus="virtio", format="qcow2"
    ):
        self.manager.node_attach_volume(
            node=node,
            volume=self.manager.volume_create(
                name=name, capacity=capacity, environment=self.environment(), format=format
            ),
            device=device,
            bus=bus,
        )

    def add_node(self, memory, name, boot=None):
        return self.manager.node_create(name=name, memory=memory, environment=self.environment(), boot=boot)

    def create_interfaces(self, networks, node):
        for network in networks:
            self.manager.interface_create(network, node=node)

    def describe_admin_node(self, name, networks, memory=1024):
        node = self.add_node(memory=memory, name=name, boot=["hd", "cdrom"])
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + "-system")
        self.add_empty_volume(
            node, name + "-iso", capacity=_get_file_size(ISO_PATH), format="raw", device="cdrom", bus="ide"
        )
        return node

    def describe_empty_node(self, name, networks, memory=1024):
        node = self.add_node(memory, name)
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + "-system")

        if USE_ALL_DISKS:
            self.add_empty_volume(node, name + "-cinder")
            self.add_empty_volume(node, name + "-swift")

        return node

    @abstractmethod
    def setup_environment(self):
        """
        :rtype : None
        """
        pass

    def get_empty_environment(self):
        if not (self.get_state(EMPTY_SNAPSHOT)):
            self.setup_environment()
            self.environment().snapshot(EMPTY_SNAPSHOT)

    def generate_state_hash(self, settings):
        return hashlib.md5(str(settings)).hexdigest()

    def revert_to_state(self, settings={}):
        state_hash = self.generate_state_hash(settings)

        if state_hash in self.saved_environment_states:
            # revert to matching state
            state = self.saved_environment_states[state_hash]
            if not (self.get_state(state["snapshot_name"])):
                return False
            self.environment().resume()
            return True

        return False

    def snapshot_state(self, name, settings={}):
        state_hash = self.generate_state_hash(settings)

        snapshot_name = "{0}_{1}".format(name.replace(" ", "_")[:17], state_hash)

        self.environment().suspend(verbose=False)
        self.environment().snapshot(name=snapshot_name, description=name, force=True)
        self.environment().resume(verbose=False)
        self.saved_environment_states[state_hash] = {
            "snapshot_name": snapshot_name,
            "cluster_name": name,
            "settings": settings,
        }

    def internal_virtual_ip(self):
        return str(IPNetwork(self.environment().network_by_name("internal").ip_network)[-2])

    def public_router(self):
        return str(IPNetwork(self.environment().network_by_name("public").ip_network)[1])

    def internal_router(self):
        return self._router("internal")

    def nat_router(self):
        return self._router("nat")

    def _router(self, router_name):
        return str(IPNetwork(self.environment().network_by_name(router_name).ip_network)[1])

    def get_host_node_ip(self):
        return self.internal_router()

    def internal_network(self):
        return str(IPNetwork(self.environment().network_by_name("internal").ip_network))

    def internal_net_mask(self):
        return str(IPNetwork(self.environment().network_by_name("internal").ip_network).netmask)

    def public_net_mask(self):
        return str(IPNetwork(self.environment().network_by_name("public").ip_network).netmask)

    def public_network(self):
        return str(IPNetwork(self.environment().network_by_name("public").ip_network))
Exemple #10
0
class EnvironmentModel(object):
    hostname = 'nailgun'
    domain = 'test.domain.local'
    installation_timeout = 1800
    deployment_timeout = 1800
    puppet_timeout = 2000
    nat_interface = ''  # INTERFACES.get('admin')
    admin_net = 'admin'

    def __init__(self, os_image=None):
        self._virtual_environment = None
        self._keys = None
        self.manager = Manager()
        self.os_image = os_image
        self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self)

    def _get_or_create(self):
        try:
            return self.manager.environment_get(self.env_name)
        except Exception:
            self._virtual_environment = self.describe_environment()
            self._virtual_environment.define()
            return self._virtual_environment

    def router(self, router_name=None):
        router_name = router_name or self.admin_net
        return str(
            IPNetwork(self.get_virtual_environment().network_by_name(
                router_name).ip_network)[1])

    @property
    def fuel_web(self):
        """FuelWebClient
        :rtype: FuelWebClient
        """
        return self._fuel_web

    @property
    def node_roles(self):
        return NodeRoles(admin_names=['admin'],
                         other_names=[
                             'slave-%02d' % x
                             for x in range(1, int(settings.NODES_COUNT))
                         ])

    @property
    def env_name(self):
        return settings.ENV_NAME

    def add_empty_volume(self,
                         node,
                         name,
                         capacity=settings.NODE_VOLUME_SIZE * 1024 * 1024 *
                         1024,
                         device='disk',
                         bus='virtio',
                         format='qcow2'):
        self.manager.node_attach_volume(
            node=node,
            volume=self.manager.volume_create(
                name=name,
                capacity=capacity,
                environment=self.get_virtual_environment(),
                format=format),
            device=device,
            bus=bus)

    def add_node(self, memory, name, vcpu=1, boot=None):
        return self.manager.node_create(
            name=name,
            memory=memory,
            vcpu=vcpu,
            environment=self.get_virtual_environment(),
            boot=boot)

    @logwrap
    def add_syslog_server(self, cluster_id, port=5514):
        self.fuel_web.add_syslog_server(cluster_id, self.get_host_node_ip(),
                                        port)

    def bootstrap_nodes(self, devops_nodes, timeout=600):
        """Lists registered nailgun nodes
        Start vms and wait until they are registered on nailgun.
        :rtype : List of registered nailgun nodes
        """
        self.dhcrelay_check()

        for node in devops_nodes:
            node.start()
            #TODO(aglarendil): LP#1317213 temporary sleep
            #remove after better fix is applied
            time.sleep(2)
        wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout)

        for node in self.nailgun_nodes(devops_nodes):
            self.sync_node_time(self.get_ssh_to_remote(node["ip"]))

        return self.nailgun_nodes(devops_nodes)

    def create_interfaces(self,
                          networks,
                          node,
                          model=settings.INTERFACE_MODEL):
        for network in networks:
            self.manager.interface_create(network, node=node, model=model)

    def describe_environment(self):
        """Environment
        :rtype : Environment
        """
        environment = self.manager.environment_create(self.env_name)
        networks = []
        for name in settings.INTERFACE_ORDER:
            ip_networks = [
                IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',')
            ]
            new_prefix = int(settings.POOLS.get(name)[1])
            pool = self.manager.create_network_pool(networks=ip_networks,
                                                    prefix=int(new_prefix))
            networks.append(
                self.manager.network_create(
                    name=name,
                    environment=environment,
                    pool=pool,
                    forward=settings.FORWARDING.get(name),
                    has_dhcp_server=settings.DHCP.get(name)))

        for name in self.node_roles.admin_names:
            self.describe_admin_node(name, networks)
        for name in self.node_roles.other_names:
            self.describe_empty_node(name, networks)
        return environment

    def devops_nodes_by_names(self, devops_node_names):
        return map(
            lambda name: self.get_virtual_environment().node_by_name(name),
            devops_node_names)

    @logwrap
    def describe_admin_node(self, name, networks):
        node = self.add_node(memory=settings.HARDWARE.get(
            "admin_node_memory", 1024),
                             vcpu=settings.HARDWARE.get("admin_node_cpu", 1),
                             name=name,
                             boot=['hd', 'cdrom'])
        self.create_interfaces(networks, node)

        if self.os_image is None:
            self.add_empty_volume(node, name + '-system')
            self.add_empty_volume(node,
                                  name + '-iso',
                                  capacity=_get_file_size(settings.ISO_PATH),
                                  format='raw',
                                  device='cdrom',
                                  bus='ide')
        else:
            volume = self.manager.volume_get_predefined(self.os_image)
            vol_child = self.manager.volume_create_child(
                name=name + '-system',
                backing_store=volume,
                environment=self.get_virtual_environment())
            self.manager.node_attach_volume(node=node, volume=vol_child)
        return node

    def describe_empty_node(self, name, networks):
        node = self.add_node(name=name,
                             memory=settings.HARDWARE.get(
                                 "slave_node_memory", 1024),
                             vcpu=settings.HARDWARE.get("slave_node_cpu", 1))
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + '-system')

        if settings.USE_ALL_DISKS:
            self.add_empty_volume(node, name + '-cinder')
            self.add_empty_volume(node, name + '-swift')

        return node

    @logwrap
    def get_admin_remote(self):
        """SSH to admin node
        :rtype : SSHClient
        """
        return self.nodes().admin.remote(self.admin_net,
                                         login='******',
                                         password='******')

    @logwrap
    def get_admin_node_ip(self):
        return str(self.nodes().admin.get_ip_address_by_network_name(
            self.admin_net))

    @logwrap
    def get_ebtables(self, cluster_id, devops_nodes):
        return Ebtables(self.get_target_devs(devops_nodes),
                        self.fuel_web.client.get_cluster_vlans(cluster_id))

    def get_host_node_ip(self):
        return self.router()

    def get_keys(self, node):
        params = {
            'ip': node.get_ip_address_by_network_name(self.admin_net),
            'mask': self.get_net_mask(self.admin_net),
            'gw': self.router(),
            'hostname': '.'.join((self.hostname, self.domain)),
            'nat_interface': self.nat_interface,
            'dns1': settings.DNS
        }
        keys = ("<Esc><Enter>\n"
                "<Wait>\n"
                "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n"
                " ip=%(ip)s\n"
                " netmask=%(mask)s\n"
                " gw=%(gw)s\n"
                " dns1=%(dns1)s\n"
                " hostname=%(hostname)s\n"
                " dhcp_interface=%(nat_interface)s\n"
                " <Enter>\n") % params
        return keys

    @logwrap
    def get_private_keys(self, force=False):
        if force or self._keys is None:
            self._keys = []
            for key_string in [
                    '/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa'
            ]:
                with self.get_admin_remote().open(key_string) as f:
                    self._keys.append(RSAKey.from_private_key(f))
        return self._keys

    @logwrap
    def get_ssh_to_remote(self, ip):
        return SSHClient(ip,
                         username='******',
                         password='******',
                         private_keys=self.get_private_keys())

    @logwrap
    def get_ssh_to_remote_by_name(self, node_name):
        return self.get_ssh_to_remote(
            self.fuel_web.get_nailgun_node_by_devops_node(
                self.get_virtual_environment().node_by_name(node_name))['ip'])

    def get_target_devs(self, devops_nodes):
        return [
            interface.target_dev for interface in [
                val for var in map(lambda node: node.interfaces, devops_nodes)
                for val in var
            ]
        ]

    def get_virtual_environment(self):
        """Returns virtual environment
        :rtype : devops.models.Environment
        """
        if self._virtual_environment is None:
            self._virtual_environment = self._get_or_create()
        return self._virtual_environment

    def get_network(self, net_name):
        return str(
            IPNetwork(self.get_virtual_environment().network_by_name(
                net_name).ip_network))

    def get_net_mask(self, net_name):
        return str(
            IPNetwork(self.get_virtual_environment().network_by_name(
                net_name).ip_network).netmask)

    def make_snapshot(self, snapshot_name, description="", is_make=False):
        if settings.MAKE_SNAPSHOT or is_make:
            self.get_virtual_environment().suspend(verbose=False)
            self.get_virtual_environment().snapshot(snapshot_name, force=True)
            revert_info(snapshot_name, description)

    def nailgun_nodes(self, devops_nodes):
        return map(
            lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node),
            devops_nodes)

    def nodes(self):
        return Nodes(self.get_virtual_environment(), self.node_roles)

    def revert_snapshot(self, name):
        if self.get_virtual_environment().has_snapshot(name):
            logger.info('We have snapshot with such name %s' % name)

            self.get_virtual_environment().revert(name)
            logger.info('Starting snapshot reverting ....')

            self.get_virtual_environment().resume()
            logger.info('Starting snapshot resuming ...')

            self.nodes().admin. await (self.admin_net, timeout=10 * 60)

            self.sync_time_admin_node()

            for node in self.nodes().slaves:
                if not node.driver.node_active(node):
                    continue
                try:
                    logger.info("Sync time on revert for node %s" % node.name)
                    self.sync_node_time(
                        self.get_ssh_to_remote_by_name(node.name))
                except Exception as e:
                    logger.warn('Paramiko exception catched while'
                                ' trying to run ntpdate: %s' % e)

                self.run_nailgun_agent(
                    self.get_ssh_to_remote_by_name(node.name))
            return True
        return False

    def setup_environment(self):
        # start admin node
        admin = self.nodes().admin
        admin.disk_devices.get(device='cdrom').volume.upload(settings.ISO_PATH)
        self.get_virtual_environment().start(self.nodes().admins)
        # update network parameters at boot screen
        time.sleep(float(settings.ADMIN_NODE_SETUP_TIMEOUT))
        admin.send_keys(self.get_keys(admin))
        # wait while installation complete
        admin. await (self.admin_net, timeout=10 * 60)
        self.wait_bootstrap()
        time.sleep(10)
        self.sync_time_admin_node()

    @retry()
    @logwrap
    def sync_node_time(self, remote):
        remote.execute('hwclock --hctosys')
        ntpd_exit = remote.execute("ntpdate -u $(egrep '^server' "
                                   "/etc/ntp.conf | sed '/^#/d' | "
                                   "awk '{print $2}')")['exit_code']
        assert_equal(0, ntpd_exit)
        remote.execute('hwclock -w')
        remote_date = remote.execute('date')['stdout']
        logger.info("Node time: %s" % remote_date)

    def sync_time_admin_node(self):
        logger.info("Sync time on revert for admin")
        self.sync_node_time(self.get_admin_remote())

    def verify_node_service_list(self, node_name, smiles_count):
        remote = self.get_ssh_to_remote_by_name(node_name)
        checkers.verify_service_list(remote, smiles_count)

    def verify_network_configuration(self, node_name):
        checkers.verify_network_configuration(
            node=self.fuel_web.get_nailgun_node_by_name(node_name),
            remote=self.get_ssh_to_remote_by_name(node_name))

    def wait_bootstrap(self):
        logger.info("Waiting while bootstrapping is in progress")
        log_path = "/var/log/puppet/bootstrap_admin_node.log"
        logger.info("Puppet timeout set in {0}".format(
            float(settings.PUPPET_TIMEOUT)))
        wait(lambda: not self.get_admin_remote().execute(
            "grep 'Fuel node deployment complete' '%s'" % log_path)['exit_code'
                                                                    ],
             timeout=(float(settings.PUPPET_TIMEOUT)))

    def dhcrelay_check(self):
        admin_remote = self.get_admin_remote()
        out = admin_remote.execute("dhcpcheck discover "
                                   "--ifaces eth0 "
                                   "--repeat 3 "
                                   "--timeout 10")['stdout']
        master_ip = filter(lambda x: self.get_admin_node_ip() in x, out)
        logger.info("dhcpcheck discover: %s" % master_ip)
        assert_equal(len(master_ip), 1)

    def run_nailgun_agent(self, remote):
        agent = remote.execute('/opt/nailgun/bin/agent')['exit_code']
        logger.info("Nailgun agent run with exit_code: %s" % agent)
Exemple #11
0
class EnvironmentModel(object):
    hostname = "nailgun"
    domain = "test.domain.local"
    installation_timeout = 1800
    deployment_timeout = 1800
    puppet_timeout = 1000
    nat_interface = ""  # INTERFACES.get('admin')
    admin_net = "admin"

    def __init__(self):
        self._virtual_environment = None
        self._keys = None
        self.manager = Manager()
        self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self)

    def _get_or_create(self):
        try:
            return self.manager.environment_get(self.env_name)
        except:
            self._virtual_environment = self.describe_environment()
            self._virtual_environment.define()
            return self._virtual_environment

    def router(self, router_name=None):
        router_name = router_name or self.admin_net
        return str(IPNetwork(self.get_virtual_environment().network_by_name(router_name).ip_network)[1])

    @property
    def fuel_web(self):
        """
        :rtype: FuelWebClient
        """
        return self._fuel_web

    @property
    def node_roles(self):
        return NodeRoles(admin_names=["admin"], other_names=["slave-%02d" % x for x in range(1, 10)])

    @property
    def env_name(self):
        return ENV_NAME

    def add_empty_volume(
        self, node, name, capacity=NODE_VOLUME_SIZE * 1024 * 1024 * 1024, device="disk", bus="virtio", format="qcow2"
    ):
        self.manager.node_attach_volume(
            node=node,
            volume=self.manager.volume_create(
                name=name, capacity=capacity, environment=self.get_virtual_environment(), format=format
            ),
            device=device,
            bus=bus,
        )

    def add_node(self, memory, name, vcpu=1, boot=None):
        return self.manager.node_create(
            name=name, memory=memory, vcpu=vcpu, environment=self.get_virtual_environment(), boot=boot
        )

    @logwrap
    def add_syslog_server(self, cluster_id, port=5514):
        self.fuel_web.add_syslog_server(cluster_id, self.get_host_node_ip(), port)

    def bootstrap_nodes(self, devops_nodes, timeout=600):
        """
        Start vms and wait they are registered on nailgun.
        :rtype : List of registered nailgun nodes
        """
        for node in devops_nodes:
            node.start()
        wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout)

        for node in self.nailgun_nodes(devops_nodes):
            self.sync_node_time(self.get_ssh_to_remote(node["ip"]))

        return self.nailgun_nodes(devops_nodes)

    def create_interfaces(self, networks, node, model=INTERFACE_MODEL):
        for network in networks:
            self.manager.interface_create(network, node=node, model=model)

    def describe_environment(self):
        """
        :rtype : Environment
        """
        environment = self.manager.environment_create(self.env_name)
        networks = []
        for name in INTERFACE_ORDER:
            ip_networks = [IPNetwork(x) for x in POOLS.get(name)[0].split(",")]
            new_prefix = int(POOLS.get(name)[1])
            pool = self.manager.create_network_pool(networks=ip_networks, prefix=int(new_prefix))
            networks.append(
                self.manager.network_create(
                    name=name,
                    environment=environment,
                    pool=pool,
                    forward=FORWARDING.get(name),
                    has_dhcp_server=DHCP.get(name),
                )
            )

        for name in self.node_roles.admin_names:
            self.describe_admin_node(name, networks)
        for name in self.node_roles.other_names:
            self.describe_empty_node(name, networks)
        return environment

    def devops_nodes_by_names(self, devops_node_names):
        return map(lambda name: self.get_virtual_environment().node_by_name(name), devops_node_names)

    @logwrap
    def describe_admin_node(self, name, networks):
        node = self.add_node(
            memory=HARDWARE.get("admin_node_memory", 1024),
            vcpu=HARDWARE.get("admin_node_cpu", 1),
            name=name,
            boot=["hd", "cdrom"],
        )
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + "-system")
        self.add_empty_volume(
            node, name + "-iso", capacity=_get_file_size(ISO_PATH), format="raw", device="cdrom", bus="ide"
        )
        return node

    def describe_empty_node(self, name, networks):
        node = self.add_node(
            name=name, memory=HARDWARE.get("slave_node_memory", 1024), vcpu=HARDWARE.get("slave_node_cpu", 1)
        )
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + "-system")

        if USE_ALL_DISKS:
            self.add_empty_volume(node, name + "-cinder")
            self.add_empty_volume(node, name + "-swift")

        return node

    @logwrap
    def get_admin_remote(self):
        """
        :rtype : SSHClient
        """
        return self.nodes().admin.remote(self.admin_net, login="******", password="******")

    @logwrap
    def get_admin_node_ip(self):
        return str(self.nodes().admin.get_ip_address_by_network_name(self.admin_net))

    @logwrap
    def get_ebtables(self, cluster_id, devops_nodes):
        return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id))

    def get_host_node_ip(self):
        return self.router()

    def get_keys(self, node):
        params = {
            "ip": node.get_ip_address_by_network_name(self.admin_net),
            "mask": self.get_net_mask(self.admin_net),
            "gw": self.router(),
            "hostname": ".".join((self.hostname, self.domain)),
            "nat_interface": self.nat_interface,
            "dns1": DNS,
        }
        keys = (
            "<Esc><Enter>\n"
            "<Wait>\n"
            "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n"
            " ip=%(ip)s\n"
            " netmask=%(mask)s\n"
            " gw=%(gw)s\n"
            " dns1=%(dns1)s\n"
            " hostname=%(hostname)s\n"
            " dhcp_interface=%(nat_interface)s\n"
            " <Enter>\n"
        ) % params
        return keys

    @logwrap
    def get_private_keys(self, force=False):
        if force or self._keys is None:
            self._keys = []
            for key_string in ["/root/.ssh/id_rsa", "/root/.ssh/bootstrap.rsa"]:
                with self.get_admin_remote().open(key_string) as f:
                    self._keys.append(RSAKey.from_private_key(f))
        return self._keys

    @logwrap
    def get_ssh_to_remote(self, ip):
        return SSHClient(ip, username="******", password="******", private_keys=self.get_private_keys())

    @logwrap
    def get_ssh_to_remote_by_name(self, node_name):
        return self.get_ssh_to_remote(
            self.fuel_web.get_nailgun_node_by_devops_node(self.get_virtual_environment().node_by_name(node_name))["ip"]
        )

    def get_target_devs(self, devops_nodes):
        return [
            interface.target_dev
            for interface in [val for var in map(lambda node: node.interfaces, devops_nodes) for val in var]
        ]

    def get_virtual_environment(self):
        """
        :rtype : devops.models.Environment
        """
        if self._virtual_environment is None:
            self._virtual_environment = self._get_or_create()
        return self._virtual_environment

    def get_network(self, net_name):
        return str(IPNetwork(self.get_virtual_environment().network_by_name(net_name).ip_network))

    def get_net_mask(self, net_name):
        return str(IPNetwork(self.get_virtual_environment().network_by_name(net_name).ip_network).netmask)

    def make_snapshot(self, snapshot_name):
        self.get_virtual_environment().suspend(verbose=False)
        self.get_virtual_environment().snapshot(snapshot_name, force=True)

    def nailgun_nodes(self, devops_nodes):
        return map(lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node), devops_nodes)

    def nodes(self):
        return Nodes(self.get_virtual_environment(), self.node_roles)

    def revert_snapshot(self, name):
        if self.get_virtual_environment().has_snapshot(name):
            logging.info("We have snapshot with such name %s" % name)

            self.get_virtual_environment().revert(name)
            logging.info("Starting snapshot reverting ....")

            self.get_virtual_environment().resume()
            logging.info("Starting snapshot resuming ...")

            time.sleep(10)

            list = self.nodes().slaves
            for node in list:
                try:
                    self.sync_node_time(self.get_ssh_to_remote(node.get_ip_address_by_network_name(self.admin_net)))
                except Exception, e:
                    logging.warn("Exception got: %s" % e)
                    traceback.print_exc()
            return True
        return False
Exemple #12
0
 def run(self):
     Manager().network_create(str(random.randint(1, 5000)))
Exemple #13
0
 def __init__(self, name=name, base_image=None):
     self.manager = Manager()
     self.name = name
     self.base_image = base_image
     self.environment = self._get_or_create()
Exemple #14
0
class Environment(object):
    capacity = 20 * 1024 * 1024 * 1024
    boot = ['hd', 'cdrom']
    login = "******"
    password = "******"
    name = 'fuel'

    def __init__(self, name=name, base_image=None):
        self.manager = Manager()
        self.name = name
        self.base_image = base_image
        self.environment = self._get_or_create()

    def _get_or_create(self):
        try:
            return self.manager.environment_get(self.name)
        except:
            self.environment = self._create()
            self.environment.define()
            return self.environment

    def nodes(self):
        return Nodes(self.environment, self._node_roles())

    def get_empty_state(self):
        if self.environment.has_snapshot(EMPTY_SNAPSHOT):
            self.environment.revert(EMPTY_SNAPSHOT)
            return True

        return None

    def get_env(self):
        return self.environment

    def add_empty_volume(self, node, name, capacity=capacity, device='disk', bus='virtio', format='qcow2'):
        self.manager.node_attach_volume(node=node,
                                        volume=self.manager.volume_create(name=name,
                                                                          capacity=capacity,
                                                                          environment=self.environment,
                                                                          format=format),
                                        device=device,
                                        bus=bus)

    def add_node(self, name, memory, boot=None):
        return self.manager.node_create(name=name,
                                        memory=memory,
                                        environment=self.environment,
                                        boot=boot)

    def create_interfaces(self, node, networks):
        for network in networks:
            self.manager.interface_create(network, node=node)

    def describe_admin_node(self, name, networks, memory=DEFAULT_RAM_SIZE, boot=boot):
        node = self.add_node(memory=memory, name=name, boot=boot)
        self.create_interfaces(node, networks)

        if self.base_image is None:
            self.add_empty_volume(node, name + '-system')
            self.add_empty_volume(node, name + '-iso', capacity=_get_file_size(ISO_PATH), format='raw', device='cdrom', bus='ide')
        else:
            volume = self.manager.volume_get_predefined(self.base_image)
            v = self.manager.volume_create_child(name + '-system', backing_store=volume, environment=self.environment)
            self.manager.node_attach_volume(node=node, volume=v)

        return node

    def describe_node(self, name, networks, memory=DEFAULT_RAM_SIZE):
        node = self.add_node(name, memory)
        self.create_interfaces(node, networks)
        self.add_empty_volume(node, name + '-system')
        # self.add_empty_volume(node, name + '-cinder')
        #self.add_empty_volume(node, name + '-swift')

        return node

    def _node_roles(self):
        controllers = ['fuel-controller-%02d' % x for x in range(1, 1 + COUNT_NODES[DEPLOYMENT_MODE]['CONTROLLERS'])]
        computes = ['fuel-compute-%02d' % x for x in range(1, 1 + COUNT_NODES[DEPLOYMENT_MODE]['COMPUTES'])]
        storages = ['fuel-swift-%02d' % x for x in range(1, 1 + COUNT_NODES[DEPLOYMENT_MODE]['STORAGES'])]
        proxies = ['fuel-swift-proxy-%02d' % x for x in range(1, 1 + COUNT_NODES[DEPLOYMENT_MODE]['PROXIES'])]
        quantums = ['fuel-quantum-%02d' % x for x in range(1, 1 + COUNT_NODES[DEPLOYMENT_MODE]['QUANTUMS'])]

        return NodeRoles(admin_names=['master'],
                         other_names=controllers + computes + storages + proxies + quantums
        )

    def _create(self):
        self.environment = self.manager.environment_create(self.name)
        networks = []
        for name in INTERFACE_ORDER:
            ip_networks = [IPNetwork(x) for x in POOLS.get(name)[0].split(',')]
            new_prefix = int(POOLS.get(name)[1])
            pool = self.manager.create_network_pool(networks=ip_networks, prefix=int(new_prefix))
            networks.append(self.manager.network_create(name=name,
                                                        environment=self.environment,
                                                        pool=pool,
                                                        forward=FORWARDING.get(name),
                                                        has_dhcp_server=DHCP.get(name)))

        for name in self._node_roles().admin_names:
            self.describe_admin_node(name, networks)

        for name in self._node_roles().other_names:
            self.describe_node(name, networks, memory=COMPUTE_RAM_SIZE)

        return self.environment

    def start(self):
        admin = self.nodes().admin
        admin.disk_devices.get(device='cdrom').volume.upload(ISO_PATH)
        self.environment.start([admin])
        #self._environment.snapshot(EMPTY_SNAPSHOT)

    def start_all(self):
        self.environment.start(self.nodes())

    def get_netmask_by_netname(self, netname):
        return str(IPNetwork(self.environment.network_by_name(netname).ip_network).netmask)

    def get_router_by_netname(self, netname):
        return str(IPNetwork(self.environment.network_by_name(netname).ip_network)[1])

    def internal_virtual_ip(self):
        return self._get_virtual_ip_by_netname('internal')

    def public_virtual_ip(self):
        return self._get_virtual_ip_by_netname('public')

    def _get_virtual_ip_by_netname(self, netname):
        return str(IPNetwork(self.environment.network_by_name(netname).ip_network)[-2])

    def erase(self):
        self.environment.erase()

    def get_master_ssh(self):
        """
        :rtype : SSHClient
        """
        return self.nodes().admin.remote('internal', login=self.login, password=self.password)

    def get_master_ip(self, net_name='internal'):
        return str(self.nodes().admin.get_ip_address_by_network_name(net_name))

    def get_volume_capacity(self, node, volume_name='system'):
        for d in node.disk_devices:
            if volume_name in d.volume.get_path():
                return d.volume.get_capacity()

        return None


    # def public_router(self):
    #     return str(IPNetwork(self.get().network_by_name('public').ip_network)[1])
    #
    # def internal_router(self):
    #     return str(IPNetwork(self.get().network_by_name('internal').ip_network)[1])
    #
    # def internal_network(self):
    #     return str(IPNetwork(self.get().network_by_name('internal').ip_network))
    #
    # def public_network(self):
    #     return str(IPNetwork(self.get().network_by_name('public').ip_network))
    #
    def internal_net_mask(self):
        return str(IPNetwork(self.environment.network_by_name('internal').ip_network).netmask)
Exemple #15
0
class CiBase(object):
    def __init__(self):
        self.manager = Manager()
        self._environment = None

    def _get_or_create(self):
        try:
            return self.manager.environment_get(self.env_name())
        except:
            self._environment = self.describe_environment()
            self._environment.define()
            return self._environment

    def get_empty_state(self):
        if self.environment().has_snapshot(EMPTY_SNAPSHOT):
            self.environment().revert(EMPTY_SNAPSHOT)
        else:
            self.setup_environment()

    def environment(self):
        """
        :rtype : devops.models.Environment
        """
        self._environment = self._environment or self._get_or_create()
        return self._environment

    @abstractproperty
    def env_name(self):
        """
        :rtype : string
        """
        pass

    @abstractmethod
    def describe_environment(self):
        """
        :rtype : devops.models.Environment
        """
        pass

    @abstractproperty
    def node_roles(self):
        """
        :rtype : NodeRoles
        """
        pass

    def nodes(self):
        return Nodes(self.environment(), self.node_roles())

    # noinspection PyShadowingBuiltins
    def add_empty_volume(self,
                         node,
                         name,
                         capacity=20 * 1024 * 1024 * 1024,
                         device='disk',
                         bus='virtio',
                         format='qcow2'):
        self.manager.node_attach_volume(node=node,
                                        volume=self.manager.volume_create(
                                            name=name,
                                            capacity=capacity,
                                            environment=self.environment(),
                                            format=format),
                                        device=device,
                                        bus=bus)

    def add_node(self, memory, name, boot=None):
        return self.manager.node_create(name=name,
                                        memory=memory,
                                        environment=self.environment(),
                                        boot=boot)

    def create_interfaces(self, networks, node):
        for network in networks:
            if network.name == 'internal':
                self.manager.interface_create(network, node=node)
                self.manager.interface_create(network, node=node)
            self.manager.interface_create(network, node=node)

    def describe_admin_node(self, name, networks, memory=1024):
        node = self.add_node(memory=memory, name=name, boot=['hd', 'cdrom'])
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + '-system')
        self.add_empty_volume(node,
                              name + '-iso',
                              capacity=_get_file_size(ISO_PATH),
                              format='raw',
                              device='cdrom',
                              bus='ide')
        return node

    def describe_empty_node(self, name, networks, memory=1024):
        node = self.add_node(memory, name)
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + '-system')
        self.add_empty_volume(node, name + '-cinder')
        self.add_empty_volume(node, name + '-swift')
        return node

    @abstractmethod
    def setup_environment(self):
        """
        :rtype : None
        """
        pass

    def internal_virtual_ip(self):
        return str(
            IPNetwork(
                self.environment().network_by_name('internal').ip_network)[-2])

    def public_router(self):
        return str(
            IPNetwork(
                self.environment().network_by_name('public').ip_network)[1])

    def internal_router(self):
        return str(
            IPNetwork(
                self.environment().network_by_name('internal').ip_network)[1])

    def get_host_node_ip(self):
        return self.internal_router()

    def internal_network(self):
        return str(
            IPNetwork(
                self.environment().network_by_name('internal').ip_network))

    def internal_net_mask(self):
        return str(
            IPNetwork(self.environment().network_by_name(
                'internal').ip_network).netmask)

    def public_net_mask(self):
        return str(
            IPNetwork(self.environment().network_by_name(
                'public').ip_network).netmask)

    def public_network(self):
        return str(
            IPNetwork(self.environment().network_by_name('public').ip_network))
Exemple #16
0
class Shell(object):
    def __init__(self):
        super(Shell, self).__init__()
        self.params = self.get_params()
        self.manager = Manager()

    def execute(self):
        self.commands.get(self.params.command)(self)

    def do_list(self):
        env_list = self.manager.environment_list().values('name')
        for env in env_list:
            print env['name']

        return env_list

    def node_dict(self, node):
        return {'name': node.name,
                'vnc': node.get_vnc_port()}

    def do_show(self):
        environment = self.manager.environment_get(self.params.name)

        print '%5s %25s' % ("VNC", "NODE-NAME")
        for item in map(lambda x: self.node_dict(x), environment.nodes):
            print '%5s %25s' % (item['vnc'], item['name'])

    def do_erase(self):
        self.manager.environment_get(self.params.name).erase()

    def do_start(self):
        self.manager.environment_get(self.params.name).start()

    def do_destroy(self):
        self.manager.environment_get(self.params.name).destroy(verbose=False)

    def do_suspend(self):
        self.manager.environment_get(self.params.name).suspend(verbose=False)

    def do_resume(self):
        self.manager.environment_get(self.params.name).resume(verbose=False)

    def do_revert(self):
        self.manager.environment_get(self.params.name).revert(
            self.params.snapshot_name)

    def do_snapshot(self):
        self.manager.environment_get(self.params.name).snapshot(
            self.params.snapshot_name)

    def do_synchronize(self):
        self.manager.synchronize_environments()

    commands = {
        'list': do_list,
        'show': do_show,
        'erase': do_erase,
        'start': do_start,
        'destroy': do_destroy,
        'suspend': do_suspend,
        'resume': do_resume,
        'revert': do_revert,
        'snapshot': do_snapshot,
        'sync': do_synchronize
    }

    def get_params(self):
        name_parser = argparse.ArgumentParser(add_help=False)
        name_parser.add_argument('name', help='environment name',
                                 default=environ.get('ENV_NAME'))
        snapshot_name_parser = argparse.ArgumentParser(add_help=False)
        snapshot_name_parser.add_argument('--snapshot-name',
                                          help='snapshot name',
                                          default=environ.get('SNAPSHOT_NAME'))
        parser = argparse.ArgumentParser(
            description="Manage virtual environments")
        subparsers = parser.add_subparsers(help='commands', dest='command')
        subparsers.add_parser('list')
        subparsers.add_parser('show', parents=[name_parser])
        subparsers.add_parser('erase', parents=[name_parser])
        subparsers.add_parser('start', parents=[name_parser])
        subparsers.add_parser('destroy', parents=[name_parser])
        subparsers.add_parser('suspend', parents=[name_parser])
        subparsers.add_parser('resume', parents=[name_parser])
        subparsers.add_parser('revert',
                              parents=[name_parser, snapshot_name_parser])
        subparsers.add_parser('snapshot',
                              parents=[name_parser, snapshot_name_parser])
        subparsers.add_parser('sync')
        return parser.parse_args()
Exemple #17
0
class CiBase(object):
    """
    Base class for creating environment -- deploy openstack in various mode.
    """
    def __init__(self):
        """
        Constructor. Define environment and manager for manipulation with it.
        """
        self._environment = None
        self.manager = Manager()

    def get_or_create(self):
        """
        Get prepared environment or create it.
        """
        try:
            return self.manager.environment_get(self.env_name())
        except:
            self._environment = self.describe_environment()
            self._environment.define()
            return self._environment

    def get_empty_state(self):
        """
        Get 'empty' snapshot for virtual machines.
        """
        if self.environment().has_snapshot(EMPTY_SNAPSHOT):
            self.environment().revert(EMPTY_SNAPSHOT)
        else:
            self.setup_environment()

    def environment(self):
        """
        :rtype : devops.models.Environment
        """
        self._environment = self._environment or self.get_or_create()
        return self._environment

    @abstractproperty
    def env_name(self):
        """
        :rtype : string
        """
        pass

    @abstractmethod
    def define(self):
        """
        :rtype : devops.models.Environment
        """
        pass

    @abstractmethod
    def describe_environment(self):
        """
        :rtype : devops.models.Environment
        """
        pass

    @abstractproperty
    def node_roles(self):
        """
        :rtype : NodeRoles
        """
        pass

    def add_empty_volume(self, node, name, capacity=20 * 1024 * 1024 * 1024, format="qcow2", device="disk", bus='virtio'):
        """
        Attach a empty volume to virtual machine.
        """
        self.manager.node_attach_volume(node=node,
                                        device=device,
                                        bus=bus,
                                        volume=self.manager.volume_create(
                                        name=name,
                                        capacity=capacity,
                                        format=format,
                                        environment=self.environment()))

    def add_node(self, memory, name, boot=None):
        """
        Create a virtual machine in environment.
        """
        return self.manager.node_create(name=name,
                                        memory=memory,
                                        environment=self.environment())

    def describe_master_node(self, name, networks, memory=DEFAULT_RAM_SIZE):
        """
        Define master node.
        """
        node = self.add_node(memory, name, boot=['cdrom', 'hd'])
        for network in networks:
            self.manager.interface_create(network, node=node)
        self.add_empty_volume(node, name + '-system')
        self.add_empty_volume(node, name + '-iso', capacity=_get_file_size(ISO_IMAGE), format='raw', device='cdrom', bus='ide')
        return node

    def describe_empty_node(self, name, networks, memory=DEFAULT_RAM_SIZE):
        """
        Define node with default settings.
        """
        node = self.add_node(memory, name)
        for network in networks:
            self.manager.interface_create(network, node=node)
        self.add_empty_volume(node, name + '-system')
        self.add_empty_volume(node, name + '-cinder')
        return node

    def nodes(self):
        """
        Get all virtual machines of environment.
        """
        return Nodes(self.environment(), self.node_roles())

    def add_nodes_to_hosts(self, remote, nodes):
        """
        Append to /etc/hosts list of ip addresses by internal interface for all vms.
        """
        for node in nodes:
            add_to_hosts(remote,
                         node.get_ip_address_by_network_name('internal'),
                         node.name,
                         node.name + '.localdomain')

    def setup_master_node(self, master_remote, nodes):
        """
        Make the basic settings for master vm(node).
        """
        setup_puppet_master(master_remote)
        add_nmap(master_remote)
        switch_off_ip_tables(master_remote)
        self.add_nodes_to_hosts(master_remote, nodes)

    def setup_agent_nodes(self, nodes):
        """
        Make the basic settings for default vm(node).
        """
        agent_config = load(root('fuel-test', 'config', 'puppet.agent.config'))
        for node in nodes:
            if node.name != 'master':
                remote = node.remote('public', login='******', password='******')
                self.add_nodes_to_hosts(remote, self.environment().nodes)
                setup_puppet_client(remote)
                write_config(remote, '/etc/puppet/puppet.conf', agent_config)
                request_cerificate(remote)

    def rename_nodes(self, nodes):
        """
        Change host name for nodes.
        """
        for node in nodes:
            remote = node.remote('public', login='******', password='******')
            change_host_name(remote,
                             node.name,
                             node.name + '.localdomain')
            LOG.info("Renamed %s" % node.name)

    @abstractmethod
    def setup_environment(self):
        """
        :rtype : None
        """
        pass

    def internal_virtual_ip(self):
        """
        Get virtual ip in internal network.
        """
        return str(IPNetwork(self.environment().network_by_name('internal').ip_network)[-2])

    def floating_network(self):
        """
        Get floating subnet of public network.
        """
        prefix = IPNetwork(self.environment().network_by_name('public').ip_network).prefixlen
        return str(IPNetwork(self.environment().network_by_name('public').ip_network).subnet(new_prefix=prefix + 2)[-1])

    def public_virtual_ip(self):
        """
        Get virtual ip in public network.
        """
        prefix = IPNetwork(self.environment().network_by_name('public').ip_network).prefixlen
        return str(IPNetwork(self.environment().network_by_name('public').ip_network).subnet(new_prefix=prefix + 2)[-2][-1])

    def public_router(self):
        """
        Get route for public network.
        """
        return str(IPNetwork(self.environment().network_by_name('public').ip_network)[1])

    def internal_router(self):
        """
        Get route for internal network.
        """
        return str(IPNetwork(self.environment().network_by_name('internal').ip_network)[1])

    def fixed_network(self):
        """
        Get fixed subnet of private network.
        """
        return str(IPNetwork(self.environment().network_by_name('private').ip_network).subnet(
                new_prefix=27)[0])

    def internal_network(self):
        """
        Get internal network.
        """
        return str(IPNetwork(self.environment().network_by_name('internal').ip_network))

    def internal_net_mask(self):
        """
        Get internal netmask.
        """
        return str(IPNetwork(self.environment().network_by_name('internal').ip_network).netmask)

    def public_net_mask(self):
        """
        Get public netmask.
        """
        return str(IPNetwork(self.environment().network_by_name('public').ip_network).netmask)

    def public_network(self):
        """
        Get public network.
        """
        return str(IPNetwork(self.environment().network_by_name('public').ip_network))
Exemple #18
0
class EnvManager:
    """
    Class for create environment in puppet modules testing.
    """

    env_name = os.environ.get("ENV_NAME", "puppet-integration")
    env_node_name = "node"
    env_net_public = NET_PUBLIC
    env_net_internal = NET_INTERNAL
    env_net_private = NET_PRIVATE
    env_vol = "vol"
    login = "******"
    password = "******"

    def __init__(self, base_image=None):
        """
        Constructor for create environment.
        """
        self.manager = Manager()
        self.base_image = base_image or BASE_IMAGE
        self.environment = self.create_env()

    def create_env(self):
        try:
            return self.manager.environment_get(self.env_name)
        except:
            return self._define_env()

    def _define_env(self):
        """
        Create environment with default settings.
        """
        self.environment = self.manager.environment_create(self.env_name)

        internal = self.manager.network_create(environment=self.environment, name=self.env_net_internal, pool=None)
        external = self.manager.network_create(environment=self.environment, name=self.env_net_public, pool=None)
        private = self.manager.network_create(environment=self.environment, name=self.env_net_private, pool=None)

        node = self.manager.node_create(name=self.env_node_name, environment=self.environment)

        self.manager.interface_create(node=node, network=internal)
        self.manager.interface_create(node=node, network=external)
        self.manager.interface_create(node=node, network=private)

        volume = self.manager.volume_get_predefined(self.base_image)

        v3 = self.manager.volume_create_child(self.env_vol, backing_store=volume, environment=self.environment)

        self.manager.node_attach_volume(node=node, volume=v3)

        self.environment.define()

        self.environment.start()

        return self.environment

    def _get_public_ip(self):
        return self.environment.node_by_name(self.env_node_name).get_ip_address_by_network_name(self.env_net_public)

    def _ssh(self):
        return ssh(self._get_public_ip(), username=self.login, password=self.password).sudo.ssh

    def remote(self):
        """
        Return remote access to node by name with default login/password.
        """
        return self._ssh()

    def snapshot_exist(self, snap_name="before_test"):
        return self.environment.has_snapshot(name=snap_name)

    def create_snapshot_env(self, snap_name="", description="", force=True):
        """
        Create snapshot for environment.
        """
        self.environment.snapshot(name=snap_name, description=description, force=force)

    def revert_snapshot_env(self, snap_name="", destroy=True):
        """
        Revert environment to snapshot by name.
        """
        self.environment.revert(name=snap_name, destroy=destroy)

    def erase_env(self):
        """
        Erase environment.
        """
        self.environment.erase()

    def execute_cmd(self, command, debug=True):
        """
        Execute command on node.
        """
        return self.remote().execute(command, verbose=debug)["exit_code"]

    def upload_files(self, source, dest):
        """
        Upload file(s) to node.
        """
        self.remote().upload(source, dest)

    def upload_modules(self, local_dir, remote_dir="/etc/puppet/modules/"):
        """
        Upload puppet modules.
        """
        upload_recipes(remote=self.remote(), local_dir=local_dir, remote_dir=remote_dir)

    def await(self, timeout=1200):
Exemple #19
0
class Shell(object):
    def __init__(self):
        super(Shell, self).__init__()
        self.params = self.get_params()
        self.manager = Manager()

    def execute(self):
        self.commands.get(self.params.command)(self)

    def do_list(self):
        print self.manager.environment_list().values('name')

    def node_dict(self, node):
        return {
            'name': node.name,
            'vnc': node.get_vnc_port(),
        }

    def do_show(self):
        environment = self.manager.environment_get(self.params.name)
        print {
            'name': environment.name,
            'nodes': map(lambda x: {'node': self.node_dict(x)},
                         environment.nodes)
        }

    def do_erase(self):
        self.manager.environment_get(self.params.name).erase()

    def do_start(self):
        self.manager.environment_get(self.params.name).start()

    def do_destroy(self):
        self.manager.environment_get(self.params.name).destroy(verbose=False)

    def do_suspend(self):
        self.manager.environment_get(self.params.name).suspend()

    def do_resume(self):
        self.manager.environment_get(self.params.name).resume()

    def do_revert(self):
        self.manager.environment_get(self.params.name).revert(
            self.params.snapshot_name)

    def do_snapshot(self):
        self.manager.environment_get(self.params.name).snapshot(
            self.params.snapshot_name)

    commands = {
        'list': do_list,
        'show': do_show,
        'erase': do_erase,
        'start': do_start,
        'destroy': do_destroy,
        'suspend': do_suspend,
        'resume': do_resume,
        'revert': do_revert,
        'snapshot': do_snapshot
    }

    def get_params(self):
        name_parser = argparse.ArgumentParser(add_help=False)
        name_parser.add_argument('name',
                                 help='environment name',
                                 default=os.getenv('ENV_NAME'))
        snapshot_name_parser = argparse.ArgumentParser(add_help=False)
        snapshot_name_parser.add_argument('--snapshot-name',
                                          help='snapshot name',
                                          default=os.getenv('SNAPSHOT_NAME'))
        parser = argparse.ArgumentParser(
            description="Manage virtual environments")
        subparsers = parser.add_subparsers(help='commands', dest='command')
        subparsers.add_parser('list')
        subparsers.add_parser('show', parents=[name_parser])
        subparsers.add_parser('erase', parents=[name_parser])
        subparsers.add_parser('start', parents=[name_parser])
        subparsers.add_parser('destroy', parents=[name_parser])
        subparsers.add_parser('suspend', parents=[name_parser])
        subparsers.add_parser('resume', parents=[name_parser])
        subparsers.add_parser('revert',
                              parents=[name_parser, snapshot_name_parser])
        subparsers.add_parser('snapshot',
                              parents=[name_parser, snapshot_name_parser])
        return parser.parse_args()
Exemple #20
0
class EnvironmentModel(object):
    hostname = 'nailgun'
    domain = 'test.domain.local'
    installation_timeout = 1800
    deployment_timeout = 1800
    puppet_timeout = 1000
    nat_interface = ''  # INTERFACES.get('admin')
    admin_net = 'admin'

    def __init__(self):
        self._virtual_environment = None
        self._keys = None
        self.manager = Manager()
        self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self)

    def _get_or_create(self):
        try:
            return self.manager.environment_get(self.env_name)
        except:
            self._virtual_environment = self.describe_environment()
            self._virtual_environment.define()
            return self._virtual_environment

    def router(self, router_name=None):
        router_name = router_name or self.admin_net
        return str(
            IPNetwork(
                self.get_virtual_environment().network_by_name(router_name).
                ip_network)[1])

    @property
    def fuel_web(self):
        """
        :rtype: FuelWebClient
        """
        return self._fuel_web

    @property
    def node_roles(self):
        return NodeRoles(
            admin_names=['admin'],
            other_names=['slave-%02d' % x for x in range(1, int(
                settings.NODES_COUNT))]
        )

    @property
    def env_name(self):
        return settings.ENV_NAME

    def add_empty_volume(self, node, name,
                         capacity=settings.NODE_VOLUME_SIZE*1024*1024*1024,
                         device='disk', bus='virtio', format='qcow2'):
        self.manager.node_attach_volume(
            node=node,
            volume=self.manager.volume_create(
                name=name,
                capacity=capacity,
                environment=self.get_virtual_environment(),
                format=format),
            device=device,
            bus=bus)

    def add_node(self, memory, name, vcpu=1, boot=None):
        return self.manager.node_create(
            name=name,
            memory=memory,
            vcpu=vcpu,
            environment=self.get_virtual_environment(),
            boot=boot)

    @logwrap
    def add_syslog_server(self, cluster_id, port=5514):
        self.fuel_web.add_syslog_server(
            cluster_id, self.get_host_node_ip(), port)

    def bootstrap_nodes(self, devops_nodes, timeout=600):
        """
        Start vms and wait they are registered on nailgun.
        :rtype : List of registered nailgun nodes
        """
        for node in devops_nodes:
            node.start()
        wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout)

        for node in self.nailgun_nodes(devops_nodes):
            self.sync_node_time(self.get_ssh_to_remote(node["ip"]))

        return self.nailgun_nodes(devops_nodes)

    def create_interfaces(self, networks, node,
                          model=settings.INTERFACE_MODEL):
        for network in networks:
            self.manager.interface_create(network, node=node, model=model)

    def describe_environment(self):
        """
        :rtype : Environment
        """
        environment = self.manager.environment_create(self.env_name)
        networks = []
        for name in settings.INTERFACE_ORDER:
            ip_networks = [
                IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',')]
            new_prefix = int(settings.POOLS.get(name)[1])
            pool = self.manager.create_network_pool(networks=ip_networks,
                                                    prefix=int(new_prefix))
            networks.append(self.manager.network_create(
                name=name,
                environment=environment,
                pool=pool,
                forward=settings.FORWARDING.get(name),
                has_dhcp_server=settings.DHCP.get(name)))

        for name in self.node_roles.admin_names:
            self.describe_admin_node(name, networks)
        for name in self.node_roles.other_names:
            self.describe_empty_node(name, networks)
        return environment

    def devops_nodes_by_names(self, devops_node_names):
        return map(
            lambda name:
            self.get_virtual_environment().node_by_name(name),
            devops_node_names)

    @logwrap
    def describe_admin_node(self, name, networks):
        node = self.add_node(
            memory=settings.HARDWARE.get("admin_node_memory", 1024),
            vcpu=settings.HARDWARE.get("admin_node_cpu", 1),
            name=name,
            boot=['hd', 'cdrom'])
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + '-system')
        self.add_empty_volume(
            node,
            name + '-iso',
            capacity=_get_file_size(settings.ISO_PATH),
            format='raw',
            device='cdrom',
            bus='ide')
        return node

    def describe_empty_node(self, name, networks):
        node = self.add_node(
            name=name,
            memory=settings.HARDWARE.get("slave_node_memory", 1024),
            vcpu=settings.HARDWARE.get("slave_node_cpu", 1))
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + '-system')

        if settings.USE_ALL_DISKS:
            self.add_empty_volume(node, name + '-cinder')
            self.add_empty_volume(node, name + '-swift')

        return node

    @logwrap
    def get_admin_remote(self):
        """
        :rtype : SSHClient
        """
        return self.nodes().admin.remote(self.admin_net,
                                         login='******',
                                         password='******')

    @logwrap
    def get_admin_node_ip(self):
        return str(
            self.nodes().admin.get_ip_address_by_network_name(self.admin_net))

    @logwrap
    def get_ebtables(self, cluster_id, devops_nodes):
        return Ebtables(self.get_target_devs(devops_nodes),
                        self.fuel_web.client.get_cluster_vlans(cluster_id))

    def get_host_node_ip(self):
        return self.router()

    def get_keys(self, node):
        params = {
            'ip': node.get_ip_address_by_network_name(self.admin_net),
            'mask': self.get_net_mask(self.admin_net),
            'gw': self.router(),
            'hostname': '.'.join((self.hostname, self.domain)),
            'nat_interface': self.nat_interface,
            'dns1': settings.DNS

        }
        keys = (
            "<Esc><Enter>\n"
            "<Wait>\n"
            "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n"
            " ip=%(ip)s\n"
            " netmask=%(mask)s\n"
            " gw=%(gw)s\n"
            " dns1=%(dns1)s\n"
            " hostname=%(hostname)s\n"
            " dhcp_interface=%(nat_interface)s\n"
            " <Enter>\n"
        ) % params
        return keys

    @logwrap
    def get_private_keys(self, force=False):
        if force or self._keys is None:
            self._keys = []
            for key_string in ['/root/.ssh/id_rsa',
                               '/root/.ssh/bootstrap.rsa']:
                with self.get_admin_remote().open(key_string) as f:
                    self._keys.append(RSAKey.from_private_key(f))
        return self._keys

    @logwrap
    def get_ssh_to_remote(self, ip):
        return SSHClient(ip,
                         username='******',
                         password='******',
                         private_keys=self.get_private_keys())

    @logwrap
    def get_ssh_to_remote_by_name(self, node_name):
        return self.get_ssh_to_remote(
            self.fuel_web.get_nailgun_node_by_devops_node(
                self.get_virtual_environment().node_by_name(node_name))['ip']
        )

    def get_target_devs(self, devops_nodes):
        return [
            interface.target_dev for interface in [
                val for var in map(lambda node: node.interfaces, devops_nodes)
                for val in var]]

    def get_virtual_environment(self):
        """
        :rtype : devops.models.Environment
        """
        if self._virtual_environment is None:
            self._virtual_environment = self._get_or_create()
        return self._virtual_environment

    def get_network(self, net_name):
        return str(
            IPNetwork(
                self.get_virtual_environment().network_by_name(net_name).
                ip_network))

    def get_net_mask(self, net_name):
        return str(
            IPNetwork(
                self.get_virtual_environment().network_by_name(
                    net_name).ip_network).netmask)

    def make_snapshot(self, snapshot_name):
        self.get_virtual_environment().suspend(verbose=False)
        self.get_virtual_environment().snapshot(snapshot_name, force=True)

    def nailgun_nodes(self, devops_nodes):
        return map(
            lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node),
            devops_nodes
        )

    def nodes(self):
        return Nodes(self.get_virtual_environment(), self.node_roles)

    def revert_snapshot(self, name):
        if self.get_virtual_environment().has_snapshot(name):
            logging.info('We have snapshot with such name %s' % name)

            self.get_virtual_environment().revert(name)
            logging.info('Starting snapshot reverting ....')

            self.get_virtual_environment().resume()
            logging.info('Starting snapshot resuming ...')

            time.sleep(10)

            list = self.nodes().slaves
            for node in list:
                if not node.driver.node_active(node):
                    continue
                try:
                    self.sync_node_time(self.get_ssh_to_remote(
                        node.get_ip_address_by_network_name(self.admin_net)))
                except Exception, e:
                    logging.warn(
                        'Paramiko exception catched while'
                        ' trying to run ntpdate: %s' % e)
            return True
        return False
Exemple #21
0
class CiBase(object):
    def __init__(self):
        self.manager = Manager()
        self._environment = None
        self.saved_environment_states = {}

    def _get_or_create(self):
        try:
            return self.manager.environment_get(self.env_name())
        except:
            self._environment = self.describe_environment()
            self._environment.define()
            return self._environment

    def get_state(self, name):
        if self.environment().has_snapshot(name):
            self.environment().revert(name)
            return True
        return False

    def environment(self):
        """
        :rtype : devops.models.Environment
        """
        self._environment = self._environment or self._get_or_create()
        return self._environment

    @abstractproperty
    def env_name(self):
        """
        :rtype : string
        """
        pass

    @abstractmethod
    def describe_environment(self):
        """
        :rtype : devops.models.Environment
        """
        pass

    @abstractproperty
    def node_roles(self):
        """
        :rtype : NodeRoles
        """
        pass

    def nodes(self):
        return Nodes(self.environment(), self.node_roles())

    # noinspection PyShadowingBuiltins
    def add_empty_volume(self,
                         node,
                         name,
                         capacity=20 * 1024 * 1024 * 1024,
                         device='disk',
                         bus='virtio',
                         format='qcow2'):
        self.manager.node_attach_volume(node=node,
                                        volume=self.manager.volume_create(
                                            name=name,
                                            capacity=capacity,
                                            environment=self.environment(),
                                            format=format),
                                        device=device,
                                        bus=bus)

    def add_node(self, memory, name, boot=None):
        return self.manager.node_create(name=name,
                                        memory=memory,
                                        environment=self.environment(),
                                        boot=boot)

    def create_interfaces(self, networks, node):
        for network in networks:
            self.manager.interface_create(network, node=node)

    def describe_admin_node(self, name, networks, memory=1024):
        node = self.add_node(memory=memory, name=name, boot=['hd', 'cdrom'])
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + '-system')
        self.add_empty_volume(node,
                              name + '-iso',
                              capacity=_get_file_size(ISO_PATH),
                              format='raw',
                              device='cdrom',
                              bus='ide')
        return node

    def describe_empty_node(self, name, networks, memory=1024):
        node = self.add_node(memory, name)
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + '-system')

        if USE_ALL_DISKS:
            self.add_empty_volume(node, name + '-cinder')
            self.add_empty_volume(node, name + '-swift')

        return node

    @abstractmethod
    def setup_environment(self):
        """
        :rtype : None
        """
        pass

    def get_empty_environment(self):
        if not (self.get_state(EMPTY_SNAPSHOT)):
            self.setup_environment()
            self.environment().snapshot(EMPTY_SNAPSHOT)

    def generate_state_hash(self, settings):
        return hashlib.md5(str(settings)).hexdigest()

    def revert_to_state(self, settings={}):
        state_hash = self.generate_state_hash(settings)

        if state_hash in self.saved_environment_states:
            # revert to matching state
            state = self.saved_environment_states[state_hash]
            if not (self.get_state(state['snapshot_name'])):
                return False
            self.environment().resume()
            return True

        return False

    def snapshot_state(self, name, settings={}):
        state_hash = self.generate_state_hash(settings)

        snapshot_name = '{0}_{1}'.format(
            name.replace(' ', '_')[:17], state_hash)

        self.environment().suspend(verbose=False)
        self.environment().snapshot(
            name=snapshot_name,
            description=name,
            force=True,
        )
        self.environment().resume(verbose=False)
        self.saved_environment_states[state_hash] = {
            'snapshot_name': snapshot_name,
            'cluster_name': name,
            'settings': settings
        }

    def internal_virtual_ip(self):
        return str(
            IPNetwork(
                self.environment().network_by_name('internal').ip_network)[-2])

    def public_router(self):
        return str(
            IPNetwork(
                self.environment().network_by_name('public').ip_network)[1])

    def internal_router(self):
        return self._router('internal')

    def nat_router(self):
        return self._router('nat')

    def _router(self, router_name):
        return str(
            IPNetwork(
                self.environment().network_by_name(router_name).ip_network)[1])

    def get_host_node_ip(self):
        return self.internal_router()

    def internal_network(self):
        return str(
            IPNetwork(
                self.environment().network_by_name('internal').ip_network))

    def internal_net_mask(self):
        return str(
            IPNetwork(self.environment().network_by_name(
                'internal').ip_network).netmask)

    def public_net_mask(self):
        return str(
            IPNetwork(self.environment().network_by_name(
                'public').ip_network).netmask)

    def public_network(self):
        return str(
            IPNetwork(self.environment().network_by_name('public').ip_network))
Exemple #22
0
class TestManager(TestCase):
    manager = Manager()

    def tearDown(self):
        for environment in self.manager.environment_list():
            environment.erase()

    def test_getting_subnetworks(self):
        pool = IpNetworksPool(networks=[IPNetwork('10.1.0.0/22')], prefix=24)
        pool.set_allocated_networks([IPv4Network('10.1.1.0/24')])
        networks = list(pool)
        self.assertTrue(IPv4Network('10.1.0.0/24') in networks)
        self.assertFalse(IPv4Network('10.1.1.0/24') in networks)
        self.assertTrue(IPv4Network('10.1.2.0/24') in networks)
        self.assertTrue(IPv4Network('10.1.3.0/24') in networks)

    def test_getting_ips(self):
        self.assertEquals('10.1.0.254', str(IPv4Network('10.1.0.0/24')[-2]))

    def test_network_iterator(self):
        environment = self.manager.environment_create('test_env')
        node = self.manager.node_create('test_node', environment)
        network = self.manager.network_create(environment=environment,
                                              name='internal',
                                              ip_network='10.1.0.0/24')
        interface = self.manager.interface_create(network=network, node=node)
        self.manager.network_create_address(str('10.1.0.1'),
                                            interface=interface)
        ip = network.next_ip()
        self.manager.network_create_address(str('10.1.0.3'),
                                            interface=interface)
        ip = network.next_ip()
        self.assertEquals('10.1.0.4', str(ip))

    def test_network_model(self):
        environment = self.manager.environment_create('test_env')
        node = self.manager.node_create('test_node', environment)
        network = self.manager.network_create(environment=environment,
                                              name='internal',
                                              ip_network='10.1.0.0/24')
        interface1 = self.manager.interface_create(network=network, node=node)
        self.assertEquals('virtio', interface1.model)
        interface2 = self.manager.interface_create(network=network,
                                                   node=node,
                                                   model='e1000')
        self.assertEquals('e1000', interface2.model)

    def test_environment_values(self):
        environment = self.manager.environment_create('test_env')
        print environment.volumes

    def test_network_pool(self):
        environment = self.manager.environment_create('test_env2')
        self.assertEqual(
            '10.0.0.0/24',
            str(
                self.manager.network_create(environment=environment,
                                            name='internal',
                                            pool=None).ip_network))
        self.assertEqual(
            '10.0.1.0/24',
            str(
                self.manager.network_create(environment=environment,
                                            name='external',
                                            pool=None).ip_network))
        self.assertEqual(
            '10.0.2.0/24',
            str(
                self.manager.network_create(environment=environment,
                                            name='private',
                                            pool=None).ip_network))
        environment = self.manager.environment_create('test_env2')
        self.assertEqual(
            '10.0.3.0/24',
            str(
                self.manager.network_create(environment=environment,
                                            name='internal',
                                            pool=None).ip_network))
        self.assertEqual(
            '10.0.4.0/24',
            str(
                self.manager.network_create(environment=environment,
                                            name='external',
                                            pool=None).ip_network))
        self.assertEqual(
            '10.0.5.0/24',
            str(
                self.manager.network_create(environment=environment,
                                            name='private',
                                            pool=None).ip_network))

    def test_node_creationw(self):
        environment = self.manager.environment_create('test_env55')
        node = self.manager.node_create(name='test_node4',
                                        environment=environment)
        node.define()

    def test_node_creation(self):
        environment = self.manager.environment_create('test_env3')
        internal = self.manager.network_create(environment=environment,
                                               name='internal',
                                               pool=None)
        node = self.manager.node_create(name='test_node',
                                        environment=environment)
        self.manager.interface_create(node=node, network=internal)
        environment.define()

    def test_create_volume(self):
        environment = self.manager.environment_create('test_env3')
        volume = self.manager.volume_get_predefined(
            '/var/lib/libvirt/images/disk-135824657433.qcow2')
        v3 = self.manager.volume_create_child('test_vp89',
                                              backing_store=volume,
                                              environment=environment)
        v3.define()

    def test_create_node3(self):
        environment = self.manager.environment_create('test_env3')
        internal = self.manager.network_create(environment=environment,
                                               name='internal',
                                               pool=None)
        external = self.manager.network_create(environment=environment,
                                               name='external',
                                               pool=None)
        private = self.manager.network_create(environment=environment,
                                              name='private',
                                              pool=None)
        node = self.manager.node_create(name='test_node',
                                        environment=environment)
        self.manager.interface_create(node=node, network=internal)
        self.manager.interface_create(node=node, network=external)
        self.manager.interface_create(node=node, network=private)
        volume = self.manager.volume_get_predefined(
            '/var/lib/libvirt/images/disk-135824657433.qcow2')
        v3 = self.manager.volume_create_child('test_vp892',
                                              backing_store=volume,
                                              environment=environment)
        v4 = self.manager.volume_create_child('test_vp891',
                                              backing_store=volume,
                                              environment=environment)
        self.manager.node_attach_volume(node=node, volume=v3)
        self.manager.node_attach_volume(node, v4)
        environment.define()
Exemple #23
0
class EnvironmentModel(object):
    hostname = 'nailgun'
    domain = 'test.domain.local'
    installation_timeout = 1800
    deployment_timeout = 1800
    puppet_timeout = 2000
    nat_interface = ''  # INTERFACES.get('admin')
    admin_net = 'admin'

    def __init__(self, os_image=None):
        self._virtual_environment = None
        self._keys = None
        self.manager = Manager()
        self.os_image = os_image
        self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self)

    def _get_or_create(self):
        try:
            return self.manager.environment_get(self.env_name)
        except Exception:
            self._virtual_environment = self.describe_environment()
            self._virtual_environment.define()
            return self._virtual_environment

    def router(self, router_name=None):
        router_name = router_name or self.admin_net
        return str(
            IPNetwork(
                self.get_virtual_environment().network_by_name(router_name).
                ip_network)[1])

    @property
    def fuel_web(self):
        """FuelWebClient
        :rtype: FuelWebClient
        """
        return self._fuel_web

    @property
    def node_roles(self):
        return NodeRoles(
            admin_names=['admin'],
            other_names=['slave-%02d' % x for x in range(1, int(
                settings.NODES_COUNT))]
        )

    @property
    def env_name(self):
        return settings.ENV_NAME

    def add_empty_volume(self, node, name,
                         capacity=settings.NODE_VOLUME_SIZE * 1024 * 1024
                         * 1024, device='disk', bus='virtio', format='qcow2'):
        self.manager.node_attach_volume(
            node=node,
            volume=self.manager.volume_create(
                name=name,
                capacity=capacity,
                environment=self.get_virtual_environment(),
                format=format),
            device=device,
            bus=bus)

    def add_node(self, memory, name, vcpu=1, boot=None):
        return self.manager.node_create(
            name=name,
            memory=memory,
            vcpu=vcpu,
            environment=self.get_virtual_environment(),
            boot=boot)

    @logwrap
    def add_syslog_server(self, cluster_id, port=5514):
        self.fuel_web.add_syslog_server(
            cluster_id, self.get_host_node_ip(), port)

    def bootstrap_nodes(self, devops_nodes, timeout=600):
        """Lists registered nailgun nodes
        Start vms and wait until they are registered on nailgun.
        :rtype : List of registered nailgun nodes
        """
        self.dhcrelay_check()

        for node in devops_nodes:
            node.start()
            #TODO(aglarendil): LP#1317213 temporary sleep
            #remove after better fix is applied
            time.sleep(2)
        wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout)

        for node in self.nailgun_nodes(devops_nodes):
            self.sync_node_time(self.get_ssh_to_remote(node["ip"]))

        return self.nailgun_nodes(devops_nodes)

    def create_interfaces(self, networks, node,
                          model=settings.INTERFACE_MODEL):
        if settings.BONDING:
            for network in networks:
                self.manager.interface_create(
                    network, node=node, model=model,
                    interface_map=settings.BONDING_INTERFACES)
        else:
            for network in networks:
                self.manager.interface_create(network, node=node, model=model)

    def describe_environment(self):
        """Environment
        :rtype : Environment
        """
        environment = self.manager.environment_create(self.env_name)
        networks = []
        interfaces = settings.INTERFACE_ORDER
        if settings.BONDING:
            interfaces = settings.BONDING_INTERFACES.keys()

        for name in interfaces:
            networks.append(self.create_networks(name, environment))
        for name in self.node_roles.admin_names:
            self.describe_admin_node(name, networks)
        for name in self.node_roles.other_names:
            self.describe_empty_node(name, networks)
        return environment

    def create_networks(self, name, environment):
        ip_networks = [
            IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',')]
        new_prefix = int(settings.POOLS.get(name)[1])
        pool = self.manager.create_network_pool(networks=ip_networks,
                                                prefix=int(new_prefix))
        return self.manager.network_create(
            name=name,
            environment=environment,
            pool=pool,
            forward=settings.FORWARDING.get(name),
            has_dhcp_server=settings.DHCP.get(name))

    def devops_nodes_by_names(self, devops_node_names):
        return map(
            lambda name:
            self.get_virtual_environment().node_by_name(name),
            devops_node_names)

    @logwrap
    def describe_admin_node(self, name, networks):
        node = self.add_node(
            memory=settings.HARDWARE.get("admin_node_memory", 1024),
            vcpu=settings.HARDWARE.get("admin_node_cpu", 1),
            name=name,
            boot=['hd', 'cdrom'])
        self.create_interfaces(networks, node)

        if self.os_image is None:
            self.add_empty_volume(node, name + '-system')
            self.add_empty_volume(
                node,
                name + '-iso',
                capacity=_get_file_size(settings.ISO_PATH),
                format='raw',
                device='cdrom',
                bus='ide')
        else:
            volume = self.manager.volume_get_predefined(self.os_image)
            vol_child = self.manager.volume_create_child(
                name=name + '-system',
                backing_store=volume,
                environment=self.get_virtual_environment()
            )
            self.manager.node_attach_volume(
                node=node,
                volume=vol_child
            )
        return node

    def describe_empty_node(self, name, networks):
        node = self.add_node(
            name=name,
            memory=settings.HARDWARE.get("slave_node_memory", 1024),
            vcpu=settings.HARDWARE.get("slave_node_cpu", 1))
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + '-system')

        if settings.USE_ALL_DISKS:
            self.add_empty_volume(node, name + '-cinder')
            self.add_empty_volume(node, name + '-swift')

        return node

    @logwrap
    def get_admin_remote(self):
        """SSH to admin node
        :rtype : SSHClient
        """
        return self.nodes().admin.remote(self.admin_net,
                                         login='******',
                                         password='******')

    @logwrap
    def get_admin_node_ip(self):
        return str(
            self.nodes().admin.get_ip_address_by_network_name(self.admin_net))

    @logwrap
    def get_ebtables(self, cluster_id, devops_nodes):
        return Ebtables(self.get_target_devs(devops_nodes),
                        self.fuel_web.client.get_cluster_vlans(cluster_id))

    def get_host_node_ip(self):
        return self.router()

    def get_keys(self, node):
        params = {
            'ip': node.get_ip_address_by_network_name(self.admin_net),
            'mask': self.get_net_mask(self.admin_net),
            'gw': self.router(),
            'hostname': '.'.join((self.hostname, self.domain)),
            'nat_interface': self.nat_interface,
            'dns1': settings.DNS

        }
        keys = (
            "<Wait>\n"
            "<Esc><Enter>\n"
            "<Wait>\n"
            "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n"
            " ip=%(ip)s\n"
            " netmask=%(mask)s\n"
            " gw=%(gw)s\n"
            " dns1=%(dns1)s\n"
            " hostname=%(hostname)s\n"
            " dhcp_interface=%(nat_interface)s\n"
            " <Enter>\n"
        ) % params
        return keys

    @logwrap
    def get_private_keys(self, force=False):
        if force or self._keys is None:
            self._keys = []
            for key_string in ['/root/.ssh/id_rsa',
                               '/root/.ssh/bootstrap.rsa']:
                with self.get_admin_remote().open(key_string) as f:
                    self._keys.append(RSAKey.from_private_key(f))
        return self._keys

    @logwrap
    def get_ssh_to_remote(self, ip):
        return SSHClient(ip,
                         username='******',
                         password='******',
                         private_keys=self.get_private_keys())

    @logwrap
    def get_ssh_to_remote_by_name(self, node_name):
        return self.get_ssh_to_remote(
            self.fuel_web.get_nailgun_node_by_devops_node(
                self.get_virtual_environment().node_by_name(node_name))['ip']
        )

    def get_target_devs(self, devops_nodes):
        return [
            interface.target_dev for interface in [
                val for var in map(lambda node: node.interfaces, devops_nodes)
                for val in var]]

    def get_virtual_environment(self):
        """Returns virtual environment
        :rtype : devops.models.Environment
        """
        if self._virtual_environment is None:
            self._virtual_environment = self._get_or_create()
        return self._virtual_environment

    def get_network(self, net_name):
        return str(
            IPNetwork(
                self.get_virtual_environment().network_by_name(net_name).
                ip_network))

    def get_net_mask(self, net_name):
        return str(
            IPNetwork(
                self.get_virtual_environment().network_by_name(
                    net_name).ip_network).netmask)

    def make_snapshot(self, snapshot_name, description="", is_make=False):
        if settings.MAKE_SNAPSHOT or is_make:
            self.get_virtual_environment().suspend(verbose=False)
            self.get_virtual_environment().snapshot(snapshot_name, force=True)
            revert_info(snapshot_name, description)

    def nailgun_nodes(self, devops_nodes):
        return map(
            lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node),
            devops_nodes
        )

    def nodes(self):
        return Nodes(self.get_virtual_environment(), self.node_roles)

    def revert_snapshot(self, name):
        if self.get_virtual_environment().has_snapshot(name):
            logger.info('We have snapshot with such name %s' % name)

            self.get_virtual_environment().revert(name)
            logger.info('Starting snapshot reverting ....')

            self.get_virtual_environment().resume()
            logger.info('Starting snapshot resuming ...')

            admin = self.nodes().admin

            try:
                admin.await(
                    self.admin_net, timeout=10 * 60, by_port=8000)
            except Exception as e:
                logger.warning("From first time admin isn't reverted: "
                               "{0}".format(e))
                admin.destroy()
                logger.info('Admin node was destroyed. Wait 10 sec.')
                time.sleep(10)
                self.get_virtual_environment().start(self.nodes().admins)
                logger.info('Admin node started second time.')
                self.nodes().admin.await(
                    self.admin_net, timeout=10 * 60, by_port=8000)

            self.sync_time_admin_node()

            for node in self.nodes().slaves:
                if not node.driver.node_active(node):
                    continue
                try:
                    logger.info("Sync time on revert for node %s" % node.name)
                    self.sync_node_time(
                        self.get_ssh_to_remote_by_name(node.name))
                except Exception as e:
                    logger.warning(
                        'Exception caught while trying to sync time on {0}:'
                        ' {1}'.format(node.name, e))
                self.run_nailgun_agent(
                    self.get_ssh_to_remote_by_name(node.name))
            return True
        return False

    def setup_environment(self):
        # start admin node
        admin = self.nodes().admin
        admin.disk_devices.get(device='cdrom').volume.upload(settings.ISO_PATH)
        self.get_virtual_environment().start(self.nodes().admins)
        logger.info("Waiting for admin node to start up")
        wait(lambda: admin.driver.node_active(admin), 60)
        logger.info("Proceed with installation")
        # update network parameters at boot screen
        admin.send_keys(self.get_keys(admin))
        # wait while installation complete
        admin.await(self.admin_net, timeout=10 * 60)
        self.wait_bootstrap()
        time.sleep(10)
        self.sync_time_admin_node()

    @retry()
    @logwrap
    def sync_node_time(self, remote):
        self.execute_remote_cmd(remote, 'hwclock -s')
        self.execute_remote_cmd(remote, 'NTPD=$(find /etc/init.d/ -regex \''
                                        '/etc/init.d/ntp.?\'); $NTPD stop; '
                                        'killall ntpd; ntpd -qg && '
                                        '$NTPD start')
        self.execute_remote_cmd(remote, 'hwclock -w')
        remote_date = remote.execute('date')['stdout']
        logger.info("Node time: %s" % remote_date)

    @logwrap
    def sync_time_admin_node(self):
        logger.info("Sync time on revert for admin")
        remote = self.get_admin_remote()
        self.execute_remote_cmd(remote, 'hwclock -s')
        # Sync time using ntpd
        try:
            # If public NTP servers aren't accessible ntpdate will fail and
            # ntpd daemon shouldn't be restarted to avoid 'Server has gone
            # too long without sync' error while syncing time from slaves
            self.execute_remote_cmd(remote, "ntpdate -d $(awk '/^server/{print"
                                            " $2}' /etc/ntp.conf)")
        except AssertionError as e:
            logger.warning('Error occurred while synchronizing time on master'
                           ': {0}'.format(e))
        else:
            self.execute_remote_cmd(remote, 'service ntpd stop && ntpd -qg && '
                                            'service ntpd start')
            self.execute_remote_cmd(remote, 'hwclock -w')

        remote_date = remote.execute('date')['stdout']
        logger.info("Master node time: {0}".format(remote_date))

    def verify_node_service_list(self, node_name, smiles_count):
        remote = self.get_ssh_to_remote_by_name(node_name)
        checkers.verify_service_list(remote, smiles_count)

    def verify_network_configuration(self, node_name):
        checkers.verify_network_configuration(
            node=self.fuel_web.get_nailgun_node_by_name(node_name),
            remote=self.get_ssh_to_remote_by_name(node_name)
        )

    def wait_bootstrap(self):
        logger.info("Waiting while bootstrapping is in progress")
        log_path = "/var/log/puppet/bootstrap_admin_node.log"
        logger.info("Puppet timeout set in {0}".format(
            float(settings.PUPPET_TIMEOUT)))
        wait(
            lambda: not
            self.get_admin_remote().execute(
                "grep 'Fuel node deployment complete' '%s'" % log_path
            )['exit_code'],
            timeout=(float(settings.PUPPET_TIMEOUT))
        )

    def dhcrelay_check(self):
        admin_remote = self.get_admin_remote()
        out = admin_remote.execute("dhcpcheck discover "
                                   "--ifaces eth0 "
                                   "--repeat 3 "
                                   "--timeout 10")['stdout']

        assert_true(self.get_admin_node_ip() in "".join(out),
                    "dhcpcheck doesn't discover master ip")

    def run_nailgun_agent(self, remote):
        agent = remote.execute('/opt/nailgun/bin/agent')['exit_code']
        logger.info("Nailgun agent run with exit_code: %s" % agent)

    def get_fuel_settings(self, remote=None):
        if not remote:
            remote = self.get_admin_remote()
        cmd = 'cat {cfg_file}'.format(cfg_file=settings.FUEL_SETTINGS_YAML)
        result = remote.execute(cmd)
        if result['exit_code'] == 0:
            fuel_settings = yaml.load(''.join(result['stdout']))
        else:
            raise Exception('Can\'t output {cfg_file} file: {error}'.
                            format(cfg_file=settings.FUEL_SETTINGS_YAML,
                                   error=result['stderr']))
        return fuel_settings

    def admin_install_pkg(self, pkg_name):
        """Install a package <pkg_name> on the admin node"""
        admin_remote = self.get_admin_remote()
        remote_status = admin_remote.execute("rpm -q {0}'".format(pkg_name))
        if remote_status['exit_code'] == 0:
            logger.info("Package '{0}' already installed.".format(pkg_name))
        else:
            logger.info("Installing package '{0}' ...".format(pkg_name))
            remote_status = admin_remote.execute("yum -y install {0}"
                                                 .format(pkg_name))
            logger.info("Installation of the package '{0}' has been"
                        " completed with exit code {1}"
                        .format(pkg_name, remote_status['exit_code']))
        return remote_status['exit_code']

    def admin_run_service(self, service_name):
        """Start a service <service_name> on the admin node"""
        admin_remote = self.get_admin_remote()
        admin_remote.execute("service {0} start".format(service_name))
        remote_status = admin_remote.execute("service {0} status"
                                             .format(service_name))
        if any('running...' in status for status in remote_status['stdout']):
            logger.info("Service '{0}' is running".format(service_name))
        else:
            logger.info("Service '{0}' failed to start"
                        " with exit code {1} :\n{2}"
                        .format(service_name,
                                remote_status['exit_code'],
                                remote_status['stdout']))

    @logwrap
    def execute_remote_cmd(self, remote, cmd, exit_code=0):
        result = remote.execute(cmd)
        assert_equal(result['exit_code'], exit_code,
                     'Failed to execute "{0}" on remote host: {1}'.
                     format(cmd, result['stderr']))
        return result['stdout']
Exemple #24
0
class EnvironmentModel(object):
    hostname = 'nailgun'
    domain = 'test.domain.local'
    installation_timeout = 1800
    deployment_timeout = 1800
    puppet_timeout = 2000
    nat_interface = ''  # INTERFACES.get('admin')
    admin_net = 'admin'

    def __init__(self, os_image=None):
        self._virtual_environment = None
        self._keys = None
        self.manager = Manager()
        self.os_image = os_image
        self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self)

    def _get_or_create(self):
        try:
            return self.manager.environment_get(self.env_name)
        except Exception:
            self._virtual_environment = self.describe_environment()
            self._virtual_environment.define()
            return self._virtual_environment

    def router(self, router_name=None):
        router_name = router_name or self.admin_net
        return str(
            IPNetwork(self.get_virtual_environment().network_by_name(
                router_name).ip_network)[1])

    @property
    def fuel_web(self):
        """FuelWebClient
        :rtype: FuelWebClient
        """
        return self._fuel_web

    @property
    def admin_node_ip(self):
        return self.fuel_web.admin_node_ip

    @property
    def node_roles(self):
        return NodeRoles(admin_names=['admin'],
                         other_names=[
                             'slave-%02d' % x
                             for x in range(1, int(settings.NODES_COUNT))
                         ])

    @property
    def env_name(self):
        return settings.ENV_NAME

    def add_empty_volume(self,
                         node,
                         name,
                         capacity=settings.NODE_VOLUME_SIZE * 1024 * 1024 *
                         1024,
                         device='disk',
                         bus='virtio',
                         format='qcow2'):
        self.manager.node_attach_volume(
            node=node,
            volume=self.manager.volume_create(
                name=name,
                capacity=capacity,
                environment=self.get_virtual_environment(),
                format=format),
            device=device,
            bus=bus)

    def add_node(self, memory, name, vcpu=1, boot=None):
        return self.manager.node_create(
            name=name,
            memory=memory,
            vcpu=vcpu,
            environment=self.get_virtual_environment(),
            boot=boot)

    @logwrap
    def add_syslog_server(self, cluster_id, port=5514):
        self.fuel_web.add_syslog_server(cluster_id, self.get_host_node_ip(),
                                        port)

    def bootstrap_nodes(self, devops_nodes, timeout=600):
        """Lists registered nailgun nodes
        Start vms and wait until they are registered on nailgun.
        :rtype : List of registered nailgun nodes
        """
        #self.dhcrelay_check()

        for node in devops_nodes:
            node.start()
            #TODO(aglarendil): LP#1317213 temporary sleep
            #remove after better fix is applied
            time.sleep(2)
        wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout)

        for node in self.nailgun_nodes(devops_nodes):
            self.sync_node_time(self.get_ssh_to_remote(node["ip"]))

        return self.nailgun_nodes(devops_nodes)

    def create_interfaces(self,
                          networks,
                          node,
                          model=settings.INTERFACE_MODEL):
        if settings.BONDING:
            for network in networks:
                self.manager.interface_create(
                    network,
                    node=node,
                    model=model,
                    interface_map=settings.BONDING_INTERFACES)
        else:
            for network in networks:
                self.manager.interface_create(network, node=node, model=model)

    def describe_environment(self):
        """Environment
        :rtype : Environment
        """
        environment = self.manager.environment_create(self.env_name)
        networks = []
        interfaces = settings.INTERFACE_ORDER
        if settings.BONDING:
            interfaces = settings.BONDING_INTERFACES.keys()

        for name in interfaces:
            networks.append(self.create_networks(name, environment))
        for name in self.node_roles.admin_names:
            self.describe_admin_node(name, networks)
        for name in self.node_roles.other_names:
            self.describe_empty_node(name, networks)
        return environment

    def create_networks(self, name, environment):
        ip_networks = [
            IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',')
        ]
        new_prefix = int(settings.POOLS.get(name)[1])
        pool = self.manager.create_network_pool(networks=ip_networks,
                                                prefix=int(new_prefix))
        return self.manager.network_create(
            name=name,
            environment=environment,
            pool=pool,
            forward=settings.FORWARDING.get(name),
            has_dhcp_server=settings.DHCP.get(name))

    def devops_nodes_by_names(self, devops_node_names):
        return map(
            lambda name: self.get_virtual_environment().node_by_name(name),
            devops_node_names)

    @logwrap
    def describe_admin_node(self, name, networks):
        node = self.add_node(memory=settings.HARDWARE.get(
            "admin_node_memory", 1024),
                             vcpu=settings.HARDWARE.get("admin_node_cpu", 1),
                             name=name,
                             boot=['hd', 'cdrom'])
        self.create_interfaces(networks, node)

        if self.os_image is None:
            self.add_empty_volume(node, name + '-system')
            self.add_empty_volume(node,
                                  name + '-iso',
                                  capacity=_get_file_size(settings.ISO_PATH),
                                  format='raw',
                                  device='cdrom',
                                  bus='ide')
        else:
            volume = self.manager.volume_get_predefined(self.os_image)
            vol_child = self.manager.volume_create_child(
                name=name + '-system',
                backing_store=volume,
                environment=self.get_virtual_environment())
            self.manager.node_attach_volume(node=node, volume=vol_child)
        return node

    def describe_empty_node(self, name, networks):
        node = self.add_node(name=name,
                             memory=settings.HARDWARE.get(
                                 "slave_node_memory", 1024),
                             vcpu=settings.HARDWARE.get("slave_node_cpu", 1))
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + '-system')

        if settings.USE_ALL_DISKS:
            self.add_empty_volume(node, name + '-cinder')
            self.add_empty_volume(node, name + '-swift')

        return node

    @logwrap
    def get_admin_remote(self):
        """SSH to admin node
        :rtype : SSHClient
        """
        return self.nodes().admin.remote(self.admin_net,
                                         login='******',
                                         password='******')

    @logwrap
    def get_admin_node_ip(self):
        return str(self.nodes().admin.get_ip_address_by_network_name(
            self.admin_net))

    @logwrap
    def get_ebtables(self, cluster_id, devops_nodes):
        return Ebtables(self.get_target_devs(devops_nodes),
                        self.fuel_web.client.get_cluster_vlans(cluster_id))

    def get_host_node_ip(self):
        return self.router()

    def get_keys(self, node, custom=None):
        params = {
            'ip': node.get_ip_address_by_network_name(self.admin_net),
            'mask': self.get_net_mask(self.admin_net),
            'gw': self.router(),
            'hostname': '.'.join((self.hostname, self.domain)),
            'nat_interface': self.nat_interface,
            'dns1': settings.DNS,
            'showmenu': 'yes' if custom else 'no'
        }
        keys = ("<Wait>\n"
                "<Esc><Enter>\n"
                "<Wait>\n"
                "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n"
                " ip=%(ip)s\n"
                " netmask=%(mask)s\n"
                " gw=%(gw)s\n"
                " dns1=%(dns1)s\n"
                " hostname=%(hostname)s\n"
                " dhcp_interface=%(nat_interface)s\n"
                " showmenu=%(showmenu)s\n"
                " <Enter>\n") % params
        return keys

    @logwrap
    def get_private_keys(self, force=False):
        if force or self._keys is None:
            self._keys = []
            for key_string in [
                    '/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa'
            ]:
                with self.get_admin_remote().open(key_string) as f:
                    self._keys.append(RSAKey.from_private_key(f))
        return self._keys

    @logwrap
    def get_ssh_to_remote(self, ip):
        return SSHClient(ip,
                         username='******',
                         password='******',
                         private_keys=self.get_private_keys())

    @logwrap
    def get_ssh_to_remote_by_name(self, node_name):
        return self.get_ssh_to_remote(
            self.fuel_web.get_nailgun_node_by_devops_node(
                self.get_virtual_environment().node_by_name(node_name))['ip'])

    def get_target_devs(self, devops_nodes):
        return [
            interface.target_dev for interface in [
                val for var in map(lambda node: node.interfaces, devops_nodes)
                for val in var
            ]
        ]

    def get_virtual_environment(self):
        """Returns virtual environment
        :rtype : devops.models.Environment
        """
        if self._virtual_environment is None:
            self._virtual_environment = self._get_or_create()
        return self._virtual_environment

    def get_network(self, net_name):
        return str(
            IPNetwork(self.get_virtual_environment().network_by_name(
                net_name).ip_network))

    def get_net_mask(self, net_name):
        return str(
            IPNetwork(self.get_virtual_environment().network_by_name(
                net_name).ip_network).netmask)

    def make_snapshot(self, snapshot_name, description="", is_make=False):
        if settings.MAKE_SNAPSHOT or is_make:
            self.get_virtual_environment().suspend(verbose=False)
            self.get_virtual_environment().snapshot(snapshot_name, force=True)
            revert_info(snapshot_name, description)

    def nailgun_nodes(self, devops_nodes):
        return map(
            lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node),
            devops_nodes)

    def nodes(self):
        return Nodes(self.get_virtual_environment(), self.node_roles)

    def revert_snapshot(self, name):
        if self.get_virtual_environment().has_snapshot(name):
            logger.info('We have snapshot with such name %s' % name)

            self.get_virtual_environment().revert(name)
            logger.info('Starting snapshot reverting ....')

            self.get_virtual_environment().resume()
            logger.info('Starting snapshot resuming ...')

            admin = self.nodes().admin

            try:
                admin. await (self.admin_net, timeout=10 * 60, by_port=8000)
            except Exception as e:
                logger.warning("From first time admin isn't reverted: "
                               "{0}".format(e))
                admin.destroy()
                logger.info('Admin node was destroyed. Wait 10 sec.')
                time.sleep(10)
                self.get_virtual_environment().start(self.nodes().admins)
                logger.info('Admin node started second time.')
                self.nodes().admin. await (self.admin_net,
                                           timeout=10 * 60,
                                           by_port=8000)
                _wait(self._fuel_web.get_nailgun_version, timeout=120)

            self.sync_time_admin_node()

            for node in self.nodes().slaves:
                if not node.driver.node_active(node):
                    continue
                try:
                    logger.info("Sync time on revert for node %s" % node.name)
                    self.sync_node_time(
                        self.get_ssh_to_remote_by_name(node.name))
                except Exception as e:
                    logger.warning(
                        'Exception caught while trying to sync time on {0}:'
                        ' {1}'.format(node.name, e))
                self.run_nailgun_agent(
                    self.get_ssh_to_remote_by_name(node.name))
            return True
        return False

    def setup_environment(self, custom=False):
        # start admin node
        admin = self.nodes().admin
        admin.disk_devices.get(device='cdrom').volume.upload(settings.ISO_PATH)
        self.get_virtual_environment().start(self.nodes().admins)
        logger.info("Waiting for admin node to start up")
        wait(lambda: admin.driver.node_active(admin), 60)
        logger.info("Proceed with installation")
        # update network parameters at boot screen
        admin.send_keys(self.get_keys(admin, custom=custom))
        if custom:
            self.setup_customisation()
        # wait while installation complete
        admin. await (self.admin_net, timeout=10 * 60)
        self.wait_bootstrap()
        time.sleep(10)
        self.sync_time_admin_node()

    @upload_manifests
    def wait_for_provisioning(self):
        _wait(lambda: _tcp_ping(
            self.nodes().admin.get_ip_address_by_network_name(self.admin_net),
            22),
              timeout=5 * 60)

    def setup_customisation(self):
        self.wait_for_provisioning()
        try:
            remote = self.get_admin_remote()
            pid = remote.execute("pgrep 'fuelmenu'")['stdout'][0]
            pid.rstrip('\n')
            remote.execute("kill -sigusr1 {0}".format(pid))
        except Exception:
            logger.error("Could not kill pid of fuelmenu")
            raise

    @retry(count=10, delay=60)
    @logwrap
    def sync_node_time(self, remote):
        self.execute_remote_cmd(remote, 'hwclock -s')
        self.execute_remote_cmd(
            remote, 'NTPD=$(find /etc/init.d/ -regex \''
            '/etc/init.d/ntp.?\'); $NTPD stop; '
            'killall ntpd; ntpd -qg && '
            '$NTPD start')
        self.execute_remote_cmd(remote, 'hwclock -w')
        remote_date = remote.execute('date')['stdout']
        logger.info("Node time: %s" % remote_date)

    @retry(count=10, delay=60)
    @logwrap
    def sync_time_admin_node(self):
        logger.info("Sync time on revert for admin")
        remote = self.get_admin_remote()
        self.execute_remote_cmd(remote, 'hwclock -s')
        # Sync time using ntpd
        try:
            # If public NTP servers aren't accessible ntpdate will fail and
            # ntpd daemon shouldn't be restarted to avoid 'Server has gone
            # too long without sync' error while syncing time from slaves
            self.execute_remote_cmd(
                remote, "ntpdate -d $(awk '/^server/{print"
                " $2}' /etc/ntp.conf)")
        except AssertionError as e:
            logger.warning('Error occurred while synchronizing time on master'
                           ': {0}'.format(e))
            raise
        else:
            self.execute_remote_cmd(
                remote, 'service ntpd stop && ntpd -qg && '
                'service ntpd start')
            self.execute_remote_cmd(remote, 'hwclock -w')

        remote_date = remote.execute('date')['stdout']
        logger.info("Master node time: {0}".format(remote_date))

    def verify_network_configuration(self, node_name):
        checkers.verify_network_configuration(
            node=self.fuel_web.get_nailgun_node_by_name(node_name),
            remote=self.get_ssh_to_remote_by_name(node_name))

    def wait_bootstrap(self):
        logger.info("Waiting while bootstrapping is in progress")
        log_path = "/var/log/puppet/bootstrap_admin_node.log"
        logger.info("Puppet timeout set in {0}".format(
            float(settings.PUPPET_TIMEOUT)))
        wait(lambda: not self.get_admin_remote().execute(
            "grep 'Fuel node deployment' '%s'" % log_path)['exit_code'],
             timeout=(float(settings.PUPPET_TIMEOUT)))
        result = self.get_admin_remote().execute("grep 'Fuel node deployment "
                                                 "complete' '%s'" %
                                                 log_path)['exit_code']
        if result != 0:
            raise Exception('Fuel node deployment failed.')

    def dhcrelay_check(self):
        admin_remote = self.get_admin_remote()
        out = admin_remote.execute("dhcpcheck discover "
                                   "--ifaces eth0 "
                                   "--repeat 3 "
                                   "--timeout 10")['stdout']

        assert_true(self.get_admin_node_ip() in "".join(out),
                    "dhcpcheck doesn't discover master ip")

    def run_nailgun_agent(self, remote):
        agent = remote.execute('/opt/nailgun/bin/agent')['exit_code']
        logger.info("Nailgun agent run with exit_code: %s" % agent)

    def get_fuel_settings(self, remote=None):
        if not remote:
            remote = self.get_admin_remote()
        cmd = 'cat {cfg_file}'.format(cfg_file=settings.FUEL_SETTINGS_YAML)
        result = remote.execute(cmd)
        if result['exit_code'] == 0:
            fuel_settings = yaml.load(''.join(result['stdout']))
        else:
            raise Exception('Can\'t output {cfg_file} file: {error}'.format(
                cfg_file=settings.FUEL_SETTINGS_YAML, error=result['stderr']))
        return fuel_settings

    def admin_install_pkg(self, pkg_name):
        """Install a package <pkg_name> on the admin node"""
        admin_remote = self.get_admin_remote()
        remote_status = admin_remote.execute("rpm -q {0}'".format(pkg_name))
        if remote_status['exit_code'] == 0:
            logger.info("Package '{0}' already installed.".format(pkg_name))
        else:
            logger.info("Installing package '{0}' ...".format(pkg_name))
            remote_status = admin_remote.execute(
                "yum -y install {0}".format(pkg_name))
            logger.info("Installation of the package '{0}' has been"
                        " completed with exit code {1}".format(
                            pkg_name, remote_status['exit_code']))
        return remote_status['exit_code']

    def admin_run_service(self, service_name):
        """Start a service <service_name> on the admin node"""
        admin_remote = self.get_admin_remote()
        admin_remote.execute("service {0} start".format(service_name))
        remote_status = admin_remote.execute(
            "service {0} status".format(service_name))
        if any('running...' in status for status in remote_status['stdout']):
            logger.info("Service '{0}' is running".format(service_name))
        else:
            logger.info("Service '{0}' failed to start"
                        " with exit code {1} :\n{2}".format(
                            service_name, remote_status['exit_code'],
                            remote_status['stdout']))

    # Modifies a resolv.conf on the Fuel master node and returns
    # its original content.
    # * adds 'nameservers' at start of resolv.conf if merge=True
    # * replaces resolv.conf with 'nameservers' if merge=False
    def modify_resolv_conf(self, nameservers=[], merge=True):
        remote = self.get_admin_remote()
        resolv_conf = remote.execute('cat /etc/resolv.conf')
        assert_equal(
            0, resolv_conf['exit_code'], 'Executing "{0}" on the '
            'admin node has failed with: {1}'.format('cat /etc/resolv.conf',
                                                     resolv_conf['stderr']))
        if merge:
            nameservers.extend(resolv_conf['stdout'])

        resolv_keys = ['search', 'domain', 'nameserver']
        resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers
                             if any(x in ns for x in resolv_keys))
        logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new))
        echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new)
        echo_result = remote.execute(echo_cmd)
        assert_equal(
            0, echo_result['exit_code'], 'Executing "{0}" on the '
            'admin node has failed with: {1}'.format(echo_cmd,
                                                     echo_result['stderr']))
        return resolv_conf['stdout']

    @logwrap
    def execute_remote_cmd(self, remote, cmd, exit_code=0):
        result = remote.execute(cmd)
        assert_equal(
            result['exit_code'], exit_code,
            'Failed to execute "{0}" on remote host: {1}'.format(
                cmd, result['stderr']))
        return result['stdout']
Exemple #25
0
class CiBase(object):
    def __init__(self):
        self.manager = Manager()
        self._environment = None

    def _get_or_create(self):
        try:
            return self.manager.environment_get(self.env_name())
        except:
            self._environment = self.describe_environment()
            self._environment.define()
            return self._environment

    def get_empty_state(self):
        return self.get_state(EMPTY_SNAPSHOT)

    def get_state(self, name):
        if self.environment().has_snapshot(name):
            self.environment().revert(name)
            return True
        return False

    def environment(self):
        """
        :rtype : devops.models.Environment
        """
        self._environment = self._environment or self._get_or_create()
        return self._environment

    @abstractproperty
    def env_name(self):
        """
        :rtype : string
        """
        pass

    @abstractmethod
    def describe_environment(self):
        """
        :rtype : devops.models.Environment
        """
        pass

    @abstractproperty
    def node_roles(self):
        """
        :rtype : NodeRoles
        """
        pass

    def nodes(self):
        return Nodes(self.environment(), self.node_roles())

    # noinspection PyShadowingBuiltins
    def add_empty_volume(self, node, name, capacity=20 * 1024 * 1024 * 1024,
                         device='disk', bus='virtio', format='qcow2'):
        self.manager.node_attach_volume(
            node=node,
            volume=self.manager.volume_create(
                name=name, capacity=capacity,
                environment=self.environment(),
                format=format),
            device=device, bus=bus)

    def add_node(self, memory, name, boot=None):
        return self.manager.node_create(
            name=name,
            memory=memory,
            environment=self.environment(),
            boot=boot)

    def create_interfaces(self, networks, node):
        for network in networks:
            if network.name == 'internal':
                self.manager.interface_create(network, node=node)
                self.manager.interface_create(network, node=node)
            self.manager.interface_create(network, node=node)

    def describe_admin_node(self, name, networks, memory=1024):
        node = self.add_node(memory=memory, name=name, boot=['hd', 'cdrom'])
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + '-system')
        self.add_empty_volume(
            node, name + '-iso', capacity=_get_file_size(ISO_PATH),
            format='raw', device='cdrom', bus='ide')
        return node

    def describe_empty_node(self, name, networks, memory=1024):
        node = self.add_node(memory, name)
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + '-system')
        self.add_empty_volume(node, name + '-cinder')
        self.add_empty_volume(node, name + '-swift')
        return node

    @abstractmethod
    def setup_environment(self):
        """
        :rtype : None
        """
        pass

    def internal_virtual_ip(self):
        return str(IPNetwork(
            self.environment().network_by_name('internal').ip_network)[-2])

    def public_router(self):
        return str(
            IPNetwork(
                self.environment().network_by_name('public').ip_network)[1])

    def internal_router(self):
        return self._router('internal')

    def nat_router(self):
        return self._router('nat')

    def _router(self, router_name):
        return str(
            IPNetwork(
                self.environment().network_by_name(router_name).ip_network)[1])

    def get_host_node_ip(self):
        return self.internal_router()

    def internal_network(self):
        return str(
            IPNetwork(
                self.environment().network_by_name('internal').ip_network))

    def internal_net_mask(self):
        return str(IPNetwork(
            self.environment().network_by_name('internal').ip_network).netmask)

    def public_net_mask(self):
        return str(IPNetwork(
            self.environment().network_by_name('public').ip_network).netmask)

    def public_network(self):
        return str(
            IPNetwork(self.environment().network_by_name('public').ip_network))
Exemple #26
0
class Shell(object):
    def __init__(self):
        super(Shell, self).__init__()
        self.params = self.get_params()
        self.manager = Manager()

    def execute(self):
        self.commands.get(self.params.command)(self)

    def do_list(self):
        env_list = self.manager.environment_list().values('name')
        for env in env_list:
            print env['name']

        return env_list

    def node_dict(self, node):
        return {'name': node.name, 'vnc': node.get_vnc_port()}

    def do_show(self):
        environment = self.manager.environment_get(self.params.name)

        print '%5s %25s' % ("VNC", "NODE-NAME")
        for item in map(lambda x: self.node_dict(x), environment.nodes):
            print '%5s %25s' % (item['vnc'], item['name'])

    def do_erase(self):
        self.manager.environment_get(self.params.name).erase()

    def do_start(self):
        self.manager.environment_get(self.params.name).start()

    def do_destroy(self):
        self.manager.environment_get(self.params.name).destroy(verbose=False)

    def do_suspend(self):
        self.manager.environment_get(self.params.name).suspend(verbose=False)

    def do_resume(self):
        self.manager.environment_get(self.params.name).resume(verbose=False)

    def do_revert(self):
        self.manager.environment_get(self.params.name).revert(
            self.params.snapshot_name)

    def do_snapshot(self):
        self.manager.environment_get(self.params.name).snapshot(
            self.params.snapshot_name)

    def do_synchronize(self):
        self.manager.synchronize_environments()

    commands = {
        'list': do_list,
        'show': do_show,
        'erase': do_erase,
        'start': do_start,
        'destroy': do_destroy,
        'suspend': do_suspend,
        'resume': do_resume,
        'revert': do_revert,
        'snapshot': do_snapshot,
        'sync': do_synchronize
    }

    def get_params(self):
        name_parser = argparse.ArgumentParser(add_help=False)
        name_parser.add_argument('name',
                                 help='environment name',
                                 default=environ.get('ENV_NAME'))
        snapshot_name_parser = argparse.ArgumentParser(add_help=False)
        snapshot_name_parser.add_argument('--snapshot-name',
                                          help='snapshot name',
                                          default=environ.get('SNAPSHOT_NAME'))
        parser = argparse.ArgumentParser(
            description="Manage virtual environments")
        subparsers = parser.add_subparsers(help='commands', dest='command')
        subparsers.add_parser('list')
        subparsers.add_parser('show', parents=[name_parser])
        subparsers.add_parser('erase', parents=[name_parser])
        subparsers.add_parser('start', parents=[name_parser])
        subparsers.add_parser('destroy', parents=[name_parser])
        subparsers.add_parser('suspend', parents=[name_parser])
        subparsers.add_parser('resume', parents=[name_parser])
        subparsers.add_parser('revert',
                              parents=[name_parser, snapshot_name_parser])
        subparsers.add_parser('snapshot',
                              parents=[name_parser, snapshot_name_parser])
        subparsers.add_parser('sync')
        return parser.parse_args()
Exemple #27
0
 def __init__(self):
     self._environment = None
     self.manager = Manager()
Exemple #28
0
class Shell(object):
    def __init__(self):
        super(Shell, self).__init__()
        self.params = self.get_params()
        self.manager = Manager()

    def execute(self):
        self.commands.get(self.params.command)(self)

    def do_list(self):
        print self.manager.environment_list().values('name')

    def node_dict(self, node):
        return {'name': node.name,
                'vnc': node.get_vnc_port(),
        }

    def do_show(self):
        environment = self.manager.environment_get(self.params.name)
        print {
            'name': environment.name,
            'nodes': map(lambda x: {'node': self.node_dict(x)},
                         environment.nodes)
        }

    def do_erase(self):
        self.manager.environment_get(self.params.name).erase()

    def do_start(self):
        self.manager.environment_get(self.params.name).start()

    def do_destroy(self):
        self.manager.environment_get(self.params.name).destroy(verbose=False)

    def do_suspend(self):
        self.manager.environment_get(self.params.name).suspend()

    def do_resume(self):
        self.manager.environment_get(self.params.name).resume()

    def do_revert(self):
        self.manager.environment_get(self.params.name).revert(
            self.params.snapshot_name)

    def do_snapshot(self):
        self.manager.environment_get(self.params.name).snapshot(
            self.params.snapshot_name)

    commands = {
        'list': do_list,
        'show': do_show,
        'erase': do_erase,
        'start': do_start,
        'destroy': do_destroy,
        'suspend': do_suspend,
        'resume': do_resume,
        'revert': do_revert,
        'snapshot': do_snapshot
    }

    def get_params(self):
        name_parser = argparse.ArgumentParser(add_help=False)
        name_parser.add_argument('name', help='environment name',
                                 default=os.getenv('ENV_NAME'))
        snapshot_name_parser = argparse.ArgumentParser(add_help=False)
        snapshot_name_parser.add_argument('--snapshot-name',
                                          help='snapshot name',
                                          default=os.getenv('SNAPSHOT_NAME'))
        parser = argparse.ArgumentParser(
            description="Manage virtual environments")
        subparsers = parser.add_subparsers(help='commands', dest='command')
        subparsers.add_parser('list')
        subparsers.add_parser('show', parents=[name_parser])
        subparsers.add_parser('erase', parents=[name_parser])
        subparsers.add_parser('start', parents=[name_parser])
        subparsers.add_parser('destroy', parents=[name_parser])
        subparsers.add_parser('suspend', parents=[name_parser])
        subparsers.add_parser('resume', parents=[name_parser])
        subparsers.add_parser('revert',
                              parents=[name_parser, snapshot_name_parser])
        subparsers.add_parser('snapshot',
                              parents=[name_parser, snapshot_name_parser])
        return parser.parse_args()
Exemple #29
0
class EnvironmentModel(object):
    hostname = 'nailgun'
    domain = 'test.domain.local'
    installation_timeout = 1800
    deployment_timeout = 1800
    puppet_timeout = 2000
    nat_interface = ''  # INTERFACES.get('admin')
    admin_net = 'admin'
    admin_net2 = 'admin2'
    multiple_cluster_networks = settings.MULTIPLE_NETWORKS

    def __init__(self, os_image=None):
        self._virtual_environment = None
        self._keys = None
        self.manager = Manager()
        self.os_image = os_image
        self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self)

    @property
    def nailgun_actions(self):
        return FuelActions.Nailgun(self.get_admin_remote())

    @property
    def postgres_actions(self):
        return FuelActions.Postgres(self.get_admin_remote())

    def _get_or_create(self):
        try:
            return self.manager.environment_get(self.env_name)
        except Exception:
            self._virtual_environment = self.describe_environment()
            self._virtual_environment.define()
            return self._virtual_environment

    def router(self, router_name=None):
        router_name = router_name or self.admin_net
        if router_name == self.admin_net2:
            return str(
                IPNetwork(self.get_virtual_environment().network_by_name(
                    router_name).ip_network)[2])
        return str(
            IPNetwork(self.get_virtual_environment().network_by_name(
                router_name).ip_network)[1])

    @property
    def fuel_web(self):
        """FuelWebClient
        :rtype: FuelWebClient
        """
        return self._fuel_web

    @property
    def admin_node_ip(self):
        return self.fuel_web.admin_node_ip

    @property
    def node_roles(self):
        return NodeRoles(admin_names=['admin'],
                         other_names=[
                             'slave-%02d' % x
                             for x in range(1, int(settings.NODES_COUNT))
                         ])

    @property
    def env_name(self):
        return settings.ENV_NAME

    def add_empty_volume(self,
                         node,
                         name,
                         capacity=settings.NODE_VOLUME_SIZE * 1024 * 1024 *
                         1024,
                         device='disk',
                         bus='virtio',
                         format='qcow2'):
        self.manager.node_attach_volume(
            node=node,
            volume=self.manager.volume_create(
                name=name,
                capacity=capacity,
                environment=self.get_virtual_environment(),
                format=format),
            device=device,
            bus=bus)

    def add_node(self, memory, name, vcpu=1, boot=None):
        return self.manager.node_create(
            name=name,
            memory=memory,
            vcpu=vcpu,
            environment=self.get_virtual_environment(),
            boot=boot)

    @logwrap
    def add_syslog_server(self, cluster_id, port=5514):
        self.fuel_web.add_syslog_server(cluster_id, self.get_host_node_ip(),
                                        port)

    def bootstrap_nodes(self, devops_nodes, timeout=600):
        """Lists registered nailgun nodes
        Start vms and wait until they are registered on nailgun.
        :rtype : List of registered nailgun nodes
        """
        # self.dhcrelay_check()

        for node in devops_nodes:
            node.start()
            # TODO(aglarendil): LP#1317213 temporary sleep
            # remove after better fix is applied
            time.sleep(2)
        wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout)

        for node in self.nailgun_nodes(devops_nodes):
            self.sync_node_time(self.get_ssh_to_remote(node["ip"]))

        return self.nailgun_nodes(devops_nodes)

    def create_interfaces(self,
                          networks,
                          node,
                          model=settings.INTERFACE_MODEL):
        if settings.BONDING:
            for network in networks:
                self.manager.interface_create(
                    network,
                    node=node,
                    model=model,
                    interface_map=settings.BONDING_INTERFACES)
        else:
            for network in networks:
                self.manager.interface_create(network, node=node, model=model)

    def describe_environment(self):
        """Environment
        :rtype : Environment
        """
        environment = self.manager.environment_create(self.env_name)
        networks = []
        interfaces = settings.INTERFACE_ORDER
        if self.multiple_cluster_networks:
            logger.info('Multiple cluster networks feature is enabled!')
        if settings.BONDING:
            interfaces = settings.BONDING_INTERFACES.keys()

        for name in interfaces:
            networks.append(self.create_networks(name, environment))
        for name in self.node_roles.admin_names:
            self.describe_admin_node(name, networks)
        for name in self.node_roles.other_names:
            if self.multiple_cluster_networks:
                networks1 = [
                    net for net in networks
                    if net.name in settings.NODEGROUPS[0]['pools']
                ]
                networks2 = [
                    net for net in networks
                    if net.name in settings.NODEGROUPS[1]['pools']
                ]
                # If slave index is even number, then attach to
                # it virtual networks from the second network group.
                if int(name[-2:]) % 2 == 1:
                    self.describe_empty_node(name, networks1)
                elif int(name[-2:]) % 2 == 0:
                    self.describe_empty_node(name, networks2)
            else:
                self.describe_empty_node(name, networks)
        return environment

    def create_networks(self, name, environment):
        ip_networks = [
            IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',')
        ]
        new_prefix = int(settings.POOLS.get(name)[1])
        pool = self.manager.create_network_pool(networks=ip_networks,
                                                prefix=int(new_prefix))
        return self.manager.network_create(
            name=name,
            environment=environment,
            pool=pool,
            forward=settings.FORWARDING.get(name),
            has_dhcp_server=settings.DHCP.get(name))

    def devops_nodes_by_names(self, devops_node_names):
        return map(
            lambda name: self.get_virtual_environment().node_by_name(name),
            devops_node_names)

    @logwrap
    def describe_admin_node(self, name, networks):
        node = self.add_node(memory=settings.HARDWARE.get(
            "admin_node_memory", 1024),
                             vcpu=settings.HARDWARE.get("admin_node_cpu", 1),
                             name=name,
                             boot=['hd', 'cdrom'])
        self.create_interfaces(networks, node)

        if self.os_image is None:
            self.add_empty_volume(node, name + '-system')
            self.add_empty_volume(node,
                                  name + '-iso',
                                  capacity=_get_file_size(settings.ISO_PATH),
                                  format='raw',
                                  device='cdrom',
                                  bus='ide')
        else:
            volume = self.manager.volume_get_predefined(self.os_image)
            vol_child = self.manager.volume_create_child(
                name=name + '-system',
                backing_store=volume,
                environment=self.get_virtual_environment())
            self.manager.node_attach_volume(node=node, volume=vol_child)
        return node

    def describe_empty_node(self, name, networks):
        node = self.add_node(name=name,
                             memory=settings.HARDWARE.get(
                                 "slave_node_memory", 1024),
                             vcpu=settings.HARDWARE.get("slave_node_cpu", 1))
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + '-system')

        if settings.USE_ALL_DISKS:
            self.add_empty_volume(node, name + '-cinder')
            self.add_empty_volume(node, name + '-swift')

        return node

    @logwrap
    def get_admin_remote(self,
                         login=settings.SSH_CREDENTIALS['login'],
                         password=settings.SSH_CREDENTIALS['password']):
        """SSH to admin node
        :rtype : SSHClient
        """
        return self.nodes().admin.remote(self.admin_net,
                                         login=login,
                                         password=password)

    @logwrap
    def get_admin_node_ip(self):
        return str(self.nodes().admin.get_ip_address_by_network_name(
            self.admin_net))

    @logwrap
    def get_ebtables(self, cluster_id, devops_nodes):
        return Ebtables(self.get_target_devs(devops_nodes),
                        self.fuel_web.client.get_cluster_vlans(cluster_id))

    def get_host_node_ip(self):
        return self.router()

    def get_keys(self, node, custom=None, build_images=None):
        params = {
            'ip': node.get_ip_address_by_network_name(self.admin_net),
            'mask': self.get_net_mask(self.admin_net),
            'gw': self.router(),
            'hostname': '.'.join((self.hostname, self.domain)),
            'nat_interface': self.nat_interface,
            'dns1': settings.DNS,
            'showmenu': 'yes' if custom else 'no',
            'build_images': '1' if build_images else '0'
        }
        keys = ("<Wait>\n"
                "<Esc><Enter>\n"
                "<Wait>\n"
                "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n"
                " ip=%(ip)s\n"
                " netmask=%(mask)s\n"
                " gw=%(gw)s\n"
                " dns1=%(dns1)s\n"
                " hostname=%(hostname)s\n"
                " dhcp_interface=%(nat_interface)s\n"
                " showmenu=%(showmenu)s\n"
                " build_images=%(build_images)s\n"
                " <Enter>\n") % params
        return keys

    @logwrap
    def get_private_keys(self, force=False):
        if force or self._keys is None:
            self._keys = []
            for key_string in [
                    '/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa'
            ]:
                with self.get_admin_remote().open(key_string) as f:
                    self._keys.append(RSAKey.from_private_key(f))
        return self._keys

    @logwrap
    def get_ssh_to_remote(self, ip):
        return SSHClient(ip,
                         username=settings.SSH_CREDENTIALS['login'],
                         password=settings.SSH_CREDENTIALS['password'],
                         private_keys=self.get_private_keys())

    @logwrap
    def get_ssh_to_remote_by_key(self, ip, keyfile):
        try:
            with open(keyfile) as f:
                keys = [RSAKey.from_private_key(f)]
                return SSHClient(ip, private_keys=keys)
        except IOError:
            logger.warning('Loading of SSH key from file failed. Trying to use'
                           ' SSH agent ...')
            keys = Agent().get_keys()
            return SSHClient(ip, private_keys=keys)

    @logwrap
    def get_ssh_to_remote_by_name(self, node_name):
        return self.get_ssh_to_remote(
            self.fuel_web.get_nailgun_node_by_devops_node(
                self.get_virtual_environment().node_by_name(node_name))['ip'])

    def get_target_devs(self, devops_nodes):
        return [
            interface.target_dev for interface in [
                val for var in map(lambda node: node.interfaces, devops_nodes)
                for val in var
            ]
        ]

    def get_virtual_environment(self):
        """Returns virtual environment
        :rtype : devops.models.Environment
        """
        if self._virtual_environment is None:
            self._virtual_environment = self._get_or_create()
        return self._virtual_environment

    def get_network(self, net_name):
        return str(
            IPNetwork(self.get_virtual_environment().network_by_name(
                net_name).ip_network))

    def get_net_mask(self, net_name):
        return str(
            IPNetwork(self.get_virtual_environment().network_by_name(
                net_name).ip_network).netmask)

    def make_snapshot(self, snapshot_name, description="", is_make=False):
        if settings.MAKE_SNAPSHOT or is_make:
            self.get_virtual_environment().suspend(verbose=False)
            time.sleep(10)
            self.get_virtual_environment().snapshot(snapshot_name, force=True)
            revert_info(snapshot_name, description)
        if settings.FUEL_STATS_CHECK:
            self.get_virtual_environment().resume()
            try:
                self.nodes().admin. await (self.admin_net, timeout=60)
            except Exception:
                logger.error('Admin node is unavailable via SSH after '
                             'environment resume ')
                raise

    def nailgun_nodes(self, devops_nodes):
        return map(
            lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node),
            devops_nodes)

    def nodes(self):
        return Nodes(self.get_virtual_environment(), self.node_roles)

    def revert_snapshot(self, name):
        if self.get_virtual_environment().has_snapshot(name):
            logger.info('We have snapshot with such name %s' % name)

            self.get_virtual_environment().revert(name)
            logger.info('Starting snapshot reverting ....')

            self.get_virtual_environment().resume()
            logger.info('Starting snapshot resuming ...')

            admin = self.nodes().admin

            try:
                admin. await (self.admin_net, timeout=10 * 60, by_port=8000)
            except Exception as e:
                logger.warning("From first time admin isn't reverted: "
                               "{0}".format(e))
                admin.destroy()
                logger.info('Admin node was destroyed. Wait 10 sec.')
                time.sleep(10)
                self.get_virtual_environment().start(self.nodes().admins)
                logger.info('Admin node started second time.')
                self.nodes().admin. await (self.admin_net,
                                           timeout=10 * 60,
                                           by_port=8000)

            self.set_admin_ssh_password()
            try:
                _wait(self._fuel_web.client.get_releases,
                      expected=EnvironmentError,
                      timeout=300)
            except exceptions.Unauthorized:
                self.set_admin_keystone_password()
                self._fuel_web.get_nailgun_version()

            self.sync_time_admin_node()

            for node in self.nodes().slaves:
                if not node.driver.node_active(node):
                    continue
                try:
                    logger.info("Sync time on revert for node %s" % node.name)
                    self.sync_node_time(
                        self.get_ssh_to_remote_by_name(node.name))
                except Exception as e:
                    logger.warning(
                        'Exception caught while trying to sync time on {0}:'
                        ' {1}'.format(node.name, e))
                self.run_nailgun_agent(
                    self.get_ssh_to_remote_by_name(node.name))
            return True
        return False

    def set_admin_ssh_password(self):
        try:
            remote = self.get_admin_remote(
                login=settings.SSH_CREDENTIALS['login'],
                password=settings.SSH_CREDENTIALS['password'])
            self.execute_remote_cmd(remote, 'date')
            logger.debug('Accessing admin node using SSH: SUCCESS')
        except Exception:
            logger.debug('Accessing admin node using SSH credentials:'
                         ' FAIL, trying to change password from default')
            remote = self.get_admin_remote(login='******', password='******')
            self.execute_remote_cmd(
                remote, 'echo -e "{1}\\n{1}" | passwd {0}'.format(
                    settings.SSH_CREDENTIALS['login'],
                    settings.SSH_CREDENTIALS['password']))
            logger.debug("Admin node password has changed.")
        logger.info("Admin node login name: '{0}' , password: '******'".format(
            settings.SSH_CREDENTIALS['login'],
            settings.SSH_CREDENTIALS['password']))

    def set_admin_keystone_password(self):
        remote = self.get_admin_remote()
        try:
            self._fuel_web.client.get_releases()
        except exceptions.Unauthorized:
            self.execute_remote_cmd(
                remote, 'fuel user --newpass {0} --change-password'.format(
                    settings.KEYSTONE_CREDS['password']))
            logger.info(
                'New Fuel UI (keystone) username: "******", password: "******"'.
                format(settings.KEYSTONE_CREDS['username'],
                       settings.KEYSTONE_CREDS['password']))

    def setup_environment(self,
                          custom=settings.CUSTOM_ENV,
                          build_images=settings.BUILD_IMAGES):
        # start admin node
        admin = self.nodes().admin
        admin.disk_devices.get(device='cdrom').volume.upload(settings.ISO_PATH)
        self.get_virtual_environment().start(self.nodes().admins)
        logger.info("Waiting for admin node to start up")
        wait(lambda: admin.driver.node_active(admin), 60)
        logger.info("Proceed with installation")
        # update network parameters at boot screen
        admin.send_keys(
            self.get_keys(admin, custom=custom, build_images=build_images))
        if custom:
            self.setup_customisation()
        # wait while installation complete
        admin. await (self.admin_net, timeout=10 * 60)
        self.set_admin_ssh_password()
        self.wait_bootstrap()
        time.sleep(10)
        self.set_admin_keystone_password()
        self.sync_time_admin_node()
        if settings.MULTIPLE_NETWORKS:
            self.describe_second_admin_interface()
            multiple_networks_hacks.configure_second_admin_cobbler(self)
            multiple_networks_hacks.configure_second_dhcrelay(self)
        self.nailgun_actions.set_collector_address(settings.FUEL_STATS_HOST,
                                                   settings.FUEL_STATS_PORT,
                                                   settings.FUEL_STATS_SSL)
        if settings.FUEL_STATS_ENABLED:
            self.fuel_web.client.send_fuel_stats(enabled=True)
            logger.info('Enabled sending of statistics to {0}:{1}'.format(
                settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT))

    @upload_manifests
    def wait_for_provisioning(self):
        _wait(lambda: _tcp_ping(
            self.nodes().admin.get_ip_address_by_network_name(self.admin_net),
            22),
              timeout=5 * 60)

    def setup_customisation(self):
        self.wait_for_provisioning()
        try:
            remote = self.get_admin_remote()
            pid = remote.execute("pgrep 'fuelmenu'")['stdout'][0]
            pid.rstrip('\n')
            remote.execute("kill -sigusr1 {0}".format(pid))
        except Exception:
            logger.error("Could not kill pid of fuelmenu")
            raise

    @retry(count=10, delay=60)
    @logwrap
    def sync_node_time(self, remote):
        self.execute_remote_cmd(remote, 'hwclock -s')
        self.execute_remote_cmd(
            remote, 'NTPD=$(find /etc/init.d/ -regex \''
            '/etc/init.d/\(ntp.?\|ntp-dev\)\');'
            '$NTPD stop && ntpd -dqg && $NTPD '
            'start')
        self.execute_remote_cmd(remote, 'hwclock -w')
        remote_date = remote.execute('date')['stdout']
        logger.info("Node time: %s" % remote_date)

    @retry(count=10, delay=60)
    @logwrap
    def sync_time_admin_node(self):
        logger.info("Sync time on revert for admin")
        remote = self.get_admin_remote()
        self.execute_remote_cmd(remote, 'hwclock -s')
        # Sync time using ntpd
        try:
            # If public NTP servers aren't accessible ntpdate will fail and
            # ntpd daemon shouldn't be restarted to avoid 'Server has gone
            # too long without sync' error while syncing time from slaves
            self.execute_remote_cmd(
                remote, "ntpdate -vu $(awk '/^server/ && "
                "$2 !~ /127.*/ {print $2}' "
                "/etc/ntp.conf)")
        except AssertionError as e:
            logger.warning('Error occurred while synchronizing time on master'
                           ': {0}'.format(e))
            raise
        else:
            self.execute_remote_cmd(
                remote, 'service ntpd stop && ntpd -dqg &&'
                ' service ntpd start')
            self.execute_remote_cmd(remote, 'hwclock -w')

        remote_date = remote.execute('date')['stdout']
        logger.info("Master node time: {0}".format(remote_date))

    def verify_network_configuration(self, node_name):
        checkers.verify_network_configuration(
            node=self.fuel_web.get_nailgun_node_by_name(node_name),
            remote=self.get_ssh_to_remote_by_name(node_name))

    def wait_bootstrap(self):
        logger.info("Waiting while bootstrapping is in progress")
        log_path = "/var/log/puppet/bootstrap_admin_node.log"
        logger.info("Puppet timeout set in {0}".format(
            float(settings.PUPPET_TIMEOUT)))
        wait(lambda: not self.get_admin_remote().execute(
            "grep 'Fuel node deployment' '%s'" % log_path)['exit_code'],
             timeout=(float(settings.PUPPET_TIMEOUT)))
        result = self.get_admin_remote().execute("grep 'Fuel node deployment "
                                                 "complete' '%s'" %
                                                 log_path)['exit_code']
        if result != 0:
            raise Exception('Fuel node deployment failed.')

    def dhcrelay_check(self):
        admin_remote = self.get_admin_remote()
        out = admin_remote.execute("dhcpcheck discover "
                                   "--ifaces eth0 "
                                   "--repeat 3 "
                                   "--timeout 10")['stdout']

        assert_true(self.get_admin_node_ip() in "".join(out),
                    "dhcpcheck doesn't discover master ip")

    def run_nailgun_agent(self, remote):
        agent = remote.execute('/opt/nailgun/bin/agent')['exit_code']
        logger.info("Nailgun agent run with exit_code: %s" % agent)

    def get_fuel_settings(self, remote=None):
        if not remote:
            remote = self.get_admin_remote()
        cmd = 'cat {cfg_file}'.format(cfg_file=settings.FUEL_SETTINGS_YAML)
        result = remote.execute(cmd)
        if result['exit_code'] == 0:
            fuel_settings = yaml.load(''.join(result['stdout']))
        else:
            raise Exception('Can\'t output {cfg_file} file: {error}'.format(
                cfg_file=settings.FUEL_SETTINGS_YAML, error=result['stderr']))
        return fuel_settings

    def admin_install_pkg(self, pkg_name):
        """Install a package <pkg_name> on the admin node"""
        admin_remote = self.get_admin_remote()
        remote_status = admin_remote.execute("rpm -q {0}'".format(pkg_name))
        if remote_status['exit_code'] == 0:
            logger.info("Package '{0}' already installed.".format(pkg_name))
        else:
            logger.info("Installing package '{0}' ...".format(pkg_name))
            remote_status = admin_remote.execute(
                "yum -y install {0}".format(pkg_name))
            logger.info("Installation of the package '{0}' has been"
                        " completed with exit code {1}".format(
                            pkg_name, remote_status['exit_code']))
        return remote_status['exit_code']

    def admin_run_service(self, service_name):
        """Start a service <service_name> on the admin node"""
        admin_remote = self.get_admin_remote()
        admin_remote.execute("service {0} start".format(service_name))
        remote_status = admin_remote.execute(
            "service {0} status".format(service_name))
        if any('running...' in status for status in remote_status['stdout']):
            logger.info("Service '{0}' is running".format(service_name))
        else:
            logger.info("Service '{0}' failed to start"
                        " with exit code {1} :\n{2}".format(
                            service_name, remote_status['exit_code'],
                            remote_status['stdout']))

    # Modifies a resolv.conf on the Fuel master node and returns
    # its original content.
    # * adds 'nameservers' at start of resolv.conf if merge=True
    # * replaces resolv.conf with 'nameservers' if merge=False
    def modify_resolv_conf(self, nameservers=[], merge=True):
        remote = self.get_admin_remote()
        resolv_conf = remote.execute('cat /etc/resolv.conf')
        assert_equal(
            0, resolv_conf['exit_code'], 'Executing "{0}" on the '
            'admin node has failed with: {1}'.format('cat /etc/resolv.conf',
                                                     resolv_conf['stderr']))
        if merge:
            nameservers.extend(resolv_conf['stdout'])

        resolv_keys = ['search', 'domain', 'nameserver']
        resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers
                             if any(x in ns for x in resolv_keys))
        logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new))
        echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new)
        echo_result = remote.execute(echo_cmd)
        assert_equal(
            0, echo_result['exit_code'], 'Executing "{0}" on the '
            'admin node has failed with: {1}'.format(echo_cmd,
                                                     echo_result['stderr']))
        return resolv_conf['stdout']

    @logwrap
    def execute_remote_cmd(self, remote, cmd, exit_code=0):
        result = remote.execute(cmd)
        assert_equal(
            result['exit_code'], exit_code,
            'Failed to execute "{0}" on remote host: {1}'.format(cmd, result))
        return result['stdout']

    @logwrap
    def describe_second_admin_interface(self):
        remote = self.get_admin_remote()
        second_admin_network = self.get_network(self.admin_net2).split('/')[0]
        second_admin_netmask = self.get_net_mask(self.admin_net2)
        second_admin_if = settings.INTERFACES.get(self.admin_net2)
        second_admin_ip = str(
            self.nodes().admin.get_ip_address_by_network_name(self.admin_net2))
        logger.info(
            ('Parameters for second admin interface configuration: '
             'Network - {0}, Netmask - {1}, Interface - {2}, '
             'IP Address - {3}').format(second_admin_network,
                                        second_admin_netmask, second_admin_if,
                                        second_admin_ip))
        add_second_admin_ip = ('DEVICE={0}\\n'
                               'ONBOOT=yes\\n'
                               'NM_CONTROLLED=no\\n'
                               'USERCTL=no\\n'
                               'PEERDNS=no\\n'
                               'BOOTPROTO=static\\n'
                               'IPADDR={1}\\n'
                               'NETMASK={2}\\n').format(
                                   second_admin_if, second_admin_ip,
                                   second_admin_netmask)
        cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};'
               'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format(
                   add_second_admin_ip, second_admin_if, second_admin_ip)
        logger.debug(
            'Trying to assign {0} IP to the {1} on master node...'.format(
                second_admin_ip, second_admin_if))
        result = remote.execute(cmd)
        assert_equal(result['exit_code'], 0,
                     ('Failed to assign second admin '
                      'IP address on master node: {0}').format(result))
        logger.debug('Done: {0}'.format(result['stdout']))
        multiple_networks_hacks.configure_second_admin_firewall(
            self, second_admin_network, second_admin_netmask)

    @logwrap
    def get_masternode_uuid(self):
        return self.postgres_actions.run_query(
            db='nailgun',
            query="select master_node_uid from master_node_settings limit 1;")
Exemple #30
0
class CiBase(object):
    def __init__(self):
        self.manager = Manager()
        self._environment = None
        self.saved_environment_states = {}

    def _get_or_create(self):
        try:
            return self.manager.environment_get(self.env_name())
        except:
            self._environment = self.describe_environment()
            self._environment.define()
            return self._environment

    def get_state(self, name):
        if self.environment().has_snapshot(name):
            self.environment().revert(name)
            return True
        return False

    def environment(self):
        """
        :rtype : devops.models.Environment
        """
        self._environment = self._environment or self._get_or_create()
        return self._environment

    @abstractproperty
    def env_name(self):
        """
        :rtype : string
        """
        pass

    @abstractmethod
    def describe_environment(self):
        """
        :rtype : devops.models.Environment
        """
        pass

    @abstractproperty
    def node_roles(self):
        """
        :rtype : NodeRoles
        """
        pass

    def nodes(self):
        return Nodes(self.environment(), self.node_roles())

    # noinspection PyShadowingBuiltins
    def add_empty_volume(self, node, name, capacity=20 * 1024 * 1024 * 1024,
                         device='disk', bus='virtio', format='qcow2'):
        self.manager.node_attach_volume(
            node=node,
            volume=self.manager.volume_create(
                name=name, capacity=capacity,
                environment=self.environment(),
                format=format),
            device=device, bus=bus)

    def add_node(self, memory, name, boot=None):
        return self.manager.node_create(
            name=name,
            memory=memory,
            environment=self.environment(),
            boot=boot)

    def create_interfaces(self, networks, node):
        for network in networks:
            if network.name == 'internal':
                self.manager.interface_create(network, node=node)
                self.manager.interface_create(network, node=node)
            self.manager.interface_create(network, node=node)

    def describe_admin_node(self, name, networks, memory=1024):
        node = self.add_node(memory=memory, name=name, boot=['hd', 'cdrom'])
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + '-system')
        self.add_empty_volume(
            node, name + '-iso', capacity=_get_file_size(ISO_PATH),
            format='raw', device='cdrom', bus='ide')
        return node

    def describe_empty_node(self, name, networks, memory=1024):
        node = self.add_node(memory, name)
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + '-system')
        self.add_empty_volume(node, name + '-cinder')
        self.add_empty_volume(node, name + '-swift')
        return node

    @abstractmethod
    def setup_environment(self):
        """
        :rtype : None
        """
        pass

    def get_empty_environment(self):
        if not(self.get_state(EMPTY_SNAPSHOT)):
            self.setup_environment()
            self.environment().snapshot(EMPTY_SNAPSHOT)

    def generate_state_hash(self, settings):
        return hashlib.md5(str(settings)).hexdigest()

    def revert_to_state(self, settings):
        state_hash = self.generate_state_hash(settings)

        empty_state_hash = self.generate_state_hash({})
        if state_hash == empty_state_hash:
            # revert to empty state
            self.get_empty_environment()
            return True

        if state_hash in self.saved_environment_states:
            # revert to matching state
            state = self.saved_environment_states[state_hash]
            if not(self.get_state(state['snapshot_name'])):
                return False
            self.environment().resume()
            return True

        return False

    def snapshot_state(self, name, settings):
        state_hash = self.generate_state_hash(settings)
        snapshot_name = '{0}_{1}'.format(
                name.replace(' ', '_')[:17], state_hash)
        self.environment().suspend(verbose=False)
        self.environment().snapshot(
            name=snapshot_name,
            description=name,
            force=True,
        )
        self.environment().resume(verbose=False)
        self.saved_environment_states[state_hash] = {
            'snapshot_name': snapshot_name,
            'cluster_name': name,
            'settings': settings
        }

    def internal_virtual_ip(self):
        return str(IPNetwork(
            self.environment().network_by_name('internal').ip_network)[-2])

    def public_router(self):
        return str(
            IPNetwork(
                self.environment().network_by_name('public').ip_network)[1])

    def internal_router(self):
        return self._router('internal')

    def nat_router(self):
        return self._router('nat')

    def _router(self, router_name):
        return str(
            IPNetwork(
                self.environment().network_by_name(router_name).ip_network)[1])

    def get_host_node_ip(self):
        return self.internal_router()

    def internal_network(self):
        return str(
            IPNetwork(
                self.environment().network_by_name('internal').ip_network))

    def internal_net_mask(self):
        return str(IPNetwork(
            self.environment().network_by_name('internal').ip_network).netmask)

    def public_net_mask(self):
        return str(IPNetwork(
            self.environment().network_by_name('public').ip_network).netmask)

    def public_network(self):
        return str(
            IPNetwork(self.environment().network_by_name('public').ip_network))
Exemple #31
0
class EnvironmentModel(object):
    hostname = 'nailgun'
    domain = 'test.domain.local'
    installation_timeout = 1800
    deployment_timeout = 1800
    puppet_timeout = 1000
    nat_interface = ''  # INTERFACES.get('admin')
    admin_net = 'admin'

    def __init__(self, os_image=None):
        self._virtual_environment = None
        self._keys = None
        self.manager = Manager()
        self.os_image = os_image
        self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self)

    def _get_or_create(self):
        try:
            return self.manager.environment_get(self.env_name)
        except Exception:
            self._virtual_environment = self.describe_environment()
            self._virtual_environment.define()
            return self._virtual_environment

    def router(self, router_name=None):
        router_name = router_name or self.admin_net
        return str(
            IPNetwork(
                self.get_virtual_environment().network_by_name(router_name).
                ip_network)[1])

    @property
    def fuel_web(self):
        """FuelWebClient
        :rtype: FuelWebClient
        """
        return self._fuel_web

    @property
    def node_roles(self):
        return NodeRoles(
            admin_names=['admin'],
            other_names=['slave-%02d' % x for x in range(1, int(
                settings.NODES_COUNT))]
        )

    @property
    def env_name(self):
        return settings.ENV_NAME

    def add_empty_volume(self, node, name,
                         capacity=settings.NODE_VOLUME_SIZE * 1024 * 1024
                         * 1024, device='disk', bus='virtio', format='qcow2'):
        self.manager.node_attach_volume(
            node=node,
            volume=self.manager.volume_create(
                name=name,
                capacity=capacity,
                environment=self.get_virtual_environment(),
                format=format),
            device=device,
            bus=bus)

    def add_node(self, memory, name, vcpu=1, boot=None):
        return self.manager.node_create(
            name=name,
            memory=memory,
            vcpu=vcpu,
            environment=self.get_virtual_environment(),
            boot=boot)

    @logwrap
    def add_syslog_server(self, cluster_id, port=5514):
        self.fuel_web.add_syslog_server(
            cluster_id, self.get_host_node_ip(), port)

    def bootstrap_nodes(self, devops_nodes, timeout=600):
        """Lists registered nailgun nodes
        Start vms and wait until they are registered on nailgun.
        :rtype : List of registered nailgun nodes
        """
        for node in devops_nodes:
            node.start()
        wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout)

        for node in self.nailgun_nodes(devops_nodes):
            self.sync_node_time(self.get_ssh_to_remote(node["ip"]))

        return self.nailgun_nodes(devops_nodes)

    def create_interfaces(self, networks, node,
                          model=settings.INTERFACE_MODEL):
        for network in networks:
            self.manager.interface_create(network, node=node, model=model)

    def describe_environment(self):
        """Environment
        :rtype : Environment
        """
        environment = self.manager.environment_create(self.env_name)
        networks = []
        for name in settings.INTERFACE_ORDER:
            ip_networks = [
                IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',')]
            new_prefix = int(settings.POOLS.get(name)[1])
            pool = self.manager.create_network_pool(networks=ip_networks,
                                                    prefix=int(new_prefix))
            networks.append(self.manager.network_create(
                name=name,
                environment=environment,
                pool=pool,
                forward=settings.FORWARDING.get(name),
                has_dhcp_server=settings.DHCP.get(name)))

        for name in self.node_roles.admin_names:
            self.describe_admin_node(name, networks)
        for name in self.node_roles.other_names:
            self.describe_empty_node(name, networks)
        return environment

    def devops_nodes_by_names(self, devops_node_names):
        return map(
            lambda name:
            self.get_virtual_environment().node_by_name(name),
            devops_node_names)

    @logwrap
    def describe_admin_node(self, name, networks):
        node = self.add_node(
            memory=settings.HARDWARE.get("admin_node_memory", 1024),
            vcpu=settings.HARDWARE.get("admin_node_cpu", 1),
            name=name,
            boot=['hd', 'cdrom'])
        self.create_interfaces(networks, node)

        if self.os_image is None:
            self.add_empty_volume(node, name + '-system')
            self.add_empty_volume(
                node,
                name + '-iso',
                capacity=_get_file_size(settings.ISO_PATH),
                format='raw',
                device='cdrom',
                bus='ide')
        else:
            volume = self.manager.volume_get_predefined(self.os_image)
            vol_child = self.manager.volume_create_child(
                name=name + '-system',
                backing_store=volume,
                environment=self.get_virtual_environment()
            )
            self.manager.node_attach_volume(
                node=node,
                volume=vol_child
            )
        return node

    def describe_empty_node(self, name, networks):
        node = self.add_node(
            name=name,
            memory=settings.HARDWARE.get("slave_node_memory", 1024),
            vcpu=settings.HARDWARE.get("slave_node_cpu", 1))
        self.create_interfaces(networks, node)
        self.add_empty_volume(node, name + '-system')

        if settings.USE_ALL_DISKS:
            self.add_empty_volume(node, name + '-cinder')
            self.add_empty_volume(node, name + '-swift')

        return node

    @logwrap
    def get_admin_remote(self):
        """SSH to admin node
        :rtype : SSHClient
        """
        return self.nodes().admin.remote(self.admin_net,
                                         login='******',
                                         password='******')

    @logwrap
    def get_admin_node_ip(self):
        return str(
            self.nodes().admin.get_ip_address_by_network_name(self.admin_net))

    @logwrap
    def get_ebtables(self, cluster_id, devops_nodes):
        return Ebtables(self.get_target_devs(devops_nodes),
                        self.fuel_web.client.get_cluster_vlans(cluster_id))

    def get_host_node_ip(self):
        return self.router()

    def get_keys(self, node):
        params = {
            'ip': node.get_ip_address_by_network_name(self.admin_net),
            'mask': self.get_net_mask(self.admin_net),
            'gw': self.router(),
            'hostname': '.'.join((self.hostname, self.domain)),
            'nat_interface': self.nat_interface,
            'dns1': settings.DNS

        }
        keys = (
            "<Esc><Enter>\n"
            "<Wait>\n"
            "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n"
            " ip=%(ip)s\n"
            " netmask=%(mask)s\n"
            " gw=%(gw)s\n"
            " dns1=%(dns1)s\n"
            " hostname=%(hostname)s\n"
            " dhcp_interface=%(nat_interface)s\n"
            " <Enter>\n"
        ) % params
        return keys

    @logwrap
    def get_private_keys(self, force=False):
        if force or self._keys is None:
            self._keys = []
            for key_string in ['/root/.ssh/id_rsa',
                               '/root/.ssh/bootstrap.rsa']:
                with self.get_admin_remote().open(key_string) as f:
                    self._keys.append(RSAKey.from_private_key(f))
        return self._keys

    @logwrap
    def get_ssh_to_remote(self, ip):
        return SSHClient(ip,
                         username='******',
                         password='******',
                         private_keys=self.get_private_keys())

    @logwrap
    def get_ssh_to_remote_by_name(self, node_name):
        return self.get_ssh_to_remote(
            self.fuel_web.get_nailgun_node_by_devops_node(
                self.get_virtual_environment().node_by_name(node_name))['ip']
        )

    def get_target_devs(self, devops_nodes):
        return [
            interface.target_dev for interface in [
                val for var in map(lambda node: node.interfaces, devops_nodes)
                for val in var]]

    def get_virtual_environment(self):
        """Returns virtual environment
        :rtype : devops.models.Environment
        """
        if self._virtual_environment is None:
            self._virtual_environment = self._get_or_create()
        return self._virtual_environment

    def get_network(self, net_name):
        return str(
            IPNetwork(
                self.get_virtual_environment().network_by_name(net_name).
                ip_network))

    def get_net_mask(self, net_name):
        return str(
            IPNetwork(
                self.get_virtual_environment().network_by_name(
                    net_name).ip_network).netmask)

    def make_snapshot(self, snapshot_name, description=""):
        self.get_virtual_environment().suspend(verbose=False)
        self.get_virtual_environment().snapshot(snapshot_name, force=True)
        revert_info(snapshot_name, description)

    def nailgun_nodes(self, devops_nodes):
        return map(
            lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node),
            devops_nodes
        )

    def nodes(self):
        return Nodes(self.get_virtual_environment(), self.node_roles)

    def revert_snapshot(self, name):
        if self.get_virtual_environment().has_snapshot(name):
            logger.info('We have snapshot with such name %s' % name)

            self.get_virtual_environment().revert(name)
            logger.info('Starting snapshot reverting ....')

            self.get_virtual_environment().resume()
            logger.info('Starting snapshot resuming ...')

            self.nodes().admin.await(self.admin_net, timeout=10 * 60)

            list = self.nodes().slaves
            for node in list:
                nailgun_node = self.fuel_web.get_nailgun_node_by_devops_node(
                    node)
                if not node.driver.node_active(node) or \
                        nailgun_node['status'] not in ['provisioned', 'ready']:
                    continue
                try:
                    self.sync_node_time(self.get_ssh_to_remote(
                        node.get_ip_address_by_network_name(self.admin_net)))
                except Exception as e:
                    logger.warn(
                        'Paramiko exception catched while'
                        ' trying to run ntpdate: %s' % e)
            return True
        return False

    def setup_environment(self):
        # start admin node
        admin = self.nodes().admin
        admin.disk_devices.get(device='cdrom').volume.upload(settings.ISO_PATH)
        self.get_virtual_environment().start(self.nodes().admins)
        # update network parameters at boot screen
        time.sleep(float(settings.ADMIN_NODE_SETUP_TIMEOUT))
        admin.send_keys(self.get_keys(admin))
        # wait while installation complete
        admin.await(self.admin_net, timeout=10 * 60)
        self.wait_bootstrap()
        time.sleep(10)
        self.sync_time_admin_node()

    @logwrap
    def sync_node_time(self, remote):
        remote.execute('hwclock --hctosys')
        remote.execute("ntpdate -u "
                       "$(egrep '^server' /etc/ntp.conf |"
                       " sed '/^#/d' | awk '{print $2}' |"
                       " tail -n 1)")
        remote.execute('hwclock -w')

    def sync_time_admin_node(self):
        self.sync_node_time(self.get_admin_remote())

    def verify_node_service_list(self, node_name, smiles_count):
        remote = self.get_ssh_to_remote_by_name(node_name)
        checkers.verify_service_list(remote, smiles_count)

    def verify_network_configuration(self, node_name):
        checkers.verify_network_configuration(
            node=self.fuel_web.get_nailgun_node_by_name(node_name),
            remote=self.get_ssh_to_remote_by_name(node_name)
        )

    def wait_bootstrap(self):
        logger.info("Waiting while bootstrapping is in progress")
        log_path = "/var/log/puppet/bootstrap_admin_node.log"
        wait(
            lambda: not
            self.get_admin_remote().execute(
                "grep 'Finished catalog run' '%s'" % log_path
            )['exit_code'],
            timeout=self.puppet_timeout
        )
Exemple #32
0
 def __init__(self):
     self.manager = Manager()
     self._environment = None
     self.saved_environment_states = {}
Exemple #33
0
 def __init__(self):
     super(Shell, self).__init__()
     self.params = self.get_params()
     self.manager = Manager()
Exemple #34
0
class CiBase(object):
    def __init__(self):
        self._environment = None
        self.manager = Manager()
        # self.base_image = self.manager.volume_get_predefined(BASE_IMAGE)

    def get_or_create(self):
        try:
            return self.manager.environment_get(self.env_name())
        except:
            self._environment = self.describe_environment()
            self._environment.define()
            return self._environment

    def get_empty_state(self):
        if self.environment().has_snapshot(EMPTY_SNAPSHOT):
            self.environment().revert(EMPTY_SNAPSHOT)
        else:
            self.setup_environment()

    def environment(self):
        """
        :rtype : devops.models.Environment
        """
        self._environment = self._environment or self.get_or_create()
        return self._environment

    @abstractproperty
    def env_name(self):
        """
        :rtype : string
        """
        pass

    @abstractmethod
    def define(self):
        """
        :rtype : devops.models.Environment
        """
        pass

    @abstractmethod
    def describe_environment(self):
        """
        :rtype : devops.models.Environment
        """
        pass

    @abstractproperty
    def node_roles(self):
        """
        :rtype : NodeRoles
        """
        pass

    def add_empty_volume(self,
                         node,
                         name,
                         capacity=20 * 1024 * 1024 * 1024,
                         format="qcow2",
                         device="disk",
                         bus='virtio'):
        self.manager.node_attach_volume(node=node,
                                        device=device,
                                        bus=bus,
                                        volume=self.manager.volume_create(
                                            name=name,
                                            capacity=capacity,
                                            format=format,
                                            environment=self.environment()))

    def add_node(self, memory, name, boot=None):
        return self.manager.node_create(name=name,
                                        memory=memory,
                                        environment=self.environment())

    def describe_master_node(self, name, networks, memory=1024):
        node = self.add_node(memory, name, boot=['cdrom', 'hd'])
        for network in networks:
            self.manager.interface_create(network, node=node)
        self.add_empty_volume(node, name + '-system')
        self.add_empty_volume(node,
                              name + '-iso',
                              capacity=_get_file_size(ISO_IMAGE),
                              format='raw',
                              device='cdrom',
                              bus='ide')
        return node

    def describe_empty_node(self, name, networks, memory=1024):
        node = self.add_node(memory, name)
        for network in networks:
            self.manager.interface_create(network, node=node)
        self.add_empty_volume(node, name + '-system')
        self.add_empty_volume(node, name + '-cinder')
        return node

    def nodes(self):
        return Nodes(self.environment(), self.node_roles())

    def add_nodes_to_hosts(self, remote, nodes):
        for node in nodes:
            add_to_hosts(remote,
                         node.get_ip_address_by_network_name('internal'),
                         node.name, node.name + '.localdomain')

    def setup_master_node(self, master_remote, nodes):
        setup_puppet_master(master_remote)
        add_nmap(master_remote)
        switch_off_ip_tables(master_remote)
        self.add_nodes_to_hosts(master_remote, nodes)

    def setup_agent_nodes(self, nodes):
        agent_config = load(root('fuel_test', 'config', 'puppet.agent.config'))
        for node in nodes:
            if node.name != 'master':
                remote = node.remote('public', login='******', password='******')
                self.add_nodes_to_hosts(remote, self.environment().nodes)
                setup_puppet_client(remote)
                write_config(remote, '/etc/puppet/puppet.conf', agent_config)
                request_cerificate(remote)

    def rename_nodes(self, nodes):
        for node in nodes:
            remote = node.remote('public', login='******', password='******')
            change_host_name(remote, node.name, node.name + '.localdomain')
            logging.info("Renamed %s" % node.name)

    @abstractmethod
    def setup_environment(self):
        """
        :rtype : None
        """
        pass

    def internal_virtual_ip(self):
        return str(
            IPNetwork(
                self.environment().network_by_name('internal').ip_network)[-2])

    def floating_network(self):
        return str(
            IPNetwork(self.environment().network_by_name(
                'public').ip_network).subnet(new_prefix=29)[-1])

    def public_virtual_ip(self):
        return str(
            IPNetwork(self.environment().network_by_name(
                'public').ip_network).subnet(new_prefix=29)[-2][-1])

    def public_router(self):
        return str(
            IPNetwork(
                self.environment().network_by_name('public').ip_network)[1])

    def internal_router(self):
        return str(
            IPNetwork(
                self.environment().network_by_name('internal').ip_network)[1])

    def fixed_network(self):
        return str(
            IPNetwork(self.environment().network_by_name(
                'private').ip_network).subnet(new_prefix=27)[0])

    def internal_network(self):
        return str(
            IPNetwork(
                self.environment().network_by_name('internal').ip_network))

    def internal_net_mask(self):
        return str(
            IPNetwork(self.environment().network_by_name(
                'internal').ip_network).netmask)

    def public_net_mask(self):
        return str(
            IPNetwork(self.environment().network_by_name(
                'public').ip_network).netmask)

    def public_network(self):
        return str(
            IPNetwork(self.environment().network_by_name('public').ip_network))
Exemple #35
0
 def __init__(self):
     """
     Constructor. Define environment and manager for manipulation with it.
     """
     self._environment = None
     self.manager = Manager()
Exemple #36
0
 def __init__(self):
     self._environment = None
     self.manager = Manager()
 def __init__(self, os_image=None):
     self._virtual_environment = None
     self._keys = None
     self.manager = Manager()
     self.os_image = os_image
     self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self)
Exemple #38
0
        #'r00tme').check_stderr('ls -la', verbose=True)
        #remotes.append(node.remote('external', 'root', 'r00tme'))
    #SSHClient.execute_together(remotes, 'hostname -f; ifconfig eth0')


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Create environment or add'
                                     'VM to environment created via devops.')
    parser.add_argument('command',
                        choices=['create-env', 'add-node'],
                        help='command')
    parser.add_argument('-e',
                        '--env-name',
                        help='name of the environment to create')
    parser.add_argument('-n', '--name', help='name of the VM to create')
    args = parser.parse_args()

    env_name = args.env_name
    node_name = args.name

    template_volume = '/media/build/libvirt_default/vm_tempest_template.img'
    #template_volume = '/media/build/libvirt_default/vm_ubuntu_initial.img'
    #template_volume = '/var/lib/libvirt/images/vm_ubuntu_initial.img'
    #template_volume = '/var/lib/libvirt/images/vm_tempest_template.img'

    from devops.manager import Manager
    if args.command == 'create-env':
        create_env(Manager())
    elif args.command == 'add-node':
        add_node(Manager(), env_name, node_name, template_volume)
Exemple #39
0
 def __init__(self):
     super(Shell, self).__init__()
     self.params = self.get_params()
     self.manager = Manager()
Exemple #40
0
class CiBase(object):
    def __init__(self):
        self.manager = Manager()
        self.base_image = self.manager.volume_get_predefined(BASE_IMAGE)
        self._environment = None

    def _get_or_create(self):
        try:
            return self.manager.environment_get(self.env_name())
        except:
            self._environment = self.describe_environment()
            self._environment.define()
            return self._environment

    def get_empty_state(self):
        if self.environment().has_snapshot(EMPTY_SNAPSHOT):
            self.environment().revert(EMPTY_SNAPSHOT)
        else:
            self.setup_environment()

    def environment(self):
        """
        :rtype : devops.models.Environment
        """
        self._environment = self._environment or self._get_or_create()
        return self._environment

    @abstractproperty
    def env_name(self):
        """
        :rtype : string
        """
        pass

    @abstractmethod
    def describe_environment(self):
        """
        :rtype : devops.models.Environment
        """
        pass

    @abstractproperty
    def node_roles(self):
        """
        :rtype : NodeRoles
        """
        pass

    def nodes(self):
        return Nodes(self.environment(), self.node_roles())

    def add_empty_volume(self, node, name):
        self.manager.node_attach_volume(
            node=node,
            volume=self.manager.volume_create(
                name=name, capacity=20 * 1024 * 1024 * 1024, environment=self.environment()
            ),
        )

    def add_node(self, memory, name):
        return self.manager.node_create(name=name, memory=memory, environment=self.environment())

    def describe_node(self, name, networks, memory=1024):
        node = self.add_node(memory, name)
        for network in networks:
            self.manager.interface_create(network, node=node)
        self.manager.node_attach_volume(
            node=node,
            volume=self.manager.volume_create_child(
                name=name + "-system", backing_store=self.base_image, environment=self.environment()
            ),
        )
        self.add_empty_volume(node, name + "-cinder")
        return node

    def describe_empty_node(self, name, networks, memory=1024):
        node = self.add_node(memory, name)
        for network in networks:
            self.manager.interface_create(network, node=node)
        self.add_empty_volume(node, name + "-system")
        self.add_empty_volume(node, name + "-cinder")
        return node

    def add_nodes_to_hosts(self, remote, nodes):
        for node in nodes:
            add_to_hosts(
                remote, node.get_ip_address_by_network_name("internal"), node.name, node.name + ".your-domain-name.com"
            )

    def setup_master_node(self, master_remote, nodes):
        setup_puppet_master(master_remote)
        add_nmap(master_remote)
        switch_off_ip_tables(master_remote)
        self.add_nodes_to_hosts(master_remote, nodes)

    def setup_agent_nodes(self, nodes):
        agent_config = load(root("fuel_test", "config", "puppet.agent.config"))
        for node in nodes:
            if node.name != "master":
                remote = node.remote("public", login="******", password="******")
                self.add_nodes_to_hosts(remote, self.environment().nodes)
                setup_puppet_client(remote)
                write_config(remote, "/etc/puppet/puppet.conf", agent_config)
                request_cerificate(remote)

    def rename_nodes(self, nodes):
        for node in nodes:
            remote = node.remote("public", login="******", password="******")
            change_host_name(remote, node.name, node.name + ".your-domain-name.com")
            logging.info("Renamed %s" % node.name)

    @abstractmethod
    def setup_environment(self):
        """
        :rtype : None
        """
        pass

    def internal_virtual_ip(self):
        return str(IPNetwork(self.environment().network_by_name("internal").ip_network)[-2])

    def floating_network(self):
        return str(IPNetwork(self.environment().network_by_name("public").ip_network).subnet(new_prefix=29)[-1])

    def public_virtual_ip(self):
        return str(IPNetwork(self.environment().network_by_name("public").ip_network).subnet(new_prefix=29)[-2][-1])

    def public_router(self):
        return str(IPNetwork(self.environment().network_by_name("public").ip_network)[1])

    def internal_router(self):
        return str(IPNetwork(self.environment().network_by_name("internal").ip_network)[1])

    def fixed_network(self):
        return str(IPNetwork(self.environment().network_by_name("private").ip_network).subnet(new_prefix=27)[0])

    def internal_network(self):
        return str(IPNetwork(self.environment().network_by_name("internal").ip_network))

    def internal_net_mask(self):
        return str(IPNetwork(self.environment().network_by_name("internal").ip_network).netmask)

    def public_net_mask(self):
        return str(IPNetwork(self.environment().network_by_name("public").ip_network).netmask)

    def public_network(self):
        return str(IPNetwork(self.environment().network_by_name("public").ip_network))