class Shell(object): def __init__(self): super(Shell, self).__init__() self.params = self.get_params() self.manager = Manager() def execute(self): self.commands.get(self.params.command)(self) def do_list(self): env_list = self.manager.environment_list().values('name') for env in env_list: print(env['name']) return env_list def node_dict(self, node): return {'name': node.name, 'vnc': node.get_vnc_port()} def do_show(self): environment = self.manager.environment_get(self.params.name) print('%5s %25s' % ("VNC", "NODE-NAME")) for item in map(lambda x: self.node_dict(x), environment.nodes): print ('%5s %25s' % (item['vnc'], item['name'])) def do_erase(self): self.manager.environment_get(self.params.name).erase() def do_start(self): self.manager.environment_get(self.params.name).start() def do_destroy(self): self.manager.environment_get(self.params.name).destroy(verbose=False) def do_suspend(self): self.manager.environment_get(self.params.name).suspend(verbose=False) def do_resume(self): self.manager.environment_get(self.params.name).resume(verbose=False) def do_revert(self): self.manager.environment_get(self.params.name).revert( self.params.snapshot_name) def do_snapshot(self): self.manager.environment_get(self.params.name).snapshot( self.params.snapshot_name) def do_synchronize(self): self.manager.synchronize_environments() def do_snapshot_list(self): environment = self.manager.environment_get(self.params.name) snap_nodes = {} max_len = 0 for node in environment.nodes: snaps = sorted(node.get_snapshots()) for snap in snaps: if len(snap) > max_len: max_len = len(snap) if snap in snap_nodes: snap_nodes[snap].append(node.name) else: snap_nodes[snap] = [node.name, ] print("%*s %50s" % (max_len, "SNAPSHOT", "NODES-NAME")) for snap in snap_nodes: print("%*s %50s" % (max_len, snap, ', '.join(snap_nodes[snap]))) def do_snapshot_delete(self): environment = self.manager.environment_get(self.params.name) for node in environment.nodes: snaps = sorted(node.get_snapshots()) if self.params.snapshot_name in snaps: node.erase_snapshot(name=self.params.snapshot_name) def do_net_list(self): environment = self.manager.environment_get(self.params.name) networks = environment.networks print("%15s %10s" % ("NETWORK NAME", "IP NET")) for network in networks: print("%15s %10s" % (network.name, network.ip_network)) def do_timesync(self): env = self.manager.environment_get(self.params.name) if not self.params.node_name: _nodes = {node.name: node.get_vnc_port() for node in env.nodes} for node_name in sorted(_nodes.keys()): if _nodes[node_name] != '-1': sync_node_time(env, node_name) else: sync_node_time(env, self.params.node_name) def do_revert_resume(self): self.manager.environment_get(self.params.name).revert( self.params.snapshot_name) self.manager.environment_get(self.params.name).resume(verbose=False) if not self.params.no_timesync: print('time synchronization is starting') self.do_timesync() commands = { 'list': do_list, 'show': do_show, 'erase': do_erase, 'start': do_start, 'destroy': do_destroy, 'suspend': do_suspend, 'resume': do_resume, 'revert': do_revert, 'snapshot': do_snapshot, 'sync': do_synchronize, 'snapshot-list': do_snapshot_list, 'snapshot-delete': do_snapshot_delete, 'net-list': do_net_list, 'time-sync': do_timesync, 'revert-resume': do_revert_resume } def get_params(self): name_parser = argparse.ArgumentParser(add_help=False) name_parser.add_argument('name', help='environment name', default=environ.get('ENV_NAME'), metavar='ENV_NAME') snapshot_name_parser = argparse.ArgumentParser(add_help=False) snapshot_name_parser.add_argument('--snapshot-name', help='snapshot name', default=environ.get('SNAPSHOT_NAME')) node_name_parser = argparse.ArgumentParser(add_help=False) node_name_parser.add_argument('--node-name', help='node name', default=None) no_timesync_parser = argparse.ArgumentParser(add_help=False) no_timesync_parser.add_argument('--no-timesync', dest='no_timesync', action='store_const', const=True, help='revert without timesync', default=False) parser = argparse.ArgumentParser( description="Manage virtual environments. " "For addional help use command with -h/--help") subparsers = parser.add_subparsers(title="Operation commands", help='available commands', dest='command') subparsers.add_parser('list', help="Show virtual environments", description="Show virtual environments on host") subparsers.add_parser('show', parents=[name_parser], help="Show VMs in environment", description="Show VMs in environment") subparsers.add_parser('erase', parents=[name_parser], help="Delete environment", description="Delete environment and VMs on it") subparsers.add_parser('start', parents=[name_parser], help="Start VMs", description="Start VMs in selected environment") subparsers.add_parser('destroy', parents=[name_parser], help="Destroy(stop) VMs", description="Stop VMs in selected environment") subparsers.add_parser('suspend', parents=[name_parser], help="Suspend VMs", description="Suspend VMs in selected " "environment") subparsers.add_parser('resume', parents=[name_parser], help="Resume VMs", description="Resume VMs in selected environment") subparsers.add_parser('revert', parents=[name_parser, snapshot_name_parser], help="Apply snapshot to environment", description="Apply selected snapshot to " "environment") subparsers.add_parser('snapshot', parents=[name_parser, snapshot_name_parser], help="Make environment snapshot", description="Make environment snapshot") subparsers.add_parser('sync', help="Synchronization environment and devops", description="Synchronization environment " "and devops"), subparsers.add_parser('snapshot-list', parents=[name_parser], help="Show snapshots in environment", description="Show snapshots in selected " "environment") subparsers.add_parser('snapshot-delete', parents=[name_parser, snapshot_name_parser], help="Delete snapshot from environment", description="Delete snapshot from selected " "environment") subparsers.add_parser('net-list', parents=[name_parser], help="Show networks in environment", description="Display allocated networks for " "environment") subparsers.add_parser('time-sync', parents=[name_parser, node_name_parser], help="Sync time on all env nodes", description="Sync time on all active nodes " "of environment starting from " "admin") subparsers.add_parser('revert-resume', parents=[name_parser, snapshot_name_parser, node_name_parser, no_timesync_parser], help="Revert, resume, sync time on VMs", description="Revert and resume VMs in selected" "environment, then" " sync time on VMs") return parser.parse_args()
class CiBase(object): def __init__(self): self.manager = Manager() self._environment = None self.saved_environment_states = {} def _get_or_create(self): try: return self.manager.environment_get(self.env_name()) except: self._environment = self.describe_environment() self._environment.define() return self._environment def get_state(self, name): if self.environment().has_snapshot(name): self.environment().revert(name) return True return False def environment(self): """ :rtype : devops.models.Environment """ self._environment = self._environment or self._get_or_create() return self._environment @abstractproperty def env_name(self): """ :rtype : string """ pass @abstractmethod def describe_environment(self): """ :rtype : devops.models.Environment """ pass @abstractproperty def node_roles(self): """ :rtype : NodeRoles """ pass def nodes(self): return Nodes(self.environment(), self.node_roles()) # noinspection PyShadowingBuiltins def add_empty_volume(self, node, name, capacity=20 * 1024 * 1024 * 1024, device='disk', bus='virtio', format='qcow2'): self.manager.node_attach_volume(node=node, volume=self.manager.volume_create( name=name, capacity=capacity, environment=self.environment(), format=format), device=device, bus=bus) def add_node(self, memory, name, boot=None): return self.manager.node_create(name=name, memory=memory, environment=self.environment(), boot=boot) def create_interfaces(self, networks, node): for network in networks: self.manager.interface_create(network, node=node) def describe_admin_node(self, name, networks, memory=1024): node = self.add_node(memory=memory, name=name, boot=['hd', 'cdrom']) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') self.add_empty_volume(node, name + '-iso', capacity=_get_file_size(ISO_PATH), format='raw', device='cdrom', bus='ide') return node def describe_empty_node(self, name, networks, memory=1024): node = self.add_node(memory, name) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') if USE_ALL_DISKS: self.add_empty_volume(node, name + '-cinder') self.add_empty_volume(node, name + '-swift') return node @abstractmethod def setup_environment(self): """ :rtype : None """ pass def get_empty_environment(self): if not (self.get_state(EMPTY_SNAPSHOT)): self.setup_environment() self.environment().snapshot(EMPTY_SNAPSHOT) def generate_state_hash(self, settings): return hashlib.md5(str(settings)).hexdigest() def revert_to_state(self, settings={}): state_hash = self.generate_state_hash(settings) if state_hash in self.saved_environment_states: # revert to matching state state = self.saved_environment_states[state_hash] if not (self.get_state(state['snapshot_name'])): return False self.environment().resume() return True return False def snapshot_state(self, name, settings={}): state_hash = self.generate_state_hash(settings) snapshot_name = '{0}_{1}'.format( name.replace(' ', '_')[:17], state_hash) self.environment().suspend(verbose=False) self.environment().snapshot( name=snapshot_name, description=name, force=True, ) self.environment().resume(verbose=False) self.saved_environment_states[state_hash] = { 'snapshot_name': snapshot_name, 'cluster_name': name, 'settings': settings } def internal_virtual_ip(self): return str( IPNetwork( self.environment().network_by_name('internal').ip_network)[-2]) def public_router(self): return str( IPNetwork( self.environment().network_by_name('public').ip_network)[1]) def internal_router(self): return self._router('internal') def nat_router(self): return self._router('nat') def _router(self, router_name): return str( IPNetwork( self.environment().network_by_name(router_name).ip_network)[1]) def get_host_node_ip(self): return self.internal_router() def internal_network(self): return str( IPNetwork( self.environment().network_by_name('internal').ip_network)) def internal_net_mask(self): return str( IPNetwork(self.environment().network_by_name( 'internal').ip_network).netmask) def public_net_mask(self): return str( IPNetwork(self.environment().network_by_name( 'public').ip_network).netmask) def public_network(self): return str( IPNetwork(self.environment().network_by_name('public').ip_network))
class EnvironmentModel(object): hostname = 'nailgun' domain = 'test.domain.local' installation_timeout = 1800 deployment_timeout = 1800 puppet_timeout = 2000 nat_interface = '' # INTERFACES.get('admin') admin_net = 'admin' def __init__(self, os_image=None): self._virtual_environment = None self._keys = None self.manager = Manager() self.os_image = os_image self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self) def _get_or_create(self): try: return self.manager.environment_get(self.env_name) except Exception: self._virtual_environment = self.describe_environment() self._virtual_environment.define() return self._virtual_environment def router(self, router_name=None): router_name = router_name or self.admin_net return str( IPNetwork( self.get_virtual_environment().network_by_name(router_name). ip_network)[1]) @property def fuel_web(self): """FuelWebClient :rtype: FuelWebClient """ return self._fuel_web @property def node_roles(self): return NodeRoles( admin_names=['admin'], other_names=['slave-%02d' % x for x in range(1, int( settings.NODES_COUNT))] ) @property def env_name(self): return settings.ENV_NAME def add_empty_volume(self, node, name, capacity=settings.NODE_VOLUME_SIZE * 1024 * 1024 * 1024, device='disk', bus='virtio', format='qcow2'): self.manager.node_attach_volume( node=node, volume=self.manager.volume_create( name=name, capacity=capacity, environment=self.get_virtual_environment(), format=format), device=device, bus=bus) def add_node(self, memory, name, vcpu=1, boot=None): return self.manager.node_create( name=name, memory=memory, vcpu=vcpu, environment=self.get_virtual_environment(), boot=boot) @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server( cluster_id, self.get_host_node_ip(), port) def bootstrap_nodes(self, devops_nodes, timeout=600): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ self.dhcrelay_check() for node in devops_nodes: node.start() #TODO(aglarendil): LP#1317213 temporary sleep #remove after better fix is applied time.sleep(2) wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) for node in self.nailgun_nodes(devops_nodes): self.sync_node_time(self.get_ssh_to_remote(node["ip"])) return self.nailgun_nodes(devops_nodes) def create_interfaces(self, networks, node, model=settings.INTERFACE_MODEL): if settings.BONDING: for network in networks: self.manager.interface_create( network, node=node, model=model, interface_map=settings.BONDING_INTERFACES) else: for network in networks: self.manager.interface_create(network, node=node, model=model) def describe_environment(self): """Environment :rtype : Environment """ environment = self.manager.environment_create(self.env_name) networks = [] interfaces = settings.INTERFACE_ORDER if settings.BONDING: interfaces = settings.BONDING_INTERFACES.keys() for name in interfaces: networks.append(self.create_networks(name, environment)) for name in self.node_roles.admin_names: self.describe_admin_node(name, networks) for name in self.node_roles.other_names: self.describe_empty_node(name, networks) return environment def create_networks(self, name, environment): ip_networks = [ IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',')] new_prefix = int(settings.POOLS.get(name)[1]) pool = self.manager.create_network_pool(networks=ip_networks, prefix=int(new_prefix)) return self.manager.network_create( name=name, environment=environment, pool=pool, forward=settings.FORWARDING.get(name), has_dhcp_server=settings.DHCP.get(name)) def devops_nodes_by_names(self, devops_node_names): return map( lambda name: self.get_virtual_environment().node_by_name(name), devops_node_names) @logwrap def describe_admin_node(self, name, networks): node = self.add_node( memory=settings.HARDWARE.get("admin_node_memory", 1024), vcpu=settings.HARDWARE.get("admin_node_cpu", 1), name=name, boot=['hd', 'cdrom']) self.create_interfaces(networks, node) if self.os_image is None: self.add_empty_volume(node, name + '-system') self.add_empty_volume( node, name + '-iso', capacity=_get_file_size(settings.ISO_PATH), format='raw', device='cdrom', bus='ide') else: volume = self.manager.volume_get_predefined(self.os_image) vol_child = self.manager.volume_create_child( name=name + '-system', backing_store=volume, environment=self.get_virtual_environment() ) self.manager.node_attach_volume( node=node, volume=vol_child ) return node def describe_empty_node(self, name, networks): node = self.add_node( name=name, memory=settings.HARDWARE.get("slave_node_memory", 1024), vcpu=settings.HARDWARE.get("slave_node_cpu", 1)) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') if settings.USE_ALL_DISKS: self.add_empty_volume(node, name + '-cinder') self.add_empty_volume(node, name + '-swift') return node @logwrap def get_admin_remote(self): """SSH to admin node :rtype : SSHClient """ return self.nodes().admin.remote(self.admin_net, login='******', password='******') @logwrap def get_admin_node_ip(self): return str( self.nodes().admin.get_ip_address_by_network_name(self.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_host_node_ip(self): return self.router() def get_keys(self, node): params = { 'ip': node.get_ip_address_by_network_name(self.admin_net), 'mask': self.get_net_mask(self.admin_net), 'gw': self.router(), 'hostname': '.'.join((self.hostname, self.domain)), 'nat_interface': self.nat_interface, 'dns1': settings.DNS } keys = ( "<Wait>\n" "<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " <Enter>\n" ) % params return keys @logwrap def get_private_keys(self, force=False): if force or self._keys is None: self._keys = [] for key_string in ['/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa']: with self.get_admin_remote().open(key_string) as f: self._keys.append(RSAKey.from_private_key(f)) return self._keys @logwrap def get_ssh_to_remote(self, ip): return SSHClient(ip, username='******', password='******', private_keys=self.get_private_keys()) @logwrap def get_ssh_to_remote_by_name(self, node_name): return self.get_ssh_to_remote( self.fuel_web.get_nailgun_node_by_devops_node( self.get_virtual_environment().node_by_name(node_name))['ip'] ) def get_target_devs(self, devops_nodes): return [ interface.target_dev for interface in [ val for var in map(lambda node: node.interfaces, devops_nodes) for val in var]] def get_virtual_environment(self): """Returns virtual environment :rtype : devops.models.Environment """ if self._virtual_environment is None: self._virtual_environment = self._get_or_create() return self._virtual_environment def get_network(self, net_name): return str( IPNetwork( self.get_virtual_environment().network_by_name(net_name). ip_network)) def get_net_mask(self, net_name): return str( IPNetwork( self.get_virtual_environment().network_by_name( net_name).ip_network).netmask) def make_snapshot(self, snapshot_name, description="", is_make=False): if settings.MAKE_SNAPSHOT or is_make: self.get_virtual_environment().suspend(verbose=False) self.get_virtual_environment().snapshot(snapshot_name, force=True) revert_info(snapshot_name, description) def nailgun_nodes(self, devops_nodes): return map( lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node), devops_nodes ) def nodes(self): return Nodes(self.get_virtual_environment(), self.node_roles) def revert_snapshot(self, name): if self.get_virtual_environment().has_snapshot(name): logger.info('We have snapshot with such name %s' % name) self.get_virtual_environment().revert(name) logger.info('Starting snapshot reverting ....') self.get_virtual_environment().resume() logger.info('Starting snapshot resuming ...') admin = self.nodes().admin try: admin.await( self.admin_net, timeout=10 * 60, by_port=8000) except Exception as e: logger.warning("From first time admin isn't reverted: " "{0}".format(e)) admin.destroy() logger.info('Admin node was destroyed. Wait 10 sec.') time.sleep(10) self.get_virtual_environment().start(self.nodes().admins) logger.info('Admin node started second time.') self.nodes().admin.await( self.admin_net, timeout=10 * 60, by_port=8000) self.sync_time_admin_node() for node in self.nodes().slaves: if not node.driver.node_active(node): continue try: logger.info("Sync time on revert for node %s" % node.name) self.sync_node_time( self.get_ssh_to_remote_by_name(node.name)) except Exception as e: logger.warning( 'Exception caught while trying to sync time on {0}:' ' {1}'.format(node.name, e)) self.run_nailgun_agent( self.get_ssh_to_remote_by_name(node.name)) return True return False def setup_environment(self): # start admin node admin = self.nodes().admin admin.disk_devices.get(device='cdrom').volume.upload(settings.ISO_PATH) self.get_virtual_environment().start(self.nodes().admins) logger.info("Waiting for admin node to start up") wait(lambda: admin.driver.node_active(admin), 60) logger.info("Proceed with installation") # update network parameters at boot screen admin.send_keys(self.get_keys(admin)) # wait while installation complete admin.await(self.admin_net, timeout=10 * 60) self.wait_bootstrap() time.sleep(10) self.sync_time_admin_node() @retry() @logwrap def sync_node_time(self, remote): self.execute_remote_cmd(remote, 'hwclock -s') self.execute_remote_cmd(remote, 'NTPD=$(find /etc/init.d/ -regex \'' '/etc/init.d/ntp.?\'); $NTPD stop; ' 'killall ntpd; ntpd -qg && ' '$NTPD start') self.execute_remote_cmd(remote, 'hwclock -w') remote_date = remote.execute('date')['stdout'] logger.info("Node time: %s" % remote_date) @logwrap def sync_time_admin_node(self): logger.info("Sync time on revert for admin") remote = self.get_admin_remote() self.execute_remote_cmd(remote, 'hwclock -s') # Sync time using ntpd try: # If public NTP servers aren't accessible ntpdate will fail and # ntpd daemon shouldn't be restarted to avoid 'Server has gone # too long without sync' error while syncing time from slaves self.execute_remote_cmd(remote, "ntpdate -d $(awk '/^server/{print" " $2}' /etc/ntp.conf)") except AssertionError as e: logger.warning('Error occurred while synchronizing time on master' ': {0}'.format(e)) else: self.execute_remote_cmd(remote, 'service ntpd stop && ntpd -qg && ' 'service ntpd start') self.execute_remote_cmd(remote, 'hwclock -w') remote_date = remote.execute('date')['stdout'] logger.info("Master node time: {0}".format(remote_date)) def verify_node_service_list(self, node_name, smiles_count): remote = self.get_ssh_to_remote_by_name(node_name) checkers.verify_service_list(remote, smiles_count) def verify_network_configuration(self, node_name): checkers.verify_network_configuration( node=self.fuel_web.get_nailgun_node_by_name(node_name), remote=self.get_ssh_to_remote_by_name(node_name) ) def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" logger.info("Puppet timeout set in {0}".format( float(settings.PUPPET_TIMEOUT))) wait( lambda: not self.get_admin_remote().execute( "grep 'Fuel node deployment complete' '%s'" % log_path )['exit_code'], timeout=(float(settings.PUPPET_TIMEOUT)) ) def dhcrelay_check(self): admin_remote = self.get_admin_remote() out = admin_remote.execute("dhcpcheck discover " "--ifaces eth0 " "--repeat 3 " "--timeout 10")['stdout'] assert_true(self.get_admin_node_ip() in "".join(out), "dhcpcheck doesn't discover master ip") def run_nailgun_agent(self, remote): agent = remote.execute('/opt/nailgun/bin/agent')['exit_code'] logger.info("Nailgun agent run with exit_code: %s" % agent) def get_fuel_settings(self, remote=None): if not remote: remote = self.get_admin_remote() cmd = 'cat {cfg_file}'.format(cfg_file=settings.FUEL_SETTINGS_YAML) result = remote.execute(cmd) if result['exit_code'] == 0: fuel_settings = yaml.load(''.join(result['stdout'])) else: raise Exception('Can\'t output {cfg_file} file: {error}'. format(cfg_file=settings.FUEL_SETTINGS_YAML, error=result['stderr'])) return fuel_settings def admin_install_pkg(self, pkg_name): """Install a package <pkg_name> on the admin node""" admin_remote = self.get_admin_remote() remote_status = admin_remote.execute("rpm -q {0}'".format(pkg_name)) if remote_status['exit_code'] == 0: logger.info("Package '{0}' already installed.".format(pkg_name)) else: logger.info("Installing package '{0}' ...".format(pkg_name)) remote_status = admin_remote.execute("yum -y install {0}" .format(pkg_name)) logger.info("Installation of the package '{0}' has been" " completed with exit code {1}" .format(pkg_name, remote_status['exit_code'])) return remote_status['exit_code'] def admin_run_service(self, service_name): """Start a service <service_name> on the admin node""" admin_remote = self.get_admin_remote() admin_remote.execute("service {0} start".format(service_name)) remote_status = admin_remote.execute("service {0} status" .format(service_name)) if any('running...' in status for status in remote_status['stdout']): logger.info("Service '{0}' is running".format(service_name)) else: logger.info("Service '{0}' failed to start" " with exit code {1} :\n{2}" .format(service_name, remote_status['exit_code'], remote_status['stdout'])) @logwrap def execute_remote_cmd(self, remote, cmd, exit_code=0): result = remote.execute(cmd) assert_equal(result['exit_code'], exit_code, 'Failed to execute "{0}" on remote host: {1}'. format(cmd, result['stderr'])) return result['stdout']
class EnvironmentModel(object): hostname = 'nailgun' domain = 'test.domain.local' installation_timeout = 1800 deployment_timeout = 1800 puppet_timeout = 2000 nat_interface = '' # INTERFACES.get('admin') admin_net = 'admin' admin_net2 = 'admin2' multiple_cluster_networks = settings.MULTIPLE_NETWORKS def __init__(self, os_image=None): self._virtual_environment = None self._keys = None self.manager = Manager() self.os_image = os_image self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self) @property def nailgun_actions(self): return FuelActions.Nailgun(self.get_admin_remote()) @property def postgres_actions(self): return FuelActions.Postgres(self.get_admin_remote()) def _get_or_create(self): try: return self.manager.environment_get(self.env_name) except Exception: self._virtual_environment = self.describe_environment() self._virtual_environment.define() return self._virtual_environment def router(self, router_name=None): router_name = router_name or self.admin_net if router_name == self.admin_net2: return str(IPNetwork(self.get_virtual_environment(). network_by_name(router_name).ip_network)[2]) return str( IPNetwork( self.get_virtual_environment().network_by_name(router_name). ip_network)[1]) @property def fuel_web(self): """FuelWebClient :rtype: FuelWebClient """ return self._fuel_web @property def admin_node_ip(self): return self.fuel_web.admin_node_ip @property def node_roles(self): return NodeRoles( admin_names=['admin'], other_names=['slave-%02d' % x for x in range(1, int( settings.NODES_COUNT))] ) @property def env_name(self): return settings.ENV_NAME def add_empty_volume(self, node, name, capacity=settings.NODE_VOLUME_SIZE * 1024 * 1024 * 1024, device='disk', bus='virtio', format='qcow2'): self.manager.node_attach_volume( node=node, volume=self.manager.volume_create( name=name, capacity=capacity, environment=self.get_virtual_environment(), format=format), device=device, bus=bus) def add_node(self, memory, name, vcpu=1, boot=None): return self.manager.node_create( name=name, memory=memory, vcpu=vcpu, environment=self.get_virtual_environment(), boot=boot) @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server( cluster_id, self.get_host_node_ip(), port) def bootstrap_nodes(self, devops_nodes, timeout=600): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ # self.dhcrelay_check() for node in devops_nodes: node.start() # TODO(aglarendil): LP#1317213 temporary sleep # remove after better fix is applied time.sleep(2) wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) for node in self.nailgun_nodes(devops_nodes): self.sync_node_time(self.get_ssh_to_remote(node["ip"])) return self.nailgun_nodes(devops_nodes) def create_interfaces(self, networks, node, model=settings.INTERFACE_MODEL): if settings.BONDING: for network in networks: self.manager.interface_create( network, node=node, model=model, interface_map=settings.BONDING_INTERFACES) else: for network in networks: self.manager.interface_create(network, node=node, model=model) def describe_environment(self): """Environment :rtype : Environment """ environment = self.manager.environment_create(self.env_name) networks = [] interfaces = settings.INTERFACE_ORDER if self.multiple_cluster_networks: logger.info('Multiple cluster networks feature is enabled!') if settings.BONDING: interfaces = settings.BONDING_INTERFACES.keys() for name in interfaces: networks.append(self.create_networks(name, environment)) for name in self.node_roles.admin_names: self.describe_admin_node(name, networks) for name in self.node_roles.other_names: if self.multiple_cluster_networks: networks1 = [net for net in networks if net.name in settings.NODEGROUPS[0]['pools']] networks2 = [net for net in networks if net.name in settings.NODEGROUPS[1]['pools']] # If slave index is even number, then attach to # it virtual networks from the second network group. if int(name[-2:]) % 2 == 1: self.describe_empty_node(name, networks1) elif int(name[-2:]) % 2 == 0: self.describe_empty_node(name, networks2) else: self.describe_empty_node(name, networks) return environment def create_networks(self, name, environment): ip_networks = [ IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',')] new_prefix = int(settings.POOLS.get(name)[1]) pool = self.manager.create_network_pool(networks=ip_networks, prefix=int(new_prefix)) return self.manager.network_create( name=name, environment=environment, pool=pool, forward=settings.FORWARDING.get(name), has_dhcp_server=settings.DHCP.get(name)) def devops_nodes_by_names(self, devops_node_names): return map( lambda name: self.get_virtual_environment().node_by_name(name), devops_node_names) @logwrap def describe_admin_node(self, name, networks): node = self.add_node( memory=settings.HARDWARE.get("admin_node_memory", 1024), vcpu=settings.HARDWARE.get("admin_node_cpu", 1), name=name, boot=['hd', 'cdrom']) self.create_interfaces(networks, node) if self.os_image is None: self.add_empty_volume(node, name + '-system') self.add_empty_volume( node, name + '-iso', capacity=_get_file_size(settings.ISO_PATH), format='raw', device='cdrom', bus='ide') else: volume = self.manager.volume_get_predefined(self.os_image) vol_child = self.manager.volume_create_child( name=name + '-system', backing_store=volume, environment=self.get_virtual_environment() ) self.manager.node_attach_volume( node=node, volume=vol_child ) return node def describe_empty_node(self, name, networks): node = self.add_node( name=name, memory=settings.HARDWARE.get("slave_node_memory", 1024), vcpu=settings.HARDWARE.get("slave_node_cpu", 1)) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') if settings.USE_ALL_DISKS: self.add_empty_volume(node, name + '-cinder') self.add_empty_volume(node, name + '-swift') return node @logwrap def get_admin_remote(self, login=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password']): """SSH to admin node :rtype : SSHClient """ return self.nodes().admin.remote(self.admin_net, login=login, password=password) @logwrap def get_admin_node_ip(self): return str( self.nodes().admin.get_ip_address_by_network_name(self.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_host_node_ip(self): return self.router() def get_keys(self, node, custom=None, build_images=None): params = { 'ip': node.get_ip_address_by_network_name(self.admin_net), 'mask': self.get_net_mask(self.admin_net), 'gw': self.router(), 'hostname': '.'.join((self.hostname, self.domain)), 'nat_interface': self.nat_interface, 'dns1': settings.DNS, 'showmenu': 'yes' if custom else 'no', 'build_images': '1' if build_images else '0' } keys = ( "<Wait>\n" "<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " showmenu=%(showmenu)s\n" " build_images=%(build_images)s\n" " <Enter>\n" ) % params return keys @logwrap def get_private_keys(self, force=False): if force or self._keys is None: self._keys = [] for key_string in ['/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa']: with self.get_admin_remote().open(key_string) as f: self._keys.append(RSAKey.from_private_key(f)) return self._keys @logwrap def get_ssh_to_remote(self, ip): return SSHClient(ip, username=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password'], private_keys=self.get_private_keys()) @logwrap def get_ssh_to_remote_by_key(self, ip, keyfile): try: with open(keyfile) as f: keys = [RSAKey.from_private_key(f)] return SSHClient(ip, private_keys=keys) except IOError: logger.warning('Loading of SSH key from file failed. Trying to use' ' SSH agent ...') keys = Agent().get_keys() return SSHClient(ip, private_keys=keys) @logwrap def get_ssh_to_remote_by_name(self, node_name): return self.get_ssh_to_remote( self.fuel_web.get_nailgun_node_by_devops_node( self.get_virtual_environment().node_by_name(node_name))['ip'] ) def get_target_devs(self, devops_nodes): return [ interface.target_dev for interface in [ val for var in map(lambda node: node.interfaces, devops_nodes) for val in var]] def get_virtual_environment(self): """Returns virtual environment :rtype : devops.models.Environment """ if self._virtual_environment is None: self._virtual_environment = self._get_or_create() return self._virtual_environment def get_network(self, net_name): return str( IPNetwork( self.get_virtual_environment().network_by_name(net_name). ip_network)) def get_net_mask(self, net_name): return str( IPNetwork( self.get_virtual_environment().network_by_name( net_name).ip_network).netmask) def make_snapshot(self, snapshot_name, description="", is_make=False): if settings.MAKE_SNAPSHOT or is_make: self.get_virtual_environment().suspend(verbose=False) time.sleep(10) self.get_virtual_environment().snapshot(snapshot_name, force=True) revert_info(snapshot_name, description) if settings.FUEL_STATS_CHECK: self.get_virtual_environment().resume() try: self.nodes().admin.await(self.admin_net, timeout=60) except Exception: logger.error('Admin node is unavailable via SSH after ' 'environment resume ') raise def nailgun_nodes(self, devops_nodes): return map( lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node), devops_nodes ) def nodes(self): return Nodes(self.get_virtual_environment(), self.node_roles) def revert_snapshot(self, name): if self.get_virtual_environment().has_snapshot(name): logger.info('We have snapshot with such name %s' % name) self.get_virtual_environment().revert(name) logger.info('Starting snapshot reverting ....') self.get_virtual_environment().resume() logger.info('Starting snapshot resuming ...') admin = self.nodes().admin try: admin.await( self.admin_net, timeout=10 * 60, by_port=8000) except Exception as e: logger.warning("From first time admin isn't reverted: " "{0}".format(e)) admin.destroy() logger.info('Admin node was destroyed. Wait 10 sec.') time.sleep(10) self.get_virtual_environment().start(self.nodes().admins) logger.info('Admin node started second time.') self.nodes().admin.await( self.admin_net, timeout=10 * 60, by_port=8000) self.set_admin_ssh_password() try: _wait(self._fuel_web.client.get_releases, expected=EnvironmentError, timeout=300) except exceptions.Unauthorized: self.set_admin_keystone_password() self._fuel_web.get_nailgun_version() self.sync_time_admin_node() for node in self.nodes().slaves: if not node.driver.node_active(node): continue try: logger.info("Sync time on revert for node %s" % node.name) self.sync_node_time( self.get_ssh_to_remote_by_name(node.name)) except Exception as e: logger.warning( 'Exception caught while trying to sync time on {0}:' ' {1}'.format(node.name, e)) self.run_nailgun_agent( self.get_ssh_to_remote_by_name(node.name)) return True return False def set_admin_ssh_password(self): try: remote = self.get_admin_remote( login=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password']) self.execute_remote_cmd(remote, 'date') logger.debug('Accessing admin node using SSH: SUCCESS') except Exception: logger.debug('Accessing admin node using SSH credentials:' ' FAIL, trying to change password from default') remote = self.get_admin_remote(login='******', password='******') self.execute_remote_cmd( remote, 'echo -e "{1}\\n{1}" | passwd {0}' .format(settings.SSH_CREDENTIALS['login'], settings.SSH_CREDENTIALS['password'])) logger.debug("Admin node password has changed.") logger.info("Admin node login name: '{0}' , password: '******'". format(settings.SSH_CREDENTIALS['login'], settings.SSH_CREDENTIALS['password'])) def set_admin_keystone_password(self): remote = self.get_admin_remote() try: self._fuel_web.client.get_releases() except exceptions.Unauthorized: self.execute_remote_cmd( remote, 'fuel user --newpass {0} --change-password' .format(settings.KEYSTONE_CREDS['password'])) logger.info( 'New Fuel UI (keystone) username: "******", password: "******"' .format(settings.KEYSTONE_CREDS['username'], settings.KEYSTONE_CREDS['password'])) def setup_environment(self, custom=settings.CUSTOM_ENV, build_images=settings.BUILD_IMAGES): # start admin node admin = self.nodes().admin admin.disk_devices.get(device='cdrom').volume.upload(settings.ISO_PATH) self.get_virtual_environment().start(self.nodes().admins) logger.info("Waiting for admin node to start up") wait(lambda: admin.driver.node_active(admin), 60) logger.info("Proceed with installation") # update network parameters at boot screen admin.send_keys(self.get_keys(admin, custom=custom, build_images=build_images)) if custom: self.setup_customisation() # wait while installation complete admin.await(self.admin_net, timeout=10 * 60) self.set_admin_ssh_password() self.wait_bootstrap() time.sleep(10) self.set_admin_keystone_password() self.sync_time_admin_node() if settings.MULTIPLE_NETWORKS: self.describe_second_admin_interface() multiple_networks_hacks.configure_second_admin_cobbler(self) multiple_networks_hacks.configure_second_dhcrelay(self) self.nailgun_actions.set_collector_address( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL) if settings.FUEL_STATS_ENABLED: self.fuel_web.client.send_fuel_stats(enabled=True) logger.info('Enabled sending of statistics to {0}:{1}'.format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT )) @upload_manifests def wait_for_provisioning(self): _wait(lambda: _tcp_ping( self.nodes().admin.get_ip_address_by_network_name (self.admin_net), 22), timeout=5 * 60) def setup_customisation(self): self.wait_for_provisioning() try: remote = self.get_admin_remote() pid = remote.execute("pgrep 'fuelmenu'")['stdout'][0] pid.rstrip('\n') remote.execute("kill -sigusr1 {0}".format(pid)) except Exception: logger.error("Could not kill pid of fuelmenu") raise @retry(count=10, delay=60) @logwrap def sync_node_time(self, remote): self.execute_remote_cmd(remote, 'hwclock -s') self.execute_remote_cmd(remote, 'NTPD=$(find /etc/init.d/ -regex \'' '/etc/init.d/\(ntp.?\|ntp-dev\)\');' '$NTPD stop && ntpd -dqg && $NTPD ' 'start') self.execute_remote_cmd(remote, 'hwclock -w') remote_date = remote.execute('date')['stdout'] logger.info("Node time: %s" % remote_date) @retry(count=10, delay=60) @logwrap def sync_time_admin_node(self): logger.info("Sync time on revert for admin") remote = self.get_admin_remote() self.execute_remote_cmd(remote, 'hwclock -s') # Sync time using ntpd try: # If public NTP servers aren't accessible ntpdate will fail and # ntpd daemon shouldn't be restarted to avoid 'Server has gone # too long without sync' error while syncing time from slaves self.execute_remote_cmd(remote, "ntpdate -vu $(awk '/^server/ && " "$2 !~ /127.*/ {print $2}' " "/etc/ntp.conf)") except AssertionError as e: logger.warning('Error occurred while synchronizing time on master' ': {0}'.format(e)) raise else: self.execute_remote_cmd(remote, 'service ntpd stop && ntpd -dqg &&' ' service ntpd start') self.execute_remote_cmd(remote, 'hwclock -w') remote_date = remote.execute('date')['stdout'] logger.info("Master node time: {0}".format(remote_date)) def verify_network_configuration(self, node_name): checkers.verify_network_configuration( node=self.fuel_web.get_nailgun_node_by_name(node_name), remote=self.get_ssh_to_remote_by_name(node_name) ) def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" logger.info("Puppet timeout set in {0}".format( float(settings.PUPPET_TIMEOUT))) wait( lambda: not self.get_admin_remote().execute( "grep 'Fuel node deployment' '%s'" % log_path )['exit_code'], timeout=(float(settings.PUPPET_TIMEOUT)) ) result = self.get_admin_remote().execute("grep 'Fuel node deployment " "complete' '%s'" % log_path )['exit_code'] if result != 0: raise Exception('Fuel node deployment failed.') def dhcrelay_check(self): admin_remote = self.get_admin_remote() out = admin_remote.execute("dhcpcheck discover " "--ifaces eth0 " "--repeat 3 " "--timeout 10")['stdout'] assert_true(self.get_admin_node_ip() in "".join(out), "dhcpcheck doesn't discover master ip") def run_nailgun_agent(self, remote): agent = remote.execute('/opt/nailgun/bin/agent')['exit_code'] logger.info("Nailgun agent run with exit_code: %s" % agent) def get_fuel_settings(self, remote=None): if not remote: remote = self.get_admin_remote() cmd = 'cat {cfg_file}'.format(cfg_file=settings.FUEL_SETTINGS_YAML) result = remote.execute(cmd) if result['exit_code'] == 0: fuel_settings = yaml.load(''.join(result['stdout'])) else: raise Exception('Can\'t output {cfg_file} file: {error}'. format(cfg_file=settings.FUEL_SETTINGS_YAML, error=result['stderr'])) return fuel_settings def admin_install_pkg(self, pkg_name): """Install a package <pkg_name> on the admin node""" admin_remote = self.get_admin_remote() remote_status = admin_remote.execute("rpm -q {0}'".format(pkg_name)) if remote_status['exit_code'] == 0: logger.info("Package '{0}' already installed.".format(pkg_name)) else: logger.info("Installing package '{0}' ...".format(pkg_name)) remote_status = admin_remote.execute("yum -y install {0}" .format(pkg_name)) logger.info("Installation of the package '{0}' has been" " completed with exit code {1}" .format(pkg_name, remote_status['exit_code'])) return remote_status['exit_code'] def admin_run_service(self, service_name): """Start a service <service_name> on the admin node""" admin_remote = self.get_admin_remote() admin_remote.execute("service {0} start".format(service_name)) remote_status = admin_remote.execute("service {0} status" .format(service_name)) if any('running...' in status for status in remote_status['stdout']): logger.info("Service '{0}' is running".format(service_name)) else: logger.info("Service '{0}' failed to start" " with exit code {1} :\n{2}" .format(service_name, remote_status['exit_code'], remote_status['stdout'])) # Modifies a resolv.conf on the Fuel master node and returns # its original content. # * adds 'nameservers' at start of resolv.conf if merge=True # * replaces resolv.conf with 'nameservers' if merge=False def modify_resolv_conf(self, nameservers=[], merge=True): remote = self.get_admin_remote() resolv_conf = remote.execute('cat /etc/resolv.conf') assert_equal(0, resolv_conf['exit_code'], 'Executing "{0}" on the ' 'admin node has failed with: {1}' .format('cat /etc/resolv.conf', resolv_conf['stderr'])) if merge: nameservers.extend(resolv_conf['stdout']) resolv_keys = ['search', 'domain', 'nameserver'] resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers if any(x in ns for x in resolv_keys)) logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new)) echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new) echo_result = remote.execute(echo_cmd) assert_equal(0, echo_result['exit_code'], 'Executing "{0}" on the ' 'admin node has failed with: {1}' .format(echo_cmd, echo_result['stderr'])) return resolv_conf['stdout'] @logwrap def execute_remote_cmd(self, remote, cmd, exit_code=0): result = remote.execute(cmd) assert_equal(result['exit_code'], exit_code, 'Failed to execute "{0}" on remote host: {1}'. format(cmd, result)) return result['stdout'] @logwrap def describe_second_admin_interface(self): remote = self.get_admin_remote() second_admin_network = self.get_network(self.admin_net2).split('/')[0] second_admin_netmask = self.get_net_mask(self.admin_net2) second_admin_if = settings.INTERFACES.get(self.admin_net2) second_admin_ip = str(self.nodes().admin. get_ip_address_by_network_name(self.admin_net2)) logger.info(('Parameters for second admin interface configuration: ' 'Network - {0}, Netmask - {1}, Interface - {2}, ' 'IP Address - {3}').format(second_admin_network, second_admin_netmask, second_admin_if, second_admin_ip)) add_second_admin_ip = ('DEVICE={0}\\n' 'ONBOOT=yes\\n' 'NM_CONTROLLED=no\\n' 'USERCTL=no\\n' 'PEERDNS=no\\n' 'BOOTPROTO=static\\n' 'IPADDR={1}\\n' 'NETMASK={2}\\n').format(second_admin_if, second_admin_ip, second_admin_netmask) cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};' 'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format( add_second_admin_ip, second_admin_if, second_admin_ip) logger.debug('Trying to assign {0} IP to the {1} on master node...'. format(second_admin_ip, second_admin_if)) result = remote.execute(cmd) assert_equal(result['exit_code'], 0, ('Failed to assign second admin ' 'IP address on master node: {0}').format(result)) logger.debug('Done: {0}'.format(result['stdout'])) multiple_networks_hacks.configure_second_admin_firewall( self, second_admin_network, second_admin_netmask) @logwrap def get_masternode_uuid(self): return self.postgres_actions.run_query( db='nailgun', query="select master_node_uid from master_node_settings limit 1;")
class CiBase(object): def __init__(self): self.manager = Manager() self._environment = None def _get_or_create(self): try: return self.manager.environment_get(self.env_name()) except: self._environment = self.describe_environment() self._environment.define() return self._environment def get_empty_state(self): if self.environment().has_snapshot(EMPTY_SNAPSHOT): self.environment().revert(EMPTY_SNAPSHOT) else: self.setup_environment() def environment(self): """ :rtype : devops.models.Environment """ self._environment = self._environment or self._get_or_create() return self._environment @abstractproperty def env_name(self): """ :rtype : string """ pass @abstractmethod def describe_environment(self): """ :rtype : devops.models.Environment """ pass @abstractproperty def node_roles(self): """ :rtype : NodeRoles """ pass def nodes(self): return Nodes(self.environment(), self.node_roles()) # noinspection PyShadowingBuiltins def add_empty_volume(self, node, name, capacity=20 * 1024 * 1024 * 1024, device='disk', bus='virtio', format='qcow2'): self.manager.node_attach_volume(node=node, volume=self.manager.volume_create( name=name, capacity=capacity, environment=self.environment(), format=format), device=device, bus=bus) def add_node(self, memory, name, boot=None): return self.manager.node_create(name=name, memory=memory, environment=self.environment(), boot=boot) def create_interfaces(self, networks, node): for network in networks: if network.name == 'internal': self.manager.interface_create(network, node=node) self.manager.interface_create(network, node=node) self.manager.interface_create(network, node=node) def describe_admin_node(self, name, networks, memory=1024): node = self.add_node(memory=memory, name=name, boot=['hd', 'cdrom']) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') self.add_empty_volume(node, name + '-iso', capacity=_get_file_size(ISO_PATH), format='raw', device='cdrom', bus='ide') return node def describe_empty_node(self, name, networks, memory=1024): node = self.add_node(memory, name) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') self.add_empty_volume(node, name + '-cinder') self.add_empty_volume(node, name + '-swift') return node @abstractmethod def setup_environment(self): """ :rtype : None """ pass def internal_virtual_ip(self): return str( IPNetwork( self.environment().network_by_name('internal').ip_network)[-2]) def public_router(self): return str( IPNetwork( self.environment().network_by_name('public').ip_network)[1]) def internal_router(self): return str( IPNetwork( self.environment().network_by_name('internal').ip_network)[1]) def get_host_node_ip(self): return self.internal_router() def internal_network(self): return str( IPNetwork( self.environment().network_by_name('internal').ip_network)) def internal_net_mask(self): return str( IPNetwork(self.environment().network_by_name( 'internal').ip_network).netmask) def public_net_mask(self): return str( IPNetwork(self.environment().network_by_name( 'public').ip_network).netmask) def public_network(self): return str( IPNetwork(self.environment().network_by_name('public').ip_network))
class EnvironmentModel(object): hostname = 'nailgun' domain = 'test.domain.local' installation_timeout = 1800 deployment_timeout = 1800 puppet_timeout = 2000 nat_interface = '' # INTERFACES.get('admin') admin_net = 'admin' admin_net2 = 'admin2' multiple_cluster_networks = settings.MULTIPLE_NETWORKS def __init__(self, os_image=None): self._virtual_environment = None self._keys = None self.manager = Manager() self.os_image = os_image self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self) @property def nailgun_actions(self): return FuelActions.Nailgun(self.get_admin_remote()) @property def postgres_actions(self): return FuelActions.Postgres(self.get_admin_remote()) def _get_or_create(self): try: return self.manager.environment_get(self.env_name) except Exception: self._virtual_environment = self.describe_environment() self._virtual_environment.define() return self._virtual_environment def router(self, router_name=None): router_name = router_name or self.admin_net if router_name == self.admin_net2: return str( IPNetwork(self.get_virtual_environment().network_by_name( router_name).ip_network)[2]) return str( IPNetwork(self.get_virtual_environment().network_by_name( router_name).ip_network)[1]) @property def fuel_web(self): """FuelWebClient :rtype: FuelWebClient """ return self._fuel_web @property def admin_node_ip(self): return self.fuel_web.admin_node_ip @property def node_roles(self): return NodeRoles(admin_names=['admin'], other_names=[ 'slave-%02d' % x for x in range(1, int(settings.NODES_COUNT)) ]) @property def env_name(self): return settings.ENV_NAME def add_empty_volume(self, node, name, capacity=settings.NODE_VOLUME_SIZE * 1024 * 1024 * 1024, device='disk', bus='virtio', format='qcow2'): self.manager.node_attach_volume( node=node, volume=self.manager.volume_create( name=name, capacity=capacity, environment=self.get_virtual_environment(), format=format), device=device, bus=bus) def add_node(self, memory, name, vcpu=1, boot=None): return self.manager.node_create( name=name, memory=memory, vcpu=vcpu, environment=self.get_virtual_environment(), boot=boot) @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server(cluster_id, self.get_host_node_ip(), port) def bootstrap_nodes(self, devops_nodes, timeout=600): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ # self.dhcrelay_check() for node in devops_nodes: node.start() # TODO(aglarendil): LP#1317213 temporary sleep # remove after better fix is applied time.sleep(2) wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) for node in self.nailgun_nodes(devops_nodes): self.sync_node_time(self.get_ssh_to_remote(node["ip"])) return self.nailgun_nodes(devops_nodes) def create_interfaces(self, networks, node, model=settings.INTERFACE_MODEL): if settings.BONDING: for network in networks: self.manager.interface_create( network, node=node, model=model, interface_map=settings.BONDING_INTERFACES) else: for network in networks: self.manager.interface_create(network, node=node, model=model) def describe_environment(self): """Environment :rtype : Environment """ environment = self.manager.environment_create(self.env_name) networks = [] interfaces = settings.INTERFACE_ORDER if self.multiple_cluster_networks: logger.info('Multiple cluster networks feature is enabled!') if settings.BONDING: interfaces = settings.BONDING_INTERFACES.keys() for name in interfaces: networks.append(self.create_networks(name, environment)) for name in self.node_roles.admin_names: self.describe_admin_node(name, networks) for name in self.node_roles.other_names: if self.multiple_cluster_networks: networks1 = [ net for net in networks if net.name in settings.NODEGROUPS[0]['pools'] ] networks2 = [ net for net in networks if net.name in settings.NODEGROUPS[1]['pools'] ] # If slave index is even number, then attach to # it virtual networks from the second network group. if int(name[-2:]) % 2 == 1: self.describe_empty_node(name, networks1) elif int(name[-2:]) % 2 == 0: self.describe_empty_node(name, networks2) else: self.describe_empty_node(name, networks) return environment def create_networks(self, name, environment): ip_networks = [ IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',') ] new_prefix = int(settings.POOLS.get(name)[1]) pool = self.manager.create_network_pool(networks=ip_networks, prefix=int(new_prefix)) return self.manager.network_create( name=name, environment=environment, pool=pool, forward=settings.FORWARDING.get(name), has_dhcp_server=settings.DHCP.get(name)) def devops_nodes_by_names(self, devops_node_names): return map( lambda name: self.get_virtual_environment().node_by_name(name), devops_node_names) @logwrap def describe_admin_node(self, name, networks): node = self.add_node(memory=settings.HARDWARE.get( "admin_node_memory", 1024), vcpu=settings.HARDWARE.get("admin_node_cpu", 1), name=name, boot=['hd', 'cdrom']) self.create_interfaces(networks, node) if self.os_image is None: self.add_empty_volume(node, name + '-system') self.add_empty_volume(node, name + '-iso', capacity=_get_file_size(settings.ISO_PATH), format='raw', device='cdrom', bus='ide') else: volume = self.manager.volume_get_predefined(self.os_image) vol_child = self.manager.volume_create_child( name=name + '-system', backing_store=volume, environment=self.get_virtual_environment()) self.manager.node_attach_volume(node=node, volume=vol_child) return node def describe_empty_node(self, name, networks): node = self.add_node(name=name, memory=settings.HARDWARE.get( "slave_node_memory", 1024), vcpu=settings.HARDWARE.get("slave_node_cpu", 1)) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') if settings.USE_ALL_DISKS: self.add_empty_volume(node, name + '-cinder') self.add_empty_volume(node, name + '-swift') return node @logwrap def get_admin_remote(self, login=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password']): """SSH to admin node :rtype : SSHClient """ return self.nodes().admin.remote(self.admin_net, login=login, password=password) @logwrap def get_admin_node_ip(self): return str(self.nodes().admin.get_ip_address_by_network_name( self.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_host_node_ip(self): return self.router() def get_keys(self, node, custom=None, build_images=None): params = { 'ip': node.get_ip_address_by_network_name(self.admin_net), 'mask': self.get_net_mask(self.admin_net), 'gw': self.router(), 'hostname': '.'.join((self.hostname, self.domain)), 'nat_interface': self.nat_interface, 'dns1': settings.DNS, 'showmenu': 'yes' if custom else 'no', 'build_images': '1' if build_images else '0' } keys = ("<Wait>\n" "<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " showmenu=%(showmenu)s\n" " build_images=%(build_images)s\n" " <Enter>\n") % params return keys @logwrap def get_private_keys(self, force=False): if force or self._keys is None: self._keys = [] for key_string in [ '/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa' ]: with self.get_admin_remote().open(key_string) as f: self._keys.append(RSAKey.from_private_key(f)) return self._keys @logwrap def get_ssh_to_remote(self, ip): return SSHClient(ip, username=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password'], private_keys=self.get_private_keys()) @logwrap def get_ssh_to_remote_by_key(self, ip, keyfile): try: with open(keyfile) as f: keys = [RSAKey.from_private_key(f)] return SSHClient(ip, private_keys=keys) except IOError: logger.warning('Loading of SSH key from file failed. Trying to use' ' SSH agent ...') keys = Agent().get_keys() return SSHClient(ip, private_keys=keys) @logwrap def get_ssh_to_remote_by_name(self, node_name): return self.get_ssh_to_remote( self.fuel_web.get_nailgun_node_by_devops_node( self.get_virtual_environment().node_by_name(node_name))['ip']) def get_target_devs(self, devops_nodes): return [ interface.target_dev for interface in [ val for var in map(lambda node: node.interfaces, devops_nodes) for val in var ] ] def get_virtual_environment(self): """Returns virtual environment :rtype : devops.models.Environment """ if self._virtual_environment is None: self._virtual_environment = self._get_or_create() return self._virtual_environment def get_network(self, net_name): return str( IPNetwork(self.get_virtual_environment().network_by_name( net_name).ip_network)) def get_net_mask(self, net_name): return str( IPNetwork(self.get_virtual_environment().network_by_name( net_name).ip_network).netmask) def make_snapshot(self, snapshot_name, description="", is_make=False): if settings.MAKE_SNAPSHOT or is_make: self.get_virtual_environment().suspend(verbose=False) time.sleep(10) self.get_virtual_environment().snapshot(snapshot_name, force=True) revert_info(snapshot_name, description) if settings.FUEL_STATS_CHECK: self.get_virtual_environment().resume() try: self.nodes().admin. await (self.admin_net, timeout=60) except Exception: logger.error('Admin node is unavailable via SSH after ' 'environment resume ') raise def nailgun_nodes(self, devops_nodes): return map( lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node), devops_nodes) def nodes(self): return Nodes(self.get_virtual_environment(), self.node_roles) def revert_snapshot(self, name): if self.get_virtual_environment().has_snapshot(name): logger.info('We have snapshot with such name %s' % name) self.get_virtual_environment().revert(name) logger.info('Starting snapshot reverting ....') self.get_virtual_environment().resume() logger.info('Starting snapshot resuming ...') admin = self.nodes().admin try: admin. await (self.admin_net, timeout=10 * 60, by_port=8000) except Exception as e: logger.warning("From first time admin isn't reverted: " "{0}".format(e)) admin.destroy() logger.info('Admin node was destroyed. Wait 10 sec.') time.sleep(10) self.get_virtual_environment().start(self.nodes().admins) logger.info('Admin node started second time.') self.nodes().admin. await (self.admin_net, timeout=10 * 60, by_port=8000) self.set_admin_ssh_password() try: _wait(self._fuel_web.client.get_releases, expected=EnvironmentError, timeout=300) except exceptions.Unauthorized: self.set_admin_keystone_password() self._fuel_web.get_nailgun_version() self.sync_time_admin_node() for node in self.nodes().slaves: if not node.driver.node_active(node): continue try: logger.info("Sync time on revert for node %s" % node.name) self.sync_node_time( self.get_ssh_to_remote_by_name(node.name)) except Exception as e: logger.warning( 'Exception caught while trying to sync time on {0}:' ' {1}'.format(node.name, e)) self.run_nailgun_agent( self.get_ssh_to_remote_by_name(node.name)) return True return False def set_admin_ssh_password(self): try: remote = self.get_admin_remote( login=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password']) self.execute_remote_cmd(remote, 'date') logger.debug('Accessing admin node using SSH: SUCCESS') except Exception: logger.debug('Accessing admin node using SSH credentials:' ' FAIL, trying to change password from default') remote = self.get_admin_remote(login='******', password='******') self.execute_remote_cmd( remote, 'echo -e "{1}\\n{1}" | passwd {0}'.format( settings.SSH_CREDENTIALS['login'], settings.SSH_CREDENTIALS['password'])) logger.debug("Admin node password has changed.") logger.info("Admin node login name: '{0}' , password: '******'".format( settings.SSH_CREDENTIALS['login'], settings.SSH_CREDENTIALS['password'])) def set_admin_keystone_password(self): remote = self.get_admin_remote() try: self._fuel_web.client.get_releases() except exceptions.Unauthorized: self.execute_remote_cmd( remote, 'fuel user --newpass {0} --change-password'.format( settings.KEYSTONE_CREDS['password'])) logger.info( 'New Fuel UI (keystone) username: "******", password: "******"'. format(settings.KEYSTONE_CREDS['username'], settings.KEYSTONE_CREDS['password'])) def setup_environment(self, custom=settings.CUSTOM_ENV, build_images=settings.BUILD_IMAGES): # start admin node admin = self.nodes().admin admin.disk_devices.get(device='cdrom').volume.upload(settings.ISO_PATH) self.get_virtual_environment().start(self.nodes().admins) logger.info("Waiting for admin node to start up") wait(lambda: admin.driver.node_active(admin), 60) logger.info("Proceed with installation") # update network parameters at boot screen admin.send_keys( self.get_keys(admin, custom=custom, build_images=build_images)) if custom: self.setup_customisation() # wait while installation complete admin. await (self.admin_net, timeout=10 * 60) self.set_admin_ssh_password() self.wait_bootstrap() time.sleep(10) self.set_admin_keystone_password() self.sync_time_admin_node() if settings.MULTIPLE_NETWORKS: self.describe_second_admin_interface() multiple_networks_hacks.configure_second_admin_cobbler(self) multiple_networks_hacks.configure_second_dhcrelay(self) self.nailgun_actions.set_collector_address(settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL) if settings.FUEL_STATS_ENABLED: self.fuel_web.client.send_fuel_stats(enabled=True) logger.info('Enabled sending of statistics to {0}:{1}'.format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT)) @upload_manifests def wait_for_provisioning(self): _wait(lambda: _tcp_ping( self.nodes().admin.get_ip_address_by_network_name(self.admin_net), 22), timeout=5 * 60) def setup_customisation(self): self.wait_for_provisioning() try: remote = self.get_admin_remote() pid = remote.execute("pgrep 'fuelmenu'")['stdout'][0] pid.rstrip('\n') remote.execute("kill -sigusr1 {0}".format(pid)) except Exception: logger.error("Could not kill pid of fuelmenu") raise @retry(count=10, delay=60) @logwrap def sync_node_time(self, remote): self.execute_remote_cmd(remote, 'hwclock -s') self.execute_remote_cmd( remote, 'NTPD=$(find /etc/init.d/ -regex \'' '/etc/init.d/\(ntp.?\|ntp-dev\)\');' '$NTPD stop && ntpd -dqg && $NTPD ' 'start') self.execute_remote_cmd(remote, 'hwclock -w') remote_date = remote.execute('date')['stdout'] logger.info("Node time: %s" % remote_date) @retry(count=10, delay=60) @logwrap def sync_time_admin_node(self): logger.info("Sync time on revert for admin") remote = self.get_admin_remote() self.execute_remote_cmd(remote, 'hwclock -s') # Sync time using ntpd try: # If public NTP servers aren't accessible ntpdate will fail and # ntpd daemon shouldn't be restarted to avoid 'Server has gone # too long without sync' error while syncing time from slaves self.execute_remote_cmd( remote, "ntpdate -vu $(awk '/^server/ && " "$2 !~ /127.*/ {print $2}' " "/etc/ntp.conf)") except AssertionError as e: logger.warning('Error occurred while synchronizing time on master' ': {0}'.format(e)) raise else: self.execute_remote_cmd( remote, 'service ntpd stop && ntpd -dqg &&' ' service ntpd start') self.execute_remote_cmd(remote, 'hwclock -w') remote_date = remote.execute('date')['stdout'] logger.info("Master node time: {0}".format(remote_date)) def verify_network_configuration(self, node_name): checkers.verify_network_configuration( node=self.fuel_web.get_nailgun_node_by_name(node_name), remote=self.get_ssh_to_remote_by_name(node_name)) def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" logger.info("Puppet timeout set in {0}".format( float(settings.PUPPET_TIMEOUT))) wait(lambda: not self.get_admin_remote().execute( "grep 'Fuel node deployment' '%s'" % log_path)['exit_code'], timeout=(float(settings.PUPPET_TIMEOUT))) result = self.get_admin_remote().execute("grep 'Fuel node deployment " "complete' '%s'" % log_path)['exit_code'] if result != 0: raise Exception('Fuel node deployment failed.') def dhcrelay_check(self): admin_remote = self.get_admin_remote() out = admin_remote.execute("dhcpcheck discover " "--ifaces eth0 " "--repeat 3 " "--timeout 10")['stdout'] assert_true(self.get_admin_node_ip() in "".join(out), "dhcpcheck doesn't discover master ip") def run_nailgun_agent(self, remote): agent = remote.execute('/opt/nailgun/bin/agent')['exit_code'] logger.info("Nailgun agent run with exit_code: %s" % agent) def get_fuel_settings(self, remote=None): if not remote: remote = self.get_admin_remote() cmd = 'cat {cfg_file}'.format(cfg_file=settings.FUEL_SETTINGS_YAML) result = remote.execute(cmd) if result['exit_code'] == 0: fuel_settings = yaml.load(''.join(result['stdout'])) else: raise Exception('Can\'t output {cfg_file} file: {error}'.format( cfg_file=settings.FUEL_SETTINGS_YAML, error=result['stderr'])) return fuel_settings def admin_install_pkg(self, pkg_name): """Install a package <pkg_name> on the admin node""" admin_remote = self.get_admin_remote() remote_status = admin_remote.execute("rpm -q {0}'".format(pkg_name)) if remote_status['exit_code'] == 0: logger.info("Package '{0}' already installed.".format(pkg_name)) else: logger.info("Installing package '{0}' ...".format(pkg_name)) remote_status = admin_remote.execute( "yum -y install {0}".format(pkg_name)) logger.info("Installation of the package '{0}' has been" " completed with exit code {1}".format( pkg_name, remote_status['exit_code'])) return remote_status['exit_code'] def admin_run_service(self, service_name): """Start a service <service_name> on the admin node""" admin_remote = self.get_admin_remote() admin_remote.execute("service {0} start".format(service_name)) remote_status = admin_remote.execute( "service {0} status".format(service_name)) if any('running...' in status for status in remote_status['stdout']): logger.info("Service '{0}' is running".format(service_name)) else: logger.info("Service '{0}' failed to start" " with exit code {1} :\n{2}".format( service_name, remote_status['exit_code'], remote_status['stdout'])) # Modifies a resolv.conf on the Fuel master node and returns # its original content. # * adds 'nameservers' at start of resolv.conf if merge=True # * replaces resolv.conf with 'nameservers' if merge=False def modify_resolv_conf(self, nameservers=[], merge=True): remote = self.get_admin_remote() resolv_conf = remote.execute('cat /etc/resolv.conf') assert_equal( 0, resolv_conf['exit_code'], 'Executing "{0}" on the ' 'admin node has failed with: {1}'.format('cat /etc/resolv.conf', resolv_conf['stderr'])) if merge: nameservers.extend(resolv_conf['stdout']) resolv_keys = ['search', 'domain', 'nameserver'] resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers if any(x in ns for x in resolv_keys)) logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new)) echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new) echo_result = remote.execute(echo_cmd) assert_equal( 0, echo_result['exit_code'], 'Executing "{0}" on the ' 'admin node has failed with: {1}'.format(echo_cmd, echo_result['stderr'])) return resolv_conf['stdout'] @logwrap def execute_remote_cmd(self, remote, cmd, exit_code=0): result = remote.execute(cmd) assert_equal( result['exit_code'], exit_code, 'Failed to execute "{0}" on remote host: {1}'.format(cmd, result)) return result['stdout'] @logwrap def describe_second_admin_interface(self): remote = self.get_admin_remote() second_admin_network = self.get_network(self.admin_net2).split('/')[0] second_admin_netmask = self.get_net_mask(self.admin_net2) second_admin_if = settings.INTERFACES.get(self.admin_net2) second_admin_ip = str( self.nodes().admin.get_ip_address_by_network_name(self.admin_net2)) logger.info( ('Parameters for second admin interface configuration: ' 'Network - {0}, Netmask - {1}, Interface - {2}, ' 'IP Address - {3}').format(second_admin_network, second_admin_netmask, second_admin_if, second_admin_ip)) add_second_admin_ip = ('DEVICE={0}\\n' 'ONBOOT=yes\\n' 'NM_CONTROLLED=no\\n' 'USERCTL=no\\n' 'PEERDNS=no\\n' 'BOOTPROTO=static\\n' 'IPADDR={1}\\n' 'NETMASK={2}\\n').format( second_admin_if, second_admin_ip, second_admin_netmask) cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};' 'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format( add_second_admin_ip, second_admin_if, second_admin_ip) logger.debug( 'Trying to assign {0} IP to the {1} on master node...'.format( second_admin_ip, second_admin_if)) result = remote.execute(cmd) assert_equal(result['exit_code'], 0, ('Failed to assign second admin ' 'IP address on master node: {0}').format(result)) logger.debug('Done: {0}'.format(result['stdout'])) multiple_networks_hacks.configure_second_admin_firewall( self, second_admin_network, second_admin_netmask) @logwrap def get_masternode_uuid(self): return self.postgres_actions.run_query( db='nailgun', query="select master_node_uid from master_node_settings limit 1;")
class Shell(object): def __init__(self): super(Shell, self).__init__() self.params = self.get_params() self.manager = Manager() def execute(self): self.commands.get(self.params.command)(self) def do_list(self): env_list = self.manager.environment_list().values('name') for env in env_list: print env['name'] return env_list def node_dict(self, node): return {'name': node.name, 'vnc': node.get_vnc_port()} def do_show(self): environment = self.manager.environment_get(self.params.name) print '%5s %25s' % ("VNC", "NODE-NAME") for item in map(lambda x: self.node_dict(x), environment.nodes): print '%5s %25s' % (item['vnc'], item['name']) def do_erase(self): self.manager.environment_get(self.params.name).erase() def do_start(self): self.manager.environment_get(self.params.name).start() def do_destroy(self): self.manager.environment_get(self.params.name).destroy(verbose=False) def do_suspend(self): self.manager.environment_get(self.params.name).suspend(verbose=False) def do_resume(self): self.manager.environment_get(self.params.name).resume(verbose=False) def do_revert(self): self.manager.environment_get(self.params.name).revert( self.params.snapshot_name) def do_snapshot(self): self.manager.environment_get(self.params.name).snapshot( self.params.snapshot_name) def do_synchronize(self): self.manager.synchronize_environments() commands = { 'list': do_list, 'show': do_show, 'erase': do_erase, 'start': do_start, 'destroy': do_destroy, 'suspend': do_suspend, 'resume': do_resume, 'revert': do_revert, 'snapshot': do_snapshot, 'sync': do_synchronize } def get_params(self): name_parser = argparse.ArgumentParser(add_help=False) name_parser.add_argument('name', help='environment name', default=environ.get('ENV_NAME')) snapshot_name_parser = argparse.ArgumentParser(add_help=False) snapshot_name_parser.add_argument('--snapshot-name', help='snapshot name', default=environ.get('SNAPSHOT_NAME')) parser = argparse.ArgumentParser( description="Manage virtual environments") subparsers = parser.add_subparsers(help='commands', dest='command') subparsers.add_parser('list') subparsers.add_parser('show', parents=[name_parser]) subparsers.add_parser('erase', parents=[name_parser]) subparsers.add_parser('start', parents=[name_parser]) subparsers.add_parser('destroy', parents=[name_parser]) subparsers.add_parser('suspend', parents=[name_parser]) subparsers.add_parser('resume', parents=[name_parser]) subparsers.add_parser('revert', parents=[name_parser, snapshot_name_parser]) subparsers.add_parser('snapshot', parents=[name_parser, snapshot_name_parser]) subparsers.add_parser('sync') return parser.parse_args()
class EnvironmentModel(object): hostname = 'nailgun' domain = 'test.domain.local' installation_timeout = 1800 deployment_timeout = 1800 puppet_timeout = 1000 nat_interface = '' # INTERFACES.get('admin') admin_net = 'admin' def __init__(self): self._virtual_environment = None self._keys = None self.manager = Manager() self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self) def _get_or_create(self): try: return self.manager.environment_get(self.env_name) except: self._virtual_environment = self.describe_environment() self._virtual_environment.define() return self._virtual_environment def router(self, router_name=None): router_name = router_name or self.admin_net return str( IPNetwork( self.get_virtual_environment().network_by_name(router_name). ip_network)[1]) @property def fuel_web(self): """ :rtype: FuelWebClient """ return self._fuel_web @property def node_roles(self): return NodeRoles( admin_names=['admin'], other_names=['slave-%02d' % x for x in range(1, int( settings.NODES_COUNT))] ) @property def env_name(self): return settings.ENV_NAME def add_empty_volume(self, node, name, capacity=settings.NODE_VOLUME_SIZE*1024*1024*1024, device='disk', bus='virtio', format='qcow2'): self.manager.node_attach_volume( node=node, volume=self.manager.volume_create( name=name, capacity=capacity, environment=self.get_virtual_environment(), format=format), device=device, bus=bus) def add_node(self, memory, name, vcpu=1, boot=None): return self.manager.node_create( name=name, memory=memory, vcpu=vcpu, environment=self.get_virtual_environment(), boot=boot) @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server( cluster_id, self.get_host_node_ip(), port) def bootstrap_nodes(self, devops_nodes, timeout=600): """ Start vms and wait they are registered on nailgun. :rtype : List of registered nailgun nodes """ for node in devops_nodes: node.start() wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) for node in self.nailgun_nodes(devops_nodes): self.sync_node_time(self.get_ssh_to_remote(node["ip"])) return self.nailgun_nodes(devops_nodes) def create_interfaces(self, networks, node, model=settings.INTERFACE_MODEL): for network in networks: self.manager.interface_create(network, node=node, model=model) def describe_environment(self): """ :rtype : Environment """ environment = self.manager.environment_create(self.env_name) networks = [] for name in settings.INTERFACE_ORDER: ip_networks = [ IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',')] new_prefix = int(settings.POOLS.get(name)[1]) pool = self.manager.create_network_pool(networks=ip_networks, prefix=int(new_prefix)) networks.append(self.manager.network_create( name=name, environment=environment, pool=pool, forward=settings.FORWARDING.get(name), has_dhcp_server=settings.DHCP.get(name))) for name in self.node_roles.admin_names: self.describe_admin_node(name, networks) for name in self.node_roles.other_names: self.describe_empty_node(name, networks) return environment def devops_nodes_by_names(self, devops_node_names): return map( lambda name: self.get_virtual_environment().node_by_name(name), devops_node_names) @logwrap def describe_admin_node(self, name, networks): node = self.add_node( memory=settings.HARDWARE.get("admin_node_memory", 1024), vcpu=settings.HARDWARE.get("admin_node_cpu", 1), name=name, boot=['hd', 'cdrom']) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') self.add_empty_volume( node, name + '-iso', capacity=_get_file_size(settings.ISO_PATH), format='raw', device='cdrom', bus='ide') return node def describe_empty_node(self, name, networks): node = self.add_node( name=name, memory=settings.HARDWARE.get("slave_node_memory", 1024), vcpu=settings.HARDWARE.get("slave_node_cpu", 1)) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') if settings.USE_ALL_DISKS: self.add_empty_volume(node, name + '-cinder') self.add_empty_volume(node, name + '-swift') return node @logwrap def get_admin_remote(self): """ :rtype : SSHClient """ return self.nodes().admin.remote(self.admin_net, login='******', password='******') @logwrap def get_admin_node_ip(self): return str( self.nodes().admin.get_ip_address_by_network_name(self.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_host_node_ip(self): return self.router() def get_keys(self, node): params = { 'ip': node.get_ip_address_by_network_name(self.admin_net), 'mask': self.get_net_mask(self.admin_net), 'gw': self.router(), 'hostname': '.'.join((self.hostname, self.domain)), 'nat_interface': self.nat_interface, 'dns1': settings.DNS } keys = ( "<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " <Enter>\n" ) % params return keys @logwrap def get_private_keys(self, force=False): if force or self._keys is None: self._keys = [] for key_string in ['/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa']: with self.get_admin_remote().open(key_string) as f: self._keys.append(RSAKey.from_private_key(f)) return self._keys @logwrap def get_ssh_to_remote(self, ip): return SSHClient(ip, username='******', password='******', private_keys=self.get_private_keys()) @logwrap def get_ssh_to_remote_by_name(self, node_name): return self.get_ssh_to_remote( self.fuel_web.get_nailgun_node_by_devops_node( self.get_virtual_environment().node_by_name(node_name))['ip'] ) def get_target_devs(self, devops_nodes): return [ interface.target_dev for interface in [ val for var in map(lambda node: node.interfaces, devops_nodes) for val in var]] def get_virtual_environment(self): """ :rtype : devops.models.Environment """ if self._virtual_environment is None: self._virtual_environment = self._get_or_create() return self._virtual_environment def get_network(self, net_name): return str( IPNetwork( self.get_virtual_environment().network_by_name(net_name). ip_network)) def get_net_mask(self, net_name): return str( IPNetwork( self.get_virtual_environment().network_by_name( net_name).ip_network).netmask) def make_snapshot(self, snapshot_name): self.get_virtual_environment().suspend(verbose=False) self.get_virtual_environment().snapshot(snapshot_name, force=True) def nailgun_nodes(self, devops_nodes): return map( lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node), devops_nodes ) def nodes(self): return Nodes(self.get_virtual_environment(), self.node_roles) def revert_snapshot(self, name): if self.get_virtual_environment().has_snapshot(name): logging.info('We have snapshot with such name %s' % name) self.get_virtual_environment().revert(name) logging.info('Starting snapshot reverting ....') self.get_virtual_environment().resume() logging.info('Starting snapshot resuming ...') time.sleep(10) list = self.nodes().slaves for node in list: if not node.driver.node_active(node): continue try: self.sync_node_time(self.get_ssh_to_remote( node.get_ip_address_by_network_name(self.admin_net))) except Exception, e: logging.warn( 'Paramiko exception catched while' ' trying to run ntpdate: %s' % e) return True return False
class EnvironmentModel(object): hostname = 'nailgun' domain = 'test.domain.local' installation_timeout = 1800 deployment_timeout = 1800 puppet_timeout = 2000 nat_interface = '' # INTERFACES.get('admin') admin_net = 'admin' def __init__(self, os_image=None): self._virtual_environment = None self._keys = None self.manager = Manager() self.os_image = os_image self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self) def _get_or_create(self): try: return self.manager.environment_get(self.env_name) except Exception: self._virtual_environment = self.describe_environment() self._virtual_environment.define() return self._virtual_environment def router(self, router_name=None): router_name = router_name or self.admin_net return str( IPNetwork(self.get_virtual_environment().network_by_name( router_name).ip_network)[1]) @property def fuel_web(self): """FuelWebClient :rtype: FuelWebClient """ return self._fuel_web @property def admin_node_ip(self): return self.fuel_web.admin_node_ip @property def node_roles(self): return NodeRoles(admin_names=['admin'], other_names=[ 'slave-%02d' % x for x in range(1, int(settings.NODES_COUNT)) ]) @property def env_name(self): return settings.ENV_NAME def add_empty_volume(self, node, name, capacity=settings.NODE_VOLUME_SIZE * 1024 * 1024 * 1024, device='disk', bus='virtio', format='qcow2'): self.manager.node_attach_volume( node=node, volume=self.manager.volume_create( name=name, capacity=capacity, environment=self.get_virtual_environment(), format=format), device=device, bus=bus) def add_node(self, memory, name, vcpu=1, boot=None): return self.manager.node_create( name=name, memory=memory, vcpu=vcpu, environment=self.get_virtual_environment(), boot=boot) @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server(cluster_id, self.get_host_node_ip(), port) def bootstrap_nodes(self, devops_nodes, timeout=600): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ #self.dhcrelay_check() for node in devops_nodes: node.start() #TODO(aglarendil): LP#1317213 temporary sleep #remove after better fix is applied time.sleep(2) wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) for node in self.nailgun_nodes(devops_nodes): self.sync_node_time(self.get_ssh_to_remote(node["ip"])) return self.nailgun_nodes(devops_nodes) def create_interfaces(self, networks, node, model=settings.INTERFACE_MODEL): if settings.BONDING: for network in networks: self.manager.interface_create( network, node=node, model=model, interface_map=settings.BONDING_INTERFACES) else: for network in networks: self.manager.interface_create(network, node=node, model=model) def describe_environment(self): """Environment :rtype : Environment """ environment = self.manager.environment_create(self.env_name) networks = [] interfaces = settings.INTERFACE_ORDER if settings.BONDING: interfaces = settings.BONDING_INTERFACES.keys() for name in interfaces: networks.append(self.create_networks(name, environment)) for name in self.node_roles.admin_names: self.describe_admin_node(name, networks) for name in self.node_roles.other_names: self.describe_empty_node(name, networks) return environment def create_networks(self, name, environment): ip_networks = [ IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',') ] new_prefix = int(settings.POOLS.get(name)[1]) pool = self.manager.create_network_pool(networks=ip_networks, prefix=int(new_prefix)) return self.manager.network_create( name=name, environment=environment, pool=pool, forward=settings.FORWARDING.get(name), has_dhcp_server=settings.DHCP.get(name)) def devops_nodes_by_names(self, devops_node_names): return map( lambda name: self.get_virtual_environment().node_by_name(name), devops_node_names) @logwrap def describe_admin_node(self, name, networks): node = self.add_node(memory=settings.HARDWARE.get( "admin_node_memory", 1024), vcpu=settings.HARDWARE.get("admin_node_cpu", 1), name=name, boot=['hd', 'cdrom']) self.create_interfaces(networks, node) if self.os_image is None: self.add_empty_volume(node, name + '-system') self.add_empty_volume(node, name + '-iso', capacity=_get_file_size(settings.ISO_PATH), format='raw', device='cdrom', bus='ide') else: volume = self.manager.volume_get_predefined(self.os_image) vol_child = self.manager.volume_create_child( name=name + '-system', backing_store=volume, environment=self.get_virtual_environment()) self.manager.node_attach_volume(node=node, volume=vol_child) return node def describe_empty_node(self, name, networks): node = self.add_node(name=name, memory=settings.HARDWARE.get( "slave_node_memory", 1024), vcpu=settings.HARDWARE.get("slave_node_cpu", 1)) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') if settings.USE_ALL_DISKS: self.add_empty_volume(node, name + '-cinder') self.add_empty_volume(node, name + '-swift') return node @logwrap def get_admin_remote(self): """SSH to admin node :rtype : SSHClient """ return self.nodes().admin.remote(self.admin_net, login='******', password='******') @logwrap def get_admin_node_ip(self): return str(self.nodes().admin.get_ip_address_by_network_name( self.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_host_node_ip(self): return self.router() def get_keys(self, node, custom=None): params = { 'ip': node.get_ip_address_by_network_name(self.admin_net), 'mask': self.get_net_mask(self.admin_net), 'gw': self.router(), 'hostname': '.'.join((self.hostname, self.domain)), 'nat_interface': self.nat_interface, 'dns1': settings.DNS, 'showmenu': 'yes' if custom else 'no' } keys = ("<Wait>\n" "<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " showmenu=%(showmenu)s\n" " <Enter>\n") % params return keys @logwrap def get_private_keys(self, force=False): if force or self._keys is None: self._keys = [] for key_string in [ '/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa' ]: with self.get_admin_remote().open(key_string) as f: self._keys.append(RSAKey.from_private_key(f)) return self._keys @logwrap def get_ssh_to_remote(self, ip): return SSHClient(ip, username='******', password='******', private_keys=self.get_private_keys()) @logwrap def get_ssh_to_remote_by_name(self, node_name): return self.get_ssh_to_remote( self.fuel_web.get_nailgun_node_by_devops_node( self.get_virtual_environment().node_by_name(node_name))['ip']) def get_target_devs(self, devops_nodes): return [ interface.target_dev for interface in [ val for var in map(lambda node: node.interfaces, devops_nodes) for val in var ] ] def get_virtual_environment(self): """Returns virtual environment :rtype : devops.models.Environment """ if self._virtual_environment is None: self._virtual_environment = self._get_or_create() return self._virtual_environment def get_network(self, net_name): return str( IPNetwork(self.get_virtual_environment().network_by_name( net_name).ip_network)) def get_net_mask(self, net_name): return str( IPNetwork(self.get_virtual_environment().network_by_name( net_name).ip_network).netmask) def make_snapshot(self, snapshot_name, description="", is_make=False): if settings.MAKE_SNAPSHOT or is_make: self.get_virtual_environment().suspend(verbose=False) self.get_virtual_environment().snapshot(snapshot_name, force=True) revert_info(snapshot_name, description) def nailgun_nodes(self, devops_nodes): return map( lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node), devops_nodes) def nodes(self): return Nodes(self.get_virtual_environment(), self.node_roles) def revert_snapshot(self, name): if self.get_virtual_environment().has_snapshot(name): logger.info('We have snapshot with such name %s' % name) self.get_virtual_environment().revert(name) logger.info('Starting snapshot reverting ....') self.get_virtual_environment().resume() logger.info('Starting snapshot resuming ...') admin = self.nodes().admin try: admin. await (self.admin_net, timeout=10 * 60, by_port=8000) except Exception as e: logger.warning("From first time admin isn't reverted: " "{0}".format(e)) admin.destroy() logger.info('Admin node was destroyed. Wait 10 sec.') time.sleep(10) self.get_virtual_environment().start(self.nodes().admins) logger.info('Admin node started second time.') self.nodes().admin. await (self.admin_net, timeout=10 * 60, by_port=8000) _wait(self._fuel_web.get_nailgun_version, timeout=120) self.sync_time_admin_node() for node in self.nodes().slaves: if not node.driver.node_active(node): continue try: logger.info("Sync time on revert for node %s" % node.name) self.sync_node_time( self.get_ssh_to_remote_by_name(node.name)) except Exception as e: logger.warning( 'Exception caught while trying to sync time on {0}:' ' {1}'.format(node.name, e)) self.run_nailgun_agent( self.get_ssh_to_remote_by_name(node.name)) return True return False def setup_environment(self, custom=False): # start admin node admin = self.nodes().admin admin.disk_devices.get(device='cdrom').volume.upload(settings.ISO_PATH) self.get_virtual_environment().start(self.nodes().admins) logger.info("Waiting for admin node to start up") wait(lambda: admin.driver.node_active(admin), 60) logger.info("Proceed with installation") # update network parameters at boot screen admin.send_keys(self.get_keys(admin, custom=custom)) if custom: self.setup_customisation() # wait while installation complete admin. await (self.admin_net, timeout=10 * 60) self.wait_bootstrap() time.sleep(10) self.sync_time_admin_node() @upload_manifests def wait_for_provisioning(self): _wait(lambda: _tcp_ping( self.nodes().admin.get_ip_address_by_network_name(self.admin_net), 22), timeout=5 * 60) def setup_customisation(self): self.wait_for_provisioning() try: remote = self.get_admin_remote() pid = remote.execute("pgrep 'fuelmenu'")['stdout'][0] pid.rstrip('\n') remote.execute("kill -sigusr1 {0}".format(pid)) except Exception: logger.error("Could not kill pid of fuelmenu") raise @retry(count=10, delay=60) @logwrap def sync_node_time(self, remote): self.execute_remote_cmd(remote, 'hwclock -s') self.execute_remote_cmd( remote, 'NTPD=$(find /etc/init.d/ -regex \'' '/etc/init.d/ntp.?\'); $NTPD stop; ' 'killall ntpd; ntpd -qg && ' '$NTPD start') self.execute_remote_cmd(remote, 'hwclock -w') remote_date = remote.execute('date')['stdout'] logger.info("Node time: %s" % remote_date) @retry(count=10, delay=60) @logwrap def sync_time_admin_node(self): logger.info("Sync time on revert for admin") remote = self.get_admin_remote() self.execute_remote_cmd(remote, 'hwclock -s') # Sync time using ntpd try: # If public NTP servers aren't accessible ntpdate will fail and # ntpd daemon shouldn't be restarted to avoid 'Server has gone # too long without sync' error while syncing time from slaves self.execute_remote_cmd( remote, "ntpdate -d $(awk '/^server/{print" " $2}' /etc/ntp.conf)") except AssertionError as e: logger.warning('Error occurred while synchronizing time on master' ': {0}'.format(e)) raise else: self.execute_remote_cmd( remote, 'service ntpd stop && ntpd -qg && ' 'service ntpd start') self.execute_remote_cmd(remote, 'hwclock -w') remote_date = remote.execute('date')['stdout'] logger.info("Master node time: {0}".format(remote_date)) def verify_network_configuration(self, node_name): checkers.verify_network_configuration( node=self.fuel_web.get_nailgun_node_by_name(node_name), remote=self.get_ssh_to_remote_by_name(node_name)) def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" logger.info("Puppet timeout set in {0}".format( float(settings.PUPPET_TIMEOUT))) wait(lambda: not self.get_admin_remote().execute( "grep 'Fuel node deployment' '%s'" % log_path)['exit_code'], timeout=(float(settings.PUPPET_TIMEOUT))) result = self.get_admin_remote().execute("grep 'Fuel node deployment " "complete' '%s'" % log_path)['exit_code'] if result != 0: raise Exception('Fuel node deployment failed.') def dhcrelay_check(self): admin_remote = self.get_admin_remote() out = admin_remote.execute("dhcpcheck discover " "--ifaces eth0 " "--repeat 3 " "--timeout 10")['stdout'] assert_true(self.get_admin_node_ip() in "".join(out), "dhcpcheck doesn't discover master ip") def run_nailgun_agent(self, remote): agent = remote.execute('/opt/nailgun/bin/agent')['exit_code'] logger.info("Nailgun agent run with exit_code: %s" % agent) def get_fuel_settings(self, remote=None): if not remote: remote = self.get_admin_remote() cmd = 'cat {cfg_file}'.format(cfg_file=settings.FUEL_SETTINGS_YAML) result = remote.execute(cmd) if result['exit_code'] == 0: fuel_settings = yaml.load(''.join(result['stdout'])) else: raise Exception('Can\'t output {cfg_file} file: {error}'.format( cfg_file=settings.FUEL_SETTINGS_YAML, error=result['stderr'])) return fuel_settings def admin_install_pkg(self, pkg_name): """Install a package <pkg_name> on the admin node""" admin_remote = self.get_admin_remote() remote_status = admin_remote.execute("rpm -q {0}'".format(pkg_name)) if remote_status['exit_code'] == 0: logger.info("Package '{0}' already installed.".format(pkg_name)) else: logger.info("Installing package '{0}' ...".format(pkg_name)) remote_status = admin_remote.execute( "yum -y install {0}".format(pkg_name)) logger.info("Installation of the package '{0}' has been" " completed with exit code {1}".format( pkg_name, remote_status['exit_code'])) return remote_status['exit_code'] def admin_run_service(self, service_name): """Start a service <service_name> on the admin node""" admin_remote = self.get_admin_remote() admin_remote.execute("service {0} start".format(service_name)) remote_status = admin_remote.execute( "service {0} status".format(service_name)) if any('running...' in status for status in remote_status['stdout']): logger.info("Service '{0}' is running".format(service_name)) else: logger.info("Service '{0}' failed to start" " with exit code {1} :\n{2}".format( service_name, remote_status['exit_code'], remote_status['stdout'])) # Modifies a resolv.conf on the Fuel master node and returns # its original content. # * adds 'nameservers' at start of resolv.conf if merge=True # * replaces resolv.conf with 'nameservers' if merge=False def modify_resolv_conf(self, nameservers=[], merge=True): remote = self.get_admin_remote() resolv_conf = remote.execute('cat /etc/resolv.conf') assert_equal( 0, resolv_conf['exit_code'], 'Executing "{0}" on the ' 'admin node has failed with: {1}'.format('cat /etc/resolv.conf', resolv_conf['stderr'])) if merge: nameservers.extend(resolv_conf['stdout']) resolv_keys = ['search', 'domain', 'nameserver'] resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers if any(x in ns for x in resolv_keys)) logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new)) echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new) echo_result = remote.execute(echo_cmd) assert_equal( 0, echo_result['exit_code'], 'Executing "{0}" on the ' 'admin node has failed with: {1}'.format(echo_cmd, echo_result['stderr'])) return resolv_conf['stdout'] @logwrap def execute_remote_cmd(self, remote, cmd, exit_code=0): result = remote.execute(cmd) assert_equal( result['exit_code'], exit_code, 'Failed to execute "{0}" on remote host: {1}'.format( cmd, result['stderr'])) return result['stdout']
class CiBase(object): def __init__(self): self.manager = Manager() self._environment = None self.saved_environment_states = {} def _get_or_create(self): try: return self.manager.environment_get(self.env_name()) except: self._environment = self.describe_environment() self._environment.define() return self._environment def get_state(self, name): if self.environment().has_snapshot(name): self.environment().revert(name) return True return False def environment(self): """ :rtype : devops.models.Environment """ self._environment = self._environment or self._get_or_create() return self._environment @abstractproperty def env_name(self): """ :rtype : string """ pass @abstractmethod def describe_environment(self): """ :rtype : devops.models.Environment """ pass @abstractproperty def node_roles(self): """ :rtype : NodeRoles """ pass def nodes(self): return Nodes(self.environment(), self.node_roles()) # noinspection PyShadowingBuiltins def add_empty_volume( self, node, name, capacity=20 * 1024 * 1024 * 1024, device="disk", bus="virtio", format="qcow2" ): self.manager.node_attach_volume( node=node, volume=self.manager.volume_create( name=name, capacity=capacity, environment=self.environment(), format=format ), device=device, bus=bus, ) def add_node(self, memory, name, boot=None): return self.manager.node_create(name=name, memory=memory, environment=self.environment(), boot=boot) def create_interfaces(self, networks, node): for network in networks: self.manager.interface_create(network, node=node) def describe_admin_node(self, name, networks, memory=1024): node = self.add_node(memory=memory, name=name, boot=["hd", "cdrom"]) self.create_interfaces(networks, node) self.add_empty_volume(node, name + "-system") self.add_empty_volume( node, name + "-iso", capacity=_get_file_size(ISO_PATH), format="raw", device="cdrom", bus="ide" ) return node def describe_empty_node(self, name, networks, memory=1024): node = self.add_node(memory, name) self.create_interfaces(networks, node) self.add_empty_volume(node, name + "-system") if USE_ALL_DISKS: self.add_empty_volume(node, name + "-cinder") self.add_empty_volume(node, name + "-swift") return node @abstractmethod def setup_environment(self): """ :rtype : None """ pass def get_empty_environment(self): if not (self.get_state(EMPTY_SNAPSHOT)): self.setup_environment() self.environment().snapshot(EMPTY_SNAPSHOT) def generate_state_hash(self, settings): return hashlib.md5(str(settings)).hexdigest() def revert_to_state(self, settings={}): state_hash = self.generate_state_hash(settings) if state_hash in self.saved_environment_states: # revert to matching state state = self.saved_environment_states[state_hash] if not (self.get_state(state["snapshot_name"])): return False self.environment().resume() return True return False def snapshot_state(self, name, settings={}): state_hash = self.generate_state_hash(settings) snapshot_name = "{0}_{1}".format(name.replace(" ", "_")[:17], state_hash) self.environment().suspend(verbose=False) self.environment().snapshot(name=snapshot_name, description=name, force=True) self.environment().resume(verbose=False) self.saved_environment_states[state_hash] = { "snapshot_name": snapshot_name, "cluster_name": name, "settings": settings, } def internal_virtual_ip(self): return str(IPNetwork(self.environment().network_by_name("internal").ip_network)[-2]) def public_router(self): return str(IPNetwork(self.environment().network_by_name("public").ip_network)[1]) def internal_router(self): return self._router("internal") def nat_router(self): return self._router("nat") def _router(self, router_name): return str(IPNetwork(self.environment().network_by_name(router_name).ip_network)[1]) def get_host_node_ip(self): return self.internal_router() def internal_network(self): return str(IPNetwork(self.environment().network_by_name("internal").ip_network)) def internal_net_mask(self): return str(IPNetwork(self.environment().network_by_name("internal").ip_network).netmask) def public_net_mask(self): return str(IPNetwork(self.environment().network_by_name("public").ip_network).netmask) def public_network(self): return str(IPNetwork(self.environment().network_by_name("public").ip_network))
class EnvManager: """ Class for create environment in puppet modules testing. """ env_name = os.environ.get("ENV_NAME", "puppet-integration") env_node_name = "node" env_net_public = NET_PUBLIC env_net_internal = NET_INTERNAL env_net_private = NET_PRIVATE env_vol = "vol" login = "******" password = "******" def __init__(self, base_image=None): """ Constructor for create environment. """ self.manager = Manager() self.base_image = base_image or BASE_IMAGE self.environment = self.create_env() def create_env(self): try: return self.manager.environment_get(self.env_name) except: return self._define_env() def _define_env(self): """ Create environment with default settings. """ self.environment = self.manager.environment_create(self.env_name) internal = self.manager.network_create(environment=self.environment, name=self.env_net_internal, pool=None) external = self.manager.network_create(environment=self.environment, name=self.env_net_public, pool=None) private = self.manager.network_create(environment=self.environment, name=self.env_net_private, pool=None) node = self.manager.node_create(name=self.env_node_name, environment=self.environment) self.manager.interface_create(node=node, network=internal) self.manager.interface_create(node=node, network=external) self.manager.interface_create(node=node, network=private) volume = self.manager.volume_get_predefined(self.base_image) v3 = self.manager.volume_create_child(self.env_vol, backing_store=volume, environment=self.environment) self.manager.node_attach_volume(node=node, volume=v3) self.environment.define() self.environment.start() return self.environment def _get_public_ip(self): return self.environment.node_by_name(self.env_node_name).get_ip_address_by_network_name(self.env_net_public) def _ssh(self): return ssh(self._get_public_ip(), username=self.login, password=self.password).sudo.ssh def remote(self): """ Return remote access to node by name with default login/password. """ return self._ssh() def snapshot_exist(self, snap_name="before_test"): return self.environment.has_snapshot(name=snap_name) def create_snapshot_env(self, snap_name="", description="", force=True): """ Create snapshot for environment. """ self.environment.snapshot(name=snap_name, description=description, force=force) def revert_snapshot_env(self, snap_name="", destroy=True): """ Revert environment to snapshot by name. """ self.environment.revert(name=snap_name, destroy=destroy) def erase_env(self): """ Erase environment. """ self.environment.erase() def execute_cmd(self, command, debug=True): """ Execute command on node. """ return self.remote().execute(command, verbose=debug)["exit_code"] def upload_files(self, source, dest): """ Upload file(s) to node. """ self.remote().upload(source, dest) def upload_modules(self, local_dir, remote_dir="/etc/puppet/modules/"): """ Upload puppet modules. """ upload_recipes(remote=self.remote(), local_dir=local_dir, remote_dir=remote_dir) def await(self, timeout=1200):
class EnvironmentModel(object): hostname = 'nailgun' domain = 'test.domain.local' installation_timeout = 1800 deployment_timeout = 1800 puppet_timeout = 2000 nat_interface = '' # INTERFACES.get('admin') admin_net = 'admin' def __init__(self, os_image=None): self._virtual_environment = None self._keys = None self.manager = Manager() self.os_image = os_image self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self) def _get_or_create(self): try: return self.manager.environment_get(self.env_name) except Exception: self._virtual_environment = self.describe_environment() self._virtual_environment.define() return self._virtual_environment def router(self, router_name=None): router_name = router_name or self.admin_net return str( IPNetwork(self.get_virtual_environment().network_by_name( router_name).ip_network)[1]) @property def fuel_web(self): """FuelWebClient :rtype: FuelWebClient """ return self._fuel_web @property def node_roles(self): return NodeRoles(admin_names=['admin'], other_names=[ 'slave-%02d' % x for x in range(1, int(settings.NODES_COUNT)) ]) @property def env_name(self): return settings.ENV_NAME def add_empty_volume(self, node, name, capacity=settings.NODE_VOLUME_SIZE * 1024 * 1024 * 1024, device='disk', bus='virtio', format='qcow2'): self.manager.node_attach_volume( node=node, volume=self.manager.volume_create( name=name, capacity=capacity, environment=self.get_virtual_environment(), format=format), device=device, bus=bus) def add_node(self, memory, name, vcpu=1, boot=None): return self.manager.node_create( name=name, memory=memory, vcpu=vcpu, environment=self.get_virtual_environment(), boot=boot) @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server(cluster_id, self.get_host_node_ip(), port) def bootstrap_nodes(self, devops_nodes, timeout=600): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ self.dhcrelay_check() for node in devops_nodes: node.start() #TODO(aglarendil): LP#1317213 temporary sleep #remove after better fix is applied time.sleep(2) wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) for node in self.nailgun_nodes(devops_nodes): self.sync_node_time(self.get_ssh_to_remote(node["ip"])) return self.nailgun_nodes(devops_nodes) def create_interfaces(self, networks, node, model=settings.INTERFACE_MODEL): for network in networks: self.manager.interface_create(network, node=node, model=model) def describe_environment(self): """Environment :rtype : Environment """ environment = self.manager.environment_create(self.env_name) networks = [] for name in settings.INTERFACE_ORDER: ip_networks = [ IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',') ] new_prefix = int(settings.POOLS.get(name)[1]) pool = self.manager.create_network_pool(networks=ip_networks, prefix=int(new_prefix)) networks.append( self.manager.network_create( name=name, environment=environment, pool=pool, forward=settings.FORWARDING.get(name), has_dhcp_server=settings.DHCP.get(name))) for name in self.node_roles.admin_names: self.describe_admin_node(name, networks) for name in self.node_roles.other_names: self.describe_empty_node(name, networks) return environment def devops_nodes_by_names(self, devops_node_names): return map( lambda name: self.get_virtual_environment().node_by_name(name), devops_node_names) @logwrap def describe_admin_node(self, name, networks): node = self.add_node(memory=settings.HARDWARE.get( "admin_node_memory", 1024), vcpu=settings.HARDWARE.get("admin_node_cpu", 1), name=name, boot=['hd', 'cdrom']) self.create_interfaces(networks, node) if self.os_image is None: self.add_empty_volume(node, name + '-system') self.add_empty_volume(node, name + '-iso', capacity=_get_file_size(settings.ISO_PATH), format='raw', device='cdrom', bus='ide') else: volume = self.manager.volume_get_predefined(self.os_image) vol_child = self.manager.volume_create_child( name=name + '-system', backing_store=volume, environment=self.get_virtual_environment()) self.manager.node_attach_volume(node=node, volume=vol_child) return node def describe_empty_node(self, name, networks): node = self.add_node(name=name, memory=settings.HARDWARE.get( "slave_node_memory", 1024), vcpu=settings.HARDWARE.get("slave_node_cpu", 1)) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') if settings.USE_ALL_DISKS: self.add_empty_volume(node, name + '-cinder') self.add_empty_volume(node, name + '-swift') return node @logwrap def get_admin_remote(self): """SSH to admin node :rtype : SSHClient """ return self.nodes().admin.remote(self.admin_net, login='******', password='******') @logwrap def get_admin_node_ip(self): return str(self.nodes().admin.get_ip_address_by_network_name( self.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_host_node_ip(self): return self.router() def get_keys(self, node): params = { 'ip': node.get_ip_address_by_network_name(self.admin_net), 'mask': self.get_net_mask(self.admin_net), 'gw': self.router(), 'hostname': '.'.join((self.hostname, self.domain)), 'nat_interface': self.nat_interface, 'dns1': settings.DNS } keys = ("<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " <Enter>\n") % params return keys @logwrap def get_private_keys(self, force=False): if force or self._keys is None: self._keys = [] for key_string in [ '/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa' ]: with self.get_admin_remote().open(key_string) as f: self._keys.append(RSAKey.from_private_key(f)) return self._keys @logwrap def get_ssh_to_remote(self, ip): return SSHClient(ip, username='******', password='******', private_keys=self.get_private_keys()) @logwrap def get_ssh_to_remote_by_name(self, node_name): return self.get_ssh_to_remote( self.fuel_web.get_nailgun_node_by_devops_node( self.get_virtual_environment().node_by_name(node_name))['ip']) def get_target_devs(self, devops_nodes): return [ interface.target_dev for interface in [ val for var in map(lambda node: node.interfaces, devops_nodes) for val in var ] ] def get_virtual_environment(self): """Returns virtual environment :rtype : devops.models.Environment """ if self._virtual_environment is None: self._virtual_environment = self._get_or_create() return self._virtual_environment def get_network(self, net_name): return str( IPNetwork(self.get_virtual_environment().network_by_name( net_name).ip_network)) def get_net_mask(self, net_name): return str( IPNetwork(self.get_virtual_environment().network_by_name( net_name).ip_network).netmask) def make_snapshot(self, snapshot_name, description="", is_make=False): if settings.MAKE_SNAPSHOT or is_make: self.get_virtual_environment().suspend(verbose=False) self.get_virtual_environment().snapshot(snapshot_name, force=True) revert_info(snapshot_name, description) def nailgun_nodes(self, devops_nodes): return map( lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node), devops_nodes) def nodes(self): return Nodes(self.get_virtual_environment(), self.node_roles) def revert_snapshot(self, name): if self.get_virtual_environment().has_snapshot(name): logger.info('We have snapshot with such name %s' % name) self.get_virtual_environment().revert(name) logger.info('Starting snapshot reverting ....') self.get_virtual_environment().resume() logger.info('Starting snapshot resuming ...') self.nodes().admin. await (self.admin_net, timeout=10 * 60) self.sync_time_admin_node() for node in self.nodes().slaves: if not node.driver.node_active(node): continue try: logger.info("Sync time on revert for node %s" % node.name) self.sync_node_time( self.get_ssh_to_remote_by_name(node.name)) except Exception as e: logger.warn('Paramiko exception catched while' ' trying to run ntpdate: %s' % e) self.run_nailgun_agent( self.get_ssh_to_remote_by_name(node.name)) return True return False def setup_environment(self): # start admin node admin = self.nodes().admin admin.disk_devices.get(device='cdrom').volume.upload(settings.ISO_PATH) self.get_virtual_environment().start(self.nodes().admins) # update network parameters at boot screen time.sleep(float(settings.ADMIN_NODE_SETUP_TIMEOUT)) admin.send_keys(self.get_keys(admin)) # wait while installation complete admin. await (self.admin_net, timeout=10 * 60) self.wait_bootstrap() time.sleep(10) self.sync_time_admin_node() @retry() @logwrap def sync_node_time(self, remote): remote.execute('hwclock --hctosys') ntpd_exit = remote.execute("ntpdate -u $(egrep '^server' " "/etc/ntp.conf | sed '/^#/d' | " "awk '{print $2}')")['exit_code'] assert_equal(0, ntpd_exit) remote.execute('hwclock -w') remote_date = remote.execute('date')['stdout'] logger.info("Node time: %s" % remote_date) def sync_time_admin_node(self): logger.info("Sync time on revert for admin") self.sync_node_time(self.get_admin_remote()) def verify_node_service_list(self, node_name, smiles_count): remote = self.get_ssh_to_remote_by_name(node_name) checkers.verify_service_list(remote, smiles_count) def verify_network_configuration(self, node_name): checkers.verify_network_configuration( node=self.fuel_web.get_nailgun_node_by_name(node_name), remote=self.get_ssh_to_remote_by_name(node_name)) def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" logger.info("Puppet timeout set in {0}".format( float(settings.PUPPET_TIMEOUT))) wait(lambda: not self.get_admin_remote().execute( "grep 'Fuel node deployment complete' '%s'" % log_path)['exit_code' ], timeout=(float(settings.PUPPET_TIMEOUT))) def dhcrelay_check(self): admin_remote = self.get_admin_remote() out = admin_remote.execute("dhcpcheck discover " "--ifaces eth0 " "--repeat 3 " "--timeout 10")['stdout'] master_ip = filter(lambda x: self.get_admin_node_ip() in x, out) logger.info("dhcpcheck discover: %s" % master_ip) assert_equal(len(master_ip), 1) def run_nailgun_agent(self, remote): agent = remote.execute('/opt/nailgun/bin/agent')['exit_code'] logger.info("Nailgun agent run with exit_code: %s" % agent)
class EnvironmentModel(object): hostname = "nailgun" domain = "test.domain.local" installation_timeout = 1800 deployment_timeout = 1800 puppet_timeout = 1000 nat_interface = "" # INTERFACES.get('admin') admin_net = "admin" def __init__(self): self._virtual_environment = None self._keys = None self.manager = Manager() self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self) def _get_or_create(self): try: return self.manager.environment_get(self.env_name) except: self._virtual_environment = self.describe_environment() self._virtual_environment.define() return self._virtual_environment def router(self, router_name=None): router_name = router_name or self.admin_net return str(IPNetwork(self.get_virtual_environment().network_by_name(router_name).ip_network)[1]) @property def fuel_web(self): """ :rtype: FuelWebClient """ return self._fuel_web @property def node_roles(self): return NodeRoles(admin_names=["admin"], other_names=["slave-%02d" % x for x in range(1, 10)]) @property def env_name(self): return ENV_NAME def add_empty_volume( self, node, name, capacity=NODE_VOLUME_SIZE * 1024 * 1024 * 1024, device="disk", bus="virtio", format="qcow2" ): self.manager.node_attach_volume( node=node, volume=self.manager.volume_create( name=name, capacity=capacity, environment=self.get_virtual_environment(), format=format ), device=device, bus=bus, ) def add_node(self, memory, name, vcpu=1, boot=None): return self.manager.node_create( name=name, memory=memory, vcpu=vcpu, environment=self.get_virtual_environment(), boot=boot ) @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server(cluster_id, self.get_host_node_ip(), port) def bootstrap_nodes(self, devops_nodes, timeout=600): """ Start vms and wait they are registered on nailgun. :rtype : List of registered nailgun nodes """ for node in devops_nodes: node.start() wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) for node in self.nailgun_nodes(devops_nodes): self.sync_node_time(self.get_ssh_to_remote(node["ip"])) return self.nailgun_nodes(devops_nodes) def create_interfaces(self, networks, node, model=INTERFACE_MODEL): for network in networks: self.manager.interface_create(network, node=node, model=model) def describe_environment(self): """ :rtype : Environment """ environment = self.manager.environment_create(self.env_name) networks = [] for name in INTERFACE_ORDER: ip_networks = [IPNetwork(x) for x in POOLS.get(name)[0].split(",")] new_prefix = int(POOLS.get(name)[1]) pool = self.manager.create_network_pool(networks=ip_networks, prefix=int(new_prefix)) networks.append( self.manager.network_create( name=name, environment=environment, pool=pool, forward=FORWARDING.get(name), has_dhcp_server=DHCP.get(name), ) ) for name in self.node_roles.admin_names: self.describe_admin_node(name, networks) for name in self.node_roles.other_names: self.describe_empty_node(name, networks) return environment def devops_nodes_by_names(self, devops_node_names): return map(lambda name: self.get_virtual_environment().node_by_name(name), devops_node_names) @logwrap def describe_admin_node(self, name, networks): node = self.add_node( memory=HARDWARE.get("admin_node_memory", 1024), vcpu=HARDWARE.get("admin_node_cpu", 1), name=name, boot=["hd", "cdrom"], ) self.create_interfaces(networks, node) self.add_empty_volume(node, name + "-system") self.add_empty_volume( node, name + "-iso", capacity=_get_file_size(ISO_PATH), format="raw", device="cdrom", bus="ide" ) return node def describe_empty_node(self, name, networks): node = self.add_node( name=name, memory=HARDWARE.get("slave_node_memory", 1024), vcpu=HARDWARE.get("slave_node_cpu", 1) ) self.create_interfaces(networks, node) self.add_empty_volume(node, name + "-system") if USE_ALL_DISKS: self.add_empty_volume(node, name + "-cinder") self.add_empty_volume(node, name + "-swift") return node @logwrap def get_admin_remote(self): """ :rtype : SSHClient """ return self.nodes().admin.remote(self.admin_net, login="******", password="******") @logwrap def get_admin_node_ip(self): return str(self.nodes().admin.get_ip_address_by_network_name(self.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_host_node_ip(self): return self.router() def get_keys(self, node): params = { "ip": node.get_ip_address_by_network_name(self.admin_net), "mask": self.get_net_mask(self.admin_net), "gw": self.router(), "hostname": ".".join((self.hostname, self.domain)), "nat_interface": self.nat_interface, "dns1": DNS, } keys = ( "<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " <Enter>\n" ) % params return keys @logwrap def get_private_keys(self, force=False): if force or self._keys is None: self._keys = [] for key_string in ["/root/.ssh/id_rsa", "/root/.ssh/bootstrap.rsa"]: with self.get_admin_remote().open(key_string) as f: self._keys.append(RSAKey.from_private_key(f)) return self._keys @logwrap def get_ssh_to_remote(self, ip): return SSHClient(ip, username="******", password="******", private_keys=self.get_private_keys()) @logwrap def get_ssh_to_remote_by_name(self, node_name): return self.get_ssh_to_remote( self.fuel_web.get_nailgun_node_by_devops_node(self.get_virtual_environment().node_by_name(node_name))["ip"] ) def get_target_devs(self, devops_nodes): return [ interface.target_dev for interface in [val for var in map(lambda node: node.interfaces, devops_nodes) for val in var] ] def get_virtual_environment(self): """ :rtype : devops.models.Environment """ if self._virtual_environment is None: self._virtual_environment = self._get_or_create() return self._virtual_environment def get_network(self, net_name): return str(IPNetwork(self.get_virtual_environment().network_by_name(net_name).ip_network)) def get_net_mask(self, net_name): return str(IPNetwork(self.get_virtual_environment().network_by_name(net_name).ip_network).netmask) def make_snapshot(self, snapshot_name): self.get_virtual_environment().suspend(verbose=False) self.get_virtual_environment().snapshot(snapshot_name, force=True) def nailgun_nodes(self, devops_nodes): return map(lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node), devops_nodes) def nodes(self): return Nodes(self.get_virtual_environment(), self.node_roles) def revert_snapshot(self, name): if self.get_virtual_environment().has_snapshot(name): logging.info("We have snapshot with such name %s" % name) self.get_virtual_environment().revert(name) logging.info("Starting snapshot reverting ....") self.get_virtual_environment().resume() logging.info("Starting snapshot resuming ...") time.sleep(10) list = self.nodes().slaves for node in list: try: self.sync_node_time(self.get_ssh_to_remote(node.get_ip_address_by_network_name(self.admin_net))) except Exception, e: logging.warn("Exception got: %s" % e) traceback.print_exc() return True return False
class Environment(object): capacity = 20 * 1024 * 1024 * 1024 boot = ['hd', 'cdrom'] login = "******" password = "******" name = 'fuel' def __init__(self, name=name, base_image=None): self.manager = Manager() self.name = name self.base_image = base_image self.environment = self._get_or_create() def _get_or_create(self): try: return self.manager.environment_get(self.name) except: self.environment = self._create() self.environment.define() return self.environment def nodes(self): return Nodes(self.environment, self._node_roles()) def get_empty_state(self): if self.environment.has_snapshot(EMPTY_SNAPSHOT): self.environment.revert(EMPTY_SNAPSHOT) return True return None def get_env(self): return self.environment def add_empty_volume(self, node, name, capacity=capacity, device='disk', bus='virtio', format='qcow2'): self.manager.node_attach_volume(node=node, volume=self.manager.volume_create(name=name, capacity=capacity, environment=self.environment, format=format), device=device, bus=bus) def add_node(self, name, memory, boot=None): return self.manager.node_create(name=name, memory=memory, environment=self.environment, boot=boot) def create_interfaces(self, node, networks): for network in networks: self.manager.interface_create(network, node=node) def describe_admin_node(self, name, networks, memory=DEFAULT_RAM_SIZE, boot=boot): node = self.add_node(memory=memory, name=name, boot=boot) self.create_interfaces(node, networks) if self.base_image is None: self.add_empty_volume(node, name + '-system') self.add_empty_volume(node, name + '-iso', capacity=_get_file_size(ISO_PATH), format='raw', device='cdrom', bus='ide') else: volume = self.manager.volume_get_predefined(self.base_image) v = self.manager.volume_create_child(name + '-system', backing_store=volume, environment=self.environment) self.manager.node_attach_volume(node=node, volume=v) return node def describe_node(self, name, networks, memory=DEFAULT_RAM_SIZE): node = self.add_node(name, memory) self.create_interfaces(node, networks) self.add_empty_volume(node, name + '-system') # self.add_empty_volume(node, name + '-cinder') #self.add_empty_volume(node, name + '-swift') return node def _node_roles(self): controllers = ['fuel-controller-%02d' % x for x in range(1, 1 + COUNT_NODES[DEPLOYMENT_MODE]['CONTROLLERS'])] computes = ['fuel-compute-%02d' % x for x in range(1, 1 + COUNT_NODES[DEPLOYMENT_MODE]['COMPUTES'])] storages = ['fuel-swift-%02d' % x for x in range(1, 1 + COUNT_NODES[DEPLOYMENT_MODE]['STORAGES'])] proxies = ['fuel-swift-proxy-%02d' % x for x in range(1, 1 + COUNT_NODES[DEPLOYMENT_MODE]['PROXIES'])] quantums = ['fuel-quantum-%02d' % x for x in range(1, 1 + COUNT_NODES[DEPLOYMENT_MODE]['QUANTUMS'])] return NodeRoles(admin_names=['master'], other_names=controllers + computes + storages + proxies + quantums ) def _create(self): self.environment = self.manager.environment_create(self.name) networks = [] for name in INTERFACE_ORDER: ip_networks = [IPNetwork(x) for x in POOLS.get(name)[0].split(',')] new_prefix = int(POOLS.get(name)[1]) pool = self.manager.create_network_pool(networks=ip_networks, prefix=int(new_prefix)) networks.append(self.manager.network_create(name=name, environment=self.environment, pool=pool, forward=FORWARDING.get(name), has_dhcp_server=DHCP.get(name))) for name in self._node_roles().admin_names: self.describe_admin_node(name, networks) for name in self._node_roles().other_names: self.describe_node(name, networks, memory=COMPUTE_RAM_SIZE) return self.environment def start(self): admin = self.nodes().admin admin.disk_devices.get(device='cdrom').volume.upload(ISO_PATH) self.environment.start([admin]) #self._environment.snapshot(EMPTY_SNAPSHOT) def start_all(self): self.environment.start(self.nodes()) def get_netmask_by_netname(self, netname): return str(IPNetwork(self.environment.network_by_name(netname).ip_network).netmask) def get_router_by_netname(self, netname): return str(IPNetwork(self.environment.network_by_name(netname).ip_network)[1]) def internal_virtual_ip(self): return self._get_virtual_ip_by_netname('internal') def public_virtual_ip(self): return self._get_virtual_ip_by_netname('public') def _get_virtual_ip_by_netname(self, netname): return str(IPNetwork(self.environment.network_by_name(netname).ip_network)[-2]) def erase(self): self.environment.erase() def get_master_ssh(self): """ :rtype : SSHClient """ return self.nodes().admin.remote('internal', login=self.login, password=self.password) def get_master_ip(self, net_name='internal'): return str(self.nodes().admin.get_ip_address_by_network_name(net_name)) def get_volume_capacity(self, node, volume_name='system'): for d in node.disk_devices: if volume_name in d.volume.get_path(): return d.volume.get_capacity() return None # def public_router(self): # return str(IPNetwork(self.get().network_by_name('public').ip_network)[1]) # # def internal_router(self): # return str(IPNetwork(self.get().network_by_name('internal').ip_network)[1]) # # def internal_network(self): # return str(IPNetwork(self.get().network_by_name('internal').ip_network)) # # def public_network(self): # return str(IPNetwork(self.get().network_by_name('public').ip_network)) # def internal_net_mask(self): return str(IPNetwork(self.environment.network_by_name('internal').ip_network).netmask)
class CiBase(object): def __init__(self): self.manager = Manager() self._environment = None def _get_or_create(self): try: return self.manager.environment_get(self.env_name()) except: self._environment = self.describe_environment() self._environment.define() return self._environment def get_empty_state(self): return self.get_state(EMPTY_SNAPSHOT) def get_state(self, name): if self.environment().has_snapshot(name): self.environment().revert(name) return True return False def environment(self): """ :rtype : devops.models.Environment """ self._environment = self._environment or self._get_or_create() return self._environment @abstractproperty def env_name(self): """ :rtype : string """ pass @abstractmethod def describe_environment(self): """ :rtype : devops.models.Environment """ pass @abstractproperty def node_roles(self): """ :rtype : NodeRoles """ pass def nodes(self): return Nodes(self.environment(), self.node_roles()) # noinspection PyShadowingBuiltins def add_empty_volume(self, node, name, capacity=20 * 1024 * 1024 * 1024, device='disk', bus='virtio', format='qcow2'): self.manager.node_attach_volume( node=node, volume=self.manager.volume_create( name=name, capacity=capacity, environment=self.environment(), format=format), device=device, bus=bus) def add_node(self, memory, name, boot=None): return self.manager.node_create( name=name, memory=memory, environment=self.environment(), boot=boot) def create_interfaces(self, networks, node): for network in networks: if network.name == 'internal': self.manager.interface_create(network, node=node) self.manager.interface_create(network, node=node) self.manager.interface_create(network, node=node) def describe_admin_node(self, name, networks, memory=1024): node = self.add_node(memory=memory, name=name, boot=['hd', 'cdrom']) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') self.add_empty_volume( node, name + '-iso', capacity=_get_file_size(ISO_PATH), format='raw', device='cdrom', bus='ide') return node def describe_empty_node(self, name, networks, memory=1024): node = self.add_node(memory, name) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') self.add_empty_volume(node, name + '-cinder') self.add_empty_volume(node, name + '-swift') return node @abstractmethod def setup_environment(self): """ :rtype : None """ pass def internal_virtual_ip(self): return str(IPNetwork( self.environment().network_by_name('internal').ip_network)[-2]) def public_router(self): return str( IPNetwork( self.environment().network_by_name('public').ip_network)[1]) def internal_router(self): return self._router('internal') def nat_router(self): return self._router('nat') def _router(self, router_name): return str( IPNetwork( self.environment().network_by_name(router_name).ip_network)[1]) def get_host_node_ip(self): return self.internal_router() def internal_network(self): return str( IPNetwork( self.environment().network_by_name('internal').ip_network)) def internal_net_mask(self): return str(IPNetwork( self.environment().network_by_name('internal').ip_network).netmask) def public_net_mask(self): return str(IPNetwork( self.environment().network_by_name('public').ip_network).netmask) def public_network(self): return str( IPNetwork(self.environment().network_by_name('public').ip_network))
class Shell(object): def __init__(self): super(Shell, self).__init__() self.params = self.get_params() self.manager = Manager() def execute(self): self.commands.get(self.params.command)(self) def do_list(self): env_list = self.manager.environment_list().values('name') for env in env_list: print env['name'] return env_list def node_dict(self, node): return {'name': node.name, 'vnc': node.get_vnc_port()} def do_show(self): environment = self.manager.environment_get(self.params.name) print '%5s %25s' % ("VNC", "NODE-NAME") for item in map(lambda x: self.node_dict(x), environment.nodes): print '%5s %25s' % (item['vnc'], item['name']) def do_erase(self): self.manager.environment_get(self.params.name).erase() def do_start(self): self.manager.environment_get(self.params.name).start() def do_destroy(self): self.manager.environment_get(self.params.name).destroy(verbose=False) def do_suspend(self): self.manager.environment_get(self.params.name).suspend(verbose=False) def do_resume(self): self.manager.environment_get(self.params.name).resume(verbose=False) def do_revert(self): self.manager.environment_get(self.params.name).revert( self.params.snapshot_name) def do_snapshot(self): self.manager.environment_get(self.params.name).snapshot( self.params.snapshot_name) def do_synchronize(self): self.manager.synchronize_environments() commands = { 'list': do_list, 'show': do_show, 'erase': do_erase, 'start': do_start, 'destroy': do_destroy, 'suspend': do_suspend, 'resume': do_resume, 'revert': do_revert, 'snapshot': do_snapshot, 'sync': do_synchronize } def get_params(self): name_parser = argparse.ArgumentParser(add_help=False) name_parser.add_argument('name', help='environment name', default=environ.get('ENV_NAME')) snapshot_name_parser = argparse.ArgumentParser(add_help=False) snapshot_name_parser.add_argument('--snapshot-name', help='snapshot name', default=environ.get('SNAPSHOT_NAME')) parser = argparse.ArgumentParser( description="Manage virtual environments") subparsers = parser.add_subparsers(help='commands', dest='command') subparsers.add_parser('list') subparsers.add_parser('show', parents=[name_parser]) subparsers.add_parser('erase', parents=[name_parser]) subparsers.add_parser('start', parents=[name_parser]) subparsers.add_parser('destroy', parents=[name_parser]) subparsers.add_parser('suspend', parents=[name_parser]) subparsers.add_parser('resume', parents=[name_parser]) subparsers.add_parser('revert', parents=[name_parser, snapshot_name_parser]) subparsers.add_parser('snapshot', parents=[name_parser, snapshot_name_parser]) subparsers.add_parser('sync') return parser.parse_args()
class CiBase(object): def __init__(self): self._environment = None self.manager = Manager() # self.base_image = self.manager.volume_get_predefined(BASE_IMAGE) def get_or_create(self): try: return self.manager.environment_get(self.env_name()) except: self._environment = self.describe_environment() self._environment.define() return self._environment def get_empty_state(self): if self.environment().has_snapshot(EMPTY_SNAPSHOT): self.environment().revert(EMPTY_SNAPSHOT) else: self.setup_environment() def environment(self): """ :rtype : devops.models.Environment """ self._environment = self._environment or self.get_or_create() return self._environment @abstractproperty def env_name(self): """ :rtype : string """ pass @abstractmethod def define(self): """ :rtype : devops.models.Environment """ pass @abstractmethod def describe_environment(self): """ :rtype : devops.models.Environment """ pass @abstractproperty def node_roles(self): """ :rtype : NodeRoles """ pass def add_empty_volume(self, node, name, capacity=20 * 1024 * 1024 * 1024, format="qcow2", device="disk", bus='virtio'): self.manager.node_attach_volume( node=node, device=device, bus=bus, volume=self.manager.volume_create( name=name, capacity=capacity, format=format, environment=self.environment())) def add_node(self, memory, name, boot=None): return self.manager.node_create( name=name, memory=memory, environment=self.environment()) def describe_master_node(self, name, networks, memory=1024): node = self.add_node(memory, name, boot=['cdrom', 'hd']) for network in networks: self.manager.interface_create(network, node=node) self.add_empty_volume(node, name + '-system') self.add_empty_volume(node, name + '-iso', capacity=_get_file_size(ISO_IMAGE), format='raw', device='cdrom', bus='ide') return node def describe_empty_node(self, name, networks, memory=1024): node = self.add_node(memory, name) for network in networks: self.manager.interface_create(network, node=node) self.add_empty_volume(node, name + '-system') self.add_empty_volume(node, name + '-cinder') return node def nodes(self): return Nodes(self.environment(), self.node_roles()) def add_nodes_to_hosts(self, remote, nodes): for node in nodes: add_to_hosts(remote, node.get_ip_address_by_network_name('internal'), node.name, node.name + '.localdomain') def setup_master_node(self, master_remote, nodes): setup_puppet_master(master_remote) add_nmap(master_remote) switch_off_ip_tables(master_remote) self.add_nodes_to_hosts(master_remote, nodes) def setup_agent_nodes(self, nodes): agent_config = load( root('fuel_test', 'config', 'puppet.agent.config')) for node in nodes: if node.name != 'master': remote = node.remote('public', login='******', password='******') self.add_nodes_to_hosts(remote, self.environment().nodes) setup_puppet_client(remote) write_config(remote, '/etc/puppet/puppet.conf', agent_config) request_cerificate(remote) def rename_nodes(self, nodes): for node in nodes: remote = node.remote('public', login='******', password='******') change_host_name(remote, node.name, node.name + '.localdomain') logging.info("Renamed %s" % node.name) @abstractmethod def setup_environment(self): """ :rtype : None """ pass def internal_virtual_ip(self): return str(IPNetwork( self.environment().network_by_name('internal').ip_network)[-2]) def floating_network(self): prefix = IPNetwork(self.environment().network_by_name('public').ip_network).prefixlen return str( IPNetwork(self.environment().network_by_name('public').ip_network).subnet(new_prefix=prefix + 2)[-1]) def public_virtual_ip(self): prefix = IPNetwork(self.environment().network_by_name('public').ip_network).prefixlen return str( IPNetwork(self.environment().network_by_name('public').ip_network).subnet(new_prefix=prefix + 2)[-2][ -1]) def public_router(self): return str( IPNetwork( self.environment().network_by_name('public').ip_network)[1]) def internal_router(self): return str( IPNetwork( self.environment().network_by_name('internal').ip_network)[1]) def fixed_network(self): return str( IPNetwork(self.environment().network_by_name('private').ip_network).subnet( new_prefix=27)[0]) def internal_network(self): return str(IPNetwork(self.environment().network_by_name('internal').ip_network)) def internal_net_mask(self): return str(IPNetwork(self.environment().network_by_name('internal').ip_network).netmask) def public_net_mask(self): return str(IPNetwork(self.environment().network_by_name('public').ip_network).netmask) def public_network(self): return str(IPNetwork(self.environment().network_by_name('public').ip_network))
class Shell(object): def __init__(self): super(Shell, self).__init__() self.params = self.get_params() self.manager = Manager() def execute(self): self.commands.get(self.params.command)(self) def do_list(self): print self.manager.environment_list().values('name') def node_dict(self, node): return {'name': node.name, 'vnc': node.get_vnc_port(), } def do_show(self): environment = self.manager.environment_get(self.params.name) print { 'name': environment.name, 'nodes': map(lambda x: {'node': self.node_dict(x)}, environment.nodes) } def do_erase(self): self.manager.environment_get(self.params.name).erase() def do_start(self): self.manager.environment_get(self.params.name).start() def do_destroy(self): self.manager.environment_get(self.params.name).destroy(verbose=False) def do_suspend(self): self.manager.environment_get(self.params.name).suspend() def do_resume(self): self.manager.environment_get(self.params.name).resume() def do_revert(self): self.manager.environment_get(self.params.name).revert( self.params.snapshot_name) def do_snapshot(self): self.manager.environment_get(self.params.name).snapshot( self.params.snapshot_name) commands = { 'list': do_list, 'show': do_show, 'erase': do_erase, 'start': do_start, 'destroy': do_destroy, 'suspend': do_suspend, 'resume': do_resume, 'revert': do_revert, 'snapshot': do_snapshot } def get_params(self): name_parser = argparse.ArgumentParser(add_help=False) name_parser.add_argument('name', help='environment name', default=os.getenv('ENV_NAME')) snapshot_name_parser = argparse.ArgumentParser(add_help=False) snapshot_name_parser.add_argument('--snapshot-name', help='snapshot name', default=os.getenv('SNAPSHOT_NAME')) parser = argparse.ArgumentParser( description="Manage virtual environments") subparsers = parser.add_subparsers(help='commands', dest='command') subparsers.add_parser('list') subparsers.add_parser('show', parents=[name_parser]) subparsers.add_parser('erase', parents=[name_parser]) subparsers.add_parser('start', parents=[name_parser]) subparsers.add_parser('destroy', parents=[name_parser]) subparsers.add_parser('suspend', parents=[name_parser]) subparsers.add_parser('resume', parents=[name_parser]) subparsers.add_parser('revert', parents=[name_parser, snapshot_name_parser]) subparsers.add_parser('snapshot', parents=[name_parser, snapshot_name_parser]) return parser.parse_args()
class EnvironmentModel(object): hostname = 'nailgun' domain = 'test.domain.local' installation_timeout = 1800 deployment_timeout = 1800 puppet_timeout = 1000 nat_interface = '' # INTERFACES.get('admin') admin_net = 'admin' def __init__(self, os_image=None): self._virtual_environment = None self._keys = None self.manager = Manager() self.os_image = os_image self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self) def _get_or_create(self): try: return self.manager.environment_get(self.env_name) except Exception: self._virtual_environment = self.describe_environment() self._virtual_environment.define() return self._virtual_environment def router(self, router_name=None): router_name = router_name or self.admin_net return str( IPNetwork( self.get_virtual_environment().network_by_name(router_name). ip_network)[1]) @property def fuel_web(self): """FuelWebClient :rtype: FuelWebClient """ return self._fuel_web @property def node_roles(self): return NodeRoles( admin_names=['admin'], other_names=['slave-%02d' % x for x in range(1, int( settings.NODES_COUNT))] ) @property def env_name(self): return settings.ENV_NAME def add_empty_volume(self, node, name, capacity=settings.NODE_VOLUME_SIZE * 1024 * 1024 * 1024, device='disk', bus='virtio', format='qcow2'): self.manager.node_attach_volume( node=node, volume=self.manager.volume_create( name=name, capacity=capacity, environment=self.get_virtual_environment(), format=format), device=device, bus=bus) def add_node(self, memory, name, vcpu=1, boot=None): return self.manager.node_create( name=name, memory=memory, vcpu=vcpu, environment=self.get_virtual_environment(), boot=boot) @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server( cluster_id, self.get_host_node_ip(), port) def bootstrap_nodes(self, devops_nodes, timeout=600): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ for node in devops_nodes: node.start() wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) for node in self.nailgun_nodes(devops_nodes): self.sync_node_time(self.get_ssh_to_remote(node["ip"])) return self.nailgun_nodes(devops_nodes) def create_interfaces(self, networks, node, model=settings.INTERFACE_MODEL): for network in networks: self.manager.interface_create(network, node=node, model=model) def describe_environment(self): """Environment :rtype : Environment """ environment = self.manager.environment_create(self.env_name) networks = [] for name in settings.INTERFACE_ORDER: ip_networks = [ IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',')] new_prefix = int(settings.POOLS.get(name)[1]) pool = self.manager.create_network_pool(networks=ip_networks, prefix=int(new_prefix)) networks.append(self.manager.network_create( name=name, environment=environment, pool=pool, forward=settings.FORWARDING.get(name), has_dhcp_server=settings.DHCP.get(name))) for name in self.node_roles.admin_names: self.describe_admin_node(name, networks) for name in self.node_roles.other_names: self.describe_empty_node(name, networks) return environment def devops_nodes_by_names(self, devops_node_names): return map( lambda name: self.get_virtual_environment().node_by_name(name), devops_node_names) @logwrap def describe_admin_node(self, name, networks): node = self.add_node( memory=settings.HARDWARE.get("admin_node_memory", 1024), vcpu=settings.HARDWARE.get("admin_node_cpu", 1), name=name, boot=['hd', 'cdrom']) self.create_interfaces(networks, node) if self.os_image is None: self.add_empty_volume(node, name + '-system') self.add_empty_volume( node, name + '-iso', capacity=_get_file_size(settings.ISO_PATH), format='raw', device='cdrom', bus='ide') else: volume = self.manager.volume_get_predefined(self.os_image) vol_child = self.manager.volume_create_child( name=name + '-system', backing_store=volume, environment=self.get_virtual_environment() ) self.manager.node_attach_volume( node=node, volume=vol_child ) return node def describe_empty_node(self, name, networks): node = self.add_node( name=name, memory=settings.HARDWARE.get("slave_node_memory", 1024), vcpu=settings.HARDWARE.get("slave_node_cpu", 1)) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') if settings.USE_ALL_DISKS: self.add_empty_volume(node, name + '-cinder') self.add_empty_volume(node, name + '-swift') return node @logwrap def get_admin_remote(self): """SSH to admin node :rtype : SSHClient """ return self.nodes().admin.remote(self.admin_net, login='******', password='******') @logwrap def get_admin_node_ip(self): return str( self.nodes().admin.get_ip_address_by_network_name(self.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_host_node_ip(self): return self.router() def get_keys(self, node): params = { 'ip': node.get_ip_address_by_network_name(self.admin_net), 'mask': self.get_net_mask(self.admin_net), 'gw': self.router(), 'hostname': '.'.join((self.hostname, self.domain)), 'nat_interface': self.nat_interface, 'dns1': settings.DNS } keys = ( "<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " <Enter>\n" ) % params return keys @logwrap def get_private_keys(self, force=False): if force or self._keys is None: self._keys = [] for key_string in ['/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa']: with self.get_admin_remote().open(key_string) as f: self._keys.append(RSAKey.from_private_key(f)) return self._keys @logwrap def get_ssh_to_remote(self, ip): return SSHClient(ip, username='******', password='******', private_keys=self.get_private_keys()) @logwrap def get_ssh_to_remote_by_name(self, node_name): return self.get_ssh_to_remote( self.fuel_web.get_nailgun_node_by_devops_node( self.get_virtual_environment().node_by_name(node_name))['ip'] ) def get_target_devs(self, devops_nodes): return [ interface.target_dev for interface in [ val for var in map(lambda node: node.interfaces, devops_nodes) for val in var]] def get_virtual_environment(self): """Returns virtual environment :rtype : devops.models.Environment """ if self._virtual_environment is None: self._virtual_environment = self._get_or_create() return self._virtual_environment def get_network(self, net_name): return str( IPNetwork( self.get_virtual_environment().network_by_name(net_name). ip_network)) def get_net_mask(self, net_name): return str( IPNetwork( self.get_virtual_environment().network_by_name( net_name).ip_network).netmask) def make_snapshot(self, snapshot_name, description=""): self.get_virtual_environment().suspend(verbose=False) self.get_virtual_environment().snapshot(snapshot_name, force=True) revert_info(snapshot_name, description) def nailgun_nodes(self, devops_nodes): return map( lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node), devops_nodes ) def nodes(self): return Nodes(self.get_virtual_environment(), self.node_roles) def revert_snapshot(self, name): if self.get_virtual_environment().has_snapshot(name): logger.info('We have snapshot with such name %s' % name) self.get_virtual_environment().revert(name) logger.info('Starting snapshot reverting ....') self.get_virtual_environment().resume() logger.info('Starting snapshot resuming ...') self.nodes().admin.await(self.admin_net, timeout=10 * 60) list = self.nodes().slaves for node in list: nailgun_node = self.fuel_web.get_nailgun_node_by_devops_node( node) if not node.driver.node_active(node) or \ nailgun_node['status'] not in ['provisioned', 'ready']: continue try: self.sync_node_time(self.get_ssh_to_remote( node.get_ip_address_by_network_name(self.admin_net))) except Exception as e: logger.warn( 'Paramiko exception catched while' ' trying to run ntpdate: %s' % e) return True return False def setup_environment(self): # start admin node admin = self.nodes().admin admin.disk_devices.get(device='cdrom').volume.upload(settings.ISO_PATH) self.get_virtual_environment().start(self.nodes().admins) # update network parameters at boot screen time.sleep(float(settings.ADMIN_NODE_SETUP_TIMEOUT)) admin.send_keys(self.get_keys(admin)) # wait while installation complete admin.await(self.admin_net, timeout=10 * 60) self.wait_bootstrap() time.sleep(10) self.sync_time_admin_node() @logwrap def sync_node_time(self, remote): remote.execute('hwclock --hctosys') remote.execute("ntpdate -u " "$(egrep '^server' /etc/ntp.conf |" " sed '/^#/d' | awk '{print $2}' |" " tail -n 1)") remote.execute('hwclock -w') def sync_time_admin_node(self): self.sync_node_time(self.get_admin_remote()) def verify_node_service_list(self, node_name, smiles_count): remote = self.get_ssh_to_remote_by_name(node_name) checkers.verify_service_list(remote, smiles_count) def verify_network_configuration(self, node_name): checkers.verify_network_configuration( node=self.fuel_web.get_nailgun_node_by_name(node_name), remote=self.get_ssh_to_remote_by_name(node_name) ) def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" wait( lambda: not self.get_admin_remote().execute( "grep 'Finished catalog run' '%s'" % log_path )['exit_code'], timeout=self.puppet_timeout )
class CiBase(object): def __init__(self): self.manager = Manager() self._environment = None self.saved_environment_states = {} def _get_or_create(self): try: return self.manager.environment_get(self.env_name()) except: self._environment = self.describe_environment() self._environment.define() return self._environment def get_state(self, name): if self.environment().has_snapshot(name): self.environment().revert(name) return True return False def environment(self): """ :rtype : devops.models.Environment """ self._environment = self._environment or self._get_or_create() return self._environment @abstractproperty def env_name(self): """ :rtype : string """ pass @abstractmethod def describe_environment(self): """ :rtype : devops.models.Environment """ pass @abstractproperty def node_roles(self): """ :rtype : NodeRoles """ pass def nodes(self): return Nodes(self.environment(), self.node_roles()) # noinspection PyShadowingBuiltins def add_empty_volume(self, node, name, capacity=20 * 1024 * 1024 * 1024, device='disk', bus='virtio', format='qcow2'): self.manager.node_attach_volume( node=node, volume=self.manager.volume_create( name=name, capacity=capacity, environment=self.environment(), format=format), device=device, bus=bus) def add_node(self, memory, name, boot=None): return self.manager.node_create( name=name, memory=memory, environment=self.environment(), boot=boot) def create_interfaces(self, networks, node): for network in networks: if network.name == 'internal': self.manager.interface_create(network, node=node) self.manager.interface_create(network, node=node) self.manager.interface_create(network, node=node) def describe_admin_node(self, name, networks, memory=1024): node = self.add_node(memory=memory, name=name, boot=['hd', 'cdrom']) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') self.add_empty_volume( node, name + '-iso', capacity=_get_file_size(ISO_PATH), format='raw', device='cdrom', bus='ide') return node def describe_empty_node(self, name, networks, memory=1024): node = self.add_node(memory, name) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') self.add_empty_volume(node, name + '-cinder') self.add_empty_volume(node, name + '-swift') return node @abstractmethod def setup_environment(self): """ :rtype : None """ pass def get_empty_environment(self): if not(self.get_state(EMPTY_SNAPSHOT)): self.setup_environment() self.environment().snapshot(EMPTY_SNAPSHOT) def generate_state_hash(self, settings): return hashlib.md5(str(settings)).hexdigest() def revert_to_state(self, settings): state_hash = self.generate_state_hash(settings) empty_state_hash = self.generate_state_hash({}) if state_hash == empty_state_hash: # revert to empty state self.get_empty_environment() return True if state_hash in self.saved_environment_states: # revert to matching state state = self.saved_environment_states[state_hash] if not(self.get_state(state['snapshot_name'])): return False self.environment().resume() return True return False def snapshot_state(self, name, settings): state_hash = self.generate_state_hash(settings) snapshot_name = '{0}_{1}'.format( name.replace(' ', '_')[:17], state_hash) self.environment().suspend(verbose=False) self.environment().snapshot( name=snapshot_name, description=name, force=True, ) self.environment().resume(verbose=False) self.saved_environment_states[state_hash] = { 'snapshot_name': snapshot_name, 'cluster_name': name, 'settings': settings } def internal_virtual_ip(self): return str(IPNetwork( self.environment().network_by_name('internal').ip_network)[-2]) def public_router(self): return str( IPNetwork( self.environment().network_by_name('public').ip_network)[1]) def internal_router(self): return self._router('internal') def nat_router(self): return self._router('nat') def _router(self, router_name): return str( IPNetwork( self.environment().network_by_name(router_name).ip_network)[1]) def get_host_node_ip(self): return self.internal_router() def internal_network(self): return str( IPNetwork( self.environment().network_by_name('internal').ip_network)) def internal_net_mask(self): return str(IPNetwork( self.environment().network_by_name('internal').ip_network).netmask) def public_net_mask(self): return str(IPNetwork( self.environment().network_by_name('public').ip_network).netmask) def public_network(self): return str( IPNetwork(self.environment().network_by_name('public').ip_network))
class CiBase(object): """ Base class for creating environment -- deploy openstack in various mode. """ def __init__(self): """ Constructor. Define environment and manager for manipulation with it. """ self._environment = None self.manager = Manager() def get_or_create(self): """ Get prepared environment or create it. """ try: return self.manager.environment_get(self.env_name()) except: self._environment = self.describe_environment() self._environment.define() return self._environment def get_empty_state(self): """ Get 'empty' snapshot for virtual machines. """ if self.environment().has_snapshot(EMPTY_SNAPSHOT): self.environment().revert(EMPTY_SNAPSHOT) else: self.setup_environment() def environment(self): """ :rtype : devops.models.Environment """ self._environment = self._environment or self.get_or_create() return self._environment @abstractproperty def env_name(self): """ :rtype : string """ pass @abstractmethod def define(self): """ :rtype : devops.models.Environment """ pass @abstractmethod def describe_environment(self): """ :rtype : devops.models.Environment """ pass @abstractproperty def node_roles(self): """ :rtype : NodeRoles """ pass def add_empty_volume(self, node, name, capacity=20 * 1024 * 1024 * 1024, format="qcow2", device="disk", bus='virtio'): """ Attach a empty volume to virtual machine. """ self.manager.node_attach_volume(node=node, device=device, bus=bus, volume=self.manager.volume_create( name=name, capacity=capacity, format=format, environment=self.environment())) def add_node(self, memory, name, boot=None): """ Create a virtual machine in environment. """ return self.manager.node_create(name=name, memory=memory, environment=self.environment()) def describe_master_node(self, name, networks, memory=DEFAULT_RAM_SIZE): """ Define master node. """ node = self.add_node(memory, name, boot=['cdrom', 'hd']) for network in networks: self.manager.interface_create(network, node=node) self.add_empty_volume(node, name + '-system') self.add_empty_volume(node, name + '-iso', capacity=_get_file_size(ISO_IMAGE), format='raw', device='cdrom', bus='ide') return node def describe_empty_node(self, name, networks, memory=DEFAULT_RAM_SIZE): """ Define node with default settings. """ node = self.add_node(memory, name) for network in networks: self.manager.interface_create(network, node=node) self.add_empty_volume(node, name + '-system') self.add_empty_volume(node, name + '-cinder') return node def nodes(self): """ Get all virtual machines of environment. """ return Nodes(self.environment(), self.node_roles()) def add_nodes_to_hosts(self, remote, nodes): """ Append to /etc/hosts list of ip addresses by internal interface for all vms. """ for node in nodes: add_to_hosts(remote, node.get_ip_address_by_network_name('internal'), node.name, node.name + '.localdomain') def setup_master_node(self, master_remote, nodes): """ Make the basic settings for master vm(node). """ setup_puppet_master(master_remote) add_nmap(master_remote) switch_off_ip_tables(master_remote) self.add_nodes_to_hosts(master_remote, nodes) def setup_agent_nodes(self, nodes): """ Make the basic settings for default vm(node). """ agent_config = load(root('fuel-test', 'config', 'puppet.agent.config')) for node in nodes: if node.name != 'master': remote = node.remote('public', login='******', password='******') self.add_nodes_to_hosts(remote, self.environment().nodes) setup_puppet_client(remote) write_config(remote, '/etc/puppet/puppet.conf', agent_config) request_cerificate(remote) def rename_nodes(self, nodes): """ Change host name for nodes. """ for node in nodes: remote = node.remote('public', login='******', password='******') change_host_name(remote, node.name, node.name + '.localdomain') LOG.info("Renamed %s" % node.name) @abstractmethod def setup_environment(self): """ :rtype : None """ pass def internal_virtual_ip(self): """ Get virtual ip in internal network. """ return str(IPNetwork(self.environment().network_by_name('internal').ip_network)[-2]) def floating_network(self): """ Get floating subnet of public network. """ prefix = IPNetwork(self.environment().network_by_name('public').ip_network).prefixlen return str(IPNetwork(self.environment().network_by_name('public').ip_network).subnet(new_prefix=prefix + 2)[-1]) def public_virtual_ip(self): """ Get virtual ip in public network. """ prefix = IPNetwork(self.environment().network_by_name('public').ip_network).prefixlen return str(IPNetwork(self.environment().network_by_name('public').ip_network).subnet(new_prefix=prefix + 2)[-2][-1]) def public_router(self): """ Get route for public network. """ return str(IPNetwork(self.environment().network_by_name('public').ip_network)[1]) def internal_router(self): """ Get route for internal network. """ return str(IPNetwork(self.environment().network_by_name('internal').ip_network)[1]) def fixed_network(self): """ Get fixed subnet of private network. """ return str(IPNetwork(self.environment().network_by_name('private').ip_network).subnet( new_prefix=27)[0]) def internal_network(self): """ Get internal network. """ return str(IPNetwork(self.environment().network_by_name('internal').ip_network)) def internal_net_mask(self): """ Get internal netmask. """ return str(IPNetwork(self.environment().network_by_name('internal').ip_network).netmask) def public_net_mask(self): """ Get public netmask. """ return str(IPNetwork(self.environment().network_by_name('public').ip_network).netmask) def public_network(self): """ Get public network. """ return str(IPNetwork(self.environment().network_by_name('public').ip_network))
class CiBase(object): def __init__(self): self._environment = None self.manager = Manager() # self.base_image = self.manager.volume_get_predefined(BASE_IMAGE) def get_or_create(self): try: return self.manager.environment_get(self.env_name()) except: self._environment = self.describe_environment() self._environment.define() return self._environment def get_empty_state(self): if self.environment().has_snapshot(EMPTY_SNAPSHOT): self.environment().revert(EMPTY_SNAPSHOT) else: self.setup_environment() def environment(self): """ :rtype : devops.models.Environment """ self._environment = self._environment or self.get_or_create() return self._environment @abstractproperty def env_name(self): """ :rtype : string """ pass @abstractmethod def define(self): """ :rtype : devops.models.Environment """ pass @abstractmethod def describe_environment(self): """ :rtype : devops.models.Environment """ pass @abstractproperty def node_roles(self): """ :rtype : NodeRoles """ pass def add_empty_volume(self, node, name, capacity=20 * 1024 * 1024 * 1024, format="qcow2", device="disk", bus='virtio'): self.manager.node_attach_volume(node=node, device=device, bus=bus, volume=self.manager.volume_create( name=name, capacity=capacity, format=format, environment=self.environment())) def add_node(self, memory, name, boot=None): return self.manager.node_create(name=name, memory=memory, environment=self.environment()) def describe_master_node(self, name, networks, memory=1024): node = self.add_node(memory, name, boot=['cdrom', 'hd']) for network in networks: self.manager.interface_create(network, node=node) self.add_empty_volume(node, name + '-system') self.add_empty_volume(node, name + '-iso', capacity=_get_file_size(ISO_IMAGE), format='raw', device='cdrom', bus='ide') return node def describe_empty_node(self, name, networks, memory=1024): node = self.add_node(memory, name) for network in networks: self.manager.interface_create(network, node=node) self.add_empty_volume(node, name + '-system') self.add_empty_volume(node, name + '-cinder') return node def nodes(self): return Nodes(self.environment(), self.node_roles()) def add_nodes_to_hosts(self, remote, nodes): for node in nodes: add_to_hosts(remote, node.get_ip_address_by_network_name('internal'), node.name, node.name + '.localdomain') def setup_master_node(self, master_remote, nodes): setup_puppet_master(master_remote) add_nmap(master_remote) switch_off_ip_tables(master_remote) self.add_nodes_to_hosts(master_remote, nodes) def setup_agent_nodes(self, nodes): agent_config = load(root('fuel_test', 'config', 'puppet.agent.config')) for node in nodes: if node.name != 'master': remote = node.remote('public', login='******', password='******') self.add_nodes_to_hosts(remote, self.environment().nodes) setup_puppet_client(remote) write_config(remote, '/etc/puppet/puppet.conf', agent_config) request_cerificate(remote) def rename_nodes(self, nodes): for node in nodes: remote = node.remote('public', login='******', password='******') change_host_name(remote, node.name, node.name + '.localdomain') logging.info("Renamed %s" % node.name) @abstractmethod def setup_environment(self): """ :rtype : None """ pass def internal_virtual_ip(self): return str( IPNetwork( self.environment().network_by_name('internal').ip_network)[-2]) def floating_network(self): return str( IPNetwork(self.environment().network_by_name( 'public').ip_network).subnet(new_prefix=29)[-1]) def public_virtual_ip(self): return str( IPNetwork(self.environment().network_by_name( 'public').ip_network).subnet(new_prefix=29)[-2][-1]) def public_router(self): return str( IPNetwork( self.environment().network_by_name('public').ip_network)[1]) def internal_router(self): return str( IPNetwork( self.environment().network_by_name('internal').ip_network)[1]) def fixed_network(self): return str( IPNetwork(self.environment().network_by_name( 'private').ip_network).subnet(new_prefix=27)[0]) def internal_network(self): return str( IPNetwork( self.environment().network_by_name('internal').ip_network)) def internal_net_mask(self): return str( IPNetwork(self.environment().network_by_name( 'internal').ip_network).netmask) def public_net_mask(self): return str( IPNetwork(self.environment().network_by_name( 'public').ip_network).netmask) def public_network(self): return str( IPNetwork(self.environment().network_by_name('public').ip_network))
class Shell(object): def __init__(self): super(Shell, self).__init__() self.params = self.get_params() self.manager = Manager() def execute(self): self.commands.get(self.params.command)(self) def do_list(self): print self.manager.environment_list().values('name') def node_dict(self, node): return { 'name': node.name, 'vnc': node.get_vnc_port(), } def do_show(self): environment = self.manager.environment_get(self.params.name) print { 'name': environment.name, 'nodes': map(lambda x: {'node': self.node_dict(x)}, environment.nodes) } def do_erase(self): self.manager.environment_get(self.params.name).erase() def do_start(self): self.manager.environment_get(self.params.name).start() def do_destroy(self): self.manager.environment_get(self.params.name).destroy(verbose=False) def do_suspend(self): self.manager.environment_get(self.params.name).suspend() def do_resume(self): self.manager.environment_get(self.params.name).resume() def do_revert(self): self.manager.environment_get(self.params.name).revert( self.params.snapshot_name) def do_snapshot(self): self.manager.environment_get(self.params.name).snapshot( self.params.snapshot_name) commands = { 'list': do_list, 'show': do_show, 'erase': do_erase, 'start': do_start, 'destroy': do_destroy, 'suspend': do_suspend, 'resume': do_resume, 'revert': do_revert, 'snapshot': do_snapshot } def get_params(self): name_parser = argparse.ArgumentParser(add_help=False) name_parser.add_argument('name', help='environment name', default=os.getenv('ENV_NAME')) snapshot_name_parser = argparse.ArgumentParser(add_help=False) snapshot_name_parser.add_argument('--snapshot-name', help='snapshot name', default=os.getenv('SNAPSHOT_NAME')) parser = argparse.ArgumentParser( description="Manage virtual environments") subparsers = parser.add_subparsers(help='commands', dest='command') subparsers.add_parser('list') subparsers.add_parser('show', parents=[name_parser]) subparsers.add_parser('erase', parents=[name_parser]) subparsers.add_parser('start', parents=[name_parser]) subparsers.add_parser('destroy', parents=[name_parser]) subparsers.add_parser('suspend', parents=[name_parser]) subparsers.add_parser('resume', parents=[name_parser]) subparsers.add_parser('revert', parents=[name_parser, snapshot_name_parser]) subparsers.add_parser('snapshot', parents=[name_parser, snapshot_name_parser]) return parser.parse_args()
class CiBase(object): def __init__(self): self.manager = Manager() self.base_image = self.manager.volume_get_predefined(BASE_IMAGE) self._environment = None def _get_or_create(self): try: return self.manager.environment_get(self.env_name()) except: self._environment = self.describe_environment() self._environment.define() return self._environment def get_empty_state(self): if self.environment().has_snapshot(EMPTY_SNAPSHOT): self.environment().revert(EMPTY_SNAPSHOT) else: self.setup_environment() def environment(self): """ :rtype : devops.models.Environment """ self._environment = self._environment or self._get_or_create() return self._environment @abstractproperty def env_name(self): """ :rtype : string """ pass @abstractmethod def describe_environment(self): """ :rtype : devops.models.Environment """ pass @abstractproperty def node_roles(self): """ :rtype : NodeRoles """ pass def nodes(self): return Nodes(self.environment(), self.node_roles()) def add_empty_volume(self, node, name): self.manager.node_attach_volume( node=node, volume=self.manager.volume_create( name=name, capacity=20 * 1024 * 1024 * 1024, environment=self.environment() ), ) def add_node(self, memory, name): return self.manager.node_create(name=name, memory=memory, environment=self.environment()) def describe_node(self, name, networks, memory=1024): node = self.add_node(memory, name) for network in networks: self.manager.interface_create(network, node=node) self.manager.node_attach_volume( node=node, volume=self.manager.volume_create_child( name=name + "-system", backing_store=self.base_image, environment=self.environment() ), ) self.add_empty_volume(node, name + "-cinder") return node def describe_empty_node(self, name, networks, memory=1024): node = self.add_node(memory, name) for network in networks: self.manager.interface_create(network, node=node) self.add_empty_volume(node, name + "-system") self.add_empty_volume(node, name + "-cinder") return node def add_nodes_to_hosts(self, remote, nodes): for node in nodes: add_to_hosts( remote, node.get_ip_address_by_network_name("internal"), node.name, node.name + ".your-domain-name.com" ) def setup_master_node(self, master_remote, nodes): setup_puppet_master(master_remote) add_nmap(master_remote) switch_off_ip_tables(master_remote) self.add_nodes_to_hosts(master_remote, nodes) def setup_agent_nodes(self, nodes): agent_config = load(root("fuel_test", "config", "puppet.agent.config")) for node in nodes: if node.name != "master": remote = node.remote("public", login="******", password="******") self.add_nodes_to_hosts(remote, self.environment().nodes) setup_puppet_client(remote) write_config(remote, "/etc/puppet/puppet.conf", agent_config) request_cerificate(remote) def rename_nodes(self, nodes): for node in nodes: remote = node.remote("public", login="******", password="******") change_host_name(remote, node.name, node.name + ".your-domain-name.com") logging.info("Renamed %s" % node.name) @abstractmethod def setup_environment(self): """ :rtype : None """ pass def internal_virtual_ip(self): return str(IPNetwork(self.environment().network_by_name("internal").ip_network)[-2]) def floating_network(self): return str(IPNetwork(self.environment().network_by_name("public").ip_network).subnet(new_prefix=29)[-1]) def public_virtual_ip(self): return str(IPNetwork(self.environment().network_by_name("public").ip_network).subnet(new_prefix=29)[-2][-1]) def public_router(self): return str(IPNetwork(self.environment().network_by_name("public").ip_network)[1]) def internal_router(self): return str(IPNetwork(self.environment().network_by_name("internal").ip_network)[1]) def fixed_network(self): return str(IPNetwork(self.environment().network_by_name("private").ip_network).subnet(new_prefix=27)[0]) def internal_network(self): return str(IPNetwork(self.environment().network_by_name("internal").ip_network)) def internal_net_mask(self): return str(IPNetwork(self.environment().network_by_name("internal").ip_network).netmask) def public_net_mask(self): return str(IPNetwork(self.environment().network_by_name("public").ip_network).netmask) def public_network(self): return str(IPNetwork(self.environment().network_by_name("public").ip_network))