class EnvironmentModel(object): hostname = 'nailgun' domain = 'test.domain.local' installation_timeout = 1800 deployment_timeout = 1800 puppet_timeout = 2000 nat_interface = '' # INTERFACES.get('admin') admin_net = 'admin' admin_net2 = 'admin2' multiple_cluster_networks = settings.MULTIPLE_NETWORKS def __init__(self, os_image=None): self._virtual_environment = None self._keys = None self.manager = Manager() self.os_image = os_image self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self) @property def nailgun_actions(self): return FuelActions.Nailgun(self.get_admin_remote()) @property def postgres_actions(self): return FuelActions.Postgres(self.get_admin_remote()) def _get_or_create(self): try: return self.manager.environment_get(self.env_name) except Exception: self._virtual_environment = self.describe_environment() self._virtual_environment.define() return self._virtual_environment def router(self, router_name=None): router_name = router_name or self.admin_net if router_name == self.admin_net2: return str(IPNetwork(self.get_virtual_environment(). network_by_name(router_name).ip_network)[2]) return str( IPNetwork( self.get_virtual_environment().network_by_name(router_name). ip_network)[1]) @property def fuel_web(self): """FuelWebClient :rtype: FuelWebClient """ return self._fuel_web @property def admin_node_ip(self): return self.fuel_web.admin_node_ip @property def node_roles(self): return NodeRoles( admin_names=['admin'], other_names=['slave-%02d' % x for x in range(1, int( settings.NODES_COUNT))] ) @property def env_name(self): return settings.ENV_NAME def add_empty_volume(self, node, name, capacity=settings.NODE_VOLUME_SIZE * 1024 * 1024 * 1024, device='disk', bus='virtio', format='qcow2'): self.manager.node_attach_volume( node=node, volume=self.manager.volume_create( name=name, capacity=capacity, environment=self.get_virtual_environment(), format=format), device=device, bus=bus) def add_node(self, memory, name, vcpu=1, boot=None): return self.manager.node_create( name=name, memory=memory, vcpu=vcpu, environment=self.get_virtual_environment(), boot=boot) @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server( cluster_id, self.get_host_node_ip(), port) def bootstrap_nodes(self, devops_nodes, timeout=600): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ # self.dhcrelay_check() for node in devops_nodes: node.start() # TODO(aglarendil): LP#1317213 temporary sleep # remove after better fix is applied time.sleep(2) wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) for node in self.nailgun_nodes(devops_nodes): self.sync_node_time(self.get_ssh_to_remote(node["ip"])) return self.nailgun_nodes(devops_nodes) def create_interfaces(self, networks, node, model=settings.INTERFACE_MODEL): if settings.BONDING: for network in networks: self.manager.interface_create( network, node=node, model=model, interface_map=settings.BONDING_INTERFACES) else: for network in networks: self.manager.interface_create(network, node=node, model=model) def describe_environment(self): """Environment :rtype : Environment """ environment = self.manager.environment_create(self.env_name) networks = [] interfaces = settings.INTERFACE_ORDER if self.multiple_cluster_networks: logger.info('Multiple cluster networks feature is enabled!') if settings.BONDING: interfaces = settings.BONDING_INTERFACES.keys() for name in interfaces: networks.append(self.create_networks(name, environment)) for name in self.node_roles.admin_names: self.describe_admin_node(name, networks) for name in self.node_roles.other_names: if self.multiple_cluster_networks: networks1 = [net for net in networks if net.name in settings.NODEGROUPS[0]['pools']] networks2 = [net for net in networks if net.name in settings.NODEGROUPS[1]['pools']] # If slave index is even number, then attach to # it virtual networks from the second network group. if int(name[-2:]) % 2 == 1: self.describe_empty_node(name, networks1) elif int(name[-2:]) % 2 == 0: self.describe_empty_node(name, networks2) else: self.describe_empty_node(name, networks) return environment def create_networks(self, name, environment): ip_networks = [ IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',')] new_prefix = int(settings.POOLS.get(name)[1]) pool = self.manager.create_network_pool(networks=ip_networks, prefix=int(new_prefix)) return self.manager.network_create( name=name, environment=environment, pool=pool, forward=settings.FORWARDING.get(name), has_dhcp_server=settings.DHCP.get(name)) def devops_nodes_by_names(self, devops_node_names): return map( lambda name: self.get_virtual_environment().node_by_name(name), devops_node_names) @logwrap def describe_admin_node(self, name, networks): node = self.add_node( memory=settings.HARDWARE.get("admin_node_memory", 1024), vcpu=settings.HARDWARE.get("admin_node_cpu", 1), name=name, boot=['hd', 'cdrom']) self.create_interfaces(networks, node) if self.os_image is None: self.add_empty_volume(node, name + '-system') self.add_empty_volume( node, name + '-iso', capacity=_get_file_size(settings.ISO_PATH), format='raw', device='cdrom', bus='ide') else: volume = self.manager.volume_get_predefined(self.os_image) vol_child = self.manager.volume_create_child( name=name + '-system', backing_store=volume, environment=self.get_virtual_environment() ) self.manager.node_attach_volume( node=node, volume=vol_child ) return node def describe_empty_node(self, name, networks): node = self.add_node( name=name, memory=settings.HARDWARE.get("slave_node_memory", 1024), vcpu=settings.HARDWARE.get("slave_node_cpu", 1)) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') if settings.USE_ALL_DISKS: self.add_empty_volume(node, name + '-cinder') self.add_empty_volume(node, name + '-swift') return node @logwrap def get_admin_remote(self, login=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password']): """SSH to admin node :rtype : SSHClient """ return self.nodes().admin.remote(self.admin_net, login=login, password=password) @logwrap def get_admin_node_ip(self): return str( self.nodes().admin.get_ip_address_by_network_name(self.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_host_node_ip(self): return self.router() def get_keys(self, node, custom=None, build_images=None): params = { 'ip': node.get_ip_address_by_network_name(self.admin_net), 'mask': self.get_net_mask(self.admin_net), 'gw': self.router(), 'hostname': '.'.join((self.hostname, self.domain)), 'nat_interface': self.nat_interface, 'dns1': settings.DNS, 'showmenu': 'yes' if custom else 'no', 'build_images': '1' if build_images else '0' } keys = ( "<Wait>\n" "<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " showmenu=%(showmenu)s\n" " build_images=%(build_images)s\n" " <Enter>\n" ) % params return keys @logwrap def get_private_keys(self, force=False): if force or self._keys is None: self._keys = [] for key_string in ['/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa']: with self.get_admin_remote().open(key_string) as f: self._keys.append(RSAKey.from_private_key(f)) return self._keys @logwrap def get_ssh_to_remote(self, ip): return SSHClient(ip, username=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password'], private_keys=self.get_private_keys()) @logwrap def get_ssh_to_remote_by_key(self, ip, keyfile): try: with open(keyfile) as f: keys = [RSAKey.from_private_key(f)] return SSHClient(ip, private_keys=keys) except IOError: logger.warning('Loading of SSH key from file failed. Trying to use' ' SSH agent ...') keys = Agent().get_keys() return SSHClient(ip, private_keys=keys) @logwrap def get_ssh_to_remote_by_name(self, node_name): return self.get_ssh_to_remote( self.fuel_web.get_nailgun_node_by_devops_node( self.get_virtual_environment().node_by_name(node_name))['ip'] ) def get_target_devs(self, devops_nodes): return [ interface.target_dev for interface in [ val for var in map(lambda node: node.interfaces, devops_nodes) for val in var]] def get_virtual_environment(self): """Returns virtual environment :rtype : devops.models.Environment """ if self._virtual_environment is None: self._virtual_environment = self._get_or_create() return self._virtual_environment def get_network(self, net_name): return str( IPNetwork( self.get_virtual_environment().network_by_name(net_name). ip_network)) def get_net_mask(self, net_name): return str( IPNetwork( self.get_virtual_environment().network_by_name( net_name).ip_network).netmask) def make_snapshot(self, snapshot_name, description="", is_make=False): if settings.MAKE_SNAPSHOT or is_make: self.get_virtual_environment().suspend(verbose=False) time.sleep(10) self.get_virtual_environment().snapshot(snapshot_name, force=True) revert_info(snapshot_name, description) if settings.FUEL_STATS_CHECK: self.get_virtual_environment().resume() try: self.nodes().admin.await(self.admin_net, timeout=60) except Exception: logger.error('Admin node is unavailable via SSH after ' 'environment resume ') raise def nailgun_nodes(self, devops_nodes): return map( lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node), devops_nodes ) def nodes(self): return Nodes(self.get_virtual_environment(), self.node_roles) def revert_snapshot(self, name): if self.get_virtual_environment().has_snapshot(name): logger.info('We have snapshot with such name %s' % name) self.get_virtual_environment().revert(name) logger.info('Starting snapshot reverting ....') self.get_virtual_environment().resume() logger.info('Starting snapshot resuming ...') admin = self.nodes().admin try: admin.await( self.admin_net, timeout=10 * 60, by_port=8000) except Exception as e: logger.warning("From first time admin isn't reverted: " "{0}".format(e)) admin.destroy() logger.info('Admin node was destroyed. Wait 10 sec.') time.sleep(10) self.get_virtual_environment().start(self.nodes().admins) logger.info('Admin node started second time.') self.nodes().admin.await( self.admin_net, timeout=10 * 60, by_port=8000) self.set_admin_ssh_password() try: _wait(self._fuel_web.client.get_releases, expected=EnvironmentError, timeout=300) except exceptions.Unauthorized: self.set_admin_keystone_password() self._fuel_web.get_nailgun_version() self.sync_time_admin_node() for node in self.nodes().slaves: if not node.driver.node_active(node): continue try: logger.info("Sync time on revert for node %s" % node.name) self.sync_node_time( self.get_ssh_to_remote_by_name(node.name)) except Exception as e: logger.warning( 'Exception caught while trying to sync time on {0}:' ' {1}'.format(node.name, e)) self.run_nailgun_agent( self.get_ssh_to_remote_by_name(node.name)) return True return False def set_admin_ssh_password(self): try: remote = self.get_admin_remote( login=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password']) self.execute_remote_cmd(remote, 'date') logger.debug('Accessing admin node using SSH: SUCCESS') except Exception: logger.debug('Accessing admin node using SSH credentials:' ' FAIL, trying to change password from default') remote = self.get_admin_remote(login='******', password='******') self.execute_remote_cmd( remote, 'echo -e "{1}\\n{1}" | passwd {0}' .format(settings.SSH_CREDENTIALS['login'], settings.SSH_CREDENTIALS['password'])) logger.debug("Admin node password has changed.") logger.info("Admin node login name: '{0}' , password: '******'". format(settings.SSH_CREDENTIALS['login'], settings.SSH_CREDENTIALS['password'])) def set_admin_keystone_password(self): remote = self.get_admin_remote() try: self._fuel_web.client.get_releases() except exceptions.Unauthorized: self.execute_remote_cmd( remote, 'fuel user --newpass {0} --change-password' .format(settings.KEYSTONE_CREDS['password'])) logger.info( 'New Fuel UI (keystone) username: "******", password: "******"' .format(settings.KEYSTONE_CREDS['username'], settings.KEYSTONE_CREDS['password'])) def setup_environment(self, custom=settings.CUSTOM_ENV, build_images=settings.BUILD_IMAGES): # start admin node admin = self.nodes().admin admin.disk_devices.get(device='cdrom').volume.upload(settings.ISO_PATH) self.get_virtual_environment().start(self.nodes().admins) logger.info("Waiting for admin node to start up") wait(lambda: admin.driver.node_active(admin), 60) logger.info("Proceed with installation") # update network parameters at boot screen admin.send_keys(self.get_keys(admin, custom=custom, build_images=build_images)) if custom: self.setup_customisation() # wait while installation complete admin.await(self.admin_net, timeout=10 * 60) self.set_admin_ssh_password() self.wait_bootstrap() time.sleep(10) self.set_admin_keystone_password() self.sync_time_admin_node() if settings.MULTIPLE_NETWORKS: self.describe_second_admin_interface() multiple_networks_hacks.configure_second_admin_cobbler(self) multiple_networks_hacks.configure_second_dhcrelay(self) self.nailgun_actions.set_collector_address( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL) if settings.FUEL_STATS_ENABLED: self.fuel_web.client.send_fuel_stats(enabled=True) logger.info('Enabled sending of statistics to {0}:{1}'.format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT )) @upload_manifests def wait_for_provisioning(self): _wait(lambda: _tcp_ping( self.nodes().admin.get_ip_address_by_network_name (self.admin_net), 22), timeout=5 * 60) def setup_customisation(self): self.wait_for_provisioning() try: remote = self.get_admin_remote() pid = remote.execute("pgrep 'fuelmenu'")['stdout'][0] pid.rstrip('\n') remote.execute("kill -sigusr1 {0}".format(pid)) except Exception: logger.error("Could not kill pid of fuelmenu") raise @retry(count=10, delay=60) @logwrap def sync_node_time(self, remote): self.execute_remote_cmd(remote, 'hwclock -s') self.execute_remote_cmd(remote, 'NTPD=$(find /etc/init.d/ -regex \'' '/etc/init.d/\(ntp.?\|ntp-dev\)\');' '$NTPD stop && ntpd -dqg && $NTPD ' 'start') self.execute_remote_cmd(remote, 'hwclock -w') remote_date = remote.execute('date')['stdout'] logger.info("Node time: %s" % remote_date) @retry(count=10, delay=60) @logwrap def sync_time_admin_node(self): logger.info("Sync time on revert for admin") remote = self.get_admin_remote() self.execute_remote_cmd(remote, 'hwclock -s') # Sync time using ntpd try: # If public NTP servers aren't accessible ntpdate will fail and # ntpd daemon shouldn't be restarted to avoid 'Server has gone # too long without sync' error while syncing time from slaves self.execute_remote_cmd(remote, "ntpdate -vu $(awk '/^server/ && " "$2 !~ /127.*/ {print $2}' " "/etc/ntp.conf)") except AssertionError as e: logger.warning('Error occurred while synchronizing time on master' ': {0}'.format(e)) raise else: self.execute_remote_cmd(remote, 'service ntpd stop && ntpd -dqg &&' ' service ntpd start') self.execute_remote_cmd(remote, 'hwclock -w') remote_date = remote.execute('date')['stdout'] logger.info("Master node time: {0}".format(remote_date)) def verify_network_configuration(self, node_name): checkers.verify_network_configuration( node=self.fuel_web.get_nailgun_node_by_name(node_name), remote=self.get_ssh_to_remote_by_name(node_name) ) def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" logger.info("Puppet timeout set in {0}".format( float(settings.PUPPET_TIMEOUT))) wait( lambda: not self.get_admin_remote().execute( "grep 'Fuel node deployment' '%s'" % log_path )['exit_code'], timeout=(float(settings.PUPPET_TIMEOUT)) ) result = self.get_admin_remote().execute("grep 'Fuel node deployment " "complete' '%s'" % log_path )['exit_code'] if result != 0: raise Exception('Fuel node deployment failed.') def dhcrelay_check(self): admin_remote = self.get_admin_remote() out = admin_remote.execute("dhcpcheck discover " "--ifaces eth0 " "--repeat 3 " "--timeout 10")['stdout'] assert_true(self.get_admin_node_ip() in "".join(out), "dhcpcheck doesn't discover master ip") def run_nailgun_agent(self, remote): agent = remote.execute('/opt/nailgun/bin/agent')['exit_code'] logger.info("Nailgun agent run with exit_code: %s" % agent) def get_fuel_settings(self, remote=None): if not remote: remote = self.get_admin_remote() cmd = 'cat {cfg_file}'.format(cfg_file=settings.FUEL_SETTINGS_YAML) result = remote.execute(cmd) if result['exit_code'] == 0: fuel_settings = yaml.load(''.join(result['stdout'])) else: raise Exception('Can\'t output {cfg_file} file: {error}'. format(cfg_file=settings.FUEL_SETTINGS_YAML, error=result['stderr'])) return fuel_settings def admin_install_pkg(self, pkg_name): """Install a package <pkg_name> on the admin node""" admin_remote = self.get_admin_remote() remote_status = admin_remote.execute("rpm -q {0}'".format(pkg_name)) if remote_status['exit_code'] == 0: logger.info("Package '{0}' already installed.".format(pkg_name)) else: logger.info("Installing package '{0}' ...".format(pkg_name)) remote_status = admin_remote.execute("yum -y install {0}" .format(pkg_name)) logger.info("Installation of the package '{0}' has been" " completed with exit code {1}" .format(pkg_name, remote_status['exit_code'])) return remote_status['exit_code'] def admin_run_service(self, service_name): """Start a service <service_name> on the admin node""" admin_remote = self.get_admin_remote() admin_remote.execute("service {0} start".format(service_name)) remote_status = admin_remote.execute("service {0} status" .format(service_name)) if any('running...' in status for status in remote_status['stdout']): logger.info("Service '{0}' is running".format(service_name)) else: logger.info("Service '{0}' failed to start" " with exit code {1} :\n{2}" .format(service_name, remote_status['exit_code'], remote_status['stdout'])) # Modifies a resolv.conf on the Fuel master node and returns # its original content. # * adds 'nameservers' at start of resolv.conf if merge=True # * replaces resolv.conf with 'nameservers' if merge=False def modify_resolv_conf(self, nameservers=[], merge=True): remote = self.get_admin_remote() resolv_conf = remote.execute('cat /etc/resolv.conf') assert_equal(0, resolv_conf['exit_code'], 'Executing "{0}" on the ' 'admin node has failed with: {1}' .format('cat /etc/resolv.conf', resolv_conf['stderr'])) if merge: nameservers.extend(resolv_conf['stdout']) resolv_keys = ['search', 'domain', 'nameserver'] resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers if any(x in ns for x in resolv_keys)) logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new)) echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new) echo_result = remote.execute(echo_cmd) assert_equal(0, echo_result['exit_code'], 'Executing "{0}" on the ' 'admin node has failed with: {1}' .format(echo_cmd, echo_result['stderr'])) return resolv_conf['stdout'] @logwrap def execute_remote_cmd(self, remote, cmd, exit_code=0): result = remote.execute(cmd) assert_equal(result['exit_code'], exit_code, 'Failed to execute "{0}" on remote host: {1}'. format(cmd, result)) return result['stdout'] @logwrap def describe_second_admin_interface(self): remote = self.get_admin_remote() second_admin_network = self.get_network(self.admin_net2).split('/')[0] second_admin_netmask = self.get_net_mask(self.admin_net2) second_admin_if = settings.INTERFACES.get(self.admin_net2) second_admin_ip = str(self.nodes().admin. get_ip_address_by_network_name(self.admin_net2)) logger.info(('Parameters for second admin interface configuration: ' 'Network - {0}, Netmask - {1}, Interface - {2}, ' 'IP Address - {3}').format(second_admin_network, second_admin_netmask, second_admin_if, second_admin_ip)) add_second_admin_ip = ('DEVICE={0}\\n' 'ONBOOT=yes\\n' 'NM_CONTROLLED=no\\n' 'USERCTL=no\\n' 'PEERDNS=no\\n' 'BOOTPROTO=static\\n' 'IPADDR={1}\\n' 'NETMASK={2}\\n').format(second_admin_if, second_admin_ip, second_admin_netmask) cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};' 'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format( add_second_admin_ip, second_admin_if, second_admin_ip) logger.debug('Trying to assign {0} IP to the {1} on master node...'. format(second_admin_ip, second_admin_if)) result = remote.execute(cmd) assert_equal(result['exit_code'], 0, ('Failed to assign second admin ' 'IP address on master node: {0}').format(result)) logger.debug('Done: {0}'.format(result['stdout'])) multiple_networks_hacks.configure_second_admin_firewall( self, second_admin_network, second_admin_netmask) @logwrap def get_masternode_uuid(self): return self.postgres_actions.run_query( db='nailgun', query="select master_node_uid from master_node_settings limit 1;")
def __init__(self, os_image=None): self._virtual_environment = None self._keys = None self.manager = Manager() self.os_image = os_image self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self)
class EnvironmentModel(object): hostname = 'nailgun' domain = 'test.domain.local' installation_timeout = 1800 deployment_timeout = 1800 puppet_timeout = 2000 nat_interface = '' # INTERFACES.get('admin') admin_net = 'admin' admin_net2 = 'admin2' multiple_cluster_networks = settings.MULTIPLE_NETWORKS def __init__(self, os_image=None): self._virtual_environment = None self._keys = None self.manager = Manager() self.os_image = os_image self._fuel_web = FuelWebClient(self.get_admin_node_ip(), self) @property def nailgun_actions(self): return FuelActions.Nailgun(self.get_admin_remote()) @property def postgres_actions(self): return FuelActions.Postgres(self.get_admin_remote()) def _get_or_create(self): try: return self.manager.environment_get(self.env_name) except Exception: self._virtual_environment = self.describe_environment() self._virtual_environment.define() return self._virtual_environment def router(self, router_name=None): router_name = router_name or self.admin_net if router_name == self.admin_net2: return str( IPNetwork(self.get_virtual_environment().network_by_name( router_name).ip_network)[2]) return str( IPNetwork(self.get_virtual_environment().network_by_name( router_name).ip_network)[1]) @property def fuel_web(self): """FuelWebClient :rtype: FuelWebClient """ return self._fuel_web @property def admin_node_ip(self): return self.fuel_web.admin_node_ip @property def node_roles(self): return NodeRoles(admin_names=['admin'], other_names=[ 'slave-%02d' % x for x in range(1, int(settings.NODES_COUNT)) ]) @property def env_name(self): return settings.ENV_NAME def add_empty_volume(self, node, name, capacity=settings.NODE_VOLUME_SIZE * 1024 * 1024 * 1024, device='disk', bus='virtio', format='qcow2'): self.manager.node_attach_volume( node=node, volume=self.manager.volume_create( name=name, capacity=capacity, environment=self.get_virtual_environment(), format=format), device=device, bus=bus) def add_node(self, memory, name, vcpu=1, boot=None): return self.manager.node_create( name=name, memory=memory, vcpu=vcpu, environment=self.get_virtual_environment(), boot=boot) @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server(cluster_id, self.get_host_node_ip(), port) def bootstrap_nodes(self, devops_nodes, timeout=600): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ # self.dhcrelay_check() for node in devops_nodes: node.start() # TODO(aglarendil): LP#1317213 temporary sleep # remove after better fix is applied time.sleep(2) wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) for node in self.nailgun_nodes(devops_nodes): self.sync_node_time(self.get_ssh_to_remote(node["ip"])) return self.nailgun_nodes(devops_nodes) def create_interfaces(self, networks, node, model=settings.INTERFACE_MODEL): if settings.BONDING: for network in networks: self.manager.interface_create( network, node=node, model=model, interface_map=settings.BONDING_INTERFACES) else: for network in networks: self.manager.interface_create(network, node=node, model=model) def describe_environment(self): """Environment :rtype : Environment """ environment = self.manager.environment_create(self.env_name) networks = [] interfaces = settings.INTERFACE_ORDER if self.multiple_cluster_networks: logger.info('Multiple cluster networks feature is enabled!') if settings.BONDING: interfaces = settings.BONDING_INTERFACES.keys() for name in interfaces: networks.append(self.create_networks(name, environment)) for name in self.node_roles.admin_names: self.describe_admin_node(name, networks) for name in self.node_roles.other_names: if self.multiple_cluster_networks: networks1 = [ net for net in networks if net.name in settings.NODEGROUPS[0]['pools'] ] networks2 = [ net for net in networks if net.name in settings.NODEGROUPS[1]['pools'] ] # If slave index is even number, then attach to # it virtual networks from the second network group. if int(name[-2:]) % 2 == 1: self.describe_empty_node(name, networks1) elif int(name[-2:]) % 2 == 0: self.describe_empty_node(name, networks2) else: self.describe_empty_node(name, networks) return environment def create_networks(self, name, environment): ip_networks = [ IPNetwork(x) for x in settings.POOLS.get(name)[0].split(',') ] new_prefix = int(settings.POOLS.get(name)[1]) pool = self.manager.create_network_pool(networks=ip_networks, prefix=int(new_prefix)) return self.manager.network_create( name=name, environment=environment, pool=pool, forward=settings.FORWARDING.get(name), has_dhcp_server=settings.DHCP.get(name)) def devops_nodes_by_names(self, devops_node_names): return map( lambda name: self.get_virtual_environment().node_by_name(name), devops_node_names) @logwrap def describe_admin_node(self, name, networks): node = self.add_node(memory=settings.HARDWARE.get( "admin_node_memory", 1024), vcpu=settings.HARDWARE.get("admin_node_cpu", 1), name=name, boot=['hd', 'cdrom']) self.create_interfaces(networks, node) if self.os_image is None: self.add_empty_volume(node, name + '-system') self.add_empty_volume(node, name + '-iso', capacity=_get_file_size(settings.ISO_PATH), format='raw', device='cdrom', bus='ide') else: volume = self.manager.volume_get_predefined(self.os_image) vol_child = self.manager.volume_create_child( name=name + '-system', backing_store=volume, environment=self.get_virtual_environment()) self.manager.node_attach_volume(node=node, volume=vol_child) return node def describe_empty_node(self, name, networks): node = self.add_node(name=name, memory=settings.HARDWARE.get( "slave_node_memory", 1024), vcpu=settings.HARDWARE.get("slave_node_cpu", 1)) self.create_interfaces(networks, node) self.add_empty_volume(node, name + '-system') if settings.USE_ALL_DISKS: self.add_empty_volume(node, name + '-cinder') self.add_empty_volume(node, name + '-swift') return node @logwrap def get_admin_remote(self, login=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password']): """SSH to admin node :rtype : SSHClient """ return self.nodes().admin.remote(self.admin_net, login=login, password=password) @logwrap def get_admin_node_ip(self): return str(self.nodes().admin.get_ip_address_by_network_name( self.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_host_node_ip(self): return self.router() def get_keys(self, node, custom=None, build_images=None): params = { 'ip': node.get_ip_address_by_network_name(self.admin_net), 'mask': self.get_net_mask(self.admin_net), 'gw': self.router(), 'hostname': '.'.join((self.hostname, self.domain)), 'nat_interface': self.nat_interface, 'dns1': settings.DNS, 'showmenu': 'yes' if custom else 'no', 'build_images': '1' if build_images else '0' } keys = ("<Wait>\n" "<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=cdrom:/ks.cfg\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " showmenu=%(showmenu)s\n" " build_images=%(build_images)s\n" " <Enter>\n") % params return keys @logwrap def get_private_keys(self, force=False): if force or self._keys is None: self._keys = [] for key_string in [ '/root/.ssh/id_rsa', '/root/.ssh/bootstrap.rsa' ]: with self.get_admin_remote().open(key_string) as f: self._keys.append(RSAKey.from_private_key(f)) return self._keys @logwrap def get_ssh_to_remote(self, ip): return SSHClient(ip, username=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password'], private_keys=self.get_private_keys()) @logwrap def get_ssh_to_remote_by_key(self, ip, keyfile): try: with open(keyfile) as f: keys = [RSAKey.from_private_key(f)] return SSHClient(ip, private_keys=keys) except IOError: logger.warning('Loading of SSH key from file failed. Trying to use' ' SSH agent ...') keys = Agent().get_keys() return SSHClient(ip, private_keys=keys) @logwrap def get_ssh_to_remote_by_name(self, node_name): return self.get_ssh_to_remote( self.fuel_web.get_nailgun_node_by_devops_node( self.get_virtual_environment().node_by_name(node_name))['ip']) def get_target_devs(self, devops_nodes): return [ interface.target_dev for interface in [ val for var in map(lambda node: node.interfaces, devops_nodes) for val in var ] ] def get_virtual_environment(self): """Returns virtual environment :rtype : devops.models.Environment """ if self._virtual_environment is None: self._virtual_environment = self._get_or_create() return self._virtual_environment def get_network(self, net_name): return str( IPNetwork(self.get_virtual_environment().network_by_name( net_name).ip_network)) def get_net_mask(self, net_name): return str( IPNetwork(self.get_virtual_environment().network_by_name( net_name).ip_network).netmask) def make_snapshot(self, snapshot_name, description="", is_make=False): if settings.MAKE_SNAPSHOT or is_make: self.get_virtual_environment().suspend(verbose=False) time.sleep(10) self.get_virtual_environment().snapshot(snapshot_name, force=True) revert_info(snapshot_name, description) if settings.FUEL_STATS_CHECK: self.get_virtual_environment().resume() try: self.nodes().admin. await (self.admin_net, timeout=60) except Exception: logger.error('Admin node is unavailable via SSH after ' 'environment resume ') raise def nailgun_nodes(self, devops_nodes): return map( lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node), devops_nodes) def nodes(self): return Nodes(self.get_virtual_environment(), self.node_roles) def revert_snapshot(self, name): if self.get_virtual_environment().has_snapshot(name): logger.info('We have snapshot with such name %s' % name) self.get_virtual_environment().revert(name) logger.info('Starting snapshot reverting ....') self.get_virtual_environment().resume() logger.info('Starting snapshot resuming ...') admin = self.nodes().admin try: admin. await (self.admin_net, timeout=10 * 60, by_port=8000) except Exception as e: logger.warning("From first time admin isn't reverted: " "{0}".format(e)) admin.destroy() logger.info('Admin node was destroyed. Wait 10 sec.') time.sleep(10) self.get_virtual_environment().start(self.nodes().admins) logger.info('Admin node started second time.') self.nodes().admin. await (self.admin_net, timeout=10 * 60, by_port=8000) self.set_admin_ssh_password() try: _wait(self._fuel_web.client.get_releases, expected=EnvironmentError, timeout=300) except exceptions.Unauthorized: self.set_admin_keystone_password() self._fuel_web.get_nailgun_version() self.sync_time_admin_node() for node in self.nodes().slaves: if not node.driver.node_active(node): continue try: logger.info("Sync time on revert for node %s" % node.name) self.sync_node_time( self.get_ssh_to_remote_by_name(node.name)) except Exception as e: logger.warning( 'Exception caught while trying to sync time on {0}:' ' {1}'.format(node.name, e)) self.run_nailgun_agent( self.get_ssh_to_remote_by_name(node.name)) return True return False def set_admin_ssh_password(self): try: remote = self.get_admin_remote( login=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password']) self.execute_remote_cmd(remote, 'date') logger.debug('Accessing admin node using SSH: SUCCESS') except Exception: logger.debug('Accessing admin node using SSH credentials:' ' FAIL, trying to change password from default') remote = self.get_admin_remote(login='******', password='******') self.execute_remote_cmd( remote, 'echo -e "{1}\\n{1}" | passwd {0}'.format( settings.SSH_CREDENTIALS['login'], settings.SSH_CREDENTIALS['password'])) logger.debug("Admin node password has changed.") logger.info("Admin node login name: '{0}' , password: '******'".format( settings.SSH_CREDENTIALS['login'], settings.SSH_CREDENTIALS['password'])) def set_admin_keystone_password(self): remote = self.get_admin_remote() try: self._fuel_web.client.get_releases() except exceptions.Unauthorized: self.execute_remote_cmd( remote, 'fuel user --newpass {0} --change-password'.format( settings.KEYSTONE_CREDS['password'])) logger.info( 'New Fuel UI (keystone) username: "******", password: "******"'. format(settings.KEYSTONE_CREDS['username'], settings.KEYSTONE_CREDS['password'])) def setup_environment(self, custom=settings.CUSTOM_ENV, build_images=settings.BUILD_IMAGES): # start admin node admin = self.nodes().admin admin.disk_devices.get(device='cdrom').volume.upload(settings.ISO_PATH) self.get_virtual_environment().start(self.nodes().admins) logger.info("Waiting for admin node to start up") wait(lambda: admin.driver.node_active(admin), 60) logger.info("Proceed with installation") # update network parameters at boot screen admin.send_keys( self.get_keys(admin, custom=custom, build_images=build_images)) if custom: self.setup_customisation() # wait while installation complete admin. await (self.admin_net, timeout=10 * 60) self.set_admin_ssh_password() self.wait_bootstrap() time.sleep(10) self.set_admin_keystone_password() self.sync_time_admin_node() if settings.MULTIPLE_NETWORKS: self.describe_second_admin_interface() multiple_networks_hacks.configure_second_admin_cobbler(self) multiple_networks_hacks.configure_second_dhcrelay(self) self.nailgun_actions.set_collector_address(settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL) if settings.FUEL_STATS_ENABLED: self.fuel_web.client.send_fuel_stats(enabled=True) logger.info('Enabled sending of statistics to {0}:{1}'.format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT)) @upload_manifests def wait_for_provisioning(self): _wait(lambda: _tcp_ping( self.nodes().admin.get_ip_address_by_network_name(self.admin_net), 22), timeout=5 * 60) def setup_customisation(self): self.wait_for_provisioning() try: remote = self.get_admin_remote() pid = remote.execute("pgrep 'fuelmenu'")['stdout'][0] pid.rstrip('\n') remote.execute("kill -sigusr1 {0}".format(pid)) except Exception: logger.error("Could not kill pid of fuelmenu") raise @retry(count=10, delay=60) @logwrap def sync_node_time(self, remote): self.execute_remote_cmd(remote, 'hwclock -s') self.execute_remote_cmd( remote, 'NTPD=$(find /etc/init.d/ -regex \'' '/etc/init.d/\(ntp.?\|ntp-dev\)\');' '$NTPD stop && ntpd -dqg && $NTPD ' 'start') self.execute_remote_cmd(remote, 'hwclock -w') remote_date = remote.execute('date')['stdout'] logger.info("Node time: %s" % remote_date) @retry(count=10, delay=60) @logwrap def sync_time_admin_node(self): logger.info("Sync time on revert for admin") remote = self.get_admin_remote() self.execute_remote_cmd(remote, 'hwclock -s') # Sync time using ntpd try: # If public NTP servers aren't accessible ntpdate will fail and # ntpd daemon shouldn't be restarted to avoid 'Server has gone # too long without sync' error while syncing time from slaves self.execute_remote_cmd( remote, "ntpdate -vu $(awk '/^server/ && " "$2 !~ /127.*/ {print $2}' " "/etc/ntp.conf)") except AssertionError as e: logger.warning('Error occurred while synchronizing time on master' ': {0}'.format(e)) raise else: self.execute_remote_cmd( remote, 'service ntpd stop && ntpd -dqg &&' ' service ntpd start') self.execute_remote_cmd(remote, 'hwclock -w') remote_date = remote.execute('date')['stdout'] logger.info("Master node time: {0}".format(remote_date)) def verify_network_configuration(self, node_name): checkers.verify_network_configuration( node=self.fuel_web.get_nailgun_node_by_name(node_name), remote=self.get_ssh_to_remote_by_name(node_name)) def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" logger.info("Puppet timeout set in {0}".format( float(settings.PUPPET_TIMEOUT))) wait(lambda: not self.get_admin_remote().execute( "grep 'Fuel node deployment' '%s'" % log_path)['exit_code'], timeout=(float(settings.PUPPET_TIMEOUT))) result = self.get_admin_remote().execute("grep 'Fuel node deployment " "complete' '%s'" % log_path)['exit_code'] if result != 0: raise Exception('Fuel node deployment failed.') def dhcrelay_check(self): admin_remote = self.get_admin_remote() out = admin_remote.execute("dhcpcheck discover " "--ifaces eth0 " "--repeat 3 " "--timeout 10")['stdout'] assert_true(self.get_admin_node_ip() in "".join(out), "dhcpcheck doesn't discover master ip") def run_nailgun_agent(self, remote): agent = remote.execute('/opt/nailgun/bin/agent')['exit_code'] logger.info("Nailgun agent run with exit_code: %s" % agent) def get_fuel_settings(self, remote=None): if not remote: remote = self.get_admin_remote() cmd = 'cat {cfg_file}'.format(cfg_file=settings.FUEL_SETTINGS_YAML) result = remote.execute(cmd) if result['exit_code'] == 0: fuel_settings = yaml.load(''.join(result['stdout'])) else: raise Exception('Can\'t output {cfg_file} file: {error}'.format( cfg_file=settings.FUEL_SETTINGS_YAML, error=result['stderr'])) return fuel_settings def admin_install_pkg(self, pkg_name): """Install a package <pkg_name> on the admin node""" admin_remote = self.get_admin_remote() remote_status = admin_remote.execute("rpm -q {0}'".format(pkg_name)) if remote_status['exit_code'] == 0: logger.info("Package '{0}' already installed.".format(pkg_name)) else: logger.info("Installing package '{0}' ...".format(pkg_name)) remote_status = admin_remote.execute( "yum -y install {0}".format(pkg_name)) logger.info("Installation of the package '{0}' has been" " completed with exit code {1}".format( pkg_name, remote_status['exit_code'])) return remote_status['exit_code'] def admin_run_service(self, service_name): """Start a service <service_name> on the admin node""" admin_remote = self.get_admin_remote() admin_remote.execute("service {0} start".format(service_name)) remote_status = admin_remote.execute( "service {0} status".format(service_name)) if any('running...' in status for status in remote_status['stdout']): logger.info("Service '{0}' is running".format(service_name)) else: logger.info("Service '{0}' failed to start" " with exit code {1} :\n{2}".format( service_name, remote_status['exit_code'], remote_status['stdout'])) # Modifies a resolv.conf on the Fuel master node and returns # its original content. # * adds 'nameservers' at start of resolv.conf if merge=True # * replaces resolv.conf with 'nameservers' if merge=False def modify_resolv_conf(self, nameservers=[], merge=True): remote = self.get_admin_remote() resolv_conf = remote.execute('cat /etc/resolv.conf') assert_equal( 0, resolv_conf['exit_code'], 'Executing "{0}" on the ' 'admin node has failed with: {1}'.format('cat /etc/resolv.conf', resolv_conf['stderr'])) if merge: nameservers.extend(resolv_conf['stdout']) resolv_keys = ['search', 'domain', 'nameserver'] resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers if any(x in ns for x in resolv_keys)) logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new)) echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new) echo_result = remote.execute(echo_cmd) assert_equal( 0, echo_result['exit_code'], 'Executing "{0}" on the ' 'admin node has failed with: {1}'.format(echo_cmd, echo_result['stderr'])) return resolv_conf['stdout'] @logwrap def execute_remote_cmd(self, remote, cmd, exit_code=0): result = remote.execute(cmd) assert_equal( result['exit_code'], exit_code, 'Failed to execute "{0}" on remote host: {1}'.format(cmd, result)) return result['stdout'] @logwrap def describe_second_admin_interface(self): remote = self.get_admin_remote() second_admin_network = self.get_network(self.admin_net2).split('/')[0] second_admin_netmask = self.get_net_mask(self.admin_net2) second_admin_if = settings.INTERFACES.get(self.admin_net2) second_admin_ip = str( self.nodes().admin.get_ip_address_by_network_name(self.admin_net2)) logger.info( ('Parameters for second admin interface configuration: ' 'Network - {0}, Netmask - {1}, Interface - {2}, ' 'IP Address - {3}').format(second_admin_network, second_admin_netmask, second_admin_if, second_admin_ip)) add_second_admin_ip = ('DEVICE={0}\\n' 'ONBOOT=yes\\n' 'NM_CONTROLLED=no\\n' 'USERCTL=no\\n' 'PEERDNS=no\\n' 'BOOTPROTO=static\\n' 'IPADDR={1}\\n' 'NETMASK={2}\\n').format( second_admin_if, second_admin_ip, second_admin_netmask) cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};' 'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format( add_second_admin_ip, second_admin_if, second_admin_ip) logger.debug( 'Trying to assign {0} IP to the {1} on master node...'.format( second_admin_ip, second_admin_if)) result = remote.execute(cmd) assert_equal(result['exit_code'], 0, ('Failed to assign second admin ' 'IP address on master node: {0}').format(result)) logger.debug('Done: {0}'.format(result['stdout'])) multiple_networks_hacks.configure_second_admin_firewall( self, second_admin_network, second_admin_netmask) @logwrap def get_masternode_uuid(self): return self.postgres_actions.run_query( db='nailgun', query="select master_node_uid from master_node_settings limit 1;")
def fuel_web(self): if self._fuel_web is None: self._fuel_web = FuelWebClient(self) return self._fuel_web
def __init__(self): self._virtual_environment = None self.fuel_web = FuelWebClient(self.get_admin_node_ip(), self)
class EnvironmentModel(object): """EnvironmentModel.""" # TODO documentation def __init__(self): self._virtual_environment = None self.fuel_web = FuelWebClient(self.get_admin_node_ip(), self) @property def admin_actions(self): return AdminActions(self.d_env.get_admin_remote()) @property def nailgun_actions(self): return NailgunActions(self.d_env.get_admin_remote()) @property def postgres_actions(self): return PostgresActions(self.d_env.get_admin_remote()) @property def cobbler_actions(self): return CobblerActions(self.d_env.get_admin_remote()) @property def docker_actions(self): return DockerActions(self.d_env.get_admin_remote()) @property def admin_node_ip(self): return self.fuel_web.admin_node_ip @property def collector(self): return CollectorClient(settings.ANALYTICS_IP, 'api/v1/json') @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server( cluster_id, self.d_env.router(), port) def bootstrap_nodes(self, devops_nodes, timeout=600, skip_timesync=False): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ # self.dhcrelay_check() for node in devops_nodes: logger.info("Bootstrapping node: {}".format(node.name)) node.start() # TODO(aglarendil): LP#1317213 temporary sleep # remove after better fix is applied time.sleep(2) with timestat("wait_for_nodes_to_start_and_register_in_nailgun"): wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) if not skip_timesync: self.sync_time([node for node in self.nailgun_nodes(devops_nodes)]) return self.nailgun_nodes(devops_nodes) @logwrap def get_admin_node_ip(self): return str( self.d_env.nodes( ).admin.get_ip_address_by_network_name( self.d_env.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_keys(self, node, custom=None, build_images=None, iso_connect_as='cdrom'): params = { 'ks': 'hd:LABEL="Mirantis_Fuel":/ks.cfg' if iso_connect_as == 'usb' else 'cdrom:/ks.cfg', 'repo': 'hd:LABEL="Mirantis_Fuel":/', # only required for USB boot 'ip': node.get_ip_address_by_network_name( self.d_env.admin_net), 'mask': self.d_env.get_network( name=self.d_env.admin_net).ip.netmask, 'gw': self.d_env.router(), 'hostname': '.'.join((self.d_env.hostname, self.d_env.domain)), 'nat_interface': self.d_env.nat_interface, 'dns1': settings.DNS, 'showmenu': 'yes' if custom else 'no', 'build_images': '1' if build_images else '0' } keys = '' if(iso_connect_as == 'usb'): keys = ( "<Wait>\n" # USB boot uses boot_menu=yes for master node "<F12>\n" "2\n" "<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=%(ks)s\n" " repo=%(repo)s\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " showmenu=%(showmenu)s\n" " build_images=%(build_images)s\n" " <Enter>\n" ) % params else: # cdrom case is default keys = ( "<Wait>\n" "<Esc>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=%(ks)s\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " showmenu=%(showmenu)s\n" " build_images=%(build_images)s\n" " <Enter>\n" ) % params return keys def get_target_devs(self, devops_nodes): return [ interface.target_dev for interface in [ val for var in map(lambda node: node.interfaces, devops_nodes) for val in var]] @property def d_env(self): if self._virtual_environment is None: try: return Environment.get(name=settings.ENV_NAME) except Exception: self._virtual_environment = Environment.describe_environment( boot_from=settings.ADMIN_BOOT_DEVICE) self._virtual_environment.define() return self._virtual_environment def resume_environment(self): self.d_env.resume() admin = self.d_env.nodes().admin try: admin.await(self.d_env.admin_net, timeout=30, by_port=8000) except Exception as e: logger.warning("From first time admin isn't reverted: " "{0}".format(e)) admin.destroy() logger.info('Admin node was destroyed. Wait 10 sec.') time.sleep(10) admin.start() logger.info('Admin node started second time.') self.d_env.nodes().admin.await(self.d_env.admin_net) self.set_admin_ssh_password() self.docker_actions.wait_for_ready_containers(timeout=600) # set collector address in case of admin node destroy if settings.FUEL_STATS_ENABLED: self.nailgun_actions.set_collector_address( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL) # Restart statsenderd in order to apply new collector address self.nailgun_actions.force_fuel_stats_sending() self.fuel_web.client.send_fuel_stats(enabled=True) logger.info('Enabled sending of statistics to {0}:{1}'.format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT )) self.set_admin_ssh_password() self.docker_actions.wait_for_ready_containers() def make_snapshot(self, snapshot_name, description="", is_make=False): if settings.MAKE_SNAPSHOT or is_make: self.d_env.suspend(verbose=False) time.sleep(10) self.d_env.snapshot(snapshot_name, force=True) revert_info(snapshot_name, self.get_admin_node_ip(), description) if settings.FUEL_STATS_CHECK: self.resume_environment() def nailgun_nodes(self, devops_nodes): return map( lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node), devops_nodes ) def check_slaves_are_ready(self): devops_nodes = [node for node in self.d_env.nodes().slaves if node.driver.node_active(node)] # Bug: 1455753 time.sleep(30) for node in devops_nodes: try: wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node( node)['online'], timeout=60 * 6) except TimeoutError: raise TimeoutError( "Node {0} does not become online".format(node.name)) return True def revert_snapshot(self, name, skip_timesync=False): if not self.d_env.has_snapshot(name): return False logger.info('We have snapshot with such name: %s' % name) logger.info("Reverting the snapshot '{0}' ....".format(name)) self.d_env.revert(name) logger.info("Resuming the snapshot '{0}' ....".format(name)) self.resume_environment() if not skip_timesync: nailgun_nodes = [self.fuel_web.get_nailgun_node_by_name(node.name) for node in self.d_env.nodes().slaves if node.driver.node_active(node)] self.sync_time(nailgun_nodes) try: _wait(self.fuel_web.client.get_releases, expected=EnvironmentError, timeout=300) except exceptions.Unauthorized: self.set_admin_keystone_password() self.fuel_web.get_nailgun_version() _wait(lambda: self.check_slaves_are_ready(), timeout=60 * 6) return True def set_admin_ssh_password(self): try: remote = self.d_env.get_admin_remote( login=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password']) self.execute_remote_cmd(remote, 'date') logger.debug('Accessing admin node using SSH: SUCCESS') except Exception: logger.debug('Accessing admin node using SSH credentials:' ' FAIL, trying to change password from default') remote = self.d_env.get_admin_remote( login='******', password='******') self.execute_remote_cmd( remote, 'echo -e "{1}\\n{1}" | passwd {0}' .format(settings.SSH_CREDENTIALS['login'], settings.SSH_CREDENTIALS['password'])) logger.debug("Admin node password has changed.") logger.info("Admin node login name: '{0}' , password: '******'". format(settings.SSH_CREDENTIALS['login'], settings.SSH_CREDENTIALS['password'])) def set_admin_keystone_password(self): remote = self.d_env.get_admin_remote() try: self.fuel_web.client.get_releases() except exceptions.Unauthorized: self.execute_remote_cmd( remote, 'fuel user --newpass {0} --change-password' .format(settings.KEYSTONE_CREDS['password'])) logger.info( 'New Fuel UI (keystone) username: "******", password: "******"' .format(settings.KEYSTONE_CREDS['username'], settings.KEYSTONE_CREDS['password'])) def setup_environment(self, custom=settings.CUSTOM_ENV, build_images=settings.BUILD_IMAGES, iso_connect_as=settings.ADMIN_BOOT_DEVICE): # start admin node admin = self.d_env.nodes().admin if(iso_connect_as == 'usb'): admin.disk_devices.get(device='disk', bus='usb').volume.upload(settings.ISO_PATH) else: # cdrom is default admin.disk_devices.get( device='cdrom').volume.upload(settings.ISO_PATH) self.d_env.start(self.d_env.nodes().admins) logger.info("Waiting for admin node to start up") wait(lambda: admin.driver.node_active(admin), 60) logger.info("Proceed with installation") # update network parameters at boot screen admin.send_keys(self.get_keys(admin, custom=custom, build_images=build_images, iso_connect_as=iso_connect_as)) if custom: self.setup_customisation() # wait while installation complete admin.await(self.d_env.admin_net, timeout=10 * 60) self.set_admin_ssh_password() self.admin_actions.modify_configs(self.d_env.router()) self.wait_bootstrap() self.docker_actions.wait_for_ready_containers() time.sleep(10) self.set_admin_keystone_password() self.sync_time() if settings.MULTIPLE_NETWORKS: self.describe_second_admin_interface() multiple_networks_hacks.configure_second_admin_cobbler(self) self.nailgun_actions.set_collector_address( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL) # Restart statsenderd in order to apply new settings(Collector address) self.nailgun_actions.force_fuel_stats_sending() if settings.FUEL_STATS_ENABLED: self.fuel_web.client.send_fuel_stats(enabled=True) logger.info('Enabled sending of statistics to {0}:{1}'.format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT )) @update_packages @upload_manifests def wait_for_provisioning(self): _wait(lambda: _tcp_ping( self.d_env.nodes( ).admin.get_ip_address_by_network_name (self.d_env.admin_net), 22), timeout=7 * 60) def setup_customisation(self): self.wait_for_provisioning() try: remote = self.d_env.get_admin_remote() cmd = "pkill -sigusr1 -f '^.*/fuelmenu$'" wait(lambda: remote.execute(cmd)['exit_code'] == 0, timeout=60) except Exception: logger.error("Could not kill process of fuelmenu") raise @retry(count=3, delay=60) def sync_time(self, nailgun_nodes=[]): # with @retry, failure on any step of time synchronization causes # restart the time synchronization starting from the admin node admin_ntp = Ntp.get_ntp(self.d_env.get_admin_remote(), 'admin') controller_nodes = [ n for n in nailgun_nodes if "controller" in n['roles']] other_nodes = [ n for n in nailgun_nodes if "controller" not in n['roles']] # 1. The first time source for the environment: admin node logger.info("Synchronizing time on Fuel admin node") GroupNtpSync().do_sync_time([admin_ntp]) # 2. Controllers should be synchronized before providing time to others if controller_nodes: logger.info("Synchronizing time on all controllers") GroupNtpSync(self, controller_nodes).do_sync_time() # 3. Synchronize time on all the rest nodes if other_nodes: logger.info("Synchronizing time on other active nodes") GroupNtpSync(self, other_nodes).do_sync_time() def verify_network_configuration(self, node_name): node = self.fuel_web.get_nailgun_node_by_name(node_name) checkers.verify_network_configuration( node=node, remote=self.d_env.get_ssh_to_remote(node['ip']) ) def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" logger.info("Puppet timeout set in {0}".format( float(settings.PUPPET_TIMEOUT))) wait( lambda: not self.d_env.get_admin_remote().execute( "grep 'Fuel node deployment' '%s'" % log_path )['exit_code'], timeout=(float(settings.PUPPET_TIMEOUT)) ) result = self.d_env.get_admin_remote().execute( "grep 'Fuel node deployment " "complete' '%s'" % log_path)['exit_code'] if result != 0: raise Exception('Fuel node deployment failed.') def dhcrelay_check(self): admin_remote = self.d_env.get_admin_remote() out = admin_remote.execute("dhcpcheck discover " "--ifaces eth0 " "--repeat 3 " "--timeout 10")['stdout'] assert_true(self.get_admin_node_ip() in "".join(out), "dhcpcheck doesn't discover master ip") def run_nailgun_agent(self, remote): agent = remote.execute('/opt/nailgun/bin/agent')['exit_code'] logger.info("Nailgun agent run with exit_code: %s" % agent) def get_fuel_settings(self, remote=None): if not remote: remote = self.d_env.get_admin_remote() cmd = 'cat {cfg_file}'.format(cfg_file=settings.FUEL_SETTINGS_YAML) result = remote.execute(cmd) if result['exit_code'] == 0: fuel_settings = yaml.load(''.join(result['stdout'])) else: raise Exception('Can\'t output {cfg_file} file: {error}'. format(cfg_file=settings.FUEL_SETTINGS_YAML, error=result['stderr'])) return fuel_settings def admin_install_pkg(self, pkg_name): """Install a package <pkg_name> on the admin node""" admin_remote = self.d_env.get_admin_remote() remote_status = admin_remote.execute("rpm -q {0}'".format(pkg_name)) if remote_status['exit_code'] == 0: logger.info("Package '{0}' already installed.".format(pkg_name)) else: logger.info("Installing package '{0}' ...".format(pkg_name)) remote_status = admin_remote.execute("yum -y install {0}" .format(pkg_name)) logger.info("Installation of the package '{0}' has been" " completed with exit code {1}" .format(pkg_name, remote_status['exit_code'])) return remote_status['exit_code'] def admin_run_service(self, service_name): """Start a service <service_name> on the admin node""" admin_remote = self.d_env.get_admin_remote() admin_remote.execute("service {0} start".format(service_name)) remote_status = admin_remote.execute("service {0} status" .format(service_name)) if any('running...' in status for status in remote_status['stdout']): logger.info("Service '{0}' is running".format(service_name)) else: logger.info("Service '{0}' failed to start" " with exit code {1} :\n{2}" .format(service_name, remote_status['exit_code'], remote_status['stdout'])) # Modifies a resolv.conf on the Fuel master node and returns # its original content. # * adds 'nameservers' at start of resolv.conf if merge=True # * replaces resolv.conf with 'nameservers' if merge=False def modify_resolv_conf(self, nameservers=[], merge=True): remote = self.d_env.get_admin_remote() resolv_conf = remote.execute('cat /etc/resolv.conf') assert_equal(0, resolv_conf['exit_code'], 'Executing "{0}" on the ' 'admin node has failed with: {1}' .format('cat /etc/resolv.conf', resolv_conf['stderr'])) if merge: nameservers.extend(resolv_conf['stdout']) resolv_keys = ['search', 'domain', 'nameserver'] resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers if any(x in ns for x in resolv_keys)) logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new)) echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new) echo_result = remote.execute(echo_cmd) assert_equal(0, echo_result['exit_code'], 'Executing "{0}" on the ' 'admin node has failed with: {1}' .format(echo_cmd, echo_result['stderr'])) return resolv_conf['stdout'] @logwrap def execute_remote_cmd(self, remote, cmd, exit_code=0): result = remote.execute(cmd) assert_equal(result['exit_code'], exit_code, 'Failed to execute "{0}" on remote host: {1}'. format(cmd, result)) return result['stdout'] @logwrap def describe_second_admin_interface(self): remote = self.d_env.get_admin_remote() admin_net2_object = self.d_env.get_network(name=self.d_env.admin_net2) second_admin_network = admin_net2_object.ip.network second_admin_netmask = admin_net2_object.ip.netmask second_admin_if = settings.INTERFACES.get(self.d_env.admin_net2) second_admin_ip = str(self.d_env.nodes( ).admin.get_ip_address_by_network_name(self.d_env.admin_net2)) logger.info(('Parameters for second admin interface configuration: ' 'Network - {0}, Netmask - {1}, Interface - {2}, ' 'IP Address - {3}').format(second_admin_network, second_admin_netmask, second_admin_if, second_admin_ip)) add_second_admin_ip = ('DEVICE={0}\\n' 'ONBOOT=yes\\n' 'NM_CONTROLLED=no\\n' 'USERCTL=no\\n' 'PEERDNS=no\\n' 'BOOTPROTO=static\\n' 'IPADDR={1}\\n' 'NETMASK={2}\\n').format(second_admin_if, second_admin_ip, second_admin_netmask) cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};' 'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format( add_second_admin_ip, second_admin_if, second_admin_ip) logger.debug('Trying to assign {0} IP to the {1} on master node...'. format(second_admin_ip, second_admin_if)) result = remote.execute(cmd) assert_equal(result['exit_code'], 0, ('Failed to assign second admin ' 'IP address on master node: {0}').format(result)) logger.debug('Done: {0}'.format(result['stdout'])) multiple_networks_hacks.configure_second_admin_firewall( self, second_admin_network, second_admin_netmask) @logwrap def get_masternode_uuid(self): return self.postgres_actions.run_query( db='nailgun', query="select master_node_uid from master_node_settings limit 1;")
class EnvironmentModel(object): """EnvironmentModel.""" # TODO documentation def __init__(self): self._virtual_environment = None self.fuel_web = FuelWebClient(self.get_admin_node_ip(), self) def __repr__(self): klass, obj_id = type(self), hex(id(self)) if hasattr(self, 'fuel_web'): ip = self.fuel_web.admin_node_ip else: ip = None return "[{klass}({obj_id}), ip:{ip}]".format(klass=klass, obj_id=obj_id, ip=ip) @property def admin_actions(self): return AdminActions(self.d_env.get_admin_remote()) @property def nailgun_actions(self): return NailgunActions(self.d_env.get_admin_remote()) @property def postgres_actions(self): return PostgresActions(self.d_env.get_admin_remote()) @property def cobbler_actions(self): return CobblerActions(self.d_env.get_admin_remote()) @property def docker_actions(self): return DockerActions(self.d_env.get_admin_remote()) @property def admin_node_ip(self): return self.fuel_web.admin_node_ip @property def collector(self): return CollectorClient(settings.ANALYTICS_IP, 'api/v1/json') @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server(cluster_id, self.d_env.router(), port) def bootstrap_nodes(self, devops_nodes, timeout=900, skip_timesync=False): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ # self.dhcrelay_check() for node in devops_nodes: logger.info("Bootstrapping node: {}".format(node.name)) node.start() # TODO(aglarendil): LP#1317213 temporary sleep # remove after better fix is applied time.sleep(2) with timestat("wait_for_nodes_to_start_and_register_in_nailgun"): wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) if not skip_timesync: self.sync_time([node for node in self.nailgun_nodes(devops_nodes)]) return self.nailgun_nodes(devops_nodes) @logwrap def get_admin_node_ip(self): return str(self.d_env.nodes().admin.get_ip_address_by_network_name( self.d_env.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_keys(self, node, custom=None, build_images=None, iso_connect_as='cdrom'): params = { 'ks': 'hd:LABEL="Mirantis_Fuel":/ks.cfg' if iso_connect_as == 'usb' else 'cdrom:/ks.cfg', 'repo': 'hd:LABEL="Mirantis_Fuel":/', # only required for USB boot 'ip': node.get_ip_address_by_network_name(self.d_env.admin_net), 'mask': self.d_env.get_network(name=self.d_env.admin_net).ip.netmask, 'gw': self.d_env.router(), 'hostname': ''.join((settings.FUEL_MASTER_HOSTNAME, settings.DNS_SUFFIX)), 'nat_interface': self.d_env.nat_interface, 'dns1': settings.DNS, 'showmenu': 'yes' if custom else 'no', 'build_images': '1' if build_images else '0' } keys = '' if (iso_connect_as == 'usb'): keys = ( "<Wait>\n" # USB boot uses boot_menu=yes for master node "<F12>\n" "2\n" "<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=%(ks)s\n" " repo=%(repo)s\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " showmenu=%(showmenu)s\n" " build_images=%(build_images)s\n" " <Enter>\n") % params else: # cdrom case is default keys = ("<Wait>\n" "<Wait>\n" "<Wait>\n" "<Esc>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=%(ks)s\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " showmenu=%(showmenu)s\n" " build_images=%(build_images)s\n" " <Enter>\n") % params return keys def get_target_devs(self, devops_nodes): return [ interface.target_dev for interface in [ val for var in map(lambda node: node.interfaces, devops_nodes) for val in var ] ] @property def d_env(self): if self._virtual_environment is None: try: return Environment.get(name=settings.ENV_NAME) except Exception: self._virtual_environment = Environment.describe_environment( boot_from=settings.ADMIN_BOOT_DEVICE) self._virtual_environment.define() return self._virtual_environment def resume_environment(self): self.d_env.resume() admin = self.d_env.nodes().admin try: admin. await (self.d_env.admin_net, timeout=30, by_port=8000) except Exception as e: logger.warning("From first time admin isn't reverted: " "{0}".format(e)) admin.destroy() logger.info('Admin node was destroyed. Wait 10 sec.') time.sleep(10) admin.start() logger.info('Admin node started second time.') self.d_env.nodes().admin. await (self.d_env.admin_net) self.set_admin_ssh_password() self.docker_actions.wait_for_ready_containers(timeout=600) # set collector address in case of admin node destroy if settings.FUEL_STATS_ENABLED: self.nailgun_actions.set_collector_address( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL) # Restart statsenderd in order to apply new collector address self.nailgun_actions.force_fuel_stats_sending() self.fuel_web.client.send_fuel_stats(enabled=True) logger.info('Enabled sending of statistics to {0}:{1}'.format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT)) self.set_admin_ssh_password() self.docker_actions.wait_for_ready_containers() def make_snapshot(self, snapshot_name, description="", is_make=False): if settings.MAKE_SNAPSHOT or is_make: self.d_env.suspend(verbose=False) time.sleep(10) self.d_env.snapshot(snapshot_name, force=True) revert_info(snapshot_name, self.get_admin_node_ip(), description) if settings.FUEL_STATS_CHECK: self.resume_environment() def nailgun_nodes(self, devops_nodes): return map( lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node), devops_nodes) def check_slaves_are_ready(self): devops_nodes = [ node for node in self.d_env.nodes().slaves if node.driver.node_active(node) ] # Bug: 1455753 time.sleep(30) for node in devops_nodes: try: wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node( node)['online'], timeout=60 * 6) except TimeoutError: raise TimeoutError("Node {0} does not become online".format( node.name)) return True def revert_snapshot(self, name, skip_timesync=False): if not self.d_env.has_snapshot(name): return False logger.info('We have snapshot with such name: %s' % name) logger.info("Reverting the snapshot '{0}' ....".format(name)) self.d_env.revert(name) logger.info("Resuming the snapshot '{0}' ....".format(name)) self.resume_environment() if not skip_timesync: nailgun_nodes = [ self.fuel_web.get_nailgun_node_by_name(node.name) for node in self.d_env.nodes().slaves if node.driver.node_active(node) ] self.sync_time(nailgun_nodes) try: _wait(self.fuel_web.client.get_releases, expected=EnvironmentError, timeout=300) except exceptions.Unauthorized: self.set_admin_keystone_password() self.fuel_web.get_nailgun_version() _wait(lambda: self.check_slaves_are_ready(), timeout=60 * 6) return True def set_admin_ssh_password(self): try: with self.d_env.get_admin_remote( login=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password']) as remote: self.execute_remote_cmd(remote, 'date') logger.debug('Accessing admin node using SSH: SUCCESS') except Exception: logger.debug('Accessing admin node using SSH credentials:' ' FAIL, trying to change password from default') with self.d_env.get_admin_remote(login='******', password='******') as remote: self.execute_remote_cmd( remote, 'echo -e "{1}\\n{1}" | passwd {0}'.format( settings.SSH_CREDENTIALS['login'], settings.SSH_CREDENTIALS['password'])) logger.debug("Admin node password has changed.") logger.info("Admin node login name: '{0}' , password: '******'".format( settings.SSH_CREDENTIALS['login'], settings.SSH_CREDENTIALS['password'])) def set_admin_keystone_password(self): try: self.fuel_web.client.get_releases() except exceptions.Unauthorized: with self.d_env.get_admin_remote() as remote: self.execute_remote_cmd( remote, 'fuel user --newpass {0} --change-password'.format( settings.KEYSTONE_CREDS['password'])) logger.info( 'New Fuel UI (keystone) username: "******", password: "******"'. format(settings.KEYSTONE_CREDS['username'], settings.KEYSTONE_CREDS['password'])) def setup_environment(self, custom=settings.CUSTOM_ENV, build_images=settings.BUILD_IMAGES, iso_connect_as=settings.ADMIN_BOOT_DEVICE, security=settings.SECURITY_TEST): # start admin node admin = self.d_env.nodes().admin if (iso_connect_as == 'usb'): admin.disk_devices.get(device='disk', bus='usb').volume.upload(settings.ISO_PATH) else: # cdrom is default admin.disk_devices.get(device='cdrom').volume.upload( settings.ISO_PATH) self.d_env.start(self.d_env.nodes().admins) logger.info("Waiting for admin node to start up") wait(lambda: admin.driver.node_active(admin), 60) logger.info("Proceed with installation") # update network parameters at boot screen admin.send_keys( self.get_keys(admin, custom=custom, build_images=build_images, iso_connect_as=iso_connect_as)) if custom: self.setup_customisation() if security: nessus_node = NessusActions(self.d_env) nessus_node.add_nessus_node() # wait while installation complete admin. await (self.d_env.admin_net, timeout=10 * 60) self.set_admin_ssh_password() self.admin_actions.modify_configs(self.d_env.router()) self.wait_bootstrap() self.docker_actions.wait_for_ready_containers() time.sleep(10) self.set_admin_keystone_password() self.sync_time() if settings.UPDATE_MASTER: if settings.UPDATE_FUEL_MIRROR: for i, url in enumerate(settings.UPDATE_FUEL_MIRROR): conf_file = '/etc/yum.repos.d/temporary-{}.repo'.format(i) cmd = ("echo -e" " '[temporary-{0}]\nname=" "temporary-{0}\nbaseurl={1}/" "\ngpgcheck=0\npriority=" "1' > {2}").format(i, url, conf_file) with self.d_env.get_admin_remote() as remote: remote.execute(cmd) self.admin_install_updates() if settings.MULTIPLE_NETWORKS: self.describe_second_admin_interface() multiple_networks_hacks.configure_second_admin_cobbler(self) self.nailgun_actions.set_collector_address(settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL) # Restart statsenderd in order to apply new settings(Collector address) self.nailgun_actions.force_fuel_stats_sending() if settings.FUEL_STATS_ENABLED: self.fuel_web.client.send_fuel_stats(enabled=True) logger.info('Enabled sending of statistics to {0}:{1}'.format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT)) if settings.PATCHING_DISABLE_UPDATES: with self.d_env.get_admin_remote() as remote: cmd = "find /etc/yum.repos.d/ -type f -regextype posix-egrep" \ " -regex '.*/mos[0-9,\.]+\-(updates|security).repo' | " \ "xargs -n1 -i sed '$aenabled=0' -i {}" self.execute_remote_cmd(remote, cmd) @update_packages @upload_manifests def wait_for_provisioning(self): _wait(lambda: _tcp_ping( self.d_env.nodes().admin.get_ip_address_by_network_name( self.d_env.admin_net), 22), timeout=7 * 60) def setup_customisation(self): self.wait_for_provisioning() try: cmd = "pkill -sigusr1 -f '^.*/fuelmenu$'" with self.d_env.get_admin_remote() as remote: wait(lambda: remote.execute(cmd)['exit_code'] == 0, timeout=60) except Exception: logger.error("Could not kill process of fuelmenu") raise @retry(count=3, delay=60) def sync_time(self, nailgun_nodes=[]): # with @retry, failure on any step of time synchronization causes # restart the time synchronization starting from the admin node controller_nodes = [ n for n in nailgun_nodes if "controller" in n['roles'] ] other_nodes = [ n for n in nailgun_nodes if "controller" not in n['roles'] ] # 1. The first time source for the environment: admin node logger.info("Synchronizing time on Fuel admin node") with GroupNtpSync(self, sync_admin_node=True) as g_ntp: g_ntp.do_sync_time() # 2. Controllers should be synchronized before providing time to others if controller_nodes: logger.info("Synchronizing time on all controllers") with GroupNtpSync(self, nailgun_nodes=controller_nodes) as g_ntp: g_ntp.do_sync_time() # 3. Synchronize time on all the rest nodes if other_nodes: logger.info("Synchronizing time on other active nodes") with GroupNtpSync(self, nailgun_nodes=other_nodes) as g_ntp: g_ntp.do_sync_time() def verify_network_configuration(self, node_name): node = self.fuel_web.get_nailgun_node_by_name(node_name) with self.fuel_web.get_ssh_for_node(node_name) as ssh: checkers.verify_network_configuration(node=node, remote=ssh) def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" logger.info("Puppet timeout set in {0}".format( float(settings.PUPPET_TIMEOUT))) wait(lambda: not self.d_env.get_admin_remote().execute( "grep 'Fuel node deployment' '%s'" % log_path)['exit_code'], timeout=(float(settings.PUPPET_TIMEOUT))) result = self.d_env.get_admin_remote().execute( "grep 'Fuel node deployment " "complete' '%s'" % log_path)['exit_code'] if result != 0: raise Exception('Fuel node deployment failed.') def dhcrelay_check(self): with self.d_env.get_admin_remote() as admin_remote: out = admin_remote.execute("dhcpcheck discover " "--ifaces eth0 " "--repeat 3 " "--timeout 10")['stdout'] assert_true(self.get_admin_node_ip() in "".join(out), "dhcpcheck doesn't discover master ip") def get_fuel_settings(self, remote=None): cmd = 'cat {cfg_file}'.format(cfg_file=settings.FUEL_SETTINGS_YAML) if not remote: with self.d_env.get_admin_remote() as remote: remote.execute(cmd) else: result = remote.execute(cmd) if result['exit_code'] == 0: fuel_settings = yaml.load(''.join(result['stdout'])) else: raise Exception('Can\'t output {cfg_file} file: {error}'.format( cfg_file=settings.FUEL_SETTINGS_YAML, error=result['stderr'])) return fuel_settings def admin_install_pkg(self, pkg_name): """Install a package <pkg_name> on the admin node""" with self.d_env.get_admin_remote() as remote: remote_status = remote.execute("rpm -q {0}'".format(pkg_name)) if remote_status['exit_code'] == 0: logger.info( "Package '{0}' already installed.".format(pkg_name)) else: logger.info("Installing package '{0}' ...".format(pkg_name)) remote_status = remote.execute( "yum -y install {0}".format(pkg_name)) logger.info("Installation of the package '{0}' has been" " completed with exit code {1}".format( pkg_name, remote_status['exit_code'])) return remote_status['exit_code'] def admin_run_service(self, service_name): """Start a service <service_name> on the admin node""" with self.d_env.get_admin_remote() as admin_remote: admin_remote.execute("service {0} start".format(service_name)) remote_status = admin_remote.execute( "service {0} status".format(service_name)) if any('running...' in status for status in remote_status['stdout']): logger.info("Service '{0}' is running".format(service_name)) else: logger.info("Service '{0}' failed to start" " with exit code {1} :\n{2}".format( service_name, remote_status['exit_code'], remote_status['stdout'])) # Execute yum updates # If updates installed, # then `dockerctl destroy all; bootstrap_admin_node.sh;` def admin_install_updates(self): logger.info('Searching for updates..') update_command = 'yum clean expire-cache; yum update -y' with self.d_env.get_admin_remote() as admin_remote: update_result = admin_remote.execute(update_command) logger.info('Result of "{1}" command on master node: ' '{0}'.format(update_result, update_command)) assert_equal(int(update_result['exit_code']), 0, 'Packages update failed, ' 'inspect logs for details') # Check if any packets were updated and update was successful for str_line in update_result['stdout']: match_updated_count = re.search("Upgrade(?:\s*)(\d+).*Package", str_line) if match_updated_count: updates_count = match_updated_count.group(1) match_complete_message = re.search("(Complete!)", str_line) match_no_updates = re.search("No Packages marked for Update", str_line) if (not match_updated_count or match_no_updates)\ and not match_complete_message: logger.warning('No updates were found or update was incomplete.') return logger.info('{0} packet(s) were updated'.format(updates_count)) cmd = 'dockerctl destroy all; bootstrap_admin_node.sh;' with self.d_env.get_admin_remote() as admin_remote: result = admin_remote.execute(cmd) logger.info('Result of "{1}" command on master node: ' '{0}'.format(result, cmd)) assert_equal(int(result['exit_code']), 0, 'bootstrap failed, ' 'inspect logs for details') # Modifies a resolv.conf on the Fuel master node and returns # its original content. # * adds 'nameservers' at start of resolv.conf if merge=True # * replaces resolv.conf with 'nameservers' if merge=False def modify_resolv_conf(self, nameservers=[], merge=True): with self.d_env.get_admin_remote() as remote: resolv_conf = remote.execute('cat /etc/resolv.conf') assert_equal( 0, resolv_conf['exit_code'], 'Executing "{0}" on the ' 'admin node has failed with: {1}'.format( 'cat /etc/resolv.conf', resolv_conf['stderr'])) if merge: nameservers.extend(resolv_conf['stdout']) resolv_keys = ['search', 'domain', 'nameserver'] resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers if any(x in ns for x in resolv_keys)) logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new)) echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new) echo_result = remote.execute(echo_cmd) assert_equal( 0, echo_result['exit_code'], 'Executing "{0}" on the ' 'admin node has failed with: {1}'.format( echo_cmd, echo_result['stderr'])) return resolv_conf['stdout'] @logwrap def execute_remote_cmd(self, remote, cmd, exit_code=0): result = remote.execute(cmd) assert_equal( result['exit_code'], exit_code, 'Failed to execute "{0}" on remote host: {1}'.format(cmd, result)) return result['stdout'] @logwrap def describe_second_admin_interface(self): admin_net2_object = self.d_env.get_network(name=self.d_env.admin_net2) second_admin_network = admin_net2_object.ip.network second_admin_netmask = admin_net2_object.ip.netmask second_admin_if = settings.INTERFACES.get(self.d_env.admin_net2) second_admin_ip = str( self.d_env.nodes().admin.get_ip_address_by_network_name( self.d_env.admin_net2)) logger.info( ('Parameters for second admin interface configuration: ' 'Network - {0}, Netmask - {1}, Interface - {2}, ' 'IP Address - {3}').format(second_admin_network, second_admin_netmask, second_admin_if, second_admin_ip)) add_second_admin_ip = ('DEVICE={0}\\n' 'ONBOOT=yes\\n' 'NM_CONTROLLED=no\\n' 'USERCTL=no\\n' 'PEERDNS=no\\n' 'BOOTPROTO=static\\n' 'IPADDR={1}\\n' 'NETMASK={2}\\n').format( second_admin_if, second_admin_ip, second_admin_netmask) cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};' 'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format( add_second_admin_ip, second_admin_if, second_admin_ip) logger.debug( 'Trying to assign {0} IP to the {1} on master node...'.format( second_admin_ip, second_admin_if)) with self.d_env.get_admin_remote() as remote: result = remote.execute(cmd) assert_equal(result['exit_code'], 0, ('Failed to assign second admin ' 'IP address on master node: {0}').format(result)) logger.debug('Done: {0}'.format(result['stdout'])) multiple_networks_hacks.configure_second_admin_firewall( self, second_admin_network, second_admin_netmask) @logwrap def get_masternode_uuid(self): return self.postgres_actions.run_query( db='nailgun', query="select master_node_uid from master_node_settings limit 1;")