def _check_known_hosts(host_list): """Ensure all hosts have entries in 'known_hosts' to avoid Ansible's clunky yes/no prompting to accept keys (all prompts are printed at once). If any hosts are missing the user will be prompted to add it. Args: host_list (list): List of hostnames or IP addresses """ known_hosts_files = [os.path.join(Path.home(), ".ssh", "known_hosts")] user_name, user_home_dir = get_user_and_home() if os.environ['USER'] == 'root' and user_name != 'root': known_hosts_files.append('/root/.ssh/known_hosts') if not os.path.isdir('/root/.ssh'): os.mkdir('/root/.ssh') os.chmod('/root/.ssh', 0o700) for host in host_list: for known_hosts in known_hosts_files: cmd = (f'ssh-keygen -F {host} -f {known_hosts}') resp, err, rc = sub_proc_exec(cmd) if rc != 0: cmd = (f'ssh-keyscan -H {host}') resp, err, rc = sub_proc_exec(cmd) print(f'Adding \'{host}\' host keys to \'{known_hosts}\'') append_line(known_hosts, resp, check_exists=False)
def add_software_hosts_global_var(software_hosts_file_path, entry): """Copy an SSH public key into software hosts authorized_keys files Add entry to software_hosts '[all:vars]' section. Any existing entries with the same key name (string before '=') will be overwritten. Args: software_hosts_file_path (str): Path to software inventory file entry (str) : Entry to write in software_hosts '[all:vars]' """ remove_line(software_hosts_file_path, '^ansible_ssh_private_key_file=.*') append_line(software_hosts_file_path, '[all:vars]') with open(software_hosts_file_path, 'r') as software_hosts_read: software_hosts = software_hosts_read.readlines() in_all_vars = False prev_line = "BOF" with open(software_hosts_file_path, 'w') as software_hosts_write: for line in software_hosts: if line.startswith("[all:vars]"): if prev_line != "\n": line = "\n" + line line = line + f'{entry}\n' in_all_vars = True elif in_all_vars and line.startswith('['): in_all_vars = False elif in_all_vars and line.startswith(entry.split('=')[0]): continue software_hosts_write.write(line) prev_line = line _set_software_hosts_owner_mode(software_hosts_file_path)
def _check_known_hosts(host_list): """Ensure all hosts have entries in 'known_hosts' to avoid Ansible's clunky yes/no prompting to accept keys (all prompts are printed at once). If any hosts are missing the user will be prompted to add it. Args: host_list (list): List of hostnames or IP addresses """ log = logger.getlogger() known_hosts_files = set() user_name, user_home_dir = get_user_and_home() user_uid = pwd.getpwnam(user_name).pw_uid user_gid = grp.getgrnam(user_name).gr_gid user_known_hosts = os.path.join(Path.home(), ".ssh", "known_hosts") known_hosts_files.add(user_known_hosts) if not os.path.isdir(os.path.join(Path.home(), ".ssh")): log.debug(f"Creating user ~/.ssh dir") os.mkdir(os.path.join(Path.home(), ".ssh")) os.chmod(os.path.join(Path.home(), ".ssh"), 0o700) os.chown(os.path.join(Path.home(), ".ssh"), user_uid, user_gid) if not os.path.isfile(user_known_hosts): log.debug(f"Creating user known hosts '{user_known_hosts}' file") Path(user_known_hosts).touch(mode=0o600) os.chown(user_known_hosts, user_uid, user_gid) if os.environ['USER'] == 'root' and user_name != 'root': known_hosts_files.add('/root/.ssh/known_hosts') if not os.path.isdir('/root/.ssh'): log.debug("Creating root '/root/.ssh' dir") os.mkdir('/root/.ssh') os.chmod('/root/.ssh', 0o700) if not os.path.isfile('/root/.ssh/known_hosts'): log.debug("Creating root '/root/.ssh/known_hosts' file") Path('/root/.ssh/known_hosts').touch(mode=0o600) for host in host_list: for known_hosts in known_hosts_files: cmd = (f'ssh-keygen -F {host} -f {known_hosts}') resp, err, rc = sub_proc_exec(cmd) if rc != 0: cmd = (f'ssh-keyscan -H {host}') resp, err, rc = sub_proc_exec(cmd) print(f'Adding \'{host}\' host keys to \'{known_hosts}\'') append_line(known_hosts, resp, check_exists=False)
def cobbler_install(config_path=None): """Install and configure Cobbler in container. This function must be called within the container 'pup-venv' python virtual environment. Cobbler will be installed within this environment. """ cfg = Config(config_path) log = logger.getlogger() # Check to see if cobbler is already installed try: util.bash_cmd('cobbler check') log.info("Cobbler is already installed") return except util.CalledProcessError as error: if error.returncode == 127: log.debug("'cobbler' command not found, continuing with " "installation") else: log.warning("Cobbler is installed but not working:") log.warning(error.output) print("\nPress enter to remove Cobbler and attempt to ") print("re-install, or 'T' to terminate.") resp = input("\nEnter or 'T': ") log.debug("User response = \'{}\'".format(resp)) if resp == 'T': sys.exit('POWER-Up stopped at user request') # Clone cobbler github repo cobbler_url = URL cobbler_branch = BRANCH install_dir = gen.get_cobbler_install_dir() if os.path.exists(install_dir): log.info( "Removing Cobbler source directory \'{}\'".format(install_dir)) util.bash_cmd('rm -rf %s' % install_dir) log.info("Cloning Cobbler branch \'%s\' from \'%s\'" % (cobbler_branch, cobbler_url)) repo = Repo.clone_from(cobbler_url, install_dir, branch=cobbler_branch, single_branch=True) log.info("Cobbler branch \'%s\' cloned into \'%s\'" % (repo.active_branch, repo.working_dir)) # Modify Cobbler scrpit that write DHCP reservations so that the # lease time is included. dhcp_lease_time = cfg.get_globals_dhcp_lease_time() util.replace_regex( MANAGE_DNSMASQ, r'systxt \= systxt \+ \"\\\\n\"', "systxt = systxt + \",{}\\\\n\"".format(dhcp_lease_time)) # Use non-secure http to download network boot-loaders util.replace_regex(COBBLER_DLCONTENT, 'https://cobbler.github.io', 'http://cobbler.github.io') # Use non-secure http to download signatures util.replace_regex(COBBLER_SETTINGS_PY, 'https://cobbler.github.io', 'http://cobbler.github.io') # Run cobbler make install util.bash_cmd('cd %s; make install' % install_dir) # Backup original files util.backup_file(DNSMASQ_TEMPLATE) util.backup_file(MODULES_CONF) util.backup_file(COBBLER_WEB_SETTINGS) util.backup_file(COBBLER_CONF_ORIG) util.backup_file(COBBLER_WEB_CONF_ORIG) util.backup_file(COBBLER_SETTINGS) util.backup_file(PXEDEFAULT_TEMPLATE) util.backup_file(KICKSTART_DONE) util.backup_file(NTP_CONF) util.backup_file(APACHE2_CONF) # Create tftp root directory if not os.path.exists(TFTPBOOT): mode = 0o755 os.mkdir(TFTPBOOT, mode) # Set IP address range to use for unrecognized DHCP clients dhcp_range = 'dhcp-range=%s,%s,%s # %s' util.remove_line(DNSMASQ_TEMPLATE, 'dhcp-range') dhcp_pool_start = gen.get_dhcp_pool_start() for index, netw_type in enumerate(cfg.yield_depl_netw_client_type()): depl_netw_client_ip = cfg.get_depl_netw_client_cont_ip(index) depl_netw_client_netmask = cfg.get_depl_netw_client_netmask(index) network = IPNetwork(depl_netw_client_ip + '/' + depl_netw_client_netmask) entry = dhcp_range % (str(network.network + dhcp_pool_start), str(network.network + network.size - 1), str(dhcp_lease_time), str(network.cidr)) util.append_line(DNSMASQ_TEMPLATE, entry) # Save PXE client network information for later if netw_type == 'pxe': cont_pxe_ipaddr = depl_netw_client_ip cont_pxe_netmask = depl_netw_client_netmask bridge_pxe_ipaddr = cfg.get_depl_netw_client_brg_ip(index) # Configure dnsmasq to enable TFTP server util.append_line(DNSMASQ_TEMPLATE, 'enable-tftp') util.append_line(DNSMASQ_TEMPLATE, 'tftp-root=%s' % TFTPBOOT) util.append_line(DNSMASQ_TEMPLATE, 'user=root') # Configure dnsmasq to use deployer as gateway if cfg.get_depl_gateway(): util.remove_line(DNSMASQ_TEMPLATE, 'dhcp-option') util.append_line(DNSMASQ_TEMPLATE, 'dhcp-option=3,%s' % bridge_pxe_ipaddr) # Cobbler modules configuration util.replace_regex(MODULES_CONF, 'module = manage_bind', 'module = manage_dnsmasq') util.replace_regex(MODULES_CONF, 'module = manage_isc', 'module = manage_dnsmasq') # Copy cobbler.conf into apache2/conf-available copy2(COBBLER_CONF_ORIG, COBBLER_CONF) # Copy cobbler_web.conf into apache2/conf-available copy2(COBBLER_WEB_CONF_ORIG, COBBLER_WEB_CONF) # Apache2 configuration util.bash_cmd('%s cobbler cobbler_web' % A2ENCONF) util.bash_cmd('%s proxy' % A2ENMOD) util.bash_cmd('%s proxy_http' % A2ENMOD) # Set secret key in web settings secret_key = _generate_random_characters() util.replace_regex(COBBLER_WEB_SETTINGS, '^SECRET_KEY = .*', 'SECRET_KEY = "%s"' % secret_key) # Remove "Order allow,deny" lines from cobbler configuration regex = '.*Order allow,deny' util.remove_line(COBBLER_CONF, regex) util.remove_line(COBBLER_WEB_CONF, regex) # Replace "Allow from all" with "Require all granted" in regex = 'Allow from all' replace = 'Require all granted' util.replace_regex(COBBLER_CONF, regex, replace) util.replace_regex(COBBLER_WEB_CONF, regex, replace) # chown www-data WEBUI_SESSIONS uid = pwd.getpwnam("www-data").pw_uid gid = -1 # unchanged os.chown(WEBUI_SESSIONS, uid, gid) # Cobbler settings util.replace_regex(COBBLER_SETTINGS, '127.0.0.1', cont_pxe_ipaddr) util.replace_regex(COBBLER_SETTINGS, 'manage_dhcp: 0', 'manage_dhcp: 1') util.replace_regex(COBBLER_SETTINGS, 'manage_dns: 0', 'manage_dns: 1') util.replace_regex(COBBLER_SETTINGS, 'pxe_just_once: 0', 'pxe_just_once: 1') globals_env_variables = cfg.get_globals_env_variables() if globals_env_variables and 'http_proxy' in globals_env_variables: util.replace_regex( COBBLER_SETTINGS, 'proxy_url_ext: ""', 'proxy_url_ext: %s' % globals_env_variables['http_proxy']) util.replace_regex( COBBLER_SETTINGS, 'default_password_crypted:', 'default_password_crypted: ' '$1$clusterp$/gd3ep3.36A2808GGdHUz.') # Create link to if not os.path.exists(PY_DIST_PKGS): util.bash_cmd('ln -s %s/cobbler %s' % (LOCAL_PY_DIST_PKGS, PY_DIST_PKGS)) # Set PXE timeout to maximum util.replace_regex(PXEDEFAULT_TEMPLATE, r'TIMEOUT \d+', 'TIMEOUT 35996') util.replace_regex(PXEDEFAULT_TEMPLATE, r'TOTALTIMEOUT \d+', 'TOTALTIMEOUT 35996') # Fix line break escape in kickstart_done snippet util.replace_regex(KICKSTART_DONE, "\\\\nwget", "wget") util.replace_regex(KICKSTART_DONE, r"\$saveks", "$saveks + \"; \\\\\\\"\n") util.replace_regex(KICKSTART_DONE, r"\$runpost", "$runpost + \"; \\\\\\\"\n") # Copy authorized_keys ssh key file to web repo directory copy2(ROOT_AUTH_KEYS, WWW_AUTH_KEYS) os.chmod(WWW_AUTH_KEYS, 0o444) # Add mgmt subnet to NTP service configuration cont_pxe_broadcast = str( IPNetwork(cont_pxe_ipaddr + '/' + cont_pxe_netmask).broadcast) util.append_line(NTP_CONF, 'broadcast %s' % cont_pxe_broadcast) # Add 'required-stop' line to cobblerd init.d to avoid warning util.replace_regex(INITD + 'cobblerd', '### END INIT INFO', '# Required-Stop:\n### END INIT INFO') # Set Apache2 'ServerName' util.append_line(APACHE2_CONF, "ServerName localhost") # Restart services _restart_service('ntp') _restart_service('cobblerd') _restart_service('apache2') # Update Cobbler boot-loader files util.bash_cmd('%s get-loaders' % COBBLER) # Update cobbler list of OS signatures util.bash_cmd('%s signature update' % COBBLER) # Run Cobbler sync util.bash_cmd('%s sync' % COBBLER) # Restart services (again) _restart_service('apache2') _restart_service('cobblerd') _restart_service('dnsmasq') # Set services to start on boot _service_start_on_boot('cobblerd') _service_start_on_boot('ntp')
def inv_set_ipmi_pxe_ip(config_path): """Configure DHCP IP reservations for IPMI and PXE interfaces IP addresses are assigned sequentially within the appropriate client networks starting with the DHCP pool start offset defined in 'lib.genesis'. Raises: UserException: - No IPMI or PXE client networks defined within the 'config.yml' - Unable to connect to BMC at new IPMI IP address """ log = logger.getlogger() cfg = Config(config_path) inv = Inventory(cfg_file=config_path) ipmiNetwork = None pxeNetwork = None nodes_list = [] # All nodes should be powered off before starting ipmi_set_power('off', config_path, wait=POWER_WAIT) # Create IPManager object for IPMI and/or PXE networks start_offset = gen.get_dhcp_pool_start() for index, netw_type in enumerate(cfg.yield_depl_netw_client_type()): ip = cfg.get_depl_netw_client_cont_ip(index) netmask = cfg.get_depl_netw_client_netmask(index) if netw_type == 'ipmi': ipmiNetwork = IPManager(IPNetwork(ip + '/' + netmask), start_offset) elif netw_type == 'pxe': pxeNetwork = IPManager(IPNetwork(ip + '/' + netmask), start_offset) # If only one network is defined use the same IPManager for both if ipmiNetwork is None and pxeNetwork is not None: ipmiNetwork = pxeNetwork elif ipmiNetwork is not None and pxeNetwork is None: pxeNetwork = ipmiNetwork elif ipmiNetwork is None and pxeNetwork is None: raise UserException('No IPMI or PXE client network found') # Modify IP addresses for each node dhcp_lease_time = cfg.get_globals_dhcp_lease_time() for index, hostname in enumerate(inv.yield_nodes_hostname()): # IPMI reservations are written directly to the dnsmasq template ipmi_ipaddr = inv.get_nodes_ipmi_ipaddr(0, index) ipmi_mac = inv.get_nodes_ipmi_mac(0, index) ipmi_new_ipaddr = ipmiNetwork.get_next_ip() util.remove_line(DNSMASQ_TEMPLATE, "^dhcp-host=" + ipmi_mac + ".*") util.append_line( DNSMASQ_TEMPLATE, 'dhcp-host=%s,%s-bmc,%s,%s\n' % (ipmi_mac, hostname, ipmi_new_ipaddr, dhcp_lease_time)) _adjust_dhcp_pool(ipmiNetwork.network, ipmiNetwork.get_next_ip(reserve=False), dhcp_lease_time) # PXE reservations are handled by Cobbler pxe_ipaddr = inv.get_nodes_pxe_ipaddr(0, index) pxe_mac = inv.get_nodes_pxe_mac(0, index) pxe_new_ipaddr = pxeNetwork.get_next_ip() log.info('Modifying Inventory PXE IP - Node: %s MAC: %s ' 'Original IP: %s New IP: %s' % (hostname, pxe_mac, pxe_ipaddr, pxe_new_ipaddr)) inv.set_nodes_pxe_ipaddr(0, index, pxe_new_ipaddr) _adjust_dhcp_pool(pxeNetwork.network, pxeNetwork.get_next_ip(reserve=False), dhcp_lease_time) # Run Cobbler sync to process DNSMASQ template cobbler_server = xmlrpclib.Server("http://127.0.0.1/cobbler_api") token = cobbler_server.login(COBBLER_USER, COBBLER_PASS) cobbler_server.sync(token) log.debug("Running Cobbler sync") # Save info to verify connection come back up ipmi_userid = inv.get_nodes_ipmi_userid(index) ipmi_password = inv.get_nodes_ipmi_password(index) # No need to reset and check if the IP does not change if ipmi_new_ipaddr != ipmi_ipaddr: nodes_list.append({ 'hostname': hostname, 'index': index, 'ipmi_userid': ipmi_userid, 'ipmi_password': ipmi_password, 'ipmi_new_ipaddr': ipmi_new_ipaddr, 'ipmi_ipaddr': ipmi_ipaddr, 'ipmi_mac': ipmi_mac }) # Issue MC cold reset to force refresh of IPMI interfaces for node in nodes_list: ipmi_userid = node['ipmi_userid'] ipmi_password = node['ipmi_password'] ipmi_ipaddr = node['ipmi_ipaddr'] ipmi_cmd = ipmi_command.Command(bmc=ipmi_ipaddr, userid=ipmi_userid, password=ipmi_password) ipmi_cmd.reset_bmc() del ipmi_cmd log.debug('BMC Cold Reset Issued - Node: %s - IP: %s' % (hostname, ipmi_ipaddr)) # Check connections for set amount of time end_time = time() + WAIT_TIME while time() < end_time and len(nodes_list) > 0: success_list = [] for list_index, node in enumerate(nodes_list): hostname = node['hostname'] index = node['index'] ipmi_userid = node['ipmi_userid'] ipmi_password = node['ipmi_password'] ipmi_new_ipaddr = node['ipmi_new_ipaddr'] ipmi_ipaddr = node['ipmi_ipaddr'] ipmi_mac = node['ipmi_mac'] # Attempt to connect to new IPMI IP address try: ipmi_cmd = ipmi_command.Command(bmc=ipmi_new_ipaddr, userid=ipmi_userid, password=ipmi_password) status = ipmi_cmd.get_power() except pyghmi_exception.IpmiException as error: log.debug('BMC connection failed - Node: %s IP: %s, %s ' '(Retrying for %s seconds)' % (hostname, ipmi_new_ipaddr, str(error), WAIT_TIME)) continue # If connection sucessful modify inventory if status.get('powerstate') in ['on', 'off']: log.debug('BMC connection success - Node: %s IP: %s' % (hostname, ipmi_new_ipaddr)) log.info('Modifying Inventory IPMI IP - Node: %s MAC: %s ' 'Original IP: %s New IP: %s' % (hostname, ipmi_mac, ipmi_ipaddr, ipmi_new_ipaddr)) inv.set_nodes_ipmi_ipaddr(0, index, ipmi_new_ipaddr) success_list.append(list_index) # Remove nodes that connected successfully for remove_index in sorted(success_list, reverse=True): del nodes_list[remove_index] for node in nodes_list: log.error('Unable to connect to BMC at new IPMI IP address- Node: %s ' 'MAC: %s Original IP: %s New IP: %s' % (hostname, ipmi_mac, ipmi_ipaddr, ipmi_new_ipaddr)) if len(nodes_list) > 0: raise UserException('%d BMC(s) not responding after IP modification' % len(nodes_list))
def _validate_ansible_ping(software_hosts_file_path, hosts_list): """Validate Ansible connectivity and functionality on all hosts Args: software_hosts_file_path (str): Path to software inventory file host_list (list): List of hostnames or IP addresses Returns: bool: True if Ansible can connect to all hosts Raises: UserException: If any host fails """ log = logger.getlogger() cmd = ('{} -i {} -m ping all'.format(get_ansible_path(), software_hosts_file_path)) resp, err, rc = sub_proc_exec(cmd) if str(rc) != "0": msg = f'Ansible ping validation failed:\n{resp}' log.debug(msg) if 'WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!' in msg: print( '@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n' '@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED @\n' '@ ON ONE OR MORE CLIENT NODES! @\n' '@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n' 'IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY!\n' 'Someone could be eavesdropping on you right now ' '(man-in-the-middle attack)!\n' 'It is also possible that a host key has just been changed.\n') if get_yesno('Remove the existing known host keys? '): known_hosts_files = ( [os.path.join(Path.home(), ".ssh", "known_hosts")]) user_name, user_home_dir = get_user_and_home() if user_home_dir != str(Path.home()): known_hosts_files.append(os.path.join(user_home_dir, ".ssh", "known_hosts")) for host in hosts_list: print(f'Collecting new host key(s) for {host}') cmd = (f'ssh-keyscan -H {host}') new_host_key, err, rc = sub_proc_exec(cmd) for known_hosts in known_hosts_files: print(f'Removing host keys for {host} ' f'from {known_hosts}') cmd = (f'ssh-keygen -R {host} -f {known_hosts}') resp, err, rc = sub_proc_exec(cmd) print(f'Appending new host key for {host} to ' f'{known_hosts}') append_line(known_hosts, new_host_key, check_exists=False) if user_home_dir != str(Path.home()): user_known_hosts = os.path.join(user_home_dir, ".ssh", "known_hosts") user_uid = pwd.getpwnam(user_name).pw_uid user_gid = grp.getgrnam(user_name).gr_gid os.chown(user_known_hosts, user_uid, user_gid) os.chmod(user_known_hosts, 0o600) os.chown(user_known_hosts + '.old', user_uid, user_gid) os.chmod(user_known_hosts + '.old', 0o600) return _validate_ansible_ping(software_hosts_file_path, hosts_list) elif 'Permission denied' in msg: msg = ('The PowerUp software installer attempted to log into the ' 'the client node(s) but was unsuccessful. SSH key access ' 'may need to be configured.\n') print(msg) if get_yesno('OK to configure Client Nodes for SSH Key Access? '): configure_ssh_keys(software_hosts_file_path) return _validate_ansible_ping(software_hosts_file_path, hosts_list) raise UserException(msg) log.debug("Software inventory Ansible ping validation passed") return True
def inv_set_ipmi_pxe_ip(config_path): """Configure DHCP IP reservations for IPMI and PXE interfaces IP addresses are assigned sequentially within the appropriate client networks starting with the DHCP pool start offset defined in 'lib.genesis'. Raises: UserException: - No IPMI or PXE client networks defined within the 'config.yml' - Unable to connect to BMC at new IPMI IP address """ log = logger.getlogger() cfg = Config(config_path) inv = Inventory(cfg_file=config_path) ipmiNetwork = None pxeNetwork = None nodes_list = [] # All nodes should be powered off before starting set_power_clients('off', config_path, wait=POWER_WAIT) # Create IPManager object for IPMI and/or PXE networks start_offset = gen.get_dhcp_pool_start() for index, netw_type in enumerate(cfg.yield_depl_netw_client_type()): ip = cfg.get_depl_netw_client_cont_ip(index) netmask = cfg.get_depl_netw_client_netmask(index) if netw_type == 'ipmi': ipmiNetwork = IPManager(IPNetwork(ip + '/' + netmask), start_offset) elif netw_type == 'pxe': pxeNetwork = IPManager(IPNetwork(ip + '/' + netmask), start_offset) # If only one network is defined use the same IPManager for both if ipmiNetwork is None and pxeNetwork is not None: ipmiNetwork = pxeNetwork elif ipmiNetwork is not None and pxeNetwork is None: pxeNetwork = ipmiNetwork elif ipmiNetwork is None and pxeNetwork is None: raise UserException('No IPMI or PXE client network found') # Modify IP addresses for each node dhcp_lease_time = cfg.get_globals_dhcp_lease_time() for index, hostname in enumerate(inv.yield_nodes_hostname()): # IPMI reservations are written directly to the dnsmasq template ipmi_ipaddr = inv.get_nodes_ipmi_ipaddr(0, index) ipmi_mac = inv.get_nodes_ipmi_mac(0, index) ipmi_new_ipaddr = ipmiNetwork.get_next_ip() util.remove_line(DNSMASQ_TEMPLATE, "^dhcp-host=" + ipmi_mac + ".*") util.append_line( DNSMASQ_TEMPLATE, 'dhcp-host=%s,%s-bmc,%s,%s\n' % (ipmi_mac, hostname, ipmi_new_ipaddr, dhcp_lease_time)) _adjust_dhcp_pool(ipmiNetwork.network, ipmiNetwork.get_next_ip(reserve=False), dhcp_lease_time) # PXE reservations are handled by Cobbler pxe_ipaddr = inv.get_nodes_pxe_ipaddr(0, index) pxe_mac = inv.get_nodes_pxe_mac(0, index) pxe_new_ipaddr = pxeNetwork.get_next_ip() log.info('Modifying Inventory PXE IP - Node: %s MAC: %s ' 'Original IP: %s New IP: %s' % (hostname, pxe_mac, pxe_ipaddr, pxe_new_ipaddr)) inv.set_nodes_pxe_ipaddr(0, index, pxe_new_ipaddr) _adjust_dhcp_pool(pxeNetwork.network, pxeNetwork.get_next_ip(reserve=False), dhcp_lease_time) # Run Cobbler sync to process DNSMASQ template cobbler_server = xmlrpc.client.Server("http://127.0.0.1/cobbler_api") token = cobbler_server.login(COBBLER_USER, COBBLER_PASS) cobbler_server.sync(token) log.debug("Running Cobbler sync") # Save info to verify connection come back up ipmi_userid = inv.get_nodes_ipmi_userid(index) ipmi_password = inv.get_nodes_ipmi_password(index) bmc_type = inv.get_nodes_bmc_type(index) # No need to reset and check if the IP does not change if ipmi_new_ipaddr != ipmi_ipaddr: nodes_list.append({ 'hostname': hostname, 'index': index, 'ipmi_userid': ipmi_userid, 'ipmi_password': ipmi_password, 'ipmi_new_ipaddr': ipmi_new_ipaddr, 'ipmi_ipaddr': ipmi_ipaddr, 'ipmi_mac': ipmi_mac, 'bmc_type': bmc_type }) # Issue MC cold reset to force refresh of IPMI interfaces for node in nodes_list: ipmi_userid = node['ipmi_userid'] ipmi_password = node['ipmi_password'] ipmi_ipaddr = node['ipmi_ipaddr'] bmc_type = node['bmc_type'] bmc = _bmc.Bmc(ipmi_ipaddr, ipmi_userid, ipmi_password, bmc_type) if bmc.is_connected(): log.debug(f'Issuing BMC Cold Reset - Node: {node["hostname"]} ' f'- IP: {ipmi_ipaddr}') if not bmc.bmc_reset('cold'): log.error( f'Failed attempting BMC reset on {node["ipmi_ipaddr"]}') bmc.logout() log.info('Pausing 1 minute for BMCs to begin reset') sleep(60) # Check connections for set amount of time end_time = time() + WAIT_TIME while time() < end_time and len(nodes_list) > 0: print(f'\rTimeout count down: {int(end_time - time())} ', end='') sys.stdout.flush() success_list = [] sleep(2) for list_index, node in enumerate(nodes_list): hostname = node['hostname'] index = node['index'] ipmi_userid = node['ipmi_userid'] ipmi_password = node['ipmi_password'] ipmi_new_ipaddr = node['ipmi_new_ipaddr'] ipmi_ipaddr = node['ipmi_ipaddr'] ipmi_mac = node['ipmi_mac'] bmc_type = node['bmc_type'] # Attempt to connect to new IPMI IP address bmc = _bmc.Bmc(ipmi_new_ipaddr, ipmi_userid, ipmi_password, bmc_type) if bmc.is_connected(): if bmc.chassis_power('status') in ('on', 'off'): log.debug(f'BMC connection success - Node: {hostname} ' f'IP: {ipmi_ipaddr}') else: log.debug(f'BMC communication failed - Node: {hostname} ' f'IP: {ipmi_ipaddr}') continue log.info( f'Modifying Inventory IPMI IP - Node: {hostname} MAC: ' f'{ipmi_mac} Original IP: {ipmi_ipaddr} New IP: ' f'{ipmi_new_ipaddr}') inv.set_nodes_ipmi_ipaddr(0, index, ipmi_new_ipaddr) success_list.append(list_index) else: log.debug(f'BMC connection failed - Node: {hostname} ' f'IP: {ipmi_ipaddr}') continue # Remove nodes that connected successfully for remove_index in sorted(success_list, reverse=True): del nodes_list[remove_index] for node in nodes_list: log.error('Unable to connect to BMC at new IPMI IP address- Node: %s ' 'MAC: %s Original IP: %s New IP: %s' % (hostname, ipmi_mac, ipmi_ipaddr, ipmi_new_ipaddr)) if len(nodes_list) > 0: raise UserException('%d BMC(s) not responding after IP modification' % len(nodes_list))