def set_power_clients(state, config_path=None, clients=None, max_attempts=5, wait=6): """Set power on or off for multiple clients. If a list of ip addresses are given or no clients given then the credentials are looked up in an inventory file. If clients is a dictionary, then the credentials are taken from the dictionary values. Args: state (str) : 'on' or 'off' config_path (str): path to a config file clients (dict or list of str): list of IP addresses or dict of ip addresses with values of credentials as tuple ie {'192.168.1.2': ('user', 'password', 'bmc_type')} """ log = logger.getlogger() if config_path: inv = Inventory(config_path) wait = float(wait) max_attempts = int(max_attempts) def _get_cred_list(client_list=None): """Returns dict with values of tuples. Each tuple has the credentials for a node (userid, password, bmc_type). If no client list is passed, all nodes are returned Args: client_list (list of str): each list item is an ipv4 address """ cred_list = {} for index, hostname in enumerate(inv.yield_nodes_hostname()): ipv4 = inv.get_nodes_ipmi_ipaddr(0, index) if client_list and ipv4 not in client_list: continue # rack_id = inv.get_nodes_rack_id(index) userid = inv.get_nodes_ipmi_userid(index) password = inv.get_nodes_ipmi_password(index) bmc_type = inv.get_nodes_bmc_type(index) cred_list[ipv4] = (userid, password, bmc_type) return cred_list if isinstance(clients, list) or not clients: log.debug('Retrieving IPMI address list from inventory') cred_list = _get_cred_list(clients) else: # insure cred info in tuple cred_list = {} for client in clients: cred_list[client] = tuple(clients[client]) clients_left = list(cred_list.keys()) attempt = 0 clients_left.sort() while clients_left and attempt < max_attempts: attempt += 1 if attempt > 1: print('Retrying set power {}. Attempt {} of {}'.format( state, attempt, max_attempts)) print('Clients remaining: {}'.format(clients_left)) clients_set = [] bmc_dict = {} for client in clients_left: for i in range(3): log.debug(f'Attempting login to BMC: {client}') tmp = _bmc.Bmc(client, *cred_list[client]) if tmp.is_connected(): bmc_dict[client] = tmp break else: log.debug( f'Failed BMC login attempt {i + 1} BMC: {client}') if i > 0: log.info(f'BMC login attempt {i + 1} BMC: {client}') if attempt == max_attempts and i == 2: log.error(f'Failed BMC login. BMC: {client}') time.sleep(1) del tmp for client in clients_left: if client in bmc_dict: log.debug(f'Setting power state to {state}. ' f'Device: {client}') status = bmc_dict[client].chassis_power(state, wait) if status: if attempt in [2, 4, 8]: print(f'{client} - Power status: {status}') # Allow delay between turn on to limit power surge if state == 'on': time.sleep(0.5) elif attempt == max_attempts: log.error(f'Failed attempt {attempt} set power {state} ' f'for node {client}') time.sleep(wait + attempt) for client in clients_left: if client in bmc_dict: status = bmc_dict[client].chassis_power('status') if status: if attempt in [2, 4, 8]: print(f'{client} - Power status: {status}, ' f'required state: {state}') if status == state: log.debug( f'Successfully set power {state} for node {client}' ) clients_set += [client] elif attempt == max_attempts: log.error(f'Failed attempt {attempt} get power {state} ' f'for node {client}') for client in clients_set: clients_left.remove(client) if attempt == max_attempts and clients_left: log.error(f'Failed to power {state} some clients') log.error(f'Clients left: {clients_left}') del bmc_dict log.info('Powered {} {} of {} client devices.'.format( state, len(cred_list) - len(clients_left), len(cred_list))) if state == 'off': print('Pausing 60 sec for client power off') time.sleep(60) if clients_left: return False return True
def inv_set_ipmi_pxe_ip(config_path): """Configure DHCP IP reservations for IPMI and PXE interfaces IP addresses are assigned sequentially within the appropriate client networks starting with the DHCP pool start offset defined in 'lib.genesis'. Raises: UserException: - No IPMI or PXE client networks defined within the 'config.yml' - Unable to connect to BMC at new IPMI IP address """ log = logger.getlogger() cfg = Config(config_path) inv = Inventory(cfg_file=config_path) ipmiNetwork = None pxeNetwork = None nodes_list = [] # All nodes should be powered off before starting set_power_clients('off', config_path, wait=POWER_WAIT) # Create IPManager object for IPMI and/or PXE networks start_offset = gen.get_dhcp_pool_start() for index, netw_type in enumerate(cfg.yield_depl_netw_client_type()): ip = cfg.get_depl_netw_client_cont_ip(index) netmask = cfg.get_depl_netw_client_netmask(index) if netw_type == 'ipmi': ipmiNetwork = IPManager(IPNetwork(ip + '/' + netmask), start_offset) elif netw_type == 'pxe': pxeNetwork = IPManager(IPNetwork(ip + '/' + netmask), start_offset) # If only one network is defined use the same IPManager for both if ipmiNetwork is None and pxeNetwork is not None: ipmiNetwork = pxeNetwork elif ipmiNetwork is not None and pxeNetwork is None: pxeNetwork = ipmiNetwork elif ipmiNetwork is None and pxeNetwork is None: raise UserException('No IPMI or PXE client network found') # Modify IP addresses for each node dhcp_lease_time = cfg.get_globals_dhcp_lease_time() for index, hostname in enumerate(inv.yield_nodes_hostname()): # IPMI reservations are written directly to the dnsmasq template ipmi_ipaddr = inv.get_nodes_ipmi_ipaddr(0, index) ipmi_mac = inv.get_nodes_ipmi_mac(0, index) ipmi_new_ipaddr = ipmiNetwork.get_next_ip() util.remove_line(DNSMASQ_TEMPLATE, "^dhcp-host=" + ipmi_mac + ".*") util.append_line( DNSMASQ_TEMPLATE, 'dhcp-host=%s,%s-bmc,%s,%s\n' % (ipmi_mac, hostname, ipmi_new_ipaddr, dhcp_lease_time)) _adjust_dhcp_pool(ipmiNetwork.network, ipmiNetwork.get_next_ip(reserve=False), dhcp_lease_time) # PXE reservations are handled by Cobbler pxe_ipaddr = inv.get_nodes_pxe_ipaddr(0, index) pxe_mac = inv.get_nodes_pxe_mac(0, index) pxe_new_ipaddr = pxeNetwork.get_next_ip() log.info('Modifying Inventory PXE IP - Node: %s MAC: %s ' 'Original IP: %s New IP: %s' % (hostname, pxe_mac, pxe_ipaddr, pxe_new_ipaddr)) inv.set_nodes_pxe_ipaddr(0, index, pxe_new_ipaddr) _adjust_dhcp_pool(pxeNetwork.network, pxeNetwork.get_next_ip(reserve=False), dhcp_lease_time) # Run Cobbler sync to process DNSMASQ template cobbler_server = xmlrpc.client.Server("http://127.0.0.1/cobbler_api") token = cobbler_server.login(COBBLER_USER, COBBLER_PASS) cobbler_server.sync(token) log.debug("Running Cobbler sync") # Save info to verify connection come back up ipmi_userid = inv.get_nodes_ipmi_userid(index) ipmi_password = inv.get_nodes_ipmi_password(index) bmc_type = inv.get_nodes_bmc_type(index) # No need to reset and check if the IP does not change if ipmi_new_ipaddr != ipmi_ipaddr: nodes_list.append({ 'hostname': hostname, 'index': index, 'ipmi_userid': ipmi_userid, 'ipmi_password': ipmi_password, 'ipmi_new_ipaddr': ipmi_new_ipaddr, 'ipmi_ipaddr': ipmi_ipaddr, 'ipmi_mac': ipmi_mac, 'bmc_type': bmc_type }) # Issue MC cold reset to force refresh of IPMI interfaces for node in nodes_list: ipmi_userid = node['ipmi_userid'] ipmi_password = node['ipmi_password'] ipmi_ipaddr = node['ipmi_ipaddr'] bmc_type = node['bmc_type'] bmc = _bmc.Bmc(ipmi_ipaddr, ipmi_userid, ipmi_password, bmc_type) if bmc.is_connected(): log.debug(f'Issuing BMC Cold Reset - Node: {node["hostname"]} ' f'- IP: {ipmi_ipaddr}') if not bmc.bmc_reset('cold'): log.error( f'Failed attempting BMC reset on {node["ipmi_ipaddr"]}') bmc.logout() log.info('Pausing 1 minute for BMCs to begin reset') sleep(60) # Check connections for set amount of time end_time = time() + WAIT_TIME while time() < end_time and len(nodes_list) > 0: print(f'\rTimeout count down: {int(end_time - time())} ', end='') sys.stdout.flush() success_list = [] sleep(2) for list_index, node in enumerate(nodes_list): hostname = node['hostname'] index = node['index'] ipmi_userid = node['ipmi_userid'] ipmi_password = node['ipmi_password'] ipmi_new_ipaddr = node['ipmi_new_ipaddr'] ipmi_ipaddr = node['ipmi_ipaddr'] ipmi_mac = node['ipmi_mac'] bmc_type = node['bmc_type'] # Attempt to connect to new IPMI IP address bmc = _bmc.Bmc(ipmi_new_ipaddr, ipmi_userid, ipmi_password, bmc_type) if bmc.is_connected(): if bmc.chassis_power('status') in ('on', 'off'): log.debug(f'BMC connection success - Node: {hostname} ' f'IP: {ipmi_ipaddr}') else: log.debug(f'BMC communication failed - Node: {hostname} ' f'IP: {ipmi_ipaddr}') continue log.info( f'Modifying Inventory IPMI IP - Node: {hostname} MAC: ' f'{ipmi_mac} Original IP: {ipmi_ipaddr} New IP: ' f'{ipmi_new_ipaddr}') inv.set_nodes_ipmi_ipaddr(0, index, ipmi_new_ipaddr) success_list.append(list_index) else: log.debug(f'BMC connection failed - Node: {hostname} ' f'IP: {ipmi_ipaddr}') continue # Remove nodes that connected successfully for remove_index in sorted(success_list, reverse=True): del nodes_list[remove_index] for node in nodes_list: log.error('Unable to connect to BMC at new IPMI IP address- Node: %s ' 'MAC: %s Original IP: %s New IP: %s' % (hostname, ipmi_mac, ipmi_ipaddr, ipmi_new_ipaddr)) if len(nodes_list) > 0: raise UserException('%d BMC(s) not responding after IP modification' % len(nodes_list))
def set_bootdev_clients(bootdev, persist=False, config_path=None, clients=None, max_attempts=5): log = logger.getlogger() if config_path: inv = Inventory(cfg_file=config_path) if type(persist) is not bool: persist = (persist == 'True') def _get_cred_list(client_list=None): """Returns dict with values of tuples or list. Each tuple/list has the credentials for a node (userid, password, bmc_type). If no client list is passed, all nodes are returned Args: client_list (list of str): each list item is an ipv4 address """ cred_list = {} for index, hostname in enumerate(inv.yield_nodes_hostname()): ipv4 = inv.get_nodes_ipmi_ipaddr(0, index) if client_list and ipv4 not in client_list: continue userid = inv.get_nodes_ipmi_userid(index) password = inv.get_nodes_ipmi_password(index) bmc_type = inv.get_nodes_bmc_type(index) cred_list[ipv4] = (userid, password, bmc_type) return cred_list # if client list passed, it is assumed to be pxe addresses which # are used to look up the associated bmc addresses for the node. # otherwise use the entire ipmi inventory list. This allows a # subset of nodes to have their bootdev updated during install if isinstance(clients, list): # Get corresponing ipmi addresses _clients = [] for index, hostname in enumerate(inv.yield_nodes_hostname()): ipv4_ipmi = inv.get_nodes_ipmi_ipaddr(0, index) ipv4_pxe = inv.get_nodes_pxe_ipaddr(0, index) if ipv4_pxe is not None and ipv4_pxe in clients: _clients.append(ipv4_ipmi) if not clients: log.debug('Retrieving IPMI address list from inventory') clients = inv.get_nodes_ipmi_ipaddr(0) _clients = clients[:] if isinstance(clients, list): log.debug('Retrieving client credentials from inventory') cred_list = _get_cred_list(_clients) else: # insure cred info in tuple cred_list = {} for client in clients: cred_list[client] = tuple(clients[client]) clients_left = list(cred_list.keys()) attempt = 0 clients_left.sort() while clients_left and attempt < max_attempts: attempt += 1 if attempt > 1: print('Retrying set bootdev. Attempt {} of {}'.format( attempt, max_attempts)) print('Clients remaining: {}'.format(clients_left)) clients_set = [] bmc_dict = {} for client in clients_left: for i in range(2): tmp = _bmc.Bmc(client, *cred_list[client]) if tmp.is_connected(): bmc_dict[client] = tmp break else: log.error( f'Failed BMC login attempt {i + 1}, BMC {client}') time.sleep(1) del tmp for client in clients_left: if client in bmc_dict: log.debug(f'Setting boot device to {bootdev}. ' f'Device: {client}') if bootdev in ('setup'): status = bmc_dict[client].host_boot_mode(bootdev) else: status = bmc_dict[client].host_boot_source(bootdev) log.debug(f'status1 from set bootdev: {status}') if status: if attempt in [2, 4, 8]: print( f'{client} - Boot source: {status} Required source: ' f'{bootdev}') else: log.error( f'Failed attempt {attempt} set boot source {bootdev} ' f'for node {client}') time.sleep(1 + attempt) for client in clients_left: if client in bmc_dict: if bootdev in ('setup'): status = bmc_dict[client].host_boot_mode() else: status = bmc_dict[client].host_boot_source() log.debug(f'status2 from set bootdev: {status}') if status: if attempt in [2, 4, 8]: print(f'{client} - Boot source: {bootdev}') if status == bootdev: log.debug( f'Successfully set boot source to {bootdev} for ' f'node {client}') clients_set += [client] else: log.error( f'Failed attempt {attempt} set host boot source to' f'{bootdev} for node {client}') bmc_dict[client].logout() for client in clients_set: clients_left.remove(client) if attempt == max_attempts and clients_left: log.error('Failed to set boot device for some clients') log.debug(clients_left) del bmc_dict log.info('Set boot device to {} on {} of {} client devices.'.format( bootdev, len(cred_list) - len(clients_left), len(cred_list)))
def _get_credentials(self, node_addr_list, cred_list): """ Attempts to discover bmc credentials and generate a list of all discovered nodes. For each node try all available credentials. If no credentials allow access, the node is not marked as succesful. Args: node_addr_list (list): list of ipv4 addresses for the discovered nodes. (ie those that previously fetched an address from the DHCP server. cred_list (list of lists): Each list item is a list containing the the userid, password, bmc_type and number of nodes for a node template. return: bmc access info (dict) : Values hold tuple of userid, password, bmc_type """ tot = [cred_list[x][3] for x in range(len(cred_list))] tot = sum(tot) left = tot max_attempts = 20 delay = 5 attempt = 0 timeout = 4 print() self.log.info("Discover BMC credentials and verify communications") print() nodes = {} bmc_ai = {} for node in node_addr_list: nodes[node] = False while not all([x for x in nodes.values()]) and attempt <= max_attempts: print(f'\rAttempt count: {max_attempts - attempt} ', end='') sys.stdout.flush() attempt += 1 timeout += 1 node_list = [x for x in nodes if not nodes[x]] for node in node_list: # each time through re-sort cred_list based on nodes left with # those credentials to maximize the probability # of using the correct credentials with minimum attempts cred_list.sort(key=lambda x: x[3], reverse=True) for j, creds in enumerate(cred_list): self.log.debug(f'BMC {node} - Trying userid: {creds[0]} | ' f'password: {creds[1]} | bmc type: {creds[2]}') bmc = _bmc.Bmc(node, *creds[:-1], timeout=timeout) if bmc.is_connected(): r = bmc.chassis_power('status') self.log.debug(f'Chassis power status: {r}') if r: nodes[node] = True time.sleep(1) self.log.debug(f'Node {node} is powered {r}') bmc_ai[node] = tuple(cred_list[j][:-1]) cred_list[j][3] -= 1 left -= 1 print(f'\r{tot - left} of {tot} nodes communicating via IPMI', end='') sys.stdout.flush() bmc.logout() else: self.log.debug(f'No power status response from node {node}') time.sleep(delay) if left != 0: self.log.error('IPMI communication successful with only ' f'{tot - left} of {tot} nodes') raise UserException('Unable to validate the following IPMI IP ' 'Addresses :' f'{[x for x in nodes if not nodes[x]]}') print('\n') return bmc_ai