def cobbler_add_systems(cfg_file=None): LOG = logger.getlogger() cobbler_user = gen.get_cobbler_user() cobbler_pass = gen.get_cobbler_pass() cobbler_server = xmlrpclib.Server("http://127.0.0.1/cobbler_api") token = cobbler_server.login(cobbler_user, cobbler_pass) inv = Inventory(cfg_file=cfg_file) for index, hostname in enumerate(inv.yield_nodes_hostname()): ipv4_ipmi = inv.get_nodes_ipmi_ipaddr(0, index) userid_ipmi = inv.get_nodes_ipmi_userid(index) password_ipmi = inv.get_nodes_ipmi_password(index) ipv4_pxe = inv.get_nodes_pxe_ipaddr(0, index) mac_pxe = inv.get_nodes_pxe_mac(0, index) cobbler_profile = gen.check_os_profile( re.sub("[.]iso", "", inv.get_nodes_os_profile(index))) raid1_enabled = False new_system_create = cobbler_server.new_system(token) cobbler_server.modify_system(new_system_create, "name", hostname, token) cobbler_server.modify_system(new_system_create, "hostname", hostname, token) cobbler_server.modify_system(new_system_create, "power_address", ipv4_ipmi, token) cobbler_server.modify_system(new_system_create, "power_user", userid_ipmi, token) cobbler_server.modify_system(new_system_create, "power_pass", password_ipmi, token) cobbler_server.modify_system(new_system_create, "power_type", "ipmilan", token) cobbler_server.modify_system(new_system_create, "profile", cobbler_profile, token) cobbler_server.modify_system( new_system_create, 'modify_interface', { "macaddress-eth0": mac_pxe, "ipaddress-eth0": ipv4_pxe, "dnsname-eth0": hostname }, token) ks_meta = "" disks = inv.get_nodes_os_install_device(index) if disks is not None: if isinstance(disks, basestring): ks_meta += 'install_disk=%s ' % disks elif isinstance(disks, list) and len(disks) == 2: ks_meta += ('install_disk=%s install_disk_2=%s ' % (disks[0], disks[1])) raid1_enabled = True else: LOG.error('%s: Invalid install_device value: %s ' 'Must be string or two item list.' % (hostname, disks)) if raid1_enabled: ks_meta += 'raid1_enabled=true ' users = inv.get_nodes_os_users(index) if users is not None: for user in users: if 'name' in user and user['name'] != 'root': ks_meta += 'default_user=%s ' % user['name'] LOG.debug("%s: Using \'%s\' as default user" % (hostname, user['name'])) if 'password' in user: ks_meta += ('passwd=%s passwdcrypted=true ' % user['password']) break else: LOG.debug("%s: No default user found" % hostname) else: LOG.debug("%s: No users defined" % hostname) if ks_meta != "": cobbler_server.modify_system(new_system_create, "ks_meta", ks_meta, token) kernel_options = inv.get_nodes_os_kernel_options(index) if kernel_options is not None: cobbler_server.modify_system(new_system_create, "kernel_options", kernel_options, token) comment = "" cobbler_server.modify_system(new_system_create, "comment", comment, token) cobbler_server.save_system(new_system_create, token) LOG.info("Cobbler Add System: name=%s, profile=%s" % (hostname, cobbler_profile)) cobbler_server.sync(token) LOG.info("Running Cobbler sync")
def ipmi_set_bootdev(bootdev, persist=False, config_path=None, client_list=None): log = logger.getlogger() inv = Inventory(cfg_file=config_path) if type(persist) is not bool: persist = (persist == 'True') # if client list passed, then use the passed client list, # otherwise use the entire inventory list. This allows a # subset of nodes to have their bootdev updated during install if not client_list: client_list = inv.get_nodes_pxe_ipaddr(0) max_attempts = 5 attempt = 0 clients_left = client_list[:] clients_left.sort() while clients_left and attempt < max_attempts: nodes = {} attempt += 1 if attempt > 1: print('Retrying set bootdev. Attempt {} of {}'.format( attempt, max_attempts)) print('Clients remaining: {}'.format(clients_left)) clients_set = [] bmc_dict = {} for index, hostname in enumerate(inv.yield_nodes_hostname()): ipv4 = inv.get_nodes_ipmi_ipaddr(0, index) ipv4_pxe = inv.get_nodes_pxe_ipaddr(0, index) if ipv4_pxe not in clients_left: continue rack_id = inv.get_nodes_rack_id(index) userid = inv.get_nodes_ipmi_userid(index) password = inv.get_nodes_ipmi_password(index) nodes[ipv4_pxe] = [rack_id, ipv4] for i in range(2): try: bmc_dict[ipv4_pxe] = ipmi_command.Command( bmc=ipv4, userid=userid, password=password) except pyghmi_exception.IpmiException as error: log.error('IPMI login try {}, address {} - {}'.format( i, ipv4, error.message)) time.sleep(1) else: break for client in clients_left: if client in bmc_dict: try: status = bmc_dict[client].set_bootdev(bootdev, persist) if attempt in [2, 4, 8]: print('{} - {}'.format(client, status)) except pyghmi_exception.IpmiException as error: msg = ('set_bootdev failed (device=%s persist=%s), ' 'Rack: %s - IP: %s, %s' % (bootdev, persist, nodes[client][0], nodes[client][1], str(error))) log.warning(msg) finally: if 'error' in status: log.error(status) time.sleep(2) for client in clients_left: if client in bmc_dict: try: status = bmc_dict[client].get_bootdev() if attempt in [2, 4, 8]: print('{} - {}'.format(client, status)) except pyghmi_exception.IpmiException as error: msg = ('get_bootdev failed - ' 'Rack: %s - IP: %s, %s' % (rack_id, ipv4, str(error))) log.error(msg) else: if status['bootdev'] == bootdev and str(status['persistent']) \ == str(persist): log.debug( 'set_bootdev successful (device=%s persist=%s) - ' 'Rack: %s - IP: %s' % (bootdev, persist, nodes[client][0], nodes[client][1])) clients_set += [client] finally: if 'error' in status: log.error(status) bmc_dict[client].ipmi_session.logout() for client in clients_set: clients_left.remove(client) if attempt == max_attempts and clients_left: log.error('Failed to set boot device for some clients') log.debug(clients_left) del bmc_dict log.info('Set boot device to {} on {} of {} client devices.'.format( bootdev, len(client_list) - len(clients_left), len(client_list)))
def inv_set_ipmi_pxe_ip(config_path): """Configure DHCP IP reservations for IPMI and PXE interfaces IP addresses are assigned sequentially within the appropriate client networks starting with the DHCP pool start offset defined in 'lib.genesis'. Raises: UserException: - No IPMI or PXE client networks defined within the 'config.yml' - Unable to connect to BMC at new IPMI IP address """ log = logger.getlogger() cfg = Config(config_path) inv = Inventory(cfg_file=config_path) ipmiNetwork = None pxeNetwork = None nodes_list = [] # All nodes should be powered off before starting set_power_clients('off', config_path, wait=POWER_WAIT) # Create IPManager object for IPMI and/or PXE networks start_offset = gen.get_dhcp_pool_start() for index, netw_type in enumerate(cfg.yield_depl_netw_client_type()): ip = cfg.get_depl_netw_client_cont_ip(index) netmask = cfg.get_depl_netw_client_netmask(index) if netw_type == 'ipmi': ipmiNetwork = IPManager(IPNetwork(ip + '/' + netmask), start_offset) elif netw_type == 'pxe': pxeNetwork = IPManager(IPNetwork(ip + '/' + netmask), start_offset) # If only one network is defined use the same IPManager for both if ipmiNetwork is None and pxeNetwork is not None: ipmiNetwork = pxeNetwork elif ipmiNetwork is not None and pxeNetwork is None: pxeNetwork = ipmiNetwork elif ipmiNetwork is None and pxeNetwork is None: raise UserException('No IPMI or PXE client network found') # Modify IP addresses for each node dhcp_lease_time = cfg.get_globals_dhcp_lease_time() for index, hostname in enumerate(inv.yield_nodes_hostname()): # IPMI reservations are written directly to the dnsmasq template ipmi_ipaddr = inv.get_nodes_ipmi_ipaddr(0, index) ipmi_mac = inv.get_nodes_ipmi_mac(0, index) ipmi_new_ipaddr = ipmiNetwork.get_next_ip() util.remove_line(DNSMASQ_TEMPLATE, "^dhcp-host=" + ipmi_mac + ".*") util.append_line( DNSMASQ_TEMPLATE, 'dhcp-host=%s,%s-bmc,%s,%s\n' % (ipmi_mac, hostname, ipmi_new_ipaddr, dhcp_lease_time)) _adjust_dhcp_pool(ipmiNetwork.network, ipmiNetwork.get_next_ip(reserve=False), dhcp_lease_time) # PXE reservations are handled by Cobbler pxe_ipaddr = inv.get_nodes_pxe_ipaddr(0, index) pxe_mac = inv.get_nodes_pxe_mac(0, index) pxe_new_ipaddr = pxeNetwork.get_next_ip() log.info('Modifying Inventory PXE IP - Node: %s MAC: %s ' 'Original IP: %s New IP: %s' % (hostname, pxe_mac, pxe_ipaddr, pxe_new_ipaddr)) inv.set_nodes_pxe_ipaddr(0, index, pxe_new_ipaddr) _adjust_dhcp_pool(pxeNetwork.network, pxeNetwork.get_next_ip(reserve=False), dhcp_lease_time) # Run Cobbler sync to process DNSMASQ template cobbler_server = xmlrpc.client.Server("http://127.0.0.1/cobbler_api") token = cobbler_server.login(COBBLER_USER, COBBLER_PASS) cobbler_server.sync(token) log.debug("Running Cobbler sync") # Save info to verify connection come back up ipmi_userid = inv.get_nodes_ipmi_userid(index) ipmi_password = inv.get_nodes_ipmi_password(index) bmc_type = inv.get_nodes_bmc_type(index) # No need to reset and check if the IP does not change if ipmi_new_ipaddr != ipmi_ipaddr: nodes_list.append({ 'hostname': hostname, 'index': index, 'ipmi_userid': ipmi_userid, 'ipmi_password': ipmi_password, 'ipmi_new_ipaddr': ipmi_new_ipaddr, 'ipmi_ipaddr': ipmi_ipaddr, 'ipmi_mac': ipmi_mac, 'bmc_type': bmc_type }) # Issue MC cold reset to force refresh of IPMI interfaces for node in nodes_list: ipmi_userid = node['ipmi_userid'] ipmi_password = node['ipmi_password'] ipmi_ipaddr = node['ipmi_ipaddr'] bmc_type = node['bmc_type'] bmc = _bmc.Bmc(ipmi_ipaddr, ipmi_userid, ipmi_password, bmc_type) if bmc.is_connected(): log.debug(f'Issuing BMC Cold Reset - Node: {node["hostname"]} ' f'- IP: {ipmi_ipaddr}') if not bmc.bmc_reset('cold'): log.error( f'Failed attempting BMC reset on {node["ipmi_ipaddr"]}') bmc.logout() log.info('Pausing 1 minute for BMCs to begin reset') sleep(60) # Check connections for set amount of time end_time = time() + WAIT_TIME while time() < end_time and len(nodes_list) > 0: print(f'\rTimeout count down: {int(end_time - time())} ', end='') sys.stdout.flush() success_list = [] sleep(2) for list_index, node in enumerate(nodes_list): hostname = node['hostname'] index = node['index'] ipmi_userid = node['ipmi_userid'] ipmi_password = node['ipmi_password'] ipmi_new_ipaddr = node['ipmi_new_ipaddr'] ipmi_ipaddr = node['ipmi_ipaddr'] ipmi_mac = node['ipmi_mac'] bmc_type = node['bmc_type'] # Attempt to connect to new IPMI IP address bmc = _bmc.Bmc(ipmi_new_ipaddr, ipmi_userid, ipmi_password, bmc_type) if bmc.is_connected(): if bmc.chassis_power('status') in ('on', 'off'): log.debug(f'BMC connection success - Node: {hostname} ' f'IP: {ipmi_ipaddr}') else: log.debug(f'BMC communication failed - Node: {hostname} ' f'IP: {ipmi_ipaddr}') continue log.info( f'Modifying Inventory IPMI IP - Node: {hostname} MAC: ' f'{ipmi_mac} Original IP: {ipmi_ipaddr} New IP: ' f'{ipmi_new_ipaddr}') inv.set_nodes_ipmi_ipaddr(0, index, ipmi_new_ipaddr) success_list.append(list_index) else: log.debug(f'BMC connection failed - Node: {hostname} ' f'IP: {ipmi_ipaddr}') continue # Remove nodes that connected successfully for remove_index in sorted(success_list, reverse=True): del nodes_list[remove_index] for node in nodes_list: log.error('Unable to connect to BMC at new IPMI IP address- Node: %s ' 'MAC: %s Original IP: %s New IP: %s' % (hostname, ipmi_mac, ipmi_ipaddr, ipmi_new_ipaddr)) if len(nodes_list) > 0: raise UserException('%d BMC(s) not responding after IP modification' % len(nodes_list))
def inv_set_ipmi_pxe_ip(config_path): """Configure DHCP IP reservations for IPMI and PXE interfaces IP addresses are assigned sequentially within the appropriate client networks starting with the DHCP pool start offset defined in 'lib.genesis'. Raises: UserException: - No IPMI or PXE client networks defined within the 'config.yml' - Unable to connect to BMC at new IPMI IP address """ log = logger.getlogger() cfg = Config(config_path) inv = Inventory(cfg_file=config_path) ipmiNetwork = None pxeNetwork = None nodes_list = [] # All nodes should be powered off before starting ipmi_set_power('off', config_path, wait=POWER_WAIT) # Create IPManager object for IPMI and/or PXE networks start_offset = gen.get_dhcp_pool_start() for index, netw_type in enumerate(cfg.yield_depl_netw_client_type()): ip = cfg.get_depl_netw_client_cont_ip(index) netmask = cfg.get_depl_netw_client_netmask(index) if netw_type == 'ipmi': ipmiNetwork = IPManager(IPNetwork(ip + '/' + netmask), start_offset) elif netw_type == 'pxe': pxeNetwork = IPManager(IPNetwork(ip + '/' + netmask), start_offset) # If only one network is defined use the same IPManager for both if ipmiNetwork is None and pxeNetwork is not None: ipmiNetwork = pxeNetwork elif ipmiNetwork is not None and pxeNetwork is None: pxeNetwork = ipmiNetwork elif ipmiNetwork is None and pxeNetwork is None: raise UserException('No IPMI or PXE client network found') # Modify IP addresses for each node dhcp_lease_time = cfg.get_globals_dhcp_lease_time() for index, hostname in enumerate(inv.yield_nodes_hostname()): # IPMI reservations are written directly to the dnsmasq template ipmi_ipaddr = inv.get_nodes_ipmi_ipaddr(0, index) ipmi_mac = inv.get_nodes_ipmi_mac(0, index) ipmi_new_ipaddr = ipmiNetwork.get_next_ip() util.remove_line(DNSMASQ_TEMPLATE, "^dhcp-host=" + ipmi_mac + ".*") util.append_line( DNSMASQ_TEMPLATE, 'dhcp-host=%s,%s-bmc,%s,%s\n' % (ipmi_mac, hostname, ipmi_new_ipaddr, dhcp_lease_time)) _adjust_dhcp_pool(ipmiNetwork.network, ipmiNetwork.get_next_ip(reserve=False), dhcp_lease_time) # PXE reservations are handled by Cobbler pxe_ipaddr = inv.get_nodes_pxe_ipaddr(0, index) pxe_mac = inv.get_nodes_pxe_mac(0, index) pxe_new_ipaddr = pxeNetwork.get_next_ip() log.info('Modifying Inventory PXE IP - Node: %s MAC: %s ' 'Original IP: %s New IP: %s' % (hostname, pxe_mac, pxe_ipaddr, pxe_new_ipaddr)) inv.set_nodes_pxe_ipaddr(0, index, pxe_new_ipaddr) _adjust_dhcp_pool(pxeNetwork.network, pxeNetwork.get_next_ip(reserve=False), dhcp_lease_time) # Run Cobbler sync to process DNSMASQ template cobbler_server = xmlrpclib.Server("http://127.0.0.1/cobbler_api") token = cobbler_server.login(COBBLER_USER, COBBLER_PASS) cobbler_server.sync(token) log.debug("Running Cobbler sync") # Save info to verify connection come back up ipmi_userid = inv.get_nodes_ipmi_userid(index) ipmi_password = inv.get_nodes_ipmi_password(index) # No need to reset and check if the IP does not change if ipmi_new_ipaddr != ipmi_ipaddr: nodes_list.append({ 'hostname': hostname, 'index': index, 'ipmi_userid': ipmi_userid, 'ipmi_password': ipmi_password, 'ipmi_new_ipaddr': ipmi_new_ipaddr, 'ipmi_ipaddr': ipmi_ipaddr, 'ipmi_mac': ipmi_mac }) # Issue MC cold reset to force refresh of IPMI interfaces for node in nodes_list: ipmi_userid = node['ipmi_userid'] ipmi_password = node['ipmi_password'] ipmi_ipaddr = node['ipmi_ipaddr'] ipmi_cmd = ipmi_command.Command(bmc=ipmi_ipaddr, userid=ipmi_userid, password=ipmi_password) ipmi_cmd.reset_bmc() del ipmi_cmd log.debug('BMC Cold Reset Issued - Node: %s - IP: %s' % (hostname, ipmi_ipaddr)) # Check connections for set amount of time end_time = time() + WAIT_TIME while time() < end_time and len(nodes_list) > 0: success_list = [] for list_index, node in enumerate(nodes_list): hostname = node['hostname'] index = node['index'] ipmi_userid = node['ipmi_userid'] ipmi_password = node['ipmi_password'] ipmi_new_ipaddr = node['ipmi_new_ipaddr'] ipmi_ipaddr = node['ipmi_ipaddr'] ipmi_mac = node['ipmi_mac'] # Attempt to connect to new IPMI IP address try: ipmi_cmd = ipmi_command.Command(bmc=ipmi_new_ipaddr, userid=ipmi_userid, password=ipmi_password) status = ipmi_cmd.get_power() except pyghmi_exception.IpmiException as error: log.debug('BMC connection failed - Node: %s IP: %s, %s ' '(Retrying for %s seconds)' % (hostname, ipmi_new_ipaddr, str(error), WAIT_TIME)) continue # If connection sucessful modify inventory if status.get('powerstate') in ['on', 'off']: log.debug('BMC connection success - Node: %s IP: %s' % (hostname, ipmi_new_ipaddr)) log.info('Modifying Inventory IPMI IP - Node: %s MAC: %s ' 'Original IP: %s New IP: %s' % (hostname, ipmi_mac, ipmi_ipaddr, ipmi_new_ipaddr)) inv.set_nodes_ipmi_ipaddr(0, index, ipmi_new_ipaddr) success_list.append(list_index) # Remove nodes that connected successfully for remove_index in sorted(success_list, reverse=True): del nodes_list[remove_index] for node in nodes_list: log.error('Unable to connect to BMC at new IPMI IP address- Node: %s ' 'MAC: %s Original IP: %s New IP: %s' % (hostname, ipmi_mac, ipmi_ipaddr, ipmi_new_ipaddr)) if len(nodes_list) > 0: raise UserException('%d BMC(s) not responding after IP modification' % len(nodes_list))
def ipmi_set_power(state, config_path=None, client_list=None, max_attempts=5, wait=6): """Set power on or off Args: state (str) : 'on' or 'off' client_list (list of str): list of IP addresses """ log = logger.getlogger() inv = Inventory(config_path) wait = float(wait) max_attempts = int(max_attempts) if not client_list: log.debug('Retrieving IPMI address list from inventory') client_list = inv.get_nodes_ipmi_ipaddr(0) clients_left = client_list[:] attempt = 0 none_cnt = 0 for client in client_list: if client is None: none_cnt += 1 log.warning('client node ip address is "None"') clients_left.remove(None) clients_left.sort() while clients_left and attempt < max_attempts: nodes = {} attempt += 1 if attempt > 1: print('Retrying set power {}. Attempt {} of {}' .format(state, attempt, max_attempts)) print('Clients remaining: {}'.format(clients_left)) clients_set = [] bmc_dict = {} for index, hostname in enumerate(inv.yield_nodes_hostname()): ipv4 = inv.get_nodes_ipmi_ipaddr(0, index) if ipv4 is None or ipv4 not in clients_left: continue rack_id = inv.get_nodes_rack_id(index) userid = inv.get_nodes_ipmi_userid(index) password = inv.get_nodes_ipmi_password(index) nodes[ipv4] = [rack_id, ipv4] for i in range(2): try: bmc_dict[ipv4] = ipmi_command.Command( bmc=ipv4, userid=userid, password=password) except pyghmi_exception.IpmiException as error: log.error('IPMI login attempt {}, address {}\nIPMI error' 'message: {}'.format(i, ipv4, error.message)) time.sleep(1) else: break for client in clients_left: if client in bmc_dict: try: log.debug('Setting power state to {}. Device: {}' .format(state, client)) status = bmc_dict[client].set_power(state, wait) if attempt in [2, 4, 8]: print('{} - {}'.format(client, status)) except pyghmi_exception.IpmiException as error: msg = ('set_power failed Rack: %s - IP: %s, \n%s' % (nodes[client][0], nodes[client][1], str(error))) log.error(msg) else: # Allow delay between turn on to limit power surge if state == 'on': time.sleep(0.5) finally: if 'error' in status: log.error(status) time.sleep(wait + attempt) for client in clients_left: if client in bmc_dict: try: status = bmc_dict[client].get_power() if attempt in [2, 4, 8]: print('{} - {}'.format(client, status)) except pyghmi_exception.IpmiException as error: msg = ('get_power failed - Rack: %s - IP: %s, %s' % (rack_id, ipv4, str(error))) log.error(msg) else: if status['powerstate'] == state: log.debug('set_power successful Rack: %s - IP: %s' % (nodes[client][0], nodes[client][1])) clients_set += [client] finally: if 'error' in status: log.error(status) bmc_dict[client].ipmi_session.logout() for client in clients_set: clients_left.remove(client) if attempt == max_attempts and clients_left: log.error('Failed to power {} some clients'.format(state)) log.error(clients_left) del bmc_dict log.info('Powered {} {} of {} client devices.' .format(state, len(client_list) - (len(clients_left) + none_cnt), len(client_list))) if state == 'off': print('Pausing 60 sec for client power off') time.sleep(60) if clients_left: return False return True