def cobbler_add_systems(cfg_file=None):
LOG = logger.getlogger()
cobbler_user = gen.get_cobbler_user()
cobbler_pass = gen.get_cobbler_pass()
cobbler_server = xmlrpclib.Server("http://127.0.0.1/cobbler_api")
token = cobbler_server.login(cobbler_user, cobbler_pass)
inv = Inventory(cfg_file=cfg_file)
for index, hostname in enumerate(inv.yield_nodes_hostname()):
ipv4_ipmi = inv.get_nodes_ipmi_ipaddr(0, index)
userid_ipmi = inv.get_nodes_ipmi_userid(index)
password_ipmi = inv.get_nodes_ipmi_password(index)
ipv4_pxe = inv.get_nodes_pxe_ipaddr(0, index)
mac_pxe = inv.get_nodes_pxe_mac(0, index)
cobbler_profile = gen.check_os_profile(
re.sub("[.]iso", "", inv.get_nodes_os_profile(index)))
raid1_enabled = False
new_system_create = cobbler_server.new_system(token)
cobbler_server.modify_system(new_system_create, "name", hostname,
token)
cobbler_server.modify_system(new_system_create, "hostname", hostname,
token)
cobbler_server.modify_system(new_system_create, "power_address",
ipv4_ipmi, token)
cobbler_server.modify_system(new_system_create, "power_user",
userid_ipmi, token)
cobbler_server.modify_system(new_system_create, "power_pass",
password_ipmi, token)
cobbler_server.modify_system(new_system_create, "power_type",
"ipmilan", token)
cobbler_server.modify_system(new_system_create, "profile",
cobbler_profile, token)
cobbler_server.modify_system(
new_system_create, 'modify_interface', {
"macaddress-eth0": mac_pxe,
"ipaddress-eth0": ipv4_pxe,
"dnsname-eth0": hostname
}, token)
ks_meta = ""
disks = inv.get_nodes_os_install_device(index)
if disks is not None:
if isinstance(disks, basestring):
ks_meta += 'install_disk=%s ' % disks
elif isinstance(disks, list) and len(disks) == 2:
ks_meta += ('install_disk=%s install_disk_2=%s ' %
(disks[0], disks[1]))
raid1_enabled = True
else:
LOG.error('%s: Invalid install_device value: %s '
'Must be string or two item list.' %
(hostname, disks))
if raid1_enabled:
ks_meta += 'raid1_enabled=true '
users = inv.get_nodes_os_users(index)
if users is not None:
for user in users:
if 'name' in user and user['name'] != 'root':
ks_meta += 'default_user=%s ' % user['name']
LOG.debug("%s: Using \'%s\' as default user" %
(hostname, user['name']))
if 'password' in user:
ks_meta += ('passwd=%s passwdcrypted=true ' %
user['password'])
break
else:
LOG.debug("%s: No default user found" % hostname)
else:
LOG.debug("%s: No users defined" % hostname)
if ks_meta != "":
cobbler_server.modify_system(new_system_create, "ks_meta", ks_meta,
token)
kernel_options = inv.get_nodes_os_kernel_options(index)
if kernel_options is not None:
cobbler_server.modify_system(new_system_create, "kernel_options",
kernel_options, token)
comment = ""
cobbler_server.modify_system(new_system_create, "comment", comment,
token)
cobbler_server.save_system(new_system_create, token)
LOG.info("Cobbler Add System: name=%s, profile=%s" %
(hostname, cobbler_profile))
cobbler_server.sync(token)
LOG.info("Running Cobbler sync")
def ipmi_set_bootdev(bootdev,
persist=False,
config_path=None,
client_list=None):
log = logger.getlogger()
inv = Inventory(cfg_file=config_path)
if type(persist) is not bool:
persist = (persist == 'True')
# if client list passed, then use the passed client list,
# otherwise use the entire inventory list. This allows a
# subset of nodes to have their bootdev updated during install
if not client_list:
client_list = inv.get_nodes_pxe_ipaddr(0)
max_attempts = 5
attempt = 0
clients_left = client_list[:]
clients_left.sort()
while clients_left and attempt < max_attempts:
nodes = {}
attempt += 1
if attempt > 1:
print('Retrying set bootdev. Attempt {} of {}'.format(
attempt, max_attempts))
print('Clients remaining: {}'.format(clients_left))
clients_set = []
bmc_dict = {}
for index, hostname in enumerate(inv.yield_nodes_hostname()):
ipv4 = inv.get_nodes_ipmi_ipaddr(0, index)
ipv4_pxe = inv.get_nodes_pxe_ipaddr(0, index)
if ipv4_pxe not in clients_left:
continue
rack_id = inv.get_nodes_rack_id(index)
userid = inv.get_nodes_ipmi_userid(index)
password = inv.get_nodes_ipmi_password(index)
nodes[ipv4_pxe] = [rack_id, ipv4]
for i in range(2):
try:
bmc_dict[ipv4_pxe] = ipmi_command.Command(
bmc=ipv4, userid=userid, password=password)
except pyghmi_exception.IpmiException as error:
log.error('IPMI login try {}, address {} - {}'.format(
i, ipv4, error.message))
time.sleep(1)
else:
break
for client in clients_left:
if client in bmc_dict:
try:
status = bmc_dict[client].set_bootdev(bootdev, persist)
if attempt in [2, 4, 8]:
print('{} - {}'.format(client, status))
except pyghmi_exception.IpmiException as error:
msg = ('set_bootdev failed (device=%s persist=%s), '
'Rack: %s - IP: %s, %s' %
(bootdev, persist, nodes[client][0],
nodes[client][1], str(error)))
log.warning(msg)
finally:
if 'error' in status:
log.error(status)
time.sleep(2)
for client in clients_left:
if client in bmc_dict:
try:
status = bmc_dict[client].get_bootdev()
if attempt in [2, 4, 8]:
print('{} - {}'.format(client, status))
except pyghmi_exception.IpmiException as error:
msg = ('get_bootdev failed - '
'Rack: %s - IP: %s, %s' %
(rack_id, ipv4, str(error)))
log.error(msg)
else:
if status['bootdev'] == bootdev and str(status['persistent']) \
== str(persist):
log.debug(
'set_bootdev successful (device=%s persist=%s) - '
'Rack: %s - IP: %s' %
(bootdev, persist, nodes[client][0],
nodes[client][1]))
clients_set += [client]
finally:
if 'error' in status:
log.error(status)
bmc_dict[client].ipmi_session.logout()
for client in clients_set:
clients_left.remove(client)
if attempt == max_attempts and clients_left:
log.error('Failed to set boot device for some clients')
log.debug(clients_left)
del bmc_dict
log.info('Set boot device to {} on {} of {} client devices.'.format(
bootdev,
len(client_list) - len(clients_left), len(client_list)))
def inv_set_ipmi_pxe_ip(config_path):
"""Configure DHCP IP reservations for IPMI and PXE interfaces
IP addresses are assigned sequentially within the appropriate
client networks starting with the DHCP pool start offset defined
in 'lib.genesis'.
Raises:
UserException: - No IPMI or PXE client networks defined within
the 'config.yml'
- Unable to connect to BMC at new IPMI IP address
"""
log = logger.getlogger()
cfg = Config(config_path)
inv = Inventory(cfg_file=config_path)
ipmiNetwork = None
pxeNetwork = None
nodes_list = []
# All nodes should be powered off before starting
set_power_clients('off', config_path, wait=POWER_WAIT)
# Create IPManager object for IPMI and/or PXE networks
start_offset = gen.get_dhcp_pool_start()
for index, netw_type in enumerate(cfg.yield_depl_netw_client_type()):
ip = cfg.get_depl_netw_client_cont_ip(index)
netmask = cfg.get_depl_netw_client_netmask(index)
if netw_type == 'ipmi':
ipmiNetwork = IPManager(IPNetwork(ip + '/' + netmask),
start_offset)
elif netw_type == 'pxe':
pxeNetwork = IPManager(IPNetwork(ip + '/' + netmask), start_offset)
# If only one network is defined use the same IPManager for both
if ipmiNetwork is None and pxeNetwork is not None:
ipmiNetwork = pxeNetwork
elif ipmiNetwork is not None and pxeNetwork is None:
pxeNetwork = ipmiNetwork
elif ipmiNetwork is None and pxeNetwork is None:
raise UserException('No IPMI or PXE client network found')
# Modify IP addresses for each node
dhcp_lease_time = cfg.get_globals_dhcp_lease_time()
for index, hostname in enumerate(inv.yield_nodes_hostname()):
# IPMI reservations are written directly to the dnsmasq template
ipmi_ipaddr = inv.get_nodes_ipmi_ipaddr(0, index)
ipmi_mac = inv.get_nodes_ipmi_mac(0, index)
ipmi_new_ipaddr = ipmiNetwork.get_next_ip()
util.remove_line(DNSMASQ_TEMPLATE, "^dhcp-host=" + ipmi_mac + ".*")
util.append_line(
DNSMASQ_TEMPLATE, 'dhcp-host=%s,%s-bmc,%s,%s\n' %
(ipmi_mac, hostname, ipmi_new_ipaddr, dhcp_lease_time))
_adjust_dhcp_pool(ipmiNetwork.network,
ipmiNetwork.get_next_ip(reserve=False),
dhcp_lease_time)
# PXE reservations are handled by Cobbler
pxe_ipaddr = inv.get_nodes_pxe_ipaddr(0, index)
pxe_mac = inv.get_nodes_pxe_mac(0, index)
pxe_new_ipaddr = pxeNetwork.get_next_ip()
log.info('Modifying Inventory PXE IP - Node: %s MAC: %s '
'Original IP: %s New IP: %s' %
(hostname, pxe_mac, pxe_ipaddr, pxe_new_ipaddr))
inv.set_nodes_pxe_ipaddr(0, index, pxe_new_ipaddr)
_adjust_dhcp_pool(pxeNetwork.network,
pxeNetwork.get_next_ip(reserve=False),
dhcp_lease_time)
# Run Cobbler sync to process DNSMASQ template
cobbler_server = xmlrpc.client.Server("http://127.0.0.1/cobbler_api")
token = cobbler_server.login(COBBLER_USER, COBBLER_PASS)
cobbler_server.sync(token)
log.debug("Running Cobbler sync")
# Save info to verify connection come back up
ipmi_userid = inv.get_nodes_ipmi_userid(index)
ipmi_password = inv.get_nodes_ipmi_password(index)
bmc_type = inv.get_nodes_bmc_type(index)
# No need to reset and check if the IP does not change
if ipmi_new_ipaddr != ipmi_ipaddr:
nodes_list.append({
'hostname': hostname,
'index': index,
'ipmi_userid': ipmi_userid,
'ipmi_password': ipmi_password,
'ipmi_new_ipaddr': ipmi_new_ipaddr,
'ipmi_ipaddr': ipmi_ipaddr,
'ipmi_mac': ipmi_mac,
'bmc_type': bmc_type
})
# Issue MC cold reset to force refresh of IPMI interfaces
for node in nodes_list:
ipmi_userid = node['ipmi_userid']
ipmi_password = node['ipmi_password']
ipmi_ipaddr = node['ipmi_ipaddr']
bmc_type = node['bmc_type']
bmc = _bmc.Bmc(ipmi_ipaddr, ipmi_userid, ipmi_password, bmc_type)
if bmc.is_connected():
log.debug(f'Issuing BMC Cold Reset - Node: {node["hostname"]} '
f'- IP: {ipmi_ipaddr}')
if not bmc.bmc_reset('cold'):
log.error(
f'Failed attempting BMC reset on {node["ipmi_ipaddr"]}')
bmc.logout()
log.info('Pausing 1 minute for BMCs to begin reset')
sleep(60)
# Check connections for set amount of time
end_time = time() + WAIT_TIME
while time() < end_time and len(nodes_list) > 0:
print(f'\rTimeout count down: {int(end_time - time())} ', end='')
sys.stdout.flush()
success_list = []
sleep(2)
for list_index, node in enumerate(nodes_list):
hostname = node['hostname']
index = node['index']
ipmi_userid = node['ipmi_userid']
ipmi_password = node['ipmi_password']
ipmi_new_ipaddr = node['ipmi_new_ipaddr']
ipmi_ipaddr = node['ipmi_ipaddr']
ipmi_mac = node['ipmi_mac']
bmc_type = node['bmc_type']
# Attempt to connect to new IPMI IP address
bmc = _bmc.Bmc(ipmi_new_ipaddr, ipmi_userid, ipmi_password,
bmc_type)
if bmc.is_connected():
if bmc.chassis_power('status') in ('on', 'off'):
log.debug(f'BMC connection success - Node: {hostname} '
f'IP: {ipmi_ipaddr}')
else:
log.debug(f'BMC communication failed - Node: {hostname} '
f'IP: {ipmi_ipaddr}')
continue
log.info(
f'Modifying Inventory IPMI IP - Node: {hostname} MAC: '
f'{ipmi_mac} Original IP: {ipmi_ipaddr} New IP: '
f'{ipmi_new_ipaddr}')
inv.set_nodes_ipmi_ipaddr(0, index, ipmi_new_ipaddr)
success_list.append(list_index)
else:
log.debug(f'BMC connection failed - Node: {hostname} '
f'IP: {ipmi_ipaddr}')
continue
# Remove nodes that connected successfully
for remove_index in sorted(success_list, reverse=True):
del nodes_list[remove_index]
for node in nodes_list:
log.error('Unable to connect to BMC at new IPMI IP address- Node: %s '
'MAC: %s Original IP: %s New IP: %s' %
(hostname, ipmi_mac, ipmi_ipaddr, ipmi_new_ipaddr))
if len(nodes_list) > 0:
raise UserException('%d BMC(s) not responding after IP modification' %
len(nodes_list))
def inv_set_ipmi_pxe_ip(config_path):
"""Configure DHCP IP reservations for IPMI and PXE interfaces
IP addresses are assigned sequentially within the appropriate
client networks starting with the DHCP pool start offset defined
in 'lib.genesis'.
Raises:
UserException: - No IPMI or PXE client networks defined within
the 'config.yml'
- Unable to connect to BMC at new IPMI IP address
"""
log = logger.getlogger()
cfg = Config(config_path)
inv = Inventory(cfg_file=config_path)
ipmiNetwork = None
pxeNetwork = None
nodes_list = []
# All nodes should be powered off before starting
ipmi_set_power('off', config_path, wait=POWER_WAIT)
# Create IPManager object for IPMI and/or PXE networks
start_offset = gen.get_dhcp_pool_start()
for index, netw_type in enumerate(cfg.yield_depl_netw_client_type()):
ip = cfg.get_depl_netw_client_cont_ip(index)
netmask = cfg.get_depl_netw_client_netmask(index)
if netw_type == 'ipmi':
ipmiNetwork = IPManager(IPNetwork(ip + '/' + netmask),
start_offset)
elif netw_type == 'pxe':
pxeNetwork = IPManager(IPNetwork(ip + '/' + netmask), start_offset)
# If only one network is defined use the same IPManager for both
if ipmiNetwork is None and pxeNetwork is not None:
ipmiNetwork = pxeNetwork
elif ipmiNetwork is not None and pxeNetwork is None:
pxeNetwork = ipmiNetwork
elif ipmiNetwork is None and pxeNetwork is None:
raise UserException('No IPMI or PXE client network found')
# Modify IP addresses for each node
dhcp_lease_time = cfg.get_globals_dhcp_lease_time()
for index, hostname in enumerate(inv.yield_nodes_hostname()):
# IPMI reservations are written directly to the dnsmasq template
ipmi_ipaddr = inv.get_nodes_ipmi_ipaddr(0, index)
ipmi_mac = inv.get_nodes_ipmi_mac(0, index)
ipmi_new_ipaddr = ipmiNetwork.get_next_ip()
util.remove_line(DNSMASQ_TEMPLATE, "^dhcp-host=" + ipmi_mac + ".*")
util.append_line(
DNSMASQ_TEMPLATE, 'dhcp-host=%s,%s-bmc,%s,%s\n' %
(ipmi_mac, hostname, ipmi_new_ipaddr, dhcp_lease_time))
_adjust_dhcp_pool(ipmiNetwork.network,
ipmiNetwork.get_next_ip(reserve=False),
dhcp_lease_time)
# PXE reservations are handled by Cobbler
pxe_ipaddr = inv.get_nodes_pxe_ipaddr(0, index)
pxe_mac = inv.get_nodes_pxe_mac(0, index)
pxe_new_ipaddr = pxeNetwork.get_next_ip()
log.info('Modifying Inventory PXE IP - Node: %s MAC: %s '
'Original IP: %s New IP: %s' %
(hostname, pxe_mac, pxe_ipaddr, pxe_new_ipaddr))
inv.set_nodes_pxe_ipaddr(0, index, pxe_new_ipaddr)
_adjust_dhcp_pool(pxeNetwork.network,
pxeNetwork.get_next_ip(reserve=False),
dhcp_lease_time)
# Run Cobbler sync to process DNSMASQ template
cobbler_server = xmlrpclib.Server("http://127.0.0.1/cobbler_api")
token = cobbler_server.login(COBBLER_USER, COBBLER_PASS)
cobbler_server.sync(token)
log.debug("Running Cobbler sync")
# Save info to verify connection come back up
ipmi_userid = inv.get_nodes_ipmi_userid(index)
ipmi_password = inv.get_nodes_ipmi_password(index)
# No need to reset and check if the IP does not change
if ipmi_new_ipaddr != ipmi_ipaddr:
nodes_list.append({
'hostname': hostname,
'index': index,
'ipmi_userid': ipmi_userid,
'ipmi_password': ipmi_password,
'ipmi_new_ipaddr': ipmi_new_ipaddr,
'ipmi_ipaddr': ipmi_ipaddr,
'ipmi_mac': ipmi_mac
})
# Issue MC cold reset to force refresh of IPMI interfaces
for node in nodes_list:
ipmi_userid = node['ipmi_userid']
ipmi_password = node['ipmi_password']
ipmi_ipaddr = node['ipmi_ipaddr']
ipmi_cmd = ipmi_command.Command(bmc=ipmi_ipaddr,
userid=ipmi_userid,
password=ipmi_password)
ipmi_cmd.reset_bmc()
del ipmi_cmd
log.debug('BMC Cold Reset Issued - Node: %s - IP: %s' %
(hostname, ipmi_ipaddr))
# Check connections for set amount of time
end_time = time() + WAIT_TIME
while time() < end_time and len(nodes_list) > 0:
success_list = []
for list_index, node in enumerate(nodes_list):
hostname = node['hostname']
index = node['index']
ipmi_userid = node['ipmi_userid']
ipmi_password = node['ipmi_password']
ipmi_new_ipaddr = node['ipmi_new_ipaddr']
ipmi_ipaddr = node['ipmi_ipaddr']
ipmi_mac = node['ipmi_mac']
# Attempt to connect to new IPMI IP address
try:
ipmi_cmd = ipmi_command.Command(bmc=ipmi_new_ipaddr,
userid=ipmi_userid,
password=ipmi_password)
status = ipmi_cmd.get_power()
except pyghmi_exception.IpmiException as error:
log.debug('BMC connection failed - Node: %s IP: %s, %s '
'(Retrying for %s seconds)' %
(hostname, ipmi_new_ipaddr, str(error), WAIT_TIME))
continue
# If connection sucessful modify inventory
if status.get('powerstate') in ['on', 'off']:
log.debug('BMC connection success - Node: %s IP: %s' %
(hostname, ipmi_new_ipaddr))
log.info('Modifying Inventory IPMI IP - Node: %s MAC: %s '
'Original IP: %s New IP: %s' %
(hostname, ipmi_mac, ipmi_ipaddr, ipmi_new_ipaddr))
inv.set_nodes_ipmi_ipaddr(0, index, ipmi_new_ipaddr)
success_list.append(list_index)
# Remove nodes that connected successfully
for remove_index in sorted(success_list, reverse=True):
del nodes_list[remove_index]
for node in nodes_list:
log.error('Unable to connect to BMC at new IPMI IP address- Node: %s '
'MAC: %s Original IP: %s New IP: %s' %
(hostname, ipmi_mac, ipmi_ipaddr, ipmi_new_ipaddr))
if len(nodes_list) > 0:
raise UserException('%d BMC(s) not responding after IP modification' %
len(nodes_list))
def ipmi_set_power(state, config_path=None, client_list=None, max_attempts=5,
wait=6):
"""Set power on or off
Args:
state (str) : 'on' or 'off'
client_list (list of str): list of IP addresses
"""
log = logger.getlogger()
inv = Inventory(config_path)
wait = float(wait)
max_attempts = int(max_attempts)
if not client_list:
log.debug('Retrieving IPMI address list from inventory')
client_list = inv.get_nodes_ipmi_ipaddr(0)
clients_left = client_list[:]
attempt = 0
none_cnt = 0
for client in client_list:
if client is None:
none_cnt += 1
log.warning('client node ip address is "None"')
clients_left.remove(None)
clients_left.sort()
while clients_left and attempt < max_attempts:
nodes = {}
attempt += 1
if attempt > 1:
print('Retrying set power {}. Attempt {} of {}'
.format(state, attempt, max_attempts))
print('Clients remaining: {}'.format(clients_left))
clients_set = []
bmc_dict = {}
for index, hostname in enumerate(inv.yield_nodes_hostname()):
ipv4 = inv.get_nodes_ipmi_ipaddr(0, index)
if ipv4 is None or ipv4 not in clients_left:
continue
rack_id = inv.get_nodes_rack_id(index)
userid = inv.get_nodes_ipmi_userid(index)
password = inv.get_nodes_ipmi_password(index)
nodes[ipv4] = [rack_id, ipv4]
for i in range(2):
try:
bmc_dict[ipv4] = ipmi_command.Command(
bmc=ipv4,
userid=userid,
password=password)
except pyghmi_exception.IpmiException as error:
log.error('IPMI login attempt {}, address {}\nIPMI error'
'message: {}'.format(i, ipv4, error.message))
time.sleep(1)
else:
break
for client in clients_left:
if client in bmc_dict:
try:
log.debug('Setting power state to {}. Device: {}'
.format(state, client))
status = bmc_dict[client].set_power(state, wait)
if attempt in [2, 4, 8]:
print('{} - {}'.format(client, status))
except pyghmi_exception.IpmiException as error:
msg = ('set_power failed Rack: %s - IP: %s, \n%s' %
(nodes[client][0], nodes[client][1], str(error)))
log.error(msg)
else:
# Allow delay between turn on to limit power surge
if state == 'on':
time.sleep(0.5)
finally:
if 'error' in status:
log.error(status)
time.sleep(wait + attempt)
for client in clients_left:
if client in bmc_dict:
try:
status = bmc_dict[client].get_power()
if attempt in [2, 4, 8]:
print('{} - {}'.format(client, status))
except pyghmi_exception.IpmiException as error:
msg = ('get_power failed - Rack: %s - IP: %s, %s' %
(rack_id, ipv4, str(error)))
log.error(msg)
else:
if status['powerstate'] == state:
log.debug('set_power successful Rack: %s - IP: %s' %
(nodes[client][0], nodes[client][1]))
clients_set += [client]
finally:
if 'error' in status:
log.error(status)
bmc_dict[client].ipmi_session.logout()
for client in clients_set:
clients_left.remove(client)
if attempt == max_attempts and clients_left:
log.error('Failed to power {} some clients'.format(state))
log.error(clients_left)
del bmc_dict
log.info('Powered {} {} of {} client devices.'
.format(state, len(client_list) - (len(clients_left) + none_cnt),
len(client_list)))
if state == 'off':
print('Pausing 60 sec for client power off')
time.sleep(60)
if clients_left:
return False
return True