def set_power_clients(state,
config_path=None,
clients=None,
max_attempts=5,
wait=6):
"""Set power on or off for multiple clients. If a list of ip addresses
are given or no clients given then the credentials are looked up in an
inventory file. If clients is a dictionary, then the credentials are
taken from the dictionary values.
Args:
state (str) : 'on' or 'off'
config_path (str): path to a config file
clients (dict or list of str): list of IP addresses or
dict of ip addresses with values of credentials as tuple
ie {'192.168.1.2': ('user', 'password', 'bmc_type')}
"""
log = logger.getlogger()
if config_path:
inv = Inventory(config_path)
wait = float(wait)
max_attempts = int(max_attempts)
def _get_cred_list(client_list=None):
"""Returns dict with values of tuples. Each tuple has the credentials
for a node (userid, password, bmc_type). If no client list
is passed, all nodes are returned
Args:
client_list (list of str): each list item is an ipv4 address
"""
cred_list = {}
for index, hostname in enumerate(inv.yield_nodes_hostname()):
ipv4 = inv.get_nodes_ipmi_ipaddr(0, index)
if client_list and ipv4 not in client_list:
continue
# rack_id = inv.get_nodes_rack_id(index)
userid = inv.get_nodes_ipmi_userid(index)
password = inv.get_nodes_ipmi_password(index)
bmc_type = inv.get_nodes_bmc_type(index)
cred_list[ipv4] = (userid, password, bmc_type)
return cred_list
if isinstance(clients, list) or not clients:
log.debug('Retrieving IPMI address list from inventory')
cred_list = _get_cred_list(clients)
else:
# insure cred info in tuple
cred_list = {}
for client in clients:
cred_list[client] = tuple(clients[client])
clients_left = list(cred_list.keys())
attempt = 0
clients_left.sort()
while clients_left and attempt < max_attempts:
attempt += 1
if attempt > 1:
print('Retrying set power {}. Attempt {} of {}'.format(
state, attempt, max_attempts))
print('Clients remaining: {}'.format(clients_left))
clients_set = []
bmc_dict = {}
for client in clients_left:
for i in range(3):
log.debug(f'Attempting login to BMC: {client}')
tmp = _bmc.Bmc(client, *cred_list[client])
if tmp.is_connected():
bmc_dict[client] = tmp
break
else:
log.debug(
f'Failed BMC login attempt {i + 1} BMC: {client}')
if i > 0:
log.info(f'BMC login attempt {i + 1} BMC: {client}')
if attempt == max_attempts and i == 2:
log.error(f'Failed BMC login. BMC: {client}')
time.sleep(1)
del tmp
for client in clients_left:
if client in bmc_dict:
log.debug(f'Setting power state to {state}. '
f'Device: {client}')
status = bmc_dict[client].chassis_power(state, wait)
if status:
if attempt in [2, 4, 8]:
print(f'{client} - Power status: {status}')
# Allow delay between turn on to limit power surge
if state == 'on':
time.sleep(0.5)
elif attempt == max_attempts:
log.error(f'Failed attempt {attempt} set power {state} '
f'for node {client}')
time.sleep(wait + attempt)
for client in clients_left:
if client in bmc_dict:
status = bmc_dict[client].chassis_power('status')
if status:
if attempt in [2, 4, 8]:
print(f'{client} - Power status: {status}, '
f'required state: {state}')
if status == state:
log.debug(
f'Successfully set power {state} for node {client}'
)
clients_set += [client]
elif attempt == max_attempts:
log.error(f'Failed attempt {attempt} get power {state} '
f'for node {client}')
for client in clients_set:
clients_left.remove(client)
if attempt == max_attempts and clients_left:
log.error(f'Failed to power {state} some clients')
log.error(f'Clients left: {clients_left}')
del bmc_dict
log.info('Powered {} {} of {} client devices.'.format(
state,
len(cred_list) - len(clients_left), len(cred_list)))
if state == 'off':
print('Pausing 60 sec for client power off')
time.sleep(60)
if clients_left:
return False
return True
def inv_set_ipmi_pxe_ip(config_path):
"""Configure DHCP IP reservations for IPMI and PXE interfaces
IP addresses are assigned sequentially within the appropriate
client networks starting with the DHCP pool start offset defined
in 'lib.genesis'.
Raises:
UserException: - No IPMI or PXE client networks defined within
the 'config.yml'
- Unable to connect to BMC at new IPMI IP address
"""
log = logger.getlogger()
cfg = Config(config_path)
inv = Inventory(cfg_file=config_path)
ipmiNetwork = None
pxeNetwork = None
nodes_list = []
# All nodes should be powered off before starting
set_power_clients('off', config_path, wait=POWER_WAIT)
# Create IPManager object for IPMI and/or PXE networks
start_offset = gen.get_dhcp_pool_start()
for index, netw_type in enumerate(cfg.yield_depl_netw_client_type()):
ip = cfg.get_depl_netw_client_cont_ip(index)
netmask = cfg.get_depl_netw_client_netmask(index)
if netw_type == 'ipmi':
ipmiNetwork = IPManager(IPNetwork(ip + '/' + netmask),
start_offset)
elif netw_type == 'pxe':
pxeNetwork = IPManager(IPNetwork(ip + '/' + netmask), start_offset)
# If only one network is defined use the same IPManager for both
if ipmiNetwork is None and pxeNetwork is not None:
ipmiNetwork = pxeNetwork
elif ipmiNetwork is not None and pxeNetwork is None:
pxeNetwork = ipmiNetwork
elif ipmiNetwork is None and pxeNetwork is None:
raise UserException('No IPMI or PXE client network found')
# Modify IP addresses for each node
dhcp_lease_time = cfg.get_globals_dhcp_lease_time()
for index, hostname in enumerate(inv.yield_nodes_hostname()):
# IPMI reservations are written directly to the dnsmasq template
ipmi_ipaddr = inv.get_nodes_ipmi_ipaddr(0, index)
ipmi_mac = inv.get_nodes_ipmi_mac(0, index)
ipmi_new_ipaddr = ipmiNetwork.get_next_ip()
util.remove_line(DNSMASQ_TEMPLATE, "^dhcp-host=" + ipmi_mac + ".*")
util.append_line(
DNSMASQ_TEMPLATE, 'dhcp-host=%s,%s-bmc,%s,%s\n' %
(ipmi_mac, hostname, ipmi_new_ipaddr, dhcp_lease_time))
_adjust_dhcp_pool(ipmiNetwork.network,
ipmiNetwork.get_next_ip(reserve=False),
dhcp_lease_time)
# PXE reservations are handled by Cobbler
pxe_ipaddr = inv.get_nodes_pxe_ipaddr(0, index)
pxe_mac = inv.get_nodes_pxe_mac(0, index)
pxe_new_ipaddr = pxeNetwork.get_next_ip()
log.info('Modifying Inventory PXE IP - Node: %s MAC: %s '
'Original IP: %s New IP: %s' %
(hostname, pxe_mac, pxe_ipaddr, pxe_new_ipaddr))
inv.set_nodes_pxe_ipaddr(0, index, pxe_new_ipaddr)
_adjust_dhcp_pool(pxeNetwork.network,
pxeNetwork.get_next_ip(reserve=False),
dhcp_lease_time)
# Run Cobbler sync to process DNSMASQ template
cobbler_server = xmlrpc.client.Server("http://127.0.0.1/cobbler_api")
token = cobbler_server.login(COBBLER_USER, COBBLER_PASS)
cobbler_server.sync(token)
log.debug("Running Cobbler sync")
# Save info to verify connection come back up
ipmi_userid = inv.get_nodes_ipmi_userid(index)
ipmi_password = inv.get_nodes_ipmi_password(index)
bmc_type = inv.get_nodes_bmc_type(index)
# No need to reset and check if the IP does not change
if ipmi_new_ipaddr != ipmi_ipaddr:
nodes_list.append({
'hostname': hostname,
'index': index,
'ipmi_userid': ipmi_userid,
'ipmi_password': ipmi_password,
'ipmi_new_ipaddr': ipmi_new_ipaddr,
'ipmi_ipaddr': ipmi_ipaddr,
'ipmi_mac': ipmi_mac,
'bmc_type': bmc_type
})
# Issue MC cold reset to force refresh of IPMI interfaces
for node in nodes_list:
ipmi_userid = node['ipmi_userid']
ipmi_password = node['ipmi_password']
ipmi_ipaddr = node['ipmi_ipaddr']
bmc_type = node['bmc_type']
bmc = _bmc.Bmc(ipmi_ipaddr, ipmi_userid, ipmi_password, bmc_type)
if bmc.is_connected():
log.debug(f'Issuing BMC Cold Reset - Node: {node["hostname"]} '
f'- IP: {ipmi_ipaddr}')
if not bmc.bmc_reset('cold'):
log.error(
f'Failed attempting BMC reset on {node["ipmi_ipaddr"]}')
bmc.logout()
log.info('Pausing 1 minute for BMCs to begin reset')
sleep(60)
# Check connections for set amount of time
end_time = time() + WAIT_TIME
while time() < end_time and len(nodes_list) > 0:
print(f'\rTimeout count down: {int(end_time - time())} ', end='')
sys.stdout.flush()
success_list = []
sleep(2)
for list_index, node in enumerate(nodes_list):
hostname = node['hostname']
index = node['index']
ipmi_userid = node['ipmi_userid']
ipmi_password = node['ipmi_password']
ipmi_new_ipaddr = node['ipmi_new_ipaddr']
ipmi_ipaddr = node['ipmi_ipaddr']
ipmi_mac = node['ipmi_mac']
bmc_type = node['bmc_type']
# Attempt to connect to new IPMI IP address
bmc = _bmc.Bmc(ipmi_new_ipaddr, ipmi_userid, ipmi_password,
bmc_type)
if bmc.is_connected():
if bmc.chassis_power('status') in ('on', 'off'):
log.debug(f'BMC connection success - Node: {hostname} '
f'IP: {ipmi_ipaddr}')
else:
log.debug(f'BMC communication failed - Node: {hostname} '
f'IP: {ipmi_ipaddr}')
continue
log.info(
f'Modifying Inventory IPMI IP - Node: {hostname} MAC: '
f'{ipmi_mac} Original IP: {ipmi_ipaddr} New IP: '
f'{ipmi_new_ipaddr}')
inv.set_nodes_ipmi_ipaddr(0, index, ipmi_new_ipaddr)
success_list.append(list_index)
else:
log.debug(f'BMC connection failed - Node: {hostname} '
f'IP: {ipmi_ipaddr}')
continue
# Remove nodes that connected successfully
for remove_index in sorted(success_list, reverse=True):
del nodes_list[remove_index]
for node in nodes_list:
log.error('Unable to connect to BMC at new IPMI IP address- Node: %s '
'MAC: %s Original IP: %s New IP: %s' %
(hostname, ipmi_mac, ipmi_ipaddr, ipmi_new_ipaddr))
if len(nodes_list) > 0:
raise UserException('%d BMC(s) not responding after IP modification' %
len(nodes_list))
def set_bootdev_clients(bootdev,
persist=False,
config_path=None,
clients=None,
max_attempts=5):
log = logger.getlogger()
if config_path:
inv = Inventory(cfg_file=config_path)
if type(persist) is not bool:
persist = (persist == 'True')
def _get_cred_list(client_list=None):
"""Returns dict with values of tuples or list. Each tuple/list
has the credentials for a node (userid, password, bmc_type).
If no client list is passed, all nodes are returned
Args:
client_list (list of str): each list item is an ipv4 address
"""
cred_list = {}
for index, hostname in enumerate(inv.yield_nodes_hostname()):
ipv4 = inv.get_nodes_ipmi_ipaddr(0, index)
if client_list and ipv4 not in client_list:
continue
userid = inv.get_nodes_ipmi_userid(index)
password = inv.get_nodes_ipmi_password(index)
bmc_type = inv.get_nodes_bmc_type(index)
cred_list[ipv4] = (userid, password, bmc_type)
return cred_list
# if client list passed, it is assumed to be pxe addresses which
# are used to look up the associated bmc addresses for the node.
# otherwise use the entire ipmi inventory list. This allows a
# subset of nodes to have their bootdev updated during install
if isinstance(clients, list):
# Get corresponing ipmi addresses
_clients = []
for index, hostname in enumerate(inv.yield_nodes_hostname()):
ipv4_ipmi = inv.get_nodes_ipmi_ipaddr(0, index)
ipv4_pxe = inv.get_nodes_pxe_ipaddr(0, index)
if ipv4_pxe is not None and ipv4_pxe in clients:
_clients.append(ipv4_ipmi)
if not clients:
log.debug('Retrieving IPMI address list from inventory')
clients = inv.get_nodes_ipmi_ipaddr(0)
_clients = clients[:]
if isinstance(clients, list):
log.debug('Retrieving client credentials from inventory')
cred_list = _get_cred_list(_clients)
else:
# insure cred info in tuple
cred_list = {}
for client in clients:
cred_list[client] = tuple(clients[client])
clients_left = list(cred_list.keys())
attempt = 0
clients_left.sort()
while clients_left and attempt < max_attempts:
attempt += 1
if attempt > 1:
print('Retrying set bootdev. Attempt {} of {}'.format(
attempt, max_attempts))
print('Clients remaining: {}'.format(clients_left))
clients_set = []
bmc_dict = {}
for client in clients_left:
for i in range(2):
tmp = _bmc.Bmc(client, *cred_list[client])
if tmp.is_connected():
bmc_dict[client] = tmp
break
else:
log.error(
f'Failed BMC login attempt {i + 1}, BMC {client}')
time.sleep(1)
del tmp
for client in clients_left:
if client in bmc_dict:
log.debug(f'Setting boot device to {bootdev}. '
f'Device: {client}')
if bootdev in ('setup'):
status = bmc_dict[client].host_boot_mode(bootdev)
else:
status = bmc_dict[client].host_boot_source(bootdev)
log.debug(f'status1 from set bootdev: {status}')
if status:
if attempt in [2, 4, 8]:
print(
f'{client} - Boot source: {status} Required source: '
f'{bootdev}')
else:
log.error(
f'Failed attempt {attempt} set boot source {bootdev} '
f'for node {client}')
time.sleep(1 + attempt)
for client in clients_left:
if client in bmc_dict:
if bootdev in ('setup'):
status = bmc_dict[client].host_boot_mode()
else:
status = bmc_dict[client].host_boot_source()
log.debug(f'status2 from set bootdev: {status}')
if status:
if attempt in [2, 4, 8]:
print(f'{client} - Boot source: {bootdev}')
if status == bootdev:
log.debug(
f'Successfully set boot source to {bootdev} for '
f'node {client}')
clients_set += [client]
else:
log.error(
f'Failed attempt {attempt} set host boot source to'
f'{bootdev} for node {client}')
bmc_dict[client].logout()
for client in clients_set:
clients_left.remove(client)
if attempt == max_attempts and clients_left:
log.error('Failed to set boot device for some clients')
log.debug(clients_left)
del bmc_dict
log.info('Set boot device to {} on {} of {} client devices.'.format(
bootdev,
len(cred_list) - len(clients_left), len(cred_list)))
def _get_credentials(self, node_addr_list, cred_list):
""" Attempts to discover bmc credentials and generate a list of all
discovered nodes. For each node try all available credentials. If no
credentials allow access, the node is not marked as succesful.
Args:
node_addr_list (list): list of ipv4 addresses for the discovered
nodes. (ie those that previously fetched an address from the DHCP
server.
cred_list (list of lists): Each list item is a list containing the
the userid, password, bmc_type and number of nodes for a node template.
return: bmc access info (dict) : Values hold tuple of userid, password,
bmc_type
"""
tot = [cred_list[x][3] for x in range(len(cred_list))]
tot = sum(tot)
left = tot
max_attempts = 20
delay = 5
attempt = 0
timeout = 4
print()
self.log.info("Discover BMC credentials and verify communications")
print()
nodes = {}
bmc_ai = {}
for node in node_addr_list:
nodes[node] = False
while not all([x for x in nodes.values()]) and attempt <= max_attempts:
print(f'\rAttempt count: {max_attempts - attempt} ', end='')
sys.stdout.flush()
attempt += 1
timeout += 1
node_list = [x for x in nodes if not nodes[x]]
for node in node_list:
# each time through re-sort cred_list based on nodes left with
# those credentials to maximize the probability
# of using the correct credentials with minimum attempts
cred_list.sort(key=lambda x: x[3], reverse=True)
for j, creds in enumerate(cred_list):
self.log.debug(f'BMC {node} - Trying userid: {creds[0]} | '
f'password: {creds[1]} | bmc type: {creds[2]}')
bmc = _bmc.Bmc(node, *creds[:-1], timeout=timeout)
if bmc.is_connected():
r = bmc.chassis_power('status')
self.log.debug(f'Chassis power status: {r}')
if r:
nodes[node] = True
time.sleep(1)
self.log.debug(f'Node {node} is powered {r}')
bmc_ai[node] = tuple(cred_list[j][:-1])
cred_list[j][3] -= 1
left -= 1
print(f'\r{tot - left} of {tot} nodes communicating via IPMI',
end='')
sys.stdout.flush()
bmc.logout()
else:
self.log.debug(f'No power status response from node {node}')
time.sleep(delay)
if left != 0:
self.log.error('IPMI communication successful with only '
f'{tot - left} of {tot} nodes')
raise UserException('Unable to validate the following IPMI IP '
'Addresses :'
f'{[x for x in nodes if not nodes[x]]}')
print('\n')
return bmc_ai