def _check_known_hosts(host_list):
"""Ensure all hosts have entries in 'known_hosts' to avoid
Ansible's clunky yes/no prompting to accept keys (all prompts are
printed at once).
If any hosts are missing the user will be prompted to add it.
Args:
host_list (list): List of hostnames or IP addresses
"""
known_hosts_files = [os.path.join(Path.home(), ".ssh", "known_hosts")]
user_name, user_home_dir = get_user_and_home()
if os.environ['USER'] == 'root' and user_name != 'root':
known_hosts_files.append('/root/.ssh/known_hosts')
if not os.path.isdir('/root/.ssh'):
os.mkdir('/root/.ssh')
os.chmod('/root/.ssh', 0o700)
for host in host_list:
for known_hosts in known_hosts_files:
cmd = (f'ssh-keygen -F {host} -f {known_hosts}')
resp, err, rc = sub_proc_exec(cmd)
if rc != 0:
cmd = (f'ssh-keyscan -H {host}')
resp, err, rc = sub_proc_exec(cmd)
print(f'Adding \'{host}\' host keys to \'{known_hosts}\'')
append_line(known_hosts, resp, check_exists=False)
def add_software_hosts_global_var(software_hosts_file_path, entry):
"""Copy an SSH public key into software hosts authorized_keys files
Add entry to software_hosts '[all:vars]' section. Any existing
entries with the same key name (string before '=') will be
overwritten.
Args:
software_hosts_file_path (str): Path to software inventory file
entry (str) : Entry to write in software_hosts '[all:vars]'
"""
remove_line(software_hosts_file_path, '^ansible_ssh_private_key_file=.*')
append_line(software_hosts_file_path, '[all:vars]')
with open(software_hosts_file_path, 'r') as software_hosts_read:
software_hosts = software_hosts_read.readlines()
in_all_vars = False
prev_line = "BOF"
with open(software_hosts_file_path, 'w') as software_hosts_write:
for line in software_hosts:
if line.startswith("[all:vars]"):
if prev_line != "\n":
line = "\n" + line
line = line + f'{entry}\n'
in_all_vars = True
elif in_all_vars and line.startswith('['):
in_all_vars = False
elif in_all_vars and line.startswith(entry.split('=')[0]):
continue
software_hosts_write.write(line)
prev_line = line
_set_software_hosts_owner_mode(software_hosts_file_path)
def _check_known_hosts(host_list):
"""Ensure all hosts have entries in 'known_hosts' to avoid
Ansible's clunky yes/no prompting to accept keys (all prompts are
printed at once).
If any hosts are missing the user will be prompted to add it.
Args:
host_list (list): List of hostnames or IP addresses
"""
log = logger.getlogger()
known_hosts_files = set()
user_name, user_home_dir = get_user_and_home()
user_uid = pwd.getpwnam(user_name).pw_uid
user_gid = grp.getgrnam(user_name).gr_gid
user_known_hosts = os.path.join(Path.home(), ".ssh", "known_hosts")
known_hosts_files.add(user_known_hosts)
if not os.path.isdir(os.path.join(Path.home(), ".ssh")):
log.debug(f"Creating user ~/.ssh dir")
os.mkdir(os.path.join(Path.home(), ".ssh"))
os.chmod(os.path.join(Path.home(), ".ssh"), 0o700)
os.chown(os.path.join(Path.home(), ".ssh"), user_uid, user_gid)
if not os.path.isfile(user_known_hosts):
log.debug(f"Creating user known hosts '{user_known_hosts}' file")
Path(user_known_hosts).touch(mode=0o600)
os.chown(user_known_hosts, user_uid, user_gid)
if os.environ['USER'] == 'root' and user_name != 'root':
known_hosts_files.add('/root/.ssh/known_hosts')
if not os.path.isdir('/root/.ssh'):
log.debug("Creating root '/root/.ssh' dir")
os.mkdir('/root/.ssh')
os.chmod('/root/.ssh', 0o700)
if not os.path.isfile('/root/.ssh/known_hosts'):
log.debug("Creating root '/root/.ssh/known_hosts' file")
Path('/root/.ssh/known_hosts').touch(mode=0o600)
for host in host_list:
for known_hosts in known_hosts_files:
cmd = (f'ssh-keygen -F {host} -f {known_hosts}')
resp, err, rc = sub_proc_exec(cmd)
if rc != 0:
cmd = (f'ssh-keyscan -H {host}')
resp, err, rc = sub_proc_exec(cmd)
print(f'Adding \'{host}\' host keys to \'{known_hosts}\'')
append_line(known_hosts, resp, check_exists=False)
def cobbler_install(config_path=None):
"""Install and configure Cobbler in container.
This function must be called within the container 'pup-venv'
python virtual environment. Cobbler will be installed within
this environment.
"""
cfg = Config(config_path)
log = logger.getlogger()
# Check to see if cobbler is already installed
try:
util.bash_cmd('cobbler check')
log.info("Cobbler is already installed")
return
except util.CalledProcessError as error:
if error.returncode == 127:
log.debug("'cobbler' command not found, continuing with "
"installation")
else:
log.warning("Cobbler is installed but not working:")
log.warning(error.output)
print("\nPress enter to remove Cobbler and attempt to ")
print("re-install, or 'T' to terminate.")
resp = input("\nEnter or 'T': ")
log.debug("User response = \'{}\'".format(resp))
if resp == 'T':
sys.exit('POWER-Up stopped at user request')
# Clone cobbler github repo
cobbler_url = URL
cobbler_branch = BRANCH
install_dir = gen.get_cobbler_install_dir()
if os.path.exists(install_dir):
log.info(
"Removing Cobbler source directory \'{}\'".format(install_dir))
util.bash_cmd('rm -rf %s' % install_dir)
log.info("Cloning Cobbler branch \'%s\' from \'%s\'" %
(cobbler_branch, cobbler_url))
repo = Repo.clone_from(cobbler_url,
install_dir,
branch=cobbler_branch,
single_branch=True)
log.info("Cobbler branch \'%s\' cloned into \'%s\'" %
(repo.active_branch, repo.working_dir))
# Modify Cobbler scrpit that write DHCP reservations so that the
# lease time is included.
dhcp_lease_time = cfg.get_globals_dhcp_lease_time()
util.replace_regex(
MANAGE_DNSMASQ, r'systxt \= systxt \+ \"\\\\n\"',
"systxt = systxt + \",{}\\\\n\"".format(dhcp_lease_time))
# Use non-secure http to download network boot-loaders
util.replace_regex(COBBLER_DLCONTENT, 'https://cobbler.github.io',
'http://cobbler.github.io')
# Use non-secure http to download signatures
util.replace_regex(COBBLER_SETTINGS_PY, 'https://cobbler.github.io',
'http://cobbler.github.io')
# Run cobbler make install
util.bash_cmd('cd %s; make install' % install_dir)
# Backup original files
util.backup_file(DNSMASQ_TEMPLATE)
util.backup_file(MODULES_CONF)
util.backup_file(COBBLER_WEB_SETTINGS)
util.backup_file(COBBLER_CONF_ORIG)
util.backup_file(COBBLER_WEB_CONF_ORIG)
util.backup_file(COBBLER_SETTINGS)
util.backup_file(PXEDEFAULT_TEMPLATE)
util.backup_file(KICKSTART_DONE)
util.backup_file(NTP_CONF)
util.backup_file(APACHE2_CONF)
# Create tftp root directory
if not os.path.exists(TFTPBOOT):
mode = 0o755
os.mkdir(TFTPBOOT, mode)
# Set IP address range to use for unrecognized DHCP clients
dhcp_range = 'dhcp-range=%s,%s,%s # %s'
util.remove_line(DNSMASQ_TEMPLATE, 'dhcp-range')
dhcp_pool_start = gen.get_dhcp_pool_start()
for index, netw_type in enumerate(cfg.yield_depl_netw_client_type()):
depl_netw_client_ip = cfg.get_depl_netw_client_cont_ip(index)
depl_netw_client_netmask = cfg.get_depl_netw_client_netmask(index)
network = IPNetwork(depl_netw_client_ip + '/' +
depl_netw_client_netmask)
entry = dhcp_range % (str(network.network + dhcp_pool_start),
str(network.network + network.size - 1),
str(dhcp_lease_time), str(network.cidr))
util.append_line(DNSMASQ_TEMPLATE, entry)
# Save PXE client network information for later
if netw_type == 'pxe':
cont_pxe_ipaddr = depl_netw_client_ip
cont_pxe_netmask = depl_netw_client_netmask
bridge_pxe_ipaddr = cfg.get_depl_netw_client_brg_ip(index)
# Configure dnsmasq to enable TFTP server
util.append_line(DNSMASQ_TEMPLATE, 'enable-tftp')
util.append_line(DNSMASQ_TEMPLATE, 'tftp-root=%s' % TFTPBOOT)
util.append_line(DNSMASQ_TEMPLATE, 'user=root')
# Configure dnsmasq to use deployer as gateway
if cfg.get_depl_gateway():
util.remove_line(DNSMASQ_TEMPLATE, 'dhcp-option')
util.append_line(DNSMASQ_TEMPLATE,
'dhcp-option=3,%s' % bridge_pxe_ipaddr)
# Cobbler modules configuration
util.replace_regex(MODULES_CONF, 'module = manage_bind',
'module = manage_dnsmasq')
util.replace_regex(MODULES_CONF, 'module = manage_isc',
'module = manage_dnsmasq')
# Copy cobbler.conf into apache2/conf-available
copy2(COBBLER_CONF_ORIG, COBBLER_CONF)
# Copy cobbler_web.conf into apache2/conf-available
copy2(COBBLER_WEB_CONF_ORIG, COBBLER_WEB_CONF)
# Apache2 configuration
util.bash_cmd('%s cobbler cobbler_web' % A2ENCONF)
util.bash_cmd('%s proxy' % A2ENMOD)
util.bash_cmd('%s proxy_http' % A2ENMOD)
# Set secret key in web settings
secret_key = _generate_random_characters()
util.replace_regex(COBBLER_WEB_SETTINGS, '^SECRET_KEY = .*',
'SECRET_KEY = "%s"' % secret_key)
# Remove "Order allow,deny" lines from cobbler configuration
regex = '.*Order allow,deny'
util.remove_line(COBBLER_CONF, regex)
util.remove_line(COBBLER_WEB_CONF, regex)
# Replace "Allow from all" with "Require all granted" in
regex = 'Allow from all'
replace = 'Require all granted'
util.replace_regex(COBBLER_CONF, regex, replace)
util.replace_regex(COBBLER_WEB_CONF, regex, replace)
# chown www-data WEBUI_SESSIONS
uid = pwd.getpwnam("www-data").pw_uid
gid = -1 # unchanged
os.chown(WEBUI_SESSIONS, uid, gid)
# Cobbler settings
util.replace_regex(COBBLER_SETTINGS, '127.0.0.1', cont_pxe_ipaddr)
util.replace_regex(COBBLER_SETTINGS, 'manage_dhcp: 0', 'manage_dhcp: 1')
util.replace_regex(COBBLER_SETTINGS, 'manage_dns: 0', 'manage_dns: 1')
util.replace_regex(COBBLER_SETTINGS, 'pxe_just_once: 0',
'pxe_just_once: 1')
globals_env_variables = cfg.get_globals_env_variables()
if globals_env_variables and 'http_proxy' in globals_env_variables:
util.replace_regex(
COBBLER_SETTINGS, 'proxy_url_ext: ""',
'proxy_url_ext: %s' % globals_env_variables['http_proxy'])
util.replace_regex(
COBBLER_SETTINGS, 'default_password_crypted:',
'default_password_crypted: '
'$1$clusterp$/gd3ep3.36A2808GGdHUz.')
# Create link to
if not os.path.exists(PY_DIST_PKGS):
util.bash_cmd('ln -s %s/cobbler %s' %
(LOCAL_PY_DIST_PKGS, PY_DIST_PKGS))
# Set PXE timeout to maximum
util.replace_regex(PXEDEFAULT_TEMPLATE, r'TIMEOUT \d+', 'TIMEOUT 35996')
util.replace_regex(PXEDEFAULT_TEMPLATE, r'TOTALTIMEOUT \d+',
'TOTALTIMEOUT 35996')
# Fix line break escape in kickstart_done snippet
util.replace_regex(KICKSTART_DONE, "\\\\nwget", "wget")
util.replace_regex(KICKSTART_DONE, r"\$saveks", "$saveks + \"; \\\\\\\"\n")
util.replace_regex(KICKSTART_DONE, r"\$runpost",
"$runpost + \"; \\\\\\\"\n")
# Copy authorized_keys ssh key file to web repo directory
copy2(ROOT_AUTH_KEYS, WWW_AUTH_KEYS)
os.chmod(WWW_AUTH_KEYS, 0o444)
# Add mgmt subnet to NTP service configuration
cont_pxe_broadcast = str(
IPNetwork(cont_pxe_ipaddr + '/' + cont_pxe_netmask).broadcast)
util.append_line(NTP_CONF, 'broadcast %s' % cont_pxe_broadcast)
# Add 'required-stop' line to cobblerd init.d to avoid warning
util.replace_regex(INITD + 'cobblerd', '### END INIT INFO',
'# Required-Stop:\n### END INIT INFO')
# Set Apache2 'ServerName'
util.append_line(APACHE2_CONF, "ServerName localhost")
# Restart services
_restart_service('ntp')
_restart_service('cobblerd')
_restart_service('apache2')
# Update Cobbler boot-loader files
util.bash_cmd('%s get-loaders' % COBBLER)
# Update cobbler list of OS signatures
util.bash_cmd('%s signature update' % COBBLER)
# Run Cobbler sync
util.bash_cmd('%s sync' % COBBLER)
# Restart services (again)
_restart_service('apache2')
_restart_service('cobblerd')
_restart_service('dnsmasq')
# Set services to start on boot
_service_start_on_boot('cobblerd')
_service_start_on_boot('ntp')
def inv_set_ipmi_pxe_ip(config_path):
"""Configure DHCP IP reservations for IPMI and PXE interfaces
IP addresses are assigned sequentially within the appropriate
client networks starting with the DHCP pool start offset defined
in 'lib.genesis'.
Raises:
UserException: - No IPMI or PXE client networks defined within
the 'config.yml'
- Unable to connect to BMC at new IPMI IP address
"""
log = logger.getlogger()
cfg = Config(config_path)
inv = Inventory(cfg_file=config_path)
ipmiNetwork = None
pxeNetwork = None
nodes_list = []
# All nodes should be powered off before starting
ipmi_set_power('off', config_path, wait=POWER_WAIT)
# Create IPManager object for IPMI and/or PXE networks
start_offset = gen.get_dhcp_pool_start()
for index, netw_type in enumerate(cfg.yield_depl_netw_client_type()):
ip = cfg.get_depl_netw_client_cont_ip(index)
netmask = cfg.get_depl_netw_client_netmask(index)
if netw_type == 'ipmi':
ipmiNetwork = IPManager(IPNetwork(ip + '/' + netmask),
start_offset)
elif netw_type == 'pxe':
pxeNetwork = IPManager(IPNetwork(ip + '/' + netmask), start_offset)
# If only one network is defined use the same IPManager for both
if ipmiNetwork is None and pxeNetwork is not None:
ipmiNetwork = pxeNetwork
elif ipmiNetwork is not None and pxeNetwork is None:
pxeNetwork = ipmiNetwork
elif ipmiNetwork is None and pxeNetwork is None:
raise UserException('No IPMI or PXE client network found')
# Modify IP addresses for each node
dhcp_lease_time = cfg.get_globals_dhcp_lease_time()
for index, hostname in enumerate(inv.yield_nodes_hostname()):
# IPMI reservations are written directly to the dnsmasq template
ipmi_ipaddr = inv.get_nodes_ipmi_ipaddr(0, index)
ipmi_mac = inv.get_nodes_ipmi_mac(0, index)
ipmi_new_ipaddr = ipmiNetwork.get_next_ip()
util.remove_line(DNSMASQ_TEMPLATE, "^dhcp-host=" + ipmi_mac + ".*")
util.append_line(
DNSMASQ_TEMPLATE, 'dhcp-host=%s,%s-bmc,%s,%s\n' %
(ipmi_mac, hostname, ipmi_new_ipaddr, dhcp_lease_time))
_adjust_dhcp_pool(ipmiNetwork.network,
ipmiNetwork.get_next_ip(reserve=False),
dhcp_lease_time)
# PXE reservations are handled by Cobbler
pxe_ipaddr = inv.get_nodes_pxe_ipaddr(0, index)
pxe_mac = inv.get_nodes_pxe_mac(0, index)
pxe_new_ipaddr = pxeNetwork.get_next_ip()
log.info('Modifying Inventory PXE IP - Node: %s MAC: %s '
'Original IP: %s New IP: %s' %
(hostname, pxe_mac, pxe_ipaddr, pxe_new_ipaddr))
inv.set_nodes_pxe_ipaddr(0, index, pxe_new_ipaddr)
_adjust_dhcp_pool(pxeNetwork.network,
pxeNetwork.get_next_ip(reserve=False),
dhcp_lease_time)
# Run Cobbler sync to process DNSMASQ template
cobbler_server = xmlrpclib.Server("http://127.0.0.1/cobbler_api")
token = cobbler_server.login(COBBLER_USER, COBBLER_PASS)
cobbler_server.sync(token)
log.debug("Running Cobbler sync")
# Save info to verify connection come back up
ipmi_userid = inv.get_nodes_ipmi_userid(index)
ipmi_password = inv.get_nodes_ipmi_password(index)
# No need to reset and check if the IP does not change
if ipmi_new_ipaddr != ipmi_ipaddr:
nodes_list.append({
'hostname': hostname,
'index': index,
'ipmi_userid': ipmi_userid,
'ipmi_password': ipmi_password,
'ipmi_new_ipaddr': ipmi_new_ipaddr,
'ipmi_ipaddr': ipmi_ipaddr,
'ipmi_mac': ipmi_mac
})
# Issue MC cold reset to force refresh of IPMI interfaces
for node in nodes_list:
ipmi_userid = node['ipmi_userid']
ipmi_password = node['ipmi_password']
ipmi_ipaddr = node['ipmi_ipaddr']
ipmi_cmd = ipmi_command.Command(bmc=ipmi_ipaddr,
userid=ipmi_userid,
password=ipmi_password)
ipmi_cmd.reset_bmc()
del ipmi_cmd
log.debug('BMC Cold Reset Issued - Node: %s - IP: %s' %
(hostname, ipmi_ipaddr))
# Check connections for set amount of time
end_time = time() + WAIT_TIME
while time() < end_time and len(nodes_list) > 0:
success_list = []
for list_index, node in enumerate(nodes_list):
hostname = node['hostname']
index = node['index']
ipmi_userid = node['ipmi_userid']
ipmi_password = node['ipmi_password']
ipmi_new_ipaddr = node['ipmi_new_ipaddr']
ipmi_ipaddr = node['ipmi_ipaddr']
ipmi_mac = node['ipmi_mac']
# Attempt to connect to new IPMI IP address
try:
ipmi_cmd = ipmi_command.Command(bmc=ipmi_new_ipaddr,
userid=ipmi_userid,
password=ipmi_password)
status = ipmi_cmd.get_power()
except pyghmi_exception.IpmiException as error:
log.debug('BMC connection failed - Node: %s IP: %s, %s '
'(Retrying for %s seconds)' %
(hostname, ipmi_new_ipaddr, str(error), WAIT_TIME))
continue
# If connection sucessful modify inventory
if status.get('powerstate') in ['on', 'off']:
log.debug('BMC connection success - Node: %s IP: %s' %
(hostname, ipmi_new_ipaddr))
log.info('Modifying Inventory IPMI IP - Node: %s MAC: %s '
'Original IP: %s New IP: %s' %
(hostname, ipmi_mac, ipmi_ipaddr, ipmi_new_ipaddr))
inv.set_nodes_ipmi_ipaddr(0, index, ipmi_new_ipaddr)
success_list.append(list_index)
# Remove nodes that connected successfully
for remove_index in sorted(success_list, reverse=True):
del nodes_list[remove_index]
for node in nodes_list:
log.error('Unable to connect to BMC at new IPMI IP address- Node: %s '
'MAC: %s Original IP: %s New IP: %s' %
(hostname, ipmi_mac, ipmi_ipaddr, ipmi_new_ipaddr))
if len(nodes_list) > 0:
raise UserException('%d BMC(s) not responding after IP modification' %
len(nodes_list))
def _validate_ansible_ping(software_hosts_file_path, hosts_list):
"""Validate Ansible connectivity and functionality on all hosts
Args:
software_hosts_file_path (str): Path to software inventory file
host_list (list): List of hostnames or IP addresses
Returns:
bool: True if Ansible can connect to all hosts
Raises:
UserException: If any host fails
"""
log = logger.getlogger()
cmd = ('{} -i {} -m ping all'.format(get_ansible_path(),
software_hosts_file_path))
resp, err, rc = sub_proc_exec(cmd)
if str(rc) != "0":
msg = f'Ansible ping validation failed:\n{resp}'
log.debug(msg)
if 'WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!' in msg:
print(
'@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n'
'@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED @\n'
'@ ON ONE OR MORE CLIENT NODES! @\n'
'@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n'
'IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY!\n'
'Someone could be eavesdropping on you right now '
'(man-in-the-middle attack)!\n'
'It is also possible that a host key has just been changed.\n')
if get_yesno('Remove the existing known host keys? '):
known_hosts_files = (
[os.path.join(Path.home(), ".ssh", "known_hosts")])
user_name, user_home_dir = get_user_and_home()
if user_home_dir != str(Path.home()):
known_hosts_files.append(os.path.join(user_home_dir,
".ssh",
"known_hosts"))
for host in hosts_list:
print(f'Collecting new host key(s) for {host}')
cmd = (f'ssh-keyscan -H {host}')
new_host_key, err, rc = sub_proc_exec(cmd)
for known_hosts in known_hosts_files:
print(f'Removing host keys for {host} '
f'from {known_hosts}')
cmd = (f'ssh-keygen -R {host} -f {known_hosts}')
resp, err, rc = sub_proc_exec(cmd)
print(f'Appending new host key for {host} to '
f'{known_hosts}')
append_line(known_hosts, new_host_key,
check_exists=False)
if user_home_dir != str(Path.home()):
user_known_hosts = os.path.join(user_home_dir, ".ssh",
"known_hosts")
user_uid = pwd.getpwnam(user_name).pw_uid
user_gid = grp.getgrnam(user_name).gr_gid
os.chown(user_known_hosts, user_uid, user_gid)
os.chmod(user_known_hosts, 0o600)
os.chown(user_known_hosts + '.old', user_uid, user_gid)
os.chmod(user_known_hosts + '.old', 0o600)
return _validate_ansible_ping(software_hosts_file_path,
hosts_list)
elif 'Permission denied' in msg:
msg = ('The PowerUp software installer attempted to log into the '
'the client node(s) but was unsuccessful. SSH key access '
'may need to be configured.\n')
print(msg)
if get_yesno('OK to configure Client Nodes for SSH Key Access? '):
configure_ssh_keys(software_hosts_file_path)
return _validate_ansible_ping(software_hosts_file_path,
hosts_list)
raise UserException(msg)
log.debug("Software inventory Ansible ping validation passed")
return True
def inv_set_ipmi_pxe_ip(config_path):
"""Configure DHCP IP reservations for IPMI and PXE interfaces
IP addresses are assigned sequentially within the appropriate
client networks starting with the DHCP pool start offset defined
in 'lib.genesis'.
Raises:
UserException: - No IPMI or PXE client networks defined within
the 'config.yml'
- Unable to connect to BMC at new IPMI IP address
"""
log = logger.getlogger()
cfg = Config(config_path)
inv = Inventory(cfg_file=config_path)
ipmiNetwork = None
pxeNetwork = None
nodes_list = []
# All nodes should be powered off before starting
set_power_clients('off', config_path, wait=POWER_WAIT)
# Create IPManager object for IPMI and/or PXE networks
start_offset = gen.get_dhcp_pool_start()
for index, netw_type in enumerate(cfg.yield_depl_netw_client_type()):
ip = cfg.get_depl_netw_client_cont_ip(index)
netmask = cfg.get_depl_netw_client_netmask(index)
if netw_type == 'ipmi':
ipmiNetwork = IPManager(IPNetwork(ip + '/' + netmask),
start_offset)
elif netw_type == 'pxe':
pxeNetwork = IPManager(IPNetwork(ip + '/' + netmask), start_offset)
# If only one network is defined use the same IPManager for both
if ipmiNetwork is None and pxeNetwork is not None:
ipmiNetwork = pxeNetwork
elif ipmiNetwork is not None and pxeNetwork is None:
pxeNetwork = ipmiNetwork
elif ipmiNetwork is None and pxeNetwork is None:
raise UserException('No IPMI or PXE client network found')
# Modify IP addresses for each node
dhcp_lease_time = cfg.get_globals_dhcp_lease_time()
for index, hostname in enumerate(inv.yield_nodes_hostname()):
# IPMI reservations are written directly to the dnsmasq template
ipmi_ipaddr = inv.get_nodes_ipmi_ipaddr(0, index)
ipmi_mac = inv.get_nodes_ipmi_mac(0, index)
ipmi_new_ipaddr = ipmiNetwork.get_next_ip()
util.remove_line(DNSMASQ_TEMPLATE, "^dhcp-host=" + ipmi_mac + ".*")
util.append_line(
DNSMASQ_TEMPLATE, 'dhcp-host=%s,%s-bmc,%s,%s\n' %
(ipmi_mac, hostname, ipmi_new_ipaddr, dhcp_lease_time))
_adjust_dhcp_pool(ipmiNetwork.network,
ipmiNetwork.get_next_ip(reserve=False),
dhcp_lease_time)
# PXE reservations are handled by Cobbler
pxe_ipaddr = inv.get_nodes_pxe_ipaddr(0, index)
pxe_mac = inv.get_nodes_pxe_mac(0, index)
pxe_new_ipaddr = pxeNetwork.get_next_ip()
log.info('Modifying Inventory PXE IP - Node: %s MAC: %s '
'Original IP: %s New IP: %s' %
(hostname, pxe_mac, pxe_ipaddr, pxe_new_ipaddr))
inv.set_nodes_pxe_ipaddr(0, index, pxe_new_ipaddr)
_adjust_dhcp_pool(pxeNetwork.network,
pxeNetwork.get_next_ip(reserve=False),
dhcp_lease_time)
# Run Cobbler sync to process DNSMASQ template
cobbler_server = xmlrpc.client.Server("http://127.0.0.1/cobbler_api")
token = cobbler_server.login(COBBLER_USER, COBBLER_PASS)
cobbler_server.sync(token)
log.debug("Running Cobbler sync")
# Save info to verify connection come back up
ipmi_userid = inv.get_nodes_ipmi_userid(index)
ipmi_password = inv.get_nodes_ipmi_password(index)
bmc_type = inv.get_nodes_bmc_type(index)
# No need to reset and check if the IP does not change
if ipmi_new_ipaddr != ipmi_ipaddr:
nodes_list.append({
'hostname': hostname,
'index': index,
'ipmi_userid': ipmi_userid,
'ipmi_password': ipmi_password,
'ipmi_new_ipaddr': ipmi_new_ipaddr,
'ipmi_ipaddr': ipmi_ipaddr,
'ipmi_mac': ipmi_mac,
'bmc_type': bmc_type
})
# Issue MC cold reset to force refresh of IPMI interfaces
for node in nodes_list:
ipmi_userid = node['ipmi_userid']
ipmi_password = node['ipmi_password']
ipmi_ipaddr = node['ipmi_ipaddr']
bmc_type = node['bmc_type']
bmc = _bmc.Bmc(ipmi_ipaddr, ipmi_userid, ipmi_password, bmc_type)
if bmc.is_connected():
log.debug(f'Issuing BMC Cold Reset - Node: {node["hostname"]} '
f'- IP: {ipmi_ipaddr}')
if not bmc.bmc_reset('cold'):
log.error(
f'Failed attempting BMC reset on {node["ipmi_ipaddr"]}')
bmc.logout()
log.info('Pausing 1 minute for BMCs to begin reset')
sleep(60)
# Check connections for set amount of time
end_time = time() + WAIT_TIME
while time() < end_time and len(nodes_list) > 0:
print(f'\rTimeout count down: {int(end_time - time())} ', end='')
sys.stdout.flush()
success_list = []
sleep(2)
for list_index, node in enumerate(nodes_list):
hostname = node['hostname']
index = node['index']
ipmi_userid = node['ipmi_userid']
ipmi_password = node['ipmi_password']
ipmi_new_ipaddr = node['ipmi_new_ipaddr']
ipmi_ipaddr = node['ipmi_ipaddr']
ipmi_mac = node['ipmi_mac']
bmc_type = node['bmc_type']
# Attempt to connect to new IPMI IP address
bmc = _bmc.Bmc(ipmi_new_ipaddr, ipmi_userid, ipmi_password,
bmc_type)
if bmc.is_connected():
if bmc.chassis_power('status') in ('on', 'off'):
log.debug(f'BMC connection success - Node: {hostname} '
f'IP: {ipmi_ipaddr}')
else:
log.debug(f'BMC communication failed - Node: {hostname} '
f'IP: {ipmi_ipaddr}')
continue
log.info(
f'Modifying Inventory IPMI IP - Node: {hostname} MAC: '
f'{ipmi_mac} Original IP: {ipmi_ipaddr} New IP: '
f'{ipmi_new_ipaddr}')
inv.set_nodes_ipmi_ipaddr(0, index, ipmi_new_ipaddr)
success_list.append(list_index)
else:
log.debug(f'BMC connection failed - Node: {hostname} '
f'IP: {ipmi_ipaddr}')
continue
# Remove nodes that connected successfully
for remove_index in sorted(success_list, reverse=True):
del nodes_list[remove_index]
for node in nodes_list:
log.error('Unable to connect to BMC at new IPMI IP address- Node: %s '
'MAC: %s Original IP: %s New IP: %s' %
(hostname, ipmi_mac, ipmi_ipaddr, ipmi_new_ipaddr))
if len(nodes_list) > 0:
raise UserException('%d BMC(s) not responding after IP modification' %
len(nodes_list))
