def _check_known_hosts(host_list):
"""Ensure all hosts have entries in 'known_hosts' to avoid
Ansible's clunky yes/no prompting to accept keys (all prompts are
printed at once).
If any hosts are missing the user will be prompted to add it.
Args:
host_list (list): List of hostnames or IP addresses
"""
known_hosts_files = [os.path.join(Path.home(), ".ssh", "known_hosts")]
user_name, user_home_dir = get_user_and_home()
if os.environ['USER'] == 'root' and user_name != 'root':
known_hosts_files.append('/root/.ssh/known_hosts')
if not os.path.isdir('/root/.ssh'):
os.mkdir('/root/.ssh')
os.chmod('/root/.ssh', 0o700)
for host in host_list:
for known_hosts in known_hosts_files:
cmd = (f'ssh-keygen -F {host} -f {known_hosts}')
resp, err, rc = sub_proc_exec(cmd)
if rc != 0:
cmd = (f'ssh-keyscan -H {host}')
resp, err, rc = sub_proc_exec(cmd)
print(f'Adding \'{host}\' host keys to \'{known_hosts}\'')
append_line(known_hosts, resp, check_exists=False)
def create_base_dir(base_dir):
log = logger.getlogger()
print('\nMove or Copy the existing software server directories?')
ch, action = get_selection('move\ncopy', ('m', 'c'))
if action == 'copy':
statvfs = os.statvfs(base_dir)
freespace = statvfs.f_frsize * statvfs.f_bavail
if freespace < 18000000000:
sys.exit('Insufficient space on disk')
arch = ''
exists = glob('/srv/repos/dependencies/rhel7/*')
if not exists:
log.error('\nNo dependencies folder found. Unable to perform move.\n')
sys.exit()
for path in exists:
if 'p8' in path or 'p9' in path:
arch = 'ppc64le'
break
elif 'x86_64' in path:
arch = 'x86_64'
break
if not arch:
log.error('\nUnable to determine architecture. Unable to perform move.\n')
sys.exit()
if os.path.exists(f'{base_dir}/wmla120-{arch}'):
print(f'Destination path {base_dir}/wmla120-{arch} already exists.')
if action == 'copy':
if not get_yesno('Okay to proceed with force copy? '):
sys.exit('Exit at user request')
else:
os.mkdir(f'{base_dir}/wmla120-{arch}/')
for _dir in (('repos', 'anaconda', 'spectrum-conductor', 'spectrum-dli',
'wmla-license',)):
path = os.path.join('/srv/', _dir, '')
if os.path.isdir(path):
print(f'Found dir: {path}')
if action == 'move':
try:
_dir = f'{base_dir}/wmla120-{arch}/'
move(path, _dir)
cmd = f'sudo chcon -Rv --type=httpd_sys_content_t {_dir}'
_, err, rc = sub_proc_exec(cmd)
if rc != 0:
log.error(f'chtype of directory {_dir} failed {err}')
except shutil_Error as exc:
print(exc)
elif action == 'copy':
cmd = f'cp -rf {path} {base_dir}/wmla120-{arch}/'
try:
_, err, rc = sub_proc_exec(cmd)
except:
pass
if rc != 0:
log.error('Copy error {err}')
else:
log.error(f'Path {path} missing')
print('Done')
def _update_firewall(br_label):
"""Update iptables FORWARD table to forward all traffic coming into
the specified bridge.
"""
if _is_firewall_running():
fwd_tbl, err, rc = sub_proc_exec('iptables -vL FORWARD')
if br_label not in fwd_tbl:
LOG.debug(f'Updating firewall. Forward {br_label} packets.')
cmd = (f'iptables -I FORWARD -p all -i {br_label} '
f'-s 0.0.0.0/0 -d 0.0.0.0/0 -j ACCEPT')
res, err, rc = sub_proc_exec(cmd)
if rc:
LOG.warning('An error occured while updating the firewall. '
f'Error {err}. RC: {rc}')
def _validate_client_hostnames(software_hosts_file_path, hosts_list):
"""Validate hostnames listed in inventory match client hostnames
Args:
software_hosts_file_path (str): Path to software inventory file
host_list (list): List of hostnames or IP addresses
Returns:
bool: True if all client hostnames match
Raises:
UserException: If any hostname does not match
"""
base_cmd = (f'{get_ansible_path()} -i {software_hosts_file_path} ')
msg = ""
for host in hosts_list:
cmd = base_cmd + f'{host} -a "hostname --fqdn"'
resp, err, rc = sub_proc_exec(cmd, shell=True)
hostname = resp.splitlines()[-1]
if hostname != host:
msg += (f"Inventory hostname mis-match: '{host}' is reporting "
f"an FQDN of '{hostname}'\n")
if msg != "":
raise UserException(msg)
else:
return True
def create_pup_repo_mirror(repo_base_dir):
""" Download POWER-Up public repository in full
Args:
repo_base_dir (str): Base directory path
arch (str, optional): Select package architecture to save
py_ver (int, optional): Python version number
"""
log = logger.getlogger()
if not repo_base_dir.endswith('.git'):
repo_base_dir = path.join(repo_base_dir, 'power-up.git')
makedirs(repo_base_dir, exist_ok=True)
if len(listdir(repo_base_dir)) != 0:
log.info(f"The directory '{repo_base_dir}' already exists and is not"
" empty.")
if get_yesno("Permanently remove existing contents and re-clone? "):
rmtree(repo_base_dir)
makedirs(repo_base_dir, exist_ok=True)
else:
log.debug("User selected to continue without re-cloning")
return
url_path = rlinput("POWER-Up Repository path/URL: ", get_github_url())
resp, err, rc = sub_proc_exec(f'git clone --mirror {url_path} '
f'{repo_base_dir}')
if rc != 0:
log.error('An error occurred while cloning mirror of power-up repo: '
f'{err}')
def create_ssh_key_pair(name):
"""Create an SSH private/public key pair in ~/.ssh/
If an SSH key pair exists with "name" then the private key path is
returned *without* creating anything new.
Args:
name (str): Filename of private key file
Returns:
str: Private ssh key path
Raises:
UserException: If ssh-keygen command fails
"""
log = logger.getlogger()
ssh_dir = os.path.join(Path.home(), ".ssh")
private_key_path = os.path.join(ssh_dir, name)
if not os.path.isdir(ssh_dir):
os.mkdir(ssh_dir, mode=0o700)
if os.path.isfile(private_key_path):
log.info(f'SSH key \'{private_key_path}\' already exists, continuing')
else:
print(bold(f'Creating SSH key \'{private_key_path}\''))
cmd = ('ssh-keygen -t rsa -b 4096 '
'-C "Generated by Power-Up Software Installer" '
f'-f {private_key_path} -N ""')
resp, err, rc = sub_proc_exec(cmd, shell=True)
if str(rc) != "0":
msg = 'ssh-keygen failed:\n{}'.format(resp)
log.debug(msg)
raise UserException(msg)
return private_key_path
def _check_known_hosts(host_list):
"""Ensure all hosts have entries in 'known_hosts' to avoid
Ansible's clunky yes/no prompting to accept keys (all prompts are
printed at once).
If any hosts are missing the user will be prompted to add it.
Args:
host_list (list): List of hostnames or IP addresses
"""
log = logger.getlogger()
known_hosts_files = set()
user_name, user_home_dir = get_user_and_home()
user_uid = pwd.getpwnam(user_name).pw_uid
user_gid = grp.getgrnam(user_name).gr_gid
user_known_hosts = os.path.join(Path.home(), ".ssh", "known_hosts")
known_hosts_files.add(user_known_hosts)
if not os.path.isdir(os.path.join(Path.home(), ".ssh")):
log.debug(f"Creating user ~/.ssh dir")
os.mkdir(os.path.join(Path.home(), ".ssh"))
os.chmod(os.path.join(Path.home(), ".ssh"), 0o700)
os.chown(os.path.join(Path.home(), ".ssh"), user_uid, user_gid)
if not os.path.isfile(user_known_hosts):
log.debug(f"Creating user known hosts '{user_known_hosts}' file")
Path(user_known_hosts).touch(mode=0o600)
os.chown(user_known_hosts, user_uid, user_gid)
if os.environ['USER'] == 'root' and user_name != 'root':
known_hosts_files.add('/root/.ssh/known_hosts')
if not os.path.isdir('/root/.ssh'):
log.debug("Creating root '/root/.ssh' dir")
os.mkdir('/root/.ssh')
os.chmod('/root/.ssh', 0o700)
if not os.path.isfile('/root/.ssh/known_hosts'):
log.debug("Creating root '/root/.ssh/known_hosts' file")
Path('/root/.ssh/known_hosts').touch(mode=0o600)
for host in host_list:
for known_hosts in known_hosts_files:
cmd = (f'ssh-keygen -F {host} -f {known_hosts}')
resp, err, rc = sub_proc_exec(cmd)
if rc != 0:
cmd = (f'ssh-keyscan -H {host}')
resp, err, rc = sub_proc_exec(cmd)
print(f'Adding \'{host}\' host keys to \'{known_hosts}\'')
append_line(known_hosts, resp, check_exists=False)
def get_system_inventory(host, user, pw):
log = logger.getlogger()
cmd = f'ipmitool -I lanplus -H {host} -U {user} -P {pw} fru'
res, err, rc = u.sub_proc_exec(cmd)
if rc == 0:
return res
else:
log.debug(f'Unable to read system information from {host}, rc: {rc}')
def _exec_cmd(self, cmd, stdout=PIPE, stderr=PIPE):
"""Execute a command in the namespace
Args:
log (object): Log
cmd (string)
"""
cmd = 'ip netns exec {} {}'.format(self.name, cmd)
std_out, std_err, rc = sub_proc_exec(cmd, stdout, stderr)
return std_out, std_err, rc
def __init__(self, args):
self.args = args
self.config_file_path = gen.GEN_PATH
self.cont_config_file_path = gen.CONTAINER_PACKAGE_PATH + '/'
ssh_log = os.path.join(gen.GEN_LOGS_PATH, 'ssh_paramiko')
if not os.path.isfile(ssh_log):
os.mknod(ssh_log)
if not os.access(ssh_log, os.W_OK):
cmd = f'sudo chmod 666 {ssh_log}'
res, err, rc = sub_proc_exec(cmd)
def _update_firewall(br_label):
"""gets the FORWARD iptable and remove all entries which contain 'br_label'
"""
if _is_firewall_running():
rmv = []
fwd_tbl, err, rc = sub_proc_exec(
'iptables -nvL FORWARD --line-numbers')
for line in fwd_tbl.splitlines():
if br_label in line:
rmv.append(int(line[0:2]))
# reverse the entry number list so that they can be removed sequentially
# without affecting the number earlier entries.
rmv.reverse()
for item in rmv:
cmd = f'iptables -D FORWARD {item}'
res, err, rc = sub_proc_exec(cmd)
if rc:
LOG.warning(
f'An error occured while removing {br_label} from the '
f'iptables FORWARD table')
def find_unused_addr_and_add_to_ifc(self, ifc, cidr, offset=4, loc='top'):
""" Finds an available address in the given subnet. nmap -PR is used to
scan the subnet on the specified interface. Searching starts at either the
top or the bottom of the subnet at an offset specified by offset.
"""
status = False
mult = 1 if loc == 'bot' else -1
# check for an existing address on the interface in the subnet
for addr in self.get_interface_addresses(ifc):
if is_overlapping_addr(addr, cidr):
status = True
break
if not status:
# Find an available address on the subnet.
# if no route exists, add one temporarily so the subnet can be scanned
routes = self.get_interfaces_routes()
if ifc not in routes or cidr not in routes[ifc]:
self.route('add',
dst=cidr,
oif=self.link_lookup(ifname=ifc)[0])
# Get an address near the top of the subnet
if loc == 'top':
st_addr = add_offset_to_address(
cidr,
get_network_size(cidr) - offset)
else:
st_addr = add_offset_to_address(cidr, offset)
for i in range(get_network_size(cidr) - offset):
addr = add_offset_to_address(st_addr, mult * i)
cmd = f'nmap -PR {addr}'
res, err, rc = sub_proc_exec(cmd)
if not re.search(r'\d+\.\d+\.\d+\.\d+', res, re.DOTALL):
# Found an unused address
# First remove the temp route
res = self.route('del',
dst=cidr,
oif=self.link_lookup(ifname=ifc)[0])
if res[0]['header']['error']:
self.log.error(
f'Error occurred removing route from {ifc}')
# Add the address to the BMC interface
self.log.info(f'Adding address {addr} to ifc {ifc}')
idx = self.link_lookup(ifname=ifc)[0]
self.addr('add',
index=idx,
address=addr,
mask=int(cidr.rsplit('/')[1]))
status = True
break
# Update self.ifcs
self.ifcs = self.get_interfaces_dict()
return status
def setup_nginx_yum_repo(root_dir='/srv', repo_id='nginx'):
"""Install and setup nginx http server
Args:
root_dir (str): Path to root directory for requests
repo_id (str): Name of nginx yum repository
Returns:
int: Return code from 'yum makecache'
"""
log = logger.getlogger()
baseurl = 'http://nginx.org/packages/rhel/7/' + platform.machine()
repo_file = os.path.join('/etc/yum.repos.d', repo_id + '.repo')
if os.path.isfile(repo_file):
with open(repo_file, 'r') as f:
content = f.read()
if baseurl in content:
rc = 0
else:
line_in_file(repo_file, r'^baseurl=.+', f'baseurl={baseurl}')
for cmd in ['yum clean all', 'yum makecache']:
resp, err, rc = sub_proc_exec(cmd)
if rc != 0:
log.error(f"A problem occured while running '{cmd}'")
log.error(f'Response: {resp}\nError: {err}\nRC: {rc}')
else:
repo_name = 'nginx.org public'
repo = PowerupRepo(repo_id, repo_name, root_dir)
content = repo.get_yum_dotrepo_content(baseurl, gpgcheck=0)
repo.write_yum_dot_repo_file(content)
cmd = 'yum makecache'
resp, err, rc = sub_proc_exec(cmd)
if rc != 0:
log.error('A problem occured while creating the yum ' 'caches')
log.error(f'Response: {resp}\nError: {err}\nRC: {rc}')
return rc
def _teardown_ns(self, ns):
# kill dnsmasq
dns_list, stderr, rc = sub_proc_exec('pgrep dnsmasq')
dns_list = dns_list.splitlines()
for pid in dns_list:
ns_name, stderr, rc = sub_proc_exec('ip netns identify ' + pid)
if ns._get_name_sp_name() in ns_name:
self.log.debug('Killing dnsmasq {}'.format(pid))
stdout, stderr, rc = sub_proc_exec('kill -15 ' + pid)
# kill tcpdump
tcpdump_list, stderr, rc = sub_proc_exec('pgrep tcpdump')
tcpdump_list = tcpdump_list.splitlines()
for pid in tcpdump_list:
ns_name, stderr, rc = sub_proc_exec('ip netns identify ' + pid)
if ns._get_name_sp_name() in ns_name:
self.log.debug('Killing tcpdump {}'.format(pid))
stdout, stderr, rc = sub_proc_exec('kill -15 ' + pid)
# reconnect the veth pair to the container
ns._reconnect_container()
# Destroy the namespace
self.log.debug('Destroying namespace')
ns._destroy_name_sp()
def get_ansible_hostvars(software_hosts_file_path):
"""Get Ansible generated 'hostvars' dictionary
Args:
software_hosts_file_path (str): Path to software inventory file
Returns:
dict: Ansible 'hostvars' dictionary
"""
cmd = (f'ansible-inventory --inventory {software_hosts_file_path} --list')
resp, err, rc = sub_proc_exec(cmd, shell=True)
hostvars = json.loads(resp)['_meta']['hostvars']
return hostvars
def create_meta(self, update=False):
action = ('update', 'Updating') if update else ('create', 'Creating')
self.log.info(f'{action[1]} repository metadata and databases')
print('This may take a few minutes.')
if not update:
cmd = f'createrepo -v {self.repo_dir}'
else:
cmd = f'createrepo -v --update {self.repo_dir}'
resp, err, rc = sub_proc_exec(cmd)
if rc != 0:
self.log.error(f'Repo creation error: rc: {rc} stderr: {err}')
else:
self.log.info(
f'Repo {action[0]} process for {self.repo_id} finished'
' succesfully')
def get_rpm_path(self, filepath='/home/**/*.rpm'):
"""Interactive search for the rpm path.
Returns: Path to file or None
"""
while True:
self.rpm_path = get_file_path(filepath)
# Check for .rpm files in the chosen file
cmd = 'rpm -qlp self.rpm_path'
resp, err, rc = sub_proc_exec(cmd)
if self.rpm_path:
if '.rpm' not in resp:
print('There are no ".rpm" files in the selected path')
if get_yesno('Use selected path? ', default='n'):
return self.rpm_path
else:
return None
def _is_ifc_attached_elsewhere(ifc, bridge):
""" Checks to see if ifc is in use on a bridge other than that specified
Args:
ifc (str) interface name
bridge (str) name of bridge the interface is intended for
Returns:
True if the interface is already being used (is unavailable)
"""
br_list, err, rc = sub_proc_exec('brctl show')
br_list = br_list.splitlines()
output = []
for line in br_list[1:]:
if line.startswith('\t'):
output[len(output) - 1] = output[len(output) - 1] + line
else:
output.append(line)
if ifc in output[len(output) - 1] \
and bridge not in output[len(output) - 1]:
return True
return False
def copy_ssh_key_pair_to_hosts(private_key_path,
software_hosts_file_path,
global_pass=None):
"""Copy an SSH public key into software hosts authorized_keys files
TODO: detailed description
Args:
private_key_path (str) : Filename of private key file
software_hosts_file_path (str): Path to software inventory file
global_pass (str, optional): Global client default SSH password
Returns:
bool: True iff rc of all commands are "0"
"""
hosts_list = _validate_inventory_count(software_hosts_file_path, 0)
all_zero_returns = True
hostvars = get_ansible_hostvars(software_hosts_file_path)
for host in hosts_list:
print(bold(f'Copy SSH Public Key to {host}'))
cmd = f'ssh-copy-id -i {private_key_path} '
if "ansible_port" in hostvars[host]:
cmd += f'-p {hostvars[host]["ansible_port"]} '
if "ansible_ssh_common_args" in hostvars[host]:
cmd += f'{hostvars[host]["ansible_ssh_common_args"]} '
cmd += f'{hostvars[host]["ansible_user"]}@{host}'
if 'ansible_ssh_pass' not in hostvars[host]:
cmd = f'SSHPASS=\'{global_pass}\' sshpass -e ' + cmd
resp, err, rc = sub_proc_exec(cmd, shell=True)
if rc != 0:
all_zero_returns = False
print(err)
return all_zero_returns
def extract_rpm(self, src_path):
"""Extracts files from the selected rpm file to a repository directory
under /srv/repoid/rhel7/repoid. If a repodata directory is included in
the extracted data, then the path to repodata directory is returned
Inputs: Uses self.repo_dir and self.repo_id
Outputs:
repodata_dir : absolute path to repodata directory if one exists
"""
extract_dir = self.repo_dir
if not os.path.exists(extract_dir):
os.makedirs(extract_dir)
os.chdir(extract_dir)
cmd = f'rpm2cpio {src_path} | sudo cpio -div'
resp, err, rc = sub_proc_exec(cmd, shell=True)
if rc != 0:
self.log.error(f'Failed extracting {src_path}')
repodata_dir = glob.glob(f'{extract_dir}/**/repodata', recursive=True)
if repodata_dir:
self.repo_dir = os.path.dirname(repodata_dir[0])
return self.repo_dir
else:
return None
def get_user_and_home():
"""Get user name and home directory path
Returns the user account calling the script, *not* 'root' even
when called with 'sudo'.
Returns:
user_name, user_home_dir (tuple): User name and home dir path
Raises:
UserException: If 'getent' command fails
"""
log = logger.getlogger()
user_name = getlogin()
cmd = f'getent passwd {user_name}'
resp, err, rc = sub_proc_exec(cmd, shell=True)
if str(rc) != "0":
msg = 'getent failed:\n{}'.format(err)
log.debug(msg)
raise UserException(msg)
user_home_dir = resp.split(':')[5].rstrip()
return (user_name, user_home_dir)
def validate_pxe(self, bootdev='default', persist=True):
# if self.inv.check_all_nodes_pxe_macs():
# self.log.info("Inventory exists with PXE MACs populated.")
# if not self.ran_ipmi:
# return
# print("\nPress Enter to continue cluster deployment without "
# "running PXE hardware validation.")
# print("Type 'C' to validate cluster nodes defined in current "
# "'config.yml'")
# resp = input("Type 'T' to terminate Power-Up ")
# if resp == 'T':
# resp = input("Type 'y' to confirm ")
# if resp == 'y':
# self.log.info("'{}' entered. Terminating Power-Up at user "
# "request".format(resp))
# sys.exit(1)
# elif resp == 'C':
# self.log.info("'{}' entered. Continuing with hardware "
# "validation".format(resp))
# else:
# print()
# return
# if not self.ran_ipmi:
# return
if not self.node_table_ipmi:
raise UserCriticalException('No BMCs discovered')
self.log.debug("Checking PXE networks and client PXE"
" ports ________\n")
self.log.debug('Boot device: {}'.format(bootdev))
ipmi_cnt, pxe_cnt = self._get_port_cnts()
pxe_addr, bridge_addr, pxe_prefix, pxe_vlan = self._get_network('pxe')
pxe_network = pxe_addr + '/' + str(pxe_prefix)
addr = IPNetwork(bridge_addr + '/' + str(pxe_prefix))
netmask = str(addr.netmask)
addr.value += NAME_SPACE_OFFSET_ADDR
addr = str(addr)
foundall = False
dhcp_st = get_dhcp_pool_start()
pxe_ns = NetNameSpace('pxe-ns-', 'br-pxe-' + str(pxe_vlan), addr)
# setup DHCP. save start and end addr raw numeric values
self.log.debug('Installing DHCP server in network namespace')
addr_st = self._add_offset_to_address(pxe_network, dhcp_st)
addr_end = self._add_offset_to_address(pxe_network,
dhcp_st + pxe_cnt + 2)
dns_list, stderr, rc = sub_proc_exec('pgrep dnsmasq')
dns_list = dns_list.splitlines()
if os.path.exists(self.dhcp_pxe_leases_file):
os.remove(self.dhcp_pxe_leases_file)
# delete any remnant dnsmasq processes
for pid in dns_list:
ns_name, stderr, rc = sub_proc_exec(
'ip netns identify {}'.format(pid))
if pxe_ns._get_name_sp_name() in ns_name:
self.log.debug('Killing dnsmasq. pid {}'.format(pid))
stdout, stderr, rc = sub_proc_exec('kill -15 ' + pid)
cmd = (f'dnsmasq --dhcp-leasefile={self.dhcp_pxe_leases_file} '
f'--interface={pxe_ns._get_name_sp_ifc_name()} '
f'--dhcp-range={addr_st},{addr_end},{netmask},3600')
stdout, stderr, rc = pxe_ns._exec_cmd(cmd)
if rc != 0:
self.log.warning(f'Error configuring dnsmasq. rc: {rc}')
if os.path.exists(self.tcp_dump_file):
os.remove(self.tcp_dump_file)
tcpdump_list, stderr, rc = sub_proc_exec('pgrep tcpdump')
tcpdump_list = tcpdump_list.splitlines()
# delete any remnant tcpdump processes
for pid in tcpdump_list:
ns_name, stderr, rc = sub_proc_exec('ip netns identify ' + pid)
if pxe_ns._get_name_sp_name() in ns_name:
self.log.debug('Killing tcpdump. pid {}'.format(pid))
stdout, stderr, rc = sub_proc_exec('kill -15 ' + pid)
cmd = (f'sudo tcpdump -X -U -i {pxe_ns._get_name_sp_ifc_name()} '
f'-w {self.tcp_dump_file} --immediate-mode port 67')
proc = pxe_ns._launch_cmd(cmd)
if not isinstance(proc, object):
self.log.error(
f'Failure to launch process of tcpdump monitor {proc}')
# Scan up to 25 times. Delay 10 seconds between scans
# Allow infinite number of retries
self.log.info('Scanning pxe network on 10 s intervals.')
cnt = 0
cnt_prev = 0
cnt_down = 25
mac_list = []
dump = ''
while cnt < pxe_cnt:
print()
cmd = 'sudo tcpdump -r {} -xx'.format(self.tcp_dump_file)
for i in range(cnt_down):
print('\r{} of {} nodes requesting PXE boot. Scan cnt: {} '.
format(cnt, pxe_cnt, cnt_down - i),
end="")
sys.stdout.flush()
time.sleep(10)
# read the tcpdump file if size is not 0
if os.path.exists(self.tcp_dump_file) and os.path.getsize(
self.tcp_dump_file):
dump, stderr, rc = sub_proc_exec(cmd)
if rc != 0:
self.log.warning(
f'Error reading tcpdump file. rc: {rc}')
if 'reading' not in stderr:
self.log.warning(
f'Failure reading tcpdump file - {stderr}')
mac_list = self._get_macs(mac_list, dump)
cnt = len(mac_list)
if cnt > cnt_prev:
cnt_prev = cnt
# Pause briefly for in flight DHCP to complete and lease file to update
time.sleep(5)
self._build_port_table_pxe(mac_list)
if cnt >= pxe_cnt:
foundall = True
print(
'\r{} of {} nodes requesting PXE boot. Scan count: {} '
.format(cnt, pxe_cnt, cnt_down - i),
end="")
break
self.log.debug('Table of found PXE ports: {}'.format(
self.node_table_pxe))
for switch in self.node_table_pxe:
print('\n\nSwitch: {}'.format(switch))
print(
tabulate(self.node_table_pxe[switch],
headers=('port', 'MAC address', 'IP address')))
print()
if cnt >= pxe_cnt:
break
print('\n\nPress Enter to continue scanning for cluster nodes.')
print(
"Or enter 'C' to continue cluster deployment with a subset of nodes"
)
print("Or enter 'R' to cycle power to missing nodes")
resp = input("Or enter 'T' to terminate Power-Up ")
if resp == 'T':
resp = input("Enter 'y' to confirm ")
if resp == 'y':
self.log.info("'{}' entered. Terminating Power-Up at user"
" request".format(resp))
self._teardown_ns(self.ipmi_ns)
self._teardown_ns(pxe_ns)
sys.exit(1)
elif resp == 'R':
self._reset_unfound_nodes()
elif resp == 'C':
print('\nNot all nodes have been discovered')
resp = input("Enter 'y' to confirm continuation of"
" deployment without all nodes ")
if resp == 'y':
self.log.info(
"'{}' entered. Continuing Power-Up".format(resp))
break
if cnt < pxe_cnt:
self.log.warning('Failed to validate expected number of nodes')
self._teardown_ns(pxe_ns)
# Cycle power on all discovered nodes if bootdev set to 'network'
if bootdev == 'network':
self.log.debug('\nCycling power to discovered nodes.\n')
set_power_clients('off', clients=self.bmc_ai)
set_power_clients('on', clients=self.bmc_ai)
set_bootdev_clients('network', clients=self.bmc_ai)
self._teardown_ns(self.ipmi_ns)
# Reset BMCs to insure they acquire a new address from container
# during inv_add_ports. Avoids conflicting addresses during redeploy
self._reset_existing_bmcs(self.node_list, self._get_cred_list())
self.log.info('Cluster nodes validation complete')
if not foundall:
raise UserException('Not all node PXE ports validated')
def __init__(self, name, bridge, addr):
"""
Args:
name (str): namespace name
bridge (str): name of bridge to attach to
addr (str): cidr of namespace address
"""
self.log = logger.getlogger()
self.addr = addr
self.bridge = bridge
self.vlan = bridge.split('-')[-1]
self.name = name + self.vlan
self.ip = IPRoute()
self._disconnect_container()
self.log.debug('Creating network namespace {}'.format(self.name))
stdout, stderr, rc = sub_proc_exec('ip netns add {}'.format(self.name))
if rc:
self.log.debug(
'An error occurred while creating namespace '
f' {self.name}.\nreturn code: {rc}\nWarning: {stderr}')
if stderr:
if 'File exists' in stderr:
self.log.debug(stderr)
else:
self.log.error('Unable to create namespace')
sys.exit(1)
self.br_ifc = 'veth-br-' + self.name.split('-')[0] + '-' + self.vlan
self.peer_ifc = 'veth-' + self.name
try:
self.ip.link("add",
ifname=self.br_ifc,
peer=self.peer_ifc,
kind='veth')
except NetlinkError as exc:
if 'File exists' not in str(exc):
self.log.error('Failed creating veth pair. {}'.format(exc))
sys.exit(1)
try:
# peer interface side disappears from host space once attached to
# the namespace
idx_ns_ifc = self.ip.link_lookup(ifname=self.peer_ifc)[0]
self.ip.link('set', index=idx_ns_ifc, net_ns_fd=self.name)
except IndexError:
self.log.debug('Peer ifc already attached.')
except NetlinkError:
self.log.debug('Peer ifc already attached.')
idx_br = self.ip.link_lookup(ifname=bridge)[0]
self.idx_br_ifc = self.ip.link_lookup(ifname=self.br_ifc)[0]
self.ip.link('set', index=self.idx_br_ifc, master=idx_br)
# bring up the interfaces
cmd = 'ip netns exec {} ip link set dev {} up'.format(
self.name, self.peer_ifc)
stdout, stderr, rc = sub_proc_exec(cmd)
cmd = 'ip netns exec {} ip link set dev lo up'.format(self.name)
stdout, stderr, rc = sub_proc_exec(cmd)
cmd = 'ip netns exec {} ip addr add {} dev {} brd +' \
.format(self.name, addr, self.peer_ifc)
stdout, stderr, rc = sub_proc_exec(cmd)
# verify address setup
# cmd = 'ip netns exec {} ip addr show'.format(self.name)
# proc = Popen(cmd.split(), stdout=PIPE, stderr=PIPE)
# stdout, stderr = proc.communicate()
self.ip.link('set', index=self.idx_br_ifc, state='up')
def sync(self, pkg_list, alt_url=None, py_ver=27):
"""
inputs:
pkg_list (str): list of packages separated by space(s). Packages can
include versions. ie Keras==2.0.5
"""
if not os.path.isdir(self.pypirepo_dir):
os.mkdir(self.pypirepo_dir)
pkg_cnt = len(pkg_list.split())
print(
f'Downloading {pkg_cnt} python{py_ver} packages plus dependencies:\n'
)
pkg_list2 = pkg_list.split()
if alt_url:
host = re.search(r'http://([^/]+)', alt_url).group(1)
cmd = host # Dummy assign to silence tox
# wait on 'f' string formatting since 'pkg' is not available yet
cmd = ("f'python -m pip download --python-version {py_ver} "
"--platform ppc64le --no-deps --index-url={alt_url} "
"-d {self.pypirepo_dir} {pkg} --trusted-host {host}'")
else:
cmd = (
"f'python -m pip download --python-version {py_ver} "
"--platform ppc64le --no-deps -d {self.pypirepo_dir} {pkg}'")
for pkg in pkg_list2:
print(pkg)
resp, err, rc = sub_proc_exec(eval(cmd), shell=True)
if rc != 0:
if 'functools32' in resp and 'for Python 2.7 only' in resp:
pass
else:
self.log.error(
'Error occured while downloading python packages: '
f'\nResp: {resp} \nRet code: {rc} \nerr: {err}')
if not os.path.isdir(self.pypirepo_dir + '/simple'):
os.mkdir(self.pypirepo_dir + '/simple')
dir_list = os.listdir(self.pypirepo_dir)
cnt = 0
for item in dir_list:
if item[0] != '.' and os.path.isfile(self.pypirepo_dir + '/' +
item):
res = re.search(r'([-_+\w\.]+)(?=-(\d+\.\d+){1,3}).+', item)
if res:
cnt += 1
name = res.group(1)
name = name.replace('.', '-')
name = name.replace('_', '-')
name = name.lower()
if not os.path.isdir(self.pypirepo_dir +
f'/simple/{name}'):
os.mkdir(self.pypirepo_dir + f'/simple/{name}')
if not os.path.islink(self.pypirepo_dir +
f'/simple/{name}/{item}'):
os.symlink(
self.pypirepo_dir + f'/{item}',
self.pypirepo_dir + f'/simple/{name}/{item}')
else:
self.log.error(
f'mismatch: {item}. There was a problem entering '
f'{item}\ninto the python package index')
self.log.info(
f'A total of {cnt} packages exist or were added to the python '
'package repository')
def validate_ipmi(self):
self.log.info("Discover and validate cluster nodes")
# if self.inv.check_all_nodes_ipmi_macs() and self.inv.check_all_nodes_pxe_macs():
# self.log.info("Inventory exists with IPMI and PXE MACs populated.")
# print("\nPress Enter to continue cluster deployment without "
# "running IPMI hardware validation.")
# print("Type 'C' to validate cluster nodes defined in current "
# "'config.yml'")
# resp = input("Type 'T' to terminate Power-Up ")
# if resp == 'T':
# resp = input("Type 'y' to confirm ")
# if resp == 'y':
# self.log.info("'{}' entered. Terminating Power-Up at user "
# "request".format(resp))
# sys.exit(1)
# elif resp == 'C':
# self.log.info("'{}' entered. Continuing with hardware "
# "validation".format(resp))
# else:
# print()
# return
ipmi_cnt, pxe_cnt = self._get_port_cnts()
ipmi_addr, bridge_addr, ipmi_prefix, ipmi_vlan = self._get_network(
'ipmi')
ipmi_network = ipmi_addr + '/' + str(ipmi_prefix)
addr = IPNetwork(bridge_addr + '/' + str(ipmi_prefix))
netmask = str(addr.netmask)
ipmi_size = addr.size
addr.value += NAME_SPACE_OFFSET_ADDR
addr = str(addr)
cred_list = self._get_cred_list()
rc = False
dhcp_st = get_dhcp_pool_start()
self.ipmi_ns = NetNameSpace('ipmi-ns-', 'br-ipmi-' + str(ipmi_vlan),
addr)
# setup DHCP, unless already running in namesapce
# save start and end addr raw numeric values
self.log.debug('Installing DHCP server in network namespace')
addr_st = self._add_offset_to_address(ipmi_network, dhcp_st)
addr_end = self._add_offset_to_address(ipmi_network, ipmi_size - 2)
dhcp_end = self._add_offset_to_address(ipmi_network,
dhcp_st + ipmi_cnt + 2)
# scan ipmi network for nodes with pre-existing ip addresses
cmd = 'fping -r0 -a -g {} {}'.format(addr_st, addr_end)
node_list, stderr, rc = sub_proc_exec(cmd)
if rc not in (0, 1):
self.log.warning(f'Error scanning IPMI network. rc: {rc}')
self.log.debug('Pre-existing node list: \n{}'.format(node_list))
node_list = node_list.splitlines()
self._reset_existing_bmcs(node_list, cred_list)
if len(node_list) > 0:
print('Pause 60s for BMCs to begin reset')
time.sleep(60)
dns_list, stderr, rc = sub_proc_exec('pgrep dnsmasq')
if rc not in [0, 1]:
self.log.warning(f'Error looking for dnsmasq. rc: {rc}')
dns_list = dns_list.splitlines()
for pid in dns_list:
ns_name, stderr, rc = sub_proc_exec(
'ip netns identify {}'.format(pid))
if self.ipmi_ns._get_name_sp_name() in ns_name:
self.log.debug('DHCP already running in {}'.format(ns_name))
break
else:
cmd = (f'dnsmasq --dhcp-leasefile={self.dhcp_ipmi_leases_file} '
f'--interface={self.ipmi_ns._get_name_sp_ifc_name()} '
f'--dhcp-range={addr_st},{dhcp_end},{netmask},600')
stdout, stderr, rc = self.ipmi_ns._exec_cmd(cmd)
if rc != 0:
self.log.warning(f'Error setting up dnsmasq. rc: {rc}')
print(stderr)
# Scan up to 25 times. Delay 5 seconds between scans
# Allow infinite number of retries
self.log.info('Scanning BMC network on 5 s intervals')
cnt = 0
cnt_down = 25
while cnt < ipmi_cnt:
print()
for i in range(cnt_down):
print(
'\r{} of {} nodes requesting DHCP address. Scan count: {} '
.format(cnt, ipmi_cnt, cnt_down - i),
end="")
sys.stdout.flush()
time.sleep(5)
cmd = 'fping -r0 -a -g {} {}'.format(addr_st, dhcp_end)
stdout, stderr, rc = sub_proc_exec(cmd)
node_list = stdout.splitlines()
cnt = len(node_list)
if cnt >= ipmi_cnt:
rc = True
print(
'\r{} of {} nodes requesting DHCP address. Scan count: {} '
.format(cnt, ipmi_cnt, cnt_down - i),
end="")
break
self._get_port_table_ipmi(node_list)
self.log.debug('Table of found IPMI ports: {}'.format(
self.node_table_ipmi))
for switch in self.node_table_ipmi:
print('\n\nSwitch: {} '.format(switch))
print(
tabulate(self.node_table_ipmi[switch],
headers=('port', 'MAC address', 'IP address')))
print()
if cnt >= ipmi_cnt:
break
print(
'\n\nPress Enter to continue scanning for cluster nodes.\nOr')
print(
"Or enter 'C' to continue cluster deployment with a subset of nodes"
)
resp = input("Or Enter 'T' to terminate Power-Up ")
if resp == 'T':
resp = input("Enter 'y' to confirm ")
if resp == 'y':
self.log.info(
"'{}' entered. Terminating Power-Up at user request".
format(resp))
self._teardown_ns(self.ipmi_ns)
sys.exit(1)
elif resp == 'C':
print('\nNot all nodes have been discovered')
resp = input("Enter 'y' to confirm continuation of"
" deployment without all nodes ")
if resp == 'y':
self.log.info(
"'{}' entered. Continuing PowerUp".format(resp))
break
self.node_list = node_list
if cnt < ipmi_cnt:
self.log.warning('Failed to validate expected number of nodes')
if len(node_list) > 0 and len(cred_list) > 0:
# Verify and power off nodes
self.bmc_ai = self._get_credentials(node_list, cred_list)
if not self.bmc_ai:
self.log.error(
'Critical error. Unable to establish BMC communication '
'with any cluster nodes.\n.')
sys.exit('Exiting.')
# set_power_cients('off') has built in 60 s delay
self.log.info('\nPowering off cluster nodes')
set_power_clients('off', clients=self.bmc_ai)
set_power_clients('on', clients=self.bmc_ai)
self.log.debug('\nSetting "network" boot device on all nodes')
set_bootdev_clients('network', clients=self.bmc_ai)
self.log.debug('Cluster nodes IPMI validation complete')
self.ran_ipmi = True
if not rc:
raise UserException('Not all node IPMI ports validated')
def _destroy_name_sp(self):
self.ip.link('set', index=self.idx_br_ifc, state='down')
self.ip.link('del', index=self.idx_br_ifc)
self.ip.close()
stdout, stderr, rc = sub_proc_exec('ip netns del {}'.format(self.name))
def nginx_setup(root_dir='/srv', repo_id='nginx'):
"""Install and setup nginx http server
Args:
root_dir (str): Path to root directory for requests
repo_id (str): Name of nginx yum repository
Returns:
int: Return code from 'systemctl restart nginx.service'
"""
log = logger.getlogger()
# Check if nginx installed. Install if necessary.
cmd = 'nginx -v'
try:
resp, err, rc = sub_proc_exec(cmd)
except OSError:
if 'rhel' in linux_distribution(full_distribution_name=False):
cmd = 'yum -y install nginx'
resp, err, rc = sub_proc_exec(cmd)
if rc != 0:
setup_nginx_yum_repo(root_dir, repo_id)
cmd = 'yum -y install nginx'
resp, err, rc = sub_proc_exec(cmd)
if rc != 0:
log.error('Failed installing nginx')
log.error(resp)
sys.exit(1)
elif 'ubuntu' in linux_distribution(full_distribution_name=False):
for cmd in ['apt-get update', 'apt-get install -y nginx']:
resp, err, rc = sub_proc_exec(cmd)
if rc != 0:
log.error(f"A problem occured while running '{cmd}'")
log.error(f'Response: {resp}\nError: {err}\nRC: {rc}')
cmd = 'systemctl enable nginx.service'
resp, err, rc = sub_proc_exec(cmd)
if rc != 0:
log.error('Failed to enable nginx service')
cmd = 'systemctl start nginx.service'
resp, err, rc = sub_proc_exec(cmd)
if rc != 0:
log.error('Failed to start nginx service')
if os.path.isfile('/etc/nginx/conf.d/default.conf'):
try:
os.rename('/etc/nginx/conf.d/default.conf',
'/etc/nginx/conf.d/default.conf.bak')
except OSError:
log.warning('Failed renaming /etc/nginx/conf.d/default.conf')
nginx_location = {'/': [f'root {root_dir}', 'autoindex on']}
nginx_directives = {'listen': '80', 'server_name': 'powerup'}
rc = nginx_modify_conf('/etc/nginx/conf.d/server1.conf',
directives=nginx_directives,
locations=nginx_location)
return rc
def _is_firewall_running():
res, err, rc = sub_proc_exec('systemctl status firewalld')
if not rc:
if 'Active: active' in res or 'active (running)' in res:
return True
def main(args):
log = logger.getlogger()
log.debug('log this')
user_input = len(args) > 0
if user_input:
args = parse_input(args)
dep_base_path = gen.get_dependencies_path()
dirs = [
d for d in os.listdir(dep_base_path)
if os.path.isdir(os.path.join(dep_base_path, d))
]
dirs = [os.path.join(dep_base_path, _dir) for _dir in dirs]
dep_dir = ""
if user_input:
dep_dir = dirs[0]
if not user_input:
ch, dep_dir = get_selection(dirs,
prompt='Select a directory to aggregate '
'dependencies from: ')
dep_files = {}
pip_pre_files = [
'client_pip_pre_install.txt',
'dlipy3_pip_pre_install.txt',
'dlipy2_pip_pre_install.txt',
'dlinsights_pip_pre_install.txt',
]
dep_files['pip_pre_files'] = pip_pre_files
pip_post_files = [
'client_pip_post_install.txt',
'dlipy3_pip_post_install.txt',
'dlipy2_pip_post_install.txt',
'dlinsights_pip_post_install.txt',
]
dep_files['pip_post_files'] = pip_post_files
conda_pre_files = [
'dlipy3_conda_pre_install.txt',
'dlipy2_conda_pre_install.txt',
'dlinsights_conda_pre_install.txt',
]
dep_files['conda_pre_files'] = conda_pre_files
conda_post_files = [
'dlipy3_conda_post_install.txt',
'dlipy2_conda_post_install.txt',
'dlinsights_conda_post_install.txt',
]
dep_files['conda_post_files'] = conda_post_files
yum_pre_files = ['client_yum_pre_install.txt']
dep_files['yum_pre_files'] = yum_pre_files
yum_post_files = ['client_yum_post_install.txt']
dep_files['yum_post_files'] = yum_post_files
exists = glob.glob(f'{dep_dir}/**/{yum_pre_files[0]}', recursive=True)
if exists:
dep_dir = os.path.dirname(exists[0])
else:
log.error('No client yum pre file found')
sys.exit()
# # Change file ownership to current user
# if not os.access(dep_dir, os.W_OK):
username = getpass.getuser()
cmd = f'sudo chown -R {username}:{username} {dep_dir}'
sub_proc_exec(cmd, shell=True)
# Clear comments and other known header content from files
for item in dep_files:
for _file in dep_files[item]:
_file_path = os.path.join(dep_dir, _file)
with open(_file_path, 'r') as f:
lines = f.read().splitlines()
with open(_file_path, 'w') as f:
for line in lines:
if line.startswith('#') or line.startswith('@'):
continue
else:
f.write(line + '\n')
def file_check(file_list):
for f in file_list:
file_path = os.path.join(dep_dir, f)
my_file = os.path.isfile(file_path)
if my_file:
pass
else:
input(f'\nINFO - {f} Does not exist\n')
def get_pkg_repo(pkg, pkg_type):
if pkg_type == 'yum':
pkg_items = pkg.split()
repo = pkg_items[2]
if repo.endswith('-powerup'):
repo = repo[:-8]
elif pkg_type == 'pip':
pkg_items = pkg.split()
repo = pkg_items[2]
if pkg_type in repo:
repo = 'pypi'
elif pkg_type == 'conda':
pkg_dir = pkg.rpartition('/')[0]
if 'ibm-ai' in pkg or 'ibmai' in pkg:
if 'linux-ppc64le' in pkg:
repo = 'ibmai_linux_ppc64le'
elif 'noarch' in pkg:
repo = 'ibmai_noarch'
elif 'x86_64' in pkg:
repo = 'ibmai_linux_x86_64'
else:
repo = 'ibm_ai_unresolved_reponame'
elif 'repo.anaconda' in pkg:
repo = '-'.join(pkg_dir.rsplit('/', 2)[-2:])
repo = 'anaconda_' + repo.replace('-', '_')
else:
pkg_dir = pkg.rpartition('/')[0]
repo = '_'.join(pkg_dir.rsplit('/', 2)[-2:])
return repo
def format_pkg_name(pkg, pkg_type):
if pkg_type == 'yum':
pkg_items = pkg.split()
pkg_repo = get_pkg_repo(pkg, pkg_type)
pkg_fmt_name = (pkg_items[0].rsplit('.', 1)[0] + '-' +
pkg_items[1] + '.' +
pkg_items[0].rsplit('.', 1)[1])
elif pkg_type == 'conda':
pkg_fmt_name = pkg.rpartition('/')[-1]
pkg_repo = get_pkg_repo(pkg, pkg_type)
elif pkg_type == 'pip':
pkg_items = pkg.split()
pkg_repo = get_pkg_repo(pkg, pkg_type)
version = pkg_items[1].replace('(', '')
version = version.replace(')', '')
pkg_fmt_name = pkg_items[0] + '==' + version
return pkg_fmt_name, pkg_repo
def write_merged_files(merged_sets, pkg_type):
repo_list = {}
if pkg_type == 'yum':
repo_list[pkg_type] = []
for repo in merged_sets:
file_name = repo.replace('/', '')
file_name = file_name.replace('@', '')
file_name = f'{file_name}.yml'
file_path = os.path.join(dep_dir, file_name)
with open(file_path, 'w') as f:
d = {file_name: sorted(merged_sets[repo], key=str.lower)}
repo_list[pkg_type].append({
"path": file_path,
"filename": file_name,
"hash": d
})
yaml.dump(d, f, indent=4, default_flow_style=False)
elif pkg_type == 'conda':
repo_list[pkg_type] = []
for repo in merged_sets:
file_name = f'{repo}.yml'
file_path = os.path.join(dep_dir, file_name)
with open(file_path, 'w') as f:
d = {
file_name: sorted(list(merged_sets[repo]),
key=str.lower)
}
repo_list[pkg_type].append({
"path": file_path,
"filename": file_name,
"hash": d
})
yaml.dump(d, f, indent=4, default_flow_style=False)
elif pkg_type == 'pip':
repo_list[pkg_type] = []
for repo in merged_sets:
file_name = 'pypi.yml'
file_path = os.path.join(dep_dir, file_name)
with open(file_path, 'w') as f:
d = {file_name: sorted(merged_sets[repo], key=str.lower)}
repo_list[pkg_type].append({
"path": file_path,
"filename": file_name,
"hash": d
})
yaml.dump(d, f, indent=4, default_flow_style=False)
return repo_list
def get_repo_list(pkgs, pkg_type):
repo_list = []
if pkg_type == 'yum':
for pkg in pkgs:
repo = get_pkg_repo(pkg, pkg_type)
if repo not in repo_list:
repo_list.append(repo)
if pkg_type == 'conda':
for pkg in pkgs:
repo = get_pkg_repo(pkg, pkg_type)
if repo not in repo_list:
repo_list.append(repo)
if pkg_type == 'pip':
for pkg in pkgs:
if '<pip>' in pkg:
repo = get_pkg_repo(pkg, pkg_type)
if repo not in repo_list:
repo_list.append(repo)
return repo_list
def merge_function(pre_files, post_files, pkg_type):
""" Merges packages of a given type listed in a collection of files
collected 'post' installation and 'pre' installation for various
environments.
The merged set of 'pre' packages is removed from the merge set of
'post' packages to arrive at the list of installed packages across
all environments.
"""
# generate pre paths
pre_paths = []
for file in pre_files:
pre_paths.append(os.path.join(dep_dir, file))
# Generate post paths
post_paths = []
for file in post_files:
post_paths.append(os.path.join(dep_dir, file))
# Loop through the files
pkgs = {} # # {file:{repo:{pre:[], post: []}
for i, pre_file in enumerate(pre_paths):
file_name = os.path.basename(pre_file)
file_key = file_name.split('_')[0] + '_' + file_name.split('_')[1]
pkgs[file_key] = {}
post_file = post_paths[i]
try:
with open(pre_file, 'r') as f:
pre_pkgs = f.read().splitlines()
except FileNotFoundError as exc:
print(f'File not found: {pre_file}. Err: {exc}')
try:
with open(post_file, 'r') as f:
post_pkgs = f.read().splitlines()
except FileNotFoundError as exc:
print(f'File not found: {post_file}. Err: {exc}')
# Get the repo list
repo_list = get_repo_list(post_pkgs, pkg_type)
for repo in repo_list:
pkgs[file_key][repo] = {}
pkgs[file_key][repo]['pre'] = []
pkgs[file_key][repo]['post'] = []
for pkg in pre_pkgs:
pkg_fmt_name, pkg_repo = format_pkg_name(pkg, pkg_type)
if pkg_repo == repo:
pkgs[file_key][repo]['pre'].append(pkg_fmt_name)
for pkg in post_pkgs:
# Format the name
pkg_fmt_name, pkg_repo = format_pkg_name(pkg, pkg_type)
if pkg_repo == repo:
pkgs[file_key][repo]['post'].append(pkg_fmt_name)
diff_sets = {}
# Post - pre pkg sets. (may need adjustment for different repo type)
for _file in pkgs:
diff_sets[_file] = {}
for repo in pkgs[_file]:
post_minus_pre = set(pkgs[_file][repo]['post']) # -
# set(pkgs[_file][repo]['pre']))
diff_sets[_file][repo] = post_minus_pre
# Merge by repository
merged_sets = {}
for _file in diff_sets:
for repo in diff_sets[_file]:
if repo not in merged_sets:
merged_sets[repo] = set()
merged_sets[repo] = merged_sets[repo] | diff_sets[_file][repo]
return write_merged_files(merged_sets, pkg_type)
file_check(yum_pre_files)
file_check(yum_post_files)
main_repo_list = merge_function(yum_pre_files, yum_post_files, 'yum')
file_check(conda_pre_files)
file_check(conda_post_files)
conda_repo_list = merge_function(conda_pre_files, conda_post_files,
'conda')
merge_dicts(conda_repo_list, main_repo_list)
file_check(pip_pre_files)
file_check(pip_post_files)
pip_repo_list = merge_function(pip_pre_files, pip_post_files, 'pip')
merge_dicts(pip_repo_list, main_repo_list)
software_type = args.software if user_input else None
proc_family = ""
if software_type:
try:
file_path = GEN_SOFTWARE_PATH + SOFT_FILE
yaml_file = load_yamlfile(file_path)
proc_family = "_" + yaml_file["proc_family"]
except:
proc_family = ""
pass
lists, arch = parse_pkg_list(main_repo_list, software_type, proc_family)
generate_pkg_list(lists, software_type, arch, dep_dir)
def _validate_ansible_ping(software_hosts_file_path, hosts_list):
"""Validate Ansible connectivity and functionality on all hosts
Args:
software_hosts_file_path (str): Path to software inventory file
host_list (list): List of hostnames or IP addresses
Returns:
bool: True if Ansible can connect to all hosts
Raises:
UserException: If any host fails
"""
log = logger.getlogger()
cmd = ('{} -i {} -m ping all'.format(get_ansible_path(),
software_hosts_file_path))
resp, err, rc = sub_proc_exec(cmd)
if str(rc) != "0":
msg = f'Ansible ping validation failed:\n{resp}'
log.debug(msg)
if 'WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!' in msg:
print(
'@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n'
'@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED @\n'
'@ ON ONE OR MORE CLIENT NODES! @\n'
'@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n'
'IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY!\n'
'Someone could be eavesdropping on you right now '
'(man-in-the-middle attack)!\n'
'It is also possible that a host key has just been changed.\n')
if get_yesno('Remove the existing known host keys? '):
known_hosts_files = (
[os.path.join(Path.home(), ".ssh", "known_hosts")])
user_name, user_home_dir = get_user_and_home()
if user_home_dir != str(Path.home()):
known_hosts_files.append(os.path.join(user_home_dir,
".ssh",
"known_hosts"))
for host in hosts_list:
print(f'Collecting new host key(s) for {host}')
cmd = (f'ssh-keyscan -H {host}')
new_host_key, err, rc = sub_proc_exec(cmd)
for known_hosts in known_hosts_files:
print(f'Removing host keys for {host} '
f'from {known_hosts}')
cmd = (f'ssh-keygen -R {host} -f {known_hosts}')
resp, err, rc = sub_proc_exec(cmd)
print(f'Appending new host key for {host} to '
f'{known_hosts}')
append_line(known_hosts, new_host_key,
check_exists=False)
if user_home_dir != str(Path.home()):
user_known_hosts = os.path.join(user_home_dir, ".ssh",
"known_hosts")
user_uid = pwd.getpwnam(user_name).pw_uid
user_gid = grp.getgrnam(user_name).gr_gid
os.chown(user_known_hosts, user_uid, user_gid)
os.chmod(user_known_hosts, 0o600)
os.chown(user_known_hosts + '.old', user_uid, user_gid)
os.chmod(user_known_hosts + '.old', 0o600)
return _validate_ansible_ping(software_hosts_file_path,
hosts_list)
elif 'Permission denied' in msg:
msg = ('The PowerUp software installer attempted to log into the '
'the client node(s) but was unsuccessful. SSH key access '
'may need to be configured.\n')
print(msg)
if get_yesno('OK to configure Client Nodes for SSH Key Access? '):
configure_ssh_keys(software_hosts_file_path)
return _validate_ansible_ping(software_hosts_file_path,
hosts_list)
raise UserException(msg)
log.debug("Software inventory Ansible ping validation passed")
return True
