def _validate_master_node_count(software_hosts_file_path, min_count, max_count=0): """Validate number of nodes are defined in inventory's 'master' group. Either an exact or minimum count can be validated. Args: software_hosts_file_path (str): Path to software inventory file min_count (int): Minimum number of master nodes max_count (int, optional): Maximum number of master nodes. If set to 0 no maximum value is checked. Returns: bool: True validation passes Raises: UserException: Minimum or exact count is not present """ host_count = len(_validate_inventory_count(software_hosts_file_path, 0, group='master')) if host_count < min_count: raise UserException(f'Inventory requires at least {min_count} master ' f'node(s) ({host_count} found)!') elif max_count != 0 and host_count > max_count: raise UserException(f'Inventory requires at most {max_count} master ' f'node(s) ({host_count} found)!') else: return True
def _add_macs(self, macs, type_): for node in self.inv.nodes: for index, _port in enumerate(node[type_][self.InvKey.PORTS]): port = str(_port) switch = node[type_][self.InvKey.SWITCHES][index] # If switch is not found if switch not in macs: msg = "Switch '{}' not found".format(switch) self.log.error(msg) raise UserException(msg) # If port is not found if port not in macs[switch]: msg = "Switch '{}' port '{}' not found".format( switch, port) self.log.debug(msg) continue # If port has no MAC if not macs[switch][port]: msg = "Switch '{}' port '{}' no MAC".format(switch, port) self.log.debug(msg) continue # If port has more than one MAC if len(macs[switch][port]) > 1: msg = "Switch '{}' port '{}' too many MACs '{}'".format( switch, port, macs[switch][port]) self.log.error(msg) raise UserException(msg) if macs[switch][port][0] not in node[type_][self.InvKey.MACS]: node[type_][self.InvKey.MACS][index] = \ macs[switch][port][0]
def _validate_client_hostnames(software_hosts_file_path, hosts_list): """Validate hostnames listed in inventory match client hostnames Args: software_hosts_file_path (str): Path to software inventory file host_list (list): List of hostnames or IP addresses Returns: bool: True if all client hostnames match Raises: UserException: If any hostname does not match """ base_cmd = (f'{get_ansible_path()} -i {software_hosts_file_path} ') msg = "" for host in hosts_list: cmd = base_cmd + f'{host} -a "hostname --fqdn"' resp, err, rc = sub_proc_exec(cmd, shell=True) hostname = resp.splitlines()[-1] if hostname != host: msg += (f"Inventory hostname mis-match: '{host}' is reporting " f"an FQDN of '{hostname}'\n") if msg != "": raise UserException(msg) else: return True
def download_os_images(config_path=None): """Download OS installation images""" log = logger.getlogger() os_images_path = get_os_images_path() + "/" os_image_urls_yaml_path = os_images_path + OS_IMAGES_URLS_FILENAME cfg = Config(config_path) os_image_urls = yaml.load(open(os_image_urls_yaml_path), Loader=AttrDictYAMLLoader).os_image_urls for os_profile in cfg.yield_ntmpl_os_profile(): for os_image_url in os_image_urls: if check_os_profile(os_profile) in os_image_url.name: for image in os_image_url.images: dest = os_images_path if 'filename' in image: dest += image.filename else: dest += image.url.split("/")[-1] if not os.path.isfile(dest): log.info('Downloading OS image: %s' % image.url) wget.download(image.url, out=dest) print('') sys.stdout.flush() log.info('Verifying OS image sha1sum: %s' % dest) sha1sum = _sha1sum(dest) if image.sha1sum != sha1sum: msg = ('OS image sha1sum verification failed: %s' % dest) log.error(msg) raise UserException(msg)
def create_ssh_key_pair(name): """Create an SSH private/public key pair in ~/.ssh/ If an SSH key pair exists with "name" then the private key path is returned *without* creating anything new. Args: name (str): Filename of private key file Returns: str: Private ssh key path Raises: UserException: If ssh-keygen command fails """ log = logger.getlogger() ssh_dir = os.path.join(Path.home(), ".ssh") private_key_path = os.path.join(ssh_dir, name) if not os.path.isdir(ssh_dir): os.mkdir(ssh_dir, mode=0o700) if os.path.isfile(private_key_path): log.info(f'SSH key \'{private_key_path}\' already exists, continuing') else: print(bold(f'Creating SSH key \'{private_key_path}\'')) cmd = ('ssh-keygen -t rsa -b 4096 ' '-C "Generated by Power-Up Software Installer" ' f'-f {private_key_path} -N ""') resp, err, rc = sub_proc_exec(cmd, shell=True) if str(rc) != "0": msg = 'ssh-keygen failed:\n{}'.format(resp) log.debug(msg) raise UserException(msg) return private_key_path
def run_command(self, cmd, stdout=None): if stdout: print('.', end="") sys.stdout.flush() rc = self.cont.attach_wait( lxc.attach_run_command, cmd, stdout=stdout, stderr=stdout, extra_env_vars=[ logger.get_log_level_env_var_file(), logger.get_log_level_env_var_print()]) else: rc = self.cont.attach_wait( lxc.attach_run_command, cmd, extra_env_vars=[ logger.get_log_level_env_var_file(), logger.get_log_level_env_var_print()]) if rc: error = "Failed running '{}' in the container '{}'".format( ' '.join(cmd), self.name) raise UserException(error) self.log.debug( "Successfully ran '{}' in the container '{}'".format( ' '.join(cmd), self.name))
def validate_config_schema(self): """Config schema validation Exception: If schema validation fails """ schema = SchemaDefinition.get_schema(ordered=True) try: validate(self.config, schema, format_checker=jsonschema.FormatChecker()) except jsonschema.exceptions.ValidationError as error: if error.cause is None: path = None for index, element in enumerate(error.path): if isinstance(element, int): path += '[{}]'.format(element) else: if index == 0: path = '{}'.format(element) else: path += '.{}'.format(element) exc = 'Schema validation failed - {} - {}'.format( path, error.message) else: exc = 'Schema validation failed - {} - {}'.format( error.cause, error.message) if 'Additional properties are not allowed' in error.message: raise UserException(exc) else: raise UserCriticalException(exc)
def run_command(self, cmd, interactive=False): self.log.debug(f"Exec container:'{self.cont.name}' cmd:'{cmd}'") if interactive: # TODO: Use docker.Container.exec_run() method for interactive cmds cmd_string = ' '.join(cmd) rc = sub_proc_display(f'docker exec -it {self.cont.name} ' f'{cmd_string}') output = None else: environment = [logger.get_log_level_env_var_file(), logger.get_log_level_env_var_print()] print('.', end="") rc, output = self.cont.exec_run(cmd, stderr=True, stdout=True, stdin=True, stream=interactive, detach=False, tty=True, environment=environment) print('.', end="") self.log.debug(f"rc:'{rc}' output:'{output.decode('utf-8')}'") sys.stdout.flush() if rc: msg = f"Failed running '{cmd}' in the container '{self.name}'" if output is not None: msg += f": {output}" self.log.error(msg) raise UserException(msg) else: self.log.debug(f"Successfully ran '{cmd}' in the container " f"'{self.name}'")
def factory(switch_type=None, host=None, userid=None, password=None, mode='active', outfile='switch_cmds.txt'): """Return management switch model object. Args: inv (:obj:`Inventory`): Inventory object. switch_type (enum): Switch type. host (str): Switch ipv4 address userid (str): Switch userid. (This user must have configuration authority on the switch) password (str): Switch password. Raises: Exception: If management switch class is invalid. """ if switch_type in 'lenovo Lenovo LENOVO': return lenovo.switch.factory(host, userid, password, mode, outfile) if switch_type in 'mellanox Mellanox MELLANOX': return mellanox.switch.factory(host, userid, password, mode, outfile) if switch_type in 'cisco Cisco CISCO': return cisco.switch.factory(host, userid, password, mode, outfile) msg = 'Invalid switch class' LOG.error(msg) raise UserException(msg)
def download_os_images(config_path=None): """Download OS installation images""" log = logger.getlogger() cfg = Config(config_path) os_images_path = get_os_images_path() + "/" os_image_urls = get_os_image_urls() for os_profile in cfg.yield_ntmpl_os_profile(): for os_image_url in os_image_urls: if check_os_profile(os_profile) in os_image_url['name']: for image in os_image_url['images']: dest = os_images_path if 'filename' in image: dest += image['filename'] else: dest += image['url'].split("/")[-1] if not os.path.isfile(dest): log.info(f"Downloading OS image: {image['url']}") wget.download(image['url'], out=dest) print('') sys.stdout.flush() log.info('Verifying OS image sha1sum: %s' % dest) if image['sha1sum'] != sha1sum(dest): msg = ('OS image sha1sum verification failed: %s' % dest) log.error(msg) raise UserException(msg)
def _load_yaml_file(self, yaml_file): """Load from YAML file Exception: If load from file fails """ msg = "Failed to load '{}'".format(yaml_file) try: return yaml.load(open(yaml_file), Loader=AttrDictYAMLLoader) except yaml.parser.ParserError as exc: self.log.error("Failed to parse JSON '{}' - {}".format( yaml_file, exc)) raise UserException(msg) except Exception as exc: self.log.error("Failed to load '{}' - {}".format(yaml_file, exc)) raise UserException(msg)
def _is_config_file(self, config_file): """ Check if config file exists Exception: If config file does not exist """ if not os.path.isfile(config_file): msg = 'Could not find config file: ' + config_file self.log.error(msg) raise UserException(msg)
def _get_pxe_ips(inv): ip_list = '' for index, hostname in enumerate(inv.yield_nodes_hostname()): ip = inv.get_nodes_pxe_ipaddr(0, index) if ip is None: raise UserException('No PXE IP Address in Inventory for client ' '\'%s\'' % hostname) if ip_list != '': ip = ',' + ip ip_list += ip return ip_list
def _create_network( self, dev_label, interface_ipaddr, netprefix, container_ipaddr=None, bridge_ipaddr=None, vlan=None, type_='mgmt', remove=False): network = None if container_ipaddr is not None and bridge_ipaddr is not None: name = 'pup-' + type_ br_name = 'br-' + type_ if vlan is not None: name += '-' + str(vlan) br_name += '-' + str(vlan) try: network = self.client.networks.get(name) if remove: for container in network.containers: self.log.debug("Disconnecting Docker network " f"'{network.name}' from container" f"'{container.name}'") network.disconnect(container, force=True) self.log.debug(f"Removing Docker network '{network.name}'") network.remove() network = None except docker.errors.NotFound: if not remove: self.log.debug(f"Creating Docker network '{name}'") subnet = str(IPNetwork(bridge_ipaddr + '/' + str(netprefix)).cidr) ipam_pool = docker.types.IPAMPool(subnet=subnet, gateway=bridge_ipaddr) ipam_config = docker.types.IPAMConfig( pool_configs=[ipam_pool]) try: network = self.client.networks.create( name=name, driver='bridge', ipam=ipam_config, options={'com.docker.network.bridge.name': br_name}) except docker.errors.APIError as exc: msg = (f"Failed to create network '{name}': {exc}") self.log.error(msg) raise UserException(msg) return network
def _is_config_file(self, config_file): """ Check if config file exists Exception: If config file does not exist """ if not os.path.isfile(config_file): if os.path.isfile(os.path.join(gen.GEN_PATH, config_file)): self.cfg = os.path.join(gen.GEN_PATH, config_file) msg = 'Could not find config file: ' + config_file self.log.error(msg) raise UserException(msg)
def set_interface_name(self, set_mac, set_name): """Set physical interface name Args: macs (str): Interface MAC address name (str): Device name """ old_name = '' for index, node in enumerate(self.inv.nodes): for if_index, mac in enumerate(node.pxe.macs): if set_mac == mac: old_name = node.pxe.devices[if_index] self.log.debug("Renaming node \'%s\' PXE physical " "interface \'%s\' to \'%s\' (MAC:%s)" % (node.hostname, old_name, set_name, mac)) node.pxe.devices[if_index] = set_name break else: for if_index, mac in enumerate(node.data.macs): if set_mac == mac: old_name = node.data.devices[if_index] self.log.debug( "Renaming node \'%s\' data physical " "interface \'%s\' to \'%s\' (MAC:%s)" % (node.hostname, old_name, set_name, mac)) node.data.devices[if_index] = set_name break if old_name != '': node_index = index break else: raise UserException( "No physical interface found in inventory with " "MAC: %s" % set_mac) for interface in self.inv.nodes[node_index][self.InvKey.INTERFACES]: for key, value in interface.iteritems(): if isinstance(value, basestring): value_split = [] for _value in value.split(): if old_name == _value or old_name in _value.split('.'): _value = _value.replace(old_name, set_name) value_split.append(_value) new_value = " ".join(value_split) self.log.debug("Renaming node \'%s\' interface key \'%s\' " "from \'%s\' to \'%s\'" % (self.inv.nodes[node_index].hostname, key, value, new_value)) interface[key] = new_value self.dbase.dump_inventory(self.inv)
def build_image(self): self.log.info("Building Docker image " f"'{self.DEFAULT_CONTAINER_NAME}'") try: self.image, build_logs = self.client.images.build( path=gen.get_package_path(), tag=self.DEFAULT_CONTAINER_NAME, rm=True) except docker.errors.APIError as exc: msg = ("Failed to create image " f"'{self.DEFAULT_CONTAINER_NAME}': {exc}") self.log.error(msg) raise UserException(msg) self.log.debug("Created image " f"'{self.DEFAULT_CONTAINER_NAME}'")
def _validate_host_list_network(host_list): """Validate all hosts in list are pingable Args: host_list (list): List of hostnames or IP addresses Returns: bool: True if all hosts are pingable Raises: UserException: If list item will not resolve or ping """ log = logger.getlogger() for host in host_list: # Check if host is given as IP address if not netaddr.valid_ipv4(host, flags=0): try: socket.gethostbyname(host) except socket.gaierror as exc: log.debug("Unable to resolve host to IP: '{}' exception: '{}'" .format(host, exc)) raise UserException("Unable to resolve hostname '{}'!" .format(host)) else: raise UserException('Client nodes must be defined using hostnames ' f'(IP address found: {host})!') # Ping IP try: bash_cmd('fping -u {}'.format(' '.join(host_list))) except CalledProcessError as exc: msg = "Ping failed on hosts:\n{}".format(exc.output) log.debug(msg) raise UserException(msg) log.debug("Software inventory host fping validation passed") return True
def check_permissions(self, user): # Enumerate LXC bridge entry = AttrDict({ 'user': user, 'type': 'veth', 'bridge': 'lxcbr0'}) allows = [] allows.append(entry.copy()) # Enumerate management bridges for vlan in self.cfg.yield_depl_netw_mgmt_vlan(): if vlan is not None: entry.bridge = 'br-mgmt-%d' % vlan allows.append(entry.copy()) # Enumerate client bridges for index, vlan in enumerate(self.cfg.yield_depl_netw_client_vlan()): if vlan is not None: type_ = self.cfg.get_depl_netw_client_type(index) entry.bridge = 'br-%s-%d' % (type_, vlan) allows.append(entry.copy()) # Check bridge permissions for line in open(self.LXC_USERNET, 'r'): match = re.search( r'^\s*(\w+)\s+(\w+)\s+([\w-]+)\s+(\d+)\s*$', line) if match is not None: allows[:] = [ allow for allow in allows if not ( allow.user == match.group(1) and allow.type == match.group(2) and allow.bridge == match.group(3))] # If bridge permissions are missing if allows: msg = "Missing entries in '%s':" % self.LXC_USERNET for allow in allows: msg += ' (%s %s %s <number>)' % \ (allow.user, allow.type, allow.bridge) self.log.error(msg) raise UserException(msg) # Success self.log.debug( "Unprivileged/non-root container bridge support found in '%s'" % self.LXC_USERNET)
def _dump_yaml_file(self, yaml_file, content): """Dump to YAML file Exception: If dump to file fails """ try: yaml.safe_dump(content, open(yaml_file, 'w'), indent=4, default_flow_style=False) except Exception as exc: self.log.error("Failed to dump inventory to '{}' - {}".format( yaml_file, exc)) raise UserException( "Failed to dump inventory to '{}'".format(yaml_file))
def build_image(self): repo_name = self.DEFAULT_CONTAINER_NAME dockerfile_tag = sha1sum(self.depl_dockerfile_path) tag = f"{repo_name}:{dockerfile_tag}" try: self.client.images.get(tag) self.log.info(f"Using existing Docker image '{tag}'") except docker.errors.ImageNotFound: self.log.info(f"Building Docker image '{repo_name}'") try: self.image, build_logs = self.client.images.build( path=gen.get_package_path(), tag=tag, rm=True) except docker.errors.APIError as exc: msg = ("Failed to create image " f"'{self.DEFAULT_CONTAINER_NAME}': {exc}") self.log.error(msg) raise UserException(msg) self.log.debug("Created image " f"'{self.DEFAULT_CONTAINER_NAME}'") return tag
def _validate_dhcp_lease_time(self): """Validate DHCP lease time value Lease time can be given as an int (seconds), int + m (minutes), int + h (hours) or "infinite". Exception: Invalid lease time value """ dhcp_lease_time = self.cfg.get_globals_dhcp_lease_time() if not (re.match('^\d+[mh]{0,1}$', dhcp_lease_time) or dhcp_lease_time == "infinite"): exc = ("Config 'Globals: dhcp_lease_time: {}' has invalid value!" "\n".format(dhcp_lease_time)) exc += ('Value can be in seconds, minutes (e.g. "15m"),\n' 'hours (e.g. "1h") or "infinite" (lease does not expire).') raise UserException(exc)
def get_next_ip(self, reserve=True): """Get next available sequential IP address Args: reserve (bool): If true the IP will be considered reserved Returns: ip_address (str): Next IP address Raises: UserException: No more IP addresses available """ if self.next_ip == self.network.network + self.network.size: raise UserException('Not enough IP addresses in network \'%s\'' % str(self.network.cidr)) ip_address = str(self.next_ip) if reserve: self.next_ip += 1 return ip_address
def _validate_inventory_count(software_hosts_file_path, min_hosts, group='all'): """Validate minimum number of hosts are defined in inventory Calls Ansible to process inventory which validates file syntax. Args: software_hosts_file_path (str): Path to software inventory file min_hosts (int): Minimum number of hosts required to pass group (str, optional): Ansible group name (defaults to 'all') Returns: list: List of hosts defined in software inventory file Raises: UserException: Ansible reports host count of less than min_hosts """ log = logger.getlogger() host_count = None host_list = [] raw_host_list = bash_cmd(f'ansible {group} -i {software_hosts_file_path} ' '--list-hosts') # Iterate over ansible '--list-hosts' output count_verified = False host_count_pattern = re.compile(r'.*\((\d+)\)\:$') for host in raw_host_list.splitlines(): if not count_verified: # Verify host count is > 0 match = host_count_pattern.match(host) if match: host_count = int(match.group(1)) log.debug("Ansible host count: {}".format(host_count)) if host_count < min_hosts: raise UserException("Ansible reporting host count of less " "than one ({})!".format(host_count)) count_verified = True else: host_list.append(host.strip()) log.debug("Software inventory host count validation passed") log.debug("Ansible host list: {}".format(host_list)) return host_list
def _validate_installer_is_not_client(host_list): """Validate the installer node is not listed as a client Args: host_list (list): List of hostnames Returns: bool: True validation passes Raises: UserException: If installer is listed as client """ hostname = gethostname() fqdn = getfqdn() if hostname in host_list or fqdn in host_list: raise UserException('Installer can not be a target for install') else: return True
def __init__(self, dhcp_leases_file): dhcp_leases_file = os.path.abspath( os.path.dirname(os.path.abspath(dhcp_leases_file)) + os.path.sep + os.path.basename(dhcp_leases_file)) log = logger.getlogger() try: fds = open(dhcp_leases_file, 'r') except: msg = 'DHCP leases file not found: %s' log.error(msg % (dhcp_leases_file)) raise UserException(msg % dhcp_leases_file) self.mac_ip = AttrDict() for line in fds: match = re.search(r'^\S+\s+(\S+)\s+(\S+)', line) mac = match.group(1) ipaddr = match.group(2) self.mac_ip[mac] = ipaddr log.debug('Lease found - MAC: %s - IP: %s' % (mac, ipaddr))
def _assign_interface_ips(interfaces, interface_ip_lists): for interface in interfaces: list_key = '' if 'address' in interface.keys(): list_key = 'address' if 'IPADDR' in interface.keys(): list_key = 'IPADDR' if list_key: try: ip = interface_ip_lists[interface.label].pop(0) except IndexError: raise UserException("Not enough IP addresses listed for " "interface \'%s\'" % interface.label) if isinstance(ip, IPAddress): interface[list_key] = str(ip) interface_ip_lists[interface.label].append(IPAddress(ip + 1)) else: interface[list_key] = ip return interfaces, interface_ip_lists
def __init__(self, config_file=None): self.log = logger.getlogger() try: self.cfg = Config(config_file) self.inv = Inventory(None, config_file) except UserException as exc: self.log.critical(exc) raise UserException(exc) # initialize ipmi list of access info self.ran_ipmi = False self.bmc_ai = {} vlan_ipmi = self.cfg.get_depl_netw_client_vlan(if_type='ipmi')[0] vlan_pxe = self.cfg.get_depl_netw_client_vlan(if_type='pxe')[0] self.dhcp_pxe_leases_file = GEN_PATH + \ 'logs/dnsmasq{}.leases'.format(vlan_pxe) self.dhcp_ipmi_leases_file = GEN_PATH + \ 'logs/dnsmasq{}.leases'.format(vlan_ipmi) self.tcp_dump_file = GEN_PATH + \ 'logs/tcpdump{}.out'.format(vlan_pxe) self.node_table_ipmi = AttrDict() self.node_table_pxe = AttrDict() self.node_list = []
def get_user_and_home(): """Get user name and home directory path Returns the user account calling the script, *not* 'root' even when called with 'sudo'. Returns: user_name, user_home_dir (tuple): User name and home dir path Raises: UserException: If 'getent' command fails """ log = logger.getlogger() user_name = getlogin() cmd = f'getent passwd {user_name}' resp, err, rc = sub_proc_exec(cmd, shell=True) if str(rc) != "0": msg = 'getent failed:\n{}'.format(err) log.debug(msg) raise UserException(msg) user_home_dir = resp.split(':')[5].rstrip() return (user_name, user_home_dir)
def __init__(self, config_path=None, name=None): self.log = logger.getlogger() self.cfg = Config(config_path) self.cont_package_path = gen.get_container_package_path() self.cont_id_file = gen.get_container_id_file() self.cont_venv_path = gen.get_container_venv_path() self.cont_scripts_path = gen.get_container_scripts_path() self.cont_python_path = gen.get_container_python_path() self.cont_os_images_path = gen.get_container_os_images_path() self.cont_playbooks_path = gen.get_container_playbooks_path() self.depl_package_path = gen.get_package_path() self.depl_python_path = gen.get_python_path() self.depl_playbooks_path = gen.get_playbooks_path() self.cont_ini = os.path.join(self.depl_package_path, 'container.ini') self.rootfs = self.ROOTFS # Check if architecture is supported arch = platform.machine() if arch not in self.ARCHITECTURE.keys(): msg = "Unsupported architecture '{}'".format(arch) self.log.error(msg) raise UserException(msg) self.rootfs.arch = self.ARCHITECTURE[arch] if name is True or name is None: for vlan in self.cfg.yield_depl_netw_client_vlan('pxe'): break self.name = '{}-pxe{}'.format(self.DEFAULT_CONTAINER_NAME, vlan) else: self.name = name self.cont = lxc.Container(self.name) # Get a file descriptor for stdout self.fd = open(os.path.join(gen.GEN_LOGS_PATH, self.name + '.stdout.log'), 'w')