class NodeDbApi(TortugaDbApi): """ Nodes DB API class. """ def __init__(self): TortugaDbApi.__init__(self) self._nodesDbHandler = NodesDbHandler() self._globalParameterDbApi = GlobalParameterDbApi() def getNode(self, name: str, optionDict: Optional[Union[dict, None]] = None): """ Get node from the db. Returns: node Throws: NodeNotFound DbError """ session = DbManager().openSession() try: dbNode = self._nodesDbHandler.getNode(session, name) self.loadRelations(dbNode, optionDict) self.loadRelations(dbNode, { 'softwareprofile': True, 'hardwareprofile': True, 'tags': True, }) return Node.getFromDbDict(dbNode.__dict__) except TortugaException as ex: raise except Exception as ex: self.getLogger().exception('%s' % ex) raise finally: DbManager().closeSession() def getNodesByAddHostSession(self, ahSession): """ Get node(s) from db based their addhost session """ session = DbManager().openSession() try: return self.__convert_nodes_to_TortugaObjectList( self._nodesDbHandler.getNodesByAddHostSession( session, ahSession)) except TortugaException as ex: raise except Exception as ex: self.getLogger().exception('%s' % ex) raise finally: DbManager().closeSession() def getNodesByNameFilter(self, _filter): """ Get node(s) from db based on the name filter """ session = DbManager().openSession() try: dbNodes = self._nodesDbHandler.getNodesByNameFilter( session, _filter) return self.getTortugaObjectList(Node, dbNodes) except TortugaException as ex: raise except Exception as ex: self.getLogger().exception('%s' % ex) raise finally: DbManager().closeSession() def getNodeById(self, nodeId: int, optionDict: Optional[Union[dict, None]] = None): session = DbManager().openSession() try: dbNode = self._nodesDbHandler.getNodeById(session, nodeId) self.loadRelations(dbNode, optionDict) return Node.getFromDbDict(dbNode.__dict__) except TortugaException as ex: raise except Exception as ex: self.getLogger().exception('%s' % ex) raise finally: DbManager().closeSession() def getNodeByIp(self, ip): session = DbManager().openSession() try: node = self._nodesDbHandler.getNodeByIp(session, ip) return Node.getFromDbDict(node.__dict__) except TortugaException as ex: raise except Exception as ex: self.getLogger().exception('%s' % ex) raise finally: DbManager().closeSession() def __convert_nodes_to_TortugaObjectList( self, nodes, relations: Optional[Union[dict, None]] = None) -> TortugaObjectList: nodeList = TortugaObjectList() relations = relations or dict(softwareprofile=True, hardwareprofile=True) for t in nodes: self.loadRelations(t, relations) # Always load 'tags' relation self.loadRelations(t, {'tags': True}) node = Node.getFromDbDict(t.__dict__) nodeList.append(node) return nodeList def getNodeList(self, tags=None): """ Get list of all available nodes from the db. Returns: [node] Throws: DbError """ session = DbManager().openSession() try: return self.__convert_nodes_to_TortugaObjectList( self._nodesDbHandler.getNodeList(session, tags=tags)) except TortugaException as ex: raise except Exception as ex: self.getLogger().exception('%s' % ex) raise finally: DbManager().closeSession() def getProvisioningInfo(self, nodeName): """ Get the provisioing information for a given provisioned address Returns: [provisioningInformation structure] Throws: NodeNotFound DbError """ session = DbManager().openSession() try: provisioningInfo = ProvisioningInfo() dbNode = self._nodesDbHandler.getNode(session, nodeName) if dbNode.softwareprofile: self.loadRelations(dbNode.softwareprofile, { 'partitions': True, 'packages': True, }) for component in dbNode.softwareprofile.components: self.loadRelations( component, { 'kit': True, 'os': True, 'family': True, 'os_components': True, 'osfamily_components': True, }) self.loadRelation(dbNode, 'hardwareprofile') provisioningInfo.setNode(Node.getFromDbDict(dbNode.__dict__)) globalParameters = self._globalParameterDbApi.getParameterList() # TODO: this is a terrible hack until something better comes # along. p = Parameter() p.setName('Installer') hostName = socket.gethostname().split('.', 1)[0] if '.' in dbNode.name: nodeDomain = dbNode.name.split('.', 1)[1] priInstaller = hostName + '.%s' % (nodeDomain) else: priInstaller = hostName p.setValue(priInstaller) globalParameters.append(p) provisioningInfo.setGlobalParameters(globalParameters) return provisioningInfo except TortugaException as ex: raise except Exception as ex: self.getLogger().exception('%s' % ex) raise finally: DbManager().closeSession() def startupNode(self, nodespec, remainingNodeList=None, bootMethod='n'): """ Start Node """ session = DbManager().openSession() try: dbNodes = self.__expand_nodespec(session, nodespec) if not dbNodes: raise NodeNotFound('No matching nodes for nodespec [%s]' % (nodespec)) self._nodesDbHandler.startupNode( session, dbNodes, remainingNodeList=remainingNodeList or [], bootMethod=bootMethod) session.commit() except TortugaException as ex: session.rollback() raise except Exception as ex: session.rollback() self.getLogger().exception('%s' % ex) raise finally: DbManager().closeSession() def shutdownNode(self, nodespec, bSoftShutdown=False): """ Shutdown Node Raises: NodeNotFound """ session = DbManager().openSession() try: dbNodes = self.__expand_nodespec(session, nodespec) if not dbNodes: raise NodeNotFound('No matching nodes for nodespec [%s]' % (nodespec)) self._nodesDbHandler.shutdownNode(session, dbNodes, bSoftShutdown) session.commit() except TortugaException as ex: session.rollback() raise except Exception as ex: session.rollback() self.getLogger().exception('%s' % ex) raise finally: DbManager().closeSession() def __expand_nodespec(self, session, nodespec): # Expand wildcards in nodespec. Each token in the nodespec can # be wildcard that expands into one or more nodes. filter_spec = [] for nodespec_token in nodespec.split(','): # Convert shell-style wildcards into SQL wildcards if '*' in nodespec_token or '?' in nodespec_token: filter_spec.append( nodespec_token.replace('*', '%').replace('?', '_')) continue # Add nodespec "AS IS" filter_spec.append(nodespec_token) return self._nodesDbHandler.getNodesByNameFilter(session, filter_spec) def evacuateChildren(self, nodeName): """ Evacuate Children of node """ session = DbManager().openSession() try: dbNode = self._nodesDbHandler.getNode(session, nodeName) self._nodesDbHandler.evacuateChildren(session, dbNode) session.commit() except TortugaException as ex: session.rollback() raise except Exception as ex: session.rollback() self.getLogger().exception('%s' % ex) raise finally: DbManager().closeSession() def getChildrenList(self, nodeName): """ Get children of node Raises: NodeNotFound """ session = DbManager().openSession() try: dbNode = self._nodesDbHandler.getNode(session, nodeName) return self.getTortugaObjectList(Node, dbNode.children) except TortugaException as ex: raise except Exception as ex: self.getLogger().exception('%s' % ex) raise finally: DbManager().closeSession() def checkpointNode(self, nodeName): """ Checkpoint Node """ session = DbManager().openSession() try: self._nodesDbHandler.checkpointNode(session, nodeName) session.commit() except TortugaException as ex: session.rollback() raise except Exception as ex: session.rollback() self.getLogger().exception('%s' % ex) raise finally: DbManager().closeSession() def revertNodeToCheckpoint(self, nodeName): """ Revert Node to Checkpoint """ session = DbManager().openSession() try: self._nodesDbHandler.revertNodeToCheckpoint(session, nodeName) session.commit() except TortugaException as ex: session.rollback() raise except Exception as ex: session.rollback() self.getLogger().exception('%s' % ex) raise finally: DbManager().closeSession() def migrateNode(self, nodeName, remainingNodeList, liveMigrate): """ Migrate Node """ session = DbManager().openSession() try: self._nodesDbHandler.migrateNode(session, nodeName, remainingNodeList, liveMigrate) session.commit() except TortugaException as ex: session.rollback() raise except Exception as ex: session.rollback() self.getLogger().exception('%s' % ex) raise finally: DbManager().closeSession() def setParentNode(self, nodeName, parentNodeName): ''' Raises: NodeNotFound ''' session = DbManager().openSession() try: dbNode = self._nodesDbHandler.getNode(session, nodeName) # Setting the parent to 'None' is equivalent to unsetting it dbNode.parentnode = self._nodesDbHandler.getNode( session, parentNodeName) if parentNodeName else None session.commit() except TortugaException as ex: session.rollback() raise except Exception as ex: session.rollback() self.getLogger().exception('%s' % ex) raise finally: DbManager().closeSession() def getNodesByNodeState(self, state): session = DbManager().openSession() try: return self.getTortugaObjectList( Node, self._nodesDbHandler.getNodesByNodeState(session, state)) except TortugaException as ex: raise except Exception as ex: self.getLogger().exception('%s' % (ex)) raise finally: DbManager().closeSession()
class AddHostServerLocal(TortugaApi): def __init__(self): super(AddHostServerLocal, self).__init__() self._nodesDbHandler = NodesDbHandler() self._logger = logging.getLogger(ADD_HOST_NAMESPACE) @staticmethod def clear_session_nodes(nodes: List[Node]) -> None: """Remove session entries for specified nodes""" with session_nodes_lock: for node in nodes: AddHostServerLocal.clear_session_node(node, lock=False) @staticmethod def clear_session_node(node: Node, lock: bool = True) -> None: if lock: session_nodes_lock.acquire() try: if not node.name: return hostname = get_host_name(node.name) logger.debug('clear_session_node(): session_nodes=[{0}]'.format( ' '.join([session_node for session_node in session_nodes]))) if hostname in session_nodes: logger.debug( 'DELETING session_nodes entry: {0}'.format(hostname)) session_nodes.remove(hostname) prov_nics = get_provisioning_nics(node) if prov_nics: if prov_nics[0].ip in reservedIps: reservedIps.remove(prov_nics[0].ip) finally: if lock: session_nodes_lock.release() def initializeNode(self, session: Session, dbNode: Node, dbHardwareProfile: HardwareProfile, dbSoftwareProfile: Optional[SoftwareProfile], nic_defs: List[dict], bValidateIp: bool = True, bGenerateIp: bool = True, dns_zone: Optional[str] = None) -> None: \ # pylint: disable=unused-argument """ Assigns hostname and IP address, and inserts new record into Node table. Raises: InvalidArgument NodeAlreadyExists """ # Do not attempt to validate IP addresses for hardware profiles # representing remote nodes. bValidateIp &= dbHardwareProfile.location != 'remote' if dbHardwareProfile.location != 'remote' and \ not dbHardwareProfile.hardwareprofilenetworks: raise NetworkNotFound( 'Hardware profile [{}] does not have a provisioning' ' network'.format(dbHardwareProfile.name)) try: if not dbNode.name: # Generate unique name for new node dbNode.name = self.generate_node_name( session, dbHardwareProfile.nameFormat, rackNumber=dbNode.rack, dns_zone=dns_zone) # Create NIC entries dbNode.nics = self._initializeNics(session, dbNode, dbHardwareProfile, nic_defs, bValidateIp=bValidateIp, bGenerateIp=bGenerateIp) self._logger.debug('initializeNode(): initialized new node [%s]' % (dbNode.name)) except Exception: with session_nodes_lock: self.clear_session_node(dbNode) raise def generate_node_name(self, session: Session, nameFormat: str, rackNumber: Optional[str] = None, randomize: bool = False, dns_zone: Optional[str] = None) -> str: ''' Generate unique node name for the specified nameFormat. Raises: InvalidArgument ''' try: base_name = nameFormat if rackNumber is None else \ self._substituteHashSpecifier( nameFormat, '#R', rackNumber) # Find all pre-existing nodes + nodes in the session if not randomize: nodes = self._nodesDbHandler.getNodesByNameFilter( session, self._substituteHashSpecifier(base_name, '#N', '_')) # Get list of all existing nodes names node_names = [get_host_name(tmpNode.name) for tmpNode in nodes] else: # Get all nodes matching name format WITH a random suffix nodes = self._nodesDbHandler.getNodesByNameFilter( session, self._substituteHashSpecifier(base_name, '#N', '_') + '-_____') node_names = [ strip_random_node_name_suffix(get_host_name(node.name)) for node in nodes ] with session_nodes_lock: # Build a list of all nodes in database and session if randomize: all_session_nodes = node_names + \ strip_random_node_name_suffixes(session_nodes) else: all_session_nodes = node_names + session_nodes for slot in itertools.count(1): name = self._substituteHashSpecifier(base_name, '#N', slot) if name not in all_session_nodes: break if randomize: # Add random 5 letter suffix to generated host name name += '-%s' % (''.join( random.sample(string.ascii_lowercase, 5))) with session_nodes_lock: # Add only host name to session_nodes cache session_nodes.append(name) return '{}.{}'.format(name, dns_zone) if dns_zone else name except InvalidArgument as exc: raise InvalidArgument('%s (format=[%s])' % (exc, nameFormat)) def _substituteHashSpecifier(self, s, specifier, replacement): ''' Replace the given specifier, '#R' or '#N', with the given replacement, which is either an integer (rack number or node number), or the string literal '_'. Don't touch this unless you're willing to rewrite it! Raises: InvalidArgument ''' # Check arguments. if specifier not in ('#R', '#N'): raise InvalidArgument('specifier must be one of "#R" or "#N"') if isinstance(replacement, str): if replacement != '_': raise InvalidArgument( 'replacement must be "_" when type == str') elif not isinstance(replacement, int): raise InvalidArgument('replacement must an integer or "_"') # Point to the '#' (of '#R' or '#N') in the string. hashIdx = s.find(specifier) if hashIdx < 0: return s # Capture the portion to the left of the specifier. left = s[:hashIdx] # Point to the last 'R' or 'N' in the specifier. lastIdx = hashIdx + 1 finalIdx = len(s) - 1 while lastIdx < finalIdx and s[lastIdx + 1] == specifier[1]: lastIdx += 1 # The width of the specifier (without the '#'). width = lastIdx - hashIdx if replacement != '_' and replacement > (10**width - 1): raise InvalidArgument('Unable to generate unique host name') # Capture the portion to the right of the specifier. right = s[lastIdx + 1:] # Put it together. s = left if replacement == '_': s += '_' * width else: s += '%0*d' % (width, replacement) s += right return s def _initializeNics(self, session: Session, dbNode: Node, dbHardwareProfile: HardwareProfile, nic_defs: List[dict], bValidateIp: bool = True, bGenerateIp: bool = True) -> List[Nic]: """ Return list of Nic objects reflecting the configuration of dbNode and nic definitions provided in nic_defs. :raises NetaNotFound: :raises InvalidMacAddress: :raises MacAddressAlreadyExists: """ nics = [] hwpnetworks = dbHardwareProfile.hardwareprofilenetworks[:] hwpnetworks.sort(key=lambda a: a.networkdevice.name) for nic_def, dbHardwareProfileNetwork in itertools.zip_longest( nic_defs, hwpnetworks, fillvalue=None): # Create a nic for each associated hardware profile network dbNic = Nic() dbNic.node = dbNode if nic_def and 'mac' in nic_def: # # MAC addresses are generated for virtualization platforms # such as libvirt and VMware # dbNic.mac = self._validate_mac_address( session, nic_def['mac'], dbHardwareProfileNetwork.network) # # Validate IP, if specified, otherwise generate an IP, if # requested # if nic_def and 'ip' in nic_def: if bValidateIp: self._validate_ip_address(nic_def['ip'], dbHardwareProfileNetwork.network) dbNic.ip = nic_def['ip'] else: if dbHardwareProfile.location == 'local' and \ not dbHardwareProfileNetwork: raise NicNotFound( 'Hardware profile [%s] does not have a provisioning' ' network' % (dbHardwareProfile.name)) if bGenerateIp and \ dbHardwareProfileNetwork.network.type == 'provision': # Generate an IP address for the specified nic dbNic.ip = self.generate_provisioning_ip_address( dbHardwareProfileNetwork.network) self._logger.debug('Generated IP [%s] for node [%s]' % (dbNic.ip, dbNode.name)) if dbNic.ip or \ dbHardwareProfileNetwork and \ dbHardwareProfileNetwork.network.type != 'provision': # Only add a network and network device for nodes that have # managed IP addresses. if dbHardwareProfileNetwork: dbNic.network = dbHardwareProfileNetwork.network dbNic.networkdevice = \ dbHardwareProfileNetwork.networkdevice # Set the 'boot' flag if this is a provisioning network dbNic.boot = dbNic.network and dbNic.network.type == 'provision' if dbNic.ip: reservedIps.append(dbNic.ip) nics.append(dbNic) return nics def _validate_mac_address(self, session: Session, mac_address: str, network: Network) -> str: """ Validate a MAC address and make sure it is not a duplicate. :raises InvalidMacAddress: :raises MacAddressAlreadyExists: """ if not mac_address: raise InvalidMacAddress('MAC address is empty/undefined') # # Normalize and Validate MAC address formatting # mac_regex = re.compile( r'^([0-9A-Fa-f]{2}[:-]?){5}[0-9A-Fa-f]{2}([:-]?.*)?$') if not mac_regex.match(mac_address): raise InvalidMacAddress( 'MAC address [{}] is invalid/malformed'.format(mac_address)) if '-' in mac_address: mac_address = mac_address.replace('-', ':').lower() if ':' not in mac_address: mac_address = '{}:{}:{}:{}:{}:{}'.format( mac_address[0:2], mac_address[2:4], mac_address[4:6], mac_address[6:8], mac_address[8:10], mac_address[10:12]) # # Make sure that the MAC address does not already exist on the # same provisioning network # db_host_list = self._nodesDbHandler.getNodesByMac( session, [mac_address]) for db_host in db_host_list: for db_nic in db_host.nics: if db_nic.mac == mac_address and \ db_nic.networkId == network.id: raise MacAddressAlreadyExists( 'The MAC address [{}] already exists ' 'on the network [{}/{}]'.format( mac_address, network.address, network.netmask)) return mac_address def _validate_ip_address(self, ip, network) -> None: """ :raises NetworkNotFound: """ # # Only validate IP addresses in 'local' hardware profiles # try: requested_ip = ipaddress.IPv4Address(str(ip)) provisioning_network = ipaddress.IPv4Network('{}/{}'.format( network.address, network.netmask)) if requested_ip not in provisioning_network: raise NetworkNotFound( 'IP address [{}] not on network [{}]'.format( ip, provisioning_network)) except ipaddress.AddressValueError: # # malformed ip address # raise NetworkNotFound('IP address [{}] is invalid'.format(ip)) def generate_provisioning_ip_address(self, network): """ Raises: InvalidArgument """ if not network or network.usingDhcp: # This hardwareProfile uses an external DHCP server # (we do not assign the IP address for this hardwareProfile.) return None n = ipaddress.IPv4Network('%s/%s' % (network.address, network.netmask)) # Get list of all currently allocated IPs on this network ips = [ ipaddress.IPv4Address(str(dbNic.ip)) for dbNic in network.nics if dbNic and dbNic.ip ] with session_nodes_lock: # Include reserved IPs ips.extend([ipaddress.IPv4Address(str(ip)) for ip in reservedIps]) if not network.startIp: # Assume the starting IP address is the first IP address in # the subnet try: ip = n[1] except IndexError: raise InvalidArgument('IP address space exhausted') else: ip = ipaddress.IPv4Address(str(network.startIp)) # Ensure there's a valid increment if none is defined in the # network object. inc = network.increment if network.increment else 1 for _ in range(n.num_addresses): if ip not in ips: # Found available IP address within address space break ip += int(inc) else: raise InvalidArgument('IP address space exhausted') if ip == n.broadcast_address: raise InvalidArgument('IP address space exhausted') reservedIps.append(ip.exploded) self._logger.debug('Assigning IP address [%s] on network [%s]' % (ip.exploded, str(n))) return ip.exploded