def getQueue(site, ce, queue): """ Get parameters of the specified queue """ grid = site.split('.')[0] result = gConfig.getOptionsDict('/Resources/Sites/%s/%s/CEs/%s' % (grid, site, ce)) if not result['OK']: return result resultDict = result['Value'] # Get queue defaults result = gConfig.getOptionsDict('/Resources/Sites/%s/%s/CEs/%s/Queues/%s' % (grid, site, ce, queue)) if not result['OK']: return result resultDict.update(result['Value']) # Handle tag lists for the queue for tagFieldName in ('Tag', 'RequiredTag'): tags = [] ceTags = resultDict.get(tagFieldName) if ceTags: tags = fromChar(ceTags) queueTags = resultDict.get(tagFieldName) if queueTags: queueTags = fromChar(queueTags) tags = list(set(tags + queueTags)) if tags: resultDict[tagFieldName] = tags resultDict['Queue'] = queue return S_OK(resultDict)
def _handleDestination( self, paramsDict, getSitesForSE = None ): """ Handle Sites and TargetSE in the parameters """ try: sites = ['ANY'] if paramsDict['Site']: # 'Site' comes from the XML and therefore is ; separated sites = fromChar( paramsDict['Site'], sepChar = ';' ) except KeyError: pass try: seList = ['Unknown'] if paramsDict['TargetSE']: seList = fromChar( paramsDict['TargetSE'] ) except KeyError: pass if not seList or seList == ['Unknown']: return sites # from now on we know there is some TargetSE requested if not getSitesForSE: from DIRAC.Core.Utilities.SiteSEMapping import getSitesForSE seSites = [] for se in seList: res = getSitesForSE( se ) if not res['OK']: self.log.warn( 'Could not get Sites associated to SE', res['Message'] ) else: thisSESites = res['Value'] if not thisSESites: continue if seSites == []: seSites = copy.deepcopy( thisSESites ) else: # We make an OR of the possible sites for nSE in list( thisSESites ): if nSE not in seSites: seSites.append( nSE ) # Now we need to make the AND with the sites, if defined if sites == ['ANY']: return seSites else: # Need to get the AND for nSE in list( seSites ): if nSE not in sites: seSites.remove( nSE ) return seSites
def _BySE( self ): """ Matches using TargetSE. This is the standard plugin. """ destSites = set() try: seList = ['Unknown'] if self.params['TargetSE']: if isinstance( self.params['TargetSE'], basestring ): seList = fromChar( self.params['TargetSE'] ) elif isinstance( self.params['TargetSE'], list ): seList = self.params['TargetSE'] except KeyError: pass if not seList or seList == ['Unknown']: return destSites for se in seList: res = getSitesForSE( se ) if not res['OK']: gLogger.warn( "Could not get Sites associated to SE", res['Message'] ) else: thisSESites = res['Value'] if thisSESites: # We make an OR of the possible sites destSites.update( thisSESites ) gLogger.debug( "Destinations: %s" % ','.join ( destSites ) ) return destSites
def _handleDestination( self, paramsDict ): """ Handle Sites and TargetSE in the parameters """ try: sites = ['ANY'] if paramsDict['Site']: # 'Site' comes from the XML and therefore is ; separated sites = fromChar( paramsDict['Site'], sepChar = ';' ) except KeyError: pass if self.destinationPlugin_o: destinationPlugin_o = self.destinationPlugin_o else: res = self.__generatePluginObject( self.destinationPlugin ) if not res['OK']: self._logFatal( "Could not generate a destination plugin object" ) return res destinationPlugin_o = res['Value'] destinationPlugin_o.setParameters( paramsDict ) destSites = destinationPlugin_o.run() if not destSites: return sites # Now we need to make the AND with the sites, if defined if sites != ['ANY']: # Need to get the AND destSites &= set( sites ) return list( destSites )
def __init__( self, useCertificates = False ): """c'tor :param self: self reference :param bool useCertificates: flag to enable/disable certificates """ Client.__init__( self ) ## setup logger self.log = gLogger.getSubLogger( "RequestManagement/RequestClient" ) ## dict to store all RPC clients for easy reuse self.__requestRPCClientsDict = {} ## local if any defined local = PathFinder.getServiceURL( "RequestManagement/localURL" ) if local: self.__requestRPCClientsDict.setdefault( "local" , [ self.__requestRPCClient( local ) ] ) ## central if any defined central = PathFinder.getServiceURL( "RequestManagement/centralURL" ) if central: self.__requestRPCClientsDict.setdefault( "central", [ self.__requestRPCClient( central ) ] ) ## voboxes if any defined voBoxUrls = fromChar( PathFinder.getServiceURL( "RequestManagement/voBoxURLs" ) ) if voBoxUrls: self.__requestRPCClientsDict.setdefault( "voboxes", [] ) for voBoxURL in randomize( voBoxUrls ): self.__requestRPCClientsDict["voboxes"].append( self.__requestRPCClient( voBoxURL ) ) self.setServer( 'RequestManagement/centralURL' )
def requestProxies( self, timeout = 120 ): """ get request proxies dict """ if not self.__requestProxiesDict: self.__requestProxiesDict = {} proxiesURLs = fromChar( PathFinder.getServiceURL( "RequestManagement/ReqProxyURLs" ) ) if not proxiesURLs: self.log.warn( "CS option RequestManagement/ReqProxyURLs is not set!" ) for proxyURL in proxiesURLs: self.log.debug( "creating RequestProxy for url = %s" % proxyURL ) self.__requestProxiesDict[proxyURL] = RPCClient( proxyURL, timeout = timeout ) return self.__requestProxiesDict
def getNumberOfPayloadProcessors(siteName=None, gridCE=None, queue=None): """ Gets the number of processors allowed for a single JobAgent (so for a "inner" CE). (NB: this does not refer to the job processors). This is normally used ONLY when a pilot instantiates more than one JobAgent (MultiLaunchAgent pilot command). The siteName/gridCE/queue parameters are normally not necessary. Tries to find it in this order: 1) from the /Resources/Computing/CEDefaults/NumberOfPayloadProcessors (which is what pilot 3 fills up) 2) if not present but there's WholeNode tag, use the getNumberOfProcessors function above 3) otherwise returns 1 """ # 1) from /Resources/Computing/CEDefaults/NumberOfPayloadProcessors gLogger.info( "Getting NumberOfPayloadProcessors from /Resources/Computing/CEDefaults/NumberOfPayloadProcessors" ) NumberOfPayloadProcessors = gConfig.getValue( '/Resources/Computing/CEDefaults/NumberOfPayloadProcessors') if NumberOfPayloadProcessors: return NumberOfPayloadProcessors # 2) Checks if 'Whole' is one of the used tags # Tags of the CE tags = fromChar( gConfig.getValue( '/Resources/Sites/%s/%s/CEs/%s/Tag' % (siteName.split('.')[0], siteName, gridCE), '')) # Tags of the Queue tags += fromChar( gConfig.getValue( '/Resources/Sites/%s/%s/CEs/%s/Queues/%s/Tag' % (siteName.split('.')[0], siteName, gridCE, queue), '')) if 'WholeNode' in tags: return getNumberOfProcessors() # 3) Just returns a conservative "1" return 1
def getVOUserReport( self, vo ): """ :param str vo: VO name :return: report string """ result = self.getVOUserData( vo, refreshFlag = True ) if not result['OK']: return result userDict = result['Value'] # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping( vo ) if not result['OK']: return result diracVOMSMapping = result['Value']['DIRACVOMS'] records = [] groupDict = defaultdict( int ) multiDNUsers = {} suspendedUsers = [] for user in userDict: for group in userDict[user]['Groups']: groupDict[group] += 1 dnList = fromChar( userDict[user]['DN'] ) if len( dnList ) > 1: multiDNUsers[user] = dnList if userDict[user].get( 'Status', 'Active' ) == 'Suspended': suspendedUsers.append( user ) for group in diracVOMSMapping: records.append( ( group, str( groupDict[group] ), diracVOMSMapping.get( group, '' ) ) ) fields = [ 'Group', 'Number of users', 'VOMS Role' ] output = printTable( fields, records, sortField = 'Group', printOut = False, numbering = False ) if multiDNUsers: output += '\nUsers with multiple DNs:\n' for user in multiDNUsers: output += ' %s:\n' % user for dn in multiDNUsers[user]: output += ' %s\n' % dn if suspendedUsers: output += '\n%d suspended users:\n' % len( suspendedUsers ) output += ' %s' % ','.join( suspendedUsers ) return S_OK( output )
def getVOUserReport(self): """Get a report string with the current status of the DIRAC Registry for the Virtual Organization :return: S_OK with the report string as Value """ result = self.getVOUserData(refreshFlag=True) if not result["OK"]: return result userDict = result["Value"] # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping(self.vo) if not result["OK"]: return result diracVOMSMapping = result["Value"]["DIRACVOMS"] records = [] groupDict = defaultdict(int) multiDNUsers = {} suspendedUsers = [] for user in userDict: for group in userDict[user]["Groups"]: groupDict[group] += 1 dnList = fromChar(userDict[user]["DN"]) if len(dnList) > 1: multiDNUsers[user] = dnList if userDict[user].get("Status", "Active") == "Suspended": suspendedUsers.append(user) for group in diracVOMSMapping: records.append((group, str(groupDict[group]), diracVOMSMapping.get(group, ""))) fields = ["Group", "Number of users", "VOMS Role"] output = printTable(fields, records, sortField="Group", printOut=False, numbering=False) if multiDNUsers: output += "\nUsers with multiple DNs:\n" for user in multiDNUsers: output += " %s:\n" % user for dn in multiDNUsers[user]: output += " %s\n" % dn if suspendedUsers: output += "\n%d suspended users:\n" % len(suspendedUsers) output += " %s" % ",".join(suspendedUsers) return S_OK(output)
def requestProxies(self, timeout=120): """ get request proxies dict """ if not self.__requestProxiesDict: self.__requestProxiesDict = {} proxiesURLs = fromChar( PathFinder.getServiceURL("RequestManagement/ReqProxyURLs")) if not proxiesURLs: self.log.warn( "CS option RequestManagement/ReqProxyURLs is not set!") for proxyURL in proxiesURLs: self.log.debug("creating RequestProxy for url = %s" % proxyURL) self.__requestProxiesDict[proxyURL] = RPCClient( proxyURL, timeout=timeout) return self.__requestProxiesDict
def getVOUserReport(self): """ Get a report string with the current status of the DIRAC Registry for the Virtual Organization :return: S_OK with the report string as Value """ result = self.getVOUserData(refreshFlag=True) if not result['OK']: return result userDict = result['Value'] # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping(self.vo) if not result['OK']: return result diracVOMSMapping = result['Value']['DIRACVOMS'] records = [] groupDict = defaultdict(int) multiDNUsers = {} suspendedUsers = [] for user in userDict: for group in userDict[user]['Groups']: groupDict[group] += 1 dnList = fromChar(userDict[user]['DN']) if len(dnList) > 1: multiDNUsers[user] = dnList if userDict[user].get('Status', 'Active') == 'Suspended': suspendedUsers.append(user) for group in diracVOMSMapping: records.append((group, str(groupDict[group]), diracVOMSMapping.get(group, ''))) fields = ['Group', 'Number of users', 'VOMS Role'] output = printTable(fields, records, sortField='Group', printOut=False, numbering=False) if multiDNUsers: output += '\nUsers with multiple DNs:\n' for user in multiDNUsers: output += ' %s:\n' % user for dn in multiDNUsers[user]: output += ' %s\n' % dn if suspendedUsers: output += '\n%d suspended users:\n' % len(suspendedUsers) output += ' %s' % ','.join(suspendedUsers) return S_OK(output)
def getQueue( site, ce, queue ): """ Get parameters of the specified queue """ Tags = [] grid = site.split( '.' )[0] result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/CEs/%s' % ( grid, site, ce ) ) if not result['OK']: return result resultDict = result['Value'] ceTags = resultDict.get( 'Tag' ) if ceTags: Tags = fromChar( ceTags ) result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/CEs/%s/Queues/%s' % ( grid, site, ce, queue ) ) if not result['OK']: return result resultDict.update( result['Value'] ) queueTags = resultDict.get( 'Tag' ) if queueTags: queueTags = fromChar( queueTags ) Tags = list( set( Tags + queueTags ) ) if Tags: resultDict['Tag'] = Tags resultDict['Queue'] = queue return S_OK( resultDict )
def requestProxies(self, timeout=120): """get request proxies dict""" # Forward all the connection options to the requestClient # (e.g. the userDN to use) kwargs = self.getClientKWArgs() kwargs["timeout"] = timeout if not self.__requestProxiesDict: self.__requestProxiesDict = {} proxiesURLs = fromChar(PathFinder.getServiceURL("RequestManagement/ReqProxyURLs")) if not proxiesURLs: self.log.warn("CS option RequestManagement/ReqProxyURLs is not set!") for proxyURL in proxiesURLs: self.log.debug("creating RequestProxy for url = %s" % proxyURL) pc = Client(**kwargs) pc.setServer(proxyURL) self.__requestProxiesDict[proxyURL] = pc return self.__requestProxiesDict
def __init__( self, useCertificates = False ): """ Constructor of the RequestClient class """ voBoxUrls = fromChar( PathFinder.getServiceURL( "RequestManagement/voBoxURLs" ) ) self.voBoxUrls = [] if voBoxUrls: self.voBoxUrls = randomize( voBoxUrls ) local = PathFinder.getServiceURL( "RequestManagement/localURL" ) self.local = False if local: self.local = local central = PathFinder.getServiceURL( "RequestManagement/centralURL" ) self.central = False if central: self.central = central self.setServer( 'RequestManagement/centralURL' )
def __init__(self, useCertificates=False): """ Constructor of the RequestClient class """ voBoxUrls = fromChar( PathFinder.getServiceURL("RequestManagement/voBoxURLs")) self.voBoxUrls = [] if voBoxUrls: self.voBoxUrls = randomize(voBoxUrls) local = PathFinder.getServiceURL("RequestManagement/localURL") self.local = False if local: self.local = local central = PathFinder.getServiceURL("RequestManagement/centralURL") self.central = False if central: self.central = central self.setServer('RequestManagement/centralURL')
def getJobInformation(self, diracAPI, jobMon, jdlOnly=False): """Get all the information for the job. The InputData, TaskID, OutputData can either be taken from properly filled JDL or * inputData from jobMonitor getInputData * TaskID from the name of The job via jobMonitor getJobAttribute JobName * ProductionOutputData: from jobMonitor getJobParameter ProductionOutputData This would be faster if we could do bulk calls for all of these """ if not jdlOnly: # this is actually slower than just getting the jdl, because getting the jdl is one service call # this is three service calls to three different DBs resInputData = jobMon.getInputData(self.jobID) if resInputData['OK']: self.inputFiles = resInputData['Value'] resName = jobMon.getJobAttribute(self.jobID, 'JobName') if resName['OK'] and '_' in resName['Value']: self.__getTaskID(resName['Value']) resOutput = jobMon.getJobParameter(self.jobID, 'ProductionOutputData') if resOutput['OK']: self.outputFiles = fromChar(resOutput['Value'].get( 'ProductionOutputData', '')) if not self.outputFiles: LOG.verbose('Did not find outputFiles for', str(self)) if not (self.inputFiles and self.outputFiles and self.taskID): LOG.verbose('Have to check JDL') jdlParameters = self.__getJDL(diracAPI) # get taskID from JobName, get inputfile(s) from DownloadInputdata self.__getOutputFiles(jdlParameters) self.__getTaskID(jdlParameters) if not self.inputFiles: self.__getInputFile(jdlParameters)
def _setSystems(self, val): self.systems = fromChar(val) return S_OK()
def getNumberOfProcessors(siteName=None, gridCE=None, queue=None): """gets the number of processors on a certain CE/queue/node (what the pilot administers) The siteName/gridCE/queue parameters are normally not necessary. Tries to find it in this order: 1) from the /Resources/Computing/CEDefaults/NumberOfProcessors (which is what the pilot fills up) 2) if not present from JobFeatures 3) if not present looks in CS for "NumberOfProcessors" Queue or CE option 4) if not present but there's WholeNode tag, look what the WN provides using multiprocessing.cpu_count() 5) return 1 """ # 1) from /Resources/Computing/CEDefaults/NumberOfProcessors gLogger.info("Getting numberOfProcessors from /Resources/Computing/CEDefaults/NumberOfProcessors") numberOfProcessors = gConfig.getValue("/Resources/Computing/CEDefaults/NumberOfProcessors", 0) if numberOfProcessors: return numberOfProcessors # 2) from MJF gLogger.info("Getting numberOfProcessors from MJF") numberOfProcessors = getProcessorFromMJF() if numberOfProcessors: return numberOfProcessors gLogger.info("NumberOfProcessors could not be found in MJF") # 3) looks in CS for "NumberOfProcessors" Queue or CE or site option if not siteName: siteName = gConfig.getValue("/LocalSite/Site", "") if not gridCE: gridCE = gConfig.getValue("/LocalSite/GridCE", "") if not queue: queue = gConfig.getValue("/LocalSite/CEQueue", "") if not (siteName and gridCE and queue): gLogger.error("Could not find NumberOfProcessors: missing siteName or gridCE or queue. Returning '1'") return 1 grid = siteName.split(".")[0] csPaths = [ "/Resources/Sites/%s/%s/CEs/%s/Queues/%s/NumberOfProcessors" % (grid, siteName, gridCE, queue), "/Resources/Sites/%s/%s/CEs/%s/NumberOfProcessors" % (grid, siteName, gridCE), "/Resources/Sites/%s/%s/Cloud/%s/VMTypes/%s/NumberOfProcessors" % (grid, siteName, gridCE, queue), "/Resources/Sites/%s/%s/Cloud/%s/NumberOfProcessors" % (grid, siteName, gridCE), "/Resources/Sites/%s/%s/NumberOfProcessors" % (grid, siteName), ] for csPath in csPaths: gLogger.info("Looking in", csPath) numberOfProcessors = gConfig.getValue(csPath, 0) if numberOfProcessors: return numberOfProcessors # 4) looks in CS for tags gLogger.info("Getting tags" "for %s: %s: %s" % (siteName, gridCE, queue)) # Tags of the CE tags = fromChar( gConfig.getValue("/Resources/Sites/%s/%s/CEs/%s/Tag" % (siteName.split(".")[0], siteName, gridCE), "") ) + fromChar( gConfig.getValue("/Resources/Sites/%s/%s/Cloud/%s/Tag" % (siteName.split(".")[0], siteName, gridCE), "") ) # Tags of the Queue tags += fromChar( gConfig.getValue( "/Resources/Sites/%s/%s/CEs/%s/Queues/%s/Tag" % (siteName.split(".")[0], siteName, gridCE, queue), "" ) ) + fromChar( gConfig.getValue( "/Resources/Sites/%s/%s/Cloud/%s/VMTypes/%s/Tag" % (siteName.split(".")[0], siteName, gridCE, queue), "" ) ) gLogger.info("NumberOfProcessors could not be found in CS") if "WholeNode" in tags: gLogger.info("Found WholeNode tag, using multiprocessing.cpu_count()") return multiprocessing.cpu_count() # 5) return the default return 1
def syncCSWithVOMS(self): """ Performs the synchronization of the DIRAC registry with the VOMS data. The resulting CSAPI object containing modifications is returned as part of the output dictionary. Those changes can be applied by the caller depending on the mode (dry or a real run) :return: S_OK with a dictionary containing the results of the synchronization operation """ resultDict = defaultdict(list) # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping(self.vo) if not result['OK']: return result vomsDIRACMapping = result['Value']['VOMSDIRAC'] diracVOMSMapping = result['Value']['DIRACVOMS'] noVOMSGroups = result['Value']['NoVOMS'] noSyncVOMSGroups = result['Value']['NoSyncVOMS'] vomsSrv = VOMSService(self.vo) # Get VOMS users result = vomsSrv.getUsers() if not result['OK']: self.log.error('Could not retrieve user information from VOMS', result['Message']) return result self.vomsUserDict = result['Value'] message = "There are %s user entries in VOMS for VO %s" % (len( self.vomsUserDict), self.vomsVOName) self.adminMsgs['Info'].append(message) self.log.info(message) # Get DIRAC users result = self.getVOUserData(self.vo) if not result['OK']: return result diracUserDict = result['Value'] self.adminMsgs['Info'].append( "There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), self.vo)) self.log.info("There are %s registered users in DIRAC VO %s" % (len(diracUserDict), self.vo)) # Find new and obsoleted user DNs existingDNs = [] obsoletedDNs = [] newDNs = [] for user in diracUserDict: dn = diracUserDict[user]['DN'] # We can have users with more than one DN registered dnList = fromChar(dn) existingDNs.extend(dnList) for dn in dnList: if dn not in self.vomsUserDict: obsoletedDNs.append(dn) for dn in self.vomsUserDict: if dn not in existingDNs: newDNs.append(dn) allDiracUsers = getAllUsers() nonVOUserDict = {} nonVOUsers = list(set(allDiracUsers) - set(diracUserDict)) if nonVOUsers: result = self.csapi.describeUsers(nonVOUsers) if not result['OK']: self.log.error('Could not retrieve CS User description') return result nonVOUserDict = result['Value'] # Process users defaultVOGroup = getVOOption(self.vo, "DefaultGroup", "%s_user" % self.vo) # If a user is (previously put by hand) in a "QuarantineGroup", # then the default group will be ignored. # So, this option is only considered for the case of existing users. quarantineVOGroup = getVOOption(self.vo, "QuarantineGroup") newAddedUserDict = {} for dn in self.vomsUserDict: newDNForExistingUser = '' diracName = '' if dn in existingDNs: for user in diracUserDict: if dn == diracUserDict[user]['DN']: diracName = user if dn in newDNs: # Find if the DN is already registered in the DIRAC CS for user in nonVOUserDict: if dn == nonVOUserDict[user]['DN']: diracName = user # Check the nickName in the same VO to see if the user is already registered # with another DN nickName = self.vomsUserDict[dn].get('nickname') if nickName in diracUserDict or nickName in newAddedUserDict: diracName = nickName # This is a flag for adding the new DN to an already existing user newDNForExistingUser = dn # We have a real new user if not diracName: if nickName: newDiracName = nickName else: newDiracName = self.getUserName(dn) # Do not consider users with Suspended status in VOMS if self.vomsUserDict[dn]['suspended'] or self.vomsUserDict[ dn]['certSuspended']: resultDict["SuspendedUsers"].append(newDiracName) continue # If the chosen user name exists already, append a distinguishing suffix ind = 1 trialName = newDiracName while newDiracName in allDiracUsers: # We have a user with the same name but with a different DN newDiracName = "%s_%d" % (trialName, ind) ind += 1 # We now have everything to add the new user userDict = { "DN": dn, "CA": self.vomsUserDict[dn]['CA'], "Email": self.vomsUserDict[dn]['mail'] } groupsWithRole = [] for role in self.vomsUserDict[dn]['Roles']: groupList = vomsDIRACMapping.get(role, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) userDict['Groups'] = list( set(groupsWithRole + [defaultVOGroup])) message = "\n Added new user %s:\n" % newDiracName for key in userDict: message += " %s: %s\n" % (key, str(userDict[key])) self.adminMsgs['Info'].append(message) self.voChanged = True if self.autoAddUsers: self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict))) result = self.csapi.modifyUser( newDiracName, userDict, createIfNonExistant=True) if not result['OK']: self.log.warn('Failed adding new user %s' % newDiracName) resultDict['NewUsers'].append(newDiracName) newAddedUserDict[newDiracName] = userDict continue # We have an already existing user modified = False userDict = { "DN": dn, "CA": self.vomsUserDict[dn]['CA'], "Email": self.vomsUserDict[dn]['mail'] } if newDNForExistingUser: userDict['DN'] = ','.join([ dn, diracUserDict.get(diracName, newAddedUserDict.get(diracName))['DN'] ]) modified = True existingGroups = diracUserDict.get(diracName, {}).get('Groups', []) nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping)) groupsWithRole = [] for role in self.vomsUserDict[dn]['Roles']: groupList = vomsDIRACMapping.get(role, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) keepGroups = nonVOGroups + groupsWithRole if not quarantineVOGroup or quarantineVOGroup not in existingGroups: keepGroups += [defaultVOGroup] for group in existingGroups: if group in nonVOGroups: continue role = diracVOMSMapping.get(group, '') # Among already existing groups for the user keep those without a special VOMS Role # because this membership is done by hand in the CS if "Role" not in role: keepGroups.append(group) # Keep existing groups with no VOMS attribute if any if group in noVOMSGroups: keepGroups.append(group) # Keep groups for which syncronization with VOMS is forbidden if group in noSyncVOMSGroups: keepGroups.append(group) userDict['Groups'] = list(set(keepGroups)) # Merge together groups for the same user but different DNs if diracName in newAddedUserDict: otherGroups = newAddedUserDict[diracName].get('Groups', []) userDict['Groups'] = list(set(keepGroups + otherGroups)) modified = True # Check if something changed before asking CSAPI to modify if diracName in diracUserDict: message = "\n Modified user %s:\n" % diracName modMsg = '' for key in userDict: if key == "Groups": addedGroups = set(userDict[key]) - set( diracUserDict.get(diracName, {}).get(key, [])) removedGroups = set( diracUserDict.get(diracName, {}).get( key, [])) - set(userDict[key]) if addedGroups: modMsg += " Added to group(s) %s\n" % ','.join( addedGroups) if removedGroups: modMsg += " Removed from group(s) %s\n" % ','.join( removedGroups) else: oldValue = str( diracUserDict.get(diracName, {}).get(key, '')) if str(userDict[key]) != oldValue: modMsg += " %s: %s -> %s\n" % ( key, oldValue, str(userDict[key])) if modMsg: self.adminMsgs['Info'].append(message + modMsg) modified = True if self.autoModifyUsers and modified: result = self.csapi.modifyUser(diracName, userDict) if result['OK'] and result['Value']: self.log.info("Modified user %s: %s" % (diracName, str(userDict))) self.voChanged = True resultDict['ModifiedUsers'].append(diracName) # Check if there are potentially obsoleted users oldUsers = set() for user in diracUserDict: dnSet = set(fromChar(diracUserDict[user]['DN'])) if not dnSet.intersection(set( self.vomsUserDict)) and user not in nonVOUserDict: for group in diracUserDict[user]['Groups']: if group not in noVOMSGroups: oldUsers.add(user) # Check for obsoleted DNs for user in diracUserDict: dnSet = set(fromChar(diracUserDict[user]['DN'])) for dn in dnSet: if dn in obsoletedDNs and user not in oldUsers: self.log.verbose("Modified user %s: dropped DN %s" % (user, dn)) if self.autoModifyUsers: userDict = diracUserDict[user] modDNSet = dnSet - set([dn]) if modDNSet: userDict['DN'] = ','.join(modDNSet) result = self.csapi.modifyUser(user, userDict) if result['OK'] and result['Value']: self.log.info( "Modified user %s: dropped DN %s" % (user, dn)) self.adminMsgs['Info'].append( "Modified user %s: dropped DN %s" % (user, dn)) self.voChanged = True resultDict['ModifiedUsers'].append(diracName) else: oldUsers.add(user) if oldUsers: self.voChanged = True if self.autoDeleteUsers: self.log.info('The following users will be deleted: %s' % str(oldUsers)) result = self.csapi.deleteUsers(oldUsers) if result['OK']: self.adminMsgs['Info'].append( 'The following users are deleted from CS:\n %s\n' % str(oldUsers)) resultDict['DeletedUsers'] = oldUsers else: self.adminMsgs['Errors'].append( 'Error in deleting users from CS:\n %s' % str(oldUsers)) self.log.error('Error while user deletion from CS', result) else: self.adminMsgs['Info'].append( 'The following users to be checked for deletion:\n %s' % str(oldUsers)) self.log.info( 'The following users to be checked for deletion: %s' % str(oldUsers)) resultDict['CSAPI'] = self.csapi resultDict['AdminMessages'] = self.adminMsgs return S_OK(resultDict)
def getQueues( self, resourceDict ): """ Get the list of relevant CEs and their descriptions """ self.queueDict = {} ceFactory = ComputingElementFactory() for site in resourceDict: for ce in resourceDict[site]: ceDict = resourceDict[site][ce] ceTags = ceDict.get( 'Tag' ) if isinstance( ceTags, basestring ): ceTags = fromChar( ceTags ) qDict = ceDict.pop( 'Queues' ) for queue in qDict: queueName = '%s_%s' % ( ce, queue ) self.queueDict[queueName] = {} self.queueDict[queueName]['ParametersDict'] = qDict[queue] self.queueDict[queueName]['ParametersDict']['Queue'] = queue self.queueDict[queueName]['ParametersDict']['Site'] = site self.queueDict[queueName]['ParametersDict']['GridEnv'] = self.gridEnv self.queueDict[queueName]['ParametersDict']['Setup'] = gConfig.getValue( '/DIRAC/Setup', 'unknown' ) # Evaluate the CPU limit of the queue according to the Glue convention # To Do: should be a utility if "maxCPUTime" in self.queueDict[queueName]['ParametersDict'] and \ "SI00" in self.queueDict[queueName]['ParametersDict']: maxCPUTime = float( self.queueDict[queueName]['ParametersDict']['maxCPUTime'] ) # For some sites there are crazy values in the CS maxCPUTime = max( maxCPUTime, 0 ) maxCPUTime = min( maxCPUTime, 86400 * 12.5 ) si00 = float( self.queueDict[queueName]['ParametersDict']['SI00'] ) queueCPUTime = 60. / 250. * maxCPUTime * si00 self.queueDict[queueName]['ParametersDict']['CPUTime'] = int( queueCPUTime ) queueTags = self.queueDict[queueName]['ParametersDict'].get( 'Tag' ) if queueTags and isinstance( queueTags, basestring ): queueTags = fromChar( queueTags ) self.queueDict[queueName]['ParametersDict']['Tag'] = queueTags if ceTags: if queueTags: allTags = list( set( ceTags + queueTags ) ) self.queueDict[queueName]['ParametersDict']['Tag'] = allTags else: self.queueDict[queueName]['ParametersDict']['Tag'] = ceTags maxMemory = self.queueDict[queueName]['ParametersDict'].get( 'MaxRAM', None ) if maxMemory: # MaxRAM value is supposed to be in MB maxMemoryList = range( 1, int( maxMemory )/1000 + 1 ) memoryTags = [ '%dGB' % mem for mem in maxMemoryList ] if memoryTags: self.queueDict[queueName]['ParametersDict'].setdefault( 'Tag', [] ) self.queueDict[queueName]['ParametersDict']['Tag'] += memoryTags qwDir = os.path.join( self.workingDirectory, queue ) if not os.path.exists( qwDir ): os.makedirs( qwDir ) self.queueDict[queueName]['ParametersDict']['WorkingDirectory'] = qwDir platform = '' if "Platform" in self.queueDict[queueName]['ParametersDict']: platform = self.queueDict[queueName]['ParametersDict']['Platform'] elif "Platform" in ceDict: platform = ceDict['Platform'] elif "OS" in ceDict: architecture = ceDict.get( 'architecture', 'x86_64' ) OS = ceDict['OS'] platform = '_'.join( [architecture, OS] ) if platform and not platform in self.platforms: self.platforms.append( platform ) if not "Platform" in self.queueDict[queueName]['ParametersDict'] and platform: result = Resources.getDIRACPlatform( platform ) if result['OK']: self.queueDict[queueName]['ParametersDict']['Platform'] = result['Value'][0] ceQueueDict = dict( ceDict ) ceQueueDict.update( self.queueDict[queueName]['ParametersDict'] ) # Generate the CE object for the queue or pick the already existing one # if the queue definition did not change queueHash = self.__generateQueueHash( ceQueueDict ) if queueName in self.queueCECache and self.queueCECache[queueName]['Hash'] == queueHash: queueCE = self.queueCECache[queueName]['CE'] else: result = ceFactory.getCE( ceName = ce, ceType = ceDict['CEType'], ceParametersDict = ceQueueDict ) if not result['OK']: return result self.queueCECache.setdefault( queueName, {} ) self.queueCECache[queueName]['Hash'] = queueHash self.queueCECache[queueName]['CE'] = result['Value'] queueCE = self.queueCECache[queueName]['CE'] self.queueDict[queueName]['CE'] = queueCE self.queueDict[queueName]['CEName'] = ce self.queueDict[queueName]['CEType'] = ceDict['CEType'] self.queueDict[queueName]['Site'] = site self.queueDict[queueName]['QueueName'] = queue self.queueDict[queueName]['Platform'] = platform result = self.queueDict[queueName]['CE'].isValid() if not result['OK']: self.log.fatal( result['Message'] ) return result if 'BundleProxy' in self.queueDict[queueName]['ParametersDict']: if self.queueDict[queueName]['ParametersDict']['BundleProxy'].lower() in ['true','yes','1']: self.queueDict[queueName]['BundleProxy'] = True elif 'BundleProxy' in ceDict: if ceDict['BundleProxy'].lower() in ['true','yes','1']: self.queueDict[queueName]['BundleProxy'] = True if site not in self.sites: self.sites.append( site ) return S_OK()
def submitJobs( self ): """ Go through defined computing elements and submit jobs if necessary """ # Check that there is some work at all setup = CSGlobals.getSetup() tqDict = { 'Setup':setup, 'CPUTime': 9999999, 'SubmitPool' : self.defaultSubmitPools } if self.vo: tqDict['Community'] = self.vo if self.voGroups: tqDict['OwnerGroup'] = self.voGroups result = Resources.getCompatiblePlatforms( self.platforms ) if not result['OK']: return result tqDict['Platform'] = result['Value'] tqDict['Site'] = self.sites tqDict['Tag'] = [] self.log.verbose( 'Checking overall TQ availability with requirements' ) self.log.verbose( tqDict ) rpcMatcher = RPCClient( "WorkloadManagement/Matcher" ) result = rpcMatcher.getMatchingTaskQueues( tqDict ) if not result[ 'OK' ]: return result if not result['Value']: self.log.verbose( 'No Waiting jobs suitable for the director' ) return S_OK() jobSites = set() anySite = False testSites = set() totalWaitingJobs = 0 for tqID in result['Value']: if "Sites" in result['Value'][tqID]: for site in result['Value'][tqID]['Sites']: if site.lower() != 'any': jobSites.add( site ) else: anySite = True else: anySite = True if "JobTypes" in result['Value'][tqID]: if "Sites" in result['Value'][tqID]: for site in result['Value'][tqID]['Sites']: if site.lower() != 'any': testSites.add( site ) totalWaitingJobs += result['Value'][tqID]['Jobs'] tqIDList = result['Value'].keys() result = pilotAgentsDB.countPilots( { 'TaskQueueID': tqIDList, 'Status': WAITING_PILOT_STATUS }, None ) totalWaitingPilots = 0 if result['OK']: totalWaitingPilots = result['Value'] self.log.info( 'Total %d jobs in %d task queues with %d waiting pilots' % (totalWaitingJobs, len( tqIDList ), totalWaitingPilots ) ) #if totalWaitingPilots >= totalWaitingJobs: # self.log.info( 'No more pilots to be submitted in this cycle' ) # return S_OK() # Check if the site is allowed in the mask result = jobDB.getSiteMask() if not result['OK']: return S_ERROR( 'Can not get the site mask' ) siteMaskList = result['Value'] queues = self.queueDict.keys() random.shuffle( queues ) totalSubmittedPilots = 0 for queue in queues: # Check if the queue failed previously failedCount = self.failedQueues.setdefault( queue, 0 ) % self.failedQueueCycleFactor if failedCount != 0: self.log.warn( "%s queue failed recently, skipping %d cycles" % ( queue, 10-failedCount ) ) self.failedQueues[queue] += 1 continue ce = self.queueDict[queue]['CE'] ceName = self.queueDict[queue]['CEName'] ceType = self.queueDict[queue]['CEType'] queueName = self.queueDict[queue]['QueueName'] siteName = self.queueDict[queue]['Site'] platform = self.queueDict[queue]['Platform'] siteMask = siteName in siteMaskList if not anySite and siteName not in jobSites: self.log.verbose( "Skipping queue %s at %s: no workload expected" % (queueName, siteName) ) continue if not siteMask and siteName not in testSites: self.log.verbose( "Skipping queue %s at site %s not in the mask" % (queueName, siteName) ) continue if 'CPUTime' in self.queueDict[queue]['ParametersDict'] : queueCPUTime = int( self.queueDict[queue]['ParametersDict']['CPUTime'] ) else: self.log.warn( 'CPU time limit is not specified for queue %s, skipping...' % queue ) continue if queueCPUTime > self.maxQueueLength: queueCPUTime = self.maxQueueLength # Prepare the queue description to look for eligible jobs ceDict = ce.getParameterDict() ceDict[ 'GridCE' ] = ceName #if not siteMask and 'Site' in ceDict: # self.log.info( 'Site not in the mask %s' % siteName ) # self.log.info( 'Removing "Site" from matching Dict' ) # del ceDict[ 'Site' ] if not siteMask: ceDict['JobType'] = "Test" if self.vo: ceDict['Community'] = self.vo if self.voGroups: ceDict['OwnerGroup'] = self.voGroups # This is a hack to get rid of ! ceDict['SubmitPool'] = self.defaultSubmitPools if "Tag" in ceDict and type( ceDict['Tag'] ) in types.StringTypes: ceDict['Tag'] = fromChar( ceDict['Tag'] ) result = Resources.getCompatiblePlatforms( platform ) if not result['OK']: continue ceDict['Platform'] = result['Value'] # Get the number of eligible jobs for the target site/queue result = rpcMatcher.getMatchingTaskQueues( ceDict ) if not result['OK']: self.log.error( 'Could not retrieve TaskQueues from TaskQueueDB', result['Message'] ) return result taskQueueDict = result['Value'] if not taskQueueDict: self.log.verbose( 'No matching TQs found for %s' % queue ) continue totalTQJobs = 0 tqIDList = taskQueueDict.keys() for tq in taskQueueDict: totalTQJobs += taskQueueDict[tq]['Jobs'] self.log.verbose( '%d job(s) from %d task queue(s) are eligible for %s queue' % (totalTQJobs, len( tqIDList ), queue) ) # Get the number of already waiting pilots for these task queues totalWaitingPilots = 0 if self.pilotWaitingFlag: lastUpdateTime = dateTime() - self.pilotWaitingTime * second result = pilotAgentsDB.countPilots( { 'TaskQueueID': tqIDList, 'Status': WAITING_PILOT_STATUS }, None, lastUpdateTime ) if not result['OK']: self.log.error( 'Failed to get Number of Waiting pilots', result['Message'] ) totalWaitingPilots = 0 else: totalWaitingPilots = result['Value'] self.log.verbose( 'Waiting Pilots for TaskQueue %s:' % tqIDList, totalWaitingPilots ) if totalWaitingPilots >= totalTQJobs: self.log.verbose( "%d waiting pilots already for all the available jobs" % totalWaitingPilots ) continue self.log.verbose( "%d waiting pilots for the total of %d eligible jobs for %s" % (totalWaitingPilots, totalTQJobs, queue) ) # Get the working proxy cpuTime = queueCPUTime + 86400 self.log.verbose( "Getting pilot proxy for %s/%s %d long" % ( self.pilotDN, self.pilotGroup, cpuTime ) ) result = gProxyManager.getPilotProxyFromDIRACGroup( self.pilotDN, self.pilotGroup, cpuTime ) if not result['OK']: return result self.proxy = result['Value'] ce.setProxy( self.proxy, cpuTime - 60 ) # Get the number of available slots on the target site/queue totalSlots = self.__getQueueSlots( queue ) if totalSlots == 0: continue pilotsToSubmit = max( 0, min( totalSlots, totalTQJobs - totalWaitingPilots ) ) self.log.info( '%s: Slots=%d, TQ jobs=%d, Pilots: waiting %d, to submit=%d' % \ ( queue, totalSlots, totalTQJobs, totalWaitingPilots, pilotsToSubmit ) ) # Limit the number of pilots to submit to MAX_PILOTS_TO_SUBMIT pilotsToSubmit = min( self.maxPilotsToSubmit, pilotsToSubmit ) while pilotsToSubmit > 0: self.log.info( 'Going to submit %d pilots to %s queue' % ( pilotsToSubmit, queue ) ) bundleProxy = self.queueDict[queue].get( 'BundleProxy', False ) jobExecDir = '' if ceType == 'CREAM': jobExecDir = '.' jobExecDir = self.queueDict[queue].get( 'JobExecDir', jobExecDir ) httpProxy = self.queueDict[queue].get( 'HttpProxy', '' ) result = self.__getExecutable( queue, pilotsToSubmit, bundleProxy, httpProxy, jobExecDir ) if not result['OK']: return result executable, pilotSubmissionChunk = result['Value'] result = ce.submitJob( executable, '', pilotSubmissionChunk ) os.unlink( executable ) if not result['OK']: self.log.error( 'Failed submission to queue %s:\n' % queue, result['Message'] ) pilotsToSubmit = 0 self.failedQueues[queue] += 1 continue pilotsToSubmit = pilotsToSubmit - pilotSubmissionChunk # Add pilots to the PilotAgentsDB assign pilots to TaskQueue proportionally to the # task queue priorities pilotList = result['Value'] self.queueSlots[queue]['AvailableSlots'] -= len( pilotList ) totalSubmittedPilots += len( pilotList ) self.log.info( 'Submitted %d pilots to %s@%s' % ( len( pilotList ), queueName, ceName ) ) stampDict = {} if result.has_key( 'PilotStampDict' ): stampDict = result['PilotStampDict'] tqPriorityList = [] sumPriority = 0. for tq in taskQueueDict: sumPriority += taskQueueDict[tq]['Priority'] tqPriorityList.append( ( tq, sumPriority ) ) rndm = random.random()*sumPriority tqDict = {} for pilotID in pilotList: rndm = random.random()*sumPriority for tq, prio in tqPriorityList: if rndm < prio: tqID = tq break if not tqDict.has_key( tqID ): tqDict[tqID] = [] tqDict[tqID].append( pilotID ) for tqID, pilotList in tqDict.items(): result = pilotAgentsDB.addPilotTQReference( pilotList, tqID, self.pilotDN, self.pilotGroup, self.localhost, ceType, '', stampDict ) if not result['OK']: self.log.error( 'Failed add pilots to the PilotAgentsDB: ', result['Message'] ) continue for pilot in pilotList: result = pilotAgentsDB.setPilotStatus( pilot, 'Submitted', ceName, 'Successfully submitted by the SiteDirector', siteName, queueName ) if not result['OK']: self.log.error( 'Failed to set pilot status: ', result['Message'] ) continue self.log.info( "%d pilots submitted in total in this cycle" % totalSubmittedPilots ) return S_OK()
def _ByJobType(self): """ By default, all sites are allowed to do every job. The actual rules are freely specified in the Operation JobTypeMapping section. The content of the section may look like this: User { Exclude = PAK Exclude += Ferrara Exclude += Bologna Exclude += Paris Exclude += CERN Exclude += IN2P3 Allow { Paris = IN2P3 CERN = CERN IN2P3 = IN2P3 } } DataReconstruction { Exclude = PAK Exclude += Ferrara Exclude += CERN Exclude += IN2P3 Allow { Ferrara = CERN CERN = CERN IN2P3 = IN2P3 IN2P3 += CERN } } Merge { Exclude = ALL Allow { CERN = CERN IN2P3 = IN2P3 } } The sites in the exclusion list will be removed. The allow section says where each site may help another site """ # 1. get sites list res = getSites() if not res['OK']: gLogger.error("Could not get the list of sites", res['Message']) return res destSites = set(res['Value']) # 2. get JobTypeMapping "Exclude" value (and add autoAddedSites) gLogger.debug( "Getting JobTypeMapping 'Exclude' value (and add autoAddedSites)") jobType = self.params['JobType'] if not jobType: raise RuntimeError("No jobType specified") excludedSites = set( self.opsH.getValue('JobTypeMapping/%s/Exclude' % jobType, [])) gLogger.debug("Explicitly excluded sites for %s task: %s" % (jobType, ','.join(excludedSites))) autoAddedSites = self.opsH.getValue('JobTypeMapping/AutoAddedSites', []) if 'WithStorage' in autoAddedSites: # Add all sites with storage, such that jobs can run wherever data is autoAddedSites.remove('WithStorage') autoAddedSites += DMSHelpers().getTiers(withStorage=True) # 3. removing sites in Exclude if not excludedSites: pass elif 'ALL' in excludedSites: destSites = set() else: destSites -= excludedSites # 4. get JobTypeMapping "Allow" section res = self.opsH.getOptionsDict('JobTypeMapping/%s/Allow' % jobType) if not res['OK']: gLogger.debug(res['Message']) allowed = {} else: allowed = dict((site, set(fromChar(fromSites))) for site, fromSites in res['Value'].items()) autoAddedSites = set( self.opsH.getValue('JobTypeMapping/%s/AutoAddedSites' % jobType, autoAddedSites)) gLogger.debug("Auto-added sites for %s task: %s" % (jobType, ','.join(autoAddedSites))) # 5. add autoAddedSites, if requested for autoAddedSite in autoAddedSites: allowed.setdefault(autoAddedSite, set()).add(autoAddedSite) gLogger.debug("Allowed sites for %s task: %s" % (jobType, ','.join(allowed))) # 6. Allowing sites that should be allowed taskSiteDestination = self._BySE() for destSite, fromSites in allowed.items(): for fromSite in fromSites: if not taskSiteDestination or fromSite in taskSiteDestination: destSites.add(destSite) gLogger.debug( "Computed list of destination sites for %s task with TargetSE %s: %s" % (jobType, self.params['TargetSE'], ','.join(destSites))) return destSites
def getQueuesResolved(siteDict): """ Get the list of queue descriptions merging site/ce/queue parameters and adding some derived parameters. :param dict siteDict: dictionary with configuration data as returned by Resources.getQueues() method :return: S_OK/S_ERROR, Value dictionary per queue with configuration data updated, e.g. for SiteDirector """ queueFinalDict = {} for site in siteDict: for ce, ceDict in siteDict[site].items(): qDict = ceDict.pop('Queues') for queue in qDict: queueName = '%s_%s' % (ce, queue) queueDict = qDict[queue] queueDict['Queue'] = queue queueDict['Site'] = site # Evaluate the CPU limit of the queue according to the Glue convention # To Do: should be a utility if "maxCPUTime" in queueDict and "SI00" in queueDict: maxCPUTime = float(queueDict['maxCPUTime']) # For some sites there are crazy values in the CS maxCPUTime = max(maxCPUTime, 0) maxCPUTime = min(maxCPUTime, 86400 * 12.5) si00 = float(queueDict['SI00']) queueCPUTime = 60 / 250 * maxCPUTime * si00 queueDict['CPUTime'] = int(queueCPUTime) # Tags & RequiredTags defined on the Queue level and on the CE level are concatenated # This also converts them from a string to a list if required. for tagFieldName in ('Tag', 'RequiredTag'): ceTags = ceDict.get(tagFieldName, []) if isinstance(ceTags, six.string_types): ceTags = fromChar(ceTags) queueTags = queueDict.get(tagFieldName, []) if isinstance(queueTags, six.string_types): queueTags = fromChar(queueTags) queueDict[tagFieldName] = list(set(ceTags + queueTags)) # Some parameters can be defined on the CE level and are inherited by all Queues for parameter in ['MaxRAM', 'NumberOfProcessors', 'WholeNode']: queueParameter = queueDict.get(parameter, ceDict.get(parameter)) if queueParameter: queueDict[parameter] = queueParameter # If we have a multi-core queue add MultiProcessor tag if queueDict.get('NumberOfProcessors', 1) > 1: queueDict.setdefault('Tag', []).append('MultiProcessor') queueDict['CEName'] = ce queueDict['GridCE'] = ce queueDict['CEType'] = ceDict['CEType'] queueDict['GridMiddleware'] = ceDict['CEType'] queueDict['QueueName'] = queue platform = queueDict.get('Platform', ceDict.get('Platform', '')) if not platform and "OS" in ceDict: architecture = ceDict.get('architecture', 'x86_64') platform = '_'.join([architecture, ceDict['OS']]) queueDict['Platform'] = platform if platform: result = getDIRACPlatform(platform) if result['OK']: queueDict['Platform'] = result['Value'][0] queueFinalDict[queueName] = queueDict return S_OK(queueFinalDict)
def syncCSWithVOMS(self): """ Performs the synchronization of the DIRAC registry with the VOMS data. The resulting CSAPI object containing modifications is returned as part of the output dictionary. Those changes can be applied by the caller depending on the mode (dry or a real run) :return: S_OK with a dictionary containing the results of the synchronization operation """ resultDict = defaultdict(list) # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping(self.vo) if not result['OK']: return result vomsDIRACMapping = result['Value']['VOMSDIRAC'] diracVOMSMapping = result['Value']['DIRACVOMS'] noVOMSGroups = result['Value']['NoVOMS'] noSyncVOMSGroups = result['Value']['NoSyncVOMS'] vomsSrv = VOMSService(self.vo) # Get VOMS users result = vomsSrv.getUsers() if not result['OK']: self.log.error('Could not retrieve user information from VOMS', result['Message']) return result self.vomsUserDict = result['Value'] message = "There are %s user entries in VOMS for VO %s" % (len(self.vomsUserDict), self.vomsVOName) self.adminMsgs['Info'].append(message) self.log.info(message) # Get DIRAC users result = self.getVOUserData(self.vo) if not result['OK']: return result diracUserDict = result['Value'] self.adminMsgs['Info'].append("There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), self.vo)) self.log.info("There are %s registered users in DIRAC VO %s" % (len(diracUserDict), self.vo)) # Find new and obsoleted user DNs existingDNs = [] obsoletedDNs = [] newDNs = [] for user in diracUserDict: dn = diracUserDict[user]['DN'] # We can have users with more than one DN registered dnList = fromChar(dn) existingDNs.extend(dnList) for dn in dnList: if dn not in self.vomsUserDict: obsoletedDNs.append(dn) for dn in self.vomsUserDict: if dn not in existingDNs: newDNs.append(dn) allDiracUsers = getAllUsers() nonVOUserDict = {} nonVOUsers = list(set(allDiracUsers) - set(diracUserDict)) if nonVOUsers: result = self.csapi.describeUsers(nonVOUsers) if not result['OK']: self.log.error('Could not retrieve CS User description') return result nonVOUserDict = result['Value'] # Process users defaultVOGroup = getVOOption(self.vo, "DefaultGroup", "%s_user" % self.vo) # If a user is (previously put by hand) in a "QuarantineGroup", # then the default group will be ignored. # So, this option is only considered for the case of existing users. quarantineVOGroup = getVOOption(self.vo, "QuarantineGroup") newAddedUserDict = {} for dn in self.vomsUserDict: newDNForExistingUser = '' diracName = '' if dn in existingDNs: for user in diracUserDict: if dn == diracUserDict[user]['DN']: diracName = user if dn in newDNs: # Find if the DN is already registered in the DIRAC CS for user in nonVOUserDict: if dn == nonVOUserDict[user]['DN']: diracName = user # Check the nickName in the same VO to see if the user is already registered # with another DN nickName = self.vomsUserDict[dn].get('nickname') if nickName in diracUserDict or nickName in newAddedUserDict: diracName = nickName # This is a flag for adding the new DN to an already existing user newDNForExistingUser = dn # We have a real new user if not diracName: if nickName: newDiracName = nickName else: newDiracName = self.getUserName(dn) # Do not consider users with Suspended status in VOMS if self.vomsUserDict[dn]['suspended'] or self.vomsUserDict[dn]['certSuspended']: resultDict["SuspendedUsers"].append(newDiracName) continue # If the chosen user name exists already, append a distinguishing suffix ind = 1 trialName = newDiracName while newDiracName in allDiracUsers: # We have a user with the same name but with a different DN newDiracName = "%s_%d" % (trialName, ind) ind += 1 # We now have everything to add the new user userDict = {"DN": dn, "CA": self.vomsUserDict[dn]['CA'], "Email": self.vomsUserDict[dn]['mail']} groupsWithRole = [] for role in self.vomsUserDict[dn]['Roles']: groupList = vomsDIRACMapping.get(role, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) userDict['Groups'] = list(set(groupsWithRole + [defaultVOGroup])) message = "\n Added new user %s:\n" % newDiracName for key in userDict: message += " %s: %s\n" % (key, str(userDict[key])) self.adminMsgs['Info'].append(message) self.voChanged = True if self.autoAddUsers: self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict))) result = self.csapi.modifyUser(newDiracName, userDict, createIfNonExistant=True) if not result['OK']: self.log.warn('Failed adding new user %s' % newDiracName) resultDict['NewUsers'].append(newDiracName) newAddedUserDict[newDiracName] = userDict continue # We have an already existing user modified = False userDict = {"DN": dn, "CA": self.vomsUserDict[dn]['CA'], "Email": self.vomsUserDict[dn]['mail']} if newDNForExistingUser: userDict['DN'] = ','.join([dn, diracUserDict.get(diracName, newAddedUserDict.get(diracName))['DN']]) modified = True existingGroups = diracUserDict.get(diracName, {}).get('Groups', []) nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping)) groupsWithRole = [] for role in self.vomsUserDict[dn]['Roles']: groupList = vomsDIRACMapping.get(role, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) keepGroups = nonVOGroups + groupsWithRole if not quarantineVOGroup or quarantineVOGroup not in existingGroups: keepGroups += [defaultVOGroup] for group in existingGroups: if group in nonVOGroups: continue role = diracVOMSMapping.get(group, '') # Among already existing groups for the user keep those without a special VOMS Role # because this membership is done by hand in the CS if "Role" not in role: keepGroups.append(group) # Keep existing groups with no VOMS attribute if any if group in noVOMSGroups: keepGroups.append(group) # Keep groups for which syncronization with VOMS is forbidden if group in noSyncVOMSGroups: keepGroups.append(group) userDict['Groups'] = list(set(keepGroups)) # Merge together groups for the same user but different DNs if diracName in newAddedUserDict: otherGroups = newAddedUserDict[diracName].get('Groups', []) userDict['Groups'] = list(set(keepGroups + otherGroups)) modified = True # Check if something changed before asking CSAPI to modify if diracName in diracUserDict: message = "\n Modified user %s:\n" % diracName modMsg = '' for key in userDict: if key == "Groups": addedGroups = set(userDict[key]) - set(diracUserDict.get(diracName, {}).get(key, [])) removedGroups = set(diracUserDict.get(diracName, {}).get(key, [])) - set(userDict[key]) if addedGroups: modMsg += " Added to group(s) %s\n" % ','.join(addedGroups) if removedGroups: modMsg += " Removed from group(s) %s\n" % ','.join(removedGroups) else: oldValue = str(diracUserDict.get(diracName, {}).get(key, '')) if str(userDict[key]) != oldValue: modMsg += " %s: %s -> %s\n" % (key, oldValue, str(userDict[key])) if modMsg: self.adminMsgs['Info'].append(message + modMsg) modified = True if self.autoModifyUsers and modified: result = self.csapi.modifyUser(diracName, userDict) if result['OK'] and result['Value']: self.log.info("Modified user %s: %s" % (diracName, str(userDict))) self.voChanged = True resultDict['ModifiedUsers'].append(diracName) # Check if there are potentially obsoleted users oldUsers = set() for user in diracUserDict: dnSet = set(fromChar(diracUserDict[user]['DN'])) if not dnSet.intersection(set(self.vomsUserDict)) and user not in nonVOUserDict: for group in diracUserDict[user]['Groups']: if group not in noVOMSGroups: oldUsers.add(user) # Check for obsoleted DNs for user in diracUserDict: dnSet = set(fromChar(diracUserDict[user]['DN'])) for dn in dnSet: if dn in obsoletedDNs and user not in oldUsers: self.log.verbose("Modified user %s: dropped DN %s" % (user, dn)) if self.autoModifyUsers: userDict = diracUserDict[user] modDNSet = dnSet - set([dn]) if modDNSet: userDict['DN'] = ','.join(modDNSet) result = self.csapi.modifyUser(user, userDict) if result['OK'] and result['Value']: self.log.info("Modified user %s: dropped DN %s" % (user, dn)) self.adminMsgs['Info'].append("Modified user %s: dropped DN %s" % (user, dn)) self.voChanged = True resultDict['ModifiedUsers'].append(diracName) else: oldUsers.add(user) if oldUsers: self.voChanged = True if self.autoDeleteUsers: self.log.info('The following users will be deleted: %s' % str(oldUsers)) result = self.csapi.deleteUsers(oldUsers) if result['OK']: self.adminMsgs['Info'].append('The following users are deleted from CS:\n %s\n' % str(oldUsers)) resultDict['DeletedUsers'] = oldUsers else: self.adminMsgs['Errors'].append('Error in deleting users from CS:\n %s' % str(oldUsers)) self.log.error('Error while user deletion from CS', result) else: self.adminMsgs['Info'].append('The following users to be checked for deletion:\n %s' % str(oldUsers)) self.log.info('The following users to be checked for deletion: %s' % str(oldUsers)) resultDict['CSAPI'] = self.csapi resultDict['AdminMessages'] = self.adminMsgs return S_OK(resultDict)
def getNumberOfProcessors(siteName=None, gridCE=None, queue=None): """ gets the number of processors on a certain CE/queue/node (what the pilot administers) The siteName/gridCE/queue parameters are normally not necessary. Tries to find it in this order: 1) from the /Resources/Computing/CEDefaults/NumberOfProcessors (which is what the pilot fills up) 2) if not present from JobFeatures 3) if not present looks in CS for "NumberOfProcessors" Queue or CE option 4) if not present looks in CS for "%dProcessors" Queue or CE Tag 5) if not present but there's WholeNode tag, look what the WN provides using multiprocessing.cpu_count() 6) return 1 """ # 1) from /Resources/Computing/CEDefaults/NumberOfProcessors gLogger.info( "Getting numberOfProcessors from /Resources/Computing/CEDefaults/NumberOfProcessors" ) numberOfProcessors = gConfig.getValue( '/Resources/Computing/CEDefaults/NumberOfProcessors') if numberOfProcessors: return numberOfProcessors # 2) from MJF gLogger.info("Getting numberOfProcessors from MJF") numberOfProcessors = getProcessorFromMJF() if numberOfProcessors: return numberOfProcessors # 3) looks in CS for "NumberOfProcessors" Queue or CE or site option grid = siteName.split('.')[0] gLogger.info( "NumberOfProcessors could not be found in MJF, trying from CS (queue definition)" ) numberOfProcessors = gConfig.getValue( '/Resources/Sites/%s/%s/CEs/%s/Queues/%s/NumberOfProcessors' % (grid, siteName, gridCE, queue)) if numberOfProcessors: return numberOfProcessors gLogger.info( "NumberOfProcessors could not be found in CS queue definition, ", "trying from /Resources/Sites/%s/%s/CEs/%s/NumberOfProcessors" % (grid, siteName, gridCE)) numberOfProcessors = gConfig.getValue( '/Resources/Sites/%s/%s/CEs/%s/NumberOfProcessors' % (grid, siteName, gridCE)) if numberOfProcessors: return numberOfProcessors gLogger.info( "NumberOfProcessors could not be found in CS CE definition, ", "trying from /Resources/Sites/%s/%s/NumberOfProcessors" % (grid, siteName)) numberOfProcessors = gConfig.getValue( '/Resources/Sites/%s/%s/NumberOfProcessors' % (grid, siteName)) if numberOfProcessors: return numberOfProcessors # 3) looks in CS for tags gLogger.info("Getting number of processors" "from tags for %s: %s: %s" % (siteName, gridCE, queue)) # Tags of the CE tags = fromChar( gConfig.getValue( '/Resources/Sites/%s/%s/CEs/%s/Tag' % (siteName.split('.')[0], siteName, gridCE), '')) # Tags of the Queue tags += fromChar( gConfig.getValue( '/Resources/Sites/%s/%s/CEs/%s/Queues/%s/Tag' % (siteName.split('.')[0], siteName, gridCE, queue), '')) for tag in tags: numberOfProcessorsTag = re.search('[0-9]Processors', tag) if numberOfProcessorsTag: gLogger.info("Number of processors from tags", numberOfProcessorsTag.string) return int(numberOfProcessorsTag.string.replace('Processors', '')) gLogger.info("NumberOfProcessors could not be found in CS") if 'WholeNode' in tags: gLogger.info("Found WholeNode tag, using multiprocessing.cpu_count()") return multiprocessing.cpu_count() return 1
def getEndpoints(self, resourceDict): """ Get the list of relevant CEs and their descriptions """ self.vmTypeDict = {} ceFactory = EndpointFactory() result = getPilotBootstrapParameters(vo=self.vo, runningPod=self.runningPod) if not result['OK']: return result opParameters = result['Value'] for site in resourceDict: for ce in resourceDict[site]: ceDict = resourceDict[site][ce] ceTags = ceDict.get('Tag', []) if isinstance(ceTags, basestring): ceTags = fromChar(ceTags) ceMaxRAM = ceDict.get('MaxRAM', None) qDict = ceDict.pop('VMTypes') for vmType in qDict: vmTypeName = '%s_%s' % (ce, vmType) self.vmTypeDict[vmTypeName] = {} self.vmTypeDict[vmTypeName]['ParametersDict'] = qDict[ vmType] self.vmTypeDict[vmTypeName]['ParametersDict'][ 'VMType'] = vmType self.vmTypeDict[vmTypeName]['ParametersDict'][ 'Site'] = site self.vmTypeDict[vmTypeName]['ParametersDict'][ 'Setup'] = gConfig.getValue('/DIRAC/Setup', 'unknown') self.vmTypeDict[vmTypeName]['ParametersDict'][ 'CPUTime'] = 99999999 vmTypeTags = self.vmTypeDict[vmTypeName][ 'ParametersDict'].get('Tag') if vmTypeTags and isinstance(vmTypeTags, basestring): vmTypeTags = fromChar(vmTypeTags) self.vmTypeDict[vmTypeName]['ParametersDict'][ 'Tag'] = vmTypeTags if ceTags: if vmTypeTags: allTags = list(set(ceTags + vmTypeTags)) self.vmTypeDict[vmTypeName]['ParametersDict'][ 'Tag'] = allTags else: self.vmTypeDict[vmTypeName]['ParametersDict'][ 'Tag'] = ceTags maxRAM = self.vmTypeDict[vmTypeName]['ParametersDict'].get( 'MaxRAM') maxRAM = ceMaxRAM if not maxRAM else maxRAM if maxRAM: self.vmTypeDict[vmTypeName]['ParametersDict'][ 'MaxRAM'] = maxRAM ceWholeNode = ceDict.get('WholeNode', 'true') wholeNode = self.vmTypeDict[vmTypeName][ 'ParametersDict'].get('WholeNode', ceWholeNode) if wholeNode.lower() in ('yes', 'true'): self.vmTypeDict[vmTypeName][ 'ParametersDict'].setdefault('Tag', []) self.vmTypeDict[vmTypeName]['ParametersDict'][ 'Tag'].append('WholeNode') platform = '' if "Platform" in self.vmTypeDict[vmTypeName][ 'ParametersDict']: platform = self.vmTypeDict[vmTypeName][ 'ParametersDict']['Platform'] elif "Platform" in ceDict: platform = ceDict['Platform'] if platform and platform not in self.platforms: self.platforms.append(platform) if "Platform" not in self.vmTypeDict[vmTypeName][ 'ParametersDict'] and platform: result = Resources.getDIRACPlatform(platform) if result['OK']: self.vmTypeDict[vmTypeName]['ParametersDict'][ 'Platform'] = result['Value'][0] ceVMTypeDict = dict(ceDict) ceVMTypeDict['CEName'] = ce ceVMTypeDict['VO'] = self.vo ceVMTypeDict['VMType'] = vmType ceVMTypeDict['RunningPod'] = self.runningPod ceVMTypeDict['CSServers'] = gConfig.getValue( "/DIRAC/Configuration/Servers", []) ceVMTypeDict.update( self.vmTypeDict[vmTypeName]['ParametersDict']) # Allow a resource-specifc CAPath to be set (as some clouds have their own CAs) # Otherwise fall back to the system-wide default(s) if 'CAPath' not in ceVMTypeDict: ceVMTypeDict['CAPath'] = gConfig.getValue( '/DIRAC/Security/CAPath', "/opt/dirac/etc/grid-security/certificates/cas.pem" ) # Generate the CE object for the vmType or pick the already existing one # if the vmType definition did not change vmTypeHash = self.__generateVMTypeHash(ceVMTypeDict) if vmTypeName in self.vmTypeCECache and self.vmTypeCECache[ vmTypeName]['Hash'] == vmTypeHash: vmTypeCE = self.vmTypeCECache[vmTypeName]['CE'] else: result = ceFactory.getCEObject(parameters=ceVMTypeDict) if not result['OK']: return result self.vmTypeCECache.setdefault(vmTypeName, {}) self.vmTypeCECache[vmTypeName]['Hash'] = vmTypeHash self.vmTypeCECache[vmTypeName]['CE'] = result['Value'] vmTypeCE = self.vmTypeCECache[vmTypeName]['CE'] vmTypeCE.setBootstrapParameters(opParameters) self.vmTypeDict[vmTypeName]['CE'] = vmTypeCE self.vmTypeDict[vmTypeName]['CEName'] = ce self.vmTypeDict[vmTypeName]['CEType'] = ceDict['CEType'] self.vmTypeDict[vmTypeName]['Site'] = site self.vmTypeDict[vmTypeName]['VMType'] = vmType self.vmTypeDict[vmTypeName]['Platform'] = platform self.vmTypeDict[vmTypeName]['MaxInstances'] = ceDict[ 'MaxInstances'] if not self.vmTypeDict[vmTypeName]['CE'].isValid(): self.log.error( 'Failed to instantiate CloudEndpoint for %s' % vmTypeName) continue if site not in self.sites: self.sites.append(site) return S_OK()
def _ByJobType( self ): """ By default, all sites are allowed to do every job. The actual rules are freely specified in the Operation JobTypeMapping section. The content of the section may look like this: User { Exclude = PAK Exclude += Ferrara Exclude += Bologna Exclude += Paris Exclude += CERN Exclude += IN2P3 Allow { Paris = IN2P3 CERN = CERN IN2P3 = IN2P3 } } DataReconstruction { Exclude = PAK Exclude += Ferrara Exclude += CERN Exclude += IN2P3 Allow { Ferrara = CERN CERN = CERN IN2P3 = IN2P3 IN2P3 += CERN } } Merge { Exclude = ALL Allow { CERN = CERN IN2P3 = IN2P3 } } The sites in the exclusion list will be removed. The allow section says where each site may help another site """ # 1. get sites list res = getSites() if not res['OK']: gLogger.error( "Could not get the list of sites", res['Message'] ) return res destSites = set( res['Value'] ) # 2. get JobTypeMapping "Exclude" value (and add autoAddedSites) gLogger.debug( "Getting JobTypeMapping 'Exclude' value (and add autoAddedSites)" ) jobType = self.params['JobType'] if not jobType: raise RuntimeError( "No jobType specified" ) excludedSites = self.opsH.getValue( 'JobTypeMapping/%s/Exclude' % jobType, [] ) gLogger.debug( "Explicitly excluded sites for %s task: %s" % ( jobType, ','.join( excludedSites ) ) ) excludedSites += self.opsH.getValue( 'JobTypeMapping/AutoAddedSites', [] ) gLogger.debug( "Full list of excluded sites for %s task: %s" % ( jobType, ','.join( excludedSites ) ) ) # 3. removing sites in Exclude if not excludedSites: pass elif 'ALL' in excludedSites: destSites = set() else: destSites = destSites.difference( set( excludedSites ) ) # 4. get JobTypeMapping "Allow" section res = self.opsH.getOptionsDict( 'JobTypeMapping/%s/Allow' % jobType ) if not res['OK']: gLogger.verbose( res['Message'] ) allowed = {} else: allowed = res['Value'] for site in allowed: allowed[site] = fromChar( allowed[site] ) # 5. add autoAddedSites, if requested autoAddedSites = self.opsH.getValue( 'JobTypeMapping/AutoAddedSites', [] ) if autoAddedSites: for autoAddedSite in autoAddedSites: allowed.setdefault( autoAddedSite, [autoAddedSite] ) if autoAddedSite not in allowed: allowed[autoAddedSite] = [autoAddedSite] else: allowed[autoAddedSite] = [autoAddedSite] + allowed[autoAddedSite] gLogger.debug( "Allowed sites for %s task: %s" % ( jobType, ','.join( allowed ) ) ) # 6. Allowing sites that should be allowed if not self.params['TargetSE'] or self.params['TargetSE'] == 'Unknown': gLogger.warn( "TargetSE is not set: the destination sites list will be incomplete" ) taskSiteDestination = self._BySE() for destSite, fromSites in allowed.iteritems(): for fromSite in fromSites: if taskSiteDestination: if fromSite in taskSiteDestination: destSites.add( destSite ) else: destSites.add( destSite ) gLogger.verbose( "Computed list of destination sites for %s task with TargetSE %s: %s" % ( jobType, self.params['TargetSE'], ','.join( destSites ) ) ) return destSites
def registerUser( self ): """ This function is used to notify DIRAC admins about user registration request The logic is simple: 0) Check if request from this e-mail has already registered or not 1) Send mail to VO admin of requested VO 2) Send mail to users in group with UserAdministrator property 3) Send mail to users indicated in /Website/UserRegistrationAdmin option """ gLogger.info("Start processing a registration request") checkUserCredentials() # Check for having a DN but no username dn = getUserDN() if not dn: error = "Certificate is not loaded in the browser or DN is absent" gLogger.error( "Service response: %s" % error ) return { "success" : "false" , "error" : error } username = getUsername() if not username == "anonymous": error = "You are already registered in DIRAC with username: %s" % username gLogger.error( "Service response: %s" % error ) return { "success" : "false" , "error" : error } gLogger.info( "DN: %s" % dn ) if not request.params.has_key( "email" ): error = "Can not get your email address from the request" gLogger.debug( "Service response: %s" % error ) return { "success" : "false" , "error" : error } userMail = request.params[ "email" ] if self.alreadyRequested( userMail ): error = "Request associated with %s already registered" % userMail gLogger.debug( "Service response: %s" % error ) return { "success" : "false" , "error" : error } vo = fromChar( request.params[ "vo" ] ) if not vo: error = "You should indicate a VirtualOrganization for membership" gLogger.debug( "Service response: %s" % error ) return { "success" : "false" , "error" : error } gLogger.info( "User want to be register in VO(s): %s" % vo ) body = str() for i in request.params: if not i in [ "registration_request" , "email" , "vo" ]: info = self.__checkUnicode( i , request.params[ i ] ) body = body + info + "\n" body = body + "DN - " + dn gLogger.debug( "email body: %s" % body ) adminList = self.__getAdminList( vo ) if not len( adminList ) > 0: error = "Can't get in contact with administrators about your request\n" error = error + "Most likely this DIRAC instance is not configured yet" gLogger.debug( "Service response: %s" % error ) return { "success" : "false" , "error" : error } adminList = uniqueElements( adminList ) gLogger.info( "Chosen admin(s): %s" % adminList ) sendDict = self.__getMailDict( adminList ) if not len(sendDict) > 0: error = "Can't get in contact with administrators about your request\n" error = error + "Most likely this DIRAC instance is not configured yet" gLogger.debug( "Service response: %s" % error ) return { "success" : "false" , "error" : error } gLogger.debug( "Final dictionary with mails to be used %s" % sendDict ) return self.__sendAMail( sendDict , body , userMail )
def getQueues( self, resourceDict ): """ Get the list of relevant CEs and their descriptions """ self.queueDict = {} ceFactory = ComputingElementFactory() for site in resourceDict: for ce in resourceDict[site]: ceDict = resourceDict[site][ce] ceTags = ceDict.get( 'Tag', [] ) pilotRunDirectory = ceDict.get( 'PilotRunDirectory', '' ) if isinstance( ceTags, basestring ): ceTags = fromChar( ceTags ) ceMaxRAM = ceDict.get( 'MaxRAM', None ) qDict = ceDict.pop( 'Queues' ) for queue in qDict: queueName = '%s_%s' % ( ce, queue ) self.queueDict[queueName] = {} self.queueDict[queueName]['ParametersDict'] = qDict[queue] self.queueDict[queueName]['ParametersDict']['Queue'] = queue self.queueDict[queueName]['ParametersDict']['Site'] = site self.queueDict[queueName]['ParametersDict']['GridEnv'] = self.gridEnv self.queueDict[queueName]['ParametersDict']['Setup'] = gConfig.getValue( '/DIRAC/Setup', 'unknown' ) # Evaluate the CPU limit of the queue according to the Glue convention # To Do: should be a utility if "maxCPUTime" in self.queueDict[queueName]['ParametersDict'] and \ "SI00" in self.queueDict[queueName]['ParametersDict']: maxCPUTime = float( self.queueDict[queueName]['ParametersDict']['maxCPUTime'] ) # For some sites there are crazy values in the CS maxCPUTime = max( maxCPUTime, 0 ) maxCPUTime = min( maxCPUTime, 86400 * 12.5 ) si00 = float( self.queueDict[queueName]['ParametersDict']['SI00'] ) queueCPUTime = 60. / 250. * maxCPUTime * si00 self.queueDict[queueName]['ParametersDict']['CPUTime'] = int( queueCPUTime ) queueTags = self.queueDict[queueName]['ParametersDict'].get( 'Tag' ) if queueTags and isinstance( queueTags, basestring ): queueTags = fromChar( queueTags ) self.queueDict[queueName]['ParametersDict']['Tag'] = queueTags if ceTags: if queueTags: allTags = list( set( ceTags + queueTags ) ) self.queueDict[queueName]['ParametersDict']['Tag'] = allTags else: self.queueDict[queueName]['ParametersDict']['Tag'] = ceTags maxRAM = self.queueDict[queueName]['ParametersDict'].get( 'MaxRAM' ) maxRAM = ceMaxRAM if not maxRAM else maxRAM if maxRAM: self.queueDict[queueName]['ParametersDict']['MaxRAM'] = maxRAM if pilotRunDirectory: self.queueDict[queueName]['ParametersDict']['JobExecDir'] = pilotRunDirectory qwDir = os.path.join( self.workingDirectory, queue ) mkDir(qwDir) self.queueDict[queueName]['ParametersDict']['WorkingDirectory'] = qwDir platform = '' if "Platform" in self.queueDict[queueName]['ParametersDict']: platform = self.queueDict[queueName]['ParametersDict']['Platform'] elif "Platform" in ceDict: platform = ceDict['Platform'] elif "OS" in ceDict: architecture = ceDict.get( 'architecture', 'x86_64' ) OS = ceDict['OS'] platform = '_'.join( [architecture, OS] ) if platform and not platform in self.platforms: self.platforms.append( platform ) if not "Platform" in self.queueDict[queueName]['ParametersDict'] and platform: result = Resources.getDIRACPlatform( platform ) if result['OK']: self.queueDict[queueName]['ParametersDict']['Platform'] = result['Value'][0] ceQueueDict = dict( ceDict ) ceQueueDict.update( self.queueDict[queueName]['ParametersDict'] ) # Generate the CE object for the queue or pick the already existing one # if the queue definition did not change queueHash = self.__generateQueueHash( ceQueueDict ) if queueName in self.queueCECache and self.queueCECache[queueName]['Hash'] == queueHash: queueCE = self.queueCECache[queueName]['CE'] else: result = ceFactory.getCE( ceName = ce, ceType = ceDict['CEType'], ceParametersDict = ceQueueDict ) if not result['OK']: return result self.queueCECache.setdefault( queueName, {} ) self.queueCECache[queueName]['Hash'] = queueHash self.queueCECache[queueName]['CE'] = result['Value'] queueCE = self.queueCECache[queueName]['CE'] self.queueDict[queueName]['CE'] = queueCE self.queueDict[queueName]['CEName'] = ce self.queueDict[queueName]['CEType'] = ceDict['CEType'] self.queueDict[queueName]['Site'] = site self.queueDict[queueName]['QueueName'] = queue self.queueDict[queueName]['Platform'] = platform result = self.queueDict[queueName]['CE'].isValid() if not result['OK']: self.log.fatal( result['Message'] ) return result if 'BundleProxy' in self.queueDict[queueName]['ParametersDict']: if self.queueDict[queueName]['ParametersDict']['BundleProxy'].lower() in ['true','yes','1']: self.queueDict[queueName]['BundleProxy'] = True elif 'BundleProxy' in ceDict: if ceDict['BundleProxy'].lower() in ['true','yes','1']: self.queueDict[queueName]['BundleProxy'] = True if site not in self.sites: self.sites.append( site ) return S_OK()
def __syncCSWithVOMS(self, vo): self.__adminMsgs = {'Errors': [], 'Info': []} resultDict = defaultdict(list) # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping(vo) if not result['OK']: return result vomsDIRACMapping = result['Value']['VOMSDIRAC'] diracVOMSMapping = result['Value']['DIRACVOMS'] noVOMSGroups = result['Value']['NoVOMS'] noSyncVOMSGroups = result['Value']['NoSyncVOMS'] vomsSrv = VOMSService(vo) # Get VOMS VO name result = vomsSrv.admGetVOName() if not result['OK']: self.log.error('Could not retrieve VOMS VO name', "for %s" % vo) return result vomsVOName = result['Value'].lstrip('/') self.log.verbose("VOMS VO Name for %s is %s" % (vo, vomsVOName)) # Get VOMS users result = vomsSrv.getUsers() if not result['OK']: self.log.error('Could not retrieve user information from VOMS', result['Message']) return result vomsUserDict = result['Value'] message = "There are %s user entries in VOMS for VO %s" % ( len(vomsUserDict), vomsVOName) self.__adminMsgs['Info'].append(message) self.log.info(message) # Get DIRAC users result = self.getVOUserData(vo) if not result['OK']: return result diracUserDict = result['Value'] self.__adminMsgs['Info'].append( "There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), vo)) self.log.info("There are %s registered users in DIRAC VO %s" % (len(diracUserDict), vo)) # Find new and obsoleted user DNs existingDNs = [] obsoletedDNs = [] newDNs = [] for user in diracUserDict: dn = diracUserDict[user]['DN'] # We can have users with more than one DN registered dnList = fromChar(dn) existingDNs.extend(dnList) for dn in dnList: if dn not in vomsUserDict: obsoletedDNs.append(dn) for dn in vomsUserDict: if dn not in existingDNs: newDNs.append(dn) allDiracUsers = getAllUsers() nonVOUserDict = {} nonVOUsers = list(set(allDiracUsers) - set(diracUserDict.keys())) if nonVOUsers: result = self.csapi.describeUsers(nonVOUsers) if not result['OK']: self.log.error('Could not retrieve CS User description') return result nonVOUserDict = result['Value'] # Process users defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo) newAddedUserDict = {} for dn in vomsUserDict: nickName = '' newDNForExistingUser = '' diracName = '' if dn in existingDNs: for user in diracUserDict: if dn == diracUserDict[user]['DN']: diracName = user if dn in newDNs: # Find if the DN is already registered in the DIRAC CS for user in nonVOUserDict: if dn == nonVOUserDict[user]['DN']: diracName = user # Check the nickName in the same VO to see if the user is already registered # with another DN result = vomsSrv.attGetUserNickname(dn, vomsUserDict[dn]['CA']) if result['OK']: nickName = result['Value'] if nickName in diracUserDict or nickName in newAddedUserDict: diracName = nickName # This is a flag for adding the new DN to an already existing user newDNForExistingUser = dn # We have a real new user if not diracName: if nickName: newDiracName = nickName else: newDiracName = getUserName(dn, vomsUserDict[dn]['mail'], vo) # If the chosen user name exists already, append a distinguishing suffix ind = 1 trialName = newDiracName while newDiracName in allDiracUsers: # We have a user with the same name but with a different DN newDiracName = "%s_%d" % (trialName, ind) ind += 1 # We now have everything to add the new user userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] } groupsWithRole = [] for role in vomsUserDict[dn]['Roles']: fullRole = "/%s/%s" % (vomsVOName, role) groupList = vomsDIRACMapping.get(fullRole, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) userDict['Groups'] = list( set(groupsWithRole + [defaultVOGroup])) message = "\n Added new user %s:\n" % newDiracName for key in userDict: message += " %s: %s\n" % (key, str(userDict[key])) self.__adminMsgs['Info'].append(message) self.voChanged = True if self.autoAddUsers: self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict))) result = self.csapi.modifyUser( newDiracName, userDict, createIfNonExistant=True) if not result['OK']: self.log.warn('Failed adding new user %s' % newDiracName) resultDict['NewUsers'].append(newDiracName) newAddedUserDict[newDiracName] = userDict continue # We have an already existing user modified = False userDict = { "DN": dn, "CA": vomsUserDict[dn]['CA'], "Email": vomsUserDict[dn]['mail'] } if newDNForExistingUser: userDict['DN'] = ','.join([dn, diracUserDict[diracName]['DN']]) modified = True existingGroups = diracUserDict.get(diracName, {}).get('Groups', []) nonVOGroups = list( set(existingGroups) - set(diracVOMSMapping.keys())) groupsWithRole = [] for role in vomsUserDict[dn]['Roles']: fullRole = "/%s/%s" % (vomsVOName, role) groupList = vomsDIRACMapping.get(fullRole, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup] for group in existingGroups: if group in nonVOGroups: continue role = diracVOMSMapping.get(group, '') # Among already existing groups for the user keep those without a special VOMS Role # because this membership is done by hand in the CS if not "Role" in role: keepGroups.append(group) # Keep existing groups with no VOMS attribute if any if group in noVOMSGroups: keepGroups.append(group) # Keep groups for which syncronization with VOMS is forbidden if group in noSyncVOMSGroups: keepGroups.append(group) userDict['Groups'] = list(set(keepGroups)) # Merge together groups for the same user but different DNs if diracName in newAddedUserDict: otherGroups = newAddedUserDict[diracName].get('Groups', []) userDict['Groups'] = list(set(keepGroups + otherGroups)) modified = True # Check if something changed before asking CSAPI to modify if diracName in diracUserDict: message = "\n Modified user %s:\n" % diracName modMsg = '' for key in userDict: if key == "Groups": addedGroups = set(userDict[key]) - set( diracUserDict.get(diracName, {}).get(key, [])) removedGroups = set( diracUserDict.get(diracName, {}).get( key, [])) - set(userDict[key]) if addedGroups: modMsg += " Added to group(s) %s\n" % ','.join( addedGroups) if removedGroups: modMsg += " Removed from group(s) %s\n" % ','.join( removedGroups) else: oldValue = str( diracUserDict.get(diracName, {}).get(key, '')) if str(userDict[key]) != oldValue: modMsg += " %s: %s -> %s\n" % ( key, oldValue, str(userDict[key])) if modMsg: self.__adminMsgs['Info'].append(message + modMsg) modified = True if self.autoModifyUsers and modified: result = self.csapi.modifyUser(diracName, userDict) if result['OK'] and result['Value']: self.log.info("Modified user %s: %s" % (diracName, str(userDict))) self.voChanged = True resultDict['ModifiedUsers'].append(diracName) # Check if there are potentially obsoleted users oldUsers = set() for user in diracUserDict: dnSet = set(fromChar(diracUserDict[user]['DN'])) if not dnSet.intersection(set( vomsUserDict.keys())) and user not in nonVOUserDict: for group in diracUserDict[user]['Groups']: if group not in noVOMSGroups: oldUsers.add(user) if oldUsers: self.voChanged = True if self.autoDeleteUsers: self.log.info('The following users will be deleted: %s' % str(oldUsers)) result = self.csapi.deleteUsers(oldUsers) if result['OK']: self.__adminMsgs['Info'].append( 'The following users are deleted from CS:\n %s\n' % str(oldUsers)) resultDict['DeletedUsers'] = oldUsers else: self.__adminMsgs['Errors'].append( 'Error in deleting users from CS:\n %s' % str(oldUsers)) self.log.error('Error while user deletion from CS', result) else: self.__adminMsgs['Info'].append( 'The following users to be checked for deletion:\n %s' % str(oldUsers)) self.log.info( 'The following users to be checked for deletion: %s' % str(oldUsers)) return S_OK(resultDict)
def registerUser( self ): """ This function is used to notify DIRAC admins about user registration request The logic is simple: 0) Check if request from this e-mail has already registered or not 1) Send mail to VO admin of requested VO 2) Send mail to users in group with UserAdministrator property 3) Send mail to users indicated in /Website/UserRegistrationAdmin option """ gLogger.info("Start processing a registration request") checkUserCredentials() # Check for having a DN but no username dn = getUserDN() if not dn: error = "Certificate is not loaded in the browser or DN is absent" gLogger.error( "Service response: %s" % error ) return { "success" : "false" , "error" : error } username = getUsername() if not username == "anonymous": error = "You are already registered in DIRAC with username: %s" % username gLogger.error( "Service response: %s" % error ) return { "success" : "false" , "error" : error } gLogger.info( "DN: %s" % dn ) if not "email" in request.params: error = "Can not get your email address from the request" gLogger.debug( "Service response: %s" % error ) return { "success" : "false" , "error" : error } userMail = request.params[ "email" ] result = self.isRequested( userMail ) gLogger.debug( result ) if result[ "OK" ]: return render( "/reg_done.mako" ) result = self.registerRequest( dn , userMail ) gLogger.debug( result ) if not result[ "OK" ]: return { "success" : "false" , "error" : result[ "Message" ] } vo = fromChar( request.params[ "vo" ] ) if not vo: error = "You should indicate a VirtualOrganization for membership" gLogger.debug( "Service response: %s" % error ) return { "success" : "false" , "error" : error } gLogger.info( "User want to be register in VO(s): %s" % vo ) body = str() for i in request.params: if not i in [ "registration_request" , "email" , "vo" ]: text = self.checkUnicode( request.params[ i ] ) info = "%s - %s" % ( i , text ) body = body + info + "\n" body = body + "DN - " + dn gLogger.debug( "email body: %s" % body ) adminList = self.__getAdminList( vo ) if not len( adminList ) > 0: error = "Can't get in contact with administrators about your request\n" error = error + "Most likely this DIRAC instance is not configured yet" gLogger.debug( "Service response: %s" % error ) return { "success" : "false" , "error" : error } adminList = uniqueElements( adminList ) gLogger.info( "Chosen admin(s): %s" % adminList ) sendDict = self.getMailDict( adminList ) if not len( sendDict ) > 0: error = "Can't get in contact with administrators about your request\n" error = error + "Most likely this DIRAC instance is not configured yet" gLogger.debug( "Service response: %s" % error ) return { "success" : "false" , "error" : error } gLogger.debug( "Final dictionary with mails to be used %s" % sendDict ) if socket.gethostname().find( '.' ) >= 0: hostname = socket.gethostname() else: hostname = socket.gethostbyaddr( socket.gethostname() )[ 0 ] title = "New user has sent registration request to %s" % hostname return self.sendMail( sendDict , title , body , userMail )
def getImages(self, resourceDict): """ Get the list of relevant CEs and their descriptions """ self.imageDict = {} ceFactory = EndpointFactory() result = getPilotBootstrapParameters(vo=self.vo, runningPod=self.runningPod) if not result['OK']: return result opParameters = result['Value'] for site in resourceDict: for ce in resourceDict[site]: ceDict = resourceDict[site][ce] ceTags = ceDict.get('Tag', []) if isinstance(ceTags, basestring): ceTags = fromChar(ceTags) ceMaxRAM = ceDict.get('MaxRAM', None) qDict = ceDict.pop('Images') for image in qDict: imageName = '%s_%s' % (ce, image) self.imageDict[imageName] = {} self.imageDict[imageName]['ParametersDict'] = qDict[image] self.imageDict[imageName]['ParametersDict'][ 'Image'] = image self.imageDict[imageName]['ParametersDict']['Site'] = site self.imageDict[imageName]['ParametersDict'][ 'Setup'] = gConfig.getValue('/DIRAC/Setup', 'unknown') self.imageDict[imageName]['ParametersDict'][ 'CPUTime'] = 99999999 imageTags = self.imageDict[imageName][ 'ParametersDict'].get('Tag') if imageTags and isinstance(imageTags, basestring): imageTags = fromChar(imageTags) self.imageDict[imageName]['ParametersDict'][ 'Tag'] = imageTags if ceTags: if imageTags: allTags = list(set(ceTags + imageTags)) self.imageDict[imageName]['ParametersDict'][ 'Tag'] = allTags else: self.imageDict[imageName]['ParametersDict'][ 'Tag'] = ceTags maxRAM = self.imageDict[imageName]['ParametersDict'].get( 'MaxRAM') maxRAM = ceMaxRAM if not maxRAM else maxRAM if maxRAM: self.imageDict[imageName]['ParametersDict'][ 'MaxRAM'] = maxRAM platform = '' if "Platform" in self.imageDict[imageName][ 'ParametersDict']: platform = self.imageDict[imageName]['ParametersDict'][ 'Platform'] elif "Platform" in ceDict: platform = ceDict['Platform'] if platform and not platform in self.platforms: self.platforms.append(platform) if not "Platform" in self.imageDict[imageName][ 'ParametersDict'] and platform: result = Resources.getDIRACPlatform(platform) if result['OK']: self.imageDict[imageName]['ParametersDict'][ 'Platform'] = result['Value'][0] ceImageDict = dict(ceDict) ceImageDict['CEName'] = ce ceImageDict['VO'] = self.vo ceImageDict['Image'] = image ceImageDict['RunningPod'] = self.runningPod ceImageDict['CSServers'] = gConfig.getValue( "/DIRAC/Configuration/Servers", []) ceImageDict.update( self.imageDict[imageName]['ParametersDict']) ceImageDict.update(opParameters) # Generate the CE object for the image or pick the already existing one # if the image definition did not change imageHash = self.__generateImageHash(ceImageDict) if imageName in self.imageCECache and self.imageCECache[ imageName]['Hash'] == imageHash: imageCE = self.imageCECache[imageName]['CE'] else: result = ceFactory.getCEObject(parameters=ceImageDict) if not result['OK']: return result self.imageCECache.setdefault(imageName, {}) self.imageCECache[imageName]['Hash'] = imageHash self.imageCECache[imageName]['CE'] = result['Value'] imageCE = self.imageCECache[imageName]['CE'] self.imageDict[imageName]['CE'] = imageCE self.imageDict[imageName]['CEName'] = ce self.imageDict[imageName]['CEType'] = ceDict['CEType'] self.imageDict[imageName]['Site'] = site self.imageDict[imageName]['ImageName'] = image self.imageDict[imageName]['Platform'] = platform self.imageDict[imageName]['MaxInstances'] = ceDict[ 'MaxInstances'] if not self.imageDict[imageName]['CE'].isValid(): self.log.error( 'Failed to instantiate CloudEndpoint for %s' % imageName) continue if site not in self.sites: self.sites.append(site) return S_OK()
def registerUser(self): """ This function is used to notify DIRAC admins about user registration request The logic is simple: 0) Check if request from this e-mail has already registered or not 1) Send mail to VO admin of requested VO 2) Send mail to users in group with UserAdministrator property 3) Send mail to users indicated in /Website/UserRegistrationAdmin option """ gLogger.info("Start processing a registration request") checkUserCredentials() # Check for having a DN but no username dn = getUserDN() if not dn: error = "Certificate is not loaded in the browser or DN is absent" gLogger.error("Service response: %s" % error) return {"success": "false", "error": error} username = getUsername() if not username == "anonymous": error = "You are already registered in DIRAC with username: %s" % username gLogger.error("Service response: %s" % error) return {"success": "false", "error": error} gLogger.info("DN: %s" % dn) if not "email" in request.params: error = "Can not get your email address from the request" gLogger.debug("Service response: %s" % error) return {"success": "false", "error": error} userMail = request.params["email"] result = self.isRequested(userMail) gLogger.debug(result) if result["OK"]: return render("/reg_done.mako") result = self.registerRequest(dn, userMail) gLogger.debug(result) if not result["OK"]: return {"success": "false", "error": result["Message"]} vo = fromChar(request.params["vo"]) if not vo: error = "You should indicate a VirtualOrganization for membership" gLogger.debug("Service response: %s" % error) return {"success": "false", "error": error} gLogger.info("User want to be register in VO(s): %s" % vo) body = str() for i in request.params: if not i in ["registration_request", "email", "vo"]: text = self.checkUnicode(request.params[i]) info = "%s - %s" % (i, text) body = body + info + "\n" body = body + "DN - " + dn gLogger.debug("email body: %s" % body) adminList = self.__getAdminList(vo) if not len(adminList) > 0: error = "Can't get in contact with administrators about your request\n" error = error + "Most likely this DIRAC instance is not configured yet" gLogger.debug("Service response: %s" % error) return {"success": "false", "error": error} adminList = uniqueElements(adminList) gLogger.info("Chosen admin(s): %s" % adminList) sendDict = self.getMailDict(adminList) if not len(sendDict) > 0: error = "Can't get in contact with administrators about your request\n" error = error + "Most likely this DIRAC instance is not configured yet" gLogger.debug("Service response: %s" % error) return {"success": "false", "error": error} gLogger.debug("Final dictionary with mails to be used %s" % sendDict) if socket.gethostname().find('.') >= 0: hostname = socket.gethostname() else: hostname = socket.gethostbyaddr(socket.gethostname())[0] title = "New user has sent registration request to %s" % hostname return self.sendMail(sendDict, title, body, userMail)
def syncCSWithVOMS(self): """Performs the synchronization of the DIRAC registry with the VOMS data. The resulting CSAPI object containing modifications is returned as part of the output dictionary. Those changes can be applied by the caller depending on the mode (dry or a real run) :return: S_OK with a dictionary containing the results of the synchronization operation """ resultDict = defaultdict(list) # Get DIRAC group vs VOMS Role Mappings result = getVOMSRoleGroupMapping(self.vo) if not result["OK"]: return result vomsDIRACMapping = result["Value"]["VOMSDIRAC"] diracVOMSMapping = result["Value"]["DIRACVOMS"] noVOMSGroups = result["Value"]["NoVOMS"] noSyncVOMSGroups = result["Value"]["NoSyncVOMS"] vomsSrv = VOMSService(self.vo) # Get VOMS users result = vomsSrv.getUsers() if not result["OK"]: self.log.error("Could not retrieve user information from VOMS", result["Message"]) return result self.vomsUserDict = result["Value"] message = "There are %s user entries in VOMS for VO %s" % (len(self.vomsUserDict), self.vomsVOName) self.adminMsgs["Info"].append(message) self.log.info("VOMS user entries", message) self.log.debug(self.vomsUserDict) # Get DIRAC users result = self.getVOUserData(self.vo) if not result["OK"]: return result diracUserDict = result["Value"] self.adminMsgs["Info"].append( "There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), self.vo) ) self.log.info( "Users already registered", ": there are %s registered users in DIRAC VO %s" % (len(diracUserDict), self.vo) ) # Find new and obsoleted user DNs existingDNs = [] obsoletedDNs = [] newDNs = [] for user in diracUserDict: dn = diracUserDict[user]["DN"] # We can have users with more than one DN registered dnList = fromChar(dn) existingDNs.extend(dnList) for dn in dnList: if dn not in self.vomsUserDict: obsoletedDNs.append(dn) for dn in self.vomsUserDict: if dn not in existingDNs: newDNs.append(dn) allDiracUsers = getAllUsers() nonVOUserDict = {} nonVOUsers = list(set(allDiracUsers) - set(diracUserDict)) if nonVOUsers: result = self.csapi.describeUsers(nonVOUsers) if not result["OK"]: self.log.error("Could not retrieve CS User description") return result nonVOUserDict = result["Value"] # Process users defaultVOGroup = getVOOption(self.vo, "DefaultGroup", "%s_user" % self.vo) # If a user is (previously put by hand) in a "QuarantineGroup", # then the default group will be ignored. # So, this option is only considered for the case of existing users. quarantineVOGroup = getVOOption(self.vo, "QuarantineGroup") newAddedUserDict = {} for dn in self.vomsUserDict: newDNForExistingUser = "" diracName = "" if dn in existingDNs: for user in diracUserDict: if dn in fromChar(diracUserDict[user]["DN"]): diracName = user break if dn in newDNs: # Find if the DN is already registered in the DIRAC CS for user in nonVOUserDict: if dn in fromChar(nonVOUserDict[user]["DN"]): diracName = user diracUserDict[diracName] = nonVOUserDict[user] break # Check the nickName in the same VO to see if the user is already registered # with another DN nickName = self.vomsUserDict[dn].get("nickname") if nickName in diracUserDict or nickName in newAddedUserDict: diracName = nickName # This is a flag for adding the new DN to an already existing user newDNForExistingUser = dn # We have a real new user if not diracName: if nickName: newDiracName = nickName else: newDiracName = self.getUserName(dn) # Do not consider users with Suspended status in VOMS if self.vomsUserDict[dn]["suspended"] or self.vomsUserDict[dn]["certSuspended"]: resultDict["SuspendedUsers"].append(newDiracName) continue # If the chosen user name exists already, append a distinguishing suffix ind = 1 trialName = newDiracName while newDiracName in allDiracUsers: # We have a user with the same name but with a different DN newDiracName = "%s_%d" % (trialName, ind) ind += 1 # We now have everything to add the new user userDict = {"DN": dn, "CA": self.vomsUserDict[dn]["CA"], "Email": self.vomsUserDict[dn]["mail"]} groupsWithRole = [] for role in self.vomsUserDict[dn]["Roles"]: groupList = vomsDIRACMapping.get(role, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) userDict["Groups"] = list(set(groupsWithRole + [defaultVOGroup])) # Run the sync plugins for extra info and/or validations if self.syncPlugin: try: self.syncPlugin.verifyAndUpdateUserInfo(newDiracName, userDict) except ValueError as e: self.log.error("Error validating new user", "nickname %s\n error %s" % (newDiracName, e)) self.adminMsgs["Errors"].append( "Error validating new user %s: %s\n %s" % (newDiracName, userDict, e) ) continue message = "\n Added new user %s:\n" % newDiracName for key in userDict: message += " %s: %s\n" % (key, str(userDict[key])) self.adminMsgs["Info"].append(message) self.voChanged = True if self.autoAddUsers: self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict))) result = self.csapi.modifyUser(newDiracName, userDict, createIfNonExistant=True) if not result["OK"]: self.log.warn("Failed adding new user %s" % newDiracName) resultDict["NewUsers"].append(newDiracName) newAddedUserDict[newDiracName] = userDict continue # We have an already existing user modified = False suspendedInVOMS = self.vomsUserDict[dn]["suspended"] or self.vomsUserDict[dn]["certSuspended"] suspendedVOList = getUserOption(diracName, "Suspended", []) knownEmail = getUserOption(diracName, "Email", None) userDict = { "DN": diracUserDict[diracName]["DN"], "CA": diracUserDict[diracName]["CA"], "Email": self.vomsUserDict[dn].get("mail", self.vomsUserDict[dn].get("emailAddress")) or knownEmail, } # Set Suspended status for the user for this particular VO if suspendedInVOMS and self.vo not in suspendedVOList: suspendedVOList.append(self.vo) userDict["Suspended"] = ",".join(suspendedVOList) modified = True # Remove the lifted Suspended status if not suspendedInVOMS and self.vo in suspendedVOList and self.autoLiftSuspendedStatus: newList = [] for vo in suspendedVOList: if vo != self.vo: newList.append(vo) if not newList: newList = ["None"] userDict["Suspended"] = ",".join(newList) modified = True if newDNForExistingUser: userDict["DN"] = ",".join([dn, diracUserDict.get(diracName, newAddedUserDict.get(diracName))["DN"]]) userDict["CA"] = ",".join( [self.vomsUserDict[dn]["CA"], diracUserDict.get(diracName, newAddedUserDict.get(diracName))["CA"]] ) modified = True existingGroups = diracUserDict.get(diracName, {}).get("Groups", []) nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping)) groupsWithRole = [] for role in self.vomsUserDict[dn]["Roles"]: groupList = vomsDIRACMapping.get(role, []) for group in groupList: if group not in noSyncVOMSGroups: groupsWithRole.append(group) keepGroups = nonVOGroups + groupsWithRole if not quarantineVOGroup or quarantineVOGroup not in existingGroups: keepGroups += [defaultVOGroup] if quarantineVOGroup and quarantineVOGroup in existingGroups: keepGroups = [quarantineVOGroup] for group in existingGroups: if group in nonVOGroups: continue role = diracVOMSMapping.get(group, "") # Among already existing groups for the user keep those without a special VOMS Role # because this membership is done by hand in the CS if "Role" not in role: keepGroups.append(group) # Keep existing groups with no VOMS attribute if any if group in noVOMSGroups: keepGroups.append(group) # Keep groups for which syncronization with VOMS is forbidden if group in noSyncVOMSGroups: keepGroups.append(group) userDict["Groups"] = list(set(keepGroups)) # Merge together groups for the same user but different DNs if diracName in newAddedUserDict: otherGroups = newAddedUserDict[diracName].get("Groups", []) userDict["Groups"] = list(set(keepGroups + otherGroups)) modified = True if not existingGroups and diracName in allDiracUsers: groups = getGroupsForUser(diracName) if groups["OK"]: self.log.info("Found groups for user %s %s" % (diracName, groups["Value"])) userDict["Groups"] = list(set(groups["Value"] + keepGroups)) addedGroups = list(set(userDict["Groups"]) - set(groups["Value"])) modified = True message = "\n Modified user %s:\n" % diracName message += " Added to group(s) %s\n" % ",".join(addedGroups) self.adminMsgs["Info"].append(message) # Check if something changed before asking CSAPI to modify if diracName in diracUserDict: message = "\n Modified user %s:\n" % diracName modMsg = "" for key in userDict: if key == "Groups": addedGroups = set(userDict[key]) - set(diracUserDict.get(diracName, {}).get(key, [])) removedGroups = set(diracUserDict.get(diracName, {}).get(key, [])) - set(userDict[key]) if addedGroups: modMsg += " Added to group(s) %s\n" % ",".join(addedGroups) if removedGroups: modMsg += " Removed from group(s) %s\n" % ",".join(removedGroups) elif key == "Suspended": if userDict["Suspended"] == "None": modMsg += " Suspended status removed\n" else: modMsg += " User Suspended in VOs: %s\n" % userDict["Suspended"] else: oldValue = str(diracUserDict.get(diracName, {}).get(key, "")) if str(userDict[key]) != oldValue: modMsg += " %s: %s -> %s\n" % (key, oldValue, str(userDict[key])) if modMsg: self.adminMsgs["Info"].append(message + modMsg) modified = True if self.autoModifyUsers and modified: result = self.csapi.modifyUser(diracName, userDict) if result["OK"] and result["Value"]: self.log.info("Modified user %s: %s" % (diracName, str(userDict))) self.voChanged = True resultDict["ModifiedUsers"].append(diracName) # Check if there are potentially obsoleted users oldUsers = set() for user in diracUserDict: dnSet = set(fromChar(diracUserDict[user]["DN"])) if not dnSet.intersection(set(self.vomsUserDict)) and user not in nonVOUserDict: existingGroups = diracUserDict.get(user, {}).get("Groups", []) nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping)) removedGroups = list(set(existingGroups) - set(nonVOGroups)) if removedGroups: self.log.info("Checking user for deletion", "%s: %s" % (user, existingGroups)) self.log.info("User has groups in other VOs", "%s: %s" % (user, nonVOGroups)) userDict = diracUserDict[user] userDict["Groups"] = nonVOGroups if self.autoModifyUsers: result = self.csapi.modifyUser(user, userDict) if result["OK"] and result["Value"]: self.log.info("Modified user %s: %s" % (user, str(userDict))) self.voChanged = True message = "\n Modified user %s:\n" % user modMsg = " Removed from group(s) %s\n" % ",".join(removedGroups) self.adminMsgs["Info"].append(message + modMsg) resultDict["ModifiedUsers"].append(user) continue if not any(group in noVOMSGroups for group in existingGroups): oldUsers.add(user) # Check for obsoleted DNs for user in diracUserDict: dnSet = set(fromChar(diracUserDict[user]["DN"])) for dn in dnSet: if dn in obsoletedDNs and user not in oldUsers: existingGroups = diracUserDict.get(user, {}).get("Groups", []) nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping)) if nonVOGroups: self.log.verbose("User has groups in other VOs", "%s: %s" % (user, nonVOGroups)) continue self.log.verbose("Modified user %s: dropped DN %s" % (user, dn)) if self.autoModifyUsers: userDict = diracUserDict[user] modDNSet = dnSet - set([dn]) if modDNSet: userDict["DN"] = ",".join(modDNSet) result = self.csapi.modifyUser(user, userDict) if result["OK"] and result["Value"]: self.log.info("Modified user %s: dropped DN %s" % (user, dn)) self.adminMsgs["Info"].append("Modified user %s: dropped DN %s" % (user, dn)) self.voChanged = True resultDict["ModifiedUsers"].append(diracName) else: oldUsers.add(user) if oldUsers: self.voChanged = True if self.autoDeleteUsers: self.log.info("The following users will be deleted: %s" % str(oldUsers)) result = self.csapi.deleteUsers(oldUsers) if result["OK"]: self.adminMsgs["Info"].append("The following users are deleted from CS:\n %s\n" % str(oldUsers)) resultDict["DeletedUsers"] = oldUsers else: self.adminMsgs["Errors"].append("Error in deleting users from CS:\n %s" % str(oldUsers)) self.log.error("Error while user deletion from CS", result) else: self.adminMsgs["Info"].append( "The following users to be checked for deletion:\n\t%s" % "\n\t".join(sorted(oldUsers)) ) self.log.info("The following users to be checked for deletion:", "\n\t".join(sorted(oldUsers))) resultDict["CSAPI"] = self.csapi resultDict["AdminMessages"] = self.adminMsgs resultDict["VOChanged"] = self.voChanged return S_OK(resultDict)
def getEndpoints(self, resourceDict): """Get the list of relevant CEs and their descriptions""" self.vmTypeDict = {} ceFactory = EndpointFactory() result = getPilotBootstrapParameters(vo=self.vo, runningPod=self.runningPod) if not result["OK"]: return result opParameters = result["Value"] for site in resourceDict: for ce in resourceDict[site]: ceDict = resourceDict[site][ce] ceTags = ceDict.get("Tag", []) if isinstance(ceTags, six.string_types): ceTags = fromChar(ceTags) ceMaxRAM = ceDict.get("MaxRAM", None) qDict = ceDict.pop("VMTypes") for vmType in qDict: vmTypeName = "%s_%s" % (ce, vmType) self.vmTypeDict[vmTypeName] = {} self.vmTypeDict[vmTypeName]["ParametersDict"] = qDict[vmType] self.vmTypeDict[vmTypeName]["ParametersDict"]["VMType"] = vmType self.vmTypeDict[vmTypeName]["ParametersDict"]["Site"] = site self.vmTypeDict[vmTypeName]["ParametersDict"]["Setup"] = gConfig.getValue("/DIRAC/Setup", "unknown") self.vmTypeDict[vmTypeName]["ParametersDict"]["CPUTime"] = 99999999 vmTypeTags = self.vmTypeDict[vmTypeName]["ParametersDict"].get("Tag") if vmTypeTags and isinstance(vmTypeTags, six.string_types): vmTypeTags = fromChar(vmTypeTags) self.vmTypeDict[vmTypeName]["ParametersDict"]["Tag"] = vmTypeTags if ceTags: if vmTypeTags: allTags = list(set(ceTags + vmTypeTags)) self.vmTypeDict[vmTypeName]["ParametersDict"]["Tag"] = allTags else: self.vmTypeDict[vmTypeName]["ParametersDict"]["Tag"] = ceTags maxRAM = self.vmTypeDict[vmTypeName]["ParametersDict"].get("MaxRAM") maxRAM = ceMaxRAM if not maxRAM else maxRAM if maxRAM: self.vmTypeDict[vmTypeName]["ParametersDict"]["MaxRAM"] = maxRAM ceWholeNode = ceDict.get("WholeNode", "true") wholeNode = self.vmTypeDict[vmTypeName]["ParametersDict"].get("WholeNode", ceWholeNode) if wholeNode.lower() in ("yes", "true"): self.vmTypeDict[vmTypeName]["ParametersDict"].setdefault("Tag", []) self.vmTypeDict[vmTypeName]["ParametersDict"]["Tag"].append("WholeNode") platform = "" if "Platform" in self.vmTypeDict[vmTypeName]["ParametersDict"]: platform = self.vmTypeDict[vmTypeName]["ParametersDict"]["Platform"] elif "Platform" in ceDict: platform = ceDict["Platform"] if platform and platform not in self.platforms: self.platforms.append(platform) if "Platform" not in self.vmTypeDict[vmTypeName]["ParametersDict"] and platform: result = Resources.getDIRACPlatform(platform) if result["OK"]: self.vmTypeDict[vmTypeName]["ParametersDict"]["Platform"] = result["Value"][0] ceVMTypeDict = dict(ceDict) ceVMTypeDict["CEName"] = ce ceVMTypeDict["VO"] = self.vo ceVMTypeDict["VMType"] = vmType ceVMTypeDict["RunningPod"] = self.runningPod ceVMTypeDict["CSServers"] = gConfig.getValue("/DIRAC/Configuration/Servers", []) ceVMTypeDict.update(self.vmTypeDict[vmTypeName]["ParametersDict"]) # Allow a resource-specifc CAPath to be set (as some clouds have their own CAs) # Otherwise fall back to the system-wide default(s) if "CAPath" not in ceVMTypeDict: ceVMTypeDict["CAPath"] = gConfig.getValue( "/DIRAC/Security/CAPath", "/opt/dirac/etc/grid-security/certificates/cas.pem" ) # Generate the CE object for the vmType or pick the already existing one # if the vmType definition did not change vmTypeHash = self.__generateVMTypeHash(ceVMTypeDict) if vmTypeName in self.vmTypeCECache and self.vmTypeCECache[vmTypeName]["Hash"] == vmTypeHash: vmTypeCE = self.vmTypeCECache[vmTypeName]["CE"] else: result = ceFactory.getCEObject(parameters=ceVMTypeDict) if not result["OK"]: return result self.vmTypeCECache.setdefault(vmTypeName, {}) self.vmTypeCECache[vmTypeName]["Hash"] = vmTypeHash self.vmTypeCECache[vmTypeName]["CE"] = result["Value"] vmTypeCE = self.vmTypeCECache[vmTypeName]["CE"] vmTypeCE.setBootstrapParameters(opParameters) self.vmTypeDict[vmTypeName]["CE"] = vmTypeCE self.vmTypeDict[vmTypeName]["CEName"] = ce self.vmTypeDict[vmTypeName]["CEType"] = ceDict["CEType"] self.vmTypeDict[vmTypeName]["Site"] = site self.vmTypeDict[vmTypeName]["VMType"] = vmType self.vmTypeDict[vmTypeName]["Platform"] = platform self.vmTypeDict[vmTypeName]["MaxInstances"] = ceDict["MaxInstances"] if not self.vmTypeDict[vmTypeName]["CE"].isValid(): self.log.error("Failed to instantiate CloudEndpoint for %s" % vmTypeName) continue if site not in self.sites: self.sites.append(site) return S_OK()
def __getPilotOptions( self, queue, pilotsToSubmit ): """ Prepare pilot options """ queueDict = self.queueDict[queue]['ParametersDict'] pilotOptions = [] setup = gConfig.getValue( "/DIRAC/Setup", "unknown" ) if setup == 'unknown': self.log.error( 'Setup is not defined in the configuration' ) return [ None, None ] pilotOptions.append( '-S %s' % setup ) opsHelper = Operations.Operations( group = self.pilotGroup, setup = setup ) #Installation defined? installationName = opsHelper.getValue( "Pilot/Installation", "" ) if installationName: pilotOptions.append( '-V %s' % installationName ) #Project defined? projectName = opsHelper.getValue( "Pilot/Project", "" ) if projectName: pilotOptions.append( '-l %s' % projectName ) else: self.log.info( 'DIRAC project will be installed by pilots' ) #Request a release diracVersion = opsHelper.getValue( "Pilot/Version", [] ) if not diracVersion: self.log.error( 'Pilot/Version is not defined in the configuration' ) return [ None, None ] #diracVersion is a list of accepted releases. Just take the first one pilotOptions.append( '-r %s' % diracVersion[0] ) ownerDN = self.pilotDN ownerGroup = self.pilotGroup # Request token for maximum pilot efficiency result = gProxyManager.requestToken( ownerDN, ownerGroup, pilotsToSubmit * self.maxJobsInFillMode ) if not result[ 'OK' ]: self.log.error( 'Invalid proxy token request', result['Message'] ) return [ None, None ] ( token, numberOfUses ) = result[ 'Value' ] pilotOptions.append( '-o /Security/ProxyToken=%s' % token ) # Use Filling mode pilotOptions.append( '-M %s' % min( numberOfUses, self.maxJobsInFillMode ) ) # Since each pilot will execute min( numberOfUses, self.maxJobsInFillMode ) # with numberOfUses tokens we can submit at most: # numberOfUses / min( numberOfUses, self.maxJobsInFillMode ) # pilots newPilotsToSubmit = numberOfUses / min( numberOfUses, self.maxJobsInFillMode ) if newPilotsToSubmit != pilotsToSubmit: self.log.info( 'Number of pilots to submit is changed to %d after getting the proxy token' % newPilotsToSubmit ) pilotsToSubmit = newPilotsToSubmit # Debug if self.pilotLogLevel.lower() == 'debug': pilotOptions.append( '-d' ) # CS Servers csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] ) pilotOptions.append( '-C %s' % ",".join( csServers ) ) # DIRAC Extensions to be used in pilots pilotExtensionsList = opsHelper.getValue( "Pilot/Extensions", [] ) extensionsList = [] if pilotExtensionsList: if pilotExtensionsList[0] != 'None': extensionsList = pilotExtensionsList else: extensionsList = CSGlobals.getCSExtensions() if extensionsList: pilotOptions.append( '-e %s' % ",".join( extensionsList ) ) # Requested CPU time pilotOptions.append( '-T %s' % queueDict['CPUTime'] ) # CEName pilotOptions.append( '-N %s' % self.queueDict[queue]['CEName'] ) # Queue pilotOptions.append( '-Q %s' % self.queueDict[queue]['QueueName'] ) # SiteName pilotOptions.append( '-n %s' % queueDict['Site'] ) if 'ClientPlatform' in queueDict: pilotOptions.append( "-p '%s'" % queueDict['ClientPlatform'] ) if 'SharedArea' in queueDict: pilotOptions.append( "-o '/LocalSite/SharedArea=%s'" % queueDict['SharedArea'] ) if 'SI00' in queueDict: factor = float( queueDict['SI00'] ) / 250. pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % factor ) pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % factor ) else: if 'CPUScalingFactor' in queueDict: pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % queueDict['CPUScalingFactor'] ) if 'CPUNormalizationFactor' in queueDict: pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % queueDict['CPUNormalizationFactor'] ) if "ExtraPilotOptions" in queueDict: pilotOptions.append( queueDict['ExtraPilotOptions'] ) # Hack if self.defaultSubmitPools: pilotOptions.append( '-o /Resources/Computing/CEDefaults/SubmitPool=%s' % self.defaultSubmitPools ) if "Tag" in queueDict: tagString = ','.join( fromChar( queueDict['Tag'] ) ) pilotOptions.append( '-o /Resources/Computing/CEDefaults/Tag=%s' % tagString ) if self.group: pilotOptions.append( '-G %s' % self.group ) self.log.verbose( "pilotOptions: ", ' '.join( pilotOptions ) ) return [ pilotOptions, pilotsToSubmit ]
def matchQueue(jobJDL, queueDict, fullMatch=False): """ Match the job description to the queue definition :param str job: JDL job description :param bool fullMatch: test matching on all the criteria :param dict queueDict: queue parameters dictionary :return: S_OK/S_ERROR, Value - result of matching, S_OK if matched or S_ERROR with the reason for no match """ # Check the job description validity job = ClassAd(jobJDL) if not job.isOK(): return S_ERROR("Invalid job description") noMatchReasons = [] # Check job requirements to resource # 1. CPUTime cpuTime = job.getAttributeInt("CPUTime") if not cpuTime: cpuTime = 84600 if cpuTime > int(queueDict.get("CPUTime", 0)): noMatchReasons.append("Job CPUTime requirement not satisfied") if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 2. Multi-value match requirements for parameter in ["Site", "GridCE", "Platform", "JobType"]: if parameter in queueDict: valueSet = set(job.getListFromExpression(parameter)) if not valueSet: valueSet = set(job.getListFromExpression("%ss" % parameter)) queueSet = set(fromChar(queueDict[parameter])) if valueSet and queueSet and not valueSet.intersection(queueSet): valueToPrint = ",".join(valueSet) if len(valueToPrint) > 20: valueToPrint = "%s..." % valueToPrint[:20] noMatchReasons.append("Job %s %s requirement not satisfied" % (parameter, valueToPrint)) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 3. Banned multi-value match requirements for par in ["Site", "GridCE", "Platform", "JobType"]: parameter = "Banned%s" % par if par in queueDict: valueSet = set(job.getListFromExpression(parameter)) if not valueSet: valueSet = set(job.getListFromExpression("%ss" % parameter)) queueSet = set(fromChar(queueDict[par])) if valueSet and queueSet and valueSet.issubset(queueSet): valueToPrint = ",".join(valueSet) if len(valueToPrint) > 20: valueToPrint = "%s..." % valueToPrint[:20] noMatchReasons.append("Job %s %s requirement not satisfied" % (parameter, valueToPrint)) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 4. Tags tags = set(job.getListFromExpression("Tag")) nProc = job.getAttributeInt("NumberOfProcessors") if nProc and nProc > 1: tags.add("MultiProcessor") wholeNode = job.getAttributeString("WholeNode") if wholeNode: tags.add("WholeNode") queueTags = set(queueDict.get("Tag", [])) if not tags.issubset(queueTags): noMatchReasons.append("Job Tag %s not satisfied" % ",".join(tags)) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 4. MultiProcessor requirements if nProc and nProc > int(queueDict.get("NumberOfProcessors", 1)): noMatchReasons.append("Job NumberOfProcessors %d requirement not satisfied" % nProc) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 5. RAM ram = job.getAttributeInt("RAM") # If MaxRAM is not specified in the queue description, assume 2GB if ram and ram > int(queueDict.get("MaxRAM", 2048) / 1024): noMatchReasons.append("Job RAM %d requirement not satisfied" % ram) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # Check resource requirements to job # 1. OwnerGroup - rare case but still if "OwnerGroup" in queueDict: result = getProxyInfo(disableVOMS=True) if not result["OK"]: return S_ERROR("No valid proxy available") ownerGroup = result["Value"]["group"] if ownerGroup != queueDict["OwnerGroup"]: noMatchReasons.append("Resource OwnerGroup %s requirement not satisfied" % queueDict["OwnerGroup"]) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 2. Required tags requiredTags = set(queueDict.get("RequiredTags", [])) if not requiredTags.issubset(tags): noMatchReasons.append("Resource RequiredTags %s not satisfied" % ",".join(requiredTags)) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) # 3. RunningLimit site = queueDict["Site"] ce = queueDict.get("GridCE") opsHelper = Operations() result = opsHelper.getSections("JobScheduling/RunningLimit") if result["OK"] and site in result["Value"]: result = opsHelper.getSections("JobScheduling/RunningLimit/%s" % site) if result["OK"]: for parameter in result["Value"]: value = job.getAttributeString(parameter) if ( value and ( opsHelper.getValue("JobScheduling/RunningLimit/%s/%s/%s" % (site, parameter, value), 1) or opsHelper.getValue( "JobScheduling/RunningLimit/%s/CEs/%s/%s/%s" % (site, ce, parameter, value), 1 ) ) == 0 ): noMatchReasons.append("Resource operational %s requirement not satisfied" % parameter) if not fullMatch: return S_OK({"Match": False, "Reason": noMatchReasons[0]}) return S_OK({"Match": not bool(noMatchReasons), "Reason": noMatchReasons})
def _ByJobType(self): """ By default, all sites are allowed to do every job. The actual rules are freely specified in the Operation JobTypeMapping section. The content of the section may look like this: User { Exclude = PAK Exclude += Ferrara Exclude += Bologna Exclude += Paris Exclude += CERN Exclude += IN2P3 Allow { Paris = IN2P3 CERN = CERN IN2P3 = IN2P3 } } DataReconstruction { Exclude = PAK Exclude += Ferrara Exclude += CERN Exclude += IN2P3 Allow { Ferrara = CERN CERN = CERN IN2P3 = IN2P3 IN2P3 += CERN } } Merge { Exclude = ALL Allow { CERN = CERN IN2P3 = IN2P3 } } The sites in the exclusion list will be removed. The allow section says where each site may help another site """ # 1. get sites list res = getSites() if not res['OK']: gLogger.error("Could not get the list of sites", res['Message']) return res destSites = set(res['Value']) # 2. get JobTypeMapping "Exclude" value (and add autoAddedSites) gLogger.debug("Getting JobTypeMapping 'Exclude' value (and add autoAddedSites)") jobType = self.params['JobType'] if not jobType: raise RuntimeError("No jobType specified") excludedSites = set(self.opsH.getValue('JobTypeMapping/%s/Exclude' % jobType, [])) gLogger.debug("Explicitly excluded sites for %s task: %s" % (jobType, ','.join(excludedSites))) autoAddedSites = self.opsH.getValue('JobTypeMapping/AutoAddedSites', []) if 'WithStorage' in autoAddedSites: # Add all sites with storage, such that jobs can run wherever data is autoAddedSites.remove('WithStorage') autoAddedSites += DMSHelpers().getTiers(withStorage=True, tier=(0, 1, 2)) # 3. removing sites in Exclude if not excludedSites: pass elif 'ALL' in excludedSites: destSites = set() else: destSites -= excludedSites # 4. get JobTypeMapping "Allow" section res = self.opsH.getOptionsDict('JobTypeMapping/%s/Allow' % jobType) if not res['OK']: gLogger.debug(res['Message']) allowed = {} else: allowed = dict((site, set(fromChar(fromSites))) for site, fromSites in res['Value'].iteritems()) autoAddedSites = set(self.opsH.getValue('JobTypeMapping/%s/AutoAddedSites' % jobType, autoAddedSites)) gLogger.debug("Auto-added sites for %s task: %s" % (jobType, ','.join(autoAddedSites))) # 5. add autoAddedSites, if requested for autoAddedSite in autoAddedSites: allowed.setdefault(autoAddedSite, set()).add(autoAddedSite) gLogger.debug("Allowed sites for %s task: %s" % (jobType, ','.join(allowed))) # 6. Allowing sites that should be allowed taskSiteDestination = self._BySE() for destSite, fromSites in allowed.iteritems(): for fromSite in fromSites: if not taskSiteDestination or fromSite in taskSiteDestination: destSites.add(destSite) gLogger.debug("Computed list of destination sites for %s task with TargetSE %s: %s" % (jobType, self.params['TargetSE'], ','.join(destSites))) return destSites
def matchQueue(jobJDL, queueDict, fullMatch=False): """ Match the job description to the queue definition :param str job: JDL job description :param bool fullMatch: test matching on all the criteria :param dict queueDict: queue parameters dictionary :return: S_OK/S_ERROR, Value - result of matching, S_OK if matched or S_ERROR with the reason for no match """ # Check the job description validity job = ClassAd(jobJDL) if not job.isOK(): return S_ERROR('Invalid job description') noMatchReasons = [] # Check job requirements to resource # 1. CPUTime cpuTime = job.getAttributeInt('CPUTime') if not cpuTime: cpuTime = 84600 if cpuTime > queueDict.get('CPUTime', 0.): noMatchReasons.append('Job CPUTime requirement not satisfied') if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 2. Multi-value match requirements for parameter in [ 'Site', 'GridCE', 'Platform', 'GridMiddleware', 'PilotType', 'SubmitPool', 'JobType' ]: if parameter in queueDict: valueSet = set(job.getListFromExpression(parameter)) if not valueSet: valueSet = set(job.getListFromExpression('%ss' % parameter)) queueSet = set(fromChar(queueDict[parameter])) if valueSet and queueSet and not valueSet.intersection(queueSet): valueToPrint = ','.join(valueSet) if len(valueToPrint) > 20: valueToPrint = "%s..." % valueToPrint[:20] noMatchReasons.append('Job %s %s requirement not satisfied' % (parameter, valueToPrint)) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 3. Banned multi-value match requirements for par in [ 'Site', 'GridCE', 'Platform', 'GridMiddleware', 'PilotType', 'SubmitPool', 'JobType' ]: parameter = "Banned%s" % par if par in queueDict: valueSet = set(job.getListFromExpression(parameter)) if not valueSet: valueSet = set(job.getListFromExpression('%ss' % parameter)) queueSet = set(fromChar(queueDict[par])) if valueSet and queueSet and valueSet.issubset(queueSet): valueToPrint = ','.join(valueSet) if len(valueToPrint) > 20: valueToPrint = "%s..." % valueToPrint[:20] noMatchReasons.append('Job %s %s requirement not satisfied' % (parameter, valueToPrint)) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 4. Tags tags = set(job.getListFromExpression('Tag')) nProc = job.getAttributeInt('NumberOfProcessors') if nProc and nProc > 1: tags.add('MultiProcessor') wholeNode = job.getAttributeString('WholeNode') if wholeNode: tags.add('WholeNode') queueTags = set(queueDict.get('Tags', [])) if not tags.issubset(queueTags): noMatchReasons.append('Job Tag %s not satisfied' % ','.join(tags)) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 4. MultiProcessor requirements if nProc and nProc > int(queueDict.get('NumberOfProcessors', 1)): noMatchReasons.append( 'Job NumberOfProcessors %d requirement not satisfied' % nProc) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 5. RAM ram = job.getAttributeInt('RAM') # If MaxRAM is not specified in the queue description, assume 2GB if ram and ram > int(queueDict.get('MaxRAM', 2048)) / 1024: noMatchReasons.append('Job RAM %d requirement not satisfied' % ram) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # Check resource requirements to job # 1. OwnerGroup - rare case but still if "OwnerGroup" in queueDict: result = getProxyInfo(disableVOMS=True) if not result['OK']: return S_ERROR('No valid proxy available') ownerGroup = result['Value']['group'] if ownerGroup != queueDict['OwnerGroup']: noMatchReasons.append( 'Resource OwnerGroup %s requirement not satisfied' % queueDict['OwnerGroup']) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 2. Required tags requiredTags = set(queueDict.get('RequiredTags', [])) if not requiredTags.issubset(tags): noMatchReasons.append('Resource RequiredTags %s not satisfied' % ','.join(requiredTags)) if not fullMatch: return S_OK({'Match': False, 'Reason': noMatchReasons[0]}) # 3. RunningLimit site = queueDict['Site'] opsHelper = Operations() result = opsHelper.getSections('JobScheduling/RunningLimit') if result['OK'] and site in result['Value']: result = opsHelper.getSections('JobScheduling/RunningLimit/%s' % site) if result['OK']: for parameter in result['Value']: value = job.getAttributeString(parameter) if value and opsHelper.getValue( 'JobScheduling/RunningLimit/%s/%s/%s' % (site, parameter, value), 1) == 0: noMatchReasons.append( 'Resource operational %s requirement not satisfied' % parameter) if not fullMatch: return S_OK({ 'Match': False, 'Reason': noMatchReasons[0] }) return S_OK({'Match': not bool(noMatchReasons), 'Reason': noMatchReasons})