def getQueue(site, ce, queue):
""" Get parameters of the specified queue
"""
grid = site.split('.')[0]
result = gConfig.getOptionsDict('/Resources/Sites/%s/%s/CEs/%s' % (grid, site, ce))
if not result['OK']:
return result
resultDict = result['Value']
# Get queue defaults
result = gConfig.getOptionsDict('/Resources/Sites/%s/%s/CEs/%s/Queues/%s' % (grid, site, ce, queue))
if not result['OK']:
return result
resultDict.update(result['Value'])
# Handle tag lists for the queue
for tagFieldName in ('Tag', 'RequiredTag'):
tags = []
ceTags = resultDict.get(tagFieldName)
if ceTags:
tags = fromChar(ceTags)
queueTags = resultDict.get(tagFieldName)
if queueTags:
queueTags = fromChar(queueTags)
tags = list(set(tags + queueTags))
if tags:
resultDict[tagFieldName] = tags
resultDict['Queue'] = queue
return S_OK(resultDict)
def _handleDestination( self, paramsDict, getSitesForSE = None ):
""" Handle Sites and TargetSE in the parameters
"""
try:
sites = ['ANY']
if paramsDict['Site']:
# 'Site' comes from the XML and therefore is ; separated
sites = fromChar( paramsDict['Site'], sepChar = ';' )
except KeyError:
pass
try:
seList = ['Unknown']
if paramsDict['TargetSE']:
seList = fromChar( paramsDict['TargetSE'] )
except KeyError:
pass
if not seList or seList == ['Unknown']:
return sites
# from now on we know there is some TargetSE requested
if not getSitesForSE:
from DIRAC.Core.Utilities.SiteSEMapping import getSitesForSE
seSites = []
for se in seList:
res = getSitesForSE( se )
if not res['OK']:
self.log.warn( 'Could not get Sites associated to SE', res['Message'] )
else:
thisSESites = res['Value']
if not thisSESites:
continue
if seSites == []:
seSites = copy.deepcopy( thisSESites )
else:
# We make an OR of the possible sites
for nSE in list( thisSESites ):
if nSE not in seSites:
seSites.append( nSE )
# Now we need to make the AND with the sites, if defined
if sites == ['ANY']:
return seSites
else:
# Need to get the AND
for nSE in list( seSites ):
if nSE not in sites:
seSites.remove( nSE )
return seSites
def _BySE( self ):
""" Matches using TargetSE. This is the standard plugin.
"""
destSites = set()
try:
seList = ['Unknown']
if self.params['TargetSE']:
if isinstance( self.params['TargetSE'], basestring ):
seList = fromChar( self.params['TargetSE'] )
elif isinstance( self.params['TargetSE'], list ):
seList = self.params['TargetSE']
except KeyError:
pass
if not seList or seList == ['Unknown']:
return destSites
for se in seList:
res = getSitesForSE( se )
if not res['OK']:
gLogger.warn( "Could not get Sites associated to SE", res['Message'] )
else:
thisSESites = res['Value']
if thisSESites:
# We make an OR of the possible sites
destSites.update( thisSESites )
gLogger.debug( "Destinations: %s" % ','.join ( destSites ) )
return destSites
def _handleDestination( self, paramsDict ):
""" Handle Sites and TargetSE in the parameters
"""
try:
sites = ['ANY']
if paramsDict['Site']:
# 'Site' comes from the XML and therefore is ; separated
sites = fromChar( paramsDict['Site'], sepChar = ';' )
except KeyError:
pass
if self.destinationPlugin_o:
destinationPlugin_o = self.destinationPlugin_o
else:
res = self.__generatePluginObject( self.destinationPlugin )
if not res['OK']:
self._logFatal( "Could not generate a destination plugin object" )
return res
destinationPlugin_o = res['Value']
destinationPlugin_o.setParameters( paramsDict )
destSites = destinationPlugin_o.run()
if not destSites:
return sites
# Now we need to make the AND with the sites, if defined
if sites != ['ANY']:
# Need to get the AND
destSites &= set( sites )
return list( destSites )
def __init__( self, useCertificates = False ):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__( self )
## setup logger
self.log = gLogger.getSubLogger( "RequestManagement/RequestClient" )
## dict to store all RPC clients for easy reuse
self.__requestRPCClientsDict = {}
## local if any defined
local = PathFinder.getServiceURL( "RequestManagement/localURL" )
if local:
self.__requestRPCClientsDict.setdefault( "local" , [ self.__requestRPCClient( local ) ] )
## central if any defined
central = PathFinder.getServiceURL( "RequestManagement/centralURL" )
if central:
self.__requestRPCClientsDict.setdefault( "central", [ self.__requestRPCClient( central ) ] )
## voboxes if any defined
voBoxUrls = fromChar( PathFinder.getServiceURL( "RequestManagement/voBoxURLs" ) )
if voBoxUrls:
self.__requestRPCClientsDict.setdefault( "voboxes", [] )
for voBoxURL in randomize( voBoxUrls ):
self.__requestRPCClientsDict["voboxes"].append( self.__requestRPCClient( voBoxURL ) )
self.setServer( 'RequestManagement/centralURL' )
def requestProxies( self, timeout = 120 ):
""" get request proxies dict """
if not self.__requestProxiesDict:
self.__requestProxiesDict = {}
proxiesURLs = fromChar( PathFinder.getServiceURL( "RequestManagement/ReqProxyURLs" ) )
if not proxiesURLs:
self.log.warn( "CS option RequestManagement/ReqProxyURLs is not set!" )
for proxyURL in proxiesURLs:
self.log.debug( "creating RequestProxy for url = %s" % proxyURL )
self.__requestProxiesDict[proxyURL] = RPCClient( proxyURL, timeout = timeout )
return self.__requestProxiesDict
def getNumberOfPayloadProcessors(siteName=None, gridCE=None, queue=None):
""" Gets the number of processors allowed for a single JobAgent (so for a "inner" CE).
(NB: this does not refer to the job processors).
This is normally used ONLY when a pilot instantiates more than one JobAgent (MultiLaunchAgent pilot command).
The siteName/gridCE/queue parameters are normally not necessary.
Tries to find it in this order:
1) from the /Resources/Computing/CEDefaults/NumberOfPayloadProcessors (which is what pilot 3 fills up)
2) if not present but there's WholeNode tag, use the getNumberOfProcessors function above
3) otherwise returns 1
"""
# 1) from /Resources/Computing/CEDefaults/NumberOfPayloadProcessors
gLogger.info(
"Getting NumberOfPayloadProcessors from /Resources/Computing/CEDefaults/NumberOfPayloadProcessors"
)
NumberOfPayloadProcessors = gConfig.getValue(
'/Resources/Computing/CEDefaults/NumberOfPayloadProcessors')
if NumberOfPayloadProcessors:
return NumberOfPayloadProcessors
# 2) Checks if 'Whole' is one of the used tags
# Tags of the CE
tags = fromChar(
gConfig.getValue(
'/Resources/Sites/%s/%s/CEs/%s/Tag' %
(siteName.split('.')[0], siteName, gridCE), ''))
# Tags of the Queue
tags += fromChar(
gConfig.getValue(
'/Resources/Sites/%s/%s/CEs/%s/Queues/%s/Tag' %
(siteName.split('.')[0], siteName, gridCE, queue), ''))
if 'WholeNode' in tags:
return getNumberOfProcessors()
# 3) Just returns a conservative "1"
return 1
def getVOUserReport( self, vo ):
"""
:param str vo: VO name
:return: report string
"""
result = self.getVOUserData( vo, refreshFlag = True )
if not result['OK']:
return result
userDict = result['Value']
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping( vo )
if not result['OK']:
return result
diracVOMSMapping = result['Value']['DIRACVOMS']
records = []
groupDict = defaultdict( int )
multiDNUsers = {}
suspendedUsers = []
for user in userDict:
for group in userDict[user]['Groups']:
groupDict[group] += 1
dnList = fromChar( userDict[user]['DN'] )
if len( dnList ) > 1:
multiDNUsers[user] = dnList
if userDict[user].get( 'Status', 'Active' ) == 'Suspended':
suspendedUsers.append( user )
for group in diracVOMSMapping:
records.append( ( group, str( groupDict[group] ), diracVOMSMapping.get( group, '' ) ) )
fields = [ 'Group', 'Number of users', 'VOMS Role' ]
output = printTable( fields, records, sortField = 'Group', printOut = False, numbering = False )
if multiDNUsers:
output += '\nUsers with multiple DNs:\n'
for user in multiDNUsers:
output += ' %s:\n' % user
for dn in multiDNUsers[user]:
output += ' %s\n' % dn
if suspendedUsers:
output += '\n%d suspended users:\n' % len( suspendedUsers )
output += ' %s' % ','.join( suspendedUsers )
return S_OK( output )
def getVOUserReport(self):
"""Get a report string with the current status of the DIRAC Registry for the
Virtual Organization
:return: S_OK with the report string as Value
"""
result = self.getVOUserData(refreshFlag=True)
if not result["OK"]:
return result
userDict = result["Value"]
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(self.vo)
if not result["OK"]:
return result
diracVOMSMapping = result["Value"]["DIRACVOMS"]
records = []
groupDict = defaultdict(int)
multiDNUsers = {}
suspendedUsers = []
for user in userDict:
for group in userDict[user]["Groups"]:
groupDict[group] += 1
dnList = fromChar(userDict[user]["DN"])
if len(dnList) > 1:
multiDNUsers[user] = dnList
if userDict[user].get("Status", "Active") == "Suspended":
suspendedUsers.append(user)
for group in diracVOMSMapping:
records.append((group, str(groupDict[group]), diracVOMSMapping.get(group, "")))
fields = ["Group", "Number of users", "VOMS Role"]
output = printTable(fields, records, sortField="Group", printOut=False, numbering=False)
if multiDNUsers:
output += "\nUsers with multiple DNs:\n"
for user in multiDNUsers:
output += " %s:\n" % user
for dn in multiDNUsers[user]:
output += " %s\n" % dn
if suspendedUsers:
output += "\n%d suspended users:\n" % len(suspendedUsers)
output += " %s" % ",".join(suspendedUsers)
return S_OK(output)
def requestProxies(self, timeout=120):
""" get request proxies dict """
if not self.__requestProxiesDict:
self.__requestProxiesDict = {}
proxiesURLs = fromChar(
PathFinder.getServiceURL("RequestManagement/ReqProxyURLs"))
if not proxiesURLs:
self.log.warn(
"CS option RequestManagement/ReqProxyURLs is not set!")
for proxyURL in proxiesURLs:
self.log.debug("creating RequestProxy for url = %s" % proxyURL)
self.__requestProxiesDict[proxyURL] = RPCClient(
proxyURL, timeout=timeout)
return self.__requestProxiesDict
def getVOUserReport(self):
""" Get a report string with the current status of the DIRAC Registry for the
Virtual Organization
:return: S_OK with the report string as Value
"""
result = self.getVOUserData(refreshFlag=True)
if not result['OK']:
return result
userDict = result['Value']
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(self.vo)
if not result['OK']:
return result
diracVOMSMapping = result['Value']['DIRACVOMS']
records = []
groupDict = defaultdict(int)
multiDNUsers = {}
suspendedUsers = []
for user in userDict:
for group in userDict[user]['Groups']:
groupDict[group] += 1
dnList = fromChar(userDict[user]['DN'])
if len(dnList) > 1:
multiDNUsers[user] = dnList
if userDict[user].get('Status', 'Active') == 'Suspended':
suspendedUsers.append(user)
for group in diracVOMSMapping:
records.append((group, str(groupDict[group]), diracVOMSMapping.get(group, '')))
fields = ['Group', 'Number of users', 'VOMS Role']
output = printTable(fields, records, sortField='Group', printOut=False, numbering=False)
if multiDNUsers:
output += '\nUsers with multiple DNs:\n'
for user in multiDNUsers:
output += ' %s:\n' % user
for dn in multiDNUsers[user]:
output += ' %s\n' % dn
if suspendedUsers:
output += '\n%d suspended users:\n' % len(suspendedUsers)
output += ' %s' % ','.join(suspendedUsers)
return S_OK(output)
def getQueue( site, ce, queue ):
""" Get parameters of the specified queue
"""
Tags = []
grid = site.split( '.' )[0]
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/CEs/%s' % ( grid, site, ce ) )
if not result['OK']:
return result
resultDict = result['Value']
ceTags = resultDict.get( 'Tag' )
if ceTags:
Tags = fromChar( ceTags )
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/CEs/%s/Queues/%s' % ( grid, site, ce, queue ) )
if not result['OK']:
return result
resultDict.update( result['Value'] )
queueTags = resultDict.get( 'Tag' )
if queueTags:
queueTags = fromChar( queueTags )
Tags = list( set( Tags + queueTags ) )
if Tags:
resultDict['Tag'] = Tags
resultDict['Queue'] = queue
return S_OK( resultDict )
def getQueue( site, ce, queue ):
""" Get parameters of the specified queue
"""
Tags = []
grid = site.split( '.' )[0]
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/CEs/%s' % ( grid, site, ce ) )
if not result['OK']:
return result
resultDict = result['Value']
ceTags = resultDict.get( 'Tag' )
if ceTags:
Tags = fromChar( ceTags )
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/CEs/%s/Queues/%s' % ( grid, site, ce, queue ) )
if not result['OK']:
return result
resultDict.update( result['Value'] )
queueTags = resultDict.get( 'Tag' )
if queueTags:
queueTags = fromChar( queueTags )
Tags = list( set( Tags + queueTags ) )
if Tags:
resultDict['Tag'] = Tags
resultDict['Queue'] = queue
return S_OK( resultDict )
def requestProxies(self, timeout=120):
"""get request proxies dict"""
# Forward all the connection options to the requestClient
# (e.g. the userDN to use)
kwargs = self.getClientKWArgs()
kwargs["timeout"] = timeout
if not self.__requestProxiesDict:
self.__requestProxiesDict = {}
proxiesURLs = fromChar(PathFinder.getServiceURL("RequestManagement/ReqProxyURLs"))
if not proxiesURLs:
self.log.warn("CS option RequestManagement/ReqProxyURLs is not set!")
for proxyURL in proxiesURLs:
self.log.debug("creating RequestProxy for url = %s" % proxyURL)
pc = Client(**kwargs)
pc.setServer(proxyURL)
self.__requestProxiesDict[proxyURL] = pc
return self.__requestProxiesDict
def __init__( self, useCertificates = False ):
""" Constructor of the RequestClient class
"""
voBoxUrls = fromChar( PathFinder.getServiceURL( "RequestManagement/voBoxURLs" ) )
self.voBoxUrls = []
if voBoxUrls:
self.voBoxUrls = randomize( voBoxUrls )
local = PathFinder.getServiceURL( "RequestManagement/localURL" )
self.local = False
if local:
self.local = local
central = PathFinder.getServiceURL( "RequestManagement/centralURL" )
self.central = False
if central:
self.central = central
self.setServer( 'RequestManagement/centralURL' )
def __init__(self, useCertificates=False):
""" Constructor of the RequestClient class
"""
voBoxUrls = fromChar(
PathFinder.getServiceURL("RequestManagement/voBoxURLs"))
self.voBoxUrls = []
if voBoxUrls:
self.voBoxUrls = randomize(voBoxUrls)
local = PathFinder.getServiceURL("RequestManagement/localURL")
self.local = False
if local:
self.local = local
central = PathFinder.getServiceURL("RequestManagement/centralURL")
self.central = False
if central:
self.central = central
self.setServer('RequestManagement/centralURL')
def getJobInformation(self, diracAPI, jobMon, jdlOnly=False):
"""Get all the information for the job.
The InputData, TaskID, OutputData can either be taken from properly filled JDL or
* inputData from jobMonitor getInputData
* TaskID from the name of The job via jobMonitor getJobAttribute JobName
* ProductionOutputData: from jobMonitor getJobParameter ProductionOutputData
This would be faster if we could do bulk calls for all of these
"""
if not jdlOnly:
# this is actually slower than just getting the jdl, because getting the jdl is one service call
# this is three service calls to three different DBs
resInputData = jobMon.getInputData(self.jobID)
if resInputData['OK']:
self.inputFiles = resInputData['Value']
resName = jobMon.getJobAttribute(self.jobID, 'JobName')
if resName['OK'] and '_' in resName['Value']:
self.__getTaskID(resName['Value'])
resOutput = jobMon.getJobParameter(self.jobID,
'ProductionOutputData')
if resOutput['OK']:
self.outputFiles = fromChar(resOutput['Value'].get(
'ProductionOutputData', ''))
if not self.outputFiles:
LOG.verbose('Did not find outputFiles for', str(self))
if not (self.inputFiles and self.outputFiles and self.taskID):
LOG.verbose('Have to check JDL')
jdlParameters = self.__getJDL(diracAPI)
# get taskID from JobName, get inputfile(s) from DownloadInputdata
self.__getOutputFiles(jdlParameters)
self.__getTaskID(jdlParameters)
if not self.inputFiles:
self.__getInputFile(jdlParameters)
def _setSystems(self, val):
self.systems = fromChar(val)
return S_OK()
def getNumberOfProcessors(siteName=None, gridCE=None, queue=None):
"""gets the number of processors on a certain CE/queue/node (what the pilot administers)
The siteName/gridCE/queue parameters are normally not necessary.
Tries to find it in this order:
1) from the /Resources/Computing/CEDefaults/NumberOfProcessors (which is what the pilot fills up)
2) if not present from JobFeatures
3) if not present looks in CS for "NumberOfProcessors" Queue or CE option
4) if not present but there's WholeNode tag, look what the WN provides using multiprocessing.cpu_count()
5) return 1
"""
# 1) from /Resources/Computing/CEDefaults/NumberOfProcessors
gLogger.info("Getting numberOfProcessors from /Resources/Computing/CEDefaults/NumberOfProcessors")
numberOfProcessors = gConfig.getValue("/Resources/Computing/CEDefaults/NumberOfProcessors", 0)
if numberOfProcessors:
return numberOfProcessors
# 2) from MJF
gLogger.info("Getting numberOfProcessors from MJF")
numberOfProcessors = getProcessorFromMJF()
if numberOfProcessors:
return numberOfProcessors
gLogger.info("NumberOfProcessors could not be found in MJF")
# 3) looks in CS for "NumberOfProcessors" Queue or CE or site option
if not siteName:
siteName = gConfig.getValue("/LocalSite/Site", "")
if not gridCE:
gridCE = gConfig.getValue("/LocalSite/GridCE", "")
if not queue:
queue = gConfig.getValue("/LocalSite/CEQueue", "")
if not (siteName and gridCE and queue):
gLogger.error("Could not find NumberOfProcessors: missing siteName or gridCE or queue. Returning '1'")
return 1
grid = siteName.split(".")[0]
csPaths = [
"/Resources/Sites/%s/%s/CEs/%s/Queues/%s/NumberOfProcessors" % (grid, siteName, gridCE, queue),
"/Resources/Sites/%s/%s/CEs/%s/NumberOfProcessors" % (grid, siteName, gridCE),
"/Resources/Sites/%s/%s/Cloud/%s/VMTypes/%s/NumberOfProcessors" % (grid, siteName, gridCE, queue),
"/Resources/Sites/%s/%s/Cloud/%s/NumberOfProcessors" % (grid, siteName, gridCE),
"/Resources/Sites/%s/%s/NumberOfProcessors" % (grid, siteName),
]
for csPath in csPaths:
gLogger.info("Looking in", csPath)
numberOfProcessors = gConfig.getValue(csPath, 0)
if numberOfProcessors:
return numberOfProcessors
# 4) looks in CS for tags
gLogger.info("Getting tags" "for %s: %s: %s" % (siteName, gridCE, queue))
# Tags of the CE
tags = fromChar(
gConfig.getValue("/Resources/Sites/%s/%s/CEs/%s/Tag" % (siteName.split(".")[0], siteName, gridCE), "")
) + fromChar(
gConfig.getValue("/Resources/Sites/%s/%s/Cloud/%s/Tag" % (siteName.split(".")[0], siteName, gridCE), "")
)
# Tags of the Queue
tags += fromChar(
gConfig.getValue(
"/Resources/Sites/%s/%s/CEs/%s/Queues/%s/Tag" % (siteName.split(".")[0], siteName, gridCE, queue), ""
)
) + fromChar(
gConfig.getValue(
"/Resources/Sites/%s/%s/Cloud/%s/VMTypes/%s/Tag" % (siteName.split(".")[0], siteName, gridCE, queue), ""
)
)
gLogger.info("NumberOfProcessors could not be found in CS")
if "WholeNode" in tags:
gLogger.info("Found WholeNode tag, using multiprocessing.cpu_count()")
return multiprocessing.cpu_count()
# 5) return the default
return 1
def syncCSWithVOMS(self):
""" Performs the synchronization of the DIRAC registry with the VOMS data. The resulting
CSAPI object containing modifications is returned as part of the output dictionary.
Those changes can be applied by the caller depending on the mode (dry or a real run)
:return: S_OK with a dictionary containing the results of the synchronization operation
"""
resultDict = defaultdict(list)
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(self.vo)
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
noSyncVOMSGroups = result['Value']['NoSyncVOMS']
vomsSrv = VOMSService(self.vo)
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error('Could not retrieve user information from VOMS',
result['Message'])
return result
self.vomsUserDict = result['Value']
message = "There are %s user entries in VOMS for VO %s" % (len(
self.vomsUserDict), self.vomsVOName)
self.adminMsgs['Info'].append(message)
self.log.info(message)
# Get DIRAC users
result = self.getVOUserData(self.vo)
if not result['OK']:
return result
diracUserDict = result['Value']
self.adminMsgs['Info'].append(
"There are %s registered users in DIRAC for VO %s" %
(len(diracUserDict), self.vo))
self.log.info("There are %s registered users in DIRAC VO %s" %
(len(diracUserDict), self.vo))
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
# We can have users with more than one DN registered
dnList = fromChar(dn)
existingDNs.extend(dnList)
for dn in dnList:
if dn not in self.vomsUserDict:
obsoletedDNs.append(dn)
for dn in self.vomsUserDict:
if dn not in existingDNs:
newDNs.append(dn)
allDiracUsers = getAllUsers()
nonVOUserDict = {}
nonVOUsers = list(set(allDiracUsers) - set(diracUserDict))
if nonVOUsers:
result = self.csapi.describeUsers(nonVOUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption(self.vo, "DefaultGroup",
"%s_user" % self.vo)
# If a user is (previously put by hand) in a "QuarantineGroup",
# then the default group will be ignored.
# So, this option is only considered for the case of existing users.
quarantineVOGroup = getVOOption(self.vo, "QuarantineGroup")
newAddedUserDict = {}
for dn in self.vomsUserDict:
newDNForExistingUser = ''
diracName = ''
if dn in existingDNs:
for user in diracUserDict:
if dn == diracUserDict[user]['DN']:
diracName = user
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# Check the nickName in the same VO to see if the user is already registered
# with another DN
nickName = self.vomsUserDict[dn].get('nickname')
if nickName in diracUserDict or nickName in newAddedUserDict:
diracName = nickName
# This is a flag for adding the new DN to an already existing user
newDNForExistingUser = dn
# We have a real new user
if not diracName:
if nickName:
newDiracName = nickName
else:
newDiracName = self.getUserName(dn)
# Do not consider users with Suspended status in VOMS
if self.vomsUserDict[dn]['suspended'] or self.vomsUserDict[
dn]['certSuspended']:
resultDict["SuspendedUsers"].append(newDiracName)
continue
# If the chosen user name exists already, append a distinguishing suffix
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % (trialName, ind)
ind += 1
# We now have everything to add the new user
userDict = {
"DN": dn,
"CA": self.vomsUserDict[dn]['CA'],
"Email": self.vomsUserDict[dn]['mail']
}
groupsWithRole = []
for role in self.vomsUserDict[dn]['Roles']:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
userDict['Groups'] = list(
set(groupsWithRole + [defaultVOGroup]))
message = "\n Added new user %s:\n" % newDiracName
for key in userDict:
message += " %s: %s\n" % (key, str(userDict[key]))
self.adminMsgs['Info'].append(message)
self.voChanged = True
if self.autoAddUsers:
self.log.info("Adding new user %s: %s" %
(newDiracName, str(userDict)))
result = self.csapi.modifyUser(
newDiracName, userDict, createIfNonExistant=True)
if not result['OK']:
self.log.warn('Failed adding new user %s' %
newDiracName)
resultDict['NewUsers'].append(newDiracName)
newAddedUserDict[newDiracName] = userDict
continue
# We have an already existing user
modified = False
userDict = {
"DN": dn,
"CA": self.vomsUserDict[dn]['CA'],
"Email": self.vomsUserDict[dn]['mail']
}
if newDNForExistingUser:
userDict['DN'] = ','.join([
dn,
diracUserDict.get(diracName,
newAddedUserDict.get(diracName))['DN']
])
modified = True
existingGroups = diracUserDict.get(diracName, {}).get('Groups', [])
nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping))
groupsWithRole = []
for role in self.vomsUserDict[dn]['Roles']:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
keepGroups = nonVOGroups + groupsWithRole
if not quarantineVOGroup or quarantineVOGroup not in existingGroups:
keepGroups += [defaultVOGroup]
for group in existingGroups:
if group in nonVOGroups:
continue
role = diracVOMSMapping.get(group, '')
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if "Role" not in role:
keepGroups.append(group)
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append(group)
# Keep groups for which syncronization with VOMS is forbidden
if group in noSyncVOMSGroups:
keepGroups.append(group)
userDict['Groups'] = list(set(keepGroups))
# Merge together groups for the same user but different DNs
if diracName in newAddedUserDict:
otherGroups = newAddedUserDict[diracName].get('Groups', [])
userDict['Groups'] = list(set(keepGroups + otherGroups))
modified = True
# Check if something changed before asking CSAPI to modify
if diracName in diracUserDict:
message = "\n Modified user %s:\n" % diracName
modMsg = ''
for key in userDict:
if key == "Groups":
addedGroups = set(userDict[key]) - set(
diracUserDict.get(diracName, {}).get(key, []))
removedGroups = set(
diracUserDict.get(diracName, {}).get(
key, [])) - set(userDict[key])
if addedGroups:
modMsg += " Added to group(s) %s\n" % ','.join(
addedGroups)
if removedGroups:
modMsg += " Removed from group(s) %s\n" % ','.join(
removedGroups)
else:
oldValue = str(
diracUserDict.get(diracName, {}).get(key, ''))
if str(userDict[key]) != oldValue:
modMsg += " %s: %s -> %s\n" % (
key, oldValue, str(userDict[key]))
if modMsg:
self.adminMsgs['Info'].append(message + modMsg)
modified = True
if self.autoModifyUsers and modified:
result = self.csapi.modifyUser(diracName, userDict)
if result['OK'] and result['Value']:
self.log.info("Modified user %s: %s" %
(diracName, str(userDict)))
self.voChanged = True
resultDict['ModifiedUsers'].append(diracName)
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]['DN']))
if not dnSet.intersection(set(
self.vomsUserDict)) and user not in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if group not in noVOMSGroups:
oldUsers.add(user)
# Check for obsoleted DNs
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]['DN']))
for dn in dnSet:
if dn in obsoletedDNs and user not in oldUsers:
self.log.verbose("Modified user %s: dropped DN %s" %
(user, dn))
if self.autoModifyUsers:
userDict = diracUserDict[user]
modDNSet = dnSet - set([dn])
if modDNSet:
userDict['DN'] = ','.join(modDNSet)
result = self.csapi.modifyUser(user, userDict)
if result['OK'] and result['Value']:
self.log.info(
"Modified user %s: dropped DN %s" %
(user, dn))
self.adminMsgs['Info'].append(
"Modified user %s: dropped DN %s" %
(user, dn))
self.voChanged = True
resultDict['ModifiedUsers'].append(diracName)
else:
oldUsers.add(user)
if oldUsers:
self.voChanged = True
if self.autoDeleteUsers:
self.log.info('The following users will be deleted: %s' %
str(oldUsers))
result = self.csapi.deleteUsers(oldUsers)
if result['OK']:
self.adminMsgs['Info'].append(
'The following users are deleted from CS:\n %s\n' %
str(oldUsers))
resultDict['DeletedUsers'] = oldUsers
else:
self.adminMsgs['Errors'].append(
'Error in deleting users from CS:\n %s' %
str(oldUsers))
self.log.error('Error while user deletion from CS', result)
else:
self.adminMsgs['Info'].append(
'The following users to be checked for deletion:\n %s' %
str(oldUsers))
self.log.info(
'The following users to be checked for deletion: %s' %
str(oldUsers))
resultDict['CSAPI'] = self.csapi
resultDict['AdminMessages'] = self.adminMsgs
return S_OK(resultDict)
def getQueues( self, resourceDict ):
""" Get the list of relevant CEs and their descriptions
"""
self.queueDict = {}
ceFactory = ComputingElementFactory()
for site in resourceDict:
for ce in resourceDict[site]:
ceDict = resourceDict[site][ce]
ceTags = ceDict.get( 'Tag' )
if isinstance( ceTags, basestring ):
ceTags = fromChar( ceTags )
qDict = ceDict.pop( 'Queues' )
for queue in qDict:
queueName = '%s_%s' % ( ce, queue )
self.queueDict[queueName] = {}
self.queueDict[queueName]['ParametersDict'] = qDict[queue]
self.queueDict[queueName]['ParametersDict']['Queue'] = queue
self.queueDict[queueName]['ParametersDict']['Site'] = site
self.queueDict[queueName]['ParametersDict']['GridEnv'] = self.gridEnv
self.queueDict[queueName]['ParametersDict']['Setup'] = gConfig.getValue( '/DIRAC/Setup', 'unknown' )
# Evaluate the CPU limit of the queue according to the Glue convention
# To Do: should be a utility
if "maxCPUTime" in self.queueDict[queueName]['ParametersDict'] and \
"SI00" in self.queueDict[queueName]['ParametersDict']:
maxCPUTime = float( self.queueDict[queueName]['ParametersDict']['maxCPUTime'] )
# For some sites there are crazy values in the CS
maxCPUTime = max( maxCPUTime, 0 )
maxCPUTime = min( maxCPUTime, 86400 * 12.5 )
si00 = float( self.queueDict[queueName]['ParametersDict']['SI00'] )
queueCPUTime = 60. / 250. * maxCPUTime * si00
self.queueDict[queueName]['ParametersDict']['CPUTime'] = int( queueCPUTime )
queueTags = self.queueDict[queueName]['ParametersDict'].get( 'Tag' )
if queueTags and isinstance( queueTags, basestring ):
queueTags = fromChar( queueTags )
self.queueDict[queueName]['ParametersDict']['Tag'] = queueTags
if ceTags:
if queueTags:
allTags = list( set( ceTags + queueTags ) )
self.queueDict[queueName]['ParametersDict']['Tag'] = allTags
else:
self.queueDict[queueName]['ParametersDict']['Tag'] = ceTags
maxMemory = self.queueDict[queueName]['ParametersDict'].get( 'MaxRAM', None )
if maxMemory:
# MaxRAM value is supposed to be in MB
maxMemoryList = range( 1, int( maxMemory )/1000 + 1 )
memoryTags = [ '%dGB' % mem for mem in maxMemoryList ]
if memoryTags:
self.queueDict[queueName]['ParametersDict'].setdefault( 'Tag', [] )
self.queueDict[queueName]['ParametersDict']['Tag'] += memoryTags
qwDir = os.path.join( self.workingDirectory, queue )
if not os.path.exists( qwDir ):
os.makedirs( qwDir )
self.queueDict[queueName]['ParametersDict']['WorkingDirectory'] = qwDir
platform = ''
if "Platform" in self.queueDict[queueName]['ParametersDict']:
platform = self.queueDict[queueName]['ParametersDict']['Platform']
elif "Platform" in ceDict:
platform = ceDict['Platform']
elif "OS" in ceDict:
architecture = ceDict.get( 'architecture', 'x86_64' )
OS = ceDict['OS']
platform = '_'.join( [architecture, OS] )
if platform and not platform in self.platforms:
self.platforms.append( platform )
if not "Platform" in self.queueDict[queueName]['ParametersDict'] and platform:
result = Resources.getDIRACPlatform( platform )
if result['OK']:
self.queueDict[queueName]['ParametersDict']['Platform'] = result['Value'][0]
ceQueueDict = dict( ceDict )
ceQueueDict.update( self.queueDict[queueName]['ParametersDict'] )
# Generate the CE object for the queue or pick the already existing one
# if the queue definition did not change
queueHash = self.__generateQueueHash( ceQueueDict )
if queueName in self.queueCECache and self.queueCECache[queueName]['Hash'] == queueHash:
queueCE = self.queueCECache[queueName]['CE']
else:
result = ceFactory.getCE( ceName = ce,
ceType = ceDict['CEType'],
ceParametersDict = ceQueueDict )
if not result['OK']:
return result
self.queueCECache.setdefault( queueName, {} )
self.queueCECache[queueName]['Hash'] = queueHash
self.queueCECache[queueName]['CE'] = result['Value']
queueCE = self.queueCECache[queueName]['CE']
self.queueDict[queueName]['CE'] = queueCE
self.queueDict[queueName]['CEName'] = ce
self.queueDict[queueName]['CEType'] = ceDict['CEType']
self.queueDict[queueName]['Site'] = site
self.queueDict[queueName]['QueueName'] = queue
self.queueDict[queueName]['Platform'] = platform
result = self.queueDict[queueName]['CE'].isValid()
if not result['OK']:
self.log.fatal( result['Message'] )
return result
if 'BundleProxy' in self.queueDict[queueName]['ParametersDict']:
if self.queueDict[queueName]['ParametersDict']['BundleProxy'].lower() in ['true','yes','1']:
self.queueDict[queueName]['BundleProxy'] = True
elif 'BundleProxy' in ceDict:
if ceDict['BundleProxy'].lower() in ['true','yes','1']:
self.queueDict[queueName]['BundleProxy'] = True
if site not in self.sites:
self.sites.append( site )
return S_OK()
def submitJobs( self ):
""" Go through defined computing elements and submit jobs if necessary
"""
# Check that there is some work at all
setup = CSGlobals.getSetup()
tqDict = { 'Setup':setup,
'CPUTime': 9999999,
'SubmitPool' : self.defaultSubmitPools }
if self.vo:
tqDict['Community'] = self.vo
if self.voGroups:
tqDict['OwnerGroup'] = self.voGroups
result = Resources.getCompatiblePlatforms( self.platforms )
if not result['OK']:
return result
tqDict['Platform'] = result['Value']
tqDict['Site'] = self.sites
tqDict['Tag'] = []
self.log.verbose( 'Checking overall TQ availability with requirements' )
self.log.verbose( tqDict )
rpcMatcher = RPCClient( "WorkloadManagement/Matcher" )
result = rpcMatcher.getMatchingTaskQueues( tqDict )
if not result[ 'OK' ]:
return result
if not result['Value']:
self.log.verbose( 'No Waiting jobs suitable for the director' )
return S_OK()
jobSites = set()
anySite = False
testSites = set()
totalWaitingJobs = 0
for tqID in result['Value']:
if "Sites" in result['Value'][tqID]:
for site in result['Value'][tqID]['Sites']:
if site.lower() != 'any':
jobSites.add( site )
else:
anySite = True
else:
anySite = True
if "JobTypes" in result['Value'][tqID]:
if "Sites" in result['Value'][tqID]:
for site in result['Value'][tqID]['Sites']:
if site.lower() != 'any':
testSites.add( site )
totalWaitingJobs += result['Value'][tqID]['Jobs']
tqIDList = result['Value'].keys()
result = pilotAgentsDB.countPilots( { 'TaskQueueID': tqIDList,
'Status': WAITING_PILOT_STATUS },
None )
totalWaitingPilots = 0
if result['OK']:
totalWaitingPilots = result['Value']
self.log.info( 'Total %d jobs in %d task queues with %d waiting pilots' % (totalWaitingJobs, len( tqIDList ), totalWaitingPilots ) )
#if totalWaitingPilots >= totalWaitingJobs:
# self.log.info( 'No more pilots to be submitted in this cycle' )
# return S_OK()
# Check if the site is allowed in the mask
result = jobDB.getSiteMask()
if not result['OK']:
return S_ERROR( 'Can not get the site mask' )
siteMaskList = result['Value']
queues = self.queueDict.keys()
random.shuffle( queues )
totalSubmittedPilots = 0
for queue in queues:
# Check if the queue failed previously
failedCount = self.failedQueues.setdefault( queue, 0 ) % self.failedQueueCycleFactor
if failedCount != 0:
self.log.warn( "%s queue failed recently, skipping %d cycles" % ( queue, 10-failedCount ) )
self.failedQueues[queue] += 1
continue
ce = self.queueDict[queue]['CE']
ceName = self.queueDict[queue]['CEName']
ceType = self.queueDict[queue]['CEType']
queueName = self.queueDict[queue]['QueueName']
siteName = self.queueDict[queue]['Site']
platform = self.queueDict[queue]['Platform']
siteMask = siteName in siteMaskList
if not anySite and siteName not in jobSites:
self.log.verbose( "Skipping queue %s at %s: no workload expected" % (queueName, siteName) )
continue
if not siteMask and siteName not in testSites:
self.log.verbose( "Skipping queue %s at site %s not in the mask" % (queueName, siteName) )
continue
if 'CPUTime' in self.queueDict[queue]['ParametersDict'] :
queueCPUTime = int( self.queueDict[queue]['ParametersDict']['CPUTime'] )
else:
self.log.warn( 'CPU time limit is not specified for queue %s, skipping...' % queue )
continue
if queueCPUTime > self.maxQueueLength:
queueCPUTime = self.maxQueueLength
# Prepare the queue description to look for eligible jobs
ceDict = ce.getParameterDict()
ceDict[ 'GridCE' ] = ceName
#if not siteMask and 'Site' in ceDict:
# self.log.info( 'Site not in the mask %s' % siteName )
# self.log.info( 'Removing "Site" from matching Dict' )
# del ceDict[ 'Site' ]
if not siteMask:
ceDict['JobType'] = "Test"
if self.vo:
ceDict['Community'] = self.vo
if self.voGroups:
ceDict['OwnerGroup'] = self.voGroups
# This is a hack to get rid of !
ceDict['SubmitPool'] = self.defaultSubmitPools
if "Tag" in ceDict and type( ceDict['Tag'] ) in types.StringTypes:
ceDict['Tag'] = fromChar( ceDict['Tag'] )
result = Resources.getCompatiblePlatforms( platform )
if not result['OK']:
continue
ceDict['Platform'] = result['Value']
# Get the number of eligible jobs for the target site/queue
result = rpcMatcher.getMatchingTaskQueues( ceDict )
if not result['OK']:
self.log.error( 'Could not retrieve TaskQueues from TaskQueueDB', result['Message'] )
return result
taskQueueDict = result['Value']
if not taskQueueDict:
self.log.verbose( 'No matching TQs found for %s' % queue )
continue
totalTQJobs = 0
tqIDList = taskQueueDict.keys()
for tq in taskQueueDict:
totalTQJobs += taskQueueDict[tq]['Jobs']
self.log.verbose( '%d job(s) from %d task queue(s) are eligible for %s queue' % (totalTQJobs, len( tqIDList ), queue) )
# Get the number of already waiting pilots for these task queues
totalWaitingPilots = 0
if self.pilotWaitingFlag:
lastUpdateTime = dateTime() - self.pilotWaitingTime * second
result = pilotAgentsDB.countPilots( { 'TaskQueueID': tqIDList,
'Status': WAITING_PILOT_STATUS },
None, lastUpdateTime )
if not result['OK']:
self.log.error( 'Failed to get Number of Waiting pilots', result['Message'] )
totalWaitingPilots = 0
else:
totalWaitingPilots = result['Value']
self.log.verbose( 'Waiting Pilots for TaskQueue %s:' % tqIDList, totalWaitingPilots )
if totalWaitingPilots >= totalTQJobs:
self.log.verbose( "%d waiting pilots already for all the available jobs" % totalWaitingPilots )
continue
self.log.verbose( "%d waiting pilots for the total of %d eligible jobs for %s" % (totalWaitingPilots, totalTQJobs, queue) )
# Get the working proxy
cpuTime = queueCPUTime + 86400
self.log.verbose( "Getting pilot proxy for %s/%s %d long" % ( self.pilotDN, self.pilotGroup, cpuTime ) )
result = gProxyManager.getPilotProxyFromDIRACGroup( self.pilotDN, self.pilotGroup, cpuTime )
if not result['OK']:
return result
self.proxy = result['Value']
ce.setProxy( self.proxy, cpuTime - 60 )
# Get the number of available slots on the target site/queue
totalSlots = self.__getQueueSlots( queue )
if totalSlots == 0:
continue
pilotsToSubmit = max( 0, min( totalSlots, totalTQJobs - totalWaitingPilots ) )
self.log.info( '%s: Slots=%d, TQ jobs=%d, Pilots: waiting %d, to submit=%d' % \
( queue, totalSlots, totalTQJobs, totalWaitingPilots, pilotsToSubmit ) )
# Limit the number of pilots to submit to MAX_PILOTS_TO_SUBMIT
pilotsToSubmit = min( self.maxPilotsToSubmit, pilotsToSubmit )
while pilotsToSubmit > 0:
self.log.info( 'Going to submit %d pilots to %s queue' % ( pilotsToSubmit, queue ) )
bundleProxy = self.queueDict[queue].get( 'BundleProxy', False )
jobExecDir = ''
if ceType == 'CREAM':
jobExecDir = '.'
jobExecDir = self.queueDict[queue].get( 'JobExecDir', jobExecDir )
httpProxy = self.queueDict[queue].get( 'HttpProxy', '' )
result = self.__getExecutable( queue, pilotsToSubmit, bundleProxy, httpProxy, jobExecDir )
if not result['OK']:
return result
executable, pilotSubmissionChunk = result['Value']
result = ce.submitJob( executable, '', pilotSubmissionChunk )
os.unlink( executable )
if not result['OK']:
self.log.error( 'Failed submission to queue %s:\n' % queue, result['Message'] )
pilotsToSubmit = 0
self.failedQueues[queue] += 1
continue
pilotsToSubmit = pilotsToSubmit - pilotSubmissionChunk
# Add pilots to the PilotAgentsDB assign pilots to TaskQueue proportionally to the
# task queue priorities
pilotList = result['Value']
self.queueSlots[queue]['AvailableSlots'] -= len( pilotList )
totalSubmittedPilots += len( pilotList )
self.log.info( 'Submitted %d pilots to %s@%s' % ( len( pilotList ), queueName, ceName ) )
stampDict = {}
if result.has_key( 'PilotStampDict' ):
stampDict = result['PilotStampDict']
tqPriorityList = []
sumPriority = 0.
for tq in taskQueueDict:
sumPriority += taskQueueDict[tq]['Priority']
tqPriorityList.append( ( tq, sumPriority ) )
rndm = random.random()*sumPriority
tqDict = {}
for pilotID in pilotList:
rndm = random.random()*sumPriority
for tq, prio in tqPriorityList:
if rndm < prio:
tqID = tq
break
if not tqDict.has_key( tqID ):
tqDict[tqID] = []
tqDict[tqID].append( pilotID )
for tqID, pilotList in tqDict.items():
result = pilotAgentsDB.addPilotTQReference( pilotList,
tqID,
self.pilotDN,
self.pilotGroup,
self.localhost,
ceType,
'',
stampDict )
if not result['OK']:
self.log.error( 'Failed add pilots to the PilotAgentsDB: ', result['Message'] )
continue
for pilot in pilotList:
result = pilotAgentsDB.setPilotStatus( pilot, 'Submitted', ceName,
'Successfully submitted by the SiteDirector',
siteName, queueName )
if not result['OK']:
self.log.error( 'Failed to set pilot status: ', result['Message'] )
continue
self.log.info( "%d pilots submitted in total in this cycle" % totalSubmittedPilots )
return S_OK()
def _ByJobType(self):
""" By default, all sites are allowed to do every job.
The actual rules are freely specified in the Operation JobTypeMapping section.
The content of the section may look like this:
User
{
Exclude = PAK
Exclude += Ferrara
Exclude += Bologna
Exclude += Paris
Exclude += CERN
Exclude += IN2P3
Allow
{
Paris = IN2P3
CERN = CERN
IN2P3 = IN2P3
}
}
DataReconstruction
{
Exclude = PAK
Exclude += Ferrara
Exclude += CERN
Exclude += IN2P3
Allow
{
Ferrara = CERN
CERN = CERN
IN2P3 = IN2P3
IN2P3 += CERN
}
}
Merge
{
Exclude = ALL
Allow
{
CERN = CERN
IN2P3 = IN2P3
}
}
The sites in the exclusion list will be removed.
The allow section says where each site may help another site
"""
# 1. get sites list
res = getSites()
if not res['OK']:
gLogger.error("Could not get the list of sites", res['Message'])
return res
destSites = set(res['Value'])
# 2. get JobTypeMapping "Exclude" value (and add autoAddedSites)
gLogger.debug(
"Getting JobTypeMapping 'Exclude' value (and add autoAddedSites)")
jobType = self.params['JobType']
if not jobType:
raise RuntimeError("No jobType specified")
excludedSites = set(
self.opsH.getValue('JobTypeMapping/%s/Exclude' % jobType, []))
gLogger.debug("Explicitly excluded sites for %s task: %s" %
(jobType, ','.join(excludedSites)))
autoAddedSites = self.opsH.getValue('JobTypeMapping/AutoAddedSites',
[])
if 'WithStorage' in autoAddedSites:
# Add all sites with storage, such that jobs can run wherever data is
autoAddedSites.remove('WithStorage')
autoAddedSites += DMSHelpers().getTiers(withStorage=True)
# 3. removing sites in Exclude
if not excludedSites:
pass
elif 'ALL' in excludedSites:
destSites = set()
else:
destSites -= excludedSites
# 4. get JobTypeMapping "Allow" section
res = self.opsH.getOptionsDict('JobTypeMapping/%s/Allow' % jobType)
if not res['OK']:
gLogger.debug(res['Message'])
allowed = {}
else:
allowed = dict((site, set(fromChar(fromSites)))
for site, fromSites in res['Value'].items())
autoAddedSites = set(
self.opsH.getValue('JobTypeMapping/%s/AutoAddedSites' % jobType,
autoAddedSites))
gLogger.debug("Auto-added sites for %s task: %s" %
(jobType, ','.join(autoAddedSites)))
# 5. add autoAddedSites, if requested
for autoAddedSite in autoAddedSites:
allowed.setdefault(autoAddedSite, set()).add(autoAddedSite)
gLogger.debug("Allowed sites for %s task: %s" %
(jobType, ','.join(allowed)))
# 6. Allowing sites that should be allowed
taskSiteDestination = self._BySE()
for destSite, fromSites in allowed.items():
for fromSite in fromSites:
if not taskSiteDestination or fromSite in taskSiteDestination:
destSites.add(destSite)
gLogger.debug(
"Computed list of destination sites for %s task with TargetSE %s: %s"
% (jobType, self.params['TargetSE'], ','.join(destSites)))
return destSites
def getQueuesResolved(siteDict):
"""
Get the list of queue descriptions merging site/ce/queue parameters and adding some
derived parameters.
:param dict siteDict: dictionary with configuration data as returned by Resources.getQueues() method
:return: S_OK/S_ERROR, Value dictionary per queue with configuration data updated, e.g. for SiteDirector
"""
queueFinalDict = {}
for site in siteDict:
for ce, ceDict in siteDict[site].items():
qDict = ceDict.pop('Queues')
for queue in qDict:
queueName = '%s_%s' % (ce, queue)
queueDict = qDict[queue]
queueDict['Queue'] = queue
queueDict['Site'] = site
# Evaluate the CPU limit of the queue according to the Glue convention
# To Do: should be a utility
if "maxCPUTime" in queueDict and "SI00" in queueDict:
maxCPUTime = float(queueDict['maxCPUTime'])
# For some sites there are crazy values in the CS
maxCPUTime = max(maxCPUTime, 0)
maxCPUTime = min(maxCPUTime, 86400 * 12.5)
si00 = float(queueDict['SI00'])
queueCPUTime = 60 / 250 * maxCPUTime * si00
queueDict['CPUTime'] = int(queueCPUTime)
# Tags & RequiredTags defined on the Queue level and on the CE level are concatenated
# This also converts them from a string to a list if required.
for tagFieldName in ('Tag', 'RequiredTag'):
ceTags = ceDict.get(tagFieldName, [])
if isinstance(ceTags, six.string_types):
ceTags = fromChar(ceTags)
queueTags = queueDict.get(tagFieldName, [])
if isinstance(queueTags, six.string_types):
queueTags = fromChar(queueTags)
queueDict[tagFieldName] = list(set(ceTags + queueTags))
# Some parameters can be defined on the CE level and are inherited by all Queues
for parameter in ['MaxRAM', 'NumberOfProcessors', 'WholeNode']:
queueParameter = queueDict.get(parameter,
ceDict.get(parameter))
if queueParameter:
queueDict[parameter] = queueParameter
# If we have a multi-core queue add MultiProcessor tag
if queueDict.get('NumberOfProcessors', 1) > 1:
queueDict.setdefault('Tag', []).append('MultiProcessor')
queueDict['CEName'] = ce
queueDict['GridCE'] = ce
queueDict['CEType'] = ceDict['CEType']
queueDict['GridMiddleware'] = ceDict['CEType']
queueDict['QueueName'] = queue
platform = queueDict.get('Platform',
ceDict.get('Platform', ''))
if not platform and "OS" in ceDict:
architecture = ceDict.get('architecture', 'x86_64')
platform = '_'.join([architecture, ceDict['OS']])
queueDict['Platform'] = platform
if platform:
result = getDIRACPlatform(platform)
if result['OK']:
queueDict['Platform'] = result['Value'][0]
queueFinalDict[queueName] = queueDict
return S_OK(queueFinalDict)
def syncCSWithVOMS(self):
""" Performs the synchronization of the DIRAC registry with the VOMS data. The resulting
CSAPI object containing modifications is returned as part of the output dictionary.
Those changes can be applied by the caller depending on the mode (dry or a real run)
:return: S_OK with a dictionary containing the results of the synchronization operation
"""
resultDict = defaultdict(list)
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(self.vo)
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
noSyncVOMSGroups = result['Value']['NoSyncVOMS']
vomsSrv = VOMSService(self.vo)
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error('Could not retrieve user information from VOMS', result['Message'])
return result
self.vomsUserDict = result['Value']
message = "There are %s user entries in VOMS for VO %s" % (len(self.vomsUserDict), self.vomsVOName)
self.adminMsgs['Info'].append(message)
self.log.info(message)
# Get DIRAC users
result = self.getVOUserData(self.vo)
if not result['OK']:
return result
diracUserDict = result['Value']
self.adminMsgs['Info'].append("There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), self.vo))
self.log.info("There are %s registered users in DIRAC VO %s" % (len(diracUserDict), self.vo))
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
# We can have users with more than one DN registered
dnList = fromChar(dn)
existingDNs.extend(dnList)
for dn in dnList:
if dn not in self.vomsUserDict:
obsoletedDNs.append(dn)
for dn in self.vomsUserDict:
if dn not in existingDNs:
newDNs.append(dn)
allDiracUsers = getAllUsers()
nonVOUserDict = {}
nonVOUsers = list(set(allDiracUsers) - set(diracUserDict))
if nonVOUsers:
result = self.csapi.describeUsers(nonVOUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption(self.vo, "DefaultGroup", "%s_user" % self.vo)
# If a user is (previously put by hand) in a "QuarantineGroup",
# then the default group will be ignored.
# So, this option is only considered for the case of existing users.
quarantineVOGroup = getVOOption(self.vo, "QuarantineGroup")
newAddedUserDict = {}
for dn in self.vomsUserDict:
newDNForExistingUser = ''
diracName = ''
if dn in existingDNs:
for user in diracUserDict:
if dn == diracUserDict[user]['DN']:
diracName = user
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# Check the nickName in the same VO to see if the user is already registered
# with another DN
nickName = self.vomsUserDict[dn].get('nickname')
if nickName in diracUserDict or nickName in newAddedUserDict:
diracName = nickName
# This is a flag for adding the new DN to an already existing user
newDNForExistingUser = dn
# We have a real new user
if not diracName:
if nickName:
newDiracName = nickName
else:
newDiracName = self.getUserName(dn)
# Do not consider users with Suspended status in VOMS
if self.vomsUserDict[dn]['suspended'] or self.vomsUserDict[dn]['certSuspended']:
resultDict["SuspendedUsers"].append(newDiracName)
continue
# If the chosen user name exists already, append a distinguishing suffix
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % (trialName, ind)
ind += 1
# We now have everything to add the new user
userDict = {"DN": dn, "CA": self.vomsUserDict[dn]['CA'], "Email": self.vomsUserDict[dn]['mail']}
groupsWithRole = []
for role in self.vomsUserDict[dn]['Roles']:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
userDict['Groups'] = list(set(groupsWithRole + [defaultVOGroup]))
message = "\n Added new user %s:\n" % newDiracName
for key in userDict:
message += " %s: %s\n" % (key, str(userDict[key]))
self.adminMsgs['Info'].append(message)
self.voChanged = True
if self.autoAddUsers:
self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict)))
result = self.csapi.modifyUser(newDiracName, userDict, createIfNonExistant=True)
if not result['OK']:
self.log.warn('Failed adding new user %s' % newDiracName)
resultDict['NewUsers'].append(newDiracName)
newAddedUserDict[newDiracName] = userDict
continue
# We have an already existing user
modified = False
userDict = {"DN": dn, "CA": self.vomsUserDict[dn]['CA'], "Email": self.vomsUserDict[dn]['mail']}
if newDNForExistingUser:
userDict['DN'] = ','.join([dn, diracUserDict.get(diracName, newAddedUserDict.get(diracName))['DN']])
modified = True
existingGroups = diracUserDict.get(diracName, {}).get('Groups', [])
nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping))
groupsWithRole = []
for role in self.vomsUserDict[dn]['Roles']:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
keepGroups = nonVOGroups + groupsWithRole
if not quarantineVOGroup or quarantineVOGroup not in existingGroups:
keepGroups += [defaultVOGroup]
for group in existingGroups:
if group in nonVOGroups:
continue
role = diracVOMSMapping.get(group, '')
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if "Role" not in role:
keepGroups.append(group)
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append(group)
# Keep groups for which syncronization with VOMS is forbidden
if group in noSyncVOMSGroups:
keepGroups.append(group)
userDict['Groups'] = list(set(keepGroups))
# Merge together groups for the same user but different DNs
if diracName in newAddedUserDict:
otherGroups = newAddedUserDict[diracName].get('Groups', [])
userDict['Groups'] = list(set(keepGroups + otherGroups))
modified = True
# Check if something changed before asking CSAPI to modify
if diracName in diracUserDict:
message = "\n Modified user %s:\n" % diracName
modMsg = ''
for key in userDict:
if key == "Groups":
addedGroups = set(userDict[key]) - set(diracUserDict.get(diracName, {}).get(key, []))
removedGroups = set(diracUserDict.get(diracName, {}).get(key, [])) - set(userDict[key])
if addedGroups:
modMsg += " Added to group(s) %s\n" % ','.join(addedGroups)
if removedGroups:
modMsg += " Removed from group(s) %s\n" % ','.join(removedGroups)
else:
oldValue = str(diracUserDict.get(diracName, {}).get(key, ''))
if str(userDict[key]) != oldValue:
modMsg += " %s: %s -> %s\n" % (key, oldValue, str(userDict[key]))
if modMsg:
self.adminMsgs['Info'].append(message + modMsg)
modified = True
if self.autoModifyUsers and modified:
result = self.csapi.modifyUser(diracName, userDict)
if result['OK'] and result['Value']:
self.log.info("Modified user %s: %s" % (diracName, str(userDict)))
self.voChanged = True
resultDict['ModifiedUsers'].append(diracName)
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]['DN']))
if not dnSet.intersection(set(self.vomsUserDict)) and user not in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if group not in noVOMSGroups:
oldUsers.add(user)
# Check for obsoleted DNs
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]['DN']))
for dn in dnSet:
if dn in obsoletedDNs and user not in oldUsers:
self.log.verbose("Modified user %s: dropped DN %s" % (user, dn))
if self.autoModifyUsers:
userDict = diracUserDict[user]
modDNSet = dnSet - set([dn])
if modDNSet:
userDict['DN'] = ','.join(modDNSet)
result = self.csapi.modifyUser(user, userDict)
if result['OK'] and result['Value']:
self.log.info("Modified user %s: dropped DN %s" % (user, dn))
self.adminMsgs['Info'].append("Modified user %s: dropped DN %s" % (user, dn))
self.voChanged = True
resultDict['ModifiedUsers'].append(diracName)
else:
oldUsers.add(user)
if oldUsers:
self.voChanged = True
if self.autoDeleteUsers:
self.log.info('The following users will be deleted: %s' % str(oldUsers))
result = self.csapi.deleteUsers(oldUsers)
if result['OK']:
self.adminMsgs['Info'].append('The following users are deleted from CS:\n %s\n' % str(oldUsers))
resultDict['DeletedUsers'] = oldUsers
else:
self.adminMsgs['Errors'].append('Error in deleting users from CS:\n %s' % str(oldUsers))
self.log.error('Error while user deletion from CS', result)
else:
self.adminMsgs['Info'].append('The following users to be checked for deletion:\n %s' % str(oldUsers))
self.log.info('The following users to be checked for deletion: %s' % str(oldUsers))
resultDict['CSAPI'] = self.csapi
resultDict['AdminMessages'] = self.adminMsgs
return S_OK(resultDict)
def getNumberOfProcessors(siteName=None, gridCE=None, queue=None):
""" gets the number of processors on a certain CE/queue/node (what the pilot administers)
The siteName/gridCE/queue parameters are normally not necessary.
Tries to find it in this order:
1) from the /Resources/Computing/CEDefaults/NumberOfProcessors (which is what the pilot fills up)
2) if not present from JobFeatures
3) if not present looks in CS for "NumberOfProcessors" Queue or CE option
4) if not present looks in CS for "%dProcessors" Queue or CE Tag
5) if not present but there's WholeNode tag, look what the WN provides using multiprocessing.cpu_count()
6) return 1
"""
# 1) from /Resources/Computing/CEDefaults/NumberOfProcessors
gLogger.info(
"Getting numberOfProcessors from /Resources/Computing/CEDefaults/NumberOfProcessors"
)
numberOfProcessors = gConfig.getValue(
'/Resources/Computing/CEDefaults/NumberOfProcessors')
if numberOfProcessors:
return numberOfProcessors
# 2) from MJF
gLogger.info("Getting numberOfProcessors from MJF")
numberOfProcessors = getProcessorFromMJF()
if numberOfProcessors:
return numberOfProcessors
# 3) looks in CS for "NumberOfProcessors" Queue or CE or site option
grid = siteName.split('.')[0]
gLogger.info(
"NumberOfProcessors could not be found in MJF, trying from CS (queue definition)"
)
numberOfProcessors = gConfig.getValue(
'/Resources/Sites/%s/%s/CEs/%s/Queues/%s/NumberOfProcessors' %
(grid, siteName, gridCE, queue))
if numberOfProcessors:
return numberOfProcessors
gLogger.info(
"NumberOfProcessors could not be found in CS queue definition, ",
"trying from /Resources/Sites/%s/%s/CEs/%s/NumberOfProcessors" %
(grid, siteName, gridCE))
numberOfProcessors = gConfig.getValue(
'/Resources/Sites/%s/%s/CEs/%s/NumberOfProcessors' %
(grid, siteName, gridCE))
if numberOfProcessors:
return numberOfProcessors
gLogger.info(
"NumberOfProcessors could not be found in CS CE definition, ",
"trying from /Resources/Sites/%s/%s/NumberOfProcessors" %
(grid, siteName))
numberOfProcessors = gConfig.getValue(
'/Resources/Sites/%s/%s/NumberOfProcessors' % (grid, siteName))
if numberOfProcessors:
return numberOfProcessors
# 3) looks in CS for tags
gLogger.info("Getting number of processors"
"from tags for %s: %s: %s" % (siteName, gridCE, queue))
# Tags of the CE
tags = fromChar(
gConfig.getValue(
'/Resources/Sites/%s/%s/CEs/%s/Tag' %
(siteName.split('.')[0], siteName, gridCE), ''))
# Tags of the Queue
tags += fromChar(
gConfig.getValue(
'/Resources/Sites/%s/%s/CEs/%s/Queues/%s/Tag' %
(siteName.split('.')[0], siteName, gridCE, queue), ''))
for tag in tags:
numberOfProcessorsTag = re.search('[0-9]Processors', tag)
if numberOfProcessorsTag:
gLogger.info("Number of processors from tags",
numberOfProcessorsTag.string)
return int(numberOfProcessorsTag.string.replace('Processors', ''))
gLogger.info("NumberOfProcessors could not be found in CS")
if 'WholeNode' in tags:
gLogger.info("Found WholeNode tag, using multiprocessing.cpu_count()")
return multiprocessing.cpu_count()
return 1
def _setSystems(self, val): self.systems = fromChar(val) return S_OK()
def getEndpoints(self, resourceDict):
""" Get the list of relevant CEs and their descriptions
"""
self.vmTypeDict = {}
ceFactory = EndpointFactory()
result = getPilotBootstrapParameters(vo=self.vo,
runningPod=self.runningPod)
if not result['OK']:
return result
opParameters = result['Value']
for site in resourceDict:
for ce in resourceDict[site]:
ceDict = resourceDict[site][ce]
ceTags = ceDict.get('Tag', [])
if isinstance(ceTags, basestring):
ceTags = fromChar(ceTags)
ceMaxRAM = ceDict.get('MaxRAM', None)
qDict = ceDict.pop('VMTypes')
for vmType in qDict:
vmTypeName = '%s_%s' % (ce, vmType)
self.vmTypeDict[vmTypeName] = {}
self.vmTypeDict[vmTypeName]['ParametersDict'] = qDict[
vmType]
self.vmTypeDict[vmTypeName]['ParametersDict'][
'VMType'] = vmType
self.vmTypeDict[vmTypeName]['ParametersDict'][
'Site'] = site
self.vmTypeDict[vmTypeName]['ParametersDict'][
'Setup'] = gConfig.getValue('/DIRAC/Setup', 'unknown')
self.vmTypeDict[vmTypeName]['ParametersDict'][
'CPUTime'] = 99999999
vmTypeTags = self.vmTypeDict[vmTypeName][
'ParametersDict'].get('Tag')
if vmTypeTags and isinstance(vmTypeTags, basestring):
vmTypeTags = fromChar(vmTypeTags)
self.vmTypeDict[vmTypeName]['ParametersDict'][
'Tag'] = vmTypeTags
if ceTags:
if vmTypeTags:
allTags = list(set(ceTags + vmTypeTags))
self.vmTypeDict[vmTypeName]['ParametersDict'][
'Tag'] = allTags
else:
self.vmTypeDict[vmTypeName]['ParametersDict'][
'Tag'] = ceTags
maxRAM = self.vmTypeDict[vmTypeName]['ParametersDict'].get(
'MaxRAM')
maxRAM = ceMaxRAM if not maxRAM else maxRAM
if maxRAM:
self.vmTypeDict[vmTypeName]['ParametersDict'][
'MaxRAM'] = maxRAM
ceWholeNode = ceDict.get('WholeNode', 'true')
wholeNode = self.vmTypeDict[vmTypeName][
'ParametersDict'].get('WholeNode', ceWholeNode)
if wholeNode.lower() in ('yes', 'true'):
self.vmTypeDict[vmTypeName][
'ParametersDict'].setdefault('Tag', [])
self.vmTypeDict[vmTypeName]['ParametersDict'][
'Tag'].append('WholeNode')
platform = ''
if "Platform" in self.vmTypeDict[vmTypeName][
'ParametersDict']:
platform = self.vmTypeDict[vmTypeName][
'ParametersDict']['Platform']
elif "Platform" in ceDict:
platform = ceDict['Platform']
if platform and platform not in self.platforms:
self.platforms.append(platform)
if "Platform" not in self.vmTypeDict[vmTypeName][
'ParametersDict'] and platform:
result = Resources.getDIRACPlatform(platform)
if result['OK']:
self.vmTypeDict[vmTypeName]['ParametersDict'][
'Platform'] = result['Value'][0]
ceVMTypeDict = dict(ceDict)
ceVMTypeDict['CEName'] = ce
ceVMTypeDict['VO'] = self.vo
ceVMTypeDict['VMType'] = vmType
ceVMTypeDict['RunningPod'] = self.runningPod
ceVMTypeDict['CSServers'] = gConfig.getValue(
"/DIRAC/Configuration/Servers", [])
ceVMTypeDict.update(
self.vmTypeDict[vmTypeName]['ParametersDict'])
# Allow a resource-specifc CAPath to be set (as some clouds have their own CAs)
# Otherwise fall back to the system-wide default(s)
if 'CAPath' not in ceVMTypeDict:
ceVMTypeDict['CAPath'] = gConfig.getValue(
'/DIRAC/Security/CAPath',
"/opt/dirac/etc/grid-security/certificates/cas.pem"
)
# Generate the CE object for the vmType or pick the already existing one
# if the vmType definition did not change
vmTypeHash = self.__generateVMTypeHash(ceVMTypeDict)
if vmTypeName in self.vmTypeCECache and self.vmTypeCECache[
vmTypeName]['Hash'] == vmTypeHash:
vmTypeCE = self.vmTypeCECache[vmTypeName]['CE']
else:
result = ceFactory.getCEObject(parameters=ceVMTypeDict)
if not result['OK']:
return result
self.vmTypeCECache.setdefault(vmTypeName, {})
self.vmTypeCECache[vmTypeName]['Hash'] = vmTypeHash
self.vmTypeCECache[vmTypeName]['CE'] = result['Value']
vmTypeCE = self.vmTypeCECache[vmTypeName]['CE']
vmTypeCE.setBootstrapParameters(opParameters)
self.vmTypeDict[vmTypeName]['CE'] = vmTypeCE
self.vmTypeDict[vmTypeName]['CEName'] = ce
self.vmTypeDict[vmTypeName]['CEType'] = ceDict['CEType']
self.vmTypeDict[vmTypeName]['Site'] = site
self.vmTypeDict[vmTypeName]['VMType'] = vmType
self.vmTypeDict[vmTypeName]['Platform'] = platform
self.vmTypeDict[vmTypeName]['MaxInstances'] = ceDict[
'MaxInstances']
if not self.vmTypeDict[vmTypeName]['CE'].isValid():
self.log.error(
'Failed to instantiate CloudEndpoint for %s' %
vmTypeName)
continue
if site not in self.sites:
self.sites.append(site)
return S_OK()
def _ByJobType( self ):
""" By default, all sites are allowed to do every job. The actual rules are freely specified in the Operation JobTypeMapping section.
The content of the section may look like this:
User
{
Exclude = PAK
Exclude += Ferrara
Exclude += Bologna
Exclude += Paris
Exclude += CERN
Exclude += IN2P3
Allow
{
Paris = IN2P3
CERN = CERN
IN2P3 = IN2P3
}
}
DataReconstruction
{
Exclude = PAK
Exclude += Ferrara
Exclude += CERN
Exclude += IN2P3
Allow
{
Ferrara = CERN
CERN = CERN
IN2P3 = IN2P3
IN2P3 += CERN
}
}
Merge
{
Exclude = ALL
Allow
{
CERN = CERN
IN2P3 = IN2P3
}
}
The sites in the exclusion list will be removed.
The allow section says where each site may help another site
"""
# 1. get sites list
res = getSites()
if not res['OK']:
gLogger.error( "Could not get the list of sites", res['Message'] )
return res
destSites = set( res['Value'] )
# 2. get JobTypeMapping "Exclude" value (and add autoAddedSites)
gLogger.debug( "Getting JobTypeMapping 'Exclude' value (and add autoAddedSites)" )
jobType = self.params['JobType']
if not jobType:
raise RuntimeError( "No jobType specified" )
excludedSites = self.opsH.getValue( 'JobTypeMapping/%s/Exclude' % jobType, [] )
gLogger.debug( "Explicitly excluded sites for %s task: %s" % ( jobType, ','.join( excludedSites ) ) )
excludedSites += self.opsH.getValue( 'JobTypeMapping/AutoAddedSites', [] )
gLogger.debug( "Full list of excluded sites for %s task: %s" % ( jobType, ','.join( excludedSites ) ) )
# 3. removing sites in Exclude
if not excludedSites:
pass
elif 'ALL' in excludedSites:
destSites = set()
else:
destSites = destSites.difference( set( excludedSites ) )
# 4. get JobTypeMapping "Allow" section
res = self.opsH.getOptionsDict( 'JobTypeMapping/%s/Allow' % jobType )
if not res['OK']:
gLogger.verbose( res['Message'] )
allowed = {}
else:
allowed = res['Value']
for site in allowed:
allowed[site] = fromChar( allowed[site] )
# 5. add autoAddedSites, if requested
autoAddedSites = self.opsH.getValue( 'JobTypeMapping/AutoAddedSites', [] )
if autoAddedSites:
for autoAddedSite in autoAddedSites:
allowed.setdefault( autoAddedSite, [autoAddedSite] )
if autoAddedSite not in allowed:
allowed[autoAddedSite] = [autoAddedSite]
else:
allowed[autoAddedSite] = [autoAddedSite] + allowed[autoAddedSite]
gLogger.debug( "Allowed sites for %s task: %s" % ( jobType, ','.join( allowed ) ) )
# 6. Allowing sites that should be allowed
if not self.params['TargetSE'] or self.params['TargetSE'] == 'Unknown':
gLogger.warn( "TargetSE is not set: the destination sites list will be incomplete" )
taskSiteDestination = self._BySE()
for destSite, fromSites in allowed.iteritems():
for fromSite in fromSites:
if taskSiteDestination:
if fromSite in taskSiteDestination:
destSites.add( destSite )
else:
destSites.add( destSite )
gLogger.verbose( "Computed list of destination sites for %s task with TargetSE %s: %s" % ( jobType,
self.params['TargetSE'],
','.join( destSites ) ) )
return destSites
def registerUser( self ):
"""
This function is used to notify DIRAC admins about user registration request
The logic is simple:
0) Check if request from this e-mail has already registered or not
1) Send mail to VO admin of requested VO
2) Send mail to users in group with UserAdministrator property
3) Send mail to users indicated in /Website/UserRegistrationAdmin option
"""
gLogger.info("Start processing a registration request")
checkUserCredentials()
# Check for having a DN but no username
dn = getUserDN()
if not dn:
error = "Certificate is not loaded in the browser or DN is absent"
gLogger.error( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
username = getUsername()
if not username == "anonymous":
error = "You are already registered in DIRAC with username: %s" % username
gLogger.error( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
gLogger.info( "DN: %s" % dn )
if not request.params.has_key( "email" ):
error = "Can not get your email address from the request"
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
userMail = request.params[ "email" ]
if self.alreadyRequested( userMail ):
error = "Request associated with %s already registered" % userMail
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
vo = fromChar( request.params[ "vo" ] )
if not vo:
error = "You should indicate a VirtualOrganization for membership"
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
gLogger.info( "User want to be register in VO(s): %s" % vo )
body = str()
for i in request.params:
if not i in [ "registration_request" , "email" , "vo" ]:
info = self.__checkUnicode( i , request.params[ i ] )
body = body + info + "\n"
body = body + "DN - " + dn
gLogger.debug( "email body: %s" % body )
adminList = self.__getAdminList( vo )
if not len( adminList ) > 0:
error = "Can't get in contact with administrators about your request\n"
error = error + "Most likely this DIRAC instance is not configured yet"
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
adminList = uniqueElements( adminList )
gLogger.info( "Chosen admin(s): %s" % adminList )
sendDict = self.__getMailDict( adminList )
if not len(sendDict) > 0:
error = "Can't get in contact with administrators about your request\n"
error = error + "Most likely this DIRAC instance is not configured yet"
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
gLogger.debug( "Final dictionary with mails to be used %s" % sendDict )
return self.__sendAMail( sendDict , body , userMail )
def getQueues( self, resourceDict ):
""" Get the list of relevant CEs and their descriptions
"""
self.queueDict = {}
ceFactory = ComputingElementFactory()
for site in resourceDict:
for ce in resourceDict[site]:
ceDict = resourceDict[site][ce]
ceTags = ceDict.get( 'Tag', [] )
pilotRunDirectory = ceDict.get( 'PilotRunDirectory', '' )
if isinstance( ceTags, basestring ):
ceTags = fromChar( ceTags )
ceMaxRAM = ceDict.get( 'MaxRAM', None )
qDict = ceDict.pop( 'Queues' )
for queue in qDict:
queueName = '%s_%s' % ( ce, queue )
self.queueDict[queueName] = {}
self.queueDict[queueName]['ParametersDict'] = qDict[queue]
self.queueDict[queueName]['ParametersDict']['Queue'] = queue
self.queueDict[queueName]['ParametersDict']['Site'] = site
self.queueDict[queueName]['ParametersDict']['GridEnv'] = self.gridEnv
self.queueDict[queueName]['ParametersDict']['Setup'] = gConfig.getValue( '/DIRAC/Setup', 'unknown' )
# Evaluate the CPU limit of the queue according to the Glue convention
# To Do: should be a utility
if "maxCPUTime" in self.queueDict[queueName]['ParametersDict'] and \
"SI00" in self.queueDict[queueName]['ParametersDict']:
maxCPUTime = float( self.queueDict[queueName]['ParametersDict']['maxCPUTime'] )
# For some sites there are crazy values in the CS
maxCPUTime = max( maxCPUTime, 0 )
maxCPUTime = min( maxCPUTime, 86400 * 12.5 )
si00 = float( self.queueDict[queueName]['ParametersDict']['SI00'] )
queueCPUTime = 60. / 250. * maxCPUTime * si00
self.queueDict[queueName]['ParametersDict']['CPUTime'] = int( queueCPUTime )
queueTags = self.queueDict[queueName]['ParametersDict'].get( 'Tag' )
if queueTags and isinstance( queueTags, basestring ):
queueTags = fromChar( queueTags )
self.queueDict[queueName]['ParametersDict']['Tag'] = queueTags
if ceTags:
if queueTags:
allTags = list( set( ceTags + queueTags ) )
self.queueDict[queueName]['ParametersDict']['Tag'] = allTags
else:
self.queueDict[queueName]['ParametersDict']['Tag'] = ceTags
maxRAM = self.queueDict[queueName]['ParametersDict'].get( 'MaxRAM' )
maxRAM = ceMaxRAM if not maxRAM else maxRAM
if maxRAM:
self.queueDict[queueName]['ParametersDict']['MaxRAM'] = maxRAM
if pilotRunDirectory:
self.queueDict[queueName]['ParametersDict']['JobExecDir'] = pilotRunDirectory
qwDir = os.path.join( self.workingDirectory, queue )
mkDir(qwDir)
self.queueDict[queueName]['ParametersDict']['WorkingDirectory'] = qwDir
platform = ''
if "Platform" in self.queueDict[queueName]['ParametersDict']:
platform = self.queueDict[queueName]['ParametersDict']['Platform']
elif "Platform" in ceDict:
platform = ceDict['Platform']
elif "OS" in ceDict:
architecture = ceDict.get( 'architecture', 'x86_64' )
OS = ceDict['OS']
platform = '_'.join( [architecture, OS] )
if platform and not platform in self.platforms:
self.platforms.append( platform )
if not "Platform" in self.queueDict[queueName]['ParametersDict'] and platform:
result = Resources.getDIRACPlatform( platform )
if result['OK']:
self.queueDict[queueName]['ParametersDict']['Platform'] = result['Value'][0]
ceQueueDict = dict( ceDict )
ceQueueDict.update( self.queueDict[queueName]['ParametersDict'] )
# Generate the CE object for the queue or pick the already existing one
# if the queue definition did not change
queueHash = self.__generateQueueHash( ceQueueDict )
if queueName in self.queueCECache and self.queueCECache[queueName]['Hash'] == queueHash:
queueCE = self.queueCECache[queueName]['CE']
else:
result = ceFactory.getCE( ceName = ce,
ceType = ceDict['CEType'],
ceParametersDict = ceQueueDict )
if not result['OK']:
return result
self.queueCECache.setdefault( queueName, {} )
self.queueCECache[queueName]['Hash'] = queueHash
self.queueCECache[queueName]['CE'] = result['Value']
queueCE = self.queueCECache[queueName]['CE']
self.queueDict[queueName]['CE'] = queueCE
self.queueDict[queueName]['CEName'] = ce
self.queueDict[queueName]['CEType'] = ceDict['CEType']
self.queueDict[queueName]['Site'] = site
self.queueDict[queueName]['QueueName'] = queue
self.queueDict[queueName]['Platform'] = platform
result = self.queueDict[queueName]['CE'].isValid()
if not result['OK']:
self.log.fatal( result['Message'] )
return result
if 'BundleProxy' in self.queueDict[queueName]['ParametersDict']:
if self.queueDict[queueName]['ParametersDict']['BundleProxy'].lower() in ['true','yes','1']:
self.queueDict[queueName]['BundleProxy'] = True
elif 'BundleProxy' in ceDict:
if ceDict['BundleProxy'].lower() in ['true','yes','1']:
self.queueDict[queueName]['BundleProxy'] = True
if site not in self.sites:
self.sites.append( site )
return S_OK()
def __syncCSWithVOMS(self, vo):
self.__adminMsgs = {'Errors': [], 'Info': []}
resultDict = defaultdict(list)
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(vo)
if not result['OK']:
return result
vomsDIRACMapping = result['Value']['VOMSDIRAC']
diracVOMSMapping = result['Value']['DIRACVOMS']
noVOMSGroups = result['Value']['NoVOMS']
noSyncVOMSGroups = result['Value']['NoSyncVOMS']
vomsSrv = VOMSService(vo)
# Get VOMS VO name
result = vomsSrv.admGetVOName()
if not result['OK']:
self.log.error('Could not retrieve VOMS VO name', "for %s" % vo)
return result
vomsVOName = result['Value'].lstrip('/')
self.log.verbose("VOMS VO Name for %s is %s" % (vo, vomsVOName))
# Get VOMS users
result = vomsSrv.getUsers()
if not result['OK']:
self.log.error('Could not retrieve user information from VOMS',
result['Message'])
return result
vomsUserDict = result['Value']
message = "There are %s user entries in VOMS for VO %s" % (
len(vomsUserDict), vomsVOName)
self.__adminMsgs['Info'].append(message)
self.log.info(message)
# Get DIRAC users
result = self.getVOUserData(vo)
if not result['OK']:
return result
diracUserDict = result['Value']
self.__adminMsgs['Info'].append(
"There are %s registered users in DIRAC for VO %s" %
(len(diracUserDict), vo))
self.log.info("There are %s registered users in DIRAC VO %s" %
(len(diracUserDict), vo))
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]['DN']
# We can have users with more than one DN registered
dnList = fromChar(dn)
existingDNs.extend(dnList)
for dn in dnList:
if dn not in vomsUserDict:
obsoletedDNs.append(dn)
for dn in vomsUserDict:
if dn not in existingDNs:
newDNs.append(dn)
allDiracUsers = getAllUsers()
nonVOUserDict = {}
nonVOUsers = list(set(allDiracUsers) - set(diracUserDict.keys()))
if nonVOUsers:
result = self.csapi.describeUsers(nonVOUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
nonVOUserDict = result['Value']
# Process users
defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo)
newAddedUserDict = {}
for dn in vomsUserDict:
nickName = ''
newDNForExistingUser = ''
diracName = ''
if dn in existingDNs:
for user in diracUserDict:
if dn == diracUserDict[user]['DN']:
diracName = user
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
for user in nonVOUserDict:
if dn == nonVOUserDict[user]['DN']:
diracName = user
# Check the nickName in the same VO to see if the user is already registered
# with another DN
result = vomsSrv.attGetUserNickname(dn, vomsUserDict[dn]['CA'])
if result['OK']:
nickName = result['Value']
if nickName in diracUserDict or nickName in newAddedUserDict:
diracName = nickName
# This is a flag for adding the new DN to an already existing user
newDNForExistingUser = dn
# We have a real new user
if not diracName:
if nickName:
newDiracName = nickName
else:
newDiracName = getUserName(dn,
vomsUserDict[dn]['mail'],
vo)
# If the chosen user name exists already, append a distinguishing suffix
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % (trialName, ind)
ind += 1
# We now have everything to add the new user
userDict = {
"DN": dn,
"CA": vomsUserDict[dn]['CA'],
"Email": vomsUserDict[dn]['mail']
}
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % (vomsVOName, role)
groupList = vomsDIRACMapping.get(fullRole, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
userDict['Groups'] = list(
set(groupsWithRole + [defaultVOGroup]))
message = "\n Added new user %s:\n" % newDiracName
for key in userDict:
message += " %s: %s\n" % (key, str(userDict[key]))
self.__adminMsgs['Info'].append(message)
self.voChanged = True
if self.autoAddUsers:
self.log.info("Adding new user %s: %s" %
(newDiracName, str(userDict)))
result = self.csapi.modifyUser(
newDiracName, userDict, createIfNonExistant=True)
if not result['OK']:
self.log.warn('Failed adding new user %s' %
newDiracName)
resultDict['NewUsers'].append(newDiracName)
newAddedUserDict[newDiracName] = userDict
continue
# We have an already existing user
modified = False
userDict = {
"DN": dn,
"CA": vomsUserDict[dn]['CA'],
"Email": vomsUserDict[dn]['mail']
}
if newDNForExistingUser:
userDict['DN'] = ','.join([dn, diracUserDict[diracName]['DN']])
modified = True
existingGroups = diracUserDict.get(diracName, {}).get('Groups', [])
nonVOGroups = list(
set(existingGroups) - set(diracVOMSMapping.keys()))
groupsWithRole = []
for role in vomsUserDict[dn]['Roles']:
fullRole = "/%s/%s" % (vomsVOName, role)
groupList = vomsDIRACMapping.get(fullRole, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
keepGroups = nonVOGroups + groupsWithRole + [defaultVOGroup]
for group in existingGroups:
if group in nonVOGroups:
continue
role = diracVOMSMapping.get(group, '')
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if not "Role" in role:
keepGroups.append(group)
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append(group)
# Keep groups for which syncronization with VOMS is forbidden
if group in noSyncVOMSGroups:
keepGroups.append(group)
userDict['Groups'] = list(set(keepGroups))
# Merge together groups for the same user but different DNs
if diracName in newAddedUserDict:
otherGroups = newAddedUserDict[diracName].get('Groups', [])
userDict['Groups'] = list(set(keepGroups + otherGroups))
modified = True
# Check if something changed before asking CSAPI to modify
if diracName in diracUserDict:
message = "\n Modified user %s:\n" % diracName
modMsg = ''
for key in userDict:
if key == "Groups":
addedGroups = set(userDict[key]) - set(
diracUserDict.get(diracName, {}).get(key, []))
removedGroups = set(
diracUserDict.get(diracName, {}).get(
key, [])) - set(userDict[key])
if addedGroups:
modMsg += " Added to group(s) %s\n" % ','.join(
addedGroups)
if removedGroups:
modMsg += " Removed from group(s) %s\n" % ','.join(
removedGroups)
else:
oldValue = str(
diracUserDict.get(diracName, {}).get(key, ''))
if str(userDict[key]) != oldValue:
modMsg += " %s: %s -> %s\n" % (
key, oldValue, str(userDict[key]))
if modMsg:
self.__adminMsgs['Info'].append(message + modMsg)
modified = True
if self.autoModifyUsers and modified:
result = self.csapi.modifyUser(diracName, userDict)
if result['OK'] and result['Value']:
self.log.info("Modified user %s: %s" %
(diracName, str(userDict)))
self.voChanged = True
resultDict['ModifiedUsers'].append(diracName)
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]['DN']))
if not dnSet.intersection(set(
vomsUserDict.keys())) and user not in nonVOUserDict:
for group in diracUserDict[user]['Groups']:
if group not in noVOMSGroups:
oldUsers.add(user)
if oldUsers:
self.voChanged = True
if self.autoDeleteUsers:
self.log.info('The following users will be deleted: %s' %
str(oldUsers))
result = self.csapi.deleteUsers(oldUsers)
if result['OK']:
self.__adminMsgs['Info'].append(
'The following users are deleted from CS:\n %s\n' %
str(oldUsers))
resultDict['DeletedUsers'] = oldUsers
else:
self.__adminMsgs['Errors'].append(
'Error in deleting users from CS:\n %s' %
str(oldUsers))
self.log.error('Error while user deletion from CS', result)
else:
self.__adminMsgs['Info'].append(
'The following users to be checked for deletion:\n %s' %
str(oldUsers))
self.log.info(
'The following users to be checked for deletion: %s' %
str(oldUsers))
return S_OK(resultDict)
def registerUser( self ):
"""
This function is used to notify DIRAC admins about user registration request
The logic is simple:
0) Check if request from this e-mail has already registered or not
1) Send mail to VO admin of requested VO
2) Send mail to users in group with UserAdministrator property
3) Send mail to users indicated in /Website/UserRegistrationAdmin option
"""
gLogger.info("Start processing a registration request")
checkUserCredentials()
# Check for having a DN but no username
dn = getUserDN()
if not dn:
error = "Certificate is not loaded in the browser or DN is absent"
gLogger.error( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
username = getUsername()
if not username == "anonymous":
error = "You are already registered in DIRAC with username: %s" % username
gLogger.error( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
gLogger.info( "DN: %s" % dn )
if not "email" in request.params:
error = "Can not get your email address from the request"
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
userMail = request.params[ "email" ]
result = self.isRequested( userMail )
gLogger.debug( result )
if result[ "OK" ]:
return render( "/reg_done.mako" )
result = self.registerRequest( dn , userMail )
gLogger.debug( result )
if not result[ "OK" ]:
return { "success" : "false" , "error" : result[ "Message" ] }
vo = fromChar( request.params[ "vo" ] )
if not vo:
error = "You should indicate a VirtualOrganization for membership"
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
gLogger.info( "User want to be register in VO(s): %s" % vo )
body = str()
for i in request.params:
if not i in [ "registration_request" , "email" , "vo" ]:
text = self.checkUnicode( request.params[ i ] )
info = "%s - %s" % ( i , text )
body = body + info + "\n"
body = body + "DN - " + dn
gLogger.debug( "email body: %s" % body )
adminList = self.__getAdminList( vo )
if not len( adminList ) > 0:
error = "Can't get in contact with administrators about your request\n"
error = error + "Most likely this DIRAC instance is not configured yet"
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
adminList = uniqueElements( adminList )
gLogger.info( "Chosen admin(s): %s" % adminList )
sendDict = self.getMailDict( adminList )
if not len( sendDict ) > 0:
error = "Can't get in contact with administrators about your request\n"
error = error + "Most likely this DIRAC instance is not configured yet"
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
gLogger.debug( "Final dictionary with mails to be used %s" % sendDict )
if socket.gethostname().find( '.' ) >= 0:
hostname = socket.gethostname()
else:
hostname = socket.gethostbyaddr( socket.gethostname() )[ 0 ]
title = "New user has sent registration request to %s" % hostname
return self.sendMail( sendDict , title , body , userMail )
def getImages(self, resourceDict):
""" Get the list of relevant CEs and their descriptions
"""
self.imageDict = {}
ceFactory = EndpointFactory()
result = getPilotBootstrapParameters(vo=self.vo,
runningPod=self.runningPod)
if not result['OK']:
return result
opParameters = result['Value']
for site in resourceDict:
for ce in resourceDict[site]:
ceDict = resourceDict[site][ce]
ceTags = ceDict.get('Tag', [])
if isinstance(ceTags, basestring):
ceTags = fromChar(ceTags)
ceMaxRAM = ceDict.get('MaxRAM', None)
qDict = ceDict.pop('Images')
for image in qDict:
imageName = '%s_%s' % (ce, image)
self.imageDict[imageName] = {}
self.imageDict[imageName]['ParametersDict'] = qDict[image]
self.imageDict[imageName]['ParametersDict'][
'Image'] = image
self.imageDict[imageName]['ParametersDict']['Site'] = site
self.imageDict[imageName]['ParametersDict'][
'Setup'] = gConfig.getValue('/DIRAC/Setup', 'unknown')
self.imageDict[imageName]['ParametersDict'][
'CPUTime'] = 99999999
imageTags = self.imageDict[imageName][
'ParametersDict'].get('Tag')
if imageTags and isinstance(imageTags, basestring):
imageTags = fromChar(imageTags)
self.imageDict[imageName]['ParametersDict'][
'Tag'] = imageTags
if ceTags:
if imageTags:
allTags = list(set(ceTags + imageTags))
self.imageDict[imageName]['ParametersDict'][
'Tag'] = allTags
else:
self.imageDict[imageName]['ParametersDict'][
'Tag'] = ceTags
maxRAM = self.imageDict[imageName]['ParametersDict'].get(
'MaxRAM')
maxRAM = ceMaxRAM if not maxRAM else maxRAM
if maxRAM:
self.imageDict[imageName]['ParametersDict'][
'MaxRAM'] = maxRAM
platform = ''
if "Platform" in self.imageDict[imageName][
'ParametersDict']:
platform = self.imageDict[imageName]['ParametersDict'][
'Platform']
elif "Platform" in ceDict:
platform = ceDict['Platform']
if platform and not platform in self.platforms:
self.platforms.append(platform)
if not "Platform" in self.imageDict[imageName][
'ParametersDict'] and platform:
result = Resources.getDIRACPlatform(platform)
if result['OK']:
self.imageDict[imageName]['ParametersDict'][
'Platform'] = result['Value'][0]
ceImageDict = dict(ceDict)
ceImageDict['CEName'] = ce
ceImageDict['VO'] = self.vo
ceImageDict['Image'] = image
ceImageDict['RunningPod'] = self.runningPod
ceImageDict['CSServers'] = gConfig.getValue(
"/DIRAC/Configuration/Servers", [])
ceImageDict.update(
self.imageDict[imageName]['ParametersDict'])
ceImageDict.update(opParameters)
# Generate the CE object for the image or pick the already existing one
# if the image definition did not change
imageHash = self.__generateImageHash(ceImageDict)
if imageName in self.imageCECache and self.imageCECache[
imageName]['Hash'] == imageHash:
imageCE = self.imageCECache[imageName]['CE']
else:
result = ceFactory.getCEObject(parameters=ceImageDict)
if not result['OK']:
return result
self.imageCECache.setdefault(imageName, {})
self.imageCECache[imageName]['Hash'] = imageHash
self.imageCECache[imageName]['CE'] = result['Value']
imageCE = self.imageCECache[imageName]['CE']
self.imageDict[imageName]['CE'] = imageCE
self.imageDict[imageName]['CEName'] = ce
self.imageDict[imageName]['CEType'] = ceDict['CEType']
self.imageDict[imageName]['Site'] = site
self.imageDict[imageName]['ImageName'] = image
self.imageDict[imageName]['Platform'] = platform
self.imageDict[imageName]['MaxInstances'] = ceDict[
'MaxInstances']
if not self.imageDict[imageName]['CE'].isValid():
self.log.error(
'Failed to instantiate CloudEndpoint for %s' %
imageName)
continue
if site not in self.sites:
self.sites.append(site)
return S_OK()
def registerUser(self):
"""
This function is used to notify DIRAC admins about user registration request
The logic is simple:
0) Check if request from this e-mail has already registered or not
1) Send mail to VO admin of requested VO
2) Send mail to users in group with UserAdministrator property
3) Send mail to users indicated in /Website/UserRegistrationAdmin option
"""
gLogger.info("Start processing a registration request")
checkUserCredentials()
# Check for having a DN but no username
dn = getUserDN()
if not dn:
error = "Certificate is not loaded in the browser or DN is absent"
gLogger.error("Service response: %s" % error)
return {"success": "false", "error": error}
username = getUsername()
if not username == "anonymous":
error = "You are already registered in DIRAC with username: %s" % username
gLogger.error("Service response: %s" % error)
return {"success": "false", "error": error}
gLogger.info("DN: %s" % dn)
if not "email" in request.params:
error = "Can not get your email address from the request"
gLogger.debug("Service response: %s" % error)
return {"success": "false", "error": error}
userMail = request.params["email"]
result = self.isRequested(userMail)
gLogger.debug(result)
if result["OK"]:
return render("/reg_done.mako")
result = self.registerRequest(dn, userMail)
gLogger.debug(result)
if not result["OK"]:
return {"success": "false", "error": result["Message"]}
vo = fromChar(request.params["vo"])
if not vo:
error = "You should indicate a VirtualOrganization for membership"
gLogger.debug("Service response: %s" % error)
return {"success": "false", "error": error}
gLogger.info("User want to be register in VO(s): %s" % vo)
body = str()
for i in request.params:
if not i in ["registration_request", "email", "vo"]:
text = self.checkUnicode(request.params[i])
info = "%s - %s" % (i, text)
body = body + info + "\n"
body = body + "DN - " + dn
gLogger.debug("email body: %s" % body)
adminList = self.__getAdminList(vo)
if not len(adminList) > 0:
error = "Can't get in contact with administrators about your request\n"
error = error + "Most likely this DIRAC instance is not configured yet"
gLogger.debug("Service response: %s" % error)
return {"success": "false", "error": error}
adminList = uniqueElements(adminList)
gLogger.info("Chosen admin(s): %s" % adminList)
sendDict = self.getMailDict(adminList)
if not len(sendDict) > 0:
error = "Can't get in contact with administrators about your request\n"
error = error + "Most likely this DIRAC instance is not configured yet"
gLogger.debug("Service response: %s" % error)
return {"success": "false", "error": error}
gLogger.debug("Final dictionary with mails to be used %s" % sendDict)
if socket.gethostname().find('.') >= 0:
hostname = socket.gethostname()
else:
hostname = socket.gethostbyaddr(socket.gethostname())[0]
title = "New user has sent registration request to %s" % hostname
return self.sendMail(sendDict, title, body, userMail)
def syncCSWithVOMS(self):
"""Performs the synchronization of the DIRAC registry with the VOMS data. The resulting
CSAPI object containing modifications is returned as part of the output dictionary.
Those changes can be applied by the caller depending on the mode (dry or a real run)
:return: S_OK with a dictionary containing the results of the synchronization operation
"""
resultDict = defaultdict(list)
# Get DIRAC group vs VOMS Role Mappings
result = getVOMSRoleGroupMapping(self.vo)
if not result["OK"]:
return result
vomsDIRACMapping = result["Value"]["VOMSDIRAC"]
diracVOMSMapping = result["Value"]["DIRACVOMS"]
noVOMSGroups = result["Value"]["NoVOMS"]
noSyncVOMSGroups = result["Value"]["NoSyncVOMS"]
vomsSrv = VOMSService(self.vo)
# Get VOMS users
result = vomsSrv.getUsers()
if not result["OK"]:
self.log.error("Could not retrieve user information from VOMS", result["Message"])
return result
self.vomsUserDict = result["Value"]
message = "There are %s user entries in VOMS for VO %s" % (len(self.vomsUserDict), self.vomsVOName)
self.adminMsgs["Info"].append(message)
self.log.info("VOMS user entries", message)
self.log.debug(self.vomsUserDict)
# Get DIRAC users
result = self.getVOUserData(self.vo)
if not result["OK"]:
return result
diracUserDict = result["Value"]
self.adminMsgs["Info"].append(
"There are %s registered users in DIRAC for VO %s" % (len(diracUserDict), self.vo)
)
self.log.info(
"Users already registered", ": there are %s registered users in DIRAC VO %s" % (len(diracUserDict), self.vo)
)
# Find new and obsoleted user DNs
existingDNs = []
obsoletedDNs = []
newDNs = []
for user in diracUserDict:
dn = diracUserDict[user]["DN"]
# We can have users with more than one DN registered
dnList = fromChar(dn)
existingDNs.extend(dnList)
for dn in dnList:
if dn not in self.vomsUserDict:
obsoletedDNs.append(dn)
for dn in self.vomsUserDict:
if dn not in existingDNs:
newDNs.append(dn)
allDiracUsers = getAllUsers()
nonVOUserDict = {}
nonVOUsers = list(set(allDiracUsers) - set(diracUserDict))
if nonVOUsers:
result = self.csapi.describeUsers(nonVOUsers)
if not result["OK"]:
self.log.error("Could not retrieve CS User description")
return result
nonVOUserDict = result["Value"]
# Process users
defaultVOGroup = getVOOption(self.vo, "DefaultGroup", "%s_user" % self.vo)
# If a user is (previously put by hand) in a "QuarantineGroup",
# then the default group will be ignored.
# So, this option is only considered for the case of existing users.
quarantineVOGroup = getVOOption(self.vo, "QuarantineGroup")
newAddedUserDict = {}
for dn in self.vomsUserDict:
newDNForExistingUser = ""
diracName = ""
if dn in existingDNs:
for user in diracUserDict:
if dn in fromChar(diracUserDict[user]["DN"]):
diracName = user
break
if dn in newDNs:
# Find if the DN is already registered in the DIRAC CS
for user in nonVOUserDict:
if dn in fromChar(nonVOUserDict[user]["DN"]):
diracName = user
diracUserDict[diracName] = nonVOUserDict[user]
break
# Check the nickName in the same VO to see if the user is already registered
# with another DN
nickName = self.vomsUserDict[dn].get("nickname")
if nickName in diracUserDict or nickName in newAddedUserDict:
diracName = nickName
# This is a flag for adding the new DN to an already existing user
newDNForExistingUser = dn
# We have a real new user
if not diracName:
if nickName:
newDiracName = nickName
else:
newDiracName = self.getUserName(dn)
# Do not consider users with Suspended status in VOMS
if self.vomsUserDict[dn]["suspended"] or self.vomsUserDict[dn]["certSuspended"]:
resultDict["SuspendedUsers"].append(newDiracName)
continue
# If the chosen user name exists already, append a distinguishing suffix
ind = 1
trialName = newDiracName
while newDiracName in allDiracUsers:
# We have a user with the same name but with a different DN
newDiracName = "%s_%d" % (trialName, ind)
ind += 1
# We now have everything to add the new user
userDict = {"DN": dn, "CA": self.vomsUserDict[dn]["CA"], "Email": self.vomsUserDict[dn]["mail"]}
groupsWithRole = []
for role in self.vomsUserDict[dn]["Roles"]:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
userDict["Groups"] = list(set(groupsWithRole + [defaultVOGroup]))
# Run the sync plugins for extra info and/or validations
if self.syncPlugin:
try:
self.syncPlugin.verifyAndUpdateUserInfo(newDiracName, userDict)
except ValueError as e:
self.log.error("Error validating new user", "nickname %s\n error %s" % (newDiracName, e))
self.adminMsgs["Errors"].append(
"Error validating new user %s: %s\n %s" % (newDiracName, userDict, e)
)
continue
message = "\n Added new user %s:\n" % newDiracName
for key in userDict:
message += " %s: %s\n" % (key, str(userDict[key]))
self.adminMsgs["Info"].append(message)
self.voChanged = True
if self.autoAddUsers:
self.log.info("Adding new user %s: %s" % (newDiracName, str(userDict)))
result = self.csapi.modifyUser(newDiracName, userDict, createIfNonExistant=True)
if not result["OK"]:
self.log.warn("Failed adding new user %s" % newDiracName)
resultDict["NewUsers"].append(newDiracName)
newAddedUserDict[newDiracName] = userDict
continue
# We have an already existing user
modified = False
suspendedInVOMS = self.vomsUserDict[dn]["suspended"] or self.vomsUserDict[dn]["certSuspended"]
suspendedVOList = getUserOption(diracName, "Suspended", [])
knownEmail = getUserOption(diracName, "Email", None)
userDict = {
"DN": diracUserDict[diracName]["DN"],
"CA": diracUserDict[diracName]["CA"],
"Email": self.vomsUserDict[dn].get("mail", self.vomsUserDict[dn].get("emailAddress")) or knownEmail,
}
# Set Suspended status for the user for this particular VO
if suspendedInVOMS and self.vo not in suspendedVOList:
suspendedVOList.append(self.vo)
userDict["Suspended"] = ",".join(suspendedVOList)
modified = True
# Remove the lifted Suspended status
if not suspendedInVOMS and self.vo in suspendedVOList and self.autoLiftSuspendedStatus:
newList = []
for vo in suspendedVOList:
if vo != self.vo:
newList.append(vo)
if not newList:
newList = ["None"]
userDict["Suspended"] = ",".join(newList)
modified = True
if newDNForExistingUser:
userDict["DN"] = ",".join([dn, diracUserDict.get(diracName, newAddedUserDict.get(diracName))["DN"]])
userDict["CA"] = ",".join(
[self.vomsUserDict[dn]["CA"], diracUserDict.get(diracName, newAddedUserDict.get(diracName))["CA"]]
)
modified = True
existingGroups = diracUserDict.get(diracName, {}).get("Groups", [])
nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping))
groupsWithRole = []
for role in self.vomsUserDict[dn]["Roles"]:
groupList = vomsDIRACMapping.get(role, [])
for group in groupList:
if group not in noSyncVOMSGroups:
groupsWithRole.append(group)
keepGroups = nonVOGroups + groupsWithRole
if not quarantineVOGroup or quarantineVOGroup not in existingGroups:
keepGroups += [defaultVOGroup]
if quarantineVOGroup and quarantineVOGroup in existingGroups:
keepGroups = [quarantineVOGroup]
for group in existingGroups:
if group in nonVOGroups:
continue
role = diracVOMSMapping.get(group, "")
# Among already existing groups for the user keep those without a special VOMS Role
# because this membership is done by hand in the CS
if "Role" not in role:
keepGroups.append(group)
# Keep existing groups with no VOMS attribute if any
if group in noVOMSGroups:
keepGroups.append(group)
# Keep groups for which syncronization with VOMS is forbidden
if group in noSyncVOMSGroups:
keepGroups.append(group)
userDict["Groups"] = list(set(keepGroups))
# Merge together groups for the same user but different DNs
if diracName in newAddedUserDict:
otherGroups = newAddedUserDict[diracName].get("Groups", [])
userDict["Groups"] = list(set(keepGroups + otherGroups))
modified = True
if not existingGroups and diracName in allDiracUsers:
groups = getGroupsForUser(diracName)
if groups["OK"]:
self.log.info("Found groups for user %s %s" % (diracName, groups["Value"]))
userDict["Groups"] = list(set(groups["Value"] + keepGroups))
addedGroups = list(set(userDict["Groups"]) - set(groups["Value"]))
modified = True
message = "\n Modified user %s:\n" % diracName
message += " Added to group(s) %s\n" % ",".join(addedGroups)
self.adminMsgs["Info"].append(message)
# Check if something changed before asking CSAPI to modify
if diracName in diracUserDict:
message = "\n Modified user %s:\n" % diracName
modMsg = ""
for key in userDict:
if key == "Groups":
addedGroups = set(userDict[key]) - set(diracUserDict.get(diracName, {}).get(key, []))
removedGroups = set(diracUserDict.get(diracName, {}).get(key, [])) - set(userDict[key])
if addedGroups:
modMsg += " Added to group(s) %s\n" % ",".join(addedGroups)
if removedGroups:
modMsg += " Removed from group(s) %s\n" % ",".join(removedGroups)
elif key == "Suspended":
if userDict["Suspended"] == "None":
modMsg += " Suspended status removed\n"
else:
modMsg += " User Suspended in VOs: %s\n" % userDict["Suspended"]
else:
oldValue = str(diracUserDict.get(diracName, {}).get(key, ""))
if str(userDict[key]) != oldValue:
modMsg += " %s: %s -> %s\n" % (key, oldValue, str(userDict[key]))
if modMsg:
self.adminMsgs["Info"].append(message + modMsg)
modified = True
if self.autoModifyUsers and modified:
result = self.csapi.modifyUser(diracName, userDict)
if result["OK"] and result["Value"]:
self.log.info("Modified user %s: %s" % (diracName, str(userDict)))
self.voChanged = True
resultDict["ModifiedUsers"].append(diracName)
# Check if there are potentially obsoleted users
oldUsers = set()
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]["DN"]))
if not dnSet.intersection(set(self.vomsUserDict)) and user not in nonVOUserDict:
existingGroups = diracUserDict.get(user, {}).get("Groups", [])
nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping))
removedGroups = list(set(existingGroups) - set(nonVOGroups))
if removedGroups:
self.log.info("Checking user for deletion", "%s: %s" % (user, existingGroups))
self.log.info("User has groups in other VOs", "%s: %s" % (user, nonVOGroups))
userDict = diracUserDict[user]
userDict["Groups"] = nonVOGroups
if self.autoModifyUsers:
result = self.csapi.modifyUser(user, userDict)
if result["OK"] and result["Value"]:
self.log.info("Modified user %s: %s" % (user, str(userDict)))
self.voChanged = True
message = "\n Modified user %s:\n" % user
modMsg = " Removed from group(s) %s\n" % ",".join(removedGroups)
self.adminMsgs["Info"].append(message + modMsg)
resultDict["ModifiedUsers"].append(user)
continue
if not any(group in noVOMSGroups for group in existingGroups):
oldUsers.add(user)
# Check for obsoleted DNs
for user in diracUserDict:
dnSet = set(fromChar(diracUserDict[user]["DN"]))
for dn in dnSet:
if dn in obsoletedDNs and user not in oldUsers:
existingGroups = diracUserDict.get(user, {}).get("Groups", [])
nonVOGroups = list(set(existingGroups) - set(diracVOMSMapping))
if nonVOGroups:
self.log.verbose("User has groups in other VOs", "%s: %s" % (user, nonVOGroups))
continue
self.log.verbose("Modified user %s: dropped DN %s" % (user, dn))
if self.autoModifyUsers:
userDict = diracUserDict[user]
modDNSet = dnSet - set([dn])
if modDNSet:
userDict["DN"] = ",".join(modDNSet)
result = self.csapi.modifyUser(user, userDict)
if result["OK"] and result["Value"]:
self.log.info("Modified user %s: dropped DN %s" % (user, dn))
self.adminMsgs["Info"].append("Modified user %s: dropped DN %s" % (user, dn))
self.voChanged = True
resultDict["ModifiedUsers"].append(diracName)
else:
oldUsers.add(user)
if oldUsers:
self.voChanged = True
if self.autoDeleteUsers:
self.log.info("The following users will be deleted: %s" % str(oldUsers))
result = self.csapi.deleteUsers(oldUsers)
if result["OK"]:
self.adminMsgs["Info"].append("The following users are deleted from CS:\n %s\n" % str(oldUsers))
resultDict["DeletedUsers"] = oldUsers
else:
self.adminMsgs["Errors"].append("Error in deleting users from CS:\n %s" % str(oldUsers))
self.log.error("Error while user deletion from CS", result)
else:
self.adminMsgs["Info"].append(
"The following users to be checked for deletion:\n\t%s" % "\n\t".join(sorted(oldUsers))
)
self.log.info("The following users to be checked for deletion:", "\n\t".join(sorted(oldUsers)))
resultDict["CSAPI"] = self.csapi
resultDict["AdminMessages"] = self.adminMsgs
resultDict["VOChanged"] = self.voChanged
return S_OK(resultDict)
def getEndpoints(self, resourceDict):
"""Get the list of relevant CEs and their descriptions"""
self.vmTypeDict = {}
ceFactory = EndpointFactory()
result = getPilotBootstrapParameters(vo=self.vo, runningPod=self.runningPod)
if not result["OK"]:
return result
opParameters = result["Value"]
for site in resourceDict:
for ce in resourceDict[site]:
ceDict = resourceDict[site][ce]
ceTags = ceDict.get("Tag", [])
if isinstance(ceTags, six.string_types):
ceTags = fromChar(ceTags)
ceMaxRAM = ceDict.get("MaxRAM", None)
qDict = ceDict.pop("VMTypes")
for vmType in qDict:
vmTypeName = "%s_%s" % (ce, vmType)
self.vmTypeDict[vmTypeName] = {}
self.vmTypeDict[vmTypeName]["ParametersDict"] = qDict[vmType]
self.vmTypeDict[vmTypeName]["ParametersDict"]["VMType"] = vmType
self.vmTypeDict[vmTypeName]["ParametersDict"]["Site"] = site
self.vmTypeDict[vmTypeName]["ParametersDict"]["Setup"] = gConfig.getValue("/DIRAC/Setup", "unknown")
self.vmTypeDict[vmTypeName]["ParametersDict"]["CPUTime"] = 99999999
vmTypeTags = self.vmTypeDict[vmTypeName]["ParametersDict"].get("Tag")
if vmTypeTags and isinstance(vmTypeTags, six.string_types):
vmTypeTags = fromChar(vmTypeTags)
self.vmTypeDict[vmTypeName]["ParametersDict"]["Tag"] = vmTypeTags
if ceTags:
if vmTypeTags:
allTags = list(set(ceTags + vmTypeTags))
self.vmTypeDict[vmTypeName]["ParametersDict"]["Tag"] = allTags
else:
self.vmTypeDict[vmTypeName]["ParametersDict"]["Tag"] = ceTags
maxRAM = self.vmTypeDict[vmTypeName]["ParametersDict"].get("MaxRAM")
maxRAM = ceMaxRAM if not maxRAM else maxRAM
if maxRAM:
self.vmTypeDict[vmTypeName]["ParametersDict"]["MaxRAM"] = maxRAM
ceWholeNode = ceDict.get("WholeNode", "true")
wholeNode = self.vmTypeDict[vmTypeName]["ParametersDict"].get("WholeNode", ceWholeNode)
if wholeNode.lower() in ("yes", "true"):
self.vmTypeDict[vmTypeName]["ParametersDict"].setdefault("Tag", [])
self.vmTypeDict[vmTypeName]["ParametersDict"]["Tag"].append("WholeNode")
platform = ""
if "Platform" in self.vmTypeDict[vmTypeName]["ParametersDict"]:
platform = self.vmTypeDict[vmTypeName]["ParametersDict"]["Platform"]
elif "Platform" in ceDict:
platform = ceDict["Platform"]
if platform and platform not in self.platforms:
self.platforms.append(platform)
if "Platform" not in self.vmTypeDict[vmTypeName]["ParametersDict"] and platform:
result = Resources.getDIRACPlatform(platform)
if result["OK"]:
self.vmTypeDict[vmTypeName]["ParametersDict"]["Platform"] = result["Value"][0]
ceVMTypeDict = dict(ceDict)
ceVMTypeDict["CEName"] = ce
ceVMTypeDict["VO"] = self.vo
ceVMTypeDict["VMType"] = vmType
ceVMTypeDict["RunningPod"] = self.runningPod
ceVMTypeDict["CSServers"] = gConfig.getValue("/DIRAC/Configuration/Servers", [])
ceVMTypeDict.update(self.vmTypeDict[vmTypeName]["ParametersDict"])
# Allow a resource-specifc CAPath to be set (as some clouds have their own CAs)
# Otherwise fall back to the system-wide default(s)
if "CAPath" not in ceVMTypeDict:
ceVMTypeDict["CAPath"] = gConfig.getValue(
"/DIRAC/Security/CAPath", "/opt/dirac/etc/grid-security/certificates/cas.pem"
)
# Generate the CE object for the vmType or pick the already existing one
# if the vmType definition did not change
vmTypeHash = self.__generateVMTypeHash(ceVMTypeDict)
if vmTypeName in self.vmTypeCECache and self.vmTypeCECache[vmTypeName]["Hash"] == vmTypeHash:
vmTypeCE = self.vmTypeCECache[vmTypeName]["CE"]
else:
result = ceFactory.getCEObject(parameters=ceVMTypeDict)
if not result["OK"]:
return result
self.vmTypeCECache.setdefault(vmTypeName, {})
self.vmTypeCECache[vmTypeName]["Hash"] = vmTypeHash
self.vmTypeCECache[vmTypeName]["CE"] = result["Value"]
vmTypeCE = self.vmTypeCECache[vmTypeName]["CE"]
vmTypeCE.setBootstrapParameters(opParameters)
self.vmTypeDict[vmTypeName]["CE"] = vmTypeCE
self.vmTypeDict[vmTypeName]["CEName"] = ce
self.vmTypeDict[vmTypeName]["CEType"] = ceDict["CEType"]
self.vmTypeDict[vmTypeName]["Site"] = site
self.vmTypeDict[vmTypeName]["VMType"] = vmType
self.vmTypeDict[vmTypeName]["Platform"] = platform
self.vmTypeDict[vmTypeName]["MaxInstances"] = ceDict["MaxInstances"]
if not self.vmTypeDict[vmTypeName]["CE"].isValid():
self.log.error("Failed to instantiate CloudEndpoint for %s" % vmTypeName)
continue
if site not in self.sites:
self.sites.append(site)
return S_OK()
def __getPilotOptions( self, queue, pilotsToSubmit ):
""" Prepare pilot options
"""
queueDict = self.queueDict[queue]['ParametersDict']
pilotOptions = []
setup = gConfig.getValue( "/DIRAC/Setup", "unknown" )
if setup == 'unknown':
self.log.error( 'Setup is not defined in the configuration' )
return [ None, None ]
pilotOptions.append( '-S %s' % setup )
opsHelper = Operations.Operations( group = self.pilotGroup, setup = setup )
#Installation defined?
installationName = opsHelper.getValue( "Pilot/Installation", "" )
if installationName:
pilotOptions.append( '-V %s' % installationName )
#Project defined?
projectName = opsHelper.getValue( "Pilot/Project", "" )
if projectName:
pilotOptions.append( '-l %s' % projectName )
else:
self.log.info( 'DIRAC project will be installed by pilots' )
#Request a release
diracVersion = opsHelper.getValue( "Pilot/Version", [] )
if not diracVersion:
self.log.error( 'Pilot/Version is not defined in the configuration' )
return [ None, None ]
#diracVersion is a list of accepted releases. Just take the first one
pilotOptions.append( '-r %s' % diracVersion[0] )
ownerDN = self.pilotDN
ownerGroup = self.pilotGroup
# Request token for maximum pilot efficiency
result = gProxyManager.requestToken( ownerDN, ownerGroup, pilotsToSubmit * self.maxJobsInFillMode )
if not result[ 'OK' ]:
self.log.error( 'Invalid proxy token request', result['Message'] )
return [ None, None ]
( token, numberOfUses ) = result[ 'Value' ]
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
# Use Filling mode
pilotOptions.append( '-M %s' % min( numberOfUses, self.maxJobsInFillMode ) )
# Since each pilot will execute min( numberOfUses, self.maxJobsInFillMode )
# with numberOfUses tokens we can submit at most:
# numberOfUses / min( numberOfUses, self.maxJobsInFillMode )
# pilots
newPilotsToSubmit = numberOfUses / min( numberOfUses, self.maxJobsInFillMode )
if newPilotsToSubmit != pilotsToSubmit:
self.log.info( 'Number of pilots to submit is changed to %d after getting the proxy token' % newPilotsToSubmit )
pilotsToSubmit = newPilotsToSubmit
# Debug
if self.pilotLogLevel.lower() == 'debug':
pilotOptions.append( '-d' )
# CS Servers
csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions to be used in pilots
pilotExtensionsList = opsHelper.getValue( "Pilot/Extensions", [] )
extensionsList = []
if pilotExtensionsList:
if pilotExtensionsList[0] != 'None':
extensionsList = pilotExtensionsList
else:
extensionsList = CSGlobals.getCSExtensions()
if extensionsList:
pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
# Requested CPU time
pilotOptions.append( '-T %s' % queueDict['CPUTime'] )
# CEName
pilotOptions.append( '-N %s' % self.queueDict[queue]['CEName'] )
# Queue
pilotOptions.append( '-Q %s' % self.queueDict[queue]['QueueName'] )
# SiteName
pilotOptions.append( '-n %s' % queueDict['Site'] )
if 'ClientPlatform' in queueDict:
pilotOptions.append( "-p '%s'" % queueDict['ClientPlatform'] )
if 'SharedArea' in queueDict:
pilotOptions.append( "-o '/LocalSite/SharedArea=%s'" % queueDict['SharedArea'] )
if 'SI00' in queueDict:
factor = float( queueDict['SI00'] ) / 250.
pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % factor )
pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % factor )
else:
if 'CPUScalingFactor' in queueDict:
pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % queueDict['CPUScalingFactor'] )
if 'CPUNormalizationFactor' in queueDict:
pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % queueDict['CPUNormalizationFactor'] )
if "ExtraPilotOptions" in queueDict:
pilotOptions.append( queueDict['ExtraPilotOptions'] )
# Hack
if self.defaultSubmitPools:
pilotOptions.append( '-o /Resources/Computing/CEDefaults/SubmitPool=%s' % self.defaultSubmitPools )
if "Tag" in queueDict:
tagString = ','.join( fromChar( queueDict['Tag'] ) )
pilotOptions.append( '-o /Resources/Computing/CEDefaults/Tag=%s' % tagString )
if self.group:
pilotOptions.append( '-G %s' % self.group )
self.log.verbose( "pilotOptions: ", ' '.join( pilotOptions ) )
return [ pilotOptions, pilotsToSubmit ]
def matchQueue(jobJDL, queueDict, fullMatch=False):
"""
Match the job description to the queue definition
:param str job: JDL job description
:param bool fullMatch: test matching on all the criteria
:param dict queueDict: queue parameters dictionary
:return: S_OK/S_ERROR, Value - result of matching, S_OK if matched or
S_ERROR with the reason for no match
"""
# Check the job description validity
job = ClassAd(jobJDL)
if not job.isOK():
return S_ERROR("Invalid job description")
noMatchReasons = []
# Check job requirements to resource
# 1. CPUTime
cpuTime = job.getAttributeInt("CPUTime")
if not cpuTime:
cpuTime = 84600
if cpuTime > int(queueDict.get("CPUTime", 0)):
noMatchReasons.append("Job CPUTime requirement not satisfied")
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 2. Multi-value match requirements
for parameter in ["Site", "GridCE", "Platform", "JobType"]:
if parameter in queueDict:
valueSet = set(job.getListFromExpression(parameter))
if not valueSet:
valueSet = set(job.getListFromExpression("%ss" % parameter))
queueSet = set(fromChar(queueDict[parameter]))
if valueSet and queueSet and not valueSet.intersection(queueSet):
valueToPrint = ",".join(valueSet)
if len(valueToPrint) > 20:
valueToPrint = "%s..." % valueToPrint[:20]
noMatchReasons.append("Job %s %s requirement not satisfied" % (parameter, valueToPrint))
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 3. Banned multi-value match requirements
for par in ["Site", "GridCE", "Platform", "JobType"]:
parameter = "Banned%s" % par
if par in queueDict:
valueSet = set(job.getListFromExpression(parameter))
if not valueSet:
valueSet = set(job.getListFromExpression("%ss" % parameter))
queueSet = set(fromChar(queueDict[par]))
if valueSet and queueSet and valueSet.issubset(queueSet):
valueToPrint = ",".join(valueSet)
if len(valueToPrint) > 20:
valueToPrint = "%s..." % valueToPrint[:20]
noMatchReasons.append("Job %s %s requirement not satisfied" % (parameter, valueToPrint))
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 4. Tags
tags = set(job.getListFromExpression("Tag"))
nProc = job.getAttributeInt("NumberOfProcessors")
if nProc and nProc > 1:
tags.add("MultiProcessor")
wholeNode = job.getAttributeString("WholeNode")
if wholeNode:
tags.add("WholeNode")
queueTags = set(queueDict.get("Tag", []))
if not tags.issubset(queueTags):
noMatchReasons.append("Job Tag %s not satisfied" % ",".join(tags))
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 4. MultiProcessor requirements
if nProc and nProc > int(queueDict.get("NumberOfProcessors", 1)):
noMatchReasons.append("Job NumberOfProcessors %d requirement not satisfied" % nProc)
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 5. RAM
ram = job.getAttributeInt("RAM")
# If MaxRAM is not specified in the queue description, assume 2GB
if ram and ram > int(queueDict.get("MaxRAM", 2048) / 1024):
noMatchReasons.append("Job RAM %d requirement not satisfied" % ram)
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# Check resource requirements to job
# 1. OwnerGroup - rare case but still
if "OwnerGroup" in queueDict:
result = getProxyInfo(disableVOMS=True)
if not result["OK"]:
return S_ERROR("No valid proxy available")
ownerGroup = result["Value"]["group"]
if ownerGroup != queueDict["OwnerGroup"]:
noMatchReasons.append("Resource OwnerGroup %s requirement not satisfied" % queueDict["OwnerGroup"])
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 2. Required tags
requiredTags = set(queueDict.get("RequiredTags", []))
if not requiredTags.issubset(tags):
noMatchReasons.append("Resource RequiredTags %s not satisfied" % ",".join(requiredTags))
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
# 3. RunningLimit
site = queueDict["Site"]
ce = queueDict.get("GridCE")
opsHelper = Operations()
result = opsHelper.getSections("JobScheduling/RunningLimit")
if result["OK"] and site in result["Value"]:
result = opsHelper.getSections("JobScheduling/RunningLimit/%s" % site)
if result["OK"]:
for parameter in result["Value"]:
value = job.getAttributeString(parameter)
if (
value
and (
opsHelper.getValue("JobScheduling/RunningLimit/%s/%s/%s" % (site, parameter, value), 1)
or opsHelper.getValue(
"JobScheduling/RunningLimit/%s/CEs/%s/%s/%s" % (site, ce, parameter, value), 1
)
)
== 0
):
noMatchReasons.append("Resource operational %s requirement not satisfied" % parameter)
if not fullMatch:
return S_OK({"Match": False, "Reason": noMatchReasons[0]})
return S_OK({"Match": not bool(noMatchReasons), "Reason": noMatchReasons})
def _ByJobType(self):
""" By default, all sites are allowed to do every job.
The actual rules are freely specified in the Operation JobTypeMapping section.
The content of the section may look like this:
User
{
Exclude = PAK
Exclude += Ferrara
Exclude += Bologna
Exclude += Paris
Exclude += CERN
Exclude += IN2P3
Allow
{
Paris = IN2P3
CERN = CERN
IN2P3 = IN2P3
}
}
DataReconstruction
{
Exclude = PAK
Exclude += Ferrara
Exclude += CERN
Exclude += IN2P3
Allow
{
Ferrara = CERN
CERN = CERN
IN2P3 = IN2P3
IN2P3 += CERN
}
}
Merge
{
Exclude = ALL
Allow
{
CERN = CERN
IN2P3 = IN2P3
}
}
The sites in the exclusion list will be removed.
The allow section says where each site may help another site
"""
# 1. get sites list
res = getSites()
if not res['OK']:
gLogger.error("Could not get the list of sites", res['Message'])
return res
destSites = set(res['Value'])
# 2. get JobTypeMapping "Exclude" value (and add autoAddedSites)
gLogger.debug("Getting JobTypeMapping 'Exclude' value (and add autoAddedSites)")
jobType = self.params['JobType']
if not jobType:
raise RuntimeError("No jobType specified")
excludedSites = set(self.opsH.getValue('JobTypeMapping/%s/Exclude' % jobType, []))
gLogger.debug("Explicitly excluded sites for %s task: %s" % (jobType, ','.join(excludedSites)))
autoAddedSites = self.opsH.getValue('JobTypeMapping/AutoAddedSites', [])
if 'WithStorage' in autoAddedSites:
# Add all sites with storage, such that jobs can run wherever data is
autoAddedSites.remove('WithStorage')
autoAddedSites += DMSHelpers().getTiers(withStorage=True, tier=(0, 1, 2))
# 3. removing sites in Exclude
if not excludedSites:
pass
elif 'ALL' in excludedSites:
destSites = set()
else:
destSites -= excludedSites
# 4. get JobTypeMapping "Allow" section
res = self.opsH.getOptionsDict('JobTypeMapping/%s/Allow' % jobType)
if not res['OK']:
gLogger.debug(res['Message'])
allowed = {}
else:
allowed = dict((site, set(fromChar(fromSites))) for site, fromSites in res['Value'].iteritems())
autoAddedSites = set(self.opsH.getValue('JobTypeMapping/%s/AutoAddedSites' % jobType, autoAddedSites))
gLogger.debug("Auto-added sites for %s task: %s" % (jobType, ','.join(autoAddedSites)))
# 5. add autoAddedSites, if requested
for autoAddedSite in autoAddedSites:
allowed.setdefault(autoAddedSite, set()).add(autoAddedSite)
gLogger.debug("Allowed sites for %s task: %s" % (jobType, ','.join(allowed)))
# 6. Allowing sites that should be allowed
taskSiteDestination = self._BySE()
for destSite, fromSites in allowed.iteritems():
for fromSite in fromSites:
if not taskSiteDestination or fromSite in taskSiteDestination:
destSites.add(destSite)
gLogger.debug("Computed list of destination sites for %s task with TargetSE %s: %s" % (jobType,
self.params['TargetSE'],
','.join(destSites)))
return destSites
def matchQueue(jobJDL, queueDict, fullMatch=False):
"""
Match the job description to the queue definition
:param str job: JDL job description
:param bool fullMatch: test matching on all the criteria
:param dict queueDict: queue parameters dictionary
:return: S_OK/S_ERROR, Value - result of matching, S_OK if matched or
S_ERROR with the reason for no match
"""
# Check the job description validity
job = ClassAd(jobJDL)
if not job.isOK():
return S_ERROR('Invalid job description')
noMatchReasons = []
# Check job requirements to resource
# 1. CPUTime
cpuTime = job.getAttributeInt('CPUTime')
if not cpuTime:
cpuTime = 84600
if cpuTime > queueDict.get('CPUTime', 0.):
noMatchReasons.append('Job CPUTime requirement not satisfied')
if not fullMatch:
return S_OK({'Match': False, 'Reason': noMatchReasons[0]})
# 2. Multi-value match requirements
for parameter in [
'Site', 'GridCE', 'Platform', 'GridMiddleware', 'PilotType',
'SubmitPool', 'JobType'
]:
if parameter in queueDict:
valueSet = set(job.getListFromExpression(parameter))
if not valueSet:
valueSet = set(job.getListFromExpression('%ss' % parameter))
queueSet = set(fromChar(queueDict[parameter]))
if valueSet and queueSet and not valueSet.intersection(queueSet):
valueToPrint = ','.join(valueSet)
if len(valueToPrint) > 20:
valueToPrint = "%s..." % valueToPrint[:20]
noMatchReasons.append('Job %s %s requirement not satisfied' %
(parameter, valueToPrint))
if not fullMatch:
return S_OK({'Match': False, 'Reason': noMatchReasons[0]})
# 3. Banned multi-value match requirements
for par in [
'Site', 'GridCE', 'Platform', 'GridMiddleware', 'PilotType',
'SubmitPool', 'JobType'
]:
parameter = "Banned%s" % par
if par in queueDict:
valueSet = set(job.getListFromExpression(parameter))
if not valueSet:
valueSet = set(job.getListFromExpression('%ss' % parameter))
queueSet = set(fromChar(queueDict[par]))
if valueSet and queueSet and valueSet.issubset(queueSet):
valueToPrint = ','.join(valueSet)
if len(valueToPrint) > 20:
valueToPrint = "%s..." % valueToPrint[:20]
noMatchReasons.append('Job %s %s requirement not satisfied' %
(parameter, valueToPrint))
if not fullMatch:
return S_OK({'Match': False, 'Reason': noMatchReasons[0]})
# 4. Tags
tags = set(job.getListFromExpression('Tag'))
nProc = job.getAttributeInt('NumberOfProcessors')
if nProc and nProc > 1:
tags.add('MultiProcessor')
wholeNode = job.getAttributeString('WholeNode')
if wholeNode:
tags.add('WholeNode')
queueTags = set(queueDict.get('Tags', []))
if not tags.issubset(queueTags):
noMatchReasons.append('Job Tag %s not satisfied' % ','.join(tags))
if not fullMatch:
return S_OK({'Match': False, 'Reason': noMatchReasons[0]})
# 4. MultiProcessor requirements
if nProc and nProc > int(queueDict.get('NumberOfProcessors', 1)):
noMatchReasons.append(
'Job NumberOfProcessors %d requirement not satisfied' % nProc)
if not fullMatch:
return S_OK({'Match': False, 'Reason': noMatchReasons[0]})
# 5. RAM
ram = job.getAttributeInt('RAM')
# If MaxRAM is not specified in the queue description, assume 2GB
if ram and ram > int(queueDict.get('MaxRAM', 2048)) / 1024:
noMatchReasons.append('Job RAM %d requirement not satisfied' % ram)
if not fullMatch:
return S_OK({'Match': False, 'Reason': noMatchReasons[0]})
# Check resource requirements to job
# 1. OwnerGroup - rare case but still
if "OwnerGroup" in queueDict:
result = getProxyInfo(disableVOMS=True)
if not result['OK']:
return S_ERROR('No valid proxy available')
ownerGroup = result['Value']['group']
if ownerGroup != queueDict['OwnerGroup']:
noMatchReasons.append(
'Resource OwnerGroup %s requirement not satisfied' %
queueDict['OwnerGroup'])
if not fullMatch:
return S_OK({'Match': False, 'Reason': noMatchReasons[0]})
# 2. Required tags
requiredTags = set(queueDict.get('RequiredTags', []))
if not requiredTags.issubset(tags):
noMatchReasons.append('Resource RequiredTags %s not satisfied' %
','.join(requiredTags))
if not fullMatch:
return S_OK({'Match': False, 'Reason': noMatchReasons[0]})
# 3. RunningLimit
site = queueDict['Site']
opsHelper = Operations()
result = opsHelper.getSections('JobScheduling/RunningLimit')
if result['OK'] and site in result['Value']:
result = opsHelper.getSections('JobScheduling/RunningLimit/%s' % site)
if result['OK']:
for parameter in result['Value']:
value = job.getAttributeString(parameter)
if value and opsHelper.getValue(
'JobScheduling/RunningLimit/%s/%s/%s' %
(site, parameter, value), 1) == 0:
noMatchReasons.append(
'Resource operational %s requirement not satisfied' %
parameter)
if not fullMatch:
return S_OK({
'Match': False,
'Reason': noMatchReasons[0]
})
return S_OK({'Match': not bool(noMatchReasons), 'Reason': noMatchReasons})
