def _submitPilots( self, workDir, taskQueueDict, pilotOptions, pilotsToSubmit,
ceMask, submitPrivatePilot, privateTQ, proxy, pilotsPerJob ):
"""
This method does the actual pilot submission to the Grid RB
The logic is as follows:
- If there are no available RB it return error
- If there is no VOMS extension in the proxy, return error
- It creates a temp directory
- Prepare a JDL
it has some part common to gLite and LCG (the payload description)
it has some part specific to each middleware
"""
taskQueueID = taskQueueDict['TaskQueueID']
# ownerDN = taskQueueDict['OwnerDN']
credDict = proxy.getCredentials()['Value']
ownerDN = credDict['identity']
ownerGroup = credDict[ 'group' ]
if not self.resourceBrokers:
# Since we can exclude RBs from the list, it may become empty
return S_ERROR( ERROR_RB )
# Need to get VOMS extension for the later interactions with WMS
ret = gProxyManager.getVOMSAttributes( proxy )
if not ret['OK']:
self.log.error( ERROR_VOMS, ret['Message'] )
return S_ERROR( ERROR_VOMS )
if not ret['Value']:
return S_ERROR( ERROR_VOMS )
vomsGroup = ret['Value'][0]
workingDirectory = tempfile.mkdtemp( prefix = 'TQ_%s_' % taskQueueID, dir = workDir )
self.log.verbose( 'Using working Directory:', workingDirectory )
# Write JDL
retDict = self._prepareJDL( taskQueueDict, workingDirectory, pilotOptions, pilotsPerJob,
ceMask, submitPrivatePilot, privateTQ )
jdl = retDict['JDL']
pilotRequirements = retDict['Requirements']
rb = retDict['RB']
if not jdl:
try:
shutil.rmtree( workingDirectory )
except:
pass
return S_ERROR( ERROR_JDL )
# Check that there are available queues for the Job:
if self.enableListMatch:
availableCEs = []
now = Time.dateTime()
availableCEs = self.listMatchCache.get( pilotRequirements )
if availableCEs == False:
availableCEs = self._listMatch( proxy, jdl, taskQueueID, rb )
if availableCEs != False:
self.log.verbose( 'LastListMatch', now )
self.log.verbose( 'AvailableCEs ', availableCEs )
self.listMatchCache.add( pilotRequirements, self.listMatchDelay * 60,
value = availableCEs ) # it is given in minutes
if not availableCEs:
try:
shutil.rmtree( workingDirectory )
except:
pass
return S_ERROR( ERROR_CE + ' TQ: %d' % taskQueueID )
# Now we are ready for the actual submission, so
self.log.verbose( 'Submitting Pilots for TaskQueue', taskQueueID )
submitRet = self._submitPilot( proxy, pilotsPerJob, jdl, taskQueueID, rb )
try:
shutil.rmtree( workingDirectory )
except:
pass
if not submitRet:
return S_ERROR( 'Pilot Submission Failed for TQ %d ' % taskQueueID )
# pilotReference, resourceBroker = submitRet
submittedPilots = 0
if pilotsPerJob != 1 and len( submitRet ) != pilotsPerJob:
# Parametric jobs are used
for pilotReference, resourceBroker in submitRet:
pilotReference = self._getChildrenReferences( proxy, pilotReference, taskQueueID )
submittedPilots += len( pilotReference )
pilotAgentsDB.addPilotTQReference( pilotReference, taskQueueID, ownerDN,
ownerGroup, resourceBroker, self.gridMiddleware,
pilotRequirements )
else:
for pilotReference, resourceBroker in submitRet:
pilotReference = [pilotReference]
submittedPilots += len( pilotReference )
pilotAgentsDB.addPilotTQReference( pilotReference, taskQueueID, ownerDN,
ownerGroup, resourceBroker, self.gridMiddleware, pilotRequirements )
# add some sleep here
time.sleep( 0.1 * submittedPilots )
if pilotsToSubmit > pilotsPerJob:
# Additional submissions are necessary, need to get a new token and iterate.
pilotsToSubmit -= pilotsPerJob
result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) )
if not result[ 'OK' ]:
self.log.error( ERROR_TOKEN, result['Message'] )
result = S_ERROR( ERROR_TOKEN )
result['Value'] = submittedPilots
return result
( token, numberOfUses ) = result[ 'Value' ]
for option in pilotOptions:
if option.find( '-o /Security/ProxyToken=' ) == 0:
pilotOptions.remove( option )
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
pilotsPerJob = max( 1, min( pilotsPerJob, int( numberOfUses / self.maxJobsInFillMode ) ) )
result = self._submitPilots( workDir, taskQueueDict, pilotOptions,
pilotsToSubmit, ceMask,
submitPrivatePilot, privateTQ,
proxy, pilotsPerJob )
if not result['OK']:
if 'Value' not in result:
result['Value'] = 0
result['Value'] += submittedPilots
return result
submittedPilots += result['Value']
return S_OK( submittedPilots )
def _getPilotOptions( self, queue, pilotsToSubmit ):
""" Prepare pilot options
"""
queueDict = self.queueDict[queue]['ParametersDict']
pilotOptions = []
setup = gConfig.getValue( "/DIRAC/Setup", "unknown" )
if setup == 'unknown':
self.log.error( 'Setup is not defined in the configuration' )
return [ None, None ]
pilotOptions.append( '-S %s' % setup )
opsHelper = Operations.Operations( group = self.pilotGroup, setup = setup )
#Installation defined?
installationName = opsHelper.getValue( "Pilot/Installation", "" )
if installationName:
pilotOptions.append( '-V %s' % installationName )
#Project defined?
projectName = opsHelper.getValue( "Pilot/Project", "" )
if projectName:
pilotOptions.append( '-l %s' % projectName )
else:
self.log.info( 'DIRAC project will be installed by pilots' )
#Request a release
diracVersion = opsHelper.getValue( "Pilot/Version", [] )
if not diracVersion:
self.log.error( 'Pilot/Version is not defined in the configuration' )
return [ None, None ]
# diracVersion is a list of accepted releases
pilotOptions.append( '-r %s' % ','.join( str( it ) for it in diracVersion ) )
ownerDN = self.pilotDN
ownerGroup = self.pilotGroup
# Request token for maximum pilot efficiency
result = gProxyManager.requestToken( ownerDN, ownerGroup, pilotsToSubmit * self.maxJobsInFillMode )
if not result[ 'OK' ]:
self.log.error( 'Invalid proxy token request', result['Message'] )
return [ None, None ]
( token, numberOfUses ) = result[ 'Value' ]
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
# Use Filling mode
pilotOptions.append( '-M %s' % min( numberOfUses, self.maxJobsInFillMode ) )
# Since each pilot will execute min( numberOfUses, self.maxJobsInFillMode )
# with numberOfUses tokens we can submit at most:
# numberOfUses / min( numberOfUses, self.maxJobsInFillMode )
# pilots
newPilotsToSubmit = numberOfUses / min( numberOfUses, self.maxJobsInFillMode )
if newPilotsToSubmit != pilotsToSubmit:
self.log.info( 'Number of pilots to submit is changed to %d after getting the proxy token' % newPilotsToSubmit )
pilotsToSubmit = newPilotsToSubmit
# Debug
if self.pilotLogLevel.lower() == 'debug':
pilotOptions.append( '-d' )
# CS Servers
csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions to be used in pilots
pilotExtensionsList = opsHelper.getValue( "Pilot/Extensions", [] )
extensionsList = []
if pilotExtensionsList:
if pilotExtensionsList[0] != 'None':
extensionsList = pilotExtensionsList
else:
extensionsList = CSGlobals.getCSExtensions()
if extensionsList:
pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
# Requested CPU time
pilotOptions.append( '-T %s' % queueDict['CPUTime'] )
# CEName
pilotOptions.append( '-N %s' % self.queueDict[queue]['CEName'] )
# Queue
pilotOptions.append( '-Q %s' % self.queueDict[queue]['QueueName'] )
# SiteName
pilotOptions.append( '-n %s' % queueDict['Site'] )
if 'ClientPlatform' in queueDict:
pilotOptions.append( "-p '%s'" % queueDict['ClientPlatform'] )
if 'SharedArea' in queueDict:
pilotOptions.append( "-o '/LocalSite/SharedArea=%s'" % queueDict['SharedArea'] )
# if 'SI00' in queueDict:
# factor = float( queueDict['SI00'] ) / 250.
# pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % factor )
# pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % factor )
# else:
# if 'CPUScalingFactor' in queueDict:
# pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % queueDict['CPUScalingFactor'] )
# if 'CPUNormalizationFactor' in queueDict:
# pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % queueDict['CPUNormalizationFactor'] )
if "ExtraPilotOptions" in queueDict:
pilotOptions.append( queueDict['ExtraPilotOptions'] )
# Hack
if self.defaultSubmitPools:
pilotOptions.append( '-o /Resources/Computing/CEDefaults/SubmitPool=%s' % self.defaultSubmitPools )
if "Tag" in queueDict:
tagString = ','.join( queueDict['Tag'] )
pilotOptions.append( '-o /Resources/Computing/CEDefaults/Tag=%s' % tagString )
if self.group:
pilotOptions.append( '-G %s' % self.group )
return [ pilotOptions, pilotsToSubmit ]
def __getPilotOptions( self, queue, pilotsToSubmit ):
""" Prepare pilot options
"""
queueDict = self.queueDict[queue]['ParametersDict']
pilotOptions = []
setup = gConfig.getValue( "/DIRAC/Setup", "unknown" )
if setup == 'unknown':
self.log.error( 'Setup is not defined in the configuration' )
return None
pilotOptions.append( '-S %s' % setup )
opsHelper = Operations.Operations( group = self.genericPilotGroup, setup = setup )
#Installation defined?
installationName = opsHelper.getValue( "Pilot/Installation", "" )
if installationName:
pilotOptions.append( '-V %s' % installationName )
#Project defined?
projectName = opsHelper.getValue( "Pilot/Project", "" )
if projectName:
pilotOptions.append( '-l %s' % projectName )
else:
self.log.info( 'DIRAC project will be installed by pilots' )
#Request a release
diracVersion = opsHelper.getValue( "Pilot/Version", [] )
#####borrar
diracVersion = "v6r4"
if not diracVersion:
self.log.error( 'Pilot/Version is not defined in the configuration' )
return None
#diracVersion is a list of accepted releases. Just take the first one
pilotOptions.append( '-r %s' % diracVersion )
ownerDN = self.genericPilotDN
ownerGroup = self.genericPilotGroup
result = gProxyManager.requestToken( ownerDN, ownerGroup, pilotsToSubmit * 5 )
if not result[ 'OK' ]:
self.log.error( ERROR_TOKEN, result['Message'] )
return S_ERROR( ERROR_TOKEN )
( token, numberOfUses ) = result[ 'Value' ]
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
# Use Filling mode
pilotOptions.append( '-M %s' % 5 )
# Debug
if self.pilotLogLevel.lower() == 'debug':
pilotOptions.append( '-d' )
# CS Servers
csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions
# extensionsList = CSGlobals.getCSExtensions()
# if extensionsList:
# pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
# Requested CPU time
pilotOptions.append( '-T %s' % queueDict['CPUTime'] )
# CEName
pilotOptions.append( '-N %s' % self.queueDict[queue]['CEName'] )
# SiteName
pilotOptions.append( '-n %s' % queueDict['Site'] )
if 'ClientPlatform' in queueDict:
pilotOptions.append( "-p '%s'" % queueDict['ClientPlatform'] )
if 'SharedArea' in queueDict:
pilotOptions.append( "-o '/LocalSite/SharedArea=%s'" % queueDict['SharedArea'] )
group = "lhcb_pilot"
if group:
pilotOptions.append( '-G %s' % group )
self.log.verbose( "pilotOptions: ", ' '.join( pilotOptions ) )
return pilotOptions
def _getPilotOptions( self, taskQueueDict, pilotsToSubmit ):
# Need to limit the maximum number of pilots to submit at once
# For generic pilots this is limited by the number of use of the tokens and the
# maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation:
pilotsToSubmit = min( pilotsToSubmit, int( 50 / self.maxJobsInFillMode ) )
pilotOptions = [ "-V %s" % self.virtualOrganization ]
privateIfGenericTQ = self.privatePilotFraction > random.random()
privateTQ = ( 'PilotTypes' in taskQueueDict and 'private' in [ t.lower() for t in taskQueueDict['PilotTypes'] ] )
forceGeneric = 'ForceGeneric' in taskQueueDict
submitPrivatePilot = ( privateIfGenericTQ or privateTQ ) and not forceGeneric
if submitPrivatePilot:
self.log.verbose( 'Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
ownerDN = taskQueueDict['OwnerDN']
ownerGroup = taskQueueDict['OwnerGroup']
# User Group requirement
pilotOptions.append( '-G %s' % taskQueueDict['OwnerGroup'] )
# check if group allows jobsharing
ownerGroupProperties = getPropertiesForGroup( ownerGroup )
if not 'JobSharing' in ownerGroupProperties:
# Add Owner requirement to pilot
pilotOptions.append( "-O '%s'" % ownerDN )
if privateTQ:
pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private' )
maxJobsInFillMode = self.maxJobsInFillMode
else:
#For generic jobs we'll submit mixture of generic and private pilots
self.log.verbose( 'Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
ownerDN = self.genericPilotDN
ownerGroup = self.genericPilotGroup
result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) )
if not result[ 'OK' ]:
self.log.error( ERROR_TOKEN, result['Message'] )
return S_ERROR( ERROR_TOKEN )
( token, numberOfUses ) = result[ 'Value' ]
pilotsToSubmit = min( numberOfUses, pilotsToSubmit )
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
pilotsToSubmit = ( pilotsToSubmit - 1 ) / self.maxJobsInFillMode + 1
maxJobsInFillMode = int( numberOfUses / pilotsToSubmit )
# Use Filling mode
pilotOptions.append( '-M %s' % maxJobsInFillMode )
# Debug
pilotOptions.append( '-d' )
# Setup.
pilotOptions.append( '-S %s' % taskQueueDict['Setup'] )
# CS Servers
csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
# Requested version of DIRAC
pilotOptions.append( '-r %s' % self.installVersion )
# Requested Project to install
pilotOptions.append( '-V %s' % self.installInstallation )
# Requested CPU time
pilotOptions.append( '-T %s' % taskQueueDict['CPUTime'] )
if self.extraPilotOptions:
pilotOptions.extend( self.extraPilotOptions )
return S_OK( ( pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ ) )
def _getPilotOptions( self, taskQueueDict, pilotsToSubmit ):
# Need to limit the maximum number of pilots to submit at once
# For generic pilots this is limited by the number of use of the tokens and the
# maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation:
pilotsToSubmit = max( min( pilotsToSubmit, int( 50 / self.maxJobsInFillMode ) ), 1 )
pilotOptions = []
privateIfGenericTQ = self.privatePilotFraction > random.random()
privateTQ = ( 'PilotTypes' in taskQueueDict and 'private' in [ t.lower() for t in taskQueueDict['PilotTypes'] ] )
forceGeneric = 'ForceGeneric' in taskQueueDict
submitPrivatePilot = ( privateIfGenericTQ or privateTQ ) and not forceGeneric
if submitPrivatePilot:
self.log.verbose( 'Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
ownerDN = taskQueueDict['OwnerDN']
ownerGroup = taskQueueDict['OwnerGroup']
# User Group requirement
pilotOptions.append( '-G %s' % taskQueueDict['OwnerGroup'] )
# check if group allows jobsharing
ownerGroupProperties = getPropertiesForGroup( ownerGroup )
if not 'JobSharing' in ownerGroupProperties:
# Add Owner requirement to pilot
pilotOptions.append( "-O '%s'" % ownerDN )
if privateTQ:
pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private' )
maxJobsInFillMode = self.maxJobsInFillMode
else:
#For generic jobs we'll submit mixture of generic and private pilots
self.log.verbose( 'Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
#ADRI: Find the generic group
result = findGenericPilotCredentials( group = taskQueueDict[ 'OwnerGroup' ] )
if not result[ 'OK' ]:
self.log.error( ERROR_GENERIC_CREDENTIALS, result[ 'Message' ] )
return S_ERROR( ERROR_GENERIC_CREDENTIALS )
ownerDN, ownerGroup = result[ 'Value' ]
result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) )
if not result[ 'OK' ]:
self.log.error( ERROR_TOKEN, result['Message'] )
return S_ERROR( ERROR_TOKEN )
( token, numberOfUses ) = result[ 'Value' ]
pilotsToSubmit = min( numberOfUses, pilotsToSubmit )
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
pilotsToSubmit = max( 1, ( pilotsToSubmit - 1 ) / self.maxJobsInFillMode + 1 )
maxJobsInFillMode = int( numberOfUses / pilotsToSubmit )
# Use Filling mode
pilotOptions.append( '-M %s' % maxJobsInFillMode )
# Debug
pilotOptions.append( '-d' )
# Setup.
pilotOptions.append( '-S %s' % taskQueueDict['Setup'] )
# CS Servers
csServers = gConfig.getServersList()
if len( csServers ) > 3:
# Remove the master
master = gConfigurationData.getMasterServer()
if master in csServers:
csServers.remove( master )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions to be used in pilots
# ubeda: I'm not entirely sure if we can use here the same opsHelper as in line
# line +352
pilotExtensionsList = Operations().getValue( "Pilot/Extensions", [] )
extensionsList = []
if pilotExtensionsList:
if pilotExtensionsList[0] != 'None':
extensionsList = pilotExtensionsList
else:
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
#Get DIRAC version and project, There might be global Setup defaults and per VO/Setup defaults (from configure)
opsHelper = Operations( group = taskQueueDict['OwnerGroup'], setup = taskQueueDict['Setup'] )
# Requested version of DIRAC (it can be a list, so we take the fist one)
version = opsHelper.getValue( cfgPath( 'Pilot', 'Version' ) , [ self.installVersion ] )[0]
pilotOptions.append( '-r %s' % version )
# Requested Project to install
installProject = opsHelper.getValue( cfgPath( 'Pilot', 'Project' ) , self.installProject )
if installProject:
pilotOptions.append( '-l %s' % installProject )
installation = opsHelper.getValue( cfgPath( 'Pilot', 'Installation' ), self.installation )
if installation:
pilotOptions.append( "-V %s" % installation )
# Requested CPU time
pilotOptions.append( '-T %s' % taskQueueDict['CPUTime'] )
if self.submitPoolOption not in self.extraPilotOptions:
pilotOptions.append( self.submitPoolOption )
if self.extraPilotOptions:
pilotOptions.extend( self.extraPilotOptions )
return S_OK( ( pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ ) )
def __getPilotOptions( self, queue, pilotsToSubmit ):
""" Prepare pilot options
"""
queueDict = self.queueDict[queue]['ParametersDict']
vo = getVO()
if not vo:
self.log.error( 'Virtual Organization is not defined in the configuration' )
return None
pilotOptions = [ "-V '%s'" % vo ]
setup = gConfig.getValue( "/DIRAC/Setup", "unknown" )
if setup == 'unknown':
self.log.error( 'Setup is not defined in the configuration' )
return None
pilotOptions.append( '-S %s' % setup )
diracVersion = gConfig.getValue( "/Operations/%s/%s/Versions/PilotVersion" % ( vo, setup ), "unknown" )
if diracVersion == 'unknown':
self.log.error( 'PilotVersion is not defined in the configuration' )
return None
pilotOptions.append( '-r %s' % diracVersion )
projectName = gConfig.getValue( "/Operations/%s/%s/Versions/PilotProject" % ( vo, setup ), "unknown" )
if projectName == 'unknown':
self.log.info( 'PilotProject is not defined in the configuration' )
else:
pilotOptions.append( '-l %s' % projectName )
ownerDN = self.genericPilotDN
ownerGroup = self.genericPilotGroup
result = gProxyManager.requestToken( ownerDN, ownerGroup, pilotsToSubmit * 5 )
if not result[ 'OK' ]:
self.log.error( ERROR_TOKEN, result['Message'] )
return S_ERROR( ERROR_TOKEN )
( token, numberOfUses ) = result[ 'Value' ]
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
# Use Filling mode
pilotOptions.append( '-M %s' % 5 )
# Debug
pilotOptions.append( '-d' )
# CS Servers
csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
# Requested CPU time
pilotOptions.append( '-T %s' % queueDict['CPUTime'] )
# SiteName
pilotOptions.append( '-n %s' % queueDict['Site'] )
if 'ClientPlatform' in queueDict:
pilotOptions.append( "-p '%s'" % queueDict['ClientPlatform'] )
if 'SharedArea' in queueDict:
pilotOptions.append( "-o '/LocalSite/SharedArea=%s'" % queueDict['SharedArea'] )
if 'SI00' in queueDict:
factor = float( queueDict['SI00'] ) / 250.
pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % factor )
pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % factor )
else:
if 'CPUScalingFactor' in queueDict:
pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % queueDict['CPUScalingFactor'] )
if 'CPUNormalizationFactor' in queueDict:
pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % queueDict['CPUNormalizationFactor'] )
self.log.verbose( "pilotOptions: ", ' '.join( pilotOptions ) )
return pilotOptions
def _getPilotOptions( self, taskQueueDict, pilotsToSubmit ):
# Need to limit the maximum number of pilots to submit at once
# For generic pilots this is limited by the number of use of the tokens and the
# maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation:
pilotsToSubmit = min( pilotsToSubmit, int( 50 / self.maxJobsInFillMode ) )
pilotOptions = []
privateIfGenericTQ = self.privatePilotFraction > random.random()
privateTQ = ( 'PilotTypes' in taskQueueDict and 'private' in [ t.lower() for t in taskQueueDict['PilotTypes'] ] )
forceGeneric = 'ForceGeneric' in taskQueueDict
submitPrivatePilot = ( privateIfGenericTQ or privateTQ ) and not forceGeneric
if submitPrivatePilot:
self.log.verbose( 'Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
ownerDN = taskQueueDict['OwnerDN']
ownerGroup = taskQueueDict['OwnerGroup']
# User Group requirement
pilotOptions.append( '-G %s' % taskQueueDict['OwnerGroup'] )
# check if group allows jobsharing
ownerGroupProperties = getPropertiesForGroup( ownerGroup )
if not 'JobSharing' in ownerGroupProperties:
# Add Owner requirement to pilot
pilotOptions.append( "-O '%s'" % ownerDN )
if privateTQ:
pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private' )
maxJobsInFillMode = self.maxJobsInFillMode
else:
#For generic jobs we'll submit mixture of generic and private pilots
self.log.verbose( 'Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
ownerDN = self.genericPilotDN
ownerGroup = self.genericPilotGroup
result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) )
if not result[ 'OK' ]:
self.log.error( ERROR_TOKEN, result['Message'] )
return S_ERROR( ERROR_TOKEN )
( token, numberOfUses ) = result[ 'Value' ]
pilotsToSubmit = min( numberOfUses, pilotsToSubmit )
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
pilotsToSubmit = ( pilotsToSubmit - 1 ) / self.maxJobsInFillMode + 1
maxJobsInFillMode = int( numberOfUses / pilotsToSubmit )
# Use Filling mode
pilotOptions.append( '-M %s' % maxJobsInFillMode )
# Debug
pilotOptions.append( '-d' )
# Setup.
pilotOptions.append( '-S %s' % taskQueueDict['Setup'] )
# CS Servers
csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
# Requested version of DIRAC
pilotOptions.append( '-r %s' % self.installVersion )
# Requested Project to install
if self.installInstallation:
pilotOptions.append( '-V %s' % self.installInstallation )
# Requested CPU time
pilotOptions.append( '-T %s' % taskQueueDict['CPUTime'] )
if self.extraPilotOptions:
pilotOptions.extend( self.extraPilotOptions )
return S_OK( ( pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ ) )
def _getPilotOptions(self, taskQueueDict, pilotsToSubmit):
# Need to limit the maximum number of pilots to submit at once
# For generic pilots this is limited by the number of use of the tokens and the
# maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation:
pilotsToSubmit = max(min(pilotsToSubmit, int(50 / self.maxJobsInFillMode)), 1)
pilotOptions = []
privateIfGenericTQ = self.privatePilotFraction > random.random()
privateTQ = "PilotTypes" in taskQueueDict and "private" in [t.lower() for t in taskQueueDict["PilotTypes"]]
forceGeneric = "ForceGeneric" in taskQueueDict
submitPrivatePilot = (privateIfGenericTQ or privateTQ) and not forceGeneric
if submitPrivatePilot:
self.log.verbose("Submitting private pilots for TaskQueue %s" % taskQueueDict["TaskQueueID"])
ownerDN = taskQueueDict["OwnerDN"]
ownerGroup = taskQueueDict["OwnerGroup"]
# User Group requirement
pilotOptions.append("-G %s" % taskQueueDict["OwnerGroup"])
# check if group allows jobsharing
ownerGroupProperties = getPropertiesForGroup(ownerGroup)
if not "JobSharing" in ownerGroupProperties:
# Add Owner requirement to pilot
pilotOptions.append("-O '%s'" % ownerDN)
if privateTQ:
pilotOptions.append("-o /Resources/Computing/CEDefaults/PilotType=private")
maxJobsInFillMode = self.maxJobsInFillMode
else:
# For generic jobs we'll submit mixture of generic and private pilots
self.log.verbose("Submitting generic pilots for TaskQueue %s" % taskQueueDict["TaskQueueID"])
# ADRI: Find the generic group
result = findGenericPilotCredentials(group=taskQueueDict["OwnerGroup"])
if not result["OK"]:
self.log.error(ERROR_GENERIC_CREDENTIALS, result["Message"])
return S_ERROR(ERROR_GENERIC_CREDENTIALS)
ownerDN, ownerGroup = result["Value"]
result = gProxyManager.requestToken(ownerDN, ownerGroup, max(pilotsToSubmit, self.maxJobsInFillMode))
if not result["OK"]:
self.log.error(ERROR_TOKEN, result["Message"])
return S_ERROR(ERROR_TOKEN)
(token, numberOfUses) = result["Value"]
pilotsToSubmit = min(numberOfUses, pilotsToSubmit)
pilotOptions.append("-o /Security/ProxyToken=%s" % token)
pilotsToSubmit = max(1, (pilotsToSubmit - 1) / self.maxJobsInFillMode + 1)
maxJobsInFillMode = int(numberOfUses / pilotsToSubmit)
# Use Filling mode
pilotOptions.append("-M %s" % maxJobsInFillMode)
# Debug
pilotOptions.append("-d")
# Setup.
pilotOptions.append("-S %s" % taskQueueDict["Setup"])
# CS Servers
csServers = gConfig.getServersList()
if len(csServers) > 3:
# Remove the master
master = gConfigurationData.getMasterServer()
if master in csServers:
csServers.remove(master)
pilotOptions.append("-C %s" % ",".join(csServers))
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append("-e %s" % ",".join(extensionsList))
# Get DIRAC version and project, There might be global Setup defaults and per VO/Setup defaults (from configure)
opsHelper = Operations(group=taskQueueDict["OwnerGroup"], setup=taskQueueDict["Setup"])
# Requested version of DIRAC (it can be a list, so we take the fist one)
version = opsHelper.getValue(cfgPath("Pilot", "Version"), [self.installVersion])[0]
pilotOptions.append("-r %s" % version)
# Requested Project to install
installProject = opsHelper.getValue(cfgPath("Pilot", "Project"), self.installProject)
if installProject:
pilotOptions.append("-l %s" % installProject)
installation = opsHelper.getValue(cfgPath("Pilot", "Installation"), self.installation)
if installation:
pilotOptions.append("-V %s" % installation)
# Requested CPU time
pilotOptions.append("-T %s" % taskQueueDict["CPUTime"])
if self.submitPoolOption not in self.extraPilotOptions:
pilotOptions.append(self.submitPoolOption)
if self.extraPilotOptions:
pilotOptions.extend(self.extraPilotOptions)
return S_OK((pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ))
def _getPilotOptions(self, taskQueueDict, pilotsToSubmit):
# Need to limit the maximum number of pilots to submit at once
# For generic pilots this is limited by the number of use of the tokens and the
# maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation:
pilotsToSubmit = max(
min(pilotsToSubmit, int(50 / self.maxJobsInFillMode)), 1)
pilotOptions = []
privateIfGenericTQ = self.privatePilotFraction > random.random()
privateTQ = ('PilotTypes' in taskQueueDict and 'private'
in [t.lower() for t in taskQueueDict['PilotTypes']])
forceGeneric = 'ForceGeneric' in taskQueueDict
submitPrivatePilot = (privateIfGenericTQ
or privateTQ) and not forceGeneric
if submitPrivatePilot:
self.log.verbose('Submitting private pilots for TaskQueue %s' %
taskQueueDict['TaskQueueID'])
ownerDN = taskQueueDict['OwnerDN']
ownerGroup = taskQueueDict['OwnerGroup']
# User Group requirement
pilotOptions.append('-G %s' % taskQueueDict['OwnerGroup'])
# check if group allows jobsharing
ownerGroupProperties = getPropertiesForGroup(ownerGroup)
if not 'JobSharing' in ownerGroupProperties:
# Add Owner requirement to pilot
pilotOptions.append("-O '%s'" % ownerDN)
if privateTQ:
pilotOptions.append(
'-o /Resources/Computing/CEDefaults/PilotType=private')
maxJobsInFillMode = self.maxJobsInFillMode
else:
#For generic jobs we'll submit mixture of generic and private pilots
self.log.verbose('Submitting generic pilots for TaskQueue %s' %
taskQueueDict['TaskQueueID'])
#ADRI: Find the generic group
result = findGenericPilotCredentials(
group=taskQueueDict['OwnerGroup'])
if not result['OK']:
self.log.error(ERROR_GENERIC_CREDENTIALS, result['Message'])
return S_ERROR(ERROR_GENERIC_CREDENTIALS)
ownerDN, ownerGroup = result['Value']
result = gProxyManager.requestToken(
ownerDN, ownerGroup, max(pilotsToSubmit,
self.maxJobsInFillMode))
if not result['OK']:
self.log.error(ERROR_TOKEN, result['Message'])
return S_ERROR(ERROR_TOKEN)
(token, numberOfUses) = result['Value']
pilotsToSubmit = min(numberOfUses, pilotsToSubmit)
pilotOptions.append('-o /Security/ProxyToken=%s' % token)
pilotsToSubmit = max(
1, (pilotsToSubmit - 1) / self.maxJobsInFillMode + 1)
maxJobsInFillMode = int(numberOfUses / pilotsToSubmit)
# Use Filling mode
pilotOptions.append('-M %s' % maxJobsInFillMode)
# Debug
pilotOptions.append('-d')
# Setup.
pilotOptions.append('-S %s' % taskQueueDict['Setup'])
# CS Servers
csServers = gConfig.getServersList()
if len(csServers) > 3:
# Remove the master
master = gConfigurationData.getMasterServer()
if master in csServers:
csServers.remove(master)
pilotOptions.append('-C %s' % ",".join(csServers))
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append('-e %s' % ",".join(extensionsList))
#Get DIRAC version and project, There might be global Setup defaults and per VO/Setup defaults (from configure)
opsHelper = Operations(group=taskQueueDict['OwnerGroup'],
setup=taskQueueDict['Setup'])
# Requested version of DIRAC (it can be a list, so we take the fist one)
version = opsHelper.getValue(cfgPath('Pilot', 'Version'),
[self.installVersion])[0]
pilotOptions.append('-r %s' % version)
# Requested Project to install
installProject = opsHelper.getValue(cfgPath('Pilot', 'Project'),
self.installProject)
if installProject:
pilotOptions.append('-l %s' % installProject)
installation = opsHelper.getValue(cfgPath('Pilot', 'Installation'),
self.installation)
if installation:
pilotOptions.append("-V %s" % installation)
# Requested CPU time
pilotOptions.append('-T %s' % taskQueueDict['CPUTime'])
if self.submitPoolOption not in self.extraPilotOptions:
pilotOptions.append(self.submitPoolOption)
if self.extraPilotOptions:
pilotOptions.extend(self.extraPilotOptions)
return S_OK((pilotOptions, pilotsToSubmit, ownerDN, ownerGroup,
submitPrivatePilot, privateTQ))
def _submitPilots(self, workDir, taskQueueDict, pilotOptions,
pilotsToSubmit, ceMask, submitPrivatePilot, privateTQ,
proxy, pilotsPerJob):
"""
This method does the actual pilot submission to the Grid RB
The logic is as follows:
- If there are no available RB it return error
- If there is no VOMS extension in the proxy, return error
- It creates a temp directory
- Prepare a JDL
it has some part common to gLite and LCG (the payload description)
it has some part specific to each middleware
"""
taskQueueID = taskQueueDict['TaskQueueID']
# ownerDN = taskQueueDict['OwnerDN']
ownerDN = proxy.getCredentials()['Value']['identity']
if not self.resourceBrokers:
# Since we can exclude RBs from the list, it may become empty
return S_ERROR(ERROR_RB)
# Need to get VOMS extension for the later interactions with WMS
ret = gProxyManager.getVOMSAttributes(proxy)
if not ret['OK']:
self.log.error(ERROR_VOMS, ret['Message'])
return S_ERROR(ERROR_VOMS)
if not ret['Value']:
return S_ERROR(ERROR_VOMS)
vomsGroup = ret['Value'][0]
workingDirectory = tempfile.mkdtemp(prefix='TQ_%s_' % taskQueueID,
dir=workDir)
self.log.verbose('Using working Directory:', workingDirectory)
# Write JDL
retDict = self._prepareJDL(taskQueueDict, workingDirectory,
pilotOptions, pilotsPerJob, ceMask,
submitPrivatePilot, privateTQ)
jdl = retDict['JDL']
pilotRequirements = retDict['Requirements']
rb = retDict['RB']
if not jdl:
try:
shutil.rmtree(workingDirectory)
except:
pass
return S_ERROR(ERROR_JDL)
# Check that there are available queues for the Job:
if self.enableListMatch:
availableCEs = []
now = Time.dateTime()
availableCEs = self.listMatchCache.get(pilotRequirements)
if availableCEs == False:
availableCEs = self._listMatch(proxy, jdl, taskQueueID, rb)
if availableCEs != False:
self.log.verbose('LastListMatch', now)
self.log.verbose('AvailableCEs ', availableCEs)
self.listMatchCache.add(
pilotRequirements,
self.listMatchDelay * 60,
value=availableCEs) # it is given in minutes
if not availableCEs:
try:
shutil.rmtree(workingDirectory)
except:
pass
return S_ERROR(ERROR_CE + ' TQ: %d' % taskQueueID)
# Now we are ready for the actual submission, so
self.log.verbose('Submitting Pilots for TaskQueue', taskQueueID)
submitRet = self._submitPilot(proxy, pilotsPerJob, jdl, taskQueueID,
rb)
try:
shutil.rmtree(workingDirectory)
except:
pass
if not submitRet:
return S_ERROR('Pilot Submission Failed for TQ %d ' % taskQueueID)
# pilotReference, resourceBroker = submitRet
submittedPilots = 0
if pilotsPerJob != 1 and len(submitRet) != pilotsPerJob:
# Parametric jobs are used
for pilotReference, resourceBroker in submitRet:
pilotReference = self._getChildrenReferences(
proxy, pilotReference, taskQueueID)
submittedPilots += len(pilotReference)
pilotAgentsDB.addPilotTQReference(pilotReference, taskQueueID,
ownerDN, vomsGroup,
resourceBroker,
self.gridMiddleware,
pilotRequirements)
else:
for pilotReference, resourceBroker in submitRet:
pilotReference = [pilotReference]
submittedPilots += len(pilotReference)
pilotAgentsDB.addPilotTQReference(
pilotReference,
taskQueueID,
ownerDN,
vomsGroup,
broker=resourceBroker,
gridType=self.gridMiddleware,
requirements=pilotRequirements)
# add some sleep here
time.sleep(0.1 * submittedPilots)
if pilotsToSubmit > pilotsPerJob:
# Additional submissions are necessary, need to get a new token and iterate.
pilotsToSubmit -= pilotsPerJob
ownerDN = self.genericPilotDN
ownerGroup = self.genericPilotGroup
result = gProxyManager.requestToken(
ownerDN, ownerGroup, max(pilotsToSubmit,
self.maxJobsInFillMode))
if not result['OK']:
self.log.error(ERROR_TOKEN, result['Message'])
return S_ERROR(ERROR_TOKEN)
(token, numberOfUses) = result['Value']
for option in pilotOptions:
if option.find('-o /Security/ProxyToken=') == 0:
pilotOptions.remove(option)
pilotOptions.append('-o /Security/ProxyToken=%s' % token)
pilotsPerJob = min(pilotsPerJob,
int(numberOfUses / self.maxJobsInFillMode))
result = self._submitPilots(workDir, taskQueueDict, pilotOptions,
pilotsToSubmit, ceMask,
submitPrivatePilot, privateTQ, proxy,
pilotsPerJob)
if not result['OK']:
result['Value'] = submittedPilots
return result
submittedPilots += result['Value']
return S_OK(submittedPilots)
def _getPilotOptions( self, queue, pilotsToSubmit, processors = 1 ):
""" Prepare pilot options
"""
queueDict = self.queueDict[queue]['ParametersDict']
pilotOptions = []
setup = gConfig.getValue( "/DIRAC/Setup", "unknown" )
if setup == 'unknown':
self.log.error( 'Setup is not defined in the configuration' )
return [ None, None ]
pilotOptions.append( '-S %s' % setup )
opsHelper = Operations.Operations( group = self.pilotGroup, setup = setup )
#Installation defined?
installationName = opsHelper.getValue( "Pilot/Installation", "" )
if installationName:
pilotOptions.append( '-V %s' % installationName )
#Project defined?
projectName = opsHelper.getValue( "Pilot/Project", "" )
if projectName:
pilotOptions.append( '-l %s' % projectName )
else:
self.log.info( 'DIRAC project will be installed by pilots' )
#Request a release
diracVersion = opsHelper.getValue( "Pilot/Version", [] )
if not diracVersion:
self.log.error( 'Pilot/Version is not defined in the configuration' )
return [ None, None ]
# diracVersion is a list of accepted releases
pilotOptions.append( '-r %s' % ','.join( str( it ) for it in diracVersion ) )
ownerDN = self.pilotDN
ownerGroup = self.pilotGroup
# Request token for maximum pilot efficiency
result = gProxyManager.requestToken( ownerDN, ownerGroup, pilotsToSubmit * self.maxJobsInFillMode )
if not result[ 'OK' ]:
self.log.error( 'Invalid proxy token request', result['Message'] )
return [ None, None ]
( token, numberOfUses ) = result[ 'Value' ]
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
# Use Filling mode
pilotOptions.append( '-M %s' % min( numberOfUses, self.maxJobsInFillMode ) )
# Since each pilot will execute min( numberOfUses, self.maxJobsInFillMode )
# with numberOfUses tokens we can submit at most:
# numberOfUses / min( numberOfUses, self.maxJobsInFillMode )
# pilots
newPilotsToSubmit = numberOfUses / min( numberOfUses, self.maxJobsInFillMode )
if newPilotsToSubmit != pilotsToSubmit:
self.log.info( 'Number of pilots to submit is changed to %d after getting the proxy token' % newPilotsToSubmit )
pilotsToSubmit = newPilotsToSubmit
# Debug
if self.pilotLogLevel.lower() == 'debug':
pilotOptions.append( '-ddd' )
# CS Servers
csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions to be used in pilots
pilotExtensionsList = opsHelper.getValue( "Pilot/Extensions", [] )
extensionsList = []
if pilotExtensionsList:
if pilotExtensionsList[0] != 'None':
extensionsList = pilotExtensionsList
else:
extensionsList = CSGlobals.getCSExtensions()
if extensionsList:
pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
# Requested CPU time
pilotOptions.append( '-T %s' % queueDict['CPUTime'] )
# CEName
pilotOptions.append( '-N %s' % self.queueDict[queue]['CEName'] )
# Queue
pilotOptions.append( '-Q %s' % self.queueDict[queue]['QueueName'] )
# SiteName
pilotOptions.append( '-n %s' % queueDict['Site'] )
if 'ClientPlatform' in queueDict:
pilotOptions.append( "-p '%s'" % queueDict['ClientPlatform'] )
if 'SharedArea' in queueDict:
pilotOptions.append( "-o '/LocalSite/SharedArea=%s'" % queueDict['SharedArea'] )
# if 'SI00' in queueDict:
# factor = float( queueDict['SI00'] ) / 250.
# pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % factor )
# pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % factor )
# else:
# if 'CPUScalingFactor' in queueDict:
# pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % queueDict['CPUScalingFactor'] )
# if 'CPUNormalizationFactor' in queueDict:
# pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % queueDict['CPUNormalizationFactor'] )
if "ExtraPilotOptions" in queueDict:
pilotOptions.append( queueDict['ExtraPilotOptions'] )
# Hack
if self.defaultSubmitPools:
pilotOptions.append( '-o /Resources/Computing/CEDefaults/SubmitPool=%s' % self.defaultSubmitPools )
if processors != 1:
if processors > 1:
pilotOptions.append( '-o /AgentJobRequirements/RequiredTag=%sProcessors' % processors )
pilotOptions.append( '-o /Resources/Computing/CEDefaults/Tag=%sProcessors' % processors )
else:
pilotOptions.append( '-o /AgentJobRequirements/RequiredTag=WholeNode' )
pilotOptions.append( '-o /Resources/Computing/CEDefaults/Tag=WholeNode' )
if self.group:
pilotOptions.append( '-G %s' % self.group )
return [ pilotOptions, pilotsToSubmit ]
def __getPilotOptions(self, queue, pilotsToSubmit):
""" Prepare pilot options
"""
queueDict = self.queueDict[queue]['ParametersDict']
vo = getVO()
if not vo:
self.log.error(
'Virtual Organization is not defined in the configuration')
return None
pilotOptions = ["-V '%s'" % vo]
setup = gConfig.getValue("/DIRAC/Setup", "unknown")
if setup == 'unknown':
self.log.error('Setup is not defined in the configuration')
return None
pilotOptions.append('-S %s' % setup)
diracVersion = gConfig.getValue(
"/Operations/%s/%s/Versions/PilotVersion" % (vo, setup), "unknown")
if diracVersion == 'unknown':
self.log.error('PilotVersion is not defined in the configuration')
return None
pilotOptions.append('-r %s' % diracVersion)
projectName = gConfig.getValue(
"/Operations/%s/%s/Versions/PilotProject" % (vo, setup), "unknown")
if projectName == 'unknown':
self.log.info('PilotProject is not defined in the configuration')
else:
pilotOptions.append('-l %s' % projectName)
ownerDN = self.genericPilotDN
ownerGroup = self.genericPilotGroup
result = gProxyManager.requestToken(ownerDN, ownerGroup,
pilotsToSubmit * 5)
if not result['OK']:
self.log.error(ERROR_TOKEN, result['Message'])
return S_ERROR(ERROR_TOKEN)
(token, numberOfUses) = result['Value']
pilotOptions.append('-o /Security/ProxyToken=%s' % token)
# Use Filling mode
pilotOptions.append('-M %s' % 5)
# Debug
pilotOptions.append('-d')
# CS Servers
csServers = gConfig.getValue("/DIRAC/Configuration/Servers", [])
pilotOptions.append('-C %s' % ",".join(csServers))
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append('-e %s' % ",".join(extensionsList))
# Requested CPU time
pilotOptions.append('-T %s' % queueDict['CPUTime'])
# SiteName
pilotOptions.append('-n %s' % queueDict['Site'])
if 'ClientPlatform' in queueDict:
pilotOptions.append("-p '%s'" % queueDict['ClientPlatform'])
if 'SharedArea' in queueDict:
pilotOptions.append("-o '/LocalSite/SharedArea=%s'" %
queueDict['SharedArea'])
if 'SI00' in queueDict:
factor = float(queueDict['SI00']) / 250.
pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" % factor)
pilotOptions.append("-o '/LocalSite/CPUNormalizationFactor=%s'" %
factor)
else:
if 'CPUScalingFactor' in queueDict:
pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" %
queueDict['CPUScalingFactor'])
if 'CPUNormalizationFactor' in queueDict:
pilotOptions.append(
"-o '/LocalSite/CPUNormalizationFactor=%s'" %
queueDict['CPUNormalizationFactor'])
self.log.verbose("pilotOptions: ", ' '.join(pilotOptions))
return pilotOptions
def __getPilotOptions(self, queue, pilotsToSubmit):
""" Prepare pilot options
"""
queueDict = self.queueDict[queue]["ParametersDict"]
pilotOptions = []
setup = gConfig.getValue("/DIRAC/Setup", "unknown")
if setup == "unknown":
self.log.error("Setup is not defined in the configuration")
return [None, None]
pilotOptions.append("-S %s" % setup)
opsHelper = Operations.Operations(group=self.pilotGroup, setup=setup)
# Installation defined?
installationName = opsHelper.getValue("Pilot/Installation", "")
if installationName:
pilotOptions.append("-V %s" % installationName)
# Project defined?
projectName = opsHelper.getValue("Pilot/Project", "")
if projectName:
pilotOptions.append("-l %s" % projectName)
else:
self.log.info("DIRAC project will be installed by pilots")
# Request a release
diracVersion = opsHelper.getValue("Pilot/Version", [])
if not diracVersion:
self.log.error("Pilot/Version is not defined in the configuration")
return [None, None]
# diracVersion is a list of accepted releases. Just take the first one
pilotOptions.append("-r %s" % diracVersion[0])
ownerDN = self.pilotDN
ownerGroup = self.pilotGroup
# Request token for maximum pilot efficiency
result = gProxyManager.requestToken(ownerDN, ownerGroup, pilotsToSubmit * self.maxJobsInFillMode)
if not result["OK"]:
self.log.error("Invalid proxy token request", result["Message"])
return [None, None]
(token, numberOfUses) = result["Value"]
pilotOptions.append("-o /Security/ProxyToken=%s" % token)
# Use Filling mode
pilotOptions.append("-M %s" % min(numberOfUses, self.maxJobsInFillMode))
# Since each pilot will execute min( numberOfUses, self.maxJobsInFillMode )
# with numberOfUses tokens we can submit at most:
# numberOfUses / min( numberOfUses, self.maxJobsInFillMode )
# pilots
newPilotsToSubmit = numberOfUses / min(numberOfUses, self.maxJobsInFillMode)
if newPilotsToSubmit != pilotsToSubmit:
self.log.info(
"Number of pilots to submit is changed to %d after getting the proxy token" % newPilotsToSubmit
)
pilotsToSubmit = newPilotsToSubmit
# Debug
if self.pilotLogLevel.lower() == "debug":
pilotOptions.append("-d")
# CS Servers
csServers = gConfig.getValue("/DIRAC/Configuration/Servers", [])
pilotOptions.append("-C %s" % ",".join(csServers))
# DIRAC Extensions to be used in pilots
pilotExtensionsList = opsHelper.getValue("Pilot/Extensions", [])
extensionsList = []
if pilotExtensionsList:
if pilotExtensionsList[0] != "None":
extensionsList = pilotExtensionsList
else:
extensionsList = CSGlobals.getCSExtensions()
if extensionsList:
pilotOptions.append("-e %s" % ",".join(extensionsList))
# Requested CPU time
pilotOptions.append("-T %s" % queueDict["CPUTime"])
# CEName
pilotOptions.append("-N %s" % self.queueDict[queue]["CEName"])
# SiteName
pilotOptions.append("-n %s" % queueDict["Site"])
if "ClientPlatform" in queueDict:
pilotOptions.append("-p '%s'" % queueDict["ClientPlatform"])
if "SharedArea" in queueDict:
pilotOptions.append("-o '/LocalSite/SharedArea=%s'" % queueDict["SharedArea"])
if "SI00" in queueDict:
factor = float(queueDict["SI00"]) / 250.0
pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" % factor)
pilotOptions.append("-o '/LocalSite/CPUNormalizationFactor=%s'" % factor)
else:
if "CPUScalingFactor" in queueDict:
pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" % queueDict["CPUScalingFactor"])
if "CPUNormalizationFactor" in queueDict:
pilotOptions.append("-o '/LocalSite/CPUNormalizationFactor=%s'" % queueDict["CPUNormalizationFactor"])
# Hack
if self.defaultSubmitPools:
pilotOptions.append("-o /Resources/Computing/CEDefaults/SubmitPool=%s" % self.defaultSubmitPools)
if self.group:
pilotOptions.append("-G %s" % self.group)
self.log.verbose("pilotOptions: ", " ".join(pilotOptions))
return [pilotOptions, pilotsToSubmit]