def _submitPilots( self, workDir, taskQueueDict, pilotOptions, pilotsToSubmit, ceMask, submitPrivatePilot, privateTQ, proxy, pilotsPerJob ): """ This method does the actual pilot submission to the Grid RB The logic is as follows: - If there are no available RB it return error - If there is no VOMS extension in the proxy, return error - It creates a temp directory - Prepare a JDL it has some part common to gLite and LCG (the payload description) it has some part specific to each middleware """ taskQueueID = taskQueueDict['TaskQueueID'] # ownerDN = taskQueueDict['OwnerDN'] credDict = proxy.getCredentials()['Value'] ownerDN = credDict['identity'] ownerGroup = credDict[ 'group' ] if not self.resourceBrokers: # Since we can exclude RBs from the list, it may become empty return S_ERROR( ERROR_RB ) # Need to get VOMS extension for the later interactions with WMS ret = gProxyManager.getVOMSAttributes( proxy ) if not ret['OK']: self.log.error( ERROR_VOMS, ret['Message'] ) return S_ERROR( ERROR_VOMS ) if not ret['Value']: return S_ERROR( ERROR_VOMS ) vomsGroup = ret['Value'][0] workingDirectory = tempfile.mkdtemp( prefix = 'TQ_%s_' % taskQueueID, dir = workDir ) self.log.verbose( 'Using working Directory:', workingDirectory ) # Write JDL retDict = self._prepareJDL( taskQueueDict, workingDirectory, pilotOptions, pilotsPerJob, ceMask, submitPrivatePilot, privateTQ ) jdl = retDict['JDL'] pilotRequirements = retDict['Requirements'] rb = retDict['RB'] if not jdl: try: shutil.rmtree( workingDirectory ) except: pass return S_ERROR( ERROR_JDL ) # Check that there are available queues for the Job: if self.enableListMatch: availableCEs = [] now = Time.dateTime() availableCEs = self.listMatchCache.get( pilotRequirements ) if availableCEs == False: availableCEs = self._listMatch( proxy, jdl, taskQueueID, rb ) if availableCEs != False: self.log.verbose( 'LastListMatch', now ) self.log.verbose( 'AvailableCEs ', availableCEs ) self.listMatchCache.add( pilotRequirements, self.listMatchDelay * 60, value = availableCEs ) # it is given in minutes if not availableCEs: try: shutil.rmtree( workingDirectory ) except: pass return S_ERROR( ERROR_CE + ' TQ: %d' % taskQueueID ) # Now we are ready for the actual submission, so self.log.verbose( 'Submitting Pilots for TaskQueue', taskQueueID ) submitRet = self._submitPilot( proxy, pilotsPerJob, jdl, taskQueueID, rb ) try: shutil.rmtree( workingDirectory ) except: pass if not submitRet: return S_ERROR( 'Pilot Submission Failed for TQ %d ' % taskQueueID ) # pilotReference, resourceBroker = submitRet submittedPilots = 0 if pilotsPerJob != 1 and len( submitRet ) != pilotsPerJob: # Parametric jobs are used for pilotReference, resourceBroker in submitRet: pilotReference = self._getChildrenReferences( proxy, pilotReference, taskQueueID ) submittedPilots += len( pilotReference ) pilotAgentsDB.addPilotTQReference( pilotReference, taskQueueID, ownerDN, ownerGroup, resourceBroker, self.gridMiddleware, pilotRequirements ) else: for pilotReference, resourceBroker in submitRet: pilotReference = [pilotReference] submittedPilots += len( pilotReference ) pilotAgentsDB.addPilotTQReference( pilotReference, taskQueueID, ownerDN, ownerGroup, resourceBroker, self.gridMiddleware, pilotRequirements ) # add some sleep here time.sleep( 0.1 * submittedPilots ) if pilotsToSubmit > pilotsPerJob: # Additional submissions are necessary, need to get a new token and iterate. pilotsToSubmit -= pilotsPerJob result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) ) if not result[ 'OK' ]: self.log.error( ERROR_TOKEN, result['Message'] ) result = S_ERROR( ERROR_TOKEN ) result['Value'] = submittedPilots return result ( token, numberOfUses ) = result[ 'Value' ] for option in pilotOptions: if option.find( '-o /Security/ProxyToken=' ) == 0: pilotOptions.remove( option ) pilotOptions.append( '-o /Security/ProxyToken=%s' % token ) pilotsPerJob = max( 1, min( pilotsPerJob, int( numberOfUses / self.maxJobsInFillMode ) ) ) result = self._submitPilots( workDir, taskQueueDict, pilotOptions, pilotsToSubmit, ceMask, submitPrivatePilot, privateTQ, proxy, pilotsPerJob ) if not result['OK']: if 'Value' not in result: result['Value'] = 0 result['Value'] += submittedPilots return result submittedPilots += result['Value'] return S_OK( submittedPilots )
def _getPilotOptions( self, queue, pilotsToSubmit ): """ Prepare pilot options """ queueDict = self.queueDict[queue]['ParametersDict'] pilotOptions = [] setup = gConfig.getValue( "/DIRAC/Setup", "unknown" ) if setup == 'unknown': self.log.error( 'Setup is not defined in the configuration' ) return [ None, None ] pilotOptions.append( '-S %s' % setup ) opsHelper = Operations.Operations( group = self.pilotGroup, setup = setup ) #Installation defined? installationName = opsHelper.getValue( "Pilot/Installation", "" ) if installationName: pilotOptions.append( '-V %s' % installationName ) #Project defined? projectName = opsHelper.getValue( "Pilot/Project", "" ) if projectName: pilotOptions.append( '-l %s' % projectName ) else: self.log.info( 'DIRAC project will be installed by pilots' ) #Request a release diracVersion = opsHelper.getValue( "Pilot/Version", [] ) if not diracVersion: self.log.error( 'Pilot/Version is not defined in the configuration' ) return [ None, None ] # diracVersion is a list of accepted releases pilotOptions.append( '-r %s' % ','.join( str( it ) for it in diracVersion ) ) ownerDN = self.pilotDN ownerGroup = self.pilotGroup # Request token for maximum pilot efficiency result = gProxyManager.requestToken( ownerDN, ownerGroup, pilotsToSubmit * self.maxJobsInFillMode ) if not result[ 'OK' ]: self.log.error( 'Invalid proxy token request', result['Message'] ) return [ None, None ] ( token, numberOfUses ) = result[ 'Value' ] pilotOptions.append( '-o /Security/ProxyToken=%s' % token ) # Use Filling mode pilotOptions.append( '-M %s' % min( numberOfUses, self.maxJobsInFillMode ) ) # Since each pilot will execute min( numberOfUses, self.maxJobsInFillMode ) # with numberOfUses tokens we can submit at most: # numberOfUses / min( numberOfUses, self.maxJobsInFillMode ) # pilots newPilotsToSubmit = numberOfUses / min( numberOfUses, self.maxJobsInFillMode ) if newPilotsToSubmit != pilotsToSubmit: self.log.info( 'Number of pilots to submit is changed to %d after getting the proxy token' % newPilotsToSubmit ) pilotsToSubmit = newPilotsToSubmit # Debug if self.pilotLogLevel.lower() == 'debug': pilotOptions.append( '-d' ) # CS Servers csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] ) pilotOptions.append( '-C %s' % ",".join( csServers ) ) # DIRAC Extensions to be used in pilots pilotExtensionsList = opsHelper.getValue( "Pilot/Extensions", [] ) extensionsList = [] if pilotExtensionsList: if pilotExtensionsList[0] != 'None': extensionsList = pilotExtensionsList else: extensionsList = CSGlobals.getCSExtensions() if extensionsList: pilotOptions.append( '-e %s' % ",".join( extensionsList ) ) # Requested CPU time pilotOptions.append( '-T %s' % queueDict['CPUTime'] ) # CEName pilotOptions.append( '-N %s' % self.queueDict[queue]['CEName'] ) # Queue pilotOptions.append( '-Q %s' % self.queueDict[queue]['QueueName'] ) # SiteName pilotOptions.append( '-n %s' % queueDict['Site'] ) if 'ClientPlatform' in queueDict: pilotOptions.append( "-p '%s'" % queueDict['ClientPlatform'] ) if 'SharedArea' in queueDict: pilotOptions.append( "-o '/LocalSite/SharedArea=%s'" % queueDict['SharedArea'] ) # if 'SI00' in queueDict: # factor = float( queueDict['SI00'] ) / 250. # pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % factor ) # pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % factor ) # else: # if 'CPUScalingFactor' in queueDict: # pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % queueDict['CPUScalingFactor'] ) # if 'CPUNormalizationFactor' in queueDict: # pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % queueDict['CPUNormalizationFactor'] ) if "ExtraPilotOptions" in queueDict: pilotOptions.append( queueDict['ExtraPilotOptions'] ) # Hack if self.defaultSubmitPools: pilotOptions.append( '-o /Resources/Computing/CEDefaults/SubmitPool=%s' % self.defaultSubmitPools ) if "Tag" in queueDict: tagString = ','.join( queueDict['Tag'] ) pilotOptions.append( '-o /Resources/Computing/CEDefaults/Tag=%s' % tagString ) if self.group: pilotOptions.append( '-G %s' % self.group ) return [ pilotOptions, pilotsToSubmit ]
def __getPilotOptions( self, queue, pilotsToSubmit ): """ Prepare pilot options """ queueDict = self.queueDict[queue]['ParametersDict'] pilotOptions = [] setup = gConfig.getValue( "/DIRAC/Setup", "unknown" ) if setup == 'unknown': self.log.error( 'Setup is not defined in the configuration' ) return None pilotOptions.append( '-S %s' % setup ) opsHelper = Operations.Operations( group = self.genericPilotGroup, setup = setup ) #Installation defined? installationName = opsHelper.getValue( "Pilot/Installation", "" ) if installationName: pilotOptions.append( '-V %s' % installationName ) #Project defined? projectName = opsHelper.getValue( "Pilot/Project", "" ) if projectName: pilotOptions.append( '-l %s' % projectName ) else: self.log.info( 'DIRAC project will be installed by pilots' ) #Request a release diracVersion = opsHelper.getValue( "Pilot/Version", [] ) #####borrar diracVersion = "v6r4" if not diracVersion: self.log.error( 'Pilot/Version is not defined in the configuration' ) return None #diracVersion is a list of accepted releases. Just take the first one pilotOptions.append( '-r %s' % diracVersion ) ownerDN = self.genericPilotDN ownerGroup = self.genericPilotGroup result = gProxyManager.requestToken( ownerDN, ownerGroup, pilotsToSubmit * 5 ) if not result[ 'OK' ]: self.log.error( ERROR_TOKEN, result['Message'] ) return S_ERROR( ERROR_TOKEN ) ( token, numberOfUses ) = result[ 'Value' ] pilotOptions.append( '-o /Security/ProxyToken=%s' % token ) # Use Filling mode pilotOptions.append( '-M %s' % 5 ) # Debug if self.pilotLogLevel.lower() == 'debug': pilotOptions.append( '-d' ) # CS Servers csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] ) pilotOptions.append( '-C %s' % ",".join( csServers ) ) # DIRAC Extensions # extensionsList = CSGlobals.getCSExtensions() # if extensionsList: # pilotOptions.append( '-e %s' % ",".join( extensionsList ) ) # Requested CPU time pilotOptions.append( '-T %s' % queueDict['CPUTime'] ) # CEName pilotOptions.append( '-N %s' % self.queueDict[queue]['CEName'] ) # SiteName pilotOptions.append( '-n %s' % queueDict['Site'] ) if 'ClientPlatform' in queueDict: pilotOptions.append( "-p '%s'" % queueDict['ClientPlatform'] ) if 'SharedArea' in queueDict: pilotOptions.append( "-o '/LocalSite/SharedArea=%s'" % queueDict['SharedArea'] ) group = "lhcb_pilot" if group: pilotOptions.append( '-G %s' % group ) self.log.verbose( "pilotOptions: ", ' '.join( pilotOptions ) ) return pilotOptions
def _getPilotOptions( self, taskQueueDict, pilotsToSubmit ): # Need to limit the maximum number of pilots to submit at once # For generic pilots this is limited by the number of use of the tokens and the # maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation: pilotsToSubmit = min( pilotsToSubmit, int( 50 / self.maxJobsInFillMode ) ) pilotOptions = [ "-V %s" % self.virtualOrganization ] privateIfGenericTQ = self.privatePilotFraction > random.random() privateTQ = ( 'PilotTypes' in taskQueueDict and 'private' in [ t.lower() for t in taskQueueDict['PilotTypes'] ] ) forceGeneric = 'ForceGeneric' in taskQueueDict submitPrivatePilot = ( privateIfGenericTQ or privateTQ ) and not forceGeneric if submitPrivatePilot: self.log.verbose( 'Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] ) ownerDN = taskQueueDict['OwnerDN'] ownerGroup = taskQueueDict['OwnerGroup'] # User Group requirement pilotOptions.append( '-G %s' % taskQueueDict['OwnerGroup'] ) # check if group allows jobsharing ownerGroupProperties = getPropertiesForGroup( ownerGroup ) if not 'JobSharing' in ownerGroupProperties: # Add Owner requirement to pilot pilotOptions.append( "-O '%s'" % ownerDN ) if privateTQ: pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private' ) maxJobsInFillMode = self.maxJobsInFillMode else: #For generic jobs we'll submit mixture of generic and private pilots self.log.verbose( 'Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] ) ownerDN = self.genericPilotDN ownerGroup = self.genericPilotGroup result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) ) if not result[ 'OK' ]: self.log.error( ERROR_TOKEN, result['Message'] ) return S_ERROR( ERROR_TOKEN ) ( token, numberOfUses ) = result[ 'Value' ] pilotsToSubmit = min( numberOfUses, pilotsToSubmit ) pilotOptions.append( '-o /Security/ProxyToken=%s' % token ) pilotsToSubmit = ( pilotsToSubmit - 1 ) / self.maxJobsInFillMode + 1 maxJobsInFillMode = int( numberOfUses / pilotsToSubmit ) # Use Filling mode pilotOptions.append( '-M %s' % maxJobsInFillMode ) # Debug pilotOptions.append( '-d' ) # Setup. pilotOptions.append( '-S %s' % taskQueueDict['Setup'] ) # CS Servers csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] ) pilotOptions.append( '-C %s' % ",".join( csServers ) ) # DIRAC Extensions extensionsList = getCSExtensions() if extensionsList: pilotOptions.append( '-e %s' % ",".join( extensionsList ) ) # Requested version of DIRAC pilotOptions.append( '-r %s' % self.installVersion ) # Requested Project to install pilotOptions.append( '-V %s' % self.installInstallation ) # Requested CPU time pilotOptions.append( '-T %s' % taskQueueDict['CPUTime'] ) if self.extraPilotOptions: pilotOptions.extend( self.extraPilotOptions ) return S_OK( ( pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ ) )
def _getPilotOptions( self, taskQueueDict, pilotsToSubmit ): # Need to limit the maximum number of pilots to submit at once # For generic pilots this is limited by the number of use of the tokens and the # maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation: pilotsToSubmit = max( min( pilotsToSubmit, int( 50 / self.maxJobsInFillMode ) ), 1 ) pilotOptions = [] privateIfGenericTQ = self.privatePilotFraction > random.random() privateTQ = ( 'PilotTypes' in taskQueueDict and 'private' in [ t.lower() for t in taskQueueDict['PilotTypes'] ] ) forceGeneric = 'ForceGeneric' in taskQueueDict submitPrivatePilot = ( privateIfGenericTQ or privateTQ ) and not forceGeneric if submitPrivatePilot: self.log.verbose( 'Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] ) ownerDN = taskQueueDict['OwnerDN'] ownerGroup = taskQueueDict['OwnerGroup'] # User Group requirement pilotOptions.append( '-G %s' % taskQueueDict['OwnerGroup'] ) # check if group allows jobsharing ownerGroupProperties = getPropertiesForGroup( ownerGroup ) if not 'JobSharing' in ownerGroupProperties: # Add Owner requirement to pilot pilotOptions.append( "-O '%s'" % ownerDN ) if privateTQ: pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private' ) maxJobsInFillMode = self.maxJobsInFillMode else: #For generic jobs we'll submit mixture of generic and private pilots self.log.verbose( 'Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] ) #ADRI: Find the generic group result = findGenericPilotCredentials( group = taskQueueDict[ 'OwnerGroup' ] ) if not result[ 'OK' ]: self.log.error( ERROR_GENERIC_CREDENTIALS, result[ 'Message' ] ) return S_ERROR( ERROR_GENERIC_CREDENTIALS ) ownerDN, ownerGroup = result[ 'Value' ] result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) ) if not result[ 'OK' ]: self.log.error( ERROR_TOKEN, result['Message'] ) return S_ERROR( ERROR_TOKEN ) ( token, numberOfUses ) = result[ 'Value' ] pilotsToSubmit = min( numberOfUses, pilotsToSubmit ) pilotOptions.append( '-o /Security/ProxyToken=%s' % token ) pilotsToSubmit = max( 1, ( pilotsToSubmit - 1 ) / self.maxJobsInFillMode + 1 ) maxJobsInFillMode = int( numberOfUses / pilotsToSubmit ) # Use Filling mode pilotOptions.append( '-M %s' % maxJobsInFillMode ) # Debug pilotOptions.append( '-d' ) # Setup. pilotOptions.append( '-S %s' % taskQueueDict['Setup'] ) # CS Servers csServers = gConfig.getServersList() if len( csServers ) > 3: # Remove the master master = gConfigurationData.getMasterServer() if master in csServers: csServers.remove( master ) pilotOptions.append( '-C %s' % ",".join( csServers ) ) # DIRAC Extensions to be used in pilots # ubeda: I'm not entirely sure if we can use here the same opsHelper as in line # line +352 pilotExtensionsList = Operations().getValue( "Pilot/Extensions", [] ) extensionsList = [] if pilotExtensionsList: if pilotExtensionsList[0] != 'None': extensionsList = pilotExtensionsList else: extensionsList = getCSExtensions() if extensionsList: pilotOptions.append( '-e %s' % ",".join( extensionsList ) ) #Get DIRAC version and project, There might be global Setup defaults and per VO/Setup defaults (from configure) opsHelper = Operations( group = taskQueueDict['OwnerGroup'], setup = taskQueueDict['Setup'] ) # Requested version of DIRAC (it can be a list, so we take the fist one) version = opsHelper.getValue( cfgPath( 'Pilot', 'Version' ) , [ self.installVersion ] )[0] pilotOptions.append( '-r %s' % version ) # Requested Project to install installProject = opsHelper.getValue( cfgPath( 'Pilot', 'Project' ) , self.installProject ) if installProject: pilotOptions.append( '-l %s' % installProject ) installation = opsHelper.getValue( cfgPath( 'Pilot', 'Installation' ), self.installation ) if installation: pilotOptions.append( "-V %s" % installation ) # Requested CPU time pilotOptions.append( '-T %s' % taskQueueDict['CPUTime'] ) if self.submitPoolOption not in self.extraPilotOptions: pilotOptions.append( self.submitPoolOption ) if self.extraPilotOptions: pilotOptions.extend( self.extraPilotOptions ) return S_OK( ( pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ ) )
def __getPilotOptions( self, queue, pilotsToSubmit ): """ Prepare pilot options """ queueDict = self.queueDict[queue]['ParametersDict'] vo = getVO() if not vo: self.log.error( 'Virtual Organization is not defined in the configuration' ) return None pilotOptions = [ "-V '%s'" % vo ] setup = gConfig.getValue( "/DIRAC/Setup", "unknown" ) if setup == 'unknown': self.log.error( 'Setup is not defined in the configuration' ) return None pilotOptions.append( '-S %s' % setup ) diracVersion = gConfig.getValue( "/Operations/%s/%s/Versions/PilotVersion" % ( vo, setup ), "unknown" ) if diracVersion == 'unknown': self.log.error( 'PilotVersion is not defined in the configuration' ) return None pilotOptions.append( '-r %s' % diracVersion ) projectName = gConfig.getValue( "/Operations/%s/%s/Versions/PilotProject" % ( vo, setup ), "unknown" ) if projectName == 'unknown': self.log.info( 'PilotProject is not defined in the configuration' ) else: pilotOptions.append( '-l %s' % projectName ) ownerDN = self.genericPilotDN ownerGroup = self.genericPilotGroup result = gProxyManager.requestToken( ownerDN, ownerGroup, pilotsToSubmit * 5 ) if not result[ 'OK' ]: self.log.error( ERROR_TOKEN, result['Message'] ) return S_ERROR( ERROR_TOKEN ) ( token, numberOfUses ) = result[ 'Value' ] pilotOptions.append( '-o /Security/ProxyToken=%s' % token ) # Use Filling mode pilotOptions.append( '-M %s' % 5 ) # Debug pilotOptions.append( '-d' ) # CS Servers csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] ) pilotOptions.append( '-C %s' % ",".join( csServers ) ) # DIRAC Extensions extensionsList = getCSExtensions() if extensionsList: pilotOptions.append( '-e %s' % ",".join( extensionsList ) ) # Requested CPU time pilotOptions.append( '-T %s' % queueDict['CPUTime'] ) # SiteName pilotOptions.append( '-n %s' % queueDict['Site'] ) if 'ClientPlatform' in queueDict: pilotOptions.append( "-p '%s'" % queueDict['ClientPlatform'] ) if 'SharedArea' in queueDict: pilotOptions.append( "-o '/LocalSite/SharedArea=%s'" % queueDict['SharedArea'] ) if 'SI00' in queueDict: factor = float( queueDict['SI00'] ) / 250. pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % factor ) pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % factor ) else: if 'CPUScalingFactor' in queueDict: pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % queueDict['CPUScalingFactor'] ) if 'CPUNormalizationFactor' in queueDict: pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % queueDict['CPUNormalizationFactor'] ) self.log.verbose( "pilotOptions: ", ' '.join( pilotOptions ) ) return pilotOptions
def _getPilotOptions( self, taskQueueDict, pilotsToSubmit ): # Need to limit the maximum number of pilots to submit at once # For generic pilots this is limited by the number of use of the tokens and the # maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation: pilotsToSubmit = min( pilotsToSubmit, int( 50 / self.maxJobsInFillMode ) ) pilotOptions = [] privateIfGenericTQ = self.privatePilotFraction > random.random() privateTQ = ( 'PilotTypes' in taskQueueDict and 'private' in [ t.lower() for t in taskQueueDict['PilotTypes'] ] ) forceGeneric = 'ForceGeneric' in taskQueueDict submitPrivatePilot = ( privateIfGenericTQ or privateTQ ) and not forceGeneric if submitPrivatePilot: self.log.verbose( 'Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] ) ownerDN = taskQueueDict['OwnerDN'] ownerGroup = taskQueueDict['OwnerGroup'] # User Group requirement pilotOptions.append( '-G %s' % taskQueueDict['OwnerGroup'] ) # check if group allows jobsharing ownerGroupProperties = getPropertiesForGroup( ownerGroup ) if not 'JobSharing' in ownerGroupProperties: # Add Owner requirement to pilot pilotOptions.append( "-O '%s'" % ownerDN ) if privateTQ: pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private' ) maxJobsInFillMode = self.maxJobsInFillMode else: #For generic jobs we'll submit mixture of generic and private pilots self.log.verbose( 'Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] ) ownerDN = self.genericPilotDN ownerGroup = self.genericPilotGroup result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) ) if not result[ 'OK' ]: self.log.error( ERROR_TOKEN, result['Message'] ) return S_ERROR( ERROR_TOKEN ) ( token, numberOfUses ) = result[ 'Value' ] pilotsToSubmit = min( numberOfUses, pilotsToSubmit ) pilotOptions.append( '-o /Security/ProxyToken=%s' % token ) pilotsToSubmit = ( pilotsToSubmit - 1 ) / self.maxJobsInFillMode + 1 maxJobsInFillMode = int( numberOfUses / pilotsToSubmit ) # Use Filling mode pilotOptions.append( '-M %s' % maxJobsInFillMode ) # Debug pilotOptions.append( '-d' ) # Setup. pilotOptions.append( '-S %s' % taskQueueDict['Setup'] ) # CS Servers csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] ) pilotOptions.append( '-C %s' % ",".join( csServers ) ) # DIRAC Extensions extensionsList = getCSExtensions() if extensionsList: pilotOptions.append( '-e %s' % ",".join( extensionsList ) ) # Requested version of DIRAC pilotOptions.append( '-r %s' % self.installVersion ) # Requested Project to install if self.installInstallation: pilotOptions.append( '-V %s' % self.installInstallation ) # Requested CPU time pilotOptions.append( '-T %s' % taskQueueDict['CPUTime'] ) if self.extraPilotOptions: pilotOptions.extend( self.extraPilotOptions ) return S_OK( ( pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ ) )
def _getPilotOptions(self, taskQueueDict, pilotsToSubmit): # Need to limit the maximum number of pilots to submit at once # For generic pilots this is limited by the number of use of the tokens and the # maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation: pilotsToSubmit = max(min(pilotsToSubmit, int(50 / self.maxJobsInFillMode)), 1) pilotOptions = [] privateIfGenericTQ = self.privatePilotFraction > random.random() privateTQ = "PilotTypes" in taskQueueDict and "private" in [t.lower() for t in taskQueueDict["PilotTypes"]] forceGeneric = "ForceGeneric" in taskQueueDict submitPrivatePilot = (privateIfGenericTQ or privateTQ) and not forceGeneric if submitPrivatePilot: self.log.verbose("Submitting private pilots for TaskQueue %s" % taskQueueDict["TaskQueueID"]) ownerDN = taskQueueDict["OwnerDN"] ownerGroup = taskQueueDict["OwnerGroup"] # User Group requirement pilotOptions.append("-G %s" % taskQueueDict["OwnerGroup"]) # check if group allows jobsharing ownerGroupProperties = getPropertiesForGroup(ownerGroup) if not "JobSharing" in ownerGroupProperties: # Add Owner requirement to pilot pilotOptions.append("-O '%s'" % ownerDN) if privateTQ: pilotOptions.append("-o /Resources/Computing/CEDefaults/PilotType=private") maxJobsInFillMode = self.maxJobsInFillMode else: # For generic jobs we'll submit mixture of generic and private pilots self.log.verbose("Submitting generic pilots for TaskQueue %s" % taskQueueDict["TaskQueueID"]) # ADRI: Find the generic group result = findGenericPilotCredentials(group=taskQueueDict["OwnerGroup"]) if not result["OK"]: self.log.error(ERROR_GENERIC_CREDENTIALS, result["Message"]) return S_ERROR(ERROR_GENERIC_CREDENTIALS) ownerDN, ownerGroup = result["Value"] result = gProxyManager.requestToken(ownerDN, ownerGroup, max(pilotsToSubmit, self.maxJobsInFillMode)) if not result["OK"]: self.log.error(ERROR_TOKEN, result["Message"]) return S_ERROR(ERROR_TOKEN) (token, numberOfUses) = result["Value"] pilotsToSubmit = min(numberOfUses, pilotsToSubmit) pilotOptions.append("-o /Security/ProxyToken=%s" % token) pilotsToSubmit = max(1, (pilotsToSubmit - 1) / self.maxJobsInFillMode + 1) maxJobsInFillMode = int(numberOfUses / pilotsToSubmit) # Use Filling mode pilotOptions.append("-M %s" % maxJobsInFillMode) # Debug pilotOptions.append("-d") # Setup. pilotOptions.append("-S %s" % taskQueueDict["Setup"]) # CS Servers csServers = gConfig.getServersList() if len(csServers) > 3: # Remove the master master = gConfigurationData.getMasterServer() if master in csServers: csServers.remove(master) pilotOptions.append("-C %s" % ",".join(csServers)) # DIRAC Extensions extensionsList = getCSExtensions() if extensionsList: pilotOptions.append("-e %s" % ",".join(extensionsList)) # Get DIRAC version and project, There might be global Setup defaults and per VO/Setup defaults (from configure) opsHelper = Operations(group=taskQueueDict["OwnerGroup"], setup=taskQueueDict["Setup"]) # Requested version of DIRAC (it can be a list, so we take the fist one) version = opsHelper.getValue(cfgPath("Pilot", "Version"), [self.installVersion])[0] pilotOptions.append("-r %s" % version) # Requested Project to install installProject = opsHelper.getValue(cfgPath("Pilot", "Project"), self.installProject) if installProject: pilotOptions.append("-l %s" % installProject) installation = opsHelper.getValue(cfgPath("Pilot", "Installation"), self.installation) if installation: pilotOptions.append("-V %s" % installation) # Requested CPU time pilotOptions.append("-T %s" % taskQueueDict["CPUTime"]) if self.submitPoolOption not in self.extraPilotOptions: pilotOptions.append(self.submitPoolOption) if self.extraPilotOptions: pilotOptions.extend(self.extraPilotOptions) return S_OK((pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ))
def _getPilotOptions(self, taskQueueDict, pilotsToSubmit): # Need to limit the maximum number of pilots to submit at once # For generic pilots this is limited by the number of use of the tokens and the # maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation: pilotsToSubmit = max( min(pilotsToSubmit, int(50 / self.maxJobsInFillMode)), 1) pilotOptions = [] privateIfGenericTQ = self.privatePilotFraction > random.random() privateTQ = ('PilotTypes' in taskQueueDict and 'private' in [t.lower() for t in taskQueueDict['PilotTypes']]) forceGeneric = 'ForceGeneric' in taskQueueDict submitPrivatePilot = (privateIfGenericTQ or privateTQ) and not forceGeneric if submitPrivatePilot: self.log.verbose('Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID']) ownerDN = taskQueueDict['OwnerDN'] ownerGroup = taskQueueDict['OwnerGroup'] # User Group requirement pilotOptions.append('-G %s' % taskQueueDict['OwnerGroup']) # check if group allows jobsharing ownerGroupProperties = getPropertiesForGroup(ownerGroup) if not 'JobSharing' in ownerGroupProperties: # Add Owner requirement to pilot pilotOptions.append("-O '%s'" % ownerDN) if privateTQ: pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private') maxJobsInFillMode = self.maxJobsInFillMode else: #For generic jobs we'll submit mixture of generic and private pilots self.log.verbose('Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID']) #ADRI: Find the generic group result = findGenericPilotCredentials( group=taskQueueDict['OwnerGroup']) if not result['OK']: self.log.error(ERROR_GENERIC_CREDENTIALS, result['Message']) return S_ERROR(ERROR_GENERIC_CREDENTIALS) ownerDN, ownerGroup = result['Value'] result = gProxyManager.requestToken( ownerDN, ownerGroup, max(pilotsToSubmit, self.maxJobsInFillMode)) if not result['OK']: self.log.error(ERROR_TOKEN, result['Message']) return S_ERROR(ERROR_TOKEN) (token, numberOfUses) = result['Value'] pilotsToSubmit = min(numberOfUses, pilotsToSubmit) pilotOptions.append('-o /Security/ProxyToken=%s' % token) pilotsToSubmit = max( 1, (pilotsToSubmit - 1) / self.maxJobsInFillMode + 1) maxJobsInFillMode = int(numberOfUses / pilotsToSubmit) # Use Filling mode pilotOptions.append('-M %s' % maxJobsInFillMode) # Debug pilotOptions.append('-d') # Setup. pilotOptions.append('-S %s' % taskQueueDict['Setup']) # CS Servers csServers = gConfig.getServersList() if len(csServers) > 3: # Remove the master master = gConfigurationData.getMasterServer() if master in csServers: csServers.remove(master) pilotOptions.append('-C %s' % ",".join(csServers)) # DIRAC Extensions extensionsList = getCSExtensions() if extensionsList: pilotOptions.append('-e %s' % ",".join(extensionsList)) #Get DIRAC version and project, There might be global Setup defaults and per VO/Setup defaults (from configure) opsHelper = Operations(group=taskQueueDict['OwnerGroup'], setup=taskQueueDict['Setup']) # Requested version of DIRAC (it can be a list, so we take the fist one) version = opsHelper.getValue(cfgPath('Pilot', 'Version'), [self.installVersion])[0] pilotOptions.append('-r %s' % version) # Requested Project to install installProject = opsHelper.getValue(cfgPath('Pilot', 'Project'), self.installProject) if installProject: pilotOptions.append('-l %s' % installProject) installation = opsHelper.getValue(cfgPath('Pilot', 'Installation'), self.installation) if installation: pilotOptions.append("-V %s" % installation) # Requested CPU time pilotOptions.append('-T %s' % taskQueueDict['CPUTime']) if self.submitPoolOption not in self.extraPilotOptions: pilotOptions.append(self.submitPoolOption) if self.extraPilotOptions: pilotOptions.extend(self.extraPilotOptions) return S_OK((pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ))
def _submitPilots(self, workDir, taskQueueDict, pilotOptions, pilotsToSubmit, ceMask, submitPrivatePilot, privateTQ, proxy, pilotsPerJob): """ This method does the actual pilot submission to the Grid RB The logic is as follows: - If there are no available RB it return error - If there is no VOMS extension in the proxy, return error - It creates a temp directory - Prepare a JDL it has some part common to gLite and LCG (the payload description) it has some part specific to each middleware """ taskQueueID = taskQueueDict['TaskQueueID'] # ownerDN = taskQueueDict['OwnerDN'] ownerDN = proxy.getCredentials()['Value']['identity'] if not self.resourceBrokers: # Since we can exclude RBs from the list, it may become empty return S_ERROR(ERROR_RB) # Need to get VOMS extension for the later interactions with WMS ret = gProxyManager.getVOMSAttributes(proxy) if not ret['OK']: self.log.error(ERROR_VOMS, ret['Message']) return S_ERROR(ERROR_VOMS) if not ret['Value']: return S_ERROR(ERROR_VOMS) vomsGroup = ret['Value'][0] workingDirectory = tempfile.mkdtemp(prefix='TQ_%s_' % taskQueueID, dir=workDir) self.log.verbose('Using working Directory:', workingDirectory) # Write JDL retDict = self._prepareJDL(taskQueueDict, workingDirectory, pilotOptions, pilotsPerJob, ceMask, submitPrivatePilot, privateTQ) jdl = retDict['JDL'] pilotRequirements = retDict['Requirements'] rb = retDict['RB'] if not jdl: try: shutil.rmtree(workingDirectory) except: pass return S_ERROR(ERROR_JDL) # Check that there are available queues for the Job: if self.enableListMatch: availableCEs = [] now = Time.dateTime() availableCEs = self.listMatchCache.get(pilotRequirements) if availableCEs == False: availableCEs = self._listMatch(proxy, jdl, taskQueueID, rb) if availableCEs != False: self.log.verbose('LastListMatch', now) self.log.verbose('AvailableCEs ', availableCEs) self.listMatchCache.add( pilotRequirements, self.listMatchDelay * 60, value=availableCEs) # it is given in minutes if not availableCEs: try: shutil.rmtree(workingDirectory) except: pass return S_ERROR(ERROR_CE + ' TQ: %d' % taskQueueID) # Now we are ready for the actual submission, so self.log.verbose('Submitting Pilots for TaskQueue', taskQueueID) submitRet = self._submitPilot(proxy, pilotsPerJob, jdl, taskQueueID, rb) try: shutil.rmtree(workingDirectory) except: pass if not submitRet: return S_ERROR('Pilot Submission Failed for TQ %d ' % taskQueueID) # pilotReference, resourceBroker = submitRet submittedPilots = 0 if pilotsPerJob != 1 and len(submitRet) != pilotsPerJob: # Parametric jobs are used for pilotReference, resourceBroker in submitRet: pilotReference = self._getChildrenReferences( proxy, pilotReference, taskQueueID) submittedPilots += len(pilotReference) pilotAgentsDB.addPilotTQReference(pilotReference, taskQueueID, ownerDN, vomsGroup, resourceBroker, self.gridMiddleware, pilotRequirements) else: for pilotReference, resourceBroker in submitRet: pilotReference = [pilotReference] submittedPilots += len(pilotReference) pilotAgentsDB.addPilotTQReference( pilotReference, taskQueueID, ownerDN, vomsGroup, broker=resourceBroker, gridType=self.gridMiddleware, requirements=pilotRequirements) # add some sleep here time.sleep(0.1 * submittedPilots) if pilotsToSubmit > pilotsPerJob: # Additional submissions are necessary, need to get a new token and iterate. pilotsToSubmit -= pilotsPerJob ownerDN = self.genericPilotDN ownerGroup = self.genericPilotGroup result = gProxyManager.requestToken( ownerDN, ownerGroup, max(pilotsToSubmit, self.maxJobsInFillMode)) if not result['OK']: self.log.error(ERROR_TOKEN, result['Message']) return S_ERROR(ERROR_TOKEN) (token, numberOfUses) = result['Value'] for option in pilotOptions: if option.find('-o /Security/ProxyToken=') == 0: pilotOptions.remove(option) pilotOptions.append('-o /Security/ProxyToken=%s' % token) pilotsPerJob = min(pilotsPerJob, int(numberOfUses / self.maxJobsInFillMode)) result = self._submitPilots(workDir, taskQueueDict, pilotOptions, pilotsToSubmit, ceMask, submitPrivatePilot, privateTQ, proxy, pilotsPerJob) if not result['OK']: result['Value'] = submittedPilots return result submittedPilots += result['Value'] return S_OK(submittedPilots)
def _getPilotOptions( self, queue, pilotsToSubmit, processors = 1 ): """ Prepare pilot options """ queueDict = self.queueDict[queue]['ParametersDict'] pilotOptions = [] setup = gConfig.getValue( "/DIRAC/Setup", "unknown" ) if setup == 'unknown': self.log.error( 'Setup is not defined in the configuration' ) return [ None, None ] pilotOptions.append( '-S %s' % setup ) opsHelper = Operations.Operations( group = self.pilotGroup, setup = setup ) #Installation defined? installationName = opsHelper.getValue( "Pilot/Installation", "" ) if installationName: pilotOptions.append( '-V %s' % installationName ) #Project defined? projectName = opsHelper.getValue( "Pilot/Project", "" ) if projectName: pilotOptions.append( '-l %s' % projectName ) else: self.log.info( 'DIRAC project will be installed by pilots' ) #Request a release diracVersion = opsHelper.getValue( "Pilot/Version", [] ) if not diracVersion: self.log.error( 'Pilot/Version is not defined in the configuration' ) return [ None, None ] # diracVersion is a list of accepted releases pilotOptions.append( '-r %s' % ','.join( str( it ) for it in diracVersion ) ) ownerDN = self.pilotDN ownerGroup = self.pilotGroup # Request token for maximum pilot efficiency result = gProxyManager.requestToken( ownerDN, ownerGroup, pilotsToSubmit * self.maxJobsInFillMode ) if not result[ 'OK' ]: self.log.error( 'Invalid proxy token request', result['Message'] ) return [ None, None ] ( token, numberOfUses ) = result[ 'Value' ] pilotOptions.append( '-o /Security/ProxyToken=%s' % token ) # Use Filling mode pilotOptions.append( '-M %s' % min( numberOfUses, self.maxJobsInFillMode ) ) # Since each pilot will execute min( numberOfUses, self.maxJobsInFillMode ) # with numberOfUses tokens we can submit at most: # numberOfUses / min( numberOfUses, self.maxJobsInFillMode ) # pilots newPilotsToSubmit = numberOfUses / min( numberOfUses, self.maxJobsInFillMode ) if newPilotsToSubmit != pilotsToSubmit: self.log.info( 'Number of pilots to submit is changed to %d after getting the proxy token' % newPilotsToSubmit ) pilotsToSubmit = newPilotsToSubmit # Debug if self.pilotLogLevel.lower() == 'debug': pilotOptions.append( '-ddd' ) # CS Servers csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] ) pilotOptions.append( '-C %s' % ",".join( csServers ) ) # DIRAC Extensions to be used in pilots pilotExtensionsList = opsHelper.getValue( "Pilot/Extensions", [] ) extensionsList = [] if pilotExtensionsList: if pilotExtensionsList[0] != 'None': extensionsList = pilotExtensionsList else: extensionsList = CSGlobals.getCSExtensions() if extensionsList: pilotOptions.append( '-e %s' % ",".join( extensionsList ) ) # Requested CPU time pilotOptions.append( '-T %s' % queueDict['CPUTime'] ) # CEName pilotOptions.append( '-N %s' % self.queueDict[queue]['CEName'] ) # Queue pilotOptions.append( '-Q %s' % self.queueDict[queue]['QueueName'] ) # SiteName pilotOptions.append( '-n %s' % queueDict['Site'] ) if 'ClientPlatform' in queueDict: pilotOptions.append( "-p '%s'" % queueDict['ClientPlatform'] ) if 'SharedArea' in queueDict: pilotOptions.append( "-o '/LocalSite/SharedArea=%s'" % queueDict['SharedArea'] ) # if 'SI00' in queueDict: # factor = float( queueDict['SI00'] ) / 250. # pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % factor ) # pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % factor ) # else: # if 'CPUScalingFactor' in queueDict: # pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % queueDict['CPUScalingFactor'] ) # if 'CPUNormalizationFactor' in queueDict: # pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % queueDict['CPUNormalizationFactor'] ) if "ExtraPilotOptions" in queueDict: pilotOptions.append( queueDict['ExtraPilotOptions'] ) # Hack if self.defaultSubmitPools: pilotOptions.append( '-o /Resources/Computing/CEDefaults/SubmitPool=%s' % self.defaultSubmitPools ) if processors != 1: if processors > 1: pilotOptions.append( '-o /AgentJobRequirements/RequiredTag=%sProcessors' % processors ) pilotOptions.append( '-o /Resources/Computing/CEDefaults/Tag=%sProcessors' % processors ) else: pilotOptions.append( '-o /AgentJobRequirements/RequiredTag=WholeNode' ) pilotOptions.append( '-o /Resources/Computing/CEDefaults/Tag=WholeNode' ) if self.group: pilotOptions.append( '-G %s' % self.group ) return [ pilotOptions, pilotsToSubmit ]
def __getPilotOptions(self, queue, pilotsToSubmit): """ Prepare pilot options """ queueDict = self.queueDict[queue]['ParametersDict'] vo = getVO() if not vo: self.log.error( 'Virtual Organization is not defined in the configuration') return None pilotOptions = ["-V '%s'" % vo] setup = gConfig.getValue("/DIRAC/Setup", "unknown") if setup == 'unknown': self.log.error('Setup is not defined in the configuration') return None pilotOptions.append('-S %s' % setup) diracVersion = gConfig.getValue( "/Operations/%s/%s/Versions/PilotVersion" % (vo, setup), "unknown") if diracVersion == 'unknown': self.log.error('PilotVersion is not defined in the configuration') return None pilotOptions.append('-r %s' % diracVersion) projectName = gConfig.getValue( "/Operations/%s/%s/Versions/PilotProject" % (vo, setup), "unknown") if projectName == 'unknown': self.log.info('PilotProject is not defined in the configuration') else: pilotOptions.append('-l %s' % projectName) ownerDN = self.genericPilotDN ownerGroup = self.genericPilotGroup result = gProxyManager.requestToken(ownerDN, ownerGroup, pilotsToSubmit * 5) if not result['OK']: self.log.error(ERROR_TOKEN, result['Message']) return S_ERROR(ERROR_TOKEN) (token, numberOfUses) = result['Value'] pilotOptions.append('-o /Security/ProxyToken=%s' % token) # Use Filling mode pilotOptions.append('-M %s' % 5) # Debug pilotOptions.append('-d') # CS Servers csServers = gConfig.getValue("/DIRAC/Configuration/Servers", []) pilotOptions.append('-C %s' % ",".join(csServers)) # DIRAC Extensions extensionsList = getCSExtensions() if extensionsList: pilotOptions.append('-e %s' % ",".join(extensionsList)) # Requested CPU time pilotOptions.append('-T %s' % queueDict['CPUTime']) # SiteName pilotOptions.append('-n %s' % queueDict['Site']) if 'ClientPlatform' in queueDict: pilotOptions.append("-p '%s'" % queueDict['ClientPlatform']) if 'SharedArea' in queueDict: pilotOptions.append("-o '/LocalSite/SharedArea=%s'" % queueDict['SharedArea']) if 'SI00' in queueDict: factor = float(queueDict['SI00']) / 250. pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" % factor) pilotOptions.append("-o '/LocalSite/CPUNormalizationFactor=%s'" % factor) else: if 'CPUScalingFactor' in queueDict: pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" % queueDict['CPUScalingFactor']) if 'CPUNormalizationFactor' in queueDict: pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % queueDict['CPUNormalizationFactor']) self.log.verbose("pilotOptions: ", ' '.join(pilotOptions)) return pilotOptions
def __getPilotOptions(self, queue, pilotsToSubmit): """ Prepare pilot options """ queueDict = self.queueDict[queue]["ParametersDict"] pilotOptions = [] setup = gConfig.getValue("/DIRAC/Setup", "unknown") if setup == "unknown": self.log.error("Setup is not defined in the configuration") return [None, None] pilotOptions.append("-S %s" % setup) opsHelper = Operations.Operations(group=self.pilotGroup, setup=setup) # Installation defined? installationName = opsHelper.getValue("Pilot/Installation", "") if installationName: pilotOptions.append("-V %s" % installationName) # Project defined? projectName = opsHelper.getValue("Pilot/Project", "") if projectName: pilotOptions.append("-l %s" % projectName) else: self.log.info("DIRAC project will be installed by pilots") # Request a release diracVersion = opsHelper.getValue("Pilot/Version", []) if not diracVersion: self.log.error("Pilot/Version is not defined in the configuration") return [None, None] # diracVersion is a list of accepted releases. Just take the first one pilotOptions.append("-r %s" % diracVersion[0]) ownerDN = self.pilotDN ownerGroup = self.pilotGroup # Request token for maximum pilot efficiency result = gProxyManager.requestToken(ownerDN, ownerGroup, pilotsToSubmit * self.maxJobsInFillMode) if not result["OK"]: self.log.error("Invalid proxy token request", result["Message"]) return [None, None] (token, numberOfUses) = result["Value"] pilotOptions.append("-o /Security/ProxyToken=%s" % token) # Use Filling mode pilotOptions.append("-M %s" % min(numberOfUses, self.maxJobsInFillMode)) # Since each pilot will execute min( numberOfUses, self.maxJobsInFillMode ) # with numberOfUses tokens we can submit at most: # numberOfUses / min( numberOfUses, self.maxJobsInFillMode ) # pilots newPilotsToSubmit = numberOfUses / min(numberOfUses, self.maxJobsInFillMode) if newPilotsToSubmit != pilotsToSubmit: self.log.info( "Number of pilots to submit is changed to %d after getting the proxy token" % newPilotsToSubmit ) pilotsToSubmit = newPilotsToSubmit # Debug if self.pilotLogLevel.lower() == "debug": pilotOptions.append("-d") # CS Servers csServers = gConfig.getValue("/DIRAC/Configuration/Servers", []) pilotOptions.append("-C %s" % ",".join(csServers)) # DIRAC Extensions to be used in pilots pilotExtensionsList = opsHelper.getValue("Pilot/Extensions", []) extensionsList = [] if pilotExtensionsList: if pilotExtensionsList[0] != "None": extensionsList = pilotExtensionsList else: extensionsList = CSGlobals.getCSExtensions() if extensionsList: pilotOptions.append("-e %s" % ",".join(extensionsList)) # Requested CPU time pilotOptions.append("-T %s" % queueDict["CPUTime"]) # CEName pilotOptions.append("-N %s" % self.queueDict[queue]["CEName"]) # SiteName pilotOptions.append("-n %s" % queueDict["Site"]) if "ClientPlatform" in queueDict: pilotOptions.append("-p '%s'" % queueDict["ClientPlatform"]) if "SharedArea" in queueDict: pilotOptions.append("-o '/LocalSite/SharedArea=%s'" % queueDict["SharedArea"]) if "SI00" in queueDict: factor = float(queueDict["SI00"]) / 250.0 pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" % factor) pilotOptions.append("-o '/LocalSite/CPUNormalizationFactor=%s'" % factor) else: if "CPUScalingFactor" in queueDict: pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" % queueDict["CPUScalingFactor"]) if "CPUNormalizationFactor" in queueDict: pilotOptions.append("-o '/LocalSite/CPUNormalizationFactor=%s'" % queueDict["CPUNormalizationFactor"]) # Hack if self.defaultSubmitPools: pilotOptions.append("-o /Resources/Computing/CEDefaults/SubmitPool=%s" % self.defaultSubmitPools) if self.group: pilotOptions.append("-G %s" % self.group) self.log.verbose("pilotOptions: ", " ".join(pilotOptions)) return [pilotOptions, pilotsToSubmit]