def initialize( self ): self.siteJobLimits = self.getCSOption( "SiteJobLimits", False ) self.checkPilotVersion = self.getCSOption( "CheckPilotVersion", True ) self.setup = gConfig.getValue( '/DIRAC/Setup', '' ) self.vo = getVO() self.pilotVersion = gConfig.getValue( '/Operations/%s/%s/Versions/PilotVersion' % ( self.vo, self.setup ), '' )
def configure(self, csSection, submitPool): """ Here goes common configuration for all PilotDirectors """ self.configureFromSection(csSection) self.reloadConfiguration(csSection, submitPool) setup = gConfig.getValue('/DIRAC/Setup', '') vo = getVO() self.installVersion = gConfig.getValue( '/Operations/%s/%s/Versions/PilotVersion' % (vo, setup), self.installVersion) self.installProject = gConfig.getValue( '/Operations/%s/%s/Versions/PilotProject' % (vo, setup), "") self.log.info('===============================================') self.log.info('Configuration:') self.log.info('') self.log.info(' Install script: ', self.install) self.log.info(' Pilot script: ', self.pilot) self.log.info(' Install Ver: ', self.installVersion) if self.installProject: self.log.info(' Project: ', self.installProject) if self.extraPilotOptions: self.log.info(' Exta Options: ', ' '.join(self.extraPilotOptions)) self.log.info(' ListMatch: ', self.enableListMatch) self.log.info(' Private %: ', self.privatePilotFraction * 100) if self.enableListMatch: self.log.info(' ListMatch Delay:', self.listMatchDelay) self.listMatchCache.purgeExpired()
def initializeOptimizer(self): """Initialize specific parameters for JobSanityAgent. """ #Test control flags N.B. JDL check is mandatory self.inputDataCheck = self.am_getOption('InputDataCheck', 1) self.outputDataCheck = self.am_getOption('OutputDataCheck', 0) self.inputSandboxCheck = self.am_getOption('InputSandboxCheck', 1) self.platformCheck = self.am_getOption('PlatformCheck', 0) #Other parameters self.voName = getVO('lhcb') self.successStatus = self.am_getOption('SuccessfulJobStatus', 'OutputReady') self.maxDataPerJob = self.am_getOption('MaxInputDataPerJob', 100) #Sandbox self.sandboxClient = SandboxStoreClient(useCertificates=True) self.log.debug('JDL Check ==> Enabled') if self.inputDataCheck: self.log.debug('Input Data Check ==> Enabled') else: self.log.debug('Input Data Check ==> Disabled') if self.outputDataCheck: self.log.debug('Output Data Check ==> Enabled') else: self.log.debug('Output Data Check ==> Disabled') if self.inputSandboxCheck: self.log.debug('Input Sbox Check ==> Enabled') else: self.log.debug('Input Sbox Check ==> Disabled') if self.platformCheck: self.log.debug('Platform Check ==> Enabled') else: self.log.debug('Platform Check ==> Disabled') return S_OK()
def initialize(self): self.siteJobLimits = self.getCSOption("SiteJobLimits", False) self.checkPilotVersion = self.getCSOption("CheckPilotVersion", True) self.setup = gConfig.getValue("/DIRAC/Setup", "") self.vo = getVO() self.pilotVersion = gConfig.getValue("/Operations/%s/%s/Versions/PilotVersion" % (self.vo, self.setup), "")
def configure( self, csSection, submitPool ): """ Here goes common configuration for all PilotDirectors """ self.configureFromSection( csSection ) self.reloadConfiguration( csSection, submitPool ) setup = gConfig.getValue( '/DIRAC/Setup', '' ) vo = getVO() self.installVersion = gConfig.getValue( '/Operations/%s/%s/Versions/PilotVersion' % ( vo, setup ), self.installVersion ) self.installProject = gConfig.getValue( '/Operations/%s/%s/Versions/PilotProject' % ( vo, setup ), "" ) self.log.info( '===============================================' ) self.log.info( 'Configuration:' ) self.log.info( '' ) self.log.info( ' Install script: ', self.install ) self.log.info( ' Pilot script: ', self.pilot ) self.log.info( ' Install Ver: ', self.installVersion ) if self.installProject: self.log.info( ' Project: ', self.installProject ) if self.extraPilotOptions: self.log.info( ' Exta Options: ', ' '.join( self.extraPilotOptions ) ) self.log.info( ' ListMatch: ', self.enableListMatch ) self.log.info( ' Private %: ', self.privatePilotFraction * 100 ) if self.enableListMatch: self.log.info( ' ListMatch Delay:', self.listMatchDelay ) self.listMatchCache.purgeExpired()
def initializeOptimizer( self ): """Initialize specific parameters for JobSanityAgent. """ #Test control flags N.B. JDL check is mandatory self.inputDataCheck = self.am_getOption( 'InputDataCheck', 1 ) self.outputDataCheck = self.am_getOption( 'OutputDataCheck', 0 ) self.inputSandboxCheck = self.am_getOption( 'InputSandboxCheck', 1 ) self.platformCheck = self.am_getOption( 'PlatformCheck', 0 ) #Other parameters self.voName = getVO( 'lhcb' ) self.successStatus = self.am_getOption( 'SuccessfulJobStatus', 'OutputReady' ) self.maxDataPerJob = self.am_getOption( 'MaxInputDataPerJob', 100 ) #Sandbox self.sandboxClient = SandboxStoreClient( useCertificates = True ) self.log.debug( 'JDL Check ==> Enabled' ) if self.inputDataCheck: self.log.debug( 'Input Data Check ==> Enabled' ) else: self.log.debug( 'Input Data Check ==> Disabled' ) if self.outputDataCheck: self.log.debug( 'Output Data Check ==> Enabled' ) else: self.log.debug( 'Output Data Check ==> Disabled' ) if self.inputSandboxCheck: self.log.debug( 'Input Sbox Check ==> Enabled' ) else: self.log.debug( 'Input Sbox Check ==> Disabled' ) if self.platformCheck: self.log.debug( 'Platform Check ==> Enabled' ) else: self.log.debug( 'Platform Check ==> Disabled' ) return S_OK()
def __init__( self, server = False, serverCert = False, serverKey = False, voName = False, timeout = False ): if timeout: self._secCmdTimeout = timeout else: self._secCmdTimeout = 30 if not server: self._secServer = gConfig.getValue( "/DIRAC/VOPolicy/MyProxyServer", "myproxy.cern.ch" ) else: self._secServer = server if not voName: self._secVO = getVO( "unknown" ) else: self._secVO = voName ckLoc = Locations.getHostCertificateAndKeyLocation() if serverCert: self._secCertLoc = serverCert else: if ckLoc: self._secCertLoc = ckLoc[0] else: self._secCertLoc = "%s/etc/grid-security/servercert.pem" % DIRAC.rootPath if serverKey: self._secKeyLoc = serverKey else: if ckLoc: self._secKeyLoc = ckLoc[1] else: self._secKeyLoc = "%s/etc/grid-security/serverkey.pem" % DIRAC.rootPath self._secRunningFromTrustedHost = gConfig.getValue( "/DIRAC/VOPolicy/MyProxyTrustedHost", "True" ).lower() in ( "y", "yes", "true" ) self._secMaxProxyHours = gConfig.getValue( "/DIRAC/VOPolicy/MyProxyMaxDelegationTime", 168 )
def _prepareJDL( self, taskQueueDict, workingDirectory, pilotOptions, pilotsToSubmit, ceMask, submitPrivatePilot, privateTQ ): """ Write JDL for Pilot Submission """ # RB = List.randomize( self.resourceBrokers )[0] LDs = [] NSs = [] LBs = [] # Select Randomly one RB from the list RB = List.randomize( self.resourceBrokers )[0] LDs.append( '"%s:9002"' % RB ) LBs.append( '"%s:9000"' % RB ) for LB in self.loggingServers: NSs.append( '"%s:7772"' % LB ) LD = ', '.join( LDs ) NS = ', '.join( NSs ) LB = ', '.join( LBs ) vo = getVO() if privateTQ or vo not in ['lhcb']: extraReq = "True" else: if submitPrivatePilot: extraReq = "! AllowsGenericPilot" else: extraReq = "AllowsGenericPilot" rbJDL = """ AllowsGenericPilot = Member( "VO-lhcb-pilot" , other.GlueHostApplicationSoftwareRunTimeEnvironment ); Requirements = pilotRequirements && other.GlueCEStateStatus == "Production" && %s; RetryCount = 0; ErrorStorage = "%s/pilotError"; OutputStorage = "%s/pilotOutput"; # ListenerPort = 44000; ListenerStorage = "%s/Storage"; VirtualOrganisation = "lhcb"; LoggingTimeout = 30; LoggingSyncTimeout = 30; LoggingDestination = { %s }; # Default NS logger level is set to 0 (null) # max value is 6 (very ugly) NSLoggerLevel = 0; DefaultLogInfoLevel = 0; DefaultStatusLevel = 0; NSAddresses = { %s }; LBAddresses = { %s }; MyProxyServer = "no-myproxy.cern.ch"; """ % ( extraReq, workingDirectory, workingDirectory, workingDirectory, LD, NS, LB ) pilotJDL, pilotRequirements = self._JobJDL( taskQueueDict, pilotOptions, ceMask ) jdl = os.path.join( workingDirectory, '%s.jdl' % taskQueueDict['TaskQueueID'] ) jdl = self._writeJDL( jdl, [pilotJDL, rbJDL] ) return {'JDL':jdl, 'Requirements':pilotRequirements + " && " + extraReq, 'Pilots': pilotsToSubmit, 'RB':RB }
def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ): """Instantiates the Workflow object and some default parameters. """ self.log = gLogger self.section = COMPONENT_NAME self.dbg = False if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG': self.dbg = True #gConfig.getValue('Tier0SE-tape','SEName') self.stepCount = 0 self.owner = 'NotSpecified' self.name = 'Name' self.type = 'User' self.priority = 1 self.group = getVO( 'lhcb' ) self.site = 'ANY' #ANY #self.setup = 'Development' self.origin = 'DIRAC' self.stdout = stdout self.stderr = stderr self.logLevel = 'info' self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified self.addToInputSandbox = [] self.addToOutputSandbox = [] self.addToInputData = [] self.systemConfig = 'ANY' self.reqParams = {'MaxCPUTime': 'other.NAME>=VALUE', 'MinCPUTime': 'other.NAME<=VALUE', 'Site': 'other.NAME=="VALUE"', 'Platform': 'other.NAME=="VALUE"', #'BannedSites': '!Member(other.Site,BannedSites)', #doesn't work unfortunately 'BannedSites': 'other.Site!="VALUE"', 'SystemConfig': 'Member("VALUE",other.CompatiblePlatforms)'} ##Add member to handle Parametric jobs self.parametric = {} self.script = script if not script: self.workflow = Workflow() self.__setJobDefaults() else: self.workflow = Workflow( script ) #Global error dictionary self.errorDict = {}
def __init__(self, submitPool): """ Define the logger and some defaults """ if submitPool == self.gridMiddleware: self.log = gLogger.getSubLogger('%sPilotDirector' % self.gridMiddleware) else: self.log = gLogger.getSubLogger('%sPilotDirector/%s' % (self.gridMiddleware, submitPool)) self.pilot = DIRAC_PILOT self.extraPilotOptions = [] setup = gConfig.getValue('/DIRAC/Setup', '') vo = getVO() self.installVersion = gConfig.getValue( '/Operations/%s/%s/Versions/PilotVersion' % (vo, setup), DIRAC_VERSION) self.installProject = gConfig.getValue( '/Operations/%s/%s/Versions/PilotProject' % (vo, setup), "") self.install = DIRAC_INSTALL self.maxJobsInFillMode = MAX_JOBS_IN_FILLMODE self.genericPilotDN = PILOT_DN self.genericPilotGroup = PILOT_GROUP self.enableListMatch = ENABLE_LISTMATCH self.listMatchDelay = LISTMATCH_DELAY self.listMatchCache = DictCache() self.privatePilotFraction = PRIVATE_PILOT_FRACTION self.errorClearTime = ERROR_CLEAR_TIME self.errorTicketTime = ERROR_TICKET_TIME self.errorMailAddress = DIRAC.errorMail self.alarmMailAddress = DIRAC.alarmMail self.mailFromAddress = FROM_MAIL if not 'log' in self.__dict__: self.log = gLogger.getSubLogger('PilotDirector') self.log.info('Initialized')
def __init__( self, submitPool ): """ Define the logger and some defaults """ if submitPool == self.gridMiddleware: self.log = gLogger.getSubLogger( '%sPilotDirector' % self.gridMiddleware ) else: self.log = gLogger.getSubLogger( '%sPilotDirector/%s' % ( self.gridMiddleware, submitPool ) ) self.pilot = DIRAC_PILOT self.extraPilotOptions = [] setup = gConfig.getValue( '/DIRAC/Setup', '' ) vo = getVO() self.installVersion = gConfig.getValue( '/Operations/%s/%s/Versions/PilotVersion' % ( vo, setup ), DIRAC_VERSION ) self.installProject = gConfig.getValue( '/Operations/%s/%s/Versions/PilotProject' % ( vo, setup ), "" ) self.install = DIRAC_INSTALL self.maxJobsInFillMode = MAX_JOBS_IN_FILLMODE self.genericPilotDN = PILOT_DN self.genericPilotGroup = PILOT_GROUP self.enableListMatch = ENABLE_LISTMATCH self.listMatchDelay = LISTMATCH_DELAY self.listMatchCache = DictCache() self.privatePilotFraction = PRIVATE_PILOT_FRACTION self.errorClearTime = ERROR_CLEAR_TIME self.errorTicketTime = ERROR_TICKET_TIME self.errorMailAddress = DIRAC.errorMail self.alarmMailAddress = DIRAC.alarmMail self.mailFromAddress = FROM_MAIL if not 'log' in self.__dict__: self.log = gLogger.getSubLogger( 'PilotDirector' ) self.log.info( 'Initialized' )
def __init__(self, server=False, serverCert=False, serverKey=False, voName=False, timeout=False): if timeout: self._secCmdTimeout = timeout else: self._secCmdTimeout = 30 if not server: self._secServer = gConfig.getValue("/DIRAC/VOPolicy/MyProxyServer", "myproxy.cern.ch") else: self._secServer = server if not voName: self._secVO = getVO("unknown") else: self._secVO = voName ckLoc = Locations.getHostCertificateAndKeyLocation() if serverCert: self._secCertLoc = serverCert else: if ckLoc: self._secCertLoc = ckLoc[0] else: self._secCertLoc = "%s/etc/grid-security/servercert.pem" % DIRAC.rootPath if serverKey: self._secKeyLoc = serverKey else: if ckLoc: self._secKeyLoc = ckLoc[1] else: self._secKeyLoc = "%s/etc/grid-security/serverkey.pem" % DIRAC.rootPath self._secRunningFromTrustedHost = gConfig.getValue( "/DIRAC/VOPolicy/MyProxyTrustedHost", "True").lower() in ("y", "yes", "true") self._secMaxProxyHours = gConfig.getValue( "/DIRAC/VOPolicy/MyProxyMaxDelegationTime", 168)
# File : dirac-info # Author : Andrei Tsaregorodtsev ######################################################################## """ Report info about local DIRAC installation """ __RCSID__ = "$Id$" from pprint import pprint import DIRAC from DIRAC import gConfig from DIRAC.Core.Base import Script from DIRAC.ConfigurationSystem.Client.Helpers import getVO Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1], 'Usage:', ' %s [option|cfgfile] ... Site' % Script.scriptName, ] ) ) Script.parseCommandLine( ignoreErrors = True ) args = Script.getPositionalArgs() infoDict = {} infoDict['Setup'] = gConfig.getValue( '/DIRAC/Setup', 'Unknown' ) infoDict['ConfigurationServer'] = gConfig.getValue( '/DIRAC/Configuration/Servers', [] ) infoDict['VirtualOrganization'] = getVO( 'Unknown' ) print 'DIRAC version'.rjust( 20 ), ':', DIRAC.version for k, v in infoDict.items(): print k.rjust( 20 ), ':', str( v )
' %s [option|cfgfile] ... CE' % Script.scriptName, 'Arguments:', ' CE: Name of the CE(ie: ce111.cern.ch)'] ) ) Script.parseCommandLine( ignoreErrors = True ) args = Script.getPositionalArgs() from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin if not len( args ) == 1: Script.showHelp() ce = args[0] host = None vo = getVO( 'lhcb' ) for unprocSw in Script.getUnprocessedSwitches(): if unprocSw[0] in ( "H", "host" ): host = unprocSw[1] if unprocSw[0] in ( "V", "vo" ): vo = unprocSw[1] diracAdmin = DiracAdmin() result = diracAdmin.getBDIICEState( ce, useVO = vo, host = host ) if not result['OK']: print result['Message'] DIRAC.exit( 2 ) ces = result['Value']
else: Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate', 'yes') Script.enableCS() if includeAllServers: DIRAC.gConfig.setOptionValue('/DIRAC/Configuration/Servers', ','.join(DIRAC.gConfig.getServersList())) DIRAC.gLogger.debug('/DIRAC/Configuration/Servers =', ','.join(DIRAC.gConfig.getServersList())) if update: DIRAC.gConfig.dumpLocalCFGToFile(DIRAC.gConfig.diracConfigFilePath) #Do the vomsdir magic voName = getVO() if not voName: sys.exit(0) result = DIRAC.gConfig.getSections("/Registry/VOMS/Servers/%s" % voName) if not result['OK']: sys.exit(0) DIRAC.gLogger.notice("Creating VOMSDIR/VOMSES files") vomsDirHosts = result['Value'] vomsDirPath = os.path.join(DIRAC.rootPath, 'etc', 'grid-security', 'vomsdir', voName) vomsesDirPath = os.path.join(DIRAC.rootPath, 'etc', 'grid-security', 'vomses') for path in (vomsDirPath, vomsesDirPath): if not os.path.isdir(path): try: os.makedirs(path) except Exception, e:
def _getPilotOptions( self, taskQueueDict, pilotsToSubmit ): # Need to limit the maximum number of pilots to submit at once # For generic pilots this is limited by the number of use of the tokens and the # maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation: pilotsToSubmit = min( pilotsToSubmit, int( 50 / self.maxJobsInFillMode ) ) pilotOptions = [ "-V %s" % getVO( "lhcb" ) ] privateIfGenericTQ = self.privatePilotFraction > random.random() privateTQ = ( 'PilotTypes' in taskQueueDict and 'private' in [ t.lower() for t in taskQueueDict['PilotTypes'] ] ) forceGeneric = 'ForceGeneric' in taskQueueDict submitPrivatePilot = ( privateIfGenericTQ or privateTQ ) and not forceGeneric if submitPrivatePilot: self.log.verbose( 'Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] ) ownerDN = taskQueueDict['OwnerDN'] ownerGroup = taskQueueDict['OwnerGroup'] # User Group requirement pilotOptions.append( '-G %s' % taskQueueDict['OwnerGroup'] ) # check if group allows jobsharing ownerGroupProperties = getPropertiesForGroup( ownerGroup ) if not 'JobSharing' in ownerGroupProperties: # Add Owner requirement to pilot pilotOptions.append( "-O '%s'" % ownerDN ) if privateTQ: pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private' ) maxJobsInFillMode = self.maxJobsInFillMode else: #For generic jobs we'll submit mixture of generic and private pilots self.log.verbose( 'Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] ) ownerDN = self.genericPilotDN ownerGroup = self.genericPilotGroup result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) ) if not result[ 'OK' ]: self.log.error( ERROR_TOKEN, result['Message'] ) return S_ERROR( ERROR_TOKEN ) ( token, numberOfUses ) = result[ 'Value' ] pilotsToSubmit = min( numberOfUses, pilotsToSubmit ) pilotOptions.append( '-o /Security/ProxyToken=%s' % token ) pilotsToSubmit = ( pilotsToSubmit - 1 ) / self.maxJobsInFillMode + 1 maxJobsInFillMode = int( numberOfUses / pilotsToSubmit ) # Use Filling mode pilotOptions.append( '-M %s' % maxJobsInFillMode ) # Debug pilotOptions.append( '-d' ) # Setup. pilotOptions.append( '-S %s' % taskQueueDict['Setup'] ) # CS Servers csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] ) pilotOptions.append( '-C %s' % ",".join( csServers ) ) # DIRAC Extensions extensionsList = getCSExtensions() if extensionsList: pilotOptions.append( '-e %s' % ",".join( extensionsList ) ) # Requested version of DIRAC pilotOptions.append( '-r %s' % self.installVersion ) # Requested Project to install pilotOptions.append( '-l %s' % self.installProject ) # Requested CPU time pilotOptions.append( '-T %s' % taskQueueDict['CPUTime'] ) if self.extraPilotOptions: pilotOptions.extend( self.extraPilotOptions ) return S_OK( ( pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ ) )
def __getPilotOptions(self, queue, pilotsToSubmit): """ Prepare pilot options """ queueDict = self.queueDict[queue]['ParametersDict'] vo = getVO() if not vo: self.log.error( 'Virtual Organization is not defined in the configuration') return None pilotOptions = ["-V '%s'" % vo] setup = gConfig.getValue("/DIRAC/Setup", "unknown") if setup == 'unknown': self.log.error('Setup is not defined in the configuration') return None pilotOptions.append('-S %s' % setup) diracVersion = gConfig.getValue( "/Operations/%s/%s/Versions/PilotVersion" % (vo, setup), "unknown") if diracVersion == 'unknown': self.log.error('PilotVersion is not defined in the configuration') return None pilotOptions.append('-r %s' % diracVersion) projectName = gConfig.getValue( "/Operations/%s/%s/Versions/PilotInstallation" % (vo, setup), "") if projectName == '': self.log.info('DIRAC installation will be installed by pilots') else: pilotOptions.append('-l %s' % projectName) ownerDN = self.genericPilotDN ownerGroup = self.genericPilotGroup result = gProxyManager.requestToken(ownerDN, ownerGroup, pilotsToSubmit * 5) if not result['OK']: self.log.error(ERROR_TOKEN, result['Message']) return S_ERROR(ERROR_TOKEN) (token, numberOfUses) = result['Value'] pilotOptions.append('-o /Security/ProxyToken=%s' % token) # Use Filling mode pilotOptions.append('-M %s' % 5) # Debug pilotOptions.append('-d') # CS Servers csServers = gConfig.getValue("/DIRAC/Configuration/Servers", []) pilotOptions.append('-C %s' % ",".join(csServers)) # DIRAC Extensions extensionsList = getCSExtensions() if extensionsList: pilotOptions.append('-e %s' % ",".join(extensionsList)) # Requested CPU time pilotOptions.append('-T %s' % queueDict['CPUTime']) # SiteName pilotOptions.append('-n %s' % queueDict['Site']) if 'ClientPlatform' in queueDict: pilotOptions.append("-p '%s'" % queueDict['ClientPlatform']) if 'SharedArea' in queueDict: pilotOptions.append("-o '/LocalSite/SharedArea=%s'" % queueDict['SharedArea']) if 'SI00' in queueDict: factor = float(queueDict['SI00']) / 250. pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" % factor) pilotOptions.append("-o '/LocalSite/CPUNormalizationFactor=%s'" % factor) else: if 'CPUScalingFactor' in queueDict: pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" % queueDict['CPUScalingFactor']) if 'CPUNormalizationFactor' in queueDict: pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % queueDict['CPUNormalizationFactor']) self.log.verbose("pilotOptions: ", ' '.join(pilotOptions)) return pilotOptions
def __getPilotOptions( self, queue, pilotsToSubmit ): """ Prepare pilot options """ queueDict = self.queueDict[queue]['ParametersDict'] vo = getVO() if not vo: self.log.error( 'Virtual Organization is not defined in the configuration' ) return None pilotOptions = [ "-V '%s'" % vo ] setup = gConfig.getValue( "/DIRAC/Setup", "unknown" ) if setup == 'unknown': self.log.error( 'Setup is not defined in the configuration' ) return None pilotOptions.append( '-S %s' % setup ) diracVersion = gConfig.getValue( "/Operations/%s/%s/Versions/PilotVersion" % ( vo, setup ), "unknown" ) if diracVersion == 'unknown': self.log.error( 'PilotVersion is not defined in the configuration' ) return None pilotOptions.append( '-r %s' % diracVersion ) projectName = gConfig.getValue( "/Operations/%s/%s/Versions/PilotProject" % ( vo, setup ), "unknown" ) if projectName == 'unknown': self.log.info( 'PilotProject is not defined in the configuration' ) else: pilotOptions.append( '-l %s' % projectName ) ownerDN = self.genericPilotDN ownerGroup = self.genericPilotGroup result = gProxyManager.requestToken( ownerDN, ownerGroup, pilotsToSubmit * 5 ) if not result[ 'OK' ]: self.log.error( ERROR_TOKEN, result['Message'] ) return S_ERROR( ERROR_TOKEN ) ( token, numberOfUses ) = result[ 'Value' ] pilotOptions.append( '-o /Security/ProxyToken=%s' % token ) # Use Filling mode pilotOptions.append( '-M %s' % 5 ) # Debug pilotOptions.append( '-d' ) # CS Servers csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] ) pilotOptions.append( '-C %s' % ",".join( csServers ) ) # DIRAC Extensions extensionsList = getCSExtensions() if extensionsList: pilotOptions.append( '-e %s' % ",".join( extensionsList ) ) # Requested CPU time pilotOptions.append( '-T %s' % queueDict['CPUTime'] ) # SiteName pilotOptions.append( '-n %s' % queueDict['Site'] ) if 'ClientPlatform' in queueDict: pilotOptions.append( "-p '%s'" % queueDict['ClientPlatform'] ) if 'SharedArea' in queueDict: pilotOptions.append( "-o '/LocalSite/SharedArea=%s'" % queueDict['SharedArea'] ) if 'SI00' in queueDict: factor = float( queueDict['SI00'] ) / 250. pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % factor ) pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % factor ) else: if 'CPUScalingFactor' in queueDict: pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % queueDict['CPUScalingFactor'] ) if 'CPUNormalizationFactor' in queueDict: pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % queueDict['CPUNormalizationFactor'] ) self.log.verbose( "pilotOptions: ", ' '.join( pilotOptions ) ) return pilotOptions
def _prepareJDL(self, taskQueueDict, workingDirectory, pilotOptions, pilotsToSubmit, ceMask, submitPrivatePilot, privateTQ): """ Write JDL for Pilot Submission """ # RB = List.randomize( self.resourceBrokers )[0] LDs = [] NSs = [] LBs = [] # Select Randomly one RB from the list RB = List.randomize(self.resourceBrokers)[0] LDs.append('"%s:9002"' % RB) LBs.append('"%s:9000"' % RB) for LB in self.loggingServers: NSs.append('"%s:7772"' % LB) LD = ', '.join(LDs) NS = ', '.join(NSs) LB = ', '.join(LBs) vo = getVO() if privateTQ or vo not in ['lhcb']: extraReq = "True" else: if submitPrivatePilot: extraReq = "! AllowsGenericPilot" else: extraReq = "AllowsGenericPilot" rbJDL = """ AllowsGenericPilot = Member( "VO-lhcb-pilot" , other.GlueHostApplicationSoftwareRunTimeEnvironment ); Requirements = pilotRequirements && other.GlueCEStateStatus == "Production" && %s; RetryCount = 0; ErrorStorage = "%s/pilotError"; OutputStorage = "%s/pilotOutput"; # ListenerPort = 44000; ListenerStorage = "%s/Storage"; VirtualOrganisation = "lhcb"; LoggingTimeout = 30; LoggingSyncTimeout = 30; LoggingDestination = { %s }; # Default NS logger level is set to 0 (null) # max value is 6 (very ugly) NSLoggerLevel = 0; DefaultLogInfoLevel = 0; DefaultStatusLevel = 0; NSAddresses = { %s }; LBAddresses = { %s }; MyProxyServer = "no-myproxy.cern.ch"; """ % (extraReq, workingDirectory, workingDirectory, workingDirectory, LD, NS, LB) pilotJDL, pilotRequirements = self._JobJDL(taskQueueDict, pilotOptions, ceMask) jdl = os.path.join(workingDirectory, '%s.jdl' % taskQueueDict['TaskQueueID']) jdl = self._writeJDL(jdl, [pilotJDL, rbJDL]) return { 'JDL': jdl, 'Requirements': pilotRequirements + " && " + extraReq, 'Pilots': pilotsToSubmit, 'RB': RB }
def getDefaultVOMSVO(): vomsVO = gConfig.getValue("%s/DefaultVOMSVO" % gBaseSecuritySection, "") if vomsVO: return vomsVO return getVO()
else: Script.enableCS() else: Script.localCfg.addDefaultEntry( '/DIRAC/Security/UseServerCertificate', 'yes' ) Script.enableCS() if includeAllServers: DIRAC.gConfig.setOptionValue( '/DIRAC/Configuration/Servers', ','.join( DIRAC.gConfig.getServersList() ) ) DIRAC.gLogger.debug( '/DIRAC/Configuration/Servers =', ','.join( DIRAC.gConfig.getServersList() ) ) if update: DIRAC.gConfig.dumpLocalCFGToFile( DIRAC.gConfig.diracConfigFilePath ) #Do the vomsdir magic voName = getVO() if not voName: sys.exit( 0 ) result = DIRAC.gConfig.getSections( "/Registry/VOMS/Servers/%s" % voName ) if not result[ 'OK' ]: sys.exit( 0 ) DIRAC.gLogger.notice( "Creating VOMSDIR/VOMSES files" ) vomsDirHosts = result[ 'Value' ] vomsDirPath = os.path.join( DIRAC.rootPath, 'etc', 'grid-security', 'vomsdir', voName ) vomsesDirPath = os.path.join( DIRAC.rootPath, 'etc', 'grid-security', 'vomses' ) for path in ( vomsDirPath, vomsesDirPath ): if not os.path.isdir( path ): try: os.makedirs( path ) except Exception, e: DIRAC.gLogger.error( "Could not create directory", str( e ) )
baseDir = switch[1] if switch[0].lower() == "e" or switch[0].lower() == "emptydirs": emptyDirsFlag = True import DIRAC from DIRAC import gLogger from DIRAC.ConfigurationSystem.Client.Helpers import getVO from DIRAC.Core.Security.Misc import getProxyInfo from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager from DIRAC.Core.Utilities.List import sortList from datetime import datetime, timedelta import sys, os, time, fnmatch rm = ReplicaManager() vo = getVO('lhcb') def isOlderThan(cTimeStruct, days): timeDelta = timedelta(days=days) maxCTime = datetime.utcnow() - timeDelta if cTimeStruct < maxCTime: return True return False verbose = False if days or months or years: verbose = True totalDays = 0 if years:
def _getPilotOptions(self, taskQueueDict, pilotsToSubmit): # Need to limit the maximum number of pilots to submit at once # For generic pilots this is limited by the number of use of the tokens and the # maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation: pilotsToSubmit = min(pilotsToSubmit, int(50 / self.maxJobsInFillMode)) pilotOptions = ["-V %s" % getVO("lhcb")] privateIfGenericTQ = self.privatePilotFraction > random.random() privateTQ = ('PilotTypes' in taskQueueDict and 'private' in [t.lower() for t in taskQueueDict['PilotTypes']]) forceGeneric = 'ForceGeneric' in taskQueueDict submitPrivatePilot = (privateIfGenericTQ or privateTQ) and not forceGeneric if submitPrivatePilot: self.log.verbose('Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID']) ownerDN = taskQueueDict['OwnerDN'] ownerGroup = taskQueueDict['OwnerGroup'] # User Group requirement pilotOptions.append('-G %s' % taskQueueDict['OwnerGroup']) # check if group allows jobsharing ownerGroupProperties = getPropertiesForGroup(ownerGroup) if not 'JobSharing' in ownerGroupProperties: # Add Owner requirement to pilot pilotOptions.append("-O '%s'" % ownerDN) if privateTQ: pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private') maxJobsInFillMode = self.maxJobsInFillMode else: #For generic jobs we'll submit mixture of generic and private pilots self.log.verbose('Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID']) ownerDN = self.genericPilotDN ownerGroup = self.genericPilotGroup result = gProxyManager.requestToken( ownerDN, ownerGroup, max(pilotsToSubmit, self.maxJobsInFillMode)) if not result['OK']: self.log.error(ERROR_TOKEN, result['Message']) return S_ERROR(ERROR_TOKEN) (token, numberOfUses) = result['Value'] pilotsToSubmit = min(numberOfUses, pilotsToSubmit) pilotOptions.append('-o /Security/ProxyToken=%s' % token) pilotsToSubmit = (pilotsToSubmit - 1) / self.maxJobsInFillMode + 1 maxJobsInFillMode = int(numberOfUses / pilotsToSubmit) # Use Filling mode pilotOptions.append('-M %s' % maxJobsInFillMode) # Debug pilotOptions.append('-d') # Setup. pilotOptions.append('-S %s' % taskQueueDict['Setup']) # CS Servers csServers = gConfig.getValue("/DIRAC/Configuration/Servers", []) pilotOptions.append('-C %s' % ",".join(csServers)) # DIRAC Extensions extensionsList = getCSExtensions() if extensionsList: pilotOptions.append('-e %s' % ",".join(extensionsList)) # Requested version of DIRAC pilotOptions.append('-r %s' % self.installVersion) # Requested Project to install pilotOptions.append('-l %s' % self.installProject) # Requested CPU time pilotOptions.append('-T %s' % taskQueueDict['CPUTime']) if self.extraPilotOptions: pilotOptions.extend(self.extraPilotOptions) return S_OK((pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ))
def getDefaultVOMSVO(): vomsVO = gConfig.getValue( "%s/DefaultVOMSVO" % gBaseSecuritySection, "" ) if vomsVO: return vomsVO return getVO()
""" Report info about local DIRAC installation """ __RCSID__ = "$Id$" from pprint import pprint import DIRAC from DIRAC import gConfig from DIRAC.Core.Base import Script from DIRAC.ConfigurationSystem.Client.Helpers import getVO Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... Site' % Script.scriptName, ])) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() infoDict = {} infoDict['Setup'] = gConfig.getValue('/DIRAC/Setup', 'Unknown') infoDict['ConfigurationServer'] = gConfig.getValue( '/DIRAC/Configuration/Servers', []) infoDict['VirtualOrganization'] = getVO('Unknown') print 'DIRAC version'.rjust(20), ':', DIRAC.version for k, v in infoDict.items(): print k.rjust(20), ':', str(v)