def initialize( self ):
self.siteJobLimits = self.getCSOption( "SiteJobLimits", False )
self.checkPilotVersion = self.getCSOption( "CheckPilotVersion", True )
self.setup = gConfig.getValue( '/DIRAC/Setup', '' )
self.vo = getVO()
self.pilotVersion = gConfig.getValue( '/Operations/%s/%s/Versions/PilotVersion' % ( self.vo, self.setup ), '' )
def configure(self, csSection, submitPool):
"""
Here goes common configuration for all PilotDirectors
"""
self.configureFromSection(csSection)
self.reloadConfiguration(csSection, submitPool)
setup = gConfig.getValue('/DIRAC/Setup', '')
vo = getVO()
self.installVersion = gConfig.getValue(
'/Operations/%s/%s/Versions/PilotVersion' % (vo, setup),
self.installVersion)
self.installProject = gConfig.getValue(
'/Operations/%s/%s/Versions/PilotProject' % (vo, setup), "")
self.log.info('===============================================')
self.log.info('Configuration:')
self.log.info('')
self.log.info(' Install script: ', self.install)
self.log.info(' Pilot script: ', self.pilot)
self.log.info(' Install Ver: ', self.installVersion)
if self.installProject:
self.log.info(' Project: ', self.installProject)
if self.extraPilotOptions:
self.log.info(' Exta Options: ',
' '.join(self.extraPilotOptions))
self.log.info(' ListMatch: ', self.enableListMatch)
self.log.info(' Private %: ', self.privatePilotFraction * 100)
if self.enableListMatch:
self.log.info(' ListMatch Delay:', self.listMatchDelay)
self.listMatchCache.purgeExpired()
def initializeOptimizer(self):
"""Initialize specific parameters for JobSanityAgent.
"""
#Test control flags N.B. JDL check is mandatory
self.inputDataCheck = self.am_getOption('InputDataCheck', 1)
self.outputDataCheck = self.am_getOption('OutputDataCheck', 0)
self.inputSandboxCheck = self.am_getOption('InputSandboxCheck', 1)
self.platformCheck = self.am_getOption('PlatformCheck', 0)
#Other parameters
self.voName = getVO('lhcb')
self.successStatus = self.am_getOption('SuccessfulJobStatus',
'OutputReady')
self.maxDataPerJob = self.am_getOption('MaxInputDataPerJob', 100)
#Sandbox
self.sandboxClient = SandboxStoreClient(useCertificates=True)
self.log.debug('JDL Check ==> Enabled')
if self.inputDataCheck:
self.log.debug('Input Data Check ==> Enabled')
else:
self.log.debug('Input Data Check ==> Disabled')
if self.outputDataCheck:
self.log.debug('Output Data Check ==> Enabled')
else:
self.log.debug('Output Data Check ==> Disabled')
if self.inputSandboxCheck:
self.log.debug('Input Sbox Check ==> Enabled')
else:
self.log.debug('Input Sbox Check ==> Disabled')
if self.platformCheck:
self.log.debug('Platform Check ==> Enabled')
else:
self.log.debug('Platform Check ==> Disabled')
return S_OK()
def initialize(self):
self.siteJobLimits = self.getCSOption("SiteJobLimits", False)
self.checkPilotVersion = self.getCSOption("CheckPilotVersion", True)
self.setup = gConfig.getValue("/DIRAC/Setup", "")
self.vo = getVO()
self.pilotVersion = gConfig.getValue("/Operations/%s/%s/Versions/PilotVersion" % (self.vo, self.setup), "")
def configure( self, csSection, submitPool ):
"""
Here goes common configuration for all PilotDirectors
"""
self.configureFromSection( csSection )
self.reloadConfiguration( csSection, submitPool )
setup = gConfig.getValue( '/DIRAC/Setup', '' )
vo = getVO()
self.installVersion = gConfig.getValue( '/Operations/%s/%s/Versions/PilotVersion' % ( vo, setup ),
self.installVersion )
self.installProject = gConfig.getValue( '/Operations/%s/%s/Versions/PilotProject' % ( vo, setup ),
"" )
self.log.info( '===============================================' )
self.log.info( 'Configuration:' )
self.log.info( '' )
self.log.info( ' Install script: ', self.install )
self.log.info( ' Pilot script: ', self.pilot )
self.log.info( ' Install Ver: ', self.installVersion )
if self.installProject:
self.log.info( ' Project: ', self.installProject )
if self.extraPilotOptions:
self.log.info( ' Exta Options: ', ' '.join( self.extraPilotOptions ) )
self.log.info( ' ListMatch: ', self.enableListMatch )
self.log.info( ' Private %: ', self.privatePilotFraction * 100 )
if self.enableListMatch:
self.log.info( ' ListMatch Delay:', self.listMatchDelay )
self.listMatchCache.purgeExpired()
def initializeOptimizer( self ):
"""Initialize specific parameters for JobSanityAgent.
"""
#Test control flags N.B. JDL check is mandatory
self.inputDataCheck = self.am_getOption( 'InputDataCheck', 1 )
self.outputDataCheck = self.am_getOption( 'OutputDataCheck', 0 )
self.inputSandboxCheck = self.am_getOption( 'InputSandboxCheck', 1 )
self.platformCheck = self.am_getOption( 'PlatformCheck', 0 )
#Other parameters
self.voName = getVO( 'lhcb' )
self.successStatus = self.am_getOption( 'SuccessfulJobStatus', 'OutputReady' )
self.maxDataPerJob = self.am_getOption( 'MaxInputDataPerJob', 100 )
#Sandbox
self.sandboxClient = SandboxStoreClient( useCertificates = True )
self.log.debug( 'JDL Check ==> Enabled' )
if self.inputDataCheck:
self.log.debug( 'Input Data Check ==> Enabled' )
else:
self.log.debug( 'Input Data Check ==> Disabled' )
if self.outputDataCheck:
self.log.debug( 'Output Data Check ==> Enabled' )
else:
self.log.debug( 'Output Data Check ==> Disabled' )
if self.inputSandboxCheck:
self.log.debug( 'Input Sbox Check ==> Enabled' )
else:
self.log.debug( 'Input Sbox Check ==> Disabled' )
if self.platformCheck:
self.log.debug( 'Platform Check ==> Enabled' )
else:
self.log.debug( 'Platform Check ==> Disabled' )
return S_OK()
def __init__( self,
server = False,
serverCert = False,
serverKey = False,
voName = False,
timeout = False ):
if timeout:
self._secCmdTimeout = timeout
else:
self._secCmdTimeout = 30
if not server:
self._secServer = gConfig.getValue( "/DIRAC/VOPolicy/MyProxyServer", "myproxy.cern.ch" )
else:
self._secServer = server
if not voName:
self._secVO = getVO( "unknown" )
else:
self._secVO = voName
ckLoc = Locations.getHostCertificateAndKeyLocation()
if serverCert:
self._secCertLoc = serverCert
else:
if ckLoc:
self._secCertLoc = ckLoc[0]
else:
self._secCertLoc = "%s/etc/grid-security/servercert.pem" % DIRAC.rootPath
if serverKey:
self._secKeyLoc = serverKey
else:
if ckLoc:
self._secKeyLoc = ckLoc[1]
else:
self._secKeyLoc = "%s/etc/grid-security/serverkey.pem" % DIRAC.rootPath
self._secRunningFromTrustedHost = gConfig.getValue( "/DIRAC/VOPolicy/MyProxyTrustedHost", "True" ).lower() in ( "y", "yes", "true" )
self._secMaxProxyHours = gConfig.getValue( "/DIRAC/VOPolicy/MyProxyMaxDelegationTime", 168 )
def _prepareJDL( self, taskQueueDict, workingDirectory, pilotOptions, pilotsToSubmit, ceMask, submitPrivatePilot, privateTQ ):
"""
Write JDL for Pilot Submission
"""
# RB = List.randomize( self.resourceBrokers )[0]
LDs = []
NSs = []
LBs = []
# Select Randomly one RB from the list
RB = List.randomize( self.resourceBrokers )[0]
LDs.append( '"%s:9002"' % RB )
LBs.append( '"%s:9000"' % RB )
for LB in self.loggingServers:
NSs.append( '"%s:7772"' % LB )
LD = ', '.join( LDs )
NS = ', '.join( NSs )
LB = ', '.join( LBs )
vo = getVO()
if privateTQ or vo not in ['lhcb']:
extraReq = "True"
else:
if submitPrivatePilot:
extraReq = "! AllowsGenericPilot"
else:
extraReq = "AllowsGenericPilot"
rbJDL = """
AllowsGenericPilot = Member( "VO-lhcb-pilot" , other.GlueHostApplicationSoftwareRunTimeEnvironment );
Requirements = pilotRequirements && other.GlueCEStateStatus == "Production" && %s;
RetryCount = 0;
ErrorStorage = "%s/pilotError";
OutputStorage = "%s/pilotOutput";
# ListenerPort = 44000;
ListenerStorage = "%s/Storage";
VirtualOrganisation = "lhcb";
LoggingTimeout = 30;
LoggingSyncTimeout = 30;
LoggingDestination = { %s };
# Default NS logger level is set to 0 (null)
# max value is 6 (very ugly)
NSLoggerLevel = 0;
DefaultLogInfoLevel = 0;
DefaultStatusLevel = 0;
NSAddresses = { %s };
LBAddresses = { %s };
MyProxyServer = "no-myproxy.cern.ch";
""" % ( extraReq, workingDirectory, workingDirectory, workingDirectory, LD, NS, LB )
pilotJDL, pilotRequirements = self._JobJDL( taskQueueDict, pilotOptions, ceMask )
jdl = os.path.join( workingDirectory, '%s.jdl' % taskQueueDict['TaskQueueID'] )
jdl = self._writeJDL( jdl, [pilotJDL, rbJDL] )
return {'JDL':jdl, 'Requirements':pilotRequirements + " && " + extraReq, 'Pilots': pilotsToSubmit, 'RB':RB }
def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ):
"""Instantiates the Workflow object and some default parameters.
"""
self.log = gLogger
self.section = COMPONENT_NAME
self.dbg = False
if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG':
self.dbg = True
#gConfig.getValue('Tier0SE-tape','SEName')
self.stepCount = 0
self.owner = 'NotSpecified'
self.name = 'Name'
self.type = 'User'
self.priority = 1
self.group = getVO( 'lhcb' )
self.site = 'ANY' #ANY
#self.setup = 'Development'
self.origin = 'DIRAC'
self.stdout = stdout
self.stderr = stderr
self.logLevel = 'info'
self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified
self.addToInputSandbox = []
self.addToOutputSandbox = []
self.addToInputData = []
self.systemConfig = 'ANY'
self.reqParams = {'MaxCPUTime': 'other.NAME>=VALUE',
'MinCPUTime': 'other.NAME<=VALUE',
'Site': 'other.NAME=="VALUE"',
'Platform': 'other.NAME=="VALUE"',
#'BannedSites': '!Member(other.Site,BannedSites)', #doesn't work unfortunately
'BannedSites': 'other.Site!="VALUE"',
'SystemConfig': 'Member("VALUE",other.CompatiblePlatforms)'}
##Add member to handle Parametric jobs
self.parametric = {}
self.script = script
if not script:
self.workflow = Workflow()
self.__setJobDefaults()
else:
self.workflow = Workflow( script )
#Global error dictionary
self.errorDict = {}
def __init__(self, submitPool):
"""
Define the logger and some defaults
"""
if submitPool == self.gridMiddleware:
self.log = gLogger.getSubLogger('%sPilotDirector' %
self.gridMiddleware)
else:
self.log = gLogger.getSubLogger('%sPilotDirector/%s' %
(self.gridMiddleware, submitPool))
self.pilot = DIRAC_PILOT
self.extraPilotOptions = []
setup = gConfig.getValue('/DIRAC/Setup', '')
vo = getVO()
self.installVersion = gConfig.getValue(
'/Operations/%s/%s/Versions/PilotVersion' % (vo, setup),
DIRAC_VERSION)
self.installProject = gConfig.getValue(
'/Operations/%s/%s/Versions/PilotProject' % (vo, setup), "")
self.install = DIRAC_INSTALL
self.maxJobsInFillMode = MAX_JOBS_IN_FILLMODE
self.genericPilotDN = PILOT_DN
self.genericPilotGroup = PILOT_GROUP
self.enableListMatch = ENABLE_LISTMATCH
self.listMatchDelay = LISTMATCH_DELAY
self.listMatchCache = DictCache()
self.privatePilotFraction = PRIVATE_PILOT_FRACTION
self.errorClearTime = ERROR_CLEAR_TIME
self.errorTicketTime = ERROR_TICKET_TIME
self.errorMailAddress = DIRAC.errorMail
self.alarmMailAddress = DIRAC.alarmMail
self.mailFromAddress = FROM_MAIL
if not 'log' in self.__dict__:
self.log = gLogger.getSubLogger('PilotDirector')
self.log.info('Initialized')
def __init__( self, submitPool ):
"""
Define the logger and some defaults
"""
if submitPool == self.gridMiddleware:
self.log = gLogger.getSubLogger( '%sPilotDirector' % self.gridMiddleware )
else:
self.log = gLogger.getSubLogger( '%sPilotDirector/%s' % ( self.gridMiddleware, submitPool ) )
self.pilot = DIRAC_PILOT
self.extraPilotOptions = []
setup = gConfig.getValue( '/DIRAC/Setup', '' )
vo = getVO()
self.installVersion = gConfig.getValue( '/Operations/%s/%s/Versions/PilotVersion' % ( vo, setup ),
DIRAC_VERSION )
self.installProject = gConfig.getValue( '/Operations/%s/%s/Versions/PilotProject' % ( vo, setup ),
"" )
self.install = DIRAC_INSTALL
self.maxJobsInFillMode = MAX_JOBS_IN_FILLMODE
self.genericPilotDN = PILOT_DN
self.genericPilotGroup = PILOT_GROUP
self.enableListMatch = ENABLE_LISTMATCH
self.listMatchDelay = LISTMATCH_DELAY
self.listMatchCache = DictCache()
self.privatePilotFraction = PRIVATE_PILOT_FRACTION
self.errorClearTime = ERROR_CLEAR_TIME
self.errorTicketTime = ERROR_TICKET_TIME
self.errorMailAddress = DIRAC.errorMail
self.alarmMailAddress = DIRAC.alarmMail
self.mailFromAddress = FROM_MAIL
if not 'log' in self.__dict__:
self.log = gLogger.getSubLogger( 'PilotDirector' )
self.log.info( 'Initialized' )
def __init__(self,
server=False,
serverCert=False,
serverKey=False,
voName=False,
timeout=False):
if timeout:
self._secCmdTimeout = timeout
else:
self._secCmdTimeout = 30
if not server:
self._secServer = gConfig.getValue("/DIRAC/VOPolicy/MyProxyServer",
"myproxy.cern.ch")
else:
self._secServer = server
if not voName:
self._secVO = getVO("unknown")
else:
self._secVO = voName
ckLoc = Locations.getHostCertificateAndKeyLocation()
if serverCert:
self._secCertLoc = serverCert
else:
if ckLoc:
self._secCertLoc = ckLoc[0]
else:
self._secCertLoc = "%s/etc/grid-security/servercert.pem" % DIRAC.rootPath
if serverKey:
self._secKeyLoc = serverKey
else:
if ckLoc:
self._secKeyLoc = ckLoc[1]
else:
self._secKeyLoc = "%s/etc/grid-security/serverkey.pem" % DIRAC.rootPath
self._secRunningFromTrustedHost = gConfig.getValue(
"/DIRAC/VOPolicy/MyProxyTrustedHost",
"True").lower() in ("y", "yes", "true")
self._secMaxProxyHours = gConfig.getValue(
"/DIRAC/VOPolicy/MyProxyMaxDelegationTime", 168)
# File : dirac-info
# Author : Andrei Tsaregorodtsev
########################################################################
"""
Report info about local DIRAC installation
"""
__RCSID__ = "$Id$"
from pprint import pprint
import DIRAC
from DIRAC import gConfig
from DIRAC.Core.Base import Script
from DIRAC.ConfigurationSystem.Client.Helpers import getVO
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ... Site' % Script.scriptName, ] ) )
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
infoDict = {}
infoDict['Setup'] = gConfig.getValue( '/DIRAC/Setup', 'Unknown' )
infoDict['ConfigurationServer'] = gConfig.getValue( '/DIRAC/Configuration/Servers', [] )
infoDict['VirtualOrganization'] = getVO( 'Unknown' )
print 'DIRAC version'.rjust( 20 ), ':', DIRAC.version
for k, v in infoDict.items():
print k.rjust( 20 ), ':', str( v )
' %s [option|cfgfile] ... CE' % Script.scriptName,
'Arguments:',
' CE: Name of the CE(ie: ce111.cern.ch)'] ) )
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
if not len( args ) == 1:
Script.showHelp()
ce = args[0]
host = None
vo = getVO( 'lhcb' )
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ( "H", "host" ):
host = unprocSw[1]
if unprocSw[0] in ( "V", "vo" ):
vo = unprocSw[1]
diracAdmin = DiracAdmin()
result = diracAdmin.getBDIICEState( ce, useVO = vo, host = host )
if not result['OK']:
print result['Message']
DIRAC.exit( 2 )
ces = result['Value']
else:
Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate',
'yes')
Script.enableCS()
if includeAllServers:
DIRAC.gConfig.setOptionValue('/DIRAC/Configuration/Servers',
','.join(DIRAC.gConfig.getServersList()))
DIRAC.gLogger.debug('/DIRAC/Configuration/Servers =',
','.join(DIRAC.gConfig.getServersList()))
if update:
DIRAC.gConfig.dumpLocalCFGToFile(DIRAC.gConfig.diracConfigFilePath)
#Do the vomsdir magic
voName = getVO()
if not voName:
sys.exit(0)
result = DIRAC.gConfig.getSections("/Registry/VOMS/Servers/%s" % voName)
if not result['OK']:
sys.exit(0)
DIRAC.gLogger.notice("Creating VOMSDIR/VOMSES files")
vomsDirHosts = result['Value']
vomsDirPath = os.path.join(DIRAC.rootPath, 'etc', 'grid-security', 'vomsdir',
voName)
vomsesDirPath = os.path.join(DIRAC.rootPath, 'etc', 'grid-security', 'vomses')
for path in (vomsDirPath, vomsesDirPath):
if not os.path.isdir(path):
try:
os.makedirs(path)
except Exception, e:
def _getPilotOptions( self, taskQueueDict, pilotsToSubmit ):
# Need to limit the maximum number of pilots to submit at once
# For generic pilots this is limited by the number of use of the tokens and the
# maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation:
pilotsToSubmit = min( pilotsToSubmit, int( 50 / self.maxJobsInFillMode ) )
pilotOptions = [ "-V %s" % getVO( "lhcb" ) ]
privateIfGenericTQ = self.privatePilotFraction > random.random()
privateTQ = ( 'PilotTypes' in taskQueueDict and 'private' in [ t.lower() for t in taskQueueDict['PilotTypes'] ] )
forceGeneric = 'ForceGeneric' in taskQueueDict
submitPrivatePilot = ( privateIfGenericTQ or privateTQ ) and not forceGeneric
if submitPrivatePilot:
self.log.verbose( 'Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
ownerDN = taskQueueDict['OwnerDN']
ownerGroup = taskQueueDict['OwnerGroup']
# User Group requirement
pilotOptions.append( '-G %s' % taskQueueDict['OwnerGroup'] )
# check if group allows jobsharing
ownerGroupProperties = getPropertiesForGroup( ownerGroup )
if not 'JobSharing' in ownerGroupProperties:
# Add Owner requirement to pilot
pilotOptions.append( "-O '%s'" % ownerDN )
if privateTQ:
pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private' )
maxJobsInFillMode = self.maxJobsInFillMode
else:
#For generic jobs we'll submit mixture of generic and private pilots
self.log.verbose( 'Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
ownerDN = self.genericPilotDN
ownerGroup = self.genericPilotGroup
result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) )
if not result[ 'OK' ]:
self.log.error( ERROR_TOKEN, result['Message'] )
return S_ERROR( ERROR_TOKEN )
( token, numberOfUses ) = result[ 'Value' ]
pilotsToSubmit = min( numberOfUses, pilotsToSubmit )
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
pilotsToSubmit = ( pilotsToSubmit - 1 ) / self.maxJobsInFillMode + 1
maxJobsInFillMode = int( numberOfUses / pilotsToSubmit )
# Use Filling mode
pilotOptions.append( '-M %s' % maxJobsInFillMode )
# Debug
pilotOptions.append( '-d' )
# Setup.
pilotOptions.append( '-S %s' % taskQueueDict['Setup'] )
# CS Servers
csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
# Requested version of DIRAC
pilotOptions.append( '-r %s' % self.installVersion )
# Requested Project to install
pilotOptions.append( '-l %s' % self.installProject )
# Requested CPU time
pilotOptions.append( '-T %s' % taskQueueDict['CPUTime'] )
if self.extraPilotOptions:
pilotOptions.extend( self.extraPilotOptions )
return S_OK( ( pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ ) )
def __getPilotOptions(self, queue, pilotsToSubmit):
""" Prepare pilot options
"""
queueDict = self.queueDict[queue]['ParametersDict']
vo = getVO()
if not vo:
self.log.error(
'Virtual Organization is not defined in the configuration')
return None
pilotOptions = ["-V '%s'" % vo]
setup = gConfig.getValue("/DIRAC/Setup", "unknown")
if setup == 'unknown':
self.log.error('Setup is not defined in the configuration')
return None
pilotOptions.append('-S %s' % setup)
diracVersion = gConfig.getValue(
"/Operations/%s/%s/Versions/PilotVersion" % (vo, setup), "unknown")
if diracVersion == 'unknown':
self.log.error('PilotVersion is not defined in the configuration')
return None
pilotOptions.append('-r %s' % diracVersion)
projectName = gConfig.getValue(
"/Operations/%s/%s/Versions/PilotInstallation" % (vo, setup), "")
if projectName == '':
self.log.info('DIRAC installation will be installed by pilots')
else:
pilotOptions.append('-l %s' % projectName)
ownerDN = self.genericPilotDN
ownerGroup = self.genericPilotGroup
result = gProxyManager.requestToken(ownerDN, ownerGroup,
pilotsToSubmit * 5)
if not result['OK']:
self.log.error(ERROR_TOKEN, result['Message'])
return S_ERROR(ERROR_TOKEN)
(token, numberOfUses) = result['Value']
pilotOptions.append('-o /Security/ProxyToken=%s' % token)
# Use Filling mode
pilotOptions.append('-M %s' % 5)
# Debug
pilotOptions.append('-d')
# CS Servers
csServers = gConfig.getValue("/DIRAC/Configuration/Servers", [])
pilotOptions.append('-C %s' % ",".join(csServers))
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append('-e %s' % ",".join(extensionsList))
# Requested CPU time
pilotOptions.append('-T %s' % queueDict['CPUTime'])
# SiteName
pilotOptions.append('-n %s' % queueDict['Site'])
if 'ClientPlatform' in queueDict:
pilotOptions.append("-p '%s'" % queueDict['ClientPlatform'])
if 'SharedArea' in queueDict:
pilotOptions.append("-o '/LocalSite/SharedArea=%s'" %
queueDict['SharedArea'])
if 'SI00' in queueDict:
factor = float(queueDict['SI00']) / 250.
pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" % factor)
pilotOptions.append("-o '/LocalSite/CPUNormalizationFactor=%s'" %
factor)
else:
if 'CPUScalingFactor' in queueDict:
pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" %
queueDict['CPUScalingFactor'])
if 'CPUNormalizationFactor' in queueDict:
pilotOptions.append(
"-o '/LocalSite/CPUNormalizationFactor=%s'" %
queueDict['CPUNormalizationFactor'])
self.log.verbose("pilotOptions: ", ' '.join(pilotOptions))
return pilotOptions
def __getPilotOptions( self, queue, pilotsToSubmit ):
""" Prepare pilot options
"""
queueDict = self.queueDict[queue]['ParametersDict']
vo = getVO()
if not vo:
self.log.error( 'Virtual Organization is not defined in the configuration' )
return None
pilotOptions = [ "-V '%s'" % vo ]
setup = gConfig.getValue( "/DIRAC/Setup", "unknown" )
if setup == 'unknown':
self.log.error( 'Setup is not defined in the configuration' )
return None
pilotOptions.append( '-S %s' % setup )
diracVersion = gConfig.getValue( "/Operations/%s/%s/Versions/PilotVersion" % ( vo, setup ), "unknown" )
if diracVersion == 'unknown':
self.log.error( 'PilotVersion is not defined in the configuration' )
return None
pilotOptions.append( '-r %s' % diracVersion )
projectName = gConfig.getValue( "/Operations/%s/%s/Versions/PilotProject" % ( vo, setup ), "unknown" )
if projectName == 'unknown':
self.log.info( 'PilotProject is not defined in the configuration' )
else:
pilotOptions.append( '-l %s' % projectName )
ownerDN = self.genericPilotDN
ownerGroup = self.genericPilotGroup
result = gProxyManager.requestToken( ownerDN, ownerGroup, pilotsToSubmit * 5 )
if not result[ 'OK' ]:
self.log.error( ERROR_TOKEN, result['Message'] )
return S_ERROR( ERROR_TOKEN )
( token, numberOfUses ) = result[ 'Value' ]
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
# Use Filling mode
pilotOptions.append( '-M %s' % 5 )
# Debug
pilotOptions.append( '-d' )
# CS Servers
csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
# Requested CPU time
pilotOptions.append( '-T %s' % queueDict['CPUTime'] )
# SiteName
pilotOptions.append( '-n %s' % queueDict['Site'] )
if 'ClientPlatform' in queueDict:
pilotOptions.append( "-p '%s'" % queueDict['ClientPlatform'] )
if 'SharedArea' in queueDict:
pilotOptions.append( "-o '/LocalSite/SharedArea=%s'" % queueDict['SharedArea'] )
if 'SI00' in queueDict:
factor = float( queueDict['SI00'] ) / 250.
pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % factor )
pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % factor )
else:
if 'CPUScalingFactor' in queueDict:
pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % queueDict['CPUScalingFactor'] )
if 'CPUNormalizationFactor' in queueDict:
pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % queueDict['CPUNormalizationFactor'] )
self.log.verbose( "pilotOptions: ", ' '.join( pilotOptions ) )
return pilotOptions
def _prepareJDL(self, taskQueueDict, workingDirectory, pilotOptions,
pilotsToSubmit, ceMask, submitPrivatePilot, privateTQ):
"""
Write JDL for Pilot Submission
"""
# RB = List.randomize( self.resourceBrokers )[0]
LDs = []
NSs = []
LBs = []
# Select Randomly one RB from the list
RB = List.randomize(self.resourceBrokers)[0]
LDs.append('"%s:9002"' % RB)
LBs.append('"%s:9000"' % RB)
for LB in self.loggingServers:
NSs.append('"%s:7772"' % LB)
LD = ', '.join(LDs)
NS = ', '.join(NSs)
LB = ', '.join(LBs)
vo = getVO()
if privateTQ or vo not in ['lhcb']:
extraReq = "True"
else:
if submitPrivatePilot:
extraReq = "! AllowsGenericPilot"
else:
extraReq = "AllowsGenericPilot"
rbJDL = """
AllowsGenericPilot = Member( "VO-lhcb-pilot" , other.GlueHostApplicationSoftwareRunTimeEnvironment );
Requirements = pilotRequirements && other.GlueCEStateStatus == "Production" && %s;
RetryCount = 0;
ErrorStorage = "%s/pilotError";
OutputStorage = "%s/pilotOutput";
# ListenerPort = 44000;
ListenerStorage = "%s/Storage";
VirtualOrganisation = "lhcb";
LoggingTimeout = 30;
LoggingSyncTimeout = 30;
LoggingDestination = { %s };
# Default NS logger level is set to 0 (null)
# max value is 6 (very ugly)
NSLoggerLevel = 0;
DefaultLogInfoLevel = 0;
DefaultStatusLevel = 0;
NSAddresses = { %s };
LBAddresses = { %s };
MyProxyServer = "no-myproxy.cern.ch";
""" % (extraReq, workingDirectory, workingDirectory, workingDirectory, LD, NS,
LB)
pilotJDL, pilotRequirements = self._JobJDL(taskQueueDict, pilotOptions,
ceMask)
jdl = os.path.join(workingDirectory,
'%s.jdl' % taskQueueDict['TaskQueueID'])
jdl = self._writeJDL(jdl, [pilotJDL, rbJDL])
return {
'JDL': jdl,
'Requirements': pilotRequirements + " && " + extraReq,
'Pilots': pilotsToSubmit,
'RB': RB
}
def getDefaultVOMSVO():
vomsVO = gConfig.getValue("%s/DefaultVOMSVO" % gBaseSecuritySection, "")
if vomsVO:
return vomsVO
return getVO()
else:
Script.enableCS()
else:
Script.localCfg.addDefaultEntry( '/DIRAC/Security/UseServerCertificate', 'yes' )
Script.enableCS()
if includeAllServers:
DIRAC.gConfig.setOptionValue( '/DIRAC/Configuration/Servers', ','.join( DIRAC.gConfig.getServersList() ) )
DIRAC.gLogger.debug( '/DIRAC/Configuration/Servers =', ','.join( DIRAC.gConfig.getServersList() ) )
if update:
DIRAC.gConfig.dumpLocalCFGToFile( DIRAC.gConfig.diracConfigFilePath )
#Do the vomsdir magic
voName = getVO()
if not voName:
sys.exit( 0 )
result = DIRAC.gConfig.getSections( "/Registry/VOMS/Servers/%s" % voName )
if not result[ 'OK' ]:
sys.exit( 0 )
DIRAC.gLogger.notice( "Creating VOMSDIR/VOMSES files" )
vomsDirHosts = result[ 'Value' ]
vomsDirPath = os.path.join( DIRAC.rootPath, 'etc', 'grid-security', 'vomsdir', voName )
vomsesDirPath = os.path.join( DIRAC.rootPath, 'etc', 'grid-security', 'vomses' )
for path in ( vomsDirPath, vomsesDirPath ):
if not os.path.isdir( path ):
try:
os.makedirs( path )
except Exception, e:
DIRAC.gLogger.error( "Could not create directory", str( e ) )
baseDir = switch[1]
if switch[0].lower() == "e" or switch[0].lower() == "emptydirs":
emptyDirsFlag = True
import DIRAC
from DIRAC import gLogger
from DIRAC.ConfigurationSystem.Client.Helpers import getVO
from DIRAC.Core.Security.Misc import getProxyInfo
from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager
from DIRAC.Core.Utilities.List import sortList
from datetime import datetime, timedelta
import sys, os, time, fnmatch
rm = ReplicaManager()
vo = getVO('lhcb')
def isOlderThan(cTimeStruct, days):
timeDelta = timedelta(days=days)
maxCTime = datetime.utcnow() - timeDelta
if cTimeStruct < maxCTime:
return True
return False
verbose = False
if days or months or years:
verbose = True
totalDays = 0
if years:
def _getPilotOptions(self, taskQueueDict, pilotsToSubmit):
# Need to limit the maximum number of pilots to submit at once
# For generic pilots this is limited by the number of use of the tokens and the
# maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation:
pilotsToSubmit = min(pilotsToSubmit, int(50 / self.maxJobsInFillMode))
pilotOptions = ["-V %s" % getVO("lhcb")]
privateIfGenericTQ = self.privatePilotFraction > random.random()
privateTQ = ('PilotTypes' in taskQueueDict and 'private'
in [t.lower() for t in taskQueueDict['PilotTypes']])
forceGeneric = 'ForceGeneric' in taskQueueDict
submitPrivatePilot = (privateIfGenericTQ
or privateTQ) and not forceGeneric
if submitPrivatePilot:
self.log.verbose('Submitting private pilots for TaskQueue %s' %
taskQueueDict['TaskQueueID'])
ownerDN = taskQueueDict['OwnerDN']
ownerGroup = taskQueueDict['OwnerGroup']
# User Group requirement
pilotOptions.append('-G %s' % taskQueueDict['OwnerGroup'])
# check if group allows jobsharing
ownerGroupProperties = getPropertiesForGroup(ownerGroup)
if not 'JobSharing' in ownerGroupProperties:
# Add Owner requirement to pilot
pilotOptions.append("-O '%s'" % ownerDN)
if privateTQ:
pilotOptions.append(
'-o /Resources/Computing/CEDefaults/PilotType=private')
maxJobsInFillMode = self.maxJobsInFillMode
else:
#For generic jobs we'll submit mixture of generic and private pilots
self.log.verbose('Submitting generic pilots for TaskQueue %s' %
taskQueueDict['TaskQueueID'])
ownerDN = self.genericPilotDN
ownerGroup = self.genericPilotGroup
result = gProxyManager.requestToken(
ownerDN, ownerGroup, max(pilotsToSubmit,
self.maxJobsInFillMode))
if not result['OK']:
self.log.error(ERROR_TOKEN, result['Message'])
return S_ERROR(ERROR_TOKEN)
(token, numberOfUses) = result['Value']
pilotsToSubmit = min(numberOfUses, pilotsToSubmit)
pilotOptions.append('-o /Security/ProxyToken=%s' % token)
pilotsToSubmit = (pilotsToSubmit - 1) / self.maxJobsInFillMode + 1
maxJobsInFillMode = int(numberOfUses / pilotsToSubmit)
# Use Filling mode
pilotOptions.append('-M %s' % maxJobsInFillMode)
# Debug
pilotOptions.append('-d')
# Setup.
pilotOptions.append('-S %s' % taskQueueDict['Setup'])
# CS Servers
csServers = gConfig.getValue("/DIRAC/Configuration/Servers", [])
pilotOptions.append('-C %s' % ",".join(csServers))
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append('-e %s' % ",".join(extensionsList))
# Requested version of DIRAC
pilotOptions.append('-r %s' % self.installVersion)
# Requested Project to install
pilotOptions.append('-l %s' % self.installProject)
# Requested CPU time
pilotOptions.append('-T %s' % taskQueueDict['CPUTime'])
if self.extraPilotOptions:
pilotOptions.extend(self.extraPilotOptions)
return S_OK((pilotOptions, pilotsToSubmit, ownerDN, ownerGroup,
submitPrivatePilot, privateTQ))
def getDefaultVOMSVO():
vomsVO = gConfig.getValue( "%s/DefaultVOMSVO" % gBaseSecuritySection, "" )
if vomsVO:
return vomsVO
return getVO()
"""
Report info about local DIRAC installation
"""
__RCSID__ = "$Id$"
from pprint import pprint
import DIRAC
from DIRAC import gConfig
from DIRAC.Core.Base import Script
from DIRAC.ConfigurationSystem.Client.Helpers import getVO
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1],
'Usage:',
' %s [option|cfgfile] ... Site' % Script.scriptName,
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
infoDict = {}
infoDict['Setup'] = gConfig.getValue('/DIRAC/Setup', 'Unknown')
infoDict['ConfigurationServer'] = gConfig.getValue(
'/DIRAC/Configuration/Servers', [])
infoDict['VirtualOrganization'] = getVO('Unknown')
print 'DIRAC version'.rjust(20), ':', DIRAC.version
for k, v in infoDict.items():
print k.rjust(20), ':', str(v)