def getGroupsToUpload( self ):
uploadGroups = []
if self.__piParams.uploadProxy or Registry.getGroupOption( self.__piParams.diracGroup, "AutoUploadProxy", False ):
uploadGroups.append( self.__piParams.diracGroup )
if not self.__piParams.uploadPilot:
if not Registry.getGroupOption( self.__piParams.diracGroup, "AutoUploadPilotProxy", False ):
return uploadGroups
issuerCert = self.getIssuerCert()
resultUserDN = issuerCert.getSubjectDN() #pylint: disable=no-member
if not resultUserDN['OK']:
return resultUserDN
userDN = resultUserDN[ 'Value' ]
resultGroups = Registry.getGroupsForDN( userDN )
if not resultGroups[ 'OK' ]:
gLogger.error( "No groups defined for DN %s" % userDN )
return []
availableGroups = resultGroups[ 'Value' ]
for group in availableGroups:
groupProps = Registry.getPropertiesForGroup( group )
if Properties.PILOT in groupProps or Properties.GENERIC_PILOT in groupProps:
uploadGroups.append( group )
return uploadGroups
def getGroupsToUpload(self):
uploadGroups = []
if self.__piParams.uploadProxy or Registry.getGroupOption(
self.__piParams.diracGroup, "AutoUploadProxy", False):
uploadGroups.append(self.__piParams.diracGroup)
if not self.__piParams.uploadPilot:
if not Registry.getGroupOption(self.__piParams.diracGroup,
"AutoUploadPilotProxy", False):
return uploadGroups
issuerCert = self.getIssuerCert()
userDN = issuerCert.getSubjectDN()['Value']
result = Registry.getGroupsForDN(userDN)
if not result['OK']:
gLogger.error("No groups defined for DN %s" % userDN)
return []
availableGroups = result['Value']
pilotGroups = []
for group in availableGroups:
groupProps = Registry.getPropertiesForGroup(group)
if Properties.PILOT in groupProps or Properties.GENERIC_PILOT in groupProps:
uploadGroups.append(group)
return uploadGroups
def addVOMSExtIfNeeded(self):
addVOMS = self.__piParams.addVOMSExt or Registry.getGroupOption(
self.__piParams.diracGroup, "AutoAddVOMS", False
)
if not addVOMS:
return S_OK()
vomsAttr = Registry.getVOMSAttributeForGroup(self.__piParams.diracGroup)
if not vomsAttr:
return S_ERROR(
"Requested adding a VOMS extension but no VOMS attribute defined for group %s"
% self.__piParams.diracGroup
)
result = VOMS.VOMS().setVOMSAttributes(
self.__proxyGenerated, attribute=vomsAttr, vo=Registry.getVOForGroup(self.__piParams.diracGroup)
)
if not result["OK"]:
return S_ERROR(
"Could not add VOMS extensions to the proxy\nFailed adding VOMS attribute: %s" % result["Message"]
)
gLogger.notice("Added VOMS attribute %s" % vomsAttr)
chain = result["Value"]
chain.dumpAllToFile(self.__proxyGenerated)
return S_OK()
def addVOMSExtIfNeeded(self):
addVOMS = self.__piParams.addVOMSExt or Registry.getGroupOption(
self.__piParams.diracGroup, "AutoAddVOMS", False)
if not addVOMS:
return S_OK()
vomsAttr = Registry.getVOMSAttributeForGroup(
self.__piParams.diracGroup)
if not vomsAttr:
return S_ERROR(
"Requested adding a VOMS extension but no VOMS attribute defined for group %s"
% self.__piParams.diracGroup)
result = VOMS.VOMS().setVOMSAttributes(self.__proxyGenerated,
attribute=vomsAttr,
vo=Registry.getVOMSVOForGroup(
self.__piParams.diracGroup))
if not result['OK']:
return S_ERROR(
"Could not add VOMS extensions to the proxy\nFailed adding VOMS attribute: %s"
% result['Message'])
gLogger.notice("Added VOMS attribute %s" % vomsAttr)
chain = result['Value']
chain.dumpAllToFile(self.__proxyGenerated)
return S_OK()
def setMetadata(self, dPath, metaDict, credDict):
"""Set the value of a given metadata field for the the given directory path
:param str dPath: directory path
:param dict metaDict: dictionary with metadata
:param dict credDict: client credential dictionary
:return: S_OK/S_ERROR
"""
result = self._getMetadataFields(credDict)
if not result["OK"]:
return result
metaFields = result["Value"]
result = self.db.dtree.findDir(dPath)
if not result["OK"]:
return result
if not result["Value"]:
return S_ERROR("Path not found: %s" % dPath)
dirID = result["Value"]
dirmeta = self.getDirectoryMetadata(dPath, credDict, ownData=False)
if not dirmeta["OK"]:
return dirmeta
voName = Registry.getGroupOption(credDict["group"], "VO")
forceIndex = Operations(vo=voName).getValue(
"DataManagement/ForceIndexedMetadata", False)
for metaName, metaValue in metaDict.items():
if metaName not in metaFields:
if forceIndex:
return S_ERROR(
"Field %s not indexed, but ForceIndexedMetadata is set"
% metaName,
callStack=[])
result = self.setMetaParameter(dPath, metaName, metaValue,
credDict)
if not result["OK"]:
return result
continue
# Check that the metadata is not defined for the parent directories
if metaName in dirmeta["Value"]:
return S_ERROR(
"Metadata conflict detected for %s for directory %s" %
(metaName, dPath))
result = self.db.insertFields("FC_Meta_%s" % metaName,
["DirID", "Value"],
[dirID, metaValue])
if not result["OK"]:
if result["Message"].find("Duplicate") != -1:
req = "UPDATE FC_Meta_%s SET Value='%s' WHERE DirID=%d" % (
metaName, metaValue, dirID)
result = self.db._update(req)
if not result["OK"]:
return result
else:
return result
return S_OK()
def sendExpirationNotifications( self ):
result = self.__cleanExpNotifs()
if not result[ 'OK' ]:
return result
cmd = "SELECT UserDN, UserGroup, LifeLimit FROM `ProxyDB_ExpNotifs`"
result = self._query( cmd )
if not result[ 'OK' ]:
return result
notifDone = dict( [ ( ( row[0], row[1] ), row[2] ) for row in result[ 'Value' ] ] )
notifLimits = sorted( [ int( x ) for x in self.getCSOption( "NotificationTimes", ProxyDB.NOTIFICATION_TIMES ) ] )
sqlSel = "UserDN, UserGroup, TIMESTAMPDIFF( SECOND, UTC_TIMESTAMP(), ExpirationTime )"
sqlCond = "TIMESTAMPDIFF( SECOND, UTC_TIMESTAMP(), ExpirationTime ) < %d" % max( notifLimits )
cmd = "SELECT %s FROM `ProxyDB_Proxies` WHERE %s" % ( sqlSel, sqlCond )
result = self._query( cmd )
if not result[ 'OK' ]:
return result
pilotProps = ( Properties.GENERIC_PILOT, Properties.PILOT )
data = result[ 'Value' ]
sent = []
for row in data:
userDN, group, lTime = row
#If it's a pilot proxy, skip it
if Registry.groupHasProperties( group, pilotProps ):
continue
#IF it dosn't hace the auto upload proxy, skip it
if not Registry.getGroupOption( group, "AutoUploadProxy", False ):
continue
notKey = ( userDN, group )
for notifLimit in notifLimits:
if notifLimit < lTime:
#Not yet in this notification limit
continue
if notKey in notifDone and notifDone[ notKey ] <= notifLimit:
#Already notified for this notification limit
break
if not self._notifyProxyAboutToExpire( userDN, group, lTime, notifLimit ):
#Cannot send notification, retry later
break
try:
sUserDN = self._escapeString( userDN )[ 'Value' ]
sGroup = self._escapeString( group )[ 'Value' ]
except KeyError:
return S_ERROR( "OOPS" )
if notKey not in notifDone:
values = "( %s, %s, %d, TIMESTAMPADD( SECOND, %s, UTC_TIMESTAMP() ) )" % ( sUserDN, sGroup, notifLimit, lTime )
cmd = "INSERT INTO `ProxyDB_ExpNotifs` ( UserDN, UserGroup, LifeLimit, ExpirationTime ) VALUES %s" % values
result = self._update( cmd )
if not result[ 'OK' ]:
gLogger.error( "Could not mark notification as sent", result[ 'Message' ] )
else:
values = "LifeLimit = %d, ExpirationTime = TIMESTAMPADD( SECOND, %s, UTC_TIMESTAMP() )" % ( notifLimit, lTime )
cmd = "UPDATE `ProxyDB_ExpNotifs` SET %s WHERE UserDN = %s AND UserGroup = %s" % ( values, sUserDN, sGroup )
result = self._update( cmd )
if not result[ 'OK' ]:
gLogger.error( "Could not mark notification as sent", result[ 'Message' ] )
sent.append( ( userDN, group, lTime ) )
notifDone[ notKey ] = notifLimit
return S_OK( sent )
def _getMetaNameSuffix(credDict):
"""
Get a VO specific suffix from user credentials.
:param credDict: user credentials
:return: VO specific suffix
"""
vo = Registry.getGroupOption(credDict['group'], 'VO')
return VO_SUFFIX_SEPARATOR + vo.replace('-', '_').replace('.', '_')
def _getMetaNameSuffix(credDict):
"""
Get a VO specific suffix from user credentials.
:param credDict: user credentials
:return: VO specific suffix
"""
vo = Registry.getGroupOption(credDict["group"], "VO")
return VO_SUFFIX_SEPARATOR + vo.replace("-", "_").replace(".", "_")
def certLifeTimeCheck(self):
minLife = Registry.getGroupOption(self.__piParams.diracGroup, "SafeCertificateLifeTime", 2592000)
resultIssuerCert = self.getIssuerCert()
resultRemainingSecs = resultIssuerCert.getRemainingSecs() # pylint: disable=no-member
if not resultRemainingSecs["OK"]:
gLogger.error("Could not retrieve certificate expiration time", resultRemainingSecs["Message"])
return
lifeLeft = resultRemainingSecs["Value"]
if minLife > lifeLeft:
daysLeft = int(lifeLeft / 86400)
msg = "Your certificate will expire in less than %d days. Please renew it!" % daysLeft
sep = "=" * (len(msg) + 4)
msg = "%s\n %s \n%s" % (sep, msg, sep)
gLogger.notice(msg)
def certLifeTimeCheck( self ):
minLife = Registry.getGroupOption( self.__piParams.diracGroup, "SafeCertificateLifeTime", 2592000 )
issuerCert = self.getIssuerCert()
result = issuerCert.getRemainingSecs()
if not result[ 'OK' ]:
gLogger.error( "Could not retrieve certificate expiration time", result[ 'Message' ] )
return
lifeLeft = result[ 'Value' ]
if minLife > lifeLeft:
daysLeft = int( lifeLeft / 86400 )
msg = "Your certificate will expire in less than %d days. Please renew it!" % daysLeft
sep = "=" * ( len( msg ) + 4 )
msg = "%s\n %s \n%s" % ( sep, msg, sep )
gLogger.notice( msg )
def setMetadata(self, path, metaDict, credDict):
"""Set the value of a given metadata field for the the given directory path
:param str path: file path
:param dict metaDict: dictionary with metadata
:param dict credDict: client credential dictionary
:return: S_OK/S_ERROR
"""
result = self._getFileMetadataFields(credDict)
if not result["OK"]:
return result
metaFields = result["Value"]
result = self.db.fileManager._findFiles([path])
if not result["OK"]:
return result
if result["Value"]["Successful"]:
fileID = result["Value"]["Successful"][path]["FileID"]
else:
return S_ERROR("File %s not found" % path)
voName = Registry.getGroupOption(credDict["group"], "VO")
forceIndex = Operations(vo=voName).getValue(
"DataManagement/ForceIndexedMetadata", False)
for metaName, metaValue in metaDict.items():
if metaName not in metaFields:
if forceIndex:
return S_ERROR(
"Field %s not indexed, but ForceIndexedMetadata is set"
% metaName,
callStack=[])
result = self.__setFileMetaParameter(fileID, metaName,
metaValue, credDict)
else:
result = self.db.insertFields("FC_FileMeta_%s" % metaName,
["FileID", "Value"],
[fileID, metaValue])
if not result["OK"]:
if result["Message"].find("Duplicate") != -1:
req = "UPDATE FC_FileMeta_%s SET Value='%s' WHERE FileID=%d" % (
metaName, metaValue, fileID)
result = self.db._update(req)
if not result["OK"]:
return result
else:
return result
return S_OK()
def addVOMSExtIfNeeded( self ):
addVOMS = Registry.getGroupOption( self.__piParams.diracGroup, "AutoAddVOMS", self.__piParams.addVOMSExt )
if not addVOMS:
return
vomsAttr = Registry.getVOMSAttributeForGroup( self.__piParams.diracGroup )
if not vomsAttr:
gLogger.error( "Requested adding a VOMS extension but no VOMS attribute defined for group %s" % self.__piParams.diracGroup )
return
result = VOMS.VOMS().setVOMSAttributes( vomsAttr, vo = Registry.getVOForGroup( self.__piParams.diracGroup ) )
if not result[ 'OK' ]:
gLogger.error( "Failed adding VOMS attribute:", result[ 'Message' ] )
return False
else:
gLogger.notice( "Added VOMS attribute %s" % vomsAttr )
def getPilotGroupsToUpload( self ):
pilotUpload = Registry.getGroupOption( self.__piParams.diracGroup, "AutoUploadPilotProxy", self.__piParams.uploadPilot )
if not pilotUpload:
return []
issuerCert = self.getIssuerCert()
userDN = issuerCert.getSubjectDN()[ 'Value' ]
result = Registry.getGroupsForDN( userDN )
if not result[ 'OK' ]:
gLogger.error( "No groups defined for DN %s" % userDN )
return []
availableGroups = result[ 'Value' ]
pilotGroups = []
for group in availableGroups:
groupProps = Registry.getPropertiesForGroup( group )
if Properties.PILOT in groupProps or Properties.GENERIC_PILOT in groupProps:
pilotGroups.append( group )
return pilotGroups
def __checkParams( self ):
#Check if the proxy needs to be uploaded
if not self.__piParams.uploadProxy:
self.__piParams.uploadProxy = Registry.getGroupOption( self.__piParams.diracGroup, "AutoUploadProxy", False )
if self.__piParams.uploadProxy:
gLogger.verbose( "Proxy will be uploaded to ProxyManager" )
def beginExecution(self):
self.gridEnv = self.am_getOption("GridEnv", getGridEnv())
# The SiteDirector is for a particular user community
self.vo = self.am_getOption("Community", '')
if not self.vo:
self.vo = CSGlobals.getVO()
# The SiteDirector is for a particular user group
self.group = self.am_getOption("Group", '')
# self.voGroups contain all the eligible user groups for pilots submutted by this SiteDirector
self.voGroups = []
# Choose the group for which pilots will be submitted. This is a hack until
# we will be able to match pilots to VOs.
if not self.group:
if self.vo:
result = Registry.getGroupsForVO(self.vo)
if not result['OK']:
return result
for group in result['Value']:
if 'NormalUser' in Registry.getPropertiesForGroup(group):
self.voGroups.append(group)
else:
self.voGroups = [self.group]
result = findGenericPilotCredentials(vo=self.vo)
if not result['OK']:
return result
self.pilotDN, self.pilotGroup = result['Value']
self.pilotDN = self.am_getOption("PilotDN", self.pilotDN)
self.pilotGroup = self.am_getOption("PilotGroup", self.pilotGroup)
self.platforms = []
self.sites = []
self.defaultSubmitPools = ''
if self.group:
self.defaultSubmitPools = Registry.getGroupOption(
self.group, 'SubmitPools', '')
elif self.vo:
self.defaultSubmitPools = Registry.getVOOption(
self.vo, 'SubmitPools', '')
self.pilot = self.am_getOption('PilotScript', DIRAC_PILOT)
self.install = DIRAC_INSTALL
self.workingDirectory = self.am_getOption('WorkDirectory')
self.maxQueueLength = self.am_getOption('MaxQueueLength', 86400 * 3)
self.pilotLogLevel = self.am_getOption('PilotLogLevel', 'INFO')
self.maxJobsInFillMode = self.am_getOption('MaxJobsInFillMode',
self.maxJobsInFillMode)
self.maxPilotsToSubmit = self.am_getOption('MaxPilotsToSubmit',
self.maxPilotsToSubmit)
self.pilotWaitingFlag = self.am_getOption('PilotWaitingFlag', True)
self.pilotWaitingTime = self.am_getOption('MaxPilotWaitingTime', 7200)
# Flags
self.updateStatus = self.am_getOption('UpdatePilotStatus', True)
self.getOutput = self.am_getOption('GetPilotOutput', True)
self.sendAccounting = self.am_getOption('SendPilotAccounting', True)
# Get the site description dictionary
siteNames = None
if not self.am_getOption('Site', 'Any').lower() == "any":
siteNames = self.am_getOption('Site', [])
ceTypes = None
if not self.am_getOption('CETypes', 'Any').lower() == "any":
ceTypes = self.am_getOption('CETypes', [])
ces = None
if not self.am_getOption('CEs', 'Any').lower() == "any":
ces = self.am_getOption('CEs', [])
result = Resources.getQueues(community=self.vo,
siteList=siteNames,
ceList=ces,
ceTypeList=ceTypes,
mode='Direct')
if not result['OK']:
return result
resourceDict = result['Value']
result = self.getQueues(resourceDict)
if not result['OK']:
return result
#if not siteNames:
# siteName = gConfig.getValue( '/DIRAC/Site', 'Unknown' )
# if siteName == 'Unknown':
# return S_OK( 'No site specified for the SiteDirector' )
# else:
# siteNames = [siteName]
#self.siteNames = siteNames
if self.updateStatus:
self.log.always('Pilot status update requested')
if self.getOutput:
self.log.always('Pilot output retrieval requested')
if self.sendAccounting:
self.log.always('Pilot accounting sending requested')
self.log.always('Sites:', siteNames)
self.log.always('CETypes:', ceTypes)
self.log.always('CEs:', ces)
self.log.always('PilotDN:', self.pilotDN)
self.log.always('PilotGroup:', self.pilotGroup)
self.log.always('MaxPilotsToSubmit:', self.maxPilotsToSubmit)
self.log.always('MaxJobsInFillMode:', self.maxJobsInFillMode)
self.localhost = socket.getfqdn()
self.proxy = ''
if self.queueDict:
self.log.always("Agent will serve queues:")
for queue in self.queueDict:
self.log.always("Site: %s, CE: %s, Queue: %s" %
(self.queueDict[queue]['Site'],
self.queueDict[queue]['CEName'], queue))
return S_OK()
def generateProxy( params ):
if params.checkClock:
result = getClockDeviation()
if result[ 'OK' ]:
deviation = result[ 'Value' ]
if deviation > 600:
gLogger.error( "Your host clock seems to be off by more than TEN MINUTES! Thats really bad." )
gLogger.error( "We're cowardly refusing to generate a proxy. Please fix your system time" )
sys.exit( 1 )
elif deviation > 180:
gLogger.error( "Your host clock seems to be off by more than THREE minutes! Thats bad." )
gLogger.notice( "We'll generate the proxy but please fix your system time" )
elif deviation > 60:
gLogger.error( "Your host clock seems to be off by more than a minute! Thats not good." )
gLogger.notice( "We'll generate the proxy but please fix your system time" )
certLoc = params.certLoc
keyLoc = params.keyLoc
if not certLoc or not keyLoc:
cakLoc = Locations.getCertificateAndKeyLocation()
if not cakLoc:
return S_ERROR( "Can't find user certificate and key" )
if not certLoc:
certLoc = cakLoc[0]
if not keyLoc:
keyLoc = cakLoc[1]
params.certLoc = certLoc
params.keyLoc = keyLoc
#Load password
testChain = X509Chain()
retVal = testChain.loadKeyFromFile( keyLoc, password = params.userPasswd )
if not retVal[ 'OK' ]:
passwdPrompt = "Enter Certificate password:"******"\n" )
else:
userPasswd = getpass.getpass( passwdPrompt )
params.userPasswd = userPasswd
#Find location
proxyLoc = params.proxyLoc
if not proxyLoc:
proxyLoc = Locations.getDefaultProxyLocation()
chain = X509Chain()
#Load user cert and key
retVal = chain.loadChainFromFile( certLoc )
if not retVal[ 'OK' ]:
gLogger.warn( retVal[ 'Message' ] )
return S_ERROR( "Can't load %s" % certLoc )
retVal = chain.loadKeyFromFile( keyLoc, password = params.userPasswd )
if not retVal[ 'OK' ]:
gLogger.warn( retVal[ 'Message' ] )
return S_ERROR( "Can't load %s" % keyLoc )
if params.checkWithCS:
retVal = chain.generateProxyToFile( proxyLoc,
params.proxyLifeTime,
strength = params.proxyStrength,
limited = params.limitedProxy )
gLogger.info( "Contacting CS..." )
retVal = Script.enableCS()
if not retVal[ 'OK' ]:
gLogger.warn( retVal[ 'Message' ] )
return S_ERROR( "Can't contact DIRAC CS: %s" % retVal[ 'Message' ] )
userDN = chain.getCertInChain( -1 )['Value'].getSubjectDN()['Value']
if not params.diracGroup:
result = Registry.findDefaultGroupForDN( userDN )
if not result[ 'OK' ]:
gLogger.warn( "Could not get a default group for DN %s: %s" % ( userDN, result[ 'Message' ] ) )
else:
params.diracGroup = result[ 'Value' ]
gLogger.info( "Default discovered group is %s" % params.diracGroup )
gLogger.info( "Checking DN %s" % userDN )
retVal = Registry.getUsernameForDN( userDN )
if not retVal[ 'OK' ]:
gLogger.warn( retVal[ 'Message' ] )
return S_ERROR( "DN %s is not registered" % userDN )
username = retVal[ 'Value' ]
gLogger.info( "Username is %s" % username )
retVal = Registry.getGroupsForUser( username )
if not retVal[ 'OK' ]:
gLogger.warn( retVal[ 'Message' ] )
return S_ERROR( "User %s has no groups defined" % username )
groups = retVal[ 'Value' ]
if params.diracGroup not in groups:
return S_ERROR( "Requested group %s is not valid for user %s" % ( params.diracGroup, username ) )
gLogger.info( "Creating proxy for %s@%s (%s)" % ( username, params.diracGroup, userDN ) )
#Check if the proxy needs to be uploaded
if not params.uploadProxy:
params.uploadProxy = Registry.getGroupOption( params.diracGroup, "AutoUploadProxy", False )
gLogger.verbose( "Proxy will be uploaded to ProxyManager " )
if params.summary:
h = int( params.proxyLifeTime / 3600 )
m = int( params.proxyLifeTime / 60 ) - h * 60
gLogger.notice( "Proxy lifetime will be %02d:%02d" % ( h, m ) )
gLogger.notice( "User cert is %s" % certLoc )
gLogger.notice( "User key is %s" % keyLoc )
gLogger.notice( "Proxy will be written to %s" % proxyLoc )
if params.diracGroup:
gLogger.notice( "DIRAC Group will be set to %s" % params.diracGroup )
else:
gLogger.notice( "No DIRAC Group will be set" )
gLogger.notice( "Proxy strength will be %s" % params.proxyStrength )
if params.limitedProxy:
gLogger.notice( "Proxy will be limited" )
retVal = chain.generateProxyToFile( proxyLoc,
params.proxyLifeTime,
params.diracGroup,
strength = params.proxyStrength,
limited = params.limitedProxy )
if not retVal[ 'OK' ]:
gLogger.warn( retVal[ 'Message' ] )
return S_ERROR( "Couldn't generate proxy: %s" % retVal[ 'Message' ] )
return S_OK( proxyLoc )
def getSubmitPools( group = None, vo = None ):
if group:
return Registry.getGroupOption( group, 'SubmitPools', '' )
if vo:
return Registry.getVOOption( vo, 'SubmitPools', '' )
return ''
def beginExecution(self):
self.gridEnv = self.am_getOption("GridEnv", getGridEnv())
# The SiteDirector is for a particular user community
self.vo = self.am_getOption("Community", "")
if not self.vo:
self.vo = CSGlobals.getVO()
# The SiteDirector is for a particular user group
self.group = self.am_getOption("Group", "")
# self.voGroups contain all the eligible user groups for pilots submutted by this SiteDirector
self.voGroups = []
# Choose the group for which pilots will be submitted. This is a hack until
# we will be able to match pilots to VOs.
if not self.group:
if self.vo:
result = Registry.getGroupsForVO(self.vo)
if not result["OK"]:
return result
for group in result["Value"]:
if "NormalUser" in Registry.getPropertiesForGroup(group):
self.voGroups.append(group)
else:
self.voGroups = [self.group]
result = findGenericPilotCredentials(vo=self.vo)
if not result["OK"]:
return result
self.pilotDN, self.pilotGroup = result["Value"]
self.pilotDN = self.am_getOption("PilotDN", self.pilotDN)
self.pilotGroup = self.am_getOption("PilotGroup", self.pilotGroup)
self.platforms = []
self.sites = []
self.defaultSubmitPools = ""
if self.group:
self.defaultSubmitPools = Registry.getGroupOption(self.group, "SubmitPools", "")
elif self.vo:
self.defaultSubmitPools = Registry.getVOOption(self.vo, "SubmitPools", "")
self.pilot = self.am_getOption("PilotScript", DIRAC_PILOT)
self.install = DIRAC_INSTALL
self.workingDirectory = self.am_getOption("WorkDirectory")
self.maxQueueLength = self.am_getOption("MaxQueueLength", 86400 * 3)
self.pilotLogLevel = self.am_getOption("PilotLogLevel", "INFO")
self.maxJobsInFillMode = self.am_getOption("MaxJobsInFillMode", self.maxJobsInFillMode)
self.maxPilotsToSubmit = self.am_getOption("MaxPilotsToSubmit", self.maxPilotsToSubmit)
self.pilotWaitingFlag = self.am_getOption("PilotWaitingFlag", True)
self.pilotWaitingTime = self.am_getOption("MaxPilotWaitingTime", 7200)
# Flags
self.updateStatus = self.am_getOption("UpdatePilotStatus", True)
self.getOutput = self.am_getOption("GetPilotOutput", True)
self.sendAccounting = self.am_getOption("SendPilotAccounting", True)
# Get the site description dictionary
siteNames = None
if not self.am_getOption("Site", "Any").lower() == "any":
siteNames = self.am_getOption("Site", [])
ceTypes = None
if not self.am_getOption("CETypes", "Any").lower() == "any":
ceTypes = self.am_getOption("CETypes", [])
ces = None
if not self.am_getOption("CEs", "Any").lower() == "any":
ces = self.am_getOption("CEs", [])
result = Resources.getQueues(
community=self.vo, siteList=siteNames, ceList=ces, ceTypeList=ceTypes, mode="Direct"
)
if not result["OK"]:
return result
resourceDict = result["Value"]
result = self.getQueues(resourceDict)
if not result["OK"]:
return result
# if not siteNames:
# siteName = gConfig.getValue( '/DIRAC/Site', 'Unknown' )
# if siteName == 'Unknown':
# return S_OK( 'No site specified for the SiteDirector' )
# else:
# siteNames = [siteName]
# self.siteNames = siteNames
if self.updateStatus:
self.log.always("Pilot status update requested")
if self.getOutput:
self.log.always("Pilot output retrieval requested")
if self.sendAccounting:
self.log.always("Pilot accounting sending requested")
self.log.always("Sites:", siteNames)
self.log.always("CETypes:", ceTypes)
self.log.always("CEs:", ces)
self.log.always("PilotDN:", self.pilotDN)
self.log.always("PilotGroup:", self.pilotGroup)
self.log.always("MaxPilotsToSubmit:", self.maxPilotsToSubmit)
self.log.always("MaxJobsInFillMode:", self.maxJobsInFillMode)
self.localhost = socket.getfqdn()
self.proxy = ""
if self.queueDict:
self.log.always("Agent will serve queues:")
for queue in self.queueDict:
self.log.always(
"Site: %s, CE: %s, Queue: %s"
% (self.queueDict[queue]["Site"], self.queueDict[queue]["CEName"], queue)
)
return S_OK()
def getSubmitPools( group = None, vo = None ):
if group:
return Registry.getGroupOption( group, 'SubmitPools', '' )
if vo:
return Registry.getVOOption( vo, 'SubmitPools', '' )
return ''
def beginExecution( self ):
self.gridEnv = self.am_getOption( "GridEnv", getGridEnv() )
# The SiteDirector is for a particular user community
self.vo = self.am_getOption( "VO", '' )
if not self.vo:
self.vo = self.am_getOption( "Community", '' )
if not self.vo:
self.vo = CSGlobals.getVO()
# The SiteDirector is for a particular user group
self.group = self.am_getOption( "Group", '' )
# self.voGroups contain all the eligible user groups for pilots submutted by this SiteDirector
self.voGroups = []
# Choose the group for which pilots will be submitted. This is a hack until
# we will be able to match pilots to VOs.
if not self.group:
if self.vo:
result = Registry.getGroupsForVO( self.vo )
if not result['OK']:
return result
for group in result['Value']:
if 'NormalUser' in Registry.getPropertiesForGroup( group ):
self.voGroups.append( group )
else:
self.voGroups = [ self.group ]
result = findGenericPilotCredentials( vo = self.vo )
if not result[ 'OK' ]:
return result
self.pilotDN, self.pilotGroup = result[ 'Value' ]
self.pilotDN = self.am_getOption( "PilotDN", self.pilotDN )
self.pilotGroup = self.am_getOption( "PilotGroup", self.pilotGroup )
self.platforms = []
self.sites = []
self.defaultSubmitPools = ''
if self.group:
self.defaultSubmitPools = Registry.getGroupOption( self.group, 'SubmitPools', '' )
elif self.vo:
self.defaultSubmitPools = Registry.getVOOption( self.vo, 'SubmitPools', '' )
self.pilot = self.am_getOption( 'PilotScript', DIRAC_PILOT )
self.install = DIRAC_INSTALL
self.extraModules = self.am_getOption( 'ExtraPilotModules', [] ) + DIRAC_MODULES
self.workingDirectory = self.am_getOption( 'WorkDirectory' )
self.maxQueueLength = self.am_getOption( 'MaxQueueLength', 86400 * 3 )
self.pilotLogLevel = self.am_getOption( 'PilotLogLevel', 'INFO' )
self.maxJobsInFillMode = self.am_getOption( 'MaxJobsInFillMode', self.maxJobsInFillMode )
self.maxPilotsToSubmit = self.am_getOption( 'MaxPilotsToSubmit', self.maxPilotsToSubmit )
self.pilotWaitingFlag = self.am_getOption( 'PilotWaitingFlag', True )
self.pilotWaitingTime = self.am_getOption( 'MaxPilotWaitingTime', 3600 )
self.failedQueueCycleFactor = self.am_getOption( 'FailedQueueCycleFactor', 10 )
self.pilotStatusUpdateCycleFactor = self.am_getOption( 'PilotStatusUpdateCycleFactor', 10 )
# Flags
self.updateStatus = self.am_getOption( 'UpdatePilotStatus', True )
self.getOutput = self.am_getOption( 'GetPilotOutput', True )
self.sendAccounting = self.am_getOption( 'SendPilotAccounting', True )
# Get the site description dictionary
siteNames = None
if not self.am_getOption( 'Site', 'Any' ).lower() == "any":
siteNames = self.am_getOption( 'Site', [] )
if not siteNames:
siteNames = None
ceTypes = None
if not self.am_getOption( 'CETypes', 'Any' ).lower() == "any":
ceTypes = self.am_getOption( 'CETypes', [] )
ces = None
if not self.am_getOption( 'CEs', 'Any' ).lower() == "any":
ces = self.am_getOption( 'CEs', [] )
if not ces:
ces = None
result = Resources.getQueues( community = self.vo,
siteList = siteNames,
ceList = ces,
ceTypeList = ceTypes,
mode = 'Direct' )
if not result['OK']:
return result
resourceDict = result['Value']
result = self.getQueues( resourceDict )
if not result['OK']:
return result
#if not siteNames:
# siteName = gConfig.getValue( '/DIRAC/Site', 'Unknown' )
# if siteName == 'Unknown':
# return S_OK( 'No site specified for the SiteDirector' )
# else:
# siteNames = [siteName]
#self.siteNames = siteNames
if self.updateStatus:
self.log.always( 'Pilot status update requested' )
if self.getOutput:
self.log.always( 'Pilot output retrieval requested' )
if self.sendAccounting:
self.log.always( 'Pilot accounting sending requested' )
self.log.always( 'Sites:', siteNames )
self.log.always( 'CETypes:', ceTypes )
self.log.always( 'CEs:', ces )
self.log.always( 'PilotDN:', self.pilotDN )
self.log.always( 'PilotGroup:', self.pilotGroup )
self.log.always( 'MaxPilotsToSubmit:', self.maxPilotsToSubmit )
self.log.always( 'MaxJobsInFillMode:', self.maxJobsInFillMode )
self.localhost = socket.getfqdn()
self.proxy = ''
if self.firstPass:
if self.queueDict:
self.log.always( "Agent will serve queues:" )
for queue in self.queueDict:
self.log.always( "Site: %s, CE: %s, Queue: %s" % ( self.queueDict[queue]['Site'],
self.queueDict[queue]['CEName'],
queue ) )
self.firstPass = False
return S_OK()
def getProxyWithToken(self, token):
""" Get proxy with token
:param str token: access token
:return: S_OK()/S_ERROR()
"""
# Get REST endpoints from local CS
confUrl = gConfig.getValue("/LocalInstallation/ConfigurationServerAPI")
if not confUrl:
return S_ERROR('Could not get configuration server API URL.')
setup = gConfig.getValue("/DIRAC/Setup")
if not setup:
return S_ERROR('Could not get setup name.')
# Get REST endpoints from ConfigurationService
try:
r = requests.get(
'%s/option?path=/Systems/Framework/Production/URLs/ProxyAPI' %
confUrl,
verify=False)
r.raise_for_status()
proxyAPI = decode(r.text)[0]
except requests.exceptions.Timeout:
return S_ERROR('Time out')
except requests.exceptions.RequestException as e:
return S_ERROR(str(e))
except Exception as e:
return S_ERROR('Cannot read response: %s' % e)
# Fill the proxy request URL
url = '%ss:%s/g:%s/proxy?lifetime=%s' % (proxyAPI, setup, self.group,
self.lifetime)
voms = self.voms or Registry.getGroupOption(self.group, "AutoAddVOMS",
False)
if voms:
url += '&voms=%s' % voms
# Get proxy from REST API
try:
r = requests.get(url,
headers={'Authorization': 'Bearer ' + token},
verify=False)
r.raise_for_status()
proxy = decode(r.text)[0]
except requests.exceptions.Timeout:
return S_ERROR('Time out')
except requests.exceptions.RequestException as e:
return S_ERROR(str(e))
except Exception as e:
return S_ERROR('Cannot read response: %s' % e)
if not proxy:
return S_ERROR("Result is empty.")
self.log.notice('Saving proxy.. to %s..' % self.pPath)
# Save proxy to file
try:
with open(self.pPath, 'w+') as fd:
fd.write(proxy.encode("UTF-8"))
os.chmod(self.pPath, stat.S_IRUSR | stat.S_IWUSR)
except Exception as e:
return S_ERROR("%s :%s" % (self.pPath, repr(e).replace(',)', ')')))
self.log.notice('Proxy is saved to %s.' % self.pPath)
return S_OK()