def printCEInfo(voName):
resultQueues = Resources.getQueues(community=voName)
if not resultQueues["OK"]:
gLogger.error("Failed to get CE information")
DIRACExit(-1)
fields = ("Site", "CE", "CEType", "Queue", "Status")
records = []
# get list of usable sites within this cycle
resultMask = SiteStatus().getUsableSites()
if not resultMask["OK"]:
return resultMask
siteMaskList = resultMask.get("Value", [])
rssClient = ResourceStatus()
for site in resultQueues["Value"]:
siteStatus = "Active" if site in siteMaskList else "InActive"
siteNew = True
for ce in resultQueues["Value"][site]:
ceStatus = siteStatus
if rssClient.rssFlag:
result = rssClient.getElementStatus(ce, "ComputingElement")
if result["OK"]:
ceStatus = result["Value"][ce]["all"]
ceNew = True
for queue in resultQueues["Value"][site][ce]["Queues"]:
pSite = site if siteNew else ""
pCE = ""
ceType = ""
if ceNew:
pCE = ce
ceType = resultQueues["Value"][site][ce]["CEType"]
records.append((pSite, pCE, ceType, queue, ceStatus))
ceNew = False
siteNew = False
gLogger.notice(
printTable(fields, records, printOut=False, columnSeparator=" "))
return S_OK()
def printCEInfo(voName):
resultQueues = Resources.getQueues(community=voName)
if not resultQueues['OK']:
gLogger.error('Failed to get CE information')
DIRACExit(-1)
fields = ("Site", 'CE', 'CEType', 'Queue', 'Status')
records = []
# get list of usable sites within this cycle
resultMask = SiteStatus().getUsableSites()
if not resultMask['OK']:
return resultMask
siteMaskList = resultMask.get('Value', [])
rssClient = ResourceStatus()
for site in resultQueues['Value']:
siteStatus = "Active" if site in siteMaskList else "InActive"
siteNew = True
for ce in resultQueues['Value'][site]:
ceStatus = siteStatus
if rssClient.rssFlag:
result = rssClient.getElementStatus(ce, "ComputingElement")
if result['OK']:
ceStatus = result['Value'][ce]['all']
ceNew = True
for queue in resultQueues['Value'][site][ce]['Queues']:
pSite = site if siteNew else ''
pCE = ''
ceType = ''
if ceNew:
pCE = ce
ceType = resultQueues['Value'][site][ce]['CEType']
records.append((pSite, pCE, ceType, queue, ceStatus))
ceNew = False
siteNew = False
gLogger.notice(
printTable(fields, records, printOut=False, columnSeparator=' '))
return S_OK()
def getCPUNormalizationFactorAvg():
"""
Returns the average HS06 CPU normalization factor for the LCG sites (all CEs, all queues).
Raises an Exception if it can not.
"""
factorsSum = 0.0
nQueues = 0
sites = gConfig.getSections('Resources/Sites/LCG')
if not sites['OK']:
raise RuntimeError(sites['Message'])
else:
sites = sites['Value']
queuesRequest = Resources.getQueues(sites)
if not queuesRequest['OK']:
raise RuntimeError(queuesRequest['Message'])
else:
queuesRequest = queuesRequest['Value']
for site in queuesRequest.values():
for ce in site.values():
if 'Queues' in ce:
for queue in ce['Queues'].values():
if 'SI00' in queue:
# convert from SI00 to HS06
factorsSum += float(queue['SI00']) / 250
nQueues += 1
if nQueues == 0:
raise RuntimeError("No queues to get CPU normalization factor from")
else:
CPUNormalizationFactorAvg = float(factorsSum / nQueues)
return CPUNormalizationFactorAvg
def beginExecution( self ):
self.gridEnv = self.am_getOption( "GridEnv", getGridEnv() )
# The SiteDirector is for a particular user community
self.vo = self.am_getOption( "VO", '' )
if not self.vo:
self.vo = self.am_getOption( "Community", '' )
if not self.vo:
self.vo = CSGlobals.getVO()
# The SiteDirector is for a particular user group
self.group = self.am_getOption( "Group", '' )
# self.voGroups contain all the eligible user groups for pilots submutted by this SiteDirector
self.voGroups = []
# Choose the group for which pilots will be submitted. This is a hack until
# we will be able to match pilots to VOs.
if not self.group:
if self.vo:
result = Registry.getGroupsForVO( self.vo )
if not result['OK']:
return result
for group in result['Value']:
if 'NormalUser' in Registry.getPropertiesForGroup( group ):
self.voGroups.append( group )
else:
self.voGroups = [ self.group ]
result = findGenericPilotCredentials( vo = self.vo )
if not result[ 'OK' ]:
return result
self.pilotDN, self.pilotGroup = result[ 'Value' ]
self.pilotDN = self.am_getOption( "PilotDN", self.pilotDN )
self.pilotGroup = self.am_getOption( "PilotGroup", self.pilotGroup )
self.platforms = []
self.sites = []
self.defaultSubmitPools = ''
if self.group:
self.defaultSubmitPools = Registry.getGroupOption( self.group, 'SubmitPools', '' )
elif self.vo:
self.defaultSubmitPools = Registry.getVOOption( self.vo, 'SubmitPools', '' )
self.pilot = self.am_getOption( 'PilotScript', DIRAC_PILOT )
self.install = DIRAC_INSTALL
self.extraModules = self.am_getOption( 'ExtraPilotModules', [] ) + DIRAC_MODULES
self.workingDirectory = self.am_getOption( 'WorkDirectory' )
self.maxQueueLength = self.am_getOption( 'MaxQueueLength', 86400 * 3 )
self.pilotLogLevel = self.am_getOption( 'PilotLogLevel', 'INFO' )
self.maxJobsInFillMode = self.am_getOption( 'MaxJobsInFillMode', self.maxJobsInFillMode )
self.maxPilotsToSubmit = self.am_getOption( 'MaxPilotsToSubmit', self.maxPilotsToSubmit )
self.pilotWaitingFlag = self.am_getOption( 'PilotWaitingFlag', True )
self.pilotWaitingTime = self.am_getOption( 'MaxPilotWaitingTime', 3600 )
self.failedQueueCycleFactor = self.am_getOption( 'FailedQueueCycleFactor', 10 )
self.pilotStatusUpdateCycleFactor = self.am_getOption( 'PilotStatusUpdateCycleFactor', 10 )
# Flags
self.updateStatus = self.am_getOption( 'UpdatePilotStatus', True )
self.getOutput = self.am_getOption( 'GetPilotOutput', True )
self.sendAccounting = self.am_getOption( 'SendPilotAccounting', True )
# Get the site description dictionary
siteNames = None
if not self.am_getOption( 'Site', 'Any' ).lower() == "any":
siteNames = self.am_getOption( 'Site', [] )
if not siteNames:
siteNames = None
ceTypes = None
if not self.am_getOption( 'CETypes', 'Any' ).lower() == "any":
ceTypes = self.am_getOption( 'CETypes', [] )
ces = None
if not self.am_getOption( 'CEs', 'Any' ).lower() == "any":
ces = self.am_getOption( 'CEs', [] )
if not ces:
ces = None
result = Resources.getQueues( community = self.vo,
siteList = siteNames,
ceList = ces,
ceTypeList = ceTypes,
mode = 'Direct' )
if not result['OK']:
return result
resourceDict = result['Value']
result = self.getQueues( resourceDict )
if not result['OK']:
return result
#if not siteNames:
# siteName = gConfig.getValue( '/DIRAC/Site', 'Unknown' )
# if siteName == 'Unknown':
# return S_OK( 'No site specified for the SiteDirector' )
# else:
# siteNames = [siteName]
#self.siteNames = siteNames
if self.updateStatus:
self.log.always( 'Pilot status update requested' )
if self.getOutput:
self.log.always( 'Pilot output retrieval requested' )
if self.sendAccounting:
self.log.always( 'Pilot accounting sending requested' )
self.log.always( 'Sites:', siteNames )
self.log.always( 'CETypes:', ceTypes )
self.log.always( 'CEs:', ces )
self.log.always( 'PilotDN:', self.pilotDN )
self.log.always( 'PilotGroup:', self.pilotGroup )
self.log.always( 'MaxPilotsToSubmit:', self.maxPilotsToSubmit )
self.log.always( 'MaxJobsInFillMode:', self.maxJobsInFillMode )
self.localhost = socket.getfqdn()
self.proxy = ''
if self.firstPass:
if self.queueDict:
self.log.always( "Agent will serve queues:" )
for queue in self.queueDict:
self.log.always( "Site: %s, CE: %s, Queue: %s" % ( self.queueDict[queue]['Site'],
self.queueDict[queue]['CEName'],
queue ) )
self.firstPass = False
return S_OK()
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus
from DIRAC.WorkloadManagementSystem.Utilities.QueueUtilities import getQueuesResolved, matchQueue
if __name__ == '__main__':
with open(args[0]) as f:
jdl = f.read()
# Get the current VO
result = getVOfromProxyGroup()
if not result['OK']:
gLogger.error('No proxy found, please login')
DIRACExit(-1)
voName = result['Value']
resultQueues = Resources.getQueues(siteList=sites, community=voName)
if not resultQueues['OK']:
gLogger.error('Failed to get CE information')
DIRACExit(-1)
siteDict = resultQueues['Value']
result = getQueuesResolved(siteDict)
if not resultQueues['OK']:
gLogger.error('Failed to get CE information')
DIRACExit(-1)
queueDict = result['Value']
# get list of usable sites within this cycle
resultMask = SiteStatus().getUsableSites()
if not resultMask['OK']:
gLogger.error('Failed to get Site mask information')
DIRACExit(-1)
def beginExecution(self):
self.gridEnv = self.am_getOption("GridEnv", getGridEnv())
# The SiteDirector is for a particular user community
self.vo = self.am_getOption("Community", '')
if not self.vo:
self.vo = CSGlobals.getVO()
# The SiteDirector is for a particular user group
self.group = self.am_getOption("Group", '')
# self.voGroups contain all the eligible user groups for pilots submutted by this SiteDirector
self.voGroups = []
# Choose the group for which pilots will be submitted. This is a hack until
# we will be able to match pilots to VOs.
if not self.group:
if self.vo:
result = Registry.getGroupsForVO(self.vo)
if not result['OK']:
return result
for group in result['Value']:
if 'NormalUser' in Registry.getPropertiesForGroup(group):
self.voGroups.append(group)
else:
self.voGroups = [self.group]
result = findGenericPilotCredentials(vo=self.vo)
if not result['OK']:
return result
self.pilotDN, self.pilotGroup = result['Value']
self.pilotDN = self.am_getOption("PilotDN", self.pilotDN)
self.pilotGroup = self.am_getOption("PilotGroup", self.pilotGroup)
self.platforms = []
self.sites = []
self.defaultSubmitPools = ''
if self.group:
self.defaultSubmitPools = Registry.getGroupOption(
self.group, 'SubmitPools', '')
elif self.vo:
self.defaultSubmitPools = Registry.getVOOption(
self.vo, 'SubmitPools', '')
self.pilot = self.am_getOption('PilotScript', DIRAC_PILOT)
self.install = DIRAC_INSTALL
self.workingDirectory = self.am_getOption('WorkDirectory')
self.maxQueueLength = self.am_getOption('MaxQueueLength', 86400 * 3)
self.pilotLogLevel = self.am_getOption('PilotLogLevel', 'INFO')
self.maxJobsInFillMode = self.am_getOption('MaxJobsInFillMode',
self.maxJobsInFillMode)
self.maxPilotsToSubmit = self.am_getOption('MaxPilotsToSubmit',
self.maxPilotsToSubmit)
self.pilotWaitingFlag = self.am_getOption('PilotWaitingFlag', True)
self.pilotWaitingTime = self.am_getOption('MaxPilotWaitingTime', 7200)
# Flags
self.updateStatus = self.am_getOption('UpdatePilotStatus', True)
self.getOutput = self.am_getOption('GetPilotOutput', True)
self.sendAccounting = self.am_getOption('SendPilotAccounting', True)
# Get the site description dictionary
siteNames = None
if not self.am_getOption('Site', 'Any').lower() == "any":
siteNames = self.am_getOption('Site', [])
ceTypes = None
if not self.am_getOption('CETypes', 'Any').lower() == "any":
ceTypes = self.am_getOption('CETypes', [])
ces = None
if not self.am_getOption('CEs', 'Any').lower() == "any":
ces = self.am_getOption('CEs', [])
result = Resources.getQueues(community=self.vo,
siteList=siteNames,
ceList=ces,
ceTypeList=ceTypes,
mode='Direct')
if not result['OK']:
return result
resourceDict = result['Value']
result = self.getQueues(resourceDict)
if not result['OK']:
return result
#if not siteNames:
# siteName = gConfig.getValue( '/DIRAC/Site', 'Unknown' )
# if siteName == 'Unknown':
# return S_OK( 'No site specified for the SiteDirector' )
# else:
# siteNames = [siteName]
#self.siteNames = siteNames
if self.updateStatus:
self.log.always('Pilot status update requested')
if self.getOutput:
self.log.always('Pilot output retrieval requested')
if self.sendAccounting:
self.log.always('Pilot accounting sending requested')
self.log.always('Sites:', siteNames)
self.log.always('CETypes:', ceTypes)
self.log.always('CEs:', ces)
self.log.always('PilotDN:', self.pilotDN)
self.log.always('PilotGroup:', self.pilotGroup)
self.log.always('MaxPilotsToSubmit:', self.maxPilotsToSubmit)
self.log.always('MaxJobsInFillMode:', self.maxJobsInFillMode)
self.localhost = socket.getfqdn()
self.proxy = ''
if self.queueDict:
self.log.always("Agent will serve queues:")
for queue in self.queueDict:
self.log.always("Site: %s, CE: %s, Queue: %s" %
(self.queueDict[queue]['Site'],
self.queueDict[queue]['CEName'], queue))
return S_OK()
def main():
global fullMatch
global sites
Script.registerSwitch("F", "full-match", "Check all the matching criteria",
setFullMatch)
Script.registerSwitch(
"S:", "site=", "Check matching for these sites (comma separated list)",
setSites)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) == 0:
gLogger.error("Error: No job description provided")
Script.showHelp(exitCode=1)
from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup
from DIRAC.ConfigurationSystem.Client.Helpers import Resources
from DIRAC.Core.Utilities.PrettyPrint import printTable
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus
from DIRAC.WorkloadManagementSystem.Utilities.QueueUtilities import getQueuesResolved, matchQueue
with open(args[0]) as f:
jdl = f.read()
# Get the current VO
result = getVOfromProxyGroup()
if not result['OK']:
gLogger.error('No proxy found, please login')
DIRACExit(-1)
voName = result['Value']
resultQueues = Resources.getQueues(siteList=sites, community=voName)
if not resultQueues['OK']:
gLogger.error('Failed to get CE information')
DIRACExit(-1)
siteDict = resultQueues['Value']
result = getQueuesResolved(siteDict)
if not resultQueues['OK']:
gLogger.error('Failed to get CE information')
DIRACExit(-1)
queueDict = result['Value']
# get list of usable sites within this cycle
resultMask = SiteStatus().getUsableSites()
if not resultMask['OK']:
gLogger.error('Failed to get Site mask information')
DIRACExit(-1)
siteMaskList = resultMask.get('Value', [])
rssClient = ResourceStatus()
fields = ('Site', 'CE', 'Queue', 'Status', 'Match', 'Reason')
records = []
for queue, queueInfo in queueDict.items():
site = queueInfo['Site']
ce = queueInfo['CEName']
siteStatus = "Active" if site in siteMaskList else "InActive"
ceStatus = siteStatus
if rssClient.rssFlag:
result = rssClient.getElementStatus(ce, "ComputingElement")
if result['OK']:
ceStatus = result['Value'][ce]['all']
result = matchQueue(jdl, queueInfo, fullMatch=fullMatch)
if not result['OK']:
gLogger.error('Failed in getting match data', result['Message'])
DIRACExit(-1)
status = "Active" if siteStatus == "Active" and ceStatus == "Active" else "Inactive"
if result['Value']['Match']:
records.append((site, ce, queueInfo['Queue'], status, 'Yes', ''))
else:
records.append((site, ce, queueInfo['Queue'], status, 'No',
result['Value']['Reason']))
gLogger.notice(
printTable(fields,
records,
sortField='Site',
columnSeparator=' ',
printOut=False))
def beginExecution(self):
self.gridEnv = self.am_getOption("GridEnv", getGridEnv())
# The SiteDirector is for a particular user community
self.vo = self.am_getOption("Community", "")
if not self.vo:
self.vo = CSGlobals.getVO()
# The SiteDirector is for a particular user group
self.group = self.am_getOption("Group", "")
# self.voGroups contain all the eligible user groups for pilots submutted by this SiteDirector
self.voGroups = []
# Choose the group for which pilots will be submitted. This is a hack until
# we will be able to match pilots to VOs.
if not self.group:
if self.vo:
result = Registry.getGroupsForVO(self.vo)
if not result["OK"]:
return result
for group in result["Value"]:
if "NormalUser" in Registry.getPropertiesForGroup(group):
self.voGroups.append(group)
else:
self.voGroups = [self.group]
result = findGenericPilotCredentials(vo=self.vo)
if not result["OK"]:
return result
self.pilotDN, self.pilotGroup = result["Value"]
self.pilotDN = self.am_getOption("PilotDN", self.pilotDN)
self.pilotGroup = self.am_getOption("PilotGroup", self.pilotGroup)
self.platforms = []
self.sites = []
self.defaultSubmitPools = ""
if self.group:
self.defaultSubmitPools = Registry.getGroupOption(self.group, "SubmitPools", "")
elif self.vo:
self.defaultSubmitPools = Registry.getVOOption(self.vo, "SubmitPools", "")
self.pilot = self.am_getOption("PilotScript", DIRAC_PILOT)
self.install = DIRAC_INSTALL
self.workingDirectory = self.am_getOption("WorkDirectory")
self.maxQueueLength = self.am_getOption("MaxQueueLength", 86400 * 3)
self.pilotLogLevel = self.am_getOption("PilotLogLevel", "INFO")
self.maxJobsInFillMode = self.am_getOption("MaxJobsInFillMode", self.maxJobsInFillMode)
self.maxPilotsToSubmit = self.am_getOption("MaxPilotsToSubmit", self.maxPilotsToSubmit)
self.pilotWaitingFlag = self.am_getOption("PilotWaitingFlag", True)
self.pilotWaitingTime = self.am_getOption("MaxPilotWaitingTime", 7200)
# Flags
self.updateStatus = self.am_getOption("UpdatePilotStatus", True)
self.getOutput = self.am_getOption("GetPilotOutput", True)
self.sendAccounting = self.am_getOption("SendPilotAccounting", True)
# Get the site description dictionary
siteNames = None
if not self.am_getOption("Site", "Any").lower() == "any":
siteNames = self.am_getOption("Site", [])
ceTypes = None
if not self.am_getOption("CETypes", "Any").lower() == "any":
ceTypes = self.am_getOption("CETypes", [])
ces = None
if not self.am_getOption("CEs", "Any").lower() == "any":
ces = self.am_getOption("CEs", [])
result = Resources.getQueues(
community=self.vo, siteList=siteNames, ceList=ces, ceTypeList=ceTypes, mode="Direct"
)
if not result["OK"]:
return result
resourceDict = result["Value"]
result = self.getQueues(resourceDict)
if not result["OK"]:
return result
# if not siteNames:
# siteName = gConfig.getValue( '/DIRAC/Site', 'Unknown' )
# if siteName == 'Unknown':
# return S_OK( 'No site specified for the SiteDirector' )
# else:
# siteNames = [siteName]
# self.siteNames = siteNames
if self.updateStatus:
self.log.always("Pilot status update requested")
if self.getOutput:
self.log.always("Pilot output retrieval requested")
if self.sendAccounting:
self.log.always("Pilot accounting sending requested")
self.log.always("Sites:", siteNames)
self.log.always("CETypes:", ceTypes)
self.log.always("CEs:", ces)
self.log.always("PilotDN:", self.pilotDN)
self.log.always("PilotGroup:", self.pilotGroup)
self.log.always("MaxPilotsToSubmit:", self.maxPilotsToSubmit)
self.log.always("MaxJobsInFillMode:", self.maxJobsInFillMode)
self.localhost = socket.getfqdn()
self.proxy = ""
if self.queueDict:
self.log.always("Agent will serve queues:")
for queue in self.queueDict:
self.log.always(
"Site: %s, CE: %s, Queue: %s"
% (self.queueDict[queue]["Site"], self.queueDict[queue]["CEName"], queue)
)
return S_OK()
def main():
global fullMatch
global sites
Script.registerSwitch("F", "full-match", "Check all the matching criteria",
setFullMatch)
Script.registerSwitch(
"S:", "site=", "Check matching for these sites (comma separated list)",
setSites)
Script.registerArgument("job_JDL: file with job JDL description")
_, args = Script.parseCommandLine(ignoreErrors=True)
from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup
from DIRAC.ConfigurationSystem.Client.Helpers import Resources
from DIRAC.Core.Utilities.PrettyPrint import printTable
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus
from DIRAC.WorkloadManagementSystem.Utilities.QueueUtilities import getQueuesResolved, matchQueue
with open(args[0]) as f:
jdl = f.read()
# Get the current VO
result = getVOfromProxyGroup()
if not result["OK"]:
gLogger.error("No proxy found, please login")
DIRACExit(-1)
voName = result["Value"]
resultQueues = Resources.getQueues(siteList=sites, community=voName)
if not resultQueues["OK"]:
gLogger.error("Failed to get CE information")
DIRACExit(-1)
siteDict = resultQueues["Value"]
result = getQueuesResolved(siteDict, {}, checkPlatform=True)
if not resultQueues["OK"]:
gLogger.error("Failed to get CE information")
DIRACExit(-1)
queueDict = result["Value"]
# get list of usable sites within this cycle
resultMask = SiteStatus().getUsableSites()
if not resultMask["OK"]:
gLogger.error("Failed to get Site mask information")
DIRACExit(-1)
siteMaskList = resultMask.get("Value", [])
rssClient = ResourceStatus()
fields = ("Site", "CE", "Queue", "Status", "Match", "Reason")
records = []
for queue, queueInfo in queueDict.items():
site = queueInfo["Site"]
ce = queueInfo["CEName"]
siteStatus = "Active" if site in siteMaskList else "InActive"
ceStatus = siteStatus
if rssClient.rssFlag:
result = rssClient.getElementStatus(ce, "ComputingElement")
if result["OK"]:
ceStatus = result["Value"][ce]["all"]
result = matchQueue(jdl,
queueInfo["ParametersDict"],
fullMatch=fullMatch)
if not result["OK"]:
gLogger.error("Failed in getting match data", result["Message"])
DIRACExit(-1)
status = "Active" if siteStatus == "Active" and ceStatus == "Active" else "Inactive"
if result["Value"]["Match"]:
records.append(
(site, ce, queueInfo["QueueName"], status, "Yes", ""))
else:
records.append((site, ce, queueInfo["QueueName"], status, "No",
result["Value"]["Reason"]))
gLogger.notice(
printTable(fields,
records,
sortField="Site",
columnSeparator=" ",
printOut=False))
