def _addPilotsAccountingReport(self, pilotsData):
""" fill accounting data
"""
for pRef in pilotsData:
pData = pilotsData[pRef]
pA = PilotAccounting()
pA.setEndTime(pData['LastUpdateTime'])
pA.setStartTime(pData['SubmissionTime'])
retVal = Registry.getUsernameForDN(pData['OwnerDN'])
if not retVal['OK']:
userName = '******'
self.log.error("Can't determine username for dn:", pData['OwnerDN'])
else:
userName = retVal['Value']
pA.setValueByKey('User', userName)
pA.setValueByKey('UserGroup', pData['OwnerGroup'])
result = getCESiteMapping(pData['DestinationSite'])
if result['OK'] and pData['DestinationSite'] in result['Value']:
pA.setValueByKey('Site', result['Value'][pData['DestinationSite']].strip())
else:
pA.setValueByKey('Site', 'Unknown')
pA.setValueByKey('GridCE', pData['DestinationSite'])
pA.setValueByKey('GridMiddleware', pData['GridType'])
pA.setValueByKey('GridResourceBroker', pData['Broker'])
pA.setValueByKey('GridStatus', pData['Status'])
if 'Jobs' not in pData:
pA.setValueByKey('Jobs', 0)
else:
pA.setValueByKey('Jobs', len(pData['Jobs']))
self.log.verbose("Added accounting record for pilot %s" % pData['PilotID'])
retVal = gDataStoreClient.addRegister(pA)
if not retVal['OK']:
return retVal
return S_OK()
def addQueueToCE(self, ceName, queueName, optionsDict=None):
""" Adds a new queue to a CE definition in the CS.
:param str ceName: FQN of the CE (e.g. ce503.cern.ch)
:param str queueName: name of the queue (e.g. ce503.cern.ch-condor)
:param dict optionsDict: optional dictionary of options
:returns: S_OK/S_ERROR structure
"""
res = getCESiteMapping(ceName)
if not res['OK']:
return res
if ceName not in res['Value']:
return S_ERROR("CE does not exist")
siteName = res['Value'][ceName]
# CSAPI.createSection() always returns S_OK even if the section already exists
self.__csMod.createSection(
cfgPath(self.__baseResources, 'Sites',
siteName.split('.')[0], siteName, 'CEs', ceName, 'Queues',
queueName))
# add options if requested
if optionsDict is not None:
for option, optionValue in optionsDict.items(
): # can be an iterator
self.__csMod.setOptionValue(
cfgPath(self.__baseResources, 'Sites',
siteName.split('.')[0], siteName, 'CEs', ceName,
'Queues', queueName, option), optionValue)
self.csModified = True
return S_OK(True)
def doCommand(self):
"""
Returns failed pilots using the DIRAC accounting system for every CE
for the last self.args[0] hours
:params:
:attr:`CEs`: list of CEs (when not given, take every CE)
:returns:
"""
if 'hours' not in self.args:
return S_ERROR('Number of hours not specified')
hours = self.args['hours']
ces = None
if 'ces' in self.args:
ces = self.args['ces']
if ces is None:
res = getCESiteMapping()
if not res['OK']:
return res
ces = list(res['Value'])
if not ces:
return S_ERROR('CEs is empty')
fromD = datetime.utcnow() - timedelta(hours=hours)
toD = datetime.utcnow()
failedPilots = self.rClient.getReport('Pilot', 'NumberOfPilots', fromD,
toD, {
'GridStatus': ['Aborted'],
'GridCE': ces
}, 'GridCE')
if not failedPilots['OK']:
return failedPilots
failedPilots = failedPilots['Value']
if 'data' not in failedPilots:
return S_ERROR('Missing data key')
if 'granularity' not in failedPilots:
return S_ERROR('Missing granularity key')
failedPilots['data'] = {
site: strToIntDict(value)
for site, value in failedPilots['data'].items()
}
singlePlots = {}
for ce, value in failedPilots['data'].items():
if ce in ces:
plot = {}
plot['data'] = {ce: value}
plot['granularity'] = failedPilots['granularity']
singlePlots[ce] = plot
return S_OK(singlePlots)
def __getXRSLExtraString(self, multiprocessor=False):
# For the XRSL additional string from configuration - only done at initialisation time
# If this string changes, the corresponding (ARC) site directors have to be restarted
#
# Variable = XRSLExtraString (or for multi processor mode)
# Default value = ''
# If you give a value, I think it should be of the form
# (aaa = "xxx")
# Otherwise the ARC job description parser will have a fit
# Locations searched in order :
# Top priority : Resources/Sites/<Grid>/<Site>/CEs/<CE>/XRSLExtraString
# Second priority : Resources/Sites/<Grid>/<Site>/XRSLExtraString
# Default : Resources/Computing/CEDefaults/XRSLExtraString
#
xrslExtraString = '' # Start with the default value
result = getCESiteMapping(self.ceHost)
if not result['OK'] or not result['Value']:
gLogger.error("Unknown CE ...")
return
self.site = result['Value'][self.ceHost]
# Now we know the site. Get the grid
grid = self.site.split(".")[0]
# The different possibilities that we have agreed upon
if multiprocessor:
xtraVariable = "XRSLMPExtraString"
else:
xtraVariable = "XRSLExtraString"
firstOption = "Resources/Sites/%s/%s/CEs/%s/%s" % (
grid, self.site, self.ceHost, xtraVariable)
secondOption = "Resources/Sites/%s/%s/%s" % (grid, self.site,
xtraVariable)
defaultOption = "Resources/Computing/CEDefaults/%s" % xtraVariable
# Now go about getting the string in the agreed order
gLogger.debug("Trying to get %s : first option %s" %
(xtraVariable, firstOption))
result = gConfig.getValue(firstOption, defaultValue='')
if result != '':
xrslExtraString = result
gLogger.debug("Found %s : %s" % (xtraVariable, xrslExtraString))
else:
gLogger.debug("Trying to get %s : second option %s" %
(xtraVariable, secondOption))
result = gConfig.getValue(secondOption, defaultValue='')
if result != '':
xrslExtraString = result
gLogger.debug("Found %s : %s" %
(xtraVariable, xrslExtraString))
else:
gLogger.debug("Trying to get %s : default option %s" %
(xtraVariable, defaultOption))
result = gConfig.getValue(defaultOption, defaultValue='')
if result != '':
xrslExtraString = result
gLogger.debug("Found %s : %s" %
(xtraVariable, xrslExtraString))
if xrslExtraString:
gLogger.always("%s : %s" % (xtraVariable, xrslExtraString))
gLogger.always(" --- to be added to pilots going to CE : %s" %
self.ceHost)
return xrslExtraString
def doCommand(self):
"""
Returns failed pilots using the DIRAC accounting system for every CE
for the last self.args[0] hours
:params:
:attr:`CEs`: list of CEs (when not given, take every CE)
:returns:
"""
if "hours" not in self.args:
return S_ERROR("Number of hours not specified")
hours = self.args["hours"]
ces = None
if "ces" in self.args:
ces = self.args["ces"]
if ces is None:
res = getCESiteMapping()
if not res["OK"]:
return res
ces = list(res["Value"])
if not ces:
return S_ERROR("CEs is empty")
fromD = datetime.utcnow() - timedelta(hours=hours)
toD = datetime.utcnow()
failedPilots = self.rClient.getReport("Pilot", "NumberOfPilots", fromD,
toD, {
"GridStatus": ["Aborted"],
"GridCE": ces
}, "GridCE")
if not failedPilots["OK"]:
return failedPilots
failedPilots = failedPilots["Value"]
if "data" not in failedPilots:
return S_ERROR("Missing data key")
if "granularity" not in failedPilots:
return S_ERROR("Missing granularity key")
failedPilots["data"] = {
site: strToIntDict(value)
for site, value in failedPilots["data"].items()
}
singlePlots = {}
for ce, value in failedPilots["data"].items():
if ce in ces:
plot = {}
plot["data"] = {ce: value}
plot["granularity"] = failedPilots["granularity"]
singlePlots[ce] = plot
return S_OK(singlePlots)
def __purgeSites(self, ceBdiiDict):
"""Remove all sites that are not in self.selectedSites.
Modifies the ceBdiiDict!
"""
if not self.selectedSites:
return
for site in list(ceBdiiDict):
ces = list(ceBdiiDict[site]['CEs'])
if not ces:
self.log.error("No CE information for site:", site)
continue
siteInCS = 'Not_In_CS'
for ce in ces:
res = getCESiteMapping(ce)
if not res['OK']:
self.log.error("Failed to get DIRAC site name for ce", "%s: %s" % (ce, res['Message']))
continue
# if the ce is not in the CS the returned value will be empty
if ce in res['Value']:
siteInCS = res['Value'][ce]
break
self.log.debug("Checking site %s (%s), aka %s" % (site, ces, siteInCS))
if siteInCS in self.selectedSites:
continue
self.log.info("Dropping site %s, aka %s" % (site, siteInCS))
ceBdiiDict.pop(site)
return
def setPilotDestinationSite(self, pilotRef, destination, conn=False):
"""Set the pilot agent destination site"""
gridSite = "Unknown"
res = getCESiteMapping(destination)
if res["OK"] and res["Value"]:
gridSite = res["Value"][destination]
req = "UPDATE PilotAgents SET DestinationSite='%s', GridSite='%s' WHERE PilotJobReference='%s'"
req = req % (destination, gridSite, pilotRef)
return self._update(req, conn=conn)
def doMaster(self):
""" Master method, which looks little bit spaghetti code, sorry !
- It gets all sites and transforms them into gocSites.
- It gets all the storage elements and transforms them into their hosts
- It gets the the CEs (FTS and file catalogs will come).
"""
gocSites = getGOCSites()
if not gocSites['OK']:
return gocSites
gocSites = gocSites['Value']
sesHosts = getStorageElementsHosts()
if not sesHosts['OK']:
return sesHosts
sesHosts = sesHosts['Value']
resources = sesHosts if sesHosts else []
ftsServer = getFTS3Servers(hostOnly=True)
if ftsServer['OK'] and ftsServer['Value']:
resources.extend(ftsServer['Value'])
# TODO: file catalogs need also to use their hosts
# fc = CSHelpers.getFileCatalogs()
# if fc[ 'OK' ]:
# resources = resources + fc[ 'Value' ]
res = getCESiteMapping()
if res['OK'] and res['Value']:
resources.extend(list(res['Value']))
self.log.verbose('Processing Sites',
', '.join(gocSites if gocSites else ['NONE']))
siteRes = self.doNew(('Site', gocSites))
if not siteRes['OK']:
self.metrics['failed'].append(siteRes['Message'])
self.log.verbose('Processing Resources',
', '.join(resources if resources else ['NONE']))
resourceRes = self.doNew(('Resource', resources))
if not resourceRes['OK']:
self.metrics['failed'].append(resourceRes['Message'])
return S_OK(self.metrics)
def setPilotStatus(
self,
pilotRef,
status,
destination=None,
statusReason=None,
gridSite=None,
queue=None,
benchmark=None,
currentJob=None,
updateTime=None,
conn=False,
):
"""Set pilot job status"""
setList = []
setList.append("Status='%s'" % status)
if updateTime:
setList.append("LastUpdateTime='%s'" % updateTime)
else:
setList.append("LastUpdateTime=UTC_TIMESTAMP()")
if not statusReason:
statusReason = "Not given"
setList.append("StatusReason='%s'" % statusReason)
if gridSite:
setList.append("GridSite='%s'" % gridSite)
if queue:
setList.append("Queue='%s'" % queue)
if benchmark:
setList.append("BenchMark='%s'" % float(benchmark))
if currentJob:
setList.append("CurrentJobID='%s'" % int(currentJob))
if destination:
setList.append("DestinationSite='%s'" % destination)
if not gridSite:
res = getCESiteMapping(destination)
if res["OK"] and res["Value"]:
setList.append("GridSite='%s'" % res["Value"][destination])
set_string = ",".join(setList)
req = "UPDATE PilotAgents SET " + set_string + " WHERE PilotJobReference='%s'" % pilotRef
result = self._update(req, conn=conn)
if not result["OK"]:
return result
return S_OK()
def __removeNonExistingResourcesFromRM(self):
"""
Remove resources from DowntimeCache table that no longer exist in the CS.
"""
if not getServiceURL("ResourceStatus/ResourceManagement"):
gLogger.verbose(
"ResourceManagement is not installed, skipping removal of non existing resources..."
)
return S_OK()
sesHosts = getStorageElementsHosts()
if not sesHosts["OK"]:
return sesHosts
sesHosts = sesHosts["Value"]
resources = sesHosts
ftsServer = getFTS3Servers(hostOnly=True)
if ftsServer["OK"]:
resources.extend(ftsServer["Value"])
res = getCESiteMapping()
if res["OK"]:
resources.extend(list(res["Value"]))
downtimes = self.rManagement.selectDowntimeCache()
if not downtimes["OK"]:
return downtimes
# Remove hosts that no longer exist in the CS
for host in downtimes["Value"]:
gLogger.verbose("Checking if %s is still in the CS" % host[0])
if host[0] not in resources:
gLogger.verbose("%s is no longer in CS, removing entry..." %
host[0])
result = self.rManagement.deleteDowntimeCache(name=host[0])
if not result["OK"]:
return result
return S_OK()
def getResources():
"""
Gets all resources
"""
resources = DMSHelpers().getStorageElements()
fts = getFTS()
if fts["OK"]:
resources = resources + fts["Value"]
fc = getFileCatalogs()
if fc["OK"]:
resources = resources + fc["Value"]
res = getCESiteMapping()
if res["OK"]:
resources = resources + list(res["Value"])
return S_OK(resources)
def setDestinationCE(self, ceName, diracSite=None):
""" Developer function.
Allows to direct a job to a particular Grid CE.
"""
kwargs = {'ceName': ceName}
if not diracSite:
res = getCESiteMapping(ceName)
if not res['OK']:
return self._reportError(res['Message'], **kwargs)
if not res['Value']:
return self._reportError('No DIRAC site name found for CE %s' % (ceName), **kwargs)
diracSite = res['Value'][ceName]
self.setDestination(diracSite)
# Keep GridRequiredCEs for backward compatibility
self._addJDLParameter('GridRequiredCEs', ceName)
self._addJDLParameter('GridCE', ceName)
return S_OK()
def getQueueInfo(ceUniqueID, diracSiteName=''):
"""
Extract information from full CE Name including associate DIRAC Site
"""
try:
subClusterUniqueID = ceUniqueID.split('/')[0].split(':')[0]
queueID = ceUniqueID.split('/')[1]
except IndexError:
return S_ERROR('Wrong full queue Name')
if not diracSiteName:
gLogger.debug("SiteName not given, looking in /LocaSite/Site")
diracSiteName = gConfig.getValue('/LocalSite/Site', '')
if not diracSiteName:
gLogger.debug("Can't find LocalSite name, looking in CS")
result = getCESiteMapping(subClusterUniqueID)
if not result['OK']:
return result
diracSiteName = result['Value'][subClusterUniqueID]
if not diracSiteName:
gLogger.error('Can not find corresponding Site in CS')
return S_ERROR('Can not find corresponding Site in CS')
gridType = diracSiteName.split('.')[0]
siteCSSEction = '/Resources/Sites/%s/%s/CEs/%s' % (gridType, diracSiteName,
subClusterUniqueID)
queueCSSection = '%s/Queues/%s' % (siteCSSEction, queueID)
resultDict = {
'SubClusterUniqueID': subClusterUniqueID,
'QueueID': queueID,
'SiteName': diracSiteName,
'Grid': gridType,
'SiteCSSEction': siteCSSEction,
'QueueCSSection': queueCSSection
}
return S_OK(resultDict)
def getQueueInfo(ceUniqueID, diracSiteName=""):
"""
Extract information from full CE Name including associate DIRAC Site
"""
try:
subClusterUniqueID = ceUniqueID.split("/")[0].split(":")[0]
queueID = ceUniqueID.split("/")[1]
except IndexError:
return S_ERROR("Wrong full queue Name")
if not diracSiteName:
gLogger.debug("SiteName not given, looking in /LocaSite/Site")
diracSiteName = gConfig.getValue("/LocalSite/Site", "")
if not diracSiteName:
gLogger.debug("Can't find LocalSite name, looking in CS")
result = getCESiteMapping(subClusterUniqueID)
if not result["OK"]:
return result
diracSiteName = result["Value"][subClusterUniqueID]
if not diracSiteName:
gLogger.error("Can not find corresponding Site in CS")
return S_ERROR("Can not find corresponding Site in CS")
gridType = diracSiteName.split(".")[0]
siteCSSEction = "/Resources/Sites/%s/%s/CEs/%s" % (gridType, diracSiteName,
subClusterUniqueID)
queueCSSection = "%s/Queues/%s" % (siteCSSEction, queueID)
resultDict = {
"SubClusterUniqueID": subClusterUniqueID,
"QueueID": queueID,
"SiteName": diracSiteName,
"Grid": gridType,
"SiteCSSEction": siteCSSEction,
"QueueCSSection": queueCSSection,
}
return S_OK(resultDict)
def doMaster(self):
siteNames = getSites()
if not siteNames['OK']:
return siteNames
siteNames = siteNames['Value']
res = getCESiteMapping()
if not res['OK']:
return res
ces = list(res['Value'])
pilotResults = self.doNew(('Site', siteNames))
if not pilotResults['OK']:
self.metrics['failed'].append(pilotResults['Message'])
pilotResults = self.doNew(('Resource', ces))
if not pilotResults['OK']:
self.metrics['failed'].append(pilotResults['Message'])
return S_OK(self.metrics)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument("CE: Name of the CE")
Script.parseCommandLine(ignoreErrors=True)
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
ce = Script.getPositionalArgs(group=True)
from DIRAC.ConfigurationSystem.Client.Helpers import cfgPath
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getCESiteMapping
res = getCESiteMapping(ce)
if not res["OK"]:
gLogger.error(res["Message"])
Dexit(1)
site = res["Value"][ce]
res = gConfig.getOptionsDict(cfgPath("Resources", "Sites", site.split(".")[0], site, "CEs", ce))
if not res["OK"]:
gLogger.error(res["Message"])
Dexit(1)
gLogger.notice(res["Value"])
def doMaster(self):
self.log.debug("PilotCommand doMaster")
siteNames = getSites()
if not siteNames["OK"]:
return siteNames
siteNames = siteNames["Value"]
res = getCESiteMapping()
if not res["OK"]:
return res
ces = list(res["Value"])
pilotResults = self.doNew(("Site", siteNames))
if not pilotResults["OK"]:
self.metrics["failed"].append(pilotResults["Message"])
pilotResults = self.doNew(("Resource", ces))
if not pilotResults["OK"]:
self.metrics["failed"].append(pilotResults["Message"])
return S_OK(self.metrics)
def _addPilotsAccountingReport(self, pilotsData):
"""fill accounting data"""
for pRef in pilotsData:
pData = pilotsData[pRef]
pA = PilotAccounting()
pA.setEndTime(pData["LastUpdateTime"])
pA.setStartTime(pData["SubmissionTime"])
retVal = Registry.getUsernameForDN(pData["OwnerDN"])
if not retVal["OK"]:
userName = "******"
self.log.error(
"Can't determine username for dn",
": %s : %s" % (pData["OwnerDN"], retVal["Message"]),
)
else:
userName = retVal["Value"]
pA.setValueByKey("User", userName)
pA.setValueByKey("UserGroup", pData["OwnerGroup"])
result = getCESiteMapping(pData["DestinationSite"])
if result["OK"] and pData["DestinationSite"] in result["Value"]:
pA.setValueByKey(
"Site", result["Value"][pData["DestinationSite"]].strip())
else:
pA.setValueByKey("Site", "Unknown")
pA.setValueByKey("GridCE", pData["DestinationSite"])
pA.setValueByKey("GridMiddleware", pData["GridType"])
pA.setValueByKey("GridResourceBroker", pData["Broker"])
pA.setValueByKey("GridStatus", pData["Status"])
if "Jobs" not in pData:
pA.setValueByKey("Jobs", 0)
else:
pA.setValueByKey("Jobs", len(pData["Jobs"]))
self.log.verbose("Added accounting record for pilot %s" %
pData["PilotID"])
retVal = gDataStoreClient.addRegister(pA)
if not retVal["OK"]:
return retVal
return S_OK()
def main():
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
from DIRAC.ConfigurationSystem.Client.Helpers import cfgPath
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getCESiteMapping
if len(args) < 1:
Script.showHelp(exitCode=1)
res = getCESiteMapping(args[0])
if not res['OK']:
gLogger.error(res['Message'])
Dexit(1)
site = res['Value'][args[0]]
res = gConfig.getOptionsDict(
cfgPath('Resources', 'Sites',
site.split('.')[0], site, 'CEs', args[0]))
if not res['OK']:
gLogger.error(res['Message'])
Dexit(1)
gLogger.notice(res['Value'])
from DIRAC import gConfig, gLogger, exit as Dexit
from DIRAC.Core.Base import Script
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getCESiteMapping
from DIRAC.ConfigurationSystem.Client.Helpers import cfgPath
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... CE ...' % Script.scriptName, 'Arguments:',
' CE: Name of the CE'
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
res = getCESiteMapping(args[0])
if not res['OK']:
gLogger.error(res['Message'])
Dexit(1)
site = res['Value'][args[0]]
res = gConfig.getOptionsDict(
cfgPath('Resources', 'Sites',
site.split('.')[0], site, 'CEs', args[0]))
if not res['OK']:
gLogger.error(res['Message'])
Dexit(1)
gLogger.notice(res['Value'])
def __syncComputingElements(self):
'''
Sync ComputingElements: compares CS with DB and does the necessary modifications.
'''
res = getCESiteMapping()
if not res['OK']:
return res
cesCS = list(res['Value'])
gLogger.verbose('%s Computing elements found in CS' % len(cesCS))
cesDB = self.rStatus.selectStatusElement('Resource', 'Status',
elementType='ComputingElement',
meta={'columns': ['Name']})
if not cesDB['OK']:
return cesDB
cesDB = [ceDB[0] for ceDB in cesDB['Value']]
# ComputingElements that are in DB but not in CS
toBeDeleted = list(set(cesDB).difference(set(cesCS)))
gLogger.verbose('%s Computing elements to be deleted' %
len(toBeDeleted))
# Delete storage elements
for ceName in toBeDeleted:
deleteQuery = self.rStatus._extermineStatusElement(
'Resource', ceName)
gLogger.verbose('... %s' % ceName)
if not deleteQuery['OK']:
return deleteQuery
# statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ]
statusTypes = self.rssConfig.getConfigStatusType('ComputingElement')
result = self.rStatus.selectStatusElement('Resource', 'Status',
elementType='ComputingElement',
meta={'columns': ['Name', 'StatusType']})
if not result['OK']:
return result
cesTuple = [(x[0], x[1]) for x in result['Value']]
# For each ( se, statusType ) tuple not present in the DB, add it.
cesStatusTuples = [(se, statusType)
for se in cesCS for statusType in statusTypes]
toBeAdded = list(set(cesStatusTuples).difference(set(cesTuple)))
gLogger.debug('%s Computing elements entries to be added' %
len(toBeAdded))
for ceTuple in toBeAdded:
_name = ceTuple[0]
_statusType = ceTuple[1]
_status = self.defaultStatus
_reason = 'Synchronized'
_elementType = 'ComputingElement'
query = self.rStatus.addIfNotThereStatusElement('Resource', 'Status', name=_name,
statusType=_statusType,
status=_status,
elementType=_elementType,
tokenOwner=self.tokenOwner,
reason=_reason)
if not query['OK']:
return query
return S_OK()
def run(self):
''' Checks it has the parameters it needs and writes the date to a cache file.
'''
# Minor security checks
element = self.decisionParams['element']
if element is None:
return S_ERROR('element should not be None')
name = self.decisionParams['name']
if name is None:
return S_ERROR('name should not be None')
statusType = self.decisionParams['statusType']
if statusType is None:
return S_ERROR('statusType should not be None')
previousStatus = self.decisionParams['status']
if previousStatus is None:
return S_ERROR('status should not be None')
status = self.enforcementResult['Status']
if status is None:
return S_ERROR('status should not be None')
reason = self.enforcementResult['Reason']
if reason is None:
return S_ERROR('reason should not be None')
if self.decisionParams['element'] == 'Site':
siteName = self.decisionParams['name']
else:
elementType = self.decisionParams['elementType']
if elementType == 'StorageElement':
siteName = getSitesForSE(name)
elif elementType == 'ComputingElement':
res = getCESiteMapping(name)
if not res['OK']:
return res
siteName = S_OK(res['Value'][name])
else:
siteName = {'OK': True, 'Value': 'Unassigned'}
if not siteName['OK']:
self.log.error('Resource %s does not exist at any site: %s' % (name, siteName['Message']))
siteName = "Unassigned Resources"
elif not siteName['Value']:
siteName = "Unassigned Resources"
else:
siteName = siteName['Value'] if isinstance(siteName['Value'], six.string_types) else siteName['Value'][0]
with sqlite3.connect(self.cacheFile) as conn:
try:
conn.execute('''CREATE TABLE IF NOT EXISTS ResourceStatusCache(
SiteName VARCHAR(64) NOT NULL,
ResourceName VARCHAR(64) NOT NULL,
Status VARCHAR(8) NOT NULL DEFAULT "",
PreviousStatus VARCHAR(8) NOT NULL DEFAULT "",
StatusType VARCHAR(128) NOT NULL DEFAULT "all",
Time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);''')
insertQuery = "INSERT INTO ResourceStatusCache (SiteName, ResourceName, Status, PreviousStatus, StatusType)"
insertQuery += " VALUES ('%s', '%s', '%s', '%s', '%s' ); " % (siteName, name, status,
previousStatus, statusType)
conn.execute(insertQuery)
conn.commit()
except sqlite3.OperationalError:
self.log.error('Email cache database is locked')
return S_OK()
def run(self):
"""Checks it has the parameters it needs and writes the date to a cache file."""
# Minor security checks
element = self.decisionParams["element"]
if element is None:
return S_ERROR("element should not be None")
name = self.decisionParams["name"]
if name is None:
return S_ERROR("name should not be None")
statusType = self.decisionParams["statusType"]
if statusType is None:
return S_ERROR("statusType should not be None")
previousStatus = self.decisionParams["status"]
if previousStatus is None:
return S_ERROR("status should not be None")
status = self.enforcementResult["Status"]
if status is None:
return S_ERROR("status should not be None")
reason = self.enforcementResult["Reason"]
if reason is None:
return S_ERROR("reason should not be None")
if self.decisionParams["element"] == "Site":
siteName = self.decisionParams["name"]
else:
elementType = self.decisionParams["elementType"]
if elementType == "StorageElement":
siteName = getSitesForSE(name)
elif elementType == "ComputingElement":
res = getCESiteMapping(name)
if not res["OK"]:
return res
siteName = S_OK(res["Value"][name])
else:
siteName = {"OK": True, "Value": "Unassigned"}
if not siteName["OK"]:
self.log.error("Resource %s does not exist at any site: %s" % (name, siteName["Message"]))
siteName = "Unassigned Resources"
elif not siteName["Value"]:
siteName = "Unassigned Resources"
else:
siteName = (
siteName["Value"] if isinstance(siteName["Value"], six.string_types) else siteName["Value"][0]
)
# create record for insertion
recordDict = {}
recordDict["SiteName"] = siteName
recordDict["ResourceName"] = name
recordDict["Status"] = status
recordDict["PreviousStatus"] = previousStatus
recordDict["StatusType"] = statusType
return self.rsClient.insert("ResourceStatusCache", recordDict)
def run(self):
''' Checks it has the parameters it needs and writes the date to a cache file.
'''
# Minor security checks
element = self.decisionParams['element']
if element is None:
return S_ERROR('element should not be None')
name = self.decisionParams['name']
if name is None:
return S_ERROR('name should not be None')
statusType = self.decisionParams['statusType']
if statusType is None:
return S_ERROR('statusType should not be None')
previousStatus = self.decisionParams['status']
if previousStatus is None:
return S_ERROR('status should not be None')
status = self.enforcementResult['Status']
if status is None:
return S_ERROR('status should not be None')
reason = self.enforcementResult['Reason']
if reason is None:
return S_ERROR('reason should not be None')
if self.decisionParams['element'] == 'Site':
siteName = self.decisionParams['name']
else:
elementType = self.decisionParams['elementType']
if elementType == 'StorageElement':
siteName = getSitesForSE(name)
elif elementType == 'ComputingElement':
res = getCESiteMapping(name)
if not res['OK']:
return res
siteName = S_OK(res['Value'][name])
else:
siteName = {'OK': True, 'Value': 'Unassigned'}
if not siteName['OK']:
self.log.error('Resource %s does not exist at any site: %s' %
(name, siteName['Message']))
siteName = "Unassigned Resources"
elif not siteName['Value']:
siteName = "Unassigned Resources"
else:
siteName = siteName['Value'] if isinstance(
siteName['Value'],
six.string_types) else siteName['Value'][0]
# create record for insertion
recordDict = {}
recordDict['SiteName'] = siteName
recordDict['ResourceName'] = name
recordDict['Status'] = status
recordDict['PreviousStatus'] = previousStatus
recordDict['StatusType'] = statusType
return self.rsClient.insert('ResourceStatusCache', recordDict)
def getPilotSummaryWeb(self, selectDict, sortList, startItem, maxItems):
""" Get summary of the pilot jobs status by CE/site in a standard structure
"""
stateNames = [
'Submitted', 'Ready', 'Scheduled', 'Waiting', 'Running', 'Done',
'Aborted', 'Failed'
]
allStateNames = stateNames + ['Done_Empty', 'Aborted_Hour']
paramNames = ['Site', 'CE'] + allStateNames
last_update = None
if 'LastUpdateTime' in selectDict:
last_update = selectDict['LastUpdateTime']
del selectDict['LastUpdateTime']
site_select = []
if 'GridSite' in selectDict:
site_select = selectDict['GridSite']
if not isinstance(site_select, list):
site_select = [site_select]
del selectDict['GridSite']
status_select = []
if 'Status' in selectDict:
status_select = selectDict['Status']
if not isinstance(status_select, list):
status_select = [status_select]
del selectDict['Status']
expand_site = ''
if 'ExpandSite' in selectDict:
expand_site = selectDict['ExpandSite']
site_select = [expand_site]
del selectDict['ExpandSite']
# Get all the data from the database with various selections
result = self.getCounters('PilotAgents',
['GridSite', 'DestinationSite', 'Status'],
selectDict,
newer=last_update,
timeStamp='LastUpdateTime')
if not result['OK']:
return result
last_update = Time.dateTime() - Time.hour
selectDict['Status'] = 'Aborted'
resultHour = self.getCounters(
'PilotAgents', ['GridSite', 'DestinationSite', 'Status'],
selectDict,
newer=last_update,
timeStamp='LastUpdateTime')
if not resultHour['OK']:
return resultHour
last_update = Time.dateTime() - Time.day
selectDict['Status'] = ['Aborted', 'Done']
resultDay = self.getCounters('PilotAgents',
['GridSite', 'DestinationSite', 'Status'],
selectDict,
newer=last_update,
timeStamp='LastUpdateTime')
if not resultDay['OK']:
return resultDay
selectDict['CurrentJobID'] = 0
selectDict['Status'] = 'Done'
resultDayEmpty = self.getCounters(
'PilotAgents', ['GridSite', 'DestinationSite', 'Status'],
selectDict,
newer=last_update,
timeStamp='LastUpdateTime')
if not resultDayEmpty['OK']:
return resultDayEmpty
ceMap = {}
resMap = getCESiteMapping()
if resMap['OK']:
ceMap = resMap['Value']
# Sort out different counters
resultDict = {}
resultDict['Unknown'] = {}
for attDict, count in result['Value']:
site = attDict['GridSite']
ce = attDict['DestinationSite']
state = attDict['Status']
if site == 'Unknown' and ce != "Unknown" and ce != "Multiple" and ce in ceMap:
site = ceMap[ce]
if site not in resultDict:
resultDict[site] = {}
if ce not in resultDict[site]:
resultDict[site][ce] = {}
for p in allStateNames:
resultDict[site][ce][p] = 0
resultDict[site][ce][state] = count
for attDict, count in resultDay['Value']:
site = attDict['GridSite']
ce = attDict['DestinationSite']
state = attDict['Status']
if site == 'Unknown' and ce != "Unknown" and ce in ceMap:
site = ceMap[ce]
if state == "Done":
resultDict[site][ce]["Done"] = count
if state == "Aborted":
resultDict[site][ce]["Aborted"] = count
for attDict, count in resultDayEmpty['Value']:
site = attDict['GridSite']
ce = attDict['DestinationSite']
state = attDict['Status']
if site == 'Unknown' and ce != "Unknown" and ce in ceMap:
site = ceMap[ce]
if state == "Done":
resultDict[site][ce]["Done_Empty"] = count
for attDict, count in resultHour['Value']:
site = attDict['GridSite']
ce = attDict['DestinationSite']
state = attDict['Status']
if site == 'Unknown' and ce != "Unknown" and ce in ceMap:
site = ceMap[ce]
if state == "Aborted":
resultDict[site][ce]["Aborted_Hour"] = count
records = []
siteSumDict = {}
for site in resultDict:
sumDict = {}
for state in allStateNames:
if state not in sumDict:
sumDict[state] = 0
sumDict['Total'] = 0
for ce in resultDict[site]:
itemList = [site, ce]
total = 0
for state in allStateNames:
itemList.append(resultDict[site][ce][state])
sumDict[state] += resultDict[site][ce][state]
if state == "Done":
done = resultDict[site][ce][state]
if state == "Done_Empty":
empty = resultDict[site][ce][state]
if state == "Aborted":
aborted = resultDict[site][ce][state]
if state != "Aborted_Hour" and state != "Done_Empty":
total += resultDict[site][ce][state]
sumDict['Total'] += total
# Add the total number of pilots seen in the last day
itemList.append(total)
# Add pilot submission efficiency evaluation
if (done - empty) > 0:
eff = done / (done - empty)
elif done == 0:
eff = 0.
elif empty == done:
eff = 99.
else:
eff = 0.
itemList.append('%.2f' % eff)
# Add pilot job efficiency evaluation
if total > 0:
eff = (total - aborted) / total * 100
else:
eff = 100.
itemList.append('%.2f' % eff)
# Evaluate the quality status of the CE
if total > 10:
if eff < 25.:
itemList.append('Bad')
elif eff < 60.:
itemList.append('Poor')
elif eff < 85.:
itemList.append('Fair')
else:
itemList.append('Good')
else:
itemList.append('Idle')
if len(resultDict[site]) == 1 or expand_site:
records.append(itemList)
if len(resultDict[site]) > 1 and not expand_site:
itemList = [site, 'Multiple']
for state in allStateNames + ['Total']:
if state in sumDict:
itemList.append(sumDict[state])
else:
itemList.append(0)
done = sumDict["Done"]
empty = sumDict["Done_Empty"]
aborted = sumDict["Aborted"]
total = sumDict["Total"]
# Add pilot submission efficiency evaluation
if (done - empty) > 0:
eff = done / (done - empty)
elif done == 0:
eff = 0.
elif empty == done:
eff = 99.
else:
eff = 0.
itemList.append('%.2f' % eff)
# Add pilot job efficiency evaluation
if total > 0:
eff = (total - aborted) / total * 100
else:
eff = 100.
itemList.append('%.2f' % eff)
# Evaluate the quality status of the Site
if total > 10:
if eff < 25.:
itemList.append('Bad')
elif eff < 60.:
itemList.append('Poor')
elif eff < 85.:
itemList.append('Fair')
else:
itemList.append('Good')
else:
itemList.append('Idle')
records.append(itemList)
for state in allStateNames + ['Total']:
if state not in siteSumDict:
siteSumDict[state] = sumDict[state]
else:
siteSumDict[state] += sumDict[state]
# Perform site selection
if site_select:
new_records = []
for r in records:
if r[0] in site_select:
new_records.append(r)
records = new_records
# Perform status selection
if status_select:
new_records = []
for r in records:
if r[14] in status_select:
new_records.append(r)
records = new_records
# Get the Site Mask data
result = SiteStatus().getUsableSites()
if result['OK']:
siteMask = result['Value']
for r in records:
if r[0] in siteMask:
r.append('Yes')
else:
r.append('No')
else:
for r in records:
r.append('Unknown')
finalDict = {}
finalDict['TotalRecords'] = len(records)
finalDict['ParameterNames'] = paramNames + \
['Total', 'PilotsPerJob', 'PilotJobEff', 'Status', 'InMask']
# Return all the records if maxItems == 0 or the specified number otherwise
if maxItems:
finalDict['Records'] = records[startItem:startItem + maxItems]
else:
finalDict['Records'] = records
done = siteSumDict["Done"]
empty = siteSumDict["Done_Empty"]
aborted = siteSumDict["Aborted"]
total = siteSumDict["Total"]
# Add pilot submission efficiency evaluation
if (done - empty) > 0:
eff = done / (done - empty)
elif done == 0:
eff = 0.
elif empty == done:
eff = 99.
else:
eff = 0.
siteSumDict['PilotsPerJob'] = '%.2f' % eff
# Add pilot job efficiency evaluation
if total > 0:
eff = (total - aborted) / total * 100
else:
eff = 100.
siteSumDict['PilotJobEff'] = '%.2f' % eff
# Evaluate the overall quality status
if total > 100:
if eff < 25.:
siteSumDict['Status'] = 'Bad'
elif eff < 60.:
siteSumDict['Status'] = 'Poor'
elif eff < 85.:
siteSumDict['Status'] = 'Fair'
else:
siteSumDict['Status'] = 'Good'
else:
siteSumDict['Status'] = 'Idle'
finalDict['Extras'] = siteSumDict
return S_OK(finalDict)
def getPilotMonitorWeb(self, selectDict, sortList, startItem, maxItems):
""" Get summary of the pilot job information in a standard structure
"""
resultDict = {}
if 'LastUpdateTime' in selectDict:
del selectDict['LastUpdateTime']
if 'Owner' in selectDict:
userList = selectDict['Owner']
if not isinstance(userList, list):
userList = [userList]
dnList = []
for uName in userList:
uList = getDNForUsername(uName)['Value']
dnList += uList
selectDict['OwnerDN'] = dnList
del selectDict['Owner']
startDate = selectDict.get('FromDate', None)
if startDate:
del selectDict['FromDate']
# For backward compatibility
if startDate is None:
startDate = selectDict.get('LastUpdateTime', None)
if startDate:
del selectDict['LastUpdateTime']
endDate = selectDict.get('ToDate', None)
if endDate:
del selectDict['ToDate']
# Sorting instructions. Only one for the moment.
if sortList:
orderAttribute = sortList[0][0] + ":" + sortList[0][1]
else:
orderAttribute = None
# Select pilots for the summary
result = self.selectPilots(selectDict,
orderAttribute=orderAttribute,
newer=startDate,
older=endDate,
timeStamp='LastUpdateTime')
if not result['OK']:
return S_ERROR('Failed to select pilots: ' + result['Message'])
pList = result['Value']
nPilots = len(pList)
resultDict['TotalRecords'] = nPilots
if nPilots == 0:
return S_OK(resultDict)
ini = startItem
last = ini + maxItems
if ini >= nPilots:
return S_ERROR('Item number out of range')
if last > nPilots:
last = nPilots
pilotList = pList[ini:last]
paramNames = [
'PilotJobReference', 'OwnerDN', 'OwnerGroup', 'GridType', 'Broker',
'Status', 'DestinationSite', 'BenchMark', 'ParentID',
'SubmissionTime', 'PilotID', 'LastUpdateTime', 'CurrentJobID',
'TaskQueueID', 'GridSite'
]
result = self.getPilotInfo(pilotList, paramNames=paramNames)
if not result['OK']:
return S_ERROR('Failed to get pilot info: ' + result['Message'])
pilotDict = result['Value']
records = []
for pilot in pilotList:
parList = []
for parameter in paramNames:
if not isinstance(pilotDict[pilot][parameter],
six.integer_types):
parList.append(str(pilotDict[pilot][parameter]))
else:
parList.append(pilotDict[pilot][parameter])
if parameter == 'GridSite':
gridSite = pilotDict[pilot][parameter]
# If the Grid Site is unknown try to recover it in the last moment
if gridSite == "Unknown":
ce = pilotDict[pilot]['DestinationSite']
result = getCESiteMapping(ce)
if result['OK']:
gridSite = result['Value'].get(ce)
del parList[-1]
parList.append(gridSite)
records.append(parList)
resultDict['ParameterNames'] = paramNames
resultDict['Records'] = records
return S_OK(resultDict)
def getGlue2CEInfo(vo, host):
""" call ldap for GLUE2 and get information
:param str vo: Virtual Organisation
:param str host: host to query for information
:returns: result structure with result['Value'][siteID]['CEs'][ceID]['Queues'][queueName]. For
each siteID, ceID, queueName all the GLUE2 parameters are retrieved
"""
# get all Policies allowing given VO
filt = "(&(objectClass=GLUE2Policy)(|(GLUE2PolicyRule=VO:%s)(GLUE2PolicyRule=vo:%s)))" % (
vo, vo)
polRes = __ldapsearchBDII(filt=filt,
attr=None,
host=host,
base="o=glue",
selectionString="GLUE2")
if not polRes['OK']:
return S_ERROR("Failed to get policies for this VO")
polRes = polRes['Value']
sLog.notice("Found %s policies for this VO %s" % (len(polRes), vo))
# get all shares for this policy
# create an or'ed list of all the shares and then call the search
listOfSitesWithPolicies = set()
shareFilter = ''
for policyValues in polRes:
# skip entries without GLUE2DomainID in the DN because we cannot associate them to a site
if 'GLUE2DomainID' not in policyValues['attr']['dn']:
continue
shareID = policyValues['attr'].get('GLUE2MappingPolicyShareForeignKey',
None)
policyID = policyValues['attr']['GLUE2PolicyID']
siteName = policyValues['attr']['dn'].split('GLUE2DomainID=')[1].split(
',', 1)[0]
listOfSitesWithPolicies.add(siteName)
if shareID is None: # policy not pointing to ComputingInformation
sLog.debug("Policy %s does not point to computing information" %
(policyID, ))
continue
sLog.verbose("%s policy %s pointing to %s " %
(siteName, policyID, shareID))
sLog.debug("Policy values:\n%s" % pformat(policyValues))
shareFilter += '(GLUE2ShareID=%s)' % shareID
filt = '(&(objectClass=GLUE2Share)(|%s))' % shareFilter
shareRes = __ldapsearchBDII(filt=filt,
attr=None,
host=host,
base="o=glue",
selectionString="GLUE2")
if not shareRes['OK']:
sLog.error("Could not get share information", shareRes['Message'])
return shareRes
shareInfoLists = {}
for shareInfo in shareRes['Value']:
if 'GLUE2DomainID' not in shareInfo['attr']['dn']:
continue
if 'GLUE2ComputingShare' not in shareInfo['objectClass']:
sLog.debug('Share %r is not a ComputingShare: \n%s' %
(shareID, pformat(shareInfo)))
continue
sLog.debug("Found computing share:\n%s" % pformat(shareInfo))
siteName = shareInfo['attr']['dn'].split('GLUE2DomainID=')[1].split(
',', 1)[0]
shareInfoLists.setdefault(siteName, []).append(shareInfo['attr'])
siteInfo = __getGlue2ShareInfo(host, shareInfoLists)
if not siteInfo['OK']:
sLog.error("Could not get CE info for",
"%s: %s" % (shareID, siteInfo['Message']))
return siteInfo
siteDict = siteInfo['Value']
sLog.debug("Found Sites:\n%s" % pformat(siteDict))
sitesWithoutShares = set(siteDict) - listOfSitesWithPolicies
if sitesWithoutShares:
sLog.error("Found some sites without any shares",
pformat(sitesWithoutShares))
else:
sLog.notice("Found information for all known sites")
# remap siteDict to assign CEs to known sites, in case their names differ from the "gocdb name" in
# the CS.
newSiteDict = {}
ceSiteMapping = getCESiteMapping().get('Value', {})
# FIXME: pylint thinks siteDict is a tuple, so we cast
for siteName, infoDict in dict(siteDict).items():
for ce, ceInfo in infoDict.get('CEs', {}).items():
ceSiteName = ceSiteMapping.get(ce, siteName)
gocSiteName = getGOCSiteName(ceSiteName).get('Value', siteName)
newSiteDict.setdefault(gocSiteName, {}).setdefault('CEs',
{})[ce] = ceInfo
return S_OK(newSiteDict)
def __syncComputingElements(self):
"""
Sync ComputingElements: compares CS with DB and does the necessary modifications.
"""
res = getCESiteMapping()
if not res["OK"]:
return res
cesCS = list(res["Value"])
gLogger.verbose("%s Computing elements found in CS" % len(cesCS))
cesDB = self.rStatus.selectStatusElement(
"Resource",
"Status",
elementType="ComputingElement",
meta={"columns": ["Name"]})
if not cesDB["OK"]:
return cesDB
cesDB = [ceDB[0] for ceDB in cesDB["Value"]]
# ComputingElements that are in DB but not in CS
toBeDeleted = list(set(cesDB).difference(set(cesCS)))
gLogger.verbose("%s Computing elements to be deleted" %
len(toBeDeleted))
# Delete storage elements
for ceName in toBeDeleted:
deleteQuery = self.rStatus._extermineStatusElement(
"Resource", ceName)
gLogger.verbose("... %s" % ceName)
if not deleteQuery["OK"]:
return deleteQuery
# statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ]
statusTypes = self.rssConfig.getConfigStatusType("ComputingElement")
result = self.rStatus.selectStatusElement(
"Resource",
"Status",
elementType="ComputingElement",
meta={"columns": ["Name", "StatusType"]})
if not result["OK"]:
return result
cesTuple = [(x[0], x[1]) for x in result["Value"]]
# For each ( se, statusType ) tuple not present in the DB, add it.
cesStatusTuples = [(se, statusType) for se in cesCS
for statusType in statusTypes]
toBeAdded = list(set(cesStatusTuples).difference(set(cesTuple)))
gLogger.debug("%s Computing elements entries to be added" %
len(toBeAdded))
for ceTuple in toBeAdded:
_name = ceTuple[0]
_statusType = ceTuple[1]
_status = self.defaultStatus
_reason = "Synchronized"
_elementType = "ComputingElement"
query = self.rStatus.addIfNotThereStatusElement(
"Resource",
"Status",
name=_name,
statusType=_statusType,
status=_status,
elementType=_elementType,
tokenOwner=self.tokenOwner,
reason=_reason,
)
if not query["OK"]:
return query
return S_OK()
def getPilotSummaryWeb(self, selectDict, sortList, startItem, maxItems):
"""Get summary of the pilot jobs status by CE/site in a standard structure"""
allStateNames = PilotStatus.PILOT_STATES + [
"Done_Empty", "Aborted_Hour"
]
paramNames = ["Site", "CE"] + allStateNames
last_update = None
if "LastUpdateTime" in selectDict:
last_update = selectDict["LastUpdateTime"]
del selectDict["LastUpdateTime"]
site_select = []
if "GridSite" in selectDict:
site_select = selectDict["GridSite"]
if not isinstance(site_select, list):
site_select = [site_select]
del selectDict["GridSite"]
status_select = []
if "Status" in selectDict:
status_select = selectDict["Status"]
if not isinstance(status_select, list):
status_select = [status_select]
del selectDict["Status"]
expand_site = ""
if "ExpandSite" in selectDict:
expand_site = selectDict["ExpandSite"]
site_select = [expand_site]
del selectDict["ExpandSite"]
# Get all the data from the database with various selections
result = self.getCounters(
"PilotAgents",
["GridSite", "DestinationSite", "Status"],
selectDict,
newer=last_update,
timeStamp="LastUpdateTime",
)
if not result["OK"]:
return result
last_update = Time.dateTime() - Time.hour
selectDict["Status"] = PilotStatus.ABORTED
resultHour = self.getCounters(
"PilotAgents",
["GridSite", "DestinationSite", "Status"],
selectDict,
newer=last_update,
timeStamp="LastUpdateTime",
)
if not resultHour["OK"]:
return resultHour
last_update = Time.dateTime() - Time.day
selectDict["Status"] = [PilotStatus.ABORTED, PilotStatus.DONE]
resultDay = self.getCounters(
"PilotAgents",
["GridSite", "DestinationSite", "Status"],
selectDict,
newer=last_update,
timeStamp="LastUpdateTime",
)
if not resultDay["OK"]:
return resultDay
selectDict["CurrentJobID"] = 0
selectDict["Status"] = PilotStatus.DONE
resultDayEmpty = self.getCounters(
"PilotAgents",
["GridSite", "DestinationSite", "Status"],
selectDict,
newer=last_update,
timeStamp="LastUpdateTime",
)
if not resultDayEmpty["OK"]:
return resultDayEmpty
ceMap = {}
resMap = getCESiteMapping()
if resMap["OK"]:
ceMap = resMap["Value"]
# Sort out different counters
resultDict = {}
resultDict["Unknown"] = {}
for attDict, count in result["Value"]:
site = attDict["GridSite"]
ce = attDict["DestinationSite"]
state = attDict["Status"]
if site == "Unknown" and ce != "Unknown" and ce != "Multiple" and ce in ceMap:
site = ceMap[ce]
if site not in resultDict:
resultDict[site] = {}
if ce not in resultDict[site]:
resultDict[site][ce] = {}
for p in allStateNames:
resultDict[site][ce][p] = 0
resultDict[site][ce][state] = count
for attDict, count in resultDay["Value"]:
site = attDict["GridSite"]
ce = attDict["DestinationSite"]
state = attDict["Status"]
if site == "Unknown" and ce != "Unknown" and ce in ceMap:
site = ceMap[ce]
if state == PilotStatus.DONE:
resultDict[site][ce][PilotStatus.DONE] = count
if state == PilotStatus.ABORTED:
resultDict[site][ce][PilotStatus.ABORTED] = count
for attDict, count in resultDayEmpty["Value"]:
site = attDict["GridSite"]
ce = attDict["DestinationSite"]
state = attDict["Status"]
if site == "Unknown" and ce != "Unknown" and ce in ceMap:
site = ceMap[ce]
if state == PilotStatus.DONE:
resultDict[site][ce]["Done_Empty"] = count
for attDict, count in resultHour["Value"]:
site = attDict["GridSite"]
ce = attDict["DestinationSite"]
state = attDict["Status"]
if site == "Unknown" and ce != "Unknown" and ce in ceMap:
site = ceMap[ce]
if state == PilotStatus.ABORTED:
resultDict[site][ce]["Aborted_Hour"] = count
records = []
siteSumDict = {}
for site in resultDict:
sumDict = {}
for state in allStateNames:
if state not in sumDict:
sumDict[state] = 0
sumDict["Total"] = 0
for ce in resultDict[site]:
itemList = [site, ce]
total = 0
for state in allStateNames:
itemList.append(resultDict[site][ce][state])
sumDict[state] += resultDict[site][ce][state]
if state == PilotStatus.DONE:
done = resultDict[site][ce][state]
if state == "Done_Empty":
empty = resultDict[site][ce][state]
if state == PilotStatus.ABORTED:
aborted = resultDict[site][ce][state]
if state != "Aborted_Hour" and state != "Done_Empty":
total += resultDict[site][ce][state]
sumDict["Total"] += total
# Add the total number of pilots seen in the last day
itemList.append(total)
# Add pilot submission efficiency evaluation
if (done - empty) > 0:
eff = done / (done - empty)
elif done == 0:
eff = 0.0
elif empty == done:
eff = 99.0
else:
eff = 0.0
itemList.append("%.2f" % eff)
# Add pilot job efficiency evaluation
if total > 0:
eff = (total - aborted) / total * 100
else:
eff = 100.0
itemList.append("%.2f" % eff)
# Evaluate the quality status of the CE
if total > 10:
if eff < 25.0:
itemList.append("Bad")
elif eff < 60.0:
itemList.append("Poor")
elif eff < 85.0:
itemList.append("Fair")
else:
itemList.append("Good")
else:
itemList.append("Idle")
if len(resultDict[site]) == 1 or expand_site:
records.append(itemList)
if len(resultDict[site]) > 1 and not expand_site:
itemList = [site, "Multiple"]
for state in allStateNames + ["Total"]:
if state in sumDict:
itemList.append(sumDict[state])
else:
itemList.append(0)
done = sumDict[PilotStatus.DONE]
empty = sumDict["Done_Empty"]
aborted = sumDict[PilotStatus.ABORTED]
total = sumDict["Total"]
# Add pilot submission efficiency evaluation
if (done - empty) > 0:
eff = done / (done - empty)
elif done == 0:
eff = 0.0
elif empty == done:
eff = 99.0
else:
eff = 0.0
itemList.append("%.2f" % eff)
# Add pilot job efficiency evaluation
if total > 0:
eff = (total - aborted) / total * 100
else:
eff = 100.0
itemList.append("%.2f" % eff)
# Evaluate the quality status of the Site
if total > 10:
if eff < 25.0:
itemList.append("Bad")
elif eff < 60.0:
itemList.append("Poor")
elif eff < 85.0:
itemList.append("Fair")
else:
itemList.append("Good")
else:
itemList.append("Idle")
records.append(itemList)
for state in allStateNames + ["Total"]:
if state not in siteSumDict:
siteSumDict[state] = sumDict[state]
else:
siteSumDict[state] += sumDict[state]
# Perform site selection
if site_select:
new_records = []
for r in records:
if r[0] in site_select:
new_records.append(r)
records = new_records
# Perform status selection
if status_select:
new_records = []
for r in records:
if r[14] in status_select:
new_records.append(r)
records = new_records
# Get the Site Mask data
result = SiteStatus().getUsableSites()
if result["OK"]:
siteMask = result["Value"]
for r in records:
if r[0] in siteMask:
r.append("Yes")
else:
r.append("No")
else:
for r in records:
r.append("Unknown")
finalDict = {}
finalDict["TotalRecords"] = len(records)
finalDict["ParameterNames"] = paramNames + [
"Total", "PilotsPerJob", "PilotJobEff", "Status", "InMask"
]
# Return all the records if maxItems == 0 or the specified number otherwise
if maxItems:
finalDict["Records"] = records[startItem:startItem + maxItems]
else:
finalDict["Records"] = records
done = siteSumDict[PilotStatus.DONE]
empty = siteSumDict["Done_Empty"]
aborted = siteSumDict[PilotStatus.ABORTED]
total = siteSumDict["Total"]
# Add pilot submission efficiency evaluation
if (done - empty) > 0:
eff = done / (done - empty)
elif done == 0:
eff = 0.0
elif empty == done:
eff = 99.0
else:
eff = 0.0
siteSumDict["PilotsPerJob"] = "%.2f" % eff
# Add pilot job efficiency evaluation
if total > 0:
eff = (total - aborted) / total * 100
else:
eff = 100.0
siteSumDict["PilotJobEff"] = "%.2f" % eff
# Evaluate the overall quality status
if total > 100:
if eff < 25.0:
siteSumDict["Status"] = "Bad"
elif eff < 60.0:
siteSumDict["Status"] = "Poor"
elif eff < 85.0:
siteSumDict["Status"] = "Fair"
else:
siteSumDict["Status"] = "Good"
else:
siteSumDict["Status"] = "Idle"
finalDict["Extras"] = siteSumDict
return S_OK(finalDict)
def getPilotMonitorWeb(self, selectDict, sortList, startItem, maxItems):
"""Get summary of the pilot job information in a standard structure"""
resultDict = {}
if "LastUpdateTime" in selectDict:
del selectDict["LastUpdateTime"]
if "Owner" in selectDict:
userList = selectDict["Owner"]
if not isinstance(userList, list):
userList = [userList]
dnList = []
for uName in userList:
uList = getDNForUsername(uName)["Value"]
dnList += uList
selectDict["OwnerDN"] = dnList
del selectDict["Owner"]
startDate = selectDict.get("FromDate", None)
if startDate:
del selectDict["FromDate"]
# For backward compatibility
if startDate is None:
startDate = selectDict.get("LastUpdateTime", None)
if startDate:
del selectDict["LastUpdateTime"]
endDate = selectDict.get("ToDate", None)
if endDate:
del selectDict["ToDate"]
# Sorting instructions. Only one for the moment.
if sortList:
orderAttribute = sortList[0][0] + ":" + sortList[0][1]
else:
orderAttribute = None
# Select pilots for the summary
result = self.selectPilots(selectDict,
orderAttribute=orderAttribute,
newer=startDate,
older=endDate,
timeStamp="LastUpdateTime")
if not result["OK"]:
return S_ERROR("Failed to select pilots: " + result["Message"])
pList = result["Value"]
nPilots = len(pList)
resultDict["TotalRecords"] = nPilots
if nPilots == 0:
return S_OK(resultDict)
ini = startItem
last = ini + maxItems
if ini >= nPilots:
return S_ERROR("Item number out of range")
if last > nPilots:
last = nPilots
pilotList = pList[ini:last]
paramNames = [
"PilotJobReference",
"OwnerDN",
"OwnerGroup",
"GridType",
"Broker",
"Status",
"DestinationSite",
"BenchMark",
"ParentID",
"SubmissionTime",
"PilotID",
"LastUpdateTime",
"CurrentJobID",
"TaskQueueID",
"GridSite",
]
result = self.getPilotInfo(pilotList, paramNames=paramNames)
if not result["OK"]:
return S_ERROR("Failed to get pilot info: " + result["Message"])
pilotDict = result["Value"]
records = []
for pilot in pilotList:
parList = []
for parameter in paramNames:
if not isinstance(pilotDict[pilot][parameter],
six.integer_types):
parList.append(str(pilotDict[pilot][parameter]))
else:
parList.append(pilotDict[pilot][parameter])
if parameter == "GridSite":
gridSite = pilotDict[pilot][parameter]
# If the Grid Site is unknown try to recover it in the last moment
if gridSite == "Unknown":
ce = pilotDict[pilot]["DestinationSite"]
result = getCESiteMapping(ce)
if result["OK"]:
gridSite = result["Value"].get(ce)
del parList[-1]
parList.append(gridSite)
records.append(parList)
resultDict["ParameterNames"] = paramNames
resultDict["Records"] = records
return S_OK(resultDict)
