def getJobsSites(self, jobIDs):
res = self._getRPC().getJobsSites(jobIDs)
# Cast the str keys to int
if res["OK"]:
res["Value"] = strToIntDict(res["Value"])
return res
def doCommand(self):
"""
Return getQuality from DIRAC's accounting ReportsClient
`args`: a tuple
- args[0]: string: should be a ValidElement
- args[1]: string should be the name of the ValidElement
- args[2]: optional dateTime object: a "from" date
- args[3]: optional dateTime object: a "to" date
:returns:
{'Result': None | a float between 0.0 and 100.0}
"""
if "fromDate" not in self.args:
fromDate = datetime.utcnow() - timedelta(hours=2)
else:
fromDate = self.args["fromDate"]
if "toDate" not in self.args:
toDate = datetime.utcnow()
else:
toDate = self.args["toDate"]
if "name" not in self.args:
return S_ERROR("name not specified")
name = self.args["name"]
results = self.rClient.getReport(
"DataOperation",
"Quality",
fromDate,
toDate,
{"OperationType": "putAndRegister", "Destination": [name]},
"Channel",
)
if not results["OK"]:
return results
pr_q_d = {channel: strToIntDict(value) for channel, value in results["Value"]["data"].items()}
# FIXME: WHAT the hell is this doing ?
values = []
if len(pr_q_d) == 1:
for k in pr_q_d:
for n in pr_q_d[k].values():
values.append(n)
res = sum(values) / len(values) # FIXME: should convert to int?
else:
for n in pr_q_d["Total"].values():
values.append(n)
res = sum(values) / len(values) # FIXME: should convert to int?
return S_OK(res)
def getJobParameters(self, jobIDs, parName=None):
res = self._getRPC().getJobParameters(jobIDs, parName)
# Cast the str keys to int
if res["OK"]:
res["Value"] = strToIntDict(res["Value"])
return res
def getJobsApplicationStatus(self, jobIDs):
res = self._getRPC().getJobsApplicationStatus(jobIDs)
# Cast the str keys to int
if res['OK']:
res['Value'] = strToIntDict(res['Value'])
return res
def getAttributesForJobList(cls, *args, **kwargs):
""" Utility function for unpacking
"""
res = cls.gJobDB.getAttributesForJobList(*args, **kwargs)
if not res['OK']:
return res
return S_OK(strToIntDict(res['Value']))
def getJobsPrimarySummary(self, jobIDs):
res = self._getRPC().getJobsPrimarySummary(jobIDs)
# Cast the str keys to int
if res['OK']:
res['Value'] = strToIntDict(res['Value'])
return res
def getJobsParameters(self, jobIDs, parameters):
res = self._getRPC().getJobsParameters(jobIDs, parameters)
# Cast the str keys to int
if res['OK']:
res['Value'] = strToIntDict(res['Value'])
return res
def getBulkRequests(self, numberOfRequest=10, assigned=True):
"""get bulk requests from RequestDB
:param self: self reference
:param str numberOfRequest: size of the bulk (default 10)
:return: S_OK( Successful : { requestID, RequestInstance }, Failed : message ) or S_ERROR
"""
self.log.debug("getRequests: attempting to get request.")
getRequests = self._getRPC().getBulkRequests(numberOfRequest, assigned)
if not getRequests["OK"]:
self.log.error("getRequests: unable to get '%s' requests: %s" %
(numberOfRequest, getRequests["Message"]))
return getRequests
# No Request returned
if not getRequests["Value"]:
return getRequests
# No successful Request
if not getRequests["Value"]["Successful"]:
return getRequests
jsonReq = getRequests["Value"]["Successful"]
# Do not forget to cast back str keys to int
reqInstances = {int(rId): Request(jsonReq[rId]) for rId in jsonReq}
failed = strToIntDict(getRequests["Value"]["Failed"])
return S_OK({"Successful": reqInstances, "Failed": failed})
def doCommand(self):
"""
Returns successfull jobs using the DIRAC accounting system for every site
for the last self.args[0] hours
:params:
:attr:`sites`: list of sites (when not given, take every site)
:returns:
"""
if 'hours' not in self.args:
return S_ERROR('Number of hours not specified')
hours = self.args['hours']
sites = None
if 'sites' in self.args:
sites = self.args['sites']
if sites is None:
sites = getSites()
if not sites['OK']:
return sites
sites = sites['Value']
if not sites:
return S_ERROR('Sites is empty')
fromD = datetime.utcnow() - timedelta(hours=hours)
toD = datetime.utcnow()
successfulJobs = self.rClient.getReport('Job', 'NumberOfJobs', fromD,
toD, {
'FinalStatus': ['Done'],
'Site': sites
}, 'Site')
if not successfulJobs['OK']:
return successfulJobs
successfulJobs = successfulJobs['Value']
if 'data' not in successfulJobs:
return S_ERROR('Missing data key')
if 'granularity' not in successfulJobs:
return S_ERROR('Missing granularity key')
singlePlots = {}
successfulJobs['data'] = {
site: strToIntDict(value)
for site, value in successfulJobs['data'].items()
}
for site, value in successfulJobs['data'].items():
if site in sites:
plot = {}
plot['data'] = {site: value}
plot['granularity'] = successfulJobs['granularity']
singlePlots[site] = plot
return S_OK(singlePlots)
def doCommand(self):
"""
Returns failed pilots using the DIRAC accounting system for every CE
for the last self.args[0] hours
:params:
:attr:`CEs`: list of CEs (when not given, take every CE)
:returns:
"""
if 'hours' not in self.args:
return S_ERROR('Number of hours not specified')
hours = self.args['hours']
ces = None
if 'ces' in self.args:
ces = self.args['ces']
if ces is None:
res = getCESiteMapping()
if not res['OK']:
return res
ces = list(res['Value'])
if not ces:
return S_ERROR('CEs is empty')
fromD = datetime.utcnow() - timedelta(hours=hours)
toD = datetime.utcnow()
failedPilots = self.rClient.getReport('Pilot', 'NumberOfPilots', fromD,
toD, {
'GridStatus': ['Aborted'],
'GridCE': ces
}, 'GridCE')
if not failedPilots['OK']:
return failedPilots
failedPilots = failedPilots['Value']
if 'data' not in failedPilots:
return S_ERROR('Missing data key')
if 'granularity' not in failedPilots:
return S_ERROR('Missing granularity key')
failedPilots['data'] = {
site: strToIntDict(value)
for site, value in failedPilots['data'].items()
}
singlePlots = {}
for ce, value in failedPilots['data'].items():
if ce in ces:
plot = {}
plot['data'] = {ce: value}
plot['granularity'] = failedPilots['granularity']
singlePlots[ce] = plot
return S_OK(singlePlots)
def doCommand(self):
"""
Returns failed pilots using the DIRAC accounting system for every CE
for the last self.args[0] hours
:params:
:attr:`CEs`: list of CEs (when not given, take every CE)
:returns:
"""
if "hours" not in self.args:
return S_ERROR("Number of hours not specified")
hours = self.args["hours"]
ces = None
if "ces" in self.args:
ces = self.args["ces"]
if ces is None:
res = getCESiteMapping()
if not res["OK"]:
return res
ces = list(res["Value"])
if not ces:
return S_ERROR("CEs is empty")
fromD = datetime.utcnow() - timedelta(hours=hours)
toD = datetime.utcnow()
failedPilots = self.rClient.getReport("Pilot", "NumberOfPilots", fromD,
toD, {
"GridStatus": ["Aborted"],
"GridCE": ces
}, "GridCE")
if not failedPilots["OK"]:
return failedPilots
failedPilots = failedPilots["Value"]
if "data" not in failedPilots:
return S_ERROR("Missing data key")
if "granularity" not in failedPilots:
return S_ERROR("Missing granularity key")
failedPilots["data"] = {
site: strToIntDict(value)
for site, value in failedPilots["data"].items()
}
singlePlots = {}
for ce, value in failedPilots["data"].items():
if ce in ces:
plot = {}
plot["data"] = {ce: value}
plot["granularity"] = failedPilots["granularity"]
singlePlots[ce] = plot
return S_OK(singlePlots)
def doCommand(self):
"""
Returns successfull jobs using the DIRAC accounting system for every site
for the last self.args[0] hours
:params:
:attr:`sites`: list of sites (when not given, take every site)
:returns:
"""
if "hours" not in self.args:
return S_ERROR("Number of hours not specified")
hours = self.args["hours"]
sites = None
if "sites" in self.args:
sites = self.args["sites"]
if sites is None:
sites = getSites()
if not sites["OK"]:
return sites
sites = sites["Value"]
if not sites:
return S_ERROR("Sites is empty")
fromD = datetime.utcnow() - timedelta(hours=hours)
toD = datetime.utcnow()
successfulJobs = self.rClient.getReport("Job", "NumberOfJobs", fromD,
toD, {
"FinalStatus": ["Done"],
"Site": sites
}, "Site")
if not successfulJobs["OK"]:
return successfulJobs
successfulJobs = successfulJobs["Value"]
if "data" not in successfulJobs:
return S_ERROR("Missing data key")
if "granularity" not in successfulJobs:
return S_ERROR("Missing granularity key")
singlePlots = {}
successfulJobs["data"] = {
site: strToIntDict(value)
for site, value in successfulJobs["data"].items()
}
for site, value in successfulJobs["data"].items():
if site in sites:
plot = {}
plot["data"] = {site: value}
plot["granularity"] = successfulJobs["granularity"]
singlePlots[site] = plot
return S_OK(singlePlots)
def export_updateJobStatus(cls, jobStatusDict):
"""Update the job Status and error
:param jobStatusDict: { jobID : { status , error } }
"""
jobStatusDict = strToIntDict(jobStatusDict)
return cls.fts3db.updateJobStatus(jobStatusDict)
def doCommand(self):
"""
Return getQuality from DIRAC's accounting ReportsClient
`args`: a tuple
- args[0]: string: should be a ValidElement
- args[1]: string should be the name of the ValidElement
- args[2]: optional dateTime object: a "from" date
- args[3]: optional dateTime object: a "to" date
:returns:
{'Result': None | a float between 0.0 and 100.0}
"""
if 'fromDate' not in self.args:
fromDate = datetime.utcnow() - timedelta(hours=2)
else:
fromDate = self.args['fromDate']
if 'toDate' not in self.args:
toDate = datetime.utcnow()
else:
toDate = self.args['toDate']
if 'name' not in self.args:
return S_ERROR('name not specified')
name = self.args['name']
results = self.rClient.getReport('DataOperation', 'Quality', fromDate,
toDate, {
'OperationType': 'putAndRegister',
'Destination': [name]
}, 'Channel')
if not results['OK']:
return results
pr_q_d = {
channel: strToIntDict(value)
for channel, value in results['Value']['data'].items()
}
# FIXME: WHAT the hell is this doing ?
values = []
if len(pr_q_d) == 1:
for k in pr_q_d:
for n in pr_q_d[k].values():
values.append(n)
res = sum(values) / len(values) # FIXME: should convert to int?
else:
for n in pr_q_d['Total'].values():
values.append(n)
res = sum(values) / len(values) # FIXME: should convert to int?
return S_OK(res)
def doCommand(self):
"""
Returns running and runned jobs, querying the WMSHistory
for the last self.args[0] hours
:params:
:attr:`sites`: list of sites (when not given, take every sites)
:returns:
"""
if 'hours' not in self.args:
return S_ERROR('Number of hours not specified')
hours = self.args['hours']
sites = None
if 'sites' in self.args:
sites = self.args['sites']
if sites is None:
# FIXME: pointing to the CSHelper instead
# sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} )
# if not sources[ 'OK' ]:
# return sources
# sources = [ si[0] for si in sources[ 'Value' ] ]
sites = getSites()
if not sites['OK']:
return sites
sites = sites['Value']
if not sites:
return S_ERROR('Sites is empty')
fromD = datetime.utcnow() - timedelta(hours=hours)
toD = datetime.utcnow()
runJobs = self.rClient.getReport('WMSHistory', 'NumberOfJobs', fromD, toD,
{}, 'Site')
if not runJobs['OK']:
return runJobs
runJobs = runJobs['Value']
if 'data' not in runJobs:
return S_ERROR('Missing data key')
if 'granularity' not in runJobs:
return S_ERROR('Missing granularity key')
runJobs['data'] = {site: strToIntDict(value) for site, value in runJobs['data'].iteritems()}
singlePlots = {}
for site, value in runJobs['data'].iteritems():
if site in sites:
plot = {}
plot['data'] = {site: value}
plot['granularity'] = runJobs['granularity']
singlePlots[site] = plot
return S_OK(singlePlots)
def getActiveTaskQueues(self):
"""Return all active task queues"""
res = self._getRPC().getActiveTaskQueues()
if res["OK"]:
# Cast the string back to int
res["Value"] = strToIntDict(res["Value"])
return res
def export_updateFileStatus(cls, fileStatusDict, ftsGUID):
"""Update the file ftsStatus and error
:param fileStatusDict: { fileID : { status , error } }
:param ftsGUID: (not mandatory) If specified, only update the rows where the ftsGUID matches this value.
"""
fileStatusDict = strToIntDict(fileStatusDict)
return cls.fts3db.updateFileStatus(fileStatusDict, ftsGUID)
def getMatchingTaskQueues(self, resourceDict):
"""Return all task queues that match the resourceDict"""
res = self._getRPC().getMatchingTaskQueues(resourceDict)
if res["OK"]:
# Cast the string back to int
res["Value"] = strToIntDict(res["Value"])
return res
def export_confirmBulkSubmission(self, jobIDs):
""" Confirm the possibility to proceed with processing of the jobs specified
by the jobIDList
:param list jobIDs: list of job IDs
:return: S_OK(list)/S_ERROR() -- confirmed job IDs
"""
jobList = self.__getJobList(jobIDs)
if not jobList:
self.log.error("Issue with __getJobList",
": invalid job specification %s" % str(jobIDs))
return S_ERROR(EWMSSUBM,
'Invalid job specification: ' + str(jobIDs))
validJobList, _invalidJobList, _nonauthJobList, _ownerJobList = self.jobPolicy.evaluateJobRights(
jobList, RIGHT_SUBMIT)
# Check that all the requested jobs are eligible
if set(jobList) != set(validJobList):
return S_ERROR(
EWMSSUBM, 'Requested jobs for bulk transaction are not valid')
result = self.jobDB.getAttributesForJobList(jobList,
['Status', 'MinorStatus'])
if not result['OK']:
return S_ERROR(
EWMSSUBM, 'Requested jobs for bulk transaction are not valid')
js_dict = strToIntDict(result['Value'])
# Check if the jobs are already activated
jobEnabledList = [
jobID for jobID in jobList if js_dict[jobID]['Status'] in [
JobStatus.RECEIVED, JobStatus.CHECKING, JobStatus.WAITING,
JobStatus.MATCHED, JobStatus.RUNNING
]
]
if set(jobEnabledList) == set(jobList):
return S_OK(jobList)
# Check that requested job are in Submitting status
jobUpdateStatusList = list(
jobID for jobID in jobList
if js_dict[jobID]['Status'] == JobStatus.SUBMITTING)
if set(jobUpdateStatusList) != set(jobList):
return S_ERROR(
EWMSSUBM, 'Requested jobs for bulk transaction are not valid')
# Update status of all the requested jobs in one transaction
result = self.jobDB.setJobAttributes(
jobUpdateStatusList, ['Status', 'MinorStatus'],
[JobStatus.RECEIVED, 'Job accepted'])
if not result['OK']:
return result
self.__sendJobsToOptimizationMind(jobUpdateStatusList)
return S_OK(jobUpdateStatusList)
def getRequestIDsForJobs(self, jobIDs):
"""get the request ids for the supplied jobIDs.
:param self: self reference
:param list jobIDs: list of job IDs (integers)
:return: S_ERROR or S_OK( "Successful": { jobID1: reqID1, jobID2: requID2, ... },
"Failed" : { jobIDn: errMsg, jobIDm: errMsg, ...} )
"""
self.log.verbose("getRequestIDsForJobs: attempt to get request(s) for jobs", "(n=%d)" % len(jobIDs))
res = self._getRPC().getRequestIDsForJobs(jobIDs)
if not res["OK"]:
self.log.error(
"getRequestIDsForJobs: unable to get request(s) for jobs", "%s: %s" % (jobIDs, res["Message"])
)
return res
# Cast the JobIDs back to int
successful = strToIntDict(res["Value"]["Successful"])
failed = strToIntDict(res["Value"]["Failed"])
return S_OK({"Successful": successful, "Failed": failed})
def export_commitMarks(self,
sourceId,
activitiesDict,
componentExtraInfo={}):
"""
Adds marks for activities
:param int sourceId: component id
:param dict activitiesDict: the key is the metrice, the value is a dictionary with timestamp and value
for example::
{u'CPU': {1583244000: 10.6055594906},
u'Iteration': {1583244420: 1,
1583244480: 1,
1583244540: 1,
1583244600: 1,
1583244660: 1},
u'MEM': {1583244000: 49.03515625},
u'Processed': {1583244420: 5,
1583244480: 5,
1583244540: 5,
1583244600: 5,
1583244660: 6}}
:param dict componentExtraInfo: extra information
"""
# we have to cast the dictionary keys, because they are timestamps
activitiesDict = {
metric: strToIntDict(value)
for metric, value in activitiesDict.iteritems()
}
nowEpoch = Time.toEpoch()
maxEpoch = nowEpoch + 7200
minEpoch = nowEpoch - 86400
invalidActivities = []
for acName in activitiesDict:
for time in activitiesDict[acName]:
if time > maxEpoch or time < minEpoch:
gLogger.info(
"Time %s ( [%s,%s] ) is invalid for activity %s" %
(time, minEpoch, maxEpoch, acName))
invalidActivities.append(acName)
break
for acName in invalidActivities:
gLogger.info("Not commiting activity %s" % acName)
del activitiesDict[acName]
return gServiceInterface.commitMarks(sourceId, activitiesDict,
componentExtraInfo)
def readRequestsForJobs(self, jobIDs):
"""read requests for jobs
:param jobIDs: list with jobIDs
:type jobIDs: python:list
:return: S_OK( { "Successful" : { jobID1 : Request, ... },
"Failed" : { jobIDn : "Fail reason" } } )
"""
readReqsForJobs = self._getRPC().readRequestsForJobs(jobIDs)
if not readReqsForJobs["OK"]:
return readReqsForJobs
ret = readReqsForJobs["Value"]
# # create Requests out of JSONs for successful reads
# Do not forget to cast back str keys to int
successful = {int(jobID): Request(jsonReq) for jobID, jsonReq in ret["Successful"].items()}
failed = strToIntDict(ret["Failed"])
return S_OK({"Successful": successful, "Failed": failed})
def doNew(self, masterParams=None):
'''
Gets the parameters to run, either from the master method or from its
own arguments.
For every elementName ( cannot process bulk queries.. ) contacts the
accounting client. It reurns dictionaries like { 'X -> Y' : { id: 100%.. } }
If there are ggus tickets, are recorded and then returned.
'''
if masterParams is not None:
hours, name, direction, metric = masterParams
else:
params = self._prepareCommand()
if not params['OK']:
return params
hours, name, direction, metric = params['Value']
toD = datetime.utcnow()
fromD = toD - timedelta(hours=hours)
# dictionary with conditions for the accounting
transferDict = {'OperationType': 'putAndRegister', direction: name}
if metric == 'FailedTransfers':
transferDict['FinalStatus'] = ['Failed']
transferResults = self.rClient.getReport('DataOperation', metric,
fromD, toD, transferDict,
'Channel')
if not transferResults['OK']:
return transferResults
transferResults = transferResults['Value']
if 'data' not in transferResults:
return S_ERROR('Missing data key')
transferResults = {
channel: strToIntDict(value)
for channel, value in transferResults['data'].iteritems()
}
uniformResult = []
for channel, elementDict in transferResults.items():
try:
source, destination = channel.split(' -> ')
except ValueError:
continue
channelDict = {}
channelDict['SourceName'] = source
channelDict['DestinationName'] = destination
channelDict['Metric'] = metric
channelDict['Value'] = sum(elementDict.values()) / len(
elementDict.values())
uniformResult.append(channelDict)
storeRes = self._storeCommand(uniformResult)
if not storeRes['OK']:
return storeRes
# Compute mean of all transfer channels
value = 0
for channelDict in uniformResult:
value += channelDict['Value']
if uniformResult:
value = float(value) / len(uniformResult)
else:
value = None
return S_OK({'Mean': value, 'Name': name})
def doCommand(self):
"""
Returns failed jobs using the DIRAC accounting system for every site
for the last self.args[0] hours
:params:
:attr:`sites`: list of sites (when not given, take every site)
:returns:
"""
if "hours" not in self.args:
return S_ERROR("Number of hours not specified")
hours = self.args["hours"]
sites = None
if "sites" in self.args:
sites = self.args["sites"]
if sites is None:
# FIXME: pointing to the CSHelper instead
# sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} )
# if not sources[ 'OK' ]:
# return sources
# sources = [ si[0] for si in sources[ 'Value' ] ]
sites = getSites()
if not sites["OK"]:
return sites
sites = sites["Value"]
if not sites:
return S_ERROR("Sites is empty")
fromD = datetime.utcnow() - timedelta(hours=hours)
toD = datetime.utcnow()
failedPilots = self.rClient.getReport("Pilot", "NumberOfPilots", fromD,
toD, {
"GridStatus": ["Aborted"],
"Site": sites
}, "Site")
if not failedPilots["OK"]:
return failedPilots
failedPilots = failedPilots["Value"]
if "data" not in failedPilots:
return S_ERROR("Missing data key")
if "granularity" not in failedPilots:
return S_ERROR("Missing granularity key")
failedPilots["data"] = {
site: strToIntDict(value)
for site, value in failedPilots["data"].items()
}
singlePlots = {}
for site, value in failedPilots["data"].items():
if site in sites:
plot = {}
plot["data"] = {site: value}
plot["granularity"] = failedPilots["granularity"]
singlePlots[site] = plot
return S_OK(singlePlots)
def doCommand(self):
"""
Returns running and runned jobs, querying the WMSHistory
for the last self.args[0] hours
:params:
:attr:`sites`: list of sites (when not given, take every sites)
:returns:
"""
if "hours" not in self.args:
return S_ERROR("Number of hours not specified")
hours = self.args["hours"]
sites = None
if "sites" in self.args:
sites = self.args["sites"]
if sites is None:
# FIXME: pointing to the CSHelper instead
# sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} )
# if not sources[ 'OK' ]:
# return sources
# sources = [ si[0] for si in sources[ 'Value' ] ]
sites = getSites()
if not sites["OK"]:
return sites
sites = sites["Value"]
if not sites:
return S_ERROR("Sites is empty")
fromD = datetime.utcnow() - timedelta(hours=hours)
toD = datetime.utcnow()
runJobs = self.rClient.getReport("WMSHistory", "NumberOfJobs", fromD,
toD, {}, "Site")
if not runJobs["OK"]:
return runJobs
runJobs = runJobs["Value"]
if "data" not in runJobs:
return S_ERROR("Missing data key")
if "granularity" not in runJobs:
return S_ERROR("Missing granularity key")
runJobs["data"] = {
site: strToIntDict(value)
for site, value in runJobs["data"].items()
}
singlePlots = {}
for site, value in runJobs["data"].items():
if site in sites:
plot = {}
plot["data"] = {site: value}
plot["granularity"] = runJobs["granularity"]
singlePlots[site] = plot
return S_OK(singlePlots)
def doCommand(self):
"""
Returns failed pilots using the DIRAC accounting system for every CE
for the last self.args[0] hours
:params:
:attr:`CEs`: list of CEs (when not given, take every CE)
:returns:
"""
if 'hours' not in self.args:
return S_ERROR('Number of hours not specified')
hours = self.args['hours']
ces = None
if 'ces' in self.args:
ces = self.args['ces']
if ces is None:
# FIXME: pointing to the CSHelper instead
# meta = {'columns':'ResourceName'}
# CEs = self.rsClient.getResource( resourceType = [ 'CE','CREAMCE' ], meta = meta )
# if not CEs['OK']:
# return CEs
# CEs = [ ce[0] for ce in CEs['Value'] ]
ces = CSHelpers.getComputingElements()
if not ces['OK']:
return ces
ces = ces['Value']
if not ces:
return S_ERROR('CEs is empty')
fromD = datetime.utcnow() - timedelta(hours=hours)
toD = datetime.utcnow()
failedPilots = self.rClient.getReport('Pilot', 'NumberOfPilots', fromD,
toD, {
'GridStatus': ['Aborted'],
'GridCE': ces
}, 'GridCE')
if not failedPilots['OK']:
return failedPilots
failedPilots = failedPilots['Value']
if 'data' not in failedPilots:
return S_ERROR('Missing data key')
if 'granularity' not in failedPilots:
return S_ERROR('Missing granularity key')
failedPilots['data'] = {
site: strToIntDict(value)
for site, value in failedPilots['data'].iteritems()
}
singlePlots = {}
for ce, value in failedPilots['data'].iteritems():
if ce in ces:
plot = {}
plot['data'] = {ce: value}
plot['granularity'] = failedPilots['granularity']
singlePlots[ce] = plot
return S_OK(singlePlots)
def doCommand(self):
"""
Returns failed jobs using the DIRAC accounting system for every site
for the last self.args[0] hours
:params:
:attr:`sites`: list of sites (when not given, take every site)
:returns:
"""
if 'hours' not in self.args:
return S_ERROR('Number of hours not specified')
hours = self.args['hours']
sites = None
if 'sites' in self.args:
sites = self.args['sites']
if sites is None:
# FIXME: pointing to the CSHelper instead
# sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} )
# if not sources[ 'OK' ]:
# return sources
# sources = [ si[0] for si in sources[ 'Value' ] ]
sites = getSites()
if not sites['OK']:
return sites
sites = sites['Value']
if not sites:
return S_ERROR('Sites is empty')
fromD = datetime.utcnow() - timedelta(hours=hours)
toD = datetime.utcnow()
failedPilots = self.rClient.getReport('Pilot', 'NumberOfPilots', fromD,
toD, {
'GridStatus': ['Aborted'],
'Site': sites
}, 'Site')
if not failedPilots['OK']:
return failedPilots
failedPilots = failedPilots['Value']
if 'data' not in failedPilots:
return S_ERROR('Missing data key')
if 'granularity' not in failedPilots:
return S_ERROR('Missing granularity key')
failedPilots['data'] = {
site: strToIntDict(value)
for site, value in failedPilots['data'].iteritems()
}
singlePlots = {}
for site, value in failedPilots['data'].iteritems():
if site in sites:
plot = {}
plot['data'] = {site: value}
plot['granularity'] = failedPilots['granularity']
singlePlots[site] = plot
return S_OK(singlePlots)
def test_getReport(putAndDelete):
params = (
"WMSHistory",
"NumberOfJobs",
datetime(2016, 3, 16, 12, 30, 0, 0),
datetime(2016, 3, 17, 19, 29, 0, 0),
{
"grouping": ["Site"]
},
"Site",
{},
)
result = client.getReport(*params)
assert result["OK"], result["Message"]
result["Value"]["data"] = {
site: strToIntDict(value)
for site, value in result["Value"]["data"].items()
}
assert result["Value"] == {
"data": {
u"Multiple": {
1458198000: 227.0
},
u"LCG.RRCKI.ru": {
1458225000: 3.0
},
u"LCG.IHEP.su": {
1458217800: 18.0
},
u"LCG.CNAF.it": {
1458144000: None,
1458172800: None,
1458194400: None,
1458145800: None,
1458189000: None,
1458147600: None,
1458178200: None,
1458183600: None,
1458212400: None,
1458149400: None,
1458207000: None,
1458151200: None,
1458169200: None,
1458201600: None,
1458153000: None,
1458196200: None,
1458154800: None,
1458174600: None,
1458190800: None,
1458156600: None,
1458185400: None,
1458214200: None,
1458158400: None,
1458180000: None,
1458216000: None,
1458208800: None,
1458160200: None,
1458203400: None,
1458162000: None,
1458142200: None,
1458198000: None,
1458163800: None,
1458192600: None,
1458165600: None,
1458176400: None,
1458187200: None,
1458167400: None,
1458210600: None,
1458140400: 4.0,
1458181800: None,
1458205200: None,
1458171000: None,
1458217800: 22.0,
1458199800: None,
},
u"LCG.NIKHEF.nl": {
1458217800: 27.0
},
u"LCG.Bari.it": {
1458221400: 34.0
},
u"Group.RAL.uk": {
1458140400: 34.0
},
u"LCG.DESYZN.de": {
1458225000: 43.0
},
u"LCG.RAL.uk": {
1458144000: None,
1458158400: None,
1458194400: None,
1458145800: None,
1458223200: None,
1458189000: None,
1458221400: None,
1458225000: 5.0,
1458147600: None,
1458135000: None,
1458183600: None,
1458212400: None,
1458149400: None,
1458178200: None,
1458207000: None,
1458151200: None,
1458169200: None,
1458172800: None,
1458219600: None,
1458201600: None,
1458153000: None,
1458196200: None,
1458154800: None,
1458160200: None,
1458190800: None,
1458156600: None,
1458185400: None,
1458214200: None,
1458129600: 2.0,
1458165600: None,
1458180000: None,
1458216000: None,
1458208800: None,
1458131400: None,
1458174600: None,
1458203400: None,
1458162000: None,
1458171000: None,
1458198000: None,
1458163800: None,
1458192600: None,
1458136800: None,
1458133200: None,
1458187200: None,
1458167400: None,
1458181800: None,
1458210600: None,
1458140400: None,
1458138600: None,
1458176400: None,
1458205200: None,
1458142200: None,
1458217800: None,
1458199800: None,
},
u"LCG.PIC.es": {
1458129600: 1.0
},
u"LCG.GRIDKA.de": {
1458129600: 2.0
},
u"LCG.Bristol.uk": {
1458221400: 9.0
},
u"LCG.CERN.ch": {
1458140400: 120.0
},
u"LCG.Bologna.it": {
1458221400: 1.0
},
},
"granularity": 1800,
}
def test_getReport(self):
params = ('WMSHistory', 'NumberOfJobs',
datetime(2016, 3, 16, 12, 30, 0,
0), datetime(2016, 3, 17, 19, 29, 0, 0), {
'grouping': ['Site']
}, 'Site', {})
result = self.client.getReport(*params)
self.assertTrue(result['OK'], result.get('Message'))
result['Value']['data'] = {
site: strToIntDict(value)
for site, value in result['Value']['data'].iteritems()
}
self.assertDictEqual(
result['Value'], {
'data': {
u'Multiple': {
1458198000: 227.0
},
u'LCG.RRCKI.ru': {
1458225000: 3.0
},
u'LCG.IHEP.su': {
1458217800: 18.0
},
u'LCG.CNAF.it': {
1458144000: None,
1458172800: None,
1458194400: None,
1458145800: None,
1458189000: None,
1458147600: None,
1458178200: None,
1458183600: None,
1458212400: None,
1458149400: None,
1458207000: None,
1458151200: None,
1458169200: None,
1458201600: None,
1458153000: None,
1458196200: None,
1458154800: None,
1458174600: None,
1458190800: None,
1458156600: None,
1458185400: None,
1458214200: None,
1458158400: None,
1458180000: None,
1458216000: None,
1458208800: None,
1458160200: None,
1458203400: None,
1458162000: None,
1458142200: None,
1458198000: None,
1458163800: None,
1458192600: None,
1458165600: None,
1458176400: None,
1458187200: None,
1458167400: None,
1458210600: None,
1458140400: 4.0,
1458181800: None,
1458205200: None,
1458171000: None,
1458217800: 22.0,
1458199800: None
},
u'LCG.NIKHEF.nl': {
1458217800: 27.0
},
u'LCG.Bari.it': {
1458221400: 34.0
},
u'Group.RAL.uk': {
1458140400: 34.0
},
u'LCG.DESYZN.de': {
1458225000: 43.0
},
u'LCG.RAL.uk': {
1458144000: None,
1458158400: None,
1458194400: None,
1458145800: None,
1458223200: None,
1458189000: None,
1458221400: None,
1458225000: 5.0,
1458147600: None,
1458135000: None,
1458183600: None,
1458212400: None,
1458149400: None,
1458178200: None,
1458207000: None,
1458151200: None,
1458169200: None,
1458172800: None,
1458219600: None,
1458201600: None,
1458153000: None,
1458196200: None,
1458154800: None,
1458160200: None,
1458190800: None,
1458156600: None,
1458185400: None,
1458214200: None,
1458129600: 2.0,
1458165600: None,
1458180000: None,
1458216000: None,
1458208800: None,
1458131400: None,
1458174600: None,
1458203400: None,
1458162000: None,
1458171000: None,
1458198000: None,
1458163800: None,
1458192600: None,
1458136800: None,
1458133200: None,
1458187200: None,
1458167400: None,
1458181800: None,
1458210600: None,
1458140400: None,
1458138600: None,
1458176400: None,
1458205200: None,
1458142200: None,
1458217800: None,
1458199800: None
},
u'LCG.PIC.es': {
1458129600: 1.0
},
u'LCG.GRIDKA.de': {
1458129600: 2.0
},
u'LCG.Bristol.uk': {
1458221400: 9.0
},
u'LCG.CERN.ch': {
1458140400: 120.0
},
u'LCG.Bologna.it': {
1458221400: 1.0
}
},
'granularity': 1800
})
def doNew(self, masterParams=None):
"""
Gets the parameters to run, either from the master method or from its
own arguments.
For every elementName ( cannot process bulk queries.. ) contacts the
accounting client. It reurns dictionaries like { 'X -> Y' : { id: 100%.. } }
If there are ggus tickets, are recorded and then returned.
"""
if masterParams is not None:
hours, name, direction, metric = masterParams
else:
params = self._prepareCommand()
if not params["OK"]:
return params
hours, name, direction, metric = params["Value"]
toD = datetime.utcnow()
fromD = toD - timedelta(hours=hours)
# dictionary with conditions for the accounting
transferDict = {"OperationType": "putAndRegister", direction: name}
if metric == "FailedTransfers":
transferDict["FinalStatus"] = ["Failed"]
transferResults = self.rClient.getReport("DataOperation", metric, fromD, toD, transferDict, "Channel")
if not transferResults["OK"]:
return transferResults
transferResults = transferResults["Value"]
if "data" not in transferResults:
return S_ERROR("Missing data key")
transferResults = {channel: strToIntDict(value) for channel, value in transferResults["data"].items()}
uniformResult = []
for channel, elementDict in transferResults.items():
try:
source, destination = channel.split(" -> ")
except ValueError:
continue
channelDict = {}
channelDict["SourceName"] = source
channelDict["DestinationName"] = destination
channelDict["Metric"] = metric
channelDict["Value"] = int(sum(elementDict.values()) / len(elementDict.values()))
uniformResult.append(channelDict)
storeRes = self._storeCommand(uniformResult)
if not storeRes["OK"]:
return storeRes
# Compute mean of all transfer channels
value = 0
for channelDict in uniformResult:
value += channelDict["Value"]
if uniformResult:
value = float(value) / len(uniformResult)
else:
value = None
return S_OK({"Mean": value, "Name": name})
