def getJobsSites(self, jobIDs): res = self._getRPC().getJobsSites(jobIDs) # Cast the str keys to int if res["OK"]: res["Value"] = strToIntDict(res["Value"]) return res
def doCommand(self): """ Return getQuality from DIRAC's accounting ReportsClient `args`: a tuple - args[0]: string: should be a ValidElement - args[1]: string should be the name of the ValidElement - args[2]: optional dateTime object: a "from" date - args[3]: optional dateTime object: a "to" date :returns: {'Result': None | a float between 0.0 and 100.0} """ if "fromDate" not in self.args: fromDate = datetime.utcnow() - timedelta(hours=2) else: fromDate = self.args["fromDate"] if "toDate" not in self.args: toDate = datetime.utcnow() else: toDate = self.args["toDate"] if "name" not in self.args: return S_ERROR("name not specified") name = self.args["name"] results = self.rClient.getReport( "DataOperation", "Quality", fromDate, toDate, {"OperationType": "putAndRegister", "Destination": [name]}, "Channel", ) if not results["OK"]: return results pr_q_d = {channel: strToIntDict(value) for channel, value in results["Value"]["data"].items()} # FIXME: WHAT the hell is this doing ? values = [] if len(pr_q_d) == 1: for k in pr_q_d: for n in pr_q_d[k].values(): values.append(n) res = sum(values) / len(values) # FIXME: should convert to int? else: for n in pr_q_d["Total"].values(): values.append(n) res = sum(values) / len(values) # FIXME: should convert to int? return S_OK(res)
def getJobParameters(self, jobIDs, parName=None): res = self._getRPC().getJobParameters(jobIDs, parName) # Cast the str keys to int if res["OK"]: res["Value"] = strToIntDict(res["Value"]) return res
def getJobsApplicationStatus(self, jobIDs): res = self._getRPC().getJobsApplicationStatus(jobIDs) # Cast the str keys to int if res['OK']: res['Value'] = strToIntDict(res['Value']) return res
def getAttributesForJobList(cls, *args, **kwargs): """ Utility function for unpacking """ res = cls.gJobDB.getAttributesForJobList(*args, **kwargs) if not res['OK']: return res return S_OK(strToIntDict(res['Value']))
def getJobsPrimarySummary(self, jobIDs): res = self._getRPC().getJobsPrimarySummary(jobIDs) # Cast the str keys to int if res['OK']: res['Value'] = strToIntDict(res['Value']) return res
def getJobsParameters(self, jobIDs, parameters): res = self._getRPC().getJobsParameters(jobIDs, parameters) # Cast the str keys to int if res['OK']: res['Value'] = strToIntDict(res['Value']) return res
def getBulkRequests(self, numberOfRequest=10, assigned=True): """get bulk requests from RequestDB :param self: self reference :param str numberOfRequest: size of the bulk (default 10) :return: S_OK( Successful : { requestID, RequestInstance }, Failed : message ) or S_ERROR """ self.log.debug("getRequests: attempting to get request.") getRequests = self._getRPC().getBulkRequests(numberOfRequest, assigned) if not getRequests["OK"]: self.log.error("getRequests: unable to get '%s' requests: %s" % (numberOfRequest, getRequests["Message"])) return getRequests # No Request returned if not getRequests["Value"]: return getRequests # No successful Request if not getRequests["Value"]["Successful"]: return getRequests jsonReq = getRequests["Value"]["Successful"] # Do not forget to cast back str keys to int reqInstances = {int(rId): Request(jsonReq[rId]) for rId in jsonReq} failed = strToIntDict(getRequests["Value"]["Failed"]) return S_OK({"Successful": reqInstances, "Failed": failed})
def doCommand(self): """ Returns successfull jobs using the DIRAC accounting system for every site for the last self.args[0] hours :params: :attr:`sites`: list of sites (when not given, take every site) :returns: """ if 'hours' not in self.args: return S_ERROR('Number of hours not specified') hours = self.args['hours'] sites = None if 'sites' in self.args: sites = self.args['sites'] if sites is None: sites = getSites() if not sites['OK']: return sites sites = sites['Value'] if not sites: return S_ERROR('Sites is empty') fromD = datetime.utcnow() - timedelta(hours=hours) toD = datetime.utcnow() successfulJobs = self.rClient.getReport('Job', 'NumberOfJobs', fromD, toD, { 'FinalStatus': ['Done'], 'Site': sites }, 'Site') if not successfulJobs['OK']: return successfulJobs successfulJobs = successfulJobs['Value'] if 'data' not in successfulJobs: return S_ERROR('Missing data key') if 'granularity' not in successfulJobs: return S_ERROR('Missing granularity key') singlePlots = {} successfulJobs['data'] = { site: strToIntDict(value) for site, value in successfulJobs['data'].items() } for site, value in successfulJobs['data'].items(): if site in sites: plot = {} plot['data'] = {site: value} plot['granularity'] = successfulJobs['granularity'] singlePlots[site] = plot return S_OK(singlePlots)
def doCommand(self): """ Returns failed pilots using the DIRAC accounting system for every CE for the last self.args[0] hours :params: :attr:`CEs`: list of CEs (when not given, take every CE) :returns: """ if 'hours' not in self.args: return S_ERROR('Number of hours not specified') hours = self.args['hours'] ces = None if 'ces' in self.args: ces = self.args['ces'] if ces is None: res = getCESiteMapping() if not res['OK']: return res ces = list(res['Value']) if not ces: return S_ERROR('CEs is empty') fromD = datetime.utcnow() - timedelta(hours=hours) toD = datetime.utcnow() failedPilots = self.rClient.getReport('Pilot', 'NumberOfPilots', fromD, toD, { 'GridStatus': ['Aborted'], 'GridCE': ces }, 'GridCE') if not failedPilots['OK']: return failedPilots failedPilots = failedPilots['Value'] if 'data' not in failedPilots: return S_ERROR('Missing data key') if 'granularity' not in failedPilots: return S_ERROR('Missing granularity key') failedPilots['data'] = { site: strToIntDict(value) for site, value in failedPilots['data'].items() } singlePlots = {} for ce, value in failedPilots['data'].items(): if ce in ces: plot = {} plot['data'] = {ce: value} plot['granularity'] = failedPilots['granularity'] singlePlots[ce] = plot return S_OK(singlePlots)
def doCommand(self): """ Returns failed pilots using the DIRAC accounting system for every CE for the last self.args[0] hours :params: :attr:`CEs`: list of CEs (when not given, take every CE) :returns: """ if "hours" not in self.args: return S_ERROR("Number of hours not specified") hours = self.args["hours"] ces = None if "ces" in self.args: ces = self.args["ces"] if ces is None: res = getCESiteMapping() if not res["OK"]: return res ces = list(res["Value"]) if not ces: return S_ERROR("CEs is empty") fromD = datetime.utcnow() - timedelta(hours=hours) toD = datetime.utcnow() failedPilots = self.rClient.getReport("Pilot", "NumberOfPilots", fromD, toD, { "GridStatus": ["Aborted"], "GridCE": ces }, "GridCE") if not failedPilots["OK"]: return failedPilots failedPilots = failedPilots["Value"] if "data" not in failedPilots: return S_ERROR("Missing data key") if "granularity" not in failedPilots: return S_ERROR("Missing granularity key") failedPilots["data"] = { site: strToIntDict(value) for site, value in failedPilots["data"].items() } singlePlots = {} for ce, value in failedPilots["data"].items(): if ce in ces: plot = {} plot["data"] = {ce: value} plot["granularity"] = failedPilots["granularity"] singlePlots[ce] = plot return S_OK(singlePlots)
def doCommand(self): """ Returns successfull jobs using the DIRAC accounting system for every site for the last self.args[0] hours :params: :attr:`sites`: list of sites (when not given, take every site) :returns: """ if "hours" not in self.args: return S_ERROR("Number of hours not specified") hours = self.args["hours"] sites = None if "sites" in self.args: sites = self.args["sites"] if sites is None: sites = getSites() if not sites["OK"]: return sites sites = sites["Value"] if not sites: return S_ERROR("Sites is empty") fromD = datetime.utcnow() - timedelta(hours=hours) toD = datetime.utcnow() successfulJobs = self.rClient.getReport("Job", "NumberOfJobs", fromD, toD, { "FinalStatus": ["Done"], "Site": sites }, "Site") if not successfulJobs["OK"]: return successfulJobs successfulJobs = successfulJobs["Value"] if "data" not in successfulJobs: return S_ERROR("Missing data key") if "granularity" not in successfulJobs: return S_ERROR("Missing granularity key") singlePlots = {} successfulJobs["data"] = { site: strToIntDict(value) for site, value in successfulJobs["data"].items() } for site, value in successfulJobs["data"].items(): if site in sites: plot = {} plot["data"] = {site: value} plot["granularity"] = successfulJobs["granularity"] singlePlots[site] = plot return S_OK(singlePlots)
def export_updateJobStatus(cls, jobStatusDict): """Update the job Status and error :param jobStatusDict: { jobID : { status , error } } """ jobStatusDict = strToIntDict(jobStatusDict) return cls.fts3db.updateJobStatus(jobStatusDict)
def doCommand(self): """ Return getQuality from DIRAC's accounting ReportsClient `args`: a tuple - args[0]: string: should be a ValidElement - args[1]: string should be the name of the ValidElement - args[2]: optional dateTime object: a "from" date - args[3]: optional dateTime object: a "to" date :returns: {'Result': None | a float between 0.0 and 100.0} """ if 'fromDate' not in self.args: fromDate = datetime.utcnow() - timedelta(hours=2) else: fromDate = self.args['fromDate'] if 'toDate' not in self.args: toDate = datetime.utcnow() else: toDate = self.args['toDate'] if 'name' not in self.args: return S_ERROR('name not specified') name = self.args['name'] results = self.rClient.getReport('DataOperation', 'Quality', fromDate, toDate, { 'OperationType': 'putAndRegister', 'Destination': [name] }, 'Channel') if not results['OK']: return results pr_q_d = { channel: strToIntDict(value) for channel, value in results['Value']['data'].items() } # FIXME: WHAT the hell is this doing ? values = [] if len(pr_q_d) == 1: for k in pr_q_d: for n in pr_q_d[k].values(): values.append(n) res = sum(values) / len(values) # FIXME: should convert to int? else: for n in pr_q_d['Total'].values(): values.append(n) res = sum(values) / len(values) # FIXME: should convert to int? return S_OK(res)
def doCommand(self): """ Returns running and runned jobs, querying the WMSHistory for the last self.args[0] hours :params: :attr:`sites`: list of sites (when not given, take every sites) :returns: """ if 'hours' not in self.args: return S_ERROR('Number of hours not specified') hours = self.args['hours'] sites = None if 'sites' in self.args: sites = self.args['sites'] if sites is None: # FIXME: pointing to the CSHelper instead # sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} ) # if not sources[ 'OK' ]: # return sources # sources = [ si[0] for si in sources[ 'Value' ] ] sites = getSites() if not sites['OK']: return sites sites = sites['Value'] if not sites: return S_ERROR('Sites is empty') fromD = datetime.utcnow() - timedelta(hours=hours) toD = datetime.utcnow() runJobs = self.rClient.getReport('WMSHistory', 'NumberOfJobs', fromD, toD, {}, 'Site') if not runJobs['OK']: return runJobs runJobs = runJobs['Value'] if 'data' not in runJobs: return S_ERROR('Missing data key') if 'granularity' not in runJobs: return S_ERROR('Missing granularity key') runJobs['data'] = {site: strToIntDict(value) for site, value in runJobs['data'].iteritems()} singlePlots = {} for site, value in runJobs['data'].iteritems(): if site in sites: plot = {} plot['data'] = {site: value} plot['granularity'] = runJobs['granularity'] singlePlots[site] = plot return S_OK(singlePlots)
def getActiveTaskQueues(self): """Return all active task queues""" res = self._getRPC().getActiveTaskQueues() if res["OK"]: # Cast the string back to int res["Value"] = strToIntDict(res["Value"]) return res
def export_updateFileStatus(cls, fileStatusDict, ftsGUID): """Update the file ftsStatus and error :param fileStatusDict: { fileID : { status , error } } :param ftsGUID: (not mandatory) If specified, only update the rows where the ftsGUID matches this value. """ fileStatusDict = strToIntDict(fileStatusDict) return cls.fts3db.updateFileStatus(fileStatusDict, ftsGUID)
def getMatchingTaskQueues(self, resourceDict): """Return all task queues that match the resourceDict""" res = self._getRPC().getMatchingTaskQueues(resourceDict) if res["OK"]: # Cast the string back to int res["Value"] = strToIntDict(res["Value"]) return res
def export_confirmBulkSubmission(self, jobIDs): """ Confirm the possibility to proceed with processing of the jobs specified by the jobIDList :param list jobIDs: list of job IDs :return: S_OK(list)/S_ERROR() -- confirmed job IDs """ jobList = self.__getJobList(jobIDs) if not jobList: self.log.error("Issue with __getJobList", ": invalid job specification %s" % str(jobIDs)) return S_ERROR(EWMSSUBM, 'Invalid job specification: ' + str(jobIDs)) validJobList, _invalidJobList, _nonauthJobList, _ownerJobList = self.jobPolicy.evaluateJobRights( jobList, RIGHT_SUBMIT) # Check that all the requested jobs are eligible if set(jobList) != set(validJobList): return S_ERROR( EWMSSUBM, 'Requested jobs for bulk transaction are not valid') result = self.jobDB.getAttributesForJobList(jobList, ['Status', 'MinorStatus']) if not result['OK']: return S_ERROR( EWMSSUBM, 'Requested jobs for bulk transaction are not valid') js_dict = strToIntDict(result['Value']) # Check if the jobs are already activated jobEnabledList = [ jobID for jobID in jobList if js_dict[jobID]['Status'] in [ JobStatus.RECEIVED, JobStatus.CHECKING, JobStatus.WAITING, JobStatus.MATCHED, JobStatus.RUNNING ] ] if set(jobEnabledList) == set(jobList): return S_OK(jobList) # Check that requested job are in Submitting status jobUpdateStatusList = list( jobID for jobID in jobList if js_dict[jobID]['Status'] == JobStatus.SUBMITTING) if set(jobUpdateStatusList) != set(jobList): return S_ERROR( EWMSSUBM, 'Requested jobs for bulk transaction are not valid') # Update status of all the requested jobs in one transaction result = self.jobDB.setJobAttributes( jobUpdateStatusList, ['Status', 'MinorStatus'], [JobStatus.RECEIVED, 'Job accepted']) if not result['OK']: return result self.__sendJobsToOptimizationMind(jobUpdateStatusList) return S_OK(jobUpdateStatusList)
def getRequestIDsForJobs(self, jobIDs): """get the request ids for the supplied jobIDs. :param self: self reference :param list jobIDs: list of job IDs (integers) :return: S_ERROR or S_OK( "Successful": { jobID1: reqID1, jobID2: requID2, ... }, "Failed" : { jobIDn: errMsg, jobIDm: errMsg, ...} ) """ self.log.verbose("getRequestIDsForJobs: attempt to get request(s) for jobs", "(n=%d)" % len(jobIDs)) res = self._getRPC().getRequestIDsForJobs(jobIDs) if not res["OK"]: self.log.error( "getRequestIDsForJobs: unable to get request(s) for jobs", "%s: %s" % (jobIDs, res["Message"]) ) return res # Cast the JobIDs back to int successful = strToIntDict(res["Value"]["Successful"]) failed = strToIntDict(res["Value"]["Failed"]) return S_OK({"Successful": successful, "Failed": failed})
def export_commitMarks(self, sourceId, activitiesDict, componentExtraInfo={}): """ Adds marks for activities :param int sourceId: component id :param dict activitiesDict: the key is the metrice, the value is a dictionary with timestamp and value for example:: {u'CPU': {1583244000: 10.6055594906}, u'Iteration': {1583244420: 1, 1583244480: 1, 1583244540: 1, 1583244600: 1, 1583244660: 1}, u'MEM': {1583244000: 49.03515625}, u'Processed': {1583244420: 5, 1583244480: 5, 1583244540: 5, 1583244600: 5, 1583244660: 6}} :param dict componentExtraInfo: extra information """ # we have to cast the dictionary keys, because they are timestamps activitiesDict = { metric: strToIntDict(value) for metric, value in activitiesDict.iteritems() } nowEpoch = Time.toEpoch() maxEpoch = nowEpoch + 7200 minEpoch = nowEpoch - 86400 invalidActivities = [] for acName in activitiesDict: for time in activitiesDict[acName]: if time > maxEpoch or time < minEpoch: gLogger.info( "Time %s ( [%s,%s] ) is invalid for activity %s" % (time, minEpoch, maxEpoch, acName)) invalidActivities.append(acName) break for acName in invalidActivities: gLogger.info("Not commiting activity %s" % acName) del activitiesDict[acName] return gServiceInterface.commitMarks(sourceId, activitiesDict, componentExtraInfo)
def readRequestsForJobs(self, jobIDs): """read requests for jobs :param jobIDs: list with jobIDs :type jobIDs: python:list :return: S_OK( { "Successful" : { jobID1 : Request, ... }, "Failed" : { jobIDn : "Fail reason" } } ) """ readReqsForJobs = self._getRPC().readRequestsForJobs(jobIDs) if not readReqsForJobs["OK"]: return readReqsForJobs ret = readReqsForJobs["Value"] # # create Requests out of JSONs for successful reads # Do not forget to cast back str keys to int successful = {int(jobID): Request(jsonReq) for jobID, jsonReq in ret["Successful"].items()} failed = strToIntDict(ret["Failed"]) return S_OK({"Successful": successful, "Failed": failed})
def doNew(self, masterParams=None): ''' Gets the parameters to run, either from the master method or from its own arguments. For every elementName ( cannot process bulk queries.. ) contacts the accounting client. It reurns dictionaries like { 'X -> Y' : { id: 100%.. } } If there are ggus tickets, are recorded and then returned. ''' if masterParams is not None: hours, name, direction, metric = masterParams else: params = self._prepareCommand() if not params['OK']: return params hours, name, direction, metric = params['Value'] toD = datetime.utcnow() fromD = toD - timedelta(hours=hours) # dictionary with conditions for the accounting transferDict = {'OperationType': 'putAndRegister', direction: name} if metric == 'FailedTransfers': transferDict['FinalStatus'] = ['Failed'] transferResults = self.rClient.getReport('DataOperation', metric, fromD, toD, transferDict, 'Channel') if not transferResults['OK']: return transferResults transferResults = transferResults['Value'] if 'data' not in transferResults: return S_ERROR('Missing data key') transferResults = { channel: strToIntDict(value) for channel, value in transferResults['data'].iteritems() } uniformResult = [] for channel, elementDict in transferResults.items(): try: source, destination = channel.split(' -> ') except ValueError: continue channelDict = {} channelDict['SourceName'] = source channelDict['DestinationName'] = destination channelDict['Metric'] = metric channelDict['Value'] = sum(elementDict.values()) / len( elementDict.values()) uniformResult.append(channelDict) storeRes = self._storeCommand(uniformResult) if not storeRes['OK']: return storeRes # Compute mean of all transfer channels value = 0 for channelDict in uniformResult: value += channelDict['Value'] if uniformResult: value = float(value) / len(uniformResult) else: value = None return S_OK({'Mean': value, 'Name': name})
def doCommand(self): """ Returns failed jobs using the DIRAC accounting system for every site for the last self.args[0] hours :params: :attr:`sites`: list of sites (when not given, take every site) :returns: """ if "hours" not in self.args: return S_ERROR("Number of hours not specified") hours = self.args["hours"] sites = None if "sites" in self.args: sites = self.args["sites"] if sites is None: # FIXME: pointing to the CSHelper instead # sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} ) # if not sources[ 'OK' ]: # return sources # sources = [ si[0] for si in sources[ 'Value' ] ] sites = getSites() if not sites["OK"]: return sites sites = sites["Value"] if not sites: return S_ERROR("Sites is empty") fromD = datetime.utcnow() - timedelta(hours=hours) toD = datetime.utcnow() failedPilots = self.rClient.getReport("Pilot", "NumberOfPilots", fromD, toD, { "GridStatus": ["Aborted"], "Site": sites }, "Site") if not failedPilots["OK"]: return failedPilots failedPilots = failedPilots["Value"] if "data" not in failedPilots: return S_ERROR("Missing data key") if "granularity" not in failedPilots: return S_ERROR("Missing granularity key") failedPilots["data"] = { site: strToIntDict(value) for site, value in failedPilots["data"].items() } singlePlots = {} for site, value in failedPilots["data"].items(): if site in sites: plot = {} plot["data"] = {site: value} plot["granularity"] = failedPilots["granularity"] singlePlots[site] = plot return S_OK(singlePlots)
def doCommand(self): """ Returns running and runned jobs, querying the WMSHistory for the last self.args[0] hours :params: :attr:`sites`: list of sites (when not given, take every sites) :returns: """ if "hours" not in self.args: return S_ERROR("Number of hours not specified") hours = self.args["hours"] sites = None if "sites" in self.args: sites = self.args["sites"] if sites is None: # FIXME: pointing to the CSHelper instead # sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} ) # if not sources[ 'OK' ]: # return sources # sources = [ si[0] for si in sources[ 'Value' ] ] sites = getSites() if not sites["OK"]: return sites sites = sites["Value"] if not sites: return S_ERROR("Sites is empty") fromD = datetime.utcnow() - timedelta(hours=hours) toD = datetime.utcnow() runJobs = self.rClient.getReport("WMSHistory", "NumberOfJobs", fromD, toD, {}, "Site") if not runJobs["OK"]: return runJobs runJobs = runJobs["Value"] if "data" not in runJobs: return S_ERROR("Missing data key") if "granularity" not in runJobs: return S_ERROR("Missing granularity key") runJobs["data"] = { site: strToIntDict(value) for site, value in runJobs["data"].items() } singlePlots = {} for site, value in runJobs["data"].items(): if site in sites: plot = {} plot["data"] = {site: value} plot["granularity"] = runJobs["granularity"] singlePlots[site] = plot return S_OK(singlePlots)
def doCommand(self): """ Returns failed pilots using the DIRAC accounting system for every CE for the last self.args[0] hours :params: :attr:`CEs`: list of CEs (when not given, take every CE) :returns: """ if 'hours' not in self.args: return S_ERROR('Number of hours not specified') hours = self.args['hours'] ces = None if 'ces' in self.args: ces = self.args['ces'] if ces is None: # FIXME: pointing to the CSHelper instead # meta = {'columns':'ResourceName'} # CEs = self.rsClient.getResource( resourceType = [ 'CE','CREAMCE' ], meta = meta ) # if not CEs['OK']: # return CEs # CEs = [ ce[0] for ce in CEs['Value'] ] ces = CSHelpers.getComputingElements() if not ces['OK']: return ces ces = ces['Value'] if not ces: return S_ERROR('CEs is empty') fromD = datetime.utcnow() - timedelta(hours=hours) toD = datetime.utcnow() failedPilots = self.rClient.getReport('Pilot', 'NumberOfPilots', fromD, toD, { 'GridStatus': ['Aborted'], 'GridCE': ces }, 'GridCE') if not failedPilots['OK']: return failedPilots failedPilots = failedPilots['Value'] if 'data' not in failedPilots: return S_ERROR('Missing data key') if 'granularity' not in failedPilots: return S_ERROR('Missing granularity key') failedPilots['data'] = { site: strToIntDict(value) for site, value in failedPilots['data'].iteritems() } singlePlots = {} for ce, value in failedPilots['data'].iteritems(): if ce in ces: plot = {} plot['data'] = {ce: value} plot['granularity'] = failedPilots['granularity'] singlePlots[ce] = plot return S_OK(singlePlots)
def doCommand(self): """ Returns failed jobs using the DIRAC accounting system for every site for the last self.args[0] hours :params: :attr:`sites`: list of sites (when not given, take every site) :returns: """ if 'hours' not in self.args: return S_ERROR('Number of hours not specified') hours = self.args['hours'] sites = None if 'sites' in self.args: sites = self.args['sites'] if sites is None: # FIXME: pointing to the CSHelper instead # sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} ) # if not sources[ 'OK' ]: # return sources # sources = [ si[0] for si in sources[ 'Value' ] ] sites = getSites() if not sites['OK']: return sites sites = sites['Value'] if not sites: return S_ERROR('Sites is empty') fromD = datetime.utcnow() - timedelta(hours=hours) toD = datetime.utcnow() failedPilots = self.rClient.getReport('Pilot', 'NumberOfPilots', fromD, toD, { 'GridStatus': ['Aborted'], 'Site': sites }, 'Site') if not failedPilots['OK']: return failedPilots failedPilots = failedPilots['Value'] if 'data' not in failedPilots: return S_ERROR('Missing data key') if 'granularity' not in failedPilots: return S_ERROR('Missing granularity key') failedPilots['data'] = { site: strToIntDict(value) for site, value in failedPilots['data'].iteritems() } singlePlots = {} for site, value in failedPilots['data'].iteritems(): if site in sites: plot = {} plot['data'] = {site: value} plot['granularity'] = failedPilots['granularity'] singlePlots[site] = plot return S_OK(singlePlots)
def test_getReport(putAndDelete): params = ( "WMSHistory", "NumberOfJobs", datetime(2016, 3, 16, 12, 30, 0, 0), datetime(2016, 3, 17, 19, 29, 0, 0), { "grouping": ["Site"] }, "Site", {}, ) result = client.getReport(*params) assert result["OK"], result["Message"] result["Value"]["data"] = { site: strToIntDict(value) for site, value in result["Value"]["data"].items() } assert result["Value"] == { "data": { u"Multiple": { 1458198000: 227.0 }, u"LCG.RRCKI.ru": { 1458225000: 3.0 }, u"LCG.IHEP.su": { 1458217800: 18.0 }, u"LCG.CNAF.it": { 1458144000: None, 1458172800: None, 1458194400: None, 1458145800: None, 1458189000: None, 1458147600: None, 1458178200: None, 1458183600: None, 1458212400: None, 1458149400: None, 1458207000: None, 1458151200: None, 1458169200: None, 1458201600: None, 1458153000: None, 1458196200: None, 1458154800: None, 1458174600: None, 1458190800: None, 1458156600: None, 1458185400: None, 1458214200: None, 1458158400: None, 1458180000: None, 1458216000: None, 1458208800: None, 1458160200: None, 1458203400: None, 1458162000: None, 1458142200: None, 1458198000: None, 1458163800: None, 1458192600: None, 1458165600: None, 1458176400: None, 1458187200: None, 1458167400: None, 1458210600: None, 1458140400: 4.0, 1458181800: None, 1458205200: None, 1458171000: None, 1458217800: 22.0, 1458199800: None, }, u"LCG.NIKHEF.nl": { 1458217800: 27.0 }, u"LCG.Bari.it": { 1458221400: 34.0 }, u"Group.RAL.uk": { 1458140400: 34.0 }, u"LCG.DESYZN.de": { 1458225000: 43.0 }, u"LCG.RAL.uk": { 1458144000: None, 1458158400: None, 1458194400: None, 1458145800: None, 1458223200: None, 1458189000: None, 1458221400: None, 1458225000: 5.0, 1458147600: None, 1458135000: None, 1458183600: None, 1458212400: None, 1458149400: None, 1458178200: None, 1458207000: None, 1458151200: None, 1458169200: None, 1458172800: None, 1458219600: None, 1458201600: None, 1458153000: None, 1458196200: None, 1458154800: None, 1458160200: None, 1458190800: None, 1458156600: None, 1458185400: None, 1458214200: None, 1458129600: 2.0, 1458165600: None, 1458180000: None, 1458216000: None, 1458208800: None, 1458131400: None, 1458174600: None, 1458203400: None, 1458162000: None, 1458171000: None, 1458198000: None, 1458163800: None, 1458192600: None, 1458136800: None, 1458133200: None, 1458187200: None, 1458167400: None, 1458181800: None, 1458210600: None, 1458140400: None, 1458138600: None, 1458176400: None, 1458205200: None, 1458142200: None, 1458217800: None, 1458199800: None, }, u"LCG.PIC.es": { 1458129600: 1.0 }, u"LCG.GRIDKA.de": { 1458129600: 2.0 }, u"LCG.Bristol.uk": { 1458221400: 9.0 }, u"LCG.CERN.ch": { 1458140400: 120.0 }, u"LCG.Bologna.it": { 1458221400: 1.0 }, }, "granularity": 1800, }
def test_getReport(self): params = ('WMSHistory', 'NumberOfJobs', datetime(2016, 3, 16, 12, 30, 0, 0), datetime(2016, 3, 17, 19, 29, 0, 0), { 'grouping': ['Site'] }, 'Site', {}) result = self.client.getReport(*params) self.assertTrue(result['OK'], result.get('Message')) result['Value']['data'] = { site: strToIntDict(value) for site, value in result['Value']['data'].iteritems() } self.assertDictEqual( result['Value'], { 'data': { u'Multiple': { 1458198000: 227.0 }, u'LCG.RRCKI.ru': { 1458225000: 3.0 }, u'LCG.IHEP.su': { 1458217800: 18.0 }, u'LCG.CNAF.it': { 1458144000: None, 1458172800: None, 1458194400: None, 1458145800: None, 1458189000: None, 1458147600: None, 1458178200: None, 1458183600: None, 1458212400: None, 1458149400: None, 1458207000: None, 1458151200: None, 1458169200: None, 1458201600: None, 1458153000: None, 1458196200: None, 1458154800: None, 1458174600: None, 1458190800: None, 1458156600: None, 1458185400: None, 1458214200: None, 1458158400: None, 1458180000: None, 1458216000: None, 1458208800: None, 1458160200: None, 1458203400: None, 1458162000: None, 1458142200: None, 1458198000: None, 1458163800: None, 1458192600: None, 1458165600: None, 1458176400: None, 1458187200: None, 1458167400: None, 1458210600: None, 1458140400: 4.0, 1458181800: None, 1458205200: None, 1458171000: None, 1458217800: 22.0, 1458199800: None }, u'LCG.NIKHEF.nl': { 1458217800: 27.0 }, u'LCG.Bari.it': { 1458221400: 34.0 }, u'Group.RAL.uk': { 1458140400: 34.0 }, u'LCG.DESYZN.de': { 1458225000: 43.0 }, u'LCG.RAL.uk': { 1458144000: None, 1458158400: None, 1458194400: None, 1458145800: None, 1458223200: None, 1458189000: None, 1458221400: None, 1458225000: 5.0, 1458147600: None, 1458135000: None, 1458183600: None, 1458212400: None, 1458149400: None, 1458178200: None, 1458207000: None, 1458151200: None, 1458169200: None, 1458172800: None, 1458219600: None, 1458201600: None, 1458153000: None, 1458196200: None, 1458154800: None, 1458160200: None, 1458190800: None, 1458156600: None, 1458185400: None, 1458214200: None, 1458129600: 2.0, 1458165600: None, 1458180000: None, 1458216000: None, 1458208800: None, 1458131400: None, 1458174600: None, 1458203400: None, 1458162000: None, 1458171000: None, 1458198000: None, 1458163800: None, 1458192600: None, 1458136800: None, 1458133200: None, 1458187200: None, 1458167400: None, 1458181800: None, 1458210600: None, 1458140400: None, 1458138600: None, 1458176400: None, 1458205200: None, 1458142200: None, 1458217800: None, 1458199800: None }, u'LCG.PIC.es': { 1458129600: 1.0 }, u'LCG.GRIDKA.de': { 1458129600: 2.0 }, u'LCG.Bristol.uk': { 1458221400: 9.0 }, u'LCG.CERN.ch': { 1458140400: 120.0 }, u'LCG.Bologna.it': { 1458221400: 1.0 } }, 'granularity': 1800 })
def doNew(self, masterParams=None): """ Gets the parameters to run, either from the master method or from its own arguments. For every elementName ( cannot process bulk queries.. ) contacts the accounting client. It reurns dictionaries like { 'X -> Y' : { id: 100%.. } } If there are ggus tickets, are recorded and then returned. """ if masterParams is not None: hours, name, direction, metric = masterParams else: params = self._prepareCommand() if not params["OK"]: return params hours, name, direction, metric = params["Value"] toD = datetime.utcnow() fromD = toD - timedelta(hours=hours) # dictionary with conditions for the accounting transferDict = {"OperationType": "putAndRegister", direction: name} if metric == "FailedTransfers": transferDict["FinalStatus"] = ["Failed"] transferResults = self.rClient.getReport("DataOperation", metric, fromD, toD, transferDict, "Channel") if not transferResults["OK"]: return transferResults transferResults = transferResults["Value"] if "data" not in transferResults: return S_ERROR("Missing data key") transferResults = {channel: strToIntDict(value) for channel, value in transferResults["data"].items()} uniformResult = [] for channel, elementDict in transferResults.items(): try: source, destination = channel.split(" -> ") except ValueError: continue channelDict = {} channelDict["SourceName"] = source channelDict["DestinationName"] = destination channelDict["Metric"] = metric channelDict["Value"] = int(sum(elementDict.values()) / len(elementDict.values())) uniformResult.append(channelDict) storeRes = self._storeCommand(uniformResult) if not storeRes["OK"]: return storeRes # Compute mean of all transfer channels value = 0 for channelDict in uniformResult: value += channelDict["Value"] if uniformResult: value = float(value) / len(uniformResult) else: value = None return S_OK({"Mean": value, "Name": name})