def __addPilotsAccountingReport(self, pilotsData):
""" fill accounting data
"""
for pRef in pilotsData:
pData = pilotsData[pRef]
pA = PilotAccounting()
pA.setEndTime(pData["LastUpdateTime"])
pA.setStartTime(pData["SubmissionTime"])
retVal = CS.getUsernameForDN(pData["OwnerDN"])
if not retVal["OK"]:
userName = "******"
self.log.error("Can't determine username for dn:", pData["OwnerDN"])
else:
userName = retVal["Value"]
pA.setValueByKey("User", userName)
pA.setValueByKey("UserGroup", pData["OwnerGroup"])
result = getSiteForCE(pData["DestinationSite"])
if result["OK"] and result["Value"].strip():
pA.setValueByKey("Site", result["Value"].strip())
else:
pA.setValueByKey("Site", "Unknown")
pA.setValueByKey("GridCE", pData["DestinationSite"])
pA.setValueByKey("GridMiddleware", pData["GridType"])
pA.setValueByKey("GridResourceBroker", pData["Broker"])
pA.setValueByKey("GridStatus", pData["Status"])
if not "Jobs" in pData:
pA.setValueByKey("Jobs", 0)
else:
pA.setValueByKey("Jobs", len(pData["Jobs"]))
self.log.verbose("Added accounting record for pilot %s" % pData["PilotID"])
retVal = gDataStoreClient.addRegister(pA)
if not retVal["OK"]:
return retVal
return S_OK()
def sendPilotAccounting( self, pilotDict ):
""" Send pilot accounting record
"""
for pRef in pilotDict:
self.log.verbose( 'Preparing accounting record for pilot %s' % pRef )
pA = PilotAccounting()
pA.setEndTime( pilotDict[pRef][ 'LastUpdateTime' ] )
pA.setStartTime( pilotDict[pRef][ 'SubmissionTime' ] )
retVal = CS.getUsernameForDN( pilotDict[pRef][ 'OwnerDN' ] )
if not retVal[ 'OK' ]:
userName = '******'
self.log.error( "Can't determine username for dn:", pilotDict[pRef][ 'OwnerDN' ] )
else:
userName = retVal[ 'Value' ]
pA.setValueByKey( 'User', userName )
pA.setValueByKey( 'UserGroup', pilotDict[pRef][ 'OwnerGroup' ] )
result = getSiteForCE( pilotDict[pRef][ 'DestinationSite' ] )
if result['OK'] and result[ 'Value' ].strip():
pA.setValueByKey( 'Site', result['Value'].strip() )
else:
pA.setValueByKey( 'Site', 'Unknown' )
pA.setValueByKey( 'GridCE', pilotDict[pRef][ 'DestinationSite' ] )
pA.setValueByKey( 'GridMiddleware', pilotDict[pRef][ 'GridType' ] )
pA.setValueByKey( 'GridResourceBroker', pilotDict[pRef][ 'Broker' ] )
pA.setValueByKey( 'GridStatus', pilotDict[pRef][ 'Status' ] )
if not 'Jobs' in pilotDict[pRef]:
pA.setValueByKey( 'Jobs', 0 )
else:
pA.setValueByKey( 'Jobs', len( pilotDict[pRef]['Jobs'] ) )
self.log.verbose( "Adding accounting record for pilot %s" % pilotDict[pRef][ 'PilotID' ] )
retVal = gDataStoreClient.addRegister( pA )
if not retVal[ 'OK' ]:
self.log.error( 'Failed to send accounting info for pilot ', pRef )
else:
# Set up AccountingSent flag
result = pilotAgentsDB.setAccountingFlag( pRef )
if not result['OK']:
self.log.error( 'Failed to set accounting flag for pilot ', pRef )
self.log.info( 'Committing accounting records for %d pilots' % len( pilotDict ) )
result = gDataStoreClient.commit()
if result['OK']:
for pRef in pilotDict:
self.log.verbose( 'Setting AccountingSent flag for pilot %s' % pRef )
result = pilotAgentsDB.setAccountingFlag( pRef )
if not result['OK']:
self.log.error( 'Failed to set accounting flag for pilot ', pRef )
else:
return result
return S_OK()
def _addPilotsAccountingReport(self, pilotsData):
"""fill accounting data"""
for pRef in pilotsData:
pData = pilotsData[pRef]
pA = PilotAccounting()
pA.setEndTime(pData["LastUpdateTime"])
pA.setStartTime(pData["SubmissionTime"])
retVal = Registry.getUsernameForDN(pData["OwnerDN"])
if not retVal["OK"]:
userName = "******"
self.log.error(
"Can't determine username for dn",
": %s : %s" % (pData["OwnerDN"], retVal["Message"]),
)
else:
userName = retVal["Value"]
pA.setValueByKey("User", userName)
pA.setValueByKey("UserGroup", pData["OwnerGroup"])
result = getCESiteMapping(pData["DestinationSite"])
if result["OK"] and pData["DestinationSite"] in result["Value"]:
pA.setValueByKey(
"Site", result["Value"][pData["DestinationSite"]].strip())
else:
pA.setValueByKey("Site", "Unknown")
pA.setValueByKey("GridCE", pData["DestinationSite"])
pA.setValueByKey("GridMiddleware", pData["GridType"])
pA.setValueByKey("GridResourceBroker", pData["Broker"])
pA.setValueByKey("GridStatus", pData["Status"])
if "Jobs" not in pData:
pA.setValueByKey("Jobs", 0)
else:
pA.setValueByKey("Jobs", len(pData["Jobs"]))
self.log.verbose("Added accounting record for pilot %s" %
pData["PilotID"])
retVal = gDataStoreClient.addRegister(pA)
if not retVal["OK"]:
return retVal
return S_OK()
class PilotPlotter(BaseReporter):
_typeName = "Pilot"
_typeKeyFields = [dF[0] for dF in Pilot().definitionKeyFields]
def _reportCumulativeNumberOfJobs(self, reportRequest):
selectFields = (
self._getSelectStringForGrouping(reportRequest['groupingFields']) +
", %s, %s, SUM(%s)", reportRequest['groupingFields'][1] +
['startTime', 'bucketLength', 'Jobs'])
retVal = self._getTimedData(reportRequest['startTime'],
reportRequest['endTime'], selectFields,
reportRequest['condDict'],
reportRequest['groupingFields'], {})
if not retVal['OK']:
return retVal
dataDict, granularity = retVal['Value']
self.stripDataField(dataDict, 0)
dataDict = self._fillWithZero(granularity, reportRequest['startTime'],
reportRequest['endTime'], dataDict)
dataDict = self._accumulate(granularity, reportRequest['startTime'],
reportRequest['endTime'], dataDict)
baseDataDict, graphDataDict, maxValue, unitName = self._findSuitableUnit(
dataDict, self._getAccumulationMaxValue(dataDict), "jobs")
return S_OK({
'data': baseDataDict,
'graphDataDict': graphDataDict,
'granularity': granularity,
'unit': unitName
})
def _plotCumulativeNumberOfJobs(self, reportRequest, plotInfo, filename):
metadata = {
'title': 'Cumulative Jobs by %s' % reportRequest['grouping'],
'starttime': reportRequest['startTime'],
'endtime': reportRequest['endTime'],
'span': plotInfo['granularity'],
'ylabel': plotInfo['unit'],
'sort_labels': 'last_value'
}
return self._generateCumulativePlot(filename,
plotInfo['graphDataDict'],
metadata)
def _reportNumberOfJobs(self, reportRequest):
selectFields = (
self._getSelectStringForGrouping(reportRequest['groupingFields']) +
", %s, %s, SUM(%s)", reportRequest['groupingFields'][1] +
['startTime', 'bucketLength', 'Jobs'])
retVal = self._getTimedData(reportRequest['startTime'],
reportRequest['endTime'], selectFields,
reportRequest['condDict'],
reportRequest['groupingFields'], {})
if not retVal['OK']:
return retVal
dataDict, granularity = retVal['Value']
self.stripDataField(dataDict, 0)
dataDict, maxValue = self._divideByFactor(dataDict, granularity)
dataDict = self._fillWithZero(granularity, reportRequest['startTime'],
reportRequest['endTime'], dataDict)
baseDataDict, graphDataDict, maxValue, unitName = self._findSuitableRateUnit(
dataDict, self._getAccumulationMaxValue(dataDict), "jobs")
return S_OK({
'data': baseDataDict,
'graphDataDict': graphDataDict,
'granularity': granularity,
'unit': unitName
})
def _plotNumberOfJobs(self, reportRequest, plotInfo, filename):
metadata = {
'title': 'Jobs by %s' % reportRequest['grouping'],
'starttime': reportRequest['startTime'],
'endtime': reportRequest['endTime'],
'span': plotInfo['granularity'],
'ylabel': plotInfo['unit']
}
return self._generateTimedStackedBarPlot(filename,
plotInfo['graphDataDict'],
metadata)
def _reportCumulativeNumberOfPilots(self, reportRequest):
selectFields = (
self._getSelectStringForGrouping(reportRequest['groupingFields']) +
", %s, %s, SUM(%s)", reportRequest['groupingFields'][1] +
['startTime', 'bucketLength', 'entriesInBucket'])
retVal = self._getTimedData(reportRequest['startTime'],
reportRequest['endTime'], selectFields,
reportRequest['condDict'],
reportRequest['groupingFields'], {})
if not retVal['OK']:
return retVal
dataDict, granularity = retVal['Value']
self.stripDataField(dataDict, 0)
dataDict = self._fillWithZero(granularity, reportRequest['startTime'],
reportRequest['endTime'], dataDict)
dataDict = self._accumulate(granularity, reportRequest['startTime'],
reportRequest['endTime'], dataDict)
baseDataDict, graphDataDict, maxValue, unitName = self._findSuitableUnit(
dataDict, self._getAccumulationMaxValue(dataDict), "jobs")
return S_OK({
'data': baseDataDict,
'graphDataDict': graphDataDict,
'granularity': granularity,
'unit': unitName
})
def _plotCumulativeNumberOfPilots(self, reportRequest, plotInfo, filename):
metadata = {
'title': 'Cumulative Pilots by %s' % reportRequest['grouping'],
'starttime': reportRequest['startTime'],
'endtime': reportRequest['endTime'],
'span': plotInfo['granularity'],
'ylabel': plotInfo['unit'].replace('job', 'pilot'),
'sort_labels': 'last_value'
}
return self._generateCumulativePlot(filename,
plotInfo['graphDataDict'],
metadata)
def _reportNumberOfPilots(self, reportRequest):
selectFields = (
self._getSelectStringForGrouping(reportRequest['groupingFields']) +
", %s, %s, SUM(%s)", reportRequest['groupingFields'][1] +
['startTime', 'bucketLength', 'entriesInBucket'])
retVal = self._getTimedData(reportRequest['startTime'],
reportRequest['endTime'], selectFields,
reportRequest['condDict'],
reportRequest['groupingFields'], {})
if not retVal['OK']:
return retVal
dataDict, granularity = retVal['Value']
self.stripDataField(dataDict, 0)
dataDict, maxValue = self._divideByFactor(dataDict, granularity)
dataDict = self._fillWithZero(granularity, reportRequest['startTime'],
reportRequest['endTime'], dataDict)
baseDataDict, graphDataDict, maxValue, unitName = self._findSuitableRateUnit(
dataDict, self._getAccumulationMaxValue(dataDict), "jobs")
return S_OK({
'data': baseDataDict,
'graphDataDict': graphDataDict,
'granularity': granularity,
'unit': unitName
})
def _plotNumberOfPilots(self, reportRequest, plotInfo, filename):
metadata = {
'title': 'Pilots by %s' % reportRequest['grouping'],
'starttime': reportRequest['startTime'],
'endtime': reportRequest['endTime'],
'span': plotInfo['granularity'],
'ylabel': plotInfo['unit'].replace('job', 'pilot')
}
return self._generateTimedStackedBarPlot(filename,
plotInfo['graphDataDict'],
metadata)
def _reportJobsPerPilot(self, reportRequest):
selectFields = (
self._getSelectStringForGrouping(reportRequest['groupingFields']) +
", %s, %s, SUM(%s), SUM(%s)", reportRequest['groupingFields'][1] +
['startTime', 'bucketLength', 'Jobs', 'entriesInBucket'])
retVal = self._getTimedData(
reportRequest['startTime'], reportRequest['endTime'], selectFields,
reportRequest['condDict'], reportRequest['groupingFields'], {
'checkNone': True,
'convertToGranularity': 'sum',
'calculateProportionalGauges': False,
'consolidationFunction': self._averageConsolidation
})
if not retVal['OK']:
return retVal
dataDict, granularity = retVal['Value']
self.stripDataField(dataDict, 0)
dataDict = self._fillWithZero(granularity, reportRequest['startTime'],
reportRequest['endTime'], dataDict)
return S_OK({'data': dataDict, 'granularity': granularity})
def _plotJobsPerPilot(self, reportRequest, plotInfo, filename):
metadata = {
'title':
'Jobs per pilot by %s' % reportRequest['grouping'],
'starttime':
reportRequest['startTime'],
'endtime':
reportRequest['endTime'],
'span':
plotInfo['granularity'],
'ylabel':
"jobs/pilot",
'normalization':
max(x for y in plotInfo['data'].values() for x in y.values())
}
return self._generateQualityPlot(filename, plotInfo['data'], metadata)
def _reportTotalNumberOfPilots(self, reportRequest):
selectFields = (
self._getSelectStringForGrouping(reportRequest['groupingFields']) +
", SUM(%s)",
reportRequest['groupingFields'][1] + ['entriesInBucket'])
retVal = self._getSummaryData(reportRequest['startTime'],
reportRequest['endTime'], selectFields,
reportRequest['condDict'],
reportRequest['groupingFields'], {})
if not retVal['OK']:
return retVal
dataDict = retVal['Value']
return S_OK({'data': dataDict})
def _plotTotalNumberOfPilots(self, reportRequest, plotInfo, filename):
metadata = {
'title':
'Total Number of Pilots by %s' % reportRequest['grouping'],
'ylabel': 'Pilots',
'starttime': reportRequest['startTime'],
'endtime': reportRequest['endTime']
}
return self._generatePiePlot(filename, plotInfo['data'], metadata)
def __addPilotsAccountingReport(self, pilotsData):
""" fill accounting data
"""
for pRef in pilotsData:
pData = pilotsData[pRef]
pA = PilotAccounting()
pA.setEndTime(pData['LastUpdateTime'])
pA.setStartTime(pData['SubmissionTime'])
retVal = CS.getUsernameForDN(pData['OwnerDN'])
if not retVal['OK']:
userName = '******'
self.log.error("Can't determine username for dn:", pData['OwnerDN'])
else:
userName = retVal['Value']
pA.setValueByKey('User', userName)
pA.setValueByKey('UserGroup', pData['OwnerGroup'])
result = getSiteForCE(pData['DestinationSite'])
if result['OK'] and result['Value'].strip():
pA.setValueByKey('Site', result['Value'].strip())
else:
pA.setValueByKey('Site', 'Unknown')
pA.setValueByKey('GridCE', pData['DestinationSite'])
pA.setValueByKey('GridMiddleware', pData['GridType'])
pA.setValueByKey('GridResourceBroker', pData['Broker'])
pA.setValueByKey('GridStatus', pData['Status'])
if 'Jobs' not in pData:
pA.setValueByKey('Jobs', 0)
else:
pA.setValueByKey('Jobs', len(pData['Jobs']))
self.log.verbose("Added accounting record for pilot %s" % pData['PilotID'])
retVal = gDataStoreClient.addRegister(pA)
if not retVal['OK']:
return retVal
return S_OK()
def __addPilotsAccountingReport(self, pilotsData):
""" fill accounting data
"""
for pRef in pilotsData:
pData = pilotsData[pRef]
pA = PilotAccounting()
pA.setEndTime(pData['LastUpdateTime'])
pA.setStartTime(pData['SubmissionTime'])
retVal = CS.getUsernameForDN(pData['OwnerDN'])
if not retVal['OK']:
userName = '******'
self.log.error("Can't determine username for dn:",
pData['OwnerDN'])
else:
userName = retVal['Value']
pA.setValueByKey('User', userName)
pA.setValueByKey('UserGroup', pData['OwnerGroup'])
result = getSiteForCE(pData['DestinationSite'])
if result['OK'] and result['Value'].strip():
pA.setValueByKey('Site', result['Value'].strip())
else:
pA.setValueByKey('Site', 'Unknown')
pA.setValueByKey('GridCE', pData['DestinationSite'])
pA.setValueByKey('GridMiddleware', pData['GridType'])
pA.setValueByKey('GridResourceBroker', pData['Broker'])
pA.setValueByKey('GridStatus', pData['Status'])
if 'Jobs' not in pData:
pA.setValueByKey('Jobs', 0)
else:
pA.setValueByKey('Jobs', len(pData['Jobs']))
self.log.verbose("Added accounting record for pilot %s" %
pData['PilotID'])
retVal = gDataStoreClient.addRegister(pA)
if not retVal['OK']:
return retVal
return S_OK()
def sendPilotAccounting(self, pilotDict):
""" Send pilot accounting record
"""
for pRef in pilotDict:
self.log.verbose('Preparing accounting record for pilot %s' % pRef)
pA = PilotAccounting()
pA.setEndTime(pilotDict[pRef]['LastUpdateTime'])
pA.setStartTime(pilotDict[pRef]['SubmissionTime'])
retVal = CS.getUsernameForDN(pilotDict[pRef]['OwnerDN'])
if not retVal['OK']:
userName = '******'
self.log.error("Can't determine username for dn:",
pilotDict[pRef]['OwnerDN'])
else:
userName = retVal['Value']
pA.setValueByKey('User', userName)
pA.setValueByKey('UserGroup', pilotDict[pRef]['OwnerGroup'])
result = getSiteForCE(pilotDict[pRef]['DestinationSite'])
if result['OK'] and result['Value'].strip():
pA.setValueByKey('Site', result['Value'].strip())
else:
pA.setValueByKey('Site', 'Unknown')
pA.setValueByKey('GridCE', pilotDict[pRef]['DestinationSite'])
pA.setValueByKey('GridMiddleware', pilotDict[pRef]['GridType'])
pA.setValueByKey('GridResourceBroker', pilotDict[pRef]['Broker'])
pA.setValueByKey('GridStatus', pilotDict[pRef]['Status'])
if not 'Jobs' in pilotDict[pRef]:
pA.setValueByKey('Jobs', 0)
else:
pA.setValueByKey('Jobs', len(pilotDict[pRef]['Jobs']))
self.log.info("Adding accounting record for pilot %s" %
pilotDict[pRef]['PilotID'])
retVal = gDataStoreClient.addRegister(pA)
if not retVal['OK']:
self.log.error('Failed to send accounting info for pilot ',
pRef)
else:
# Set up AccountingSent flag
result = pilotAgentsDB.setAccountingFlag(pRef)
if not result['OK']:
self.log.error('Failed to set accounting flag for pilot ',
pRef)
self.log.info('Committing accounting records for %d pilots' %
len(pilotDict))
result = gDataStoreClient.commit()
if result['OK']:
for pRef in pilotDict:
self.log.verbose('Setting AccountingSent flag for pilot %s' %
pRef)
result = pilotAgentsDB.setAccountingFlag(pRef)
if not result['OK']:
self.log.error('Failed to set accounting flag for pilot ',
pRef)
else:
return result
return S_OK()
class PilotPlotter(BaseReporter):
_typeName = "Pilot"
_typeKeyFields = [dF[0] for dF in Pilot().definitionKeyFields]
def _reportCumulativeNumberOfJobs(self, reportRequest):
selectFields = (
self._getSelectStringForGrouping(reportRequest["groupingFields"]) +
", %s, %s, SUM(%s)",
reportRequest["groupingFields"][1] +
["startTime", "bucketLength", "Jobs"],
)
retVal = self._getTimedData(
reportRequest["startTime"],
reportRequest["endTime"],
selectFields,
reportRequest["condDict"],
reportRequest["groupingFields"],
{},
)
if not retVal["OK"]:
return retVal
dataDict, granularity = retVal["Value"]
self.stripDataField(dataDict, 0)
dataDict = self._fillWithZero(granularity, reportRequest["startTime"],
reportRequest["endTime"], dataDict)
dataDict = self._accumulate(granularity, reportRequest["startTime"],
reportRequest["endTime"], dataDict)
baseDataDict, graphDataDict, maxValue, unitName = self._findSuitableUnit(
dataDict, self._getAccumulationMaxValue(dataDict), "jobs")
return S_OK({
"data": baseDataDict,
"graphDataDict": graphDataDict,
"granularity": granularity,
"unit": unitName
})
def _plotCumulativeNumberOfJobs(self, reportRequest, plotInfo, filename):
metadata = {
"title": "Cumulative Jobs by %s" % reportRequest["grouping"],
"starttime": reportRequest["startTime"],
"endtime": reportRequest["endTime"],
"span": plotInfo["granularity"],
"ylabel": plotInfo["unit"],
"sort_labels": "last_value",
}
return self._generateCumulativePlot(filename,
plotInfo["graphDataDict"],
metadata)
def _reportNumberOfJobs(self, reportRequest):
selectFields = (
self._getSelectStringForGrouping(reportRequest["groupingFields"]) +
", %s, %s, SUM(%s)",
reportRequest["groupingFields"][1] +
["startTime", "bucketLength", "Jobs"],
)
retVal = self._getTimedData(
reportRequest["startTime"],
reportRequest["endTime"],
selectFields,
reportRequest["condDict"],
reportRequest["groupingFields"],
{},
)
if not retVal["OK"]:
return retVal
dataDict, granularity = retVal["Value"]
self.stripDataField(dataDict, 0)
dataDict, maxValue = self._divideByFactor(dataDict, granularity)
dataDict = self._fillWithZero(granularity, reportRequest["startTime"],
reportRequest["endTime"], dataDict)
baseDataDict, graphDataDict, maxValue, unitName = self._findSuitableRateUnit(
dataDict, self._getAccumulationMaxValue(dataDict), "jobs")
return S_OK({
"data": baseDataDict,
"graphDataDict": graphDataDict,
"granularity": granularity,
"unit": unitName
})
def _plotNumberOfJobs(self, reportRequest, plotInfo, filename):
metadata = {
"title": "Jobs by %s" % reportRequest["grouping"],
"starttime": reportRequest["startTime"],
"endtime": reportRequest["endTime"],
"span": plotInfo["granularity"],
"ylabel": plotInfo["unit"],
}
return self._generateTimedStackedBarPlot(filename,
plotInfo["graphDataDict"],
metadata)
def _reportCumulativeNumberOfPilots(self, reportRequest):
selectFields = (
self._getSelectStringForGrouping(reportRequest["groupingFields"]) +
", %s, %s, SUM(%s)",
reportRequest["groupingFields"][1] +
["startTime", "bucketLength", "entriesInBucket"],
)
retVal = self._getTimedData(
reportRequest["startTime"],
reportRequest["endTime"],
selectFields,
reportRequest["condDict"],
reportRequest["groupingFields"],
{},
)
if not retVal["OK"]:
return retVal
dataDict, granularity = retVal["Value"]
self.stripDataField(dataDict, 0)
dataDict = self._fillWithZero(granularity, reportRequest["startTime"],
reportRequest["endTime"], dataDict)
dataDict = self._accumulate(granularity, reportRequest["startTime"],
reportRequest["endTime"], dataDict)
baseDataDict, graphDataDict, maxValue, unitName = self._findSuitableUnit(
dataDict, self._getAccumulationMaxValue(dataDict), "jobs")
return S_OK({
"data": baseDataDict,
"graphDataDict": graphDataDict,
"granularity": granularity,
"unit": unitName
})
def _plotCumulativeNumberOfPilots(self, reportRequest, plotInfo, filename):
metadata = {
"title": "Cumulative Pilots by %s" % reportRequest["grouping"],
"starttime": reportRequest["startTime"],
"endtime": reportRequest["endTime"],
"span": plotInfo["granularity"],
"ylabel": plotInfo["unit"].replace("job", "pilot"),
"sort_labels": "last_value",
}
return self._generateCumulativePlot(filename,
plotInfo["graphDataDict"],
metadata)
def _reportNumberOfPilots(self, reportRequest):
selectFields = (
self._getSelectStringForGrouping(reportRequest["groupingFields"]) +
", %s, %s, SUM(%s)",
reportRequest["groupingFields"][1] +
["startTime", "bucketLength", "entriesInBucket"],
)
retVal = self._getTimedData(
reportRequest["startTime"],
reportRequest["endTime"],
selectFields,
reportRequest["condDict"],
reportRequest["groupingFields"],
{},
)
if not retVal["OK"]:
return retVal
dataDict, granularity = retVal["Value"]
self.stripDataField(dataDict, 0)
dataDict, maxValue = self._divideByFactor(dataDict, granularity)
dataDict = self._fillWithZero(granularity, reportRequest["startTime"],
reportRequest["endTime"], dataDict)
baseDataDict, graphDataDict, maxValue, unitName = self._findSuitableRateUnit(
dataDict, self._getAccumulationMaxValue(dataDict), "jobs")
return S_OK({
"data": baseDataDict,
"graphDataDict": graphDataDict,
"granularity": granularity,
"unit": unitName
})
def _plotNumberOfPilots(self, reportRequest, plotInfo, filename):
metadata = {
"title": "Pilots by %s" % reportRequest["grouping"],
"starttime": reportRequest["startTime"],
"endtime": reportRequest["endTime"],
"span": plotInfo["granularity"],
"ylabel": plotInfo["unit"].replace("job", "pilot"),
}
return self._generateTimedStackedBarPlot(filename,
plotInfo["graphDataDict"],
metadata)
def _reportJobsPerPilot(self, reportRequest):
selectFields = (
self._getSelectStringForGrouping(reportRequest["groupingFields"]) +
", %s, %s, SUM(%s), SUM(%s)",
reportRequest["groupingFields"][1] +
["startTime", "bucketLength", "Jobs", "entriesInBucket"],
)
retVal = self._getTimedData(
reportRequest["startTime"],
reportRequest["endTime"],
selectFields,
reportRequest["condDict"],
reportRequest["groupingFields"],
{
"checkNone": True,
"convertToGranularity": "sum",
"calculateProportionalGauges": False,
"consolidationFunction": self._averageConsolidation,
},
)
if not retVal["OK"]:
return retVal
dataDict, granularity = retVal["Value"]
self.stripDataField(dataDict, 0)
dataDict = self._fillWithZero(granularity, reportRequest["startTime"],
reportRequest["endTime"], dataDict)
return S_OK({"data": dataDict, "granularity": granularity})
def _plotJobsPerPilot(self, reportRequest, plotInfo, filename):
metadata = {
"title":
"Jobs per pilot by %s" % reportRequest["grouping"],
"starttime":
reportRequest["startTime"],
"endtime":
reportRequest["endTime"],
"span":
plotInfo["granularity"],
"ylabel":
"jobs/pilot",
"normalization":
max(x for y in plotInfo["data"].values() for x in y.values()),
}
return self._generateQualityPlot(filename, plotInfo["data"], metadata)
def _reportTotalNumberOfPilots(self, reportRequest):
selectFields = (
self._getSelectStringForGrouping(reportRequest["groupingFields"]) +
", SUM(%s)",
reportRequest["groupingFields"][1] + ["entriesInBucket"],
)
retVal = self._getSummaryData(
reportRequest["startTime"],
reportRequest["endTime"],
selectFields,
reportRequest["condDict"],
reportRequest["groupingFields"],
{},
)
if not retVal["OK"]:
return retVal
dataDict = retVal["Value"]
return S_OK({"data": dataDict})
def _plotTotalNumberOfPilots(self, reportRequest, plotInfo, filename):
metadata = {
"title":
"Total Number of Pilots by %s" % reportRequest["grouping"],
"ylabel": "Pilots",
"starttime": reportRequest["startTime"],
"endtime": reportRequest["endTime"],
}
return self._generatePiePlot(filename, plotInfo["data"], metadata)
def sendPilotAccounting(self, pilotDict):
""" Send pilot accounting record
"""
for pRef in pilotDict:
self.log.verbose("Preparing accounting record for pilot %s" % pRef)
pA = PilotAccounting()
pA.setEndTime(pilotDict[pRef]["LastUpdateTime"])
pA.setStartTime(pilotDict[pRef]["SubmissionTime"])
retVal = CS.getUsernameForDN(pilotDict[pRef]["OwnerDN"])
if not retVal["OK"]:
userName = "******"
self.log.error("Can't determine username for dn:", pilotDict[pRef]["OwnerDN"])
else:
userName = retVal["Value"]
pA.setValueByKey("User", userName)
pA.setValueByKey("UserGroup", pilotDict[pRef]["OwnerGroup"])
result = getSiteForCE(pilotDict[pRef]["DestinationSite"])
if result["OK"] and result["Value"].strip():
pA.setValueByKey("Site", result["Value"].strip())
else:
pA.setValueByKey("Site", "Unknown")
pA.setValueByKey("GridCE", pilotDict[pRef]["DestinationSite"])
pA.setValueByKey("GridMiddleware", pilotDict[pRef]["GridType"])
pA.setValueByKey("GridResourceBroker", pilotDict[pRef]["Broker"])
pA.setValueByKey("GridStatus", pilotDict[pRef]["Status"])
if not "Jobs" in pilotDict[pRef]:
pA.setValueByKey("Jobs", 0)
else:
pA.setValueByKey("Jobs", len(pilotDict[pRef]["Jobs"]))
self.log.info("Adding accounting record for pilot %s" % pilotDict[pRef]["PilotID"])
retVal = gDataStoreClient.addRegister(pA)
if not retVal["OK"]:
self.log.error("Failed to send accounting info for pilot ", pRef)
else:
# Set up AccountingSent flag
result = pilotAgentsDB.setAccountingFlag(pRef)
if not result["OK"]:
self.log.error("Failed to set accounting flag for pilot ", pRef)
self.log.info("Committing accounting records for %d pilots" % len(pilotDict))
result = gDataStoreClient.commit()
if result["OK"]:
for pRef in pilotDict:
self.log.verbose("Setting AccountingSent flag for pilot %s" % pRef)
result = pilotAgentsDB.setAccountingFlag(pRef)
if not result["OK"]:
self.log.error("Failed to set accounting flag for pilot ", pRef)
else:
return result
return S_OK()
