def sendAccountingRecord(self, job, msg, classAdJob): """ Send and accounting record for the failed job """ accountingReport = AccountingJob() accountingReport.setStartTime() accountingReport.setEndTime() owner = classAdJob.getAttributeString("Owner") userGroup = classAdJob.getAttributeString("OwnerGroup") jobGroup = classAdJob.getAttributeString("JobGroup") jobType = classAdJob.getAttributeString("JobType") jobClass = "unknown" if classAdJob.lookupAttribute("JobSplitType"): jobClass = classAdJob.getAttributeString("JobSplitType") inputData = [] processingType = "unknown" if classAdJob.lookupAttribute("ProcessingType"): processingType = classAdJob.getAttributeString("ProcessingType") if classAdJob.lookupAttribute("InputData"): inputData = classAdJob.getListFromExpression("InputData") inputDataFiles = len(inputData) outputData = [] if classAdJob.lookupAttribute("OutputData"): outputData = classAdJob.getListFromExpression("OutputData") outputDataFiles = len(outputData) acData = { "User": owner, "UserGroup": userGroup, "JobGroup": jobGroup, "JobType": jobType, "JobClass": jobClass, "ProcessingType": processingType, "FinalMajorStatus": self.failedStatus, "FinalMinorStatus": msg, "CPUTime": 0.0, "NormCPUTime": 0.0, "ExecTime": 0.0, "InputDataSize": 0.0, "OutputDataSize": 0.0, "InputDataFiles": inputDataFiles, "OutputDataFiles": outputDataFiles, "DiskSpace": 0.0, "InputSandBoxSize": 0.0, "OutputSandBoxSize": 0.0, "ProcessedEvents": 0.0, } self.log.debug("Accounting Report is:") self.log.debug(acData) accountingReport.setValuesFromDict(acData) return accountingReport.commit()
def sendAccountingRecord(self, job, msg, classAdJob): """ Send and accounting record for the failed job """ accountingReport = AccountingJob() accountingReport.setStartTime() accountingReport.setEndTime() owner = classAdJob.getAttributeString('Owner') userGroup = classAdJob.getAttributeString('OwnerGroup') jobGroup = classAdJob.getAttributeString('JobGroup') jobType = classAdJob.getAttributeString('JobType') jobClass = 'unknown' if classAdJob.lookupAttribute('JobSplitType'): jobClass = classAdJob.getAttributeString('JobSplitType') inputData = [] processingType = 'unknown' if classAdJob.lookupAttribute('ProcessingType'): processingType = classAdJob.getAttributeString('ProcessingType') if classAdJob.lookupAttribute('InputData'): inputData = classAdJob.getListFromExpression('InputData') inputDataFiles = len(inputData) outputData = [] if classAdJob.lookupAttribute('OutputData'): outputData = classAdJob.getListFromExpression('OutputData') outputDataFiles = len(outputData) acData = { 'User': owner, 'UserGroup': userGroup, 'JobGroup': jobGroup, 'JobType': jobType, 'JobClass': jobClass, 'ProcessingType': processingType, 'FinalMajorStatus': self.failedStatus, 'FinalMinorStatus': msg, 'CPUTime': 0.0, 'NormCPUTime': 0.0, 'ExecTime': 0.0, 'InputDataSize': 0.0, 'OutputDataSize': 0.0, 'InputDataFiles': inputDataFiles, 'OutputDataFiles': outputDataFiles, 'DiskSpace': 0.0, 'InputSandBoxSize': 0.0, 'OutputSandBoxSize': 0.0, 'ProcessedEvents': 0.0 } self.log.debug('Accounting Report is:') self.log.debug(acData) accountingReport.setValuesFromDict(acData) return accountingReport.commit()
def sendAccountingRecord(self, job, msg, classAdJob): """ Send and accounting record for the failed job """ accountingReport = AccountingJob() accountingReport.setStartTime() accountingReport.setEndTime() owner = classAdJob.getAttributeString('Owner') userGroup = classAdJob.getAttributeString('OwnerGroup') jobGroup = classAdJob.getAttributeString('JobGroup') jobType = classAdJob.getAttributeString('JobType') jobClass = 'unknown' if classAdJob.lookupAttribute('JobSplitType'): jobClass = classAdJob.getAttributeString('JobSplitType') inputData = [] processingType = 'unknown' if classAdJob.lookupAttribute('ProcessingType'): processingType = classAdJob.getAttributeString('ProcessingType') if classAdJob.lookupAttribute('InputData'): inputData = classAdJob.getListFromExpression('InputData') inputDataFiles = len(inputData) outputData = [] if classAdJob.lookupAttribute('OutputData'): outputData = classAdJob.getListFromExpression('OutputData') outputDataFiles = len(outputData) acData = { 'User': owner, 'UserGroup': userGroup, 'JobGroup': jobGroup, 'JobType': jobType, 'JobClass': jobClass, 'ProcessingType': processingType, 'FinalMajorStatus': self.failedStatus, 'FinalMinorStatus': msg, 'CPUTime': 0.0, 'NormCPUTime': 0.0, 'ExecTime': 0.0, 'InputDataSize': 0.0, 'OutputDataSize': 0.0, 'InputDataFiles': inputDataFiles, 'OutputDataFiles': outputDataFiles, 'DiskSpace': 0.0, 'InputSandBoxSize': 0.0, 'OutputSandBoxSize': 0.0, 'ProcessedEvents': 0.0 } self.log.verbose('Accounting Report is:') self.log.verbose(acData) accountingReport.setValuesFromDict(acData) return accountingReport.commit()
def sendJobAccounting(self, dataFromBDSoft, jobId): accountingReport = AccountingJob() accountingReport.setStartTime() result = jobDB.getJobAttributes(jobId) getting = result["Value"] if dataFromBDSoft["CPUTime"] == 0: cpuTime = 0 if getting["EndExecTime"] != "None": epoch = datetime(1970, 1, 1) td = datetime.strptime(getting["EndExecTime"], "%Y-%m-%d %H:%M:%S") - epoch EndExecTime = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / 1e6 td = datetime.strptime(getting["SubmissionTime"], "%Y-%m-%d %H:%M:%S") - epoch SubmissionTime = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / 1e6 cpuTime = EndExecTime - SubmissionTime else: cpuTime = dataFromBDSoft["CPUTime"] / 1000 acData = { "User": getting["Owner"], "UserGroup": getting["OwnerGroup"], "JobGroup": "cesga", "JobType": "User", "JobClass": "unknown", "ProcessingType": "unknown", "FinalMajorStatus": getting["Status"], "FinalMinorStatus": getting["MinorStatus"], "CPUTime": cpuTime, "Site": getting["Site"], # Based on the factor to convert raw CPU to Normalized units (based on the CPU Model) "NormCPUTime": 0, "ExecTime": cpuTime, "InputDataSize": dataFromBDSoft["InputDataSize"], "OutputDataSize": dataFromBDSoft["OutputDataSize"], "InputDataFiles": dataFromBDSoft["InputDataFiles"], "OutputDataFiles": len(self.fileList), "DiskSpace": 0, "InputSandBoxSize": 0, "OutputSandBoxSize": self.outputSandboxSize, "ProcessedEvents": 0, } accountingReport.setEndTime() accountingReport.setValuesFromDict(acData) self.log.debug("Info for accounting: ", acData) result = accountingReport.commit() self.log.debug("Accounting insertion: ", result) return result
class JobPlotter(BaseReporter): _typeName = "Job" _typeKeyFields = [dF[0] for dF in Job().definitionKeyFields] def _translateGrouping(self, grouping): if grouping == "Country": sqlRepr = 'upper( substring( %s, locate( ".", %s, length( %s ) - 4 ) + 1 ) )' return (sqlRepr, ['Site', 'Site', 'Site'], sqlRepr) elif grouping == "Grid": return ('substring_index( %s, ".", 1 )', ['Site']) else: return ("%s", [grouping]) _reportCPUEfficiencyName = "CPU efficiency" def _reportCPUEfficiency(self, reportRequest): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", %s, %s, SUM(%s), SUM(%s)", reportRequest['groupingFields'][1] + ['startTime', 'bucketLength', 'CPUTime', 'ExecTime']) retVal = self._getTimedData( reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], { 'checkNone': True, 'convertToGranularity': 'sum', 'calculateProportionalGauges': False, 'consolidationFunction': self._efficiencyConsolidation }) if not retVal['OK']: return retVal dataDict, granularity = retVal['Value'] self.stripDataField(dataDict, 0) if len(dataDict) > 1: # Get the total for the plot selectFields = ("'Total', %s, %s, SUM(%s),SUM(%s)", [ 'startTime', 'bucketLength', 'CPUTime', 'ExecTime' ]) retVal = self._getTimedData( reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], { 'scheckNone': True, 'convertToGranularity': 'sum', 'calculateProportionalGauges': False, 'consolidationFunction': self._efficiencyConsolidation }) if not retVal['OK']: return retVal totalDict = retVal['Value'][0] self.stripDataField(totalDict, 0) for key in totalDict: dataDict[key] = totalDict[key] return S_OK({'data': dataDict, 'granularity': granularity}) def _plotCPUEfficiency(self, reportRequest, plotInfo, filename): metadata = { 'title': 'Job CPU efficiency by %s' % reportRequest['grouping'], 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'], 'span': plotInfo['granularity'] } return self._generateQualityPlot(filename, plotInfo['data'], metadata) _reportCPUUsedName = "Cumulative CPU time" def _reportCPUUsed(self, reportRequest): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", %s, %s, SUM(%s)", reportRequest['groupingFields'][1] + ['startTime', 'bucketLength', 'CPUTime']) retVal = self._getTimedData(reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], {}) if not retVal['OK']: return retVal dataDict, granularity = retVal['Value'] self.stripDataField(dataDict, 0) dataDict = self._fillWithZero(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) dataDict = self._accumulate(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) baseDataDict, graphDataDict, _maxValue, unitName = self._findSuitableUnit( dataDict, self._getAccumulationMaxValue(dataDict), "time") return S_OK({ 'data': baseDataDict, 'graphDataDict': graphDataDict, 'granularity': granularity, 'unit': unitName }) def _plotCPUUsed(self, reportRequest, plotInfo, filename): metadata = { 'title': 'CPU used by %s' % reportRequest['grouping'], 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'], 'span': plotInfo['granularity'], 'ylabel': plotInfo['unit'], 'sort_labels': 'max_value' } return self._generateCumulativePlot(filename, plotInfo['graphDataDict'], metadata) _reportCPUUsageName = "CPU time" def _reportCPUUsage(self, reportRequest): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", %s, %s, SUM(%s)", reportRequest['groupingFields'][1] + ['startTime', 'bucketLength', 'CPUTime']) retVal = self._getTimedData(reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], {}) if not retVal['OK']: return retVal dataDict, granularity = retVal['Value'] self.stripDataField(dataDict, 0) dataDict, _maxValue = self._divideByFactor(dataDict, granularity) dataDict = self._fillWithZero(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) baseDataDict, graphDataDict, _maxValue, unitName = self._findSuitableRateUnit( dataDict, self._getAccumulationMaxValue(dataDict), "time") return S_OK({ 'data': baseDataDict, 'graphDataDict': graphDataDict, 'granularity': granularity, 'unit': unitName }) def _plotCPUUsage(self, reportRequest, plotInfo, filename): metadata = { 'title': 'CPU usage by %s' % reportRequest['grouping'], 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'], 'span': plotInfo['granularity'], 'ylabel': plotInfo['unit'] } return self._generateStackedLinePlot(filename, plotInfo['graphDataDict'], metadata) _reportNormCPUUsedName = "Cumulative Normalized CPU" def _reportNormCPUUsed(self, reportRequest): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", %s, %s, SUM(%s)", reportRequest['groupingFields'][1] + ['startTime', 'bucketLength', 'NormCPUTime']) retVal = self._getTimedData(reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], {}) if not retVal['OK']: return retVal dataDict, granularity = retVal['Value'] self.stripDataField(dataDict, 0) dataDict = self._fillWithZero(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) dataDict = self._accumulate(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) baseDataDict, graphDataDict, _maxValue, unitName = self._findSuitableUnit( dataDict, self._getAccumulationMaxValue(dataDict), "cpupower") return S_OK({ 'data': baseDataDict, 'graphDataDict': graphDataDict, 'granularity': granularity, 'unit': unitName }) def _plotNormCPUUsed(self, reportRequest, plotInfo, filename): metadata = { 'title': 'Normalized CPU used by %s' % reportRequest['grouping'], 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'], 'span': plotInfo['granularity'], 'ylabel': plotInfo['unit'], 'sort_labels': 'max_value' } return self._generateCumulativePlot(filename, plotInfo['graphDataDict'], metadata) _reportNormCPUUsageName = "Normalized CPU power" def _reportNormCPUUsage(self, reportRequest): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", %s, %s, SUM(%s)", reportRequest['groupingFields'][1] + ['startTime', 'bucketLength', 'NormCPUTime']) retVal = self._getTimedData(reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], {}) if not retVal['OK']: return retVal dataDict, granularity = retVal['Value'] self.stripDataField(dataDict, 0) dataDict, _maxValue = self._divideByFactor(dataDict, granularity) dataDict = self._fillWithZero(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) baseDataDict, graphDataDict, _maxValue, unitName = self._findSuitableRateUnit( dataDict, self._getAccumulationMaxValue(dataDict), "cpupower") return S_OK({ 'data': baseDataDict, 'graphDataDict': graphDataDict, 'granularity': granularity, 'unit': unitName }) def _plotNormCPUUsage(self, reportRequest, plotInfo, filename): metadata = { 'title': 'Normalized CPU usage by %s' % reportRequest['grouping'], 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'], 'span': plotInfo['granularity'], 'ylabel': plotInfo['unit'] } return self._generateStackedLinePlot(filename, plotInfo['graphDataDict'], metadata) _reportWallTimeName = "Wall time" def _reportWallTime(self, reportRequest): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", %s, %s, SUM(%s)", reportRequest['groupingFields'][1] + ['startTime', 'bucketLength', 'ExecTime']) retVal = self._getTimedData(reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], {}) if not retVal['OK']: return retVal dataDict, granularity = retVal['Value'] self.stripDataField(dataDict, 0) dataDict, _maxValue = self._divideByFactor(dataDict, granularity) dataDict = self._fillWithZero(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) baseDataDict, graphDataDict, _maxValue, unitName = self._findSuitableRateUnit( dataDict, self._getAccumulationMaxValue(dataDict), "time") return S_OK({ 'data': baseDataDict, 'graphDataDict': graphDataDict, 'granularity': granularity, 'unit': unitName }) def _plotWallTime(self, reportRequest, plotInfo, filename): metadata = { 'title': 'Wall Time by %s' % reportRequest['grouping'], 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'], 'span': plotInfo['granularity'], 'ylabel': plotInfo['unit'] } return self._generateStackedLinePlot(filename, plotInfo['graphDataDict'], metadata) _reportRunningJobsName = "Running jobs" def _reportRunningJobs(self, reportRequest): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", %s, %s, SUM(%s)", reportRequest['groupingFields'][1] + ['startTime', 'bucketLength', 'ExecTime']) retVal = self._getTimedData(reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], {}) if not retVal['OK']: return retVal dataDict, granularity = retVal['Value'] self.stripDataField(dataDict, 0) dataDict, _maxValue = self._divideByFactor(dataDict, granularity) dataDict = self._fillWithZero(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) baseDataDict, graphDataDict, _maxValue, unitName = self._findSuitableUnit( dataDict, self._getAccumulationMaxValue(dataDict), "jobs") return S_OK({ 'data': baseDataDict, 'graphDataDict': graphDataDict, 'granularity': granularity, 'unit': unitName }) def _plotRunningJobs(self, reportRequest, plotInfo, filename): metadata = { 'title': 'Running jobs by %s' % reportRequest['grouping'], 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'], 'span': plotInfo['granularity'], 'ylabel': plotInfo['unit'] } return self._generateStackedLinePlot(filename, plotInfo['graphDataDict'], metadata) _reportTotalCPUUsedName = "Pie plot of CPU used" def _reportTotalCPUUsed(self, reportRequest): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", SUM(%s)/86400", reportRequest['groupingFields'][1] + ['CPUTime']) retVal = self._getSummaryData(reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], {}) if not retVal['OK']: return retVal dataDict = retVal['Value'] return S_OK({'data': dataDict}) def _plotTotalCPUUsed(self, reportRequest, plotInfo, filename): metadata = { 'title': 'CPU days used by %s' % reportRequest['grouping'], 'ylabel': 'CPU days', 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'] } return self._generatePiePlot(filename, plotInfo['data'], metadata) _reportAccumulatedWallTimeName = "Cumulative wall time" def _reportAccumulatedWallTime(self, reportRequest): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", %s, %s, SUM(%s)", reportRequest['groupingFields'][1] + ['startTime', 'bucketLength', 'ExecTime']) retVal = self._getTimedData(reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], {}) if not retVal['OK']: return retVal dataDict, granularity = retVal['Value'] self.stripDataField(dataDict, 0) dataDict = self._fillWithZero(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) dataDict = self._accumulate(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) baseDataDict, graphDataDict, _maxValue, unitName = self._findSuitableUnit( dataDict, self._getAccumulationMaxValue(dataDict), "time") return S_OK({ 'data': baseDataDict, 'graphDataDict': graphDataDict, 'granularity': granularity, 'unit': unitName }) def _plotAccumulatedWallTime(self, reportRequest, plotInfo, filename): metadata = { 'title': 'Cumulative wall time by %s' % reportRequest['grouping'], 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'], 'span': plotInfo['granularity'], 'ylabel': plotInfo['unit'], 'sort_labels': 'max_value' } return self._generateCumulativePlot(filename, plotInfo['graphDataDict'], metadata) _reportTotalWallTimeName = "Pie plot of wall time usage" def _reportTotalWallTime(self, reportRequest): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", SUM(%s)/86400", reportRequest['groupingFields'][1] + ['ExecTime']) retVal = self._getSummaryData(reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], {}) if not retVal['OK']: return retVal dataDict = retVal['Value'] return S_OK({'data': dataDict}) def _plotTotalWallTime(self, reportRequest, plotInfo, filename): metadata = { 'title': 'Wall time days used by %s' % reportRequest['grouping'], 'ylabel': 'CPU days', 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'] } return self._generatePiePlot(filename, plotInfo['data'], metadata) ## # Jobs ## _reportCumulativeNumberOfJobsName = "Cumulative executed jobs" def _reportCumulativeNumberOfJobs(self, reportRequest): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", %s, %s, SUM(%s)", reportRequest['groupingFields'][1] + ['startTime', 'bucketLength', 'entriesInBucket']) retVal = self._getTimedData(reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], {}) if not retVal['OK']: return retVal dataDict, granularity = retVal['Value'] self.stripDataField(dataDict, 0) dataDict = self._fillWithZero(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) dataDict = self._accumulate(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) baseDataDict, graphDataDict, _maxValue, unitName = self._findSuitableUnit( dataDict, self._getAccumulationMaxValue(dataDict), "jobs") return S_OK({ 'data': baseDataDict, 'graphDataDict': graphDataDict, 'granularity': granularity, 'unit': unitName }) def _plotCumulativeNumberOfJobs(self, reportRequest, plotInfo, filename): metadata = { 'title': 'Cumulative Jobs by %s' % reportRequest['grouping'], 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'], 'span': plotInfo['granularity'], 'ylabel': plotInfo['unit'], 'sort_labels': 'max_value' } return self._generateCumulativePlot(filename, plotInfo['graphDataDict'], metadata) _reportNumberOfJobsName = "Job execution rate" def _reportNumberOfJobs(self, reportRequest): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", %s, %s, SUM(%s)", reportRequest['groupingFields'][1] + ['startTime', 'bucketLength', 'entriesInBucket']) retVal = self._getTimedData(reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], {}) if not retVal['OK']: return retVal dataDict, granularity = retVal['Value'] self.stripDataField(dataDict, 0) dataDict, _maxValue = self._divideByFactor(dataDict, granularity) dataDict = self._fillWithZero(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) baseDataDict, graphDataDict, _maxValue, unitName = self._findSuitableRateUnit( dataDict, self._getAccumulationMaxValue(dataDict), "jobs") return S_OK({ 'data': baseDataDict, 'graphDataDict': graphDataDict, 'granularity': granularity, 'unit': unitName }) def _plotNumberOfJobs(self, reportRequest, plotInfo, filename): metadata = { 'title': 'Jobs by %s' % reportRequest['grouping'], 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'], 'span': plotInfo['granularity'], 'ylabel': plotInfo['unit'] } return self._generateStackedLinePlot(filename, plotInfo['graphDataDict'], metadata) _reportTotalNumberOfJobsName = "Pie plot of executed jobs" def _reportTotalNumberOfJobs(self, reportRequest): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", SUM(%s)", reportRequest['groupingFields'][1] + ['entriesInBucket']) retVal = self._getSummaryData(reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], {}) if not retVal['OK']: return retVal dataDict = retVal['Value'] return S_OK({'data': dataDict}) def _plotTotalNumberOfJobs(self, reportRequest, plotInfo, filename): metadata = { 'title': 'Total Number of Jobs by %s' % reportRequest['grouping'], 'ylabel': 'Jobs', 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'] } return self._generatePiePlot(filename, plotInfo['data'], metadata) ## # Proc bw ## _reportProcessingBandwidthName = "Processing bandwidth" def _reportProcessingBandwidth(self, reportRequest): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", %s, %s, SUM((%s)/(%s))/SUM(%s)", reportRequest['groupingFields'][1] + [ 'startTime', 'bucketLength', 'InputDataSize', 'CPUTime', 'entriesInBucket' ]) retVal = self._getTimedData(reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], {}) if not retVal['OK']: return retVal dataDict, granularity = retVal['Value'] self.stripDataField(dataDict, 0) dataDict, _maxValue = self._divideByFactor(dataDict, granularity) dataDict = self._fillWithZero(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) baseDataDict, graphDataDict, _maxValue, unitName = self._findSuitableRateUnit( dataDict, self._getAccumulationMaxValue(dataDict), "bytes") return S_OK({ 'data': baseDataDict, 'graphDataDict': graphDataDict, 'granularity': granularity, 'unit': unitName }) def _plotProcessingBandwidth(self, reportRequest, plotInfo, filename): metadata = { 'title': 'Processing Bandwidth by %s' % reportRequest['grouping'], 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'], 'span': plotInfo['granularity'], 'ylabel': plotInfo['unit'] } return self._generateStackedLinePlot(filename, plotInfo['graphDataDict'], metadata) ## # Data sizes ## _reportInputSandboxSizeName = "Input sandbox" def _reportInputSandboxSize(self, reportRequest): return self.__reportFieldSizeinMB( reportRequest, ("InputSandBoxSize", "Input sand box size")) _reportOutputSandboxSizeName = "Ouput sandbox" def _reportOutputSandboxSize(self, reportRequest): return self.__reportFieldSizeinMB( reportRequest, ("OutputSandBoxSize", "Output sand box size")) _reportDiskSpaceSizeName = "Disk space" def _reportDiskSpaceSize(self, reportRequest): return self.__reportFieldSizeinMB(reportRequest, ("DiskSpace", "Used disk space")) _reportInputDataSizeName = "Input data" def _reportInputDataSize(self, reportRequest): return self.__reportFieldSizeinMB(reportRequest, ("InputDataSize", "Input data")) _reportOutputDataSizeName = "Output data" def _reportOutputDataSize(self, reportRequest): return self.__reportFieldSizeinMB(reportRequest, ("OutputDataSize", "Output data")) def __reportFieldSizeinMB(self, reportRequest, fieldTuple): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", %s, %s, SUM(%s)", reportRequest['groupingFields'][1] + ['startTime', 'bucketLength', fieldTuple[0]]) retVal = self._getTimedData(reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], {}) if not retVal['OK']: return retVal dataDict, granularity = retVal['Value'] self.stripDataField(dataDict, 0) dataDict, _maxValue = self._divideByFactor(dataDict, granularity) dataDict = self._fillWithZero(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) baseDataDict, graphDataDict, _maxValue, unitName = self._findSuitableRateUnit( dataDict, self._getAccumulationMaxValue(dataDict), "bytes") return S_OK({ 'data': baseDataDict, 'graphDataDict': graphDataDict, 'granularity': granularity, 'unit': unitName }) def _plotInputSandboxSize(self, reportRequest, plotInfo, filename): return self.__plotFieldSizeinMB( reportRequest, plotInfo, filename, ("InputSandBoxSize", "Input sand box size")) def _plotOutputSandboxSize(self, reportRequest, plotInfo, filename): return self.__plotFieldSizeinMB( reportRequest, plotInfo, filename, ("OutputSandBoxSize", "Output sand box size")) def _plotDiskSpaceSize(self, reportRequest, plotInfo, filename): return self.__plotFieldSizeinMB(reportRequest, plotInfo, filename, ("DiskSpace", "Used disk space")) def _plotInputDataSize(self, reportRequest, plotInfo, filename): return self.__plotFieldSizeinMB(reportRequest, plotInfo, filename, ("InputDataSize", "Input data")) def _plotOutputDataSize(self, reportRequest, plotInfo, filename): return self.__plotFieldSizeinMB(reportRequest, plotInfo, filename, ("OutputDataSize", "Output data")) def __plotFieldSizeinMB(self, reportRequest, plotInfo, filename, fieldTuple): metadata = { 'title': '%s by %s' % (fieldTuple[1], reportRequest['grouping']), 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'], 'span': plotInfo['granularity'], 'ylabel': plotInfo['unit'] } return self._generateStackedLinePlot(filename, plotInfo['graphDataDict'], metadata) ## # Cumulative data sizes ## _reportCumulativeInputSandboxSizeName = "Cumulative Input sandbox" def _reportCumulativeInputSandboxSize(self, reportRequest): return self.__reportCumulativeFieldSizeinMB( reportRequest, ("InputSandBoxSize", "Input sand box size")) _reportCumulativeOutputSandboxSizeName = "Cumulative Ouput sandbox" def _reportCumulativeOutputSandboxSize(self, reportRequest): return self.__reportCumulativeFieldSizeinMB( reportRequest, ("OutputSandBoxSize", "Output sand box size")) _reportCumulativeDiskSpaceSizeName = "Cumulative Disk space" def _reportCumulativeDiskSpaceSize(self, reportRequest): return self.__reportCumulativeFieldSizeinMB( reportRequest, ("DiskSpace", "Used disk space")) _reportCumulativeInputDataSizeName = "Cumulative Input data" def _reportCumulativeInputDataSize(self, reportRequest): return self.__reportCumulativeFieldSizeinMB( reportRequest, ("InputDataSize", "Input data")) _reportCumulativeOutputDataSizeName = "Cumulative Output data" def _reportCumulativeOutputDataSize(self, reportRequest): return self.__reportCumulativeFieldSizeinMB( reportRequest, ("OutputDataSize", "Output data")) def __reportCumulativeFieldSizeinMB(self, reportRequest, fieldTuple): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", %s, %s, SUM(%s)", reportRequest['groupingFields'][1] + ['startTime', 'bucketLength', fieldTuple[0]]) retVal = self._getTimedData(reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], {}) if not retVal['OK']: return retVal dataDict, granularity = retVal['Value'] self.stripDataField(dataDict, 0) dataDict = self._fillWithZero(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) dataDict = self._accumulate(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) baseDataDict, graphDataDict, _maxValue, unitName = self._findSuitableUnit( dataDict, self._getAccumulationMaxValue(dataDict), "bytes") return S_OK({ 'data': baseDataDict, 'graphDataDict': graphDataDict, 'granularity': granularity, 'unit': unitName }) def _plotCumulativeInputSandboxSize(self, reportRequest, plotInfo, filename): return self.__plotCumulativeFieldSizeinMB( reportRequest, plotInfo, filename, ("InputSandBoxSize", "Input sand box size")) def _plotCumulativeOutputSandboxSize(self, reportRequest, plotInfo, filename): return self.__plotCumulativeFieldSizeinMB( reportRequest, plotInfo, filename, ("OutputSandBoxSize", "Output sand box size")) def _plotCumulativeDiskSpaceSize(self, reportRequest, plotInfo, filename): return self.__plotCumulativeFieldSizeinMB( reportRequest, plotInfo, filename, ("DiskSpace", "Used disk space")) def _plotCumulativeInputDataSize(self, reportRequest, plotInfo, filename): return self.__plotCumulativeFieldSizeinMB( reportRequest, plotInfo, filename, ("InputDataSize", "Input data")) def _plotCumulativeOutputDataSize(self, reportRequest, plotInfo, filename): return self.__plotCumulativeFieldSizeinMB( reportRequest, plotInfo, filename, ("OutputDataSize", "Output data")) def __plotCumulativeFieldSizeinMB(self, reportRequest, plotInfo, filename, fieldTuple): metadata = { 'title': 'Cumulative %s by %s' % (fieldTuple[1], reportRequest['grouping']), 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'], 'span': plotInfo['granularity'], 'ylabel': plotInfo['unit'] } return self._generateCumulativePlot(filename, plotInfo['graphDataDict'], metadata) ## # Input/Ouput data files ## _reportInputDataFilesName = "Input data files" def _reportInputDataFiles(self, reportRequest): return self.__reportDataFiles(reportRequest, ("InputDataFiles", "Input files")) _reportOuputDataFilesName = "Output data files" def _reportOuputDataFiles(self, reportRequest): return self.__reportDataFiles(reportRequest, ("OutputDataFiles", "Output files")) def __reportDataFiles(self, reportRequest, fieldTuple): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", %s, %s, SUM(%s)", reportRequest['groupingFields'][1] + ['startTime', 'bucketLength', fieldTuple[0]]) retVal = self._getTimedData(reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict'], reportRequest['groupingFields'], {}) if not retVal['OK']: return retVal dataDict, granularity = retVal['Value'] self.stripDataField(dataDict, 0) dataDict, _maxValue = self._divideByFactor(dataDict, granularity) dataDict = self._fillWithZero(granularity, reportRequest['startTime'], reportRequest['endTime'], dataDict) baseDataDict, graphDataDict, _maxValue, unitName = self._findSuitableRateUnit( dataDict, self._getAccumulationMaxValue(dataDict), "files") return S_OK({ 'data': baseDataDict, 'graphDataDict': graphDataDict, 'granularity': granularity, 'unit': unitName }) def _plotInputDataFiles(self, reportRequest, plotInfo, filename): return self.__plotDataFiles(reportRequest, plotInfo, filename, ("InputDataFiles", "Input files")) def _plotOuputDataFiles(self, reportRequest, plotInfo, filename): return self.__plotDataFiles(reportRequest, plotInfo, filename, ("OutputDataFiles", "Output files")) def __plotDataFiles(self, reportRequest, plotInfo, filename, fieldTuple): metadata = { 'title': '%s by %s' % (fieldTuple[1], reportRequest['grouping']), 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'], 'span': plotInfo['granularity'], 'ylabel': plotInfo['unit'] } return self._generateStackedLinePlot(filename, plotInfo['graphDataDict'], metadata) _reportHistogramCPUUsedName = "Histogram CPU time" def _reportHistogramCPUUsed(self, reportRequest): selectFields = ( self._getSelectStringForGrouping(reportRequest['groupingFields']) + ", %s", reportRequest['groupingFields'][1] + ['CPUTime']) retVal = self._getBucketData(reportRequest['startTime'], reportRequest['endTime'], selectFields, reportRequest['condDict']) if not retVal['OK']: return retVal dataDict = retVal['Value'] return S_OK({'data': dataDict}) def _plotHistogramCPUUsed(self, reportRequest, plotInfo, filename): metadata = { 'title': 'CPU usage by %s' % reportRequest['grouping'], 'starttime': reportRequest['startTime'], 'endtime': reportRequest['endTime'] } return self._generateHistogram(filename, [plotInfo['data']], metadata)
def __sendAccounting(self, jobID): """ Send WMS accounting data for the given job """ try: accountingReport = Job() endTime = 'Unknown' lastHeartBeatTime = 'Unknown' result = self.jobDB.getJobAttributes(jobID) if not result['OK']: return result jobDict = result['Value'] startTime, endTime = self.__checkLoggingInfo(jobID, jobDict) lastCPUTime, lastWallTime, lastHeartBeatTime = self.__checkHeartBeat( jobID, jobDict) lastHeartBeatTime = fromString(lastHeartBeatTime) if lastHeartBeatTime is not None and lastHeartBeatTime > endTime: endTime = lastHeartBeatTime cpuNormalization = self.jobDB.getJobParameter( jobID, 'CPUNormalizationFactor') if not cpuNormalization['OK'] or not cpuNormalization['Value']: cpuNormalization = 0.0 else: cpuNormalization = float(cpuNormalization['Value']) except Exception: self.log.exception( "Exception in __sendAccounting for job %s: endTime=%s, lastHBTime %s" % (str(jobID), str(endTime), str(lastHeartBeatTime)), '', False) return S_ERROR("Exception") processingType = self.__getProcessingType(jobID) accountingReport.setStartTime(startTime) accountingReport.setEndTime(endTime) # execTime = toEpoch( endTime ) - toEpoch( startTime ) #Fill the accounting data acData = { 'Site': jobDict['Site'], 'User': jobDict['Owner'], 'UserGroup': jobDict['OwnerGroup'], 'JobGroup': jobDict['JobGroup'], 'JobType': jobDict['JobType'], 'JobClass': jobDict['JobSplitType'], 'ProcessingType': processingType, 'FinalMajorStatus': 'Failed', 'FinalMinorStatus': 'Stalled', 'CPUTime': lastCPUTime, 'NormCPUTime': lastCPUTime * cpuNormalization, 'ExecTime': lastWallTime, 'InputDataSize': 0.0, 'OutputDataSize': 0.0, 'InputDataFiles': 0, 'OutputDataFiles': 0, 'DiskSpace': 0.0, 'InputSandBoxSize': 0.0, 'OutputSandBoxSize': 0.0, 'ProcessedEvents': 0 } self.log.verbose('Accounting Report is:') self.log.verbose(acData) accountingReport.setValuesFromDict(acData) result = accountingReport.commit() if result['OK']: self.jobDB.setJobAttribute(jobID, 'AccountedFlag', 'True') else: self.log.error( 'Failed to send accounting report', 'Job: %d, Error: %s' % (int(jobID), result['Message'])) return result
def __sendAccounting( self, jobID ): """ Send WMS accounting data for the given job """ try: accountingReport = Job() endTime = 'Unknown' lastHeartBeatTime = 'Unknown' result = self.jobDB.getJobAttributes( jobID ) if not result['OK']: return result jobDict = result['Value'] startTime, endTime = self.__checkLoggingInfo( jobID, jobDict ) lastCPUTime, lastWallTime, lastHeartBeatTime = self.__checkHeartBeat( jobID, jobDict ) lastHeartBeatTime = fromString( lastHeartBeatTime ) if lastHeartBeatTime is not None and lastHeartBeatTime > endTime: endTime = lastHeartBeatTime cpuNormalization = self.jobDB.getJobParameter( jobID, 'CPUNormalizationFactor' ) if not cpuNormalization['OK'] or not cpuNormalization['Value']: cpuNormalization = 0.0 else: cpuNormalization = float( cpuNormalization['Value'] ) except Exception: self.log.exception( "Exception in __sendAccounting for job %s: endTime=%s, lastHBTime %s" % ( str( jobID ), str( endTime ), str( lastHeartBeatTime ) ), '' , False ) return S_ERROR( "Exception" ) processingType = self.__getProcessingType( jobID ) accountingReport.setStartTime( startTime ) accountingReport.setEndTime( endTime ) # execTime = toEpoch( endTime ) - toEpoch( startTime ) #Fill the accounting data acData = { 'Site' : jobDict['Site'], 'User' : jobDict['Owner'], 'UserGroup' : jobDict['OwnerGroup'], 'JobGroup' : jobDict['JobGroup'], 'JobType' : jobDict['JobType'], 'JobClass' : jobDict['JobSplitType'], 'ProcessingType' : processingType, 'FinalMajorStatus' : 'Failed', 'FinalMinorStatus' : 'Stalled', 'CPUTime' : lastCPUTime, 'NormCPUTime' : lastCPUTime * cpuNormalization, 'ExecTime' : lastWallTime, 'InputDataSize' : 0.0, 'OutputDataSize' : 0.0, 'InputDataFiles' : 0, 'OutputDataFiles' : 0, 'DiskSpace' : 0.0, 'InputSandBoxSize' : 0.0, 'OutputSandBoxSize' : 0.0, 'ProcessedEvents' : 0 } # For accidentally stopped jobs ExecTime can be not set if not acData['ExecTime']: acData['ExecTime'] = acData['CPUTime'] elif acData['ExecTime'] < acData['CPUTime']: acData['ExecTime'] = acData['CPUTime'] self.log.verbose( 'Accounting Report is:' ) self.log.verbose( acData ) accountingReport.setValuesFromDict( acData ) result = accountingReport.commit() if result['OK']: self.jobDB.setJobAttribute( jobID, 'AccountedFlag', 'True' ) else: self.log.error( 'Failed to send accounting report', 'Job: %d, Error: %s' % ( int( jobID ), result['Message'] ) ) return result
def monitoring( self, loop, parentthread, output ): self.initialTiming = os.times() accountingReport = AccountingJob() accountingReport.setStartTime() numberJobsFlag = True numberJobs = 0 numberStartedJobsDict = {} numberEndingJobsDict = {} job_pattern = re.compile( 'Job =.*?,' ) job_pattern_2 = re.compile( 'Job =.*?\n' ) jobid = int( re.split( "_", re.split( "/", output )[int( len( re.split( "/", output ) ) - 1 )] )[0] ) cmd = '/bin/chmod 555 ' + self.getinfo returned = self.commandLaunch( cmd ) while parentthread.isAlive(): time.sleep( loop ) if numberJobsFlag: cmd = self.getinfo + ' -c step1' returned = self.commandLaunch( cmd ) self.log.info( 'InteractiveJobMonitorThread:step1:numJobs:', returned ) if returned != None: if ( returned['Value'][1] != "" ): if re.split( "=", returned['Value'][1] )[1].strip().isdigit(): numberJobs = int( re.split( "=", returned['Value'][1] )[1] ) if ( numberJobs != 0 ): numberJobsFlag = False BigDataDB.setJobStatus( jobid, "Running" ) else: cmd = self.getinfo + ' -c step2' returned = self.commandLaunch( cmd ) self.log.info( 'InteractiveJobMonitorThread:step2:startedJobs:', returned ) if returned != "": if ( returned['Value'][1] != "" ): startedJobs = job_pattern.findall( returned['Value'][1] ) self.log.info( 'step2:startedJobs:', startedJobs ) cmd = self.getinfo + ' -c step3' returned = self.commandLaunch( cmd ) self.log.info( 'InteractiveJobMonitorThread:step3:endedJobs:', returned ) if returned != "": if ( returned['Value'][1] != "" ): finishedJobs = job_pattern_2.findall( returned['Value'][1] ) self.log.info( 'step3:finishedJobs:', finishedJobs ) if ( len( finishedJobs ) == numberJobs ): BigDataDB.setJobStatus( jobid, "Done" ) BigDataDB.setHadoopID( jobid, finishedJobs ) self.__updateSandBox( jobid, output ) #Update Accounting EXECUTION_RESULT = {} EXECUTION_RESULT['CPU'] = [] finalStat = os.times() for i in range( len( finalStat ) ): EXECUTION_RESULT['CPU'].append( finalStat[i] - self.initialTiming[i] ) utime, stime, cutime, cstime, elapsed = EXECUTION_RESULT['CPU'] cpuTime = utime + stime + cutime + cstime execTime = elapsed result = jobDB.getJobAttributes( jobid ) getting = result['Value'] acData = { 'User' : getting['Owner'], 'UserGroup' : getting['OwnerGroup'], 'JobGroup' : 'cesga', 'JobType' : 'User', 'JobClass' : 'unknown', 'ProcessingType' : 'unknown', 'FinalMajorStatus' : getting['Status'], 'FinalMinorStatus' : getting['MinorStatus'], 'CPUTime' : cpuTime, 'Site' : getting['Site'], # Based on the factor to convert raw CPU to Normalized units (based on the CPU Model) 'NormCPUTime' : 0, 'ExecTime' : cpuTime, 'InputDataSize' : 0, 'OutputDataSize' : 0, 'InputDataFiles' : 0, 'OutputDataFiles' : 0, 'DiskSpace' : 0, 'InputSandBoxSize' : 0, 'OutputSandBoxSize' : 0, 'ProcessedEvents' : 0 } accountingReport.setEndTime() accountingReport.setValuesFromDict( acData ) result = accountingReport.commit()
def _sendAccounting(self, jobID): """ Send WMS accounting data for the given job. Run inside thread. """ try: accountingReport = Job() endTime = "Unknown" lastHeartBeatTime = "Unknown" result = self.jobDB.getJobAttributes(jobID) if not result["OK"]: return result jobDict = result["Value"] startTime, endTime = self._checkLoggingInfo(jobID, jobDict) lastCPUTime, lastWallTime, lastHeartBeatTime = self._checkHeartBeat( jobID, jobDict) lastHeartBeatTime = fromString(lastHeartBeatTime) if lastHeartBeatTime is not None and lastHeartBeatTime > endTime: endTime = lastHeartBeatTime result = JobMonitoringClient().getJobParameter( jobID, "CPUNormalizationFactor") if not result["OK"] or not result["Value"]: self.log.error( "Error getting Job Parameter CPUNormalizationFactor, setting 0", result.get("Message", "No such value"), ) cpuNormalization = 0.0 else: cpuNormalization = float( result["Value"].get("CPUNormalizationFactor")) except Exception as e: self.log.exception( "Exception in _sendAccounting", "for job=%s: endTime=%s, lastHBTime=%s" % (str(jobID), str(endTime), str(lastHeartBeatTime)), lException=e, ) return S_ERROR("Exception") processingType = self._getProcessingType(jobID) accountingReport.setStartTime(startTime) accountingReport.setEndTime(endTime) # execTime = toEpoch( endTime ) - toEpoch( startTime ) # Fill the accounting data acData = { "Site": jobDict["Site"], "User": jobDict["Owner"], "UserGroup": jobDict["OwnerGroup"], "JobGroup": jobDict["JobGroup"], "JobType": jobDict["JobType"], "JobClass": jobDict["JobSplitType"], "ProcessingType": processingType, "FinalMajorStatus": JobStatus.FAILED, "FinalMinorStatus": JobMinorStatus.STALLED_PILOT_NOT_RUNNING, "CPUTime": lastCPUTime, "NormCPUTime": lastCPUTime * cpuNormalization, "ExecTime": lastWallTime, "InputDataSize": 0.0, "OutputDataSize": 0.0, "InputDataFiles": 0, "OutputDataFiles": 0, "DiskSpace": 0.0, "InputSandBoxSize": 0.0, "OutputSandBoxSize": 0.0, "ProcessedEvents": 0, } # For accidentally stopped jobs ExecTime can be not set if not acData["ExecTime"]: acData["ExecTime"] = acData["CPUTime"] elif acData["ExecTime"] < acData["CPUTime"]: acData["ExecTime"] = acData["CPUTime"] self.log.verbose("Accounting Report is:") self.log.verbose(acData) accountingReport.setValuesFromDict(acData) result = accountingReport.commit() if result["OK"]: self.jobDB.setJobAttribute(jobID, "AccountedFlag", "True") else: self.log.error( "Failed to send accounting report", "Job: %d, Error: %s" % (int(jobID), result["Message"])) return result
""" This python class move the data from BigDataDB to AccountingJobDB """ __RCSID__ = "$Id: $" from DIRAC.AccountingSystem.Client.Types.Job import Job as AccountingJob from DIRAC.Core.Utilities.File import getGlobbedTotalSize, getGlobbedFiles class MoveToAccounting: ############################################################################# def __init__( self ): """ Standard constructor """ accountingReport = AccountingJob() accountingReport.setStartTime() jobID = 489 status = "Done" minorStatus = "Completed" if status: wmsMajorStatus = status if minorStatus: wmsMinorStatus = minorStatus accountingReport.setEndTime() #CPUTime and ExecTime if not 'CPU' in EXECUTION_RESULT: # If the payload has not started execution (error with input data, SW, SB,...) # Execution result is not filled use initialTiming
def sendAccounting( self, jobID ): """Send WMS accounting data for the given job """ accountingReport = Job() result = self.jobDB.getJobAttributes( jobID ) if not result['OK']: return result jobDict = result['Value'] result = self.logDB.getJobLoggingInfo( jobID ) if not result['OK']: logList = [] else: logList = result['Value'] startTime = jobDict['StartExecTime'] endTime = '' if not startTime or startTime == 'None': for status, minor, app, stime, source in logList: if status == 'Running': startTime = stime break for status, minor, app, stime, source in logList: if status == 'Stalled': endTime = stime if not startTime or startTime == 'None': startTime = jobDict['SubmissionTime'] if type( startTime ) in types.StringTypes: startTime = fromString( startTime ) result = self.logDB.getJobLoggingInfo( jobID ) if not result['OK']: endTime = dateTime() else: for status, minor, app, stime, source in result['Value']: if status == 'Stalled': endTime = stime break if not endTime: endTime = dateTime() if type( endTime ) in types.StringTypes: endTime = fromString( endTime ) result = self.jobDB.getHeartBeatData( jobID ) lastCPUTime = 0 lastWallTime = 0 lastHeartBeatTime = jobDict['StartExecTime'] if result['OK']: for name, value, heartBeatTime in result['Value']: if 'CPUConsumed' == name: try: value = int( float( value ) ) if value > lastCPUTime: lastCPUTime = value except: pass if 'WallClockTime' == name: try: value = int( float( value ) ) if value > lastWallTime: lastWallTime = value except: pass if heartBeatTime > lastHeartBeatTime: lastHeartBeatTime = heartBeatTime accountingReport.setStartTime( startTime ) accountingReport.setEndTime() # execTime = toEpoch( endTime ) - toEpoch( startTime ) #Fill the accounting data acData = { 'Site' : jobDict['Site'], 'User' : jobDict['Owner'], 'UserGroup' : jobDict['OwnerGroup'], 'JobGroup' : jobDict['JobGroup'], 'JobType' : jobDict['JobType'], 'JobClass' : jobDict['JobSplitType'], 'ProcessingType' : 'unknown', 'FinalMajorStatus' : 'Failed', 'FinalMinorStatus' : 'Stalled', 'CPUTime' : lastCPUTime, 'NormCPUTime' : 0.0, 'ExecTime' : lastWallTime, 'InputDataSize' : 0.0, 'OutputDataSize' : 0.0, 'InputDataFiles' : 0, 'OutputDataFiles' : 0, 'DiskSpace' : 0.0, 'InputSandBoxSize' : 0.0, 'OutputSandBoxSize' : 0.0, 'ProcessedEvents' : 0 } self.log.verbose( 'Accounting Report is:' ) self.log.verbose( acData ) accountingReport.setValuesFromDict( acData ) result = accountingReport.commit() if result['OK']: self.jobDB.setJobAttribute( jobID, 'AccountedFlag', 'True' ) else: self.log.warn( 'Failed to send accounting report for job %d' % int( jobID ) ) self.log.error( result['Message'] ) return result
def __sendAccounting(self, jobID): """ Send WMS accounting data for the given job """ try: accountingReport = Job() endTime = "Unknown" lastHeartBeatTime = "Unknown" result = self.jobDB.getJobAttributes(jobID) if not result["OK"]: return result jobDict = result["Value"] startTime, endTime = self.__checkLoggingInfo(jobID, jobDict) lastCPUTime, lastWallTime, lastHeartBeatTime = self.__checkHeartBeat(jobID, jobDict) lastHeartBeatTime = fromString(lastHeartBeatTime) if lastHeartBeatTime is not None and lastHeartBeatTime > endTime: endTime = lastHeartBeatTime cpuNormalization = self.jobDB.getJobParameter(jobID, "CPUNormalizationFactor") if not cpuNormalization["OK"] or not cpuNormalization["Value"]: cpuNormalization = 0.0 else: cpuNormalization = float(cpuNormalization["Value"]) except Exception: self.log.exception( "Exception in __sendAccounting for job %s: endTime=%s, lastHBTime %s" % (str(jobID), str(endTime), str(lastHeartBeatTime)), "", False, ) return S_ERROR("Exception") processingType = self.__getProcessingType(jobID) accountingReport.setStartTime(startTime) accountingReport.setEndTime(endTime) # execTime = toEpoch( endTime ) - toEpoch( startTime ) # Fill the accounting data acData = { "Site": jobDict["Site"], "User": jobDict["Owner"], "UserGroup": jobDict["OwnerGroup"], "JobGroup": jobDict["JobGroup"], "JobType": jobDict["JobType"], "JobClass": jobDict["JobSplitType"], "ProcessingType": processingType, "FinalMajorStatus": "Failed", "FinalMinorStatus": "Stalled", "CPUTime": lastCPUTime, "NormCPUTime": lastCPUTime * cpuNormalization, "ExecTime": lastWallTime, "InputDataSize": 0.0, "OutputDataSize": 0.0, "InputDataFiles": 0, "OutputDataFiles": 0, "DiskSpace": 0.0, "InputSandBoxSize": 0.0, "OutputSandBoxSize": 0.0, "ProcessedEvents": 0, } # For accidentally stopped jobs ExecTime can be not set if not acData["ExecTime"]: acData["ExecTime"] = acData["CPUTime"] elif acData["ExecTime"] < acData["CPUTime"]: acData["ExecTime"] = acData["CPUTime"] self.log.verbose("Accounting Report is:") self.log.verbose(acData) accountingReport.setValuesFromDict(acData) result = accountingReport.commit() if result["OK"]: self.jobDB.setJobAttribute(jobID, "AccountedFlag", "True") else: self.log.error("Failed to send accounting report", "Job: %d, Error: %s" % (int(jobID), result["Message"])) return result
def __sendAccounting( self, jobID ): """ Send WMS accounting data for the given job """ accountingReport = Job() result = self.jobDB.getJobAttributes( jobID ) if not result['OK']: return result jobDict = result['Value'] startTime, endTime = self.__checkLoggingInfo( jobID, jobDict ) lastCPUTime, lastWallTime, lastHeartBeatTime = self.__checkHeartBeat( jobID, jobDict ) if lastHeartBeatTime and fromString( lastHeartBeatTime ) > endTime: endTime = fromString( lastHeartBeatTime ) cpuNormalization = self.jobDB.getJobParameter( jobID, 'CPUNormalizationFactor' ) if not cpuNormalization['OK'] or not cpuNormalization['Value']: cpuNormalization = 0.0 else: cpuNormalization = float( cpuNormalization['Value'] ) processingType = self.__getProcessingType( jobID ) accountingReport.setStartTime( startTime ) accountingReport.setEndTime( endTime ) # execTime = toEpoch( endTime ) - toEpoch( startTime ) #Fill the accounting data acData = { 'Site' : jobDict['Site'], 'User' : jobDict['Owner'], 'UserGroup' : jobDict['OwnerGroup'], 'JobGroup' : jobDict['JobGroup'], 'JobType' : jobDict['JobType'], 'JobClass' : jobDict['JobSplitType'], 'ProcessingType' : processingType, 'FinalMajorStatus' : 'Failed', 'FinalMinorStatus' : 'Stalled', 'CPUTime' : lastCPUTime, 'NormCPUTime' : lastCPUTime * cpuNormalization, 'ExecTime' : lastWallTime, 'InputDataSize' : 0.0, 'OutputDataSize' : 0.0, 'InputDataFiles' : 0, 'OutputDataFiles' : 0, 'DiskSpace' : 0.0, 'InputSandBoxSize' : 0.0, 'OutputSandBoxSize' : 0.0, 'ProcessedEvents' : 0 } self.log.verbose( 'Accounting Report is:' ) self.log.verbose( acData ) accountingReport.setValuesFromDict( acData ) result = accountingReport.commit() if result['OK']: self.jobDB.setJobAttribute( jobID, 'AccountedFlag', 'True' ) else: self.log.error( 'Failed to send accounting report', 'Job: %d, Error: %s' % ( int( jobID ), result['Message'] ) ) return result