def inFileValidationWallTime(self):
inFileValidationWallTime = None
if self._inFileValidationStart and self._inFileValidationStop:
inFileValidationWallTime = calcWallTime(
self._inFileValidationStart, self._inFileValidationStop)
return inFileValidationWallTime
def outFileValidationWallTime(self):
outFileValidationWallTime = None
if self._outFileValidationStart and self._outFileValidationStop:
outFileValidationWallTime = calcWallTime(
self._outFileValidationStart, self._outFileValidationStop)
return outFileValidationWallTime
def transformSetupWallTime(self):
transformSetupWallTime = None
if self._transformStart and self._inFileValidationStart:
transformSetupWallTime = calcWallTime(self._transformStart,
self._inFileValidationStart)
return transformSetupWallTime
def python(self, fast=False, fileReport=defaultFileReport):
myDict = {
'name': self._trf.name,
'reportVersion': self._reportVersion,
'cmdLine': ' '.join(shQuoteStrings(sys.argv)),
'exitAcronym': trfExit.codeToName(self._trf.exitCode),
'exitCode': self._trf.exitCode,
'created': isodate(),
'resource': {
'executor': {},
'transform': {}
},
'files': {}
}
if len(self._trf.exitMsg) > self._maxMsgLen:
myDict['exitMsg'] = self._trf.exitMsg[:self._maxMsgLen -
len(self._truncationMsg
)] + self._truncationMsg
myDict['exitMsgExtra'] = self._trf.exitMsg[self._maxMsgLen -
len(self._truncationMsg
):]
else:
myDict['exitMsg'] = self._trf.exitMsg
myDict['exitMsgExtra'] = ""
# Iterate over files
for fileType in ('input', 'output', 'temporary'):
if fileReport[fileType]:
myDict['files'][fileType] = []
# Should have a dataDictionary, unless something went wrong very early...
for dataType, dataArg in iteritems(self._trf._dataDictionary):
if dataArg.auxiliaryFile: # Always skip auxilliary files from the report
continue
if fileReport[dataArg.io]:
entry = {"type": dataType}
entry.update(
trfFileReport(dataArg).python(fast=fast,
type=fileReport[dataArg.io]))
# Supress RAW if all subfiles had nentries == 0
if 'subFiles' in entry and len(
entry['subFiles']) == 0 and isinstance(
dataArg, trfArgClasses.argBSFile):
msg.info(
'No subFiles for entry {0}, suppressing from report.'.
format(entry['argName']))
else:
myDict['files'][dataArg.io].append(entry)
# We report on all executors, in execution order
myDict['executor'] = []
if hasattr(self._trf, '_executorPath'):
for executionStep in self._trf._executorPath:
exe = self._trf._executorDictionary[executionStep['name']]
myDict['executor'].append(
trfExecutorReport(exe).python(fast=fast))
# Executor resources are gathered here to unify where this information is held
# and allow T0/PanDA to just store this JSON fragment on its own
myDict['resource']['executor'][exe.name] = exeResourceReport(
exe, self)
for mergeStep in exe.myMerger:
myDict['resource']['executor'][
mergeStep.name] = exeResourceReport(mergeStep, self)
if self._dbDataTotal > 0 or self._dbTimeTotal > 0:
myDict['resource']['dbDataTotal'] = self._dbDataTotal
myDict['resource']['dbTimeTotal'] = self.roundoff(
self._dbTimeTotal)
# Resource consumption
reportTime = os.times()
# Calculate total cpu time we used -
myCpuTime = reportTime[0] + reportTime[1]
childCpuTime = reportTime[2] + reportTime[3]
wallTime = reportTime[4] - self._trf.transformStart[4]
cpuTime = myCpuTime
cpuTimeTotal = 0
cpuTimePerWorker = myCpuTime
maxWorkers = 1
msg.debug(
'Raw cpu resource consumption: transform {0}, children {1}'.format(
myCpuTime, childCpuTime))
# Reduce childCpuTime by times reported in the executors (broken for MP...?)
for exeName, exeReport in iteritems(myDict['resource']['executor']):
if 'mpworkers' in exeReport:
if exeReport['mpworkers'] > maxWorkers:
maxWorkers = exeReport['mpworkers']
try:
msg.debug('Subtracting {0}s time for executor {1}'.format(
exeReport['cpuTime'], exeName))
childCpuTime -= exeReport['cpuTime']
except TypeError:
pass
try:
cpuTime += exeReport['cpuTime']
cpuTimeTotal += exeReport['total']['cpuTime']
if 'cpuTimePerWorker' in exeReport:
msg.debug('Adding {0}s to cpuTimePerWorker'.format(
exeReport['cpuTimePerWorker']))
cpuTimePerWorker += exeReport['cpuTimePerWorker']
else:
msg.debug(
'Adding nonMP cpuTime {0}s to cpuTimePerWorker'.format(
exeReport['cpuTime']))
cpuTimePerWorker += exeReport['cpuTime']
except TypeError:
pass
msg.debug(
'maxWorkers: {0}, cpuTimeTotal: {1}, cpuTimePerWorker: {2}'.format(
maxWorkers, cpuTime, cpuTimePerWorker))
reportGenerationCpuTime = reportGenerationWallTime = None
if self._trf.outFileValidationStop and reportTime:
reportGenerationCpuTime = calcCpuTime(
self._trf.outFileValidationStop, reportTime)
reportGenerationWallTime = calcWallTime(
self._trf.outFileValidationStop, reportTime)
myDict['resource']['transform'] = {
'cpuTime': self.roundoff(myCpuTime),
'cpuTimeTotal': self.roundoff(cpuTimeTotal),
'externalCpuTime': self.roundoff(childCpuTime),
'wallTime': self.roundoff(wallTime),
'transformSetup': {
'cpuTime': self.roundoff(self._trf.transformSetupCpuTime),
'wallTime': self.roundoff(self._trf.transformSetupWallTime)
},
'inFileValidation': {
'cpuTime': self.roundoff(self._trf.inFileValidationCpuTime),
'wallTime': self.roundoff(self._trf.inFileValidationWallTime)
},
'outFileValidation': {
'cpuTime': self.roundoff(self._trf.outFileValidationCpuTime),
'wallTime': self.roundoff(self._trf.outFileValidationWallTime)
},
'reportGeneration': {
'cpuTime': self.roundoff(reportGenerationCpuTime),
'wallTime': self.roundoff(reportGenerationWallTime)
},
}
if self._trf.processedEvents:
myDict['resource']['transform'][
'processedEvents'] = self._trf.processedEvents
myDict['resource']['transform']['trfPredata'] = self._trf.trfPredata
# check for devision by zero for fast jobs, unit tests
if wallTime > 0:
myDict['resource']['transform']['cpuEfficiency'] = round(
cpuTime / maxWorkers / wallTime, 4)
myDict['resource']['transform']['cpuPWEfficiency'] = round(
cpuTimePerWorker / wallTime, 4)
myDict['resource']['machine'] = machineReport().python(fast=fast)
return myDict