def testStripReport(self):
"""
_testStripReport_
Test whether or not we can strip input file information
from a FWJR and create a smaller object.
"""
myReport = Report("cmsRun1")
myReport.parse(self.xmlPath)
path1 = os.path.join(self.testDir, 'testReport1.pkl')
path2 = os.path.join(self.testDir, 'testReport2.pkl')
myReport.save(path1)
info = BasicAlgos.getFileInfo(filename=path1)
self.assertEqual(info['Size'], 6821)
inputFiles = myReport.getAllInputFiles()
self.assertEqual(len(inputFiles), 1)
myReport.stripInputFiles()
self.assertEqual(len(myReport.getAllInputFiles()), 0)
myReport.save(path2)
info = BasicAlgos.getFileInfo(filename=path2)
self.assertEqual(info['Size'], 5933)
return
def testStripReport(self):
"""
_testStripReport_
Test whether or not we can strip input file information
from a FWJR and create a smaller object.
"""
myReport = Report("cmsRun1")
myReport.parse(self.xmlPath)
path1 = os.path.join(self.testDir, 'testReport1.pkl')
path2 = os.path.join(self.testDir, 'testReport2.pkl')
myReport.save(path1)
info = BasicAlgos.getFileInfo(filename = path1)
self.assertEqual(info['Size'], 7101)
inputFiles = myReport.getAllInputFiles()
self.assertEqual(len(inputFiles), 1)
myReport.stripInputFiles()
self.assertEqual(len(myReport.getAllInputFiles()), 0)
myReport.save(path2)
info = BasicAlgos.getFileInfo(filename = path2)
self.assertEqual(info['Size'], 6210)
return
def testPileupFiles(self):
"""
_testPileupFiles_
Test that alll the pileup files end up in the report
"""
report = Report("cmsRun1")
report.parse(self.pileupXmlPath)
self.assertEqual(len(report.getAllInputFiles()), 14)
primaryCount = 0
secondaryCount = 0
mixingCount = 0
for fileEntry in report.getAllInputFiles():
if fileEntry['input_type'] == 'mixingFiles':
mixingCount += 1
elif fileEntry['input_type'] == 'primaryFiles':
primaryCount += 1
elif fileEntry['input_type'] == 'secondaryFiles':
secondaryCount += 1
self.assertEqual(primaryCount, 1)
self.assertEqual(secondaryCount, 0)
self.assertEqual(mixingCount, 13)
self.assertEqual(len(report.getAllFallbackFiles()), 1)
return
def testPileupFiles(self):
"""
_testPileupFiles_
Test that alll the pileup files end up in the report
"""
report = Report("cmsRun1")
report.parse(self.pileupXmlPath)
self.assertEqual(len(report.getAllInputFiles()), 14)
primaryCount = 0
secondaryCount = 0
mixingCount = 0
for fileEntry in report.getAllInputFiles():
if fileEntry['input_type'] == 'mixingFiles':
mixingCount += 1
elif fileEntry['input_type'] == 'primaryFiles':
primaryCount += 1
elif fileEntry['input_type'] == 'secondaryFiles':
secondaryCount += 1
self.assertEqual(primaryCount, 1)
self.assertEqual(secondaryCount, 0)
self.assertEqual(mixingCount, 13)
self.assertEqual(len(report.getAllFallbackFiles()), 1)
return
def parse_fwk_report(data, config, report_filename):
"""Extract task data from a framework report.
Analyze the CMSSW job framework report to get the CMSSW exit code,
skipped files, runs and lumis processed on a file basis, total events
written, and CPU time overall and per event.
"""
exit_code = 0
skipped = []
infos = {}
written = 0
eventsPerRun = 0
report = Report("cmsrun")
report.parse(report_filename)
exit_code = report.getExitCode()
for fn in report.getAllSkippedFiles():
fn = config['file map'].get(fn, fn)
skipped.append(fn)
outinfos = {}
for file in report.getAllFiles():
pfn = file['pfn']
outinfos[pfn] = {
'runs': {},
'events': file['events'],
}
written += int(file['events'])
for run in file['runs']:
try:
outinfos[pfn]['runs'][run.run].extend(run.lumis)
except KeyError:
outinfos[pfn]['runs'][run.run] = run.lumis
for file in report.getAllInputFiles():
filename = file['lfn'] if len(file['lfn']) > 0 else file['pfn']
filename = config['file map'].get(filename, filename)
file_lumis = []
try:
for run in file['runs']:
for lumi in run.lumis:
file_lumis.append((run.run, lumi))
except AttributeError:
logger.info('Detected file-based task')
infos[filename] = (int(file['events']), file_lumis)
eventsPerRun += infos[filename][0]
serialized = report.__to_json__(None)
cputime = float(serialized['steps']['cmsrun']['performance']['cpu'].get('TotalJobCPU', '0'))
data['files']['info'] = infos
data['files']['output_info'] = outinfos
data['files']['skipped'] = skipped
data['events_written'] = written
data['exe_exit_code'] = exit_code
# For efficiency, we care only about the CPU time spent processing
# events
data['cpu_time'] = cputime
data['events_per_run'] = eventsPerRun