def testSkippedFiles(self):
"""
_testSkippedFiles_
Test that skipped files are translated from FWJR into report
"""
# Check a report where some files were skipped but not all
originalReport = Report("cmsRun1")
originalReport.parse(self.skippedFilesxmlPath)
self.assertEqual(originalReport.getAllSkippedFiles(),
['/store/data/Run2012D/Cosmics/RAW/v1/000/206/379/1ED243E7-A611-E211-A851-0019B9F581C9.root'])
# For negative control, check a good report with no skipped files
goodReport = Report("cmsRun1")
goodReport.parse(self.xmlPath)
self.assertEqual(goodReport.getAllSkippedFiles(), [])
# Check a report where all files were skipped
badReport = Report("cmsRun1")
badReport.parse(self.skippedAllFilesxmlPath)
self.assertEqual(sorted(badReport.getAllSkippedFiles()),
['/store/data/Run2012D/Cosmics/RAW/v1/000/206/379/1ED243E7-A611-E211-A851-0019B9F581C9.root',
'/store/data/Run2012D/Cosmics/RAW/v1/000/206/379/1ED243E7-A622-E211-A851-0019B9F581C.root'])
return
def testSkippedFiles(self):
"""
_testSkippedFiles_
Test that skipped files are translated from FWJR into report
"""
# Check a report where some files were skipped but not all
originalReport = Report("cmsRun1")
originalReport.parse(self.skippedFilesxmlPath)
self.assertEqual(originalReport.getAllSkippedFiles(), [
'/store/data/Run2012D/Cosmics/RAW/v1/000/206/379/1ED243E7-A611-E211-A851-0019B9F581C9.root'
])
# For negative control, check a good report with no skipped files
goodReport = Report("cmsRun1")
goodReport.parse(self.xmlPath)
self.assertEqual(goodReport.getAllSkippedFiles(), [])
# Check a report where all files were skipped
badReport = Report("cmsRun1")
badReport.parse(self.skippedAllFilesxmlPath)
self.assertEqual(sorted(badReport.getAllSkippedFiles()), [
'/store/data/Run2012D/Cosmics/RAW/v1/000/206/379/1ED243E7-A611-E211-A851-0019B9F581C9.root',
'/store/data/Run2012D/Cosmics/RAW/v1/000/206/379/1ED243E7-A622-E211-A851-0019B9F581C.root'
])
return
def testSkippedFiles(self):
"""
_testDeleteOutputModule_
If asked delete an output module, if it doesn't
exist then do nothing
"""
# Check a report where some files were skipped but not all
originalReport = Report("cmsRun1")
originalReport.parse(self.skippedFilesxmlPath)
self.assertEqual(originalReport.getAllSkippedFiles(),
['/store/data/Run2012D/Cosmics/RAW/v1/000/206/379/1ED243E7-A611-E211-A851-0019B9F581C9.root'])
# For negative control, check a good report with no skipped files
goodReport = Report("cmsRun1")
goodReport.parse(self.xmlPath)
self.assertEqual(goodReport.getAllSkippedFiles(), [])
# Check a report where all files were skipped
badReport = Report("cmsRun1")
badReport.parse(self.skippedAllFilesxmlPath)
self.assertEqual(sorted(badReport.getAllSkippedFiles()),
['/store/data/Run2012D/Cosmics/RAW/v1/000/206/379/1ED243E7-A611-E211-A851-0019B9F581C9.root',
'/store/data/Run2012D/Cosmics/RAW/v1/000/206/379/1ED243E7-A622-E211-A851-0019B9F581C.root'])
return
def parse_fwk_report(data, config, report_filename):
"""Extract task data from a framework report.
Analyze the CMSSW job framework report to get the CMSSW exit code,
skipped files, runs and lumis processed on a file basis, total events
written, and CPU time overall and per event.
"""
exit_code = 0
skipped = []
infos = {}
written = 0
eventsPerRun = 0
report = Report("cmsrun")
report.parse(report_filename)
exit_code = report.getExitCode()
for fn in report.getAllSkippedFiles():
fn = config['file map'].get(fn, fn)
skipped.append(fn)
outinfos = {}
for file in report.getAllFiles():
pfn = file['pfn']
outinfos[pfn] = {
'runs': {},
'events': file['events'],
}
written += int(file['events'])
for run in file['runs']:
try:
outinfos[pfn]['runs'][run.run].extend(run.lumis)
except KeyError:
outinfos[pfn]['runs'][run.run] = run.lumis
for file in report.getAllInputFiles():
filename = file['lfn'] if len(file['lfn']) > 0 else file['pfn']
filename = config['file map'].get(filename, filename)
file_lumis = []
try:
for run in file['runs']:
for lumi in run.lumis:
file_lumis.append((run.run, lumi))
except AttributeError:
logger.info('Detected file-based task')
infos[filename] = (int(file['events']), file_lumis)
eventsPerRun += infos[filename][0]
serialized = report.__to_json__(None)
cputime = float(serialized['steps']['cmsrun']['performance']['cpu'].get('TotalJobCPU', '0'))
data['files']['info'] = infos
data['files']['output_info'] = outinfos
data['files']['skipped'] = skipped
data['events_written'] = written
data['exe_exit_code'] = exit_code
# For efficiency, we care only about the CPU time spent processing
# events
data['cpu_time'] = cputime
data['events_per_run'] = eventsPerRun
