def testStripReport(self): """ _testStripReport_ Test whether or not we can strip input file information from a FWJR and create a smaller object. """ myReport = Report("cmsRun1") myReport.parse(self.xmlPath) path1 = os.path.join(self.testDir, 'testReport1.pkl') path2 = os.path.join(self.testDir, 'testReport2.pkl') myReport.save(path1) info = BasicAlgos.getFileInfo(filename=path1) self.assertEqual(info['Size'], 6821) inputFiles = myReport.getAllInputFiles() self.assertEqual(len(inputFiles), 1) myReport.stripInputFiles() self.assertEqual(len(myReport.getAllInputFiles()), 0) myReport.save(path2) info = BasicAlgos.getFileInfo(filename=path2) self.assertEqual(info['Size'], 5933) return
def testStripReport(self): """ _testStripReport_ Test whether or not we can strip input file information from a FWJR and create a smaller object. """ myReport = Report("cmsRun1") myReport.parse(self.xmlPath) path1 = os.path.join(self.testDir, 'testReport1.pkl') path2 = os.path.join(self.testDir, 'testReport2.pkl') myReport.save(path1) info = BasicAlgos.getFileInfo(filename = path1) self.assertEqual(info['Size'], 7101) inputFiles = myReport.getAllInputFiles() self.assertEqual(len(inputFiles), 1) myReport.stripInputFiles() self.assertEqual(len(myReport.getAllInputFiles()), 0) myReport.save(path2) info = BasicAlgos.getFileInfo(filename = path2) self.assertEqual(info['Size'], 6210) return
def testPileupFiles(self): """ _testPileupFiles_ Test that alll the pileup files end up in the report """ report = Report("cmsRun1") report.parse(self.pileupXmlPath) self.assertEqual(len(report.getAllInputFiles()), 14) primaryCount = 0 secondaryCount = 0 mixingCount = 0 for fileEntry in report.getAllInputFiles(): if fileEntry['input_type'] == 'mixingFiles': mixingCount += 1 elif fileEntry['input_type'] == 'primaryFiles': primaryCount += 1 elif fileEntry['input_type'] == 'secondaryFiles': secondaryCount += 1 self.assertEqual(primaryCount, 1) self.assertEqual(secondaryCount, 0) self.assertEqual(mixingCount, 13) self.assertEqual(len(report.getAllFallbackFiles()), 1) return
def testPileupFiles(self): """ _testPileupFiles_ Test that alll the pileup files end up in the report """ report = Report("cmsRun1") report.parse(self.pileupXmlPath) self.assertEqual(len(report.getAllInputFiles()), 14) primaryCount = 0 secondaryCount = 0 mixingCount = 0 for fileEntry in report.getAllInputFiles(): if fileEntry['input_type'] == 'mixingFiles': mixingCount += 1 elif fileEntry['input_type'] == 'primaryFiles': primaryCount += 1 elif fileEntry['input_type'] == 'secondaryFiles': secondaryCount += 1 self.assertEqual(primaryCount, 1) self.assertEqual(secondaryCount, 0) self.assertEqual(mixingCount, 13) self.assertEqual(len(report.getAllFallbackFiles()), 1) return
def parse_fwk_report(data, config, report_filename): """Extract task data from a framework report. Analyze the CMSSW job framework report to get the CMSSW exit code, skipped files, runs and lumis processed on a file basis, total events written, and CPU time overall and per event. """ exit_code = 0 skipped = [] infos = {} written = 0 eventsPerRun = 0 report = Report("cmsrun") report.parse(report_filename) exit_code = report.getExitCode() for fn in report.getAllSkippedFiles(): fn = config['file map'].get(fn, fn) skipped.append(fn) outinfos = {} for file in report.getAllFiles(): pfn = file['pfn'] outinfos[pfn] = { 'runs': {}, 'events': file['events'], } written += int(file['events']) for run in file['runs']: try: outinfos[pfn]['runs'][run.run].extend(run.lumis) except KeyError: outinfos[pfn]['runs'][run.run] = run.lumis for file in report.getAllInputFiles(): filename = file['lfn'] if len(file['lfn']) > 0 else file['pfn'] filename = config['file map'].get(filename, filename) file_lumis = [] try: for run in file['runs']: for lumi in run.lumis: file_lumis.append((run.run, lumi)) except AttributeError: logger.info('Detected file-based task') infos[filename] = (int(file['events']), file_lumis) eventsPerRun += infos[filename][0] serialized = report.__to_json__(None) cputime = float(serialized['steps']['cmsrun']['performance']['cpu'].get('TotalJobCPU', '0')) data['files']['info'] = infos data['files']['output_info'] = outinfos data['files']['skipped'] = skipped data['events_written'] = written data['exe_exit_code'] = exit_code # For efficiency, we care only about the CPU time spent processing # events data['cpu_time'] = cputime data['events_per_run'] = eventsPerRun