def testSkippedFilesJSON(self): """ _testSkippedFilesJSON_ Test that skipped files are translated properly into JSON """ # Check a report where some files were skipped but not all originalReport = Report("cmsRun1") originalReport.parse(self.skippedFilesxmlPath) originalJSON = originalReport.__to_json__(None) self.assertEqual(len(originalJSON['skippedFiles']), 1) # For negative control, check a good report with no skipped files goodReport = Report("cmsRun1") goodReport.parse(self.xmlPath) goodJSON = goodReport.__to_json__(None) self.assertEqual(goodJSON['skippedFiles'], []) # Check a report where all files were skipped badReport = Report("cmsRun1") badReport.parse(self.skippedAllFilesxmlPath) badJSON = badReport.__to_json__(None) self.assertEqual(len(badJSON['skippedFiles']), 2) return
def testSkippedFilesJSON(self): """ _testSkippedFilesJSON_ Test that skipped files are translated properly into JSON """ # Check a report where some files were skipped but not all originalReport = Report("cmsRun1") originalReport.parse(self.skippedFilesxmlPath) originalJSON = originalReport.__to_json__(None) self.assertEqual(len(originalJSON['skippedFiles']), 1) # For negative control, check a good report with no skipped files goodReport = Report("cmsRun1") goodReport.parse(self.xmlPath) goodJSON = goodReport.__to_json__(None) self.assertEqual(goodJSON['skippedFiles'], []) # Check a report where all files were skipped badReport = Report("cmsRun1") badReport.parse(self.skippedAllFilesxmlPath) badJSON = badReport.__to_json__(None) self.assertEqual(len(badJSON['skippedFiles']), 2) return
def testMultiCoreReport(self): """ _testMultiCoreReport_ Verify that multicore reports can be json encoded and uploaded to couch. """ couchdb = CouchServer(os.environ["COUCHURL"]) fwjrdatabase = couchdb.connectDatabase("report_t/fwjrs") self.mcPath = os.path.join( WMCore.WMBase.getTestBase(), "WMCore_t/FwkJobReport_t/MulticoreReport.pkl") myReport = Report() myReport.unpersist(self.mcPath) fwjrDocument = { "_id": "303-0", "jobid": 303, "retrycount": 0, "fwjr": myReport.__to_json__(None), "type": "fwjr" } fwjrdatabase.queue(fwjrDocument, timestamp=True) fwjrdatabase.commit() return
def testJSONEncoding(self): """ _testJSONEncoding_ Verify that turning the FWJR into a JSON object works correctly. """ xmlPath = os.path.join( WMCore.WMBase.getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml") myReport = Report("cmsRun1") myReport.parse(xmlPath) jsonReport = myReport.__to_json__(None) assert "task" in jsonReport.keys(), \ "Error: Task name missing from report." assert len(jsonReport["steps"].keys()) == 1, \ "Error: Wrong number of steps in report." assert "cmsRun1" in jsonReport["steps"].keys(), \ "Error: Step missing from json report." cmsRunStep = jsonReport["steps"]["cmsRun1"] jsonReportSections = [ "status", "errors", "logs", "parameters", "site", "analysis", "cleanup", "input", "output", "start" ] for jsonReportSection in jsonReportSections: assert jsonReportSection in cmsRunStep.keys(), \ "Error: missing section: %s" % jsonReportSection return
def testPerformanceJSON(self): """ _testPerformanceJSON_ Verify that the performance section of the report is correctly converted to JSON. """ xmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/PerformanceReport.xml") myReport = Report("cmsRun1") myReport.parse(xmlPath) perfSection = myReport.__to_json__(thunker = None)["steps"]["cmsRun1"]["performance"] self.assertTrue(perfSection.has_key("storage"), "Error: Storage section is missing.") self.assertTrue(perfSection.has_key("memory"), "Error: Memory section is missing.") self.assertTrue(perfSection.has_key("cpu"), "Error: CPU section is missing.") self.assertEqual(perfSection["cpu"]["AvgEventCPU"], "0.626105", "Error: AvgEventCPU is wrong.") self.assertEqual(perfSection["cpu"]["TotalJobTime"], "23.5703", "Error: TotalJobTime is wrong.") self.assertEqual(perfSection["storage"]["readTotalMB"], 39.6166, "Error: readTotalMB is wrong.") self.assertEqual(perfSection["storage"]["readMaxMSec"], 320.653, "Error: readMaxMSec is wrong") self.assertEqual(perfSection["memory"]["PeakValueRss"], "492.293", "Error: PeakValueRss is wrong.") self.assertEqual(perfSection["memory"]["PeakValueVsize"], "643.281", "Error: PeakValueVsize is wrong.") return
def testJSONEncoding(self): """ _testJSONEncoding_ Verify that turning the FWJR into a JSON object works correctly. """ xmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml") myReport = Report("cmsRun1") myReport.parse(xmlPath) jsonReport = myReport.__to_json__(None) assert "task" in jsonReport.keys(), \ "Error: Task name missing from report." assert len(jsonReport["steps"].keys()) == 1, \ "Error: Wrong number of steps in report." assert "cmsRun1" in jsonReport["steps"].keys(), \ "Error: Step missing from json report." cmsRunStep = jsonReport["steps"]["cmsRun1"] jsonReportSections = ["status", "errors", "logs", "parameters", "site", "analysis", "cleanup", "input", "output", "start"] for jsonReportSection in jsonReportSections: assert jsonReportSection in cmsRunStep.keys(), \ "Error: missing section: %s" % jsonReportSection return
def testPerformanceJSON(self): """ _testPerformanceJSON_ Verify that the performance section of the report is correctly converted to JSON. """ xmlPath = os.path.join( getTestBase(), "WMCore_t/FwkJobReport_t/PerformanceReport.xml") myReport = Report("cmsRun1") myReport.parse(xmlPath) perfSection = myReport.__to_json__( thunker=None)["steps"]["cmsRun1"]["performance"] self.assertTrue("storage" in perfSection, "Error: Storage section is missing.") self.assertTrue("memory" in perfSection, "Error: Memory section is missing.") self.assertTrue("cpu" in perfSection, "Error: CPU section is missing.") self.assertEqual(perfSection["cpu"]["AvgEventCPU"], "0.626105", "Error: AvgEventCPU is wrong.") self.assertEqual(perfSection["cpu"]["TotalJobTime"], "23.5703", "Error: TotalJobTime is wrong.") self.assertEqual(perfSection["storage"]["readTotalMB"], 39.6166, "Error: readTotalMB is wrong.") self.assertEqual(perfSection["storage"]["readMaxMSec"], 320.653, "Error: readMaxMSec is wrong") self.assertEqual(perfSection["memory"]["PeakValueRss"], "492.293", "Error: PeakValueRss is wrong.") self.assertEqual(perfSection["memory"]["PeakValueVsize"], "643.281", "Error: PeakValueVsize is wrong.") return
def testFallbackFilesJSON(self): """ _testFallbackFilesJSON_ Test that fallback attempt files are translated properly into JSON """ # For negative control, check a good report with no skipped files goodReport = Report("cmsRun1") goodReport.parse(self.xmlPath) goodJSON = goodReport.__to_json__(None) self.assertEqual(goodJSON['fallbackFiles'], []) # Check a report where all files were skipped badReport = Report("cmsRun1") badReport.parse(self.fallbackXmlPath) badJSON = badReport.__to_json__(None) self.assertEqual(len(badJSON['fallbackFiles']), 1) return
def testFallbackFilesJSON(self): """ _testFallbackFilesJSON_ Test that fallback attempt files are translated properly into JSON """ # For negative control, check a good report with no skipped files goodReport = Report("cmsRun1") goodReport.parse(self.xmlPath) goodJSON = goodReport.__to_json__(None) self.assertEqual(goodJSON['fallbackFiles'], []) # Check a report where all files were skipped badReport = Report("cmsRun1") badReport.parse(self.fallbackXmlPath) badJSON = badReport.__to_json__(None) self.assertEqual(len(badJSON['fallbackFiles']), 1) return
def testMultiCoreReport(self): """ _testMultiCoreReport_ Verify that multicore reports can be json encoded and uploaded to couch. """ couchdb = CouchServer(os.environ["COUCHURL"]) fwjrdatabase = couchdb.connectDatabase("report_t/fwjrs") self.mcPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/MulticoreReport.pkl") myReport = Report() myReport.unpersist(self.mcPath) fwjrDocument = {"_id": "303-0", "jobid": 303, "retrycount": 0, "fwjr": myReport.__to_json__(None), "type": "fwjr"} fwjrdatabase.queue(fwjrDocument, timestamp = True) fwjrdatabase.commit() return
print("==== CMSSW Stack Execution FINISHING at %s ====" % time.asctime(time.gmtime())) logCMSSW() except WMExecutionFailure as WMex: print("ERROR: Caught WMExecutionFailure - code = %s - name = %s - detail = %s" % (WMex.code, WMex.name, WMex.detail)) exmsg = WMex.name # Try to recover what we can from the FJR. handleException will use this if possible. if os.path.exists('FrameworkJobReport.xml'): try: report = Report("cmsRun") report.parse('FrameworkJobReport.xml', "cmsRun") try: jobExitCode = report.getExitCode() except: jobExitCode = WMex.code report = report.__to_json__(None) StripReport(report) report['jobExitCode'] = jobExitCode with open('jobReport.json', 'w') as of: json.dump(report, of) except: print("WARNING: Failure when trying to parse FJR XML after job failure. Traceback follows.") print(traceback.format_exc()) handleException("FAILED", WMex.code, exmsg) mintime() sys.exit(WMex.code) except Exception as ex: #print "jobExitCode = %s" % jobExitCode handleException("FAILED", EC_CMSRunWrapper, "failed to generate cmsRun cfg file at runtime") mintime()
def parse_fwk_report(data, config, report_filename): """Extract task data from a framework report. Analyze the CMSSW job framework report to get the CMSSW exit code, skipped files, runs and lumis processed on a file basis, total events written, and CPU time overall and per event. """ exit_code = 0 skipped = [] infos = {} written = 0 eventsPerRun = 0 report = Report("cmsrun") report.parse(report_filename) exit_code = report.getExitCode() for fn in report.getAllSkippedFiles(): fn = config['file map'].get(fn, fn) skipped.append(fn) outinfos = {} for file in report.getAllFiles(): pfn = file['pfn'] outinfos[pfn] = { 'runs': {}, 'events': file['events'], } written += int(file['events']) for run in file['runs']: try: outinfos[pfn]['runs'][run.run].extend(run.lumis) except KeyError: outinfos[pfn]['runs'][run.run] = run.lumis for file in report.getAllInputFiles(): filename = file['lfn'] if len(file['lfn']) > 0 else file['pfn'] filename = config['file map'].get(filename, filename) file_lumis = [] try: for run in file['runs']: for lumi in run.lumis: file_lumis.append((run.run, lumi)) except AttributeError: logger.info('Detected file-based task') infos[filename] = (int(file['events']), file_lumis) eventsPerRun += infos[filename][0] serialized = report.__to_json__(None) cputime = float(serialized['steps']['cmsrun']['performance']['cpu'].get('TotalJobCPU', '0')) data['files']['info'] = infos data['files']['output_info'] = outinfos data['files']['skipped'] = skipped data['events_written'] = written data['exe_exit_code'] = exit_code # For efficiency, we care only about the CPU time spent processing # events data['cpu_time'] = cputime data['events_per_run'] = eventsPerRun
except WMExecutionFailure as WMex: print( "ERROR: Caught WMExecutionFailure - code = %s - name = %s - detail = %s" % (WMex.code, WMex.name, WMex.detail)) exmsg = WMex.name # Try to recover what we can from the FJR. handleException will use this if possible. if os.path.exists('FrameworkJobReport.xml'): try: rep = Report("cmsRun") rep.parse('FrameworkJobReport.xml', "cmsRun") try: jobExitCode = rep.getExitCode() except: jobExitCode = WMex.code rep = rep.__to_json__(None) #save the virgin WMArchive report with open('WMArchiveReport.json', 'w') as of: json.dump(rep, of) StripReport(rep) rep['jobExitCode'] = jobExitCode with open('jobReport.json', 'w') as of: json.dump(rep, of) except: print( "WARNING: Failure when trying to parse FJR XML after job failure." ) handleException("FAILED", WMex.code, exmsg) mintime() sys.exit(WMex.code)
except WMExecutionFailure as WMex: print( "ERROR: Caught WMExecutionFailure - code = %s - name = %s - detail = %s" % (WMex.code, WMex.name, WMex.detail)) exmsg = WMex.name # Try to recover what we can from the FJR. handleException will use this if possible. if os.path.exists('FrameworkJobReport.xml'): try: report = Report("cmsRun") report.parse('FrameworkJobReport.xml', "cmsRun") try: jobExitCode = report.getExitCode() except: jobExitCode = WMex.code report = report.__to_json__(None) StripReport(report) report['jobExitCode'] = jobExitCode with open('jobReport.json', 'w') as of: json.dump(report, of) except: print( "WARNING: Failure when trying to parse FJR XML after job failure. Traceback follows." ) print(traceback.format_exc()) handleException("FAILED", WMex.code, exmsg) mintime() sys.exit(WMex.code) except Exception as ex: #print "jobExitCode = %s" % jobExitCode
def main(): """ _main_ """ if 'WMAGENT_CONFIG' not in os.environ: os.environ['WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py' if 'manage' not in os.environ: os.environ['manage'] = '/data/srv/wmagent/current/config/wmagent/manage' ### Fetch the report pickle files from the component log command = ["tail", "-n1000", "install/wmagent/JobAccountant/ComponentLog"] p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() logFiles = [line for line in out.splitlines() if 'install/wmagent/JobCreator/JobCache' in line] logFiles = [i.split()[2] for i in logFiles] msg = "Found %d pickle files to parse " % len(logFiles) ### Now unpickle each of these files and get their output files # also check whether any of them are duplicate lfn2PklDict = {} dupOutputPkl = {} # string value with the dup LFN and keyed by the pickle file path jobReport = Report() for pklPath in logFiles: if not os.path.exists(pklPath): continue jobReport.load(pklPath) for e in jobReport.getAllFiles(): lfn2PklDict.setdefault(e['lfn'], []) lfn2PklDict[e['lfn']].append(pklPath) # now check which files contain more than one pickle path (= created by diff jobs) dupFiles = [] for lfn, pkls in lfn2PklDict.iteritems(): if len(pkls) > 1: dupFiles.append(lfn) for pkl in pkls: if pkl not in dupOutputPkl: jobReport.load(pkl) dupOutputPkl[pkl] = jobReport.__to_json__(None) dupOutputPkl[pkl]['dup_lfns'] = [] dupOutputPkl[pkl]['dup_lfns'].append(lfn) msg += "with a total of %d output files and %d duplicated" % (len(lfn2PklDict), len(dupFiles)) msg += " files to process among them." msg += "\nDuplicate files are:\n%s" % dupFiles print(msg) if dupFiles: print("See dupPickles.json for further details ...") with open('dupPickles.json', 'w') as fo: json.dump(dupOutputPkl, fo, indent=2) if dupFiles: var = raw_input("Can we automatically delete those pickle files? Y/N\n") if var == "Y": # then delete all job report files but the first one - NOT ideal for fname in dupFiles: for pklFile in lfn2PklDict[fname][1:]: if os.path.isfile(pklFile): print("Deleting %s ..." % pklFile) os.remove(pklFile) else: print(" File has probably been already deleted %s ..." % pklFile) print(" Done!") ### Time to load all - this is BAD - LFNs from WMBS database print("\nNow loading all LFNs from wmbs_file_details ...") connectToDB() myThread = threading.currentThread() formatter = DBFormatter(logging, myThread.dbi) output = myThread.transaction.processData("SELECT lfn FROM wmbs_file_details") lfnsDB = formatter.format(output) lfnsDB = [item[0] for item in lfnsDB] print("Retrieved %d lfns from wmbs_file_details" % len(lfnsDB)) ### Compare what are the duplicates dupFiles = list(set(lfn2PklDict.keys()) & set(lfnsDB)) print("\nFound %d duplicate files." % len(dupFiles)) if len(dupFiles) == 0: sys.exit(0) ### Print some basic data about these reports print("Their overview is: ") dbDupPkl = [] for fname in dupFiles: for pklPath in lfn2PklDict[fname]: jobInfo = {'lfn': fname} jobInfo['pklPath'] = pklPath jobReport.load(pklPath) jobInfo['exitCode'] = jobReport.getExitCode() jobInfo['taskSuccess'] = jobReport.taskSuccessful() jobInfo['EOSLogURL'] = jobReport.getLogURL() jobInfo['HostName'] = jobReport.getWorkerNodeInfo()['HostName'] jobInfo['Site'] = jobReport.getSiteName() jobInfo['task'] = jobReport.getTaskName() dbDupPkl.append(jobInfo) print(pformat(dbDupPkl)) print("") print("Remove them, restart the component and be happy!\n") sys.exit(0)
print("==== CMSSW Stack Execution FINISHED at %s ====" % time.asctime(time.gmtime())) logCMSSW() except WMExecutionFailure as WMex: print("ERROR: Caught WMExecutionFailure - code = %s - name = %s - detail = %s" % (WMex.code, WMex.name, WMex.detail)) exmsg = WMex.name # Try to recover what we can from the FJR. handleException will use this if possible. if os.path.exists('FrameworkJobReport.xml'): try: rep = Report("cmsRun") rep.parse('FrameworkJobReport.xml', "cmsRun") try: jobExitCode = rep.getExitCode() except: jobExitCode = WMex.code rep = rep.__to_json__(None) #save the virgin WMArchive report with open('WMArchiveReport.json', 'w') as of: json.dump(rep, of) StripReport(rep) rep['jobExitCode'] = jobExitCode with open('jobReport.json', 'w') as of: json.dump(rep, of) except: print("WARNING: Failure when trying to parse FJR XML after job failure.") handleException("FAILED", WMex.code, exmsg) mintime() sys.exit(WMex.code) except Exception as ex: #print "jobExitCode = %s" % jobExitCode