def processCleanUpJobReport(self,statusCode): """ _processCleanUpJobReport_ Arguments: None Return: None """ # // # // Generate a report # // report = FwkJobReport() report.name = "cleanUp" report.status = "Failed" if statusCode == 0 : report.status = "Success" for lfnRemovedFile in self.success: report.addRemovedFile(lfnRemovedFile, self.seName) for lfnUnremovedFile in self.failed: report.addUnremovedFile(lfnUnremovedFile, self.seName) report.exitCode = statusCode report.jobSpecId = self.state.jobSpecNode.jobName report.jobType = self.state.jobSpecNode.jobType report.workflowSpecId = self.state.jobSpecNode.workflow report.write("./FrameworkJobReport.xml") # // # // Ensure this report gets added to the job-wide report #// toplevelReport = os.path.join(os.environ['PRODAGENT_JOB_DIR'],"FrameworkJobReport.xml") newReport = os.path.join(os.getcwd(), "FrameworkJobReport.xml") mergeReports(toplevelReport, newReport)
def writeFwkJobReport( self, jobSpecId, exitCode, reportfilename ): """ __writeFwkJobReport__ write a fake reportfilename based on the status reported """ # create job report logging.debug("Creating report %s" % reportfilename) fwjr = FwkJobReport() fwjr.jobSpecId = jobSpecId if exitCode == 0 : fwjr.status = "Success" fwjr.exitCode = 0 else : fwjr.exitCode = exitCode fwjr.status = "Failed" # store job report fwjr.write(reportfilename)
def processFrameworkJobReport(): """ Runtime tool for processing the Framework Job Report produced by CMsGen executables """ theJobReport = FwkJobReport() # # If everything worked well set the success status # else record a failure and error if jobFailed == 0: theJobReport.status = "Success" theJobReport.exitCode = 0 elif jobFailed == 1: theJobReport.status = "Failed" theJobReport.exitCode = 11111 errDetail = theJobReport.addError(11111, "CmsGenFailure") errDetail['Description'] = "Error retrieving grid file" elif jobFailed == 2: theJobReport.status = "Failed" theJobReport.exitCode = 11112 errDetail = theJobReport.addError(11112, "CmsGenFailure") errDetail['Description'] = "Error producing step 1 events" elif jobFailed == 3: theJobReport.status = "Failed" theJobReport.exitCode = 11113 errDetail = theJobReport.addError(11113, "CmsGenFailure") errDetail['Description'] = "Error producing step 2 events" elif jobFailed == 4: theJobReport.status = "Failed" theJobReport.exitCode = 11114 errDetail = theJobReport.addError(11114, "CmsGenFailure") errDetail[ 'Description'] = "Error computing the number of produced events" elif jobFailed == 5: theJobReport.status = "Failed" theJobReport.exitCode = 11115 errDetail = theJobReport.addError(11115, "CmsGenFailure") errDetail['Description'] = "Error making a link" elif jobFailed == 6: theJobReport.status = "Failed" theJobReport.exitCode = 11116 errDetail = theJobReport.addError(11116, "CmsGenFailure") errDetail['Description'] = "Error preparing the executation" else: theJobReport.status = "Failed" theJobReport.exitCode = 11117 errDetail = theJobReport.addError(11117, "CmsGenFailure") errDetail['Description'] = "General cmsGen Failure" if jobFailed == 0: file = open("nCmsGenEvents.txt", 'r') nEventsLine = file.readline() nEvents = nEventsLine.split('\n') file.close() totalEvents = nEvents[0] outputFileUnw = "" if generator == "alpgen": outputFileUnw = "%s.unw" % outputFile elif generator == "comphep": outputFileUnw = "%s" % outputFile elif generator == "madgraph": outputFileUnw = "%s" % outputFile if not os.path.exists(outputFileUnw): msg = "Output file Not Found: %s" % outputFileUnw raise RuntimeError, msg totalSize = os.stat(outputFileUnw)[6] else: totalEvents = 0 totalSize = 0 newFile = theJobReport.newFile() newFile["LFN"] = "None" if generator == "alpgen": #newFile["PFN"] = "%s/%s" % (os.getcwd(),outputFile) newFile["PFN"] = "%s" % outputFile elif generator == "comphep": newFile["PFN"] = "%s" % outputFile elif generator == "madgraph": newFile["PFN"] = "%s" % outputFile newFile["Size"] = totalSize newFile["TotalEvents"] = totalEvents newFile["ModuleLabel"] = "cmsGen" theJobReport.write(jobReport)
if not os.path.exists(cfgFile): msg = "Cfg File Not Found: %s" % cfgFile raise RuntimeError, msg if generator == "alpgen": jobFailed = alpGen() processFrameworkJobReport() elif generator == "comphep": jobFailed = compHep() processFrameworkJobReport() elif generator == "madgraph": jobFailed = madgraph() processFrameworkJobReport() elif generator == "sherpa": jobFailed = sherpa() # Create a dummy FwkJobReport theJobReport = FwkJobReport() if jobFailed == 0: theJobReport.status = "Success" theJobReport.exitCode = 0 theJobReport.write(jobReport) else: theJobReport.status = "Failed" theJobReport.exitCode = jobFailed errDetail = theJobReport.addError(jobFailed, "CmsGenFailure") errDetail['Description'] = "Failed to set up Sherpa Libs" else: msg = "Generator %s Not Found" % generator raise RuntimeError, msg
def stageOut(): """ _stageOut_ Main function for this module. Loads data from the task and manages the stage out process for a single attempt """ state = TaskState(os.getcwd()) state.loadRunResDB() workflow = WorkflowSpec() workflow.load(os.environ['PRODAGENT_WORKFLOW_SPEC']) jobSpecFile = os.environ.get('PRODAGENT_JOBSPEC') jobSpecId = None if jobSpecFile is not None: jobSpec = JobSpec() jobSpec.load(jobSpecFile) jobSpecId = jobSpec.parameters.get('JobName') print workflow print state.taskName() print jobSpecId stageOutFor, override, controls = StageOutUtils.getStageOutConfig( workflow, state.taskName()) toplevelReport = os.path.join(os.environ['PRODAGENT_JOB_DIR'], "FrameworkJobReport.xml") exitCode = 0 # // # // find inputs by locating the task for which we are staging out #// and loading its TaskState for inputTask in stageOutFor: print "Attempting to stage out files for node %s" % inputTask try: inputState = getTaskState(inputTask) msg = "Loaded Input Task: %s " % inputTask except Exception, ex: msg = "Error load for TaskState for task %s" % inputTask msg += "%s\n" % str(ex) inputState = None print msg if inputState == None: # exit with init error # generate failure report in this dir, since cant find # input state dir inputReport = FwkJobReport() inputReport.name = inputTask inputReport.jobSpecId = jobSpecId exitCode = 60311 errRep = inputReport.addError( 60311, "TaskStateError") errRep['Description'] = msg inputReport.status = "Failed" inputReport.exitCode = 60311 updateReport(toplevelReport, inputReport) print "TaskState is None, exiting..." return exitCode try: inputReport = inputState.getJobReport() msg = "Loaded JobReport for Task : %s\n" % inputTask msg += "File: %s\n" % inputState.jobReport except Exception, ex: msg = "Error loading input report : %s" % str(ex) inputReport = None
msg = "Error loading input report : %s" % str(ex) inputReport = None print msg if inputReport == None: msg += "Unable to read Job Report for input task: %s\n" % inputTask msg += "Looked for file: %s\n" % inputState.jobReport print msg inputReport = FwkJobReport() inputReport.name = inputTask inputReport.jobSpecId = jobSpecId exitCode = 60311 errRep = inputReport.addError( 60311, "InputReportError") errRep['Description'] = msg inputReport.status = "Failed" inputReport.exitCode = 60311 updateReport(toplevelReport, inputReport) # exit with init error return 60311 try: manager = StageOutReport(inputReport, override, controls) except StageOutInitError, ex: exitCode = ex.data['ErrorCode'] errRep = inputReport.addError( ex.data['ErrorCode'], ex.data['ErrorType']) errRep['Description'] = ex.message inputReport.status = "Failed"
def processFrameworkJobReport(): """ Runtime tool for processing the Framework Job Report produced by CMsGen executables """ theJobReport = FwkJobReport() # # If everything worked well set the success status # else record a failure and error if jobFailed == 0: theJobReport.status = "Success" theJobReport.exitCode = 0 elif jobFailed == 1: theJobReport.status = "Failed" theJobReport.exitCode = 11111 errDetail = theJobReport.addError(11111, "CmsGenFailure") errDetail['Description'] = "Error retrieving grid file" elif jobFailed == 2: theJobReport.status = "Failed" theJobReport.exitCode = 11112 errDetail = theJobReport.addError(11112, "CmsGenFailure") errDetail['Description'] = "Error producing step 1 events" elif jobFailed == 3: theJobReport.status = "Failed" theJobReport.exitCode = 11113 errDetail = theJobReport.addError(11113, "CmsGenFailure") errDetail['Description'] = "Error producing step 2 events" elif jobFailed == 4: theJobReport.status = "Failed" theJobReport.exitCode = 11114 errDetail = theJobReport.addError(11114, "CmsGenFailure") errDetail['Description'] = "Error computing the number of produced events" elif jobFailed == 5: theJobReport.status = "Failed" theJobReport.exitCode = 11115 errDetail = theJobReport.addError(11115, "CmsGenFailure") errDetail['Description'] = "Error making a link" elif jobFailed == 6: theJobReport.status = "Failed" theJobReport.exitCode = 11116 errDetail = theJobReport.addError(11116, "CmsGenFailure") errDetail['Description'] = "Error preparing the executation" else: theJobReport.status = "Failed" theJobReport.exitCode = 11117 errDetail = theJobReport.addError(11117, "CmsGenFailure") errDetail['Description'] = "General cmsGen Failure" if jobFailed == 0: file = open("nCmsGenEvents.txt", 'r') nEventsLine = file.readline() nEvents = nEventsLine.split('\n') file.close() totalEvents = nEvents[0] outputFileUnw = "" if generator == "alpgen": outputFileUnw = "%s.unw" % outputFile elif generator == "comphep": outputFileUnw = "%s" % outputFile elif generator == "madgraph": outputFileUnw = "%s" % outputFile if not os.path.exists(outputFileUnw): msg = "Output file Not Found: %s" % outputFileUnw raise RuntimeError, msg totalSize = os.stat(outputFileUnw)[6] else: totalEvents = 0 totalSize = 0 newFile = theJobReport.newFile() newFile["LFN"] = "None" if generator == "alpgen": #newFile["PFN"] = "%s/%s" % (os.getcwd(),outputFile) newFile["PFN"] = "%s" % outputFile elif generator == "comphep": newFile["PFN"] = "%s" % outputFile elif generator == "madgraph": newFile["PFN"] = "%s" % outputFile newFile["Size"] = totalSize newFile["TotalEvents"] = totalEvents newFile["ModuleLabel"] = "cmsGen" theJobReport.write(jobReport)
if not os.path.exists(cfgFile): msg = "Cfg File Not Found: %s" % cfgFile raise RuntimeError, msg if generator == "alpgen": jobFailed = alpGen() processFrameworkJobReport() elif generator == "comphep": jobFailed = compHep() processFrameworkJobReport() elif generator == "madgraph": jobFailed = madgraph() processFrameworkJobReport() elif generator == "sherpa": jobFailed = sherpa() # Create a dummy FwkJobReport theJobReport = FwkJobReport() if jobFailed == 0: theJobReport.status = "Success" theJobReport.exitCode = 0 theJobReport.write(jobReport) else: theJobReport.status = "Failed" theJobReport.exitCode = jobFailed errDetail = theJobReport.addError(jobFailed, "CmsGenFailure") errDetail['Description'] = "Failed to set up Sherpa Libs" else: msg = "Generator %s Not Found" % generator raise RuntimeError, msg