def writeFJR(self): """ """ fwjr = FwkJobReport() fwjr.addError(self.wrapperExitCode, "WrapperExitCode") if (self.exeExitCode != ""): fwjr.addError(self.exeExitCode, "ExeExitCode") fwjr.write(self.reportFileName) return
def writeFJR(self): """ """ fwjr = FwkJobReport() fwjr.addError(self.wrapperExitCode, "WrapperExitCode") if self.exeExitCode != "": fwjr.addError(self.exeExitCode, "ExeExitCode") fwjr.write(self.reportFileName) return
def newReport(self, name, attrs): """ _newReport_ Handler method for a new FrameworkJobReport """ self.currentReport = FwkJobReport() name = attrs.get("Name", None) status = attrs.get("Status", None) jobSpec = attrs.get("JobSpecID", None) workSpec = attrs.get("WorkflowSpecID", None) jobType = attrs.get("JobType", None) if name != None: self.currentReport.name = str(name) if status != None: self.currentReport.status = str(status) if jobSpec != None: self.currentReport.jobSpecId = str(jobSpec) if jobType != None: self.currentReport.jobType = str(jobType) if workSpec != None: self.currentReport.workflowSpecId = str(workSpec) return
def writeFwkJobReport( self, jobSpecId, exitCode, reportfilename ): """ __writeFwkJobReport__ write a fake reportfilename based on the status reported """ # create job report logging.debug("Creating report %s" % reportfilename) fwjr = FwkJobReport() fwjr.jobSpecId = jobSpecId if exitCode == 0 : fwjr.status = "Success" fwjr.exitCode = 0 else : fwjr.exitCode = exitCode fwjr.status = "Failed" # store job report fwjr.write(reportfilename)
Instantiate a new ContentHandler and run it over the file producing a list of FwkJobReport instances """ #handler = FwkJobRepHandler() #parser = make_parser() #parser.setContentHandler(handler) #try: # parser.parse(filename) #except SAXParseException, ex: # msg = "Error parsing JobReport File: %s\n" % filename # msg += str(ex) # print msg # return [] #print handler.results[0] #return handler.results try: improvDoc = loadIMProvFile(filename) except Exception, ex: return [] result = [] reportQ = IMProvQuery("FrameworkJobReport") reportInstances = reportQ(improvDoc) for reportInstance in reportInstances: newReport = FwkJobReport() newReport.load(reportInstance) result.append(newReport) return result
def processFrameworkJobReport(): """ Runtime tool for processing the Framework Job Report produced by CMsGen executables """ theJobReport = FwkJobReport() # # If everything worked well set the success status # else record a failure and error if jobFailed == 0: theJobReport.status = "Success" theJobReport.exitCode = 0 elif jobFailed == 1: theJobReport.status = "Failed" theJobReport.exitCode = 11111 errDetail = theJobReport.addError(11111, "CmsGenFailure") errDetail['Description'] = "Error retrieving grid file" elif jobFailed == 2: theJobReport.status = "Failed" theJobReport.exitCode = 11112 errDetail = theJobReport.addError(11112, "CmsGenFailure") errDetail['Description'] = "Error producing step 1 events" elif jobFailed == 3: theJobReport.status = "Failed" theJobReport.exitCode = 11113 errDetail = theJobReport.addError(11113, "CmsGenFailure") errDetail['Description'] = "Error producing step 2 events" elif jobFailed == 4: theJobReport.status = "Failed" theJobReport.exitCode = 11114 errDetail = theJobReport.addError(11114, "CmsGenFailure") errDetail['Description'] = "Error computing the number of produced events" elif jobFailed == 5: theJobReport.status = "Failed" theJobReport.exitCode = 11115 errDetail = theJobReport.addError(11115, "CmsGenFailure") errDetail['Description'] = "Error making a link" elif jobFailed == 6: theJobReport.status = "Failed" theJobReport.exitCode = 11116 errDetail = theJobReport.addError(11116, "CmsGenFailure") errDetail['Description'] = "Error preparing the executation" else: theJobReport.status = "Failed" theJobReport.exitCode = 11117 errDetail = theJobReport.addError(11117, "CmsGenFailure") errDetail['Description'] = "General cmsGen Failure" if jobFailed == 0: file = open("nCmsGenEvents.txt", 'r') nEventsLine = file.readline() nEvents = nEventsLine.split('\n') file.close() totalEvents = nEvents[0] outputFileUnw = "" if generator == "alpgen": outputFileUnw = "%s.unw" % outputFile elif generator == "comphep": outputFileUnw = "%s" % outputFile elif generator == "madgraph": outputFileUnw = "%s" % outputFile if not os.path.exists(outputFileUnw): msg = "Output file Not Found: %s" % outputFileUnw raise RuntimeError, msg totalSize = os.stat(outputFileUnw)[6] else: totalEvents = 0 totalSize = 0 newFile = theJobReport.newFile() newFile["LFN"] = "None" if generator == "alpgen": #newFile["PFN"] = "%s/%s" % (os.getcwd(),outputFile) newFile["PFN"] = "%s" % outputFile elif generator == "comphep": newFile["PFN"] = "%s" % outputFile elif generator == "madgraph": newFile["PFN"] = "%s" % outputFile newFile["Size"] = totalSize newFile["TotalEvents"] = totalEvents newFile["ModuleLabel"] = "cmsGen" theJobReport.write(jobReport)
def processFrameworkJobReport(): """ Runtime tool for processing the Framework Job Report produced by CMsGen executables """ theJobReport = FwkJobReport() # # If everything worked well set the success status # else record a failure and error if jobFailed == 0: theJobReport.status = "Success" theJobReport.exitCode = 0 elif jobFailed == 1: theJobReport.status = "Failed" theJobReport.exitCode = 11111 errDetail = theJobReport.addError(11111, "CmsGenFailure") errDetail['Description'] = "Error retrieving grid file" elif jobFailed == 2: theJobReport.status = "Failed" theJobReport.exitCode = 11112 errDetail = theJobReport.addError(11112, "CmsGenFailure") errDetail['Description'] = "Error producing step 1 events" elif jobFailed == 3: theJobReport.status = "Failed" theJobReport.exitCode = 11113 errDetail = theJobReport.addError(11113, "CmsGenFailure") errDetail['Description'] = "Error producing step 2 events" elif jobFailed == 4: theJobReport.status = "Failed" theJobReport.exitCode = 11114 errDetail = theJobReport.addError(11114, "CmsGenFailure") errDetail[ 'Description'] = "Error computing the number of produced events" elif jobFailed == 5: theJobReport.status = "Failed" theJobReport.exitCode = 11115 errDetail = theJobReport.addError(11115, "CmsGenFailure") errDetail['Description'] = "Error making a link" elif jobFailed == 6: theJobReport.status = "Failed" theJobReport.exitCode = 11116 errDetail = theJobReport.addError(11116, "CmsGenFailure") errDetail['Description'] = "Error preparing the executation" else: theJobReport.status = "Failed" theJobReport.exitCode = 11117 errDetail = theJobReport.addError(11117, "CmsGenFailure") errDetail['Description'] = "General cmsGen Failure" if jobFailed == 0: file = open("nCmsGenEvents.txt", 'r') nEventsLine = file.readline() nEvents = nEventsLine.split('\n') file.close() totalEvents = nEvents[0] outputFileUnw = "" if generator == "alpgen": outputFileUnw = "%s.unw" % outputFile elif generator == "comphep": outputFileUnw = "%s" % outputFile elif generator == "madgraph": outputFileUnw = "%s" % outputFile if not os.path.exists(outputFileUnw): msg = "Output file Not Found: %s" % outputFileUnw raise RuntimeError, msg totalSize = os.stat(outputFileUnw)[6] else: totalEvents = 0 totalSize = 0 newFile = theJobReport.newFile() newFile["LFN"] = "None" if generator == "alpgen": #newFile["PFN"] = "%s/%s" % (os.getcwd(),outputFile) newFile["PFN"] = "%s" % outputFile elif generator == "comphep": newFile["PFN"] = "%s" % outputFile elif generator == "madgraph": newFile["PFN"] = "%s" % outputFile newFile["Size"] = totalSize newFile["TotalEvents"] = totalEvents newFile["ModuleLabel"] = "cmsGen" theJobReport.write(jobReport)
if not os.path.exists(cfgFile): msg = "Cfg File Not Found: %s" % cfgFile raise RuntimeError, msg if generator == "alpgen": jobFailed = alpGen() processFrameworkJobReport() elif generator == "comphep": jobFailed = compHep() processFrameworkJobReport() elif generator == "madgraph": jobFailed = madgraph() processFrameworkJobReport() elif generator == "sherpa": jobFailed = sherpa() # Create a dummy FwkJobReport theJobReport = FwkJobReport() if jobFailed == 0: theJobReport.status = "Success" theJobReport.exitCode = 0 theJobReport.write(jobReport) else: theJobReport.status = "Failed" theJobReport.exitCode = jobFailed errDetail = theJobReport.addError(jobFailed, "CmsGenFailure") errDetail['Description'] = "Failed to set up Sherpa Libs" else: msg = "Generator %s Not Found" % generator raise RuntimeError, msg
def __init__(self): ContentHandler.__init__(self) # // # // State containers used during parsing #// self.currentReport = FwkJobReport() self.currentFile = None self.currentDict = None self.currentInputDict = None self.currentCksum = None self.inTiming = False self.inGenInfo = False self._CharCache = "" # // # // container for results #// self.results = [] # // # // Response methods to start of elements based on element name #// self._StartResponses = { "FrameworkJobReport": self.newReport, "File": self.newFile, "InputFile": self.newInputFile, "Dataset": self.newDataset, "ExitCode": self.exitCode, "State": self.stateHandler, "Branch": self.noResponse, "Branches": self.noResponse, "Input": self.startInput, "Inputs": self.noResponse, "Runs": self.noResponse, "Run": self.noResponse, "SkippedEvent": self.skippedEvent, "SkippedFile": self.skippedFile, "Checksum": self.checksum, "SiteDetail": self.siteDetail, "FrameworkError": self.frameworkError, "TimingService": self.timingService, "StorageStatistics": self.noResponse, "GeneratorInfo": self.generatorInfo, "Data": self.dataHandler, "ReadBranches": self.readBranches, } # // # // Response methods to end of elements based on element name #// self._EndResponses = { "FrameworkJobReport": self.endReport, "File": self.endFile, "Dataset": self.endDataset, "ExitCode": self.noResponse, "State": self.noResponse, "Branch": self.endBranch, "Branches": self.noResponse, "InputFile": self.endInputFile, "Input": self.endInput, "Inputs": self.noResponse, "Runs": self.noResponse, "Run": self.endRun, "SkippedEvent": self.noResponse, "SkippedFile": self.noResponse, "Checksum": self.endChecksum, "SiteDetail": self.noResponse, "FrameworkError": self.endFrameworkError, "TimingService": self.endTimingService, "StorageStatistics": self.storageStatistics, "GeneratorInfo": self.endGeneratorInfo, "Data": self.noResponse, "ReadBranches": self.endReadBranches, }
class FwkJobRepHandler(ContentHandler): """ _FwkJobRepHandler_ SAX Content Handler implementation to build instances of FwkJobReport and populate it Multiple job reports in a file are supported, since the plan is to concatenate them at the end of a job from multiple outputs instances of FwkJobReport are stored in self.results as a list """ def __init__(self): ContentHandler.__init__(self) # // # // State containers used during parsing #// self.currentReport = FwkJobReport() self.currentFile = None self.currentDict = None self.currentInputDict = None self.currentCksum = None self.inTiming = False self.inGenInfo = False self._CharCache = "" # // # // container for results #// self.results = [] # // # // Response methods to start of elements based on element name #// self._StartResponses = { "FrameworkJobReport": self.newReport, "File": self.newFile, "InputFile": self.newInputFile, "Dataset": self.newDataset, "ExitCode": self.exitCode, "State": self.stateHandler, "Branch": self.noResponse, "Branches": self.noResponse, "Input": self.startInput, "Inputs": self.noResponse, "Runs": self.noResponse, "Run": self.noResponse, "SkippedEvent": self.skippedEvent, "SkippedFile": self.skippedFile, "Checksum": self.checksum, "SiteDetail": self.siteDetail, "FrameworkError": self.frameworkError, "TimingService": self.timingService, "StorageStatistics": self.noResponse, "GeneratorInfo": self.generatorInfo, "Data": self.dataHandler, "ReadBranches": self.readBranches, } # // # // Response methods to end of elements based on element name #// self._EndResponses = { "FrameworkJobReport": self.endReport, "File": self.endFile, "Dataset": self.endDataset, "ExitCode": self.noResponse, "State": self.noResponse, "Branch": self.endBranch, "Branches": self.noResponse, "InputFile": self.endInputFile, "Input": self.endInput, "Inputs": self.noResponse, "Runs": self.noResponse, "Run": self.endRun, "SkippedEvent": self.noResponse, "SkippedFile": self.noResponse, "Checksum": self.endChecksum, "SiteDetail": self.noResponse, "FrameworkError": self.endFrameworkError, "TimingService": self.endTimingService, "StorageStatistics": self.storageStatistics, "GeneratorInfo": self.endGeneratorInfo, "Data": self.noResponse, "ReadBranches": self.endReadBranches, } def noResponse(self, name, attrs={}): """some elements require no action""" if self.inTiming: value = str(attrs.get("Value")) self.currentDict[str(name)] = value pass def startElement(self, name, attrs): """ _startElement_ Override ContentHandler.startElement Start a new XML Element, call the appropriate response method based off the name of the element """ response = self._StartResponses.get(name, self.noResponse) response(name, attrs) return def endElement(self, name): """ _endElement_ Override ContentHandler.endElement End of element, invoke response based on name and flush the chardata cache """ response = self._EndResponses.get(name, self.fillDictionary) response(name) self._CharCache = "" return def characters(self, data): """ _characters_ Override ContentHandler.characters Accumulate character data from an xml element, if required the response will pick it up and insert it into the appropriate object. """ if len(data.strip()) == 0: return self._CharCache += str(data).replace("\t", "") return def inFile(self): """boolean test to see if state is in a file block""" return self.currentFile != None def newReport(self, name, attrs): """ _newReport_ Handler method for a new FrameworkJobReport """ self.currentReport = FwkJobReport() name = attrs.get("Name", None) status = attrs.get("Status", None) jobSpec = attrs.get("JobSpecID", None) workSpec = attrs.get("WorkflowSpecID", None) jobType = attrs.get("JobType", None) if name != None: self.currentReport.name = str(name) if status != None: self.currentReport.status = str(status) if jobSpec != None: self.currentReport.jobSpecId = str(jobSpec) if jobType != None: self.currentReport.jobType = str(jobType) if workSpec != None: self.currentReport.workflowSpecId = str(workSpec) return def endReport(self, name): """ _endReport_ Handler Method for finishing a FrameorkJobReport """ self.results.append(self.currentReport) self.currentReport = None return def newFile(self, name, attrs): """new File tag encountered""" self.currentFile = self.currentReport.newFile() self.currentDict = self.currentFile def endFile(self, name): """ end of file tag encountered""" self.currentFile = None self.currentDict = None def newInputFile(self, name, attrs): """new InputFile tag encountered""" self.currentFile = self.currentReport.newInputFile() self.currentDict = self.currentFile def endInputFile(self, name): """ end of InputFile tag encountered""" self.currentFile = None self.currentDict = None def newDataset(self, name, attrs): """ start of Dataset tag within a File tag""" if not self.inFile(): return self.currentDict = self.currentFile.newDataset() return def endDataset(self, name): """end of Dataset tag""" if not self.inFile(): return self.currentDict = self.currentFile def exitCode(self, name, attrs): """ handle an ExitCode node, extract the value attr and add it to the current report """ if self.currentReport == None: return value = attrs.get("Value", None) if value != None: self.currentReport.exitCode = int(value) return def fillDictionary(self, name): """ _fillDictionary_ Any object requiring population as a dictionary can use this handler to populate itself """ if self.currentDict == None: return if self.inTiming: return self.currentDict[str(name)] = str(self._CharCache) return def stateHandler(self, name, attrs): """ _stateHandler_ Handle a State Node """ self.currentFile.state = str(attrs.get('Value', "closed")) def endBranch(self, name): """ _endBranch_ End of Branch node """ if self.currentFile != None: self.currentFile.branches.append(str(self._CharCache)) def startInput(self, name, attrs): """ _startInput_ Handle Input reference for an Output File """ if self.currentFile != None: if not self.currentFile.isInput: self.currentInputDict = {} self.currentDict = self.currentInputDict def endInput(self, name): """ _endInput_ end Input node """ self.currentFile.addInputFile(self.currentInputDict['PFN'], self.currentInputDict['LFN']) self.currentInputDict = None self.currentDict = self.currentFile def endRun(self, name): """ _endRun_ Add Run Number chardata to current file """ if self.currentFile != None: self.currentFile.runs.append(str(self._CharCache)) def skippedEvent(self, name, attrs): """ _skippedEvent_ Record a Skipped Event """ if self.currentReport == None: return run = attrs.get("Run", None) if run == None: return event = attrs.get("Event", None) if event == None: return self.currentReport.addSkippedEvent(str(run), str(event)) return def skippedFile(self, name, attrs): """skipped file node""" if self.currentReport == None: return pfn = attrs.get("Pfn", None) lfn = attrs.get("Lfn", None) if pfn == None: return if lfn == None: return self.currentReport.addSkippedFile(str(pfn), str(lfn)) return def checksum(self, name, attrs): """ _checksum_ Handle a checksum element start """ self.currentCksum = attrs.get("Algorithm", None) def endChecksum(self, name): """ _endChecksum_ Handle a checksum element end """ if self.currentCksum == None: return if not self.inFile(): return self.currentFile.addChecksum(str(self.currentCksum), str(self._CharCache)) self.currentCksum = None return def siteDetail(self, name, attrs): """ _siteDetail_ Handle a site detail parameter node """ if self.currentReport == None: return detailName = attrs.get('Parameter', None) detailValue = attrs.get('Value', None) if detailName == None: return self.currentReport.siteDetails[str(detailName)] = str(detailValue) return def frameworkError(self, name, attrs): """ _frameworkError_ Start Error message in job report """ if self.currentReport == None: return errStatus = str(attrs.get("ExitStatus", "1")) errType = str(attrs.get("Type", "Unknown")) self.currentDict = self.currentReport.addError(errStatus, errType) return def endFrameworkError(self, name): """ _endFrameworkError_ End Error block in job report """ if self.currentReport == None: return if self.currentDict == None: return self.currentDict['Description'] = str(self._CharCache) self.currentDict == None return def timingService(self, name, attrs): """ _timingService_ Start of Timing Service element """ if self.currentReport == None: return self.inTiming = True self.currentDict = self.currentReport.timing return def endTimingService(self, name): """ _endTimingService_ End of Timing Service element """ if self.currentReport == None: return self.inTiming = False self.currentDict = None return def readBranches(self, name, attrs): """ _readBranches_ Start of readBranches element """ if self.currentReport == None: return self.inTiming = True self.currentDict = self.currentReport.readBranches return def endReadBraches(self, name): """ _endReadBranches_ End of ReadBranches element """ if self.currentReport == None: return self.inTiming = False self.currentDict = None return def storageStatistics(self, name): """ _storageStatistics_ Big blob of chardata... """ if self.currentReport == None: return self.currentReport.storageStatistics = str(self._CharCache) return def generatorInfo(self, name, attrs): """ _generatorInfo_ """ if self.currentReport == None: return self.inGenInfo = True self.currentDict = self.currentReport.generatorInfo return def endGeneratorInfo(self, name): """ _endGeneratorInfo_ """ self.inGenInfo = False self.currentDict = None return def dataHandler(self, name, attrs): """ _dataHandler_ """ if self.inGenInfo: if self.currentDict == None: return key = str(attrs['Name']) val = str(attrs['Value']) self.currentDict[key] = val return return
def __fwkJobReportCommon(self, jobSpecLoaded, workerNodeInfo): """ __fwkJobReportCommon_ Create a new job report and fill it in with generic information that is not dependent on the outcome of the job. The jobSpecLoaded parameter is a reference to an instance of the JobSpec class that has been initialized with the job spec that we are generating a report for. """ #workerNodeInfo = RandomAllocationPlugin().allocateJob() try: jobSpecPayload = jobSpecLoaded.payload newReport = FwkJobReport() newReport.jobSpecId = jobSpecPayload.jobName newReport.jobType = jobSpecPayload.type newReport.workflowSpecId = jobSpecPayload.workflow newReport.name = jobSpecPayload.name #get information from the super class newReport.siteDetails['SiteName'] = workerNodeInfo['SiteName'] #HostName is the same as worker_node name newReport.siteDetails['HostName'] = workerNodeInfo['HostName'] newReport.siteDetails['se-name'] = workerNodeInfo['se-name'] newReport.siteDetails['ce-name'] = workerNodeInfo['ce-name'] newReport.addLogFile("/path/to/log/archive", "some.random.se.cern.ch") return jobSpecPayload, newReport except Exception, ex: #msg = "Unable to Publish Report for %s\n" % jobSpecPayload.jobName #msg += "Since It is not known to the JobState System:\n" msg = str(ex) logging.error(msg) raise RuntimeError, msg
def __init__(self): ContentHandler.__init__(self) # // # // State containers used during parsing #// self.currentReport = FwkJobReport() self.currentFile = None self.currentDict = None self.currentInputDict = None self.currentCksum = None self.inTiming = False self.inGenInfo = False self._CharCache = "" # // # // container for results #// self.results = [] # // # // Response methods to start of elements based on element name #// self._StartResponses = { "FrameworkJobReport" : self.newReport, "File" : self.newFile, "InputFile": self.newInputFile, "Dataset" : self.newDataset, "ExitCode" : self.exitCode, "State" : self.stateHandler, "Branch" : self.noResponse, "Branches" : self.noResponse, "Input" : self.startInput, "Inputs" : self.noResponse, "Runs" : self.noResponse, "Run" : self.noResponse, "SkippedEvent" : self.skippedEvent, "SkippedFile" : self.skippedFile, "Checksum" : self.checksum, "SiteDetail" : self.siteDetail, "FrameworkError" : self.frameworkError, "TimingService" : self.timingService, "StorageStatistics" : self.noResponse, "GeneratorInfo" : self.generatorInfo, "Data" : self.dataHandler, } # // # // Response methods to end of elements based on element name #// self._EndResponses = { "FrameworkJobReport" : self.endReport, "File" : self.endFile, "Dataset" : self.endDataset, "ExitCode": self.noResponse, "State" : self.noResponse, "Branch" : self.endBranch, "Branches" : self.noResponse, "InputFile" : self.endInputFile, "Input" :self.endInput, "Inputs" : self.noResponse, "Runs" : self.noResponse, "Run" : self.endRun, "SkippedEvent" : self.noResponse, "SkippedFile" : self.noResponse, "Checksum" : self.endChecksum, "SiteDetail" : self.noResponse, "FrameworkError" : self.endFrameworkError, "TimingService" : self.endTimingService, "StorageStatistics" : self.storageStatistics, "GeneratorInfo" : self.endGeneratorInfo, "Data": self.noResponse, }
class FwkJobRepHandler(ContentHandler): """ _FwkJobRepHandler_ SAX Content Handler implementation to build instances of FwkJobReport and populate it Multiple job reports in a file are supported, since the plan is to concatenate them at the end of a job from multiple outputs instances of FwkJobReport are stored in self.results as a list """ def __init__(self): ContentHandler.__init__(self) # // # // State containers used during parsing #// self.currentReport = FwkJobReport() self.currentFile = None self.currentDict = None self.currentInputDict = None self.currentCksum = None self.inTiming = False self.inGenInfo = False self._CharCache = "" # // # // container for results #// self.results = [] # // # // Response methods to start of elements based on element name #// self._StartResponses = { "FrameworkJobReport" : self.newReport, "File" : self.newFile, "InputFile": self.newInputFile, "Dataset" : self.newDataset, "ExitCode" : self.exitCode, "State" : self.stateHandler, "Branch" : self.noResponse, "Branches" : self.noResponse, "Input" : self.startInput, "Inputs" : self.noResponse, "Runs" : self.noResponse, "Run" : self.noResponse, "SkippedEvent" : self.skippedEvent, "SkippedFile" : self.skippedFile, "Checksum" : self.checksum, "SiteDetail" : self.siteDetail, "FrameworkError" : self.frameworkError, "TimingService" : self.timingService, "StorageStatistics" : self.noResponse, "GeneratorInfo" : self.generatorInfo, "Data" : self.dataHandler, } # // # // Response methods to end of elements based on element name #// self._EndResponses = { "FrameworkJobReport" : self.endReport, "File" : self.endFile, "Dataset" : self.endDataset, "ExitCode": self.noResponse, "State" : self.noResponse, "Branch" : self.endBranch, "Branches" : self.noResponse, "InputFile" : self.endInputFile, "Input" :self.endInput, "Inputs" : self.noResponse, "Runs" : self.noResponse, "Run" : self.endRun, "SkippedEvent" : self.noResponse, "SkippedFile" : self.noResponse, "Checksum" : self.endChecksum, "SiteDetail" : self.noResponse, "FrameworkError" : self.endFrameworkError, "TimingService" : self.endTimingService, "StorageStatistics" : self.storageStatistics, "GeneratorInfo" : self.endGeneratorInfo, "Data": self.noResponse, } def noResponse(self, name, attrs = {}): """some elements require no action""" if self.inTiming: value = str(attrs.get("Value")) self.currentDict[str(name)] = value pass def startElement(self, name, attrs): """ _startElement_ Override ContentHandler.startElement Start a new XML Element, call the appropriate response method based off the name of the element """ response = self._StartResponses.get(name, self.noResponse) response(name, attrs) return def endElement(self, name): """ _endElement_ Override ContentHandler.endElement End of element, invoke response based on name and flush the chardata cache """ response = self._EndResponses.get(name, self.fillDictionary) response(name) self._CharCache = "" return def characters(self, data): """ _characters_ Override ContentHandler.characters Accumulate character data from an xml element, if required the response will pick it up and insert it into the appropriate object. """ if len(data.strip()) == 0: return self._CharCache += str(data).replace("\t", "") return def inFile(self): """boolean test to see if state is in a file block""" return self.currentFile != None def newReport(self, name, attrs): """ _newReport_ Handler method for a new FrameworkJobReport """ self.currentReport = FwkJobReport() name = attrs.get("Name", None) status = attrs.get("Status", None) jobSpec = attrs.get("JobSpecID", None) workSpec = attrs.get("WorkflowSpecID", None) jobType = attrs.get("JobType", None) if name != None: self.currentReport.name = str(name) if status != None: self.currentReport.status = str(status) if jobSpec != None: self.currentReport.jobSpecId = str(jobSpec) if jobType != None: self.currentReport.jobType = str(jobType) if workSpec != None: self.currentReport.workflowSpecId = str(workSpec) return def endReport(self, name): """ _endReport_ Handler Method for finishing a FrameorkJobReport """ self.results.append(self.currentReport) self.currentReport = None return def newFile(self, name, attrs): """new File tag encountered""" self.currentFile = self.currentReport.newFile() self.currentDict = self.currentFile def endFile(self, name): """ end of file tag encountered""" self.currentFile = None self.currentDict = None def newInputFile(self, name, attrs): """new InputFile tag encountered""" self.currentFile = self.currentReport.newInputFile() self.currentDict = self.currentFile def endInputFile(self, name): """ end of InputFile tag encountered""" self.currentFile = None self.currentDict = None def newDataset(self, name, attrs): """ start of Dataset tag within a File tag""" if not self.inFile(): return self.currentDict = self.currentFile.newDataset() return def endDataset(self, name): """end of Dataset tag""" if not self.inFile(): return self.currentDict = self.currentFile def exitCode(self, name, attrs): """ handle an ExitCode node, extract the value attr and add it to the current report """ if self.currentReport == None: return value = attrs.get("Value", None) if value != None: self.currentReport.exitCode = int(value) return def fillDictionary(self, name): """ _fillDictionary_ Any object requiring population as a dictionary can use this handler to populate itself """ if self.currentDict == None: return if self.inTiming: return self.currentDict[str(name)] = str(self._CharCache) return def stateHandler(self, name, attrs): """ _stateHandler_ Handle a State Node """ self.currentFile.state = str(attrs.get('Value', "closed")) def endBranch(self, name): """ _endBranch_ End of Branch node """ if self.currentFile != None: self.currentFile.branches.append(str(self._CharCache)) def startInput(self, name, attrs): """ _startInput_ Handle Input reference for an Output File """ if self.currentFile != None: if not self.currentFile.isInput: self.currentInputDict = {} self.currentDict = self.currentInputDict def endInput(self, name): """ _endInput_ end Input node """ self.currentFile.addInputFile(self.currentInputDict['PFN'], self.currentInputDict['LFN']) self.currentInputDict = None self.currentDict = self.currentFile def endRun(self, name): """ _endRun_ Add Run Number chardata to current file """ if self.currentFile != None: self.currentFile.runs.append(str(self._CharCache)) def skippedEvent(self, name, attrs): """ _skippedEvent_ Record a Skipped Event """ if self.currentReport == None: return run = attrs.get("Run", None) if run == None : return event = attrs.get("Event", None) if event == None: return self.currentReport.addSkippedEvent(str(run), str(event)) return def skippedFile(self, name, attrs): """skipped file node""" if self.currentReport == None: return pfn = attrs.get("Pfn", None) lfn = attrs.get("Lfn", None) if pfn == None: return if lfn == None: return self.currentReport.addSkippedFile(str(pfn), str(lfn)) return def checksum(self, name, attrs): """ _checksum_ Handle a checksum element start """ self.currentCksum = attrs.get("Algorithm", None) def endChecksum(self, name): """ _endChecksum_ Handle a checksum element end """ if self.currentCksum == None: return if not self.inFile(): return self.currentFile.addChecksum(str(self.currentCksum), str(self._CharCache)) self.currentCksum = None return def siteDetail(self, name, attrs): """ _siteDetail_ Handle a site detail parameter node """ if self.currentReport == None: return detailName = attrs.get('Parameter', None) detailValue = attrs.get('Value' , None) if detailName == None: return self.currentReport.siteDetails[str(detailName)] = str(detailValue) return def frameworkError(self, name, attrs): """ _frameworkError_ Start Error message in job report """ if self.currentReport == None: return errStatus = str(attrs.get("ExitStatus", "1")) errType = str(attrs.get("Type", "Unknown")) self.currentDict = self.currentReport.addError(errStatus, errType) return def endFrameworkError(self, name): """ _endFrameworkError_ End Error block in job report """ if self.currentReport == None: return if self.currentDict == None: return self.currentDict['Description'] = str(self._CharCache) self.currentDict == None return def timingService(self, name, attrs): """ _timingService_ Start of Timing Service element """ if self.currentReport == None: return self.inTiming = True self.currentDict = self.currentReport.timing return def endTimingService(self, name): """ _endTimingService_ End of Timing Service element """ if self.currentReport == None: return self.inTiming = False self.currentDict = None return def storageStatistics(self, name): """ _storageStatistics_ Big blob of chardata... """ if self.currentReport == None: return self.currentReport.storageStatistics = str(self._CharCache) return def generatorInfo(self, name, attrs): """ _generatorInfo_ """ if self.currentReport == None: return self.inGenInfo = True self.currentDict = self.currentReport.generatorInfo return def endGeneratorInfo(self, name): """ _endGeneratorInfo_ """ self.inGenInfo = False self.currentDict = None return def dataHandler(self, name, attrs): """ _dataHandler_ """ if self.inGenInfo: if self.currentDict == None: return key = str(attrs['Name']) val = str(attrs['Value']) self.currentDict[key] = val return return
def stageOut(): """ _stageOut_ Main function for this module. Loads data from the task and manages the stage out process for a single attempt """ state = TaskState(os.getcwd()) state.loadRunResDB() workflow = WorkflowSpec() workflow.load(os.environ['PRODAGENT_WORKFLOW_SPEC']) jobSpecFile = os.environ.get('PRODAGENT_JOBSPEC') jobSpecId = None if jobSpecFile is not None: jobSpec = JobSpec() jobSpec.load(jobSpecFile) jobSpecId = jobSpec.parameters.get('JobName') print workflow print state.taskName() print jobSpecId stageOutFor, override, controls = StageOutUtils.getStageOutConfig( workflow, state.taskName()) toplevelReport = os.path.join(os.environ['PRODAGENT_JOB_DIR'], "FrameworkJobReport.xml") exitCode = 0 # // # // find inputs by locating the task for which we are staging out #// and loading its TaskState for inputTask in stageOutFor: print "Attempting to stage out files for node %s" % inputTask try: inputState = getTaskState(inputTask) msg = "Loaded Input Task: %s " % inputTask except Exception, ex: msg = "Error load for TaskState for task %s" % inputTask msg += "%s\n" % str(ex) inputState = None print msg if inputState == None: # exit with init error # generate failure report in this dir, since cant find # input state dir inputReport = FwkJobReport() inputReport.name = inputTask inputReport.jobSpecId = jobSpecId exitCode = 60311 errRep = inputReport.addError( 60311, "TaskStateError") errRep['Description'] = msg inputReport.status = "Failed" inputReport.exitCode = 60311 updateReport(toplevelReport, inputReport) print "TaskState is None, exiting..." return exitCode try: inputReport = inputState.getJobReport() msg = "Loaded JobReport for Task : %s\n" % inputTask msg += "File: %s\n" % inputState.jobReport except Exception, ex: msg = "Error loading input report : %s" % str(ex) inputReport = None
return exitCode try: inputReport = inputState.getJobReport() msg = "Loaded JobReport for Task : %s\n" % inputTask msg += "File: %s\n" % inputState.jobReport except Exception, ex: msg = "Error loading input report : %s" % str(ex) inputReport = None print msg if inputReport == None: msg += "Unable to read Job Report for input task: %s\n" % inputTask msg += "Looked for file: %s\n" % inputState.jobReport print msg inputReport = FwkJobReport() inputReport.name = inputTask inputReport.jobSpecId = jobSpecId exitCode = 60311 errRep = inputReport.addError( 60311, "InputReportError") errRep['Description'] = msg inputReport.status = "Failed" inputReport.exitCode = 60311 updateReport(toplevelReport, inputReport) # exit with init error return 60311 try:
if fileType == 'streamer' or fileType == 'edm' or fileType == 'pixdmp': if nEvents == None: msg = "--nevents option not provided" raise RuntimeError, msg elif fileType == 'lumi': nEvents = 0 elif fileType == 'lumi-sa': nEvents = 0 elif fileType == 'lumi-vdm': nEvents = 0 # temporary, PRODCOMMON code does not support DBS registration w/0 lumi section anymore if fileType == 'pixdmp': lumiSection = 1 jobReport = FwkJobReport() jobReportFile = jobReport.newFile() if fileType == 'streamer': jobReportFile['FileType'] = 'STREAMER' elif fileType == 'edm': jobReportFile['FileType'] = 'EDM' elif fileType == 'lumi': jobReportFile['FileType'] = 'LUMI' elif fileType == 'lumi-sa': jobReportFile['FileType'] = 'LUMI-SA' elif fileType == 'lumi-vdm': jobReportFile['FileType'] = 'LUMI-VDM' elif fileType == 'pixdmp': jobReportFile['FileType'] = 'PIXDMP' else:
def processCleanUpJobReport(self,statusCode): """ _processCleanUpJobReport_ Arguments: None Return: None """ # // # // Generate a report # // report = FwkJobReport() report.name = "cleanUp" report.status = "Failed" if statusCode == 0 : report.status = "Success" for lfnRemovedFile in self.success: report.addRemovedFile(lfnRemovedFile, self.seName) for lfnUnremovedFile in self.failed: report.addUnremovedFile(lfnUnremovedFile, self.seName) report.exitCode = statusCode report.jobSpecId = self.state.jobSpecNode.jobName report.jobType = self.state.jobSpecNode.jobType report.workflowSpecId = self.state.jobSpecNode.workflow report.write("./FrameworkJobReport.xml") # // # // Ensure this report gets added to the job-wide report #// toplevelReport = os.path.join(os.environ['PRODAGENT_JOB_DIR'],"FrameworkJobReport.xml") newReport = os.path.join(os.getcwd(), "FrameworkJobReport.xml") mergeReports(toplevelReport, newReport)