def writeFJR(self):
"""
"""
fwjr = FwkJobReport()
fwjr.addError(self.wrapperExitCode, "WrapperExitCode")
if (self.exeExitCode != ""):
fwjr.addError(self.exeExitCode, "ExeExitCode")
fwjr.write(self.reportFileName)
return
def writeFJR(self):
"""
"""
fwjr = FwkJobReport()
fwjr.addError(self.wrapperExitCode, "WrapperExitCode")
if self.exeExitCode != "":
fwjr.addError(self.exeExitCode, "ExeExitCode")
fwjr.write(self.reportFileName)
return
def newReport(self, name, attrs):
"""
_newReport_
Handler method for a new FrameworkJobReport
"""
self.currentReport = FwkJobReport()
name = attrs.get("Name", None)
status = attrs.get("Status", None)
jobSpec = attrs.get("JobSpecID", None)
workSpec = attrs.get("WorkflowSpecID", None)
jobType = attrs.get("JobType", None)
if name != None:
self.currentReport.name = str(name)
if status != None:
self.currentReport.status = str(status)
if jobSpec != None:
self.currentReport.jobSpecId = str(jobSpec)
if jobType != None:
self.currentReport.jobType = str(jobType)
if workSpec != None:
self.currentReport.workflowSpecId = str(workSpec)
return
def writeFwkJobReport( self, jobSpecId, exitCode, reportfilename ):
"""
__writeFwkJobReport__
write a fake reportfilename based on the status reported
"""
# create job report
logging.debug("Creating report %s" % reportfilename)
fwjr = FwkJobReport()
fwjr.jobSpecId = jobSpecId
if exitCode == 0 :
fwjr.status = "Success"
fwjr.exitCode = 0
else :
fwjr.exitCode = exitCode
fwjr.status = "Failed"
# store job report
fwjr.write(reportfilename)
def newReport(self, name, attrs):
"""
_newReport_
Handler method for a new FrameworkJobReport
"""
self.currentReport = FwkJobReport()
name = attrs.get("Name", None)
status = attrs.get("Status", None)
jobSpec = attrs.get("JobSpecID", None)
workSpec = attrs.get("WorkflowSpecID", None)
jobType = attrs.get("JobType", None)
if name != None:
self.currentReport.name = str(name)
if status != None:
self.currentReport.status = str(status)
if jobSpec != None:
self.currentReport.jobSpecId = str(jobSpec)
if jobType != None:
self.currentReport.jobType = str(jobType)
if workSpec != None:
self.currentReport.workflowSpecId = str(workSpec)
return
Instantiate a new ContentHandler and run it over the file producing
a list of FwkJobReport instances
"""
#handler = FwkJobRepHandler()
#parser = make_parser()
#parser.setContentHandler(handler)
#try:
# parser.parse(filename)
#except SAXParseException, ex:
# msg = "Error parsing JobReport File: %s\n" % filename
# msg += str(ex)
# print msg
# return []
#print handler.results[0]
#return handler.results
try:
improvDoc = loadIMProvFile(filename)
except Exception, ex:
return []
result = []
reportQ = IMProvQuery("FrameworkJobReport")
reportInstances = reportQ(improvDoc)
for reportInstance in reportInstances:
newReport = FwkJobReport()
newReport.load(reportInstance)
result.append(newReport)
return result
def processFrameworkJobReport():
"""
Runtime tool for processing the Framework Job Report produced
by CMsGen executables
"""
theJobReport = FwkJobReport()
#
# If everything worked well set the success status
# else record a failure and error
if jobFailed == 0:
theJobReport.status = "Success"
theJobReport.exitCode = 0
elif jobFailed == 1:
theJobReport.status = "Failed"
theJobReport.exitCode = 11111
errDetail = theJobReport.addError(11111, "CmsGenFailure")
errDetail['Description'] = "Error retrieving grid file"
elif jobFailed == 2:
theJobReport.status = "Failed"
theJobReport.exitCode = 11112
errDetail = theJobReport.addError(11112, "CmsGenFailure")
errDetail['Description'] = "Error producing step 1 events"
elif jobFailed == 3:
theJobReport.status = "Failed"
theJobReport.exitCode = 11113
errDetail = theJobReport.addError(11113, "CmsGenFailure")
errDetail['Description'] = "Error producing step 2 events"
elif jobFailed == 4:
theJobReport.status = "Failed"
theJobReport.exitCode = 11114
errDetail = theJobReport.addError(11114, "CmsGenFailure")
errDetail['Description'] = "Error computing the number of produced events"
elif jobFailed == 5:
theJobReport.status = "Failed"
theJobReport.exitCode = 11115
errDetail = theJobReport.addError(11115, "CmsGenFailure")
errDetail['Description'] = "Error making a link"
elif jobFailed == 6:
theJobReport.status = "Failed"
theJobReport.exitCode = 11116
errDetail = theJobReport.addError(11116, "CmsGenFailure")
errDetail['Description'] = "Error preparing the executation"
else:
theJobReport.status = "Failed"
theJobReport.exitCode = 11117
errDetail = theJobReport.addError(11117, "CmsGenFailure")
errDetail['Description'] = "General cmsGen Failure"
if jobFailed == 0:
file = open("nCmsGenEvents.txt", 'r')
nEventsLine = file.readline()
nEvents = nEventsLine.split('\n')
file.close()
totalEvents = nEvents[0]
outputFileUnw = ""
if generator == "alpgen":
outputFileUnw = "%s.unw" % outputFile
elif generator == "comphep":
outputFileUnw = "%s" % outputFile
elif generator == "madgraph":
outputFileUnw = "%s" % outputFile
if not os.path.exists(outputFileUnw):
msg = "Output file Not Found: %s" % outputFileUnw
raise RuntimeError, msg
totalSize = os.stat(outputFileUnw)[6]
else:
totalEvents = 0
totalSize = 0
newFile = theJobReport.newFile()
newFile["LFN"] = "None"
if generator == "alpgen":
#newFile["PFN"] = "%s/%s" % (os.getcwd(),outputFile)
newFile["PFN"] = "%s" % outputFile
elif generator == "comphep":
newFile["PFN"] = "%s" % outputFile
elif generator == "madgraph":
newFile["PFN"] = "%s" % outputFile
newFile["Size"] = totalSize
newFile["TotalEvents"] = totalEvents
newFile["ModuleLabel"] = "cmsGen"
theJobReport.write(jobReport)
def processFrameworkJobReport():
"""
Runtime tool for processing the Framework Job Report produced
by CMsGen executables
"""
theJobReport = FwkJobReport()
#
# If everything worked well set the success status
# else record a failure and error
if jobFailed == 0:
theJobReport.status = "Success"
theJobReport.exitCode = 0
elif jobFailed == 1:
theJobReport.status = "Failed"
theJobReport.exitCode = 11111
errDetail = theJobReport.addError(11111, "CmsGenFailure")
errDetail['Description'] = "Error retrieving grid file"
elif jobFailed == 2:
theJobReport.status = "Failed"
theJobReport.exitCode = 11112
errDetail = theJobReport.addError(11112, "CmsGenFailure")
errDetail['Description'] = "Error producing step 1 events"
elif jobFailed == 3:
theJobReport.status = "Failed"
theJobReport.exitCode = 11113
errDetail = theJobReport.addError(11113, "CmsGenFailure")
errDetail['Description'] = "Error producing step 2 events"
elif jobFailed == 4:
theJobReport.status = "Failed"
theJobReport.exitCode = 11114
errDetail = theJobReport.addError(11114, "CmsGenFailure")
errDetail[
'Description'] = "Error computing the number of produced events"
elif jobFailed == 5:
theJobReport.status = "Failed"
theJobReport.exitCode = 11115
errDetail = theJobReport.addError(11115, "CmsGenFailure")
errDetail['Description'] = "Error making a link"
elif jobFailed == 6:
theJobReport.status = "Failed"
theJobReport.exitCode = 11116
errDetail = theJobReport.addError(11116, "CmsGenFailure")
errDetail['Description'] = "Error preparing the executation"
else:
theJobReport.status = "Failed"
theJobReport.exitCode = 11117
errDetail = theJobReport.addError(11117, "CmsGenFailure")
errDetail['Description'] = "General cmsGen Failure"
if jobFailed == 0:
file = open("nCmsGenEvents.txt", 'r')
nEventsLine = file.readline()
nEvents = nEventsLine.split('\n')
file.close()
totalEvents = nEvents[0]
outputFileUnw = ""
if generator == "alpgen":
outputFileUnw = "%s.unw" % outputFile
elif generator == "comphep":
outputFileUnw = "%s" % outputFile
elif generator == "madgraph":
outputFileUnw = "%s" % outputFile
if not os.path.exists(outputFileUnw):
msg = "Output file Not Found: %s" % outputFileUnw
raise RuntimeError, msg
totalSize = os.stat(outputFileUnw)[6]
else:
totalEvents = 0
totalSize = 0
newFile = theJobReport.newFile()
newFile["LFN"] = "None"
if generator == "alpgen":
#newFile["PFN"] = "%s/%s" % (os.getcwd(),outputFile)
newFile["PFN"] = "%s" % outputFile
elif generator == "comphep":
newFile["PFN"] = "%s" % outputFile
elif generator == "madgraph":
newFile["PFN"] = "%s" % outputFile
newFile["Size"] = totalSize
newFile["TotalEvents"] = totalEvents
newFile["ModuleLabel"] = "cmsGen"
theJobReport.write(jobReport)
if not os.path.exists(cfgFile):
msg = "Cfg File Not Found: %s" % cfgFile
raise RuntimeError, msg
if generator == "alpgen":
jobFailed = alpGen()
processFrameworkJobReport()
elif generator == "comphep":
jobFailed = compHep()
processFrameworkJobReport()
elif generator == "madgraph":
jobFailed = madgraph()
processFrameworkJobReport()
elif generator == "sherpa":
jobFailed = sherpa()
# Create a dummy FwkJobReport
theJobReport = FwkJobReport()
if jobFailed == 0:
theJobReport.status = "Success"
theJobReport.exitCode = 0
theJobReport.write(jobReport)
else:
theJobReport.status = "Failed"
theJobReport.exitCode = jobFailed
errDetail = theJobReport.addError(jobFailed, "CmsGenFailure")
errDetail['Description'] = "Failed to set up Sherpa Libs"
else:
msg = "Generator %s Not Found" % generator
raise RuntimeError, msg
def __init__(self):
ContentHandler.__init__(self)
# //
# // State containers used during parsing
#//
self.currentReport = FwkJobReport()
self.currentFile = None
self.currentDict = None
self.currentInputDict = None
self.currentCksum = None
self.inTiming = False
self.inGenInfo = False
self._CharCache = ""
# //
# // container for results
#//
self.results = []
# //
# // Response methods to start of elements based on element name
#//
self._StartResponses = {
"FrameworkJobReport": self.newReport,
"File": self.newFile,
"InputFile": self.newInputFile,
"Dataset": self.newDataset,
"ExitCode": self.exitCode,
"State": self.stateHandler,
"Branch": self.noResponse,
"Branches": self.noResponse,
"Input": self.startInput,
"Inputs": self.noResponse,
"Runs": self.noResponse,
"Run": self.noResponse,
"SkippedEvent": self.skippedEvent,
"SkippedFile": self.skippedFile,
"Checksum": self.checksum,
"SiteDetail": self.siteDetail,
"FrameworkError": self.frameworkError,
"TimingService": self.timingService,
"StorageStatistics": self.noResponse,
"GeneratorInfo": self.generatorInfo,
"Data": self.dataHandler,
"ReadBranches": self.readBranches,
}
# //
# // Response methods to end of elements based on element name
#//
self._EndResponses = {
"FrameworkJobReport": self.endReport,
"File": self.endFile,
"Dataset": self.endDataset,
"ExitCode": self.noResponse,
"State": self.noResponse,
"Branch": self.endBranch,
"Branches": self.noResponse,
"InputFile": self.endInputFile,
"Input": self.endInput,
"Inputs": self.noResponse,
"Runs": self.noResponse,
"Run": self.endRun,
"SkippedEvent": self.noResponse,
"SkippedFile": self.noResponse,
"Checksum": self.endChecksum,
"SiteDetail": self.noResponse,
"FrameworkError": self.endFrameworkError,
"TimingService": self.endTimingService,
"StorageStatistics": self.storageStatistics,
"GeneratorInfo": self.endGeneratorInfo,
"Data": self.noResponse,
"ReadBranches": self.endReadBranches,
}
class FwkJobRepHandler(ContentHandler):
"""
_FwkJobRepHandler_
SAX Content Handler implementation to build
instances of FwkJobReport and populate it
Multiple job reports in a file are supported, since the plan
is to concatenate them at the end of a job from multiple outputs
instances of FwkJobReport are stored in self.results as a list
"""
def __init__(self):
ContentHandler.__init__(self)
# //
# // State containers used during parsing
#//
self.currentReport = FwkJobReport()
self.currentFile = None
self.currentDict = None
self.currentInputDict = None
self.currentCksum = None
self.inTiming = False
self.inGenInfo = False
self._CharCache = ""
# //
# // container for results
#//
self.results = []
# //
# // Response methods to start of elements based on element name
#//
self._StartResponses = {
"FrameworkJobReport": self.newReport,
"File": self.newFile,
"InputFile": self.newInputFile,
"Dataset": self.newDataset,
"ExitCode": self.exitCode,
"State": self.stateHandler,
"Branch": self.noResponse,
"Branches": self.noResponse,
"Input": self.startInput,
"Inputs": self.noResponse,
"Runs": self.noResponse,
"Run": self.noResponse,
"SkippedEvent": self.skippedEvent,
"SkippedFile": self.skippedFile,
"Checksum": self.checksum,
"SiteDetail": self.siteDetail,
"FrameworkError": self.frameworkError,
"TimingService": self.timingService,
"StorageStatistics": self.noResponse,
"GeneratorInfo": self.generatorInfo,
"Data": self.dataHandler,
"ReadBranches": self.readBranches,
}
# //
# // Response methods to end of elements based on element name
#//
self._EndResponses = {
"FrameworkJobReport": self.endReport,
"File": self.endFile,
"Dataset": self.endDataset,
"ExitCode": self.noResponse,
"State": self.noResponse,
"Branch": self.endBranch,
"Branches": self.noResponse,
"InputFile": self.endInputFile,
"Input": self.endInput,
"Inputs": self.noResponse,
"Runs": self.noResponse,
"Run": self.endRun,
"SkippedEvent": self.noResponse,
"SkippedFile": self.noResponse,
"Checksum": self.endChecksum,
"SiteDetail": self.noResponse,
"FrameworkError": self.endFrameworkError,
"TimingService": self.endTimingService,
"StorageStatistics": self.storageStatistics,
"GeneratorInfo": self.endGeneratorInfo,
"Data": self.noResponse,
"ReadBranches": self.endReadBranches,
}
def noResponse(self, name, attrs={}):
"""some elements require no action"""
if self.inTiming:
value = str(attrs.get("Value"))
self.currentDict[str(name)] = value
pass
def startElement(self, name, attrs):
"""
_startElement_
Override ContentHandler.startElement
Start a new XML Element, call the appropriate response method based
off the name of the element
"""
response = self._StartResponses.get(name, self.noResponse)
response(name, attrs)
return
def endElement(self, name):
"""
_endElement_
Override ContentHandler.endElement
End of element, invoke response based on name and
flush the chardata cache
"""
response = self._EndResponses.get(name, self.fillDictionary)
response(name)
self._CharCache = ""
return
def characters(self, data):
"""
_characters_
Override ContentHandler.characters
Accumulate character data from an xml element, if required
the response will pick it up and insert it into the appropriate
object.
"""
if len(data.strip()) == 0:
return
self._CharCache += str(data).replace("\t", "")
return
def inFile(self):
"""boolean test to see if state is in a file block"""
return self.currentFile != None
def newReport(self, name, attrs):
"""
_newReport_
Handler method for a new FrameworkJobReport
"""
self.currentReport = FwkJobReport()
name = attrs.get("Name", None)
status = attrs.get("Status", None)
jobSpec = attrs.get("JobSpecID", None)
workSpec = attrs.get("WorkflowSpecID", None)
jobType = attrs.get("JobType", None)
if name != None:
self.currentReport.name = str(name)
if status != None:
self.currentReport.status = str(status)
if jobSpec != None:
self.currentReport.jobSpecId = str(jobSpec)
if jobType != None:
self.currentReport.jobType = str(jobType)
if workSpec != None:
self.currentReport.workflowSpecId = str(workSpec)
return
def endReport(self, name):
"""
_endReport_
Handler Method for finishing a FrameorkJobReport
"""
self.results.append(self.currentReport)
self.currentReport = None
return
def newFile(self, name, attrs):
"""new File tag encountered"""
self.currentFile = self.currentReport.newFile()
self.currentDict = self.currentFile
def endFile(self, name):
""" end of file tag encountered"""
self.currentFile = None
self.currentDict = None
def newInputFile(self, name, attrs):
"""new InputFile tag encountered"""
self.currentFile = self.currentReport.newInputFile()
self.currentDict = self.currentFile
def endInputFile(self, name):
""" end of InputFile tag encountered"""
self.currentFile = None
self.currentDict = None
def newDataset(self, name, attrs):
""" start of Dataset tag within a File tag"""
if not self.inFile():
return
self.currentDict = self.currentFile.newDataset()
return
def endDataset(self, name):
"""end of Dataset tag"""
if not self.inFile():
return
self.currentDict = self.currentFile
def exitCode(self, name, attrs):
"""
handle an ExitCode node, extract the value attr and add it to
the current report
"""
if self.currentReport == None:
return
value = attrs.get("Value", None)
if value != None:
self.currentReport.exitCode = int(value)
return
def fillDictionary(self, name):
"""
_fillDictionary_
Any object requiring population as a dictionary can use this
handler to populate itself
"""
if self.currentDict == None:
return
if self.inTiming:
return
self.currentDict[str(name)] = str(self._CharCache)
return
def stateHandler(self, name, attrs):
"""
_stateHandler_
Handle a State Node
"""
self.currentFile.state = str(attrs.get('Value', "closed"))
def endBranch(self, name):
"""
_endBranch_
End of Branch node
"""
if self.currentFile != None:
self.currentFile.branches.append(str(self._CharCache))
def startInput(self, name, attrs):
"""
_startInput_
Handle Input reference for an Output File
"""
if self.currentFile != None:
if not self.currentFile.isInput:
self.currentInputDict = {}
self.currentDict = self.currentInputDict
def endInput(self, name):
"""
_endInput_
end Input node
"""
self.currentFile.addInputFile(self.currentInputDict['PFN'],
self.currentInputDict['LFN'])
self.currentInputDict = None
self.currentDict = self.currentFile
def endRun(self, name):
"""
_endRun_
Add Run Number chardata to current file
"""
if self.currentFile != None:
self.currentFile.runs.append(str(self._CharCache))
def skippedEvent(self, name, attrs):
"""
_skippedEvent_
Record a Skipped Event
"""
if self.currentReport == None:
return
run = attrs.get("Run", None)
if run == None:
return
event = attrs.get("Event", None)
if event == None:
return
self.currentReport.addSkippedEvent(str(run), str(event))
return
def skippedFile(self, name, attrs):
"""skipped file node"""
if self.currentReport == None:
return
pfn = attrs.get("Pfn", None)
lfn = attrs.get("Lfn", None)
if pfn == None:
return
if lfn == None:
return
self.currentReport.addSkippedFile(str(pfn), str(lfn))
return
def checksum(self, name, attrs):
"""
_checksum_
Handle a checksum element start
"""
self.currentCksum = attrs.get("Algorithm", None)
def endChecksum(self, name):
"""
_endChecksum_
Handle a checksum element end
"""
if self.currentCksum == None:
return
if not self.inFile():
return
self.currentFile.addChecksum(str(self.currentCksum),
str(self._CharCache))
self.currentCksum = None
return
def siteDetail(self, name, attrs):
"""
_siteDetail_
Handle a site detail parameter node
"""
if self.currentReport == None:
return
detailName = attrs.get('Parameter', None)
detailValue = attrs.get('Value', None)
if detailName == None:
return
self.currentReport.siteDetails[str(detailName)] = str(detailValue)
return
def frameworkError(self, name, attrs):
"""
_frameworkError_
Start Error message in job report
"""
if self.currentReport == None:
return
errStatus = str(attrs.get("ExitStatus", "1"))
errType = str(attrs.get("Type", "Unknown"))
self.currentDict = self.currentReport.addError(errStatus, errType)
return
def endFrameworkError(self, name):
"""
_endFrameworkError_
End Error block in job report
"""
if self.currentReport == None:
return
if self.currentDict == None:
return
self.currentDict['Description'] = str(self._CharCache)
self.currentDict == None
return
def timingService(self, name, attrs):
"""
_timingService_
Start of Timing Service element
"""
if self.currentReport == None:
return
self.inTiming = True
self.currentDict = self.currentReport.timing
return
def endTimingService(self, name):
"""
_endTimingService_
End of Timing Service element
"""
if self.currentReport == None:
return
self.inTiming = False
self.currentDict = None
return
def readBranches(self, name, attrs):
"""
_readBranches_
Start of readBranches element
"""
if self.currentReport == None:
return
self.inTiming = True
self.currentDict = self.currentReport.readBranches
return
def endReadBraches(self, name):
"""
_endReadBranches_
End of ReadBranches element
"""
if self.currentReport == None:
return
self.inTiming = False
self.currentDict = None
return
def storageStatistics(self, name):
"""
_storageStatistics_
Big blob of chardata...
"""
if self.currentReport == None:
return
self.currentReport.storageStatistics = str(self._CharCache)
return
def generatorInfo(self, name, attrs):
"""
_generatorInfo_
"""
if self.currentReport == None:
return
self.inGenInfo = True
self.currentDict = self.currentReport.generatorInfo
return
def endGeneratorInfo(self, name):
"""
_endGeneratorInfo_
"""
self.inGenInfo = False
self.currentDict = None
return
def dataHandler(self, name, attrs):
"""
_dataHandler_
"""
if self.inGenInfo:
if self.currentDict == None:
return
key = str(attrs['Name'])
val = str(attrs['Value'])
self.currentDict[key] = val
return
return
if not os.path.exists(cfgFile):
msg = "Cfg File Not Found: %s" % cfgFile
raise RuntimeError, msg
if generator == "alpgen":
jobFailed = alpGen()
processFrameworkJobReport()
elif generator == "comphep":
jobFailed = compHep()
processFrameworkJobReport()
elif generator == "madgraph":
jobFailed = madgraph()
processFrameworkJobReport()
elif generator == "sherpa":
jobFailed = sherpa()
# Create a dummy FwkJobReport
theJobReport = FwkJobReport()
if jobFailed == 0:
theJobReport.status = "Success"
theJobReport.exitCode = 0
theJobReport.write(jobReport)
else:
theJobReport.status = "Failed"
theJobReport.exitCode = jobFailed
errDetail = theJobReport.addError(jobFailed, "CmsGenFailure")
errDetail['Description'] = "Failed to set up Sherpa Libs"
else:
msg = "Generator %s Not Found" % generator
raise RuntimeError, msg
def __fwkJobReportCommon(self, jobSpecLoaded, workerNodeInfo):
"""
__fwkJobReportCommon_
Create a new job report and fill it in with generic
information that is not dependent on the outcome of
the job.
The jobSpecLoaded parameter is a reference to an instance
of the JobSpec class that has been initialized with the
job spec that we are generating a report for.
"""
#workerNodeInfo = RandomAllocationPlugin().allocateJob()
try:
jobSpecPayload = jobSpecLoaded.payload
newReport = FwkJobReport()
newReport.jobSpecId = jobSpecPayload.jobName
newReport.jobType = jobSpecPayload.type
newReport.workflowSpecId = jobSpecPayload.workflow
newReport.name = jobSpecPayload.name
#get information from the super class
newReport.siteDetails['SiteName'] = workerNodeInfo['SiteName']
#HostName is the same as worker_node name
newReport.siteDetails['HostName'] = workerNodeInfo['HostName']
newReport.siteDetails['se-name'] = workerNodeInfo['se-name']
newReport.siteDetails['ce-name'] = workerNodeInfo['ce-name']
newReport.addLogFile("/path/to/log/archive", "some.random.se.cern.ch")
return jobSpecPayload, newReport
except Exception, ex:
#msg = "Unable to Publish Report for %s\n" % jobSpecPayload.jobName
#msg += "Since It is not known to the JobState System:\n"
msg = str(ex)
logging.error(msg)
raise RuntimeError, msg
def __init__(self):
ContentHandler.__init__(self)
# //
# // State containers used during parsing
#//
self.currentReport = FwkJobReport()
self.currentFile = None
self.currentDict = None
self.currentInputDict = None
self.currentCksum = None
self.inTiming = False
self.inGenInfo = False
self._CharCache = ""
# //
# // container for results
#//
self.results = []
# //
# // Response methods to start of elements based on element name
#//
self._StartResponses = {
"FrameworkJobReport" : self.newReport,
"File" : self.newFile,
"InputFile": self.newInputFile,
"Dataset" : self.newDataset,
"ExitCode" : self.exitCode,
"State" : self.stateHandler,
"Branch" : self.noResponse,
"Branches" : self.noResponse,
"Input" : self.startInput,
"Inputs" : self.noResponse,
"Runs" : self.noResponse,
"Run" : self.noResponse,
"SkippedEvent" : self.skippedEvent,
"SkippedFile" : self.skippedFile,
"Checksum" : self.checksum,
"SiteDetail" : self.siteDetail,
"FrameworkError" : self.frameworkError,
"TimingService" : self.timingService,
"StorageStatistics" : self.noResponse,
"GeneratorInfo" : self.generatorInfo,
"Data" : self.dataHandler,
}
# //
# // Response methods to end of elements based on element name
#//
self._EndResponses = {
"FrameworkJobReport" : self.endReport,
"File" : self.endFile,
"Dataset" : self.endDataset,
"ExitCode": self.noResponse,
"State" : self.noResponse,
"Branch" : self.endBranch,
"Branches" : self.noResponse,
"InputFile" : self.endInputFile,
"Input" :self.endInput,
"Inputs" : self.noResponse,
"Runs" : self.noResponse,
"Run" : self.endRun,
"SkippedEvent" : self.noResponse,
"SkippedFile" : self.noResponse,
"Checksum" : self.endChecksum,
"SiteDetail" : self.noResponse,
"FrameworkError" : self.endFrameworkError,
"TimingService" : self.endTimingService,
"StorageStatistics" : self.storageStatistics,
"GeneratorInfo" : self.endGeneratorInfo,
"Data": self.noResponse,
}
class FwkJobRepHandler(ContentHandler):
"""
_FwkJobRepHandler_
SAX Content Handler implementation to build
instances of FwkJobReport and populate it
Multiple job reports in a file are supported, since the plan
is to concatenate them at the end of a job from multiple outputs
instances of FwkJobReport are stored in self.results as a list
"""
def __init__(self):
ContentHandler.__init__(self)
# //
# // State containers used during parsing
#//
self.currentReport = FwkJobReport()
self.currentFile = None
self.currentDict = None
self.currentInputDict = None
self.currentCksum = None
self.inTiming = False
self.inGenInfo = False
self._CharCache = ""
# //
# // container for results
#//
self.results = []
# //
# // Response methods to start of elements based on element name
#//
self._StartResponses = {
"FrameworkJobReport" : self.newReport,
"File" : self.newFile,
"InputFile": self.newInputFile,
"Dataset" : self.newDataset,
"ExitCode" : self.exitCode,
"State" : self.stateHandler,
"Branch" : self.noResponse,
"Branches" : self.noResponse,
"Input" : self.startInput,
"Inputs" : self.noResponse,
"Runs" : self.noResponse,
"Run" : self.noResponse,
"SkippedEvent" : self.skippedEvent,
"SkippedFile" : self.skippedFile,
"Checksum" : self.checksum,
"SiteDetail" : self.siteDetail,
"FrameworkError" : self.frameworkError,
"TimingService" : self.timingService,
"StorageStatistics" : self.noResponse,
"GeneratorInfo" : self.generatorInfo,
"Data" : self.dataHandler,
}
# //
# // Response methods to end of elements based on element name
#//
self._EndResponses = {
"FrameworkJobReport" : self.endReport,
"File" : self.endFile,
"Dataset" : self.endDataset,
"ExitCode": self.noResponse,
"State" : self.noResponse,
"Branch" : self.endBranch,
"Branches" : self.noResponse,
"InputFile" : self.endInputFile,
"Input" :self.endInput,
"Inputs" : self.noResponse,
"Runs" : self.noResponse,
"Run" : self.endRun,
"SkippedEvent" : self.noResponse,
"SkippedFile" : self.noResponse,
"Checksum" : self.endChecksum,
"SiteDetail" : self.noResponse,
"FrameworkError" : self.endFrameworkError,
"TimingService" : self.endTimingService,
"StorageStatistics" : self.storageStatistics,
"GeneratorInfo" : self.endGeneratorInfo,
"Data": self.noResponse,
}
def noResponse(self, name, attrs = {}):
"""some elements require no action"""
if self.inTiming:
value = str(attrs.get("Value"))
self.currentDict[str(name)] = value
pass
def startElement(self, name, attrs):
"""
_startElement_
Override ContentHandler.startElement
Start a new XML Element, call the appropriate response method based
off the name of the element
"""
response = self._StartResponses.get(name, self.noResponse)
response(name, attrs)
return
def endElement(self, name):
"""
_endElement_
Override ContentHandler.endElement
End of element, invoke response based on name and
flush the chardata cache
"""
response = self._EndResponses.get(name, self.fillDictionary)
response(name)
self._CharCache = ""
return
def characters(self, data):
"""
_characters_
Override ContentHandler.characters
Accumulate character data from an xml element, if required
the response will pick it up and insert it into the appropriate
object.
"""
if len(data.strip()) == 0:
return
self._CharCache += str(data).replace("\t", "")
return
def inFile(self):
"""boolean test to see if state is in a file block"""
return self.currentFile != None
def newReport(self, name, attrs):
"""
_newReport_
Handler method for a new FrameworkJobReport
"""
self.currentReport = FwkJobReport()
name = attrs.get("Name", None)
status = attrs.get("Status", None)
jobSpec = attrs.get("JobSpecID", None)
workSpec = attrs.get("WorkflowSpecID", None)
jobType = attrs.get("JobType", None)
if name != None:
self.currentReport.name = str(name)
if status != None:
self.currentReport.status = str(status)
if jobSpec != None:
self.currentReport.jobSpecId = str(jobSpec)
if jobType != None:
self.currentReport.jobType = str(jobType)
if workSpec != None:
self.currentReport.workflowSpecId = str(workSpec)
return
def endReport(self, name):
"""
_endReport_
Handler Method for finishing a FrameorkJobReport
"""
self.results.append(self.currentReport)
self.currentReport = None
return
def newFile(self, name, attrs):
"""new File tag encountered"""
self.currentFile = self.currentReport.newFile()
self.currentDict = self.currentFile
def endFile(self, name):
""" end of file tag encountered"""
self.currentFile = None
self.currentDict = None
def newInputFile(self, name, attrs):
"""new InputFile tag encountered"""
self.currentFile = self.currentReport.newInputFile()
self.currentDict = self.currentFile
def endInputFile(self, name):
""" end of InputFile tag encountered"""
self.currentFile = None
self.currentDict = None
def newDataset(self, name, attrs):
""" start of Dataset tag within a File tag"""
if not self.inFile():
return
self.currentDict = self.currentFile.newDataset()
return
def endDataset(self, name):
"""end of Dataset tag"""
if not self.inFile():
return
self.currentDict = self.currentFile
def exitCode(self, name, attrs):
"""
handle an ExitCode node, extract the value attr and add it to
the current report
"""
if self.currentReport == None:
return
value = attrs.get("Value", None)
if value != None:
self.currentReport.exitCode = int(value)
return
def fillDictionary(self, name):
"""
_fillDictionary_
Any object requiring population as a dictionary can use this
handler to populate itself
"""
if self.currentDict == None:
return
if self.inTiming:
return
self.currentDict[str(name)] = str(self._CharCache)
return
def stateHandler(self, name, attrs):
"""
_stateHandler_
Handle a State Node
"""
self.currentFile.state = str(attrs.get('Value', "closed"))
def endBranch(self, name):
"""
_endBranch_
End of Branch node
"""
if self.currentFile != None:
self.currentFile.branches.append(str(self._CharCache))
def startInput(self, name, attrs):
"""
_startInput_
Handle Input reference for an Output File
"""
if self.currentFile != None:
if not self.currentFile.isInput:
self.currentInputDict = {}
self.currentDict = self.currentInputDict
def endInput(self, name):
"""
_endInput_
end Input node
"""
self.currentFile.addInputFile(self.currentInputDict['PFN'],
self.currentInputDict['LFN'])
self.currentInputDict = None
self.currentDict = self.currentFile
def endRun(self, name):
"""
_endRun_
Add Run Number chardata to current file
"""
if self.currentFile != None:
self.currentFile.runs.append(str(self._CharCache))
def skippedEvent(self, name, attrs):
"""
_skippedEvent_
Record a Skipped Event
"""
if self.currentReport == None:
return
run = attrs.get("Run", None)
if run == None :
return
event = attrs.get("Event", None)
if event == None:
return
self.currentReport.addSkippedEvent(str(run), str(event))
return
def skippedFile(self, name, attrs):
"""skipped file node"""
if self.currentReport == None:
return
pfn = attrs.get("Pfn", None)
lfn = attrs.get("Lfn", None)
if pfn == None:
return
if lfn == None:
return
self.currentReport.addSkippedFile(str(pfn), str(lfn))
return
def checksum(self, name, attrs):
"""
_checksum_
Handle a checksum element start
"""
self.currentCksum = attrs.get("Algorithm", None)
def endChecksum(self, name):
"""
_endChecksum_
Handle a checksum element end
"""
if self.currentCksum == None:
return
if not self.inFile():
return
self.currentFile.addChecksum(str(self.currentCksum),
str(self._CharCache))
self.currentCksum = None
return
def siteDetail(self, name, attrs):
"""
_siteDetail_
Handle a site detail parameter node
"""
if self.currentReport == None:
return
detailName = attrs.get('Parameter', None)
detailValue = attrs.get('Value' , None)
if detailName == None:
return
self.currentReport.siteDetails[str(detailName)] = str(detailValue)
return
def frameworkError(self, name, attrs):
"""
_frameworkError_
Start Error message in job report
"""
if self.currentReport == None:
return
errStatus = str(attrs.get("ExitStatus", "1"))
errType = str(attrs.get("Type", "Unknown"))
self.currentDict = self.currentReport.addError(errStatus, errType)
return
def endFrameworkError(self, name):
"""
_endFrameworkError_
End Error block in job report
"""
if self.currentReport == None:
return
if self.currentDict == None:
return
self.currentDict['Description'] = str(self._CharCache)
self.currentDict == None
return
def timingService(self, name, attrs):
"""
_timingService_
Start of Timing Service element
"""
if self.currentReport == None:
return
self.inTiming = True
self.currentDict = self.currentReport.timing
return
def endTimingService(self, name):
"""
_endTimingService_
End of Timing Service element
"""
if self.currentReport == None:
return
self.inTiming = False
self.currentDict = None
return
def storageStatistics(self, name):
"""
_storageStatistics_
Big blob of chardata...
"""
if self.currentReport == None:
return
self.currentReport.storageStatistics = str(self._CharCache)
return
def generatorInfo(self, name, attrs):
"""
_generatorInfo_
"""
if self.currentReport == None:
return
self.inGenInfo = True
self.currentDict = self.currentReport.generatorInfo
return
def endGeneratorInfo(self, name):
"""
_endGeneratorInfo_
"""
self.inGenInfo = False
self.currentDict = None
return
def dataHandler(self, name, attrs):
"""
_dataHandler_
"""
if self.inGenInfo:
if self.currentDict == None:
return
key = str(attrs['Name'])
val = str(attrs['Value'])
self.currentDict[key] = val
return
return
Instantiate a new ContentHandler and run it over the file producing
a list of FwkJobReport instances
"""
#handler = FwkJobRepHandler()
#parser = make_parser()
#parser.setContentHandler(handler)
#try:
# parser.parse(filename)
#except SAXParseException, ex:
# msg = "Error parsing JobReport File: %s\n" % filename
# msg += str(ex)
# print msg
# return []
#print handler.results[0]
#return handler.results
try:
improvDoc = loadIMProvFile(filename)
except Exception, ex:
return []
result = []
reportQ = IMProvQuery("FrameworkJobReport")
reportInstances = reportQ(improvDoc)
for reportInstance in reportInstances:
newReport = FwkJobReport()
newReport.load(reportInstance)
result.append(newReport)
return result
def stageOut():
"""
_stageOut_
Main function for this module. Loads data from the task
and manages the stage out process for a single attempt
"""
state = TaskState(os.getcwd())
state.loadRunResDB()
workflow = WorkflowSpec()
workflow.load(os.environ['PRODAGENT_WORKFLOW_SPEC'])
jobSpecFile = os.environ.get('PRODAGENT_JOBSPEC')
jobSpecId = None
if jobSpecFile is not None:
jobSpec = JobSpec()
jobSpec.load(jobSpecFile)
jobSpecId = jobSpec.parameters.get('JobName')
print workflow
print state.taskName()
print jobSpecId
stageOutFor, override, controls = StageOutUtils.getStageOutConfig(
workflow, state.taskName())
toplevelReport = os.path.join(os.environ['PRODAGENT_JOB_DIR'],
"FrameworkJobReport.xml")
exitCode = 0
# //
# // find inputs by locating the task for which we are staging out
#// and loading its TaskState
for inputTask in stageOutFor:
print "Attempting to stage out files for node %s" % inputTask
try:
inputState = getTaskState(inputTask)
msg = "Loaded Input Task: %s " % inputTask
except Exception, ex:
msg = "Error load for TaskState for task %s" % inputTask
msg += "%s\n" % str(ex)
inputState = None
print msg
if inputState == None:
# exit with init error
# generate failure report in this dir, since cant find
# input state dir
inputReport = FwkJobReport()
inputReport.name = inputTask
inputReport.jobSpecId = jobSpecId
exitCode = 60311
errRep = inputReport.addError(
60311, "TaskStateError")
errRep['Description'] = msg
inputReport.status = "Failed"
inputReport.exitCode = 60311
updateReport(toplevelReport, inputReport)
print "TaskState is None, exiting..."
return exitCode
try:
inputReport = inputState.getJobReport()
msg = "Loaded JobReport for Task : %s\n" % inputTask
msg += "File: %s\n" % inputState.jobReport
except Exception, ex:
msg = "Error loading input report : %s" % str(ex)
inputReport = None
return exitCode
try:
inputReport = inputState.getJobReport()
msg = "Loaded JobReport for Task : %s\n" % inputTask
msg += "File: %s\n" % inputState.jobReport
except Exception, ex:
msg = "Error loading input report : %s" % str(ex)
inputReport = None
print msg
if inputReport == None:
msg += "Unable to read Job Report for input task: %s\n" % inputTask
msg += "Looked for file: %s\n" % inputState.jobReport
print msg
inputReport = FwkJobReport()
inputReport.name = inputTask
inputReport.jobSpecId = jobSpecId
exitCode = 60311
errRep = inputReport.addError(
60311, "InputReportError")
errRep['Description'] = msg
inputReport.status = "Failed"
inputReport.exitCode = 60311
updateReport(toplevelReport, inputReport)
# exit with init error
return 60311
try:
if fileType == 'streamer' or fileType == 'edm' or fileType == 'pixdmp':
if nEvents == None:
msg = "--nevents option not provided"
raise RuntimeError, msg
elif fileType == 'lumi':
nEvents = 0
elif fileType == 'lumi-sa':
nEvents = 0
elif fileType == 'lumi-vdm':
nEvents = 0
# temporary, PRODCOMMON code does not support DBS registration w/0 lumi section anymore
if fileType == 'pixdmp':
lumiSection = 1
jobReport = FwkJobReport()
jobReportFile = jobReport.newFile()
if fileType == 'streamer':
jobReportFile['FileType'] = 'STREAMER'
elif fileType == 'edm':
jobReportFile['FileType'] = 'EDM'
elif fileType == 'lumi':
jobReportFile['FileType'] = 'LUMI'
elif fileType == 'lumi-sa':
jobReportFile['FileType'] = 'LUMI-SA'
elif fileType == 'lumi-vdm':
jobReportFile['FileType'] = 'LUMI-VDM'
elif fileType == 'pixdmp':
jobReportFile['FileType'] = 'PIXDMP'
else:
def processCleanUpJobReport(self,statusCode):
"""
_processCleanUpJobReport_
Arguments:
None
Return:
None
"""
# //
# // Generate a report
# //
report = FwkJobReport()
report.name = "cleanUp"
report.status = "Failed"
if statusCode == 0 :
report.status = "Success"
for lfnRemovedFile in self.success:
report.addRemovedFile(lfnRemovedFile, self.seName)
for lfnUnremovedFile in self.failed:
report.addUnremovedFile(lfnUnremovedFile, self.seName)
report.exitCode = statusCode
report.jobSpecId = self.state.jobSpecNode.jobName
report.jobType = self.state.jobSpecNode.jobType
report.workflowSpecId = self.state.jobSpecNode.workflow
report.write("./FrameworkJobReport.xml")
# //
# // Ensure this report gets added to the job-wide report
#//
toplevelReport = os.path.join(os.environ['PRODAGENT_JOB_DIR'],"FrameworkJobReport.xml")
newReport = os.path.join(os.getcwd(), "FrameworkJobReport.xml")
mergeReports(toplevelReport, newReport)
