def writeFwkJobReport( self, jobSpecId, exitCode, reportfilename ):
"""
__writeFwkJobReport__
write a fake reportfilename based on the status reported
"""
# create job report
logging.debug("Creating report %s" % reportfilename)
fwjr = FwkJobReport()
fwjr.jobSpecId = jobSpecId
if exitCode == 0 :
fwjr.status = "Success"
fwjr.exitCode = 0
else :
fwjr.exitCode = exitCode
fwjr.status = "Failed"
# store job report
fwjr.write(reportfilename)
def processCleanUpJobReport(self,statusCode):
"""
_processCleanUpJobReport_
Arguments:
None
Return:
None
"""
# //
# // Generate a report
# //
report = FwkJobReport()
report.name = "cleanUp"
report.status = "Failed"
if statusCode == 0 :
report.status = "Success"
for lfnRemovedFile in self.success:
report.addRemovedFile(lfnRemovedFile, self.seName)
for lfnUnremovedFile in self.failed:
report.addUnremovedFile(lfnUnremovedFile, self.seName)
report.exitCode = statusCode
report.jobSpecId = self.state.jobSpecNode.jobName
report.jobType = self.state.jobSpecNode.jobType
report.workflowSpecId = self.state.jobSpecNode.workflow
report.write("./FrameworkJobReport.xml")
# //
# // Ensure this report gets added to the job-wide report
#//
toplevelReport = os.path.join(os.environ['PRODAGENT_JOB_DIR'],"FrameworkJobReport.xml")
newReport = os.path.join(os.getcwd(), "FrameworkJobReport.xml")
mergeReports(toplevelReport, newReport)
def processFrameworkJobReport():
"""
Runtime tool for processing the Framework Job Report produced
by CMsGen executables
"""
theJobReport = FwkJobReport()
#
# If everything worked well set the success status
# else record a failure and error
if jobFailed == 0:
theJobReport.status = "Success"
theJobReport.exitCode = 0
elif jobFailed == 1:
theJobReport.status = "Failed"
theJobReport.exitCode = 11111
errDetail = theJobReport.addError(11111, "CmsGenFailure")
errDetail['Description'] = "Error retrieving grid file"
elif jobFailed == 2:
theJobReport.status = "Failed"
theJobReport.exitCode = 11112
errDetail = theJobReport.addError(11112, "CmsGenFailure")
errDetail['Description'] = "Error producing step 1 events"
elif jobFailed == 3:
theJobReport.status = "Failed"
theJobReport.exitCode = 11113
errDetail = theJobReport.addError(11113, "CmsGenFailure")
errDetail['Description'] = "Error producing step 2 events"
elif jobFailed == 4:
theJobReport.status = "Failed"
theJobReport.exitCode = 11114
errDetail = theJobReport.addError(11114, "CmsGenFailure")
errDetail[
'Description'] = "Error computing the number of produced events"
elif jobFailed == 5:
theJobReport.status = "Failed"
theJobReport.exitCode = 11115
errDetail = theJobReport.addError(11115, "CmsGenFailure")
errDetail['Description'] = "Error making a link"
elif jobFailed == 6:
theJobReport.status = "Failed"
theJobReport.exitCode = 11116
errDetail = theJobReport.addError(11116, "CmsGenFailure")
errDetail['Description'] = "Error preparing the executation"
else:
theJobReport.status = "Failed"
theJobReport.exitCode = 11117
errDetail = theJobReport.addError(11117, "CmsGenFailure")
errDetail['Description'] = "General cmsGen Failure"
if jobFailed == 0:
file = open("nCmsGenEvents.txt", 'r')
nEventsLine = file.readline()
nEvents = nEventsLine.split('\n')
file.close()
totalEvents = nEvents[0]
outputFileUnw = ""
if generator == "alpgen":
outputFileUnw = "%s.unw" % outputFile
elif generator == "comphep":
outputFileUnw = "%s" % outputFile
elif generator == "madgraph":
outputFileUnw = "%s" % outputFile
if not os.path.exists(outputFileUnw):
msg = "Output file Not Found: %s" % outputFileUnw
raise RuntimeError, msg
totalSize = os.stat(outputFileUnw)[6]
else:
totalEvents = 0
totalSize = 0
newFile = theJobReport.newFile()
newFile["LFN"] = "None"
if generator == "alpgen":
#newFile["PFN"] = "%s/%s" % (os.getcwd(),outputFile)
newFile["PFN"] = "%s" % outputFile
elif generator == "comphep":
newFile["PFN"] = "%s" % outputFile
elif generator == "madgraph":
newFile["PFN"] = "%s" % outputFile
newFile["Size"] = totalSize
newFile["TotalEvents"] = totalEvents
newFile["ModuleLabel"] = "cmsGen"
theJobReport.write(jobReport)
if not os.path.exists(cfgFile):
msg = "Cfg File Not Found: %s" % cfgFile
raise RuntimeError, msg
if generator == "alpgen":
jobFailed = alpGen()
processFrameworkJobReport()
elif generator == "comphep":
jobFailed = compHep()
processFrameworkJobReport()
elif generator == "madgraph":
jobFailed = madgraph()
processFrameworkJobReport()
elif generator == "sherpa":
jobFailed = sherpa()
# Create a dummy FwkJobReport
theJobReport = FwkJobReport()
if jobFailed == 0:
theJobReport.status = "Success"
theJobReport.exitCode = 0
theJobReport.write(jobReport)
else:
theJobReport.status = "Failed"
theJobReport.exitCode = jobFailed
errDetail = theJobReport.addError(jobFailed, "CmsGenFailure")
errDetail['Description'] = "Failed to set up Sherpa Libs"
else:
msg = "Generator %s Not Found" % generator
raise RuntimeError, msg
def stageOut():
"""
_stageOut_
Main function for this module. Loads data from the task
and manages the stage out process for a single attempt
"""
state = TaskState(os.getcwd())
state.loadRunResDB()
workflow = WorkflowSpec()
workflow.load(os.environ['PRODAGENT_WORKFLOW_SPEC'])
jobSpecFile = os.environ.get('PRODAGENT_JOBSPEC')
jobSpecId = None
if jobSpecFile is not None:
jobSpec = JobSpec()
jobSpec.load(jobSpecFile)
jobSpecId = jobSpec.parameters.get('JobName')
print workflow
print state.taskName()
print jobSpecId
stageOutFor, override, controls = StageOutUtils.getStageOutConfig(
workflow, state.taskName())
toplevelReport = os.path.join(os.environ['PRODAGENT_JOB_DIR'],
"FrameworkJobReport.xml")
exitCode = 0
# //
# // find inputs by locating the task for which we are staging out
#// and loading its TaskState
for inputTask in stageOutFor:
print "Attempting to stage out files for node %s" % inputTask
try:
inputState = getTaskState(inputTask)
msg = "Loaded Input Task: %s " % inputTask
except Exception, ex:
msg = "Error load for TaskState for task %s" % inputTask
msg += "%s\n" % str(ex)
inputState = None
print msg
if inputState == None:
# exit with init error
# generate failure report in this dir, since cant find
# input state dir
inputReport = FwkJobReport()
inputReport.name = inputTask
inputReport.jobSpecId = jobSpecId
exitCode = 60311
errRep = inputReport.addError(
60311, "TaskStateError")
errRep['Description'] = msg
inputReport.status = "Failed"
inputReport.exitCode = 60311
updateReport(toplevelReport, inputReport)
print "TaskState is None, exiting..."
return exitCode
try:
inputReport = inputState.getJobReport()
msg = "Loaded JobReport for Task : %s\n" % inputTask
msg += "File: %s\n" % inputState.jobReport
except Exception, ex:
msg = "Error loading input report : %s" % str(ex)
inputReport = None
inputReport = None
print msg
if inputReport == None:
msg += "Unable to read Job Report for input task: %s\n" % inputTask
msg += "Looked for file: %s\n" % inputState.jobReport
print msg
inputReport = FwkJobReport()
inputReport.name = inputTask
inputReport.jobSpecId = jobSpecId
exitCode = 60311
errRep = inputReport.addError(
60311, "InputReportError")
errRep['Description'] = msg
inputReport.status = "Failed"
inputReport.exitCode = 60311
updateReport(toplevelReport, inputReport)
# exit with init error
return 60311
try:
manager = StageOutReport(inputReport, override, controls)
except StageOutInitError, ex:
exitCode = ex.data['ErrorCode']
errRep = inputReport.addError(
ex.data['ErrorCode'], ex.data['ErrorType'])
errRep['Description'] = ex.message
inputReport.status = "Failed"
inputReport.exitCode = ex.data['ErrorCode']
def processFrameworkJobReport():
"""
Runtime tool for processing the Framework Job Report produced
by CMsGen executables
"""
theJobReport = FwkJobReport()
#
# If everything worked well set the success status
# else record a failure and error
if jobFailed == 0:
theJobReport.status = "Success"
theJobReport.exitCode = 0
elif jobFailed == 1:
theJobReport.status = "Failed"
theJobReport.exitCode = 11111
errDetail = theJobReport.addError(11111, "CmsGenFailure")
errDetail['Description'] = "Error retrieving grid file"
elif jobFailed == 2:
theJobReport.status = "Failed"
theJobReport.exitCode = 11112
errDetail = theJobReport.addError(11112, "CmsGenFailure")
errDetail['Description'] = "Error producing step 1 events"
elif jobFailed == 3:
theJobReport.status = "Failed"
theJobReport.exitCode = 11113
errDetail = theJobReport.addError(11113, "CmsGenFailure")
errDetail['Description'] = "Error producing step 2 events"
elif jobFailed == 4:
theJobReport.status = "Failed"
theJobReport.exitCode = 11114
errDetail = theJobReport.addError(11114, "CmsGenFailure")
errDetail['Description'] = "Error computing the number of produced events"
elif jobFailed == 5:
theJobReport.status = "Failed"
theJobReport.exitCode = 11115
errDetail = theJobReport.addError(11115, "CmsGenFailure")
errDetail['Description'] = "Error making a link"
elif jobFailed == 6:
theJobReport.status = "Failed"
theJobReport.exitCode = 11116
errDetail = theJobReport.addError(11116, "CmsGenFailure")
errDetail['Description'] = "Error preparing the executation"
else:
theJobReport.status = "Failed"
theJobReport.exitCode = 11117
errDetail = theJobReport.addError(11117, "CmsGenFailure")
errDetail['Description'] = "General cmsGen Failure"
if jobFailed == 0:
file = open("nCmsGenEvents.txt", 'r')
nEventsLine = file.readline()
nEvents = nEventsLine.split('\n')
file.close()
totalEvents = nEvents[0]
outputFileUnw = ""
if generator == "alpgen":
outputFileUnw = "%s.unw" % outputFile
elif generator == "comphep":
outputFileUnw = "%s" % outputFile
elif generator == "madgraph":
outputFileUnw = "%s" % outputFile
if not os.path.exists(outputFileUnw):
msg = "Output file Not Found: %s" % outputFileUnw
raise RuntimeError, msg
totalSize = os.stat(outputFileUnw)[6]
else:
totalEvents = 0
totalSize = 0
newFile = theJobReport.newFile()
newFile["LFN"] = "None"
if generator == "alpgen":
#newFile["PFN"] = "%s/%s" % (os.getcwd(),outputFile)
newFile["PFN"] = "%s" % outputFile
elif generator == "comphep":
newFile["PFN"] = "%s" % outputFile
elif generator == "madgraph":
newFile["PFN"] = "%s" % outputFile
newFile["Size"] = totalSize
newFile["TotalEvents"] = totalEvents
newFile["ModuleLabel"] = "cmsGen"
theJobReport.write(jobReport)
if not os.path.exists(cfgFile):
msg = "Cfg File Not Found: %s" % cfgFile
raise RuntimeError, msg
if generator == "alpgen":
jobFailed = alpGen()
processFrameworkJobReport()
elif generator == "comphep":
jobFailed = compHep()
processFrameworkJobReport()
elif generator == "madgraph":
jobFailed = madgraph()
processFrameworkJobReport()
elif generator == "sherpa":
jobFailed = sherpa()
# Create a dummy FwkJobReport
theJobReport = FwkJobReport()
if jobFailed == 0:
theJobReport.status = "Success"
theJobReport.exitCode = 0
theJobReport.write(jobReport)
else:
theJobReport.status = "Failed"
theJobReport.exitCode = jobFailed
errDetail = theJobReport.addError(jobFailed, "CmsGenFailure")
errDetail['Description'] = "Failed to set up Sherpa Libs"
else:
msg = "Generator %s Not Found" % generator
raise RuntimeError, msg
