def handleSuccessful(self, jobID, fwkJobReport, fwkJobReportPath=None):
"""
_handleSuccessful_
Handle a successful job, parsing the job report and updating the job in
WMBS.
"""
wmbsJob = Job(id=jobID)
wmbsJob.load()
wmbsJob["outcome"] = "success"
wmbsJob.getMask()
outputID = wmbsJob.loadOutputID()
wmbsJob["fwjr"] = fwkJobReport
outputMap = self.getOutputMapAction.execute(
jobID=jobID,
conn=self.getDBConn(),
transaction=self.existingTransaction())
jobType = self.getJobTypeAction.execute(
jobID=jobID,
conn=self.getDBConn(),
transaction=self.existingTransaction())
fileList = fwkJobReport.getAllFiles()
for fwjrFile in fileList:
wmbsFile = self.addFileToWMBS(jobType,
fwjrFile,
wmbsJob["mask"],
jobID=jobID,
task=fwkJobReport.getTaskName())
merged = fwjrFile['merged']
moduleLabel = fwjrFile["module_label"]
if merged:
self.mergedOutputFiles.append(wmbsFile)
self.filesetAssoc.append({
"lfn": wmbsFile["lfn"],
"fileset": outputID
})
outputFilesets = self.outputFilesetsForJob(outputMap, merged,
moduleLabel)
for outputFileset in outputFilesets:
self.filesetAssoc.append({
"lfn": wmbsFile["lfn"],
"fileset": outputFileset
})
# Only save once job is done, and we're sure we made it through okay
self._mapLocation(wmbsJob['fwjr'])
self.listOfJobsToSave.append(wmbsJob)
#wmbsJob.save()
return
def handleSuccessful(self, jobID, fwkJobReport, fwkJobReportPath = None):
"""
_handleSuccessful_
Handle a successful job, parsing the job report and updating the job in
WMBS.
"""
wmbsJob = Job(id = jobID)
wmbsJob.load()
wmbsJob["outcome"] = "success"
wmbsJob.getMask()
outputID = wmbsJob.loadOutputID()
wmbsJob["fwjr"] = fwkJobReport
outputMap = self.getOutputMapAction.execute(jobID = jobID,
conn = self.getDBConn(),
transaction = self.existingTransaction())
jobType = self.getJobTypeAction.execute(jobID = jobID,
conn = self.getDBConn(),
transaction = self.existingTransaction())
fileList = fwkJobReport.getAllFiles()
bookKeepingSuccess = True
for fwjrFile in fileList:
# associate logArchived file for parent jobs on wmstats assuming fileList is length is 1.
if jobType == "LogCollect":
try:
self.associateLogCollectToParentJobsInWMStats(fwkJobReport, fwjrFile["lfn"], fwkJobReport.getTaskName())
except Exception, ex:
bookKeepingSuccess = False
logging.error("Error occurred: associating log collect location, will try again\n %s" % str(ex))
break
wmbsFile = self.addFileToWMBS(jobType, fwjrFile, wmbsJob["mask"],
jobID = jobID, task = fwkJobReport.getTaskName())
merged = fwjrFile['merged']
moduleLabel = fwjrFile["module_label"]
if merged:
self.mergedOutputFiles.append(wmbsFile)
self.filesetAssoc.append({"lfn": wmbsFile["lfn"], "fileset": outputID})
outputFilesets = self.outputFilesetsForJob(outputMap, merged, moduleLabel)
for outputFileset in outputFilesets:
self.filesetAssoc.append({"lfn": wmbsFile["lfn"], "fileset": outputFileset})
def handleSuccessful(self, jobID, fwkJobReport, fwkJobReportPath = None):
"""
_handleSuccessful_
Handle a successful job, parsing the job report and updating the job in
WMBS.
"""
wmbsJob = Job(id = jobID)
wmbsJob.load()
wmbsJob["outcome"] = "success"
wmbsJob.getMask()
outputID = wmbsJob.loadOutputID()
wmbsJob["fwjr"] = fwkJobReport
outputMap = self.getOutputMapAction.execute(jobID = jobID,
conn = self.getDBConn(),
transaction = self.existingTransaction())
jobType = self.getJobTypeAction.execute(jobID = jobID,
conn = self.getDBConn(),
transaction = self.existingTransaction())
fileList = fwkJobReport.getAllFiles()
for fwjrFile in fileList:
wmbsFile = self.addFileToWMBS(jobType, fwjrFile, wmbsJob["mask"],
jobID = jobID, task = fwkJobReport.getTaskName())
merged = fwjrFile['merged']
moduleLabel = fwjrFile["module_label"]
if merged:
self.mergedOutputFiles.append(wmbsFile)
self.filesetAssoc.append({"lfn": wmbsFile["lfn"], "fileset": outputID})
outputFilesets = self.outputFilesetsForJob(outputMap, merged, moduleLabel)
for outputFileset in outputFilesets:
self.filesetAssoc.append({"lfn": wmbsFile["lfn"], "fileset": outputFileset})
# Only save once job is done, and we're sure we made it through okay
self._mapLocation(wmbsJob['fwjr'])
self.listOfJobsToSave.append(wmbsJob)
#wmbsJob.save()
return
def handleJob(self, jobID, fwkJobReport):
"""
_handleJob_
Figure out if a job was successful or not, handle it appropriately
(parse FWJR, update WMBS) and return the success status as a boolean
"""
jobSuccess = fwkJobReport.taskSuccessful()
outputMap = self.getOutputMapAction.execute(
jobID=jobID,
conn=self.getDBConn(),
transaction=self.existingTransaction())
jobType = self.getJobTypeAction.execute(
jobID=jobID,
conn=self.getDBConn(),
transaction=self.existingTransaction())
if jobSuccess:
fileList = fwkJobReport.getAllFiles()
# consistency check comparing outputMap to fileList
# they should match except for some limited special cases
outputModules = set([])
for fwjrFile in fileList:
outputModules.add(fwjrFile['outputModule'])
if set(outputMap.keys()) == outputModules:
pass
elif jobType == "LogCollect" and len(
outputMap.keys()) == 0 and outputModules == set(
['LogCollect']):
pass
elif jobType == "Merge" and set(outputMap.keys()) == set([
'Merged', 'MergedError', 'logArchive'
]) and outputModules == set(['Merged', 'logArchive']):
pass
elif jobType == "Merge" and set(outputMap.keys()) == set([
'Merged', 'MergedError', 'logArchive'
]) and outputModules == set(['MergedError', 'logArchive']):
pass
elif jobType == "Express" and set(
outputMap.keys()).difference(outputModules) == set(
['write_RAW']):
pass
else:
failJob = True
if jobType in ["Processing", "Production"]:
cmsRunSteps = 0
for step in fwkJobReport.listSteps():
if step.startswith("cmsRun"):
cmsRunSteps += 1
if cmsRunSteps > 1:
failJob = False
if failJob:
jobSuccess = False
logging.error(
"Job %d , list of expected outputModules does not match job report, failing job",
jobID)
logging.debug("Job %d , expected outputModules %s", jobID,
sorted(outputMap.keys()))
logging.debug("Job %d , fwjr outputModules %s", jobID,
sorted(outputModules))
fileList = fwkJobReport.getAllFilesFromStep(
step='logArch1')
else:
logging.debug(
"Job %d , list of expected outputModules does not match job report, accepted for multi-step CMSSW job",
jobID)
else:
fileList = fwkJobReport.getAllFilesFromStep(step='logArch1')
if jobSuccess:
logging.info("Job %d , handle successful job", jobID)
else:
logging.warning("Job %d , bad jobReport, failing job", jobID)
# make sure the task name is present in FWJR (recover from WMBS if needed)
if len(fileList) > 0:
if jobSuccess:
self.isTaskExistInFWJR(fwkJobReport, "success")
else:
self.isTaskExistInFWJR(fwkJobReport, "failed")
# special check for LogCollect jobs
skipLogCollect = False
if jobSuccess and jobType == "LogCollect":
for fwjrFile in fileList:
try:
# this assumes there is only one file for LogCollect jobs, not sure what happend if that changes
self.associateLogCollectToParentJobsInWMStats(
fwkJobReport, fwjrFile["lfn"],
fwkJobReport.getTaskName())
except Exception as ex:
skipLogCollect = True
logging.error(
"Error occurred: associating log collect location, will try again\n %s"
% str(ex))
break
# now handle the job (unless the special LogCollect check failed)
if not skipLogCollect:
wmbsJob = Job(id=jobID)
wmbsJob.load()
outputID = wmbsJob.loadOutputID()
wmbsJob.getMask()
wmbsJob["fwjr"] = fwkJobReport
if jobSuccess:
wmbsJob["outcome"] = "success"
else:
wmbsJob["outcome"] = "failure"
for fwjrFile in fileList:
logging.debug("Job %d , register output %s", jobID,
fwjrFile["lfn"])
wmbsFile = self.addFileToWMBS(jobType,
fwjrFile,
wmbsJob["mask"],
jobID=jobID,
task=fwkJobReport.getTaskName())
merged = fwjrFile['merged']
moduleLabel = fwjrFile["module_label"]
if merged:
self.mergedOutputFiles.append(wmbsFile)
self.filesetAssoc.append({
"lfn": wmbsFile["lfn"],
"fileset": outputID
})
# LogCollect jobs have no output fileset
if jobType == "LogCollect":
pass
# Repack jobs that wrote too large merged output skip output filesets
elif jobType == "Repack" and merged and wmbsFile[
"size"] > self.maxAllowedRepackOutputSize:
pass
else:
outputFilesets = self.outputFilesetsForJob(
outputMap, merged, moduleLabel)
for outputFileset in outputFilesets:
self.filesetAssoc.append({
"lfn": wmbsFile["lfn"],
"fileset": outputFileset
})
# Check if the job had any skipped files, put them in ACDC containers
# We assume full file processing (no job masks)
if jobSuccess:
skippedFiles = fwkJobReport.getAllSkippedFiles()
if skippedFiles and jobType not in ['LogCollect', 'Cleanup']:
self.jobsWithSkippedFiles[jobID] = skippedFiles
# Only save once job is done, and we're sure we made it through okay
self._mapLocation(wmbsJob['fwjr'])
if jobSuccess:
self.listOfJobsToSave.append(wmbsJob)
else:
self.listOfJobsToFail.append(wmbsJob)
return jobSuccess
def handleJob(self, jobID, fwkJobReport):
"""
_handleJob_
Figure out if a job was successful or not, handle it appropriately
(parse FWJR, update WMBS) and return the success status as a boolean
"""
jobSuccess = fwkJobReport.taskSuccessful()
outputMap = self.getOutputMapAction.execute(jobID = jobID,
conn = self.getDBConn(),
transaction = self.existingTransaction())
jobType = self.getJobTypeAction.execute(jobID = jobID,
conn = self.getDBConn(),
transaction = self.existingTransaction())
if jobSuccess:
fileList = fwkJobReport.getAllFiles()
# consistency check comparing outputMap to fileList
# they should match except for some limited special cases
outputModules = set([])
for fwjrFile in fileList:
outputModules.add(fwjrFile['outputModule'])
if set(outputMap.keys()) == outputModules:
pass
elif jobType == "LogCollect" and len(outputMap.keys()) == 0 and outputModules == set(['LogCollect']):
pass
elif jobType == "Merge" and set(outputMap.keys()) == set(['Merged', 'MergedError', 'logArchive']) and outputModules == set(['Merged', 'logArchive']):
pass
elif jobType == "Merge" and set(outputMap.keys()) == set(['Merged', 'MergedError', 'logArchive']) and outputModules == set(['MergedError', 'logArchive']):
pass
elif jobType == "Express" and set(outputMap.keys()).difference(outputModules) == set(['write_RAW']):
pass
else:
failJob = True
if jobType in [ "Processing", "Production" ]:
cmsRunSteps = 0
for step in fwkJobReport.listSteps():
if step.startswith("cmsRun"):
cmsRunSteps += 1
if cmsRunSteps > 1:
failJob = False
if failJob:
jobSuccess = False
logging.error("Job %d , list of expected outputModules does not match job report, failing job", jobID)
logging.debug("Job %d , expected outputModules %s", jobID, sorted(outputMap.keys()))
logging.debug("Job %d , fwjr outputModules %s", jobID, sorted(outputModules))
fileList = fwkJobReport.getAllFilesFromStep(step = 'logArch1')
else:
logging.debug("Job %d , list of expected outputModules does not match job report, accepted for multi-step CMSSW job", jobID)
else:
fileList = fwkJobReport.getAllFilesFromStep(step = 'logArch1')
if jobSuccess:
logging.info("Job %d , handle successful job", jobID)
else:
logging.error("Job %d , bad jobReport, failing job", jobID)
# make sure the task name is present in FWJR (recover from WMBS if needed)
if len(fileList) > 0:
if jobSuccess:
self.isTaskExistInFWJR(fwkJobReport, "success")
else:
self.isTaskExistInFWJR(fwkJobReport, "failed")
# special check for LogCollect jobs
skipLogCollect = False
if jobSuccess and jobType == "LogCollect":
for fwjrFile in fileList:
try:
# this assumes there is only one file for LogCollect jobs, not sure what happend if that changes
self.associateLogCollectToParentJobsInWMStats(fwkJobReport, fwjrFile["lfn"], fwkJobReport.getTaskName())
except Exception as ex:
skipLogCollect = True
logging.error("Error occurred: associating log collect location, will try again\n %s" % str(ex))
break
# now handle the job (unless the special LogCollect check failed)
if not skipLogCollect:
wmbsJob = Job(id = jobID)
wmbsJob.load()
outputID = wmbsJob.loadOutputID()
wmbsJob.getMask()
wmbsJob["fwjr"] = fwkJobReport
if jobSuccess:
wmbsJob["outcome"] = "success"
else:
wmbsJob["outcome"] = "failure"
for fwjrFile in fileList:
logging.debug("Job %d , register output %s", jobID, fwjrFile["lfn"])
wmbsFile = self.addFileToWMBS(jobType, fwjrFile, wmbsJob["mask"],
jobID = jobID, task = fwkJobReport.getTaskName())
merged = fwjrFile['merged']
moduleLabel = fwjrFile["module_label"]
if merged:
self.mergedOutputFiles.append(wmbsFile)
self.filesetAssoc.append({"lfn": wmbsFile["lfn"], "fileset": outputID})
# LogCollect jobs have no output fileset
if jobType != "LogCollect":
outputFilesets = self.outputFilesetsForJob(outputMap, merged, moduleLabel)
for outputFileset in outputFilesets:
self.filesetAssoc.append({"lfn": wmbsFile["lfn"], "fileset": outputFileset})
# Check if the job had any skipped files, put them in ACDC containers
# We assume full file processing (no job masks)
if jobSuccess:
skippedFiles = fwkJobReport.getAllSkippedFiles()
if skippedFiles:
self.jobsWithSkippedFiles[jobID] = skippedFiles
# Only save once job is done, and we're sure we made it through okay
self._mapLocation(wmbsJob['fwjr'])
if jobSuccess:
self.listOfJobsToSave.append(wmbsJob)
else:
self.listOfJobsToFail.append(wmbsJob)
return jobSuccess
def handleSuccessful(self, jobID, fwkJobReport, fwkJobReportPath=None):
"""
_handleSuccessful_
Handle a successful job, parsing the job report and updating the job in
WMBS.
"""
wmbsJob = Job(id=jobID)
wmbsJob.load()
wmbsJob["outcome"] = "success"
wmbsJob.getMask()
outputID = wmbsJob.loadOutputID()
wmbsJob["fwjr"] = fwkJobReport
outputMap = self.getOutputMapAction.execute(
jobID=jobID,
conn=self.getDBConn(),
transaction=self.existingTransaction())
jobType = self.getJobTypeAction.execute(
jobID=jobID,
conn=self.getDBConn(),
transaction=self.existingTransaction())
fileList = fwkJobReport.getAllFiles()
bookKeepingSuccess = True
for fwjrFile in fileList:
# associate logArchived file for parent jobs on wmstats assuming fileList is length is 1.
if jobType == "LogCollect":
try:
self.associateLogCollectToParentJobsInWMStats(
fwkJobReport, fwjrFile["lfn"],
fwkJobReport.getTaskName())
except Exception as ex:
bookKeepingSuccess = False
logging.error(
"Error occurred: associating log collect location, will try again\n %s"
% str(ex))
break
wmbsFile = self.addFileToWMBS(jobType,
fwjrFile,
wmbsJob["mask"],
jobID=jobID,
task=fwkJobReport.getTaskName())
merged = fwjrFile['merged']
moduleLabel = fwjrFile["module_label"]
if merged:
self.mergedOutputFiles.append(wmbsFile)
self.filesetAssoc.append({
"lfn": wmbsFile["lfn"],
"fileset": outputID
})
outputFilesets = self.outputFilesetsForJob(outputMap, merged,
moduleLabel)
for outputFileset in outputFilesets:
self.filesetAssoc.append({
"lfn": wmbsFile["lfn"],
"fileset": outputFileset
})
# Check if the job had any skipped files
# Put them in ACDC containers, we assume full file processing
# No job masks
skippedFiles = fwkJobReport.getAllSkippedFiles()
if skippedFiles:
self.jobsWithSkippedFiles[jobID] = skippedFiles
if bookKeepingSuccess:
# Only save once job is done, and we're sure we made it through okay
self._mapLocation(wmbsJob['fwjr'])
self.listOfJobsToSave.append(wmbsJob)
#wmbsJob.save()
return
def handleSuccessful(self, jobID, fwkJobReport, fwkJobReportPath=None):
"""
_handleSuccessful_
Handle a successful job, parsing the job report and updating the job in
WMBS.
"""
wmbsJob = Job(id=jobID)
wmbsJob.load()
wmbsJob["outcome"] = "success"
wmbsJob.getMask()
outputID = wmbsJob.loadOutputID()
wmbsJob["fwjr"] = fwkJobReport
outputMap = self.getOutputMapAction.execute(
jobID=jobID,
conn=self.getDBConn(),
transaction=self.existingTransaction())
jobType = self.getJobTypeAction.execute(
jobID=jobID,
conn=self.getDBConn(),
transaction=self.existingTransaction())
fileList = fwkJobReport.getAllFiles()
bookKeepingSuccess = True
for fwjrFile in fileList:
# associate logArchived file for parent jobs on wmstats assuming fileList is length is 1.
if jobType == "LogCollect":
try:
self.associateLogCollectToParentJobsInWMStats(
fwkJobReport, fwjrFile["lfn"],
fwkJobReport.getTaskName())
except Exception, ex:
bookKeepingSuccess = False
logging.error(
"Error occurred: associating log collect location, will try again\n %s"
% str(ex))
break
wmbsFile = self.addFileToWMBS(jobType,
fwjrFile,
wmbsJob["mask"],
jobID=jobID,
task=fwkJobReport.getTaskName())
merged = fwjrFile['merged']
moduleLabel = fwjrFile["module_label"]
if merged:
self.mergedOutputFiles.append(wmbsFile)
self.filesetAssoc.append({
"lfn": wmbsFile["lfn"],
"fileset": outputID
})
outputFilesets = self.outputFilesetsForJob(outputMap, merged,
moduleLabel)
for outputFileset in outputFilesets:
self.filesetAssoc.append({
"lfn": wmbsFile["lfn"],
"fileset": outputFileset
})
def handleSuccessful(self, jobID, fwkJobReport, fwkJobReportPath = None):
"""
_handleSuccessful_
Handle a successful job, parsing the job report and updating the job in
WMBS.
"""
wmbsJob = Job(id = jobID)
wmbsJob.load()
wmbsJob["outcome"] = "success"
wmbsJob.getMask()
outputID = wmbsJob.loadOutputID()
wmbsJob["fwjr"] = fwkJobReport
outputMap = self.getOutputMapAction.execute(jobID = jobID,
conn = self.getDBConn(),
transaction = self.existingTransaction())
jobType = self.getJobTypeAction.execute(jobID = jobID,
conn = self.getDBConn(),
transaction = self.existingTransaction())
fileList = fwkJobReport.getAllFiles()
bookKeepingSuccess = True
for fwjrFile in fileList:
# associate logArchived file for parent jobs on wmstats assuming fileList is length is 1.
if jobType == "LogCollect":
try:
self.associateLogCollectToParentJobsInWMStats(fwkJobReport, fwjrFile["lfn"], fwkJobReport.getTaskName())
except Exception as ex:
bookKeepingSuccess = False
logging.error("Error occurred: associating log collect location, will try again\n %s" % str(ex))
break
wmbsFile = self.addFileToWMBS(jobType, fwjrFile, wmbsJob["mask"],
jobID = jobID, task = fwkJobReport.getTaskName())
merged = fwjrFile['merged']
moduleLabel = fwjrFile["module_label"]
if merged:
self.mergedOutputFiles.append(wmbsFile)
self.filesetAssoc.append({"lfn": wmbsFile["lfn"], "fileset": outputID})
outputFilesets = self.outputFilesetsForJob(outputMap, merged, moduleLabel)
for outputFileset in outputFilesets:
self.filesetAssoc.append({"lfn": wmbsFile["lfn"], "fileset": outputFileset})
# Check if the job had any skipped files
# Put them in ACDC containers, we assume full file processing
# No job masks
skippedFiles = fwkJobReport.getAllSkippedFiles()
if skippedFiles:
self.jobsWithSkippedFiles[jobID] = skippedFiles
if bookKeepingSuccess:
# Only save once job is done, and we're sure we made it through okay
self._mapLocation(wmbsJob['fwjr'])
self.listOfJobsToSave.append(wmbsJob)
#wmbsJob.save()
return
