def test_JobStateUpdateAndJobMonitoring(self):
""" Verifying all JobStateUpdate and JobMonitoring functions
"""
wmsClient = WMSClient()
jobMonitor = JobMonitoringClient()
jobStateUpdate = JobStateUpdateClient()
# create a job and check stuff
job = helloWorldJob()
jobDescription = createFile(job)
# submitting the job. Checking few stuff
res = wmsClient.submitJob(job._toJDL(xmlFile=jobDescription))
self.assertTrue(res['OK'])
jobID = int(res['Value'])
# jobID = res['JobID']
res = jobMonitor.getJobJDL(jobID, True)
self.assertTrue(res['OK'])
res = jobMonitor.getJobJDL(jobID, False)
self.assertTrue(res['OK'])
res = jobMonitor.getJobsParameters([jobID], [])
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], {})
res = jobMonitor.getJobsParameters([jobID], ['Owner'])
self.assertTrue(res['OK'])
# Adding stuff
res = jobStateUpdate.setJobStatus(jobID, 'Matched', 'matching', 'source')
self.assertTrue(res['OK'])
res = jobStateUpdate.setJobParameters(jobID, [('par1', 'par1Value'), ('par2', 'par2Value')])
self.assertTrue(res['OK'])
res = jobStateUpdate.setJobApplicationStatus(jobID, 'app status', 'source')
self.assertTrue(res['OK'])
# res = jobStateUpdate.setJobFlag()
# self.assertTrue(res['OK'])
# res = jobStateUpdate.unsetJobFlag()
# self.assertTrue(res['OK'])
res = jobStateUpdate.setJobSite(jobID, 'Site')
self.assertTrue(res['OK'])
# res = jobMonitor.traceJobParameter( 'Site', 1, 'Status' )
# self.assertTrue(res['OK'])
# now checking few things
res = jobMonitor.getJobStatus(jobID)
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], 'Running')
res = jobMonitor.getJobParameter(jobID, 'par1')
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], {'par1': 'par1Value'})
res = jobMonitor.getJobParameters(jobID)
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], {'par1': 'par1Value', 'par2': 'par2Value'})
res = jobMonitor.getJobAttribute(jobID, 'Site')
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], 'Site')
res = jobMonitor.getJobAttributes(jobID)
self.assertTrue(res['OK'])
self.assertEqual(res['Value']['ApplicationStatus'], 'app status')
self.assertEqual(res['Value']['JobName'], 'helloWorld')
res = jobMonitor.getJobSummary(jobID)
self.assertTrue(res['OK'])
self.assertEqual(res['Value']['ApplicationStatus'], 'app status')
self.assertEqual(res['Value']['Status'], 'Running')
res = jobMonitor.getJobHeartBeatData(jobID)
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], [])
res = jobMonitor.getInputData(jobID)
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], [])
res = jobMonitor.getJobPrimarySummary(jobID)
self.assertTrue(res['OK'])
res = jobMonitor.getAtticJobParameters(jobID)
self.assertTrue(res['OK'])
res = jobStateUpdate.setJobsStatus([jobID], 'Done', 'MinorStatus', 'Unknown')
self.assertTrue(res['OK'])
res = jobMonitor.getJobSummary(jobID)
self.assertTrue(res['OK'])
self.assertEqual(res['Value']['Status'], 'Done')
self.assertEqual(res['Value']['MinorStatus'], 'MinorStatus')
self.assertEqual(res['Value']['ApplicationStatus'], 'app status')
res = jobStateUpdate.sendHeartBeat(jobID, {'bih': 'bih'}, {'boh': 'boh'})
self.assertTrue(res['OK'])
# delete the job - this will just set its status to "deleted"
wmsClient.deleteJob(jobID)
def test_JobStateUpdateAndJobMonitoring(self):
"""Verifying all JobStateUpdate and JobMonitoring functions"""
wmsClient = WMSClient()
jobMonitor = JobMonitoringClient()
jobStateUpdate = JobStateUpdateClient()
# create a job and check stuff
job = helloWorldJob()
jobDescription = createFile(job)
# submitting the job. Checking few stuff
res = wmsClient.submitJob(job._toJDL(xmlFile=jobDescription))
self.assertTrue(res["OK"], res.get("Message"))
jobID = int(res["Value"])
# jobID = res['JobID']
res = jobMonitor.getJobJDL(jobID, True)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobJDL(jobID, False)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobsParameters([jobID], [])
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobOwner(jobID)
self.assertTrue(res["OK"], res.get("Message"))
# Adding stuff
# forcing the update
res = jobStateUpdate.setJobStatus(jobID, JobStatus.RUNNING, "running",
"source", None, True)
self.assertTrue(res["OK"], res.get("Message"))
res = jobStateUpdate.setJobParameters(jobID, [("par1", "par1Value"),
("par2", "par2Value")])
time.sleep(5)
self.assertTrue(res["OK"], res.get("Message"))
res = jobStateUpdate.setJobApplicationStatus(jobID, "app status",
"source")
self.assertTrue(res["OK"], res.get("Message"))
# res = jobStateUpdate.setJobFlag()
# self.assertTrue(res['OK'], res.get('Message'))
# res = jobStateUpdate.unsetJobFlag()
# self.assertTrue(res['OK'], res.get('Message'))
res = jobStateUpdate.setJobSite(jobID, "Site")
self.assertTrue(res["OK"], res.get("Message"))
# now checking few things
res = jobMonitor.getJobsStatus(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"][jobID]["Status"],
JobStatus.RUNNING,
msg="Got %s" % str(res["Value"]))
res = jobMonitor.getJobParameter(jobID, "par1")
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"], {"par1": "par1Value"},
msg="Got %s" % str(res["Value"]))
res = jobMonitor.getJobParameters(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"],
{jobID: {
"par1": "par1Value",
"par2": "par2Value"
}},
msg="Got %s" % str(res["Value"]))
res = jobMonitor.getJobParameters(jobID, "par1")
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"], {jobID: {
"par1": "par1Value"
}},
msg="Got %s" % str(res["Value"]))
res = jobMonitor.getJobAttribute(jobID, "Site")
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"],
"Site",
msg="Got %s" % str(res["Value"]))
res = jobMonitor.getJobAttributes(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"]["ApplicationStatus"],
"app status",
msg="Got %s" % str(res["Value"]["ApplicationStatus"]))
self.assertEqual(res["Value"]["JobName"],
"helloWorld",
msg="Got %s" % str(res["Value"]["JobName"]))
res = jobMonitor.getJobSummary(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"]["ApplicationStatus"],
"app status",
msg="Got %s" % str(res["Value"]["ApplicationStatus"]))
self.assertEqual(res["Value"]["Status"],
JobStatus.RUNNING,
msg="Got %s" % str(res["Value"]["Status"]))
res = jobMonitor.getJobHeartBeatData(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"], [], msg="Got %s" % str(res["Value"]))
res = jobMonitor.getInputData(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"], [], msg="Got %s" % str(res["Value"]))
res = jobMonitor.getJobSummary(jobID)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getAtticJobParameters(jobID)
self.assertTrue(res["OK"], res.get("Message"))
res = jobStateUpdate.setJobStatus(jobID, JobStatus.DONE, "MinorStatus",
"Unknown")
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobSummary(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"]["Status"],
JobStatus.DONE,
msg="Got %s" % str(res["Value"]["Status"]))
self.assertEqual(res["Value"]["MinorStatus"],
"MinorStatus",
msg="Got %s" % str(res["Value"]["MinorStatus"]))
self.assertEqual(res["Value"]["ApplicationStatus"],
"app status",
msg="Got %s" % str(res["Value"]["ApplicationStatus"]))
res = jobStateUpdate.sendHeartBeat(jobID, {"bih": "bih"},
{"boh": "boh"})
self.assertTrue(res["OK"], res.get("Message"))
# delete the job - this will just set its status to "deleted"
wmsClient.deleteJob(jobID)
def finalizeRequest(self, requestID, jobID, useCertificates=True):
"""check request status and perform finalization if necessary
update the request status and the corresponding job parameter
:param self: self reference
:param str requestID: request id
:param int jobID: job id
"""
stateServer = JobStateUpdateClient(useCertificates=useCertificates)
# Checking if to update the job status - we should fail here, so it will be re-tried later
# Checking the state, first
res = self.getRequestStatus(requestID)
if not res["OK"]:
self.log.error(
"finalizeRequest: failed to get request",
"request: %s status: %s" % (requestID, res["Message"]))
return res
if res["Value"] != "Done":
return S_ERROR(
"The request %s isn't 'Done' but '%s', this should never happen, why are we here?"
% (requestID, res["Value"]))
# The request is 'Done', let's update the job status. If we fail, we should re-try later
monitorServer = JobMonitoringClient(useCertificates=useCertificates)
res = monitorServer.getJobSummary(int(jobID))
if not res["OK"]:
self.log.error("finalizeRequest: Failed to get job status",
"JobID: %d" % jobID)
return res
elif not res["Value"]:
self.log.info(
"finalizeRequest: job %d does not exist (anymore): finalizing"
% jobID)
return S_OK()
else:
jobStatus = res["Value"]["Status"]
jobMinorStatus = res["Value"]["MinorStatus"]
jobAppStatus = ""
newJobStatus = ""
if jobStatus == JobStatus.STALLED:
# If job is stalled, find the previous status from the logging info
res = monitorServer.getJobLoggingInfo(int(jobID))
if not res["OK"]:
self.log.error(
"finalizeRequest: Failed to get job logging info",
"JobID: %d" % jobID)
return res
# Check the last status was Stalled and get the one before
if len(res["Value"]
) >= 2 and res["Value"][-1][0] == JobStatus.STALLED:
jobStatus, jobMinorStatus, jobAppStatus = res["Value"][
-2][:3]
newJobStatus = jobStatus
# update the job pending request digest in any case since it is modified
self.log.info(
"finalizeRequest: Updating request digest for job %d" % jobID)
digest = self.getDigest(requestID)
if digest["OK"]:
digest = digest["Value"]
self.log.verbose(digest)
res = stateServer.setJobParameter(jobID, "PendingRequest",
digest)
if not res["OK"]:
self.log.info(
"finalizeRequest: Failed to set job %d parameter: %s" %
(jobID, res["Message"]))
return res
else:
self.log.error(
"finalizeRequest: Failed to get request digest for %s: %s"
% (requestID, digest["Message"]))
if jobStatus == JobStatus.COMPLETED:
# What to do? Depends on what we have in the minorStatus
if jobMinorStatus == JobMinorStatus.PENDING_REQUESTS:
newJobStatus = JobStatus.DONE
elif jobMinorStatus == JobMinorStatus.APP_ERRORS:
newJobStatus = JobStatus.FAILED
elif jobMinorStatus == JobMinorStatus.MARKED_FOR_TERMINATION:
# If the job has been Killed, set it Killed
newJobStatus = JobStatus.KILLED
else:
self.log.error(
"finalizeRequest: Unexpected jobMinorStatus",
"for %d (got %s)" % (jobID, jobMinorStatus))
return S_ERROR("Unexpected jobMinorStatus")
if newJobStatus:
self.log.info(
"finalizeRequest: Updating job status",
"for %d to '%s/%s'" %
(jobID, newJobStatus, JobMinorStatus.REQUESTS_DONE),
)
else:
self.log.info(
"finalizeRequest: Updating job minor status",
"for %d to '%s' (current status is %s)" %
(jobID, JobMinorStatus.REQUESTS_DONE, jobStatus),
)
stateUpdate = stateServer.setJobStatus(
jobID, newJobStatus, JobMinorStatus.REQUESTS_DONE, "RMS")
if jobAppStatus and stateUpdate["OK"]:
stateUpdate = stateServer.setJobApplicationStatus(
jobID, jobAppStatus, "RMS")
if not stateUpdate["OK"]:
self.log.error(
"finalizeRequest: Failed to set job status",
"JobID: %d, error: %s" % (jobID, stateUpdate["Message"]),
)
return stateUpdate
return S_OK(newJobStatus)
def test_JobStateUpdateAndJobMonitoring(self):
""" Verifying all JobStateUpdate and JobMonitoring functions
"""
wmsClient = WMSClient()
jobMonitor = JobMonitoringClient()
jobStateUpdate = JobStateUpdateClient()
# create a job and check stuff
job = helloWorldJob()
jobDescription = createFile(job)
# submitting the job. Checking few stuff
res = wmsClient.submitJob(job._toJDL(xmlFile=jobDescription))
self.assertTrue(res['OK'])
jobID = int(res['Value'])
# jobID = res['JobID']
res = jobMonitor.getJobJDL(jobID, True)
self.assertTrue(res['OK'])
res = jobMonitor.getJobJDL(jobID, False)
self.assertTrue(res['OK'])
res = jobMonitor.getJobsParameters([jobID], [])
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], {})
res = jobMonitor.getJobsParameters([jobID], ['Owner'])
self.assertTrue(res['OK'])
# Adding stuff
res = jobStateUpdate.setJobStatus(jobID, 'Matched', 'matching', 'source')
self.assertTrue(res['OK'])
res = jobStateUpdate.setJobParameters(jobID, [('par1', 'par1Value'), ('par2', 'par2Value')])
self.assertTrue(res['OK'])
res = jobStateUpdate.setJobApplicationStatus(jobID, 'app status', 'source')
self.assertTrue(res['OK'])
# res = jobStateUpdate.setJobFlag()
# self.assertTrue(res['OK'])
# res = jobStateUpdate.unsetJobFlag()
# self.assertTrue(res['OK'])
res = jobStateUpdate.setJobSite(jobID, 'Site')
self.assertTrue(res['OK'])
# res = jobMonitor.traceJobParameter( 'Site', 1, 'Status' )
# self.assertTrue(res['OK'])
# now checking few things
res = jobMonitor.getJobStatus(jobID)
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], 'Running')
res = jobMonitor.getJobParameter(jobID, 'par1')
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], {'par1': 'par1Value'})
res = jobMonitor.getJobParameters(jobID)
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], {jobID: {'par1': 'par1Value', 'par2': 'par2Value'}})
res = jobMonitor.getJobParameters(jobID, 'par1')
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], {jobID: {'par1': 'par1Value'}})
res = jobMonitor.getJobAttribute(jobID, 'Site')
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], 'Site')
res = jobMonitor.getJobAttributes(jobID)
self.assertTrue(res['OK'])
self.assertEqual(res['Value']['ApplicationStatus'], 'app status')
self.assertEqual(res['Value']['JobName'], 'helloWorld')
res = jobMonitor.getJobSummary(jobID)
self.assertTrue(res['OK'])
self.assertEqual(res['Value']['ApplicationStatus'], 'app status')
self.assertEqual(res['Value']['Status'], 'Running')
res = jobMonitor.getJobHeartBeatData(jobID)
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], [])
res = jobMonitor.getInputData(jobID)
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], [])
res = jobMonitor.getJobPrimarySummary(jobID)
self.assertTrue(res['OK'])
res = jobMonitor.getAtticJobParameters(jobID)
self.assertTrue(res['OK'])
res = jobStateUpdate.setJobsStatus([jobID], 'Done', 'MinorStatus', 'Unknown')
self.assertTrue(res['OK'])
res = jobMonitor.getJobSummary(jobID)
self.assertTrue(res['OK'])
self.assertEqual(res['Value']['Status'], 'Done')
self.assertEqual(res['Value']['MinorStatus'], 'MinorStatus')
self.assertEqual(res['Value']['ApplicationStatus'], 'app status')
res = jobStateUpdate.sendHeartBeat(jobID, {'bih': 'bih'}, {'boh': 'boh'})
self.assertTrue(res['OK'])
# delete the job - this will just set its status to "deleted"
wmsClient.deleteJob(jobID)
