def createJob():
job = helloWorldJob()
jobDescription = createFile(job)
wmsClient = WMSClient()
res = wmsClient.submitJob(job._toJDL(xmlFile=jobDescription))
assert res["OK"], res["Message"]
jobID = int(res["Value"])
return jobID
def test_matcher(self):
# insert a proper DN to run the test
resourceDescription = {
"OwnerGroup": "prod",
"OwnerDN": "/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser",
"DIRACVersion": "pippo",
"GridCE": "some.grid.ce.org",
"ReleaseVersion": "blabla",
"VirtualOrganization": "LHCb",
"PilotInfoReportedFlag": "True",
"PilotBenchmark": "anotherPilot",
"Site": "DIRAC.Jenkins.ch",
"CPUTime": 86400,
}
wmsClient = WMSClient()
job = helloWorldJob()
job.setDestination("DIRAC.Jenkins.ch")
job.setInputData("/a/bbb")
job.setType("User")
jobDescription = createFile(job)
res = wmsClient.submitJob(job._toJDL(xmlFile=jobDescription))
self.assertTrue(res["OK"], res.get("Message"))
jobID = res["Value"]
# forcing the update
res = JobStateUpdateClient().setJobStatus(jobID, JobStatus.WAITING,
"matching", "source", None,
True)
self.assertTrue(res["OK"], res.get("Message"))
tqDB = TaskQueueDB()
tqDefDict = {
"OwnerDN": "/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser",
"OwnerGroup": "prod",
"Setup": "dirac-JenkinsSetup",
"CPUTime": 86400,
}
res = tqDB.insertJob(jobID, tqDefDict, 10)
self.assertTrue(res["OK"], res.get("Message"))
res = MatcherClient().requestJob(resourceDescription)
print(res)
self.assertTrue(res["OK"], res.get("Message"))
wmsClient.deleteJob(jobID)
def test_matcher(self):
# insert a proper DN to run the test
resourceDescription = {
'OwnerGroup': 'prod',
'OwnerDN':
'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/[email protected]',
'DIRACVersion': 'pippo',
'ReleaseVersion': 'blabla',
'VirtualOrganization': 'LHCb',
'PilotInfoReportedFlag': 'True',
'PilotBenchmark': 'anotherPilot',
'Site': 'DIRAC.Jenkins.ch',
'CPUTime': 86400
}
wmsClient = WMSClient()
job = helloWorldJob()
job.setDestination('DIRAC.Jenkins.ch')
job.setInputData('/a/bbb')
job.setType('User')
jobDescription = createFile(job)
res = wmsClient.submitJob(job._toJDL(xmlFile=jobDescription))
self.assertTrue(res['OK'], res.get('Message'))
jobID = res['Value']
res = JobStateUpdateClient().setJobStatus(jobID, 'Waiting', 'matching',
'source')
self.assertTrue(res['OK'], res.get('Message'))
tqDB = TaskQueueDB()
tqDefDict = {
'OwnerDN':
'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/[email protected]',
'OwnerGroup': 'prod',
'Setup': 'dirac-JenkinsSetup',
'CPUTime': 86400
}
res = tqDB.insertJob(jobID, tqDefDict, 10)
self.assertTrue(res['OK'], res.get('Message'))
res = MatcherClient().requestJob(resourceDescription)
print(res)
self.assertTrue(res['OK'], res.get('Message'))
wmsClient.deleteJob(jobID)
def test_ParametricChain(self):
"""This test will submit a parametric job which should generate 3 actual jobs"""
wmsClient = WMSClient()
jobStateUpdate = JobStateUpdateClient()
jobMonitor = JobMonitoringClient()
# create the job
job = parametricJob()
jobDescription = createFile(job)
# submit the job
res = wmsClient.submitJob(job._toJDL(xmlFile=jobDescription))
self.assertTrue(res["OK"], res.get("Message"))
jobIDList = res["Value"]
self.assertEqual(len(jobIDList), 3, msg="Got %s" % str(jobIDList))
res = jobMonitor.getJobsParameters(jobIDList, ["JobName"])
self.assertTrue(res["OK"], res.get("Message"))
jobNames = [res["Value"][jobID]["JobName"] for jobID in res["Value"]]
self.assertEqual(
set(jobNames),
set(["parametric_helloWorld_%s" % nJob for nJob in range(3)]))
for jobID in jobIDList:
res = jobStateUpdate.setJobStatus(jobID, JobStatus.CHECKING,
"checking", "source")
self.assertTrue(res["OK"], res.get("Message"))
res = wmsClient.deleteJob(jobIDList)
self.assertTrue(res["OK"], res.get("Message"))
print(res)
for jobID in jobIDList:
res = jobMonitor.getJobsStatus(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"][jobID]["Status"],
JobStatus.DELETED,
msg="Got %s" % str(res["Value"]))
def test_ParametricChain(self):
""" This test will submit a parametric job which should generate 3 actual jobs
"""
wmsClient = WMSClient()
jobStateUpdate = JobStateUpdateClient()
jobMonitor = JobMonitoringClient()
# create the job
job = parametricJob()
jobDescription = createFile(job)
# submit the job
res = wmsClient.submitJob(job._toJDL(xmlFile=jobDescription))
self.assertTrue(res['OK'], res.get('Message'))
jobIDList = res['Value']
self.assertEqual(len(jobIDList), 3, msg="Got %s" % str(jobIDList))
res = jobMonitor.getJobsParameters(jobIDList, ['JobName'])
self.assertTrue(res['OK'], res.get('Message'))
jobNames = [res['Value'][jobID]['JobName'] for jobID in res['Value']]
self.assertEqual(
set(jobNames),
set(['parametric_helloWorld_%s' % nJob for nJob in range(3)]))
for jobID in jobIDList:
res = jobStateUpdate.setJobStatus(jobID, 'Done', 'matching',
'source')
self.assertTrue(res['OK'], res.get('Message'))
res = wmsClient.deleteJob(jobIDList)
self.assertTrue(res['OK'], res.get('Message'))
for jobID in jobIDList:
res = jobMonitor.getJobStatus(jobID)
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(res['Value'],
'Deleted',
msg="Got %s" % str(res['Value']))
def test_FullChain(self):
"""This test will
- call all the WMSClient methods
that will end up calling all the JobManager service methods
- use the JobMonitoring to verify few properties
- call the JobCleaningAgent to eliminate job entries from the DBs
"""
wmsClient = WMSClient()
jobMonitor = JobMonitoringClient()
jobStateUpdate = JobStateUpdateClient()
# create the job
job = helloWorldJob()
jobDescription = createFile(job)
# submit the job
res = wmsClient.submitJob(job._toJDL(xmlFile=jobDescription))
self.assertTrue(res["OK"], res.get("Message"))
self.assertTrue(isinstance(res["Value"], int),
msg="Got %s" % type(res["Value"]))
self.assertEqual(res["Value"],
res["JobID"],
msg="Got %s, expected %s" %
(str(res["Value"]), res["JobID"]))
jobID = res["JobID"]
jobID = res["Value"]
# updating the status
res = jobStateUpdate.setJobStatus(jobID, JobStatus.RUNNING,
"Executing Minchiapp", "source")
self.assertTrue(res["OK"], res.get("Message"))
# reset the job
res = wmsClient.resetJob(jobID)
self.assertTrue(res["OK"], res.get("Message"))
# reschedule the job
res = wmsClient.rescheduleJob(jobID)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobsStatus(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"][jobID]["Status"],
JobStatus.RECEIVED,
msg="Got %s" % str(res["Value"]))
res = jobMonitor.getJobsMinorStatus([jobID])
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"],
{jobID: {
"MinorStatus": "Job Rescheduled"
}},
msg="Got %s" % str(res["Value"]))
res = jobMonitor.getJobsApplicationStatus([jobID])
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"],
{jobID: {
"ApplicationStatus": "Unknown"
}},
msg="Got %s" % str(res["Value"]))
res = jobMonitor.getJobsStates([jobID])
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(
res["Value"],
{
jobID: {
"Status": JobStatus.RECEIVED,
"MinorStatus": "Job Rescheduled",
"ApplicationStatus": "Unknown"
}
},
msg="Got %s" % str(res["Value"]),
)
# updating the status again
res = jobStateUpdate.setJobStatus(jobID, JobStatus.CHECKING,
"checking", "source")
self.assertTrue(res["OK"], res.get("Message"))
res = jobStateUpdate.setJobStatus(jobID, JobStatus.WAITING, "waiting",
"source")
self.assertTrue(res["OK"], res.get("Message"))
res = jobStateUpdate.setJobStatus(jobID, JobStatus.MATCHED, "matched",
"source")
self.assertTrue(res["OK"], res.get("Message"))
# kill the job
res = wmsClient.killJob(jobID)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobsStatus(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"][jobID]["Status"],
JobStatus.KILLED,
msg="Got %s" % str(res["Value"]))
# delete the job - this will just set its status to "deleted"
res = wmsClient.deleteJob(jobID)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobsStatus(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"][jobID]["Status"],
JobStatus.DELETED,
msg="Got %s" % str(res["Value"]))
def test_JobStateUpdateAndJobMonitoringMultuple(self):
"""# Now, let's submit some jobs. Different sites, types, inputs"""
wmsClient = WMSClient()
jobMonitor = JobMonitoringClient()
jobStateUpdate = JobStateUpdateClient()
jobIDs = []
lfnss = [["/a/1.txt", "/a/2.txt"],
["/a/1.txt", "/a/3.txt", "/a/4.txt"], []]
types = ["User", "Test"]
for lfns in lfnss:
for jobType in types:
job = helloWorldJob()
job.setDestination("DIRAC.Jenkins.ch")
job.setInputData(lfns)
job.setType(jobType)
jobDescription = createFile(job)
res = wmsClient.submitJob(job._toJDL(xmlFile=jobDescription))
self.assertTrue(res["OK"], res.get("Message"))
jobID = res["Value"]
jobIDs.append(jobID)
res = jobMonitor.getSites()
print(res)
self.assertTrue(res["OK"], res.get("Message"))
self.assertTrue(
set(res["Value"]) <= {"ANY", "DIRAC.Jenkins.ch", "Site"},
msg="Got %s" % res["Value"])
res = jobMonitor.getJobTypes()
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(sorted(res["Value"]),
sorted(types),
msg="Got %s" % str(sorted(res["Value"])))
res = jobMonitor.getApplicationStates()
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"], ["app status", "Unknown"],
msg="Got %s" % str(res["Value"]))
res = jobMonitor.getOwners()
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getOwnerGroup()
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getProductionIds()
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobGroups()
self.assertTrue(res["OK"], res.get("Message"))
resJG_empty = res["Value"]
res = jobMonitor.getJobGroups(None, datetime.datetime.utcnow())
self.assertTrue(res["OK"], res.get("Message"))
resJG_olderThanNow = res["Value"]
self.assertEqual(resJG_empty, resJG_olderThanNow)
res = jobMonitor.getJobGroups(
None,
datetime.datetime.utcnow() - datetime.timedelta(days=365))
self.assertTrue(res["OK"], res.get("Message"))
resJG_olderThanOneYear = res["Value"]
self.assertTrue(
set(resJG_olderThanOneYear).issubset(set(resJG_olderThanNow)),
resJG_olderThanOneYear)
res = jobMonitor.getStates()
self.assertTrue(res["OK"], res.get("Message"))
self.assertTrue(
sorted(res["Value"])
in [[JobStatus.RECEIVED],
sorted([JobStatus.RECEIVED, JobStatus.KILLED])], res["Value"])
res = jobMonitor.getMinorStates()
self.assertTrue(res["OK"], res.get("Message"))
self.assertTrue(
sorted(res["Value"]) in [
["Job accepted"],
sorted(["Job accepted", "Job Rescheduled"]),
sorted(["Job accepted", "Marked for termination"]),
],
res["Value"],
)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobs()
self.assertTrue(res["OK"], res.get("Message"))
self.assertTrue(
set([str(x) for x in jobIDs]) <= set(res["Value"]), res["Value"])
# res = jobMonitor.getCounters(attrList)
# self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getJobsSummary(jobIDs)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobPageSummaryWeb({}, [], 0, 100)
self.assertTrue(res["OK"], res.get("Message"))
res = jobStateUpdate.setJobStatusBulk(
jobID,
{
str(datetime.datetime.utcnow()): {
"Status": JobStatus.CHECKING,
"MinorStatus": "MinorStatus",
"Source": "Unknown",
}
},
False,
)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobSummary(int(jobID))
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"]["Status"], JobStatus.CHECKING)
self.assertEqual(res["Value"]["MinorStatus"], "MinorStatus")
res = jobStateUpdate.setJobStatusBulk(
jobID,
{
str(datetime.datetime.utcnow() + datetime.timedelta(hours=1)):
{
"Status": JobStatus.WAITING,
"MinorStatus": "MinorStatus",
"Source": "Unknown",
},
str(datetime.datetime.utcnow() + datetime.timedelta(hours=2)):
{
"Status": JobStatus.MATCHED,
"MinorStatus": "MinorStatus-matched",
"Source": "Unknown",
},
},
False,
)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobSummary(int(jobID))
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"]["Status"], JobStatus.MATCHED)
self.assertEqual(res["Value"]["MinorStatus"], "MinorStatus-matched")
res = jobStateUpdate.setJobsParameter({jobID: ["Whatever", "booh"]})
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobSummary(int(jobID))
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"]["Status"], JobStatus.MATCHED)
self.assertEqual(res["Value"]["MinorStatus"], "MinorStatus-matched")
res = jobStateUpdate.setJobAttribute(jobID, "Status",
JobStatus.RUNNING)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobSummary(int(jobID))
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"]["Status"], JobStatus.RUNNING)
# delete the jobs - this will just set its status to "deleted"
wmsClient.deleteJob(jobIDs)
def test_JobStateUpdateAndJobMonitoring(self):
"""Verifying all JobStateUpdate and JobMonitoring functions"""
wmsClient = WMSClient()
jobMonitor = JobMonitoringClient()
jobStateUpdate = JobStateUpdateClient()
# create a job and check stuff
job = helloWorldJob()
jobDescription = createFile(job)
# submitting the job. Checking few stuff
res = wmsClient.submitJob(job._toJDL(xmlFile=jobDescription))
self.assertTrue(res["OK"], res.get("Message"))
jobID = int(res["Value"])
# jobID = res['JobID']
res = jobMonitor.getJobJDL(jobID, True)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobJDL(jobID, False)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobsParameters([jobID], [])
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobOwner(jobID)
self.assertTrue(res["OK"], res.get("Message"))
# Adding stuff
# forcing the update
res = jobStateUpdate.setJobStatus(jobID, JobStatus.RUNNING, "running",
"source", None, True)
self.assertTrue(res["OK"], res.get("Message"))
res = jobStateUpdate.setJobParameters(jobID, [("par1", "par1Value"),
("par2", "par2Value")])
time.sleep(5)
self.assertTrue(res["OK"], res.get("Message"))
res = jobStateUpdate.setJobApplicationStatus(jobID, "app status",
"source")
self.assertTrue(res["OK"], res.get("Message"))
# res = jobStateUpdate.setJobFlag()
# self.assertTrue(res['OK'], res.get('Message'))
# res = jobStateUpdate.unsetJobFlag()
# self.assertTrue(res['OK'], res.get('Message'))
res = jobStateUpdate.setJobSite(jobID, "Site")
self.assertTrue(res["OK"], res.get("Message"))
# now checking few things
res = jobMonitor.getJobsStatus(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"][jobID]["Status"],
JobStatus.RUNNING,
msg="Got %s" % str(res["Value"]))
res = jobMonitor.getJobParameter(jobID, "par1")
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"], {"par1": "par1Value"},
msg="Got %s" % str(res["Value"]))
res = jobMonitor.getJobParameters(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"],
{jobID: {
"par1": "par1Value",
"par2": "par2Value"
}},
msg="Got %s" % str(res["Value"]))
res = jobMonitor.getJobParameters(jobID, "par1")
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"], {jobID: {
"par1": "par1Value"
}},
msg="Got %s" % str(res["Value"]))
res = jobMonitor.getJobAttribute(jobID, "Site")
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"],
"Site",
msg="Got %s" % str(res["Value"]))
res = jobMonitor.getJobAttributes(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"]["ApplicationStatus"],
"app status",
msg="Got %s" % str(res["Value"]["ApplicationStatus"]))
self.assertEqual(res["Value"]["JobName"],
"helloWorld",
msg="Got %s" % str(res["Value"]["JobName"]))
res = jobMonitor.getJobSummary(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"]["ApplicationStatus"],
"app status",
msg="Got %s" % str(res["Value"]["ApplicationStatus"]))
self.assertEqual(res["Value"]["Status"],
JobStatus.RUNNING,
msg="Got %s" % str(res["Value"]["Status"]))
res = jobMonitor.getJobHeartBeatData(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"], [], msg="Got %s" % str(res["Value"]))
res = jobMonitor.getInputData(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"], [], msg="Got %s" % str(res["Value"]))
res = jobMonitor.getJobSummary(jobID)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getAtticJobParameters(jobID)
self.assertTrue(res["OK"], res.get("Message"))
res = jobStateUpdate.setJobStatus(jobID, JobStatus.DONE, "MinorStatus",
"Unknown")
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobSummary(jobID)
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"]["Status"],
JobStatus.DONE,
msg="Got %s" % str(res["Value"]["Status"]))
self.assertEqual(res["Value"]["MinorStatus"],
"MinorStatus",
msg="Got %s" % str(res["Value"]["MinorStatus"]))
self.assertEqual(res["Value"]["ApplicationStatus"],
"app status",
msg="Got %s" % str(res["Value"]["ApplicationStatus"]))
res = jobStateUpdate.sendHeartBeat(jobID, {"bih": "bih"},
{"boh": "boh"})
self.assertTrue(res["OK"], res.get("Message"))
# delete the job - this will just set its status to "deleted"
wmsClient.deleteJob(jobID)
def test_FullChain(self):
""" This test will
- call all the WMSClient methods
that will end up calling all the JobManager service methods
- use the JobMonitoring to verify few properties
- call the JobCleaningAgent to eliminate job entries from the DBs
"""
wmsClient = WMSClient()
jobMonitor = JobMonitoringClient()
jobStateUpdate = JobStateUpdateClient()
# create the job
job = helloWorldJob()
jobDescription = createFile(job)
# submit the job
res = wmsClient.submitJob(job._toJDL(xmlFile=jobDescription))
self.assertTrue(res['OK'], res.get('Message'))
self.assertTrue(isinstance(res['Value'], int),
msg="Got %s" % type(res['Value']))
self.assertEqual(res['Value'],
res['JobID'],
msg="Got %s, expected %s" %
(str(res['Value']), res['JobID']))
jobID = res['JobID']
jobID = res['Value']
# updating the status
res = jobStateUpdate.setJobStatus(jobID, 'Running',
'Executing Minchiapp', 'source')
self.assertTrue(res['OK'], res.get('Message'))
# reset the job
res = wmsClient.resetJob(jobID)
self.assertTrue(res['OK'], res.get('Message'))
# reschedule the job
res = wmsClient.rescheduleJob(jobID)
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getJobStatus(jobID)
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(res['Value'],
'Received',
msg="Got %s" % str(res['Value']))
res = jobMonitor.getJobsMinorStatus([jobID])
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(
res['Value'],
{jobID: {
'MinorStatus': 'Job Rescheduled',
'JobID': jobID
}},
msg="Got %s" % str(res['Value']))
res = jobMonitor.getJobsApplicationStatus([jobID])
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(
res['Value'],
{jobID: {
'ApplicationStatus': 'Unknown',
'JobID': jobID
}},
msg="Got %s" % str(res['Value']))
# updating the status again
res = jobStateUpdate.setJobStatus(jobID, 'Matched', 'matching',
'source')
self.assertTrue(res['OK'], res.get('Message'))
# kill the job
res = wmsClient.killJob(jobID)
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getJobStatus(jobID)
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(res['Value'],
'Killed',
msg="Got %s" % str(res['Value']))
# updating the status aaaagain
res = jobStateUpdate.setJobStatus(jobID, 'Done', 'matching', 'source')
self.assertTrue(res['OK'], res.get('Message'))
# kill the job
res = wmsClient.killJob(jobID)
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getJobStatus(jobID)
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(
res['Value'], 'Done', msg="Got %s" %
str(res['Value'])) # this time it won't kill... it's done!
# delete the job - this will just set its status to "deleted"
res = wmsClient.deleteJob(jobID)
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getJobStatus(jobID)
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(res['Value'],
'Deleted',
msg="Got %s" % str(res['Value']))
def test_JobStateUpdateAndJobMonitoringMultuple(self):
""" # Now, let's submit some jobs. Different sites, types, inputs
"""
wmsClient = WMSClient()
jobMonitor = JobMonitoringClient()
jobStateUpdate = JobStateUpdateClient()
jobIDs = []
lfnss = [['/a/1.txt', '/a/2.txt'],
['/a/1.txt', '/a/3.txt', '/a/4.txt'], []]
types = ['User', 'Test']
for lfns in lfnss:
for jobType in types:
job = helloWorldJob()
job.setDestination('DIRAC.Jenkins.ch')
job.setInputData(lfns)
job.setType(jobType)
jobDescription = createFile(job)
res = wmsClient.submitJob(job._toJDL(xmlFile=jobDescription))
self.assertTrue(res['OK'], res.get('Message'))
jobID = res['Value']
jobIDs.append(jobID)
res = jobMonitor.getSites()
print(res)
self.assertTrue(res['OK'], res.get('Message'))
self.assertTrue(
set(res['Value']) <= {'ANY', 'DIRAC.Jenkins.ch', 'Site'},
msg="Got %s" % res['Value'])
res = jobMonitor.getJobTypes()
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(sorted(res['Value']),
sorted(types),
msg="Got %s" % str(sorted(res['Value'])))
res = jobMonitor.getApplicationStates()
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(sorted(res['Value']),
sorted(['Unknown']),
msg="Got %s" % str(res['Value']))
res = jobMonitor.getOwners()
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getOwnerGroup()
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getProductionIds()
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getJobGroups()
self.assertTrue(res['OK'], res.get('Message'))
resJG_empty = res['Value']
res = jobMonitor.getJobGroups(None, datetime.datetime.utcnow())
self.assertTrue(res['OK'], res.get('Message'))
resJG_olderThanNow = res['Value']
self.assertEqual(resJG_empty, resJG_olderThanNow)
res = jobMonitor.getJobGroups(
None,
datetime.datetime.utcnow() - datetime.timedelta(days=365))
self.assertTrue(res['OK'], res.get('Message'))
resJG_olderThanOneYear = res['Value']
self.assertTrue(
set(resJG_olderThanOneYear).issubset(set(resJG_olderThanNow)))
res = jobMonitor.getStates()
self.assertTrue(res['OK'], res.get('Message'))
self.assertTrue(
sorted(res['Value']) in [['Received'],
sorted(['Received', 'Waiting'])])
res = jobMonitor.getMinorStates()
self.assertTrue(res['OK'], res.get('Message'))
self.assertTrue(
sorted(res['Value']) in [['Job accepted'],
sorted(
['Job accepted', 'Job Rescheduled'])])
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getJobs()
self.assertTrue(res['OK'], res.get('Message'))
self.assertTrue(set([str(x) for x in jobIDs]) <= set(res['Value']))
# res = jobMonitor.getCounters(attrList)
# self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getCurrentJobCounters()
self.assertTrue(res['OK'], res.get('Message'))
try:
self.assertTrue(
res['Value'].get('Received') +
res['Value'].get('Waiting') >= int(len(lfnss) * len(types)))
except TypeError:
pass
res = jobMonitor.getJobsSummary(jobIDs)
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getJobPageSummaryWeb({}, [], 0, 100)
self.assertTrue(res['OK'], res.get('Message'))
res = jobStateUpdate.setJobStatusBulk(
jobID, {
str(datetime.datetime.utcnow()): {
'Status': 'Matched',
'MinorStatus': 'MinorStatus',
'ApplicationStatus': 'ApplicationStatus',
'Source': 'Unknown'
},
str(datetime.datetime.utcnow() + datetime.timedelta(hours=1)):
{
'Status': 'Running',
'MinorStatus': 'MinorStatus',
'ApplicationStatus': 'ApplicationStatus',
'Source': 'Unknown'
},
str(datetime.datetime.utcnow() + datetime.timedelta(hours=2)):
{
'Status': 'Completed',
'MinorStatus': 'MinorStatus',
'ApplicationStatus': 'ApplicationStatus',
'Source': 'Unknown'
}
})
self.assertTrue(res['OK'], res.get('Message'))
res = jobStateUpdate.setJobsParameter({jobID: ['Status', 'Running']})
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getJobSummary(int(jobID))
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(res['Value']['Status'], 'Completed')
self.assertEqual(res['Value']['MinorStatus'], 'MinorStatus')
# delete the jobs - this will just set its status to "deleted"
wmsClient.deleteJob(jobIDs)
def test_JobStateUpdateAndJobMonitoring(self):
""" Verifying all JobStateUpdate and JobMonitoring functions
"""
wmsClient = WMSClient()
jobMonitor = JobMonitoringClient()
jobStateUpdate = JobStateUpdateClient()
# create a job and check stuff
job = helloWorldJob()
jobDescription = createFile(job)
# submitting the job. Checking few stuff
res = wmsClient.submitJob(job._toJDL(xmlFile=jobDescription))
self.assertTrue(res['OK'], res.get('Message'))
jobID = int(res['Value'])
# jobID = res['JobID']
res = jobMonitor.getJobJDL(jobID, True)
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getJobJDL(jobID, False)
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getJobsParameters([jobID], [])
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(res['Value'], {}, msg="Got %s" % str(res['Value']))
res = jobMonitor.getJobOwner(jobID)
self.assertTrue(res['OK'], res.get('Message'))
# Adding stuff
res = jobStateUpdate.setJobStatus(jobID, 'Matched', 'matching',
'source')
self.assertTrue(res['OK'], res.get('Message'))
res = jobStateUpdate.setJobParameters(jobID, [('par1', 'par1Value'),
('par2', 'par2Value')])
time.sleep(5)
self.assertTrue(res['OK'], res.get('Message'))
res = jobStateUpdate.setJobApplicationStatus(jobID, 'app status',
'source')
self.assertTrue(res['OK'], res.get('Message'))
# res = jobStateUpdate.setJobFlag()
# self.assertTrue(res['OK'], res.get('Message'))
# res = jobStateUpdate.unsetJobFlag()
# self.assertTrue(res['OK'], res.get('Message'))
res = jobStateUpdate.setJobSite(jobID, 'Site')
self.assertTrue(res['OK'], res.get('Message'))
# now checking few things
res = jobMonitor.getJobStatus(jobID)
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(res['Value'],
'Running',
msg="Got %s" % str(res['Value']))
res = jobMonitor.getJobParameter(jobID, 'par1')
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(res['Value'], {'par1': 'par1Value'},
msg="Got %s" % str(res['Value']))
res = jobMonitor.getJobParameters(jobID)
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(res['Value'],
{jobID: {
'par1': 'par1Value',
'par2': 'par2Value'
}},
msg="Got %s" % str(res['Value']))
res = jobMonitor.getJobParameters(jobID, 'par1')
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(res['Value'], {jobID: {
'par1': 'par1Value'
}},
msg="Got %s" % str(res['Value']))
res = jobMonitor.getJobAttribute(jobID, 'Site')
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(res['Value'],
'Site',
msg="Got %s" % str(res['Value']))
res = jobMonitor.getJobAttributes(jobID)
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(res['Value']['ApplicationStatus'],
'app status',
msg="Got %s" % str(res['Value']['ApplicationStatus']))
self.assertEqual(res['Value']['JobName'],
'helloWorld',
msg="Got %s" % str(res['Value']['JobName']))
res = jobMonitor.getJobSummary(jobID)
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(res['Value']['ApplicationStatus'],
'app status',
msg="Got %s" % str(res['Value']['ApplicationStatus']))
self.assertEqual(res['Value']['Status'],
'Running',
msg="Got %s" % str(res['Value']['Status']))
res = jobMonitor.getJobHeartBeatData(jobID)
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(res['Value'], [], msg="Got %s" % str(res['Value']))
res = jobMonitor.getInputData(jobID)
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(res['Value'], [], msg="Got %s" % str(res['Value']))
res = jobMonitor.getJobPrimarySummary(jobID)
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getAtticJobParameters(jobID)
self.assertTrue(res['OK'], res.get('Message'))
res = jobStateUpdate.setJobStatus(jobID, 'Done', 'MinorStatus',
'Unknown')
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getJobSummary(jobID)
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(res['Value']['Status'],
'Done',
msg="Got %s" % str(res['Value']['Status']))
self.assertEqual(res['Value']['MinorStatus'],
'MinorStatus',
msg="Got %s" % str(res['Value']['MinorStatus']))
self.assertEqual(res['Value']['ApplicationStatus'],
'app status',
msg="Got %s" % str(res['Value']['ApplicationStatus']))
res = jobStateUpdate.sendHeartBeat(jobID, {'bih': 'bih'},
{'boh': 'boh'})
self.assertTrue(res['OK'], res.get('Message'))
# delete the job - this will just set its status to "deleted"
wmsClient.deleteJob(jobID)
