def test_JobStateUpdateAndJobMonitoringMultuple(self):
""" # Now, let's submit some jobs. Different sites, types, inputs
"""
wmsClient = WMSClient()
jobMonitor = JobMonitoringClient()
jobStateUpdate = RPCClient('WorkloadManagement/JobStateUpdate')
jobIDs = []
dests = ['DIRAC.site1.org', 'DIRAC.site2.org']
lfnss = [['/a/1.txt', '/a/2.txt'],
['/a/1.txt', '/a/3.txt', '/a/4.txt'], []]
types = ['User', 'Test']
for dest in dests:
for lfns in lfnss:
for jobType in types:
job = helloWorldJob()
job.setDestination(dest)
job.setInputData(lfns)
job.setType(jobType)
jobDescription = createFile(job)
res = wmsClient.submitJob(
job._toJDL(xmlFile=jobDescription))
self.assert_(res['OK'])
jobID = res['Value']
jobIDs.append(jobID)
res = jobMonitor.getSites()
self.assert_(res['OK'])
self.assert_(
set(res['Value']) <= set(dests + ['ANY', 'DIRAC.Jenkins.ch']))
res = jobMonitor.getJobTypes()
self.assert_(res['OK'])
self.assertEqual(sorted(res['Value']), sorted(types))
res = jobMonitor.getApplicationStates()
self.assert_(res['OK'])
self.assertEqual(sorted(res['Value']), sorted(['Unknown']))
res = jobMonitor.getOwners()
self.assert_(res['OK'])
res = jobMonitor.getOwnerGroup()
self.assert_(res['OK'])
res = jobMonitor.getProductionIds()
self.assert_(res['OK'])
res = jobMonitor.getJobGroups()
self.assert_(res['OK'])
res = jobMonitor.getStates()
self.assert_(res['OK'])
self.assert_(
sorted(res['Value']) in [['Received'],
sorted(['Received', 'Waiting'])])
res = jobMonitor.getMinorStates()
self.assert_(res['OK'])
self.assert_(
sorted(res['Value']) in [['Job accepted'],
sorted(['Job accepted', 'matching'])])
self.assert_(res['OK'])
res = jobMonitor.getJobs()
self.assert_(res['OK'])
self.assert_(set([str(x) for x in jobIDs]) <= set(res['Value']))
# res = jobMonitor.getCounters(attrList)
# self.assert_( res['OK'] )
res = jobMonitor.getCurrentJobCounters()
self.assert_(res['OK'])
try:
self.assert_(
res['Value'].get('Received') + res['Value'].get('Waiting') >=
long(len(dests) * len(lfnss) * len(types)))
except TypeError:
pass
res = jobMonitor.getJobsSummary(jobIDs)
self.assert_(res['OK'])
res = jobMonitor.getJobPageSummaryWeb({}, [], 0, 100)
self.assert_(res['OK'])
res = jobStateUpdate.setJobStatusBulk(
jobID, {
str(datetime.datetime.utcnow()): {
'Status': 'Running',
'MinorStatus': 'MinorStatus',
'ApplicationStatus': 'ApplicationStatus',
'Source': 'Unknown'
}
})
self.assert_(res['OK'])
res = jobStateUpdate.setJobsParameter({jobID: ['Status', 'Running']})
self.assert_(res['OK'])
# delete the jobs - this will just set its status to "deleted"
wmsClient.deleteJob(jobIDs)
def test_JobStateUpdateAndJobMonitoringMultuple(self):
""" # Now, let's submit some jobs. Different sites, types, inputs
"""
wmsClient = WMSClient()
jobMonitor = JobMonitoringClient()
jobStateUpdate = JobStateUpdateClient()
jobIDs = []
lfnss = [['/a/1.txt', '/a/2.txt'],
['/a/1.txt', '/a/3.txt', '/a/4.txt'], []]
types = ['User', 'Test']
for lfns in lfnss:
for jobType in types:
job = helloWorldJob()
job.setDestination('DIRAC.Jenkins.ch')
job.setInputData(lfns)
job.setType(jobType)
jobDescription = createFile(job)
res = wmsClient.submitJob(job._toJDL(xmlFile=jobDescription))
self.assertTrue(res['OK'], res.get('Message'))
jobID = res['Value']
jobIDs.append(jobID)
res = jobMonitor.getSites()
print(res)
self.assertTrue(res['OK'], res.get('Message'))
self.assertTrue(set(res['Value']) <= {'ANY', 'DIRAC.Jenkins.ch'},
msg="Got %s" % res['Value'])
res = jobMonitor.getJobTypes()
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(sorted(res['Value']),
sorted(types),
msg="Got %s" % str(sorted(res['Value'])))
res = jobMonitor.getApplicationStates()
self.assertTrue(res['OK'], res.get('Message'))
self.assertEqual(sorted(res['Value']),
sorted(['Unknown']),
msg="Got %s" % sorted(str(res['Value'])))
res = jobMonitor.getOwners()
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getOwnerGroup()
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getProductionIds()
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getJobGroups()
self.assertTrue(res['OK'], res.get('Message'))
resJG_empty = res['Value']
res = jobMonitor.getJobGroups(None, datetime.datetime.utcnow())
self.assertTrue(res['OK'], res.get('Message'))
resJG_olderThanNow = res['Value']
self.assertEqual(resJG_empty, resJG_olderThanNow)
res = jobMonitor.getJobGroups(
None,
datetime.datetime.utcnow() - datetime.timedelta(days=365))
self.assertTrue(res['OK'], res.get('Message'))
resJG_olderThanOneYear = res['Value']
self.assertTrue(
set(resJG_olderThanOneYear).issubset(set(resJG_olderThanNow)))
res = jobMonitor.getStates()
self.assertTrue(res['OK'], res.get('Message'))
self.assertTrue(
sorted(res['Value']) in [['Received'],
sorted(['Received', 'Waiting'])])
res = jobMonitor.getMinorStates()
self.assertTrue(res['OK'], res.get('Message'))
self.assertTrue(
sorted(res['Value']) in [['Job accepted'],
sorted(
['Job accepted', 'Job Rescheduled'])])
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getJobs()
self.assertTrue(res['OK'], res.get('Message'))
self.assertTrue(set([str(x) for x in jobIDs]) <= set(res['Value']))
# res = jobMonitor.getCounters(attrList)
# self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getCurrentJobCounters()
self.assertTrue(res['OK'], res.get('Message'))
try:
self.assertTrue(
res['Value'].get('Received') +
res['Value'].get('Waiting') >= int(len(lfnss) * len(types)))
except TypeError:
pass
res = jobMonitor.getJobsSummary(jobIDs)
self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getJobPageSummaryWeb({}, [], 0, 100)
self.assertTrue(res['OK'], res.get('Message'))
res = jobStateUpdate.setJobStatusBulk(
jobID, {
str(datetime.datetime.utcnow()): {
'Status': 'Running',
'MinorStatus': 'MinorStatus',
'ApplicationStatus': 'ApplicationStatus',
'Source': 'Unknown'
}
})
self.assertTrue(res['OK'], res.get('Message'))
res = jobStateUpdate.setJobsParameter({jobID: ['Status', 'Running']})
self.assertTrue(res['OK'], res.get('Message'))
# delete the jobs - this will just set its status to "deleted"
wmsClient.deleteJob(jobIDs)
def test_JobStateUpdateAndJobMonitoringMultuple( self ):
""" # Now, let's submit some jobs. Different sites, types, inputs
"""
wmsClient = WMSClient()
jobMonitor = JobMonitoringClient()
jobStateUpdate = RPCClient( 'WorkloadManagement/JobStateUpdate' )
jobIDs = []
dests = ['DIRAC.site1.org', 'DIRAC.site2.org']
lfnss = [['/a/1.txt', '/a/2.txt'], ['/a/1.txt', '/a/3.txt', '/a/4.txt'], []]
types = ['User', 'Test']
for dest in dests:
for lfns in lfnss:
for jobType in types:
job = helloWorldJob()
job.setDestination( dest )
job.setInputData( lfns )
job.setType( jobType )
jobDescription = createFile( job )
res = wmsClient.submitJob( job._toJDL( xmlFile = jobDescription ) )
self.assert_( res['OK'] )
jobID = res['Value']
jobIDs.append( jobID )
res = jobMonitor.getSites()
self.assert_( res['OK'] )
self.assert_( set( res['Value'] ) <= set( dests + ['ANY', 'DIRAC.Jenkins.org'] ) )
res = jobMonitor.getJobTypes()
self.assert_( res['OK'] )
self.assertEqual( sorted( res['Value'] ), sorted( types ) )
res = jobMonitor.getApplicationStates()
self.assert_( res['OK'] )
self.assertEqual( sorted( res['Value'] ), sorted( ['Unknown'] ) )
res = jobMonitor.getOwners()
self.assert_( res['OK'] )
res = jobMonitor.getOwnerGroup()
self.assert_( res['OK'] )
res = jobMonitor.getProductionIds()
self.assert_( res['OK'] )
res = jobMonitor.getJobGroups()
self.assert_( res['OK'] )
res = jobMonitor.getStates()
self.assert_( res['OK'] )
self.assert_( sorted( res['Value'] ) in [['Received'], sorted( ['Received', 'Waiting'] )] )
res = jobMonitor.getMinorStates()
self.assert_( res['OK'] )
self.assert_( sorted( res['Value'] ) in [['Job accepted'], sorted( ['Job accepted', 'matching'] ) ] )
self.assert_( res['OK'] )
res = jobMonitor.getJobs()
self.assert_( res['OK'] )
self.assert_( set( [str( x ) for x in jobIDs] ) <= set( res['Value'] ) )
# res = jobMonitor.getCounters(attrList)
# self.assert_( res['OK'] )
res = jobMonitor.getCurrentJobCounters()
self.assert_( res['OK'] )
try:
self.assert_( res['Value'].get( 'Received' ) + res['Value'].get( 'Waiting' ) >= long( len( dests ) * len( lfnss ) * len( types ) ) )
except TypeError:
pass
res = jobMonitor.getJobsSummary( jobIDs )
self.assert_( res['OK'] )
res = jobMonitor.getJobPageSummaryWeb( {}, [], 0, 100 )
self.assert_( res['OK'] )
res = jobStateUpdate.setJobStatusBulk( jobID, {str( datetime.datetime.utcnow() ):{'Status': 'Running',
'MinorStatus': 'MinorStatus',
'ApplicationStatus': 'ApplicationStatus',
'Source': 'Unknown'}} )
self.assert_( res['OK'] )
res = jobStateUpdate.setJobsParameter( {jobID:['Status', 'Running']} )
self.assert_( res['OK'] )
# delete the jobs - this will just set its status to "deleted"
wmsClient.deleteJob( jobIDs )
def test_JobStateUpdateAndJobMonitoringMultuple(self):
"""# Now, let's submit some jobs. Different sites, types, inputs"""
wmsClient = WMSClient()
jobMonitor = JobMonitoringClient()
jobStateUpdate = JobStateUpdateClient()
jobIDs = []
lfnss = [["/a/1.txt", "/a/2.txt"],
["/a/1.txt", "/a/3.txt", "/a/4.txt"], []]
types = ["User", "Test"]
for lfns in lfnss:
for jobType in types:
job = helloWorldJob()
job.setDestination("DIRAC.Jenkins.ch")
job.setInputData(lfns)
job.setType(jobType)
jobDescription = createFile(job)
res = wmsClient.submitJob(job._toJDL(xmlFile=jobDescription))
self.assertTrue(res["OK"], res.get("Message"))
jobID = res["Value"]
jobIDs.append(jobID)
res = jobMonitor.getSites()
print(res)
self.assertTrue(res["OK"], res.get("Message"))
self.assertTrue(
set(res["Value"]) <= {"ANY", "DIRAC.Jenkins.ch", "Site"},
msg="Got %s" % res["Value"])
res = jobMonitor.getJobTypes()
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(sorted(res["Value"]),
sorted(types),
msg="Got %s" % str(sorted(res["Value"])))
res = jobMonitor.getApplicationStates()
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"], ["app status", "Unknown"],
msg="Got %s" % str(res["Value"]))
res = jobMonitor.getOwners()
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getOwnerGroup()
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getProductionIds()
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobGroups()
self.assertTrue(res["OK"], res.get("Message"))
resJG_empty = res["Value"]
res = jobMonitor.getJobGroups(None, datetime.datetime.utcnow())
self.assertTrue(res["OK"], res.get("Message"))
resJG_olderThanNow = res["Value"]
self.assertEqual(resJG_empty, resJG_olderThanNow)
res = jobMonitor.getJobGroups(
None,
datetime.datetime.utcnow() - datetime.timedelta(days=365))
self.assertTrue(res["OK"], res.get("Message"))
resJG_olderThanOneYear = res["Value"]
self.assertTrue(
set(resJG_olderThanOneYear).issubset(set(resJG_olderThanNow)),
resJG_olderThanOneYear)
res = jobMonitor.getStates()
self.assertTrue(res["OK"], res.get("Message"))
self.assertTrue(
sorted(res["Value"])
in [[JobStatus.RECEIVED],
sorted([JobStatus.RECEIVED, JobStatus.KILLED])], res["Value"])
res = jobMonitor.getMinorStates()
self.assertTrue(res["OK"], res.get("Message"))
self.assertTrue(
sorted(res["Value"]) in [
["Job accepted"],
sorted(["Job accepted", "Job Rescheduled"]),
sorted(["Job accepted", "Marked for termination"]),
],
res["Value"],
)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobs()
self.assertTrue(res["OK"], res.get("Message"))
self.assertTrue(
set([str(x) for x in jobIDs]) <= set(res["Value"]), res["Value"])
# res = jobMonitor.getCounters(attrList)
# self.assertTrue(res['OK'], res.get('Message'))
res = jobMonitor.getJobsSummary(jobIDs)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobPageSummaryWeb({}, [], 0, 100)
self.assertTrue(res["OK"], res.get("Message"))
res = jobStateUpdate.setJobStatusBulk(
jobID,
{
str(datetime.datetime.utcnow()): {
"Status": JobStatus.CHECKING,
"MinorStatus": "MinorStatus",
"Source": "Unknown",
}
},
False,
)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobSummary(int(jobID))
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"]["Status"], JobStatus.CHECKING)
self.assertEqual(res["Value"]["MinorStatus"], "MinorStatus")
res = jobStateUpdate.setJobStatusBulk(
jobID,
{
str(datetime.datetime.utcnow() + datetime.timedelta(hours=1)):
{
"Status": JobStatus.WAITING,
"MinorStatus": "MinorStatus",
"Source": "Unknown",
},
str(datetime.datetime.utcnow() + datetime.timedelta(hours=2)):
{
"Status": JobStatus.MATCHED,
"MinorStatus": "MinorStatus-matched",
"Source": "Unknown",
},
},
False,
)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobSummary(int(jobID))
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"]["Status"], JobStatus.MATCHED)
self.assertEqual(res["Value"]["MinorStatus"], "MinorStatus-matched")
res = jobStateUpdate.setJobsParameter({jobID: ["Whatever", "booh"]})
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobSummary(int(jobID))
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"]["Status"], JobStatus.MATCHED)
self.assertEqual(res["Value"]["MinorStatus"], "MinorStatus-matched")
res = jobStateUpdate.setJobAttribute(jobID, "Status",
JobStatus.RUNNING)
self.assertTrue(res["OK"], res.get("Message"))
res = jobMonitor.getJobSummary(int(jobID))
self.assertTrue(res["OK"], res.get("Message"))
self.assertEqual(res["Value"]["Status"], JobStatus.RUNNING)
# delete the jobs - this will just set its status to "deleted"
wmsClient.deleteJob(jobIDs)
