def testProcessDataEvents(self):
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),
0)
trx = EventTransmitter(brokerhost, "PostISRAvailable")
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "available")
ds = self.testDatasetFromProperty()
ds.ids["ampid"] = 0
ps.set("dataset", serializePolicy(ds.toPolicy()))
for i in xrange(15):
ps.set("dataset", serializePolicy(ds.toPolicy()))
devent = StatusEvent("testing", originatorId, ps)
trx.publishEvent(devent)
ds = copy.deepcopy(ds)
ds.ids["ampid"] += 1
# Wait for events
time.sleep(2.0)
# pdb.set_trace()
self.joboffice.processDataEvents()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),
15)
self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(),
1)
job = self.joboffice.bb.queues.jobsPossible.get(0)
self.assertEquals(job.getName(), "Job-1")
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 1)
ps.set("dataset", serializePolicy(ds.toPolicy()))
devent = StatusEvent("testing", originatorId, ps)
trx.publishEvent(devent)
time.sleep(2.0)
self.joboffice.processDataEvents()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),
16)
self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(),
1)
job = self.joboffice.bb.queues.jobsPossible.get(0)
self.assertEquals(job.getName(), "Job-1")
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 0)
def testProcessJobDoneEvents(self):
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),
0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0)
self.testAllocateJobs()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),
1)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0)
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "job:done")
ps.set("success", True)
pevent = StatusEvent("testing", originatorId, ps)
trx = EventTransmitter(brokerhost, "CcdAssemblyJob")
trx.publishEvent(pevent)
time.sleep(2.0)
self.joboffice.processDoneJobs()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),
0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 1)
def testProcessJobDoneEvent(self):
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),
0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0)
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "job:done")
ps.set("success", True)
pevent = StatusEvent("testing", originatorId, ps)
self.assert_(not self.joboffice.processJobDoneEvent(pevent))
self.testAllocateJobs()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),
1)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0)
# pdb.set_trace()
self.assert_(self.joboffice.processJobDoneEvent(pevent))
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),
0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 1)
def testProcessDataEvent(self):
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),
0)
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "available")
ds = self.testDatasetFromProperty()
ps.add("dataset", serializePolicy(ds.toPolicy()))
for i in xrange(1, 4):
ds = copy.deepcopy(ds)
ds.ids["ampid"] += 1
ps.add("dataset", serializePolicy(ds.toPolicy()))
devent = StatusEvent("testing", originatorId, ps)
# pdb.set_trace()
self.joboffice.processDataEvent(devent)
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),
4)
self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(),
1)
job = self.joboffice.bb.queues.jobsPossible.get(0)
self.assertEquals(job.getName(), "Job-1")
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 12)
def testToPipelineQueueItem(self):
pipelineName = "ccdassembly"
ps = PropertySet()
ps.set("pipelineName", pipelineName)
ps.set("STATUS", "done")
ps.set("RUNID", "testing")
pevent = StatusEvent("testing", originatorId, ps)
item = self.joboffice.toPipelineQueueItem(pevent)
self.assertEquals(item.getName(), pipelineName)
self.assertEquals(item.getProperty("status"), "done")
self.assertEquals(item.getRunId(), "testing")
def testReceiveReadyPipelines(self):
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(),
0)
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "job:ready")
pevent = StatusEvent("testing", originatorId, ps)
trx = EventTransmitter(brokerhost, "CcdAssemblyJob")
trx.publishEvent(pevent)
time.sleep(2.0)
self.joboffice.receiveReadyPipelines()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(),
1)
def makeJobStatusEvent(self, job, runId, status):
props = PropertySet()
props.set("identity", serializePolicy(job.getJobIdentity().toPolicy()))
props.set("STATUS", status)
props.set("name", job.getName())
return StatusEvent(runId, self.originatorId, props)
def makeJobOfficeStatusEvent(self, runId, status):
props = PropertySet()
props.set("STATUS", status)
return StatusEvent(runId, self.originatorId, props)
def testRunInThread(self):
self.assert_(not self.joboffice.isAlive())
self.joboffice.start()
self.assert_(self.joboffice.isAlive())
try:
with self.joboffice.bb.queues:
self.assertEquals(
self.joboffice.bb.queues.jobsInProgress.length(), 0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),
0)
self.assertEquals(
self.joboffice.bb.queues.jobsAvailable.length(), 0)
self.assertEquals(
self.joboffice.bb.queues.dataAvailable.length(), 0)
trxpipe = EventTransmitter(brokerhost, "CcdAssemblyJob")
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "job:ready")
pevent = StatusEvent("testing", originatorId, ps)
trxpipe.publishEvent(pevent)
time.sleep(2.0)
with self.joboffice.bb.queues:
self.assertEquals(
self.joboffice.bb.queues.pipelinesReady.length(), 1)
self.assertEquals(
self.joboffice.bb.queues.jobsInProgress.length(), 0)
trxdata = EventTransmitter(brokerhost, "PostISRAvailable")
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "available")
ds = self.testDatasetFromProperty()
ds.ids["ampid"] = 0
ps.set("dataset", serializePolicy(ds.toPolicy()))
# pdb.set_trace()
for i in xrange(16):
ps.set("dataset", serializePolicy(ds.toPolicy()))
devent = StatusEvent("testing", originatorId, ps)
trxdata.publishEvent(devent)
ds = copy.deepcopy(ds)
ds.ids["ampid"] += 1
time.sleep(2.0)
with self.joboffice.bb.queues:
self.assertEquals(
self.joboffice.bb.queues.jobsInProgress.length(), 1)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),
0)
self.assertEquals(
self.joboffice.bb.queues.jobsAvailable.length(), 0)
self.assertEquals(
self.joboffice.bb.queues.dataAvailable.length(), 16)
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "job:done")
ps.set("success", True)
jevent = StatusEvent("testing", originatorId, ps)
trxpipe.publishEvent(jevent)
time.sleep(2.0)
with self.joboffice.bb.queues:
self.assertEquals(
self.joboffice.bb.queues.jobsInProgress.length(), 0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),
1)
self.assertEquals(
self.joboffice.bb.queues.jobsAvailable.length(), 0)
self.assertEquals(
self.joboffice.bb.queues.dataAvailable.length(), 16)
finally:
self.joboffice.stop()
if self.joboffice.isAlive():
self.joboffice.join(10.0)