def testProcessDataEvents(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 0) trx = EventTransmitter(brokerhost, "PostISRAvailable") ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "available") ds = self.testDatasetFromProperty() ds.ids["ampid"] = 0 ps.set("dataset", serializePolicy(ds.toPolicy())) for i in xrange(15): ps.set("dataset", serializePolicy(ds.toPolicy())) devent = StatusEvent("testing", originatorId, ps) trx.publishEvent(devent) ds = copy.deepcopy(ds) ds.ids["ampid"] += 1 # Wait for events time.sleep(2.0) # pdb.set_trace() self.joboffice.processDataEvents() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 15) self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(), 1) job = self.joboffice.bb.queues.jobsPossible.get(0) self.assertEquals(job.getName(), "Job-1") self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 1) ps.set("dataset", serializePolicy(ds.toPolicy())) devent = StatusEvent("testing", originatorId, ps) trx.publishEvent(devent) time.sleep(2.0) self.joboffice.processDataEvents() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 16) self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(), 1) job = self.joboffice.bb.queues.jobsPossible.get(0) self.assertEquals(job.getName(), "Job-1") self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 0)
def testProcessJobDoneEvents(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0) self.testAllocateJobs() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 1) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:done") ps.set("success", True) pevent = StatusEvent("testing", originatorId, ps) trx = EventTransmitter(brokerhost, "CcdAssemblyJob") trx.publishEvent(pevent) time.sleep(2.0) self.joboffice.processDoneJobs() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 1)
def testProcessJobDoneEvent(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:done") ps.set("success", True) pevent = StatusEvent("testing", originatorId, ps) self.assert_(not self.joboffice.processJobDoneEvent(pevent)) self.testAllocateJobs() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 1) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0) # pdb.set_trace() self.assert_(self.joboffice.processJobDoneEvent(pevent)) with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 1)
def testProcessDataEvent(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "available") ds = self.testDatasetFromProperty() ps.add("dataset", serializePolicy(ds.toPolicy())) for i in xrange(1, 4): ds = copy.deepcopy(ds) ds.ids["ampid"] += 1 ps.add("dataset", serializePolicy(ds.toPolicy())) devent = StatusEvent("testing", originatorId, ps) # pdb.set_trace() self.joboffice.processDataEvent(devent) with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 4) self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(), 1) job = self.joboffice.bb.queues.jobsPossible.get(0) self.assertEquals(job.getName(), "Job-1") self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 12)
def testToPipelineQueueItem(self): pipelineName = "ccdassembly" ps = PropertySet() ps.set("pipelineName", pipelineName) ps.set("STATUS", "done") ps.set("RUNID", "testing") pevent = StatusEvent("testing", originatorId, ps) item = self.joboffice.toPipelineQueueItem(pevent) self.assertEquals(item.getName(), pipelineName) self.assertEquals(item.getProperty("status"), "done") self.assertEquals(item.getRunId(), "testing")
def testReceiveReadyPipelines(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(), 0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:ready") pevent = StatusEvent("testing", originatorId, ps) trx = EventTransmitter(brokerhost, "CcdAssemblyJob") trx.publishEvent(pevent) time.sleep(2.0) self.joboffice.receiveReadyPipelines() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(), 1)
def makeJobStatusEvent(self, job, runId, status): props = PropertySet() props.set("identity", serializePolicy(job.getJobIdentity().toPolicy())) props.set("STATUS", status) props.set("name", job.getName()) return StatusEvent(runId, self.originatorId, props)
def makeJobOfficeStatusEvent(self, runId, status): props = PropertySet() props.set("STATUS", status) return StatusEvent(runId, self.originatorId, props)
def testRunInThread(self): self.assert_(not self.joboffice.isAlive()) self.joboffice.start() self.assert_(self.joboffice.isAlive()) try: with self.joboffice.bb.queues: self.assertEquals( self.joboffice.bb.queues.jobsInProgress.length(), 0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0) self.assertEquals( self.joboffice.bb.queues.jobsAvailable.length(), 0) self.assertEquals( self.joboffice.bb.queues.dataAvailable.length(), 0) trxpipe = EventTransmitter(brokerhost, "CcdAssemblyJob") ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:ready") pevent = StatusEvent("testing", originatorId, ps) trxpipe.publishEvent(pevent) time.sleep(2.0) with self.joboffice.bb.queues: self.assertEquals( self.joboffice.bb.queues.pipelinesReady.length(), 1) self.assertEquals( self.joboffice.bb.queues.jobsInProgress.length(), 0) trxdata = EventTransmitter(brokerhost, "PostISRAvailable") ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "available") ds = self.testDatasetFromProperty() ds.ids["ampid"] = 0 ps.set("dataset", serializePolicy(ds.toPolicy())) # pdb.set_trace() for i in xrange(16): ps.set("dataset", serializePolicy(ds.toPolicy())) devent = StatusEvent("testing", originatorId, ps) trxdata.publishEvent(devent) ds = copy.deepcopy(ds) ds.ids["ampid"] += 1 time.sleep(2.0) with self.joboffice.bb.queues: self.assertEquals( self.joboffice.bb.queues.jobsInProgress.length(), 1) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0) self.assertEquals( self.joboffice.bb.queues.jobsAvailable.length(), 0) self.assertEquals( self.joboffice.bb.queues.dataAvailable.length(), 16) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:done") ps.set("success", True) jevent = StatusEvent("testing", originatorId, ps) trxpipe.publishEvent(jevent) time.sleep(2.0) with self.joboffice.bb.queues: self.assertEquals( self.joboffice.bb.queues.jobsInProgress.length(), 0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 1) self.assertEquals( self.joboffice.bb.queues.jobsAvailable.length(), 0) self.assertEquals( self.joboffice.bb.queues.dataAvailable.length(), 16) finally: self.joboffice.stop() if self.joboffice.isAlive(): self.joboffice.join(10.0)