def testProcessDataEvent(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "available") ds = self.testDatasetFromProperty() ps.add("dataset", serializePolicy(ds.toPolicy())) for i in xrange(1, 4): ds = copy.deepcopy(ds) ds.ids["ampid"] += 1 ps.add("dataset", serializePolicy(ds.toPolicy())) devent = StatusEvent("testing", originatorId, ps) # pdb.set_trace() self.joboffice.processDataEvent(devent) with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 4) self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(), 1) job = self.joboffice.bb.queues.jobsPossible.get(0) self.assertEquals(job.getName(), "Job-1") self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 12)
def testProcessDataEvent(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "available") ds = self.testDatasetFromProperty() ps.add("dataset", serializePolicy(ds.toPolicy())) for i in xrange(1,4): ds = copy.deepcopy(ds) ds.ids["ampid"] += 1 ps.add("dataset", serializePolicy(ds.toPolicy())) devent = StatusEvent("testing", originatorId, ps) # pdb.set_trace() self.joboffice.processDataEvent(devent) with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 4) self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(), 1) job = self.joboffice.bb.queues.jobsPossible.get(0) self.assertEquals(job.getName(), "Job-1") self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 12)
def makeit(prefs, context, saveWcs=False, plot=dict()): """This is the python wrapper for the original psfex that reads SExtractor outputs """ # Create an array of PSFs (one PSF for each extension) if prefs.getVerboseType() != prefs.QUIET: print("----- %d input catalogues:" % prefs.getNcat()) if saveWcs: # only needed for making plots wcssList = [] fields = psfexLib.vectorField() for cat in prefs.getCatalogs(): field = psfexLib.Field(cat) wcss = [] wcssList.append(wcss) with fits.open(cat) as pf: for hdu in pf: if hdu.name == "PRIMARY": pass elif hdu.name == "LDAC_IMHEAD": hdr = hdu.data[0][ 0] # the fits header from the original fits image md = PropertySet() for line in hdr: try: md.set(*splitFitsCard(line)) except AttributeError: continue if not md.exists("CRPIX1"): # no WCS; try WCSA for k in md.names(): if re.search(r"A$", k): md.set(k[:-1], md.getScalar(k)) wcs = afwGeom.makeSkyWcs(md) naxis1, naxis2 = md.getScalar("NAXIS1"), md.getScalar( "NAXIS2") elif hdu.name == "LDAC_OBJECTS": nobj = len(hdu.data) assert wcs, "LDAC_OBJECTS comes after LDAC_IMHEAD" field.addExt(wcs, naxis1, naxis2, nobj) if saveWcs: wcss.append((wcs, naxis1, naxis2)) wcs = None field.finalize() fields.append(field) sets = psfexLib.vectorSet() for set in load_samples(prefs, context, plot=plot): sets.append(set) psfexLib.makeit(fields, sets) ret = [[f.getPsfs() for f in fields], sets] if saveWcs: ret.append(wcssList) return ret
def createPropertySet(self): data = self.getDataSet() prop = PropertySet() for x in data: prop.set(x[0],x[1]) return prop
def makeit(prefs, context, saveWcs=False, plot=dict()): """This is the python wrapper for the original psfex that reads SExtractor outputs """ # Create an array of PSFs (one PSF for each extension) if prefs.getVerboseType() != prefs.QUIET: print("----- %d input catalogues:" % prefs.getNcat()) if saveWcs: # only needed for making plots wcssList = [] fields = psfexLib.vectorField() for cat in prefs.getCatalogs(): field = psfexLib.Field(cat) wcss = [] wcssList.append(wcss) with fits.open(cat) as pf: for hdu in pf: if hdu.name == "PRIMARY": pass elif hdu.name == "LDAC_IMHEAD": hdr = hdu.data[0][0] # the fits header from the original fits image md = PropertySet() for line in hdr: try: md.set(*splitFitsCard(line)) except AttributeError: continue if not md.exists("CRPIX1"): # no WCS; try WCSA for k in md.names(): if re.search(r"A$", k): md.set(k[:-1], md.getScalar(k)) wcs = afwGeom.makeSkyWcs(md) naxis1, naxis2 = md.getScalar("NAXIS1"), md.getScalar("NAXIS2") elif hdu.name == "LDAC_OBJECTS": nobj = len(hdu.data) assert wcs, "LDAC_OBJECTS comes after LDAC_IMHEAD" field.addExt(wcs, naxis1, naxis2, nobj) if saveWcs: wcss.append((wcs, naxis1, naxis2)) wcs = None field.finalize() fields.append(field) sets = psfexLib.vectorSet() for set in load_samples(prefs, context, plot=plot): sets.append(set) psfexLib.makeit(fields, sets) ret = [[f.getPsfs() for f in fields], sets] if saveWcs: ret.append(wcssList) return ret
def testEventRunID(self): runid = "testrunid" status = "my special status" root = PropertySet() MYNAME = "myname" root.set(MYNAME, MYNAME) root.set(events.Event.STATUS, status) event = events.Event(runid, root) # return the property set props = event.getPropertySet() self.assertGreater(props.get(events.Event.EVENTTIME), 0) self.assertEqual(props.get(events.Event.PUBTIME), 0) self.assertEqual(props.get(events.Event.RUNID), runid) self.assertEqual(props.get(events.Event.STATUS), status) self.assertEqual(props.get(events.Event.TOPIC), events.Event.UNINITIALIZED) self.assertEqual(props.get(events.Event.TYPE), events.EventTypes.EVENT) self.assertEqual(props.get(MYNAME), MYNAME) filterableNames = event.getFilterablePropertyNames() filterableNames.remove(events.Event.EVENTTIME) filterableNames.remove(events.Event.PUBTIME) filterableNames.remove(events.Event.RUNID) filterableNames.remove(events.Event.STATUS) filterableNames.remove(events.Event.TOPIC) filterableNames.remove(events.Event.TYPE) assert len(filterableNames) == 0 customNames = event.getCustomPropertyNames() self.assertEqual(len(customNames), 1) self.assertEqual(customNames[0], MYNAME) eventTime = event.getEventTime() self.assertEqual(event.getPubTime(), 0) self.assertEqual(event.getRunId(), runid) self.assertEqual(event.getType(), events.EventTypes.EVENT) self.assertEqual(event.getStatus(), status) # check to be sure we really update the time event.updateEventTime() self.assertLess(eventTime, event.getEventTime()) # set the time to zero, and make sure we get back the same time. event.setEventTime(0) self.assertEqual(event.getEventTime(), 0) # reset the time to eventTime, and make sure we get back the same time. event.setEventTime(eventTime) self.assertEqual(event.getEventTime(), eventTime)
def testEventRunID(self): runid = "testrunid" status = "my special status" root = PropertySet() MYNAME = "myname" root.set(MYNAME, MYNAME) root.set(events.Event.STATUS, status) event = events.Event(runid, root) # return the property set props = event.getPropertySet() self.assertGreater(props.get(events.Event.EVENTTIME), 0) self.assertEqual(props.get(events.Event.PUBTIME), 0) self.assertEqual(props.get(events.Event.RUNID), runid) self.assertEqual(props.get(events.Event.STATUS), status) self.assertEqual(props.get(events.Event.TOPIC), events.Event.UNINITIALIZED) self.assertEqual(props.get(events.Event.TYPE), events.EventTypes.EVENT) self.assertEqual(props.get(MYNAME), MYNAME) filterableNames = event.getFilterablePropertyNames() filterableNames.remove(events.Event.EVENTTIME) filterableNames.remove(events.Event.PUBTIME) filterableNames.remove(events.Event.RUNID) filterableNames.remove(events.Event.STATUS) filterableNames.remove(events.Event.TOPIC) filterableNames.remove(events.Event.TYPE) assert len(filterableNames) == 0 customNames = event.getCustomPropertyNames() self.assertEqual(len(customNames), 1) self.assertEqual(customNames[0], MYNAME) eventTime = event.getEventTime() self.assertEqual(event.getPubTime(),0) self.assertEqual(event.getRunId(), runid) self.assertEqual(event.getType(), events.EventTypes.EVENT) self.assertEqual(event.getStatus(), status) # check to be sure we really update the time event.updateEventTime() self.assertLess(eventTime, event.getEventTime()) # set the time to zero, and make sure we get back the same time. event.setEventTime(0) self.assertEqual(event.getEventTime(), 0) # reset the time to eventTime, and make sure we get back the same time. event.setEventTime(eventTime) self.assertEqual(event.getEventTime(), eventTime)
def testProcessDataEvents(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 0) trx = EventTransmitter(brokerhost, "PostISRAvailable") ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "available") ds = self.testDatasetFromProperty() ds.ids["ampid"] = 0 ps.set("dataset", serializePolicy(ds.toPolicy())) for i in xrange(15): ps.set("dataset", serializePolicy(ds.toPolicy())) devent = StatusEvent("testing", originatorId, ps) trx.publishEvent(devent) ds = copy.deepcopy(ds) ds.ids["ampid"] += 1 # Wait for events time.sleep(2.0) # pdb.set_trace() self.joboffice.processDataEvents() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 15) self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(), 1) job = self.joboffice.bb.queues.jobsPossible.get(0) self.assertEquals(job.getName(), "Job-1") self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 1) ps.set("dataset", serializePolicy(ds.toPolicy())) devent = StatusEvent("testing", originatorId, ps) trx.publishEvent(devent) time.sleep(2.0) self.joboffice.processDataEvents() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 16) self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(), 1) job = self.joboffice.bb.queues.jobsPossible.get(0) self.assertEquals(job.getName(), "Job-1") self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 0)
def testEventProperties(self): status = "my special status" root = PropertySet() MYNAME = "myname" root.set(MYNAME, MYNAME) root.set(events.Event.STATUS, status) event = events.Event(root) props = event.getPropertySet() custom = event.getCustomPropertySet() self.assertEqual(props.nameCount(), 6) self.assertEqual(custom.nameCount(), 1)
def process(self, clipboard): metadata = clipboard.get(self.policy.getString("inputKeys.metadata")) cdb = clipboard.get(self.policy.getString("inputKeys.calibrationDB")) pathPrefix = cliboard.get(self.policy.getString("inputKeys.prefixPath")) when = DateTime(metadata.get("dateObs")) ccdId = metadata.get("ccdId") ampId = metadata.get("ampId") expTime = metadata.get("expTime") darkToApply = self.policy.getString("whichdark") darkCalibList = cdb.lookup(when, "dark", ccdId, ampId, all=True) darkTimeList = [] for d in darkCalibList: darkTimeList.append(d.expTime) darkTimeList.sort() if darktoapply == "min": darkExpTime = darkTimeList[0] elif darktoapply == "max": darkExpTime = darkTimeList[-1] elif darktoapply == "closest": minDist = abs(expTime - darkTimeList[0]) minExpTime = darkTimeList[0] for i in xrange(1, len(darkTimeList)): dist = abs(expTime - darkTimeList[i]) if dist < minDist: minDist = dist minExpTime = darkTimeList[i] darkExpTime = minExpTime else: raise RuntimeError, "Unrecognized method for finding dark to apply: " + str(darktoapply) biasPath = cdb.lookup(when, "bias", ccdId, ampId) darkPath = cdb.lookup(when, "dark", ccdId, ampId, expTime=darkExpTime) defectPath = cdb.lookup(when, "defect", ccdId, ampId) flatPath = cdb.lookup(when, "flat", ccdId, ampId, filter=metadata.get("filter")) # fringePath = cdb.lookup(when, "fringe", ccdId, ampId, # filter=metadata.get("filter")) linearizePath = cdb.lookup(when, "linearize", ccdId, ampId) calibData = PropertySet() calibData.set("biasPath", os.path.join(pathPrefix, biasPath)) calibData.set("darkPath", os.path.join(pathPrefix, darkPath)) calibData.set("defectPath", os.path.join(pathPrefix, defectPath)) calibData.set("flatPath", os.path.join(pathPrefix, flatPath)) # calibData.set("fringePath", os.path.join(pathPrefix, fringePath)) calibData.set("linearizePath", os.path.join(pathPrefix, linearizePath)) outputKey = self._policy.get("outputKey") self.activeClipboard.put(outputKey, calibData) self.outputQueue.addDataset(self.activeClipboard)
def sendEvent(self, brokerName, topic): """Send an Event.""" trans = events.EventTransmitter(brokerName, topic) root = PropertySet() root.set(events.Event.TOPIC, topic) root.set("myname","myname") root.set(events.StatusEvent.STATUS, "my special status") root.set(events.StatusEvent.RUNID,"srptestrun") root.set("bazinga", "sheldon") originatorId = events.LocationId() event = events.StatusEvent("srptestrun", originatorId, root) # ok...now publish it trans.publishEvent(event)
def sendEvent(self, brokerName, topic): """Send an Event.""" trans = events.EventTransmitter(brokerName, topic) root = PropertySet() root.set(events.Event.TOPIC, topic) root.set("myname", "myname") root.set(events.StatusEvent.STATUS, "my special status") root.set(events.StatusEvent.RUNID, "srptestrun") root.set("bazinga", "sheldon") originatorId = events.LocationId() event = events.StatusEvent("srptestrun", originatorId, root) # ok...now publish it trans.publishEvent(event)
def testProcessDataEvents(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 0) trx = EventTransmitter(brokerhost, "PostISRAvailable") ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "available") ds = self.testDatasetFromProperty() ds.ids["ampid"] = 0; ps.set("dataset", serializePolicy(ds.toPolicy())) for i in xrange(15): ps.set("dataset", serializePolicy(ds.toPolicy())) devent = StatusEvent("testing", originatorId, ps) trx.publishEvent(devent); ds = copy.deepcopy(ds) ds.ids["ampid"] += 1 # Wait for events time.sleep(2.0) # pdb.set_trace() self.joboffice.processDataEvents() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),15) self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(), 1) job = self.joboffice.bb.queues.jobsPossible.get(0) self.assertEquals(job.getName(), "Job-1") self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 1) ps.set("dataset", serializePolicy(ds.toPolicy())) devent = StatusEvent("testing", originatorId, ps) trx.publishEvent(devent); time.sleep(2.0) self.joboffice.processDataEvents() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),16) self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(), 1) job = self.joboffice.bb.queues.jobsPossible.get(0) self.assertEquals(job.getName(), "Job-1") self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 0)
def testReceiveReadyPipelines(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(),0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:ready") pevent = StatusEvent("testing", originatorId, ps) trx = EventTransmitter(brokerhost, "CcdAssemblyJob") trx.publishEvent(pevent) time.sleep(2.0) self.joboffice.receiveReadyPipelines() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(),1)
def sendFilterableStatusEvent(self, broker, topic): trans = events.EventTransmitter(broker, topic) root = PropertySet() root.set(events.Event.TOPIC, topic) root.set("myname","myname") root.set(events.Event.STATUS, "my special status") filter = PropertySet() filter.set("FOO", "bar") filter.set("PLOUGH", 123) eventSystem = events.EventSystem.getDefaultEventSystem(); locationID = eventSystem.createOriginatorId() event = events.StatusEvent(locationID, root, filter) # ok...now publish it trans.publishEvent(event)
def sendPlainStatusEvent(self, broker, topic, runID=None): trans = events.EventTransmitter(broker, topic) root = PropertySet() root.set(events.Event.TOPIC, topic) root.set("myname", "myname") root.set(events.Event.STATUS, "my special status") root.set("logger.status", "my logger special status") root.set("logger.pid", "1") eventSystem = events.EventSystem.getDefaultEventSystem() locationID = eventSystem.createOriginatorId() if runID is None: event = events.StatusEvent(locationID, root) else: event = events.StatusEvent(runID, locationID, root) # ok...now publish it trans.publishEvent(event)
def testReceiveReadyPipelines(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(), 0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:ready") pevent = StatusEvent("testing", originatorId, ps) trx = EventTransmitter(brokerhost, "CcdAssemblyJob") trx.publishEvent(pevent) time.sleep(2.0) self.joboffice.receiveReadyPipelines() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(), 1)
def sendPlainStatusEvent(self, broker, topic, runID=None): trans = events.EventTransmitter(broker, topic) root = PropertySet() root.set(events.Event.TOPIC, topic) root.set("myname","myname") root.set(events.Event.STATUS, "my special status") root.set("logger.status", "my logger special status") root.set("logger.pid", "1") eventSystem = events.EventSystem.getDefaultEventSystem(); locationID = eventSystem.createOriginatorId() if runID is None: event = events.StatusEvent(locationID, root) else: event = events.StatusEvent(runID, locationID, root) # ok...now publish it trans.publishEvent(event)
class _EventFactory(object): def __init__(self, runid, props=None): """ create a generic event factor """ self.runid = runid if isinstance(props, PropertySet): self.props = props else: self.props = PropertySet() if isinstance(props, dict): for key in props.keys(): self.props.set(key, props[key]) def create(self): """create a new instance of the event""" return ev.Event(self.runid, self.props) def setRunId(self, id): """set the Run ID""" self.runid = runid def getRunId(self): """set the Run ID""" return self.runid def setProperty(self, name, val): """set the value of a named property""" self.props.set(name, val) def getProperty(self, name): """get the value of a named property""" return self.props.getString(name) def addDataset(self, propname, ds): """add a dataset to the event""" self.props.add(propname, serializeDataset(ds)) def getDatasets(self, propname): """return the datasets attached to the event""" return unserializeDatasetList(self.props.getArrayString(propname))
def testSubst(self): ad = PropertySet() ad.set("foo", "bar") ad.setInt("x", 3) LogicalLocation.setLocationMap(ad) l = LogicalLocation("%(foo)xx") self.assertEqual(l.locString(), "barxx") l = LogicalLocation("%(x)foo") self.assertEqual(l.locString(), "3foo") l = LogicalLocation("yy%04d(x)yy") self.assertEqual(l.locString(), "yy0003yy") ad2 = PropertySet() ad2.set("foo", "baz") ad2.setInt("y", 2009) l = LogicalLocation("%(foo)%(x)%(y)", ad2) self.assertEqual(l.locString(), "bar32009") LogicalLocation.setLocationMap(PropertySet()) l = LogicalLocation("%(foo)%3d(y)", ad2) self.assertEqual(l.locString(), "baz2009")
def testSubst(self): ad = PropertySet() ad.set("foo", "bar") ad.setInt("x", 3) LogicalLocation.setLocationMap(ad) loc = LogicalLocation("%(foo)xx") self.assertEqual(loc.locString(), "barxx") loc = LogicalLocation("%(x)foo") self.assertEqual(loc.locString(), "3foo") loc = LogicalLocation("yy%04d(x)yy") self.assertEqual(loc.locString(), "yy0003yy") ad2 = PropertySet() ad2.set("foo", "baz") ad2.setInt("y", 2009) loc = LogicalLocation("%(foo)%(x)%(y)", ad2) self.assertEqual(loc.locString(), "bar32009") LogicalLocation.setLocationMap(PropertySet()) loc = LogicalLocation("%(foo)%3d(y)", ad2) self.assertEqual(loc.locString(), "baz2009")
def createWcs(): metadata = PropertySet() metadata.set("SIMPLE", "T") metadata.set("BITPIX", -32) metadata.set("NAXIS", 2) metadata.set("NAXIS1", 1024) metadata.set("NAXIS2", 1153) metadata.set("RADESYS", 'FK5') metadata.set("EQUINOX", 2000.) metadata.setDouble("CRVAL1", 215.604025685476) metadata.setDouble("CRVAL2", 53.1595451514076) metadata.setDouble("CRPIX1", 1109.99981456774) metadata.setDouble("CRPIX2", 560.018167811613) metadata.set("CTYPE1", 'RA---SIN') metadata.set("CTYPE2", 'DEC--SIN') metadata.setDouble("CD1_1", 5.10808596133527E-05) metadata.setDouble("CD1_2", 1.85579539217196E-07) metadata.setDouble("CD2_2", -5.10281493481982E-05) metadata.setDouble("CD2_1", -8.27440751733828E-07) return makeSkyWcs(metadata)
def testProcessJobDoneEvents(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0) self.testAllocateJobs() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 1) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:done") ps.set("success", True) pevent = StatusEvent("testing", originatorId, ps) trx = EventTransmitter(brokerhost, "CcdAssemblyJob") trx.publishEvent(pevent) time.sleep(2.0) self.joboffice.processDoneJobs() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 1)
def testProcessJobDoneEvent(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:done") ps.set("success", True) pevent = StatusEvent("testing", originatorId, ps) self.assert_(not self.joboffice.processJobDoneEvent(pevent)) self.testAllocateJobs() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 1) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0) # pdb.set_trace() self.assert_(self.joboffice.processJobDoneEvent(pevent)) with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 1)
def sendEvent(self, broker, topic): trans = events.EventTransmitter(broker, topic) root = PropertySet() root.set(events.Event.TOPIC,topic) root.set("myname","myname") root.set(events.Event.STATUS, "my special status") root.set(events.Event.RUNID,"srptestrun") event = events.Event("srptestrun", root) # wait a short time so we can see the difference between the time # the event is created and the time it is published time.sleep(2) # ok...now publish it trans.publishEvent(event)
def sendEvent(self, broker, topic): trans = events.EventTransmitter(broker, topic) root = PropertySet() root.set(events.Event.TOPIC, topic) root.set("myname", "myname") root.set(events.Event.STATUS, "my special status") root.set(events.Event.RUNID, "srptestrun") event = events.Event("srptestrun", root) # wait a short time so we can see the difference between the time # the event is created and the time it is published time.sleep(2) # ok...now publish it trans.publishEvent(event)
def testToPipelineQueueItem(self): pipelineName = "ccdassembly" ps = PropertySet() ps.set("pipelineName", pipelineName) ps.set("STATUS", "done") ps.set("RUNID", "testing") pevent = StatusEvent("testing", originatorId, ps) item = self.joboffice.toPipelineQueueItem(pevent) self.assertEquals(item.getName(), pipelineName) self.assertEquals(item.getProperty("status"), "done") self.assertEquals(item.getRunId(), "testing")
def sendEvent(self, runid, brokerName, topic): trans = events.EventTransmitter(brokerName, topic) root = PropertySet() root.set(events.Event.TOPIC, topic) root.set("myname","myname") root.set(events.Event.STATUS, "my special status") locationID = events.LocationId() event = events.StatusEvent(runid, locationID, root) # ok...now publish it trans.publishEvent(event)
def makeJobCommandEvent(self, job, pipeline, runId=""): """ create a CommandEvent to send to a pipeline instructing it to commence working on the given job. """ props = PropertySet() for ds in job.getInputDatasets(): props.add("inputs", serializePolicy(ds.toPolicy())) for ds in job.getOutputDatasets(): props.add("outputs", serializePolicy(ds.toPolicy())) props.set("identity", serializePolicy(job.getJobIdentity().toPolicy())) props.set("STATUS", "job:assign") props.set("name", job.getName()) return CommandEvent(runId, self.originatorId, pipeline, props)
def testProcessJobDoneEvents(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),0) self.testAllocateJobs() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),1) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:done") ps.set("success", True) pevent = StatusEvent("testing", originatorId, ps) trx = EventTransmitter(brokerhost, "CcdAssemblyJob") trx.publishEvent(pevent) time.sleep(2.0) self.joboffice.processDoneJobs() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),1)
def testProcessJobDoneEvent(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:done") ps.set("success", True) pevent = StatusEvent("testing", originatorId, ps) self.assert_(not self.joboffice.processJobDoneEvent(pevent)) self.testAllocateJobs() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),1) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),0) # pdb.set_trace() self.assert_(self.joboffice.processJobDoneEvent(pevent)) with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),1)
def preprocess(self): self.activeClipboard = self.inputQueue.getNextDataset() eventName = self._policy.get("inputEvent") event = self.activeClipboard.get(eventName) visitId = event.get("visitId") exposureId = event.get("exposureId") fpaExposureId = (long(visitId) << 1) + exposureId visit = PropertySet() visit.setInt("visitId", visitId) visit.setLongLong("exposureId", fpaExposureId) self.activeClipboard.put("visit" + str(exposureId), visit) rawFpaExposure = PropertySet() rawFpaExposure.setLongLong("rawFPAExposureId", fpaExposureId) rawFpaExposure.set("ra", event.get("ra")) rawFpaExposure.set("decl", event.get("decl")) rawFpaExposure.set("filterId", self.lookupFilterId(event.get("filter"))) rawFpaExposure.set("equinox", event.get("equinox")) rawFpaExposure.set("dateObs", DateTime(event.get("dateObs"))) rawFpaExposure.set("mjdObs", DateTime(event.get("dateObs")).mjd()) rawFpaExposure.set("expTime", event.get("expTime")) rawFpaExposure.set("airmass", event.get("airmass")) self.activeClipboard.put("fpaExposure" + str(exposureId), rawFpaExposure)
def testIllegalFilterableStatusEvent(self): testEnv = TestEnvironment() broker = testEnv.getBroker() topic = self.createTopicName("test_events_10_%s.A") # # send a test event, and wait to receive it # trans = events.EventTransmitter(broker, topic) root = PropertySet() root.set(events.StatusEvent.TOPIC, topic) root.set("myname","myname") root.set(events.StatusEvent.STATUS, "my special status") filter = PropertySet() filter.set("FOO", "bar") filter.set("PLOUGH", 123) filter.set("PLOVER.FIRST", 1) filter.set("PLOVER.SECOND", 2) eventSystem = events.EventSystem.getDefaultEventSystem(); locationID = eventSystem.createOriginatorId() event = events.StatusEvent(locationID, root, filter) self.assertRaises(ex.Exception, trans.publishEvent, event)
def testRunInThread(self): self.assert_(not self.joboffice.isAlive()) self.joboffice.start() self.assert_(self.joboffice.isAlive()) try: with self.joboffice.bb.queues: self.assertEquals( self.joboffice.bb.queues.jobsInProgress.length(), 0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0) self.assertEquals( self.joboffice.bb.queues.jobsAvailable.length(), 0) self.assertEquals( self.joboffice.bb.queues.dataAvailable.length(), 0) trxpipe = EventTransmitter(brokerhost, "CcdAssemblyJob") ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:ready") pevent = StatusEvent("testing", originatorId, ps) trxpipe.publishEvent(pevent) time.sleep(2.0) with self.joboffice.bb.queues: self.assertEquals( self.joboffice.bb.queues.pipelinesReady.length(), 1) self.assertEquals( self.joboffice.bb.queues.jobsInProgress.length(), 0) trxdata = EventTransmitter(brokerhost, "PostISRAvailable") ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "available") ds = self.testDatasetFromProperty() ds.ids["ampid"] = 0 ps.set("dataset", serializePolicy(ds.toPolicy())) # pdb.set_trace() for i in xrange(16): ps.set("dataset", serializePolicy(ds.toPolicy())) devent = StatusEvent("testing", originatorId, ps) trxdata.publishEvent(devent) ds = copy.deepcopy(ds) ds.ids["ampid"] += 1 time.sleep(2.0) with self.joboffice.bb.queues: self.assertEquals( self.joboffice.bb.queues.jobsInProgress.length(), 1) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0) self.assertEquals( self.joboffice.bb.queues.jobsAvailable.length(), 0) self.assertEquals( self.joboffice.bb.queues.dataAvailable.length(), 16) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:done") ps.set("success", True) jevent = StatusEvent("testing", originatorId, ps) trxpipe.publishEvent(jevent) time.sleep(2.0) with self.joboffice.bb.queues: self.assertEquals( self.joboffice.bb.queues.jobsInProgress.length(), 0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 1) self.assertEquals( self.joboffice.bb.queues.jobsAvailable.length(), 0) self.assertEquals( self.joboffice.bb.queues.dataAvailable.length(), 16) finally: self.joboffice.stop() if self.joboffice.isAlive(): self.joboffice.join(10.0)
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the LSST License Statement and # the GNU General Public License along with this program. If not, # see <http://www.lsstcorp.org/LegalNotices/>. # import lsst.ctrl.events as events from lsst.daf.base import PropertySet import sys # usage: python bin/shutlog.py runid if __name__ == "__main__": host = "lsst8.ncsa.uiuc.edu" topic = events.LogEvent.LOGGING_TOPIC runid = sys.argv[1] eventSystem = events.EventSystem.getDefaultEventSystem() eventSystem.createTransmitter(host, topic) props = PropertySet() props.set("LOGGER","orca.control") props.set("STATUS","eol") e = events.Event(runid, props) eventSystem.publishEvent(topic, e)
def testCommandEvent(self): testEnv = EventsEnvironment() broker = testEnv.getBroker() thisHost = platform.node() topic = "test_events_command_%s_%d" % (thisHost, os.getpid()) # send a command event receiver = events.EventReceiver(broker, topic) trans = events.EventTransmitter(broker, topic) originatorId = events.LocationId() root = PropertySet() root.set(events.StatusEvent.TOPIC, topic) root.set("myname", "myname") root.set(events.StatusEvent.STATUS, "my special status") event = events.StatusEvent("srptestrun", originatorId, root) statusOriginatorId = event.getOriginator() destinationID = events.LocationId(statusOriginatorId) commandOriginatorID = events.LocationId() root2 = PropertySet() root2.set(events.CommandEvent.TOPIC, topic) root2.set("myname", "myname2") root2.set(events.CommandEvent.STATUS, "my special status2") event2 = events.CommandEvent("srptestrun", commandOriginatorID, destinationID, root2) trans.publishEvent(event2) val = receiver.receiveCommandEvent() # be sure we received an event self.assertIsNotNone(val) # these are the filterable names we expect to see names = [ events.CommandEvent.DEST_HOSTNAME, events.CommandEvent.DEST_LOCALID, events.CommandEvent.DEST_PROCESSID, events.CommandEvent.EVENTTIME, events.CommandEvent.ORIG_HOSTNAME, events.CommandEvent.ORIG_LOCALID, events.CommandEvent.ORIG_PROCESSID, events.CommandEvent.PUBTIME, events.CommandEvent.RUNID, events.CommandEvent.STATUS, events.CommandEvent.TOPIC, events.CommandEvent.TYPE ] self.assertValid(val, names, commandOriginatorID.getLocalID(), destinationID.getLocalID()) # send a command event with additional filterable properties receiver = events.EventReceiver(broker, topic) trans = events.EventTransmitter(broker, topic) originatorId = events.LocationId() event = events.StatusEvent("srptestrun", originatorId, root) statusOriginatorId = event.getOriginator() destinationID = events.LocationId(statusOriginatorId) commandOriginatorID = events.LocationId() filterable = PropertySet() filterable.set("FOO", 12.3) filterable.set("BAR", .45) event2 = events.CommandEvent("srptestrun", commandOriginatorID, destinationID, root2, filterable) trans.publishEvent(event2) val = receiver.receiveCommandEvent() # be sure we received an event self.assertIsNotNone(val) # these are the filterable names we expect to see names = [ events.CommandEvent.DEST_HOSTNAME, events.CommandEvent.DEST_LOCALID, events.CommandEvent.DEST_PROCESSID, events.CommandEvent.EVENTTIME, events.CommandEvent.ORIG_HOSTNAME, events.CommandEvent.ORIG_LOCALID, events.CommandEvent.ORIG_PROCESSID, events.CommandEvent.PUBTIME, events.CommandEvent.RUNID, events.CommandEvent.STATUS, events.CommandEvent.TOPIC, events.CommandEvent.TYPE, 'FOO', 'BAR' ] self.assertValid(val, names, commandOriginatorID.getLocalID(), destinationID.getLocalID()) self.assertTrue(event.getType(), events.EventTypes.COMMAND)
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the LSST License Statement and # the GNU General Public License along with this program. If not, # see <http://www.lsstcorp.org/LegalNotices/>. # # used to transmit status that we're not sending any more data after this import lsst.ctrl.events as events from lsst.daf.base import PropertySet import sys # usage: python dataCompleted broker topic runid if __name__ == "__main__": broker = sys.argv[1] topic = sys.argv[2] runid = sys.argv[3] status = sys.argv[4] trans = events.EventTransmitter(broker, topic) eventSystem = events.EventSystem.getDefaultEventSystem() root = PropertySet() root.set("STATUS",status) id = eventSystem.createOriginatorId() event = events.StatusEvent(runid, id, root) trans.publishEvent(event)
def makeJobStatusEvent(self, job, runId, status): props = PropertySet() props.set("identity", serializePolicy(job.getJobIdentity().toPolicy())) props.set("STATUS", status) props.set("name", job.getName()) return StatusEvent(runId, self.originatorId, props)
def testLocationId(self): testEnv = EventsEnvironment() broker = testEnv.getBroker() thisHost = platform.node() locationID = events.LocationId() # check to see if localID is 0 localId = locationID.getLocalID() self.assertEqual(localId, 0) # check host name hostname = locationID.getHostName() self.assertEqual(hostname, socket.gethostname()) # check process id processId = locationID.getProcessID() self.assertEqual(processId, os.getpid()) locationID2 = events.LocationId() # check to see if localID is 1 localId = locationID2.getLocalID() self.assertEqual(localId, 1) self.assertEqual(locationID.getLocalID(), 0) # check host name hostname = locationID.getHostName() self.assertEqual(hostname, socket.gethostname()) # check process id processId = locationID2.getProcessID() self.assertEqual(processId, os.getpid()) root = PropertySet() root.set("myname","myname") status = "my special status" root.set(events.Event.STATUS, status) statusEvent = events.StatusEvent("my runid", locationID, root) statusEvent2 = events.StatusEvent("my runid", locationID2, root) topic = "mytopic_%s_%d" % (thisHost, os.getpid()) transmitter = events.EventTransmitter(broker, topic) hostname = locationID2.getHostName() # create a receiver selector with the hostname, process id and local id sel = "%s = '%s' and %s = %d and %s = %d" % (events.StatusEvent.ORIG_HOSTNAME, hostname, events.StatusEvent.ORIG_PROCESSID, os.getpid(), events.StatusEvent.ORIG_LOCALID, 1) receiver = events.EventReceiver(broker, topic, sel) # transmit the events transmitter.publishEvent(statusEvent) transmitter.publishEvent(statusEvent2) # should receive event with with the process id returnedEvent = receiver.receiveEvent() ps = returnedEvent.getPropertySet() self.assertEqual(ps.get('ORIG_HOSTNAME'), hostname) self.assertEqual(ps.get('ORIG_PROCESSID'), os.getpid()) self.assertEqual(ps.get('ORIG_LOCALID'), 1) # should NOT receive another event, because it was filtered out by # the broker returnedEvent2 = receiver.receiveEvent(1) self.assertIsNone(returnedEvent2)
def makeJobOfficeStatusEvent(self, runId, status): props = PropertySet() props.set("STATUS", status) return StatusEvent(runId, self.originatorId, props)
def setUp(self): # make a nominal match list where the distances are 0; test can then modify # source centroid, reference coord or distance field for each match, as desired ctrPix = afwGeom.Point2I(1500, 1500) metadata = PropertySet() metadata.set("RADECSYS", "FK5") metadata.set("EQUINOX", 2000.0) metadata.set("CTYPE1", "RA---TAN") metadata.set("CTYPE2", "DEC--TAN") metadata.set("CUNIT1", "deg") metadata.set("CUNIT2", "deg") metadata.set("CRVAL1", 215.5) metadata.set("CRVAL2", 53.0) metadata.set("CRPIX1", ctrPix[0] + 1) metadata.set("CRPIX2", ctrPix[1] + 1) metadata.set("CD1_1", 5.1e-05) metadata.set("CD1_2", 0.0) metadata.set("CD2_2", -5.1e-05) metadata.set("CD2_1", 0.0) self.wcs = afwImage.makeWcs(metadata) self.bboxD = afwGeom.Box2D(afwGeom.Point2D(10, 100), afwGeom.Extent2D(1000, 1500)) self.numMatches = 25 sourceSchema = afwTable.SourceTable.makeMinimalSchema() # add centroid (and many other unwanted fields) to sourceSchema SingleFrameMeasurementTask(schema=sourceSchema) self.sourceCentroidKey = afwTable.Point2DKey( sourceSchema["slot_Centroid"]) self.sourceCat = afwTable.SourceCatalog(sourceSchema) refSchema = afwTable.SourceTable.makeMinimalSchema() self.refCoordKey = afwTable.CoordKey(refSchema["coord"]) self.refCat = afwTable.SourceCatalog(refSchema) self.matchList = [] np.random.seed(5) pixPointList = [ afwGeom.Point2D(pos) for pos in np.random.random_sample([self.numMatches, 2]) * self.bboxD.getDimensions() + self.bboxD.getMin() ] for pixPoint in pixPointList: src = self.sourceCat.addNew() src.set(self.sourceCentroidKey, pixPoint) ref = self.refCat.addNew() ref.set(self.refCoordKey, self.wcs.pixelToSky(pixPoint)) match = afwTable.ReferenceMatch(ref, src, 0) self.matchList.append(match)
def getMetadata(keyValDict): md = PropertySet() for key, val in keyValDict.items(): md.set(key, val) return md
def testLocationId(self): testEnv = EventsEnvironment() broker = testEnv.getBroker() thisHost = platform.node() locationID = events.LocationId() # check to see if localID is 0 localId = locationID.getLocalID() self.assertEqual(localId, 0) # check host name hostname = locationID.getHostName() self.assertEqual(hostname, socket.gethostname()) # check process id processId = locationID.getProcessID() self.assertEqual(processId, os.getpid()) locationID2 = events.LocationId() # check to see if localID is 1 localId = locationID2.getLocalID() self.assertEqual(localId, 1) self.assertEqual(locationID.getLocalID(), 0) # check host name hostname = locationID.getHostName() self.assertEqual(hostname, socket.gethostname()) # check process id processId = locationID2.getProcessID() self.assertEqual(processId, os.getpid()) root = PropertySet() root.set("myname", "myname") status = "my special status" root.set(events.Event.STATUS, status) statusEvent = events.StatusEvent("my runid", locationID, root) statusEvent2 = events.StatusEvent("my runid", locationID2, root) topic = "mytopic_%s_%d" % (thisHost, os.getpid()) transmitter = events.EventTransmitter(broker, topic) hostname = locationID2.getHostName() # create a receiver selector with the hostname, process id and local id sel = "%s = '%s' and %s = %d and %s = %d" % ( events.StatusEvent.ORIG_HOSTNAME, hostname, events.StatusEvent.ORIG_PROCESSID, os.getpid(), events.StatusEvent.ORIG_LOCALID, 1) receiver = events.EventReceiver(broker, topic, sel) # transmit the events transmitter.publishEvent(statusEvent) transmitter.publishEvent(statusEvent2) # should receive event with with the process id returnedEvent = receiver.receiveEvent() ps = returnedEvent.getPropertySet() self.assertEqual(ps.get('ORIG_HOSTNAME'), hostname) self.assertEqual(ps.get('ORIG_PROCESSID'), os.getpid()) self.assertEqual(ps.get('ORIG_LOCALID'), 1) # should NOT receive another event, because it was filtered out by # the broker returnedEvent2 = receiver.receiveEvent(1) self.assertIsNone(returnedEvent2)
def setUp(self): # Set up local astrometry_net_data self.datapath = setupAstrometryNetDataDir('photocal') self.config = LoadAstrometryNetObjectsTask.ConfigClass() self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(3001, 3001)) self.ctrPix = afwGeom.Point2I(1500, 1500) metadata = PropertySet() metadata.set("RADECSYS", "FK5") metadata.set("EQUINOX", 2000.0) metadata.set("CTYPE1", "RA---TAN") metadata.set("CTYPE2", "DEC--TAN") metadata.set("CUNIT1", "deg") metadata.set("CUNIT2", "deg") metadata.set("CRVAL1", 215.5) metadata.set("CRVAL2", 53.0) metadata.set("CRPIX1", self.ctrPix[0] + 1) metadata.set("CRPIX2", self.ctrPix[1] + 1) metadata.set("CD1_1", 5.1e-05) metadata.set("CD1_2", 0.0) metadata.set("CD2_2", -5.1e-05) metadata.set("CD2_1", 0.0) self.wcs = makeWcs(metadata) self.desNumStarsInPixelBox = 270 self.desNumStarsInSkyCircle = 410
def testRunInThread(self): self.assert_(not self.joboffice.isAlive()) self.joboffice.start() self.assert_(self.joboffice.isAlive()) try: with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),0) self.assertEquals(self.joboffice.bb.queues.jobsAvailable.length(),0) self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),0) trxpipe = EventTransmitter(brokerhost, "CcdAssemblyJob") ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:ready") pevent = StatusEvent("testing", originatorId, ps) trxpipe.publishEvent(pevent) time.sleep(2.0) with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(),1) self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0) trxdata = EventTransmitter(brokerhost, "PostISRAvailable") ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "available") ds = self.testDatasetFromProperty() ds.ids["ampid"] = 0; ps.set("dataset", serializePolicy(ds.toPolicy())) # pdb.set_trace() for i in xrange(16): ps.set("dataset", serializePolicy(ds.toPolicy())) devent = StatusEvent("testing", originatorId, ps) trxdata.publishEvent(devent); ds = copy.deepcopy(ds) ds.ids["ampid"] += 1 time.sleep(2.0) with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),1) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),0) self.assertEquals(self.joboffice.bb.queues.jobsAvailable.length(),0) self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),16) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:done") ps.set("success", True) jevent = StatusEvent("testing", originatorId, ps) trxpipe.publishEvent(jevent) time.sleep(2.0) with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),1) self.assertEquals(self.joboffice.bb.queues.jobsAvailable.length(),0) self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),16) finally: self.joboffice.stop() if self.joboffice.isAlive(): self.joboffice.join(10.0)
def testSendEvent(self): """Send an Event""" testEnv = EventsEnvironment() self.thisHost = platform.node() self.broker = testEnv.getBroker() topic = "test_events_3_%s_%d" % (self.thisHost, os.getpid()) runID = "test3_runid" recv = events.EventReceiver(self.broker, topic) trans = events.EventTransmitter(self.broker, topic) DATE = "date" DATE_VAL = "2007-07-01T14:28:32.546012" BLANK = "blank" BLANK_VAL = "" PID = "pid" PID_VAL = os.getpid() HOST = "host" HOST_VAL = "lsstcorp.org" IP = "ip" IP_VAL = "1.2.3.4" EVNT = "evnt" EVNT_VAL = "test" MISC1 = "misc1" MISC1_VAL = "data 1" MISC2 = "misc2" MISC2_VAL = "data 2" MISC3 = "misc3" MISC3_VAL = "" DATA = "data" DATA_VAL = 3.14 root = PropertySet() root.set(DATE, DATE_VAL) root.set(BLANK, BLANK_VAL) root.setInt(PID, PID_VAL) root.set(HOST, HOST_VAL) root.set(IP, IP_VAL) root.set(EVNT, EVNT_VAL) root.set(MISC1, MISC1_VAL) root.set(MISC2, MISC2_VAL) root.set(MISC3, MISC3_VAL) root.setDouble(DATA, DATA_VAL) event = events.Event(runID, root) trans.publishEvent(event) val = recv.receiveEvent() self.assertIsNotNone(val) # check the validity of all sent values ps = val.getPropertySet() self.assertEqual(ps.get(DATE), DATE_VAL) self.assertEqual(ps.get(BLANK), BLANK_VAL) self.assertEqual(ps.get(PID), PID_VAL) self.assertEqual(ps.get(HOST), HOST_VAL) self.assertEqual(ps.get(IP), IP_VAL) self.assertEqual(ps.get(EVNT), EVNT_VAL) self.assertEqual(ps.get(MISC1), MISC1_VAL) self.assertEqual(ps.get(MISC2), MISC2_VAL) self.assertEqual(ps.get(MISC3), MISC3_VAL) self.assertEqual(ps.get(DATA), DATA_VAL) self.assertGreater(ps.get(events.Event.EVENTTIME), 0) self.assertGreater(ps.get(events.Event.PUBTIME), 0) self.assertEqual(ps.get(events.Event.RUNID), runID) self.assertEqual(ps.get(events.Event.STATUS), "unknown") self.assertEqual(ps.get(events.Event.TOPIC), topic) self.assertEqual(ps.get(events.Event.TYPE), events.EventTypes.EVENT) # # wait a short time to receive an event. none was sent, so we should # time out and confirm that we didn't get anything # val = recv.receiveEvent(1000) self.assertIsNone(val)
def testFilterableSendEvent(self): testEnv = TestEnvironment() broker = testEnv.getBroker() thisHost = platform.node() topic = "test_events_filters_%s_%d" % (thisHost, os.getpid()) runId = "test_filters_runid" recv = events.EventReceiver(broker, topic) trans = events.EventTransmitter(broker, topic) root = PropertySet() DATE = "date" DATE_VAL = "2007-07-01T14:28:32.546012" root.set(DATE, DATE_VAL) BLANK = "blank" BLANK_VAL = "" root.set(BLANK, BLANK_VAL) PID = "pid" PID_VAL = os.getpid() root.setInt(PID, PID_VAL) HOST = "host" HOST_VAL = "lsstcorp.org" root.set(HOST, HOST_VAL) IP = "ip" IP_VAL = "1.2.3.4" root.set(IP, IP_VAL) EVNT = "evnt" EVNT_VAL = "test" root.set(EVNT, EVNT_VAL) MISC1 = "misc1" MISC1_VAL = "data 1" root.set(MISC1, MISC1_VAL) MISC2 = "misc2" MISC2_VAL = "data 2" root.set(MISC2, MISC2_VAL) MISC3 = "misc3" MISC3_VAL = "" root.set(MISC3, MISC3_VAL) DATA = "data" DATA_VAL = 3.14 root.setDouble(DATA, DATA_VAL) filterable = PropertySet() filterable.set("FOO", "bar") filterable.set("XYZZY", 123) filterable.set("PLOUGH", 0.867) event = events.Event(runId, root, filterable) trans.publishEvent(event) val = recv.receiveEvent() self.assertIsNotNone(val) ps = val.getPropertySet() self.assertEqual(ps.get(DATE), DATE_VAL) self.assertEqual(ps.get(BLANK), BLANK_VAL) self.assertEqual(ps.get(PID), PID_VAL) self.assertEqual(ps.get(HOST), HOST_VAL) self.assertEqual(ps.get(IP), IP_VAL) self.assertEqual(ps.get(EVNT), EVNT_VAL) self.assertEqual(ps.get(MISC1), MISC1_VAL) self.assertEqual(ps.get(MISC2), MISC2_VAL) self.assertEqual(ps.get(MISC3), MISC3_VAL) self.assertEqual(ps.get(DATA), DATA_VAL) self.assertGreater(ps.get(events.Event.EVENTTIME), 0) self.assertGreater(ps.get(events.Event.PUBTIME), 0) self.assertEqual(ps.get(events.Event.RUNID), runId) self.assertEqual(ps.get(events.Event.STATUS), "unknown") self.assertEqual(ps.get(events.Event.TOPIC), topic) self.assertEqual(ps.get(events.Event.TYPE), events.EventTypes.EVENT) self.assertEqual(ps.get("FOO"), "bar") self.assertEqual(ps.get("XYZZY"), 123) self.assertEqual(ps.get("PLOUGH"), 0.867) names = val.getFilterablePropertyNames() values = [events.Event.EVENTTIME, 'FOO', 'PLOUGH', events.Event.PUBTIME, events.Event.RUNID, events.Event.STATUS, events.Event.TOPIC, events.Event.TYPE, 'XYZZY'] for x in values: self.assertTrue(x in names) # # wait a short time to receive an event. none was sent, so we should # time out and confirm that we didn't get anything # val = recv.receiveEvent(1000) self.assertIsNone(val)
def testCommandEvent(self): testEnv = EventsEnvironment() broker = testEnv.getBroker() thisHost = platform.node() topic = "test_events_command_%s_%d" % (thisHost, os.getpid()) # send a command event receiver = events.EventReceiver(broker, topic) trans = events.EventTransmitter(broker, topic) originatorId = events.LocationId() root = PropertySet() root.set(events.StatusEvent.TOPIC,topic) root.set("myname","myname") root.set(events.StatusEvent.STATUS, "my special status") event = events.StatusEvent("srptestrun", originatorId, root) statusOriginatorId = event.getOriginator() destinationID = events.LocationId(statusOriginatorId) commandOriginatorID = events.LocationId() root2 = PropertySet() root2.set(events.CommandEvent.TOPIC,topic) root2.set("myname","myname2") root2.set(events.CommandEvent.STATUS, "my special status2") event2 = events.CommandEvent("srptestrun", commandOriginatorID, destinationID, root2) trans.publishEvent(event2) val = receiver.receiveCommandEvent() # be sure we received an event self.assertIsNotNone(val) # these are the filterable names we expect to see names = [events.CommandEvent.DEST_HOSTNAME, events.CommandEvent.DEST_LOCALID, events.CommandEvent.DEST_PROCESSID, events.CommandEvent.EVENTTIME, events.CommandEvent.ORIG_HOSTNAME, events.CommandEvent.ORIG_LOCALID, events.CommandEvent.ORIG_PROCESSID, events.CommandEvent.PUBTIME, events.CommandEvent.RUNID, events.CommandEvent.STATUS, events.CommandEvent.TOPIC, events.CommandEvent.TYPE] self.assertValid(val, names, commandOriginatorID.getLocalID(), destinationID.getLocalID()) # send a command event with additional filterable properties receiver = events.EventReceiver(broker, topic) trans = events.EventTransmitter(broker, topic) originatorId = events.LocationId() event = events.StatusEvent("srptestrun", originatorId, root) statusOriginatorId = event.getOriginator() destinationID = events.LocationId(statusOriginatorId) commandOriginatorID = events.LocationId() filterable = PropertySet() filterable.set("FOO", 12.3) filterable.set("BAR", .45) event2 = events.CommandEvent("srptestrun", commandOriginatorID, destinationID, root2, filterable) trans.publishEvent(event2) val = receiver.receiveCommandEvent() # be sure we received an event self.assertIsNotNone(val) # these are the filterable names we expect to see names = [events.CommandEvent.DEST_HOSTNAME, events.CommandEvent.DEST_LOCALID, events.CommandEvent.DEST_PROCESSID, events.CommandEvent.EVENTTIME, events.CommandEvent.ORIG_HOSTNAME, events.CommandEvent.ORIG_LOCALID, events.CommandEvent.ORIG_PROCESSID, events.CommandEvent.PUBTIME, events.CommandEvent.RUNID, events.CommandEvent.STATUS, events.CommandEvent.TOPIC, events.CommandEvent.TYPE, 'FOO', 'BAR'] self.assertValid(val, names, commandOriginatorID.getLocalID(), destinationID.getLocalID()) self.assertTrue(event.getType(), events.EventTypes.COMMAND)
def sendFilterableStatusEvent(self, broker, topic, runID=None): trans = events.EventTransmitter(broker, topic) root = PropertySet() root.set(events.Event.TOPIC, topic) root.set("myname", "myname") root.set(events.Event.STATUS, "my special status") filter = PropertySet() filter.set("FOO", "bar") filter.set("PLOUGH", 123) filter.set("PLOVER", 0.45) eventSystem = events.EventSystem.getDefaultEventSystem() locationID = eventSystem.createOriginatorId() if runID is None: event = events.StatusEvent(locationID, root, filter) else: event = events.StatusEvent(runID, locationID, root, filter) # ok...now publish it trans.publishEvent(event)
def propertySetFromDict(keyValDict): """Make an lsst.daf.base.PropertySet from a dict of key: value""" metadata = PropertySet() for key, val in keyValDict.items(): metadata.set(key, val) return metadata
def makeJobOfficeStatusEvent(self, runId, status): props = PropertySet() props.set("STATUS",status) return StatusEvent(runId, self.originatorId, props)