def testProcessDataEvent(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "available") ds = self.testDatasetFromProperty() ps.add("dataset", serializePolicy(ds.toPolicy())) for i in xrange(1, 4): ds = copy.deepcopy(ds) ds.ids["ampid"] += 1 ps.add("dataset", serializePolicy(ds.toPolicy())) devent = StatusEvent("testing", originatorId, ps) # pdb.set_trace() self.joboffice.processDataEvent(devent) with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 4) self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(), 1) job = self.joboffice.bb.queues.jobsPossible.get(0) self.assertEquals(job.getName(), "Job-1") self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 12)
def testValidRun(self): """Test how run delegates to the abstract methods. """ mockKey = "unitTestKey" with patch.object(self.task, "getInputMetadataKeys", return_value={"unused": mockKey}), \ patch.object(self.task, "makeMeasurement") as mockWorkhorse: if self._takesScalarMetadata(self.task): metadata1 = PropertySet() metadata1[mockKey] = 42 self.task.run(metadata1) mockWorkhorse.assert_called_once_with({"unused": 42}) mockWorkhorse.reset_mock() self.task.run(None) mockWorkhorse.assert_called_once_with({"unused": None}) else: metadata1 = PropertySet() metadata1[mockKey] = 42 metadata2 = PropertySet() metadata2[mockKey] = "Sphere" self.task.run([metadata1, None, metadata2]) mockWorkhorse.assert_called_once_with([{ "unused": value } for value in [42, None, "Sphere"]])
def createPropertySet(self): data = self.getDataSet() prop = PropertySet() for x in data: prop.set(x[0],x[1]) return prop
def process(self, clipboard): metadata = clipboard.get(self.policy.getString("inputKeys.metadata")) cdb = clipboard.get(self.policy.getString("inputKeys.calibrationDB")) pathPrefix = cliboard.get(self.policy.getString("inputKeys.prefixPath")) when = DateTime(metadata.get("dateObs")) ccdId = metadata.get("ccdId") ampId = metadata.get("ampId") expTime = metadata.get("expTime") darkToApply = self.policy.getString("whichdark") darkCalibList = cdb.lookup(when, "dark", ccdId, ampId, all=True) darkTimeList = [] for d in darkCalibList: darkTimeList.append(d.expTime) darkTimeList.sort() if darktoapply == "min": darkExpTime = darkTimeList[0] elif darktoapply == "max": darkExpTime = darkTimeList[-1] elif darktoapply == "closest": minDist = abs(expTime - darkTimeList[0]) minExpTime = darkTimeList[0] for i in xrange(1, len(darkTimeList)): dist = abs(expTime - darkTimeList[i]) if dist < minDist: minDist = dist minExpTime = darkTimeList[i] darkExpTime = minExpTime else: raise RuntimeError, "Unrecognized method for finding dark to apply: " + str(darktoapply) biasPath = cdb.lookup(when, "bias", ccdId, ampId) darkPath = cdb.lookup(when, "dark", ccdId, ampId, expTime=darkExpTime) defectPath = cdb.lookup(when, "defect", ccdId, ampId) flatPath = cdb.lookup(when, "flat", ccdId, ampId, filter=metadata.get("filter")) # fringePath = cdb.lookup(when, "fringe", ccdId, ampId, # filter=metadata.get("filter")) linearizePath = cdb.lookup(when, "linearize", ccdId, ampId) calibData = PropertySet() calibData.set("biasPath", os.path.join(pathPrefix, biasPath)) calibData.set("darkPath", os.path.join(pathPrefix, darkPath)) calibData.set("defectPath", os.path.join(pathPrefix, defectPath)) calibData.set("flatPath", os.path.join(pathPrefix, flatPath)) # calibData.set("fringePath", os.path.join(pathPrefix, fringePath)) calibData.set("linearizePath", os.path.join(pathPrefix, linearizePath)) outputKey = self._policy.get("outputKey") self.activeClipboard.put(outputKey, calibData) self.outputQueue.addDataset(self.activeClipboard)
def makeit(prefs, context, saveWcs=False, plot=dict()): """This is the python wrapper for the original psfex that reads SExtractor outputs """ # Create an array of PSFs (one PSF for each extension) if prefs.getVerboseType() != prefs.QUIET: print("----- %d input catalogues:" % prefs.getNcat()) if saveWcs: # only needed for making plots wcssList = [] fields = psfexLib.vectorField() for cat in prefs.getCatalogs(): field = psfexLib.Field(cat) wcss = [] wcssList.append(wcss) with fits.open(cat) as pf: for hdu in pf: if hdu.name == "PRIMARY": pass elif hdu.name == "LDAC_IMHEAD": hdr = hdu.data[0][0] # the fits header from the original fits image md = PropertySet() for line in hdr: try: md.set(*splitFitsCard(line)) except AttributeError: continue if not md.exists("CRPIX1"): # no WCS; try WCSA for k in md.names(): if re.search(r"A$", k): md.set(k[:-1], md.getScalar(k)) wcs = afwGeom.makeSkyWcs(md) naxis1, naxis2 = md.getScalar("NAXIS1"), md.getScalar("NAXIS2") elif hdu.name == "LDAC_OBJECTS": nobj = len(hdu.data) assert wcs, "LDAC_OBJECTS comes after LDAC_IMHEAD" field.addExt(wcs, naxis1, naxis2, nobj) if saveWcs: wcss.append((wcs, naxis1, naxis2)) wcs = None field.finalize() fields.append(field) sets = psfexLib.vectorSet() for set in load_samples(prefs, context, plot=plot): sets.append(set) psfexLib.makeit(fields, sets) ret = [[f.getPsfs() for f in fields], sets] if saveWcs: ret.append(wcssList) return ret
def stopWorkflow(self, urgency): log.debug("VanillaCondorWorkflowMonitor:stopWorkflow") transmit = events.EventTransmitter(self._eventBrokerHost, self._shutdownTopic) root = PropertySet() root.setInt("level",urgency) root.setString("STATUS","shut things down") event = events.CommandEvent(self.runid, self.originatorId, 0, root) transmit.publishEvent(event)
def testProcessDataEvent(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "available") ds = self.testDatasetFromProperty() ps.add("dataset", serializePolicy(ds.toPolicy())) for i in xrange(1,4): ds = copy.deepcopy(ds) ds.ids["ampid"] += 1 ps.add("dataset", serializePolicy(ds.toPolicy())) devent = StatusEvent("testing", originatorId, ps) # pdb.set_trace() self.joboffice.processDataEvent(devent) with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 4) self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(), 1) job = self.joboffice.bb.queues.jobsPossible.get(0) self.assertEquals(job.getName(), "Job-1") self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 12)
def testEventRunID(self): runid = "testrunid" status = "my special status" root = PropertySet() MYNAME = "myname" root.set(MYNAME, MYNAME) root.set(events.Event.STATUS, status) event = events.Event(runid, root) # return the property set props = event.getPropertySet() self.assertGreater(props.get(events.Event.EVENTTIME), 0) self.assertEqual(props.get(events.Event.PUBTIME), 0) self.assertEqual(props.get(events.Event.RUNID), runid) self.assertEqual(props.get(events.Event.STATUS), status) self.assertEqual(props.get(events.Event.TOPIC), events.Event.UNINITIALIZED) self.assertEqual(props.get(events.Event.TYPE), events.EventTypes.EVENT) self.assertEqual(props.get(MYNAME), MYNAME) filterableNames = event.getFilterablePropertyNames() filterableNames.remove(events.Event.EVENTTIME) filterableNames.remove(events.Event.PUBTIME) filterableNames.remove(events.Event.RUNID) filterableNames.remove(events.Event.STATUS) filterableNames.remove(events.Event.TOPIC) filterableNames.remove(events.Event.TYPE) assert len(filterableNames) == 0 customNames = event.getCustomPropertyNames() self.assertEqual(len(customNames), 1) self.assertEqual(customNames[0], MYNAME) eventTime = event.getEventTime() self.assertEqual(event.getPubTime(),0) self.assertEqual(event.getRunId(), runid) self.assertEqual(event.getType(), events.EventTypes.EVENT) self.assertEqual(event.getStatus(), status) # check to be sure we really update the time event.updateEventTime() self.assertLess(eventTime, event.getEventTime()) # set the time to zero, and make sure we get back the same time. event.setEventTime(0) self.assertEqual(event.getEventTime(), 0) # reset the time to eventTime, and make sure we get back the same time. event.setEventTime(eventTime) self.assertEqual(event.getEventTime(), eventTime)
def testValidRun(self): mockKey = "unitTestKey" with unittest.mock.patch.object(self.task, "getInputMetadataKeys", return_value={"unused": mockKey}): with unittest.mock.patch.object(self.task, "makeMeasurement") \ as mockWorkhorse: metadata1 = PropertySet() metadata1[mockKey] = 42 metadata2 = PropertySet() metadata2[mockKey] = "Sphere" self.task.run([metadata1, None, metadata2]) mockWorkhorse.assert_called_once_with( [{"unused": value} for value in [42, None, "Sphere"]])
def testEventProperties(self): status = "my special status" root = PropertySet() MYNAME = "myname" root.set(MYNAME, MYNAME) root.set(events.Event.STATUS, status) event = events.Event(root) props = event.getPropertySet() custom = event.getCustomPropertySet() self.assertEqual(props.nameCount(), 6) self.assertEqual(custom.nameCount(), 1)
def __init__(self, runid, props=None): """ create a generic event factor """ self.runid = runid if isinstance(props, PropertySet): self.props = props else: self.props = PropertySet() if isinstance(props, dict): for key in props.keys(): self.props.set(key, props[key])
def testProcessJobDoneEvent(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:done") ps.set("success", True) pevent = StatusEvent("testing", originatorId, ps) self.assert_(not self.joboffice.processJobDoneEvent(pevent)) self.testAllocateJobs() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 1) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0) # pdb.set_trace() self.assert_(self.joboffice.processJobDoneEvent(pevent)) with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 1)
def testProcessJobDoneEvents(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0) self.testAllocateJobs() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 1) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:done") ps.set("success", True) pevent = StatusEvent("testing", originatorId, ps) trx = EventTransmitter(brokerhost, "CcdAssemblyJob") trx.publishEvent(pevent) time.sleep(2.0) self.joboffice.processDoneJobs() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(), 0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 1)
def process(self): clipboard = self.inputQueue.getNextDataset() eventName = self._policy.get("inputEvent") event = clipboard.get(eventName) visitId = event.get("visitId") exposureId = event.get("exposureId") ccdId = clipboard.get("ccdId") ampId = clipboard.get("ampId") fpaExposureId = (long(visitId) << 1) + exposureId ccdExposureId = (fpaExposureId << 8) + ccdId ampExposureId = (ccdExposureId << 6) + ampId clipboard.put("visitId", visitId) exposureMetadata = PropertySet() exposureMetadata.setInt("filterId", self.lookupFilterId(event.get("filter"))) exposureMetadata.setLongLong("fpaExposureId", fpaExposureId) exposureMetadata.setLongLong("ccdExposureId", ccdExposureId) exposureMetadata.setLongLong("ampExposureId", ampExposureId) clipboard.put("exposureMetadata" + str(exposureId), exposureMetadata) self.outputQueue.addDataset(clipboard)
def makePluginAndCat(alg, name, control=None, metadata=False, centroid=None, psfflux=None): print("Making plugin ", alg, name) if control is None: control = alg.ConfigClass() schema = afwTable.SourceTable.makeMinimalSchema() if centroid: lsst.afw.table.Point2DKey.addFields(schema, centroid, "centroid", "pixel") schema.getAliasMap().set("slot_Centroid", centroid) if psfflux: base.PsfFluxAlgorithm(base.PsfFluxControl(), psfflux, schema) schema.getAliasMap().set("slot_PsfFlux", psfflux) if metadata: plugin = alg(control, name, schema, PropertySet()) else: plugin = alg(control, name, schema) cat = afwTable.SourceCatalog(schema) if centroid: cat.defineCentroid(centroid) return plugin, cat
def finalMessageReceived(self, propSet): log = propSet.get("LOGGER") if log != "orca.control": return False status = propSet.get("STATUS") if status != "eol": return False eventSystem = events.EventSystem.getDefaultEventSystem() id = eventSystem.createOriginatorId() root = PropertySet() root.setString("logger.status", "done") root.setInt("logger.pid", os.getpid()) event = events.StatusEvent(self.runid, id, root) self.transmitter.publishEvent(event) return True
def getCfhtImage(self): """Get a portion of a CFHT image as a MaskedImageF""" bbox = lsst.geom.Box2I(lsst.geom.Point2I(500, 2000), lsst.geom.Point2I(2079, 4609)) imagePath = os.path.join(AfwdataDir, "CFHT", "D4", "cal-53535-i-797722_1.fits") return afwImage.MaskedImageF(imagePath, PropertySet(), bbox)
def testReceiveReadyPipelines(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(),0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:ready") pevent = StatusEvent("testing", originatorId, ps) trx = EventTransmitter(brokerhost, "CcdAssemblyJob") trx.publishEvent(pevent) time.sleep(2.0) self.joboffice.receiveReadyPipelines() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(),1)
def sendEvent(self, broker, topic): trans = events.EventTransmitter(broker, topic) root = PropertySet() root.set(events.Event.TOPIC, topic) root.set("myname", "myname") root.set(events.Event.STATUS, "my special status") root.set(events.Event.RUNID, "srptestrun") event = events.Event("srptestrun", root) # wait a short time so we can see the difference between the time # the event is created and the time it is published time.sleep(2) # ok...now publish it trans.publishEvent(event)
def testAmbiguousRun(self): mockKey = "unitTestKey" with unittest.mock.patch.object(self.task, "getInputMetadataKeys", return_value={"unused": mockKey}): metadata = PropertySet() metadata[mockKey + "1"] = 42 metadata[mockKey + "2"] = "Sphere" with self.assertRaises(MetricComputationError): self.task.run([metadata])
def testToPipelineQueueItem(self): pipelineName = "ccdassembly" ps = PropertySet() ps.set("pipelineName", pipelineName) ps.set("STATUS", "done") ps.set("RUNID", "testing") pevent = StatusEvent("testing", originatorId, ps) item = self.joboffice.toPipelineQueueItem(pevent) self.assertEquals(item.getName(), pipelineName) self.assertEquals(item.getProperty("status"), "done") self.assertEquals(item.getRunId(), "testing")
def testSubst(self): ad = PropertySet() ad.set("foo", "bar") ad.setInt("x", 3) LogicalLocation.setLocationMap(ad) l = LogicalLocation("%(foo)xx") self.assertEqual(l.locString(), "barxx") l = LogicalLocation("%(x)foo") self.assertEqual(l.locString(), "3foo") l = LogicalLocation("yy%04d(x)yy") self.assertEqual(l.locString(), "yy0003yy") ad2 = PropertySet() ad2.set("foo", "baz") ad2.setInt("y", 2009) l = LogicalLocation("%(foo)%(x)%(y)", ad2) self.assertEqual(l.locString(), "bar32009") LogicalLocation.setLocationMap(PropertySet()) l = LogicalLocation("%(foo)%3d(y)", ad2) self.assertEqual(l.locString(), "baz2009")
def sendEvent(self, runid, brokerName, topic): trans = events.EventTransmitter(brokerName, topic) root = PropertySet() root.set(events.Event.TOPIC, topic) root.set("myname","myname") root.set(events.Event.STATUS, "my special status") locationID = events.LocationId() event = events.StatusEvent(runid, locationID, root) # ok...now publish it trans.publishEvent(event)
def sendEvent(self, broker, topic): trans = events.EventTransmitter(broker, topic) root = PropertySet() root.set(events.Event.TOPIC,topic) root.set("myname","myname") root.set(events.Event.STATUS, "my special status") root.set(events.Event.RUNID,"srptestrun") event = events.Event("srptestrun", root) # wait a short time so we can see the difference between the time # the event is created and the time it is published time.sleep(2) # ok...now publish it trans.publishEvent(event)
def write(self, inMemoryDataset): """Write a Python object to a file. Parameters ---------- inMemoryDataset : `object` The Python object to store. """ # Update the location with the formatter-preferred file extension self.fileDescriptor.location.updateExtension(self.extension) outputPath = self.fileDescriptor.location.path # check to see if we have a recipe requested recipeName = self.writeParameters.get("recipe") recipe = self.getImageCompressionSettings(recipeName) if recipe: # Can not construct a PropertySet from a hierarchical # dict but can update one. ps = PropertySet() ps.update(recipe) inMemoryDataset.writeFitsWithOptions(outputPath, options=ps) else: inMemoryDataset.writeFits(outputPath)
def _runStageWithPolicy(self, policy): stage = EventStage(0, policy) tester = pexHarness.SimpleStageTester.test(stage) tester.setDebugVerbosity(5) keysToPublish = policy.getArray("keysToPublish") clipboard = {} for key in keysToPublish: key = stage._parseKeysToPublish(key)[1] clipboard[key] = PropertySet() clipboard[key].setString("foo", "bar") self.assertRaises(lss.pex.exceptions.Exception, tester.runMaster, clipboard) stage.setEventBrokerHost("lsst4.ncsa.uiuc.edu") clipboard = tester.runMaster(clipboard)
def testFundamentalTypes(self) -> None: """Ensure that some fundamental stack types round trip.""" ps = PropertySet() ps["a.b"] = 5 ps["c.d.e"] = "string" self.runFundamentalTypeTest("ps", ps) pl = PropertyList() pl["A"] = 1 pl.setComment("A", "An int comment") pl["B"] = "string" pl.setComment("B", "A string comment") self.runFundamentalTypeTest("pl", pl) pkg = Packages.fromSystem() self.runFundamentalTypeTest("pkg", pkg)
class _EventFactory(object): def __init__(self, runid, props=None): """ create a generic event factor """ self.runid = runid if isinstance(props, PropertySet): self.props = props else: self.props = PropertySet() if isinstance(props, dict): for key in props.keys(): self.props.set(key, props[key]) def create(self): """create a new instance of the event""" return ev.Event(self.runid, self.props) def setRunId(self, id): """set the Run ID""" self.runid = runid def getRunId(self): """set the Run ID""" return self.runid def setProperty(self, name, val): """set the value of a named property""" self.props.set(name, val) def getProperty(self, name): """get the value of a named property""" return self.props.getString(name) def addDataset(self, propname, ds): """add a dataset to the event""" self.props.add(propname, serializeDataset(ds)) def getDatasets(self, propname): """return the datasets attached to the event""" return unserializeDatasetList(self.props.getArrayString(propname))
def testEventRunID(self): runid = "testrunid" status = "my special status" root = PropertySet() MYNAME = "myname" root.set(MYNAME, MYNAME) root.set(events.Event.STATUS, status) event = events.Event(runid, root) # return the property set props = event.getPropertySet() self.assertGreater(props.get(events.Event.EVENTTIME), 0) self.assertEqual(props.get(events.Event.PUBTIME), 0) self.assertEqual(props.get(events.Event.RUNID), runid) self.assertEqual(props.get(events.Event.STATUS), status) self.assertEqual(props.get(events.Event.TOPIC), events.Event.UNINITIALIZED) self.assertEqual(props.get(events.Event.TYPE), events.EventTypes.EVENT) self.assertEqual(props.get(MYNAME), MYNAME) filterableNames = event.getFilterablePropertyNames() filterableNames.remove(events.Event.EVENTTIME) filterableNames.remove(events.Event.PUBTIME) filterableNames.remove(events.Event.RUNID) filterableNames.remove(events.Event.STATUS) filterableNames.remove(events.Event.TOPIC) filterableNames.remove(events.Event.TYPE) assert len(filterableNames) == 0 customNames = event.getCustomPropertyNames() self.assertEqual(len(customNames), 1) self.assertEqual(customNames[0], MYNAME) eventTime = event.getEventTime() self.assertEqual(event.getPubTime(), 0) self.assertEqual(event.getRunId(), runid) self.assertEqual(event.getType(), events.EventTypes.EVENT) self.assertEqual(event.getStatus(), status) # check to be sure we really update the time event.updateEventTime() self.assertLess(eventTime, event.getEventTime()) # set the time to zero, and make sure we get back the same time. event.setEventTime(0) self.assertEqual(event.getEventTime(), 0) # reset the time to eventTime, and make sure we get back the same time. event.setEventTime(eventTime) self.assertEqual(event.getEventTime(), eventTime)
def makePluginAndCat(alg, name, control=None, metadata=False, centroid=None): print("Making plugin ", alg, name) if control is None: control = alg.ConfigClass() schema = afwTable.SourceTable.makeMinimalSchema() if centroid: schema.addField(centroid + "_x", type=float) schema.addField(centroid + "_y", type=float) schema.addField(centroid + "_flag", type='Flag') schema.getAliasMap().set("slot_Centroid", centroid) if metadata: plugin = alg(control, name, schema, PropertySet()) else: plugin = alg(control, name, schema) cat = afwTable.SourceCatalog(schema) if centroid: cat.defineCentroid(centroid) return plugin, cat
def testReceiveReadyPipelines(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(), 0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:ready") pevent = StatusEvent("testing", originatorId, ps) trx = EventTransmitter(brokerhost, "CcdAssemblyJob") trx.publishEvent(pevent) time.sleep(2.0) self.joboffice.receiveReadyPipelines() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(), 1)
def testProcessJobDoneEvents(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),0) self.testAllocateJobs() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),1) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:done") ps.set("success", True) pevent = StatusEvent("testing", originatorId, ps) trx = EventTransmitter(brokerhost, "CcdAssemblyJob") trx.publishEvent(pevent) time.sleep(2.0) self.joboffice.processDoneJobs() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),1)
def testProcessJobDoneEvent(self): with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),0) ps = PropertySet() ps.set("pipelineName", "PostISR") ps.set("STATUS", "job:done") ps.set("success", True) pevent = StatusEvent("testing", originatorId, ps) self.assert_(not self.joboffice.processJobDoneEvent(pevent)) self.testAllocateJobs() with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),1) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),0) # pdb.set_trace() self.assert_(self.joboffice.processJobDoneEvent(pevent)) with self.joboffice.bb.queues: self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0) self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),1)
def dummyMetadata(): result = PropertySet() result.add("lsst.ap.pipe.ccdProcessor.cycleCount", 42) return result
def preprocess(self): self.activeClipboard = self.inputQueue.getNextDataset() eventName = self._policy.get("inputEvent") event = self.activeClipboard.get(eventName) visitId = event.get("visitId") exposureId = event.get("exposureId") fpaExposureId = (long(visitId) << 1) + exposureId visit = PropertySet() visit.setInt("visitId", visitId) visit.setLongLong("exposureId", fpaExposureId) self.activeClipboard.put("visit" + str(exposureId), visit) rawFpaExposure = PropertySet() rawFpaExposure.setLongLong("rawFPAExposureId", fpaExposureId) rawFpaExposure.set("ra", event.get("ra")) rawFpaExposure.set("decl", event.get("decl")) rawFpaExposure.set("filterId", self.lookupFilterId(event.get("filter"))) rawFpaExposure.set("equinox", event.get("equinox")) rawFpaExposure.set("dateObs", DateTime(event.get("dateObs"))) rawFpaExposure.set("mjdObs", DateTime(event.get("dateObs")).mjd()) rawFpaExposure.set("expTime", event.get("expTime")) rawFpaExposure.set("airmass", event.get("airmass")) self.activeClipboard.put("fpaExposure" + str(exposureId), rawFpaExposure)
def makeJobCommandEvent(self, job, pipeline, runId=""): """ create a CommandEvent to send to a pipeline instructing it to commence working on the given job. """ props = PropertySet() for ds in job.getInputDatasets(): props.add("inputs", serializePolicy(ds.toPolicy())) for ds in job.getOutputDatasets(): props.add("outputs", serializePolicy(ds.toPolicy())) props.set("identity", serializePolicy(job.getJobIdentity().toPolicy())) props.set("STATUS", "job:assign") props.set("name", job.getName()) return CommandEvent(runId, self.originatorId, pipeline, props)
def sendFilterableStatusEvent(self, broker, topic): trans = events.EventTransmitter(broker, topic) root = PropertySet() root.set(events.Event.TOPIC, topic) root.set("myname","myname") root.set(events.Event.STATUS, "my special status") filter = PropertySet() filter.set("FOO", "bar") filter.set("PLOUGH", 123) eventSystem = events.EventSystem.getDefaultEventSystem(); locationID = eventSystem.createOriginatorId() event = events.StatusEvent(locationID, root, filter) # ok...now publish it trans.publishEvent(event)
def testLocationId(self): testEnv = EventsEnvironment() broker = testEnv.getBroker() thisHost = platform.node() locationID = events.LocationId() # check to see if localID is 0 localId = locationID.getLocalID() self.assertEqual(localId, 0) # check host name hostname = locationID.getHostName() self.assertEqual(hostname, socket.gethostname()) # check process id processId = locationID.getProcessID() self.assertEqual(processId, os.getpid()) locationID2 = events.LocationId() # check to see if localID is 1 localId = locationID2.getLocalID() self.assertEqual(localId, 1) self.assertEqual(locationID.getLocalID(), 0) # check host name hostname = locationID.getHostName() self.assertEqual(hostname, socket.gethostname()) # check process id processId = locationID2.getProcessID() self.assertEqual(processId, os.getpid()) root = PropertySet() root.set("myname", "myname") status = "my special status" root.set(events.Event.STATUS, status) statusEvent = events.StatusEvent("my runid", locationID, root) statusEvent2 = events.StatusEvent("my runid", locationID2, root) topic = "mytopic_%s_%d" % (thisHost, os.getpid()) transmitter = events.EventTransmitter(broker, topic) hostname = locationID2.getHostName() # create a receiver selector with the hostname, process id and local id sel = "%s = '%s' and %s = %d and %s = %d" % ( events.StatusEvent.ORIG_HOSTNAME, hostname, events.StatusEvent.ORIG_PROCESSID, os.getpid(), events.StatusEvent.ORIG_LOCALID, 1) receiver = events.EventReceiver(broker, topic, sel) # transmit the events transmitter.publishEvent(statusEvent) transmitter.publishEvent(statusEvent2) # should receive event with with the process id returnedEvent = receiver.receiveEvent() ps = returnedEvent.getPropertySet() self.assertEqual(ps.get('ORIG_HOSTNAME'), hostname) self.assertEqual(ps.get('ORIG_PROCESSID'), os.getpid()) self.assertEqual(ps.get('ORIG_LOCALID'), 1) # should NOT receive another event, because it was filtered out by # the broker returnedEvent2 = receiver.receiveEvent(1) self.assertIsNone(returnedEvent2)
def sendPlainStatusEvent(self, broker, topic, runID=None): trans = events.EventTransmitter(broker, topic) root = PropertySet() root.set(events.Event.TOPIC, topic) root.set("myname","myname") root.set(events.Event.STATUS, "my special status") root.set("value", 12) root.set("logger.status", "my logger special status") root.set("logger.name", "myname") root.set("logger.pid.xyzzy", 1) root.set("logger.pid.plover", 3.14) root.set("logger.pid.plugh", "a hollow voice says") eventSystem = events.EventSystem.getDefaultEventSystem(); locationID = eventSystem.createOriginatorId() if runID is None: event = events.StatusEvent(locationID, root) else: event = events.StatusEvent(runID, locationID, root) # ok...now publish it trans.publishEvent(event)
def createWcs(): metadata = PropertySet() metadata.set("SIMPLE", "T") metadata.set("BITPIX", -32) metadata.set("NAXIS", 2) metadata.set("NAXIS1", 1024) metadata.set("NAXIS2", 1153) metadata.set("RADESYS", 'FK5') metadata.set("EQUINOX", 2000.) metadata.setDouble("CRVAL1", 215.604025685476) metadata.setDouble("CRVAL2", 53.1595451514076) metadata.setDouble("CRPIX1", 1109.99981456774) metadata.setDouble("CRPIX2", 560.018167811613) metadata.set("CTYPE1", 'RA---SIN') metadata.set("CTYPE2", 'DEC--SIN') metadata.setDouble("CD1_1", 5.10808596133527E-05) metadata.setDouble("CD1_2", 1.85579539217196E-07) metadata.setDouble("CD2_2", -5.10281493481982E-05) metadata.setDouble("CD2_1", -8.27440751733828E-07) return makeSkyWcs(metadata)
def testFilterableSendEvent(self): testEnv = TestEnvironment() broker = testEnv.getBroker() thisHost = platform.node() topic = "test_events_filters_%s_%d" % (thisHost, os.getpid()) runId = "test_filters_runid" recv = events.EventReceiver(broker, topic) trans = events.EventTransmitter(broker, topic) root = PropertySet() DATE = "date" DATE_VAL = "2007-07-01T14:28:32.546012" root.set(DATE, DATE_VAL) BLANK = "blank" BLANK_VAL = "" root.set(BLANK, BLANK_VAL) PID = "pid" PID_VAL = os.getpid() root.setInt(PID, PID_VAL) HOST = "host" HOST_VAL = "lsstcorp.org" root.set(HOST, HOST_VAL) IP = "ip" IP_VAL = "1.2.3.4" root.set(IP, IP_VAL) EVNT = "evnt" EVNT_VAL = "test" root.set(EVNT, EVNT_VAL) MISC1 = "misc1" MISC1_VAL = "data 1" root.set(MISC1, MISC1_VAL) MISC2 = "misc2" MISC2_VAL = "data 2" root.set(MISC2, MISC2_VAL) MISC3 = "misc3" MISC3_VAL = "" root.set(MISC3, MISC3_VAL) DATA = "data" DATA_VAL = 3.14 root.setDouble(DATA, DATA_VAL) filterable = PropertySet() filterable.set("FOO", "bar") filterable.set("XYZZY", 123) filterable.set("PLOUGH", 0.867) event = events.Event(runId, root, filterable) trans.publishEvent(event) val = recv.receiveEvent() self.assertIsNotNone(val) ps = val.getPropertySet() self.assertEqual(ps.get(DATE), DATE_VAL) self.assertEqual(ps.get(BLANK), BLANK_VAL) self.assertEqual(ps.get(PID), PID_VAL) self.assertEqual(ps.get(HOST), HOST_VAL) self.assertEqual(ps.get(IP), IP_VAL) self.assertEqual(ps.get(EVNT), EVNT_VAL) self.assertEqual(ps.get(MISC1), MISC1_VAL) self.assertEqual(ps.get(MISC2), MISC2_VAL) self.assertEqual(ps.get(MISC3), MISC3_VAL) self.assertEqual(ps.get(DATA), DATA_VAL) self.assertGreater(ps.get(events.Event.EVENTTIME), 0) self.assertGreater(ps.get(events.Event.PUBTIME), 0) self.assertEqual(ps.get(events.Event.RUNID), runId) self.assertEqual(ps.get(events.Event.STATUS), "unknown") self.assertEqual(ps.get(events.Event.TOPIC), topic) self.assertEqual(ps.get(events.Event.TYPE), events.EventTypes.EVENT) self.assertEqual(ps.get("FOO"), "bar") self.assertEqual(ps.get("XYZZY"), 123) self.assertEqual(ps.get("PLOUGH"), 0.867) names = val.getFilterablePropertyNames() values = [events.Event.EVENTTIME, 'FOO', 'PLOUGH', events.Event.PUBTIME, events.Event.RUNID, events.Event.STATUS, events.Event.TOPIC, events.Event.TYPE, 'XYZZY'] for x in values: self.assertTrue(x in names) # # wait a short time to receive an event. none was sent, so we should # time out and confirm that we didn't get anything # val = recv.receiveEvent(1000) self.assertIsNone(val)
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the LSST License Statement and # the GNU General Public License along with this program. If not, # see <http://www.lsstcorp.org/LegalNotices/>. # # used to transmit status that we're not sending any more data after this import lsst.ctrl.events as events from lsst.daf.base import PropertySet import sys # usage: python dataCompleted broker topic runid if __name__ == "__main__": broker = sys.argv[1] topic = sys.argv[2] runid = sys.argv[3] status = sys.argv[4] trans = events.EventTransmitter(broker, topic) eventSystem = events.EventSystem.getDefaultEventSystem() root = PropertySet() root.set("STATUS",status) id = eventSystem.createOriginatorId() event = events.StatusEvent(runid, id, root) trans.publishEvent(event)
def sendEvent(self, brokerName, topic): """Send an Event.""" trans = events.EventTransmitter(brokerName, topic) root = PropertySet() root.set(events.Event.TOPIC, topic) root.set("myname", "myname") root.set(events.StatusEvent.STATUS, "my special status") root.set(events.StatusEvent.RUNID, "srptestrun") root.set("bazinga", "sheldon") originatorId = events.LocationId() event = events.StatusEvent("srptestrun", originatorId, root) # ok...now publish it trans.publishEvent(event)
def testSendEvent(self): """Send an Event""" testEnv = EventsEnvironment() self.thisHost = platform.node() self.broker = testEnv.getBroker() topic = "test_events_3_%s_%d" % (self.thisHost, os.getpid()) runID = "test3_runid" recv = events.EventReceiver(self.broker, topic) trans = events.EventTransmitter(self.broker, topic) DATE = "date" DATE_VAL = "2007-07-01T14:28:32.546012" BLANK = "blank" BLANK_VAL = "" PID = "pid" PID_VAL = os.getpid() HOST = "host" HOST_VAL = "lsstcorp.org" IP = "ip" IP_VAL = "1.2.3.4" EVNT = "evnt" EVNT_VAL = "test" MISC1 = "misc1" MISC1_VAL = "data 1" MISC2 = "misc2" MISC2_VAL = "data 2" MISC3 = "misc3" MISC3_VAL = "" DATA = "data" DATA_VAL = 3.14 root = PropertySet() root.set(DATE, DATE_VAL) root.set(BLANK, BLANK_VAL) root.setInt(PID, PID_VAL) root.set(HOST, HOST_VAL) root.set(IP, IP_VAL) root.set(EVNT, EVNT_VAL) root.set(MISC1, MISC1_VAL) root.set(MISC2, MISC2_VAL) root.set(MISC3, MISC3_VAL) root.setDouble(DATA, DATA_VAL) event = events.Event(runID, root) trans.publishEvent(event) val = recv.receiveEvent() self.assertIsNotNone(val) # check the validity of all sent values ps = val.getPropertySet() self.assertEqual(ps.get(DATE), DATE_VAL) self.assertEqual(ps.get(BLANK), BLANK_VAL) self.assertEqual(ps.get(PID), PID_VAL) self.assertEqual(ps.get(HOST), HOST_VAL) self.assertEqual(ps.get(IP), IP_VAL) self.assertEqual(ps.get(EVNT), EVNT_VAL) self.assertEqual(ps.get(MISC1), MISC1_VAL) self.assertEqual(ps.get(MISC2), MISC2_VAL) self.assertEqual(ps.get(MISC3), MISC3_VAL) self.assertEqual(ps.get(DATA), DATA_VAL) self.assertGreater(ps.get(events.Event.EVENTTIME), 0) self.assertGreater(ps.get(events.Event.PUBTIME), 0) self.assertEqual(ps.get(events.Event.RUNID), runID) self.assertEqual(ps.get(events.Event.STATUS), "unknown") self.assertEqual(ps.get(events.Event.TOPIC), topic) self.assertEqual(ps.get(events.Event.TYPE), events.EventTypes.EVENT) # # wait a short time to receive an event. none was sent, so we should # time out and confirm that we didn't get anything # val = recv.receiveEvent(1000) self.assertIsNone(val)
def testCommandEvent(self): testEnv = EventsEnvironment() broker = testEnv.getBroker() thisHost = platform.node() topic = "test_events_command_%s_%d" % (thisHost, os.getpid()) # send a command event receiver = events.EventReceiver(broker, topic) trans = events.EventTransmitter(broker, topic) originatorId = events.LocationId() root = PropertySet() root.set(events.StatusEvent.TOPIC,topic) root.set("myname","myname") root.set(events.StatusEvent.STATUS, "my special status") event = events.StatusEvent("srptestrun", originatorId, root) statusOriginatorId = event.getOriginator() destinationID = events.LocationId(statusOriginatorId) commandOriginatorID = events.LocationId() root2 = PropertySet() root2.set(events.CommandEvent.TOPIC,topic) root2.set("myname","myname2") root2.set(events.CommandEvent.STATUS, "my special status2") event2 = events.CommandEvent("srptestrun", commandOriginatorID, destinationID, root2) trans.publishEvent(event2) val = receiver.receiveCommandEvent() # be sure we received an event self.assertIsNotNone(val) # these are the filterable names we expect to see names = [events.CommandEvent.DEST_HOSTNAME, events.CommandEvent.DEST_LOCALID, events.CommandEvent.DEST_PROCESSID, events.CommandEvent.EVENTTIME, events.CommandEvent.ORIG_HOSTNAME, events.CommandEvent.ORIG_LOCALID, events.CommandEvent.ORIG_PROCESSID, events.CommandEvent.PUBTIME, events.CommandEvent.RUNID, events.CommandEvent.STATUS, events.CommandEvent.TOPIC, events.CommandEvent.TYPE] self.assertValid(val, names, commandOriginatorID.getLocalID(), destinationID.getLocalID()) # send a command event with additional filterable properties receiver = events.EventReceiver(broker, topic) trans = events.EventTransmitter(broker, topic) originatorId = events.LocationId() event = events.StatusEvent("srptestrun", originatorId, root) statusOriginatorId = event.getOriginator() destinationID = events.LocationId(statusOriginatorId) commandOriginatorID = events.LocationId() filterable = PropertySet() filterable.set("FOO", 12.3) filterable.set("BAR", .45) event2 = events.CommandEvent("srptestrun", commandOriginatorID, destinationID, root2, filterable) trans.publishEvent(event2) val = receiver.receiveCommandEvent() # be sure we received an event self.assertIsNotNone(val) # these are the filterable names we expect to see names = [events.CommandEvent.DEST_HOSTNAME, events.CommandEvent.DEST_LOCALID, events.CommandEvent.DEST_PROCESSID, events.CommandEvent.EVENTTIME, events.CommandEvent.ORIG_HOSTNAME, events.CommandEvent.ORIG_LOCALID, events.CommandEvent.ORIG_PROCESSID, events.CommandEvent.PUBTIME, events.CommandEvent.RUNID, events.CommandEvent.STATUS, events.CommandEvent.TOPIC, events.CommandEvent.TYPE, 'FOO', 'BAR'] self.assertValid(val, names, commandOriginatorID.getLocalID(), destinationID.getLocalID()) self.assertTrue(event.getType(), events.EventTypes.COMMAND)
def dummyMetadata(): result = PropertySet() result.add("lsst.pipe.base.calibrate.cycleCount", 42) return result
def makeWcs(pixelScale, crPixPos, crValCoord, posAng=afwGeom.Angle(0.0), doFlipX=False, projection="TAN", radDecCSys="ICRS", equinox=2000): """Make a Wcs @param[in] pixelScale: desired scale, as sky/pixel, an afwGeom.Angle @param[in] crPixPos: crPix for WCS, using the LSST standard; a pair of floats @param[in] crValCoord: crVal for WCS (afwCoord.Coord) @param[in] posAng: position angle (afwGeom.Angle) @param[in] doFlipX: flip X axis? @param[in] projection: WCS projection (e.g. "TAN" or "STG") """ if len(projection) != 3: raise RuntimeError("projection=%r; must have length 3" % (projection,)) ctypeList = [("%-5s%3s" % (("RA", "DEC")[i], projection)).replace(" ", "-") for i in range(2)] ps = PropertySet() crPixFits = [ind + 1.0 for ind in crPixPos] # convert pix position to FITS standard crValDeg = crValCoord.getPosition(afwGeom.degrees) posAngRad = posAng.asRadians() pixelScaleDeg = pixelScale.asDegrees() print("pixelScale = %s deg = %s rad" % (pixelScale.asDegrees(), pixelScale.asRadians())) print("crPixPos = ", crPixPos) print("crVal = %s deg = %s rad" % (crValCoord.getPosition(afwGeom.degrees), crValCoord.getPosition(afwGeom.radians))) cdMat = np.array([[ math.cos(posAngRad), math.sin(posAngRad)], [-math.sin(posAngRad), math.cos(posAngRad)]], dtype=float) * pixelScaleDeg if doFlipX: cdMat[:,0] = -cdMat[:,0] for i in range(2): ip1 = i + 1 ps.add("CTYPE%1d" % (ip1,), ctypeList[i]) ps.add("CRPIX%1d" % (ip1,), crPixFits[i]) ps.add("CRVAL%1d" % (ip1,), crValDeg[i]) ps.add("RADESYS", radDecCSys) ps.add("EQUINOX", equinox) ps.add("CD1_1", cdMat[0, 0]) ps.add("CD2_1", cdMat[1, 0]) ps.add("CD1_2", cdMat[0, 1]) ps.add("CD2_2", cdMat[1, 1]) return afwImage.makeWcs(ps)
def testIllegalFilterableStatusEvent(self): testEnv = TestEnvironment() broker = testEnv.getBroker() topic = self.createTopicName("test_events_10_%s.A") # # send a test event, and wait to receive it # trans = events.EventTransmitter(broker, topic) root = PropertySet() root.set(events.StatusEvent.TOPIC, topic) root.set("myname","myname") root.set(events.StatusEvent.STATUS, "my special status") filter = PropertySet() filter.set("FOO", "bar") filter.set("PLOUGH", 123) filter.set("PLOVER.FIRST", 1) filter.set("PLOVER.SECOND", 2) eventSystem = events.EventSystem.getDefaultEventSystem(); locationID = eventSystem.createOriginatorId() event = events.StatusEvent(locationID, root, filter) self.assertRaises(ex.Exception, trans.publishEvent, event)
def getMetadata(keyValDict): md = PropertySet() for key, val in keyValDict.items(): md.set(key, val) return md
def createWcs(): metadata = PropertySet() metadata.set("SIMPLE", "T") metadata.set("BITPIX", -32) metadata.set("NAXIS", 2) metadata.set("NAXIS1", 1024) metadata.set("NAXIS2", 1153) metadata.set("RADECSYS", 'FK5') metadata.set("EQUINOX", 2000.) metadata.setDouble("CRVAL1", 215.604025685476) metadata.setDouble("CRVAL2", 53.1595451514076) metadata.setDouble("CRPIX1", 1109.99981456774) metadata.setDouble("CRPIX2", 560.018167811613) metadata.set("CTYPE1", 'RA---SIN') metadata.set("CTYPE2", 'DEC--SIN') metadata.setDouble("CD1_1", 5.10808596133527E-05) metadata.setDouble("CD1_2", 1.85579539217196E-07) metadata.setDouble("CD2_2", -5.10281493481982E-05) metadata.setDouble("CD2_1", -8.27440751733828E-07) return lsst.afw.image.makeWcs(metadata)
def makeJobOfficeStatusEvent(self, runId, status): props = PropertySet() props.set("STATUS",status) return StatusEvent(runId, self.originatorId, props)
def propertySetFromDict(keyValDict): """Make an lsst.daf.base.PropertySet from a dict of key: value""" metadata = PropertySet() for key, val in keyValDict.items(): metadata.set(key, val) return metadata
def makeJobStatusEvent(self, job, runId, status): props = PropertySet() props.set("identity", serializePolicy(job.getJobIdentity().toPolicy())) props.set("STATUS", status) props.set("name", job.getName()) return StatusEvent(runId, self.originatorId, props)