def testOneVisit(self): # Create a list of clipboards, stage lists, and queue lists for each slice clipboards = [Clipboard() for i in xrange(self.universeSize)] stageLists = [[] for i in xrange(self.universeSize)] queueLists = [] for i in xrange(self.universeSize): queueList = [Queue() for j in xrange(len(self.stages) + 1)] queueList[0].addDataset(clipboards[i]) queueLists.append(queueList) # Create and initialize stages for each slice for stageClass, policy, i in izip(self.stages, self.policies, xrange(len(self.stages))): for stageList, queueList, rank in izip(stageLists, queueLists, xrange(self.universeSize)): stage = stageClass(i, policy) stage.setRun(self.runId) stage.setUniverseSize(self.universeSize) stage.setRank(rank - 1) stage.initialize(queueList[i + 1], queueList[i]) stageList.append(stage) # Create the association pipeline trigger event dateObs = DateTime.now().mjd(DateTime.TAI) triggerAssociationEvent = PropertySet() triggerAssociationEvent.setInt('visitId', self.visitId) triggerAssociationEvent.setDouble('dateObs', dateObs) triggerAssociationEvent.setString('filter', self.filter) triggerAssociationEvent.setDouble('ra', self.ra) triggerAssociationEvent.setDouble('decl', self.dec) # Create the event triggering the match against moving object predictions triggerMatchMopsPredsEvent = PropertySet() triggerMatchMopsPredsEvent.setInt('visitId', self.visitId) # Add the events to clipboard of each stage for clip in clipboards: clip.put('triggerAssociationEvent', triggerAssociationEvent) clip.put('triggerMatchMopsPredsEvent', triggerMatchMopsPredsEvent) assert self.universeSize > 1 masterStageList = stageLists.pop(0) # Run the pipeline (worker slices are run one after the other) for masterStage, workerStages in izip(masterStageList, izip(*stageLists)): masterStage.preprocess() map(lambda x: x.process(), workerStages) masterStage.postprocess() # Close log to avoid bogus memory-leak reports log.Log.closeDefaultLog()
def createWcs(): metadata = PropertySet() metadata.set("SIMPLE", "T") metadata.set("BITPIX", -32) metadata.set("NAXIS", 2) metadata.set("NAXIS1", 1024) metadata.set("NAXIS2", 1153) metadata.set("RADECSYS", 'FK5') metadata.set("EQUINOX", 2000.) metadata.setDouble("CRVAL1", 215.604025685476) metadata.setDouble("CRVAL2", 53.1595451514076) metadata.setDouble("CRPIX1", 1109.99981456774) metadata.setDouble("CRPIX2", 560.018167811613) metadata.set("CTYPE1", 'RA---SIN') metadata.set("CTYPE2", 'DEC--SIN') metadata.setDouble("CD1_1", 5.10808596133527E-05) metadata.setDouble("CD1_2", 1.85579539217196E-07) metadata.setDouble("CD2_2", -5.10281493481982E-05) metadata.setDouble("CD2_1", -8.27440751733828E-07) return lsst.afw.image.makeWcs(metadata)
def createWcs(): metadata = PropertySet() metadata.set("SIMPLE", "T") metadata.set("BITPIX", -32) metadata.set("NAXIS", 2) metadata.set("NAXIS1", 1024) metadata.set("NAXIS2", 1153) metadata.set("RADESYS", 'FK5') metadata.set("EQUINOX", 2000.) metadata.setDouble("CRVAL1", 215.604025685476) metadata.setDouble("CRVAL2", 53.1595451514076) metadata.setDouble("CRPIX1", 1109.99981456774) metadata.setDouble("CRPIX2", 560.018167811613) metadata.set("CTYPE1", 'RA---SIN') metadata.set("CTYPE2", 'DEC--SIN') metadata.setDouble("CD1_1", 5.10808596133527E-05) metadata.setDouble("CD1_2", 1.85579539217196E-07) metadata.setDouble("CD2_2", -5.10281493481982E-05) metadata.setDouble("CD2_1", -8.27440751733828E-07) return makeSkyWcs(metadata)
def testOneVisit(self): # Create a list of clipboards, stage lists, and queue lists for each slice clipboards = [Clipboard() for i in xrange(self.universeSize)] stageLists = [[] for i in xrange(self.universeSize)] queueLists = [] for i in xrange(self.universeSize): queueList = [Queue() for j in xrange(len(self.stages) + 1)] queueList[0].addDataset(clipboards[i]) queueLists.append(queueList) # Create and initialize stages for each slice for stageClass, policy, i in izip(self.stages, self.policies, xrange(len(self.stages))): for stageList, queueList, rank in izip(stageLists, queueLists, xrange(self.universeSize)): stage = stageClass(i, policy) stage.setRun(self.runId) stage.setUniverseSize(self.universeSize) stage.setRank(rank - 1) stage.initialize(queueList[i+1], queueList[i]) stageList.append(stage) # Create the association pipeline trigger event dateObs = DateTime.now().mjd(DateTime.TAI) triggerAssociationEvent = PropertySet() triggerAssociationEvent.setInt('visitId', self.visitId) triggerAssociationEvent.setDouble('dateObs', dateObs) triggerAssociationEvent.setString('filter', self.filter) triggerAssociationEvent.setDouble('ra', self.ra) triggerAssociationEvent.setDouble('decl', self.dec) # Create the event triggering the match against moving object predictions triggerMatchMopsPredsEvent = PropertySet() triggerMatchMopsPredsEvent.setInt('visitId', self.visitId) # Add the events to clipboard of each stage for clip in clipboards: clip.put('triggerAssociationEvent', triggerAssociationEvent) clip.put('triggerMatchMopsPredsEvent', triggerMatchMopsPredsEvent) assert self.universeSize > 1 masterStageList = stageLists.pop(0) # Run the pipeline (worker slices are run one after the other) for masterStage, workerStages in izip(masterStageList, izip(*stageLists)): masterStage.preprocess() map(lambda x: x.process(), workerStages) masterStage.postprocess() # Close log to avoid bogus memory-leak reports log.Log.closeDefaultLog()
def testFilterableSendEvent(self): testEnv = TestEnvironment() broker = testEnv.getBroker() thisHost = platform.node() topic = "test_events_filters_%s_%d" % (thisHost, os.getpid()) runId = "test_filters_runid" recv = events.EventReceiver(broker, topic) trans = events.EventTransmitter(broker, topic) root = PropertySet() DATE = "date" DATE_VAL = "2007-07-01T14:28:32.546012" root.set(DATE, DATE_VAL) BLANK = "blank" BLANK_VAL = "" root.set(BLANK, BLANK_VAL) PID = "pid" PID_VAL = os.getpid() root.setInt(PID, PID_VAL) HOST = "host" HOST_VAL = "lsstcorp.org" root.set(HOST, HOST_VAL) IP = "ip" IP_VAL = "1.2.3.4" root.set(IP, IP_VAL) EVNT = "evnt" EVNT_VAL = "test" root.set(EVNT, EVNT_VAL) MISC1 = "misc1" MISC1_VAL = "data 1" root.set(MISC1, MISC1_VAL) MISC2 = "misc2" MISC2_VAL = "data 2" root.set(MISC2, MISC2_VAL) MISC3 = "misc3" MISC3_VAL = "" root.set(MISC3, MISC3_VAL) DATA = "data" DATA_VAL = 3.14 root.setDouble(DATA, DATA_VAL) filterable = PropertySet() filterable.set("FOO", "bar") filterable.set("XYZZY", 123) filterable.set("PLOUGH", 0.867) event = events.Event(runId, root, filterable) trans.publishEvent(event) val = recv.receiveEvent() self.assertIsNotNone(val) ps = val.getPropertySet() self.assertEqual(ps.get(DATE), DATE_VAL) self.assertEqual(ps.get(BLANK), BLANK_VAL) self.assertEqual(ps.get(PID), PID_VAL) self.assertEqual(ps.get(HOST), HOST_VAL) self.assertEqual(ps.get(IP), IP_VAL) self.assertEqual(ps.get(EVNT), EVNT_VAL) self.assertEqual(ps.get(MISC1), MISC1_VAL) self.assertEqual(ps.get(MISC2), MISC2_VAL) self.assertEqual(ps.get(MISC3), MISC3_VAL) self.assertEqual(ps.get(DATA), DATA_VAL) self.assertGreater(ps.get(events.Event.EVENTTIME), 0) self.assertGreater(ps.get(events.Event.PUBTIME), 0) self.assertEqual(ps.get(events.Event.RUNID), runId) self.assertEqual(ps.get(events.Event.STATUS), "unknown") self.assertEqual(ps.get(events.Event.TOPIC), topic) self.assertEqual(ps.get(events.Event.TYPE), events.EventTypes.EVENT) self.assertEqual(ps.get("FOO"), "bar") self.assertEqual(ps.get("XYZZY"), 123) self.assertEqual(ps.get("PLOUGH"), 0.867) names = val.getFilterablePropertyNames() values = [events.Event.EVENTTIME, 'FOO', 'PLOUGH', events.Event.PUBTIME, events.Event.RUNID, events.Event.STATUS, events.Event.TOPIC, events.Event.TYPE, 'XYZZY'] for x in values: self.assertTrue(x in names) # # wait a short time to receive an event. none was sent, so we should # time out and confirm that we didn't get anything # val = recv.receiveEvent(1000) self.assertIsNone(val)
def testSendEvent(self): """Send an Event""" testEnv = EventsEnvironment() self.thisHost = platform.node() self.broker = testEnv.getBroker() topic = "test_events_3_%s_%d" % (self.thisHost, os.getpid()) runID = "test3_runid" recv = events.EventReceiver(self.broker, topic) trans = events.EventTransmitter(self.broker, topic) DATE = "date" DATE_VAL = "2007-07-01T14:28:32.546012" BLANK = "blank" BLANK_VAL = "" PID = "pid" PID_VAL = os.getpid() HOST = "host" HOST_VAL = "lsstcorp.org" IP = "ip" IP_VAL = "1.2.3.4" EVNT = "evnt" EVNT_VAL = "test" MISC1 = "misc1" MISC1_VAL = "data 1" MISC2 = "misc2" MISC2_VAL = "data 2" MISC3 = "misc3" MISC3_VAL = "" DATA = "data" DATA_VAL = 3.14 root = PropertySet() root.set(DATE, DATE_VAL) root.set(BLANK, BLANK_VAL) root.setInt(PID, PID_VAL) root.set(HOST, HOST_VAL) root.set(IP, IP_VAL) root.set(EVNT, EVNT_VAL) root.set(MISC1, MISC1_VAL) root.set(MISC2, MISC2_VAL) root.set(MISC3, MISC3_VAL) root.setDouble(DATA, DATA_VAL) event = events.Event(runID, root) trans.publishEvent(event) val = recv.receiveEvent() self.assertIsNotNone(val) # check the validity of all sent values ps = val.getPropertySet() self.assertEqual(ps.get(DATE), DATE_VAL) self.assertEqual(ps.get(BLANK), BLANK_VAL) self.assertEqual(ps.get(PID), PID_VAL) self.assertEqual(ps.get(HOST), HOST_VAL) self.assertEqual(ps.get(IP), IP_VAL) self.assertEqual(ps.get(EVNT), EVNT_VAL) self.assertEqual(ps.get(MISC1), MISC1_VAL) self.assertEqual(ps.get(MISC2), MISC2_VAL) self.assertEqual(ps.get(MISC3), MISC3_VAL) self.assertEqual(ps.get(DATA), DATA_VAL) self.assertGreater(ps.get(events.Event.EVENTTIME), 0) self.assertGreater(ps.get(events.Event.PUBTIME), 0) self.assertEqual(ps.get(events.Event.RUNID), runID) self.assertEqual(ps.get(events.Event.STATUS), "unknown") self.assertEqual(ps.get(events.Event.TOPIC), topic) self.assertEqual(ps.get(events.Event.TYPE), events.EventTypes.EVENT) # # wait a short time to receive an event. none was sent, so we should # time out and confirm that we didn't get anything # val = recv.receiveEvent(1000) self.assertIsNone(val)
def testFilterableSendEvent(self): testEnv = EventsEnvironment() broker = testEnv.getBroker() thisHost = platform.node() topic = "test_events_filters_%s_%d" % (thisHost, os.getpid()) runId = "test_filters_runid" recv = events.EventReceiver(broker, topic) trans = events.EventTransmitter(broker, topic) root = PropertySet() DATE = "date" DATE_VAL = "2007-07-01T14:28:32.546012" root.set(DATE, DATE_VAL) BLANK = "blank" BLANK_VAL = "" root.set(BLANK, BLANK_VAL) PID = "pid" PID_VAL = os.getpid() root.setInt(PID, PID_VAL) HOST = "host" HOST_VAL = "lsstcorp.org" root.set(HOST, HOST_VAL) IP = "ip" IP_VAL = "1.2.3.4" root.set(IP, IP_VAL) EVNT = "evnt" EVNT_VAL = "test" root.set(EVNT, EVNT_VAL) MISC1 = "misc1" MISC1_VAL = "data 1" root.set(MISC1, MISC1_VAL) MISC2 = "misc2" MISC2_VAL = "data 2" root.set(MISC2, MISC2_VAL) MISC3 = "misc3" MISC3_VAL = "" root.set(MISC3, MISC3_VAL) DATA = "data" DATA_VAL = 3.14 root.setDouble(DATA, DATA_VAL) filterable = PropertySet() filterable.set("FOO", "bar") filterable.set("XYZZY", 123) filterable.set("PLOUGH", 0.867) event = events.Event(runId, root, filterable) trans.publishEvent(event) val = recv.receiveEvent() self.assertIsNotNone(val) ps = val.getPropertySet() self.assertEqual(ps.get(DATE), DATE_VAL) self.assertEqual(ps.get(BLANK), BLANK_VAL) self.assertEqual(ps.get(PID), PID_VAL) self.assertEqual(ps.get(HOST), HOST_VAL) self.assertEqual(ps.get(IP), IP_VAL) self.assertEqual(ps.get(EVNT), EVNT_VAL) self.assertEqual(ps.get(MISC1), MISC1_VAL) self.assertEqual(ps.get(MISC2), MISC2_VAL) self.assertEqual(ps.get(MISC3), MISC3_VAL) self.assertEqual(ps.get(DATA), DATA_VAL) self.assertGreater(ps.get(events.Event.EVENTTIME), 0) self.assertGreater(ps.get(events.Event.PUBTIME), 0) self.assertEqual(ps.get(events.Event.RUNID), runId) self.assertEqual(ps.get(events.Event.STATUS), "unknown") self.assertEqual(ps.get(events.Event.TOPIC), topic) self.assertEqual(ps.get(events.Event.TYPE), events.EventTypes.EVENT) self.assertEqual(ps.get("FOO"), "bar") self.assertEqual(ps.get("XYZZY"), 123) self.assertEqual(ps.get("PLOUGH"), 0.867) names = val.getFilterablePropertyNames() values = [ events.Event.EVENTTIME, 'FOO', 'PLOUGH', events.Event.PUBTIME, events.Event.RUNID, events.Event.STATUS, events.Event.TOPIC, events.Event.TYPE, 'XYZZY' ] for x in values: self.assertTrue(x in names) # # wait a short time to receive an event. none was sent, so we should # time out and confirm that we didn't get anything # val = recv.receiveEvent(1000) self.assertIsNone(val)