def testOneVisit(self):
# Create a list of clipboards, stage lists, and queue lists for each slice
clipboards = [Clipboard() for i in xrange(self.universeSize)]
stageLists = [[] for i in xrange(self.universeSize)]
queueLists = []
for i in xrange(self.universeSize):
queueList = [Queue() for j in xrange(len(self.stages) + 1)]
queueList[0].addDataset(clipboards[i])
queueLists.append(queueList)
# Create and initialize stages for each slice
for stageClass, policy, i in izip(self.stages, self.policies,
xrange(len(self.stages))):
for stageList, queueList, rank in izip(stageLists, queueLists,
xrange(self.universeSize)):
stage = stageClass(i, policy)
stage.setRun(self.runId)
stage.setUniverseSize(self.universeSize)
stage.setRank(rank - 1)
stage.initialize(queueList[i + 1], queueList[i])
stageList.append(stage)
# Create the association pipeline trigger event
dateObs = DateTime.now().mjd(DateTime.TAI)
triggerAssociationEvent = PropertySet()
triggerAssociationEvent.setInt('visitId', self.visitId)
triggerAssociationEvent.setDouble('dateObs', dateObs)
triggerAssociationEvent.setString('filter', self.filter)
triggerAssociationEvent.setDouble('ra', self.ra)
triggerAssociationEvent.setDouble('decl', self.dec)
# Create the event triggering the match against moving object predictions
triggerMatchMopsPredsEvent = PropertySet()
triggerMatchMopsPredsEvent.setInt('visitId', self.visitId)
# Add the events to clipboard of each stage
for clip in clipboards:
clip.put('triggerAssociationEvent', triggerAssociationEvent)
clip.put('triggerMatchMopsPredsEvent', triggerMatchMopsPredsEvent)
assert self.universeSize > 1
masterStageList = stageLists.pop(0)
# Run the pipeline (worker slices are run one after the other)
for masterStage, workerStages in izip(masterStageList,
izip(*stageLists)):
masterStage.preprocess()
map(lambda x: x.process(), workerStages)
masterStage.postprocess()
# Close log to avoid bogus memory-leak reports
log.Log.closeDefaultLog()
def createWcs():
metadata = PropertySet()
metadata.set("SIMPLE", "T")
metadata.set("BITPIX", -32)
metadata.set("NAXIS", 2)
metadata.set("NAXIS1", 1024)
metadata.set("NAXIS2", 1153)
metadata.set("RADECSYS", 'FK5')
metadata.set("EQUINOX", 2000.)
metadata.setDouble("CRVAL1", 215.604025685476)
metadata.setDouble("CRVAL2", 53.1595451514076)
metadata.setDouble("CRPIX1", 1109.99981456774)
metadata.setDouble("CRPIX2", 560.018167811613)
metadata.set("CTYPE1", 'RA---SIN')
metadata.set("CTYPE2", 'DEC--SIN')
metadata.setDouble("CD1_1", 5.10808596133527E-05)
metadata.setDouble("CD1_2", 1.85579539217196E-07)
metadata.setDouble("CD2_2", -5.10281493481982E-05)
metadata.setDouble("CD2_1", -8.27440751733828E-07)
return lsst.afw.image.makeWcs(metadata)
def createWcs():
metadata = PropertySet()
metadata.set("SIMPLE", "T")
metadata.set("BITPIX", -32)
metadata.set("NAXIS", 2)
metadata.set("NAXIS1", 1024)
metadata.set("NAXIS2", 1153)
metadata.set("RADESYS", 'FK5')
metadata.set("EQUINOX", 2000.)
metadata.setDouble("CRVAL1", 215.604025685476)
metadata.setDouble("CRVAL2", 53.1595451514076)
metadata.setDouble("CRPIX1", 1109.99981456774)
metadata.setDouble("CRPIX2", 560.018167811613)
metadata.set("CTYPE1", 'RA---SIN')
metadata.set("CTYPE2", 'DEC--SIN')
metadata.setDouble("CD1_1", 5.10808596133527E-05)
metadata.setDouble("CD1_2", 1.85579539217196E-07)
metadata.setDouble("CD2_2", -5.10281493481982E-05)
metadata.setDouble("CD2_1", -8.27440751733828E-07)
return makeSkyWcs(metadata)
def testOneVisit(self):
# Create a list of clipboards, stage lists, and queue lists for each slice
clipboards = [Clipboard() for i in xrange(self.universeSize)]
stageLists = [[] for i in xrange(self.universeSize)]
queueLists = []
for i in xrange(self.universeSize):
queueList = [Queue() for j in xrange(len(self.stages) + 1)]
queueList[0].addDataset(clipboards[i])
queueLists.append(queueList)
# Create and initialize stages for each slice
for stageClass, policy, i in izip(self.stages, self.policies, xrange(len(self.stages))):
for stageList, queueList, rank in izip(stageLists, queueLists, xrange(self.universeSize)):
stage = stageClass(i, policy)
stage.setRun(self.runId)
stage.setUniverseSize(self.universeSize)
stage.setRank(rank - 1)
stage.initialize(queueList[i+1], queueList[i])
stageList.append(stage)
# Create the association pipeline trigger event
dateObs = DateTime.now().mjd(DateTime.TAI)
triggerAssociationEvent = PropertySet()
triggerAssociationEvent.setInt('visitId', self.visitId)
triggerAssociationEvent.setDouble('dateObs', dateObs)
triggerAssociationEvent.setString('filter', self.filter)
triggerAssociationEvent.setDouble('ra', self.ra)
triggerAssociationEvent.setDouble('decl', self.dec)
# Create the event triggering the match against moving object predictions
triggerMatchMopsPredsEvent = PropertySet()
triggerMatchMopsPredsEvent.setInt('visitId', self.visitId)
# Add the events to clipboard of each stage
for clip in clipboards:
clip.put('triggerAssociationEvent', triggerAssociationEvent)
clip.put('triggerMatchMopsPredsEvent', triggerMatchMopsPredsEvent)
assert self.universeSize > 1
masterStageList = stageLists.pop(0)
# Run the pipeline (worker slices are run one after the other)
for masterStage, workerStages in izip(masterStageList, izip(*stageLists)):
masterStage.preprocess()
map(lambda x: x.process(), workerStages)
masterStage.postprocess()
# Close log to avoid bogus memory-leak reports
log.Log.closeDefaultLog()
def testFilterableSendEvent(self):
testEnv = TestEnvironment()
broker = testEnv.getBroker()
thisHost = platform.node()
topic = "test_events_filters_%s_%d" % (thisHost, os.getpid())
runId = "test_filters_runid"
recv = events.EventReceiver(broker, topic)
trans = events.EventTransmitter(broker, topic)
root = PropertySet()
DATE = "date"
DATE_VAL = "2007-07-01T14:28:32.546012"
root.set(DATE, DATE_VAL)
BLANK = "blank"
BLANK_VAL = ""
root.set(BLANK, BLANK_VAL)
PID = "pid"
PID_VAL = os.getpid()
root.setInt(PID, PID_VAL)
HOST = "host"
HOST_VAL = "lsstcorp.org"
root.set(HOST, HOST_VAL)
IP = "ip"
IP_VAL = "1.2.3.4"
root.set(IP, IP_VAL)
EVNT = "evnt"
EVNT_VAL = "test"
root.set(EVNT, EVNT_VAL)
MISC1 = "misc1"
MISC1_VAL = "data 1"
root.set(MISC1, MISC1_VAL)
MISC2 = "misc2"
MISC2_VAL = "data 2"
root.set(MISC2, MISC2_VAL)
MISC3 = "misc3"
MISC3_VAL = ""
root.set(MISC3, MISC3_VAL)
DATA = "data"
DATA_VAL = 3.14
root.setDouble(DATA, DATA_VAL)
filterable = PropertySet()
filterable.set("FOO", "bar")
filterable.set("XYZZY", 123)
filterable.set("PLOUGH", 0.867)
event = events.Event(runId, root, filterable)
trans.publishEvent(event)
val = recv.receiveEvent()
self.assertIsNotNone(val)
ps = val.getPropertySet()
self.assertEqual(ps.get(DATE), DATE_VAL)
self.assertEqual(ps.get(BLANK), BLANK_VAL)
self.assertEqual(ps.get(PID), PID_VAL)
self.assertEqual(ps.get(HOST), HOST_VAL)
self.assertEqual(ps.get(IP), IP_VAL)
self.assertEqual(ps.get(EVNT), EVNT_VAL)
self.assertEqual(ps.get(MISC1), MISC1_VAL)
self.assertEqual(ps.get(MISC2), MISC2_VAL)
self.assertEqual(ps.get(MISC3), MISC3_VAL)
self.assertEqual(ps.get(DATA), DATA_VAL)
self.assertGreater(ps.get(events.Event.EVENTTIME), 0)
self.assertGreater(ps.get(events.Event.PUBTIME), 0)
self.assertEqual(ps.get(events.Event.RUNID), runId)
self.assertEqual(ps.get(events.Event.STATUS), "unknown")
self.assertEqual(ps.get(events.Event.TOPIC), topic)
self.assertEqual(ps.get(events.Event.TYPE), events.EventTypes.EVENT)
self.assertEqual(ps.get("FOO"), "bar")
self.assertEqual(ps.get("XYZZY"), 123)
self.assertEqual(ps.get("PLOUGH"), 0.867)
names = val.getFilterablePropertyNames()
values = [events.Event.EVENTTIME, 'FOO', 'PLOUGH', events.Event.PUBTIME, events.Event.RUNID, events.Event.STATUS,
events.Event.TOPIC, events.Event.TYPE, 'XYZZY']
for x in values:
self.assertTrue(x in names)
#
# wait a short time to receive an event. none was sent, so we should
# time out and confirm that we didn't get anything
#
val = recv.receiveEvent(1000)
self.assertIsNone(val)
def testSendEvent(self):
"""Send an Event"""
testEnv = EventsEnvironment()
self.thisHost = platform.node()
self.broker = testEnv.getBroker()
topic = "test_events_3_%s_%d" % (self.thisHost, os.getpid())
runID = "test3_runid"
recv = events.EventReceiver(self.broker, topic)
trans = events.EventTransmitter(self.broker, topic)
DATE = "date"
DATE_VAL = "2007-07-01T14:28:32.546012"
BLANK = "blank"
BLANK_VAL = ""
PID = "pid"
PID_VAL = os.getpid()
HOST = "host"
HOST_VAL = "lsstcorp.org"
IP = "ip"
IP_VAL = "1.2.3.4"
EVNT = "evnt"
EVNT_VAL = "test"
MISC1 = "misc1"
MISC1_VAL = "data 1"
MISC2 = "misc2"
MISC2_VAL = "data 2"
MISC3 = "misc3"
MISC3_VAL = ""
DATA = "data"
DATA_VAL = 3.14
root = PropertySet()
root.set(DATE, DATE_VAL)
root.set(BLANK, BLANK_VAL)
root.setInt(PID, PID_VAL)
root.set(HOST, HOST_VAL)
root.set(IP, IP_VAL)
root.set(EVNT, EVNT_VAL)
root.set(MISC1, MISC1_VAL)
root.set(MISC2, MISC2_VAL)
root.set(MISC3, MISC3_VAL)
root.setDouble(DATA, DATA_VAL)
event = events.Event(runID, root)
trans.publishEvent(event)
val = recv.receiveEvent()
self.assertIsNotNone(val)
# check the validity of all sent values
ps = val.getPropertySet()
self.assertEqual(ps.get(DATE), DATE_VAL)
self.assertEqual(ps.get(BLANK), BLANK_VAL)
self.assertEqual(ps.get(PID), PID_VAL)
self.assertEqual(ps.get(HOST), HOST_VAL)
self.assertEqual(ps.get(IP), IP_VAL)
self.assertEqual(ps.get(EVNT), EVNT_VAL)
self.assertEqual(ps.get(MISC1), MISC1_VAL)
self.assertEqual(ps.get(MISC2), MISC2_VAL)
self.assertEqual(ps.get(MISC3), MISC3_VAL)
self.assertEqual(ps.get(DATA), DATA_VAL)
self.assertGreater(ps.get(events.Event.EVENTTIME), 0)
self.assertGreater(ps.get(events.Event.PUBTIME), 0)
self.assertEqual(ps.get(events.Event.RUNID), runID)
self.assertEqual(ps.get(events.Event.STATUS), "unknown")
self.assertEqual(ps.get(events.Event.TOPIC), topic)
self.assertEqual(ps.get(events.Event.TYPE), events.EventTypes.EVENT)
#
# wait a short time to receive an event. none was sent, so we should
# time out and confirm that we didn't get anything
#
val = recv.receiveEvent(1000)
self.assertIsNone(val)
def testFilterableSendEvent(self):
testEnv = EventsEnvironment()
broker = testEnv.getBroker()
thisHost = platform.node()
topic = "test_events_filters_%s_%d" % (thisHost, os.getpid())
runId = "test_filters_runid"
recv = events.EventReceiver(broker, topic)
trans = events.EventTransmitter(broker, topic)
root = PropertySet()
DATE = "date"
DATE_VAL = "2007-07-01T14:28:32.546012"
root.set(DATE, DATE_VAL)
BLANK = "blank"
BLANK_VAL = ""
root.set(BLANK, BLANK_VAL)
PID = "pid"
PID_VAL = os.getpid()
root.setInt(PID, PID_VAL)
HOST = "host"
HOST_VAL = "lsstcorp.org"
root.set(HOST, HOST_VAL)
IP = "ip"
IP_VAL = "1.2.3.4"
root.set(IP, IP_VAL)
EVNT = "evnt"
EVNT_VAL = "test"
root.set(EVNT, EVNT_VAL)
MISC1 = "misc1"
MISC1_VAL = "data 1"
root.set(MISC1, MISC1_VAL)
MISC2 = "misc2"
MISC2_VAL = "data 2"
root.set(MISC2, MISC2_VAL)
MISC3 = "misc3"
MISC3_VAL = ""
root.set(MISC3, MISC3_VAL)
DATA = "data"
DATA_VAL = 3.14
root.setDouble(DATA, DATA_VAL)
filterable = PropertySet()
filterable.set("FOO", "bar")
filterable.set("XYZZY", 123)
filterable.set("PLOUGH", 0.867)
event = events.Event(runId, root, filterable)
trans.publishEvent(event)
val = recv.receiveEvent()
self.assertIsNotNone(val)
ps = val.getPropertySet()
self.assertEqual(ps.get(DATE), DATE_VAL)
self.assertEqual(ps.get(BLANK), BLANK_VAL)
self.assertEqual(ps.get(PID), PID_VAL)
self.assertEqual(ps.get(HOST), HOST_VAL)
self.assertEqual(ps.get(IP), IP_VAL)
self.assertEqual(ps.get(EVNT), EVNT_VAL)
self.assertEqual(ps.get(MISC1), MISC1_VAL)
self.assertEqual(ps.get(MISC2), MISC2_VAL)
self.assertEqual(ps.get(MISC3), MISC3_VAL)
self.assertEqual(ps.get(DATA), DATA_VAL)
self.assertGreater(ps.get(events.Event.EVENTTIME), 0)
self.assertGreater(ps.get(events.Event.PUBTIME), 0)
self.assertEqual(ps.get(events.Event.RUNID), runId)
self.assertEqual(ps.get(events.Event.STATUS), "unknown")
self.assertEqual(ps.get(events.Event.TOPIC), topic)
self.assertEqual(ps.get(events.Event.TYPE), events.EventTypes.EVENT)
self.assertEqual(ps.get("FOO"), "bar")
self.assertEqual(ps.get("XYZZY"), 123)
self.assertEqual(ps.get("PLOUGH"), 0.867)
names = val.getFilterablePropertyNames()
values = [
events.Event.EVENTTIME, 'FOO', 'PLOUGH', events.Event.PUBTIME,
events.Event.RUNID, events.Event.STATUS, events.Event.TOPIC,
events.Event.TYPE, 'XYZZY'
]
for x in values:
self.assertTrue(x in names)
#
# wait a short time to receive an event. none was sent, so we should
# time out and confirm that we didn't get anything
#
val = recv.receiveEvent(1000)
self.assertIsNone(val)
def testSendEvent(self):
"""Send an Event"""
testEnv = EventsEnvironment()
self.thisHost = platform.node()
self.broker = testEnv.getBroker()
topic = "test_events_3_%s_%d" % (self.thisHost, os.getpid())
runID = "test3_runid"
recv = events.EventReceiver(self.broker, topic)
trans = events.EventTransmitter(self.broker, topic)
DATE = "date"
DATE_VAL = "2007-07-01T14:28:32.546012"
BLANK = "blank"
BLANK_VAL = ""
PID = "pid"
PID_VAL = os.getpid()
HOST = "host"
HOST_VAL = "lsstcorp.org"
IP = "ip"
IP_VAL = "1.2.3.4"
EVNT = "evnt"
EVNT_VAL = "test"
MISC1 = "misc1"
MISC1_VAL = "data 1"
MISC2 = "misc2"
MISC2_VAL = "data 2"
MISC3 = "misc3"
MISC3_VAL = ""
DATA = "data"
DATA_VAL = 3.14
root = PropertySet()
root.set(DATE, DATE_VAL)
root.set(BLANK, BLANK_VAL)
root.setInt(PID, PID_VAL)
root.set(HOST, HOST_VAL)
root.set(IP, IP_VAL)
root.set(EVNT, EVNT_VAL)
root.set(MISC1, MISC1_VAL)
root.set(MISC2, MISC2_VAL)
root.set(MISC3, MISC3_VAL)
root.setDouble(DATA, DATA_VAL)
event = events.Event(runID, root)
trans.publishEvent(event)
val = recv.receiveEvent()
self.assertIsNotNone(val)
# check the validity of all sent values
ps = val.getPropertySet()
self.assertEqual(ps.get(DATE), DATE_VAL)
self.assertEqual(ps.get(BLANK), BLANK_VAL)
self.assertEqual(ps.get(PID), PID_VAL)
self.assertEqual(ps.get(HOST), HOST_VAL)
self.assertEqual(ps.get(IP), IP_VAL)
self.assertEqual(ps.get(EVNT), EVNT_VAL)
self.assertEqual(ps.get(MISC1), MISC1_VAL)
self.assertEqual(ps.get(MISC2), MISC2_VAL)
self.assertEqual(ps.get(MISC3), MISC3_VAL)
self.assertEqual(ps.get(DATA), DATA_VAL)
self.assertGreater(ps.get(events.Event.EVENTTIME), 0)
self.assertGreater(ps.get(events.Event.PUBTIME), 0)
self.assertEqual(ps.get(events.Event.RUNID), runID)
self.assertEqual(ps.get(events.Event.STATUS), "unknown")
self.assertEqual(ps.get(events.Event.TOPIC), topic)
self.assertEqual(ps.get(events.Event.TYPE), events.EventTypes.EVENT)
#
# wait a short time to receive an event. none was sent, so we should
# time out and confirm that we didn't get anything
#
val = recv.receiveEvent(1000)
self.assertIsNone(val)
