def process(self):
clipboard = self.inputQueue.getNextDataset()
eventName = self._policy.get("inputEvent")
event = clipboard.get(eventName)
visitId = event.get("visitId")
exposureId = event.get("exposureId")
ccdId = clipboard.get("ccdId")
ampId = clipboard.get("ampId")
fpaExposureId = (long(visitId) << 1) + exposureId
ccdExposureId = (fpaExposureId << 8) + ccdId
ampExposureId = (ccdExposureId << 6) + ampId
clipboard.put("visitId", visitId)
exposureMetadata = PropertySet()
exposureMetadata.setInt("filterId",
self.lookupFilterId(event.get("filter")))
exposureMetadata.setLongLong("fpaExposureId", fpaExposureId)
exposureMetadata.setLongLong("ccdExposureId", ccdExposureId)
exposureMetadata.setLongLong("ampExposureId", ampExposureId)
clipboard.put("exposureMetadata" + str(exposureId), exposureMetadata)
self.outputQueue.addDataset(clipboard)
def process(self):
clipboard = self.inputQueue.getNextDataset()
eventName = self._policy.get("inputEvent")
event = clipboard.get(eventName)
visitId = event.get("visitId")
exposureId = event.get("exposureId")
ccdId = clipboard.get("ccdId")
ampId = clipboard.get("ampId")
fpaExposureId = (long(visitId) << 1) + exposureId
ccdExposureId = (fpaExposureId << 8) + ccdId
ampExposureId = (ccdExposureId << 6) + ampId
clipboard.put("visitId", visitId)
exposureMetadata = PropertySet()
exposureMetadata.setInt("filterId",
self.lookupFilterId(event.get("filter")))
exposureMetadata.setLongLong("fpaExposureId", fpaExposureId)
exposureMetadata.setLongLong("ccdExposureId", ccdExposureId)
exposureMetadata.setLongLong("ampExposureId", ampExposureId)
clipboard.put("exposureMetadata" + str(exposureId), exposureMetadata)
self.outputQueue.addDataset(clipboard)
def preprocess(self):
self.activeClipboard = self.inputQueue.getNextDataset()
eventName = self._policy.get("inputEvent")
event = self.activeClipboard.get(eventName)
visitId = event.get("visitId")
exposureId = event.get("exposureId")
fpaExposureId = (long(visitId) << 1) + exposureId
visit = PropertySet()
visit.setInt("visitId", visitId)
visit.setLongLong("exposureId", fpaExposureId)
self.activeClipboard.put("visit" + str(exposureId), visit)
rawFpaExposure = PropertySet()
rawFpaExposure.setLongLong("rawFPAExposureId", fpaExposureId)
rawFpaExposure.set("ra", event.get("ra"))
rawFpaExposure.set("decl", event.get("decl"))
rawFpaExposure.set("filterId",
self.lookupFilterId(event.get("filter")))
rawFpaExposure.set("equinox", event.get("equinox"))
rawFpaExposure.set("dateObs", DateTime(event.get("dateObs")))
rawFpaExposure.set("mjdObs", DateTime(event.get("dateObs")).mjd())
rawFpaExposure.set("expTime", event.get("expTime"))
rawFpaExposure.set("airmass", event.get("airmass"))
self.activeClipboard.put("fpaExposure" + str(exposureId), rawFpaExposure)
def preprocess(self):
self.activeClipboard = self.inputQueue.getNextDataset()
eventName = self._policy.get("inputEvent")
event = self.activeClipboard.get(eventName)
visitId = event.get("visitId")
exposureId = event.get("exposureId")
fpaExposureId = (long(visitId) << 1) + exposureId
visit = PropertySet()
visit.setInt("visitId", visitId)
visit.setLongLong("exposureId", fpaExposureId)
self.activeClipboard.put("visit" + str(exposureId), visit)
rawFpaExposure = PropertySet()
rawFpaExposure.setLongLong("rawFPAExposureId", fpaExposureId)
rawFpaExposure.set("ra", event.get("ra"))
rawFpaExposure.set("decl", event.get("decl"))
rawFpaExposure.set("filterId",
self.lookupFilterId(event.get("filter")))
rawFpaExposure.set("equinox", event.get("equinox"))
rawFpaExposure.set("dateObs", DateTime(event.get("dateObs")))
rawFpaExposure.set("mjdObs", DateTime(event.get("dateObs")).mjd())
rawFpaExposure.set("expTime", event.get("expTime"))
rawFpaExposure.set("airmass", event.get("airmass"))
self.activeClipboard.put("fpaExposure" + str(exposureId),
rawFpaExposure)
def stopWorkflow(self, urgency):
log.debug("VanillaCondorWorkflowMonitor:stopWorkflow")
transmit = events.EventTransmitter(self._eventBrokerHost, self._shutdownTopic)
root = PropertySet()
root.setInt("level",urgency)
root.setString("STATUS","shut things down")
event = events.CommandEvent(self.runid, self.originatorId, 0, root)
transmit.publishEvent(event)
def testOneVisit(self):
# Create a list of clipboards, stage lists, and queue lists for each slice
clipboards = [Clipboard() for i in xrange(self.universeSize)]
stageLists = [[] for i in xrange(self.universeSize)]
queueLists = []
for i in xrange(self.universeSize):
queueList = [Queue() for j in xrange(len(self.stages) + 1)]
queueList[0].addDataset(clipboards[i])
queueLists.append(queueList)
# Create and initialize stages for each slice
for stageClass, policy, i in izip(self.stages, self.policies,
xrange(len(self.stages))):
for stageList, queueList, rank in izip(stageLists, queueLists,
xrange(self.universeSize)):
stage = stageClass(i, policy)
stage.setRun(self.runId)
stage.setUniverseSize(self.universeSize)
stage.setRank(rank - 1)
stage.initialize(queueList[i + 1], queueList[i])
stageList.append(stage)
# Create the association pipeline trigger event
dateObs = DateTime.now().mjd(DateTime.TAI)
triggerAssociationEvent = PropertySet()
triggerAssociationEvent.setInt('visitId', self.visitId)
triggerAssociationEvent.setDouble('dateObs', dateObs)
triggerAssociationEvent.setString('filter', self.filter)
triggerAssociationEvent.setDouble('ra', self.ra)
triggerAssociationEvent.setDouble('decl', self.dec)
# Create the event triggering the match against moving object predictions
triggerMatchMopsPredsEvent = PropertySet()
triggerMatchMopsPredsEvent.setInt('visitId', self.visitId)
# Add the events to clipboard of each stage
for clip in clipboards:
clip.put('triggerAssociationEvent', triggerAssociationEvent)
clip.put('triggerMatchMopsPredsEvent', triggerMatchMopsPredsEvent)
assert self.universeSize > 1
masterStageList = stageLists.pop(0)
# Run the pipeline (worker slices are run one after the other)
for masterStage, workerStages in izip(masterStageList,
izip(*stageLists)):
masterStage.preprocess()
map(lambda x: x.process(), workerStages)
masterStage.postprocess()
# Close log to avoid bogus memory-leak reports
log.Log.closeDefaultLog()
def testOneVisit(self):
# Create a list of clipboards, stage lists, and queue lists for each slice
clipboards = [Clipboard() for i in xrange(self.universeSize)]
stageLists = [[] for i in xrange(self.universeSize)]
queueLists = []
for i in xrange(self.universeSize):
queueList = [Queue() for j in xrange(len(self.stages) + 1)]
queueList[0].addDataset(clipboards[i])
queueLists.append(queueList)
# Create and initialize stages for each slice
for stageClass, policy, i in izip(self.stages, self.policies, xrange(len(self.stages))):
for stageList, queueList, rank in izip(stageLists, queueLists, xrange(self.universeSize)):
stage = stageClass(i, policy)
stage.setRun(self.runId)
stage.setUniverseSize(self.universeSize)
stage.setRank(rank - 1)
stage.initialize(queueList[i+1], queueList[i])
stageList.append(stage)
# Create the association pipeline trigger event
dateObs = DateTime.now().mjd(DateTime.TAI)
triggerAssociationEvent = PropertySet()
triggerAssociationEvent.setInt('visitId', self.visitId)
triggerAssociationEvent.setDouble('dateObs', dateObs)
triggerAssociationEvent.setString('filter', self.filter)
triggerAssociationEvent.setDouble('ra', self.ra)
triggerAssociationEvent.setDouble('decl', self.dec)
# Create the event triggering the match against moving object predictions
triggerMatchMopsPredsEvent = PropertySet()
triggerMatchMopsPredsEvent.setInt('visitId', self.visitId)
# Add the events to clipboard of each stage
for clip in clipboards:
clip.put('triggerAssociationEvent', triggerAssociationEvent)
clip.put('triggerMatchMopsPredsEvent', triggerMatchMopsPredsEvent)
assert self.universeSize > 1
masterStageList = stageLists.pop(0)
# Run the pipeline (worker slices are run one after the other)
for masterStage, workerStages in izip(masterStageList, izip(*stageLists)):
masterStage.preprocess()
map(lambda x: x.process(), workerStages)
masterStage.postprocess()
# Close log to avoid bogus memory-leak reports
log.Log.closeDefaultLog()
def finalMessageReceived(self, propSet):
log = propSet.get("LOGGER")
if log != "orca.control":
return False
status = propSet.get("STATUS")
if status != "eol":
return False
eventSystem = events.EventSystem.getDefaultEventSystem()
id = eventSystem.createOriginatorId()
root = PropertySet()
root.setString("logger.status", "done")
root.setInt("logger.pid", os.getpid())
event = events.StatusEvent(self.runid, id, root)
self.transmitter.publishEvent(event)
return True
def testSubst(self):
ad = PropertySet()
ad.set("foo", "bar")
ad.setInt("x", 3)
LogicalLocation.setLocationMap(ad)
loc = LogicalLocation("%(foo)xx")
self.assertEqual(loc.locString(), "barxx")
loc = LogicalLocation("%(x)foo")
self.assertEqual(loc.locString(), "3foo")
loc = LogicalLocation("yy%04d(x)yy")
self.assertEqual(loc.locString(), "yy0003yy")
ad2 = PropertySet()
ad2.set("foo", "baz")
ad2.setInt("y", 2009)
loc = LogicalLocation("%(foo)%(x)%(y)", ad2)
self.assertEqual(loc.locString(), "bar32009")
LogicalLocation.setLocationMap(PropertySet())
loc = LogicalLocation("%(foo)%3d(y)", ad2)
self.assertEqual(loc.locString(), "baz2009")
def testSubst(self):
ad = PropertySet()
ad.set("foo", "bar")
ad.setInt("x", 3)
LogicalLocation.setLocationMap(ad)
l = LogicalLocation("%(foo)xx")
self.assertEqual(l.locString(), "barxx")
l = LogicalLocation("%(x)foo")
self.assertEqual(l.locString(), "3foo")
l = LogicalLocation("yy%04d(x)yy")
self.assertEqual(l.locString(), "yy0003yy")
ad2 = PropertySet()
ad2.set("foo", "baz")
ad2.setInt("y", 2009)
l = LogicalLocation("%(foo)%(x)%(y)", ad2)
self.assertEqual(l.locString(), "bar32009")
LogicalLocation.setLocationMap(PropertySet())
l = LogicalLocation("%(foo)%3d(y)", ad2)
self.assertEqual(l.locString(), "baz2009")
def testFilterableSendEvent(self):
testEnv = TestEnvironment()
broker = testEnv.getBroker()
thisHost = platform.node()
topic = "test_events_filters_%s_%d" % (thisHost, os.getpid())
runId = "test_filters_runid"
recv = events.EventReceiver(broker, topic)
trans = events.EventTransmitter(broker, topic)
root = PropertySet()
DATE = "date"
DATE_VAL = "2007-07-01T14:28:32.546012"
root.set(DATE, DATE_VAL)
BLANK = "blank"
BLANK_VAL = ""
root.set(BLANK, BLANK_VAL)
PID = "pid"
PID_VAL = os.getpid()
root.setInt(PID, PID_VAL)
HOST = "host"
HOST_VAL = "lsstcorp.org"
root.set(HOST, HOST_VAL)
IP = "ip"
IP_VAL = "1.2.3.4"
root.set(IP, IP_VAL)
EVNT = "evnt"
EVNT_VAL = "test"
root.set(EVNT, EVNT_VAL)
MISC1 = "misc1"
MISC1_VAL = "data 1"
root.set(MISC1, MISC1_VAL)
MISC2 = "misc2"
MISC2_VAL = "data 2"
root.set(MISC2, MISC2_VAL)
MISC3 = "misc3"
MISC3_VAL = ""
root.set(MISC3, MISC3_VAL)
DATA = "data"
DATA_VAL = 3.14
root.setDouble(DATA, DATA_VAL)
filterable = PropertySet()
filterable.set("FOO", "bar")
filterable.set("XYZZY", 123)
filterable.set("PLOUGH", 0.867)
event = events.Event(runId, root, filterable)
trans.publishEvent(event)
val = recv.receiveEvent()
self.assertIsNotNone(val)
ps = val.getPropertySet()
self.assertEqual(ps.get(DATE), DATE_VAL)
self.assertEqual(ps.get(BLANK), BLANK_VAL)
self.assertEqual(ps.get(PID), PID_VAL)
self.assertEqual(ps.get(HOST), HOST_VAL)
self.assertEqual(ps.get(IP), IP_VAL)
self.assertEqual(ps.get(EVNT), EVNT_VAL)
self.assertEqual(ps.get(MISC1), MISC1_VAL)
self.assertEqual(ps.get(MISC2), MISC2_VAL)
self.assertEqual(ps.get(MISC3), MISC3_VAL)
self.assertEqual(ps.get(DATA), DATA_VAL)
self.assertGreater(ps.get(events.Event.EVENTTIME), 0)
self.assertGreater(ps.get(events.Event.PUBTIME), 0)
self.assertEqual(ps.get(events.Event.RUNID), runId)
self.assertEqual(ps.get(events.Event.STATUS), "unknown")
self.assertEqual(ps.get(events.Event.TOPIC), topic)
self.assertEqual(ps.get(events.Event.TYPE), events.EventTypes.EVENT)
self.assertEqual(ps.get("FOO"), "bar")
self.assertEqual(ps.get("XYZZY"), 123)
self.assertEqual(ps.get("PLOUGH"), 0.867)
names = val.getFilterablePropertyNames()
values = [events.Event.EVENTTIME, 'FOO', 'PLOUGH', events.Event.PUBTIME, events.Event.RUNID, events.Event.STATUS,
events.Event.TOPIC, events.Event.TYPE, 'XYZZY']
for x in values:
self.assertTrue(x in names)
#
# wait a short time to receive an event. none was sent, so we should
# time out and confirm that we didn't get anything
#
val = recv.receiveEvent(1000)
self.assertIsNone(val)
def testSendEvent(self):
"""Send an Event"""
testEnv = EventsEnvironment()
self.thisHost = platform.node()
self.broker = testEnv.getBroker()
topic = "test_events_3_%s_%d" % (self.thisHost, os.getpid())
runID = "test3_runid"
recv = events.EventReceiver(self.broker, topic)
trans = events.EventTransmitter(self.broker, topic)
DATE = "date"
DATE_VAL = "2007-07-01T14:28:32.546012"
BLANK = "blank"
BLANK_VAL = ""
PID = "pid"
PID_VAL = os.getpid()
HOST = "host"
HOST_VAL = "lsstcorp.org"
IP = "ip"
IP_VAL = "1.2.3.4"
EVNT = "evnt"
EVNT_VAL = "test"
MISC1 = "misc1"
MISC1_VAL = "data 1"
MISC2 = "misc2"
MISC2_VAL = "data 2"
MISC3 = "misc3"
MISC3_VAL = ""
DATA = "data"
DATA_VAL = 3.14
root = PropertySet()
root.set(DATE, DATE_VAL)
root.set(BLANK, BLANK_VAL)
root.setInt(PID, PID_VAL)
root.set(HOST, HOST_VAL)
root.set(IP, IP_VAL)
root.set(EVNT, EVNT_VAL)
root.set(MISC1, MISC1_VAL)
root.set(MISC2, MISC2_VAL)
root.set(MISC3, MISC3_VAL)
root.setDouble(DATA, DATA_VAL)
event = events.Event(runID, root)
trans.publishEvent(event)
val = recv.receiveEvent()
self.assertIsNotNone(val)
# check the validity of all sent values
ps = val.getPropertySet()
self.assertEqual(ps.get(DATE), DATE_VAL)
self.assertEqual(ps.get(BLANK), BLANK_VAL)
self.assertEqual(ps.get(PID), PID_VAL)
self.assertEqual(ps.get(HOST), HOST_VAL)
self.assertEqual(ps.get(IP), IP_VAL)
self.assertEqual(ps.get(EVNT), EVNT_VAL)
self.assertEqual(ps.get(MISC1), MISC1_VAL)
self.assertEqual(ps.get(MISC2), MISC2_VAL)
self.assertEqual(ps.get(MISC3), MISC3_VAL)
self.assertEqual(ps.get(DATA), DATA_VAL)
self.assertGreater(ps.get(events.Event.EVENTTIME), 0)
self.assertGreater(ps.get(events.Event.PUBTIME), 0)
self.assertEqual(ps.get(events.Event.RUNID), runID)
self.assertEqual(ps.get(events.Event.STATUS), "unknown")
self.assertEqual(ps.get(events.Event.TOPIC), topic)
self.assertEqual(ps.get(events.Event.TYPE), events.EventTypes.EVENT)
#
# wait a short time to receive an event. none was sent, so we should
# time out and confirm that we didn't get anything
#
val = recv.receiveEvent(1000)
self.assertIsNone(val)
def setUp(self):
# Turn on tracing
log.Trace.setVerbosity('', 10)
log.ScreenLog.createDefaultLog(True, log.Log.INFO)
# Eventually, these should be read from a policy somewhere
self.dbServer = 'lsst10.ncsa.uiuc.edu'
self.dbPort = '3306'
self.dbType = 'mysql'
if not DbAuth.available(self.dbServer, self.dbPort):
self.fail("Cannot access database server %s:%s" %
(self.dbServer, self.dbPort))
# Construct test run database name
self.runId = DbAuth.username(self.dbServer, self.dbPort) +\
time.strftime("_test_ap_%y%m%d_%H%M%S", time.gmtime())
# Tweak these to run on different input data, or with a different number of slices
self.universeSize = 2
self.visitId = 708125
self.filter = 'u'
self.ra = 333.880166667
self.dec = -17.7374166667
self.dbUrlPrefix = ''.join(
[self.dbType, '://', self.dbServer, ':', self.dbPort, '/'])
self.dbUrl = self.dbUrlPrefix + self.runId
self.substitutions = {
'visitId': self.visitId,
'filter': self.filter,
'runId': self.runId
}
# Create a database specifically for the test (copy relevant
# tables from the test_ap database)
mysqlStatements = [
"""CREATE DATABASE %(runId)s""", """USE %(runId)s""",
"""CREATE TABLE VarObject LIKE test_ap.Object""",
"""CREATE TABLE NonVarObject LIKE test_ap.Object""",
"""CREATE TABLE DIASource LIKE test_ap.DIASource""",
"""CREATE TABLE prv_Filter LIKE test_ap.prv_Filter""",
"""INSERT INTO prv_Filter SELECT * FROM test_ap.prv_Filter""",
"""CREATE TABLE _tmp_v%(visitId)d_DIASource
LIKE test_ap._tmp_v%(visitId)d_DIASource""",
"""INSERT INTO _tmp_v%(visitId)d_DIASource
SELECT * FROM test_ap._tmp_v%(visitId)d_DIASource""",
"""CREATE TABLE _tmp_v%(visitId)d_Preds
LIKE test_ap._tmp_v%(visitId)d_Preds""",
"""INSERT INTO _tmp_v%(visitId)d_Preds
SELECT * FROM test_ap._tmp_v%(visitId)d_Preds""",
"""CREATE TABLE _tmpl_MatchPair LIKE test_ap._tmpl_MatchPair""",
"""CREATE TABLE _tmpl_IdPair LIKE test_ap._tmpl_IdPair""",
"""CREATE TABLE _tmpl_InMemoryObject LIKE test_ap._tmpl_InMemoryObject""",
"""CREATE TABLE _tmpl_InMemoryMatchPair LIKE test_ap._tmpl_InMemoryMatchPair""",
"""CREATE TABLE _tmpl_InMemoryId LIKE test_ap._tmpl_InMemoryId""",
"""CREATE TABLE _ap_DIASourceToObjectMatches LIKE test_ap._ap_DIASourceToObjectMatches""",
"""CREATE TABLE _ap_PredToDIASourceMatches LIKE test_ap._ap_PredToDIASourceMatches""",
"""CREATE TABLE _ap_DIASourceToNewObject LIKE test_ap._ap_DIASourceToNewObject""",
"""CREATE TABLE _mops_Prediction LIKE test_ap._mops_Prediction"""
]
db = DbStorage()
db.setPersistLocation(LogicalLocation(self.dbUrlPrefix + 'test_ap'))
try:
for stmt in mysqlStatements:
db.executeSql(stmt % self.substitutions)
# Specify list of stages ...
self.stages = [
ap.LoadStage, InputStage, ap.MatchDiaSourcesStage, OutputStage,
InputStage, ap.MatchMopsPredsStage, OutputStage, ap.StoreStage
]
# and read in stage policy for each stage
policyDir = os.path.join(os.environ['AP_DIR'], 'pipeline',
'examples', 'policy')
self.policies = [
Policy(os.path.join(policyDir, 'LoadStage.paf')),
Policy(os.path.join(policyDir,
'MatchDiaSourcesStageInput.paf')), None,
Policy(
os.path.join(policyDir, 'MatchDiaSourcesStageOutput.paf')),
Policy(os.path.join(policyDir,
'MatchMopsPredsStageInput.paf')), None,
Policy(os.path.join(policyDir,
'MatchMopsPredsStageOutput.paf')),
Policy(os.path.join(policyDir, 'StoreStage.paf'))
]
# construct PropertySet for string interpolation
psSubs = PropertySet()
psSubs.setInt('visitId', self.visitId)
psSubs.setString('runId', self.runId)
psSubs.setString('filter', self.filter)
psSubs.setString('work', '.')
psSubs.setString('input', '/tmp')
psSubs.setString('output', '/tmp')
psSubs.setString('update', '/tmp')
psSubs.setString('dbUrl', self.dbUrl)
LogicalLocation.setLocationMap(psSubs)
except:
# cleanup database in case of error
db.executeSql("DROP DATABASE %(runId)s" % self.substitutions)
raise
def setUp(self):
# Turn on tracing
log.Trace.setVerbosity('', 10)
log.ScreenLog.createDefaultLog(True, log.Log.INFO)
# Eventually, these should be read from a policy somewhere
self.dbServer = 'lsst10.ncsa.uiuc.edu'
self.dbPort = '3306'
self.dbType = 'mysql'
if not DbAuth.available(self.dbServer, self.dbPort):
self.fail("Cannot access database server %s:%s" % (self.dbServer, self.dbPort))
# Construct test run database name
self.runId = DbAuth.username(self.dbServer, self.dbPort) +\
time.strftime("_test_ap_%y%m%d_%H%M%S", time.gmtime())
# Tweak these to run on different input data, or with a different number of slices
self.universeSize = 2
self.visitId = 708125
self.filter = 'u'
self.ra = 333.880166667
self.dec = -17.7374166667
self.dbUrlPrefix = ''.join([self.dbType, '://', self.dbServer, ':', self.dbPort, '/'])
self.dbUrl = self.dbUrlPrefix + self.runId
self.substitutions = { 'visitId': self.visitId,
'filter': self.filter,
'runId': self.runId }
# Create a database specifically for the test (copy relevant
# tables from the test_ap database)
mysqlStatements = [
"""CREATE DATABASE %(runId)s""",
"""USE %(runId)s""",
"""CREATE TABLE VarObject LIKE test_ap.Object""",
"""CREATE TABLE NonVarObject LIKE test_ap.Object""",
"""CREATE TABLE DIASource LIKE test_ap.DIASource""",
"""CREATE TABLE prv_Filter LIKE test_ap.prv_Filter""",
"""INSERT INTO prv_Filter SELECT * FROM test_ap.prv_Filter""",
"""CREATE TABLE _tmp_v%(visitId)d_DIASource
LIKE test_ap._tmp_v%(visitId)d_DIASource""",
"""INSERT INTO _tmp_v%(visitId)d_DIASource
SELECT * FROM test_ap._tmp_v%(visitId)d_DIASource""",
"""CREATE TABLE _tmp_v%(visitId)d_Preds
LIKE test_ap._tmp_v%(visitId)d_Preds""",
"""INSERT INTO _tmp_v%(visitId)d_Preds
SELECT * FROM test_ap._tmp_v%(visitId)d_Preds""",
"""CREATE TABLE _tmpl_MatchPair LIKE test_ap._tmpl_MatchPair""",
"""CREATE TABLE _tmpl_IdPair LIKE test_ap._tmpl_IdPair""",
"""CREATE TABLE _tmpl_InMemoryObject LIKE test_ap._tmpl_InMemoryObject""",
"""CREATE TABLE _tmpl_InMemoryMatchPair LIKE test_ap._tmpl_InMemoryMatchPair""",
"""CREATE TABLE _tmpl_InMemoryId LIKE test_ap._tmpl_InMemoryId""",
"""CREATE TABLE _ap_DIASourceToObjectMatches LIKE test_ap._ap_DIASourceToObjectMatches""",
"""CREATE TABLE _ap_PredToDIASourceMatches LIKE test_ap._ap_PredToDIASourceMatches""",
"""CREATE TABLE _ap_DIASourceToNewObject LIKE test_ap._ap_DIASourceToNewObject""",
"""CREATE TABLE _mops_Prediction LIKE test_ap._mops_Prediction"""
]
db = DbStorage()
db.setPersistLocation(LogicalLocation(self.dbUrlPrefix + 'test_ap'))
try:
for stmt in mysqlStatements:
db.executeSql(stmt % self.substitutions)
# Specify list of stages ...
self.stages = [ ap.LoadStage,
InputStage,
ap.MatchDiaSourcesStage,
OutputStage,
InputStage,
ap.MatchMopsPredsStage,
OutputStage,
ap.StoreStage ]
# and read in stage policy for each stage
policyDir = os.path.join(os.environ['AP_DIR'], 'pipeline', 'examples', 'policy')
self.policies = [ Policy(os.path.join(policyDir,'LoadStage.paf')),
Policy(os.path.join(policyDir,'MatchDiaSourcesStageInput.paf')),
None,
Policy(os.path.join(policyDir,'MatchDiaSourcesStageOutput.paf')),
Policy(os.path.join(policyDir,'MatchMopsPredsStageInput.paf')),
None,
Policy(os.path.join(policyDir,'MatchMopsPredsStageOutput.paf')),
Policy(os.path.join(policyDir,'StoreStage.paf')) ]
# construct PropertySet for string interpolation
psSubs = PropertySet()
psSubs.setInt('visitId', self.visitId)
psSubs.setString('runId', self.runId)
psSubs.setString('filter', self.filter)
psSubs.setString('work', '.')
psSubs.setString('input', '/tmp')
psSubs.setString('output', '/tmp')
psSubs.setString('update', '/tmp')
psSubs.setString('dbUrl', self.dbUrl)
LogicalLocation.setLocationMap(psSubs)
except:
# cleanup database in case of error
db.executeSql("DROP DATABASE %(runId)s" % self.substitutions)
raise
def testFilterableSendEvent(self):
testEnv = EventsEnvironment()
broker = testEnv.getBroker()
thisHost = platform.node()
topic = "test_events_filters_%s_%d" % (thisHost, os.getpid())
runId = "test_filters_runid"
recv = events.EventReceiver(broker, topic)
trans = events.EventTransmitter(broker, topic)
root = PropertySet()
DATE = "date"
DATE_VAL = "2007-07-01T14:28:32.546012"
root.set(DATE, DATE_VAL)
BLANK = "blank"
BLANK_VAL = ""
root.set(BLANK, BLANK_VAL)
PID = "pid"
PID_VAL = os.getpid()
root.setInt(PID, PID_VAL)
HOST = "host"
HOST_VAL = "lsstcorp.org"
root.set(HOST, HOST_VAL)
IP = "ip"
IP_VAL = "1.2.3.4"
root.set(IP, IP_VAL)
EVNT = "evnt"
EVNT_VAL = "test"
root.set(EVNT, EVNT_VAL)
MISC1 = "misc1"
MISC1_VAL = "data 1"
root.set(MISC1, MISC1_VAL)
MISC2 = "misc2"
MISC2_VAL = "data 2"
root.set(MISC2, MISC2_VAL)
MISC3 = "misc3"
MISC3_VAL = ""
root.set(MISC3, MISC3_VAL)
DATA = "data"
DATA_VAL = 3.14
root.setDouble(DATA, DATA_VAL)
filterable = PropertySet()
filterable.set("FOO", "bar")
filterable.set("XYZZY", 123)
filterable.set("PLOUGH", 0.867)
event = events.Event(runId, root, filterable)
trans.publishEvent(event)
val = recv.receiveEvent()
self.assertIsNotNone(val)
ps = val.getPropertySet()
self.assertEqual(ps.get(DATE), DATE_VAL)
self.assertEqual(ps.get(BLANK), BLANK_VAL)
self.assertEqual(ps.get(PID), PID_VAL)
self.assertEqual(ps.get(HOST), HOST_VAL)
self.assertEqual(ps.get(IP), IP_VAL)
self.assertEqual(ps.get(EVNT), EVNT_VAL)
self.assertEqual(ps.get(MISC1), MISC1_VAL)
self.assertEqual(ps.get(MISC2), MISC2_VAL)
self.assertEqual(ps.get(MISC3), MISC3_VAL)
self.assertEqual(ps.get(DATA), DATA_VAL)
self.assertGreater(ps.get(events.Event.EVENTTIME), 0)
self.assertGreater(ps.get(events.Event.PUBTIME), 0)
self.assertEqual(ps.get(events.Event.RUNID), runId)
self.assertEqual(ps.get(events.Event.STATUS), "unknown")
self.assertEqual(ps.get(events.Event.TOPIC), topic)
self.assertEqual(ps.get(events.Event.TYPE), events.EventTypes.EVENT)
self.assertEqual(ps.get("FOO"), "bar")
self.assertEqual(ps.get("XYZZY"), 123)
self.assertEqual(ps.get("PLOUGH"), 0.867)
names = val.getFilterablePropertyNames()
values = [
events.Event.EVENTTIME, 'FOO', 'PLOUGH', events.Event.PUBTIME,
events.Event.RUNID, events.Event.STATUS, events.Event.TOPIC,
events.Event.TYPE, 'XYZZY'
]
for x in values:
self.assertTrue(x in names)
#
# wait a short time to receive an event. none was sent, so we should
# time out and confirm that we didn't get anything
#
val = recv.receiveEvent(1000)
self.assertIsNone(val)
def testSendEvent(self):
"""Send an Event"""
testEnv = EventsEnvironment()
self.thisHost = platform.node()
self.broker = testEnv.getBroker()
topic = "test_events_3_%s_%d" % (self.thisHost, os.getpid())
runID = "test3_runid"
recv = events.EventReceiver(self.broker, topic)
trans = events.EventTransmitter(self.broker, topic)
DATE = "date"
DATE_VAL = "2007-07-01T14:28:32.546012"
BLANK = "blank"
BLANK_VAL = ""
PID = "pid"
PID_VAL = os.getpid()
HOST = "host"
HOST_VAL = "lsstcorp.org"
IP = "ip"
IP_VAL = "1.2.3.4"
EVNT = "evnt"
EVNT_VAL = "test"
MISC1 = "misc1"
MISC1_VAL = "data 1"
MISC2 = "misc2"
MISC2_VAL = "data 2"
MISC3 = "misc3"
MISC3_VAL = ""
DATA = "data"
DATA_VAL = 3.14
root = PropertySet()
root.set(DATE, DATE_VAL)
root.set(BLANK, BLANK_VAL)
root.setInt(PID, PID_VAL)
root.set(HOST, HOST_VAL)
root.set(IP, IP_VAL)
root.set(EVNT, EVNT_VAL)
root.set(MISC1, MISC1_VAL)
root.set(MISC2, MISC2_VAL)
root.set(MISC3, MISC3_VAL)
root.setDouble(DATA, DATA_VAL)
event = events.Event(runID, root)
trans.publishEvent(event)
val = recv.receiveEvent()
self.assertIsNotNone(val)
# check the validity of all sent values
ps = val.getPropertySet()
self.assertEqual(ps.get(DATE), DATE_VAL)
self.assertEqual(ps.get(BLANK), BLANK_VAL)
self.assertEqual(ps.get(PID), PID_VAL)
self.assertEqual(ps.get(HOST), HOST_VAL)
self.assertEqual(ps.get(IP), IP_VAL)
self.assertEqual(ps.get(EVNT), EVNT_VAL)
self.assertEqual(ps.get(MISC1), MISC1_VAL)
self.assertEqual(ps.get(MISC2), MISC2_VAL)
self.assertEqual(ps.get(MISC3), MISC3_VAL)
self.assertEqual(ps.get(DATA), DATA_VAL)
self.assertGreater(ps.get(events.Event.EVENTTIME), 0)
self.assertGreater(ps.get(events.Event.PUBTIME), 0)
self.assertEqual(ps.get(events.Event.RUNID), runID)
self.assertEqual(ps.get(events.Event.STATUS), "unknown")
self.assertEqual(ps.get(events.Event.TOPIC), topic)
self.assertEqual(ps.get(events.Event.TYPE), events.EventTypes.EVENT)
#
# wait a short time to receive an event. none was sent, so we should
# time out and confirm that we didn't get anything
#
val = recv.receiveEvent(1000)
self.assertIsNone(val)
