def testProcessDataEvent(self):
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),
0)
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "available")
ds = self.testDatasetFromProperty()
ps.add("dataset", serializePolicy(ds.toPolicy()))
for i in xrange(1, 4):
ds = copy.deepcopy(ds)
ds.ids["ampid"] += 1
ps.add("dataset", serializePolicy(ds.toPolicy()))
devent = StatusEvent("testing", originatorId, ps)
# pdb.set_trace()
self.joboffice.processDataEvent(devent)
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),
4)
self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(),
1)
job = self.joboffice.bb.queues.jobsPossible.get(0)
self.assertEquals(job.getName(), "Job-1")
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 12)
def testProcessDataEvent(self):
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 0)
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "available")
ds = self.testDatasetFromProperty()
ps.add("dataset", serializePolicy(ds.toPolicy()))
for i in xrange(1,4):
ds = copy.deepcopy(ds)
ds.ids["ampid"] += 1
ps.add("dataset", serializePolicy(ds.toPolicy()))
devent = StatusEvent("testing", originatorId, ps)
# pdb.set_trace()
self.joboffice.processDataEvent(devent)
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 4)
self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(), 1)
job = self.joboffice.bb.queues.jobsPossible.get(0)
self.assertEquals(job.getName(), "Job-1")
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 12)
def makeit(prefs, context, saveWcs=False, plot=dict()):
"""This is the python wrapper for the original psfex that reads SExtractor outputs
"""
# Create an array of PSFs (one PSF for each extension)
if prefs.getVerboseType() != prefs.QUIET:
print("----- %d input catalogues:" % prefs.getNcat())
if saveWcs: # only needed for making plots
wcssList = []
fields = psfexLib.vectorField()
for cat in prefs.getCatalogs():
field = psfexLib.Field(cat)
wcss = []
wcssList.append(wcss)
with fits.open(cat) as pf:
for hdu in pf:
if hdu.name == "PRIMARY":
pass
elif hdu.name == "LDAC_IMHEAD":
hdr = hdu.data[0][
0] # the fits header from the original fits image
md = PropertySet()
for line in hdr:
try:
md.set(*splitFitsCard(line))
except AttributeError:
continue
if not md.exists("CRPIX1"): # no WCS; try WCSA
for k in md.names():
if re.search(r"A$", k):
md.set(k[:-1], md.getScalar(k))
wcs = afwGeom.makeSkyWcs(md)
naxis1, naxis2 = md.getScalar("NAXIS1"), md.getScalar(
"NAXIS2")
elif hdu.name == "LDAC_OBJECTS":
nobj = len(hdu.data)
assert wcs, "LDAC_OBJECTS comes after LDAC_IMHEAD"
field.addExt(wcs, naxis1, naxis2, nobj)
if saveWcs:
wcss.append((wcs, naxis1, naxis2))
wcs = None
field.finalize()
fields.append(field)
sets = psfexLib.vectorSet()
for set in load_samples(prefs, context, plot=plot):
sets.append(set)
psfexLib.makeit(fields, sets)
ret = [[f.getPsfs() for f in fields], sets]
if saveWcs:
ret.append(wcssList)
return ret
def createPropertySet(self):
data = self.getDataSet()
prop = PropertySet()
for x in data:
prop.set(x[0],x[1])
return prop
def makeit(prefs, context, saveWcs=False, plot=dict()):
"""This is the python wrapper for the original psfex that reads SExtractor outputs
"""
# Create an array of PSFs (one PSF for each extension)
if prefs.getVerboseType() != prefs.QUIET:
print("----- %d input catalogues:" % prefs.getNcat())
if saveWcs: # only needed for making plots
wcssList = []
fields = psfexLib.vectorField()
for cat in prefs.getCatalogs():
field = psfexLib.Field(cat)
wcss = []
wcssList.append(wcss)
with fits.open(cat) as pf:
for hdu in pf:
if hdu.name == "PRIMARY":
pass
elif hdu.name == "LDAC_IMHEAD":
hdr = hdu.data[0][0] # the fits header from the original fits image
md = PropertySet()
for line in hdr:
try:
md.set(*splitFitsCard(line))
except AttributeError:
continue
if not md.exists("CRPIX1"): # no WCS; try WCSA
for k in md.names():
if re.search(r"A$", k):
md.set(k[:-1], md.getScalar(k))
wcs = afwGeom.makeSkyWcs(md)
naxis1, naxis2 = md.getScalar("NAXIS1"), md.getScalar("NAXIS2")
elif hdu.name == "LDAC_OBJECTS":
nobj = len(hdu.data)
assert wcs, "LDAC_OBJECTS comes after LDAC_IMHEAD"
field.addExt(wcs, naxis1, naxis2, nobj)
if saveWcs:
wcss.append((wcs, naxis1, naxis2))
wcs = None
field.finalize()
fields.append(field)
sets = psfexLib.vectorSet()
for set in load_samples(prefs, context, plot=plot):
sets.append(set)
psfexLib.makeit(fields, sets)
ret = [[f.getPsfs() for f in fields], sets]
if saveWcs:
ret.append(wcssList)
return ret
def testEventRunID(self):
runid = "testrunid"
status = "my special status"
root = PropertySet()
MYNAME = "myname"
root.set(MYNAME, MYNAME)
root.set(events.Event.STATUS, status)
event = events.Event(runid, root)
# return the property set
props = event.getPropertySet()
self.assertGreater(props.get(events.Event.EVENTTIME), 0)
self.assertEqual(props.get(events.Event.PUBTIME), 0)
self.assertEqual(props.get(events.Event.RUNID), runid)
self.assertEqual(props.get(events.Event.STATUS), status)
self.assertEqual(props.get(events.Event.TOPIC),
events.Event.UNINITIALIZED)
self.assertEqual(props.get(events.Event.TYPE), events.EventTypes.EVENT)
self.assertEqual(props.get(MYNAME), MYNAME)
filterableNames = event.getFilterablePropertyNames()
filterableNames.remove(events.Event.EVENTTIME)
filterableNames.remove(events.Event.PUBTIME)
filterableNames.remove(events.Event.RUNID)
filterableNames.remove(events.Event.STATUS)
filterableNames.remove(events.Event.TOPIC)
filterableNames.remove(events.Event.TYPE)
assert len(filterableNames) == 0
customNames = event.getCustomPropertyNames()
self.assertEqual(len(customNames), 1)
self.assertEqual(customNames[0], MYNAME)
eventTime = event.getEventTime()
self.assertEqual(event.getPubTime(), 0)
self.assertEqual(event.getRunId(), runid)
self.assertEqual(event.getType(), events.EventTypes.EVENT)
self.assertEqual(event.getStatus(), status)
# check to be sure we really update the time
event.updateEventTime()
self.assertLess(eventTime, event.getEventTime())
# set the time to zero, and make sure we get back the same time.
event.setEventTime(0)
self.assertEqual(event.getEventTime(), 0)
# reset the time to eventTime, and make sure we get back the same time.
event.setEventTime(eventTime)
self.assertEqual(event.getEventTime(), eventTime)
def testEventRunID(self):
runid = "testrunid"
status = "my special status"
root = PropertySet()
MYNAME = "myname"
root.set(MYNAME, MYNAME)
root.set(events.Event.STATUS, status)
event = events.Event(runid, root)
# return the property set
props = event.getPropertySet()
self.assertGreater(props.get(events.Event.EVENTTIME), 0)
self.assertEqual(props.get(events.Event.PUBTIME), 0)
self.assertEqual(props.get(events.Event.RUNID), runid)
self.assertEqual(props.get(events.Event.STATUS), status)
self.assertEqual(props.get(events.Event.TOPIC), events.Event.UNINITIALIZED)
self.assertEqual(props.get(events.Event.TYPE), events.EventTypes.EVENT)
self.assertEqual(props.get(MYNAME), MYNAME)
filterableNames = event.getFilterablePropertyNames()
filterableNames.remove(events.Event.EVENTTIME)
filterableNames.remove(events.Event.PUBTIME)
filterableNames.remove(events.Event.RUNID)
filterableNames.remove(events.Event.STATUS)
filterableNames.remove(events.Event.TOPIC)
filterableNames.remove(events.Event.TYPE)
assert len(filterableNames) == 0
customNames = event.getCustomPropertyNames()
self.assertEqual(len(customNames), 1)
self.assertEqual(customNames[0], MYNAME)
eventTime = event.getEventTime()
self.assertEqual(event.getPubTime(),0)
self.assertEqual(event.getRunId(), runid)
self.assertEqual(event.getType(), events.EventTypes.EVENT)
self.assertEqual(event.getStatus(), status)
# check to be sure we really update the time
event.updateEventTime()
self.assertLess(eventTime, event.getEventTime())
# set the time to zero, and make sure we get back the same time.
event.setEventTime(0)
self.assertEqual(event.getEventTime(), 0)
# reset the time to eventTime, and make sure we get back the same time.
event.setEventTime(eventTime)
self.assertEqual(event.getEventTime(), eventTime)
def testProcessDataEvents(self):
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),
0)
trx = EventTransmitter(brokerhost, "PostISRAvailable")
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "available")
ds = self.testDatasetFromProperty()
ds.ids["ampid"] = 0
ps.set("dataset", serializePolicy(ds.toPolicy()))
for i in xrange(15):
ps.set("dataset", serializePolicy(ds.toPolicy()))
devent = StatusEvent("testing", originatorId, ps)
trx.publishEvent(devent)
ds = copy.deepcopy(ds)
ds.ids["ampid"] += 1
# Wait for events
time.sleep(2.0)
# pdb.set_trace()
self.joboffice.processDataEvents()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),
15)
self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(),
1)
job = self.joboffice.bb.queues.jobsPossible.get(0)
self.assertEquals(job.getName(), "Job-1")
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 1)
ps.set("dataset", serializePolicy(ds.toPolicy()))
devent = StatusEvent("testing", originatorId, ps)
trx.publishEvent(devent)
time.sleep(2.0)
self.joboffice.processDataEvents()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),
16)
self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(),
1)
job = self.joboffice.bb.queues.jobsPossible.get(0)
self.assertEquals(job.getName(), "Job-1")
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 0)
def testEventProperties(self):
status = "my special status"
root = PropertySet()
MYNAME = "myname"
root.set(MYNAME, MYNAME)
root.set(events.Event.STATUS, status)
event = events.Event(root)
props = event.getPropertySet()
custom = event.getCustomPropertySet()
self.assertEqual(props.nameCount(), 6)
self.assertEqual(custom.nameCount(), 1)
def testEventProperties(self):
status = "my special status"
root = PropertySet()
MYNAME = "myname"
root.set(MYNAME, MYNAME)
root.set(events.Event.STATUS, status)
event = events.Event(root)
props = event.getPropertySet()
custom = event.getCustomPropertySet()
self.assertEqual(props.nameCount(), 6)
self.assertEqual(custom.nameCount(), 1)
def process(self, clipboard):
metadata = clipboard.get(self.policy.getString("inputKeys.metadata"))
cdb = clipboard.get(self.policy.getString("inputKeys.calibrationDB"))
pathPrefix =
cliboard.get(self.policy.getString("inputKeys.prefixPath"))
when = DateTime(metadata.get("dateObs"))
ccdId = metadata.get("ccdId")
ampId = metadata.get("ampId")
expTime = metadata.get("expTime")
darkToApply = self.policy.getString("whichdark")
darkCalibList = cdb.lookup(when, "dark", ccdId, ampId, all=True)
darkTimeList = []
for d in darkCalibList:
darkTimeList.append(d.expTime)
darkTimeList.sort()
if darktoapply == "min":
darkExpTime = darkTimeList[0]
elif darktoapply == "max":
darkExpTime = darkTimeList[-1]
elif darktoapply == "closest":
minDist = abs(expTime - darkTimeList[0])
minExpTime = darkTimeList[0]
for i in xrange(1, len(darkTimeList)):
dist = abs(expTime - darkTimeList[i])
if dist < minDist:
minDist = dist
minExpTime = darkTimeList[i]
darkExpTime = minExpTime
else:
raise RuntimeError, "Unrecognized method for finding dark to apply: " + str(darktoapply)
biasPath = cdb.lookup(when, "bias", ccdId, ampId)
darkPath = cdb.lookup(when, "dark", ccdId, ampId,
expTime=darkExpTime)
defectPath = cdb.lookup(when, "defect", ccdId, ampId)
flatPath = cdb.lookup(when, "flat", ccdId, ampId,
filter=metadata.get("filter"))
# fringePath = cdb.lookup(when, "fringe", ccdId, ampId,
# filter=metadata.get("filter"))
linearizePath = cdb.lookup(when, "linearize", ccdId, ampId)
calibData = PropertySet()
calibData.set("biasPath", os.path.join(pathPrefix, biasPath))
calibData.set("darkPath", os.path.join(pathPrefix, darkPath))
calibData.set("defectPath", os.path.join(pathPrefix, defectPath))
calibData.set("flatPath", os.path.join(pathPrefix, flatPath))
# calibData.set("fringePath", os.path.join(pathPrefix, fringePath))
calibData.set("linearizePath", os.path.join(pathPrefix, linearizePath))
outputKey = self._policy.get("outputKey")
self.activeClipboard.put(outputKey, calibData)
self.outputQueue.addDataset(self.activeClipboard)
def sendEvent(self, brokerName, topic):
"""Send an Event."""
trans = events.EventTransmitter(brokerName, topic)
root = PropertySet()
root.set(events.Event.TOPIC, topic)
root.set("myname","myname")
root.set(events.StatusEvent.STATUS, "my special status")
root.set(events.StatusEvent.RUNID,"srptestrun")
root.set("bazinga", "sheldon")
originatorId = events.LocationId()
event = events.StatusEvent("srptestrun", originatorId, root)
# ok...now publish it
trans.publishEvent(event)
def sendEvent(self, brokerName, topic):
"""Send an Event."""
trans = events.EventTransmitter(brokerName, topic)
root = PropertySet()
root.set(events.Event.TOPIC, topic)
root.set("myname", "myname")
root.set(events.StatusEvent.STATUS, "my special status")
root.set(events.StatusEvent.RUNID, "srptestrun")
root.set("bazinga", "sheldon")
originatorId = events.LocationId()
event = events.StatusEvent("srptestrun", originatorId, root)
# ok...now publish it
trans.publishEvent(event)
def testProcessDataEvents(self):
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(), 0)
trx = EventTransmitter(brokerhost, "PostISRAvailable")
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "available")
ds = self.testDatasetFromProperty()
ds.ids["ampid"] = 0;
ps.set("dataset", serializePolicy(ds.toPolicy()))
for i in xrange(15):
ps.set("dataset", serializePolicy(ds.toPolicy()))
devent = StatusEvent("testing", originatorId, ps)
trx.publishEvent(devent);
ds = copy.deepcopy(ds)
ds.ids["ampid"] += 1
# Wait for events
time.sleep(2.0)
# pdb.set_trace()
self.joboffice.processDataEvents()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),15)
self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(), 1)
job = self.joboffice.bb.queues.jobsPossible.get(0)
self.assertEquals(job.getName(), "Job-1")
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 1)
ps.set("dataset", serializePolicy(ds.toPolicy()))
devent = StatusEvent("testing", originatorId, ps)
trx.publishEvent(devent);
time.sleep(2.0)
self.joboffice.processDataEvents()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),16)
self.assertEquals(self.joboffice.bb.queues.jobsPossible.length(), 1)
job = self.joboffice.bb.queues.jobsPossible.get(0)
self.assertEquals(job.getName(), "Job-1")
self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 0)
def testReceiveReadyPipelines(self):
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(),0)
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "job:ready")
pevent = StatusEvent("testing", originatorId, ps)
trx = EventTransmitter(brokerhost, "CcdAssemblyJob")
trx.publishEvent(pevent)
time.sleep(2.0)
self.joboffice.receiveReadyPipelines()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(),1)
def sendFilterableStatusEvent(self, broker, topic):
trans = events.EventTransmitter(broker, topic)
root = PropertySet()
root.set(events.Event.TOPIC, topic)
root.set("myname","myname")
root.set(events.Event.STATUS, "my special status")
filter = PropertySet()
filter.set("FOO", "bar")
filter.set("PLOUGH", 123)
eventSystem = events.EventSystem.getDefaultEventSystem();
locationID = eventSystem.createOriginatorId()
event = events.StatusEvent(locationID, root, filter)
# ok...now publish it
trans.publishEvent(event)
def sendPlainStatusEvent(self, broker, topic, runID=None):
trans = events.EventTransmitter(broker, topic)
root = PropertySet()
root.set(events.Event.TOPIC, topic)
root.set("myname", "myname")
root.set(events.Event.STATUS, "my special status")
root.set("logger.status", "my logger special status")
root.set("logger.pid", "1")
eventSystem = events.EventSystem.getDefaultEventSystem()
locationID = eventSystem.createOriginatorId()
if runID is None:
event = events.StatusEvent(locationID, root)
else:
event = events.StatusEvent(runID, locationID, root)
# ok...now publish it
trans.publishEvent(event)
def testReceiveReadyPipelines(self):
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(),
0)
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "job:ready")
pevent = StatusEvent("testing", originatorId, ps)
trx = EventTransmitter(brokerhost, "CcdAssemblyJob")
trx.publishEvent(pevent)
time.sleep(2.0)
self.joboffice.receiveReadyPipelines()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(),
1)
def sendPlainStatusEvent(self, broker, topic, runID=None):
trans = events.EventTransmitter(broker, topic)
root = PropertySet()
root.set(events.Event.TOPIC, topic)
root.set("myname","myname")
root.set(events.Event.STATUS, "my special status")
root.set("logger.status", "my logger special status")
root.set("logger.pid", "1")
eventSystem = events.EventSystem.getDefaultEventSystem();
locationID = eventSystem.createOriginatorId()
if runID is None:
event = events.StatusEvent(locationID, root)
else:
event = events.StatusEvent(runID, locationID, root)
# ok...now publish it
trans.publishEvent(event)
class _EventFactory(object):
def __init__(self, runid, props=None):
"""
create a generic event factor
"""
self.runid = runid
if isinstance(props, PropertySet):
self.props = props
else:
self.props = PropertySet()
if isinstance(props, dict):
for key in props.keys():
self.props.set(key, props[key])
def create(self):
"""create a new instance of the event"""
return ev.Event(self.runid, self.props)
def setRunId(self, id):
"""set the Run ID"""
self.runid = runid
def getRunId(self):
"""set the Run ID"""
return self.runid
def setProperty(self, name, val):
"""set the value of a named property"""
self.props.set(name, val)
def getProperty(self, name):
"""get the value of a named property"""
return self.props.getString(name)
def addDataset(self, propname, ds):
"""add a dataset to the event"""
self.props.add(propname, serializeDataset(ds))
def getDatasets(self, propname):
"""return the datasets attached to the event"""
return unserializeDatasetList(self.props.getArrayString(propname))
def testSubst(self):
ad = PropertySet()
ad.set("foo", "bar")
ad.setInt("x", 3)
LogicalLocation.setLocationMap(ad)
l = LogicalLocation("%(foo)xx")
self.assertEqual(l.locString(), "barxx")
l = LogicalLocation("%(x)foo")
self.assertEqual(l.locString(), "3foo")
l = LogicalLocation("yy%04d(x)yy")
self.assertEqual(l.locString(), "yy0003yy")
ad2 = PropertySet()
ad2.set("foo", "baz")
ad2.setInt("y", 2009)
l = LogicalLocation("%(foo)%(x)%(y)", ad2)
self.assertEqual(l.locString(), "bar32009")
LogicalLocation.setLocationMap(PropertySet())
l = LogicalLocation("%(foo)%3d(y)", ad2)
self.assertEqual(l.locString(), "baz2009")
def testSubst(self):
ad = PropertySet()
ad.set("foo", "bar")
ad.setInt("x", 3)
LogicalLocation.setLocationMap(ad)
loc = LogicalLocation("%(foo)xx")
self.assertEqual(loc.locString(), "barxx")
loc = LogicalLocation("%(x)foo")
self.assertEqual(loc.locString(), "3foo")
loc = LogicalLocation("yy%04d(x)yy")
self.assertEqual(loc.locString(), "yy0003yy")
ad2 = PropertySet()
ad2.set("foo", "baz")
ad2.setInt("y", 2009)
loc = LogicalLocation("%(foo)%(x)%(y)", ad2)
self.assertEqual(loc.locString(), "bar32009")
LogicalLocation.setLocationMap(PropertySet())
loc = LogicalLocation("%(foo)%3d(y)", ad2)
self.assertEqual(loc.locString(), "baz2009")
def createWcs():
metadata = PropertySet()
metadata.set("SIMPLE", "T")
metadata.set("BITPIX", -32)
metadata.set("NAXIS", 2)
metadata.set("NAXIS1", 1024)
metadata.set("NAXIS2", 1153)
metadata.set("RADESYS", 'FK5')
metadata.set("EQUINOX", 2000.)
metadata.setDouble("CRVAL1", 215.604025685476)
metadata.setDouble("CRVAL2", 53.1595451514076)
metadata.setDouble("CRPIX1", 1109.99981456774)
metadata.setDouble("CRPIX2", 560.018167811613)
metadata.set("CTYPE1", 'RA---SIN')
metadata.set("CTYPE2", 'DEC--SIN')
metadata.setDouble("CD1_1", 5.10808596133527E-05)
metadata.setDouble("CD1_2", 1.85579539217196E-07)
metadata.setDouble("CD2_2", -5.10281493481982E-05)
metadata.setDouble("CD2_1", -8.27440751733828E-07)
return makeSkyWcs(metadata)
def testProcessJobDoneEvents(self):
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),
0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0)
self.testAllocateJobs()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),
1)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0)
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "job:done")
ps.set("success", True)
pevent = StatusEvent("testing", originatorId, ps)
trx = EventTransmitter(brokerhost, "CcdAssemblyJob")
trx.publishEvent(pevent)
time.sleep(2.0)
self.joboffice.processDoneJobs()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),
0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 1)
def testProcessJobDoneEvent(self):
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),
0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0)
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "job:done")
ps.set("success", True)
pevent = StatusEvent("testing", originatorId, ps)
self.assert_(not self.joboffice.processJobDoneEvent(pevent))
self.testAllocateJobs()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),
1)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 0)
# pdb.set_trace()
self.assert_(self.joboffice.processJobDoneEvent(pevent))
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),
0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(), 1)
class _EventFactory(object):
def __init__(self, runid, props=None):
"""
create a generic event factor
"""
self.runid = runid
if isinstance(props, PropertySet):
self.props = props
else:
self.props = PropertySet()
if isinstance(props, dict):
for key in props.keys():
self.props.set(key, props[key])
def create(self):
"""create a new instance of the event"""
return ev.Event(self.runid, self.props)
def setRunId(self, id):
"""set the Run ID"""
self.runid = runid
def getRunId(self):
"""set the Run ID"""
return self.runid
def setProperty(self, name, val):
"""set the value of a named property"""
self.props.set(name, val)
def getProperty(self, name):
"""get the value of a named property"""
return self.props.getString(name)
def addDataset(self, propname, ds):
"""add a dataset to the event"""
self.props.add(propname, serializeDataset(ds))
def getDatasets(self, propname):
"""return the datasets attached to the event"""
return unserializeDatasetList(self.props.getArrayString(propname))
def sendEvent(self, broker, topic):
trans = events.EventTransmitter(broker, topic)
root = PropertySet()
root.set(events.Event.TOPIC,topic)
root.set("myname","myname")
root.set(events.Event.STATUS, "my special status")
root.set(events.Event.RUNID,"srptestrun")
event = events.Event("srptestrun", root)
# wait a short time so we can see the difference between the time
# the event is created and the time it is published
time.sleep(2)
# ok...now publish it
trans.publishEvent(event)
def sendEvent(self, broker, topic):
trans = events.EventTransmitter(broker, topic)
root = PropertySet()
root.set(events.Event.TOPIC, topic)
root.set("myname", "myname")
root.set(events.Event.STATUS, "my special status")
root.set(events.Event.RUNID, "srptestrun")
event = events.Event("srptestrun", root)
# wait a short time so we can see the difference between the time
# the event is created and the time it is published
time.sleep(2)
# ok...now publish it
trans.publishEvent(event)
def testToPipelineQueueItem(self):
pipelineName = "ccdassembly"
ps = PropertySet()
ps.set("pipelineName", pipelineName)
ps.set("STATUS", "done")
ps.set("RUNID", "testing")
pevent = StatusEvent("testing", originatorId, ps)
item = self.joboffice.toPipelineQueueItem(pevent)
self.assertEquals(item.getName(), pipelineName)
self.assertEquals(item.getProperty("status"), "done")
self.assertEquals(item.getRunId(), "testing")
def testToPipelineQueueItem(self):
pipelineName = "ccdassembly"
ps = PropertySet()
ps.set("pipelineName", pipelineName)
ps.set("STATUS", "done")
ps.set("RUNID", "testing")
pevent = StatusEvent("testing", originatorId, ps)
item = self.joboffice.toPipelineQueueItem(pevent)
self.assertEquals(item.getName(), pipelineName)
self.assertEquals(item.getProperty("status"), "done")
self.assertEquals(item.getRunId(), "testing")
def sendEvent(self, runid, brokerName, topic):
trans = events.EventTransmitter(brokerName, topic)
root = PropertySet()
root.set(events.Event.TOPIC, topic)
root.set("myname","myname")
root.set(events.Event.STATUS, "my special status")
locationID = events.LocationId()
event = events.StatusEvent(runid, locationID, root)
# ok...now publish it
trans.publishEvent(event)
def makeJobCommandEvent(self, job, pipeline, runId=""):
"""
create a CommandEvent to send to a pipeline instructing it to
commence working on the given job.
"""
props = PropertySet()
for ds in job.getInputDatasets():
props.add("inputs", serializePolicy(ds.toPolicy()))
for ds in job.getOutputDatasets():
props.add("outputs", serializePolicy(ds.toPolicy()))
props.set("identity", serializePolicy(job.getJobIdentity().toPolicy()))
props.set("STATUS", "job:assign")
props.set("name", job.getName())
return CommandEvent(runId, self.originatorId, pipeline, props)
def makeJobCommandEvent(self, job, pipeline, runId=""):
"""
create a CommandEvent to send to a pipeline instructing it to
commence working on the given job.
"""
props = PropertySet()
for ds in job.getInputDatasets():
props.add("inputs", serializePolicy(ds.toPolicy()))
for ds in job.getOutputDatasets():
props.add("outputs", serializePolicy(ds.toPolicy()))
props.set("identity", serializePolicy(job.getJobIdentity().toPolicy()))
props.set("STATUS", "job:assign")
props.set("name", job.getName())
return CommandEvent(runId, self.originatorId, pipeline, props)
def testProcessJobDoneEvents(self):
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),0)
self.testAllocateJobs()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),1)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),0)
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "job:done")
ps.set("success", True)
pevent = StatusEvent("testing", originatorId, ps)
trx = EventTransmitter(brokerhost, "CcdAssemblyJob")
trx.publishEvent(pevent)
time.sleep(2.0)
self.joboffice.processDoneJobs()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),1)
def testProcessJobDoneEvent(self):
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),0)
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "job:done")
ps.set("success", True)
pevent = StatusEvent("testing", originatorId, ps)
self.assert_(not self.joboffice.processJobDoneEvent(pevent))
self.testAllocateJobs()
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),1)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),0)
# pdb.set_trace()
self.assert_(self.joboffice.processJobDoneEvent(pevent))
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),1)
def preprocess(self):
self.activeClipboard = self.inputQueue.getNextDataset()
eventName = self._policy.get("inputEvent")
event = self.activeClipboard.get(eventName)
visitId = event.get("visitId")
exposureId = event.get("exposureId")
fpaExposureId = (long(visitId) << 1) + exposureId
visit = PropertySet()
visit.setInt("visitId", visitId)
visit.setLongLong("exposureId", fpaExposureId)
self.activeClipboard.put("visit" + str(exposureId), visit)
rawFpaExposure = PropertySet()
rawFpaExposure.setLongLong("rawFPAExposureId", fpaExposureId)
rawFpaExposure.set("ra", event.get("ra"))
rawFpaExposure.set("decl", event.get("decl"))
rawFpaExposure.set("filterId",
self.lookupFilterId(event.get("filter")))
rawFpaExposure.set("equinox", event.get("equinox"))
rawFpaExposure.set("dateObs", DateTime(event.get("dateObs")))
rawFpaExposure.set("mjdObs", DateTime(event.get("dateObs")).mjd())
rawFpaExposure.set("expTime", event.get("expTime"))
rawFpaExposure.set("airmass", event.get("airmass"))
self.activeClipboard.put("fpaExposure" + str(exposureId),
rawFpaExposure)
def testIllegalFilterableStatusEvent(self):
testEnv = TestEnvironment()
broker = testEnv.getBroker()
topic = self.createTopicName("test_events_10_%s.A")
#
# send a test event, and wait to receive it
#
trans = events.EventTransmitter(broker, topic)
root = PropertySet()
root.set(events.StatusEvent.TOPIC, topic)
root.set("myname","myname")
root.set(events.StatusEvent.STATUS, "my special status")
filter = PropertySet()
filter.set("FOO", "bar")
filter.set("PLOUGH", 123)
filter.set("PLOVER.FIRST", 1)
filter.set("PLOVER.SECOND", 2)
eventSystem = events.EventSystem.getDefaultEventSystem();
locationID = eventSystem.createOriginatorId()
event = events.StatusEvent(locationID, root, filter)
self.assertRaises(ex.Exception, trans.publishEvent, event)
def testRunInThread(self):
self.assert_(not self.joboffice.isAlive())
self.joboffice.start()
self.assert_(self.joboffice.isAlive())
try:
with self.joboffice.bb.queues:
self.assertEquals(
self.joboffice.bb.queues.jobsInProgress.length(), 0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),
0)
self.assertEquals(
self.joboffice.bb.queues.jobsAvailable.length(), 0)
self.assertEquals(
self.joboffice.bb.queues.dataAvailable.length(), 0)
trxpipe = EventTransmitter(brokerhost, "CcdAssemblyJob")
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "job:ready")
pevent = StatusEvent("testing", originatorId, ps)
trxpipe.publishEvent(pevent)
time.sleep(2.0)
with self.joboffice.bb.queues:
self.assertEquals(
self.joboffice.bb.queues.pipelinesReady.length(), 1)
self.assertEquals(
self.joboffice.bb.queues.jobsInProgress.length(), 0)
trxdata = EventTransmitter(brokerhost, "PostISRAvailable")
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "available")
ds = self.testDatasetFromProperty()
ds.ids["ampid"] = 0
ps.set("dataset", serializePolicy(ds.toPolicy()))
# pdb.set_trace()
for i in xrange(16):
ps.set("dataset", serializePolicy(ds.toPolicy()))
devent = StatusEvent("testing", originatorId, ps)
trxdata.publishEvent(devent)
ds = copy.deepcopy(ds)
ds.ids["ampid"] += 1
time.sleep(2.0)
with self.joboffice.bb.queues:
self.assertEquals(
self.joboffice.bb.queues.jobsInProgress.length(), 1)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),
0)
self.assertEquals(
self.joboffice.bb.queues.jobsAvailable.length(), 0)
self.assertEquals(
self.joboffice.bb.queues.dataAvailable.length(), 16)
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "job:done")
ps.set("success", True)
jevent = StatusEvent("testing", originatorId, ps)
trxpipe.publishEvent(jevent)
time.sleep(2.0)
with self.joboffice.bb.queues:
self.assertEquals(
self.joboffice.bb.queues.jobsInProgress.length(), 0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),
1)
self.assertEquals(
self.joboffice.bb.queues.jobsAvailable.length(), 0)
self.assertEquals(
self.joboffice.bb.queues.dataAvailable.length(), 16)
finally:
self.joboffice.stop()
if self.joboffice.isAlive():
self.joboffice.join(10.0)
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the LSST License Statement and
# the GNU General Public License along with this program. If not,
# see <http://www.lsstcorp.org/LegalNotices/>.
#
import lsst.ctrl.events as events
from lsst.daf.base import PropertySet
import sys
# usage: python bin/shutlog.py runid
if __name__ == "__main__":
host = "lsst8.ncsa.uiuc.edu"
topic = events.LogEvent.LOGGING_TOPIC
runid = sys.argv[1]
eventSystem = events.EventSystem.getDefaultEventSystem()
eventSystem.createTransmitter(host, topic)
props = PropertySet()
props.set("LOGGER","orca.control")
props.set("STATUS","eol")
e = events.Event(runid, props)
eventSystem.publishEvent(topic, e)
def testCommandEvent(self):
testEnv = EventsEnvironment()
broker = testEnv.getBroker()
thisHost = platform.node()
topic = "test_events_command_%s_%d" % (thisHost, os.getpid())
# send a command event
receiver = events.EventReceiver(broker, topic)
trans = events.EventTransmitter(broker, topic)
originatorId = events.LocationId()
root = PropertySet()
root.set(events.StatusEvent.TOPIC, topic)
root.set("myname", "myname")
root.set(events.StatusEvent.STATUS, "my special status")
event = events.StatusEvent("srptestrun", originatorId, root)
statusOriginatorId = event.getOriginator()
destinationID = events.LocationId(statusOriginatorId)
commandOriginatorID = events.LocationId()
root2 = PropertySet()
root2.set(events.CommandEvent.TOPIC, topic)
root2.set("myname", "myname2")
root2.set(events.CommandEvent.STATUS, "my special status2")
event2 = events.CommandEvent("srptestrun", commandOriginatorID,
destinationID, root2)
trans.publishEvent(event2)
val = receiver.receiveCommandEvent()
# be sure we received an event
self.assertIsNotNone(val)
# these are the filterable names we expect to see
names = [
events.CommandEvent.DEST_HOSTNAME,
events.CommandEvent.DEST_LOCALID,
events.CommandEvent.DEST_PROCESSID, events.CommandEvent.EVENTTIME,
events.CommandEvent.ORIG_HOSTNAME,
events.CommandEvent.ORIG_LOCALID,
events.CommandEvent.ORIG_PROCESSID, events.CommandEvent.PUBTIME,
events.CommandEvent.RUNID, events.CommandEvent.STATUS,
events.CommandEvent.TOPIC, events.CommandEvent.TYPE
]
self.assertValid(val, names, commandOriginatorID.getLocalID(),
destinationID.getLocalID())
# send a command event with additional filterable properties
receiver = events.EventReceiver(broker, topic)
trans = events.EventTransmitter(broker, topic)
originatorId = events.LocationId()
event = events.StatusEvent("srptestrun", originatorId, root)
statusOriginatorId = event.getOriginator()
destinationID = events.LocationId(statusOriginatorId)
commandOriginatorID = events.LocationId()
filterable = PropertySet()
filterable.set("FOO", 12.3)
filterable.set("BAR", .45)
event2 = events.CommandEvent("srptestrun", commandOriginatorID,
destinationID, root2, filterable)
trans.publishEvent(event2)
val = receiver.receiveCommandEvent()
# be sure we received an event
self.assertIsNotNone(val)
# these are the filterable names we expect to see
names = [
events.CommandEvent.DEST_HOSTNAME,
events.CommandEvent.DEST_LOCALID,
events.CommandEvent.DEST_PROCESSID, events.CommandEvent.EVENTTIME,
events.CommandEvent.ORIG_HOSTNAME,
events.CommandEvent.ORIG_LOCALID,
events.CommandEvent.ORIG_PROCESSID, events.CommandEvent.PUBTIME,
events.CommandEvent.RUNID, events.CommandEvent.STATUS,
events.CommandEvent.TOPIC, events.CommandEvent.TYPE, 'FOO', 'BAR'
]
self.assertValid(val, names, commandOriginatorID.getLocalID(),
destinationID.getLocalID())
self.assertTrue(event.getType(), events.EventTypes.COMMAND)
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the LSST License Statement and
# the GNU General Public License along with this program. If not,
# see <http://www.lsstcorp.org/LegalNotices/>.
#
# used to transmit status that we're not sending any more data after this
import lsst.ctrl.events as events
from lsst.daf.base import PropertySet
import sys
# usage: python dataCompleted broker topic runid
if __name__ == "__main__":
broker = sys.argv[1]
topic = sys.argv[2]
runid = sys.argv[3]
status = sys.argv[4]
trans = events.EventTransmitter(broker, topic)
eventSystem = events.EventSystem.getDefaultEventSystem()
root = PropertySet()
root.set("STATUS",status)
id = eventSystem.createOriginatorId()
event = events.StatusEvent(runid, id, root)
trans.publishEvent(event)
def makeJobStatusEvent(self, job, runId, status):
props = PropertySet()
props.set("identity", serializePolicy(job.getJobIdentity().toPolicy()))
props.set("STATUS", status)
props.set("name", job.getName())
return StatusEvent(runId, self.originatorId, props)
def testLocationId(self):
testEnv = EventsEnvironment()
broker = testEnv.getBroker()
thisHost = platform.node()
locationID = events.LocationId()
# check to see if localID is 0
localId = locationID.getLocalID()
self.assertEqual(localId, 0)
# check host name
hostname = locationID.getHostName()
self.assertEqual(hostname, socket.gethostname())
# check process id
processId = locationID.getProcessID()
self.assertEqual(processId, os.getpid())
locationID2 = events.LocationId()
# check to see if localID is 1
localId = locationID2.getLocalID()
self.assertEqual(localId, 1)
self.assertEqual(locationID.getLocalID(), 0)
# check host name
hostname = locationID.getHostName()
self.assertEqual(hostname, socket.gethostname())
# check process id
processId = locationID2.getProcessID()
self.assertEqual(processId, os.getpid())
root = PropertySet()
root.set("myname","myname")
status = "my special status"
root.set(events.Event.STATUS, status)
statusEvent = events.StatusEvent("my runid", locationID, root)
statusEvent2 = events.StatusEvent("my runid", locationID2, root)
topic = "mytopic_%s_%d" % (thisHost, os.getpid())
transmitter = events.EventTransmitter(broker, topic)
hostname = locationID2.getHostName()
# create a receiver selector with the hostname, process id and local id
sel = "%s = '%s' and %s = %d and %s = %d" % (events.StatusEvent.ORIG_HOSTNAME, hostname, events.StatusEvent.ORIG_PROCESSID, os.getpid(), events.StatusEvent.ORIG_LOCALID, 1)
receiver = events.EventReceiver(broker, topic, sel)
# transmit the events
transmitter.publishEvent(statusEvent)
transmitter.publishEvent(statusEvent2)
# should receive event with with the process id
returnedEvent = receiver.receiveEvent()
ps = returnedEvent.getPropertySet()
self.assertEqual(ps.get('ORIG_HOSTNAME'), hostname)
self.assertEqual(ps.get('ORIG_PROCESSID'), os.getpid())
self.assertEqual(ps.get('ORIG_LOCALID'), 1)
# should NOT receive another event, because it was filtered out by
# the broker
returnedEvent2 = receiver.receiveEvent(1)
self.assertIsNone(returnedEvent2)
def makeJobStatusEvent(self, job, runId, status):
props = PropertySet()
props.set("identity", serializePolicy(job.getJobIdentity().toPolicy()))
props.set("STATUS", status)
props.set("name", job.getName())
return StatusEvent(runId, self.originatorId, props)
def makeJobOfficeStatusEvent(self, runId, status):
props = PropertySet()
props.set("STATUS", status)
return StatusEvent(runId, self.originatorId, props)
def setUp(self):
# make a nominal match list where the distances are 0; test can then modify
# source centroid, reference coord or distance field for each match, as desired
ctrPix = afwGeom.Point2I(1500, 1500)
metadata = PropertySet()
metadata.set("RADECSYS", "FK5")
metadata.set("EQUINOX", 2000.0)
metadata.set("CTYPE1", "RA---TAN")
metadata.set("CTYPE2", "DEC--TAN")
metadata.set("CUNIT1", "deg")
metadata.set("CUNIT2", "deg")
metadata.set("CRVAL1", 215.5)
metadata.set("CRVAL2", 53.0)
metadata.set("CRPIX1", ctrPix[0] + 1)
metadata.set("CRPIX2", ctrPix[1] + 1)
metadata.set("CD1_1", 5.1e-05)
metadata.set("CD1_2", 0.0)
metadata.set("CD2_2", -5.1e-05)
metadata.set("CD2_1", 0.0)
self.wcs = afwImage.makeWcs(metadata)
self.bboxD = afwGeom.Box2D(afwGeom.Point2D(10, 100),
afwGeom.Extent2D(1000, 1500))
self.numMatches = 25
sourceSchema = afwTable.SourceTable.makeMinimalSchema()
# add centroid (and many other unwanted fields) to sourceSchema
SingleFrameMeasurementTask(schema=sourceSchema)
self.sourceCentroidKey = afwTable.Point2DKey(
sourceSchema["slot_Centroid"])
self.sourceCat = afwTable.SourceCatalog(sourceSchema)
refSchema = afwTable.SourceTable.makeMinimalSchema()
self.refCoordKey = afwTable.CoordKey(refSchema["coord"])
self.refCat = afwTable.SourceCatalog(refSchema)
self.matchList = []
np.random.seed(5)
pixPointList = [
afwGeom.Point2D(pos)
for pos in np.random.random_sample([self.numMatches, 2]) *
self.bboxD.getDimensions() + self.bboxD.getMin()
]
for pixPoint in pixPointList:
src = self.sourceCat.addNew()
src.set(self.sourceCentroidKey, pixPoint)
ref = self.refCat.addNew()
ref.set(self.refCoordKey, self.wcs.pixelToSky(pixPoint))
match = afwTable.ReferenceMatch(ref, src, 0)
self.matchList.append(match)
def getMetadata(keyValDict):
md = PropertySet()
for key, val in keyValDict.items():
md.set(key, val)
return md
def testLocationId(self):
testEnv = EventsEnvironment()
broker = testEnv.getBroker()
thisHost = platform.node()
locationID = events.LocationId()
# check to see if localID is 0
localId = locationID.getLocalID()
self.assertEqual(localId, 0)
# check host name
hostname = locationID.getHostName()
self.assertEqual(hostname, socket.gethostname())
# check process id
processId = locationID.getProcessID()
self.assertEqual(processId, os.getpid())
locationID2 = events.LocationId()
# check to see if localID is 1
localId = locationID2.getLocalID()
self.assertEqual(localId, 1)
self.assertEqual(locationID.getLocalID(), 0)
# check host name
hostname = locationID.getHostName()
self.assertEqual(hostname, socket.gethostname())
# check process id
processId = locationID2.getProcessID()
self.assertEqual(processId, os.getpid())
root = PropertySet()
root.set("myname", "myname")
status = "my special status"
root.set(events.Event.STATUS, status)
statusEvent = events.StatusEvent("my runid", locationID, root)
statusEvent2 = events.StatusEvent("my runid", locationID2, root)
topic = "mytopic_%s_%d" % (thisHost, os.getpid())
transmitter = events.EventTransmitter(broker, topic)
hostname = locationID2.getHostName()
# create a receiver selector with the hostname, process id and local id
sel = "%s = '%s' and %s = %d and %s = %d" % (
events.StatusEvent.ORIG_HOSTNAME, hostname,
events.StatusEvent.ORIG_PROCESSID, os.getpid(),
events.StatusEvent.ORIG_LOCALID, 1)
receiver = events.EventReceiver(broker, topic, sel)
# transmit the events
transmitter.publishEvent(statusEvent)
transmitter.publishEvent(statusEvent2)
# should receive event with with the process id
returnedEvent = receiver.receiveEvent()
ps = returnedEvent.getPropertySet()
self.assertEqual(ps.get('ORIG_HOSTNAME'), hostname)
self.assertEqual(ps.get('ORIG_PROCESSID'), os.getpid())
self.assertEqual(ps.get('ORIG_LOCALID'), 1)
# should NOT receive another event, because it was filtered out by
# the broker
returnedEvent2 = receiver.receiveEvent(1)
self.assertIsNone(returnedEvent2)
def setUp(self):
# Set up local astrometry_net_data
self.datapath = setupAstrometryNetDataDir('photocal')
self.config = LoadAstrometryNetObjectsTask.ConfigClass()
self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(3001, 3001))
self.ctrPix = afwGeom.Point2I(1500, 1500)
metadata = PropertySet()
metadata.set("RADECSYS", "FK5")
metadata.set("EQUINOX", 2000.0)
metadata.set("CTYPE1", "RA---TAN")
metadata.set("CTYPE2", "DEC--TAN")
metadata.set("CUNIT1", "deg")
metadata.set("CUNIT2", "deg")
metadata.set("CRVAL1", 215.5)
metadata.set("CRVAL2", 53.0)
metadata.set("CRPIX1", self.ctrPix[0] + 1)
metadata.set("CRPIX2", self.ctrPix[1] + 1)
metadata.set("CD1_1", 5.1e-05)
metadata.set("CD1_2", 0.0)
metadata.set("CD2_2", -5.1e-05)
metadata.set("CD2_1", 0.0)
self.wcs = makeWcs(metadata)
self.desNumStarsInPixelBox = 270
self.desNumStarsInSkyCircle = 410
def preprocess(self):
self.activeClipboard = self.inputQueue.getNextDataset()
eventName = self._policy.get("inputEvent")
event = self.activeClipboard.get(eventName)
visitId = event.get("visitId")
exposureId = event.get("exposureId")
fpaExposureId = (long(visitId) << 1) + exposureId
visit = PropertySet()
visit.setInt("visitId", visitId)
visit.setLongLong("exposureId", fpaExposureId)
self.activeClipboard.put("visit" + str(exposureId), visit)
rawFpaExposure = PropertySet()
rawFpaExposure.setLongLong("rawFPAExposureId", fpaExposureId)
rawFpaExposure.set("ra", event.get("ra"))
rawFpaExposure.set("decl", event.get("decl"))
rawFpaExposure.set("filterId",
self.lookupFilterId(event.get("filter")))
rawFpaExposure.set("equinox", event.get("equinox"))
rawFpaExposure.set("dateObs", DateTime(event.get("dateObs")))
rawFpaExposure.set("mjdObs", DateTime(event.get("dateObs")).mjd())
rawFpaExposure.set("expTime", event.get("expTime"))
rawFpaExposure.set("airmass", event.get("airmass"))
self.activeClipboard.put("fpaExposure" + str(exposureId), rawFpaExposure)
def testRunInThread(self):
self.assert_(not self.joboffice.isAlive())
self.joboffice.start()
self.assert_(self.joboffice.isAlive())
try:
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),0)
self.assertEquals(self.joboffice.bb.queues.jobsAvailable.length(),0)
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),0)
trxpipe = EventTransmitter(brokerhost, "CcdAssemblyJob")
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "job:ready")
pevent = StatusEvent("testing", originatorId, ps)
trxpipe.publishEvent(pevent)
time.sleep(2.0)
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.pipelinesReady.length(),1)
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0)
trxdata = EventTransmitter(brokerhost, "PostISRAvailable")
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "available")
ds = self.testDatasetFromProperty()
ds.ids["ampid"] = 0;
ps.set("dataset", serializePolicy(ds.toPolicy()))
# pdb.set_trace()
for i in xrange(16):
ps.set("dataset", serializePolicy(ds.toPolicy()))
devent = StatusEvent("testing", originatorId, ps)
trxdata.publishEvent(devent);
ds = copy.deepcopy(ds)
ds.ids["ampid"] += 1
time.sleep(2.0)
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),1)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),0)
self.assertEquals(self.joboffice.bb.queues.jobsAvailable.length(),0)
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),16)
ps = PropertySet()
ps.set("pipelineName", "PostISR")
ps.set("STATUS", "job:done")
ps.set("success", True)
jevent = StatusEvent("testing", originatorId, ps)
trxpipe.publishEvent(jevent)
time.sleep(2.0)
with self.joboffice.bb.queues:
self.assertEquals(self.joboffice.bb.queues.jobsInProgress.length(),0)
self.assertEquals(self.joboffice.bb.queues.jobsDone.length(),1)
self.assertEquals(self.joboffice.bb.queues.jobsAvailable.length(),0)
self.assertEquals(self.joboffice.bb.queues.dataAvailable.length(),16)
finally:
self.joboffice.stop()
if self.joboffice.isAlive():
self.joboffice.join(10.0)
def testSendEvent(self):
"""Send an Event"""
testEnv = EventsEnvironment()
self.thisHost = platform.node()
self.broker = testEnv.getBroker()
topic = "test_events_3_%s_%d" % (self.thisHost, os.getpid())
runID = "test3_runid"
recv = events.EventReceiver(self.broker, topic)
trans = events.EventTransmitter(self.broker, topic)
DATE = "date"
DATE_VAL = "2007-07-01T14:28:32.546012"
BLANK = "blank"
BLANK_VAL = ""
PID = "pid"
PID_VAL = os.getpid()
HOST = "host"
HOST_VAL = "lsstcorp.org"
IP = "ip"
IP_VAL = "1.2.3.4"
EVNT = "evnt"
EVNT_VAL = "test"
MISC1 = "misc1"
MISC1_VAL = "data 1"
MISC2 = "misc2"
MISC2_VAL = "data 2"
MISC3 = "misc3"
MISC3_VAL = ""
DATA = "data"
DATA_VAL = 3.14
root = PropertySet()
root.set(DATE, DATE_VAL)
root.set(BLANK, BLANK_VAL)
root.setInt(PID, PID_VAL)
root.set(HOST, HOST_VAL)
root.set(IP, IP_VAL)
root.set(EVNT, EVNT_VAL)
root.set(MISC1, MISC1_VAL)
root.set(MISC2, MISC2_VAL)
root.set(MISC3, MISC3_VAL)
root.setDouble(DATA, DATA_VAL)
event = events.Event(runID, root)
trans.publishEvent(event)
val = recv.receiveEvent()
self.assertIsNotNone(val)
# check the validity of all sent values
ps = val.getPropertySet()
self.assertEqual(ps.get(DATE), DATE_VAL)
self.assertEqual(ps.get(BLANK), BLANK_VAL)
self.assertEqual(ps.get(PID), PID_VAL)
self.assertEqual(ps.get(HOST), HOST_VAL)
self.assertEqual(ps.get(IP), IP_VAL)
self.assertEqual(ps.get(EVNT), EVNT_VAL)
self.assertEqual(ps.get(MISC1), MISC1_VAL)
self.assertEqual(ps.get(MISC2), MISC2_VAL)
self.assertEqual(ps.get(MISC3), MISC3_VAL)
self.assertEqual(ps.get(DATA), DATA_VAL)
self.assertGreater(ps.get(events.Event.EVENTTIME), 0)
self.assertGreater(ps.get(events.Event.PUBTIME), 0)
self.assertEqual(ps.get(events.Event.RUNID), runID)
self.assertEqual(ps.get(events.Event.STATUS), "unknown")
self.assertEqual(ps.get(events.Event.TOPIC), topic)
self.assertEqual(ps.get(events.Event.TYPE), events.EventTypes.EVENT)
#
# wait a short time to receive an event. none was sent, so we should
# time out and confirm that we didn't get anything
#
val = recv.receiveEvent(1000)
self.assertIsNone(val)
def testFilterableSendEvent(self):
testEnv = TestEnvironment()
broker = testEnv.getBroker()
thisHost = platform.node()
topic = "test_events_filters_%s_%d" % (thisHost, os.getpid())
runId = "test_filters_runid"
recv = events.EventReceiver(broker, topic)
trans = events.EventTransmitter(broker, topic)
root = PropertySet()
DATE = "date"
DATE_VAL = "2007-07-01T14:28:32.546012"
root.set(DATE, DATE_VAL)
BLANK = "blank"
BLANK_VAL = ""
root.set(BLANK, BLANK_VAL)
PID = "pid"
PID_VAL = os.getpid()
root.setInt(PID, PID_VAL)
HOST = "host"
HOST_VAL = "lsstcorp.org"
root.set(HOST, HOST_VAL)
IP = "ip"
IP_VAL = "1.2.3.4"
root.set(IP, IP_VAL)
EVNT = "evnt"
EVNT_VAL = "test"
root.set(EVNT, EVNT_VAL)
MISC1 = "misc1"
MISC1_VAL = "data 1"
root.set(MISC1, MISC1_VAL)
MISC2 = "misc2"
MISC2_VAL = "data 2"
root.set(MISC2, MISC2_VAL)
MISC3 = "misc3"
MISC3_VAL = ""
root.set(MISC3, MISC3_VAL)
DATA = "data"
DATA_VAL = 3.14
root.setDouble(DATA, DATA_VAL)
filterable = PropertySet()
filterable.set("FOO", "bar")
filterable.set("XYZZY", 123)
filterable.set("PLOUGH", 0.867)
event = events.Event(runId, root, filterable)
trans.publishEvent(event)
val = recv.receiveEvent()
self.assertIsNotNone(val)
ps = val.getPropertySet()
self.assertEqual(ps.get(DATE), DATE_VAL)
self.assertEqual(ps.get(BLANK), BLANK_VAL)
self.assertEqual(ps.get(PID), PID_VAL)
self.assertEqual(ps.get(HOST), HOST_VAL)
self.assertEqual(ps.get(IP), IP_VAL)
self.assertEqual(ps.get(EVNT), EVNT_VAL)
self.assertEqual(ps.get(MISC1), MISC1_VAL)
self.assertEqual(ps.get(MISC2), MISC2_VAL)
self.assertEqual(ps.get(MISC3), MISC3_VAL)
self.assertEqual(ps.get(DATA), DATA_VAL)
self.assertGreater(ps.get(events.Event.EVENTTIME), 0)
self.assertGreater(ps.get(events.Event.PUBTIME), 0)
self.assertEqual(ps.get(events.Event.RUNID), runId)
self.assertEqual(ps.get(events.Event.STATUS), "unknown")
self.assertEqual(ps.get(events.Event.TOPIC), topic)
self.assertEqual(ps.get(events.Event.TYPE), events.EventTypes.EVENT)
self.assertEqual(ps.get("FOO"), "bar")
self.assertEqual(ps.get("XYZZY"), 123)
self.assertEqual(ps.get("PLOUGH"), 0.867)
names = val.getFilterablePropertyNames()
values = [events.Event.EVENTTIME, 'FOO', 'PLOUGH', events.Event.PUBTIME, events.Event.RUNID, events.Event.STATUS,
events.Event.TOPIC, events.Event.TYPE, 'XYZZY']
for x in values:
self.assertTrue(x in names)
#
# wait a short time to receive an event. none was sent, so we should
# time out and confirm that we didn't get anything
#
val = recv.receiveEvent(1000)
self.assertIsNone(val)
def testCommandEvent(self):
testEnv = EventsEnvironment()
broker = testEnv.getBroker()
thisHost = platform.node()
topic = "test_events_command_%s_%d" % (thisHost, os.getpid())
# send a command event
receiver = events.EventReceiver(broker, topic)
trans = events.EventTransmitter(broker, topic)
originatorId = events.LocationId()
root = PropertySet()
root.set(events.StatusEvent.TOPIC,topic)
root.set("myname","myname")
root.set(events.StatusEvent.STATUS, "my special status")
event = events.StatusEvent("srptestrun", originatorId, root)
statusOriginatorId = event.getOriginator()
destinationID = events.LocationId(statusOriginatorId)
commandOriginatorID = events.LocationId()
root2 = PropertySet()
root2.set(events.CommandEvent.TOPIC,topic)
root2.set("myname","myname2")
root2.set(events.CommandEvent.STATUS, "my special status2")
event2 = events.CommandEvent("srptestrun", commandOriginatorID, destinationID, root2)
trans.publishEvent(event2)
val = receiver.receiveCommandEvent()
# be sure we received an event
self.assertIsNotNone(val)
# these are the filterable names we expect to see
names = [events.CommandEvent.DEST_HOSTNAME, events.CommandEvent.DEST_LOCALID, events.CommandEvent.DEST_PROCESSID,
events.CommandEvent.EVENTTIME, events.CommandEvent.ORIG_HOSTNAME, events.CommandEvent.ORIG_LOCALID,
events.CommandEvent.ORIG_PROCESSID, events.CommandEvent.PUBTIME, events.CommandEvent.RUNID,
events.CommandEvent.STATUS, events.CommandEvent.TOPIC, events.CommandEvent.TYPE]
self.assertValid(val, names, commandOriginatorID.getLocalID(), destinationID.getLocalID())
# send a command event with additional filterable properties
receiver = events.EventReceiver(broker, topic)
trans = events.EventTransmitter(broker, topic)
originatorId = events.LocationId()
event = events.StatusEvent("srptestrun", originatorId, root)
statusOriginatorId = event.getOriginator()
destinationID = events.LocationId(statusOriginatorId)
commandOriginatorID = events.LocationId()
filterable = PropertySet()
filterable.set("FOO", 12.3)
filterable.set("BAR", .45)
event2 = events.CommandEvent("srptestrun", commandOriginatorID, destinationID, root2, filterable)
trans.publishEvent(event2)
val = receiver.receiveCommandEvent()
# be sure we received an event
self.assertIsNotNone(val)
# these are the filterable names we expect to see
names = [events.CommandEvent.DEST_HOSTNAME, events.CommandEvent.DEST_LOCALID, events.CommandEvent.DEST_PROCESSID,
events.CommandEvent.EVENTTIME, events.CommandEvent.ORIG_HOSTNAME, events.CommandEvent.ORIG_LOCALID,
events.CommandEvent.ORIG_PROCESSID, events.CommandEvent.PUBTIME, events.CommandEvent.RUNID,
events.CommandEvent.STATUS, events.CommandEvent.TOPIC, events.CommandEvent.TYPE, 'FOO', 'BAR']
self.assertValid(val, names, commandOriginatorID.getLocalID(), destinationID.getLocalID())
self.assertTrue(event.getType(), events.EventTypes.COMMAND)
def sendFilterableStatusEvent(self, broker, topic, runID=None):
trans = events.EventTransmitter(broker, topic)
root = PropertySet()
root.set(events.Event.TOPIC, topic)
root.set("myname", "myname")
root.set(events.Event.STATUS, "my special status")
filter = PropertySet()
filter.set("FOO", "bar")
filter.set("PLOUGH", 123)
filter.set("PLOVER", 0.45)
eventSystem = events.EventSystem.getDefaultEventSystem()
locationID = eventSystem.createOriginatorId()
if runID is None:
event = events.StatusEvent(locationID, root, filter)
else:
event = events.StatusEvent(runID, locationID, root, filter)
# ok...now publish it
trans.publishEvent(event)
def propertySetFromDict(keyValDict):
"""Make an lsst.daf.base.PropertySet from a dict of key: value"""
metadata = PropertySet()
for key, val in keyValDict.items():
metadata.set(key, val)
return metadata
def makeJobOfficeStatusEvent(self, runId, status):
props = PropertySet()
props.set("STATUS",status)
return StatusEvent(runId, self.originatorId, props)
