def testFromPolicy(self):
type = "CalExp"
path = "goob/CalExp-v88-c12.fits"
ccdid = 12
visitid = 88
p = Policy()
p.set("type", type)
# pdb.set_trace()
ds = Dataset.fromPolicy(p)
self.assertEquals(ds.type, type)
self.assert_(ds.path is None)
self.assert_(ds.ids is None)
p.set("path", path)
ds = Dataset.fromPolicy(p)
self.assertEquals(ds.type, type)
self.assertEquals(ds.path, path)
self.assert_(ds.ids is None)
p.set("ids.ccdid", ccdid)
p.set("ids.visitid", visitid)
ds = Dataset.fromPolicy(p)
self.assertEquals(ds.type, type)
self.assertEquals(ds.path, path)
self.assert_(ds.ids is not None)
self.assertEquals(ds.ids["ccdid"], ccdid)
self.assertEquals(ds.ids["visitid"], visitid)
def testFromPolicy(self):
type = "CalExp"
path = "goob/CalExp-v88-c12.fits"
ccdid = 12
visitid = 88
p = Policy()
p.set("type", type)
# pdb.set_trace()
ds = Dataset.fromPolicy(p)
self.assertEquals(ds.type, type)
self.assert_(ds.path is None)
self.assert_(ds.ids is None)
p.set("path", path)
ds = Dataset.fromPolicy(p)
self.assertEquals(ds.type, type)
self.assertEquals(ds.path, path)
self.assert_(ds.ids is None)
p.set("ids.ccdid", ccdid)
p.set("ids.visitid", visitid)
ds = Dataset.fromPolicy(p)
self.assertEquals(ds.type, type)
self.assertEquals(ds.path, path)
self.assert_(ds.ids is not None)
self.assertEquals(ds.ids["ccdid"], ccdid)
self.assertEquals(ds.ids["visitid"], visitid)
def testToString(self):
type = "CalExp"
path = "goob/CalExp-v88-c12.fits"
ccdid = 12
visitid = 88
ds = Dataset(type, ids={"ccdid": ccdid, "visitid": visitid})
self.assertEquals(ds.toString(),
"%s-ccdid%s-visitid%s" % (type, ccdid, visitid))
def testToString(self):
type = "CalExp"
path = "goob/CalExp-v88-c12.fits"
ccdid = 12
visitid = 88
ds = Dataset(type, ids={"ccdid": ccdid, "visitid": visitid })
self.assertEquals(ds.toString(),
"%s-ccdid%s-visitid%s" % (type, ccdid, visitid))
def testToPolicy(self):
type = "CalExp"
path = "goob/CalExp-v88-c12.fits"
ccdid = 12
visitid = 88
orig = Dataset(type, path, ccdid=ccdid, visitid=visitid)
pol = orig.toPolicy()
ds = Dataset.fromPolicy(pol)
self.assertEquals(ds.type, type)
self.assertEquals(ds.path, path)
self.assert_(ds.ids is not None)
self.assertEquals(ds.ids["ccdid"], ccdid)
self.assertEquals(ds.ids["visitid"], visitid)
def testToPolicy(self):
type = "CalExp"
path = "goob/CalExp-v88-c12.fits"
ccdid = 12
visitid = 88
orig = Dataset(type, path, ccdid=ccdid, visitid=visitid)
pol = orig.toPolicy()
ds = Dataset.fromPolicy(pol)
self.assertEquals(ds.type, type)
self.assertEquals(ds.path, path)
self.assert_(ds.ids is not None)
self.assertEquals(ds.ids["ccdid"], ccdid)
self.assertEquals(ds.ids["visitid"], visitid)
def testCtor(self):
type = "CalExp"
path = "goob/CalExp-v88-c12.fits"
ccdid = 12
visitid = 88
ds = Dataset(type)
self.assertEquals(ds.type, type)
self.assert_(ds.path is None)
self.assert_(ds.ids is None)
ds = Dataset(type, path)
self.assertEquals(ds.type, type)
self.assertEquals(ds.path, path)
self.assert_(ds.ids is None)
ds = Dataset(type, ccdid=ccdid, visitid=visitid)
self.assertEquals(ds.type, type)
self.assert_(ds.path is None)
self.assert_(ds.ids is not None)
self.assertEquals(ds.ids["ccdid"], ccdid)
self.assertEquals(ds.ids["visitid"], visitid)
# pdb.set_trace()
ds = Dataset(type, path, False, {"ccdid": ccdid, "visitid": visitid})
self.assertEquals(ds.type, type)
self.assertEquals(ds.path, path)
self.assert_(not ds.valid)
self.assert_(ds.ids is not None)
self.assertEquals(ds.ids["ccdid"], ccdid)
self.assertEquals(ds.ids["visitid"], visitid)
ds = Dataset(type, ids={"ccdid": ccdid, "visitid": visitid})
self.assertEquals(ds.type, type)
self.assert_(ds.path is None)
self.assert_(ds.ids is not None)
self.assertEquals(ds.ids["ccdid"], ccdid)
self.assertEquals(ds.ids["visitid"], visitid)
def setUp(self):
afwDataDir = eups.productDir("afwdata")
self.root = afwDataDir+"/ImSim"
if not os.path.exists("registry.sqlite3"):
os.symlink(os.path.join(self.root, "registry.sqlite3"),
"./registry.sqlite3")
self.clipboard = pexClipboard.Clipboard()
self.clipboard.put('jobIdentity', {
'visit': 85751839, 'snap': 0,
'raft': "2,3", 'sensor': "1,1", 'channel': "0,0", 'filter': "r"
})
self.clipboard.put('inputDatasets', [
Dataset("raw", visit=85751839, snap=0,
raft="2,3", sensor="1,1", channel="0,0", filter="r")
])
def testEquals(self):
type = "CalExp"
path = "goob/CalExp-v88-c12.fits"
ccdid = 12
visitid = 88
ds1 = Dataset(type, path, ccdid=ccdid, visitid=visitid)
ds2 = Dataset(type, path, ccdid=ccdid, visitid=visitid)
self.assert_(ds1 == ds2)
self.assertEquals(ds1, ds2)
self.assertEquals(ds2, ds1)
self.assert_(ds1 in [ds2])
ds2.ids["ccdid"] += 1
self.assertNotEquals(ds1, ds2)
self.assertNotEquals(ds2, ds1)
self.assert_(ds1 not in [ds2])
ds2 = Dataset(type, path, ccdid=ccdid, visitid=visitid, ampid=5)
self.assertNotEquals(ds1, ds2)
self.assertNotEquals(ds2, ds1)
ds2 = Dataset("junk", path, ccdid=ccdid, visitid=visitid)
self.assertNotEquals(ds1, ds2)
self.assertNotEquals(ds2, ds1)
ds2 = Dataset(type)
self.assertNotEquals(ds1, ds2)
self.assertNotEquals(ds2, ds1)
ds2 = Dataset(None, ccdid=ccdid, visitid=visitid)
self.assertNotEquals(ds1, ds2)
self.assertNotEquals(ds2, ds1)
ds1 = Dataset(None, ccdid=ccdid, visitid=visitid)
self.assertEquals(ds1, ds2)
self.assertEquals(ds2, ds1)
import lsst.pex.harness.simpleStageTester as SST
import lsst.pex.harness.IOStage as IOStage
import lsst.daf.base as dafBase
import lsst.daf.persistence as dafPersist
import lsst.afw.image as afwImage
from lsst.pex.harness import Dataset
lm = dafBase.PropertySet()
lm.add("input", "tests")
lm.add("output", ".")
dafPersist.LogicalLocation.setLocationMap(lm)
ps = dafBase.PropertySet()
ps.add("field", "D4")
clip0 = {
'inputDatasets': [
Dataset('postIsr', visit=707911, snap=0, ccd=13, amp=0),
Dataset('postIsr', visit=707911, snap=0, ccd=13, amp=1)
],
'root':
"raw",
'ps':
ps
}
policy = pexPolicy.Policy.createPolicy("tests/policy/TestIO_2.policy")
t1 = SST.SimpleStageTester(IOStage.InputStage(policy))
clip1 = t1.run(clip0, 0)
el = clip1.get("exposureSet")
assert len(el) == 2
e = afwImage.DecoratedImageU()
assert type(el[0]) == type(e)
assert type(el[1]) == type(e)
lm.add("input", "tests")
lm.add("output", ".")
dafPersist.LogicalLocation.setLocationMap(lm)
clip0 = {
'jobIdentity': {
'visit': 85751839,
'snap': 0,
'raft': "R:2,3",
'sensor': "S:1,1",
'channel': "C:0,0"
},
'inputDatasets': [
Dataset("raw",
visit=85751839,
snap=0,
raft="R:2,3",
sensor="S:1,1",
channel="C:0,0")
]
}
policy = pexPolicy.Policy.createPolicy("tests/policy/TestIO_4.policy")
t1 = SST.SimpleStageTester(IOStage.InputStage(policy))
clip1 = t1.run(clip0, 0)
rci = clip1.get("rawCameraImage")
assert rci.getMetadata().get('LSSTAMP') == "R:2,3 S:1,1 C:0,0"
bias = clip1.get("biasExposure")
assert bias.getHeight() == 2001
dark = clip1.get("darkExposure")
assert dark.getWidth() == 513
flat = clip1.get("flatExposure")
assert flat.getHeight() == 2001
def foo():
if len(sys.argv) == 3:
ix = int(sys.argv[1])
iy = int(sys.argv[2])
else:
ix = 0
iy = 0
afwDataDir = eups.productDir("afwdata")
obsDir = eups.productDir('obs_lsstSim')
isrDir = eups.productDir('ip_isr')
pipelineDir = eups.productDir('ip_pipeline')
lutpolicypath = os.path.join(isrDir, "pipeline")
pipepolicypath = os.path.join(pipelineDir, "policy")
dc3MetadataFile = 'dc3MetadataPolicy.paf'
simDatatypeFile = 'simDataTypePolicy.paf'
suffix = "Keyword"
"""Pipeline test case."""
clipboard = pexClipboard.Clipboard()
clipboard.put(
'jobIdentity', {
'visit': 2,
'snap': 0,
'raft': "R:2,3",
'sensor': "S:1,1",
'channel': "%i%i" % (ix, iy)
})
clipboard.put('inputDatasets', [
Dataset("raw",
visit=2,
snap=0,
raft="R:2,3",
sensor="S:1,1",
channel="%i%i" % (ix, iy))
])
clipboard.put("fwhm", 5.)
p0 = pexPolicy.Policy.createPolicy("MakeCalibInputStage.paf")
s0 = lsst.pex.harness.IOStage.InputStage(p0)
t0 = SimpleStageTester(s0)
file = pexPolicy.DefaultPolicyFile("ip_pipeline",
"IsrSaturationStageDictionary.paf",
"policy")
p2 = pexPolicy.Policy.createPolicy(file)
s2 = ipPipe.IsrSaturationStage(p2)
t2 = SimpleStageTester(s2)
file = pexPolicy.DefaultPolicyFile("ip_pipeline",
"IsrOverscanStageDictionary.paf",
"policy")
p3 = pexPolicy.Policy.createPolicy(file)
clipboard.put(p3.get("inputKeys.overscanfittype"), "MEDIAN")
s3 = ipPipe.IsrOverscanStage(p3)
t3 = SimpleStageTester(s3)
file = pexPolicy.DefaultPolicyFile("ip_pipeline",
"IsrBiasStageDictionary.paf", "policy")
p4 = pexPolicy.Policy.createPolicy(file)
s4 = ipPipe.IsrBiasStage(p4)
t4 = SimpleStageTester(s4)
file = pexPolicy.DefaultPolicyFile("ip_pipeline",
"IsrDarkStageDictionary.paf", "policy")
p5 = pexPolicy.Policy.createPolicy(file)
s5 = ipPipe.IsrDarkStage(p5)
t5 = SimpleStageTester(s5)
o0 = t0.runWorker(clipboard)
o2 = t2.runWorker(o0)
if display:
ds9.mtv(o2.get(p2.get("outputKeys.saturationCorrectedExposure")),
frame=0,
title="Sat")
o2.put(p3.get("inputKeys.exposure"),
o2.get(p2.get("outputKeys.saturationCorrectedExposure")))
o3 = t3.runWorker(o2)
if display:
ds9.mtv(o3.get(p3.get("outputKeys.overscanCorrectedExposure")),
frame=1,
title="Overscan")
o3.put(p4.get("inputKeys.exposure"),
o3.get(p3.get("outputKeys.overscanCorrectedExposure")))
o4 = t4.runWorker(o3)
if display:
ds9.mtv(o4.get(p4.get("outputKeys.biasSubtractedExposure")),
frame=3,
title="Bias")
o4.put(p5.get("inputKeys.exposure"),
o4.get(p4.get("outputKeys.biasSubtractedExposure")))
o5 = t5.runWorker(o4)
if display:
ds9.mtv(o5.get(p5.get("outputKeys.darkSubtractedExposure")),
frame=4,
title="Dark")
exposure = o5.get(p5.get("outputKeys.darkSubtractedExposure"))
exposure.writeFits("flat%i%i.fits" % (ix, iy))
if display:
ds9.mtv(exposure, frame=5, title="Output")
def _output(stage, policy, clipboard, log):
"""Perform the actual persistence.
@param stage The stage requesting output.
@param policy The policy for the stage.
@param clipboard The clipboard for the stage. The persisted objects are taken from this.
@param log A logger for messages.
"""
if not policy.exists('parameters.outputItems'):
# Propagate the clipboard to the output queue, but otherwise
# do nothing.
log.log(Log.WARN, "No outputItems found")
return
mainAdditionalData = lsst.pex.harness.Utils.createAdditionalData(
stage, policy, clipboard)
# Create a persistence object using policy, if present.
if policy.exists('parameters.persistence'):
persistencePolicy = pexPolicy.Policy(
policy.getPolicy('parameters.persistence'))
else:
persistencePolicy = pexPolicy.Policy()
persistence = dafPersist.Persistence.getPersistence(persistencePolicy)
# Iterate over items in OutputItems policy.
outputPolicy = policy.getPolicy('parameters.outputItems')
itemNames = outputPolicy.policyNames(True)
somethingWasOutput = False
for item in itemNames:
additionalData = mainAdditionalData.deepCopy()
itemPolicy = outputPolicy.getPolicy(item)
# Skip the item if it is not required and is not present.
itemRequired = itemPolicy.exists('required') and \
itemPolicy.getBool('required')
if not clipboard.contains(item):
if itemRequired:
raise RuntimeError, 'Missing output item: ' + item
else:
continue
itemData = clipboard.get(item)
# Add the item name to the additionalData.
additionalData.set('itemName', item)
if itemPolicy.exists('datasetId'):
dsPolicy = itemPolicy.getPolicy('datasetId')
ds = Dataset(dsPolicy.get('datasetType'))
ds.ids = {}
if dsPolicy.exists('set'):
setPolicy = dsPolicy.getPolicy('set')
for param in setPolicy.paramNames():
ds.ids[param] = setPolicy.get(param)
additionalData.set(param, setPolicy.get(param))
if dsPolicy.exists('fromJobIdentity'):
jobIdentity = clipboard.get(
policy.get('inputKeys.jobIdentity'))
for id in dsPolicy.getStringArray('fromJobIdentity'):
ds.ids[id] = jobIdentity[id]
additionalData.set(id, jobIdentity[id])
outputKey = policy.get('outputKeys.outputDatasets')
dsList = clipboard.get(outputKey)
if dsList is None:
dsList = []
clipboard.put(outputKey, dsList)
dsList.append(ds)
if stage.butler is not None:
# Use the butler to figure out storage and locations.
# Write Using Butler
iolog = BlockTimingLog(log, "write_using_butler", Log.INFO - 1)
iolog.start("persisting %s as %s with keys %s" %
(item, ds.type, ds.ids))
stage.butler.put(itemData, ds.type, dataId=ds.ids)
iolog.done()
somethingWasOutput = True
continue
# Get the item's StoragePolicy.
if itemPolicy.isArray('storagePolicy'):
policyList = itemPolicy.getPolicyArray('storagePolicy')
else:
policyList = []
policyList.append(itemPolicy.getPolicy('storagePolicy'))
# Create a list of Storages for the item based on policy.
storageList = dafPersist.StorageList()
for storagePolicy in policyList:
storageName = storagePolicy.getString('storage')
location = storagePolicy.getString('location')
logLoc = dafPersist.LogicalLocation(location, additionalData)
log.log(Log.INFO - 1,
"persisting %s to %s" % (item, logLoc.locString()))
additionalData.add('StorageLocation.' + storageName,
logLoc.locString())
mainAdditionalData.add('StorageLocation.' + storageName,
logLoc.locString())
storage = persistence.getPersistStorage(storageName, logLoc)
storageList.append(storage)
# Persist the item.
# Write Without Butler
iolog = BlockTimingLog(log, "write_without_butler", Log.INFO - 1)
iolog.start("persisting %s" % (item, ))
if hasattr(itemData, '__deref__'):
persistence.persist(itemData.__deref__(), storageList,
additionalData)
else:
persistence.persist(itemData, storageList, additionalData)
iolog.done()
somethingWasOutput = True
if not somethingWasOutput:
log.log(Log.WARN, "No items were output")