def testToString(self): type = "CalExp" path = "goob/CalExp-v88-c12.fits" ccdid = 12 visitid = 88 ds = Dataset(type, ids={"ccdid": ccdid, "visitid": visitid}) self.assertEquals(ds.toString(), "%s-ccdid%s-visitid%s" % (type, ccdid, visitid))
def testCtor(self): type = "CalExp" path = "goob/CalExp-v88-c12.fits" ccdid = 12 visitid = 88 ds = Dataset(type) self.assertEquals(ds.type, type) self.assert_(ds.path is None) self.assert_(ds.ids is None) ds = Dataset(type, path) self.assertEquals(ds.type, type) self.assertEquals(ds.path, path) self.assert_(ds.ids is None) ds = Dataset(type, ccdid=ccdid, visitid=visitid) self.assertEquals(ds.type, type) self.assert_(ds.path is None) self.assert_(ds.ids is not None) self.assertEquals(ds.ids["ccdid"], ccdid) self.assertEquals(ds.ids["visitid"], visitid) # pdb.set_trace() ds = Dataset(type, path, False, {"ccdid": ccdid, "visitid": visitid}) self.assertEquals(ds.type, type) self.assertEquals(ds.path, path) self.assert_(not ds.valid) self.assert_(ds.ids is not None) self.assertEquals(ds.ids["ccdid"], ccdid) self.assertEquals(ds.ids["visitid"], visitid) ds = Dataset(type, ids={"ccdid": ccdid, "visitid": visitid}) self.assertEquals(ds.type, type) self.assert_(ds.path is None) self.assert_(ds.ids is not None) self.assertEquals(ds.ids["ccdid"], ccdid) self.assertEquals(ds.ids["visitid"], visitid)
def testToPolicy(self): type = "CalExp" path = "goob/CalExp-v88-c12.fits" ccdid = 12 visitid = 88 orig = Dataset(type, path, ccdid=ccdid, visitid=visitid) pol = orig.toPolicy() ds = Dataset.fromPolicy(pol) self.assertEquals(ds.type, type) self.assertEquals(ds.path, path) self.assert_(ds.ids is not None) self.assertEquals(ds.ids["ccdid"], ccdid) self.assertEquals(ds.ids["visitid"], visitid)
def setUp(self): afwDataDir = eups.productDir("afwdata") self.root = afwDataDir+"/ImSim" if not os.path.exists("registry.sqlite3"): os.symlink(os.path.join(self.root, "registry.sqlite3"), "./registry.sqlite3") self.clipboard = pexClipboard.Clipboard() self.clipboard.put('jobIdentity', { 'visit': 85751839, 'snap': 0, 'raft': "2,3", 'sensor': "1,1", 'channel': "0,0", 'filter': "r" }) self.clipboard.put('inputDatasets', [ Dataset("raw", visit=85751839, snap=0, raft="2,3", sensor="1,1", channel="0,0", filter="r") ])
def testEquals(self): type = "CalExp" path = "goob/CalExp-v88-c12.fits" ccdid = 12 visitid = 88 ds1 = Dataset(type, path, ccdid=ccdid, visitid=visitid) ds2 = Dataset(type, path, ccdid=ccdid, visitid=visitid) self.assert_(ds1 == ds2) self.assertEquals(ds1, ds2) self.assertEquals(ds2, ds1) self.assert_(ds1 in [ds2]) ds2.ids["ccdid"] += 1 self.assertNotEquals(ds1, ds2) self.assertNotEquals(ds2, ds1) self.assert_(ds1 not in [ds2]) ds2 = Dataset(type, path, ccdid=ccdid, visitid=visitid, ampid=5) self.assertNotEquals(ds1, ds2) self.assertNotEquals(ds2, ds1) ds2 = Dataset("junk", path, ccdid=ccdid, visitid=visitid) self.assertNotEquals(ds1, ds2) self.assertNotEquals(ds2, ds1) ds2 = Dataset(type) self.assertNotEquals(ds1, ds2) self.assertNotEquals(ds2, ds1) ds2 = Dataset(None, ccdid=ccdid, visitid=visitid) self.assertNotEquals(ds1, ds2) self.assertNotEquals(ds2, ds1) ds1 = Dataset(None, ccdid=ccdid, visitid=visitid) self.assertEquals(ds1, ds2) self.assertEquals(ds2, ds1)
import lsst.pex.harness.simpleStageTester as SST import lsst.pex.harness.IOStage as IOStage import lsst.daf.base as dafBase import lsst.daf.persistence as dafPersist import lsst.afw.image as afwImage from lsst.pex.harness import Dataset lm = dafBase.PropertySet() lm.add("input", "tests") lm.add("output", ".") dafPersist.LogicalLocation.setLocationMap(lm) ps = dafBase.PropertySet() ps.add("field", "D4") clip0 = { 'inputDatasets': [ Dataset('postIsr', visit=707911, snap=0, ccd=13, amp=0), Dataset('postIsr', visit=707911, snap=0, ccd=13, amp=1) ], 'root': "raw", 'ps': ps } policy = pexPolicy.Policy.createPolicy("tests/policy/TestIO_2.policy") t1 = SST.SimpleStageTester(IOStage.InputStage(policy)) clip1 = t1.run(clip0, 0) el = clip1.get("exposureSet") assert len(el) == 2 e = afwImage.DecoratedImageU() assert type(el[0]) == type(e) assert type(el[1]) == type(e)
lm.add("input", "tests") lm.add("output", ".") dafPersist.LogicalLocation.setLocationMap(lm) clip0 = { 'jobIdentity': { 'visit': 85751839, 'snap': 0, 'raft': "R:2,3", 'sensor': "S:1,1", 'channel': "C:0,0" }, 'inputDatasets': [ Dataset("raw", visit=85751839, snap=0, raft="R:2,3", sensor="S:1,1", channel="C:0,0") ] } policy = pexPolicy.Policy.createPolicy("tests/policy/TestIO_4.policy") t1 = SST.SimpleStageTester(IOStage.InputStage(policy)) clip1 = t1.run(clip0, 0) rci = clip1.get("rawCameraImage") assert rci.getMetadata().get('LSSTAMP') == "R:2,3 S:1,1 C:0,0" bias = clip1.get("biasExposure") assert bias.getHeight() == 2001 dark = clip1.get("darkExposure") assert dark.getWidth() == 513 flat = clip1.get("flatExposure") assert flat.getHeight() == 2001
def foo(): if len(sys.argv) == 3: ix = int(sys.argv[1]) iy = int(sys.argv[2]) else: ix = 0 iy = 0 afwDataDir = eups.productDir("afwdata") obsDir = eups.productDir('obs_lsstSim') isrDir = eups.productDir('ip_isr') pipelineDir = eups.productDir('ip_pipeline') lutpolicypath = os.path.join(isrDir, "pipeline") pipepolicypath = os.path.join(pipelineDir, "policy") dc3MetadataFile = 'dc3MetadataPolicy.paf' simDatatypeFile = 'simDataTypePolicy.paf' suffix = "Keyword" """Pipeline test case.""" clipboard = pexClipboard.Clipboard() clipboard.put( 'jobIdentity', { 'visit': 2, 'snap': 0, 'raft': "R:2,3", 'sensor': "S:1,1", 'channel': "%i%i" % (ix, iy) }) clipboard.put('inputDatasets', [ Dataset("raw", visit=2, snap=0, raft="R:2,3", sensor="S:1,1", channel="%i%i" % (ix, iy)) ]) clipboard.put("fwhm", 5.) p0 = pexPolicy.Policy.createPolicy("MakeCalibInputStage.paf") s0 = lsst.pex.harness.IOStage.InputStage(p0) t0 = SimpleStageTester(s0) file = pexPolicy.DefaultPolicyFile("ip_pipeline", "IsrSaturationStageDictionary.paf", "policy") p2 = pexPolicy.Policy.createPolicy(file) s2 = ipPipe.IsrSaturationStage(p2) t2 = SimpleStageTester(s2) file = pexPolicy.DefaultPolicyFile("ip_pipeline", "IsrOverscanStageDictionary.paf", "policy") p3 = pexPolicy.Policy.createPolicy(file) clipboard.put(p3.get("inputKeys.overscanfittype"), "MEDIAN") s3 = ipPipe.IsrOverscanStage(p3) t3 = SimpleStageTester(s3) file = pexPolicy.DefaultPolicyFile("ip_pipeline", "IsrBiasStageDictionary.paf", "policy") p4 = pexPolicy.Policy.createPolicy(file) s4 = ipPipe.IsrBiasStage(p4) t4 = SimpleStageTester(s4) file = pexPolicy.DefaultPolicyFile("ip_pipeline", "IsrDarkStageDictionary.paf", "policy") p5 = pexPolicy.Policy.createPolicy(file) s5 = ipPipe.IsrDarkStage(p5) t5 = SimpleStageTester(s5) o0 = t0.runWorker(clipboard) o2 = t2.runWorker(o0) if display: ds9.mtv(o2.get(p2.get("outputKeys.saturationCorrectedExposure")), frame=0, title="Sat") o2.put(p3.get("inputKeys.exposure"), o2.get(p2.get("outputKeys.saturationCorrectedExposure"))) o3 = t3.runWorker(o2) if display: ds9.mtv(o3.get(p3.get("outputKeys.overscanCorrectedExposure")), frame=1, title="Overscan") o3.put(p4.get("inputKeys.exposure"), o3.get(p3.get("outputKeys.overscanCorrectedExposure"))) o4 = t4.runWorker(o3) if display: ds9.mtv(o4.get(p4.get("outputKeys.biasSubtractedExposure")), frame=3, title="Bias") o4.put(p5.get("inputKeys.exposure"), o4.get(p4.get("outputKeys.biasSubtractedExposure"))) o5 = t5.runWorker(o4) if display: ds9.mtv(o5.get(p5.get("outputKeys.darkSubtractedExposure")), frame=4, title="Dark") exposure = o5.get(p5.get("outputKeys.darkSubtractedExposure")) exposure.writeFits("flat%i%i.fits" % (ix, iy)) if display: ds9.mtv(exposure, frame=5, title="Output")
def _output(stage, policy, clipboard, log): """Perform the actual persistence. @param stage The stage requesting output. @param policy The policy for the stage. @param clipboard The clipboard for the stage. The persisted objects are taken from this. @param log A logger for messages. """ if not policy.exists('parameters.outputItems'): # Propagate the clipboard to the output queue, but otherwise # do nothing. log.log(Log.WARN, "No outputItems found") return mainAdditionalData = lsst.pex.harness.Utils.createAdditionalData( stage, policy, clipboard) # Create a persistence object using policy, if present. if policy.exists('parameters.persistence'): persistencePolicy = pexPolicy.Policy( policy.getPolicy('parameters.persistence')) else: persistencePolicy = pexPolicy.Policy() persistence = dafPersist.Persistence.getPersistence(persistencePolicy) # Iterate over items in OutputItems policy. outputPolicy = policy.getPolicy('parameters.outputItems') itemNames = outputPolicy.policyNames(True) somethingWasOutput = False for item in itemNames: additionalData = mainAdditionalData.deepCopy() itemPolicy = outputPolicy.getPolicy(item) # Skip the item if it is not required and is not present. itemRequired = itemPolicy.exists('required') and \ itemPolicy.getBool('required') if not clipboard.contains(item): if itemRequired: raise RuntimeError, 'Missing output item: ' + item else: continue itemData = clipboard.get(item) # Add the item name to the additionalData. additionalData.set('itemName', item) if itemPolicy.exists('datasetId'): dsPolicy = itemPolicy.getPolicy('datasetId') ds = Dataset(dsPolicy.get('datasetType')) ds.ids = {} if dsPolicy.exists('set'): setPolicy = dsPolicy.getPolicy('set') for param in setPolicy.paramNames(): ds.ids[param] = setPolicy.get(param) additionalData.set(param, setPolicy.get(param)) if dsPolicy.exists('fromJobIdentity'): jobIdentity = clipboard.get( policy.get('inputKeys.jobIdentity')) for id in dsPolicy.getStringArray('fromJobIdentity'): ds.ids[id] = jobIdentity[id] additionalData.set(id, jobIdentity[id]) outputKey = policy.get('outputKeys.outputDatasets') dsList = clipboard.get(outputKey) if dsList is None: dsList = [] clipboard.put(outputKey, dsList) dsList.append(ds) if stage.butler is not None: # Use the butler to figure out storage and locations. # Write Using Butler iolog = BlockTimingLog(log, "write_using_butler", Log.INFO - 1) iolog.start("persisting %s as %s with keys %s" % (item, ds.type, ds.ids)) stage.butler.put(itemData, ds.type, dataId=ds.ids) iolog.done() somethingWasOutput = True continue # Get the item's StoragePolicy. if itemPolicy.isArray('storagePolicy'): policyList = itemPolicy.getPolicyArray('storagePolicy') else: policyList = [] policyList.append(itemPolicy.getPolicy('storagePolicy')) # Create a list of Storages for the item based on policy. storageList = dafPersist.StorageList() for storagePolicy in policyList: storageName = storagePolicy.getString('storage') location = storagePolicy.getString('location') logLoc = dafPersist.LogicalLocation(location, additionalData) log.log(Log.INFO - 1, "persisting %s to %s" % (item, logLoc.locString())) additionalData.add('StorageLocation.' + storageName, logLoc.locString()) mainAdditionalData.add('StorageLocation.' + storageName, logLoc.locString()) storage = persistence.getPersistStorage(storageName, logLoc) storageList.append(storage) # Persist the item. # Write Without Butler iolog = BlockTimingLog(log, "write_without_butler", Log.INFO - 1) iolog.start("persisting %s" % (item, )) if hasattr(itemData, '__deref__'): persistence.persist(itemData.__deref__(), storageList, additionalData) else: persistence.persist(itemData, storageList, additionalData) iolog.done() somethingWasOutput = True if not somethingWasOutput: log.log(Log.WARN, "No items were output")