def test2(self): """ Test PropertySet persistence to database with policy mapping itemName to database table name. """ dp = dafBase.PropertySet() dp.addInt("intField", 2) dp.addDouble("doubleField", 2.3) dp.addString("varcharField", "gnitseT") dp.addBool("boolField", False) dp.addLongLong("int64Field", 9988776655) dp.addFloat("floatField", 2.718) pol = pexPolicy.Policy() itemPol = pexPolicy.Policy() itemPol.set("TableName", "Persistence_Test_2") pol.set("Formatter.PropertySet.testItem", itemPol) additionalData = dafBase.PropertySet() additionalData.add("itemName", "testItem") loc = dafPersist.LogicalLocation("mysql://{}:{}/test".format(HOST, PORT)) persistence = dafPersist.Persistence.getPersistence(pol) storageList = dafPersist.StorageList() storage = persistence.getPersistStorage("DbStorage", loc) storageList.append(storage) persistence.persist(dp, storageList, additionalData)
def __init__( self, # Config policy=None # Filename/file/policy for configuration ): if policy is None: self._policy = pexPolicy.Policy() elif isinstance(policy, pexPolicy.Policy): self._policy = policy elif isinstance(policy, basestring): self._policy = pexPolicy.Policy.createPolicy( pexPolicy.PolicyFile(policy), os.path.dirname(policy)) elif isinstance(policy, pexPolicy.DefaultPolicyFile): self._policy = pexPolicy.Policy.createPolicy( policy, policy.getRepositoryPath()) elif isinstance(policy, pexPolicy.PolicyFile): self._policy = pexPolicy.Policy.createPolicy(policy) elif isinstance(policy, dict): self._policy = pexPolicy.Policy() for key, value in policy.items(): if isinstance(value, dict): value = Config(value)._policy self._policy.add(key, value) else: raise RuntimeError, "Can't interpret provided policy: %s" % ( policy) return
def cr(infn, crfn, maskfn): exposure = afwImage.ExposureF(infn) #'850994p-21.fits' print 'exposure', exposure print 'w,h', exposure.getWidth(), exposure.getHeight() W, H = exposure.getWidth(), exposure.getHeight() #var = exposure.getMaskedImage().getVariance() #print 'Variance', var.get(0,0) wcs = exposure.getWcs() print 'wcs', wcs pixscale = wcs.pixelScale().asArcseconds() psf = getFakePsf(pixscale) # CRs mask = exposure.getMaskedImage().getMask() crBit = mask.getMaskPlane("CR") mask.clearMaskPlane(crBit) mi = exposure.getMaskedImage() bg = afwMath.makeStatistics(mi, afwMath.MEDIAN).getValue() print 'bg', bg varval = afwMath.makeStatistics(mi, afwMath.VARIANCE).getValue() print 'variance', varval, 'std', math.sqrt(varval) varval = afwMath.makeStatistics(mi, afwMath.VARIANCECLIP).getValue() print 'clipped variance', varval, 'std', math.sqrt(varval) var = exposure.getMaskedImage().getVariance() var.set(varval) var = exposure.getMaskedImage().getVariance() print 'Variance:', var.get(0, 0) keepCRs = False policy = pexPolicy.Policy() # policy.add('minSigma', 6.) # policy.add('min_DN', 150.) # policy.add('cond3_fac', 2.5) # policy.add('cond3_fac2', 0.6) # policy.add('niteration', 3) # policy.add('nCrPixelMax', 200000) policy.add('minSigma', 10.) policy.add('min_DN', 500.) policy.add('cond3_fac', 2.5) policy.add('cond3_fac2', 0.6) policy.add('niteration', 1) policy.add('nCrPixelMax', 100000) #psfimg = psf.computeImage(afwGeom.Point2D(W/2., H/2.)) #psfimg.writeFits('psf.fits') print 'Finding cosmics...' crs = measAlg.findCosmicRays(mi, psf, bg, policy, keepCRs) print 'got', len(crs), 'cosmic rays', mask = mi.getMask() crBit = mask.getPlaneBitMask("CR") afwDet.setMaskFromFootprintList(mask, crs, crBit) mask.writeFits(maskfn) exposure.writeFits(crfn)
def persistCovMaps(covMaps, outputDir): """Persists coverage maps for up to 6 LSST filters (ugrizy) and produces an all-filter coverage map by summing the individual maps. Output files are automatically gzipped (they tend to compress well). """ if not os.path.exists(outputDir): os.makedirs(outputDir) persistence = dafPersist.Persistence.getPersistence(pexPolicy.Policy()) for covMap, filter in izip(covMaps, "ugrizy"): # Write out coverage map outPath = os.path.join(outputDir, '%sCovMap.fits' % filter) storageList = dafPersist.StorageList() storageList.append( persistence.getPersistStorage("FitsStorage", dafPersist.LogicalLocation(outPath))) persistence.persist(covMap, storageList, dafBase.PropertySet()) subprocess.check_call(['gzip', '-f', outPath]) allCovMap = covMaps.pop() allCovMapImg = allCovMap.getImage() for covMap in covMaps: allCovMapImg += covMap.getImage() outPath = os.path.join(outputDir, 'allCovMap.fits') storageList = dafPersist.StorageList() storageList.append( persistence.getPersistStorage("FitsStorage", dafPersist.LogicalLocation(outPath))) persistence.persist(allCovMap, storageList, dafBase.PropertySet()) subprocess.check_call(['gzip', '-f', outPath])
def testSetCtr(self): """Test setCtrCol/Row""" kWidth = 3 kHeight = 4 pol = pexPolicy.Policy() additionalData = dafBase.PropertySet() loc = dafPersist.LogicalLocation( os.path.join(testPath, "data", "kernel7.boost")) persistence = dafPersist.Persistence.getPersistence(pol) gaussFunc = afwMath.GaussianFunction2D(1.0, 1.0, 0.0) k = afwMath.AnalyticKernel(kWidth, kHeight, gaussFunc) for xCtr in range(kWidth): k.setCtrX(xCtr) for yCtr in range(kHeight): k.setCtrY(yCtr) storageList = dafPersist.StorageList() storage = persistence.getPersistStorage("XmlStorage", loc) storageList.append(storage) persistence.persist(k, storageList, additionalData) storageList2 = dafPersist.StorageList() storage2 = persistence.getRetrieveStorage("XmlStorage", loc) storageList2.append(storage2) k2 = persistence.unsafeRetrieve("AnalyticKernel", storageList2, additionalData) self.kernelCheck(k, k2) self.assertEqual(k2.getCtrX(), xCtr) self.assertEqual(k2.getCtrY(), yCtr)
def __init__(self, inputPolicy=None, **kwargs): policyFile = pexPolicy.DefaultPolicyFile(self.packageName, "ctio0m9Mapper.paf", "policy") policy = pexPolicy.Policy(policyFile) CameraMapper.__init__(self, policy, policyFile.getRepositoryPath(), **kwargs) filter_pairings = [ 'NONE+SEMROCK', # list of all filter pairings found in data 'NONE+RONCHI200', 'RONCHI200+SEMROCK', 'NONE+NONE', 'NONE+g', 'NONE+r', 'NONE+i', 'NONE+z', 'RONCHI200+z', 'RONCHI200+g', 'FGB37+RONCHI200', 'NONE+RONCHI400', 'FGC715S+RONCHI400', 'FGC715S+RONCHI200' ] # default no-filter name used for biases and darks - must appear afwImageUtils.defineFilter('NONE', 0.0, alias=[]) for pairing in filter_pairings: afwImageUtils.defineFilter(pairing, 0.0, alias=[])
def testSingleExposure(self): policyFile = pexPolicy.DefaultPolicyFile( "ip_pipeline", "CrRejectStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy( policyFile, policyFile.getRepositoryPath(), True) policy = pexPolicy.Policy( os.path.join(eups.productDir("ip_pipeline"), "tests", "crReject_policy.paf")) policy.mergeDefaults(defPolicy) stage = ipPipe.CrRejectStage(policy) tester = SimpleStageTester(stage) clipboard = pexClipboard.Clipboard() clipboard.put(policy.get("inputKeys.exposure"), self.exposure) if display: ds9.mtv(self.exposure, frame=0, title="Input") # # Do the work # outWorker = tester.runWorker(clipboard) outPolicy = policy.get("outputKeys") self.assertTrue(outWorker.contains(outPolicy.get("exposure"))) self.assertEqual(outWorker.get("nCR"), 25) if display: ds9.mtv(outWorker.get(outPolicy.get("exposure")), frame=1, title="CR removed")
def __init__(self, inputPolicy=None, **kwargs): policyFile = pexPolicy.DefaultPolicyFile(self.packageName, "LsstSimMapper.paf", "policy") policy = pexPolicy.Policy(policyFile) self.doFootprints = False if inputPolicy is not None: for kw in inputPolicy.paramNames(True): if kw == "doFootprints": self.doFootprints = True else: kwargs[kw] = inputPolicy.get(kw) super(LsstSimMapper, self).__init__(policy, policyFile.getRepositoryPath(), **kwargs) self.filterIdMap = { 'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5, 'i2': 5 } # The LSST Filters from L. Jones 04/07/10 afwImageUtils.defineFilter('u', 364.59) afwImageUtils.defineFilter('g', 476.31) afwImageUtils.defineFilter('r', 619.42) afwImageUtils.defineFilter('i', 752.06) afwImageUtils.defineFilter('z', 866.85) afwImageUtils.defineFilter('y', 971.68, alias=['y4']) # official y filter
def __init__(self, label, **kwargs): """ """ import lsst.obs.hscSim as obsHsc import runHsc mapper = obsHsc.HscSimMapper dataInfo = [['visit',1], ['ccd', 0]] # find the label in the testbed path testbedDir, testdataDir = qaDataUtils.findDataInTestbed(label) #defaultConfig = runHsc.getConfigQa() argv = [] argv.extend(["--instrument=hsc", "--frame=0", "--ccd=0", "--rerun=test"]) defaultConfig, opts, args = runHsc.getConfig(argv=argv) if not 'roots' in defaultConfig: defaultConfig['roots'] = pexPolicy.Policy() roots = defaultConfig['roots'] roots['data'] = os.path.join(testdataDir, "HSC") roots['calib'] = os.path.join(testdataDir, "CALIB") roots['output'] = os.path.join(testdataDir, "HSC") rerun = "pipeQA" TestData.__init__(self, label, rerun, mapper, dataInfo, defaultConfig, kwargs)
def testPersistence(self): if persistence.DbAuth.available("lsst10.ncsa.uiuc.edu", "3306"): pol = policy.Policy() root = "Formatter.PersistableIdPairVector" pol.set(root + ".IdPair.templateTableName", "_tmpl_IdPair") pol.set(root + ".IdPair.tableNamePattern", "_tmp_v%(visitId)_IP") pers = persistence.Persistence.getPersistence(pol) loc = persistence.LogicalLocation( "mysql://lsst10.ncsa.uiuc.edu:3306/test") ps = base.PropertySet() ps.setInt("visitId", int(time.clock()) * 16384 + random.randint(0, 16383)) ps.setString("itemName", "IdPair") stl = persistence.StorageList() stl.push_back(pers.getPersistStorage("DbStorage", loc)) pers.persist(ap.PersistableIdPairVector(self.vec), stl, ps) stl = persistence.StorageList() stl.push_back(pers.getRetrieveStorage("DbStorage", loc)) res = ap.PersistableIdPairVector.swigConvert( pers.unsafeRetrieve("PersistableIdPairVector", stl, ps)) db = persistence.DbStorage() db.setPersistLocation(loc) db.startTransaction() db.dropTable(detection.getTableName(pol.getPolicy(root), ps)) db.endTransaction() assert _seqEqual(res.getIdPairs(), self.vec)
def __init__(self, inputPolicy=None, **kwargs): policyFile = pexPolicy.DefaultPolicyFile(self.packageName, "monocamMapper.paf", "policy") policy = pexPolicy.Policy(policyFile) CameraMapper.__init__(self, policy, policyFile.getRepositoryPath(), **kwargs) # Ensure each dataset type of interest knows about the full range of keys available from the registry keys = {'visit': int, 'ccd': int, 'filter': str, 'date': str, 'expTime': float, 'object': str, 'imageType': str, } for name in ("raw", "raw_amp", # processCcd outputs "postISRCCD", "calexp", "postISRCCD", "src", "icSrc", "srcMatch", ): self.mappings[name].keyDict.update(keys) # @merlin, you should swap these out for the filters you actually intend to use. self.filterIdMap = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5} # The LSST Filters from L. Jones 04/07/10 afwImageUtils.defineFilter('u', 364.59) afwImageUtils.defineFilter('g', 476.31, alias=["SDSSG"]) afwImageUtils.defineFilter('r', 619.42, alias=["SDSSR"]) afwImageUtils.defineFilter('i', 752.06, alias=["SDSSI"]) afwImageUtils.defineFilter('z', 866.85, alias=["SDSSZ"]) afwImageUtils.defineFilter('y', 971.68, alias=['y4']) # official y filter afwImageUtils.defineFilter('NONE', 0.0, alias=['no_filter', "OPEN"])
def loadId(self): dbAuth = os.path.join(os.environ["HOME"], ".lsst", "db-auth.paf") policy = pexPolicy.Policy(dbAuth) authPolicy = policy.get("database").get("authInfo") self.mySqlUser = authPolicy.get("user") self.mySqlHost = authPolicy.get("host") self.mySqlPasswd = authPolicy.get("password")
def roundTripPsf(key, psf): global psfFileNum pol = policy.Policy() additionalData = dafBase.PropertySet() if psfFileNum % 2 == 1: storageType = "Boost" else: storageType = "Xml" loc = dafPersist.LogicalLocation("tests/data/psf%d-%d.%s" % (psfFileNum, key, storageType)) psfFileNum += 1 persistence = dafPersist.Persistence.getPersistence(pol) storageList = dafPersist.StorageList() storage = persistence.getPersistStorage("%sStorage" % (storageType), loc) storageList.append(storage) persistence.persist(psf, storageList, additionalData) storageList2 = dafPersist.StorageList() storage2 = persistence.getRetrieveStorage("%sStorage" % (storageType), loc) storageList2.append(storage2) psf2 = persistence.unsafeRetrieve("Psf", storageList2, additionalData) return psf2
def setUp(self): maskedImage = afwImage.MaskedImageF(inFilePathSmall) maskedImageMD = afwImage.readMetadata(inFilePathSmall) self.smallExposure = afwImage.ExposureF(inFilePathSmall) self.width = maskedImage.getWidth() self.height = maskedImage.getHeight() self.wcs = afwImage.makeWcs(maskedImageMD) self.psf = DummyPsf(2.0) self.detector = DetectorWrapper().detector self.exposureBlank = afwImage.ExposureF() self.exposureMiOnly = afwImage.makeExposure(maskedImage) self.exposureMiWcs = afwImage.makeExposure(maskedImage, self.wcs) self.exposureCrWcs = afwImage.ExposureF( 100, 100, self.wcs) # n.b. the (100, 100, ...) form self.exposureCrOnly = afwImage.ExposureF(afwGeom.ExtentI( 100, 100)) # test with ExtentI(100, 100) too afwImage.Filter.reset() afwImage.FilterProperty.reset() filterPolicy = pexPolicy.Policy() filterPolicy.add("lambdaEff", 470.0) afwImage.Filter.define(afwImage.FilterProperty("g", filterPolicy))
def test1(self): """ Test PropertySet persistence to database without policy. """ dp = dafBase.PropertySet() dp.addInt("intField", 1) dp.addDouble("doubleField", 1.2) dp.addString("varcharField", "Testing") dp.addBool("boolField", True) dp.addLongLong("int64Field", 9876543210) dp.addFloat("floatField", 3.14) pol = pexPolicy.Policy() additionalData = dafBase.PropertySet() additionalData.add("itemName", "Persistence_Test_2") loc = dafPersist.LogicalLocation("mysql://{}:{}/test".format(HOST, PORT)) persistence = dafPersist.Persistence.getPersistence(pol) storageList = dafPersist.StorageList() storage = persistence.getPersistStorage("DbStorage", loc) storageList.append(storage) persistence.persist(dp, storageList, additionalData)
def testPersistence(self): if dafPers.DbAuth.available("lsst10.ncsa.uiuc.edu", "3306"): pol = pexPolicy.Policy() pol.set( "Formatter.PersistableMovingObjectPredictionVector.TestPreds.templateTableName", "_tmpl_mops_Prediction") pol.set( "Formatter.PersistableMovingObjectPredictionVector.TestPreds.tableNamePattern", "_tmp_v%(visitId)_Preds") pers = dafPers.Persistence.getPersistence(pol) loc = dafPers.LogicalLocation( "mysql://lsst10.ncsa.uiuc.edu:3306/test") props = dafBase.PropertySet() props.addInt("visitId", int(time.clock()) * 16384 + random.randint(0, 16383)) props.addInt("sliceId", 0) props.addInt("numSlices", 1) props.addString("itemName", "TestPreds") stl = dafPers.StorageList() stl.push_back(pers.getPersistStorage("DbStorage", loc)) pers.persist(mops.PersistableMopsPredVec(self.mpv1), stl, props) stl = dafPers.StorageList() stl.push_back(pers.getRetrieveStorage("DbStorage", loc)) persistable = pers.unsafeRetrieve( "PersistableMovingObjectPredictionVector", stl, props) res = mops.PersistableMopsPredVec.swigConvert(persistable) afwDet.dropAllSliceTables( loc, pol.getPolicy( "Formatter.PersistableMovingObjectPredictionVector"), props) checkMopsPredEqual(res.getPredictions(), self.mpv1) else: print "skipping database tests"
def __init__(self, outputRoot=None, **kwargs): policyFile = pexPolicy.DefaultPolicyFile("obs_sst", "SstMapper.paf", "policy") policy = pexPolicy.Policy(policyFile) super(SstMapper, self).__init__(policy, policyFile.getRepositoryPath(), **kwargs) afwImageUtils.defineFilter('OPEN', lambdaEff=650)
def readSourceSet(boostFile): loc = dafPersist.LogicalLocation(boostFile) storageList = dafPersist.StorageList() additionalData = dafBase.PropertySet() persistence = dafPersist.Persistence.getPersistence(pexPolicy.Policy()) storageList.append(persistence.getRetrieveStorage("BoostStorage", loc)) psv = persistence.unsafeRetrieve("PersistableSourceVector", storageList, additionalData) return psv.getSources()
def setup(self): self.log = Log(self.log, "simpleDiffImStage - parallel") policyFile = pexPolicy.DefaultPolicyFile("ip_pipeline", "SimpleDiffImStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy(policyFile, policyFile.getRepositoryPath(), True) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy.getDictionary())
def pcapsf_read_boost(fn): print('# Reading', fn, end=' ') loc = dafPersist.LogicalLocation(fn) storageList = dafPersist.StorageList() additionalData = dafBase.PropertySet() persistence = dafPersist.Persistence.getPersistence(pexPolicy.Policy()) storageList.append(persistence.getRetrieveStorage("BoostStorage", loc)) psf = persistence.unsafeRetrieve("Psf", storageList, additionalData) return psf
def setup(self): #self.log = Log(self.log, "TransformMetadataStage - parallel") policyFile = pexPolicy.DefaultPolicyFile("ip_pipeline", "TransformMetadataStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy(policyFile, policyFile.getRepositoryPath()) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy)
def __init__(self, outputRoot=None, **kwargs): policyFile = pexPolicy.DefaultPolicyFile("obs_wiyn", "WhircMapper.paf", "policy") policy = pexPolicy.Policy(policyFile) # print policyFile.getRepositoryPath() super(WhircMapper, self).__init__(policy, policyFile.getRepositoryPath(), **kwargs) afwImageUtils.defineFilter('OPEN', lambdaEff=1750) # nm afwImageUtils.defineFilter('J' , lambdaEff=1250) afwImageUtils.defineFilter('H' , lambdaEff=1650) afwImageUtils.defineFilter('KS', lambdaEff=2175, alias=['Ks'])
def setup(self): self.log = Log(self.log, "ComputeSourceSkyCoordsStage - parallel") policyFile = pexPolicy.DefaultPolicyFile( "meas_pipeline", "ComputeSourceSkyCoordsStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy( policyFile, policyFile.getRepositoryPath(), True) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy.getDictionary())
def sourceset_read_boost(fn): loc = dafPersist.LogicalLocation(fn) storageList = dafPersist.StorageList() additionalData = dafBase.PropertySet() persistence = dafPersist.Persistence.getPersistence(pexPolicy.Policy()) storageList.append(persistence.getRetrieveStorage("BoostStorage", loc)) psvptr = persistence.unsafeRetrieve("PersistableSourceVector", storageList, additionalData) print(psvptr) psv = afwDet.PersistableSourceVector.swigConvert(psvptr) return psv.getSources()
def _helper(dict_): p = pexPolicy.Policy() for k, v in dict_.items(): if isinstance(v, dict): p.set(k, _helper(v)) elif isinstance(v, list): for vi in v: p.add(k, vi) elif v is not None: p.set(k, v) return p
def pcapsf_read_boost(fn): import lsst.meas.algorithms as measAlgorithms # needed to register pcaPsf formatter print('# Reading', fn, end=' ') loc = dafPersist.LogicalLocation(fn) storageList = dafPersist.StorageList() additionalData = dafBase.PropertySet() persistence = dafPersist.Persistence.getPersistence(pexPolicy.Policy()) storageList.append(persistence.getRetrieveStorage("BoostStorage", loc)) psf = persistence.unsafeRetrieve("Psf", storageList, additionalData) return psf
def setup(self): self.log = pexLog.Log(self.log, self.__class__.__name__) policyFile = pexPolicy.DefaultPolicyFile(self.packageName, self.policyDictionaryName, "policy") defPolicy = pexPolicy.Policy.createPolicy( policyFile, policyFile.getRepositoryPath(), True) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy.getDictionary())
def __init__(self, outputRoot=None, **kwargs): policyFile = pexPolicy.DefaultPolicyFile("obs_decam", "DecamMapper.paf", "policy") policy = pexPolicy.Policy(policyFile) super(DecamMapper, self).__init__(policy, policyFile.getRepositoryPath(), **kwargs) afwImageUtils.defineFilter('u', lambdaEff=350) afwImageUtils.defineFilter('g', lambdaEff=450) afwImageUtils.defineFilter('r', lambdaEff=600) afwImageUtils.defineFilter('i', lambdaEff=750) afwImageUtils.defineFilter('z', lambdaEff=900) afwImageUtils.defineFilter('y', lambdaEff=1000, alias='Y') # Urgh!
def setup(self): self.log = Log(self.log, "CcdAssembly -- Parallel") policyFile = pexPolicy.DefaultPolicyFile( "ip_pipeline", "IsrCcdAssemblyStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy( policyFile, policyFile.getRepositoryPath()) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy)
def __init__(self, uri, create): self.log = Log.getLogger("daf.persistence.butler") self.root = self._pathFromURI(uri) if self.root and not os.path.exists(self.root): if not create: raise NoRepositroyAtRoot("No repository at {}".format(uri)) safeMakeDir(self.root) # Always use an empty Persistence policy until we can get rid of it persistencePolicy = pexPolicy.Policy() self.persistence = Persistence.getPersistence(persistencePolicy)