def processCCDs(image): from lsst.pipe.tasks.calibrate import CalibrateTask, CalibrateConfig from lsst.pipe.tasks.characterizeImage import CharacterizeImageTask calibRes = None # init tasks charImage = CharacterizeImageTask() calibrateConfig = CalibrateConfig(doPhotoCal=False, doAstrometry=False, doDeblend=False) calibrateTask = CalibrateTask(config=calibrateConfig) try: # characterize image charRes = charImage.characterize(image, exposureIdInfo=None, background=None) # calibrate image calibRes = calibrateTask.calibrate(charRes.exposure, exposureIdInfo=None, background=charRes.background, icSourceCat=None) except Exception as e: print "failed to calibrate the image" print str(e) return calibRes
def run(display=False): # # Create the tasks # charImageConfig = CharacterizeImageTask.ConfigClass() charImageTask = CharacterizeImageTask(config=charImageConfig) config = CalibrateTask.ConfigClass() config.astrometry.retarget(MyAstrometryTask) calibrateTask = CalibrateTask(config=config) # load the data # Exposure ID and the number of bits required for exposure IDs are usually obtained from a data repo, # but here we pick reasonable values (there are 64 bits to share between exposure IDs and source IDs). exposure = loadData() exposureIdInfo = ExposureIdInfo(expId=1, expBits=5) # characterize the exposure to repair cosmic rays and fit a PSF model # display now because CalibrateTask modifies the exposure in place charRes = charImageTask.characterize(exposure=exposure, exposureIdInfo=exposureIdInfo) if display: displayFunc(charRes.exposure, charRes.sourceCat, frame=1) # calibrate the exposure calRes = calibrateTask.calibrate(exposure=charRes.exposure, exposureIdInfo=exposureIdInfo) if display: displayFunc(calRes.exposure, calRes.sourceCat, frame=2)
def run(display=False): # # Create the task using a butler constructed using the obs_test repo # butler = dafPersistence.Butler(os.path.join(lsst.utils.getPackageDir("obs_test"), "data", "input")) charImageConfig = CharacterizeImageTask.ConfigClass() charImageTask = CharacterizeImageTask(butler, config=charImageConfig) config = CalibrateTask.ConfigClass() config.astrometry.retarget(MyAstrometryTask) calibrateTask = CalibrateTask(butler, config=config) # load the data # Exposure ID and the number of bits required for exposure IDs are usually obtained from a data repo, # but here we pick reasonable values (there are 64 bits to share between exposure IDs and source IDs). exposure = loadData() exposureIdInfo = ExposureIdInfo(expId=1, expBits=5) # characterize the exposure to repair cosmic rays and fit a PSF model # display now because CalibrateTask modifies the exposure in place charRes = charImageTask.characterize(exposure=exposure, exposureIdInfo=exposureIdInfo) if display: displayFunc(charRes.exposure, charRes.sourceCat, frame=1) # calibrate the exposure calRes = calibrateTask.calibrate(exposure=charRes.exposure, exposureIdInfo=exposureIdInfo) if display: displayFunc(calRes.exposure, calRes.sourceCat, frame=2)
def run(display=False): # # Create the task # config = CalibrateTask.ConfigClass() config.initialPsf.pixelScale = 0.185 # arcsec per pixel config.initialPsf.fwhm = 1.0 config.astrometry.retarget(MyAstrometryTask) calibrateTask = CalibrateTask(config=config) # # Process the data # exposure = loadData(config.initialPsf.pixelScale) result = calibrateTask.run(exposure) exposure0, exposure = exposure, result.exposure sources = result.sources if display: # display on ds9 (see also --debug argparse option) frame = 1 ds9.mtv(exposure, frame=frame) with ds9.Buffering(): for s in sources: xy = s.getCentroid() ds9.dot('+', *xy, ctype=ds9.CYAN if s.get("flags_negative") else ds9.GREEN, frame=frame)
def write_output_data(self, dataRef, calRes): """Write output data thru Butler based on config""" if self.config.doWrite: CalibrateTask.writeOutputs( CalibrateTask(config=self.config, log=self.log, icSourceSchema=self.icSourceSchema), dataRef=dataRef, exposure=calRes.exposure, background=calRes.background, sourceCat=calRes.sourceCat, astromMatches=calRes.astromMatches, matchMeta=self.matchMeta, )
def _checkDoRefcats(self, doAstrometry, doPhotoCal, ids): """Test whether run is called with the correct arguments. In the case of `CalibrateTask`, the inputs should not depend on the task configuration. Parameters ---------- doAstrometry, doPhotoCal : `bool` Values of the config flags of the same name. ids : `dict` [`str`] A mapping from the input dataset type to the data ID of the dataset to process. """ config = CalibrateConfig() config.doWriteMatches = False # no real output to write config.doAstrometry = doAstrometry config.doPhotoCal = doPhotoCal config.connections.photoRefCat = "cal_ref_cat" config.connections.astromRefCat = "cal_ref_cat" task = CalibrateTask(config=config) quantumId = ids["exposure"] quantum = testUtils.makeQuantum(task, self.butler, quantumId, ids) run = testUtils.runTestQuantum(task, self.butler, quantum) run.assert_called_once() self.assertEqual(run.call_args[0], ()) # Some arguments unprintable because we don't have a full environment # So just check which ones were passed in self.assertEqual( run.call_args[1].keys(), {"exposure", "exposureIdInfo", "background", "icSourceCat"})
def testIsPrimaryFlag(self): """Tests detect_isPrimary column gets added when run, and that sources labelled as detect_isPrimary are not sky sources and have no children. """ charImConfig = CharacterizeImageConfig() charImTask = CharacterizeImageTask(config=charImConfig) charImResults = charImTask.run(self.exposure) calibConfig = CalibrateConfig() calibConfig.doAstrometry = False calibConfig.doPhotoCal = False calibTask = CalibrateTask(config=calibConfig) calibResults = calibTask.run(charImResults.exposure) outputCat = calibResults.outputCat self.assertTrue("detect_isPrimary" in outputCat.schema.getNames()) # make sure all sky sources are flagged as not primary self.assertEqual( sum((outputCat["detect_isPrimary"]) & (outputCat["sky_source"])), 0) # make sure all parent sources are flagged as not primary self.assertEqual( sum((outputCat["detect_isPrimary"]) & (outputCat["deblend_nChild"] > 0)), 0)
def testIsSinglePrimaryFlag(self): """Tests detect_isPrimary column gets added when run, and that sources labelled as detect_isPrimary are not sky sources and have no children. """ calibConfig = CalibrateConfig() calibConfig.doAstrometry = False calibConfig.doPhotoCal = False calibTask = CalibrateTask(config=calibConfig) calibResults = calibTask.run(self.charImResults.exposure) outputCat = calibResults.outputCat self.assertTrue("detect_isPrimary" in outputCat.schema.getNames()) # make sure all sky sources are flagged as not primary self.assertEqual( sum((outputCat["detect_isPrimary"]) & (outputCat["sky_source"])), 0) # make sure all parent sources are flagged as not primary self.assertEqual( sum((outputCat["detect_isPrimary"]) & (outputCat["deblend_nChild"] > 0)), 0) with self.assertRaises(KeyError): outputCat.getSchema().find("detect_isDelendedModelPrimary")
def _checkSkySourceColumnExistence(self, doSkySources): """Implements sky_source column checking. Parameters ---------- doSkySource : `bool` Value of the config flag determining whether to insert sky sources. """ charImConfig = CharacterizeImageConfig() charImConfig.measurePsf.psfDeterminer = 'piff' charImConfig.measurePsf.psfDeterminer['piff'].spatialOrder = 0 charImTask = CharacterizeImageTask(config=charImConfig) charImResults = charImTask.run(self.exposure) calibConfig = CalibrateConfig() calibConfig.doAstrometry = False calibConfig.doPhotoCal = False calibConfig.doSkySources = doSkySources calibTask = CalibrateTask(config=calibConfig) calibResults = calibTask.run(charImResults.exposure) if doSkySources: self.assertTrue('sky_source' in calibResults.outputCat.schema.getNames()) else: self.assertFalse('sky_source' in calibResults.outputCat.schema.getNames())
def execute(self, dataRef): """!Characterize a science image @param dataRef: butler data reference @return a pipeBase Struct containing the results """ self.log.info("Performing Super Calibrate on sensor data ID %s" % (dataRef.dataId,)) self.log.info("Reading input data using dataRef") inputData = self.read_input_data(dataRef) self.log.info("Running operations. The run() method should not take anything Butler") if self.config.doWrite and self.config.doAstrometry: self.matchMeta = createMatchMetadata(inputData.getDict()['exposure'], border=self.pixelMargin) else: self.matchMeta = None result = CalibrateTask.calibrate(CalibrateTask(config=self.config, log=self.log, icSourceSchema=self.icSourceSchema), **inputData.getDict()) self.log.info("Writing output data using dataRef") self.write_output_data(dataRef, result) return result
def run(config, inputFiles, weightFiles=None, varianceFiles=None, returnCalibSources=False, display=False, verbose=False): # # Create the tasks # schema = afwTable.SourceTable.makeMinimalSchema() algMetadata = dafBase.PropertyList() calibrateTask = CalibrateTask(config=config.calibrate) sourceDetectionTask = SourceDetectionTask(config=config.detection, schema=schema) if config.doDeblend: if SourceDeblendTask: sourceDeblendTask = SourceDeblendTask(config=config.deblend, schema=schema) else: print >> sys.stderr, "Failed to import lsst.meas.deblender; setting doDeblend = False" config.doDeblend = False sourceMeasurementTask = SingleFrameMeasurementTask(config=config.measurement, schema=schema, algMetadata=algMetadata) exposureDict = {}; calibSourcesDict = {}; sourcesDict = {} for inputFile, weightFile, varianceFile in zip(inputFiles, weightFiles, varianceFiles): # # Create the output table # tab = afwTable.SourceTable.make(schema) # # read the data # if verbose: print "Reading %s" % inputFile exposure = makeExposure(inputFile, weightFile, varianceFile, config.badPixelValue, config.variance, verbose) # # process the data # calibSources = None # sources used to calibrate the frame (photom, astrom, psf) if config.doCalibrate: result = calibrateTask.run(exposure) exposure, sources = result.exposure, result.sources if returnCalibSources: calibSources = sources else: if not exposure.getPsf(): calibrateTask.installInitialPsf(exposure) if config.edgeRolloff.applyModel: if verbose: print "Adding edge rolloff distortion to the exposure WCS" addEdgeRolloffDistortion(exposure, config.edgeRolloff) exposureDict[inputFile] = exposure calibSourcesDict[inputFile] = calibSources result = sourceDetectionTask.run(tab, exposure) sources = result.sources sourcesDict[inputFile] = sources if config.doDeblend: sourceDeblendTask.run(exposure, sources, exposure.getPsf()) sourceMeasurementTask.measure(exposure, sources) if verbose: print "Detected %d objects" % len(sources) if display: # display on ds9 (see also --debug argparse option) if algMetadata.exists("base_CircularApertureFlux_radii"): radii = algMetadata.get("base_CircularApertureFlux_radii") else: radii = None frame = 1 ds9.mtv(exposure, title=os.path.split(inputFile)[1], frame=frame) with ds9.Buffering(): for s in sources: xy = s.getCentroid() ds9.dot('+', *xy, ctype=ds9.CYAN if s.get("flags_negative") else ds9.GREEN, frame=frame) ds9.dot(s.getShape(), *xy, ctype=ds9.RED, frame=frame) if radii: for radius in radii: ds9.dot('o', *xy, size=radius, ctype=ds9.YELLOW, frame=frame) return exposureDict, calibSourcesDict, sourcesDict
def testComponents(self): """Test that we can run the first-level subtasks of ProcessCcdTasks. This tests that we can run these subtasks from the command-line independently (they're all CmdLineTasks) as well as directly from Python (without giving them access to a Butler). Aside from verifying that no exceptions are raised, we simply tests that most persisted results are present and equivalent to both in-memory results. """ outPath = tempfile.mkdtemp( ) if OutputName is None else "{}-Components".format(OutputName) # We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it # to those tasks. inputButler = lsst.daf.persistence.Butler(InputDir) # Construct task instances we can use directly from Python isrTask = IsrTask(config=getObsTestConfig(IsrTask), name="isr2") # If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these # tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class # that doesn't require a Butler. If we don't, we should construct a butler-based on outside these # task constructors and pass the LoadReferenceObjectsTask instance to the task constructors. charImageTask = CharacterizeImageTask( config=getObsTestConfig(CharacterizeImageTask), name="charImage2") calibrateTask = CalibrateTask(config=getObsTestConfig(CalibrateTask), name="calibrate2", icSourceSchema=charImageTask.schema) try: dataId = dict(visit=1) dataIdStrList = [ "%s=%s" % (key, val) for key, val in dataId.items() ] isrResult1 = IsrTask.parseAndRun( args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id" ] + dataIdStrList, doReturnResults=True, ) # We'll just use the butler to get the original image and calibration frames; it's not clear # extending the test coverage to include that is worth it. dataRef = inputButler.dataRef("raw", dataId=dataId) rawExposure = dataRef.get("raw", immediate=True) camera = dataRef.get("camera") isrData = isrTask.readIsrData(dataRef, rawExposure) exposureIdInfo = inputButler.get("expIdInfo", dataId=dataId) isrResult2 = isrTask.run( rawExposure, bias=isrData.bias, linearizer=isrData.linearizer, flat=isrData.flat, defects=isrData.defects, fringes=isrData.fringes, bfKernel=isrData.bfKernel, camera=camera, ) self.assertMaskedImagesEqual( isrResult1.parsedCmd.butler.get( "postISRCCD", dataId, immediate=True).getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage()) self.assertMaskedImagesEqual( isrResult2.exposure.getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage()) icResult1 = CharacterizeImageTask.parseAndRun( args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id" ] + dataIdStrList, doReturnResults=True, ) icResult2 = charImageTask.run(isrResult2.exposure, exposureIdInfo=exposureIdInfo) self.assertMaskedImagesEqual( icResult1.parsedCmd.butler.get( "icExp", dataId, immediate=True).getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage()) self.assertMaskedImagesEqual( icResult2.exposure.getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage()) self.assertCatalogsEqual( icResult1.parsedCmd.butler.get("icSrc", dataId, immediate=True), icResult1.resultList[0].result.sourceCat) self.assertCatalogsEqual( icResult2.sourceCat, icResult1.resultList[0].result.sourceCat, ) self.assertBackgroundListsEqual( icResult1.parsedCmd.butler.get("icExpBackground", dataId, immediate=True), icResult1.resultList[0].result.background) self.assertBackgroundListsEqual( icResult2.background, icResult1.resultList[0].result.background) calResult1 = CalibrateTask.parseAndRun( args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id" ] + dataIdStrList, doReturnResults=True, ) calResult2 = calibrateTask.run( icResult2.exposure, background=icResult2.background, icSourceCat=icResult2.sourceCat, exposureIdInfo=exposureIdInfo, ) self.assertMaskedImagesEqual( calResult1.parsedCmd.butler.get( "calexp", dataId, immediate=True).getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage()) self.assertMaskedImagesEqual( calResult2.exposure.getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage()) self.assertCatalogsEqual( calResult1.parsedCmd.butler.get("src", dataId, immediate=True), calResult1.resultList[0].result.sourceCat) self.assertCatalogsEqual(calResult2.sourceCat, calResult1.resultList[0].result.sourceCat, skipCols=("id", "parent")) self.assertBackgroundListsEqual( calResult1.parsedCmd.butler.get("calexpBackground", dataId, immediate=True), calResult1.resultList[0].result.background) self.assertBackgroundListsEqual( calResult2.background, calResult1.resultList[0].result.background) finally: if OutputName is None: shutil.rmtree(outPath) else: print("testProcessCcd.py's output data saved to %r" % (OutputName, ))
####### Setup LSST begins ######################################## visit=289697 ccdnum = 1 HOME_PATH = os.path.expanduser("~") butler = dafPersist.Butler(HOME_PATH + '/lsst_data/raw') exposure = butler.get("instcal", visit=visit, ccdnum=ccdnum, filter='g', immediate=True) mask = exposure.getMaskedImage().getMask().getArray() #TODO: move this matrix to SciDB variance = exposure.getMaskedImage().getVariance().getArray() #TODO: move this matrix to SciDB butler = dafPersist.Butler(HOME_PATH + '/lsst_data/raw') filename = HOME_PATH + "/lsst_data/raw/crblasted0289697/instcal0289697.1.fits" fitsHeader = afwImage.readMetadata(filename) wcs = afwImage.makeWcs(fitsHeader) charImage = CharacterizeImageTask() calibrateConfig = CalibrateConfig(doPhotoCal=False, doAstrometry=False) calibrateTask = CalibrateTask(config=calibrateConfig) newSkyMapConfig = skymap.discreteSkyMap.DiscreteSkyMapConfig(projection='STG', decList=[-4.9325280994132905], patchInnerDimensions=[2000, 2000], radiusList=[4.488775723429071], pixelScale=0.333, rotation=0.0, patchBorder=100, raList=[154.10660740464786], tractOverlap=0.0) hits_skymap = skymap.discreteSkyMap.DiscreteSkyMap(config=newSkyMapConfig) tract = hits_skymap[0] sys.stderr.write("=====> DFZ 3/24/2017: mask.shape = " + str(mask.shape) + "\n") ######################################## ####### Setup LSST ends ######################################## cnt = 0
def run(config, inputFiles, weightFiles=None, varianceFiles=None, returnCalibSources=False, displayResults=[], verbose=False): # # Create the tasks # schema = afwTable.SourceTable.makeMinimalSchema() algMetadata = dafBase.PropertyList() isrTask = IsrTask(config=config.isr) calibrateTask = CalibrateTask(config=config.calibrate) sourceDetectionTask = SourceDetectionTask(config=config.detection, schema=schema) if config.doDeblend: if SourceDeblendTask: sourceDeblendTask = SourceDeblendTask(config=config.deblend, schema=schema) else: print >> sys.stderr, "Failed to import lsst.meas.deblender; setting doDeblend = False" config.doDeblend = False sourceMeasurementTask = SingleFrameMeasurementTask(config=config.measurement, schema=schema, algMetadata=algMetadata) sourceMeasurementTask.config.doApplyApCorr = 'yes' # # Add fields needed to identify stars while calibrating # keysToCopy = [(schema.addField(afwTable.Field["Flag"]("calib_detected", "Source was detected by calibrate")), None)] for key in calibrateTask.getCalibKeys(): keysToCopy.append((schema.addField(calibrateTask.schema.find(key).field), key)) exposureDict = {}; calibSourcesDict = {}; sourcesDict = {} for inputFile, weightFile, varianceFile in zip(inputFiles, weightFiles, varianceFiles): # # Create the output table # tab = afwTable.SourceTable.make(schema) # # read the data # if verbose: print "Reading %s" % inputFile exposure = makeExposure(inputFile, weightFile, varianceFile, config.badPixelValue, config.variance) # if config.interpPlanes: import lsst.ip.isr as ipIsr defects = ipIsr.getDefectListFromMask(exposure.getMaskedImage(), config.interpPlanes, growFootprints=0) isrTask.run(exposure, defects=defects) # # process the data # if config.doCalibrate: result = calibrateTask.run(exposure) exposure, calibSources = result.exposure, result.sources else: calibSources = None if not exposure.getPsf(): calibrateTask.installInitialPsf(exposure) exposureDict[inputFile] = exposure calibSourcesDict[inputFile] = calibSources if returnCalibSources else None result = sourceDetectionTask.run(tab, exposure) sources = result.sources sourcesDict[inputFile] = sources if config.doDeblend: sourceDeblendTask.run(exposure, sources, exposure.getPsf()) sourceMeasurementTask.measure(exposure, sources) if verbose: print "Detected %d objects" % len(sources) propagateCalibFlags(keysToCopy, calibSources, sources) if displayResults: # display results of processing (see also --debug argparse option) showApertures = "showApertures".upper() in displayResults showShapes = "showShapes".upper() in displayResults display = afwDisplay.getDisplay(frame=1) if algMetadata.exists("base_CircularApertureFlux_radii"): radii = algMetadata.get("base_CircularApertureFlux_radii") else: radii = [] display.mtv(exposure, title=os.path.split(inputFile)[1]) with display.Buffering(): for s in sources: xy = s.getCentroid() display.dot('+', *xy, ctype=afwDisplay.CYAN if s.get("flags_negative") else afwDisplay.GREEN) if showShapes: display.dot(s.getShape(), *xy, ctype=afwDisplay.RED) if showApertures: for radius in radii: display.dot('o', *xy, size=radius, ctype=afwDisplay.YELLOW) return exposureDict, calibSourcesDict, sourcesDict
def main(): # try out one exposure #visits = ["0288935","0288976"] #,"0289893","0289913","0289931","0289614","0289818","0289820", "0289850","0289851","0289871","0289892", "0288935","0288976","0289016","0289056","0289161","0289202","0289243","0289284","0289368","0289409","0289450","0289493","0289573","0289656"] visits = ["0288976", "0288935"] ccds = [] exit(0) for i in range(1, 61): ccds.append(i) filterName = 'g' DATA_PATH = "/root/extra_home/lsst_data/" #spathprefix = "/home/dongfang/download/lsst_data/" spathprefix = DATA_PATH + "raw/" #calexpsloc = "/home/dongfang/download/lsst_data/calexps/" calexpsloc = DATA_PATH + "calexps/" #coaddloc = "/home/dongfang/download/lsst_data/coadds/" coaddloc = DATA_PATH + "coadds/" #mergecoaddloc = "/home/dongfang/download/lsst_data/merge/" mergecoaddloc = DATA_PATH + "merge/" # Characterize Image charImageConfig = CharacterizeImageConfig() charImage = CharacterizeImageTask() calibrateConfig = CalibrateConfig(doPhotoCal=False, doAstrometry=False) calibrateTask = CalibrateTask(config=calibrateConfig) makeCTEConfig = MakeCoaddTempExpConfig() makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig) newSkyMapConfig = skymap.discreteSkyMap.DiscreteSkyMapConfig( projection='STG', decList=[-4.9325280994132905], patchInnerDimensions=[2000, 2000], radiusList=[4.488775723429071], pixelScale=0.333, rotation=0.0, patchBorder=100, raList=[154.10660740464786], tractOverlap=0.0) hits_skymap = skymap.discreteSkyMap.DiscreteSkyMap(config=newSkyMapConfig) tract = hits_skymap[0] coaddTempDict = {} calibResDict = {} f = open("log.txt", 'wb') start = datetime.datetime.now() #process CCDs to create calexps. for v in visits: for ccd in ccds: visit = int(v) filename = "instcal" + v + "." + str(ccd) + ".fits" calexpfn = calexpsloc + v + "/" + filename source = spathprefix + v + "/" + filename exposure = afwImg.ExposureF(source) try: # Characterize Image charRes = charImage.characterize(exposure, exposureIdInfo=None, background=None) except: f.write("DFZ DEBUG at charRes: errors in visit " + v + ", ccd " + str(ccd) + "\n") try: # Caliberate Image calibRes = calibrateTask.calibrate( charRes.exposure, exposureIdInfo=None, background=charRes.background, icSourceCat=None) except: f.write("DFZ DEBUG at calibRes: errors in visit " + v + ", ccd " + str(ccd) + "\n") try: #write out calexps calibRes.exposure.writeFits(calexpfn) #calbresDict.append((v,ccd),calibRes) except: f.write("DFZ DEBUG at calibRes.exposure: errors in visit " + v + ", ccd " + str(ccd) + "\n") end = datetime.datetime.now() d = end - start f.write("time for creating calexps: ") f.write(str(d.total_seconds())) f.write("\n") #time for creating co-add tempexps. start = datetime.datetime.now() # map calexps to patch-ids visit = visits[0] ccdsPerPatch = [] for ccd in ccds: filename = "instcal" + visit + "." + str(ccd) + ".fits" source = calexpsloc + visit + "/" + filename exposure = afwImg.ExposureF(source) bbox = exposure.getBBox() wcs = exposure.getWcs() corners = bbox.getCorners() xIndexMax, yIndexMax = tract.findPatch( wcs.pixelToSky(corners[0][0], corners[0][1])).getIndex() xIndexMin, yIndexMin = tract.findPatch( wcs.pixelToSky(corners[2][0], corners[2][1])).getIndex() yy = range(yIndexMin, yIndexMax + 1) xx = range(xIndexMin, xIndexMax + 1) for yIdx in yy: for xIdx in xx: ccdsPerPatch.append((ccd, (xIdx, yIdx))) print len(ccdsPerPatch) #import cPickle #cPickle.dump(open("ccdsinpatch.p",'wb'),ccdsPerPatch) # import cPickle # f = open("ccdsInPatch.p",'wb') # cPickle.dump(ccdsInPatch,f) #import cPickle #ccdsInPatch = cPickle.load(open("ccdsInPatch.p",'rb')) df = pd.DataFrame(ccdsPerPatch) dfgby = df.groupby(1) makeCTEConfig = MakeCoaddTempExpConfig() makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig) coaddTempExpDict = {} for visit in visits: for a in dfgby.indices: coaddTempExpDict[a] = {} xInd = a[0] yInd = a[1] skyInfo = getSkyInfo(hits_skymap, xInd, yInd) v = int(visit) coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) coaddTempExp.getMaskedImage().set( numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"), numpy.inf) totGoodPix = 0 didSetMetadata = False modelPsf = makeCTEConfig.modelPsf.apply( ) if makeCTEConfig.doPsfMatch else None setInputRecorder = False for b in dfgby.get_group(a)[0].ravel(): print a print b if not setInputRecorder: ccdsinPatch = len(dfgby.get_group(a)[0].ravel()) try: inputRecorder = makeCTE.inputRecorder.makeCoaddTempExpRecorder( v, ccdsinPatch) except: f.write("DFZ DEBUG at inputRecorder\n") setInputRecorder = True numGoodPix = 0 ccd = b filename = "instcal" + visit + "." + str(ccd) + ".fits" source = calexpsloc + visit + "/" + filename calExp = afwImg.ExposureF(source) ccdId = calExp.getId() warpedCcdExp = makeCTE.warpAndPsfMatch.run( calExp, modelPsf=modelPsf, wcs=skyInfo.wcs, maxBBox=skyInfo.bbox).exposure if didSetMetadata: mimg = calExp.getMaskedImage() mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] / calExp.getCalib().getFluxMag0()[0]) del mimg numGoodPix = coaddUtils.copyGoodPixels( coaddTempExp.getMaskedImage(), warpedCcdExp.getMaskedImage(), makeCTE.getBadPixelMask()) totGoodPix += numGoodPix if numGoodPix > 0 and not didSetMetadata: coaddTempExp.setCalib(warpedCcdExp.getCalib()) coaddTempExp.setFilter(warpedCcdExp.getFilter()) didSetMetadata = True inputRecorder.addCalExp(calExp, ccdId, numGoodPix) ##### End loop over ccds here: inputRecorder.finish(coaddTempExp, totGoodPix) if totGoodPix > 0 and didSetMetadata: coaddTempExp.setPsf( modelPsf if makeCTEConfig.doPsfMatch else CoaddPsf( inputRecorder.coaddInputs.ccds, skyInfo.wcs)) coaddTempExpDict[a][v] = coaddTempExp coaddfilename = coaddloc + visit + "/" + "instcal" + visit + "." + str( xInd) + "_" + str(yInd) + ".fits" coaddTempExp.writeFits(coaddfilename) end = datetime.datetime.now() d = end - start f.write("time for creating co-add tempexps:\n ") f.write(str(d.total_seconds())) f.write("\n") #DFZ: stop here exit(0) start = datetime.datetime.now() config = AssembleCoaddConfig() assembleTask = AssembleCoaddTask(config=config) mergcoadds = {} for a in dfgby.indices: ccdsinPatch = len(dfgby.get_group(a)[0].ravel()) xInd = a[0] yInd = a[1] imageScalerRes = prepareInputs(coaddTempExpDict[a].values(), coaddTempExpDict[a].keys(), assembleTask) mask = None doClip = False if mask is None: mask = assembleTask.getBadPixelMask() statsCtrl = afwMath.StatisticsControl() statsCtrl.setNumSigmaClip(assembleTask.config.sigmaClip) statsCtrl.setNumIter(assembleTask.config.clipIter) statsCtrl.setAndMask(mask) statsCtrl.setNanSafe(True) statsCtrl.setWeighted(True) statsCtrl.setCalcErrorFromInputVariance(True) for plane, threshold in assembleTask.config.maskPropagationThresholds.items( ): bit = afwImage.MaskU.getMaskPlane(plane) statsCtrl.setMaskPropagationThreshold(bit, threshold) if doClip: statsFlags = afwMath.MEANCLIP else: statsFlags = afwMath.MEAN coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) coaddExposure.setCalib(assembleTask.scaleZeroPoint.getCalib()) coaddExposure.getInfo().setCoaddInputs( assembleTask.inputRecorder.makeCoaddInputs()) #remember to set metadata if you want any hope of running detection and measurement on this coadd: #self.assembleMetadata(coaddExposure, tempExpRefList, weightList) #most important thing is the psf coaddExposure.setFilter(coaddTempExpDict[a].values()[0].getFilter()) coaddInputs = coaddExposure.getInfo().getCoaddInputs() for tempExp, weight in zip(coaddTempExpDict[a].values(), imageScalerRes.weightList): assembleTask.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) #takes numCcds as argument coaddInputs.ccds.reserve(ccdsinPatch) coaddInputs.visits.reserve(len(imageScalerRes.dataIdList)) psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs()) coaddExposure.setPsf(psf) maskedImageList = afwImage.vectorMaskedImageF() coaddMaskedImage = coaddExposure.getMaskedImage() for dataId, imageScaler, exposure in zip( imageScalerRes.dataIdList, imageScalerRes.imageScalerList, coaddTempExpDict[a].values()): print dataId, imageScaler, exposure maskedImage = exposure.getMaskedImage() imageScaler.scaleMaskedImage(maskedImage) maskedImageList.append(maskedImage) maskedImage = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, imageScalerRes.weightList) coaddMaskedImage.assign(maskedImage, skyInfo.bbox) coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) # write out Coadd! mergefilename = mergecoaddloc + str(xInd) + "_" + str(yInd) + ".fits" mergcoadds[a] = coaddExposure coaddExposure.writeFits(mergefilename) end = datetime.datetime.now() d = end - start f.write("time for creating merged co-adds:\n ") f.write(str(d.total_seconds())) f.write("\n") start = datetime.datetime.now() config = DetectCoaddSourcesConfig() detectCoaddSources = DetectCoaddSourcesTask(config=config) for a in dfgby.indices: # Detect on Coadd exp = mergcoadds[a] detRes = detectCoaddSources.runDetection(exp, idFactory=None) end = datetime.datetime.now() d = end - start f.write("time for detecting sources:\n ") f.write(str(d.total_seconds())) f.close()
######################################## ####### Setup LSST begins ######################################## visit = 289697 ccdnum = 1 HOME_PATH = os.path.expanduser("~") butler = dafPersist.Butler(HOME_PATH + '/lsst_data/raw') #exposure = butler.get("instcal", visit=visit, ccdnum=ccdnum, filter='g', immediate=True) #mask = exposure.getMaskedImage().getMask().getArray() #TODO: move this matrix to SciDB #variance = exposure.getMaskedImage().getVariance().getArray() #TODO: move this matrix to SciDB #filename = HOME_PATH + "/lsst_data/raw/crblasted0289697/instcal0289697.1.fits" #fitsHeader = afwImage.readMetadata(filename) #wcs = afwImage.makeWcs(fitsHeader) charImage = CharacterizeImageTask() calibrateConfig = CalibrateConfig(doPhotoCal=False, doAstrometry=False) calibrateTask = CalibrateTask(config=calibrateConfig) makeCTEConfig = MakeCoaddTempExpConfig() makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig) newSkyMapConfig = skymap.discreteSkyMap.DiscreteSkyMapConfig( projection='STG', decList=[-4.9325280994132905], patchInnerDimensions=[2000, 2000], radiusList=[4.488775723429071], pixelScale=0.333, rotation=0.0, patchBorder=100, raList=[154.10660740464786], tractOverlap=0.0) hits_skymap = skymap.discreteSkyMap.DiscreteSkyMap(config=newSkyMapConfig) tract = hits_skymap[0]
from EdgeRolloffWarper import EdgeRolloffWarper image_file='star_grid_warped.fits' schema = afwTable.SourceTable.makeMinimalSchema() schema.setVersion(0) # # CalibrateTask # cal_config = CalibrateTask.ConfigClass() cal_config.doAstrometry = False cal_config.doPhotoCal = False cal_config.doBackground = True calibrateTask = CalibrateTask(config=cal_config) calibrateTask.repair.config.doCosmicRay = False calibrateTask.detection.config.thresholdValue = 5 calibrateTask.detection.config.includeThresholdMultiplier = 1 calibrateTask.config.initialPsf.fwhm = 2.85 # # SourceDetectionTask # det_config = SourceDetectionTask.ConfigClass() det_config.reEstimateBackground = False detectionTask = SourceDetectionTask(config=det_config, schema=schema) #
def testComponents(self): """Test that we can run the first-level subtasks of ProcessCcdTasks. This tests that we can run these subtasks from the command-line independently (they're all CmdLineTasks) as well as directly from Python (without giving them access to a Butler). Aside from verifying that no exceptions are raised, we simply tests that most persisted results are present and equivalent to both in-memory results. """ outPath = tempfile.mkdtemp() if OutputName is None else "{}-Components".format(OutputName) # We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it # to those tasks. inputButler = lsst.daf.persistence.Butler(InputDir) # Construct task instances we can use directly from Python isrTask = IsrTask( config=getObsTestConfig(IsrTask), name="isr2" ) # If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these # tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class # that doesn't require a Butler. If we don't, we should construct a butler-based on outside these # task constructors and pass the LoadReferenceObjectsTask instance to the task constructors. charImageTask = CharacterizeImageTask( config=getObsTestConfig(CharacterizeImageTask), name="charImage2" ) calibrateTask = CalibrateTask( config=getObsTestConfig(CalibrateTask), name="calibrate2", icSourceSchema=charImageTask.schema ) try: dataId = dict(visit=1) dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.items()] isrResult1 = IsrTask.parseAndRun( args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList, doReturnResults=True, ) # We'll just use the butler to get the original image and calibration frames; it's not clear # extending the test coverage to include that is worth it. dataRef = inputButler.dataRef("raw", dataId=dataId) rawExposure = dataRef.get("raw", immediate=True) camera = dataRef.get("camera") isrData = isrTask.readIsrData(dataRef, rawExposure) isrResult2 = isrTask.run( rawExposure, bias=isrData.bias, linearizer=isrData.linearizer, flat=isrData.flat, defects=isrData.defects, fringes=isrData.fringes, bfKernel=isrData.bfKernel, camera=camera, ) self.assertMaskedImagesEqual( isrResult1.parsedCmd.butler.get("postISRCCD", dataId, immediate=True).getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertMaskedImagesEqual( isrResult2.exposure.getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage() ) icResult1 = CharacterizeImageTask.parseAndRun( args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList, doReturnResults=True, ) icResult2 = charImageTask.run(isrResult2.exposure) self.assertMaskedImagesEqual( icResult1.parsedCmd.butler.get("icExp", dataId, immediate=True).getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertMaskedImagesEqual( icResult2.exposure.getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertCatalogsEqual( icResult1.parsedCmd.butler.get("icSrc", dataId, immediate=True), icResult1.resultList[0].result.sourceCat ) self.assertCatalogsEqual( icResult2.sourceCat, icResult1.resultList[0].result.sourceCat, skipCols=("id", "parent") # since we didn't want to pass in an ExposureIdInfo, IDs disagree ) self.assertBackgroundListsEqual( icResult1.parsedCmd.butler.get("icExpBackground", dataId, immediate=True), icResult1.resultList[0].result.background ) self.assertBackgroundListsEqual( icResult2.background, icResult1.resultList[0].result.background ) calResult1 = CalibrateTask.parseAndRun( args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList, doReturnResults=True, ) calResult2 = calibrateTask.run( icResult2.exposure, background=icResult2.background, icSourceCat=icResult2.sourceCat ) self.assertMaskedImagesEqual( calResult1.parsedCmd.butler.get("calexp", dataId, immediate=True).getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertMaskedImagesEqual( calResult2.exposure.getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertCatalogsEqual( calResult1.parsedCmd.butler.get("src", dataId, immediate=True), calResult1.resultList[0].result.sourceCat ) self.assertCatalogsEqual( calResult2.sourceCat, calResult1.resultList[0].result.sourceCat, skipCols=("id", "parent") ) self.assertBackgroundListsEqual( calResult1.parsedCmd.butler.get("calexpBackground", dataId, immediate=True), calResult1.resultList[0].result.background ) self.assertBackgroundListsEqual( calResult2.background, calResult1.resultList[0].result.background ) finally: if OutputName is None: shutil.rmtree(outPath) else: print("testProcessCcd.py's output data saved to %r" % (OutputName,))
#!/usr/bin/env python # # LSST Data Management System # Copyright 2016 LSST/AURA # # This product includes software developed by the # LSST Project (http://www.lsst.org/). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the LSST License Statement and # the GNU General Public License along with this program. If not, # see <http://www.lsstcorp.org/LegalNotices/>. # from lsst.pipe.tasks.calibrate import CalibrateTask CalibrateTask.parseAndRun()