def makeIdFactory(self, dataRef): """Return an IdFactory for setting the detection identifiers The actual parameters used in the IdFactory are provided by the butler (through the provided data reference. """ info = ExposureIdInfo( int(dataRef.get(self.config.coaddName + datasetName)), dataRef.get(self.config.coaddName + datasetName + "_bits")) return info.makeSourceIdFactory()
def makeIdFactory(self, dataRef): """Return an IdFactory for setting the detection identifiers The actual parameters used in the IdFactory are provided by the butler (through the provided data reference. """ expId = getGen3CoaddExposureId(dataRef, coaddName=self.config.coaddName, includeBand=includeBand, log=self.log) info = ExposureIdInfo(expId, dataRef.get(self.config.coaddName + datasetName + "_bits")) return info.makeSourceIdFactory()
def makeIdFactory(self, dataRef, exposureId): """Create an object that generates globally unique source IDs. Source IDs are created based on a per-CCD ID and the ID of the CCD itself. Parameters ---------- dataRef : `lsst.daf.persistence.ButlerDataRef` Butler data reference. The "CoaddId_bits" and "CoaddId" datasets are accessed. The data ID must have tract and patch keys. """ # With the default configuration, this IdFactory doesn't do anything, # because the IDs it generates are immediately overwritten by the ID # from the reference catalog (since that's in # config.measurement.copyColumns). But we create one here anyway, to # allow us to revert back to the old behavior of generating new forced # source IDs, just by renaming the ID in config.copyColumns to # "object_id". exposureIdInfo = ExposureIdInfo( exposureId, dataRef.get(self.config.coaddName + "CoaddId_bits")) return exposureIdInfo.makeSourceIdFactory()
def run(self, exposure, exposureIdInfo=None): if not exposure.hasPsf(): self.installSimplePsf.run(exposure=exposure) if exposureIdInfo is None: exposureIdInfo = ExposureIdInfo() try: self.repair.run(exposure=exposure, keepCRs=True) except LengthError: self.log.info("Skipping cosmic ray detection: Too many CR pixels (max %0.f)" % self.repair.cosmicray.nCrPixelMax) sourceIdFactory = exposureIdInfo.makeSourceIdFactory() table = SourceTable.make(self.schema, sourceIdFactory) table.setMetadata(self.algMetadata) filtered = maximum_filter(exposure.getImage().array, size=self.config.maximumFilterBoxWidth) detected = (filtered == exposure.getImage().getArray()) & (filtered > self.config.thresholdValue) detectedImage = afwImage.ImageF(detected.astype(np.float32)) fps = afwDetect.FootprintSet(detectedImage, afwDetect.Threshold(0.5)) fp_ctrl = afwDetect.FootprintControl(True, True) fps = afwDetect.FootprintSet(fps, self.config.footprintGrowValue, fp_ctrl) sources = afwTable.SourceCatalog(table) fps.makeSources(sources) self.measurement.run(measCat=sources, exposure=exposure, exposureId=exposureIdInfo.expId) self.catalogCalculation.run(sources) ## Add metadata to source catalog md = exposure.getMetadata() sources.getMetadata().add("BOTXCAM", md["BOTXCAM"]) sources.getMetadata().add("BOTYCAM", md["BOTYCAM"]) self.display("measure", exposure=exposure, sourceCat=sources) return pipeBase.Struct(sourceCat=sources)
def run(self, diaSources, tractPatchId, skymapBits): """Associate DiaSources into a collection of DiaObjects using a brute force matching algorithm. Reproducible is for the same input data is assured by ordering the DiaSource data by ccdVisit ordering. Parameters ---------- diaSources : `pandas.DataFrame` DiaSources grouped by CcdVisitId to spatially associate into DiaObjects. tractPatchId : `int` Unique identifier for the tract patch. skymapBits : `int` Maximum number of bits used the ``tractPatchId`` integer identifier. Returns ------- results : `lsst.pipe.base.Struct` Results struct with attributes: ``assocDiaSources`` Table of DiaSources with updated values for the DiaObjects they are spatially associated to (`pandas.DataFrame`). ``diaObjects`` Table of DiaObjects from matching DiaSources (`pandas.DataFrame`). """ # Expected indexes include diaSourceId or meaningless range index # If meaningless range index, drop it, else keep it. doDropIndex = diaSources.index.names[0] is None diaSources.reset_index(inplace=True, drop=doDropIndex) # Sort by ccdVisit and diaSourceId to get a reproducible ordering for # the association. diaSources.set_index(["ccdVisitId", "diaSourceId"], inplace=True) # Empty lists to store matching and location data. diaObjectCat = [] diaObjectCoords = [] healPixIndices = [] # Create Id factory and catalog for creating DiaObjectIds. exposureIdInfo = ExposureIdInfo(tractPatchId, skymapBits) idFactory = exposureIdInfo.makeSourceIdFactory() idCat = afwTable.SourceCatalog( afwTable.SourceTable.make(afwTable.SourceTable.makeMinimalSchema(), idFactory)) for ccdVisit in diaSources.index.levels[0]: # For the first ccdVisit, just copy the DiaSource info into the # diaObject data to create the first set of Objects. ccdVisitSources = diaSources.loc[ccdVisit] if len(diaObjectCat) == 0: for diaSourceId, diaSrc in ccdVisitSources.iterrows(): self.addNewDiaObject(diaSrc, diaSources, ccdVisit, diaSourceId, diaObjectCat, idCat, diaObjectCoords, healPixIndices) continue # Temp list to store DiaObjects already used for this ccdVisit. usedMatchIndicies = [] # Run over subsequent data. for diaSourceId, diaSrc in ccdVisitSources.iterrows(): # Find matches. matchResult = self.findMatches(diaSrc["ra"], diaSrc["decl"], 2*self.config.tolerance, healPixIndices, diaObjectCat) dists = matchResult.dists matches = matchResult.matches # Create a new DiaObject if no match found. if dists is None: self.addNewDiaObject(diaSrc, diaSources, ccdVisit, diaSourceId, diaObjectCat, idCat, diaObjectCoords, healPixIndices) continue # If matched, update catalogs and arrays. if np.min(dists) < np.deg2rad(self.config.tolerance/3600): matchDistArg = np.argmin(dists) matchIndex = matches[matchDistArg] # Test to see if the DiaObject has been used. if np.isin([matchIndex], usedMatchIndicies).sum() < 1: self.updateCatalogs(matchIndex, diaSrc, diaSources, ccdVisit, diaSourceId, diaObjectCat, diaObjectCoords, healPixIndices) usedMatchIndicies.append(matchIndex) # If the matched DiaObject has already been used, create a # new DiaObject for this DiaSource. else: self.addNewDiaObject(diaSrc, diaSources, ccdVisit, diaSourceId, diaObjectCat, idCat, diaObjectCoords, healPixIndices) # Create new DiaObject if no match found within the matching # tolerance. else: self.addNewDiaObject(diaSrc, diaSources, ccdVisit, diaSourceId, diaObjectCat, idCat, diaObjectCoords, healPixIndices) # Drop indices before returning associated diaSource catalog. diaSources.reset_index(inplace=True) diaSources.set_index("diaSourceId", inplace=True, verify_integrity=True) objs = diaObjectCat if diaObjectCat else np.array([], dtype=[('diaObjectId', 'int64'), ('ra', 'float64'), ('decl', 'float64'), ('nDiaSources', 'int64')]) diaObjects = pd.DataFrame(data=objs) if "diaObjectId" in diaObjects.columns: diaObjects.set_index("diaObjectId", inplace=True, verify_integrity=True) return pipeBase.Struct( assocDiaSources=diaSources, diaObjects=diaObjects)
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None): """!Calibrate an exposure (science image or coadd) @param[in,out] exposure exposure to calibrate (an lsst.afw.image.ExposureF or similar); in: - MaskedImage - Psf out: - MaskedImage has background subtracted - Wcs is replaced - PhotoCalib is replaced @param[in] exposureIdInfo ID info for exposure (an lsst.obs.base.ExposureIdInfo) If not provided, returned SourceCatalog IDs will not be globally unique. @param[in,out] background background model already subtracted from exposure (an lsst.afw.math.BackgroundList). May be None if no background has been subtracted, though that is unusual for calibration. A refined background model is output. @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask from which we can copy some fields. @return pipe_base Struct containing these fields: - exposure calibrate science exposure with refined WCS and PhotoCalib - background model of background subtracted from exposure (an lsst.afw.math.BackgroundList) - sourceCat catalog of measured sources - astromMatches list of source/refObj matches from the astrometry solver """ # detect, deblend and measure sources if exposureIdInfo is None: exposureIdInfo = ExposureIdInfo() if background is None: background = BackgroundList() sourceIdFactory = exposureIdInfo.makeSourceIdFactory() table = SourceTable.make(self.schema, sourceIdFactory) table.setMetadata(self.algMetadata) detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True) sourceCat = detRes.sources if detRes.fpSets.background: for bg in detRes.fpSets.background: background.append(bg) if self.config.doSkySources: skySourceFootprints = self.skySources.run( mask=exposure.mask, seed=exposureIdInfo.expId) if skySourceFootprints: for foot in skySourceFootprints: s = sourceCat.addNew() s.setFootprint(foot) s.set(self.skySourceKey, True) if self.config.doDeblend: self.deblend.run(exposure=exposure, sources=sourceCat) self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId) if self.config.doApCorr: self.applyApCorr.run(catalog=sourceCat, apCorrMap=exposure.getInfo().getApCorrMap()) self.catalogCalculation.run(sourceCat) self.setPrimaryFlags.run(sourceCat) if icSourceCat is not None and \ len(self.config.icSourceFieldsToCopy) > 0: self.copyIcSourceFields(icSourceCat=icSourceCat, sourceCat=sourceCat) # TODO DM-11568: this contiguous check-and-copy could go away if we # reserve enough space during SourceDetection and/or SourceDeblend. # NOTE: sourceSelectors require contiguous catalogs, so ensure # contiguity now, so views are preserved from here on. if not sourceCat.isContiguous(): sourceCat = sourceCat.copy(deep=True) # perform astrometry calibration: # fit an improved WCS and update the exposure's WCS in place astromMatches = None matchMeta = None if self.config.doAstrometry: try: astromRes = self.astrometry.run( exposure=exposure, sourceCat=sourceCat, ) astromMatches = astromRes.matches matchMeta = astromRes.matchMeta except Exception as e: if self.config.requireAstrometry: raise self.log.warn("Unable to perform astrometric calibration " "(%s): attempting to proceed" % e) # compute photometric calibration if self.config.doPhotoCal: try: photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId) exposure.setPhotoCalib(photoRes.photoCalib) # TODO: reword this to phrase it in terms of the calibration factor? self.log.info("Photometric zero-point: %f" % photoRes.photoCalib.instFluxToMagnitude(1.0)) self.setMetadata(exposure=exposure, photoRes=photoRes) except Exception as e: if self.config.requirePhotoCal: raise self.log.warn("Unable to perform photometric calibration " "(%s): attempting to proceed" % e) self.setMetadata(exposure=exposure, photoRes=None) if self.config.doInsertFakes: self.insertFakes.run(exposure, background=background) table = SourceTable.make(self.schema, sourceIdFactory) table.setMetadata(self.algMetadata) detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True) sourceCat = detRes.sources if detRes.fpSets.background: for bg in detRes.fpSets.background: background.append(bg) if self.config.doDeblend: self.deblend.run(exposure=exposure, sources=sourceCat) self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId) if self.config.doApCorr: self.applyApCorr.run( catalog=sourceCat, apCorrMap=exposure.getInfo().getApCorrMap()) self.catalogCalculation.run(sourceCat) if icSourceCat is not None and len( self.config.icSourceFieldsToCopy) > 0: self.copyIcSourceFields(icSourceCat=icSourceCat, sourceCat=sourceCat) frame = getDebugFrame(self._display, "calibrate") if frame: displayAstrometry( sourceCat=sourceCat, exposure=exposure, matches=astromMatches, frame=frame, pause=False, ) return pipeBase.Struct( exposure=exposure, background=background, sourceCat=sourceCat, astromMatches=astromMatches, matchMeta=matchMeta, # These are duplicate entries with different names for use with # gen3 middleware outputExposure=exposure, outputCat=sourceCat, outputBackground=background, )