def make_source_catalog_from_astropy_table(out_table, debug=False): """Return an AFW SourceCatalog from an Astropy Table Written with extensive reference to https://github.com/lsst/meas_astrom/blob/master/convertToFitsTable.py """ filters = ('J', 'H', 'K') schema = makeMinimalSchema(filters=filters, debug=debug) out_cat = afwTable.SourceCatalog(schema) for row in out_table: record = out_cat.addNew() record.setId(twomass_int_id(row['2MASSID'])) record.setRa(float(row['coord_ra']) * lsst.afw.geom.degrees) record.setDec(float(row['coord_dec']) * lsst.afw.geom.degrees) for filt in filters: filtMag = '%s_mag' % filt filtMagSigma = '%s_mag_sigma' % filt filtFlux = '%s_flux' % filt filtFluxSigma = '%s_fluxSigma' % filt abMag = vegaToABMag(row[filtMag], filt) # error remains unchanged record.set(filtFlux, fluxFromABMag(abMag)) record.set(filtFluxSigma, fluxErrFromABMagErr(row[filtMagSigma], abMag)) return out_cat
def _getFluxes(self, inputData): """Compute the flux fields that will go into the output catalog. Parameters ---------- inputData : `numpy.ndarray` The input data to compute fluxes for. Returns ------- fluxes : `dict` [`str`, `numpy.ndarray`] The values that will go into the flux and fluxErr fields in the output catalog. """ result = {} for item in self.config.mag_column_list: result[item + '_flux'] = (inputData[item] * u.ABmag).to_value( u.nJy) if len(self.config.mag_err_column_map) > 0: for err_key in self.config.mag_err_column_map.keys(): error_col_name = self.config.mag_err_column_map[err_key] # TODO: multiply by 1e9 here until we have a replacement (see DM-16903) # NOTE: copy the arrays because the numpy strides may not be useable by C++. fluxErr = fluxErrFromABMagErr(inputData[error_col_name].copy(), inputData[err_key].copy()) * 1e9 result[err_key + '_fluxErr'] = fluxErr return result
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName, applyColorterms=False): """Load the necessary reference catalog sources, convert fluxes to correct units, and apply color term corrections if requested. Parameters ---------- refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` The reference catalog loader to use to get the data. referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` Source selector to apply to loaded reference catalog. center : `lsst.geom.SpherePoint` The center around which to load sources. radius : `lsst.geom.Angle` The radius around ``center`` to load sources in. filterName : `str` The name of the camera filter to load fluxes for. applyColorterms : `bool` Apply colorterm corrections to the refcat for ``filterName``? Returns ------- refCat : `lsst.afw.table.SimpleCatalog` The loaded reference catalog. fluxField : `str` The name of the reference catalog flux field appropriate for ``filterName``. """ skyCircle = refObjLoader.loadSkyCircle(center, afwGeom.Angle(radius, afwGeom.radians), filterName) selected = referenceSelector.run(skyCircle.refCat) # Need memory contiguity to get reference filters as a vector. if not selected.sourceCat.isContiguous(): refCat = selected.sourceCat.copy(deep=True) else: refCat = selected.sourceCat if applyColorterms: try: refCatName = refObjLoader.ref_dataset_name except AttributeError: # NOTE: we need this try:except: block in place until we've completely removed a.net support. raise RuntimeError("Cannot perform colorterm corrections with a.net refcats.") self.log.info("Applying color terms for filterName=%r reference catalog=%s", filterName, refCatName) colorterm = self.config.colorterms.getColorterm( filterName=filterName, photoCatName=refCatName, doRaise=True) refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat, filterName) refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy) # TODO: I didn't want to use this, but I'll deal with it in DM-16903 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9 return refCat, skyCircle.fluxField
def _set_mags(self, record, row, key_map): """!Set the flux records from the input magnitudes @param[in,out] record SourceCatalog record to modify @param[in] row dict like object containing magnitude values @param[in] key_map Map of catalog keys to use in filling the record """ for item in self.config.mag_column_list: record.set(key_map[item + "_flux"], fluxFromABMag(row[item])) if len(self.config.mag_err_column_map) > 0: for err_key in self.config.mag_err_column_map.keys(): error_col_name = self.config.mag_err_column_map[err_key] record.set(key_map[err_key + "_fluxSigma"], fluxErrFromABMagErr(row[error_col_name], row[err_key]))
def testBasics(self): for flux in (1, 210, 3210, 43210, 543210): abMag = afwImage.abMagFromFlux(flux) self.assertAlmostEqual(abMag, refABMagFromFlux(flux)) fluxRoundTrip = afwImage.fluxFromABMag(abMag) self.assertAlmostEqual(flux, fluxRoundTrip) for fluxErrFrac in (0.001, 0.01, 0.1): fluxErr = flux * fluxErrFrac abMagErr = afwImage.abMagErrFromFluxErr(fluxErr, flux) self.assertAlmostEqual(abMagErr, refABMagErrFromFluxErr(fluxErr, flux)) fluxErrRoundTrip = afwImage.fluxErrFromABMagErr(abMagErr, abMag) self.assertAlmostEqual(fluxErr, fluxErrRoundTrip)
def _set_mags(self, record, row, key_map): """!Set the flux records from the input magnitudes @param[in,out] record SourceCatalog record to modify @param[in] row dict like object containing magnitude values @param[in] key_map Map of catalog keys to use in filling the record """ for item in self.config.mag_column_list: record.set(key_map[item + '_flux'], fluxFromABMag(row[item])) if len(self.config.mag_err_column_map) > 0: for err_key in self.config.mag_err_column_map.keys(): error_col_name = self.config.mag_err_column_map[err_key] record.set( key_map[err_key + '_fluxSigma'], fluxErrFromABMagErr(row[error_col_name], row[err_key]))
def testVector(self): flux = np.array([1.0, 210.0, 3210.0, 43210.0, 543210.0]) flux.flags.writeable = False # Put the 'const' into ndarray::Array<double const, 1, 0> abMag = afwImage.abMagFromFlux(flux) self.assertFloatsAlmostEqual(abMag, refABMagFromFlux(flux)) fluxRoundTrip = afwImage.fluxFromABMag(abMag) self.assertFloatsAlmostEqual(flux, fluxRoundTrip, rtol=1.0e-15) for fluxErrFrac in (0.001, 0.01, 0.1): fluxErr = flux * fluxErrFrac abMagErr = afwImage.abMagErrFromFluxErr(fluxErr, flux) self.assertFloatsAlmostEqual(abMagErr, refABMagErrFromFluxErr(fluxErr, flux)) fluxErrRoundTrip = afwImage.fluxErrFromABMagErr(abMagErr, abMag) self.assertFloatsAlmostEqual(fluxErr, fluxErrRoundTrip, rtol=1.0e-15)
def _formatCatalog(self, fgcmStarCat, offsets): """ Turn an FGCM-formatted star catalog, applying zeropoint offsets. Parameters ---------- fgcmStarCat: `afwTable.SimpleCatalog` SimpleCatalog as output by fgcmcal offsets: `list` with len(self.bands) entries Zeropoint offsets to apply Returns ------- formattedCat: `afwTable.SimpleCatalog` SimpleCatalog suitable for using as a reference catalog """ sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema) minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema( self.bands, addCentroid=False, addIsResolved=True, coordErrDim=0) sourceMapper.addMinimalSchema(minSchema) for band in self.bands: sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32) formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) formattedCat.reserve(len(fgcmStarCat)) formattedCat.extend(fgcmStarCat, mapper=sourceMapper) # Note that we don't have to set `resolved` because the default is False for b, band in enumerate(self.bands): mag = fgcmStarCat['mag_std_noabs'][:, b] + offsets[b] # We want fluxes in Jy from calibrated AB magnitudes # (after applying offset) # TODO: Full implementation of RFC-549 will have all reference # catalogs in nJy instead of Jy. flux = afwImage.fluxFromABMag(mag) fluxErr = afwImage.fluxErrFromABMagErr( fgcmStarCat['magErr_std'][:, b], mag) formattedCat['%s_flux' % (band)][:] = flux formattedCat['%s_fluxErr' % (band)][:] = fluxErr formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b] return formattedCat
def _setFlux(self, record, row, key_map): """Set flux fields in a record of an indexed catalog. Parameters ---------- record : `lsst.afw.table.SimpleRecord` Row from indexed catalog to modify. row : structured `numpy.array` Row from catalog being ingested. key_map : `dict` mapping `str` to `lsst.afw.table.Key` Map of catalog keys. """ for item in self.config.mag_column_list: record.set(key_map[item + '_flux'], fluxFromABMag(row[item])) if len(self.config.mag_err_column_map) > 0: for err_key in self.config.mag_err_column_map.keys(): error_col_name = self.config.mag_err_column_map[err_key] record.set( key_map[err_key + '_fluxErr'], fluxErrFromABMagErr(row[error_col_name], row[err_key]))
def _formatCatalog(self, fgcmStarCat, offsets): """ Turn an FGCM-formatted star catalog, applying zp offsets. parameters ---------- fgcmStarCat: SimpleCatalog SimpleCatalog as output by fgcmcal offsets: list with len(self.bands) entries Zeropoint offsets to apply returns ------- formattedCat: SimpleCatalog SimpleCatalog suitable for ref_cat """ sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema) sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema()) for band in self.bands: sourceMapper.editOutputSchema().addField('%s_flux' % (band), type=np.float64) sourceMapper.editOutputSchema().addField('%s_fluxErr' % (band), type=np.float64) sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.float64) formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) formattedCat.reserve(len(fgcmStarCat)) formattedCat.extend(fgcmStarCat, mapper=sourceMapper) for b, band in enumerate(self.bands): mag = fgcmStarCat['mag_std_noabs'][:, b] + offsets[b] flux = afwImage.fluxFromABMag(mag) fluxErr = afwImage.fluxErrFromABMagErr( fgcmStarCat['magerr_std'][:, b], mag) formattedCat['%s_flux' % (band)][:] = flux formattedCat['%s_fluxErr' % (band)][:] = fluxErr formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b] return formattedCat
def _setFlux(self, record, row, key_map): """Set flux fields in a record of an indexed catalog. Parameters ---------- record : `lsst.afw.table.SimpleRecord` Row from indexed catalog to modify. row : structured `numpy.array` Row from catalog being ingested. key_map : `dict` mapping `str` to `lsst.afw.table.Key` Map of catalog keys. """ for item in self.config.mag_column_list: record.set(key_map[item+'_flux'], (row[item]*u.ABmag).to_value(u.nJy)) if len(self.config.mag_err_column_map) > 0: for err_key in self.config.mag_err_column_map.keys(): error_col_name = self.config.mag_err_column_map[err_key] # TODO: multiply by 1e9 here until we have a replacement (see DM-16903) fluxErr = fluxErrFromABMagErr(row[error_col_name], row[err_key]) if fluxErr is not None: fluxErr *= 1e9 record.set(key_map[err_key+'_fluxErr'], fluxErr)
def readSrc(self, dataRef): """Read source catalog etc for input dataRef The following are returned: Source catalog, matched list, and wcs will be read from 'src', 'srcMatch', and 'calexp_md', respectively. NOTE: If the detector has nQuarter%4 != 0 (i.e. it is rotated w.r.t the focal plane coordinate system), the (x, y) pixel values of the centroid slot for the source catalogs are rotated such that pixel (0, 0) is the LLC (i.e. the coordinate system expected by meas_mosaic). If color transformation information is given, it will be applied to the reference flux of the matched list. The source catalog and matched list will be converted to measMosaic's Source and SourceMatch and returned. The number of 'Source's in each cell defined by config.cellSize will be limited to brightest config.nStarPerCell. """ self.log = Log.getDefaultLogger() dataId = dataRef.dataId try: if not dataRef.datasetExists("src"): raise RuntimeError("no data for src %s" % (dataId)) if not dataRef.datasetExists("calexp_md"): raise RuntimeError("no data for calexp_md %s" % (dataId)) calexp_md = dataRef.get("calexp_md", immediate=True) detector = dataRef.get("camera")[dataRef.dataId["ccd"]] # OK for HSC; maybe not for other cameras wcs = afwGeom.makeSkyWcs(calexp_md) nQuarter = detector.getOrientation().getNQuarter() sources = dataRef.get("src", immediate=True, flags=afwTable.SOURCE_IO_NO_FOOTPRINTS) # Check if we are looking at HSC stack outputs: if so, no pixel rotation of sources is # required, but alias mapping must be set to associate HSC's schema with that of LSST. hscRun = mosaicUtils.checkHscStack(calexp_md) if hscRun is None: if nQuarter%4 != 0: dims = afwImage.bboxFromMetadata(calexp_md).getDimensions() sources = mosaicUtils.rotatePixelCoords(sources, dims.getX(), dims.getY(), nQuarter) # Set some alias maps for the source catalog where needed for # backwards compatibility if self.config.srcSchemaMap and hscRun: aliasMap = sources.schema.getAliasMap() for lsstName, otherName in self.config.srcSchemaMap.items(): aliasMap.set(lsstName, otherName) if self.config.flagsToAlias and "calib_psfUsed" in sources.schema: aliasMap = sources.schema.getAliasMap() for lsstName, otherName in self.config.flagsToAlias.items(): aliasMap.set(lsstName, otherName) refObjLoader = self.config.loadAstrom.apply(butler=dataRef.getButler()) srcMatch = dataRef.get("srcMatch", immediate=True) if hscRun is not None: # The reference object loader grows the bbox by the config parameter pixelMargin. This # is set to 50 by default but is not reflected by the radius parameter set in the # metadata, so some matches may reside outside the circle searched within this radius # Thus, increase the radius set in the metadata fed into joinMatchListWithCatalog() to # accommodate. matchmeta = srcMatch.table.getMetadata() rad = matchmeta.getDouble("RADIUS") matchmeta.setDouble("RADIUS", rad*1.05, "field radius in degrees, approximate, padded") matches = refObjLoader.joinMatchListWithCatalog(srcMatch, sources) # Set the aliap map for the matched sources (i.e. the [1] attribute schema for each match) if self.config.srcSchemaMap is not None and hscRun is not None: for mm in matches: aliasMap = mm[1].schema.getAliasMap() for lsstName, otherName in self.config.srcSchemaMap.items(): aliasMap.set(lsstName, otherName) if hscRun is not None: for slot in ("PsfFlux", "ModelFlux", "ApFlux", "GaussianFlux", "Centroid", "Shape"): getattr(matches[0][1].getTable(), "define" + slot)( getattr(sources, "get" + slot + "Definition")()) # For some reason, the CalibFlux slot in sources is coming up as centroid_sdss, so # set it to flux_naive explicitly for slot in ("CalibFlux", ): getattr(matches[0][1].getTable(), "define" + slot)("flux_naive") matches = [m for m in matches if m[0] is not None] refSchema = matches[0][0].schema if matches else None if self.cterm is not None and len(matches) != 0: # Add a "flux" field to the input schema of the first element # of the match and populate it with a colorterm correct flux. mapper = afwTable.SchemaMapper(refSchema) for key, field in refSchema: mapper.addMapping(key) fluxKey = mapper.editOutputSchema().addField("flux", type=float, doc="Reference flux") fluxErrKey = mapper.editOutputSchema().addField("fluxErr", type=float, doc="Reference flux uncertainty") table = afwTable.SimpleTable.make(mapper.getOutputSchema()) table.preallocate(len(matches)) for match in matches: newMatch = table.makeRecord() newMatch.assign(match[0], mapper) match[0] = newMatch # extract the matched refCat as a Catalog for the colorterm code refCat = afwTable.SimpleCatalog(matches[0].first.schema) refCat.reserve(len(matches)) for x in matches: record = refCat.addNew() record.assign(x.first) refMag, refMagErr = self.cterm.getCorrectedMagnitudes(refCat, afwImage.Filter(calexp_md).getName()) # NOTE: mosaic assumes fluxes are in Jy refFlux = (refMag*astropy.units.ABmag).to_value(astropy.units.Jy) refFluxErr = afwImage.fluxErrFromABMagErr(refMagErr, refMag) matches = [self.setCatFlux(m, flux, fluxKey, fluxErr, fluxErrKey) for m, flux, fluxErr in zip(matches, refFlux, refFluxErr) if flux == flux] else: filterName = afwImage.Filter(calexp_md).getName() refFluxField = measAlg.getRefFluxField(refSchema, filterName) refSchema.getAliasMap().set("flux", refFluxField) # LSST reads in a_net catalogs with flux in "janskys", so must convert back to DN. matches = mosaicUtils.matchJanskyToDn(matches) selSources = self.selectStars(sources, self.config.includeSaturated) selMatches = self.selectStars(matches, self.config.includeSaturated) retSrc = list() retMatch = list() if len(selMatches) > self.config.minNumMatch: naxis1, naxis2 = afwImage.bboxFromMetadata(calexp_md).getDimensions() if hscRun is None: if nQuarter%2 != 0: naxis1, naxis2 = naxis2, naxis1 bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(naxis1, naxis2)) cellSet = afwMath.SpatialCellSet(bbox, self.config.cellSize, self.config.cellSize) for s in selSources: if numpy.isfinite(s.getRa().asDegrees()): # get rid of NaN src = measMosaic.Source(s) src.setExp(dataId["visit"]) src.setChip(dataId["ccd"]) try: tmp = measMosaic.SpatialCellSource(src) cellSet.insertCandidate(tmp) except: self.log.info("FAILED TO INSERT CANDIDATE: visit=%d ccd=%d x=%f y=%f" % (dataRef.dataId["visit"], dataRef.dataId["ccd"], src.getX(), src.getY()) + " bbox=" + str(bbox)) for cell in cellSet.getCellList(): cell.sortCandidates() for i, cand in enumerate(cell): src = cand.getSource() retSrc.append(src) if i == self.config.nStarPerCell - 1: break for m in selMatches: if m[0] is not None and m[1] is not None: match = (measMosaic.Source(m[0], wcs), measMosaic.Source(m[1])) match[1].setExp(dataId["visit"]) match[1].setChip(dataId["ccd"]) retMatch.append(match) else: self.log.info("%8d %3d : %d/%d matches Suspicious to wrong match. Ignore this CCD" % (dataRef.dataId["visit"], dataRef.dataId["ccd"], len(selMatches), len(matches))) except Exception as e: self.log.warn("Failed to read %s: %s" % (dataId, e)) return dataId, [None, None, None] return dataId, [retSrc, retMatch, wcs]
def _computeReferenceOffsets(self, butler): """ Compute offsets relative to a reference catalog. Parameters ---------- butler: lsst.daf.persistence.Butler Returns ------- offsets: np.array of floats Per band zeropoint offsets """ # Load the stars stars = butler.get('fgcmStandardStars', fgcmcycle=self.useCycle) # Only use stars that are observed in all the bands minObs = stars['ngood'].min(axis=1) # Depending on how things work, this might need to be configurable # However, I think that best results will occur when we use the same # pixels to do the calibration for all the bands, so I think this # is the right choice. goodStars = (minObs >= 1) stars = stars[goodStars] self.log.info( "Found %d stars with at least 1 good observation in each band" % (len(stars))) # We have to make a table for each pixel with flux/fluxErr sourceMapper = afwTable.SchemaMapper(stars.schema) sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema()) sourceMapper.editOutputSchema().addField('flux', type=np.float64, doc="flux") sourceMapper.editOutputSchema().addField('fluxErr', type=np.float64, doc="flux error") badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar', type='Flag', doc="bad flag") # The exposure is used to record the filter name exposure = afwImage.ExposureF() # Split up the stars (units are radians) theta = np.pi / 2. - stars['coord_dec'] phi = stars['coord_ra'] ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi) h, rev = esutil.stat.histogram(ipring, rev=True) gdpix, = np.where(h >= self.config.referencePixelizationMinStars) self.log.info( "Found %d pixels (nside=%d) with at least %d good stars" % (gdpix.size, self.config.referencePixelizationNside, self.config.referencePixelizationMinStars)) if gdpix.size < self.config.referencePixelizationNPixels: self.log.warn( "Found fewer good pixels (%d) than preferred in configuration (%d)" % (gdpix.size, self.config.referencePixelizationNPixels)) else: # Sample out the pixels we want to use gdpix = np.random.choice( gdpix, size=self.config.referencePixelizationNPixels, replace=False) results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'), ('nstar', 'i4', len(self.bands)), ('nmatch', 'i4', len(self.bands)), ('zp', 'f4', len(self.bands)), ('zpErr', 'f4', len(self.bands))]) results['hpix'] = ipring[rev[rev[gdpix]]] # We need a boolean index to deal with catalogs... selected = np.zeros(len(stars), dtype=np.bool) refFluxFields = [None] * len(self.bands) for p, pix in enumerate(gdpix): i1a = rev[rev[pix]:rev[pix + 1]] # Need to convert to a boolean array selected[:] = False selected[i1a] = True for b, band in enumerate(self.bands): sourceCat = afwTable.SimpleCatalog( sourceMapper.getOutputSchema()) sourceCat.reserve(len(i1a)) sourceCat.extend(stars[selected], mapper=sourceMapper) sourceCat['flux'] = afwImage.fluxFromABMag( stars['mag_std_noabs'][selected, b]) sourceCat['fluxErr'] = afwImage.fluxErrFromABMagErr( stars['magerr_std'][selected, b], stars['mag_std_noabs'][selected, b]) # Make sure we only use stars that have valid measurements # (This is perhaps redundant with requirements above that the # stars be observed in all bands, but it can't hurt) badStar = (stars['mag_std_noabs'][selected, b] > 90.0) for rec in sourceCat[badStar]: rec.set(badStarKey, True) exposure.setFilter(afwImage.Filter(band)) if refFluxFields[b] is None: # Need to find the flux field in the reference catalog # to work around limitations of DirectMatch in PhotoCal ctr = stars[0].getCoord() rad = 0.05 * lsst.geom.degrees refDataTest = self.refObjLoader.loadSkyCircle( ctr, rad, band) refFluxFields[b] = refDataTest.fluxField # Make a copy of the config so that we can modify it calConfig = copy.copy(self.config.photoCal.value) calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[ b] calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[ b] + 'Err' calTask = self.config.photoCal.target( refObjLoader=self.refObjLoader, config=calConfig, schema=sourceCat.getSchema()) struct = calTask.run(exposure, sourceCat) results['nstar'][p, b] = len(i1a) results['nmatch'][p, b] = len(struct.arrays.refMag) results['zp'][p, b] = struct.zp results['zpErr'][p, b] = struct.sigma # And compute the summary statistics offsets = np.zeros(len(self.bands)) for b, band in enumerate(self.bands): # make configurable ok, = np.where( results['nmatch'][:, b] >= self.config.referenceMinMatch) offsets[b] = np.median(results['zp'][ok, b]) madSigma = 1.4826 * np.median( np.abs(results['zp'][ok, b] - offsets[b])) self.log.info( "Reference catalog offset for %s band: %.6f +/- %.6f" % (band, offsets[b], madSigma)) return offsets
def _runFgcmOutputProducts(self, visitDataRefName, ccdDataRefName, filterMapping, zpOffsets, testVisit, testCcd, testFilter, testBandIndex): """ """ if self.logLevel is not None: self.otherArgs.extend(['--loglevel', 'fgcmcal=%s' % self.logLevel]) args = [self.inputDir, '--output', self.testDir, '--doraise'] args.extend(self.otherArgs) result = fgcmcal.FgcmOutputProductsTask.parseAndRun( args=args, config=self.config, doReturnResults=True) self._checkResult(result) # Extract the offsets from the results offsets = result.resultList[0].results.offsets self.assertFloatsAlmostEqual(offsets[0], zpOffsets[0], rtol=1e-6) self.assertFloatsAlmostEqual(offsets[1], zpOffsets[1], rtol=1e-6) butler = dafPersistence.butler.Butler(self.testDir) # Test the reference catalog stars # Read in the raw stars... rawStars = butler.get('fgcmStandardStars', fgcmcycle=0) # Read in the new reference catalog... config = LoadIndexedReferenceObjectsConfig() config.ref_dataset_name = 'fgcm_stars' task = LoadIndexedReferenceObjectsTask(butler, config=config) # Read in a giant radius to get them all refStruct = task.loadSkyCircle(rawStars[0].getCoord(), 5.0 * lsst.geom.degrees, filterName='r') # Make sure all the stars are there self.assertEqual(len(rawStars), len(refStruct.refCat)) # And make sure the numbers are consistent test, = np.where(rawStars['id'][0] == refStruct.refCat['id']) mag = rawStars['mag_std_noabs'][0, 0] + offsets[0] flux = afwImage.fluxFromABMag(mag) fluxErr = afwImage.fluxErrFromABMagErr(rawStars['magerr_std'][0, 0], mag) self.assertFloatsAlmostEqual(flux, refStruct.refCat['r_flux'][test[0]], rtol=1e-6) self.assertFloatsAlmostEqual(fluxErr, refStruct.refCat['r_fluxErr'][test[0]], rtol=1e-6) # Test the joincal_photoCalib output zptCat = butler.get('fgcmZeropoints', fgcmcycle=0) selected = (zptCat['fgcmflag'] < 16) # Read in all the calibrations, these should all be there for rec in zptCat[selected]: testCal = butler.get('jointcal_photoCalib', dataId={ visitDataRefName: int(rec['visit']), ccdDataRefName: int(rec['ccd']), 'filter': filterMapping[rec['filtername']], 'tract': 0 }) # Our round-trip tests will be on this final one which is still loaded testCal = butler.get('jointcal_photoCalib', dataId={ visitDataRefName: int(testVisit), ccdDataRefName: int(testCcd), 'filter': filterMapping[testFilter], 'tract': 0 }) src = butler.get('src', dataId={ visitDataRefName: int(testVisit), ccdDataRefName: int(testCcd) }) # Only test sources with positive flux gdSrc = (src['slot_CalibFlux_flux'] > 0.0) # We need to apply the calibration offset to the fgcmzpt (which is internal # and doesn't know about that yet) testZpInd, = np.where((zptCat['visit'] == testVisit) & (zptCat['ccd'] == testCcd)) fgcmZpt = zptCat['fgcmzpt'][testZpInd] + offsets[testBandIndex] # This is the magnitude through the mean calibration photoCalMeanCalMags = np.zeros(gdSrc.sum()) # This is the magnitude through the full focal-plane variable mags photoCalMags = np.zeros_like(photoCalMeanCalMags) # This is the magnitude with the FGCM (central-ccd) zeropoint zptMeanCalMags = np.zeros_like(photoCalMeanCalMags) for i, rec in enumerate(src[gdSrc]): photoCalMeanCalMags[i] = testCal.instFluxToMagnitude( rec['slot_CalibFlux_flux']) photoCalMags[i] = testCal.instFluxToMagnitude( rec['slot_CalibFlux_flux'], rec.getCentroid()) zptMeanCalMags[i] = fgcmZpt - 2.5 * np.log10( rec['slot_CalibFlux_flux']) # These should be very close but some tiny differences because the fgcm value # is defined at the center of the bbox, and the photoCal is the mean over the box self.assertFloatsAlmostEqual(photoCalMeanCalMags, zptMeanCalMags, rtol=1e-6) # These should be roughly equal, but not precisely because of the focal-plane # variation. However, this is a useful sanity check for something going totally # wrong. self.assertFloatsAlmostEqual(photoCalMeanCalMags, photoCalMags, rtol=1e-2) # Test the transmission output visitCatalog = butler.get('fgcmVisitCatalog') lutCat = butler.get('fgcmLookUpTable') testTrans = butler.get( 'transmission_atmosphere_fgcm', dataId={visitDataRefName: visitCatalog[0]['visit']}) testResp = testTrans.sampleAt(position=afwGeom.Point2D(0, 0), wavelengths=lutCat[0]['atmlambda']) # The fit to be roughly consistent with the standard, although the # airmass is taken into account even with the "frozen" atmosphere. # This is also a rough comparison, because the interpolation does # not work well with such a coarse look-up table used for the test. self.assertFloatsAlmostEqual(testResp, lutCat[0]['atmstdtrans'], atol=0.06) # The second should be close to the first, but there is the airmass # difference so they aren't identical testTrans2 = butler.get( 'transmission_atmosphere_fgcm', dataId={visitDataRefName: visitCatalog[1]['visit']}) testResp2 = testTrans2.sampleAt(position=afwGeom.Point2D(0, 0), wavelengths=lutCat[0]['atmlambda']) self.assertFloatsAlmostEqual(testResp, testResp2, atol=1e-4)