def testPhotom(self): """Test the zero-point information""" flux, fluxErr = 1000.0, 10.0 flux0, flux0Err = 1e12, 1e10 self.calib.setFluxMag0(flux0) self.assertEqual(flux0, self.calib.getFluxMag0()[0]) self.assertEqual(0.0, self.calib.getFluxMag0()[1]) self.assertEqual(22.5, self.calib.getMagnitude(flux)) # Error just in flux self.assertAlmostEqual(self.calib.getMagnitude(flux, fluxErr)[1], 2.5/math.log(10)*fluxErr/flux) # Error just in flux0 self.calib.setFluxMag0(flux0, flux0Err) self.assertEqual(flux0Err, self.calib.getFluxMag0()[1]) self.assertAlmostEqual(self.calib.getMagnitude(flux, 0)[1], 2.5/math.log(10)*flux0Err/flux0) self.assertAlmostEqual(flux0, self.calib.getFlux(0)) self.assertAlmostEqual(flux, self.calib.getFlux(22.5)) # I don't know how to test round-trip if fluxMag0 is significant compared to fluxErr self.calib.setFluxMag0(flux0, flux0 / 1e6) for fluxErr in (flux / 1e2, flux / 1e4): mag, magErr = self.calib.getMagnitude(flux, fluxErr) self.assertAlmostEqual(flux, self.calib.getFlux(mag, magErr)[0]) self.assertTrue(abs(fluxErr - self.calib.getFlux(mag, magErr)[1]) < 1.0e-4) # Test context manager; shouldn't raise an exception within the block, should outside with imageUtils.CalibNoThrow(): self.assert_(numpy.isnan(self.calib.getMagnitude(-50.0))) self.assertRaises(pexExcept.DomainError, self.calib.getMagnitude, -50.0)
def extract_photometry(butler, dataId, forced_dataset, filt, object_id, names_to_copy, phot_type='base_PsfFlux'): # Can grab filter, mjd from 'calexp_md' call on visit md = butler.get('calexp_md', dataId=dataId, immediate=True) mjd = md.get('MJD-OBS') # filt = md.get('FILTER') # But that's not being set right now so we'll keep using f this_measurement = butler.get(forced_dataset, dataId).asAstropy() source_row, = np.where(this_measurement['objectId'] == object_id) if len(source_row) != 1: return None # 'this_measurement' is a table, but we're only extracting the first entry from each column new_row = {n: this_measurement[n][source_row] for n in names_to_copy} # new_row['filter'] = dataId['filter'] new_row['filter'] = filt new_row['mjd'] = mjd # Calibrate to magnitudes # The calibration information for the calexp # should still apply to the difference image calib = afwImage.Calib(md) with afwImageUtils.CalibNoThrow(): new_row['%s_mag' % phot_type], new_row['%s_magSigma' % phot_type] = \ calib.getMagnitude(new_row['%s_flux' % phot_type], new_row['%s_fluxSigma' % phot_type]) flux_mag_0, flux_magSigma_0 = calib.getFluxMag0() flux_mag_25 = 10**(-0.4 * 25) * flux_mag_0 flux_norm = 1 / flux_mag_25 new_row['%s_flux_zp25' % phot_type] = \ flux_norm * new_row['%s_flux' % phot_type] new_row['%s_fluxSigma_zp25' % phot_type] = \ flux_norm * new_row['%s_fluxSigma' % phot_type] return new_row
def _loadAndMatchCatalogs(self, repo, dataIds, matchRadius): """Load data from specific visit. Match with reference. Parameters ---------- repo : string The repository. This is generally the directory on disk that contains the repository and mapper. dataIds : list of dict List of `butler` data IDs of Image catalogs to compare to reference. The `calexp` cpixel image is needed for the photometric calibration. matchRadius : afwGeom.Angle(), optional Radius for matching. Default is 1 arcsecond. Returns ------- afw.table.GroupView An object of matched catalog. """ # Following # https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb butler = dafPersist.Butler(repo) dataset = 'src' # 2016-02-08 MWV: # I feel like I could be doing something more efficient with # something along the lines of the following: # dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds] ccdKeyName = getCcdKeyName(dataIds[0]) schema = butler.get(dataset + "_schema", immediate=True).schema mapper = SchemaMapper(schema) mapper.addMinimalSchema(schema) mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR')) mapper.addOutputField(Field[float]('base_PsfFlux_mag', 'PSF magnitude')) mapper.addOutputField(Field[float]('base_PsfFlux_magerr', 'PSF magnitude uncertainty')) newSchema = mapper.getOutputSchema() # Create an object that matches multiple catalogs with same schema mmatch = MultiMatch(newSchema, dataIdFormat={ 'visit': np.int32, ccdKeyName: np.int32 }, radius=matchRadius, RecordClass=SimpleRecord) # create the new extented source catalog srcVis = SourceCatalog(newSchema) for vId in dataIds: try: calexpMetadata = butler.get("calexp_md", vId, immediate=True) except (FitsError, dafPersist.NoResults) as e: print(e) print("Could not open calibrated image file for ", vId) print("Skipping %s " % repr(vId)) continue except TypeError as te: # DECam images that haven't been properly reformatted # can trigger a TypeError because of a residual FITS header # LTV2 which is a float instead of the expected integer. # This generates an error of the form: # # lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type' # # See, e.g., DM-2957 for details. print(te) print("Calibration image header information malformed.") print("Skipping %s " % repr(vId)) continue calib = afwImage.Calib(calexpMetadata) oldSrc = butler.get('src', vId, immediate=True) print( len(oldSrc), "sources in ccd %s visit %s" % (vId[ccdKeyName], vId["visit"])) # create temporary catalog tmpCat = SourceCatalog(SourceCatalog(newSchema).table) tmpCat.extend(oldSrc, mapper=mapper) tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_flux'] \ / tmpCat['base_PsfFlux_fluxSigma'] with afwImageUtils.CalibNoThrow(): _ = calib.getMagnitude(tmpCat['base_PsfFlux_flux'], tmpCat['base_PsfFlux_fluxSigma']) tmpCat['base_PsfFlux_mag'][:] = _[0] tmpCat['base_PsfFlux_magerr'][:] = _[1] srcVis.extend(tmpCat, False) mmatch.add(catalog=tmpCat, dataId=vId) # Complete the match, returning a catalog that includes # all matched sources with object IDs that can be used to group them. matchCat = mmatch.finish() # Create a mapping object that allows the matches to be manipulated # as a mapping of object ID to catalog of sources. allMatches = GroupView.build(matchCat) return allMatches
def _loadAndMatchCatalogs(self, repo, dataIds, matchRadius, useJointCal=False): """Load data from specific visit. Match with reference. Parameters ---------- repo : string or Butler A Butler or a repository URL that can be used to construct one dataIds : list of dict List of `butler` data IDs of Image catalogs to compare to reference. The `calexp` cpixel image is needed for the photometric calibration. matchRadius : afwGeom.Angle(), optional Radius for matching. Default is 1 arcsecond. Returns ------- afw.table.GroupView An object of matched catalog. """ # Following # https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb if isinstance(repo, dafPersist.Butler): butler = repo else: butler = dafPersist.Butler(repo) dataset = 'src' # 2016-02-08 MWV: # I feel like I could be doing something more efficient with # something along the lines of the following: # dataRefs = [dafPersist.ButlerDataRef(butler, vId) for vId in dataIds] ccdKeyName = getCcdKeyName(dataIds[0]) schema = butler.get(dataset + "_schema").schema mapper = SchemaMapper(schema) mapper.addMinimalSchema(schema) mapper.addOutputField(Field[float]('base_PsfFlux_snr', 'PSF flux SNR')) mapper.addOutputField(Field[float]('base_PsfFlux_mag', 'PSF magnitude')) mapper.addOutputField(Field[float]('base_PsfFlux_magErr', 'PSF magnitude uncertainty')) newSchema = mapper.getOutputSchema() newSchema.setAliasMap(schema.getAliasMap()) # Create an object that matches multiple catalogs with same schema mmatch = MultiMatch(newSchema, dataIdFormat={ 'visit': np.int32, ccdKeyName: np.int32 }, radius=matchRadius, RecordClass=SimpleRecord) # create the new extented source catalog srcVis = SourceCatalog(newSchema) for vId in dataIds: if useJointCal: try: photoCalib = butler.get("photoCalib", vId) except (FitsError, dafPersist.NoResults) as e: print(e) print("Could not open photometric calibration for ", vId) print("Skipping %s " % repr(vId)) continue try: md = butler.get("wcs_md", vId) wcs = afwImage.makeWcs(md) except (FitsError, dafPersist.NoResults) as e: print(e) print("Could not open updated WCS for ", vId) print("Skipping %s " % repr(vId)) continue else: try: calexpMetadata = butler.get("calexp_md", vId) except (FitsError, dafPersist.NoResults) as e: print(e) print("Could not open calibrated image file for ", vId) print("Skipping %s " % repr(vId)) continue except TypeError as te: # DECam images that haven't been properly reformatted # can trigger a TypeError because of a residual FITS header # LTV2 which is a float instead of the expected integer. # This generates an error of the form: # # lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type' # # See, e.g., DM-2957 for details. print(te) print("Calibration image header information malformed.") print("Skipping %s " % repr(vId)) continue calib = afwImage.Calib(calexpMetadata) # We don't want to put this above the first "if useJointCal block" # because we need to use the first `butler.get` above to quickly # catch data IDs with no usable outputs. try: # HSC supports these flags, which dramatically improve I/O # performance; support for other cameras is DM-6927. oldSrc = butler.get('src', vId, flags=SOURCE_IO_NO_FOOTPRINTS) except: oldSrc = butler.get('src', vId) print( len(oldSrc), "sources in ccd %s visit %s" % (vId[ccdKeyName], vId["visit"])) # create temporary catalog tmpCat = SourceCatalog(SourceCatalog(newSchema).table) tmpCat.extend(oldSrc, mapper=mapper) tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_flux'] \ / tmpCat['base_PsfFlux_fluxSigma'] if useJointCal: for record in tmpCat: record.updateCoord(wcs) photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux") else: with afwImageUtils.CalibNoThrow(): _ = calib.getMagnitude(tmpCat['base_PsfFlux_flux'], tmpCat['base_PsfFlux_fluxSigma']) tmpCat['base_PsfFlux_mag'][:] = _[0] tmpCat['base_PsfFlux_magErr'][:] = _[1] srcVis.extend(tmpCat, False) mmatch.add(catalog=tmpCat, dataId=vId) # Complete the match, returning a catalog that includes # all matched sources with object IDs that can be used to group them. matchCat = mmatch.finish() # Create a mapping object that allows the matches to be manipulated # as a mapping of object ID to catalog of sources. allMatches = GroupView.build(matchCat) return allMatches
def assemble_catalogs_into_lightcurve(dataIds_by_filter, repo_dir, source_row=0, dataset='calexp', debug=False): """Return Table with measurements.""" butler = dafPersist.Butler(repo_dir) names_to_copy = [ 'objectId', 'coord_ra', 'coord_dec', 'parentObjectId', 'base_RaDecCentroid_x', 'base_RaDecCentroid_y', 'base_PsfFlux_flux', 'base_PsfFlux_fluxSigma' ] # flux_zp25 is flux normalized to a zeropoint of 25. # This convention is useful and appropriate for transient sources # that are expected to be negative as well as positive # for a given lightcurve. names_to_generate = [ 'filter', 'mjd', 'base_PsfFlux_mag', 'base_PsfFlux_magSigma', 'base_PsfFlux_flux_zp25', 'base_PsfFlux_fluxSigma_zp25' ] names = names_to_generate + names_to_copy dtype = (str, float, float, float, float, float, int, float, float, int, float, float, float, float) table = Table(names=names, dtype=dtype) if dataset == 'deepDiff_differenceExp': prefix = 'deepDiff_' else: prefix = '' forced_dataset = prefix + 'forcedRaDec_src' if debug: print("FORCED_DATASET: ", forced_dataset) for f, dataIds in dataIds_by_filter.items(): for dataId in dataIds: # Can grab filter, mjd from 'calexp_md' call on visit md = butler.get('calexp_md', dataId=dataId, immediate=True) mjd = md.get('MJD-OBS') # filt = md.get('FILTER') # But that's not being set right now so we'll keep using f this_measurement = butler.get(forced_dataset, dataId) # 'this_measurement' is a table, but we're only extracting the first entry from each column cols_for_new_row = { n: this_measurement[n][source_row] for n in names_to_copy } # cols_for_new_row['filter'] = dataId['filter'] cols_for_new_row['filter'] = f cols_for_new_row['mjd'] = mjd # Calibrate to magnitudes # The calibration information for the calexp # should still apply to the difference image calib = afwImage.Calib(md) with afwImageUtils.CalibNoThrow(): cols_for_new_row['base_PsfFlux_mag'], cols_for_new_row['base_PsfFlux_magSigma'] = \ calib.getMagnitude(cols_for_new_row['base_PsfFlux_flux'], cols_for_new_row['base_PsfFlux_fluxSigma']) flux_mag_0, flux_magSigma_0 = calib.getFluxMag0() flux_mag_25 = 10**(-0.4 * 25) * flux_mag_0 flux_norm = 1 / flux_mag_25 cols_for_new_row['base_PsfFlux_flux_zp25'] = \ flux_norm * cols_for_new_row['base_PsfFlux_flux'] cols_for_new_row['base_PsfFlux_fluxSigma_zp25'] = \ flux_norm * cols_for_new_row['base_PsfFlux_fluxSigma'] table.add_row(cols_for_new_row) return table