def _photometric_rms(self, sn_cut=300, magnitude_range=3): """ Compute the photometric RMS and the photometric repeatablity values (PA1). Parameters ---------- sn_cut : float The minimum signal/noise for sources to be included in the PA1 calculation. magnitude_range : float The range of magnitudes above sn_cut to include in the PA1 calculation. """ def rms(flux, ref_flux): return np.sqrt( [np.mean((ref_flux[dd] - flux[dd])**2) for dd in flux]) self.old_rms = MatchDict(*map(rms, self.old_flux, self.old_ref_flux)) self.new_rms = MatchDict(*map(rms, self.new_flux, self.new_ref_flux)) # we want to use the absolute fluxes for all of these calculations. self.old_ref = np.fromiter(self.old_ref_flux.absolute.values(), dtype=float) self.new_ref = np.fromiter(self.new_ref_flux.absolute.values(), dtype=float) self.old_mag = np.fromiter((abMagFromFlux(r) for r in self.old_ref), dtype=float) self.new_mag = np.fromiter((abMagFromFlux(r) for r in self.new_ref), dtype=float) def signal_to_noise(sources, flux_key='slot_PsfFlux_flux', sigma_key='slot_PsfFlux_fluxSigma'): """Compute the mean signal/noise per source from a MatchDict of SourceRecords.""" result = np.empty(len(sources)) for i, src in enumerate(sources.values()): result[i] = np.mean([x[flux_key] / x[sigma_key] for x in src]) return result old_sn = signal_to_noise(self.old_source.absolute) # Find the faint/bright magnitude limits that are the "flat" part of the rms/magnitude relation. self.faint = self.old_mag[old_sn > sn_cut].max() self.bright = self.faint - magnitude_range if self.verbose: print("PA1 Magnitude range: {:.3f}, {:.3f}".format( self.bright, self.faint)) old_good = (self.old_mag < self.faint) & (self.old_mag > self.bright) new_good = (self.new_mag < self.faint) & (self.new_mag > self.bright) self.old_weighted_rms = self.old_rms.absolute / self.old_ref self.new_weighted_rms = self.new_rms.absolute / self.new_ref self.old_PA1 = np.median(self.old_weighted_rms[old_good]) self.new_PA1 = np.median(self.new_weighted_rms[new_good])
def _runTask(self): """All the common setup to actually test the results""" task = PhotoCalTask(self.refObjLoader, config=self.config, schema=self.srcCat.schema) pCal = task.run(exposure=self.exposure, sourceCat=self.srcCat) matches = pCal.matches print("Ref flux fields list =", pCal.arrays.refFluxFieldList) refFluxField = pCal.arrays.refFluxFieldList[0] # These are *all* the matches; we don't really expect to do that well. diff = [] for m in matches: refFlux = m[0].get(refFluxField) # reference catalog flux if refFlux <= 0: continue refMag = afwImage.abMagFromFlux(refFlux) # reference catalog mag instFlux = m[1].getPsfFlux() # Instrumental Flux if instFlux <= 0: continue instMag = pCal.calib.getMagnitude(instFlux) # Instrumental mag diff.append(instMag - refMag) self.diff = np.array(diff) # Differences of matched objects that were used in the fit. self.zp = pCal.calib.getMagnitude(1.) self.fitdiff = pCal.arrays.srcMag + self.zp - pCal.arrays.refMag
def plot_point_mags(output_data, visits, outfile, dataId): # get a butler butler = dp.Butler(output_data) # The following value for refcatId is "mandatory, but meaningless", # so we won't try to generalize it. refcatId = {'tract':0, 'patch':'0,0'} ref = butler.get('deepCoadd_ref', dataId=refcatId) # visits to use for lightcurves visit_list = [int(x) for x in visits.split('^')] # get the sources and calib objects for each single epoch visit forced_srcs = {} calibs = {} for visit in visit_list: dataId['visit'] = visit forced_srcs[visit] = butler.get('forced_src', dataId=dataId) calexp = butler.get('calexp', dataId=dataId) calibs[visit] = calexp.getCalib() del calexp # initialize dictionaries to hold lightcurve arrays. Get # extendedness from the coadd catalog. lightcurve_fluxes = {} extendedness = {} for idx, ext in zip(ref.get('id'), ref.get('base_ClassificationExtendedness_value')): lightcurve_fluxes[idx] = [] extendedness[idx] = ext # pivot the source tables to assemble lightcurves for visit, forced_src in forced_srcs.iteritems(): calib = calibs[visit] for idx, flux in zip(forced_src.get('objectId'), forced_src.get('base_PsfFlux_flux')): if extendedness[idx] > 0.5: continue if flux <= 0.: continue lightcurve_fluxes[idx].append(afw_image.fluxFromABMag(calib.getMagnitude(flux))) # compute aggregate quantities for each object and plot for lightcurve in lightcurve_fluxes.values(): if len(lightcurve) == len(visit_list): plt.scatter(afw_image.abMagFromFlux(numpy.median(lightcurve)), numpy.std(lightcurve)/numpy.median(lightcurve), alpha=0.5) mags, invsnrs = make_invsnr_arr() plt.plot(mags, invsnrs, color='red', linewidth=2, alpha=0.75) plt.xlabel("Calibrated magnitude of median flux") plt.ylabel("stdev(flux)/median(flux)") plt.xlim(15.5, 25) plt.ylim(0., 0.5) plt.savefig(outfile)
def testBasics(self): for flux in (1, 210, 3210, 43210, 543210): abMag = afwImage.abMagFromFlux(flux) self.assertAlmostEqual(abMag, refABMagFromFlux(flux)) fluxRoundTrip = afwImage.fluxFromABMag(abMag) self.assertAlmostEqual(flux, fluxRoundTrip) for fluxErrFrac in (0.001, 0.01, 0.1): fluxErr = flux * fluxErrFrac abMagErr = afwImage.abMagErrFromFluxErr(fluxErr, flux) self.assertAlmostEqual(abMagErr, refABMagErrFromFluxErr(fluxErr, flux)) fluxErrRoundTrip = afwImage.fluxErrFromABMagErr(abMagErr, abMag) self.assertAlmostEqual(fluxErr, fluxErrRoundTrip)
def testVector(self): flux = np.array([1.0, 210.0, 3210.0, 43210.0, 543210.0]) flux.flags.writeable = False # Put the 'const' into ndarray::Array<double const, 1, 0> abMag = afwImage.abMagFromFlux(flux) self.assertFloatsAlmostEqual(abMag, refABMagFromFlux(flux)) fluxRoundTrip = afwImage.fluxFromABMag(abMag) self.assertFloatsAlmostEqual(flux, fluxRoundTrip, rtol=1.0e-15) for fluxErrFrac in (0.001, 0.01, 0.1): fluxErr = flux * fluxErrFrac abMagErr = afwImage.abMagErrFromFluxErr(fluxErr, flux) self.assertFloatsAlmostEqual(abMagErr, refABMagErrFromFluxErr(fluxErr, flux)) fluxErrRoundTrip = afwImage.fluxErrFromABMagErr(abMagErr, abMag) self.assertFloatsAlmostEqual(fluxErr, fluxErrRoundTrip, rtol=1.0e-15)
def extractMagArrays(self, matches, filterName, sourceKeys): """!Extract magnitude and magnitude error arrays from the given matches. \param[in] matches Reference/source matches, a \link lsst::afw::table::ReferenceMatchVector\endlink \param[in] filterName Name of filter being calibrated \param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys() \return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays where magErr is an error in the magnitude; the error in srcMag - refMag If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as magErr is what is later used to determine the zero point. Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes (1 or 2 strings) \note These magnitude arrays are the \em inputs to the photometric calibration, some may have been discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813) """ srcFluxArr = np.array([m.second.get(sourceKeys.flux) for m in matches]) srcFluxErrArr = np.array([m.second.get(sourceKeys.fluxErr) for m in matches]) if not np.all(np.isfinite(srcFluxErrArr)): # this is an unpleasant hack; see DM-2308 requesting a better solution self.log.warn("Source catalog does not have flux uncertainties; using sqrt(flux).") srcFluxErrArr = np.sqrt(srcFluxArr) # convert source flux from DN to an estimate of Jy JanskysPerABFlux = 3631.0 srcFluxArr = srcFluxArr * JanskysPerABFlux srcFluxErrArr = srcFluxErrArr * JanskysPerABFlux if not matches: raise RuntimeError("No reference stars are available") refSchema = matches[0].first.schema applyColorTerms = self.config.applyColorTerms applyCTReason = "config.applyColorTerms is %s" % (self.config.applyColorTerms,) if self.config.applyColorTerms is None: # apply color terms if color term data is available and photoCatName specified ctDataAvail = len(self.config.colorterms.data) > 0 photoCatSpecified = self.config.photoCatName is not None applyCTReason += " and data %s available" % ("is" if ctDataAvail else "is not") applyCTReason += " and photoRefCat %s None" % ("is not" if photoCatSpecified else "is") applyColorTerms = ctDataAvail and photoCatSpecified if applyColorTerms: self.log.info("Applying color terms for filterName=%r, config.photoCatName=%s because %s" % (filterName, self.config.photoCatName, applyCTReason)) ct = self.config.colorterms.getColorterm( filterName=filterName, photoCatName=self.config.photoCatName, doRaise=True) else: self.log.info("Not applying color terms because %s" % (applyCTReason,)) ct = None if ct: # we have a color term to worry about fluxFieldList = [getRefFluxField(refSchema, filt) for filt in (ct.primary, ct.secondary)] missingFluxFieldList = [] for fluxField in fluxFieldList: try: refSchema.find(fluxField).key except KeyError: missingFluxFieldList.append(fluxField) if missingFluxFieldList: self.log.warn("Source catalog does not have fluxes for %s; ignoring color terms" % " ".join(missingFluxFieldList)) ct = None if not ct: fluxFieldList = [getRefFluxField(refSchema, filterName)] refFluxArrList = [] # list of ref arrays, one per flux field refFluxErrArrList = [] # list of ref flux arrays, one per flux field for fluxField in fluxFieldList: fluxKey = refSchema.find(fluxField).key refFluxArr = np.array([m.first.get(fluxKey) for m in matches]) try: fluxErrKey = refSchema.find(fluxField + "Sigma").key refFluxErrArr = np.array([m.first.get(fluxErrKey) for m in matches]) except KeyError: # Reference catalogue may not have flux uncertainties; HACK self.log.warn("Reference catalog does not have flux uncertainties for %s; using sqrt(flux)." % fluxField) refFluxErrArr = np.sqrt(refFluxArr) refFluxArrList.append(refFluxArr) refFluxErrArrList.append(refFluxErrArr) if ct: # we have a color term to worry about refMagArr1 = np.array([abMagFromFlux(rf1) for rf1 in refFluxArrList[0]]) # primary refMagArr2 = np.array([abMagFromFlux(rf2) for rf2 in refFluxArrList[1]]) # secondary refMagArr = ct.transformMags(refMagArr1, refMagArr2) refFluxErrArr = ct.propagateFluxErrors(refFluxErrArrList[0], refFluxErrArrList[1]) else: refMagArr = np.array([abMagFromFlux(rf) for rf in refFluxArrList[0]]) srcMagArr = np.array([abMagFromFlux(sf) for sf in srcFluxArr]) # Fitting with error bars in both axes is hard # for now ignore reference flux error, but ticket DM-2308 is a request for a better solution magErrArr = np.array([abMagErrFromFluxErr(fe, sf) for fe, sf in izip(srcFluxErrArr, srcFluxArr)]) if self.config.magErrFloor != 0.0: magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5 srcMagErrArr = np.array([abMagErrFromFluxErr(sfe, sf) for sfe, sf in izip(srcFluxErrArr, srcFluxArr)]) refMagErrArr = np.array([abMagErrFromFluxErr(rfe, rf) for rfe, rf in izip(refFluxErrArr, refFluxArr)]) return pipeBase.Struct( srcMag = srcMagArr, refMag = refMagArr, magErr = magErrArr, srcMagErr = srcMagErrArr, refMagErr = refMagErrArr, refFluxFieldList = fluxFieldList, )
def plot_point_mags(output_data, visit_list, dataId, minMag=17, mid_cut=20, maxMag=26, fit_curves=True): # get a butler butler = dp.Butler(output_data) # The following value for refcatId is "mandatory, but meaningless", # so we won't try to generalize it. refcatId = {'tract': 0, 'patch': '0,0'} ref = butler.get('deepCoadd_ref', dataId=refcatId) # get the sources and calib objects for each single epoch visit forced_srcs = {} calibs = {} for visit in visit_list: dataId['visit'] = visit try: my_forced_srcs = butler.get('forced_src', dataId=dataId) calexp = butler.get('calexp', dataId=dataId) my_calibs = calexp.getCalib() del calexp forced_srcs[visit] = my_forced_srcs calibs[visit] = my_calibs except FitsError as eobj: print(eobj) # initialize dictionaries to hold lightcurve arrays. Get # extendedness from the coadd catalog. lightcurve_fluxes = {} extendedness = {} for idx, ext in zip(ref.get('id'), ref.get('base_ClassificationExtendedness_value')): lightcurve_fluxes[idx] = [] extendedness[idx] = ext # pivot the source tables to assemble lightcurves for visit, forced_src in forced_srcs.items(): calib = calibs[visit] for idx, flux in zip(forced_src.get('objectId'), forced_src.get('base_PsfFlux_flux')): if extendedness[idx] > 0.5: continue if flux <= 0.: continue lightcurve_fluxes[idx].append( afw_image.fluxFromABMag(calib.getMagnitude(flux))) # compute aggregate quantities for each object and plot band = dataId['filter'] med_mags = [] med_err = [] for lightcurve in lightcurve_fluxes.values(): if len(lightcurve) == len(visit_list): median_flux = np.median(lightcurve) med_mags.append(afw_image.abMagFromFlux(median_flux)) med_err.append(np.std(lightcurve) / median_flux) print("number of objects: ", len(med_mags)) mag_stats = MagStats(band, med_mags, med_err, fit_curves=fit_curves) if mag_stats.pcov is not None: label ='filter=%s, Floor=%.1f%%, m_5=%0.2f' \ % (band, mag_stats.sys_floor*100, mag_stats.popt[1]) else: label ='filter=%s, Floor=%.1f%%' \ % (band, mag_stats.sys_floor*100) scatter = plt.scatter(med_mags, med_err, alpha=0.3, color=_filter_color[band], marker=_filter_symbol[band], label=label) plt.xlabel("Calibrated magnitude of median flux") plt.ylabel("stdev(flux)/median(flux)") plt.xlim(15.5, 25) plt.ylim(0., 0.5) return scatter, mag_stats
def test1(self): res = self.getAstrometrySolution() matches = res.matches print 'Test1' logLevel = Log.DEBUG log = Log(Log.getDefaultLog(), 'testPhotoCal', logLevel) schema = matches[0].second.schema config = PhotoCalConfig() # The test and associated data have been prepared on the basis that we # use the PsfFlux to perform photometry. config.fluxField = "base_PsfFlux_flux" config.doWriteOutput = False # schema is fixed because we already loaded the data task = PhotoCalTask(config=config, schema=schema) pCal = task.run(exposure=self.exposure, matches=matches) print "Ref flux fields list =", pCal.arrays.refFluxFieldList refFluxField = pCal.arrays.refFluxFieldList[0] # These are *all* the matches; we don't really expect to do that well. diff=[] for m in matches: refFlux = m[0].get(refFluxField) # reference catalog flux if refFlux <= 0: continue refMag = afwImage.abMagFromFlux(refFlux) # reference catalog mag instFlux = m[1].getPsfFlux() #Instrumental Flux if instFlux <= 0: continue instMag = pCal.calib.getMagnitude(instFlux) #Instrumental mag diff.append(instMag - refMag) diff = np.array(diff) self.assertGreater(len(diff), 50) log.info('%i magnitude differences; mean difference %g; mean abs diff %g' % (len(diff), np.mean(diff), np.mean(np.abs(diff)))) self.assertLess(np.mean(diff), 0.6) # Differences of matched objects that were used in the fit. zp = pCal.calib.getMagnitude(1.) log.logdebug('zeropoint: %g' % zp) fitdiff = pCal.arrays.srcMag + zp - pCal.arrays.refMag log.logdebug('number of sources used in fit: %i' % len(fitdiff)) log.logdebug('rms diff: %g' % np.mean(fitdiff**2)**0.5) log.logdebug('median abs(diff): %g' % np.median(np.abs(fitdiff))) # zeropoint: 31.3145 # number of sources used in fit: 65 # median diff: -0.009681 # mean diff: 0.00331871 # median abs(diff): 0.0368904 # mean abs(diff): 0.0516589 self.assertLess(abs(zp - 31.3145), 0.05) self.assertGreater(len(fitdiff), 50) # Tolerances are somewhat arbitrary; they're set simply to avoid regressions, and # are not based on we'd expect to get given the data quality. self.assertLess(np.mean(fitdiff**2)**0.5, 0.07) # rms difference self.assertLess(np.median(np.abs(fitdiff)), 0.06) # median absolution difference
def test1(self): res = self.getAstrometrySolution() matches = res.matches print 'Test1' logLevel = Log.DEBUG log = Log(Log.getDefaultLog(), 'testPhotoCal', logLevel) schema = matches[0].second.schema config = PhotoCalConfig() # The test and associated data have been prepared on the basis that we # use the PsfFlux to perform photometry. config.fluxField = "base_PsfFlux_flux" config.doWriteOutput = False # schema is fixed because we already loaded the data task = PhotoCalTask(config=config, schema=schema) pCal = task.run(exposure=self.exposure, matches=matches) print "Ref flux fields list =", pCal.arrays.refFluxFieldList refFluxField = pCal.arrays.refFluxFieldList[0] # These are *all* the matches; we don't really expect to do that well. diff = [] for m in matches: refFlux = m[0].get(refFluxField) # reference catalog flux if refFlux <= 0: continue refMag = afwImage.abMagFromFlux(refFlux) # reference catalog mag instFlux = m[1].getPsfFlux() #Instrumental Flux if instFlux <= 0: continue instMag = pCal.calib.getMagnitude(instFlux) #Instrumental mag diff.append(instMag - refMag) diff = np.array(diff) self.assertGreater(len(diff), 50) log.info( '%i magnitude differences; mean difference %g; mean abs diff %g' % (len(diff), np.mean(diff), np.mean(np.abs(diff)))) self.assertLess(np.mean(diff), 0.6) # Differences of matched objects that were used in the fit. zp = pCal.calib.getMagnitude(1.) log.logdebug('zeropoint: %g' % zp) fitdiff = pCal.arrays.srcMag + zp - pCal.arrays.refMag log.logdebug('number of sources used in fit: %i' % len(fitdiff)) log.logdebug('rms diff: %g' % np.mean(fitdiff**2)**0.5) log.logdebug('median abs(diff): %g' % np.median(np.abs(fitdiff))) # zeropoint: 31.3145 # number of sources used in fit: 65 # median diff: -0.009681 # mean diff: 0.00331871 # median abs(diff): 0.0368904 # mean abs(diff): 0.0516589 self.assertLess(abs(zp - 31.3145), 0.05) self.assertGreater(len(fitdiff), 50) # Tolerances are somewhat arbitrary; they're set simply to avoid regressions, and # are not based on we'd expect to get given the data quality. self.assertLess(np.mean(fitdiff**2)**0.5, 0.07) # rms difference self.assertLess(np.median(np.abs(fitdiff)), 0.06) # median absolution difference
areNans = np.logical_or.reduce(areNans) hdu[1].data = hdu[1].data[~areNans] #Limit to brighter than 19th mag in g-band bright = hdu[1].data['g'] > 9.12e-5 hdu[1].data = hdu[1].data[bright] hdu[1].data['g_err'] = abMagErrFromFluxErr(hdu[1].data['g_err'].astype(np.float), hdu[1].data['g'].astype(np.float)) hdu[1].data['r_err'] = abMagErrFromFluxErr(hdu[1].data['r_err'].astype(np.float), hdu[1].data['r'].astype(np.float)) hdu[1].data['i_err'] = abMagErrFromFluxErr(hdu[1].data['i_err'].astype(np.float), hdu[1].data['i'].astype(np.float)) hdu[1].data['z_err'] = abMagErrFromFluxErr(hdu[1].data['z_err'].astype(np.float), hdu[1].data['z'].astype(np.float)) hdu[1].data['y_err'] = abMagErrFromFluxErr(hdu[1].data['y_err'].astype(np.float), hdu[1].data['y'].astype(np.float)) hdu[1].data['g'] = abMagFromFlux(hdu[1].data['g'].astype(np.float)) hdu[1].data['r'] = abMagFromFlux(hdu[1].data['r'].astype(np.float)) hdu[1].data['i'] = abMagFromFlux(hdu[1].data['i'].astype(np.float)) hdu[1].data['z'] = abMagFromFlux(hdu[1].data['z'].astype(np.float)) hdu[1].data['y'] = abMagFromFlux(hdu[1].data['y'].astype(np.float)) hdu[1].data['coord_ra'] = (180./np.pi)*hdu[1].data['coord_ra'] hdu[1].data['coord_dec'] = (180./np.pi)*hdu[1].data['coord_dec'] base = os.path.basename(file) newFile = 'ps1_pv3_3pi_20170110_filteredAB/'+base hdu.writeto(newFile, overwrite=True)
# initialize dictionaries to hold lightcurve arrays. Get extendedness from the coadd catalog. lightcurve_fluxes = {} extendedness = {} for idx, ext in zip(ref.get('id'), ref.get('base_ClassificationExtendedness_value')): lightcurve_fluxes[idx] = [] extendedness[idx] = ext # pivot the source tables to assemble lightcurves for visit, forced_src in forced_srcs.iteritems(): calib = calibs[visit] for idx, flux in zip(forced_src.get('objectId'), forced_src.get('base_PsfFlux_flux')): if extendedness[idx] > 0.5: continue if flux <= 0.: continue lightcurve_fluxes[idx].append(afw_image.fluxFromABMag(calib.getMagnitude(flux))) # compute aggregate quantities for each object and plot for lightcurve in lightcurve_fluxes.values(): if len(lightcurve) == 9: plt.scatter(afw_image.abMagFromFlux(numpy.median(lightcurve)), numpy.std(lightcurve)/numpy.median(lightcurve), alpha=0.5) mags, invsnrs = make_invsnr_arr() plt.plot(mags, invsnrs, color='red', linewidth=2, alpha=0.75) plt.xlabel("Calibrated magnitude of median flux") plt.ylabel("stdev(flux)/median(flux)") plt.xlim(15.5, 25) plt.ylim(0., 0.5) plt.show()
# pivot the source tables to assemble lightcurves for visit, forced_src in forced_srcs.iteritems(): calib = calibs[visit] for idx, flux in zip(forced_src.get('objectId'), forced_src.get('base_PsfFlux_flux')): if extendedness[idx] > 0.5: continue if flux <= 0.: continue lightcurve_fluxes[idx].append(afw_image.fluxFromABMag(calib.getMagnitude(flux))) # compute aggregate quantities for each object and plot med_mags = [] med_err = [] for lightcurve in lightcurve_fluxes.values(): if len(lightcurve) == 10: med_mags.append(afw_image.abMagFromFlux(numpy.median(lightcurve))) med_err.append(numpy.std(lightcurve)/numpy.median(lightcurve)) fit_func = partial(fit_invsnr, bandpass_name=bandpass_name) minMag=17. mid_cut=20. maxMag=26. med_mags = numpy.array(med_mags) med_err = numpy.array(med_err) idxs = numpy.where((med_mags<mid_cut)&(med_mags>minMag)) sys_floor = numpy.median(med_err[idxs]) idxs = numpy.where((med_mags<maxMag)&(med_mags>mid_cut)) popt, pcov = curve_fit(fit_invsnr, med_mags[idxs], med_err[idxs], p0=[0.01, 24.5]) scatter = plt.scatter(med_mags, med_err, alpha=0.3, color=bandpass_color_map[bandpass_name], marker=bandpass_symbol_map[bandpass_name], label="filter=%s, Floor=%.1f%%, m_5=%0.2f"%(bandpass_name,sys_floor*100.,popt[1])) scatters.append(scatter)
def extractMagArrays(self, matches, filterName, sourceKeys): """!Extract magnitude and magnitude error arrays from the given matches. @param[in] matches Reference/source matches, a @link lsst::afw::table::ReferenceMatchVector@endlink @param[in] filterName Name of filter being calibrated @param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys() @return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays where magErr is an error in the magnitude; the error in srcMag - refMag If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as magErr is what is later used to determine the zero point. Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes (1 or 2 strings) @note These magnitude arrays are the @em inputs to the photometric calibration, some may have been discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813) """ srcFluxArr = np.array([m.second.get(sourceKeys.flux) for m in matches]) srcFluxErrArr = np.array( [m.second.get(sourceKeys.fluxErr) for m in matches]) if not np.all(np.isfinite(srcFluxErrArr)): # this is an unpleasant hack; see DM-2308 requesting a better solution self.log.warn( "Source catalog does not have flux uncertainties; using sqrt(flux)." ) srcFluxErrArr = np.sqrt(srcFluxArr) # convert source flux from DN to an estimate of Jy JanskysPerABFlux = 3631.0 srcFluxArr = srcFluxArr * JanskysPerABFlux srcFluxErrArr = srcFluxErrArr * JanskysPerABFlux if not matches: raise RuntimeError("No reference stars are available") refSchema = matches[0].first.schema applyColorTerms = self.config.applyColorTerms applyCTReason = "config.applyColorTerms is %s" % ( self.config.applyColorTerms, ) if self.config.applyColorTerms is None: # apply color terms if color term data is available and photoCatName specified ctDataAvail = len(self.config.colorterms.data) > 0 photoCatSpecified = self.config.photoCatName is not None applyCTReason += " and data %s available" % ("is" if ctDataAvail else "is not") applyCTReason += " and photoRefCat %s provided" % ( "is" if photoCatSpecified else "is not") applyColorTerms = ctDataAvail and photoCatSpecified if applyColorTerms: self.log.info( "Applying color terms for filterName=%r, config.photoCatName=%s because %s", filterName, self.config.photoCatName, applyCTReason) ct = self.config.colorterms.getColorterm( filterName=filterName, photoCatName=self.config.photoCatName, doRaise=True) else: self.log.info("Not applying color terms because %s", applyCTReason) ct = None if ct: # we have a color term to worry about fluxFieldList = [ getRefFluxField(refSchema, filt) for filt in (ct.primary, ct.secondary) ] missingFluxFieldList = [] for fluxField in fluxFieldList: try: refSchema.find(fluxField).key except KeyError: missingFluxFieldList.append(fluxField) if missingFluxFieldList: self.log.warn( "Source catalog does not have fluxes for %s; ignoring color terms", " ".join(missingFluxFieldList)) ct = None if not ct: fluxFieldList = [getRefFluxField(refSchema, filterName)] refFluxArrList = [] # list of ref arrays, one per flux field refFluxErrArrList = [] # list of ref flux arrays, one per flux field for fluxField in fluxFieldList: fluxKey = refSchema.find(fluxField).key refFluxArr = np.array([m.first.get(fluxKey) for m in matches]) try: fluxErrKey = refSchema.find(fluxField + "Sigma").key refFluxErrArr = np.array( [m.first.get(fluxErrKey) for m in matches]) except KeyError: # Reference catalogue may not have flux uncertainties; HACK self.log.warn( "Reference catalog does not have flux uncertainties for %s; using sqrt(flux).", fluxField) refFluxErrArr = np.sqrt(refFluxArr) refFluxArrList.append(refFluxArr) refFluxErrArrList.append(refFluxErrArr) if ct: # we have a color term to worry about refMagArr1 = np.array( [abMagFromFlux(rf1) for rf1 in refFluxArrList[0]]) # primary refMagArr2 = np.array( [abMagFromFlux(rf2) for rf2 in refFluxArrList[1]]) # secondary refMagArr = ct.transformMags(refMagArr1, refMagArr2) refFluxErrArr = ct.propagateFluxErrors(refFluxErrArrList[0], refFluxErrArrList[1]) else: refMagArr = np.array( [abMagFromFlux(rf) for rf in refFluxArrList[0]]) srcMagArr = np.array([abMagFromFlux(sf) for sf in srcFluxArr]) # Fitting with error bars in both axes is hard # for now ignore reference flux error, but ticket DM-2308 is a request for a better solution magErrArr = np.array([ abMagErrFromFluxErr(fe, sf) for fe, sf in zip(srcFluxErrArr, srcFluxArr) ]) if self.config.magErrFloor != 0.0: magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5 srcMagErrArr = np.array([ abMagErrFromFluxErr(sfe, sf) for sfe, sf in zip(srcFluxErrArr, srcFluxArr) ]) refMagErrArr = np.array([ abMagErrFromFluxErr(rfe, rf) for rfe, rf in zip(refFluxErrArr, refFluxArr) ]) good = np.isfinite(srcMagArr) & np.isfinite(refMagArr) return pipeBase.Struct( srcMag=srcMagArr[good], refMag=refMagArr[good], magErr=magErrArr[good], srcMagErr=srcMagErrArr[good], refMagErr=refMagErrArr[good], refFluxFieldList=fluxFieldList, )