def _photometric_rms(self, sn_cut=300, magnitude_range=3):
"""
Compute the photometric RMS and the photometric repeatablity values (PA1).
Parameters
----------
sn_cut : float
The minimum signal/noise for sources to be included in the PA1 calculation.
magnitude_range : float
The range of magnitudes above sn_cut to include in the PA1 calculation.
"""
def rms(flux, ref_flux):
return np.sqrt(
[np.mean((ref_flux[dd] - flux[dd])**2) for dd in flux])
self.old_rms = MatchDict(*map(rms, self.old_flux, self.old_ref_flux))
self.new_rms = MatchDict(*map(rms, self.new_flux, self.new_ref_flux))
# we want to use the absolute fluxes for all of these calculations.
self.old_ref = np.fromiter(self.old_ref_flux.absolute.values(),
dtype=float)
self.new_ref = np.fromiter(self.new_ref_flux.absolute.values(),
dtype=float)
self.old_mag = np.fromiter((abMagFromFlux(r) for r in self.old_ref),
dtype=float)
self.new_mag = np.fromiter((abMagFromFlux(r) for r in self.new_ref),
dtype=float)
def signal_to_noise(sources,
flux_key='slot_PsfFlux_flux',
sigma_key='slot_PsfFlux_fluxSigma'):
"""Compute the mean signal/noise per source from a MatchDict of SourceRecords."""
result = np.empty(len(sources))
for i, src in enumerate(sources.values()):
result[i] = np.mean([x[flux_key] / x[sigma_key] for x in src])
return result
old_sn = signal_to_noise(self.old_source.absolute)
# Find the faint/bright magnitude limits that are the "flat" part of the rms/magnitude relation.
self.faint = self.old_mag[old_sn > sn_cut].max()
self.bright = self.faint - magnitude_range
if self.verbose:
print("PA1 Magnitude range: {:.3f}, {:.3f}".format(
self.bright, self.faint))
old_good = (self.old_mag < self.faint) & (self.old_mag > self.bright)
new_good = (self.new_mag < self.faint) & (self.new_mag > self.bright)
self.old_weighted_rms = self.old_rms.absolute / self.old_ref
self.new_weighted_rms = self.new_rms.absolute / self.new_ref
self.old_PA1 = np.median(self.old_weighted_rms[old_good])
self.new_PA1 = np.median(self.new_weighted_rms[new_good])
def _runTask(self):
"""All the common setup to actually test the results"""
task = PhotoCalTask(self.refObjLoader,
config=self.config,
schema=self.srcCat.schema)
pCal = task.run(exposure=self.exposure, sourceCat=self.srcCat)
matches = pCal.matches
print("Ref flux fields list =", pCal.arrays.refFluxFieldList)
refFluxField = pCal.arrays.refFluxFieldList[0]
# These are *all* the matches; we don't really expect to do that well.
diff = []
for m in matches:
refFlux = m[0].get(refFluxField) # reference catalog flux
if refFlux <= 0:
continue
refMag = afwImage.abMagFromFlux(refFlux) # reference catalog mag
instFlux = m[1].getPsfFlux() # Instrumental Flux
if instFlux <= 0:
continue
instMag = pCal.calib.getMagnitude(instFlux) # Instrumental mag
diff.append(instMag - refMag)
self.diff = np.array(diff)
# Differences of matched objects that were used in the fit.
self.zp = pCal.calib.getMagnitude(1.)
self.fitdiff = pCal.arrays.srcMag + self.zp - pCal.arrays.refMag
def plot_point_mags(output_data, visits, outfile, dataId):
# get a butler
butler = dp.Butler(output_data)
# The following value for refcatId is "mandatory, but meaningless",
# so we won't try to generalize it.
refcatId = {'tract':0, 'patch':'0,0'}
ref = butler.get('deepCoadd_ref', dataId=refcatId)
# visits to use for lightcurves
visit_list = [int(x) for x in visits.split('^')]
# get the sources and calib objects for each single epoch visit
forced_srcs = {}
calibs = {}
for visit in visit_list:
dataId['visit'] = visit
forced_srcs[visit] = butler.get('forced_src', dataId=dataId)
calexp = butler.get('calexp', dataId=dataId)
calibs[visit] = calexp.getCalib()
del calexp
# initialize dictionaries to hold lightcurve arrays. Get
# extendedness from the coadd catalog.
lightcurve_fluxes = {}
extendedness = {}
for idx, ext in zip(ref.get('id'),
ref.get('base_ClassificationExtendedness_value')):
lightcurve_fluxes[idx] = []
extendedness[idx] = ext
# pivot the source tables to assemble lightcurves
for visit, forced_src in forced_srcs.iteritems():
calib = calibs[visit]
for idx, flux in zip(forced_src.get('objectId'),
forced_src.get('base_PsfFlux_flux')):
if extendedness[idx] > 0.5:
continue
if flux <= 0.:
continue
lightcurve_fluxes[idx].append(afw_image.fluxFromABMag(calib.getMagnitude(flux)))
# compute aggregate quantities for each object and plot
for lightcurve in lightcurve_fluxes.values():
if len(lightcurve) == len(visit_list):
plt.scatter(afw_image.abMagFromFlux(numpy.median(lightcurve)),
numpy.std(lightcurve)/numpy.median(lightcurve),
alpha=0.5)
mags, invsnrs = make_invsnr_arr()
plt.plot(mags, invsnrs, color='red', linewidth=2, alpha=0.75)
plt.xlabel("Calibrated magnitude of median flux")
plt.ylabel("stdev(flux)/median(flux)")
plt.xlim(15.5, 25)
plt.ylim(0., 0.5)
plt.savefig(outfile)
def testBasics(self):
for flux in (1, 210, 3210, 43210, 543210):
abMag = afwImage.abMagFromFlux(flux)
self.assertAlmostEqual(abMag, refABMagFromFlux(flux))
fluxRoundTrip = afwImage.fluxFromABMag(abMag)
self.assertAlmostEqual(flux, fluxRoundTrip)
for fluxErrFrac in (0.001, 0.01, 0.1):
fluxErr = flux * fluxErrFrac
abMagErr = afwImage.abMagErrFromFluxErr(fluxErr, flux)
self.assertAlmostEqual(abMagErr, refABMagErrFromFluxErr(fluxErr, flux))
fluxErrRoundTrip = afwImage.fluxErrFromABMagErr(abMagErr, abMag)
self.assertAlmostEqual(fluxErr, fluxErrRoundTrip)
def testVector(self):
flux = np.array([1.0, 210.0, 3210.0, 43210.0, 543210.0])
flux.flags.writeable = False # Put the 'const' into ndarray::Array<double const, 1, 0>
abMag = afwImage.abMagFromFlux(flux)
self.assertFloatsAlmostEqual(abMag, refABMagFromFlux(flux))
fluxRoundTrip = afwImage.fluxFromABMag(abMag)
self.assertFloatsAlmostEqual(flux, fluxRoundTrip, rtol=1.0e-15)
for fluxErrFrac in (0.001, 0.01, 0.1):
fluxErr = flux * fluxErrFrac
abMagErr = afwImage.abMagErrFromFluxErr(fluxErr, flux)
self.assertFloatsAlmostEqual(abMagErr, refABMagErrFromFluxErr(fluxErr, flux))
fluxErrRoundTrip = afwImage.fluxErrFromABMagErr(abMagErr, abMag)
self.assertFloatsAlmostEqual(fluxErr, fluxErrRoundTrip, rtol=1.0e-15)
def testVector(self):
flux = np.array([1.0, 210.0, 3210.0, 43210.0, 543210.0])
flux.flags.writeable = False # Put the 'const' into ndarray::Array<double const, 1, 0>
abMag = afwImage.abMagFromFlux(flux)
self.assertFloatsAlmostEqual(abMag, refABMagFromFlux(flux))
fluxRoundTrip = afwImage.fluxFromABMag(abMag)
self.assertFloatsAlmostEqual(flux, fluxRoundTrip, rtol=1.0e-15)
for fluxErrFrac in (0.001, 0.01, 0.1):
fluxErr = flux * fluxErrFrac
abMagErr = afwImage.abMagErrFromFluxErr(fluxErr, flux)
self.assertFloatsAlmostEqual(abMagErr,
refABMagErrFromFluxErr(fluxErr, flux))
fluxErrRoundTrip = afwImage.fluxErrFromABMagErr(abMagErr, abMag)
self.assertFloatsAlmostEqual(fluxErr,
fluxErrRoundTrip,
rtol=1.0e-15)
def extractMagArrays(self, matches, filterName, sourceKeys):
"""!Extract magnitude and magnitude error arrays from the given matches.
\param[in] matches Reference/source matches, a \link lsst::afw::table::ReferenceMatchVector\endlink
\param[in] filterName Name of filter being calibrated
\param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys()
\return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays
where magErr is an error in the magnitude; the error in srcMag - refMag
If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as
magErr is what is later used to determine the zero point.
Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes
(1 or 2 strings)
\note These magnitude arrays are the \em inputs to the photometric calibration, some may have been
discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813)
"""
srcFluxArr = np.array([m.second.get(sourceKeys.flux) for m in matches])
srcFluxErrArr = np.array([m.second.get(sourceKeys.fluxErr) for m in matches])
if not np.all(np.isfinite(srcFluxErrArr)):
# this is an unpleasant hack; see DM-2308 requesting a better solution
self.log.warn("Source catalog does not have flux uncertainties; using sqrt(flux).")
srcFluxErrArr = np.sqrt(srcFluxArr)
# convert source flux from DN to an estimate of Jy
JanskysPerABFlux = 3631.0
srcFluxArr = srcFluxArr * JanskysPerABFlux
srcFluxErrArr = srcFluxErrArr * JanskysPerABFlux
if not matches:
raise RuntimeError("No reference stars are available")
refSchema = matches[0].first.schema
applyColorTerms = self.config.applyColorTerms
applyCTReason = "config.applyColorTerms is %s" % (self.config.applyColorTerms,)
if self.config.applyColorTerms is None:
# apply color terms if color term data is available and photoCatName specified
ctDataAvail = len(self.config.colorterms.data) > 0
photoCatSpecified = self.config.photoCatName is not None
applyCTReason += " and data %s available" % ("is" if ctDataAvail else "is not")
applyCTReason += " and photoRefCat %s None" % ("is not" if photoCatSpecified else "is")
applyColorTerms = ctDataAvail and photoCatSpecified
if applyColorTerms:
self.log.info("Applying color terms for filterName=%r, config.photoCatName=%s because %s" %
(filterName, self.config.photoCatName, applyCTReason))
ct = self.config.colorterms.getColorterm(
filterName=filterName, photoCatName=self.config.photoCatName, doRaise=True)
else:
self.log.info("Not applying color terms because %s" % (applyCTReason,))
ct = None
if ct: # we have a color term to worry about
fluxFieldList = [getRefFluxField(refSchema, filt) for filt in (ct.primary, ct.secondary)]
missingFluxFieldList = []
for fluxField in fluxFieldList:
try:
refSchema.find(fluxField).key
except KeyError:
missingFluxFieldList.append(fluxField)
if missingFluxFieldList:
self.log.warn("Source catalog does not have fluxes for %s; ignoring color terms" %
" ".join(missingFluxFieldList))
ct = None
if not ct:
fluxFieldList = [getRefFluxField(refSchema, filterName)]
refFluxArrList = [] # list of ref arrays, one per flux field
refFluxErrArrList = [] # list of ref flux arrays, one per flux field
for fluxField in fluxFieldList:
fluxKey = refSchema.find(fluxField).key
refFluxArr = np.array([m.first.get(fluxKey) for m in matches])
try:
fluxErrKey = refSchema.find(fluxField + "Sigma").key
refFluxErrArr = np.array([m.first.get(fluxErrKey) for m in matches])
except KeyError:
# Reference catalogue may not have flux uncertainties; HACK
self.log.warn("Reference catalog does not have flux uncertainties for %s; using sqrt(flux)."
% fluxField)
refFluxErrArr = np.sqrt(refFluxArr)
refFluxArrList.append(refFluxArr)
refFluxErrArrList.append(refFluxErrArr)
if ct: # we have a color term to worry about
refMagArr1 = np.array([abMagFromFlux(rf1) for rf1 in refFluxArrList[0]]) # primary
refMagArr2 = np.array([abMagFromFlux(rf2) for rf2 in refFluxArrList[1]]) # secondary
refMagArr = ct.transformMags(refMagArr1, refMagArr2)
refFluxErrArr = ct.propagateFluxErrors(refFluxErrArrList[0], refFluxErrArrList[1])
else:
refMagArr = np.array([abMagFromFlux(rf) for rf in refFluxArrList[0]])
srcMagArr = np.array([abMagFromFlux(sf) for sf in srcFluxArr])
# Fitting with error bars in both axes is hard
# for now ignore reference flux error, but ticket DM-2308 is a request for a better solution
magErrArr = np.array([abMagErrFromFluxErr(fe, sf) for fe, sf in izip(srcFluxErrArr, srcFluxArr)])
if self.config.magErrFloor != 0.0:
magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
srcMagErrArr = np.array([abMagErrFromFluxErr(sfe, sf) for sfe, sf in izip(srcFluxErrArr, srcFluxArr)])
refMagErrArr = np.array([abMagErrFromFluxErr(rfe, rf) for rfe, rf in izip(refFluxErrArr, refFluxArr)])
return pipeBase.Struct(
srcMag = srcMagArr,
refMag = refMagArr,
magErr = magErrArr,
srcMagErr = srcMagErrArr,
refMagErr = refMagErrArr,
refFluxFieldList = fluxFieldList,
)
def plot_point_mags(output_data,
visit_list,
dataId,
minMag=17,
mid_cut=20,
maxMag=26,
fit_curves=True):
# get a butler
butler = dp.Butler(output_data)
# The following value for refcatId is "mandatory, but meaningless",
# so we won't try to generalize it.
refcatId = {'tract': 0, 'patch': '0,0'}
ref = butler.get('deepCoadd_ref', dataId=refcatId)
# get the sources and calib objects for each single epoch visit
forced_srcs = {}
calibs = {}
for visit in visit_list:
dataId['visit'] = visit
try:
my_forced_srcs = butler.get('forced_src', dataId=dataId)
calexp = butler.get('calexp', dataId=dataId)
my_calibs = calexp.getCalib()
del calexp
forced_srcs[visit] = my_forced_srcs
calibs[visit] = my_calibs
except FitsError as eobj:
print(eobj)
# initialize dictionaries to hold lightcurve arrays. Get
# extendedness from the coadd catalog.
lightcurve_fluxes = {}
extendedness = {}
for idx, ext in zip(ref.get('id'),
ref.get('base_ClassificationExtendedness_value')):
lightcurve_fluxes[idx] = []
extendedness[idx] = ext
# pivot the source tables to assemble lightcurves
for visit, forced_src in forced_srcs.items():
calib = calibs[visit]
for idx, flux in zip(forced_src.get('objectId'),
forced_src.get('base_PsfFlux_flux')):
if extendedness[idx] > 0.5:
continue
if flux <= 0.:
continue
lightcurve_fluxes[idx].append(
afw_image.fluxFromABMag(calib.getMagnitude(flux)))
# compute aggregate quantities for each object and plot
band = dataId['filter']
med_mags = []
med_err = []
for lightcurve in lightcurve_fluxes.values():
if len(lightcurve) == len(visit_list):
median_flux = np.median(lightcurve)
med_mags.append(afw_image.abMagFromFlux(median_flux))
med_err.append(np.std(lightcurve) / median_flux)
print("number of objects: ", len(med_mags))
mag_stats = MagStats(band, med_mags, med_err, fit_curves=fit_curves)
if mag_stats.pcov is not None:
label ='filter=%s, Floor=%.1f%%, m_5=%0.2f' \
% (band, mag_stats.sys_floor*100, mag_stats.popt[1])
else:
label ='filter=%s, Floor=%.1f%%' \
% (band, mag_stats.sys_floor*100)
scatter = plt.scatter(med_mags,
med_err,
alpha=0.3,
color=_filter_color[band],
marker=_filter_symbol[band],
label=label)
plt.xlabel("Calibrated magnitude of median flux")
plt.ylabel("stdev(flux)/median(flux)")
plt.xlim(15.5, 25)
plt.ylim(0., 0.5)
return scatter, mag_stats
def test1(self):
res = self.getAstrometrySolution()
matches = res.matches
print 'Test1'
logLevel = Log.DEBUG
log = Log(Log.getDefaultLog(),
'testPhotoCal',
logLevel)
schema = matches[0].second.schema
config = PhotoCalConfig()
# The test and associated data have been prepared on the basis that we
# use the PsfFlux to perform photometry.
config.fluxField = "base_PsfFlux_flux"
config.doWriteOutput = False # schema is fixed because we already loaded the data
task = PhotoCalTask(config=config, schema=schema)
pCal = task.run(exposure=self.exposure, matches=matches)
print "Ref flux fields list =", pCal.arrays.refFluxFieldList
refFluxField = pCal.arrays.refFluxFieldList[0]
# These are *all* the matches; we don't really expect to do that well.
diff=[]
for m in matches:
refFlux = m[0].get(refFluxField) # reference catalog flux
if refFlux <= 0:
continue
refMag = afwImage.abMagFromFlux(refFlux) # reference catalog mag
instFlux = m[1].getPsfFlux() #Instrumental Flux
if instFlux <= 0:
continue
instMag = pCal.calib.getMagnitude(instFlux) #Instrumental mag
diff.append(instMag - refMag)
diff = np.array(diff)
self.assertGreater(len(diff), 50)
log.info('%i magnitude differences; mean difference %g; mean abs diff %g' %
(len(diff), np.mean(diff), np.mean(np.abs(diff))))
self.assertLess(np.mean(diff), 0.6)
# Differences of matched objects that were used in the fit.
zp = pCal.calib.getMagnitude(1.)
log.logdebug('zeropoint: %g' % zp)
fitdiff = pCal.arrays.srcMag + zp - pCal.arrays.refMag
log.logdebug('number of sources used in fit: %i' % len(fitdiff))
log.logdebug('rms diff: %g' % np.mean(fitdiff**2)**0.5)
log.logdebug('median abs(diff): %g' % np.median(np.abs(fitdiff)))
# zeropoint: 31.3145
# number of sources used in fit: 65
# median diff: -0.009681
# mean diff: 0.00331871
# median abs(diff): 0.0368904
# mean abs(diff): 0.0516589
self.assertLess(abs(zp - 31.3145), 0.05)
self.assertGreater(len(fitdiff), 50)
# Tolerances are somewhat arbitrary; they're set simply to avoid regressions, and
# are not based on we'd expect to get given the data quality.
self.assertLess(np.mean(fitdiff**2)**0.5, 0.07) # rms difference
self.assertLess(np.median(np.abs(fitdiff)), 0.06) # median absolution difference
def test1(self):
res = self.getAstrometrySolution()
matches = res.matches
print 'Test1'
logLevel = Log.DEBUG
log = Log(Log.getDefaultLog(), 'testPhotoCal', logLevel)
schema = matches[0].second.schema
config = PhotoCalConfig()
# The test and associated data have been prepared on the basis that we
# use the PsfFlux to perform photometry.
config.fluxField = "base_PsfFlux_flux"
config.doWriteOutput = False # schema is fixed because we already loaded the data
task = PhotoCalTask(config=config, schema=schema)
pCal = task.run(exposure=self.exposure, matches=matches)
print "Ref flux fields list =", pCal.arrays.refFluxFieldList
refFluxField = pCal.arrays.refFluxFieldList[0]
# These are *all* the matches; we don't really expect to do that well.
diff = []
for m in matches:
refFlux = m[0].get(refFluxField) # reference catalog flux
if refFlux <= 0:
continue
refMag = afwImage.abMagFromFlux(refFlux) # reference catalog mag
instFlux = m[1].getPsfFlux() #Instrumental Flux
if instFlux <= 0:
continue
instMag = pCal.calib.getMagnitude(instFlux) #Instrumental mag
diff.append(instMag - refMag)
diff = np.array(diff)
self.assertGreater(len(diff), 50)
log.info(
'%i magnitude differences; mean difference %g; mean abs diff %g' %
(len(diff), np.mean(diff), np.mean(np.abs(diff))))
self.assertLess(np.mean(diff), 0.6)
# Differences of matched objects that were used in the fit.
zp = pCal.calib.getMagnitude(1.)
log.logdebug('zeropoint: %g' % zp)
fitdiff = pCal.arrays.srcMag + zp - pCal.arrays.refMag
log.logdebug('number of sources used in fit: %i' % len(fitdiff))
log.logdebug('rms diff: %g' % np.mean(fitdiff**2)**0.5)
log.logdebug('median abs(diff): %g' % np.median(np.abs(fitdiff)))
# zeropoint: 31.3145
# number of sources used in fit: 65
# median diff: -0.009681
# mean diff: 0.00331871
# median abs(diff): 0.0368904
# mean abs(diff): 0.0516589
self.assertLess(abs(zp - 31.3145), 0.05)
self.assertGreater(len(fitdiff), 50)
# Tolerances are somewhat arbitrary; they're set simply to avoid regressions, and
# are not based on we'd expect to get given the data quality.
self.assertLess(np.mean(fitdiff**2)**0.5, 0.07) # rms difference
self.assertLess(np.median(np.abs(fitdiff)),
0.06) # median absolution difference
areNans = np.logical_or.reduce(areNans)
hdu[1].data = hdu[1].data[~areNans]
#Limit to brighter than 19th mag in g-band
bright = hdu[1].data['g'] > 9.12e-5
hdu[1].data = hdu[1].data[bright]
hdu[1].data['g_err'] = abMagErrFromFluxErr(hdu[1].data['g_err'].astype(np.float),
hdu[1].data['g'].astype(np.float))
hdu[1].data['r_err'] = abMagErrFromFluxErr(hdu[1].data['r_err'].astype(np.float),
hdu[1].data['r'].astype(np.float))
hdu[1].data['i_err'] = abMagErrFromFluxErr(hdu[1].data['i_err'].astype(np.float),
hdu[1].data['i'].astype(np.float))
hdu[1].data['z_err'] = abMagErrFromFluxErr(hdu[1].data['z_err'].astype(np.float),
hdu[1].data['z'].astype(np.float))
hdu[1].data['y_err'] = abMagErrFromFluxErr(hdu[1].data['y_err'].astype(np.float),
hdu[1].data['y'].astype(np.float))
hdu[1].data['g'] = abMagFromFlux(hdu[1].data['g'].astype(np.float))
hdu[1].data['r'] = abMagFromFlux(hdu[1].data['r'].astype(np.float))
hdu[1].data['i'] = abMagFromFlux(hdu[1].data['i'].astype(np.float))
hdu[1].data['z'] = abMagFromFlux(hdu[1].data['z'].astype(np.float))
hdu[1].data['y'] = abMagFromFlux(hdu[1].data['y'].astype(np.float))
hdu[1].data['coord_ra'] = (180./np.pi)*hdu[1].data['coord_ra']
hdu[1].data['coord_dec'] = (180./np.pi)*hdu[1].data['coord_dec']
base = os.path.basename(file)
newFile = 'ps1_pv3_3pi_20170110_filteredAB/'+base
hdu.writeto(newFile, overwrite=True)
# initialize dictionaries to hold lightcurve arrays. Get extendedness from the coadd catalog.
lightcurve_fluxes = {}
extendedness = {}
for idx, ext in zip(ref.get('id'), ref.get('base_ClassificationExtendedness_value')):
lightcurve_fluxes[idx] = []
extendedness[idx] = ext
# pivot the source tables to assemble lightcurves
for visit, forced_src in forced_srcs.iteritems():
calib = calibs[visit]
for idx, flux in zip(forced_src.get('objectId'), forced_src.get('base_PsfFlux_flux')):
if extendedness[idx] > 0.5:
continue
if flux <= 0.:
continue
lightcurve_fluxes[idx].append(afw_image.fluxFromABMag(calib.getMagnitude(flux)))
# compute aggregate quantities for each object and plot
for lightcurve in lightcurve_fluxes.values():
if len(lightcurve) == 9:
plt.scatter(afw_image.abMagFromFlux(numpy.median(lightcurve)),
numpy.std(lightcurve)/numpy.median(lightcurve), alpha=0.5)
mags, invsnrs = make_invsnr_arr()
plt.plot(mags, invsnrs, color='red', linewidth=2, alpha=0.75)
plt.xlabel("Calibrated magnitude of median flux")
plt.ylabel("stdev(flux)/median(flux)")
plt.xlim(15.5, 25)
plt.ylim(0., 0.5)
plt.show()
# pivot the source tables to assemble lightcurves
for visit, forced_src in forced_srcs.iteritems():
calib = calibs[visit]
for idx, flux in zip(forced_src.get('objectId'), forced_src.get('base_PsfFlux_flux')):
if extendedness[idx] > 0.5:
continue
if flux <= 0.:
continue
lightcurve_fluxes[idx].append(afw_image.fluxFromABMag(calib.getMagnitude(flux)))
# compute aggregate quantities for each object and plot
med_mags = []
med_err = []
for lightcurve in lightcurve_fluxes.values():
if len(lightcurve) == 10:
med_mags.append(afw_image.abMagFromFlux(numpy.median(lightcurve)))
med_err.append(numpy.std(lightcurve)/numpy.median(lightcurve))
fit_func = partial(fit_invsnr, bandpass_name=bandpass_name)
minMag=17.
mid_cut=20.
maxMag=26.
med_mags = numpy.array(med_mags)
med_err = numpy.array(med_err)
idxs = numpy.where((med_mags<mid_cut)&(med_mags>minMag))
sys_floor = numpy.median(med_err[idxs])
idxs = numpy.where((med_mags<maxMag)&(med_mags>mid_cut))
popt, pcov = curve_fit(fit_invsnr, med_mags[idxs], med_err[idxs], p0=[0.01, 24.5])
scatter = plt.scatter(med_mags, med_err, alpha=0.3, color=bandpass_color_map[bandpass_name],
marker=bandpass_symbol_map[bandpass_name],
label="filter=%s, Floor=%.1f%%, m_5=%0.2f"%(bandpass_name,sys_floor*100.,popt[1]))
scatters.append(scatter)
def extractMagArrays(self, matches, filterName, sourceKeys):
"""!Extract magnitude and magnitude error arrays from the given matches.
@param[in] matches Reference/source matches, a @link lsst::afw::table::ReferenceMatchVector@endlink
@param[in] filterName Name of filter being calibrated
@param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys()
@return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays
where magErr is an error in the magnitude; the error in srcMag - refMag
If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as
magErr is what is later used to determine the zero point.
Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes
(1 or 2 strings)
@note These magnitude arrays are the @em inputs to the photometric calibration, some may have been
discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813)
"""
srcFluxArr = np.array([m.second.get(sourceKeys.flux) for m in matches])
srcFluxErrArr = np.array(
[m.second.get(sourceKeys.fluxErr) for m in matches])
if not np.all(np.isfinite(srcFluxErrArr)):
# this is an unpleasant hack; see DM-2308 requesting a better solution
self.log.warn(
"Source catalog does not have flux uncertainties; using sqrt(flux)."
)
srcFluxErrArr = np.sqrt(srcFluxArr)
# convert source flux from DN to an estimate of Jy
JanskysPerABFlux = 3631.0
srcFluxArr = srcFluxArr * JanskysPerABFlux
srcFluxErrArr = srcFluxErrArr * JanskysPerABFlux
if not matches:
raise RuntimeError("No reference stars are available")
refSchema = matches[0].first.schema
applyColorTerms = self.config.applyColorTerms
applyCTReason = "config.applyColorTerms is %s" % (
self.config.applyColorTerms, )
if self.config.applyColorTerms is None:
# apply color terms if color term data is available and photoCatName specified
ctDataAvail = len(self.config.colorterms.data) > 0
photoCatSpecified = self.config.photoCatName is not None
applyCTReason += " and data %s available" % ("is" if ctDataAvail
else "is not")
applyCTReason += " and photoRefCat %s provided" % (
"is" if photoCatSpecified else "is not")
applyColorTerms = ctDataAvail and photoCatSpecified
if applyColorTerms:
self.log.info(
"Applying color terms for filterName=%r, config.photoCatName=%s because %s",
filterName, self.config.photoCatName, applyCTReason)
ct = self.config.colorterms.getColorterm(
filterName=filterName,
photoCatName=self.config.photoCatName,
doRaise=True)
else:
self.log.info("Not applying color terms because %s", applyCTReason)
ct = None
if ct: # we have a color term to worry about
fluxFieldList = [
getRefFluxField(refSchema, filt)
for filt in (ct.primary, ct.secondary)
]
missingFluxFieldList = []
for fluxField in fluxFieldList:
try:
refSchema.find(fluxField).key
except KeyError:
missingFluxFieldList.append(fluxField)
if missingFluxFieldList:
self.log.warn(
"Source catalog does not have fluxes for %s; ignoring color terms",
" ".join(missingFluxFieldList))
ct = None
if not ct:
fluxFieldList = [getRefFluxField(refSchema, filterName)]
refFluxArrList = [] # list of ref arrays, one per flux field
refFluxErrArrList = [] # list of ref flux arrays, one per flux field
for fluxField in fluxFieldList:
fluxKey = refSchema.find(fluxField).key
refFluxArr = np.array([m.first.get(fluxKey) for m in matches])
try:
fluxErrKey = refSchema.find(fluxField + "Sigma").key
refFluxErrArr = np.array(
[m.first.get(fluxErrKey) for m in matches])
except KeyError:
# Reference catalogue may not have flux uncertainties; HACK
self.log.warn(
"Reference catalog does not have flux uncertainties for %s; using sqrt(flux).",
fluxField)
refFluxErrArr = np.sqrt(refFluxArr)
refFluxArrList.append(refFluxArr)
refFluxErrArrList.append(refFluxErrArr)
if ct: # we have a color term to worry about
refMagArr1 = np.array(
[abMagFromFlux(rf1) for rf1 in refFluxArrList[0]]) # primary
refMagArr2 = np.array(
[abMagFromFlux(rf2) for rf2 in refFluxArrList[1]]) # secondary
refMagArr = ct.transformMags(refMagArr1, refMagArr2)
refFluxErrArr = ct.propagateFluxErrors(refFluxErrArrList[0],
refFluxErrArrList[1])
else:
refMagArr = np.array(
[abMagFromFlux(rf) for rf in refFluxArrList[0]])
srcMagArr = np.array([abMagFromFlux(sf) for sf in srcFluxArr])
# Fitting with error bars in both axes is hard
# for now ignore reference flux error, but ticket DM-2308 is a request for a better solution
magErrArr = np.array([
abMagErrFromFluxErr(fe, sf)
for fe, sf in zip(srcFluxErrArr, srcFluxArr)
])
if self.config.magErrFloor != 0.0:
magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
srcMagErrArr = np.array([
abMagErrFromFluxErr(sfe, sf)
for sfe, sf in zip(srcFluxErrArr, srcFluxArr)
])
refMagErrArr = np.array([
abMagErrFromFluxErr(rfe, rf)
for rfe, rf in zip(refFluxErrArr, refFluxArr)
])
good = np.isfinite(srcMagArr) & np.isfinite(refMagArr)
return pipeBase.Struct(
srcMag=srcMagArr[good],
refMag=refMagArr[good],
magErr=magErrArr[good],
srcMagErr=srcMagErrArr[good],
refMagErr=refMagErrArr[good],
refFluxFieldList=fluxFieldList,
)
