def compare_brick_to_ps1(brickname, ps, name='', basedir=''):
decals = Decals()
brick = decals.get_brick_by_name(brickname)
wcs = wcs_for_brick(brick)
magrange = (15,20)
ps1 = ps1cat(ccdwcs=wcs)
ps1 = ps1.get_stars(magrange=magrange)
print 'Got', len(ps1), 'PS1 stars'
T = fits_table(os.path.join(basedir, 'tractor', brickname[:3],
'tractor-%s.fits' % brickname))
I,J,d = match_radec(T.ra, T.dec, ps1.ra, ps1.dec, 1./3600.)
print 'Matched', len(I), 'stars to PS1'
T.cut(I)
ps1.cut(J)
bands = 'z'
ap = 5
allbands = 'ugrizY'
mags = np.arange(magrange[0], 1+magrange[1])
for band in bands:
iband = allbands.index(band)
piband = ps1cat.ps1band[band]
T.flux = T.decam_flux[:,iband]
T.mag = NanoMaggies.nanomaggiesToMag(T.flux)
print 'apflux shape', T.decam_apflux.shape
T.apflux = T.decam_apflux[:, iband, ap]
T.apmag = NanoMaggies.nanomaggiesToMag(T.apflux)
ps1mag = ps1.median[:,piband]
plt.clf()
for cc,mag,label in [('b', T.mag, 'Mag'), ('r', T.apmag, 'Aper mag')]:
plt.plot(ps1mag, mag - ps1mag, '.', color=cc, label=label, alpha=0.6)
mm,dd = [],[]
for mlo,mhi in zip(mags, mags[1:]):
I = np.flatnonzero((ps1mag > mlo) * (ps1mag <= mhi))
mm.append((mlo+mhi)/2.)
dd.append(np.median(mag[I] - ps1mag[I]))
plt.plot(mm, dd, 'o-', color=cc)
plt.xlabel('PS1 %s mag' % band)
plt.ylabel('Mag - PS1 (mag)')
plt.title('%sPS1 comparison: brick %s' % (name, brickname))
plt.ylim(-0.2, 0.2)
mlo,mhi = magrange
plt.xlim(mhi, mlo)
plt.axhline(0., color='k', alpha=0.1)
plt.legend()
ps.savefig()
def set_mags(self, cat):
'''adds columns to fits_table cat'''
# Remove white spaces
cat.set('type', np.char.strip(cat.get('type')))
# AB mags
# Two kinds, as observed (including dust) and instrinsic (dust removed)
for whichmag in ['wdust', 'nodust']:
# DECam
for band in ['g', 'r', 'z', 'w1', 'w2']:
if whichmag == 'wdust':
flux = cat.get('flux_%s' % band)
flux_ivar = cat.get('flux_ivar_%s' % band)
elif whichmag == 'nodust':
flux = cat.get('flux_%s' % band) / cat.get(
'mw_transmission_%s' % band)
flux_ivar= cat.get('flux_ivar_%s' % band)*\
np.power(cat.get('mw_transmission_%s' % band),2)
else:
raise ValueError()
mag, mag_err = NanoMaggies.fluxErrorsToMagErrors(
flux, flux_ivar)
cat.set('mag_%s_%s' % (whichmag, band), mag)
cat.set('mag_ivar_%s_%s' % (whichmag, band),
1. / np.power(mag_err, 2))
# Instrinsic fluxes
whichmag = 'nodust'
# DECam
for band in ['g', 'r', 'z', 'w1', 'w2']:
flux = cat.get('flux_%s' % band) / cat.get(
'mw_transmission_%s' % band)
flux_ivar= cat.get('flux_ivar_%s' % band)*\
np.power(cat.get('mw_transmission_%s' % band),2)
cat.set('flux_%s_%s' % (whichmag, band), flux)
cat.set('flux_ivar_%s_%s' % (whichmag, band), flux_ivar)
def __init__(self, hdr, bandname=None, scale=1.):
import numpy as np
from tractor.brightness import NanoMaggies
if hdr is not None:
self.exptime = hdr['EXPTIME']
self.phot_c = hdr['PHOT_C']
self.phot_k = hdr['PHOT_K']
self.airmass = hdr['AIRMASS']
print('CFHT photometry:', self.exptime, self.phot_c, self.phot_k,
self.airmass)
zpt = (2.5 * np.log10(self.exptime) + self.phot_c + self.phot_k *
(self.airmass - 1))
print('-> zeropoint', zpt)
scale = NanoMaggies.zeropointToScale(zpt)
print('-> scale', scale)
super(CfhtLinearPhotoCal, self).__init__(scale, band=bandname)
def set_mags_OldDataModel(self, cat):
'''for tractor catalogues with columns like decam_flux.shape(many,6)
adds columns to fits_table cat'''
# Remove white spaces
cat.set('type', np.char.strip(cat.get('type')))
# AB mags
# Two kinds, as observed (including dust) and instrinsic (dust removed)
for whichmag in ['wdust', 'nodust']:
# DECam
shp = cat.get('decam_flux').shape
mag, mag_err = np.zeros(shp), np.zeros(shp)
for iband in range(shp[1]):
if whichmag == 'wdust':
flux = cat.get('decam_flux')[:, iband]
flux_ivar = cat.get('decam_flux_ivar')[:, iband]
elif whichmag == 'nodust':
flux = cat.get('decam_flux')[:, iband] / cat.get(
'decam_mw_transmission')[:, iband]
flux_ivar= cat.get('decam_flux_ivar')[:,iband]*\
np.power(cat.get('decam_mw_transmission')[:,iband],2)
else:
raise ValueError()
mag[:,
iband], mag_err[:,
iband] = NanoMaggies.fluxErrorsToMagErrors(
flux, flux_ivar)
cat.set('decam_mag_%s' % whichmag, mag)
cat.set('decam_mag_ivar_%s' % whichmag, 1. / np.power(mag_err, 2))
# WISE
if 'wise_flux' in cat.get_columns():
shp = cat.get('wise_flux').shape
mag, mag_err = np.zeros(shp), np.zeros(shp)
for iband in range(shp[1]):
if whichmag == 'wdust':
flux = cat.get('wise_flux')[:, iband]
flux_ivar = cat.get('wise_flux_ivar')[:, iband]
elif whichmag == 'nodust':
flux = cat.get('wise_flux')[:, iband] / cat.get(
'wise_mw_transmission')[:, iband]
flux_ivar= cat.get('wise_flux_ivar')[:,iband]*\
np.power(cat.get('wise_mw_transmission')[:,iband],2)
mag[:,
iband], mag_err[:,
iband] = NanoMaggies.fluxErrorsToMagErrors(
flux, flux_ivar)
cat.set('wise_mag_%s' % whichmag, mag)
cat.set('wise_mag_ivar_%s' % whichmag,
1. / np.power(mag_err, 2))
# Instrinsic fluxes
whichmag = 'nodust'
# DECam
shp = cat.get('decam_flux').shape
flux, flux_ivar = np.zeros(shp), np.zeros(shp)
for iband in range(shp[1]):
flux[:, iband] = cat.get('decam_flux')[:, iband] / cat.get(
'decam_mw_transmission')[:, iband]
flux_ivar[:,iband]= cat.get('decam_flux_ivar')[:,iband]*\
np.power(cat.get('decam_mw_transmission')[:,iband],2)
cat.set('decam_flux_%s' % whichmag, flux)
cat.set('decam_flux_ivar_%s' % whichmag, flux_ivar)
# WISE
if 'wise_flux' in cat.get_columns():
shp = cat.get('wise_flux').shape
flux, flux_err = np.zeros(shp), np.zeros(shp)
for iband in range(shp[1]):
flux[:, iband] = cat.get('wise_flux')[:, iband] / cat.get(
'wise_mw_transmission')[:, iband]
flux_ivar[:,iband]= cat.get('wise_flux_ivar')[:,iband]*\
np.power(cat.get('wise_mw_transmission')[:,iband],2)
cat.set('wise_flux_%s' % whichmag, flux)
cat.set('wise_flux_ivar_%s' % whichmag, flux_ivar)
plt.figure(figsize=(10,10))
plt.subplots_adjust(left=0.05, right=0.95, bottom=0.05, top=0.95,
hspace=0, wspace=0)
fn = os.path.join('%s/tractor/%s/tractor-%s.fits' %
(base, brick[:3], brick))
print 'Reading', fn
T = fits_table(fn)
print len(T), 'sources'
jpeg = os.path.join('%s/coadd/%s/%s/decals-%s-image.jpg' %
(base, brick[:3], brick, brick))
img = plt.imread(jpeg)
img = np.flipud(img)
mags = NanoMaggies.nanomaggiesToMag(T.decam_flux)
mags[np.logical_not(np.isfinite(mags))] = 99.
T.g = mags[:,1]
T.r = mags[:,2]
T.z = mags[:,4]
# Convert
I = np.flatnonzero((T.r < 15) * (T.type == 'SIMP'))
print 'Converting', len(I), 'bright SIMP objects into PSFs'
T.type[I] = 'PSF '
if False:
P = T[T.type == 'PSF ']
print len(P), 'PSFs'
def gim2d_catalog(cat, band):
'''
http://irsa.ipac.caltech.edu/data/COSMOS/tables/morphology/cosmos_morph_zurich_colDescriptions.html
'''
'''
ACS_MU_CLASS float Type of object.
1 = galaxy
2 = star
3 = spurious
'''
'''
ACS_CLEAN float Object useable flag.
0 = do not use this object
1 = use this object
'''
'''
FLUX_GIM2D float counts GIM2D total flux
R_GIM2D float arcseconds GIM2D psf-convolved half-light radius of object
ELL_GIM2D float GIM2D ellipticity = 1-b/a of object
PA_GIM2D float degrees GIM2D position angle of object - cw from +y-axis
DX_GIM2D float arcseconds x-offset of GIM2D-model center from ACS-coordinate center
DY_GIM2D float arcseconds y-offset of GIM2D-model center from ACS-coordinate center
SERSIC_N_GIM2D float GIM2D Sersic index
R_0P5_GIM2D float arcseconds GIM2D half-light radius of object without PSF convolution
TYPE float ZEST Type CLASS
1 = Early type
2 = Disk
3 = Irregular Galaxy
9 = no classification
'''
'''
BULG float ZEST "Bulgeness" CLASS - only for Type 2 (disk) galaxies.
0 = bulge dominated galaxy
1,2 = intermediate-bulge galaxies
3 = pure disk galaxy
9 = no classification
'''
'''
STELLARITY float Visual Stellarity flag.
0 if ACS_CLASS_STAR<0.6 (object is ASSUMED to be a galaxy; no visual inspection)
0 if ACS_CLASS_STAR>=0.6 AND object visually identified as a galaxy.
1 if ACS_CLASS_STAR>=0.6 AND visually identified as a star.
2 if ACS_CLASS_STAR>=0.8 (object is assumed to be a star and was not visually inspected)
3 if ACS_CLASS_STAR<0.6 but object is visually identified as a star (e.g. saturated star, etc)
JUNKFLAG float
0 = good object
1 = spurious
'''
print('Classifications:', Counter(cat.type).most_common())
cat.is_galaxy = (cat.stellarity == 0)
srcs = []
for t in cat:
pos = RaDecPos(t.ra, t.dec)
bright = NanoMaggies(
**{band: NanoMaggies.magToNanomaggies(t.acs_mag_auto)})
shape = GalaxyShape(t.r_0p5_gim2d, 1. - t.ell_gim2d, 90. + t.pa_gim2d)
is_galaxy = (t.is_galaxy * (shape.re >= 0) * (shape.ab <= 1.) *
(shape.phi > -999))
if is_galaxy and t.type == 1:
# deV
src = DevGalaxy(pos, bright, shape)
elif is_galaxy and t.type == 2:
# exp
src = ExpGalaxy(pos, bright, shape)
else:
src = PointSource(pos, bright)
srcs.append(src)
return srcs
print('Tim photocal:', tim.photocal)
print('Subtim photocal:', subtim.photocal)
# # Order by flux
I = np.argsort([-src.getBrightness().getFlux(band) for src in srcs])
for i in I:
src = srcs[i]
print('Source:', src)
print('-> counts',
subtim.photocal.brightnessToCounts(src.getBrightness()))
# flux in nanomaggies
flux = np.array([src.getBrightness().getFlux(band) for src in srcs])
fluxiv = R.IV
mag, magerr = NanoMaggies.fluxErrorsToMagErrors(flux, fluxiv)
typemap = {ExpGalaxy: 'E', DevGalaxy: 'D', PointSource: 'P'}
cat.tractor_type = np.array([typemap[type(src)] for src in srcs])
cat.set('cfht_forced_mag_%s' % band, mag)
cat.set('cfht_forced_magerr_%s' % band, magerr)
cat.writeto('cfht-forced.fits')
plt.clf()
plt.plot(cat.acs_mag_auto, mag, 'b.')
plt.xlabel('ACS I-band (mag)')
plt.ylabel('CFHT %s-band forced phot (mag)' % band)
plt.title('CFHT forced phot')
ps.savefig()
def main():
ps = PlotSequence('dr3-summary')
#fns = glob('/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-*.fits')
#fns = glob('/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-240p005-250p010.fits')
fns = ['dr3.1/sweep/3.1/sweep-240p005-250p010.fits',
# '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-240p010-250p015.fits',
# '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-230p005-240p010.fits',
# '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-230p010-240p015.fits'
]
plt.figure(2, figsize=(6,4))
plt.subplots_adjust(left=0.1, bottom=0.15, right=0.98, top=0.98)
plt.figure(1)
TT = []
for fn in fns:
T = fits_table(fn)
print(len(T), 'from', fn)
TT.append(T)
T = merge_tables(TT)
del TT
print('Types:', Counter(T.type))
T.flux_g = T.decam_flux[:,1]
T.flux_r = T.decam_flux[:,2]
T.flux_z = T.decam_flux[:,4]
T.flux_ivar_g = T.decam_flux_ivar[:,1]
T.flux_ivar_r = T.decam_flux_ivar[:,2]
T.flux_ivar_z = T.decam_flux_ivar[:,4]
T.mag_g, T.magerr_g = NanoMaggies.fluxErrorsToMagErrors(T.flux_g, T.flux_ivar_g)
T.mag_r, T.magerr_r = NanoMaggies.fluxErrorsToMagErrors(T.flux_r, T.flux_ivar_r)
T.mag_z, T.magerr_z = NanoMaggies.fluxErrorsToMagErrors(T.flux_z, T.flux_ivar_z)
T.deflux_g = T.flux_g / T.decam_mw_transmission[:,1]
T.deflux_r = T.flux_r / T.decam_mw_transmission[:,2]
T.deflux_z = T.flux_z / T.decam_mw_transmission[:,4]
T.deflux_ivar_g = T.flux_ivar_g * T.decam_mw_transmission[:,1]**2
T.deflux_ivar_r = T.flux_ivar_r * T.decam_mw_transmission[:,2]**2
T.deflux_ivar_z = T.flux_ivar_z * T.decam_mw_transmission[:,4]**2
T.demag_g, T.demagerr_g = NanoMaggies.fluxErrorsToMagErrors(T.deflux_g, T.deflux_ivar_g)
T.demag_r, T.demagerr_r = NanoMaggies.fluxErrorsToMagErrors(T.deflux_r, T.deflux_ivar_r)
T.demag_z, T.demagerr_z = NanoMaggies.fluxErrorsToMagErrors(T.deflux_z, T.deflux_ivar_z)
T.typex = np.array([t[0] for t in T.type])
T.mag_w1, T.magerr_w1 = NanoMaggies.fluxErrorsToMagErrors(
T.wise_flux[:,0], T.wise_flux_ivar[:,0])
T.mag_w2, T.magerr_w2 = NanoMaggies.fluxErrorsToMagErrors(
T.wise_flux[:,1], T.wise_flux_ivar[:,1])
tt = 'P'
I2 = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) *
(T.flux_ivar_z > 0) *
(T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
(T.magerr_g < 0.05) *
(T.magerr_r < 0.05) *
(T.magerr_z < 0.05) *
(T.typex == tt))
print(len(I2), 'with < 0.05-mag errs and type PSF')
plt.clf()
plt.hist(T.mag_w1[I2], bins=100, range=[12,24], histtype='step', color='b')
plt.hist(T.mag_w2[I2], bins=100, range=[12,24], histtype='step', color='g')
plt.xlabel('WISE mag')
ps.savefig()
plt.clf()
plt.hist(T.magerr_w1[I2], bins=100, range=[0,1], histtype='step', color='b')
plt.hist(T.magerr_w2[I2], bins=100, range=[0,1], histtype='step', color='g')
plt.xlabel('WISE mag errors')
ps.savefig()
I2 = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) *
(T.flux_ivar_z > 0) *
(T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
(T.magerr_g < 0.05) *
(T.magerr_r < 0.05) *
(T.magerr_z < 0.05) *
(T.typex == tt) *
(T.magerr_w1 < 0.05))
#(T.magerr_w2 < 0.1) *
print(len(I2), 'with < 0.05-mag errs and type PSF and W1 errors < 0.1')
Q1 = fits_table('dr3.1/external/survey-dr3-DR12Q.fits')
Q2 = fits_table('dr3.1/external/DR12Q.fits', columns=['z_pipe'])
Q1.add_columns_from(Q2)
I = np.flatnonzero(Q1.objid >= 0)
Q1.cut(I)
print(len(Q1), 'sources with DR12Q matches')
Q = Q1
Q.typex = np.array([t[0] for t in Q.type])
Q.mag_w1, Q.magerr_w1 = NanoMaggies.fluxErrorsToMagErrors(
Q.wise_flux[:,0], Q.wise_flux_ivar[:,0])
Q.mag_w2, Q.magerr_w2 = NanoMaggies.fluxErrorsToMagErrors(
Q.wise_flux[:,1], Q.wise_flux_ivar[:,1])
Q.flux_g = Q.decam_flux[:,1]
Q.flux_r = Q.decam_flux[:,2]
Q.flux_z = Q.decam_flux[:,4]
Q.flux_ivar_g = Q.decam_flux_ivar[:,1]
Q.flux_ivar_r = Q.decam_flux_ivar[:,2]
Q.flux_ivar_z = Q.decam_flux_ivar[:,4]
Q.deflux_g = Q.flux_g / Q.decam_mw_transmission[:,1]
Q.deflux_r = Q.flux_r / Q.decam_mw_transmission[:,2]
Q.deflux_z = Q.flux_z / Q.decam_mw_transmission[:,4]
Q.deflux_ivar_g = Q.flux_ivar_g * Q.decam_mw_transmission[:,1]**2
Q.deflux_ivar_r = Q.flux_ivar_r * Q.decam_mw_transmission[:,2]**2
Q.deflux_ivar_z = Q.flux_ivar_z * Q.decam_mw_transmission[:,4]**2
Q.demag_g, Q.demagerr_g = NanoMaggies.fluxErrorsToMagErrors(Q.deflux_g, Q.deflux_ivar_g)
Q.demag_r, Q.demagerr_r = NanoMaggies.fluxErrorsToMagErrors(Q.deflux_r, Q.deflux_ivar_r)
Q.demag_z, Q.demagerr_z = NanoMaggies.fluxErrorsToMagErrors(Q.deflux_z, Q.deflux_ivar_z)
I = np.flatnonzero((Q.flux_ivar_g > 0) * (Q.flux_ivar_r > 0) *
(Q.flux_ivar_z > 0) *
(Q.flux_g > 0) * (Q.flux_r > 0) * (Q.flux_z > 0) *
(Q.demagerr_g < 0.05) *
(Q.demagerr_r < 0.05) *
(Q.demagerr_z < 0.05) *
(Q.typex == tt) *
(Q.magerr_w1 < 0.05))
Q.cut(I)
print(len(Q), 'DR12Q-matched entries of type PSF with g,r,z,W1 errs < 0.05')
plt.clf()
loghist(T.demag_g[I2] - T.demag_r[I2], T.demag_z[I2] - T.mag_w1[I2],
nbins=200, range=((-0.5,2.0),(-3,4)))
plt.xlabel('g - r (mag)')
plt.ylabel('z - W1 (mag)')
ps.savefig()
#Q.cut(np.argsort(Q.z_pipe))
plt.clf()
plt.scatter(Q.demag_g - Q.demag_r, Q.demag_z - Q.mag_w1, c=Q.z_pipe, s=5)
plt.colorbar(label='QSO redshift')
plt.xlabel('g - r (mag)')
plt.ylabel('z - W1 (mag)')
ps.savefig()
plt.figure(2)
from scipy.ndimage.filters import gaussian_filter
import matplotlib.cm
xlo,xhi,ylo,yhi = [-0.25, 1.75, -2.5, 2.5]
plt.clf()
H,xe,ye = loghist(T.demag_g[I2] - T.demag_r[I2], T.demag_z[I2] - T.mag_w1[I2],
nbins=(400,200), range=((xlo,xhi),(ylo,yhi)), imshowargs=dict(cmap='Greys'),
hot=False, docolorbar=False)
I = np.flatnonzero(Q.z_pipe < 10.)
QH,nil,nil = np.histogram2d(Q.demag_g[I] - Q.demag_r[I], Q.demag_z[I] - Q.mag_w1[I],
bins=200, range=((xlo,xhi),(ylo,yhi)))
QH = gaussian_filter(QH.T, 3.)
c = plt.contour(QH, 8, extent=[xe.min(), xe.max(), ye.min(), ye.max()], colors='k')
plt.axis([xlo,xhi,ylo,yhi])
plt.xlabel('g - r (mag)')
plt.ylabel('z - W1 (mag)')
mx,my,mz = [],[],[]
# I = np.argsort(Q.z_pipe)
# for i in range(len(Q)/1000):
# J = I[i*1000:][:1000]
# mx.append(np.median(Q.demag_g[J] - Q.demag_r[J]))
# my.append(np.median(Q.demag_z[J] - Q.mag_w1[J]))
# mz.append(np.median(Q.z_pipe[J]))
# plt.scatter(mx,my, c=mz)
# plt.colorbar(label='QSO Redshift')
# ps.savefig()
zvals = np.arange(0, 3.61, 0.1)
for zlo,zhi in zip(zvals, zvals[1:]):
J = np.flatnonzero((Q.z_pipe >= zlo) * (Q.z_pipe < zhi))
mx.append(np.median(Q.demag_g[J] - Q.demag_r[J]))
my.append(np.median(Q.demag_z[J] - Q.mag_w1[J]))
mz.append(np.median(Q.z_pipe[J]))
# plt.scatter(mx,my, c=mz, marker='o-')
# plt.colorbar(label='QSO Redshift')
for i in range(len(mx)-1):
c = matplotlib.cm.viridis(0.8 * i / (len(mx)-1))
plt.plot([mx[i], mx[i+1]], [my[i],my[i+1]], '-', color=c, lw=4)
plt.savefig('qso-wise.png')
plt.savefig('qso-wise.pdf')
xlo,xhi,ylo,yhi = [-0.25, 1.75, -0.25, 2.5]
plt.clf()
H,xe,ye = loghist(T.demag_g[I2] - T.demag_r[I2], T.demag_r[I2] - T.demag_z[I2],
nbins=(400,200), range=((xlo,xhi),(ylo,yhi)), imshowargs=dict(cmap='Greys'),
hot=False, docolorbar=False)
#I3 = np.random.permutation(I2)[:1000]
#plt.plot(T.demag_g[I3] - T.demag_r[I3], T.demag_r[I3] - T.demag_z[I3], 'r.')
I = np.flatnonzero(Q.z_pipe < 10.)
QH,nil,nil = np.histogram2d(Q.demag_g[I] - Q.demag_r[I], Q.demag_r[I] - Q.demag_z[I],
bins=200, range=((xlo,xhi),(ylo,yhi)))
QH = gaussian_filter(QH.T, 3.)
c = plt.contour(QH, 8, extent=[xe.min(), xe.max(), ye.min(), ye.max()], colors='k')
plt.axis([xlo,xhi,ylo,yhi])
plt.xlabel('g - r (mag)')
plt.ylabel('r - z (mag)')
mx,my,mz = [],[],[]
zvals = np.arange(0, 3.61, 0.1)
for zlo,zhi in zip(zvals, zvals[1:]):
J = np.flatnonzero((Q.z_pipe >= zlo) * (Q.z_pipe < zhi))
mx.append(np.median(Q.demag_g[J] - Q.demag_r[J]))
my.append(np.median(Q.demag_r[J] - Q.demag_z[J]))
mz.append(np.median(Q.z_pipe[J]))
for i in range(len(mx)-1):
c = matplotlib.cm.viridis(0.8 * i / (len(mx)-1))
plt.plot([mx[i], mx[i+1]], [my[i],my[i+1]], '-', color=c, lw=4)
plt.savefig('qso-optical.png')
plt.savefig('qso-optical.pdf')
plt.figure(1)
# I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0))
# print(len(I), 'with g,r,z fluxes')
# I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) *
# (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0))
# print(len(I), 'with positive g,r,z fluxes')
# I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) *
# (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
# (T.magerr_g < 0.2) * (T.magerr_r < 0.2) * (T.magerr_z < 0.2))
# print(len(I), 'with < 0.2-mag errs')
# I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) *
# (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
# (T.magerr_g < 0.1) * (T.magerr_r < 0.1) * (T.magerr_z < 0.1))
# print(len(I), 'with < 0.1-mag errs')
#
# I2 = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) *
# (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
# (T.magerr_g < 0.05) * (T.magerr_r < 0.05) * (T.magerr_z < 0.05))
# print(len(I2), 'with < 0.05-mag errs')
#ha = dict(nbins=200, range=((-0.5, 2.5), (-0.5, 2.5)))
ha = dict(nbins=400, range=((-0.5, 2.5), (-0.5, 2.5)))
plt.clf()
#plothist(T.mag_g[I] - T.mag_r[I], T.mag_r[I] - T.mag_z[I], 200)
#loghist(T.mag_g[I] - T.mag_r[I], T.mag_r[I] - T.mag_z[I], **ha)
loghist(T.demag_g[I] - T.demag_r[I], T.demag_r[I] - T.demag_z[I], **ha)
plt.xlabel('g - r (mag)')
plt.ylabel('r - z (mag)')
ps.savefig()
hists = []
hists2 = []
for tt,name in [('P', 'PSF'), ('E', 'EXP'), ('D', 'DeV')]:
I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) *
(T.flux_ivar_z > 0) *
(T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
(T.magerr_g < 0.1) * (T.magerr_r < 0.1) *(T.magerr_z < 0.1) *
(T.typex == tt))
print(len(I), 'with < 0.1-mag errs and type', name)
# plt.clf()
# H,xe,ye = loghist(T.mag_g[I] - T.mag_r[I], T.mag_r[I] - T.mag_z[I], **ha)
# plt.xlabel('g - r (mag)')
# plt.ylabel('r - z (mag)')
# plt.title('Type = %s' % name)
# ps.savefig()
plt.clf()
H,xe,ye = loghist(T.demag_g[I] - T.demag_r[I], T.demag_r[I] - T.demag_z[I], **ha)
# Keeping the dereddened color-color histograms
hists.append(H.T)
plt.xlabel('g - r (mag)')
plt.ylabel('r - z (mag)')
plt.title('Type = %s, dereddened' % name)
ps.savefig()
I2 = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) *
(T.flux_ivar_z > 0) *
(T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
(T.magerr_g < 0.05) *
(T.magerr_r < 0.05) *
(T.magerr_z < 0.05) *
(T.typex == tt))
print(len(I2), 'with < 0.05-mag errs and type', name)
plt.clf()
H,xe,ye = loghist(T.demag_g[I2] - T.demag_r[I2], T.demag_r[I2] - T.demag_z[I2], **ha)
hists2.append(H.T)
plt.xlabel('g - r (mag)')
plt.ylabel('r - z (mag)')
plt.title('Type = %s, dereddened, 5%% errs' % name)
ps.savefig()
for H in hists:
H /= H.max()
#for H in hists2:
# H /= H.max()
#hists2 = [np.clip(H / np.percentile(H.ravel(), 99), 0, 1) for H in hists2]
hists2 = [np.clip(H / np.percentile(H.ravel(), 99.9), 0, 1) for H in hists2]
rgbmap = np.dstack((hists[2], hists[0], hists[1]))
plt.clf()
((xlo,xhi),(ylo,yhi)) = ha['range']
plt.imshow(rgbmap, interpolation='nearest', origin='lower', extent=[xlo,xhi,ylo,yhi])
plt.axis([0,2,0,2])
plt.xlabel('g - r (mag)')
plt.ylabel('r - z (mag)')
plt.title('Red: DeV, Blue: Exp, Green: PSF')
ps.savefig()
plt.clf()
plt.imshow(np.sqrt(rgbmap), interpolation='nearest', origin='lower', extent=[xlo,xhi,ylo,yhi])
plt.axis([0,2,0,2])
plt.xlabel('g - r (mag)')
plt.ylabel('r - z (mag)')
plt.title('Red: DeV, Blue: Exp, Green: PSF')
ps.savefig()
# For a white-background plot, mix colors like paint
for hr,hg,hb in [(hists[2], hists[0], hists[1]),
(hists2[2], hists2[0], hists2[1]),]:
H,W = hr.shape
for mapping in [lambda x: x]: #, lambda x: np.sqrt(x), lambda x: x**2]:
red = np.ones((H,W,3))
red[:,:,1] = 1 - mapping(hr)
red[:,:,2] = 1 - mapping(hr)
# in matplotlib, 'green'->(0, 0.5, 0)
green = np.ones((H,W,3))
green[:,:,0] = 1 - mapping(hg)
green[:,:,1] = 1 - 0.5*mapping(hg)
green[:,:,2] = 1 - mapping(hg)
blue = np.ones((H,W,3))
blue[:,:,0] = 1 - mapping(hb)
blue[:,:,1] = 1 - mapping(hb)
plt.clf()
plt.imshow(red * green * blue, origin='lower', interpolation='nearest',
extent=[xlo,xhi,ylo,yhi], aspect='auto')
plt.xlabel('g - r (mag)')
plt.ylabel('r - z (mag)')
plt.xticks(np.arange(0, 2.1, 0.5))
plt.yticks(np.arange(0, 2.1, 0.5))
plt.axis([-0.25,2,0,2])
plt.title('Red: DeV, Blue: Exp, Green: PSF')
ps.savefig()
plt.figure(2)
plt.clf()
plt.imshow(red * green * blue, origin='lower', interpolation='nearest',
extent=[xlo,xhi,ylo,yhi], aspect='auto')
plt.xlabel('g - r (mag)')
plt.ylabel('r - z (mag)')
plt.xticks(np.arange(0, 2.1, 0.5))
plt.yticks(np.arange(0, 2.1, 0.5))
plt.axis([0,2,-0.25,2])
plt.text(0.7, 0.1, 'PSF', ha='center', va='center', color='k')
plt.text(0.6, 0.7, 'EXP', ha='center', va='center', color='k')
plt.text(1.5, 0.6, 'DEV', ha='center', va='center', color='k')
plt.savefig('color-type.pdf')
plt.figure(1)
sys.exit(0)
np.random.seed(42)
T.gr = T.mag_g - T.mag_r
T.rz = T.mag_r - T.mag_z
for tt,name in [('P', 'PSF'), ('E', 'EXP'), ('D', 'DeV')]:
I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) *
(T.flux_ivar_z > 0) *
(T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
(T.magerr_g < 0.1) * (T.magerr_r < 0.1) *(T.magerr_z < 0.1) *
# Bright cut
(T.mag_r > 16.) *
# Contamination cut
(T.decam_fracflux[:,2] < 0.5) *
# Mag range cut
(T.mag_r > 19.) *
(T.mag_r < 20.) *
(T.typex == tt))
print(len(I), 'with < 0.1-mag errs and type', name)
# plt.clf()
# ha = dict(histtype='step', range=(16, 24), bins=100)
# plt.hist(T.mag_g[I], color='g', **ha)
# plt.hist(T.mag_r[I], color='r', **ha)
# plt.hist(T.mag_z[I], color='m', **ha)
# plt.title('Type %s' % name)
# ps.savefig()
J = np.random.permutation(I)
if tt == 'D':
J = J[T.shapedev_r[J] < 3.]
J = J[:100]
rzbin = np.argsort(T.rz[J])
rzblock = (rzbin / 10).astype(int)
K = np.lexsort((T.gr[J], rzblock))
#print('sorted rzblock', rzblock[K])
#print('sorted rzbin', rzbin[K])
J = J[K]
# plt.clf()
# plt.hist(T.decam_fracflux[J,2], label='fracflux_r', histtype='step', bins=20)
# plt.hist(T.decam_fracmasked[J,2], label='fracmasked_r', histtype='step', bins=20)
# plt.legend()
# plt.title('Type %s' % name)
# ps.savefig()
#print('fracflux r', T.decam_fracflux[J,2])
#print('fracmasked r', T.decam_fracmasked[J,2])
#print('anymasked r', T.decam_anymask[J,2])
imgs = []
imgrow = []
stackrows = []
for i in J:
fn = 'cutouts/cutout_%.4f_%.4f.jpg' % (T.ra[i], T.dec[i])
if not os.path.exists(fn):
url = 'http://legacysurvey.org/viewer/jpeg-cutout/?layer=decals-dr3&pixscale=0.262&size=25&ra=%f&dec=%f' % (T.ra[i], T.dec[i])
print('URL', url)
cmd = 'wget -O %s.tmp "%s" && mv %s.tmp %s' % (fn, url, fn, fn)
os.system(cmd)
img = plt.imread(fn)
#print('Image', img.shape)
extra = ''
if tt == 'D':
extra = ', shapedev_r %.2f' % T.shapedev_r[i]
elif tt == 'E':
extra = ', shapeexp_r %.2f' % T.shapeexp_r[i]
print(name, 'RA,Dec %.4f,%.4f, g/r/z %.2f, %.2f, %.2f, fracflux_r %.3f, fracmasked_r %.3f, anymasked_r %4i%s' % (
T.ra[i], T.dec[i], T.mag_g[i], T.mag_r[i], T.mag_z[i],
T.decam_fracflux[i,2], T.decam_fracmasked[i,2], T.decam_anymask[i,2], extra))
imgrow.append(img)
if len(imgrow) == 10:
stack = np.hstack(imgrow)
print('stacked row:', stack.shape)
stackrows.append(stack)
imgs.append(imgrow)
imgrow = []
stack = np.vstack(stackrows)
print('Stack', stack.shape)
plt.clf()
plt.imshow(stack, interpolation='nearest', origin='lower')
plt.xticks([])
plt.yticks([])
plt.title('Type %s' % name)
ps.savefig()
# Wrap-around at RA=300
# rax = T.ra + (-360 * T.ra > 300.)
#
# rabins = np.arange(-60., 300., 0.2)
# decbins = np.arange(-20, 35, 0.2)
#
# print('RA bins:', len(rabins))
# print('Dec bins:', len(decbins))
#
# for name,cut in [(None, 'All sources'),
# (T.type == 'PSF '), 'PSF'),
# (T.type == 'SIMP'), 'SIMP'),
# (T.type == 'DEV '), 'DEV'),
# (T.type == 'EXP '), 'EXP'),
# (T.type == 'COMP'), 'COMP'),
# ]:
#
# H,xe,ye = np.histogram2d(T.ra, T.dec, bins=(rabins, decbins))
# print('H shape', H.shape)
#
# H = H.T
#
# H /= (rabins[1:] - rabins[:-1])[np.newaxis, :]
# H /= ((decbins[1:] - decbins[:-1]) * np.cos(np.deg2rad((decbins[1:] + decbins[:-1]) / 2.))))[:, np.newaxis]
#
# plt.clf()
# plt.imshow(H, extent=(rabins[0], rabins[-1], decbins[0], decbins[-1]), interpolation='nearest', origin='lower',
# vmin=0)
# cbar = plt.colorbar()
# cbar.set_label('Source density per square degree')
# plt.xlim(rabins[-1], rabins[0])
# plt.xlabel('RA (deg)')
# plt.ylabel('Dec (deg)')
# ps.savefig()
'''
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--name1', help='Name for first data set')
parser.add_argument('--name2', help='Name for second data set')
parser.add_argument('--plot-prefix',
default='compare',
help='Prefix for plot filenames; default "%default"')
parser.add_argument('--match',
default=1.0,
help='Astrometric cross-match distance in arcsec')
parser.add_argument('dir1', help='First directory to compare')
parser.add_argument('dir2', help='Second directory to compare')
opt = parser.parse_args()
ps = PlotSequence(opt.plot_prefix)
name1 = opt.name1
if name1 is None:
name1 = os.path.basename(opt.dir1)
if not len(name1):
name1 = os.path.basename(os.path.dirname(opt.dir1))
name2 = opt.name2
if name2 is None:
name2 = os.path.basename(opt.dir2)
if not len(name2):
name2 = os.path.basename(os.path.dirname(opt.dir2))
tt = 'Comparing %s to %s' % (name1, name2)
# regex for tractor-*.fits catalog filename
catre = re.compile('tractor-.*.fits')
cat1, cat2 = [], []
for basedir, cat in [(opt.dir1, cat1), (opt.dir2, cat2)]:
for dirpath, dirnames, filenames in os.walk(basedir, followlinks=True):
for fn in filenames:
if not catre.match(fn):
print('Skipping', fn, 'due to filename')
continue
fn = os.path.join(dirpath, fn)
t = fits_table(fn)
print(len(t), 'from', fn)
cat.append(t)
cat1 = merge_tables(cat1, columns='fillzero')
cat2 = merge_tables(cat2, columns='fillzero')
print('Total of', len(cat1), 'from', name1)
print('Total of', len(cat2), 'from', name2)
cat1.cut(cat1.brick_primary)
cat2.cut(cat2.brick_primary)
print('Total of', len(cat1), 'BRICK_PRIMARY from', name1)
print('Total of', len(cat2), 'BRICK_PRIMARY from', name2)
cat1.cut((cat1.decam_anymask[:, 1] == 0) *
(cat1.decam_anymask[:, 2] == 0) * (cat1.decam_anymask[:, 4] == 0))
cat2.cut((cat2.decam_anymask[:, 1] == 0) *
(cat2.decam_anymask[:, 2] == 0) * (cat2.decam_anymask[:, 4] == 0))
print('Total of', len(cat1), 'unmasked from', name1)
print('Total of', len(cat2), 'unmasked from', name2)
I, J, d = match_radec(cat1.ra,
cat1.dec,
cat2.ra,
cat2.dec,
opt.match / 3600.,
nearest=True)
print(len(I), 'matched')
plt.clf()
plt.hist(d * 3600., 100)
plt.xlabel('Match distance (arcsec)')
plt.title(tt)
ps.savefig()
matched1 = cat1[I]
matched2 = cat2[J]
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
K = np.flatnonzero((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0))
print('Median mw_trans', band, 'is',
np.median(matched1.decam_mw_transmission[:, iband]))
plt.clf()
plt.errorbar(
matched1.decam_flux[K, iband],
matched2.decam_flux[K, iband],
fmt='.',
color=cc,
xerr=1. / np.sqrt(matched1.decam_flux_ivar[K, iband]),
yerr=1. / np.sqrt(matched2.decam_flux_ivar[K, iband]),
alpha=0.1,
)
plt.xlabel('%s flux: %s' % (name1, band))
plt.ylabel('%s flux: %s' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.)
plt.axis([-100, 1000, -100, 1000])
plt.title(tt)
ps.savefig()
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
good = ((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0))
K = np.flatnonzero(good)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
P = np.flatnonzero(good * psf1 * psf2)
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1. / iv1 + 1. / iv2)
plt.clf()
plt.plot(
mag1[K],
(matched2.decam_flux[K, iband] - matched1.decam_flux[K, iband]) /
std[K],
'.',
alpha=0.1,
color=cc)
plt.plot(
mag1[P],
(matched2.decam_flux[P, iband] - matched1.decam_flux[P, iband]) /
std[P],
'.',
alpha=0.1,
color='k')
plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' %
(name2, name1, band))
plt.xlabel('%s mag: %s' % (name1, band))
plt.axhline(0, color='k', alpha=0.5)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
plt.clf()
lp, lt = [], []
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
good = ((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0))
#good = True
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1. / iv1 + 1. / iv2)
#std = np.hypot(std, 0.01)
G = np.flatnonzero(good * psf1 * psf2 * np.isfinite(mag1) *
(mag1 >= 20) *
(mag1 < dict(g=24, r=23.5, z=22.5)[band]))
n, b, p = plt.hist(
(matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) /
std[G],
range=(-4, 4),
bins=50,
histtype='step',
color=cc,
normed=True)
sig = (matched2.decam_flux[G, iband] -
matched1.decam_flux[G, iband]) / std[G]
print('Raw mean and std of points:', np.mean(sig), np.std(sig))
med = np.median(sig)
rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2.
print('Median and percentile-based sigma:', med, rsigma)
lp.append(p[0])
lt.append('%s: %.2f +- %.2f' % (band, med, rsigma))
bins = []
gaussint = []
for blo, bhi in zip(b, b[1:]):
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
#bins.extend([blo,bhi])
#gaussint.extend([c,c])
bins.append((blo + bhi) / 2.)
gaussint.append(c)
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.title(tt)
plt.xlabel('Flux difference / error (sigma)')
plt.axvline(0, color='k', alpha=0.1)
plt.ylim(0, 0.45)
plt.legend(lp, lt, loc='upper right')
ps.savefig()
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
plt.clf()
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors(
matched2.decam_flux[:, iband], matched2.decam_flux_ivar[:, iband])
meanmag = NanoMaggies.nanomaggiesToMag(
(matched1.decam_flux[:, iband] + matched2.decam_flux[:, iband]) /
2.)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
good = ((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0) * np.isfinite(mag1) *
np.isfinite(mag2))
K = np.flatnonzero(good)
P = np.flatnonzero(good * psf1 * psf2)
plt.errorbar(mag1[K],
mag2[K],
fmt='.',
color=cc,
xerr=magerr1[K],
yerr=magerr2[K],
alpha=0.1)
plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s (mag)' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.)
plt.axis([24, 16, 24, 16])
plt.title(tt)
ps.savefig()
plt.clf()
plt.errorbar(mag1[K],
mag2[K] - mag1[K],
fmt='.',
color=cc,
xerr=magerr1[K],
yerr=magerr2[K],
alpha=0.1)
plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -1, 1])
plt.title(tt)
ps.savefig()
magbins = np.arange(16, 24.001, 0.5)
plt.clf()
plt.plot(mag1[K],
(mag2[K] - mag1[K]) / np.hypot(magerr1[K], magerr2[K]),
'.',
color=cc,
alpha=0.1)
plt.plot(mag1[P],
(mag2[P] - mag1[P]) / np.hypot(magerr1[P], magerr2[P]),
'k.',
alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
plt.clf()
plt.plot(meanmag[P], y[P], 'k.', alpha=0.1)
midmag = []
vals = np.zeros((len(magbins) - 1, 5))
median_err1 = []
iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25)
# FIXME -- should we do some stats after taking off the mean difference?
for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
midmag.append((mlo + mhi) / 2.)
median_err1.append(np.median(magerr1[I]))
if len(I) == 0:
continue
# median and +- 1 sigma quantiles
ybin = y[I]
vals[bini, 0] = np.percentile(ybin, 16)
vals[bini, 1] = np.median(ybin)
vals[bini, 2] = np.percentile(ybin, 84)
# +- 2 sigma quantiles
vals[bini, 3] = np.percentile(ybin, 2.3)
vals[bini, 4] = np.percentile(ybin, 97.7)
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss,
'vs expected for Gaussian;', len(ybin), 'points')
# if iqd > iqd_gauss:
# # What error adding in quadrature would you need to make the IQD match?
# err = median_err1[-1]
# target_err = err * (iqd / iqd_gauss)
# sys_err = np.sqrt(target_err**2 - err**2)
# print('--> add systematic error', sys_err)
# ~ Johan's cuts
mlo = 21.
mhi = dict(g=24., r=23.5, z=22.5)[band]
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
ybin = y[I]
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', mlo, mhi, 'band', band,
': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;',
len(ybin), 'points')
if iqd > iqd_gauss:
# What error adding in quadrature would you need to make
# the IQD match?
err = np.median(np.hypot(magerr1[I], magerr2[I]))
print('Median error (hypot):', err)
target_err = err * (iqd / iqd_gauss)
print('Target:', target_err)
sys_err = np.sqrt((target_err**2 - err**2) / 2.)
print('--> add systematic error', sys_err)
# check...
err_sys = np.hypot(np.hypot(magerr1, sys_err),
np.hypot(magerr2, sys_err))
ysys = (mag2 - mag1) / err_sys
ysys = ysys[I]
print('Resulting median error:', np.median(err_sys[I]))
iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25)
print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian')
# Hmmm, this doesn't work... totally overshoots.
plt.errorbar(midmag,
vals[:, 1],
fmt='o',
color='b',
yerr=(vals[:, 1] - vals[:, 0], vals[:, 2] - vals[:, 1]),
capthick=3,
zorder=20)
plt.errorbar(midmag,
vals[:, 1],
fmt='o',
color='b',
yerr=(vals[:, 1] - vals[:, 3], vals[:, 4] - vals[:, 1]),
capthick=2,
zorder=20)
plt.axhline(1., color='b', alpha=0.2)
plt.axhline(-1., color='b', alpha=0.2)
plt.axhline(2., color='b', alpha=0.2)
plt.axhline(-2., color='b', alpha=0.2)
for mag, err, y in zip(midmag, median_err1, vals[:, 3]):
if not np.isfinite(err):
continue
if y < -6:
continue
plt.text(mag,
y - 0.1,
'%.3f' % err,
va='top',
ha='center',
color='k',
fontsize=10)
plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axvline(21, color='k', alpha=0.3)
plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3)
plt.axis([24.1, 16, -6, 6])
plt.title(tt)
ps.savefig()
#magbins = np.append([16, 18], np.arange(20, 24.001, 0.5))
if band == 'g':
magbins = [20, 24]
elif band == 'r':
magbins = [20, 23.5]
elif band == 'z':
magbins = [20, 22.5]
slo, shi = -5, 5
plt.clf()
ha = dict(bins=25, range=(slo, shi), histtype='step', normed=True)
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
midmag = []
nn = []
rgbs = []
lt, lp = [], []
for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)]
if len(I) == 0:
continue
ybin = y[I]
rgb = [0., 0., 0.]
rgb[0] = float(bini) / (len(magbins) - 1)
rgb[2] = 1. - rgb[0]
n, b, p = plt.hist(ybin, color=rgb, **ha)
lt.append('mag %g to %g' % (mlo, mhi))
lp.append(p[0])
midmag.append((mlo + mhi) / 2.)
nn.append(n)
rgbs.append(rgb)
bins = []
gaussint = []
for blo, bhi in zip(b, b[1:]):
#midbin.append((blo+bhi)/2.)
#gaussint.append(scipy.stats.norm.cdf(bhi) -
# scipy.stats.norm.cdf(blo))
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
bins.extend([blo, bhi])
gaussint.extend([c, c])
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo, shi)
ps.savefig()
bincenters = b[:-1] + (b[1] - b[0]) / 2.
plt.clf()
lp = []
for n, rgb, mlo, mhi in zip(nn, rgbs, magbins, magbins[1:]):
p = plt.plot(bincenters, n, '-', color=rgb)
lp.append(p[0])
plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo, shi)
ps.savefig()
def plot_light_curves(pfn, ucal=False):
lightcurves = unpickle_from_file(pfn)
if ucal:
tag = 'ucal-'
else:
tag = ''
survey = LegacySurveyData()
brickname = '0364m042'
catfn = survey.find_file('tractor', brick=brickname)
print('Reading catalog from', catfn)
cat = fits_table(catfn)
print(len(cat), 'catalog entries')
cat.cut(cat.brick_primary)
print(len(cat), 'brick primary')
I = []
for i, oid in enumerate(cat.objid):
if (brickname, oid) in lightcurves:
I.append(i)
I = np.array(I)
cat.cut(I)
print('Cut to', len(cat), 'with light curves')
S = fits_table('specObj-dr12-trim-2.fits')
from astrometry.libkd.spherematch import match_radec
I, J, d = match_radec(S.ra, S.dec, cat.ra, cat.dec, 2. / 3600.)
print('Matched', len(I), 'to spectra')
plt.subplots_adjust(hspace=0)
movie_jpegs = []
movie_wcs = None
for i in range(28):
fn = os.path.join('des-sn-movie', 'epoch%i' % i, 'coadd',
brickname[:3], brickname,
'legacysurvey-%s-image.jpg' % brickname)
print(fn)
if not os.path.exists(fn):
continue
img = plt.imread(fn)
img = np.flipud(img)
h, w, d = img.shape
fn = os.path.join('des-sn-movie', 'epoch%i' % i, 'coadd',
brickname[:3], brickname,
'legacysurvey-%s-image-r.fits' % brickname)
if not os.path.exists(fn):
continue
wcs = Tan(fn)
movie_jpegs.append(img)
movie_wcs = wcs
plt.figure(figsize=(8, 6), dpi=100)
n = 0
fluxtags = [('flux', 'flux_ivar', '', 'a')]
if ucal:
fluxtags.append(('uflux', 'uflux_ivar', ': ucal', 'b'))
for oid, ii in zip(cat.objid[J], I):
print('Objid', oid)
spec = S[ii]
k = (brickname, oid)
v = lightcurves[k]
# Cut bad CCDs
v.cut(np.array([e not in [230151, 230152, 230153] for e in v.expnum]))
plt.clf()
print('obj', k, 'has', len(v), 'measurements')
T = v
for fluxtag, fluxivtag, fluxname, plottag in fluxtags:
plt.clf()
filts = np.unique(T.filter)
for i, f in enumerate(filts):
from tractor.brightness import NanoMaggies
plt.subplot(len(filts), 1, i + 1)
fluxes = np.hstack(
[T.get(ft[0])[T.filter == f] for ft in fluxtags])
fluxes = fluxes[np.isfinite(fluxes)]
mn, mx = np.percentile(fluxes, [5, 95])
print('Flux percentiles for filter', f, ':', mn, mx)
# note swap
mn, mx = NanoMaggies.nanomaggiesToMag(
mx), NanoMaggies.nanomaggiesToMag(mn)
print('-> mags', mn, mx)
cut = (T.filter == f) * (T.flux_ivar > 0)
if ucal:
cut *= np.isfinite(T.uflux)
I = np.flatnonzero(cut)
print(' ', len(I), 'in', f, 'band')
I = I[np.argsort(T.mjd[I])]
mediv = np.median(T.flux_ivar[I])
# cut really noisy ones
I = I[T.flux_ivar[I] > 0.25 * mediv]
#plt.plot(T.mjd[I], T.flux[I], '.-', color=dict(g='g',r='r',z='m')[f])
# plt.errorbar(T.mjd[I], T.flux[I], yerr=1/np.sqrt(T.fluxiv[I]),
# fmt='.-', color=dict(g='g',r='r',z='m')[f])
#plt.errorbar(T.mjd[I], T.flux[I], yerr=1/np.sqrt(T.fluxiv[I]),
# fmt='.', color=dict(g='g',r='r',z='m')[f])
# if ucal:
# mag,dmag = NanoMaggies.fluxErrorsToMagErrors(T.flux[I], T.flux_ivar[I])
# else:
# mag,dmag = NanoMaggies.fluxErrorsToMagErrors(T.uflux[I], T.uflux_ivar[I])
mag, dmag = NanoMaggies.fluxErrorsToMagErrors(
T.get(fluxtag)[I],
T.get(fluxivtag)[I])
plt.errorbar(T.mjd[I],
mag,
yerr=dmag,
fmt='.',
color=dict(g='g', r='r', z='m')[f])
#yl,yh = plt.ylim()
#plt.ylim(yh,yl)
plt.ylim(mx, mn)
plt.ylabel(f)
if i + 1 < len(filts):
plt.xticks([])
#plt.yscale('symlog')
outfn = 'cutout_%.4f_%.4f.jpg' % (spec.ra, spec.dec)
if not os.path.exists(outfn):
url = 'http://legacysurvey.org/viewer/jpeg-cutout/?ra=%.4f&dec=%.4f&zoom=14&layer=sdssco&size=128' % (
spec.ra, spec.dec)
cmd = 'wget -O %s "%s"' % (outfn, url)
print(cmd)
os.system(cmd)
pix = plt.imread(outfn)
h, w, d = pix.shape
fig = plt.gcf()
#print('fig bbox:', fig.bbox)
#print('xmax, ymax', fig.bbox.xmax, fig.bbox.ymax)
#plt.figimage(pix, 0, fig.bbox.ymax - h, zorder=10)
#plt.figimage(pix, 0, fig.bbox.ymax, zorder=10)
#plt.figimage(pix, fig.bbox.xmax - w, fig.bbox.ymax, zorder=10)
plt.figimage(pix,
fig.bbox.xmax - (w + 2),
fig.bbox.ymax - (h + 2),
zorder=10)
plt.suptitle('SDSS spectro object: %s at (%.4f, %.4f)%s' %
(spec.label.strip(), spec.ra, spec.dec, fluxname))
plt.savefig('forced-%s%i-%s.png' % (tag, n, plottag))
ok, x, y = movie_wcs.radec2pixelxy(spec.ra, spec.dec)
x = int(np.round(x - 1))
y = int(np.round(y - 1))
sz = 32
plt.clf()
plt.subplots_adjust(hspace=0, wspace=0)
k = 1
for i, img in enumerate(movie_jpegs):
stamp = img[y - sz:y + sz + 1, x - sz:x + sz + 1]
plt.subplot(5, 6, k)
plt.imshow(stamp, interpolation='nearest', origin='lower')
plt.xticks([])
plt.yticks([])
k += 1
plt.suptitle('SDSS spectro object: %s at (%.4f, %.4f): DES images' %
(spec.label.strip(), spec.ra, spec.dec))
plt.savefig('forced-%s%i-c.png' % (tag, n))
n += 1
def all(matched1, matched2, d, name1='ref', name2='test'):
tt = 'Comparing %s to %s' % (name1, name2)
plt.clf()
plt.hist(d * 3600., 100)
plt.xlabel('Match distance (arcsec)')
plt.title(tt)
plt.savefig(os.path.join(matched1.outdir, 'sep_hist.png'))
plt.close()
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
K = np.flatnonzero((matched1.t['decam_flux_ivar'][:, iband] > 0) *
(matched2.t['decam_flux_ivar'][:, iband] > 0))
print('Median mw_trans', band, 'is',
np.median(matched1.t['decam_mw_transmission'][:, iband]))
plt.clf()
plt.errorbar(
matched1.t['decam_flux'][K, iband],
matched2.t['decam_flux'][K, iband],
fmt='.',
color=cc,
xerr=1. / np.sqrt(matched1.t['decam_flux_ivar'][K, iband]),
yerr=1. / np.sqrt(matched2.t['decam_flux_ivar'][K, iband]),
alpha=0.1,
)
plt.xlabel('%s flux: %s' % (name1, band))
plt.ylabel('%s flux: %s' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.)
plt.axis([-100, 1000, -100, 1000])
plt.title(tt)
plt.savefig(os.path.join(matched1.outdir, '%s_fluxerr.png' % band))
plt.close()
print("exiting early")
sys.exit()
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
good = ((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0))
K = np.flatnonzero(good)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
P = np.flatnonzero(good * psf1 * psf2)
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1. / iv1 + 1. / iv2)
plt.clf()
plt.plot(
mag1[K],
(matched2.decam_flux[K, iband] - matched1.decam_flux[K, iband]) /
std[K],
'.',
alpha=0.1,
color=cc)
plt.plot(
mag1[P],
(matched2.decam_flux[P, iband] - matched1.decam_flux[P, iband]) /
std[P],
'.',
alpha=0.1,
color='k')
plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' %
(name2, name1, band))
plt.xlabel('%s mag: %s' % (name1, band))
plt.axhline(0, color='k', alpha=0.5)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
plt.clf()
lp, lt = [], []
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
good = ((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0))
#good = True
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1. / iv1 + 1. / iv2)
#std = np.hypot(std, 0.01)
G = np.flatnonzero(good * psf1 * psf2 * np.isfinite(mag1) *
(mag1 >= 20) *
(mag1 < dict(g=24, r=23.5, z=22.5)[band]))
n, b, p = plt.hist(
(matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) /
std[G],
range=(-4, 4),
bins=50,
histtype='step',
color=cc,
normed=True)
sig = (matched2.decam_flux[G, iband] -
matched1.decam_flux[G, iband]) / std[G]
print('Raw mean and std of points:', np.mean(sig), np.std(sig))
med = np.median(sig)
rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2.
print('Median and percentile-based sigma:', med, rsigma)
lp.append(p[0])
lt.append('%s: %.2f +- %.2f' % (band, med, rsigma))
bins = []
gaussint = []
for blo, bhi in zip(b, b[1:]):
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
#bins.extend([blo,bhi])
#gaussint.extend([c,c])
bins.append((blo + bhi) / 2.)
gaussint.append(c)
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.title(tt)
plt.xlabel('Flux difference / error (sigma)')
plt.axvline(0, color='k', alpha=0.1)
plt.ylim(0, 0.45)
plt.legend(lp, lt, loc='upper right')
ps.savefig()
for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
plt.clf()
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors(
matched2.decam_flux[:, iband], matched2.decam_flux_ivar[:, iband])
meanmag = NanoMaggies.nanomaggiesToMag(
(matched1.decam_flux[:, iband] + matched2.decam_flux[:, iband]) /
2.)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
good = ((matched1.decam_flux_ivar[:, iband] > 0) *
(matched2.decam_flux_ivar[:, iband] > 0) * np.isfinite(mag1) *
np.isfinite(mag2))
K = np.flatnonzero(good)
P = np.flatnonzero(good * psf1 * psf2)
plt.errorbar(mag1[K],
mag2[K],
fmt='.',
color=cc,
xerr=magerr1[K],
yerr=magerr2[K],
alpha=0.1)
plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s (mag)' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.)
plt.axis([24, 16, 24, 16])
plt.title(tt)
ps.savefig()
plt.clf()
plt.errorbar(mag1[K],
mag2[K] - mag1[K],
fmt='.',
color=cc,
xerr=magerr1[K],
yerr=magerr2[K],
alpha=0.1)
plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -1, 1])
plt.title(tt)
ps.savefig()
magbins = np.arange(16, 24.001, 0.5)
plt.clf()
plt.plot(mag1[K],
(mag2[K] - mag1[K]) / np.hypot(magerr1[K], magerr2[K]),
'.',
color=cc,
alpha=0.1)
plt.plot(mag1[P],
(mag2[P] - mag1[P]) / np.hypot(magerr1[P], magerr2[P]),
'k.',
alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
plt.clf()
plt.plot(meanmag[P], y[P], 'k.', alpha=0.1)
midmag = []
vals = np.zeros((len(magbins) - 1, 5))
median_err1 = []
iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25)
# FIXME -- should we do some stats after taking off the mean difference?
for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
midmag.append((mlo + mhi) / 2.)
median_err1.append(np.median(magerr1[I]))
if len(I) == 0:
continue
# median and +- 1 sigma quantiles
ybin = y[I]
vals[bini, 0] = np.percentile(ybin, 16)
vals[bini, 1] = np.median(ybin)
vals[bini, 2] = np.percentile(ybin, 84)
# +- 2 sigma quantiles
vals[bini, 3] = np.percentile(ybin, 2.3)
vals[bini, 4] = np.percentile(ybin, 97.7)
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss,
'vs expected for Gaussian;', len(ybin), 'points')
# if iqd > iqd_gauss:
# # What error adding in quadrature would you need to make the IQD match?
# err = median_err1[-1]
# target_err = err * (iqd / iqd_gauss)
# sys_err = np.sqrt(target_err**2 - err**2)
# print('--> add systematic error', sys_err)
# ~ Johan's cuts
mlo = 21.
mhi = dict(g=24., r=23.5, z=22.5)[band]
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
ybin = y[I]
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', mlo, mhi, 'band', band,
': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;',
len(ybin), 'points')
if iqd > iqd_gauss:
# What error adding in quadrature would you need to make
# the IQD match?
err = np.median(np.hypot(magerr1[I], magerr2[I]))
print('Median error (hypot):', err)
target_err = err * (iqd / iqd_gauss)
print('Target:', target_err)
sys_err = np.sqrt((target_err**2 - err**2) / 2.)
print('--> add systematic error', sys_err)
# check...
err_sys = np.hypot(np.hypot(magerr1, sys_err),
np.hypot(magerr2, sys_err))
ysys = (mag2 - mag1) / err_sys
ysys = ysys[I]
print('Resulting median error:', np.median(err_sys[I]))
iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25)
print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian')
# Hmmm, this doesn't work... totally overshoots.
plt.errorbar(midmag,
vals[:, 1],
fmt='o',
color='b',
yerr=(vals[:, 1] - vals[:, 0], vals[:, 2] - vals[:, 1]),
capthick=3,
zorder=20)
plt.errorbar(midmag,
vals[:, 1],
fmt='o',
color='b',
yerr=(vals[:, 1] - vals[:, 3], vals[:, 4] - vals[:, 1]),
capthick=2,
zorder=20)
plt.axhline(1., color='b', alpha=0.2)
plt.axhline(-1., color='b', alpha=0.2)
plt.axhline(2., color='b', alpha=0.2)
plt.axhline(-2., color='b', alpha=0.2)
for mag, err, y in zip(midmag, median_err1, vals[:, 3]):
if not np.isfinite(err):
continue
if y < -6:
continue
plt.text(mag,
y - 0.1,
'%.3f' % err,
va='top',
ha='center',
color='k',
fontsize=10)
plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axvline(21, color='k', alpha=0.3)
plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3)
plt.axis([24.1, 16, -6, 6])
plt.title(tt)
ps.savefig()
#magbins = np.append([16, 18], np.arange(20, 24.001, 0.5))
if band == 'g':
magbins = [20, 24]
elif band == 'r':
magbins = [20, 23.5]
elif band == 'z':
magbins = [20, 22.5]
slo, shi = -5, 5
plt.clf()
ha = dict(bins=25, range=(slo, shi), histtype='step', normed=True)
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
midmag = []
nn = []
rgbs = []
lt, lp = [], []
for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)]
if len(I) == 0:
continue
ybin = y[I]
rgb = [0., 0., 0.]
rgb[0] = float(bini) / (len(magbins) - 1)
rgb[2] = 1. - rgb[0]
n, b, p = plt.hist(ybin, color=rgb, **ha)
lt.append('mag %g to %g' % (mlo, mhi))
lp.append(p[0])
midmag.append((mlo + mhi) / 2.)
nn.append(n)
rgbs.append(rgb)
bins = []
gaussint = []
for blo, bhi in zip(b, b[1:]):
#midbin.append((blo+bhi)/2.)
#gaussint.append(scipy.stats.norm.cdf(bhi) -
# scipy.stats.norm.cdf(blo))
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
bins.extend([blo, bhi])
gaussint.extend([c, c])
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo, shi)
ps.savefig()
bincenters = b[:-1] + (b[1] - b[0]) / 2.
plt.clf()
lp = []
for n, rgb, mlo, mhi in zip(nn, rgbs, magbins, magbins[1:]):
p = plt.plot(bincenters, n, '-', color=rgb)
lp.append(p[0])
plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo, shi)
ps.savefig()
def compare_mags(TT, name, ps):
for i,T in enumerate(TT):
T.set('exp', np.zeros(len(T), np.uint8)+i)
plt.clf()
ap = 5
for i,T in enumerate(TT):
cc = 'rgb'[i]
plt.plot(T.flux, T.apflux[:, ap] / T.flux,
'.', color=cc, alpha=0.5)
ff, frac = [],[]
mags = np.arange(14, 24)
for mlo,mhi in zip(mags, mags[1:]):
flo = NanoMaggies.magToNanomaggies(mhi)
fhi = NanoMaggies.magToNanomaggies(mlo)
I = np.flatnonzero((T.flux > flo) * (T.flux <= fhi))
ff.append(np.sqrt(flo * fhi))
frac.append(np.median(T.apflux[I,ap] / T.flux[I]))
plt.plot(ff, frac, 'o-', color=cc)
plt.xscale('symlog')
plt.xlim(1., 1e3)
plt.ylim(0.9, 1.1)
plt.xlabel('Forced-phot flux')
plt.ylabel('Aperture / Forced-phot flux')
plt.axhline(1, color='k', alpha=0.1)
plt.title('%s region: Aperture %i fluxes' % (name, ap))
ps.savefig()
T = merge_tables(TT)
T.bobjid = T.brickid.astype(int) * 10000 + T.objid
bomap = {}
for i,bo in enumerate(T.bobjid):
try:
bomap[bo].append(i)
except KeyError:
bomap[bo] = [i]
II = []
for bo,ii in bomap.items():
if len(ii) != 3:
continue
II.append(ii)
II = np.array(II)
print 'II', II.shape
exps = T.exp[II]
print 'exposures:', exps
assert(np.all(T.exp[II[:,0]] == 0))
assert(np.all(T.exp[II[:,1]] == 1))
assert(np.all(T.exp[II[:,2]] == 2))
fluxes = T.flux[II]
print 'fluxes', fluxes.shape
meanflux = np.mean(fluxes, axis=1)
print 'meanfluxes', meanflux.shape
plt.clf()
for i in range(3):
plt.plot(meanflux, fluxes[:,i] / meanflux, '.',
color='rgb'[i], alpha=0.5)
#plt.yscale('symlog')
plt.xscale('symlog')
plt.xlabel('Mean flux (nanomaggies)')
plt.ylabel('Forced-phot flux / Mean')
#plt.ylim(0, 2)
plt.ylim(0.9, 1.1)
plt.xlim(0, 1e3)
plt.axhline(1, color='k', alpha=0.1)
plt.title('%s region: Forced-phot fluxes' % name)
ps.savefig()
for ap in [4,5,6]:
apfluxes = T.apflux[:,ap][II,]
print 'ap fluxes', apfluxes.shape
plt.clf()
for i in range(3):
plt.plot(meanflux, apfluxes[:,i] / meanflux, '.',
color='rgb'[i], alpha=0.5)
plt.xscale('symlog')
plt.xlabel('Mean flux (nanomaggies)')
plt.ylabel('Aperture(%i) flux / Mean' % ap)
plt.ylim(0.9, 1.1)
plt.xlim(0, 1e3)
plt.axhline(1, color='k', alpha=0.1)
plt.title('%s region: Aperture %i fluxes' % (name, ap))
ps.savefig()
plt.clf()
for i in range(3):
plt.plot(fluxes[:,i], apfluxes[:,i] / fluxes[:,i], '.',
color='rgb'[i], alpha=0.5)
plt.xscale('symlog')
plt.xlim(0, 1e3)
plt.ylim(0.9, 1.1)
plt.xlabel('Forced-phot flux')
plt.ylabel('Aperture / Forced-phot flux')
plt.axhline(1, color='k', alpha=0.1)
plt.title('%s region: Aperture %i fluxes' % (name, ap))
ps.savefig()
def compare_to_ps1(ps, ccds):
decals = Decals()
allplots = []
for expnum,ccdname in ccds:
ccd = decals.find_ccds(expnum=expnum, ccdname=ccdname)
assert(len(ccd) == 1)
ccd = ccd[0]
im = decals.get_image_object(ccd)
print 'Reading', im
wcs = im.get_wcs()
magrange = (15,20)
ps1 = ps1cat(ccdwcs=wcs)
ps1 = ps1.get_stars(band=im.band, magrange=magrange)
print 'Got', len(ps1), 'PS1 stars'
# ps1.about()
F = fits_table('forced-%i-%s.fits' % (expnum, ccdname))
print 'Read', len(F), 'forced-phot results'
F.ra,F.dec = wcs.pixelxy2radec(F.x+1, F.y+1)
I,J,d = match_radec(F.ra, F.dec, ps1.ra, ps1.dec, 1./3600.)
print 'Matched', len(I), 'stars to PS1'
F.cut(I)
ps1.cut(J)
F.mag = NanoMaggies.nanomaggiesToMag(F.flux)
F.apmag = NanoMaggies.nanomaggiesToMag(F.apflux[:,5])
iband = ps1cat.ps1band[im.band]
ps1mag = ps1.median[:,iband]
mags = np.arange(magrange[0], 1+magrange[1])
psf = im.read_psf_model(0, 0, pixPsf=True)
pixscale = 0.262
apertures = apertures_arcsec / pixscale
h,w = ccd.height, ccd.width
psfimg = psf.getPointSourcePatch(w/2., h/2.).patch
ph,pw = psfimg.shape
cx,cy = pw/2, ph/2
apphot = []
for rad in apertures:
aper = photutils.CircularAperture((cx,cy), rad)
p = photutils.aperture_photometry(psfimg, aper)
apphot.append(p.field('aperture_sum'))
apphot = np.hstack(apphot)
print 'aperture photometry:', apphot
skyest = apphot[6] - apphot[5]
print 'Sky estimate:', skyest
skyest /= np.pi * (apertures[6]**2 - apertures[5]**2)
print 'Sky estimate per pixel:', skyest
fraction = apphot[5] - skyest * np.pi * apertures[5]**2
print 'Fraction of flux:', fraction
zp = 2.5 * np.log10(fraction)
print 'ZP adjustment:', zp
plt.clf()
for cc,mag,label in [('b', F.mag, 'Forced mag'), ('r', F.apmag, 'Aper mag')]:
plt.plot(ps1mag, mag - ps1mag, '.', color=cc, label=label, alpha=0.6)
mm,dd = [],[]
for mlo,mhi in zip(mags, mags[1:]):
I = np.flatnonzero((ps1mag > mlo) * (ps1mag <= mhi))
mm.append((mlo+mhi)/2.)
dd.append(np.median(mag[I] - ps1mag[I]))
plt.plot(mm, dd, 'o-', color=cc)
mm = np.array(mm)
dd = np.array(dd)
plt.plot(mm, dd - zp, 'o--', lw=3, alpha=0.5, color=cc)
allplots.append((mm, dd, zp, cc, label))
plt.xlabel('PS1 %s mag' % im.band)
plt.ylabel('Mag - PS1 (mag)')
plt.title('PS1 - Single-epoch mag: %i-%s' % (expnum, ccdname))
plt.ylim(-0.2, 0.2)
mlo,mhi = magrange
plt.xlim(mhi, mlo)
plt.axhline(0., color='k', alpha=0.1)
plt.legend()
ps.savefig()
plt.clf()
# for mm,dd,zp,cc,label in allplots:
# plt.plot(mm, dd, 'o-', color=cc, label=label)
# plt.plot(mm, dd - zp, 'o--', lw=3, alpha=0.5, color=cc)
for sp,add in [(1,False),(2,True)]:
plt.subplot(2,1,sp)
for mm,dd,zp,cc,label in allplots:
if add:
plt.plot(mm, dd - zp, 'o--', lw=3, alpha=0.5, color=cc)
else:
plt.plot(mm, dd, 'o-', color=cc, label=label)
plt.ylabel('Mag - PS1 (mag)')
plt.ylim(-0.2, 0.05)
mlo,mhi = magrange
plt.xlim(mhi, mlo)
plt.axhline(0., color='k', alpha=0.1)
plt.axhline(-0.05, color='k', alpha=0.1)
plt.axhline(-0.1, color='k', alpha=0.1)
plt.xlabel('PS1 %s mag' % im.band)
plt.suptitle('PS1 - Single-epoch mags')
#plt.legend()
ps.savefig()
plt.clf()
for mm,dd,zp,cc,label in allplots:
plt.plot(mm, dd, 'o-', color=cc, label=label)
plt.plot(mm, dd - zp, 'o--', lw=3, alpha=0.5, color=cc)
plt.ylabel('Mag - PS1 (mag)')
plt.ylim(-0.2, 0.05)
mlo,mhi = magrange
plt.xlim(mhi, mlo)
plt.axhline(0., color='k', alpha=0.1)
plt.xlabel('PS1 %s mag' % im.band)
plt.suptitle('PS1 - Single-epoch mags')
ps.savefig()
import argparse
parser = argparse.ArgumentParser(description='Produce annotated CCDs file by reading CCDs file + calibration products')
parser.add_argument('--part', action='append', help='CCDs file to read, survey-ccds-X.fits.gz, default: ["decals","nondecals","extra"]. Can be repeated.', default=[])
parser.add_argument('--threads', type=int, help='Run multi-threaded', default=4)
opt = parser.parse_args()
if False:
#### FIX mistake in DR3 annotated CCDs: sig1
for part in ['decals', 'nondecals', 'extra']:
from tractor.brightness import NanoMaggies
fn = '/global/cscratch1/sd/desiproc/dr3/ccds-annotated-%s.fits.gz' % part
T = fits_table(fn)
zpt = T.ccdzpt + 2.5 * np.log10(T.exptime)
print('Median zpt:', np.median(zpt))
zpscale = NanoMaggies.zeropointToScale(zpt)
print('Median zpscale:', np.median(zpscale))
print('Zpscale limits:', zpscale.min(), zpscale.max())
print('Non-finite:', np.sum(np.logical_not(np.isfinite(zpscale))))
T.sig1 /= zpscale
T.sig1[np.logical_not(np.isfinite(T.sig1))] = 0.
print('Median sig1:', np.median(T.sig1))
print('Before:')
for col in ['psfdepth', 'galdepth', 'gausspsfdepth', 'gaussgaldepth']:
c = T.get(col)
for band in 'grz':
I = np.flatnonzero((T.filter == band) * (T.sig1 > 0))
print(col, band, 'median', np.median(c[I]))
ps.savefig()
print('Tim photocal:', tim.photocal)
print('Subtim photocal:', subtim.photocal)
# # Order by flux
I = np.argsort([-src.getBrightness().getFlux(band) for src in srcs])
for i in I:
src = srcs[i]
print('Source:', src)
print('-> counts', subtim.photocal.brightnessToCounts(src.getBrightness()))
# flux in nanomaggies
flux = np.array([src.getBrightness().getFlux(band) for src in srcs])
fluxiv = R.IV
mag, magerr = NanoMaggies.fluxErrorsToMagErrors(flux, fluxiv)
typemap = { ExpGalaxy:'E', DevGalaxy:'D', PointSource:'P' }
cat.tractor_type = np.array([typemap[type(src)] for src in srcs])
cat.set('cfht_forced_mag_%s' % band, mag)
cat.set('cfht_forced_magerr_%s' % band, magerr)
cat.writeto('cfht-forced.fits')
plt.clf()
plt.plot(cat.acs_mag_auto, mag, 'b.')
plt.xlabel('ACS I-band (mag)')
plt.ylabel('CFHT %s-band forced phot (mag)' % band)
plt.title('CFHT forced phot')
ps.savefig()
def gim2d_catalog(cat, band):
'''
http://irsa.ipac.caltech.edu/data/COSMOS/tables/morphology/cosmos_morph_zurich_colDescriptions.html
'''
'''
ACS_MU_CLASS float Type of object.
1 = galaxy
2 = star
3 = spurious
'''
'''
ACS_CLEAN float Object useable flag.
0 = do not use this object
1 = use this object
'''
'''
FLUX_GIM2D float counts GIM2D total flux
R_GIM2D float arcseconds GIM2D psf-convolved half-light radius of object
ELL_GIM2D float GIM2D ellipticity = 1-b/a of object
PA_GIM2D float degrees GIM2D position angle of object - cw from +y-axis
DX_GIM2D float arcseconds x-offset of GIM2D-model center from ACS-coordinate center
DY_GIM2D float arcseconds y-offset of GIM2D-model center from ACS-coordinate center
SERSIC_N_GIM2D float GIM2D Sersic index
R_0P5_GIM2D float arcseconds GIM2D half-light radius of object without PSF convolution
TYPE float ZEST Type CLASS
1 = Early type
2 = Disk
3 = Irregular Galaxy
9 = no classification
'''
'''
BULG float ZEST "Bulgeness" CLASS - only for Type 2 (disk) galaxies.
0 = bulge dominated galaxy
1,2 = intermediate-bulge galaxies
3 = pure disk galaxy
9 = no classification
'''
'''
STELLARITY float Visual Stellarity flag.
0 if ACS_CLASS_STAR<0.6 (object is ASSUMED to be a galaxy; no visual inspection)
0 if ACS_CLASS_STAR>=0.6 AND object visually identified as a galaxy.
1 if ACS_CLASS_STAR>=0.6 AND visually identified as a star.
2 if ACS_CLASS_STAR>=0.8 (object is assumed to be a star and was not visually inspected)
3 if ACS_CLASS_STAR<0.6 but object is visually identified as a star (e.g. saturated star, etc)
JUNKFLAG float
0 = good object
1 = spurious
'''
print('Classifications:', Counter(cat.type).most_common())
cat.is_galaxy = (cat.stellarity == 0)
srcs = []
for t in cat:
pos = RaDecPos(t.ra, t.dec)
bright = NanoMaggies(**{band:NanoMaggies.magToNanomaggies(t.acs_mag_auto)})
shape = GalaxyShape(t.r_0p5_gim2d, 1. - t.ell_gim2d, 90. + t.pa_gim2d)
is_galaxy = (t.is_galaxy * (shape.re >= 0 ) * (shape.ab <= 1.) *
(shape.phi > -999))
if is_galaxy and t.type == 1:
# deV
src = DevGalaxy(pos, bright, shape)
elif is_galaxy and t.type == 2:
# exp
src = ExpGalaxy(pos, bright, shape)
else:
src = PointSource(pos, bright)
srcs.append(src)
return srcs
def all(matched1, matched2, d, name1="ref", name2="test"):
tt = "Comparing %s to %s" % (name1, name2)
plt.clf()
plt.hist(d * 3600.0, 100)
plt.xlabel("Match distance (arcsec)")
plt.title(tt)
plt.savefig(os.path.join(matched1.outdir, "sep_hist.png"))
plt.close()
for iband, band, cc in [(1, "g", "g"), (2, "r", "r"), (4, "z", "m")]:
K = np.flatnonzero(
(matched1.t["decam_flux_ivar"][:, iband] > 0) * (matched2.t["decam_flux_ivar"][:, iband] > 0)
)
print("Median mw_trans", band, "is", np.median(matched1.t["decam_mw_transmission"][:, iband]))
plt.clf()
plt.errorbar(
matched1.t["decam_flux"][K, iband],
matched2.t["decam_flux"][K, iband],
fmt=".",
color=cc,
xerr=1.0 / np.sqrt(matched1.t["decam_flux_ivar"][K, iband]),
yerr=1.0 / np.sqrt(matched2.t["decam_flux_ivar"][K, iband]),
alpha=0.1,
)
plt.xlabel("%s flux: %s" % (name1, band))
plt.ylabel("%s flux: %s" % (name2, band))
plt.plot([-1e6, 1e6], [-1e6, 1e6], "k-", alpha=1.0)
plt.axis([-100, 1000, -100, 1000])
plt.title(tt)
plt.savefig(os.path.join(matched1.outdir, "%s_fluxerr.png" % band))
plt.close()
print("exiting early")
sys.exit()
for iband, band, cc in [(1, "g", "g"), (2, "r", "r"), (4, "z", "m")]:
good = (matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0)
K = np.flatnonzero(good)
psf1 = matched1.type == "PSF "
psf2 = matched2.type == "PSF "
P = np.flatnonzero(good * psf1 * psf2)
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband]
)
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1.0 / iv1 + 1.0 / iv2)
plt.clf()
plt.plot(
mag1[K], (matched2.decam_flux[K, iband] - matched1.decam_flux[K, iband]) / std[K], ".", alpha=0.1, color=cc
)
plt.plot(
mag1[P], (matched2.decam_flux[P, iband] - matched1.decam_flux[P, iband]) / std[P], ".", alpha=0.1, color="k"
)
plt.ylabel("(%s - %s) flux / flux errors (sigma): %s" % (name2, name1, band))
plt.xlabel("%s mag: %s" % (name1, band))
plt.axhline(0, color="k", alpha=0.5)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
plt.clf()
lp, lt = [], []
for iband, band, cc in [(1, "g", "g"), (2, "r", "r"), (4, "z", "m")]:
good = (matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0)
# good = True
psf1 = matched1.type == "PSF "
psf2 = matched2.type == "PSF "
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband]
)
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1.0 / iv1 + 1.0 / iv2)
# std = np.hypot(std, 0.01)
G = np.flatnonzero(
good * psf1 * psf2 * np.isfinite(mag1) * (mag1 >= 20) * (mag1 < dict(g=24, r=23.5, z=22.5)[band])
)
n, b, p = plt.hist(
(matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) / std[G],
range=(-4, 4),
bins=50,
histtype="step",
color=cc,
normed=True,
)
sig = (matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) / std[G]
print("Raw mean and std of points:", np.mean(sig), np.std(sig))
med = np.median(sig)
rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2.0
print("Median and percentile-based sigma:", med, rsigma)
lp.append(p[0])
lt.append("%s: %.2f +- %.2f" % (band, med, rsigma))
bins = []
gaussint = []
for blo, bhi in zip(b, b[1:]):
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= bhi - blo
# bins.extend([blo,bhi])
# gaussint.extend([c,c])
bins.append((blo + bhi) / 2.0)
gaussint.append(c)
plt.plot(bins, gaussint, "k-", lw=2, alpha=0.5)
plt.title(tt)
plt.xlabel("Flux difference / error (sigma)")
plt.axvline(0, color="k", alpha=0.1)
plt.ylim(0, 0.45)
plt.legend(lp, lt, loc="upper right")
ps.savefig()
for iband, band, cc in [(1, "g", "g"), (2, "r", "r"), (4, "z", "m")]:
plt.clf()
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband]
)
mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors(
matched2.decam_flux[:, iband], matched2.decam_flux_ivar[:, iband]
)
meanmag = NanoMaggies.nanomaggiesToMag((matched1.decam_flux[:, iband] + matched2.decam_flux[:, iband]) / 2.0)
psf1 = matched1.type == "PSF "
psf2 = matched2.type == "PSF "
good = (
(matched1.decam_flux_ivar[:, iband] > 0)
* (matched2.decam_flux_ivar[:, iband] > 0)
* np.isfinite(mag1)
* np.isfinite(mag2)
)
K = np.flatnonzero(good)
P = np.flatnonzero(good * psf1 * psf2)
plt.errorbar(mag1[K], mag2[K], fmt=".", color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1)
plt.plot(mag1[P], mag2[P], "k.", alpha=0.5)
plt.xlabel("%s %s (mag)" % (name1, band))
plt.ylabel("%s %s (mag)" % (name2, band))
plt.plot([-1e6, 1e6], [-1e6, 1e6], "k-", alpha=1.0)
plt.axis([24, 16, 24, 16])
plt.title(tt)
ps.savefig()
plt.clf()
plt.errorbar(mag1[K], mag2[K] - mag1[K], fmt=".", color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1)
plt.plot(mag1[P], mag2[P] - mag1[P], "k.", alpha=0.5)
plt.xlabel("%s %s (mag)" % (name1, band))
plt.ylabel("%s %s - %s %s (mag)" % (name2, band, name1, band))
plt.axhline(0.0, color="k", alpha=1.0)
plt.axis([24, 16, -1, 1])
plt.title(tt)
ps.savefig()
magbins = np.arange(16, 24.001, 0.5)
plt.clf()
plt.plot(mag1[K], (mag2[K] - mag1[K]) / np.hypot(magerr1[K], magerr2[K]), ".", color=cc, alpha=0.1)
plt.plot(mag1[P], (mag2[P] - mag1[P]) / np.hypot(magerr1[P], magerr2[P]), "k.", alpha=0.5)
plt.xlabel("%s %s (mag)" % (name1, band))
plt.ylabel("(%s %s - %s %s) / errors (sigma)" % (name2, band, name1, band))
plt.axhline(0.0, color="k", alpha=1.0)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
plt.clf()
plt.plot(meanmag[P], y[P], "k.", alpha=0.1)
midmag = []
vals = np.zeros((len(magbins) - 1, 5))
median_err1 = []
iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25)
# FIXME -- should we do some stats after taking off the mean difference?
for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
midmag.append((mlo + mhi) / 2.0)
median_err1.append(np.median(magerr1[I]))
if len(I) == 0:
continue
# median and +- 1 sigma quantiles
ybin = y[I]
vals[bini, 0] = np.percentile(ybin, 16)
vals[bini, 1] = np.median(ybin)
vals[bini, 2] = np.percentile(ybin, 84)
# +- 2 sigma quantiles
vals[bini, 3] = np.percentile(ybin, 2.3)
vals[bini, 4] = np.percentile(ybin, 97.7)
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print(
"Mag bin",
midmag[-1],
": IQD is factor",
iqd / iqd_gauss,
"vs expected for Gaussian;",
len(ybin),
"points",
)
# if iqd > iqd_gauss:
# # What error adding in quadrature would you need to make the IQD match?
# err = median_err1[-1]
# target_err = err * (iqd / iqd_gauss)
# sys_err = np.sqrt(target_err**2 - err**2)
# print('--> add systematic error', sys_err)
# ~ Johan's cuts
mlo = 21.0
mhi = dict(g=24.0, r=23.5, z=22.5)[band]
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
ybin = y[I]
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print(
"Mag bin",
mlo,
mhi,
"band",
band,
": IQD is factor",
iqd / iqd_gauss,
"vs expected for Gaussian;",
len(ybin),
"points",
)
if iqd > iqd_gauss:
# What error adding in quadrature would you need to make
# the IQD match?
err = np.median(np.hypot(magerr1[I], magerr2[I]))
print("Median error (hypot):", err)
target_err = err * (iqd / iqd_gauss)
print("Target:", target_err)
sys_err = np.sqrt((target_err ** 2 - err ** 2) / 2.0)
print("--> add systematic error", sys_err)
# check...
err_sys = np.hypot(np.hypot(magerr1, sys_err), np.hypot(magerr2, sys_err))
ysys = (mag2 - mag1) / err_sys
ysys = ysys[I]
print("Resulting median error:", np.median(err_sys[I]))
iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25)
print("--> IQD", iqd_sys / iqd_gauss, "vs Gaussian")
# Hmmm, this doesn't work... totally overshoots.
plt.errorbar(
midmag,
vals[:, 1],
fmt="o",
color="b",
yerr=(vals[:, 1] - vals[:, 0], vals[:, 2] - vals[:, 1]),
capthick=3,
zorder=20,
)
plt.errorbar(
midmag,
vals[:, 1],
fmt="o",
color="b",
yerr=(vals[:, 1] - vals[:, 3], vals[:, 4] - vals[:, 1]),
capthick=2,
zorder=20,
)
plt.axhline(1.0, color="b", alpha=0.2)
plt.axhline(-1.0, color="b", alpha=0.2)
plt.axhline(2.0, color="b", alpha=0.2)
plt.axhline(-2.0, color="b", alpha=0.2)
for mag, err, y in zip(midmag, median_err1, vals[:, 3]):
if not np.isfinite(err):
continue
if y < -6:
continue
plt.text(mag, y - 0.1, "%.3f" % err, va="top", ha="center", color="k", fontsize=10)
plt.xlabel("(%s + %s)/2 %s (mag), PSFs" % (name1, name2, band))
plt.ylabel("(%s %s - %s %s) / errors (sigma)" % (name2, band, name1, band))
plt.axhline(0.0, color="k", alpha=1.0)
plt.axvline(21, color="k", alpha=0.3)
plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color="k", alpha=0.3)
plt.axis([24.1, 16, -6, 6])
plt.title(tt)
ps.savefig()
# magbins = np.append([16, 18], np.arange(20, 24.001, 0.5))
if band == "g":
magbins = [20, 24]
elif band == "r":
magbins = [20, 23.5]
elif band == "z":
magbins = [20, 22.5]
slo, shi = -5, 5
plt.clf()
ha = dict(bins=25, range=(slo, shi), histtype="step", normed=True)
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
midmag = []
nn = []
rgbs = []
lt, lp = [], []
for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)]
if len(I) == 0:
continue
ybin = y[I]
rgb = [0.0, 0.0, 0.0]
rgb[0] = float(bini) / (len(magbins) - 1)
rgb[2] = 1.0 - rgb[0]
n, b, p = plt.hist(ybin, color=rgb, **ha)
lt.append("mag %g to %g" % (mlo, mhi))
lp.append(p[0])
midmag.append((mlo + mhi) / 2.0)
nn.append(n)
rgbs.append(rgb)
bins = []
gaussint = []
for blo, bhi in zip(b, b[1:]):
# midbin.append((blo+bhi)/2.)
# gaussint.append(scipy.stats.norm.cdf(bhi) -
# scipy.stats.norm.cdf(blo))
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= bhi - blo
bins.extend([blo, bhi])
gaussint.extend([c, c])
plt.plot(bins, gaussint, "k-", lw=2, alpha=0.5)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo, shi)
ps.savefig()
bincenters = b[:-1] + (b[1] - b[0]) / 2.0
plt.clf()
lp = []
for n, rgb, mlo, mhi in zip(nn, rgbs, magbins, magbins[1:]):
p = plt.plot(bincenters, n, "-", color=rgb)
lp.append(p[0])
plt.plot(bincenters, gaussint[::2], "k-", alpha=0.5, lw=2)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo, shi)
ps.savefig()
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--name1', help='Name for first data set')
parser.add_argument('--name2', help='Name for second data set')
parser.add_argument('--plot-prefix', default='compare',
help='Prefix for plot filenames; default "%default"')
parser.add_argument('--match', default=1.0,
help='Astrometric cross-match distance in arcsec')
parser.add_argument('dir1', help='First directory to compare')
parser.add_argument('dir2', help='Second directory to compare')
opt = parser.parse_args()
ps = PlotSequence(opt.plot_prefix)
name1 = opt.name1
if name1 is None:
name1 = os.path.basename(opt.dir1)
if not len(name1):
name1 = os.path.basename(os.path.dirname(opt.dir1))
name2 = opt.name2
if name2 is None:
name2 = os.path.basename(opt.dir2)
if not len(name2):
name2 = os.path.basename(os.path.dirname(opt.dir2))
tt = 'Comparing %s to %s' % (name1, name2)
# regex for tractor-*.fits catalog filename
catre = re.compile('tractor-.*.fits')
cat1,cat2 = [],[]
for basedir,cat in [(opt.dir1, cat1), (opt.dir2, cat2)]:
for dirpath,dirnames,filenames in os.walk(basedir, followlinks=True):
for fn in filenames:
if not catre.match(fn):
print('Skipping', fn, 'due to filename')
continue
fn = os.path.join(dirpath, fn)
t = fits_table(fn)
print(len(t), 'from', fn)
cat.append(t)
cat1 = merge_tables(cat1, columns='fillzero')
cat2 = merge_tables(cat2, columns='fillzero')
print('Total of', len(cat1), 'from', name1)
print('Total of', len(cat2), 'from', name2)
cat1.cut(cat1.brick_primary)
cat2.cut(cat2.brick_primary)
print('Total of', len(cat1), 'BRICK_PRIMARY from', name1)
print('Total of', len(cat2), 'BRICK_PRIMARY from', name2)
cat1.cut((cat1.decam_anymask[:,1] == 0) *
(cat1.decam_anymask[:,2] == 0) *
(cat1.decam_anymask[:,4] == 0))
cat2.cut((cat2.decam_anymask[:,1] == 0) *
(cat2.decam_anymask[:,2] == 0) *
(cat2.decam_anymask[:,4] == 0))
print('Total of', len(cat1), 'unmasked from', name1)
print('Total of', len(cat2), 'unmasked from', name2)
I,J,d = match_radec(cat1.ra, cat1.dec, cat2.ra, cat2.dec, opt.match/3600.,
nearest=True)
print(len(I), 'matched')
plt.clf()
plt.hist(d * 3600., 100)
plt.xlabel('Match distance (arcsec)')
plt.title(tt)
ps.savefig()
matched1 = cat1[I]
matched2 = cat2[J]
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
K = np.flatnonzero((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0))
print('Median mw_trans', band, 'is',
np.median(matched1.decam_mw_transmission[:,iband]))
plt.clf()
plt.errorbar(matched1.decam_flux[K,iband],
matched2.decam_flux[K,iband],
fmt='.', color=cc,
xerr=1./np.sqrt(matched1.decam_flux_ivar[K,iband]),
yerr=1./np.sqrt(matched2.decam_flux_ivar[K,iband]),
alpha=0.1,
)
plt.xlabel('%s flux: %s' % (name1, band))
plt.ylabel('%s flux: %s' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.)
plt.axis([-100, 1000, -100, 1000])
plt.title(tt)
ps.savefig()
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
good = ((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0))
K = np.flatnonzero(good)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
P = np.flatnonzero(good * psf1 * psf2)
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1./iv1 + 1./iv2)
plt.clf()
plt.plot(mag1[K],
(matched2.decam_flux[K,iband] - matched1.decam_flux[K,iband]) / std[K],
'.', alpha=0.1, color=cc)
plt.plot(mag1[P],
(matched2.decam_flux[P,iband] - matched1.decam_flux[P,iband]) / std[P],
'.', alpha=0.1, color='k')
plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' % (name2, name1, band))
plt.xlabel('%s mag: %s' % (name1, band))
plt.axhline(0, color='k', alpha=0.5)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
plt.clf()
lp,lt = [],[]
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
good = ((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0))
#good = True
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
iv1 = matched1.decam_flux_ivar[:, iband]
iv2 = matched2.decam_flux_ivar[:, iband]
std = np.sqrt(1./iv1 + 1./iv2)
#std = np.hypot(std, 0.01)
G = np.flatnonzero(good * psf1 * psf2 *
np.isfinite(mag1) *
(mag1 >= 20) * (mag1 < dict(g=24, r=23.5, z=22.5)[band]))
n,b,p = plt.hist((matched2.decam_flux[G,iband] -
matched1.decam_flux[G,iband]) / std[G],
range=(-4, 4), bins=50, histtype='step', color=cc,
normed=True)
sig = (matched2.decam_flux[G,iband] -
matched1.decam_flux[G,iband]) / std[G]
print('Raw mean and std of points:', np.mean(sig), np.std(sig))
med = np.median(sig)
rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2.
print('Median and percentile-based sigma:', med, rsigma)
lp.append(p[0])
lt.append('%s: %.2f +- %.2f' % (band, med, rsigma))
bins = []
gaussint = []
for blo,bhi in zip(b, b[1:]):
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
#bins.extend([blo,bhi])
#gaussint.extend([c,c])
bins.append((blo+bhi)/2.)
gaussint.append(c)
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.title(tt)
plt.xlabel('Flux difference / error (sigma)')
plt.axvline(0, color='k', alpha=0.1)
plt.ylim(0, 0.45)
plt.legend(lp, lt, loc='upper right')
ps.savefig()
for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
plt.clf()
mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors(
matched2.decam_flux[:,iband], matched2.decam_flux_ivar[:,iband])
meanmag = NanoMaggies.nanomaggiesToMag((
matched1.decam_flux[:,iband] + matched2.decam_flux[:,iband]) / 2.)
psf1 = (matched1.type == 'PSF ')
psf2 = (matched2.type == 'PSF ')
good = ((matched1.decam_flux_ivar[:,iband] > 0) *
(matched2.decam_flux_ivar[:,iband] > 0) *
np.isfinite(mag1) * np.isfinite(mag2))
K = np.flatnonzero(good)
P = np.flatnonzero(good * psf1 * psf2)
plt.errorbar(mag1[K], mag2[K], fmt='.', color=cc,
xerr=magerr1[K], yerr=magerr2[K], alpha=0.1)
plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s (mag)' % (name2, band))
plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.)
plt.axis([24, 16, 24, 16])
plt.title(tt)
ps.savefig()
plt.clf()
plt.errorbar(mag1[K], mag2[K] - mag1[K], fmt='.', color=cc,
xerr=magerr1[K], yerr=magerr2[K], alpha=0.1)
plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -1, 1])
plt.title(tt)
ps.savefig()
magbins = np.arange(16, 24.001, 0.5)
plt.clf()
plt.plot(mag1[K], (mag2[K]-mag1[K]) / np.hypot(magerr1[K], magerr2[K]),
'.', color=cc, alpha=0.1)
plt.plot(mag1[P], (mag2[P]-mag1[P]) / np.hypot(magerr1[P], magerr2[P]),
'k.', alpha=0.5)
plt.xlabel('%s %s (mag)' % (name1, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axis([24, 16, -10, 10])
plt.title(tt)
ps.savefig()
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
plt.clf()
plt.plot(meanmag[P], y[P], 'k.', alpha=0.1)
midmag = []
vals = np.zeros((len(magbins)-1, 5))
median_err1 = []
iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25)
# FIXME -- should we do some stats after taking off the mean difference?
for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
midmag.append((mlo+mhi)/2.)
median_err1.append(np.median(magerr1[I]))
if len(I) == 0:
continue
# median and +- 1 sigma quantiles
ybin = y[I]
vals[bini,0] = np.percentile(ybin, 16)
vals[bini,1] = np.median(ybin)
vals[bini,2] = np.percentile(ybin, 84)
# +- 2 sigma quantiles
vals[bini,3] = np.percentile(ybin, 2.3)
vals[bini,4] = np.percentile(ybin, 97.7)
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss,
'vs expected for Gaussian;', len(ybin), 'points')
# if iqd > iqd_gauss:
# # What error adding in quadrature would you need to make the IQD match?
# err = median_err1[-1]
# target_err = err * (iqd / iqd_gauss)
# sys_err = np.sqrt(target_err**2 - err**2)
# print('--> add systematic error', sys_err)
# ~ Johan's cuts
mlo = 21.
mhi = dict(g=24., r=23.5, z=22.5)[band]
I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
ybin = y[I]
iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
print('Mag bin', mlo, mhi, 'band', band, ': IQD is factor',
iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points')
if iqd > iqd_gauss:
# What error adding in quadrature would you need to make
# the IQD match?
err = np.median(np.hypot(magerr1[I], magerr2[I]))
print('Median error (hypot):', err)
target_err = err * (iqd / iqd_gauss)
print('Target:', target_err)
sys_err = np.sqrt((target_err**2 - err**2) / 2.)
print('--> add systematic error', sys_err)
# check...
err_sys = np.hypot(np.hypot(magerr1, sys_err),
np.hypot(magerr2, sys_err))
ysys = (mag2 - mag1) / err_sys
ysys = ysys[I]
print('Resulting median error:', np.median(err_sys[I]))
iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25)
print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian')
# Hmmm, this doesn't work... totally overshoots.
plt.errorbar(midmag, vals[:,1], fmt='o', color='b',
yerr=(vals[:,1]-vals[:,0], vals[:,2]-vals[:,1]),
capthick=3, zorder=20)
plt.errorbar(midmag, vals[:,1], fmt='o', color='b',
yerr=(vals[:,1]-vals[:,3], vals[:,4]-vals[:,1]),
capthick=2, zorder=20)
plt.axhline( 1., color='b', alpha=0.2)
plt.axhline(-1., color='b', alpha=0.2)
plt.axhline( 2., color='b', alpha=0.2)
plt.axhline(-2., color='b', alpha=0.2)
for mag,err,y in zip(midmag, median_err1, vals[:,3]):
if not np.isfinite(err):
continue
if y < -6:
continue
plt.text(mag, y-0.1, '%.3f' % err, va='top', ha='center', color='k',
fontsize=10)
plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band))
plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
(name2, band, name1, band))
plt.axhline(0., color='k', alpha=1.)
plt.axvline(21, color='k', alpha=0.3)
plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3)
plt.axis([24.1, 16, -6, 6])
plt.title(tt)
ps.savefig()
#magbins = np.append([16, 18], np.arange(20, 24.001, 0.5))
if band == 'g':
magbins = [20, 24]
elif band == 'r':
magbins = [20, 23.5]
elif band == 'z':
magbins = [20, 22.5]
slo,shi = -5,5
plt.clf()
ha = dict(bins=25, range=(slo,shi), histtype='step', normed=True)
y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
midmag = []
nn = []
rgbs = []
lt,lp = [],[]
for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])):
I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)]
if len(I) == 0:
continue
ybin = y[I]
rgb = [0.,0.,0.]
rgb[0] = float(bini) / (len(magbins)-1)
rgb[2] = 1. - rgb[0]
n,b,p = plt.hist(ybin, color=rgb, **ha)
lt.append('mag %g to %g' % (mlo,mhi))
lp.append(p[0])
midmag.append((mlo+mhi)/2.)
nn.append(n)
rgbs.append(rgb)
bins = []
gaussint = []
for blo,bhi in zip(b, b[1:]):
#midbin.append((blo+bhi)/2.)
#gaussint.append(scipy.stats.norm.cdf(bhi) -
# scipy.stats.norm.cdf(blo))
c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
c /= (bhi - blo)
bins.extend([blo,bhi])
gaussint.extend([c,c])
plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo,shi)
ps.savefig()
bincenters = b[:-1] + (b[1]-b[0])/2.
plt.clf()
lp = []
for n,rgb,mlo,mhi in zip(nn, rgbs, magbins, magbins[1:]):
p = plt.plot(bincenters, n, '-', color=rgb)
lp.append(p[0])
plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2)
plt.legend(lp, lt)
plt.title(tt)
plt.xlim(slo,shi)
ps.savefig()
def main():
global survey
init()
ps = PlotSequence('sky')
# export LEGACY_SURVEY_DIR=/scratch1/scratchdirs/desiproc/DRs/dr4-bootes/legacypipe-dir/
#survey = LegacySurveyData()
survey = get_survey('dr4v2')
ccds = survey.get_ccds_readonly()
print(len(ccds), 'CCDs')
ccds = ccds[ccds.camera == 'mosaic']
print(len(ccds), 'Mosaic CCDs')
# plt.clf()
# plt.hist(ccds.mjd_obs % 1.0, bins=50)
# plt.xlabel('MJD mod 1')
# ps.savefig()
ccds.imjd = np.floor(ccds.mjd_obs).astype(int)
mjds = np.unique(ccds.imjd)
print(len(mjds), 'unique MJDs')
mp = multiproc(nthreads=8, init=init)
allvals = []
medmjds = []
args = []
for kk, mjd in enumerate(mjds):
I = np.flatnonzero(ccds.imjd == mjd)
print('MJD', mjd, ' (%i of %i):' % (kk + 1, len(mjds)), len(I), 'CCDs')
if len(I) == 0:
continue
# pick one near the middle
#i = I[len(I)/2]
for i in [I[len(I) / 4], I[len(I) / 2], I[3 * len(I) / 4]]:
ccd = ccds[i]
key = dict(expnum=ccd.expnum, ccdname=ccd.ccdname)
oldvals = key
vals = cache.find(key)
print('Got', vals.count(), 'cache hits for', key)
gotone = False
for val in vals:
#if 'median_adu' in val:
if 'mode_adu' in val:
print('cache hit:', val)
allvals.append(val)
medmjds.append(mjd)
gotone = True
break
else:
print('partial cache hit:', val)
oldvals = val
###!
cache.delete_one(val)
if gotone:
continue
print('args: key', key, 'oldvals', oldvals)
args.append((mjd, key, oldvals, ccd))
# plt.clf()
# plt.hist(tim.getImage().ravel(), range=(-0.1, 0.1), bins=50,
# histtype='step', color='b')
# plt.axvline(med, color='b', alpha=0.3, lw=2)
# plt.axvline(0., color='k', alpha=0.3, lw=2)
# plt.xlabel('Sky-subtracted pixel values')
# plt.title('Date ' + ccd.date_obs + ': ' + str(im))
# ps.savefig()
if len(args):
meds = mp.map(read_sky_val, args)
print('Medians:', meds)
for (mjd, key, oldvals, ccd), val in zip(args, meds):
if val is None:
continue
allvals.append(val)
medmjds.append(mjd)
medians = []
median_adus = []
mode_adus = []
skyadus = []
keepmjds = []
for mjd, val in zip(medmjds, allvals):
madu = val['median_adu']
if madu is None:
continue
if 'median' in val:
medians.append(val['median'])
else:
from tractor.brightness import NanoMaggies
zpscale = NanoMaggies.zeropointToScale(val['ccdzpt'])
med = (val['median_adu'] - val['skyadu']) / zpscale
print('Computed median diff:', med, 'nmgy')
medians.append(med)
keepmjds.append(mjd)
median_adus.append(val['median_adu'])
mode_adus.append(val['mode_adu'])
skyadus.append(val['skyadu'])
medmjds = keepmjds
median_adus = np.array(median_adus)
mode_adus = np.array(mode_adus)
skyadus = np.array(skyadus)
medmjds = np.array(medmjds)
medians = np.array(medians)
plt.clf()
plt.plot(medmjds, median_adus - skyadus, 'b.')
plt.xlabel('MJD')
#plt.ylabel('Image median after sky subtraction (nmgy)')
plt.ylabel('Image median - SKYADU (ADU)')
#plt.ylim(-0.03, 0.03)
#plt.ylim(-10, 10)
plt.ylim(-1, 1)
plt.axhline(0, color='k', lw=2, alpha=0.5)
ps.savefig()
plt.clf()
plt.plot(medmjds, mode_adus - skyadus, 'b.')
plt.xlabel('MJD')
plt.ylabel('Image mode - SKYADU (ADU)')
plt.ylim(-1, 1)
plt.axhline(0, color='k', lw=2, alpha=0.5)
ps.savefig()
print('Median pcts:', np.percentile(medians, [0, 2, 50, 98, 100]))
mlo, mhi = np.percentile(medians, [2, 98])
I = np.flatnonzero((medians > mlo) * (medians < mhi))
d0 = np.mean(medmjds)
dmjd = medmjds[I] - d0
A = np.zeros((len(dmjd), 2))
A[:, 0] = 1.
A[:, 1] = dmjd
b = np.linalg.lstsq(A, medians[I])[0]
print('Lstsq:', b)
offset = b[0]
slope = b[1]
xx = np.array([dmjd.min(), dmjd.max()])
print('Offset', offset)
print('Slope', slope)
plt.clf()
plt.plot(medmjds, medians, 'b.')
ax = plt.axis()
plt.plot(xx + d0, offset + slope * xx, 'b-')
plt.axis(ax)
plt.xlabel('MJD')
plt.ylabel('Image median after sky subtraction (nmgy)')
plt.ylim(-0.03, 0.03)
plt.axhline(0, color='k', lw=2, alpha=0.5)
ps.savefig()
plt.clf()
plt.plot(median_adus, skyadus, 'b.')
ax = plt.axis()
lo = min(ax[0], ax[2])
hi = max(ax[1], ax[3])
plt.plot([lo, hi], [lo, hi], 'k-', alpha=0.5)
plt.xlabel('Median image ADU')
plt.ylabel('SKYADU')
plt.axis(ax)
ps.savefig()
plt.clf()
plt.plot(medmjds, skyadus / median_adus, 'b.')
plt.xlabel('MJD')
plt.ylim(0.98, 1.02)
plt.axhline(1.0, color='k', alpha=0.5)
plt.ylabel('SKYADU / median image ADU')
ps.savefig()