def set_mags(self, cat): '''adds columns to fits_table cat''' # Remove white spaces cat.set('type', np.char.strip(cat.get('type'))) # AB mags # Two kinds, as observed (including dust) and instrinsic (dust removed) for whichmag in ['wdust', 'nodust']: # DECam for band in ['g', 'r', 'z', 'w1', 'w2']: if whichmag == 'wdust': flux = cat.get('flux_%s' % band) flux_ivar = cat.get('flux_ivar_%s' % band) elif whichmag == 'nodust': flux = cat.get('flux_%s' % band) / cat.get( 'mw_transmission_%s' % band) flux_ivar= cat.get('flux_ivar_%s' % band)*\ np.power(cat.get('mw_transmission_%s' % band),2) else: raise ValueError() mag, mag_err = NanoMaggies.fluxErrorsToMagErrors( flux, flux_ivar) cat.set('mag_%s_%s' % (whichmag, band), mag) cat.set('mag_ivar_%s_%s' % (whichmag, band), 1. / np.power(mag_err, 2)) # Instrinsic fluxes whichmag = 'nodust' # DECam for band in ['g', 'r', 'z', 'w1', 'w2']: flux = cat.get('flux_%s' % band) / cat.get( 'mw_transmission_%s' % band) flux_ivar= cat.get('flux_ivar_%s' % band)*\ np.power(cat.get('mw_transmission_%s' % band),2) cat.set('flux_%s_%s' % (whichmag, band), flux) cat.set('flux_ivar_%s_%s' % (whichmag, band), flux_ivar)
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument('--name1', help='Name for first data set') parser.add_argument('--name2', help='Name for second data set') parser.add_argument('--plot-prefix', default='compare', help='Prefix for plot filenames; default "%default"') parser.add_argument('--match', default=1.0, help='Astrometric cross-match distance in arcsec') parser.add_argument('dir1', help='First directory to compare') parser.add_argument('dir2', help='Second directory to compare') opt = parser.parse_args() ps = PlotSequence(opt.plot_prefix) name1 = opt.name1 if name1 is None: name1 = os.path.basename(opt.dir1) if not len(name1): name1 = os.path.basename(os.path.dirname(opt.dir1)) name2 = opt.name2 if name2 is None: name2 = os.path.basename(opt.dir2) if not len(name2): name2 = os.path.basename(os.path.dirname(opt.dir2)) tt = 'Comparing %s to %s' % (name1, name2) # regex for tractor-*.fits catalog filename catre = re.compile('tractor-.*.fits') cat1,cat2 = [],[] for basedir,cat in [(opt.dir1, cat1), (opt.dir2, cat2)]: for dirpath,dirnames,filenames in os.walk(basedir, followlinks=True): for fn in filenames: if not catre.match(fn): print('Skipping', fn, 'due to filename') continue fn = os.path.join(dirpath, fn) t = fits_table(fn) print(len(t), 'from', fn) cat.append(t) cat1 = merge_tables(cat1, columns='fillzero') cat2 = merge_tables(cat2, columns='fillzero') print('Total of', len(cat1), 'from', name1) print('Total of', len(cat2), 'from', name2) cat1.cut(cat1.brick_primary) cat2.cut(cat2.brick_primary) print('Total of', len(cat1), 'BRICK_PRIMARY from', name1) print('Total of', len(cat2), 'BRICK_PRIMARY from', name2) cat1.cut((cat1.decam_anymask[:,1] == 0) * (cat1.decam_anymask[:,2] == 0) * (cat1.decam_anymask[:,4] == 0)) cat2.cut((cat2.decam_anymask[:,1] == 0) * (cat2.decam_anymask[:,2] == 0) * (cat2.decam_anymask[:,4] == 0)) print('Total of', len(cat1), 'unmasked from', name1) print('Total of', len(cat2), 'unmasked from', name2) I,J,d = match_radec(cat1.ra, cat1.dec, cat2.ra, cat2.dec, opt.match/3600., nearest=True) print(len(I), 'matched') plt.clf() plt.hist(d * 3600., 100) plt.xlabel('Match distance (arcsec)') plt.title(tt) ps.savefig() matched1 = cat1[I] matched2 = cat2[J] for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]: K = np.flatnonzero((matched1.decam_flux_ivar[:,iband] > 0) * (matched2.decam_flux_ivar[:,iband] > 0)) print('Median mw_trans', band, 'is', np.median(matched1.decam_mw_transmission[:,iband])) plt.clf() plt.errorbar(matched1.decam_flux[K,iband], matched2.decam_flux[K,iband], fmt='.', color=cc, xerr=1./np.sqrt(matched1.decam_flux_ivar[K,iband]), yerr=1./np.sqrt(matched2.decam_flux_ivar[K,iband]), alpha=0.1, ) plt.xlabel('%s flux: %s' % (name1, band)) plt.ylabel('%s flux: %s' % (name2, band)) plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.) plt.axis([-100, 1000, -100, 1000]) plt.title(tt) ps.savefig() for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]: good = ((matched1.decam_flux_ivar[:,iband] > 0) * (matched2.decam_flux_ivar[:,iband] > 0)) K = np.flatnonzero(good) psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') P = np.flatnonzero(good * psf1 * psf2) mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband]) iv1 = matched1.decam_flux_ivar[:, iband] iv2 = matched2.decam_flux_ivar[:, iband] std = np.sqrt(1./iv1 + 1./iv2) plt.clf() plt.plot(mag1[K], (matched2.decam_flux[K,iband] - matched1.decam_flux[K,iband]) / std[K], '.', alpha=0.1, color=cc) plt.plot(mag1[P], (matched2.decam_flux[P,iband] - matched1.decam_flux[P,iband]) / std[P], '.', alpha=0.1, color='k') plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' % (name2, name1, band)) plt.xlabel('%s mag: %s' % (name1, band)) plt.axhline(0, color='k', alpha=0.5) plt.axis([24, 16, -10, 10]) plt.title(tt) ps.savefig() plt.clf() lp,lt = [],[] for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]: good = ((matched1.decam_flux_ivar[:,iband] > 0) * (matched2.decam_flux_ivar[:,iband] > 0)) #good = True psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband]) iv1 = matched1.decam_flux_ivar[:, iband] iv2 = matched2.decam_flux_ivar[:, iband] std = np.sqrt(1./iv1 + 1./iv2) #std = np.hypot(std, 0.01) G = np.flatnonzero(good * psf1 * psf2 * np.isfinite(mag1) * (mag1 >= 20) * (mag1 < dict(g=24, r=23.5, z=22.5)[band])) n,b,p = plt.hist((matched2.decam_flux[G,iband] - matched1.decam_flux[G,iband]) / std[G], range=(-4, 4), bins=50, histtype='step', color=cc, normed=True) sig = (matched2.decam_flux[G,iband] - matched1.decam_flux[G,iband]) / std[G] print('Raw mean and std of points:', np.mean(sig), np.std(sig)) med = np.median(sig) rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2. print('Median and percentile-based sigma:', med, rsigma) lp.append(p[0]) lt.append('%s: %.2f +- %.2f' % (band, med, rsigma)) bins = [] gaussint = [] for blo,bhi in zip(b, b[1:]): c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo) c /= (bhi - blo) #bins.extend([blo,bhi]) #gaussint.extend([c,c]) bins.append((blo+bhi)/2.) gaussint.append(c) plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5) plt.title(tt) plt.xlabel('Flux difference / error (sigma)') plt.axvline(0, color='k', alpha=0.1) plt.ylim(0, 0.45) plt.legend(lp, lt, loc='upper right') ps.savefig() for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]: plt.clf() mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband]) mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors( matched2.decam_flux[:,iband], matched2.decam_flux_ivar[:,iband]) meanmag = NanoMaggies.nanomaggiesToMag(( matched1.decam_flux[:,iband] + matched2.decam_flux[:,iband]) / 2.) psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') good = ((matched1.decam_flux_ivar[:,iband] > 0) * (matched2.decam_flux_ivar[:,iband] > 0) * np.isfinite(mag1) * np.isfinite(mag2)) K = np.flatnonzero(good) P = np.flatnonzero(good * psf1 * psf2) plt.errorbar(mag1[K], mag2[K], fmt='.', color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1) plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('%s %s (mag)' % (name2, band)) plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.) plt.axis([24, 16, 24, 16]) plt.title(tt) ps.savefig() plt.clf() plt.errorbar(mag1[K], mag2[K] - mag1[K], fmt='.', color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1) plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axis([24, 16, -1, 1]) plt.title(tt) ps.savefig() magbins = np.arange(16, 24.001, 0.5) plt.clf() plt.plot(mag1[K], (mag2[K]-mag1[K]) / np.hypot(magerr1[K], magerr2[K]), '.', color=cc, alpha=0.1) plt.plot(mag1[P], (mag2[P]-mag1[P]) / np.hypot(magerr1[P], magerr2[P]), 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('(%s %s - %s %s) / errors (sigma)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axis([24, 16, -10, 10]) plt.title(tt) ps.savefig() y = (mag2 - mag1) / np.hypot(magerr1, magerr2) plt.clf() plt.plot(meanmag[P], y[P], 'k.', alpha=0.1) midmag = [] vals = np.zeros((len(magbins)-1, 5)) median_err1 = [] iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25) # FIXME -- should we do some stats after taking off the mean difference? for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])): I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)] midmag.append((mlo+mhi)/2.) median_err1.append(np.median(magerr1[I])) if len(I) == 0: continue # median and +- 1 sigma quantiles ybin = y[I] vals[bini,0] = np.percentile(ybin, 16) vals[bini,1] = np.median(ybin) vals[bini,2] = np.percentile(ybin, 84) # +- 2 sigma quantiles vals[bini,3] = np.percentile(ybin, 2.3) vals[bini,4] = np.percentile(ybin, 97.7) iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25) print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points') # if iqd > iqd_gauss: # # What error adding in quadrature would you need to make the IQD match? # err = median_err1[-1] # target_err = err * (iqd / iqd_gauss) # sys_err = np.sqrt(target_err**2 - err**2) # print('--> add systematic error', sys_err) # ~ Johan's cuts mlo = 21. mhi = dict(g=24., r=23.5, z=22.5)[band] I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)] ybin = y[I] iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25) print('Mag bin', mlo, mhi, 'band', band, ': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points') if iqd > iqd_gauss: # What error adding in quadrature would you need to make # the IQD match? err = np.median(np.hypot(magerr1[I], magerr2[I])) print('Median error (hypot):', err) target_err = err * (iqd / iqd_gauss) print('Target:', target_err) sys_err = np.sqrt((target_err**2 - err**2) / 2.) print('--> add systematic error', sys_err) # check... err_sys = np.hypot(np.hypot(magerr1, sys_err), np.hypot(magerr2, sys_err)) ysys = (mag2 - mag1) / err_sys ysys = ysys[I] print('Resulting median error:', np.median(err_sys[I])) iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25) print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian') # Hmmm, this doesn't work... totally overshoots. plt.errorbar(midmag, vals[:,1], fmt='o', color='b', yerr=(vals[:,1]-vals[:,0], vals[:,2]-vals[:,1]), capthick=3, zorder=20) plt.errorbar(midmag, vals[:,1], fmt='o', color='b', yerr=(vals[:,1]-vals[:,3], vals[:,4]-vals[:,1]), capthick=2, zorder=20) plt.axhline( 1., color='b', alpha=0.2) plt.axhline(-1., color='b', alpha=0.2) plt.axhline( 2., color='b', alpha=0.2) plt.axhline(-2., color='b', alpha=0.2) for mag,err,y in zip(midmag, median_err1, vals[:,3]): if not np.isfinite(err): continue if y < -6: continue plt.text(mag, y-0.1, '%.3f' % err, va='top', ha='center', color='k', fontsize=10) plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band)) plt.ylabel('(%s %s - %s %s) / errors (sigma)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axvline(21, color='k', alpha=0.3) plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3) plt.axis([24.1, 16, -6, 6]) plt.title(tt) ps.savefig() #magbins = np.append([16, 18], np.arange(20, 24.001, 0.5)) if band == 'g': magbins = [20, 24] elif band == 'r': magbins = [20, 23.5] elif band == 'z': magbins = [20, 22.5] slo,shi = -5,5 plt.clf() ha = dict(bins=25, range=(slo,shi), histtype='step', normed=True) y = (mag2 - mag1) / np.hypot(magerr1, magerr2) midmag = [] nn = [] rgbs = [] lt,lp = [],[] for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])): I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)] if len(I) == 0: continue ybin = y[I] rgb = [0.,0.,0.] rgb[0] = float(bini) / (len(magbins)-1) rgb[2] = 1. - rgb[0] n,b,p = plt.hist(ybin, color=rgb, **ha) lt.append('mag %g to %g' % (mlo,mhi)) lp.append(p[0]) midmag.append((mlo+mhi)/2.) nn.append(n) rgbs.append(rgb) bins = [] gaussint = [] for blo,bhi in zip(b, b[1:]): #midbin.append((blo+bhi)/2.) #gaussint.append(scipy.stats.norm.cdf(bhi) - # scipy.stats.norm.cdf(blo)) c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo) c /= (bhi - blo) bins.extend([blo,bhi]) gaussint.extend([c,c]) plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5) plt.legend(lp, lt) plt.title(tt) plt.xlim(slo,shi) ps.savefig() bincenters = b[:-1] + (b[1]-b[0])/2. plt.clf() lp = [] for n,rgb,mlo,mhi in zip(nn, rgbs, magbins, magbins[1:]): p = plt.plot(bincenters, n, '-', color=rgb) lp.append(p[0]) plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2) plt.legend(lp, lt) plt.title(tt) plt.xlim(slo,shi) ps.savefig()
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument('--name1', help='Name for first data set') parser.add_argument('--name2', help='Name for second data set') parser.add_argument('--plot-prefix', default='compare', help='Prefix for plot filenames; default "%default"') parser.add_argument('--match', default=1.0, help='Astrometric cross-match distance in arcsec') parser.add_argument('dir1', help='First directory to compare') parser.add_argument('dir2', help='Second directory to compare') opt = parser.parse_args() ps = PlotSequence(opt.plot_prefix) name1 = opt.name1 if name1 is None: name1 = os.path.basename(opt.dir1) if not len(name1): name1 = os.path.basename(os.path.dirname(opt.dir1)) name2 = opt.name2 if name2 is None: name2 = os.path.basename(opt.dir2) if not len(name2): name2 = os.path.basename(os.path.dirname(opt.dir2)) tt = 'Comparing %s to %s' % (name1, name2) # regex for tractor-*.fits catalog filename catre = re.compile('tractor-.*.fits') cat1, cat2 = [], [] for basedir, cat in [(opt.dir1, cat1), (opt.dir2, cat2)]: for dirpath, dirnames, filenames in os.walk(basedir, followlinks=True): for fn in filenames: if not catre.match(fn): print('Skipping', fn, 'due to filename') continue fn = os.path.join(dirpath, fn) t = fits_table(fn) print(len(t), 'from', fn) cat.append(t) cat1 = merge_tables(cat1, columns='fillzero') cat2 = merge_tables(cat2, columns='fillzero') print('Total of', len(cat1), 'from', name1) print('Total of', len(cat2), 'from', name2) cat1.cut(cat1.brick_primary) cat2.cut(cat2.brick_primary) print('Total of', len(cat1), 'BRICK_PRIMARY from', name1) print('Total of', len(cat2), 'BRICK_PRIMARY from', name2) cat1.cut((cat1.decam_anymask[:, 1] == 0) * (cat1.decam_anymask[:, 2] == 0) * (cat1.decam_anymask[:, 4] == 0)) cat2.cut((cat2.decam_anymask[:, 1] == 0) * (cat2.decam_anymask[:, 2] == 0) * (cat2.decam_anymask[:, 4] == 0)) print('Total of', len(cat1), 'unmasked from', name1) print('Total of', len(cat2), 'unmasked from', name2) I, J, d = match_radec(cat1.ra, cat1.dec, cat2.ra, cat2.dec, opt.match / 3600., nearest=True) print(len(I), 'matched') plt.clf() plt.hist(d * 3600., 100) plt.xlabel('Match distance (arcsec)') plt.title(tt) ps.savefig() matched1 = cat1[I] matched2 = cat2[J] for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]: K = np.flatnonzero((matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0)) print('Median mw_trans', band, 'is', np.median(matched1.decam_mw_transmission[:, iband])) plt.clf() plt.errorbar( matched1.decam_flux[K, iband], matched2.decam_flux[K, iband], fmt='.', color=cc, xerr=1. / np.sqrt(matched1.decam_flux_ivar[K, iband]), yerr=1. / np.sqrt(matched2.decam_flux_ivar[K, iband]), alpha=0.1, ) plt.xlabel('%s flux: %s' % (name1, band)) plt.ylabel('%s flux: %s' % (name2, band)) plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.) plt.axis([-100, 1000, -100, 1000]) plt.title(tt) ps.savefig() for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]: good = ((matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0)) K = np.flatnonzero(good) psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') P = np.flatnonzero(good * psf1 * psf2) mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband]) iv1 = matched1.decam_flux_ivar[:, iband] iv2 = matched2.decam_flux_ivar[:, iband] std = np.sqrt(1. / iv1 + 1. / iv2) plt.clf() plt.plot( mag1[K], (matched2.decam_flux[K, iband] - matched1.decam_flux[K, iband]) / std[K], '.', alpha=0.1, color=cc) plt.plot( mag1[P], (matched2.decam_flux[P, iband] - matched1.decam_flux[P, iband]) / std[P], '.', alpha=0.1, color='k') plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' % (name2, name1, band)) plt.xlabel('%s mag: %s' % (name1, band)) plt.axhline(0, color='k', alpha=0.5) plt.axis([24, 16, -10, 10]) plt.title(tt) ps.savefig() plt.clf() lp, lt = [], [] for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]: good = ((matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0)) #good = True psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband]) iv1 = matched1.decam_flux_ivar[:, iband] iv2 = matched2.decam_flux_ivar[:, iband] std = np.sqrt(1. / iv1 + 1. / iv2) #std = np.hypot(std, 0.01) G = np.flatnonzero(good * psf1 * psf2 * np.isfinite(mag1) * (mag1 >= 20) * (mag1 < dict(g=24, r=23.5, z=22.5)[band])) n, b, p = plt.hist( (matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) / std[G], range=(-4, 4), bins=50, histtype='step', color=cc, normed=True) sig = (matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) / std[G] print('Raw mean and std of points:', np.mean(sig), np.std(sig)) med = np.median(sig) rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2. print('Median and percentile-based sigma:', med, rsigma) lp.append(p[0]) lt.append('%s: %.2f +- %.2f' % (band, med, rsigma)) bins = [] gaussint = [] for blo, bhi in zip(b, b[1:]): c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo) c /= (bhi - blo) #bins.extend([blo,bhi]) #gaussint.extend([c,c]) bins.append((blo + bhi) / 2.) gaussint.append(c) plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5) plt.title(tt) plt.xlabel('Flux difference / error (sigma)') plt.axvline(0, color='k', alpha=0.1) plt.ylim(0, 0.45) plt.legend(lp, lt, loc='upper right') ps.savefig() for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]: plt.clf() mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband]) mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors( matched2.decam_flux[:, iband], matched2.decam_flux_ivar[:, iband]) meanmag = NanoMaggies.nanomaggiesToMag( (matched1.decam_flux[:, iband] + matched2.decam_flux[:, iband]) / 2.) psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') good = ((matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0) * np.isfinite(mag1) * np.isfinite(mag2)) K = np.flatnonzero(good) P = np.flatnonzero(good * psf1 * psf2) plt.errorbar(mag1[K], mag2[K], fmt='.', color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1) plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('%s %s (mag)' % (name2, band)) plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.) plt.axis([24, 16, 24, 16]) plt.title(tt) ps.savefig() plt.clf() plt.errorbar(mag1[K], mag2[K] - mag1[K], fmt='.', color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1) plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axis([24, 16, -1, 1]) plt.title(tt) ps.savefig() magbins = np.arange(16, 24.001, 0.5) plt.clf() plt.plot(mag1[K], (mag2[K] - mag1[K]) / np.hypot(magerr1[K], magerr2[K]), '.', color=cc, alpha=0.1) plt.plot(mag1[P], (mag2[P] - mag1[P]) / np.hypot(magerr1[P], magerr2[P]), 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('(%s %s - %s %s) / errors (sigma)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axis([24, 16, -10, 10]) plt.title(tt) ps.savefig() y = (mag2 - mag1) / np.hypot(magerr1, magerr2) plt.clf() plt.plot(meanmag[P], y[P], 'k.', alpha=0.1) midmag = [] vals = np.zeros((len(magbins) - 1, 5)) median_err1 = [] iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25) # FIXME -- should we do some stats after taking off the mean difference? for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])): I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)] midmag.append((mlo + mhi) / 2.) median_err1.append(np.median(magerr1[I])) if len(I) == 0: continue # median and +- 1 sigma quantiles ybin = y[I] vals[bini, 0] = np.percentile(ybin, 16) vals[bini, 1] = np.median(ybin) vals[bini, 2] = np.percentile(ybin, 84) # +- 2 sigma quantiles vals[bini, 3] = np.percentile(ybin, 2.3) vals[bini, 4] = np.percentile(ybin, 97.7) iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25) print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points') # if iqd > iqd_gauss: # # What error adding in quadrature would you need to make the IQD match? # err = median_err1[-1] # target_err = err * (iqd / iqd_gauss) # sys_err = np.sqrt(target_err**2 - err**2) # print('--> add systematic error', sys_err) # ~ Johan's cuts mlo = 21. mhi = dict(g=24., r=23.5, z=22.5)[band] I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)] ybin = y[I] iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25) print('Mag bin', mlo, mhi, 'band', band, ': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points') if iqd > iqd_gauss: # What error adding in quadrature would you need to make # the IQD match? err = np.median(np.hypot(magerr1[I], magerr2[I])) print('Median error (hypot):', err) target_err = err * (iqd / iqd_gauss) print('Target:', target_err) sys_err = np.sqrt((target_err**2 - err**2) / 2.) print('--> add systematic error', sys_err) # check... err_sys = np.hypot(np.hypot(magerr1, sys_err), np.hypot(magerr2, sys_err)) ysys = (mag2 - mag1) / err_sys ysys = ysys[I] print('Resulting median error:', np.median(err_sys[I])) iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25) print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian') # Hmmm, this doesn't work... totally overshoots. plt.errorbar(midmag, vals[:, 1], fmt='o', color='b', yerr=(vals[:, 1] - vals[:, 0], vals[:, 2] - vals[:, 1]), capthick=3, zorder=20) plt.errorbar(midmag, vals[:, 1], fmt='o', color='b', yerr=(vals[:, 1] - vals[:, 3], vals[:, 4] - vals[:, 1]), capthick=2, zorder=20) plt.axhline(1., color='b', alpha=0.2) plt.axhline(-1., color='b', alpha=0.2) plt.axhline(2., color='b', alpha=0.2) plt.axhline(-2., color='b', alpha=0.2) for mag, err, y in zip(midmag, median_err1, vals[:, 3]): if not np.isfinite(err): continue if y < -6: continue plt.text(mag, y - 0.1, '%.3f' % err, va='top', ha='center', color='k', fontsize=10) plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band)) plt.ylabel('(%s %s - %s %s) / errors (sigma)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axvline(21, color='k', alpha=0.3) plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3) plt.axis([24.1, 16, -6, 6]) plt.title(tt) ps.savefig() #magbins = np.append([16, 18], np.arange(20, 24.001, 0.5)) if band == 'g': magbins = [20, 24] elif band == 'r': magbins = [20, 23.5] elif band == 'z': magbins = [20, 22.5] slo, shi = -5, 5 plt.clf() ha = dict(bins=25, range=(slo, shi), histtype='step', normed=True) y = (mag2 - mag1) / np.hypot(magerr1, magerr2) midmag = [] nn = [] rgbs = [] lt, lp = [], [] for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])): I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)] if len(I) == 0: continue ybin = y[I] rgb = [0., 0., 0.] rgb[0] = float(bini) / (len(magbins) - 1) rgb[2] = 1. - rgb[0] n, b, p = plt.hist(ybin, color=rgb, **ha) lt.append('mag %g to %g' % (mlo, mhi)) lp.append(p[0]) midmag.append((mlo + mhi) / 2.) nn.append(n) rgbs.append(rgb) bins = [] gaussint = [] for blo, bhi in zip(b, b[1:]): #midbin.append((blo+bhi)/2.) #gaussint.append(scipy.stats.norm.cdf(bhi) - # scipy.stats.norm.cdf(blo)) c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo) c /= (bhi - blo) bins.extend([blo, bhi]) gaussint.extend([c, c]) plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5) plt.legend(lp, lt) plt.title(tt) plt.xlim(slo, shi) ps.savefig() bincenters = b[:-1] + (b[1] - b[0]) / 2. plt.clf() lp = [] for n, rgb, mlo, mhi in zip(nn, rgbs, magbins, magbins[1:]): p = plt.plot(bincenters, n, '-', color=rgb) lp.append(p[0]) plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2) plt.legend(lp, lt) plt.title(tt) plt.xlim(slo, shi) ps.savefig()
ps.savefig() print('Tim photocal:', tim.photocal) print('Subtim photocal:', subtim.photocal) # # Order by flux I = np.argsort([-src.getBrightness().getFlux(band) for src in srcs]) for i in I: src = srcs[i] print('Source:', src) print('-> counts', subtim.photocal.brightnessToCounts(src.getBrightness())) # flux in nanomaggies flux = np.array([src.getBrightness().getFlux(band) for src in srcs]) fluxiv = R.IV mag, magerr = NanoMaggies.fluxErrorsToMagErrors(flux, fluxiv) typemap = { ExpGalaxy:'E', DevGalaxy:'D', PointSource:'P' } cat.tractor_type = np.array([typemap[type(src)] for src in srcs]) cat.set('cfht_forced_mag_%s' % band, mag) cat.set('cfht_forced_magerr_%s' % band, magerr) cat.writeto('cfht-forced.fits') plt.clf() plt.plot(cat.acs_mag_auto, mag, 'b.') plt.xlabel('ACS I-band (mag)') plt.ylabel('CFHT %s-band forced phot (mag)' % band) plt.title('CFHT forced phot') ps.savefig()
def set_mags_OldDataModel(self, cat): '''for tractor catalogues with columns like decam_flux.shape(many,6) adds columns to fits_table cat''' # Remove white spaces cat.set('type', np.char.strip(cat.get('type'))) # AB mags # Two kinds, as observed (including dust) and instrinsic (dust removed) for whichmag in ['wdust', 'nodust']: # DECam shp = cat.get('decam_flux').shape mag, mag_err = np.zeros(shp), np.zeros(shp) for iband in range(shp[1]): if whichmag == 'wdust': flux = cat.get('decam_flux')[:, iband] flux_ivar = cat.get('decam_flux_ivar')[:, iband] elif whichmag == 'nodust': flux = cat.get('decam_flux')[:, iband] / cat.get( 'decam_mw_transmission')[:, iband] flux_ivar= cat.get('decam_flux_ivar')[:,iband]*\ np.power(cat.get('decam_mw_transmission')[:,iband],2) else: raise ValueError() mag[:, iband], mag_err[:, iband] = NanoMaggies.fluxErrorsToMagErrors( flux, flux_ivar) cat.set('decam_mag_%s' % whichmag, mag) cat.set('decam_mag_ivar_%s' % whichmag, 1. / np.power(mag_err, 2)) # WISE if 'wise_flux' in cat.get_columns(): shp = cat.get('wise_flux').shape mag, mag_err = np.zeros(shp), np.zeros(shp) for iband in range(shp[1]): if whichmag == 'wdust': flux = cat.get('wise_flux')[:, iband] flux_ivar = cat.get('wise_flux_ivar')[:, iband] elif whichmag == 'nodust': flux = cat.get('wise_flux')[:, iband] / cat.get( 'wise_mw_transmission')[:, iband] flux_ivar= cat.get('wise_flux_ivar')[:,iband]*\ np.power(cat.get('wise_mw_transmission')[:,iband],2) mag[:, iband], mag_err[:, iband] = NanoMaggies.fluxErrorsToMagErrors( flux, flux_ivar) cat.set('wise_mag_%s' % whichmag, mag) cat.set('wise_mag_ivar_%s' % whichmag, 1. / np.power(mag_err, 2)) # Instrinsic fluxes whichmag = 'nodust' # DECam shp = cat.get('decam_flux').shape flux, flux_ivar = np.zeros(shp), np.zeros(shp) for iband in range(shp[1]): flux[:, iband] = cat.get('decam_flux')[:, iband] / cat.get( 'decam_mw_transmission')[:, iband] flux_ivar[:,iband]= cat.get('decam_flux_ivar')[:,iband]*\ np.power(cat.get('decam_mw_transmission')[:,iband],2) cat.set('decam_flux_%s' % whichmag, flux) cat.set('decam_flux_ivar_%s' % whichmag, flux_ivar) # WISE if 'wise_flux' in cat.get_columns(): shp = cat.get('wise_flux').shape flux, flux_err = np.zeros(shp), np.zeros(shp) for iband in range(shp[1]): flux[:, iband] = cat.get('wise_flux')[:, iband] / cat.get( 'wise_mw_transmission')[:, iband] flux_ivar[:,iband]= cat.get('wise_flux_ivar')[:,iband]*\ np.power(cat.get('wise_mw_transmission')[:,iband],2) cat.set('wise_flux_%s' % whichmag, flux) cat.set('wise_flux_ivar_%s' % whichmag, flux_ivar)
print('Tim photocal:', tim.photocal) print('Subtim photocal:', subtim.photocal) # # Order by flux I = np.argsort([-src.getBrightness().getFlux(band) for src in srcs]) for i in I: src = srcs[i] print('Source:', src) print('-> counts', subtim.photocal.brightnessToCounts(src.getBrightness())) # flux in nanomaggies flux = np.array([src.getBrightness().getFlux(band) for src in srcs]) fluxiv = R.IV mag, magerr = NanoMaggies.fluxErrorsToMagErrors(flux, fluxiv) typemap = {ExpGalaxy: 'E', DevGalaxy: 'D', PointSource: 'P'} cat.tractor_type = np.array([typemap[type(src)] for src in srcs]) cat.set('cfht_forced_mag_%s' % band, mag) cat.set('cfht_forced_magerr_%s' % band, magerr) cat.writeto('cfht-forced.fits') plt.clf() plt.plot(cat.acs_mag_auto, mag, 'b.') plt.xlabel('ACS I-band (mag)') plt.ylabel('CFHT %s-band forced phot (mag)' % band) plt.title('CFHT forced phot') ps.savefig()
def all(matched1, matched2, d, name1="ref", name2="test"): tt = "Comparing %s to %s" % (name1, name2) plt.clf() plt.hist(d * 3600.0, 100) plt.xlabel("Match distance (arcsec)") plt.title(tt) plt.savefig(os.path.join(matched1.outdir, "sep_hist.png")) plt.close() for iband, band, cc in [(1, "g", "g"), (2, "r", "r"), (4, "z", "m")]: K = np.flatnonzero( (matched1.t["decam_flux_ivar"][:, iband] > 0) * (matched2.t["decam_flux_ivar"][:, iband] > 0) ) print("Median mw_trans", band, "is", np.median(matched1.t["decam_mw_transmission"][:, iband])) plt.clf() plt.errorbar( matched1.t["decam_flux"][K, iband], matched2.t["decam_flux"][K, iband], fmt=".", color=cc, xerr=1.0 / np.sqrt(matched1.t["decam_flux_ivar"][K, iband]), yerr=1.0 / np.sqrt(matched2.t["decam_flux_ivar"][K, iband]), alpha=0.1, ) plt.xlabel("%s flux: %s" % (name1, band)) plt.ylabel("%s flux: %s" % (name2, band)) plt.plot([-1e6, 1e6], [-1e6, 1e6], "k-", alpha=1.0) plt.axis([-100, 1000, -100, 1000]) plt.title(tt) plt.savefig(os.path.join(matched1.outdir, "%s_fluxerr.png" % band)) plt.close() print("exiting early") sys.exit() for iband, band, cc in [(1, "g", "g"), (2, "r", "r"), (4, "z", "m")]: good = (matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0) K = np.flatnonzero(good) psf1 = matched1.type == "PSF " psf2 = matched2.type == "PSF " P = np.flatnonzero(good * psf1 * psf2) mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband] ) iv1 = matched1.decam_flux_ivar[:, iband] iv2 = matched2.decam_flux_ivar[:, iband] std = np.sqrt(1.0 / iv1 + 1.0 / iv2) plt.clf() plt.plot( mag1[K], (matched2.decam_flux[K, iband] - matched1.decam_flux[K, iband]) / std[K], ".", alpha=0.1, color=cc ) plt.plot( mag1[P], (matched2.decam_flux[P, iband] - matched1.decam_flux[P, iband]) / std[P], ".", alpha=0.1, color="k" ) plt.ylabel("(%s - %s) flux / flux errors (sigma): %s" % (name2, name1, band)) plt.xlabel("%s mag: %s" % (name1, band)) plt.axhline(0, color="k", alpha=0.5) plt.axis([24, 16, -10, 10]) plt.title(tt) ps.savefig() plt.clf() lp, lt = [], [] for iband, band, cc in [(1, "g", "g"), (2, "r", "r"), (4, "z", "m")]: good = (matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0) # good = True psf1 = matched1.type == "PSF " psf2 = matched2.type == "PSF " mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband] ) iv1 = matched1.decam_flux_ivar[:, iband] iv2 = matched2.decam_flux_ivar[:, iband] std = np.sqrt(1.0 / iv1 + 1.0 / iv2) # std = np.hypot(std, 0.01) G = np.flatnonzero( good * psf1 * psf2 * np.isfinite(mag1) * (mag1 >= 20) * (mag1 < dict(g=24, r=23.5, z=22.5)[band]) ) n, b, p = plt.hist( (matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) / std[G], range=(-4, 4), bins=50, histtype="step", color=cc, normed=True, ) sig = (matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) / std[G] print("Raw mean and std of points:", np.mean(sig), np.std(sig)) med = np.median(sig) rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2.0 print("Median and percentile-based sigma:", med, rsigma) lp.append(p[0]) lt.append("%s: %.2f +- %.2f" % (band, med, rsigma)) bins = [] gaussint = [] for blo, bhi in zip(b, b[1:]): c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo) c /= bhi - blo # bins.extend([blo,bhi]) # gaussint.extend([c,c]) bins.append((blo + bhi) / 2.0) gaussint.append(c) plt.plot(bins, gaussint, "k-", lw=2, alpha=0.5) plt.title(tt) plt.xlabel("Flux difference / error (sigma)") plt.axvline(0, color="k", alpha=0.1) plt.ylim(0, 0.45) plt.legend(lp, lt, loc="upper right") ps.savefig() for iband, band, cc in [(1, "g", "g"), (2, "r", "r"), (4, "z", "m")]: plt.clf() mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband] ) mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors( matched2.decam_flux[:, iband], matched2.decam_flux_ivar[:, iband] ) meanmag = NanoMaggies.nanomaggiesToMag((matched1.decam_flux[:, iband] + matched2.decam_flux[:, iband]) / 2.0) psf1 = matched1.type == "PSF " psf2 = matched2.type == "PSF " good = ( (matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0) * np.isfinite(mag1) * np.isfinite(mag2) ) K = np.flatnonzero(good) P = np.flatnonzero(good * psf1 * psf2) plt.errorbar(mag1[K], mag2[K], fmt=".", color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1) plt.plot(mag1[P], mag2[P], "k.", alpha=0.5) plt.xlabel("%s %s (mag)" % (name1, band)) plt.ylabel("%s %s (mag)" % (name2, band)) plt.plot([-1e6, 1e6], [-1e6, 1e6], "k-", alpha=1.0) plt.axis([24, 16, 24, 16]) plt.title(tt) ps.savefig() plt.clf() plt.errorbar(mag1[K], mag2[K] - mag1[K], fmt=".", color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1) plt.plot(mag1[P], mag2[P] - mag1[P], "k.", alpha=0.5) plt.xlabel("%s %s (mag)" % (name1, band)) plt.ylabel("%s %s - %s %s (mag)" % (name2, band, name1, band)) plt.axhline(0.0, color="k", alpha=1.0) plt.axis([24, 16, -1, 1]) plt.title(tt) ps.savefig() magbins = np.arange(16, 24.001, 0.5) plt.clf() plt.plot(mag1[K], (mag2[K] - mag1[K]) / np.hypot(magerr1[K], magerr2[K]), ".", color=cc, alpha=0.1) plt.plot(mag1[P], (mag2[P] - mag1[P]) / np.hypot(magerr1[P], magerr2[P]), "k.", alpha=0.5) plt.xlabel("%s %s (mag)" % (name1, band)) plt.ylabel("(%s %s - %s %s) / errors (sigma)" % (name2, band, name1, band)) plt.axhline(0.0, color="k", alpha=1.0) plt.axis([24, 16, -10, 10]) plt.title(tt) ps.savefig() y = (mag2 - mag1) / np.hypot(magerr1, magerr2) plt.clf() plt.plot(meanmag[P], y[P], "k.", alpha=0.1) midmag = [] vals = np.zeros((len(magbins) - 1, 5)) median_err1 = [] iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25) # FIXME -- should we do some stats after taking off the mean difference? for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])): I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)] midmag.append((mlo + mhi) / 2.0) median_err1.append(np.median(magerr1[I])) if len(I) == 0: continue # median and +- 1 sigma quantiles ybin = y[I] vals[bini, 0] = np.percentile(ybin, 16) vals[bini, 1] = np.median(ybin) vals[bini, 2] = np.percentile(ybin, 84) # +- 2 sigma quantiles vals[bini, 3] = np.percentile(ybin, 2.3) vals[bini, 4] = np.percentile(ybin, 97.7) iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25) print( "Mag bin", midmag[-1], ": IQD is factor", iqd / iqd_gauss, "vs expected for Gaussian;", len(ybin), "points", ) # if iqd > iqd_gauss: # # What error adding in quadrature would you need to make the IQD match? # err = median_err1[-1] # target_err = err * (iqd / iqd_gauss) # sys_err = np.sqrt(target_err**2 - err**2) # print('--> add systematic error', sys_err) # ~ Johan's cuts mlo = 21.0 mhi = dict(g=24.0, r=23.5, z=22.5)[band] I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)] ybin = y[I] iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25) print( "Mag bin", mlo, mhi, "band", band, ": IQD is factor", iqd / iqd_gauss, "vs expected for Gaussian;", len(ybin), "points", ) if iqd > iqd_gauss: # What error adding in quadrature would you need to make # the IQD match? err = np.median(np.hypot(magerr1[I], magerr2[I])) print("Median error (hypot):", err) target_err = err * (iqd / iqd_gauss) print("Target:", target_err) sys_err = np.sqrt((target_err ** 2 - err ** 2) / 2.0) print("--> add systematic error", sys_err) # check... err_sys = np.hypot(np.hypot(magerr1, sys_err), np.hypot(magerr2, sys_err)) ysys = (mag2 - mag1) / err_sys ysys = ysys[I] print("Resulting median error:", np.median(err_sys[I])) iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25) print("--> IQD", iqd_sys / iqd_gauss, "vs Gaussian") # Hmmm, this doesn't work... totally overshoots. plt.errorbar( midmag, vals[:, 1], fmt="o", color="b", yerr=(vals[:, 1] - vals[:, 0], vals[:, 2] - vals[:, 1]), capthick=3, zorder=20, ) plt.errorbar( midmag, vals[:, 1], fmt="o", color="b", yerr=(vals[:, 1] - vals[:, 3], vals[:, 4] - vals[:, 1]), capthick=2, zorder=20, ) plt.axhline(1.0, color="b", alpha=0.2) plt.axhline(-1.0, color="b", alpha=0.2) plt.axhline(2.0, color="b", alpha=0.2) plt.axhline(-2.0, color="b", alpha=0.2) for mag, err, y in zip(midmag, median_err1, vals[:, 3]): if not np.isfinite(err): continue if y < -6: continue plt.text(mag, y - 0.1, "%.3f" % err, va="top", ha="center", color="k", fontsize=10) plt.xlabel("(%s + %s)/2 %s (mag), PSFs" % (name1, name2, band)) plt.ylabel("(%s %s - %s %s) / errors (sigma)" % (name2, band, name1, band)) plt.axhline(0.0, color="k", alpha=1.0) plt.axvline(21, color="k", alpha=0.3) plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color="k", alpha=0.3) plt.axis([24.1, 16, -6, 6]) plt.title(tt) ps.savefig() # magbins = np.append([16, 18], np.arange(20, 24.001, 0.5)) if band == "g": magbins = [20, 24] elif band == "r": magbins = [20, 23.5] elif band == "z": magbins = [20, 22.5] slo, shi = -5, 5 plt.clf() ha = dict(bins=25, range=(slo, shi), histtype="step", normed=True) y = (mag2 - mag1) / np.hypot(magerr1, magerr2) midmag = [] nn = [] rgbs = [] lt, lp = [], [] for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])): I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)] if len(I) == 0: continue ybin = y[I] rgb = [0.0, 0.0, 0.0] rgb[0] = float(bini) / (len(magbins) - 1) rgb[2] = 1.0 - rgb[0] n, b, p = plt.hist(ybin, color=rgb, **ha) lt.append("mag %g to %g" % (mlo, mhi)) lp.append(p[0]) midmag.append((mlo + mhi) / 2.0) nn.append(n) rgbs.append(rgb) bins = [] gaussint = [] for blo, bhi in zip(b, b[1:]): # midbin.append((blo+bhi)/2.) # gaussint.append(scipy.stats.norm.cdf(bhi) - # scipy.stats.norm.cdf(blo)) c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo) c /= bhi - blo bins.extend([blo, bhi]) gaussint.extend([c, c]) plt.plot(bins, gaussint, "k-", lw=2, alpha=0.5) plt.legend(lp, lt) plt.title(tt) plt.xlim(slo, shi) ps.savefig() bincenters = b[:-1] + (b[1] - b[0]) / 2.0 plt.clf() lp = [] for n, rgb, mlo, mhi in zip(nn, rgbs, magbins, magbins[1:]): p = plt.plot(bincenters, n, "-", color=rgb) lp.append(p[0]) plt.plot(bincenters, gaussint[::2], "k-", alpha=0.5, lw=2) plt.legend(lp, lt) plt.title(tt) plt.xlim(slo, shi) ps.savefig()
def plot_light_curves(pfn, ucal=False): lightcurves = unpickle_from_file(pfn) if ucal: tag = 'ucal-' else: tag = '' survey = LegacySurveyData() brickname = '0364m042' catfn = survey.find_file('tractor', brick=brickname) print('Reading catalog from', catfn) cat = fits_table(catfn) print(len(cat), 'catalog entries') cat.cut(cat.brick_primary) print(len(cat), 'brick primary') I = [] for i, oid in enumerate(cat.objid): if (brickname, oid) in lightcurves: I.append(i) I = np.array(I) cat.cut(I) print('Cut to', len(cat), 'with light curves') S = fits_table('specObj-dr12-trim-2.fits') from astrometry.libkd.spherematch import match_radec I, J, d = match_radec(S.ra, S.dec, cat.ra, cat.dec, 2. / 3600.) print('Matched', len(I), 'to spectra') plt.subplots_adjust(hspace=0) movie_jpegs = [] movie_wcs = None for i in range(28): fn = os.path.join('des-sn-movie', 'epoch%i' % i, 'coadd', brickname[:3], brickname, 'legacysurvey-%s-image.jpg' % brickname) print(fn) if not os.path.exists(fn): continue img = plt.imread(fn) img = np.flipud(img) h, w, d = img.shape fn = os.path.join('des-sn-movie', 'epoch%i' % i, 'coadd', brickname[:3], brickname, 'legacysurvey-%s-image-r.fits' % brickname) if not os.path.exists(fn): continue wcs = Tan(fn) movie_jpegs.append(img) movie_wcs = wcs plt.figure(figsize=(8, 6), dpi=100) n = 0 fluxtags = [('flux', 'flux_ivar', '', 'a')] if ucal: fluxtags.append(('uflux', 'uflux_ivar', ': ucal', 'b')) for oid, ii in zip(cat.objid[J], I): print('Objid', oid) spec = S[ii] k = (brickname, oid) v = lightcurves[k] # Cut bad CCDs v.cut(np.array([e not in [230151, 230152, 230153] for e in v.expnum])) plt.clf() print('obj', k, 'has', len(v), 'measurements') T = v for fluxtag, fluxivtag, fluxname, plottag in fluxtags: plt.clf() filts = np.unique(T.filter) for i, f in enumerate(filts): from tractor.brightness import NanoMaggies plt.subplot(len(filts), 1, i + 1) fluxes = np.hstack( [T.get(ft[0])[T.filter == f] for ft in fluxtags]) fluxes = fluxes[np.isfinite(fluxes)] mn, mx = np.percentile(fluxes, [5, 95]) print('Flux percentiles for filter', f, ':', mn, mx) # note swap mn, mx = NanoMaggies.nanomaggiesToMag( mx), NanoMaggies.nanomaggiesToMag(mn) print('-> mags', mn, mx) cut = (T.filter == f) * (T.flux_ivar > 0) if ucal: cut *= np.isfinite(T.uflux) I = np.flatnonzero(cut) print(' ', len(I), 'in', f, 'band') I = I[np.argsort(T.mjd[I])] mediv = np.median(T.flux_ivar[I]) # cut really noisy ones I = I[T.flux_ivar[I] > 0.25 * mediv] #plt.plot(T.mjd[I], T.flux[I], '.-', color=dict(g='g',r='r',z='m')[f]) # plt.errorbar(T.mjd[I], T.flux[I], yerr=1/np.sqrt(T.fluxiv[I]), # fmt='.-', color=dict(g='g',r='r',z='m')[f]) #plt.errorbar(T.mjd[I], T.flux[I], yerr=1/np.sqrt(T.fluxiv[I]), # fmt='.', color=dict(g='g',r='r',z='m')[f]) # if ucal: # mag,dmag = NanoMaggies.fluxErrorsToMagErrors(T.flux[I], T.flux_ivar[I]) # else: # mag,dmag = NanoMaggies.fluxErrorsToMagErrors(T.uflux[I], T.uflux_ivar[I]) mag, dmag = NanoMaggies.fluxErrorsToMagErrors( T.get(fluxtag)[I], T.get(fluxivtag)[I]) plt.errorbar(T.mjd[I], mag, yerr=dmag, fmt='.', color=dict(g='g', r='r', z='m')[f]) #yl,yh = plt.ylim() #plt.ylim(yh,yl) plt.ylim(mx, mn) plt.ylabel(f) if i + 1 < len(filts): plt.xticks([]) #plt.yscale('symlog') outfn = 'cutout_%.4f_%.4f.jpg' % (spec.ra, spec.dec) if not os.path.exists(outfn): url = 'http://legacysurvey.org/viewer/jpeg-cutout/?ra=%.4f&dec=%.4f&zoom=14&layer=sdssco&size=128' % ( spec.ra, spec.dec) cmd = 'wget -O %s "%s"' % (outfn, url) print(cmd) os.system(cmd) pix = plt.imread(outfn) h, w, d = pix.shape fig = plt.gcf() #print('fig bbox:', fig.bbox) #print('xmax, ymax', fig.bbox.xmax, fig.bbox.ymax) #plt.figimage(pix, 0, fig.bbox.ymax - h, zorder=10) #plt.figimage(pix, 0, fig.bbox.ymax, zorder=10) #plt.figimage(pix, fig.bbox.xmax - w, fig.bbox.ymax, zorder=10) plt.figimage(pix, fig.bbox.xmax - (w + 2), fig.bbox.ymax - (h + 2), zorder=10) plt.suptitle('SDSS spectro object: %s at (%.4f, %.4f)%s' % (spec.label.strip(), spec.ra, spec.dec, fluxname)) plt.savefig('forced-%s%i-%s.png' % (tag, n, plottag)) ok, x, y = movie_wcs.radec2pixelxy(spec.ra, spec.dec) x = int(np.round(x - 1)) y = int(np.round(y - 1)) sz = 32 plt.clf() plt.subplots_adjust(hspace=0, wspace=0) k = 1 for i, img in enumerate(movie_jpegs): stamp = img[y - sz:y + sz + 1, x - sz:x + sz + 1] plt.subplot(5, 6, k) plt.imshow(stamp, interpolation='nearest', origin='lower') plt.xticks([]) plt.yticks([]) k += 1 plt.suptitle('SDSS spectro object: %s at (%.4f, %.4f): DES images' % (spec.label.strip(), spec.ra, spec.dec)) plt.savefig('forced-%s%i-c.png' % (tag, n)) n += 1
def all(matched1, matched2, d, name1='ref', name2='test'): tt = 'Comparing %s to %s' % (name1, name2) plt.clf() plt.hist(d * 3600., 100) plt.xlabel('Match distance (arcsec)') plt.title(tt) plt.savefig(os.path.join(matched1.outdir, 'sep_hist.png')) plt.close() for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]: K = np.flatnonzero((matched1.t['decam_flux_ivar'][:, iband] > 0) * (matched2.t['decam_flux_ivar'][:, iband] > 0)) print('Median mw_trans', band, 'is', np.median(matched1.t['decam_mw_transmission'][:, iband])) plt.clf() plt.errorbar( matched1.t['decam_flux'][K, iband], matched2.t['decam_flux'][K, iband], fmt='.', color=cc, xerr=1. / np.sqrt(matched1.t['decam_flux_ivar'][K, iband]), yerr=1. / np.sqrt(matched2.t['decam_flux_ivar'][K, iband]), alpha=0.1, ) plt.xlabel('%s flux: %s' % (name1, band)) plt.ylabel('%s flux: %s' % (name2, band)) plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.) plt.axis([-100, 1000, -100, 1000]) plt.title(tt) plt.savefig(os.path.join(matched1.outdir, '%s_fluxerr.png' % band)) plt.close() print("exiting early") sys.exit() for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]: good = ((matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0)) K = np.flatnonzero(good) psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') P = np.flatnonzero(good * psf1 * psf2) mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband]) iv1 = matched1.decam_flux_ivar[:, iband] iv2 = matched2.decam_flux_ivar[:, iband] std = np.sqrt(1. / iv1 + 1. / iv2) plt.clf() plt.plot( mag1[K], (matched2.decam_flux[K, iband] - matched1.decam_flux[K, iband]) / std[K], '.', alpha=0.1, color=cc) plt.plot( mag1[P], (matched2.decam_flux[P, iband] - matched1.decam_flux[P, iband]) / std[P], '.', alpha=0.1, color='k') plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' % (name2, name1, band)) plt.xlabel('%s mag: %s' % (name1, band)) plt.axhline(0, color='k', alpha=0.5) plt.axis([24, 16, -10, 10]) plt.title(tt) ps.savefig() plt.clf() lp, lt = [], [] for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]: good = ((matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0)) #good = True psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband]) iv1 = matched1.decam_flux_ivar[:, iband] iv2 = matched2.decam_flux_ivar[:, iband] std = np.sqrt(1. / iv1 + 1. / iv2) #std = np.hypot(std, 0.01) G = np.flatnonzero(good * psf1 * psf2 * np.isfinite(mag1) * (mag1 >= 20) * (mag1 < dict(g=24, r=23.5, z=22.5)[band])) n, b, p = plt.hist( (matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) / std[G], range=(-4, 4), bins=50, histtype='step', color=cc, normed=True) sig = (matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) / std[G] print('Raw mean and std of points:', np.mean(sig), np.std(sig)) med = np.median(sig) rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2. print('Median and percentile-based sigma:', med, rsigma) lp.append(p[0]) lt.append('%s: %.2f +- %.2f' % (band, med, rsigma)) bins = [] gaussint = [] for blo, bhi in zip(b, b[1:]): c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo) c /= (bhi - blo) #bins.extend([blo,bhi]) #gaussint.extend([c,c]) bins.append((blo + bhi) / 2.) gaussint.append(c) plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5) plt.title(tt) plt.xlabel('Flux difference / error (sigma)') plt.axvline(0, color='k', alpha=0.1) plt.ylim(0, 0.45) plt.legend(lp, lt, loc='upper right') ps.savefig() for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]: plt.clf() mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband]) mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors( matched2.decam_flux[:, iband], matched2.decam_flux_ivar[:, iband]) meanmag = NanoMaggies.nanomaggiesToMag( (matched1.decam_flux[:, iband] + matched2.decam_flux[:, iband]) / 2.) psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') good = ((matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0) * np.isfinite(mag1) * np.isfinite(mag2)) K = np.flatnonzero(good) P = np.flatnonzero(good * psf1 * psf2) plt.errorbar(mag1[K], mag2[K], fmt='.', color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1) plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('%s %s (mag)' % (name2, band)) plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.) plt.axis([24, 16, 24, 16]) plt.title(tt) ps.savefig() plt.clf() plt.errorbar(mag1[K], mag2[K] - mag1[K], fmt='.', color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1) plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axis([24, 16, -1, 1]) plt.title(tt) ps.savefig() magbins = np.arange(16, 24.001, 0.5) plt.clf() plt.plot(mag1[K], (mag2[K] - mag1[K]) / np.hypot(magerr1[K], magerr2[K]), '.', color=cc, alpha=0.1) plt.plot(mag1[P], (mag2[P] - mag1[P]) / np.hypot(magerr1[P], magerr2[P]), 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('(%s %s - %s %s) / errors (sigma)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axis([24, 16, -10, 10]) plt.title(tt) ps.savefig() y = (mag2 - mag1) / np.hypot(magerr1, magerr2) plt.clf() plt.plot(meanmag[P], y[P], 'k.', alpha=0.1) midmag = [] vals = np.zeros((len(magbins) - 1, 5)) median_err1 = [] iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25) # FIXME -- should we do some stats after taking off the mean difference? for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])): I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)] midmag.append((mlo + mhi) / 2.) median_err1.append(np.median(magerr1[I])) if len(I) == 0: continue # median and +- 1 sigma quantiles ybin = y[I] vals[bini, 0] = np.percentile(ybin, 16) vals[bini, 1] = np.median(ybin) vals[bini, 2] = np.percentile(ybin, 84) # +- 2 sigma quantiles vals[bini, 3] = np.percentile(ybin, 2.3) vals[bini, 4] = np.percentile(ybin, 97.7) iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25) print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points') # if iqd > iqd_gauss: # # What error adding in quadrature would you need to make the IQD match? # err = median_err1[-1] # target_err = err * (iqd / iqd_gauss) # sys_err = np.sqrt(target_err**2 - err**2) # print('--> add systematic error', sys_err) # ~ Johan's cuts mlo = 21. mhi = dict(g=24., r=23.5, z=22.5)[band] I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)] ybin = y[I] iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25) print('Mag bin', mlo, mhi, 'band', band, ': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points') if iqd > iqd_gauss: # What error adding in quadrature would you need to make # the IQD match? err = np.median(np.hypot(magerr1[I], magerr2[I])) print('Median error (hypot):', err) target_err = err * (iqd / iqd_gauss) print('Target:', target_err) sys_err = np.sqrt((target_err**2 - err**2) / 2.) print('--> add systematic error', sys_err) # check... err_sys = np.hypot(np.hypot(magerr1, sys_err), np.hypot(magerr2, sys_err)) ysys = (mag2 - mag1) / err_sys ysys = ysys[I] print('Resulting median error:', np.median(err_sys[I])) iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25) print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian') # Hmmm, this doesn't work... totally overshoots. plt.errorbar(midmag, vals[:, 1], fmt='o', color='b', yerr=(vals[:, 1] - vals[:, 0], vals[:, 2] - vals[:, 1]), capthick=3, zorder=20) plt.errorbar(midmag, vals[:, 1], fmt='o', color='b', yerr=(vals[:, 1] - vals[:, 3], vals[:, 4] - vals[:, 1]), capthick=2, zorder=20) plt.axhline(1., color='b', alpha=0.2) plt.axhline(-1., color='b', alpha=0.2) plt.axhline(2., color='b', alpha=0.2) plt.axhline(-2., color='b', alpha=0.2) for mag, err, y in zip(midmag, median_err1, vals[:, 3]): if not np.isfinite(err): continue if y < -6: continue plt.text(mag, y - 0.1, '%.3f' % err, va='top', ha='center', color='k', fontsize=10) plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band)) plt.ylabel('(%s %s - %s %s) / errors (sigma)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axvline(21, color='k', alpha=0.3) plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3) plt.axis([24.1, 16, -6, 6]) plt.title(tt) ps.savefig() #magbins = np.append([16, 18], np.arange(20, 24.001, 0.5)) if band == 'g': magbins = [20, 24] elif band == 'r': magbins = [20, 23.5] elif band == 'z': magbins = [20, 22.5] slo, shi = -5, 5 plt.clf() ha = dict(bins=25, range=(slo, shi), histtype='step', normed=True) y = (mag2 - mag1) / np.hypot(magerr1, magerr2) midmag = [] nn = [] rgbs = [] lt, lp = [], [] for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])): I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)] if len(I) == 0: continue ybin = y[I] rgb = [0., 0., 0.] rgb[0] = float(bini) / (len(magbins) - 1) rgb[2] = 1. - rgb[0] n, b, p = plt.hist(ybin, color=rgb, **ha) lt.append('mag %g to %g' % (mlo, mhi)) lp.append(p[0]) midmag.append((mlo + mhi) / 2.) nn.append(n) rgbs.append(rgb) bins = [] gaussint = [] for blo, bhi in zip(b, b[1:]): #midbin.append((blo+bhi)/2.) #gaussint.append(scipy.stats.norm.cdf(bhi) - # scipy.stats.norm.cdf(blo)) c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo) c /= (bhi - blo) bins.extend([blo, bhi]) gaussint.extend([c, c]) plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5) plt.legend(lp, lt) plt.title(tt) plt.xlim(slo, shi) ps.savefig() bincenters = b[:-1] + (b[1] - b[0]) / 2. plt.clf() lp = [] for n, rgb, mlo, mhi in zip(nn, rgbs, magbins, magbins[1:]): p = plt.plot(bincenters, n, '-', color=rgb) lp.append(p[0]) plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2) plt.legend(lp, lt) plt.title(tt) plt.xlim(slo, shi) ps.savefig()
def main(): ps = PlotSequence('dr3-summary') #fns = glob('/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-*.fits') #fns = glob('/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-240p005-250p010.fits') fns = ['dr3.1/sweep/3.1/sweep-240p005-250p010.fits', # '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-240p010-250p015.fits', # '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-230p005-240p010.fits', # '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-230p010-240p015.fits' ] plt.figure(2, figsize=(6,4)) plt.subplots_adjust(left=0.1, bottom=0.15, right=0.98, top=0.98) plt.figure(1) TT = [] for fn in fns: T = fits_table(fn) print(len(T), 'from', fn) TT.append(T) T = merge_tables(TT) del TT print('Types:', Counter(T.type)) T.flux_g = T.decam_flux[:,1] T.flux_r = T.decam_flux[:,2] T.flux_z = T.decam_flux[:,4] T.flux_ivar_g = T.decam_flux_ivar[:,1] T.flux_ivar_r = T.decam_flux_ivar[:,2] T.flux_ivar_z = T.decam_flux_ivar[:,4] T.mag_g, T.magerr_g = NanoMaggies.fluxErrorsToMagErrors(T.flux_g, T.flux_ivar_g) T.mag_r, T.magerr_r = NanoMaggies.fluxErrorsToMagErrors(T.flux_r, T.flux_ivar_r) T.mag_z, T.magerr_z = NanoMaggies.fluxErrorsToMagErrors(T.flux_z, T.flux_ivar_z) T.deflux_g = T.flux_g / T.decam_mw_transmission[:,1] T.deflux_r = T.flux_r / T.decam_mw_transmission[:,2] T.deflux_z = T.flux_z / T.decam_mw_transmission[:,4] T.deflux_ivar_g = T.flux_ivar_g * T.decam_mw_transmission[:,1]**2 T.deflux_ivar_r = T.flux_ivar_r * T.decam_mw_transmission[:,2]**2 T.deflux_ivar_z = T.flux_ivar_z * T.decam_mw_transmission[:,4]**2 T.demag_g, T.demagerr_g = NanoMaggies.fluxErrorsToMagErrors(T.deflux_g, T.deflux_ivar_g) T.demag_r, T.demagerr_r = NanoMaggies.fluxErrorsToMagErrors(T.deflux_r, T.deflux_ivar_r) T.demag_z, T.demagerr_z = NanoMaggies.fluxErrorsToMagErrors(T.deflux_z, T.deflux_ivar_z) T.typex = np.array([t[0] for t in T.type]) T.mag_w1, T.magerr_w1 = NanoMaggies.fluxErrorsToMagErrors( T.wise_flux[:,0], T.wise_flux_ivar[:,0]) T.mag_w2, T.magerr_w2 = NanoMaggies.fluxErrorsToMagErrors( T.wise_flux[:,1], T.wise_flux_ivar[:,1]) tt = 'P' I2 = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) * (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) * (T.magerr_g < 0.05) * (T.magerr_r < 0.05) * (T.magerr_z < 0.05) * (T.typex == tt)) print(len(I2), 'with < 0.05-mag errs and type PSF') plt.clf() plt.hist(T.mag_w1[I2], bins=100, range=[12,24], histtype='step', color='b') plt.hist(T.mag_w2[I2], bins=100, range=[12,24], histtype='step', color='g') plt.xlabel('WISE mag') ps.savefig() plt.clf() plt.hist(T.magerr_w1[I2], bins=100, range=[0,1], histtype='step', color='b') plt.hist(T.magerr_w2[I2], bins=100, range=[0,1], histtype='step', color='g') plt.xlabel('WISE mag errors') ps.savefig() I2 = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) * (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) * (T.magerr_g < 0.05) * (T.magerr_r < 0.05) * (T.magerr_z < 0.05) * (T.typex == tt) * (T.magerr_w1 < 0.05)) #(T.magerr_w2 < 0.1) * print(len(I2), 'with < 0.05-mag errs and type PSF and W1 errors < 0.1') Q1 = fits_table('dr3.1/external/survey-dr3-DR12Q.fits') Q2 = fits_table('dr3.1/external/DR12Q.fits', columns=['z_pipe']) Q1.add_columns_from(Q2) I = np.flatnonzero(Q1.objid >= 0) Q1.cut(I) print(len(Q1), 'sources with DR12Q matches') Q = Q1 Q.typex = np.array([t[0] for t in Q.type]) Q.mag_w1, Q.magerr_w1 = NanoMaggies.fluxErrorsToMagErrors( Q.wise_flux[:,0], Q.wise_flux_ivar[:,0]) Q.mag_w2, Q.magerr_w2 = NanoMaggies.fluxErrorsToMagErrors( Q.wise_flux[:,1], Q.wise_flux_ivar[:,1]) Q.flux_g = Q.decam_flux[:,1] Q.flux_r = Q.decam_flux[:,2] Q.flux_z = Q.decam_flux[:,4] Q.flux_ivar_g = Q.decam_flux_ivar[:,1] Q.flux_ivar_r = Q.decam_flux_ivar[:,2] Q.flux_ivar_z = Q.decam_flux_ivar[:,4] Q.deflux_g = Q.flux_g / Q.decam_mw_transmission[:,1] Q.deflux_r = Q.flux_r / Q.decam_mw_transmission[:,2] Q.deflux_z = Q.flux_z / Q.decam_mw_transmission[:,4] Q.deflux_ivar_g = Q.flux_ivar_g * Q.decam_mw_transmission[:,1]**2 Q.deflux_ivar_r = Q.flux_ivar_r * Q.decam_mw_transmission[:,2]**2 Q.deflux_ivar_z = Q.flux_ivar_z * Q.decam_mw_transmission[:,4]**2 Q.demag_g, Q.demagerr_g = NanoMaggies.fluxErrorsToMagErrors(Q.deflux_g, Q.deflux_ivar_g) Q.demag_r, Q.demagerr_r = NanoMaggies.fluxErrorsToMagErrors(Q.deflux_r, Q.deflux_ivar_r) Q.demag_z, Q.demagerr_z = NanoMaggies.fluxErrorsToMagErrors(Q.deflux_z, Q.deflux_ivar_z) I = np.flatnonzero((Q.flux_ivar_g > 0) * (Q.flux_ivar_r > 0) * (Q.flux_ivar_z > 0) * (Q.flux_g > 0) * (Q.flux_r > 0) * (Q.flux_z > 0) * (Q.demagerr_g < 0.05) * (Q.demagerr_r < 0.05) * (Q.demagerr_z < 0.05) * (Q.typex == tt) * (Q.magerr_w1 < 0.05)) Q.cut(I) print(len(Q), 'DR12Q-matched entries of type PSF with g,r,z,W1 errs < 0.05') plt.clf() loghist(T.demag_g[I2] - T.demag_r[I2], T.demag_z[I2] - T.mag_w1[I2], nbins=200, range=((-0.5,2.0),(-3,4))) plt.xlabel('g - r (mag)') plt.ylabel('z - W1 (mag)') ps.savefig() #Q.cut(np.argsort(Q.z_pipe)) plt.clf() plt.scatter(Q.demag_g - Q.demag_r, Q.demag_z - Q.mag_w1, c=Q.z_pipe, s=5) plt.colorbar(label='QSO redshift') plt.xlabel('g - r (mag)') plt.ylabel('z - W1 (mag)') ps.savefig() plt.figure(2) from scipy.ndimage.filters import gaussian_filter import matplotlib.cm xlo,xhi,ylo,yhi = [-0.25, 1.75, -2.5, 2.5] plt.clf() H,xe,ye = loghist(T.demag_g[I2] - T.demag_r[I2], T.demag_z[I2] - T.mag_w1[I2], nbins=(400,200), range=((xlo,xhi),(ylo,yhi)), imshowargs=dict(cmap='Greys'), hot=False, docolorbar=False) I = np.flatnonzero(Q.z_pipe < 10.) QH,nil,nil = np.histogram2d(Q.demag_g[I] - Q.demag_r[I], Q.demag_z[I] - Q.mag_w1[I], bins=200, range=((xlo,xhi),(ylo,yhi))) QH = gaussian_filter(QH.T, 3.) c = plt.contour(QH, 8, extent=[xe.min(), xe.max(), ye.min(), ye.max()], colors='k') plt.axis([xlo,xhi,ylo,yhi]) plt.xlabel('g - r (mag)') plt.ylabel('z - W1 (mag)') mx,my,mz = [],[],[] # I = np.argsort(Q.z_pipe) # for i in range(len(Q)/1000): # J = I[i*1000:][:1000] # mx.append(np.median(Q.demag_g[J] - Q.demag_r[J])) # my.append(np.median(Q.demag_z[J] - Q.mag_w1[J])) # mz.append(np.median(Q.z_pipe[J])) # plt.scatter(mx,my, c=mz) # plt.colorbar(label='QSO Redshift') # ps.savefig() zvals = np.arange(0, 3.61, 0.1) for zlo,zhi in zip(zvals, zvals[1:]): J = np.flatnonzero((Q.z_pipe >= zlo) * (Q.z_pipe < zhi)) mx.append(np.median(Q.demag_g[J] - Q.demag_r[J])) my.append(np.median(Q.demag_z[J] - Q.mag_w1[J])) mz.append(np.median(Q.z_pipe[J])) # plt.scatter(mx,my, c=mz, marker='o-') # plt.colorbar(label='QSO Redshift') for i in range(len(mx)-1): c = matplotlib.cm.viridis(0.8 * i / (len(mx)-1)) plt.plot([mx[i], mx[i+1]], [my[i],my[i+1]], '-', color=c, lw=4) plt.savefig('qso-wise.png') plt.savefig('qso-wise.pdf') xlo,xhi,ylo,yhi = [-0.25, 1.75, -0.25, 2.5] plt.clf() H,xe,ye = loghist(T.demag_g[I2] - T.demag_r[I2], T.demag_r[I2] - T.demag_z[I2], nbins=(400,200), range=((xlo,xhi),(ylo,yhi)), imshowargs=dict(cmap='Greys'), hot=False, docolorbar=False) #I3 = np.random.permutation(I2)[:1000] #plt.plot(T.demag_g[I3] - T.demag_r[I3], T.demag_r[I3] - T.demag_z[I3], 'r.') I = np.flatnonzero(Q.z_pipe < 10.) QH,nil,nil = np.histogram2d(Q.demag_g[I] - Q.demag_r[I], Q.demag_r[I] - Q.demag_z[I], bins=200, range=((xlo,xhi),(ylo,yhi))) QH = gaussian_filter(QH.T, 3.) c = plt.contour(QH, 8, extent=[xe.min(), xe.max(), ye.min(), ye.max()], colors='k') plt.axis([xlo,xhi,ylo,yhi]) plt.xlabel('g - r (mag)') plt.ylabel('r - z (mag)') mx,my,mz = [],[],[] zvals = np.arange(0, 3.61, 0.1) for zlo,zhi in zip(zvals, zvals[1:]): J = np.flatnonzero((Q.z_pipe >= zlo) * (Q.z_pipe < zhi)) mx.append(np.median(Q.demag_g[J] - Q.demag_r[J])) my.append(np.median(Q.demag_r[J] - Q.demag_z[J])) mz.append(np.median(Q.z_pipe[J])) for i in range(len(mx)-1): c = matplotlib.cm.viridis(0.8 * i / (len(mx)-1)) plt.plot([mx[i], mx[i+1]], [my[i],my[i+1]], '-', color=c, lw=4) plt.savefig('qso-optical.png') plt.savefig('qso-optical.pdf') plt.figure(1) # I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0)) # print(len(I), 'with g,r,z fluxes') # I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) * # (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0)) # print(len(I), 'with positive g,r,z fluxes') # I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) * # (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) * # (T.magerr_g < 0.2) * (T.magerr_r < 0.2) * (T.magerr_z < 0.2)) # print(len(I), 'with < 0.2-mag errs') # I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) * # (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) * # (T.magerr_g < 0.1) * (T.magerr_r < 0.1) * (T.magerr_z < 0.1)) # print(len(I), 'with < 0.1-mag errs') # # I2 = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) * # (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) * # (T.magerr_g < 0.05) * (T.magerr_r < 0.05) * (T.magerr_z < 0.05)) # print(len(I2), 'with < 0.05-mag errs') #ha = dict(nbins=200, range=((-0.5, 2.5), (-0.5, 2.5))) ha = dict(nbins=400, range=((-0.5, 2.5), (-0.5, 2.5))) plt.clf() #plothist(T.mag_g[I] - T.mag_r[I], T.mag_r[I] - T.mag_z[I], 200) #loghist(T.mag_g[I] - T.mag_r[I], T.mag_r[I] - T.mag_z[I], **ha) loghist(T.demag_g[I] - T.demag_r[I], T.demag_r[I] - T.demag_z[I], **ha) plt.xlabel('g - r (mag)') plt.ylabel('r - z (mag)') ps.savefig() hists = [] hists2 = [] for tt,name in [('P', 'PSF'), ('E', 'EXP'), ('D', 'DeV')]: I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) * (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) * (T.magerr_g < 0.1) * (T.magerr_r < 0.1) *(T.magerr_z < 0.1) * (T.typex == tt)) print(len(I), 'with < 0.1-mag errs and type', name) # plt.clf() # H,xe,ye = loghist(T.mag_g[I] - T.mag_r[I], T.mag_r[I] - T.mag_z[I], **ha) # plt.xlabel('g - r (mag)') # plt.ylabel('r - z (mag)') # plt.title('Type = %s' % name) # ps.savefig() plt.clf() H,xe,ye = loghist(T.demag_g[I] - T.demag_r[I], T.demag_r[I] - T.demag_z[I], **ha) # Keeping the dereddened color-color histograms hists.append(H.T) plt.xlabel('g - r (mag)') plt.ylabel('r - z (mag)') plt.title('Type = %s, dereddened' % name) ps.savefig() I2 = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) * (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) * (T.magerr_g < 0.05) * (T.magerr_r < 0.05) * (T.magerr_z < 0.05) * (T.typex == tt)) print(len(I2), 'with < 0.05-mag errs and type', name) plt.clf() H,xe,ye = loghist(T.demag_g[I2] - T.demag_r[I2], T.demag_r[I2] - T.demag_z[I2], **ha) hists2.append(H.T) plt.xlabel('g - r (mag)') plt.ylabel('r - z (mag)') plt.title('Type = %s, dereddened, 5%% errs' % name) ps.savefig() for H in hists: H /= H.max() #for H in hists2: # H /= H.max() #hists2 = [np.clip(H / np.percentile(H.ravel(), 99), 0, 1) for H in hists2] hists2 = [np.clip(H / np.percentile(H.ravel(), 99.9), 0, 1) for H in hists2] rgbmap = np.dstack((hists[2], hists[0], hists[1])) plt.clf() ((xlo,xhi),(ylo,yhi)) = ha['range'] plt.imshow(rgbmap, interpolation='nearest', origin='lower', extent=[xlo,xhi,ylo,yhi]) plt.axis([0,2,0,2]) plt.xlabel('g - r (mag)') plt.ylabel('r - z (mag)') plt.title('Red: DeV, Blue: Exp, Green: PSF') ps.savefig() plt.clf() plt.imshow(np.sqrt(rgbmap), interpolation='nearest', origin='lower', extent=[xlo,xhi,ylo,yhi]) plt.axis([0,2,0,2]) plt.xlabel('g - r (mag)') plt.ylabel('r - z (mag)') plt.title('Red: DeV, Blue: Exp, Green: PSF') ps.savefig() # For a white-background plot, mix colors like paint for hr,hg,hb in [(hists[2], hists[0], hists[1]), (hists2[2], hists2[0], hists2[1]),]: H,W = hr.shape for mapping in [lambda x: x]: #, lambda x: np.sqrt(x), lambda x: x**2]: red = np.ones((H,W,3)) red[:,:,1] = 1 - mapping(hr) red[:,:,2] = 1 - mapping(hr) # in matplotlib, 'green'->(0, 0.5, 0) green = np.ones((H,W,3)) green[:,:,0] = 1 - mapping(hg) green[:,:,1] = 1 - 0.5*mapping(hg) green[:,:,2] = 1 - mapping(hg) blue = np.ones((H,W,3)) blue[:,:,0] = 1 - mapping(hb) blue[:,:,1] = 1 - mapping(hb) plt.clf() plt.imshow(red * green * blue, origin='lower', interpolation='nearest', extent=[xlo,xhi,ylo,yhi], aspect='auto') plt.xlabel('g - r (mag)') plt.ylabel('r - z (mag)') plt.xticks(np.arange(0, 2.1, 0.5)) plt.yticks(np.arange(0, 2.1, 0.5)) plt.axis([-0.25,2,0,2]) plt.title('Red: DeV, Blue: Exp, Green: PSF') ps.savefig() plt.figure(2) plt.clf() plt.imshow(red * green * blue, origin='lower', interpolation='nearest', extent=[xlo,xhi,ylo,yhi], aspect='auto') plt.xlabel('g - r (mag)') plt.ylabel('r - z (mag)') plt.xticks(np.arange(0, 2.1, 0.5)) plt.yticks(np.arange(0, 2.1, 0.5)) plt.axis([0,2,-0.25,2]) plt.text(0.7, 0.1, 'PSF', ha='center', va='center', color='k') plt.text(0.6, 0.7, 'EXP', ha='center', va='center', color='k') plt.text(1.5, 0.6, 'DEV', ha='center', va='center', color='k') plt.savefig('color-type.pdf') plt.figure(1) sys.exit(0) np.random.seed(42) T.gr = T.mag_g - T.mag_r T.rz = T.mag_r - T.mag_z for tt,name in [('P', 'PSF'), ('E', 'EXP'), ('D', 'DeV')]: I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) * (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) * (T.magerr_g < 0.1) * (T.magerr_r < 0.1) *(T.magerr_z < 0.1) * # Bright cut (T.mag_r > 16.) * # Contamination cut (T.decam_fracflux[:,2] < 0.5) * # Mag range cut (T.mag_r > 19.) * (T.mag_r < 20.) * (T.typex == tt)) print(len(I), 'with < 0.1-mag errs and type', name) # plt.clf() # ha = dict(histtype='step', range=(16, 24), bins=100) # plt.hist(T.mag_g[I], color='g', **ha) # plt.hist(T.mag_r[I], color='r', **ha) # plt.hist(T.mag_z[I], color='m', **ha) # plt.title('Type %s' % name) # ps.savefig() J = np.random.permutation(I) if tt == 'D': J = J[T.shapedev_r[J] < 3.] J = J[:100] rzbin = np.argsort(T.rz[J]) rzblock = (rzbin / 10).astype(int) K = np.lexsort((T.gr[J], rzblock)) #print('sorted rzblock', rzblock[K]) #print('sorted rzbin', rzbin[K]) J = J[K] # plt.clf() # plt.hist(T.decam_fracflux[J,2], label='fracflux_r', histtype='step', bins=20) # plt.hist(T.decam_fracmasked[J,2], label='fracmasked_r', histtype='step', bins=20) # plt.legend() # plt.title('Type %s' % name) # ps.savefig() #print('fracflux r', T.decam_fracflux[J,2]) #print('fracmasked r', T.decam_fracmasked[J,2]) #print('anymasked r', T.decam_anymask[J,2]) imgs = [] imgrow = [] stackrows = [] for i in J: fn = 'cutouts/cutout_%.4f_%.4f.jpg' % (T.ra[i], T.dec[i]) if not os.path.exists(fn): url = 'http://legacysurvey.org/viewer/jpeg-cutout/?layer=decals-dr3&pixscale=0.262&size=25&ra=%f&dec=%f' % (T.ra[i], T.dec[i]) print('URL', url) cmd = 'wget -O %s.tmp "%s" && mv %s.tmp %s' % (fn, url, fn, fn) os.system(cmd) img = plt.imread(fn) #print('Image', img.shape) extra = '' if tt == 'D': extra = ', shapedev_r %.2f' % T.shapedev_r[i] elif tt == 'E': extra = ', shapeexp_r %.2f' % T.shapeexp_r[i] print(name, 'RA,Dec %.4f,%.4f, g/r/z %.2f, %.2f, %.2f, fracflux_r %.3f, fracmasked_r %.3f, anymasked_r %4i%s' % ( T.ra[i], T.dec[i], T.mag_g[i], T.mag_r[i], T.mag_z[i], T.decam_fracflux[i,2], T.decam_fracmasked[i,2], T.decam_anymask[i,2], extra)) imgrow.append(img) if len(imgrow) == 10: stack = np.hstack(imgrow) print('stacked row:', stack.shape) stackrows.append(stack) imgs.append(imgrow) imgrow = [] stack = np.vstack(stackrows) print('Stack', stack.shape) plt.clf() plt.imshow(stack, interpolation='nearest', origin='lower') plt.xticks([]) plt.yticks([]) plt.title('Type %s' % name) ps.savefig() # Wrap-around at RA=300 # rax = T.ra + (-360 * T.ra > 300.) # # rabins = np.arange(-60., 300., 0.2) # decbins = np.arange(-20, 35, 0.2) # # print('RA bins:', len(rabins)) # print('Dec bins:', len(decbins)) # # for name,cut in [(None, 'All sources'), # (T.type == 'PSF '), 'PSF'), # (T.type == 'SIMP'), 'SIMP'), # (T.type == 'DEV '), 'DEV'), # (T.type == 'EXP '), 'EXP'), # (T.type == 'COMP'), 'COMP'), # ]: # # H,xe,ye = np.histogram2d(T.ra, T.dec, bins=(rabins, decbins)) # print('H shape', H.shape) # # H = H.T # # H /= (rabins[1:] - rabins[:-1])[np.newaxis, :] # H /= ((decbins[1:] - decbins[:-1]) * np.cos(np.deg2rad((decbins[1:] + decbins[:-1]) / 2.))))[:, np.newaxis] # # plt.clf() # plt.imshow(H, extent=(rabins[0], rabins[-1], decbins[0], decbins[-1]), interpolation='nearest', origin='lower', # vmin=0) # cbar = plt.colorbar() # cbar.set_label('Source density per square degree') # plt.xlim(rabins[-1], rabins[0]) # plt.xlabel('RA (deg)') # plt.ylabel('Dec (deg)') # ps.savefig() '''