def h3(): from sequels import read_photoobjs from wisecat import wise_catalog_radecbox ''' collect results via: cp h3/h3-merged-r120-d0.fits h3.fits for ((r=121;r<240; r++)); do echo $r; tabmerge h3/h3-merged-r${r}-d0.fits+1 h3.fits+1; done for ((d=1;d<10;d++)); do for ((r=121;r<240; r++)); do echo $d $r; tabmerge h3/h3-merged-r${r}-d${d}.fits+1 h3.fits+1; done; done liststruc h3.fits ''' d0 = 19. d1 = 20. r0 = 120. r1 = 121. lvl = logging.INFO #lvl = logging.DEBUG logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout) dirname = 'h3' # d0 = 19. # d1 = 20. # r0 = 120. # r1 = 121. DD = np.arange(0, 10+1) RR = np.arange(120, 240+1) RRDD = [] for d0,d1 in zip(DD, DD[1:]): for r0,r1 in zip(RR, RR[1:]): RRDD.append((r0,r1,d0,d1)) for r0,r1,d0,d1 in RRDD: print 'Reading photoObjs...' fn = os.path.join(dirname, 'h3-sdss-r%.0f-d%.0f.fits' % (r0, d0)) if not os.path.exists(fn): cols = ['objid', 'ra', 'dec', 'objc_type', #'modelflux', 'modelflux_ivar', #'psfflux', 'psfflux_ivar', 'modelmag', 'modelmagerr', 'psfmag', 'psfmagerr', 'resolve_status', 'nchild', 'flags', 'objc_flags', 'run','camcol','field','id'] SDSS = read_photoobjs(r0, r1, d0, d1, 0.01, cols=cols) print 'Got', len(SDSS) SDSS.writeto(fn) else: print 'Reading', fn SDSS = fits_table(fn) print 'Got', len(SDSS) wfn = os.path.join(dirname, 'h3-wise-r%.0f-d%.0f.fits' % (r0,d0)) print 'looking for', wfn if os.path.exists(wfn): WISE = fits_table(wfn) print 'Read', len(WISE), 'WISE sources nearby' else: cols = (['ra','dec','cntr'] + ['w%impro'%band for band in [1,2,3,4]] + ['w%isigmpro'%band for band in [1,2,3,4]] + ['w%isnr'%band for band in [1,2,3,4]]) WISE = wise_catalog_radecbox(r0, r1, d0, d1, cols=cols) WISE.writeto(wfn) print 'Found', len(WISE), 'WISE sources nearby' WISE.cut(WISE.w4snr >= 5) print 'Cut to', len(WISE), 'on W4snr' WISE.cut((WISE.w1snr >= 5) * (WISE.w2snr >= 5)) print 'Cut to', len(WISE), 'on W[12]snr' I,J,d = match_radec(WISE.ra, WISE.dec, SDSS.ra, SDSS.dec, 1./3600., nearest=True) print 'Matched', len(I) WISE.match_dist = np.zeros(len(WISE)) WISE.match_dist[I] = d WISE.matched = np.zeros(len(WISE), bool) WISE.matched[I] = True for c in SDSS.get_columns(): print 'SDSS column', c S = SDSS.get(c) sh = (len(WISE),) if len(S.shape) > 1: sh = sh + S.shape[1:] X = np.zeros(sh, S.dtype) X[I] = S[J] WISE.set('sdss_' + c, X) fn = os.path.join(dirname, 'h3-merged-r%.0f-d%.0f.fits' % (r0,d0)) WISE.writeto(fn) print 'Wrote', fn
dhi = d1 + margin if True: # HACK -- got the WISE catalogs on riemann and the WISE exposures on NERSC... from wisecat import wise_catalog_radecbox cols=['cntr', 'ra', 'dec', 'sigra', 'sigdec', 'cc_flags', 'ext_flg', 'var_flg', 'moon_lev', 'ph_qual', 'w1mpro', 'w1sigmpro', 'w1sat', 'w1nm', 'w1m', 'w1snr', 'w1cov', 'w1mag', 'w1sigm', 'w1flg', 'w2mpro', 'w2sigmpro', 'w2sat', 'w2nm', 'w2m', 'w2snr', 'w2cov', 'w2mag', 'w2sigm', 'w2flg', 'w3mpro', 'w3sigmpro', 'w3sat', 'w3nm', 'w3m', 'w3snr', 'w3cov', 'w3mag', 'w3sigm', 'w3flg', 'w4mpro', 'w4sigmpro', 'w4sat', 'w4nm', 'w4m', 'w4snr', 'w4cov', 'w4mag', 'w4sigm', 'w4flg', ] W = wise_catalog_radecbox(rlo, rhi, dlo, dhi, cols=cols) if W is None: W = fits_table() for c in cols: W.set(c, np.array([])) wfn = os.path.join(datadir, 'wise-objs-%s.fits' % dataset) W.writeto(wfn) print 'Wrote', wfn continue for band in [1,2,3,4]: pfn = os.path.join(datadir, '%s-w%i-stage108.pickle' % (dataset, band)) if os.path.exists(pfn): print 'Output exists:', pfn, '; skipping' continue args.append((dataset, band, rlo, rhi, dlo, dhi, T, sfn))
def wise_sources(bands, pred=False): S = 25 if pred: psfs = wise_psf(bands=bands, fftcentral=S*2+1, noplots=True) print 'psfs:', len(psfs) (Sx,Sy,Px,Py,sf) = psfs[0] print 'sx:', len(Sx) print 'sf', sf pitch = 0.1 print 'Pitch', pitch print 'lanczos_test...' lan3 = lanczos_test(pitch=pitch, srange=(-S-0.5+pitch/2., S+0.5))#, noplots=True) print 'lanczos_test finished.' (Slan,Plan,sflan) = lan3[0] sflan /= pitch sflan = sflan[:S+1] Plan = Plan[:S+1] Slan = Slan[:S+1] print 'lan3:', len(Slan) print 'sflan:', sflan print 'Plan:', Plan ps.basefn = 'wisestars' ps.skipto(0) wfn = 'wbox.fits' if os.path.exists(wfn): W = fits_table(wfn) print 'Read', len(W), 'from', wfn else: from wisecat import wise_catalog_radecbox W = wise_catalog_radecbox(150., 200., 45., 47.) print 'Got', len(W), 'WISE sources' W.writeto(wfn) from unwise_coadd import get_l1b_file, wisedir, zeropointToScale # plt.clf() # plt.hist(W.w1snr, 50, histtype='step', color='b') # plt.hist(W.w1snr[W.w1sat == 0], 50, histtype='step', color='r') # ps.savefig() # Find isolated stars... I,J,d = match_radec(W.ra, W.dec, W.ra, W.dec, 20. * 2.75 / 3600., notself=True) keep = np.ones(len(W), bool) keep[J] = False print np.sum(keep), 'of', len(W), 'have no neighbors' for iband,band in enumerate(bands): W.snr = W.get('w%isnr'%band) W.sat = W.get('w%isat' % band) Wi = W[keep * (W.sat == 0)] print len(Wi), 'after cuts' #(W.snr > 20) * # plt.clf() # plt.hist(Wi.snr, 50, histtype='step', color='m') Wi.cut(Wi.ext_flg == 0) print 'ext_flg:', len(Wi) Wi.cut(Wi.na == 0) print 'na:', len(Wi) Wi.cut(Wi.nb == 1) print 'nb:', len(Wi) # plt.hist(Wi.snr, 50, histtype='step', color='k') # ps.savefig() T = Wi[np.argsort(-Wi.snr)] T.cut(T.snr > 25.) print 'SN cut:', len(T) #T = Wi[np.argsort(-Wi.snr)[:25]] #print 'Coadd S/N:', T.snr # band = 1 # T = fits_table('w%i.fits' % band) # # plt.clf() # n,b,p = plt.hist(T.get('w%impro' % band), 50, histtype='step', color='b') # plt.xlabel('w%impro' % band) # # print 'blend_ext_flags:', ['0x%x' % x for x in np.unique(T.blend_ext_flags)] # # T.cut(T.blend_ext_flags & 0xf == 1) # print 'Cut to', len(T), 'on blend' # T.cut(T.get('w%isat' % band) == 0) # print 'Cut to', len(T), 'on sat' # T.cut(T.get('w%iflg' % band) == 0) # print 'Cut to', len(T), 'on flg' # # plt.hist(T.get('w%impro' % band), histtype='step', color='r', bins=b) # ps.savefig() dbs = [] l1dbs = [] #print 'Coadds', T.coadd_id for t in T: print 'coadd', t.coadd_id #print 'RA,Dec', t.ra, t.dec coadd = t.coadd_id.replace('_ab41', '').strip() for base in ['data/unwise', 'data/unwise-nersc']: dirnm = os.path.join(base, coadd[:3], coadd) fn = os.path.join(dirnm, 'unwise-%s-w%i-img-m.fits' % (coadd, band)) if os.path.exists(fn): break img = fitsio.read(fn) #print 'img', img.shape, img.dtype wcs = Tan(fn) ok,x,y = wcs.radec2pixelxy(t.ra, t.dec) x -= 1 y -= 1 #print 'x,y', x,y ix = int(np.round(x)) iy = int(np.round(y)) im = img[iy-S:iy+S+1, ix-S:ix+S+1] if im.shape != (2*S+1, 2*S+1): continue imH,imW = im.shape imax = np.argmax(im) my,mx = np.unravel_index(imax, im.shape) #print 'max pixel:', my,mx, im[my,mx] #print 'center pixel:', imH/2,imW/2, im[imH/2,imW/2] if my != imH/2 or mx != imW/2: continue # F = np.fft.fft2(im) # S = np.fft.fftshift(F) # P = S.real**2 + S.imag**2 # P /= P.max() # freqs = np.fft.fftfreq(W) # sfreqs = np.fft.fftshift(freqs) # ff = np.hypot(sfreqs.reshape(1,W), sfreqs.reshape(W,1)) # print 'ff', ff.shape #plt.clf() #plt.plot(ff.ravel(), P.ravel(), 'b.', alpha=0.1) #ps.savefig() Y = im[imH/2, :] X = im[:, imW/2] Fy = np.fft.fft(Y) Fx = np.fft.fft(X) Sy = np.fft.fftshift(Fy) Sx = np.fft.fftshift(Fx) Px = Sx.real**2 + Sx.imag**2 Py = Sy.real**2 + Sy.imag**2 # Grab "blank" lines nearby to assess noise n = im[imH/4, :] Fn = np.fft.fft(n) Sn = np.fft.fftshift(Fn) Pn = Sn.real**2 + Sn.imag**2 freqs1 = np.fft.fftfreq(len(Y)) sfreqs1 = np.fft.fftshift(freqs1) s0 = np.flatnonzero(sfreqs1 == 0) s0 = s0[0] s1 = s0 dbs.append((Sx[s1:], Sy[s1:], Px[s1:], Py[s1:], sfreqs1[s1:], Pn[s1:])) print 'sx', len(Sx[s1:]) print 'sf', sfreqs1[s1:] continue fn = os.path.join(dirnm, 'unwise-%s-w%i-frames.fits' % (coadd, band)) T = fits_table(fn) print 'Read', len(T), 'from', fn T.cut(T.included > 0) T.cut(T.use > 0) print 'Cut to', len(T), 'used' T.cut(T.coextent[:,0] < ix - S) T.cut(T.coextent[:,1] > ix + S) T.cut(T.coextent[:,2] < iy - S) T.cut(T.coextent[:,3] > iy + S) print 'Cut to', len(T), 'containing target pixels' #ims = [] #for f in T[:10]: for f in T: fn = get_l1b_file(wisedir, f.scan_id, f.frame_num, band) print 'frame', fn img,hdr = fitsio.read(fn, header=True) img -= f.sky1 gain = hdr.get('FEBGAIN') print 'Gain:', gain sig1 = np.sqrt(1./f.weight) zpscale = 1. / zeropointToScale(f.zeropoint) sig1 /= zpscale print 'Sigma1:', sig1 print 'Sky:', f.sky1 wcs = Sip(fn) mask = fitsio.read(fn.replace('-int-', '-msk-') + '.gz') #print 'RA,Dec', t.ra, t.dec ok,x,y = wcs.radec2pixelxy(t.ra, t.dec) #print 'x,y', x,y x -= 1 y -= 1 fx = int(np.round(x)) fy = int(np.round(y)) im = img[fy-S:fy+S+1, fx-S:fx+S+1] if im.shape != (2*S+1, 2*S+1): continue imH,imW = im.shape mask = mask[fy-S:fy+S+1, fx-S:fx+S+1] ok = (mask == 0) if not patch_image(im, ok): print 'Patching failed' continue Y = im[imH/2, :] X = im[:, imW/2] Fy = np.fft.fft(Y) Fx = np.fft.fft(X) Sy = np.fft.fftshift(Fy) Sx = np.fft.fftshift(Fx) Px = Sx.real**2 + Sx.imag**2 Py = Sy.real**2 + Sy.imag**2 freqs1 = np.fft.fftfreq(len(Y)) sfreqs1 = np.fft.fftshift(freqs1) s0 = np.flatnonzero(sfreqs1 == 0) s0 = s0[0] # Grab "blank" lines nearby to assess noise n = im[imH/4, :] Fn = np.fft.fft(n) Sn = np.fft.fftshift(Fn) Pn = Sn.real**2 + Sn.imag**2 l1dbs.append((Px[s0+1:], Py[s0+1:], sfreqs1[s0+1:], Pn[s0+1:])) #ims.append(im) # plt.clf() # for i,im in enumerate(ims[:9]): # plt.subplot(3,3,1+i) # plt.imshow(im, interpolation='nearest', origin='lower') # ps.savefig() print len(dbs), 'coadds' print len(l1dbs), 'L1bs' plt.figure(figsize=(4,4)) left = 0.17 right = 0.03 bot = 0.12 top = 0.03 plt.subplots_adjust(left=left, right=1.-right, bottom=bot, top=1.-top) if pred: (Sxpsf, Sypsf, Pxpsf, Pypsf, sfpsf) = psfs[iband] plt.clf() Spsf = (Sxpsf + Sypsf) / 2. Ppsf = Spsf.real**2 + Spsf.imag**2 plt.plot(sfpsf, 10.*np.log10(Pxpsf / Pxpsf[0]), 'r-') plt.plot(sfpsf, 10.*np.log10(Pypsf / Pypsf[0]), 'm-') plt.plot(sfpsf, 10.*np.log10(Ppsf / Ppsf[0]), '-', color=(1.,0.6,0)) plt.plot(sflan, 10.*np.log10(Plan / Plan[0]), 'k-') #print 'sflan', sflan #print 'Plan', Plan #print 'db', 10.*np.log10(Plan / Plan[0]) for Sx,Sy,Px,Py,sfreq,Pn in dbs: Sm = (Sx + Sy) / 2. P = Sm.real**2 + Sm.imag**2 plt.plot(sfreq, 10.*np.log10(P / P[0]), 'b-', alpha=0.1) Spred = Spsf * Slan Ppred = Spred.real**2 + Spred.imag**2 plt.plot(sfreq, 10.*np.log10(Ppred / Ppred[0]), 'g-') plt.ylabel('dB') plt.ylim(-40, 3) plt.title('PSF') ps.savefig() plt.clf() for Sx,Sy,Px,Py,sfreq,Pn in dbs: plt.plot(sfreq, 10.*np.log10(Py / Py[0]), 'b-', alpha=0.2) plt.plot(sfreq, 10.*np.log10(Px / Px[0]), 'g-', alpha=0.2) plt.plot(sfreq, 10.*np.log10(Pn / np.sqrt(Px[0]*Py[0])), 'r-', alpha=0.2) plt.ylabel('dB') plt.ylim(-40, 3) plt.title('Isolated sources in coadd: W%i' % band) ps.savefig() xx = np.zeros((len(dbs), len(dbs[0][0]))) yy = np.zeros((len(dbs), len(dbs[0][0]))) nn = np.zeros((len(dbs), len(dbs[0][0]))) mm = np.zeros((len(dbs), len(dbs[0][0]))) for i,(Sx,Sy,Px,Py,sfreq,Pn) in enumerate(dbs): xx[i,:] = 10.*np.log10(Px / Px[0]) yy[i,:] = 10.*np.log10(Py / Py[0]) nn[i,:] = 10.*np.log10(Pn / np.sqrt(Px[0]*Py[0])) Sm = (Sx + Sy) / 2. P = Sm.real**2 + Sm.imag**2 mm[i,:] = 10.*np.log10(P / P[0]) if pred: plt.clf() med = np.median(mm, axis=0) plt.plot(sfreq, med, 'g-') plt.plot(sfpsf, 10.*np.log10(Ppsf / Ppsf[0]), '-', color=(1.,0.6,0)) plt.plot(sflan, 10.*np.log10(Plan / Plan[0]), 'k-') plt.plot(sfreq, 10.*np.log10(Ppred / Ppred[0]), 'r-') plt.ylabel('dB') plt.ylim(-40, 3) plt.title('PSF') ps.savefig() plt.clf() plt.plot(sfreq, np.median(xx, axis=0), 'g-') plt.plot(sfreq, np.median(yy, axis=0), 'b-') plt.plot(sfreq, np.median(nn, axis=0), 'r-') plt.title('Isolated sources in coadd: W%i median' % band) #plt.xlabel('frequency (cycles/pixel)') plt.ylabel('dB') plt.ylim(-40, 3) ps.savefig() print 'Freqs:', sfreq if len(l1dbs) == 0: continue plt.clf() for Px,Py,sf,noise in l1dbs: plt.plot(sf, 10.*np.log10(Px / Px[0]), 'g-', alpha=0.2) plt.plot(sf, 10.*np.log10(Py / Py[0]), 'b-', alpha=0.2) plt.plot(sf, 10.*np.log10(noise / Py[0]), 'r-', alpha=0.2) plt.plot(sf, 10.*np.log10(noise / Px[0]), 'r-', alpha=0.2) plt.ylabel('dB') plt.ylim(-40, 3) plt.title('Isolated sources in L1bs: W%i' % band) ps.savefig() xx = np.zeros((len(l1dbs), len(l1dbs[0][0]))) yy = np.zeros((len(l1dbs), len(l1dbs[0][0]))) zz = np.zeros((len(l1dbs), len(l1dbs[0][0]))) zz2 = np.zeros((len(l1dbs), len(l1dbs[0][0]))) for i,(Px,Py,sfreq,noise) in enumerate(l1dbs): xx[i,:] = 10.*np.log10(Px / Px[0]) yy[i,:] = 10.*np.log10(Py / Py[0]) zz[i,:] = 10.*np.log10(noise / Py[0]) zz2[i,:] = 10.*np.log10(noise / Px[0]) plt.clf() plt.plot(sfreq, np.median(xx, axis=0), 'g-') plt.plot(sfreq, np.median(yy, axis=0), 'b-') plt.plot(sfreq, np.median(zz, axis=0), 'r-') plt.plot(sfreq, np.median(zz2, axis=0), 'r-') plt.title('Isolated sources in L1bs: W%i median' % band) #plt.xlabel('frequency (cycles/pixel)') plt.ylabel('dB') plt.ylim(-40, 3) ps.savefig()
dhi = d1 + margin if True: # HACK -- got the WISE catalogs on riemann and the WISE exposures on NERSC... from wisecat import wise_catalog_radecbox cols=['cntr', 'ra', 'dec', 'sigra', 'sigdec', 'cc_flags', 'ext_flg', 'var_flg', 'moon_lev', 'ph_qual', 'w1mpro', 'w1sigmpro', 'w1sat', 'w1nm', 'w1m', 'w1snr', 'w1cov', 'w1mag', 'w1sigm', 'w1flg', 'w2mpro', 'w2sigmpro', 'w2sat', 'w2nm', 'w2m', 'w2snr', 'w2cov', 'w2mag', 'w2sigm', 'w2flg', 'w3mpro', 'w3sigmpro', 'w3sat', 'w3nm', 'w3m', 'w3snr', 'w3cov', 'w3mag', 'w3sigm', 'w3flg', 'w4mpro', 'w4sigmpro', 'w4sat', 'w4nm', 'w4m', 'w4snr', 'w4cov', 'w4mag', 'w4sigm', 'w4flg', ] W = wise_catalog_radecbox(rlo, rhi, dlo, dhi, cols=cols) if W is None: W = fits_table() for c in cols: W.set(c, np.array([])) wfn = os.path.join(datadir, 'wise-objs-%s.fits' % dataset) W.writeto(wfn) print('Wrote', wfn) continue for band in [1,2,3,4]: pfn = os.path.join(datadir, '%s-w%i-stage108.pickle' % (dataset, band)) if os.path.exists(pfn): print('Output exists:', pfn, '; skipping') continue args.append((dataset, band, rlo, rhi, dlo, dhi, T, sfn))