def h3():
from sequels import read_photoobjs
from wisecat import wise_catalog_radecbox
'''
collect results via:
cp h3/h3-merged-r120-d0.fits h3.fits
for ((r=121;r<240; r++)); do echo $r; tabmerge h3/h3-merged-r${r}-d0.fits+1 h3.fits+1; done
for ((d=1;d<10;d++)); do for ((r=121;r<240; r++)); do echo $d $r; tabmerge h3/h3-merged-r${r}-d${d}.fits+1 h3.fits+1; done; done
liststruc h3.fits
'''
d0 = 19.
d1 = 20.
r0 = 120.
r1 = 121.
lvl = logging.INFO
#lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
dirname = 'h3'
# d0 = 19.
# d1 = 20.
# r0 = 120.
# r1 = 121.
DD = np.arange(0, 10+1)
RR = np.arange(120, 240+1)
RRDD = []
for d0,d1 in zip(DD, DD[1:]):
for r0,r1 in zip(RR, RR[1:]):
RRDD.append((r0,r1,d0,d1))
for r0,r1,d0,d1 in RRDD:
print 'Reading photoObjs...'
fn = os.path.join(dirname, 'h3-sdss-r%.0f-d%.0f.fits' % (r0, d0))
if not os.path.exists(fn):
cols = ['objid', 'ra', 'dec', 'objc_type',
#'modelflux', 'modelflux_ivar',
#'psfflux', 'psfflux_ivar',
'modelmag', 'modelmagerr',
'psfmag', 'psfmagerr',
'resolve_status', 'nchild', 'flags', 'objc_flags',
'run','camcol','field','id']
SDSS = read_photoobjs(r0, r1, d0, d1, 0.01, cols=cols)
print 'Got', len(SDSS)
SDSS.writeto(fn)
else:
print 'Reading', fn
SDSS = fits_table(fn)
print 'Got', len(SDSS)
wfn = os.path.join(dirname, 'h3-wise-r%.0f-d%.0f.fits' % (r0,d0))
print 'looking for', wfn
if os.path.exists(wfn):
WISE = fits_table(wfn)
print 'Read', len(WISE), 'WISE sources nearby'
else:
cols = (['ra','dec','cntr'] + ['w%impro'%band for band in [1,2,3,4]] +
['w%isigmpro'%band for band in [1,2,3,4]] +
['w%isnr'%band for band in [1,2,3,4]])
WISE = wise_catalog_radecbox(r0, r1, d0, d1, cols=cols)
WISE.writeto(wfn)
print 'Found', len(WISE), 'WISE sources nearby'
WISE.cut(WISE.w4snr >= 5)
print 'Cut to', len(WISE), 'on W4snr'
WISE.cut((WISE.w1snr >= 5) * (WISE.w2snr >= 5))
print 'Cut to', len(WISE), 'on W[12]snr'
I,J,d = match_radec(WISE.ra, WISE.dec, SDSS.ra, SDSS.dec, 1./3600., nearest=True)
print 'Matched', len(I)
WISE.match_dist = np.zeros(len(WISE))
WISE.match_dist[I] = d
WISE.matched = np.zeros(len(WISE), bool)
WISE.matched[I] = True
for c in SDSS.get_columns():
print 'SDSS column', c
S = SDSS.get(c)
sh = (len(WISE),)
if len(S.shape) > 1:
sh = sh + S.shape[1:]
X = np.zeros(sh, S.dtype)
X[I] = S[J]
WISE.set('sdss_' + c, X)
fn = os.path.join(dirname, 'h3-merged-r%.0f-d%.0f.fits' % (r0,d0))
WISE.writeto(fn)
print 'Wrote', fn
dhi = d1 + margin
if True:
# HACK -- got the WISE catalogs on riemann and the WISE exposures on NERSC...
from wisecat import wise_catalog_radecbox
cols=['cntr', 'ra', 'dec', 'sigra', 'sigdec', 'cc_flags',
'ext_flg', 'var_flg', 'moon_lev', 'ph_qual',
'w1mpro', 'w1sigmpro', 'w1sat', 'w1nm', 'w1m',
'w1snr', 'w1cov', 'w1mag', 'w1sigm', 'w1flg',
'w2mpro', 'w2sigmpro', 'w2sat', 'w2nm', 'w2m',
'w2snr', 'w2cov', 'w2mag', 'w2sigm', 'w2flg',
'w3mpro', 'w3sigmpro', 'w3sat', 'w3nm', 'w3m',
'w3snr', 'w3cov', 'w3mag', 'w3sigm', 'w3flg',
'w4mpro', 'w4sigmpro', 'w4sat', 'w4nm', 'w4m',
'w4snr', 'w4cov', 'w4mag', 'w4sigm', 'w4flg', ]
W = wise_catalog_radecbox(rlo, rhi, dlo, dhi, cols=cols)
if W is None:
W = fits_table()
for c in cols:
W.set(c, np.array([]))
wfn = os.path.join(datadir, 'wise-objs-%s.fits' % dataset)
W.writeto(wfn)
print 'Wrote', wfn
continue
for band in [1,2,3,4]:
pfn = os.path.join(datadir, '%s-w%i-stage108.pickle' % (dataset, band))
if os.path.exists(pfn):
print 'Output exists:', pfn, '; skipping'
continue
args.append((dataset, band, rlo, rhi, dlo, dhi, T, sfn))
def wise_sources(bands, pred=False):
S = 25
if pred:
psfs = wise_psf(bands=bands, fftcentral=S*2+1, noplots=True)
print 'psfs:', len(psfs)
(Sx,Sy,Px,Py,sf) = psfs[0]
print 'sx:', len(Sx)
print 'sf', sf
pitch = 0.1
print 'Pitch', pitch
print 'lanczos_test...'
lan3 = lanczos_test(pitch=pitch, srange=(-S-0.5+pitch/2., S+0.5))#, noplots=True)
print 'lanczos_test finished.'
(Slan,Plan,sflan) = lan3[0]
sflan /= pitch
sflan = sflan[:S+1]
Plan = Plan[:S+1]
Slan = Slan[:S+1]
print 'lan3:', len(Slan)
print 'sflan:', sflan
print 'Plan:', Plan
ps.basefn = 'wisestars'
ps.skipto(0)
wfn = 'wbox.fits'
if os.path.exists(wfn):
W = fits_table(wfn)
print 'Read', len(W), 'from', wfn
else:
from wisecat import wise_catalog_radecbox
W = wise_catalog_radecbox(150., 200., 45., 47.)
print 'Got', len(W), 'WISE sources'
W.writeto(wfn)
from unwise_coadd import get_l1b_file, wisedir, zeropointToScale
# plt.clf()
# plt.hist(W.w1snr, 50, histtype='step', color='b')
# plt.hist(W.w1snr[W.w1sat == 0], 50, histtype='step', color='r')
# ps.savefig()
# Find isolated stars...
I,J,d = match_radec(W.ra, W.dec, W.ra, W.dec, 20. * 2.75 / 3600., notself=True)
keep = np.ones(len(W), bool)
keep[J] = False
print np.sum(keep), 'of', len(W), 'have no neighbors'
for iband,band in enumerate(bands):
W.snr = W.get('w%isnr'%band)
W.sat = W.get('w%isat' % band)
Wi = W[keep * (W.sat == 0)]
print len(Wi), 'after cuts'
#(W.snr > 20) *
# plt.clf()
# plt.hist(Wi.snr, 50, histtype='step', color='m')
Wi.cut(Wi.ext_flg == 0)
print 'ext_flg:', len(Wi)
Wi.cut(Wi.na == 0)
print 'na:', len(Wi)
Wi.cut(Wi.nb == 1)
print 'nb:', len(Wi)
# plt.hist(Wi.snr, 50, histtype='step', color='k')
# ps.savefig()
T = Wi[np.argsort(-Wi.snr)]
T.cut(T.snr > 25.)
print 'SN cut:', len(T)
#T = Wi[np.argsort(-Wi.snr)[:25]]
#print 'Coadd S/N:', T.snr
# band = 1
# T = fits_table('w%i.fits' % band)
#
# plt.clf()
# n,b,p = plt.hist(T.get('w%impro' % band), 50, histtype='step', color='b')
# plt.xlabel('w%impro' % band)
#
# print 'blend_ext_flags:', ['0x%x' % x for x in np.unique(T.blend_ext_flags)]
#
# T.cut(T.blend_ext_flags & 0xf == 1)
# print 'Cut to', len(T), 'on blend'
# T.cut(T.get('w%isat' % band) == 0)
# print 'Cut to', len(T), 'on sat'
# T.cut(T.get('w%iflg' % band) == 0)
# print 'Cut to', len(T), 'on flg'
#
# plt.hist(T.get('w%impro' % band), histtype='step', color='r', bins=b)
# ps.savefig()
dbs = []
l1dbs = []
#print 'Coadds', T.coadd_id
for t in T:
print 'coadd', t.coadd_id
#print 'RA,Dec', t.ra, t.dec
coadd = t.coadd_id.replace('_ab41', '').strip()
for base in ['data/unwise', 'data/unwise-nersc']:
dirnm = os.path.join(base, coadd[:3], coadd)
fn = os.path.join(dirnm, 'unwise-%s-w%i-img-m.fits' % (coadd, band))
if os.path.exists(fn):
break
img = fitsio.read(fn)
#print 'img', img.shape, img.dtype
wcs = Tan(fn)
ok,x,y = wcs.radec2pixelxy(t.ra, t.dec)
x -= 1
y -= 1
#print 'x,y', x,y
ix = int(np.round(x))
iy = int(np.round(y))
im = img[iy-S:iy+S+1, ix-S:ix+S+1]
if im.shape != (2*S+1, 2*S+1):
continue
imH,imW = im.shape
imax = np.argmax(im)
my,mx = np.unravel_index(imax, im.shape)
#print 'max pixel:', my,mx, im[my,mx]
#print 'center pixel:', imH/2,imW/2, im[imH/2,imW/2]
if my != imH/2 or mx != imW/2:
continue
# F = np.fft.fft2(im)
# S = np.fft.fftshift(F)
# P = S.real**2 + S.imag**2
# P /= P.max()
# freqs = np.fft.fftfreq(W)
# sfreqs = np.fft.fftshift(freqs)
# ff = np.hypot(sfreqs.reshape(1,W), sfreqs.reshape(W,1))
# print 'ff', ff.shape
#plt.clf()
#plt.plot(ff.ravel(), P.ravel(), 'b.', alpha=0.1)
#ps.savefig()
Y = im[imH/2, :]
X = im[:, imW/2]
Fy = np.fft.fft(Y)
Fx = np.fft.fft(X)
Sy = np.fft.fftshift(Fy)
Sx = np.fft.fftshift(Fx)
Px = Sx.real**2 + Sx.imag**2
Py = Sy.real**2 + Sy.imag**2
# Grab "blank" lines nearby to assess noise
n = im[imH/4, :]
Fn = np.fft.fft(n)
Sn = np.fft.fftshift(Fn)
Pn = Sn.real**2 + Sn.imag**2
freqs1 = np.fft.fftfreq(len(Y))
sfreqs1 = np.fft.fftshift(freqs1)
s0 = np.flatnonzero(sfreqs1 == 0)
s0 = s0[0]
s1 = s0
dbs.append((Sx[s1:], Sy[s1:], Px[s1:], Py[s1:], sfreqs1[s1:], Pn[s1:]))
print 'sx', len(Sx[s1:])
print 'sf', sfreqs1[s1:]
continue
fn = os.path.join(dirnm, 'unwise-%s-w%i-frames.fits' % (coadd, band))
T = fits_table(fn)
print 'Read', len(T), 'from', fn
T.cut(T.included > 0)
T.cut(T.use > 0)
print 'Cut to', len(T), 'used'
T.cut(T.coextent[:,0] < ix - S)
T.cut(T.coextent[:,1] > ix + S)
T.cut(T.coextent[:,2] < iy - S)
T.cut(T.coextent[:,3] > iy + S)
print 'Cut to', len(T), 'containing target pixels'
#ims = []
#for f in T[:10]:
for f in T:
fn = get_l1b_file(wisedir, f.scan_id, f.frame_num, band)
print 'frame', fn
img,hdr = fitsio.read(fn, header=True)
img -= f.sky1
gain = hdr.get('FEBGAIN')
print 'Gain:', gain
sig1 = np.sqrt(1./f.weight)
zpscale = 1. / zeropointToScale(f.zeropoint)
sig1 /= zpscale
print 'Sigma1:', sig1
print 'Sky:', f.sky1
wcs = Sip(fn)
mask = fitsio.read(fn.replace('-int-', '-msk-') + '.gz')
#print 'RA,Dec', t.ra, t.dec
ok,x,y = wcs.radec2pixelxy(t.ra, t.dec)
#print 'x,y', x,y
x -= 1
y -= 1
fx = int(np.round(x))
fy = int(np.round(y))
im = img[fy-S:fy+S+1, fx-S:fx+S+1]
if im.shape != (2*S+1, 2*S+1):
continue
imH,imW = im.shape
mask = mask[fy-S:fy+S+1, fx-S:fx+S+1]
ok = (mask == 0)
if not patch_image(im, ok):
print 'Patching failed'
continue
Y = im[imH/2, :]
X = im[:, imW/2]
Fy = np.fft.fft(Y)
Fx = np.fft.fft(X)
Sy = np.fft.fftshift(Fy)
Sx = np.fft.fftshift(Fx)
Px = Sx.real**2 + Sx.imag**2
Py = Sy.real**2 + Sy.imag**2
freqs1 = np.fft.fftfreq(len(Y))
sfreqs1 = np.fft.fftshift(freqs1)
s0 = np.flatnonzero(sfreqs1 == 0)
s0 = s0[0]
# Grab "blank" lines nearby to assess noise
n = im[imH/4, :]
Fn = np.fft.fft(n)
Sn = np.fft.fftshift(Fn)
Pn = Sn.real**2 + Sn.imag**2
l1dbs.append((Px[s0+1:], Py[s0+1:], sfreqs1[s0+1:], Pn[s0+1:]))
#ims.append(im)
# plt.clf()
# for i,im in enumerate(ims[:9]):
# plt.subplot(3,3,1+i)
# plt.imshow(im, interpolation='nearest', origin='lower')
# ps.savefig()
print len(dbs), 'coadds'
print len(l1dbs), 'L1bs'
plt.figure(figsize=(4,4))
left = 0.17
right = 0.03
bot = 0.12
top = 0.03
plt.subplots_adjust(left=left, right=1.-right, bottom=bot, top=1.-top)
if pred:
(Sxpsf, Sypsf, Pxpsf, Pypsf, sfpsf) = psfs[iband]
plt.clf()
Spsf = (Sxpsf + Sypsf) / 2.
Ppsf = Spsf.real**2 + Spsf.imag**2
plt.plot(sfpsf, 10.*np.log10(Pxpsf / Pxpsf[0]), 'r-')
plt.plot(sfpsf, 10.*np.log10(Pypsf / Pypsf[0]), 'm-')
plt.plot(sfpsf, 10.*np.log10(Ppsf / Ppsf[0]), '-', color=(1.,0.6,0))
plt.plot(sflan, 10.*np.log10(Plan / Plan[0]), 'k-')
#print 'sflan', sflan
#print 'Plan', Plan
#print 'db', 10.*np.log10(Plan / Plan[0])
for Sx,Sy,Px,Py,sfreq,Pn in dbs:
Sm = (Sx + Sy) / 2.
P = Sm.real**2 + Sm.imag**2
plt.plot(sfreq, 10.*np.log10(P / P[0]), 'b-', alpha=0.1)
Spred = Spsf * Slan
Ppred = Spred.real**2 + Spred.imag**2
plt.plot(sfreq, 10.*np.log10(Ppred / Ppred[0]), 'g-')
plt.ylabel('dB')
plt.ylim(-40, 3)
plt.title('PSF')
ps.savefig()
plt.clf()
for Sx,Sy,Px,Py,sfreq,Pn in dbs:
plt.plot(sfreq, 10.*np.log10(Py / Py[0]), 'b-', alpha=0.2)
plt.plot(sfreq, 10.*np.log10(Px / Px[0]), 'g-', alpha=0.2)
plt.plot(sfreq, 10.*np.log10(Pn / np.sqrt(Px[0]*Py[0])), 'r-', alpha=0.2)
plt.ylabel('dB')
plt.ylim(-40, 3)
plt.title('Isolated sources in coadd: W%i' % band)
ps.savefig()
xx = np.zeros((len(dbs), len(dbs[0][0])))
yy = np.zeros((len(dbs), len(dbs[0][0])))
nn = np.zeros((len(dbs), len(dbs[0][0])))
mm = np.zeros((len(dbs), len(dbs[0][0])))
for i,(Sx,Sy,Px,Py,sfreq,Pn) in enumerate(dbs):
xx[i,:] = 10.*np.log10(Px / Px[0])
yy[i,:] = 10.*np.log10(Py / Py[0])
nn[i,:] = 10.*np.log10(Pn / np.sqrt(Px[0]*Py[0]))
Sm = (Sx + Sy) / 2.
P = Sm.real**2 + Sm.imag**2
mm[i,:] = 10.*np.log10(P / P[0])
if pred:
plt.clf()
med = np.median(mm, axis=0)
plt.plot(sfreq, med, 'g-')
plt.plot(sfpsf, 10.*np.log10(Ppsf / Ppsf[0]), '-', color=(1.,0.6,0))
plt.plot(sflan, 10.*np.log10(Plan / Plan[0]), 'k-')
plt.plot(sfreq, 10.*np.log10(Ppred / Ppred[0]), 'r-')
plt.ylabel('dB')
plt.ylim(-40, 3)
plt.title('PSF')
ps.savefig()
plt.clf()
plt.plot(sfreq, np.median(xx, axis=0), 'g-')
plt.plot(sfreq, np.median(yy, axis=0), 'b-')
plt.plot(sfreq, np.median(nn, axis=0), 'r-')
plt.title('Isolated sources in coadd: W%i median' % band)
#plt.xlabel('frequency (cycles/pixel)')
plt.ylabel('dB')
plt.ylim(-40, 3)
ps.savefig()
print 'Freqs:', sfreq
if len(l1dbs) == 0:
continue
plt.clf()
for Px,Py,sf,noise in l1dbs:
plt.plot(sf, 10.*np.log10(Px / Px[0]), 'g-', alpha=0.2)
plt.plot(sf, 10.*np.log10(Py / Py[0]), 'b-', alpha=0.2)
plt.plot(sf, 10.*np.log10(noise / Py[0]), 'r-', alpha=0.2)
plt.plot(sf, 10.*np.log10(noise / Px[0]), 'r-', alpha=0.2)
plt.ylabel('dB')
plt.ylim(-40, 3)
plt.title('Isolated sources in L1bs: W%i' % band)
ps.savefig()
xx = np.zeros((len(l1dbs), len(l1dbs[0][0])))
yy = np.zeros((len(l1dbs), len(l1dbs[0][0])))
zz = np.zeros((len(l1dbs), len(l1dbs[0][0])))
zz2 = np.zeros((len(l1dbs), len(l1dbs[0][0])))
for i,(Px,Py,sfreq,noise) in enumerate(l1dbs):
xx[i,:] = 10.*np.log10(Px / Px[0])
yy[i,:] = 10.*np.log10(Py / Py[0])
zz[i,:] = 10.*np.log10(noise / Py[0])
zz2[i,:] = 10.*np.log10(noise / Px[0])
plt.clf()
plt.plot(sfreq, np.median(xx, axis=0), 'g-')
plt.plot(sfreq, np.median(yy, axis=0), 'b-')
plt.plot(sfreq, np.median(zz, axis=0), 'r-')
plt.plot(sfreq, np.median(zz2, axis=0), 'r-')
plt.title('Isolated sources in L1bs: W%i median' % band)
#plt.xlabel('frequency (cycles/pixel)')
plt.ylabel('dB')
plt.ylim(-40, 3)
ps.savefig()
dhi = d1 + margin
if True:
# HACK -- got the WISE catalogs on riemann and the WISE exposures on NERSC...
from wisecat import wise_catalog_radecbox
cols=['cntr', 'ra', 'dec', 'sigra', 'sigdec', 'cc_flags',
'ext_flg', 'var_flg', 'moon_lev', 'ph_qual',
'w1mpro', 'w1sigmpro', 'w1sat', 'w1nm', 'w1m',
'w1snr', 'w1cov', 'w1mag', 'w1sigm', 'w1flg',
'w2mpro', 'w2sigmpro', 'w2sat', 'w2nm', 'w2m',
'w2snr', 'w2cov', 'w2mag', 'w2sigm', 'w2flg',
'w3mpro', 'w3sigmpro', 'w3sat', 'w3nm', 'w3m',
'w3snr', 'w3cov', 'w3mag', 'w3sigm', 'w3flg',
'w4mpro', 'w4sigmpro', 'w4sat', 'w4nm', 'w4m',
'w4snr', 'w4cov', 'w4mag', 'w4sigm', 'w4flg', ]
W = wise_catalog_radecbox(rlo, rhi, dlo, dhi, cols=cols)
if W is None:
W = fits_table()
for c in cols:
W.set(c, np.array([]))
wfn = os.path.join(datadir, 'wise-objs-%s.fits' % dataset)
W.writeto(wfn)
print('Wrote', wfn)
continue
for band in [1,2,3,4]:
pfn = os.path.join(datadir, '%s-w%i-stage108.pickle' % (dataset, band))
if os.path.exists(pfn):
print('Output exists:', pfn, '; skipping')
continue
args.append((dataset, band, rlo, rhi, dlo, dhi, T, sfn))
