def stage_psfplots(
T=None, sedsn=None, coimgs=None, cons=None,
detmaps=None, detivs=None,
blobsrcs=None,blobflux=None,blobslices=None, blobs=None,
tractor=None, cat=None, targetrd=None, pixscale=None, targetwcs=None,
W=None,H=None, brickid=None,
bands=None, ps=None, tims=None,
plots=False,
**kwargs):
tim = tims[0]
tim.psfex.fitSavedData(*tim.psfex.splinedata)
spl = tim.psfex.splines[0]
print 'Spline:', spl
knots = spl.get_knots()
print 'knots:', knots
tx,ty = knots
k = 3
print 'interior knots x:', tx[k+1:-k-1]
print 'additional knots x:', tx[:k+1], 'and', tx[-k-1:]
print 'interior knots y:', ty[k+1:-k-1]
print 'additional knots y:', ty[:k+1], 'and', ty[-k-1:]
for itim,tim in enumerate(tims):
psfex = tim.psfex
psfex.fitSavedData(*psfex.splinedata)
if plots:
print
print 'Tim', tim
print
pp,xx,yy = psfex.splinedata
ny,nx,nparams = pp.shape
assert(len(xx) == nx)
assert(len(yy) == ny)
psfnil = psfex.psfclass(*np.zeros(nparams))
names = psfnil.getParamNames()
xa = np.linspace(xx[0], xx[-1], 50)
ya = np.linspace(yy[0], yy[-1], 100)
#xa,ya = np.meshgrid(xa,ya)
#xa = xa.ravel()
#ya = ya.ravel()
print 'xa', xa
print 'ya', ya
for i in range(nparams):
plt.clf()
plt.subplot(1,2,1)
dimshow(pp[:,:,i])
plt.title('grid fit')
plt.colorbar()
plt.subplot(1,2,2)
sp = psfex.splines[i](xa, ya)
sp = sp.T
print 'spline shape', sp.shape
assert(sp.shape == (len(ya),len(xa)))
dimshow(sp, extent=[xx[0],xx[-1],yy[0],yy[-1]])
plt.title('spline')
plt.colorbar()
plt.suptitle('tim %s: PSF param %s' % (tim.name, names[i]))
ps.savefig()
def halo_plots_before(tims, bands, targetwcs, halostars, ps):
coimgs, _ = quick_coadds(tims, bands, targetwcs)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(halostars.ibx, halostars.iby, 'o', mec='r', ms=15, mfc='none')
plt.axis(ax)
plt.title('Before star halo subtraction')
ps.savefig()
return coimgs
def test_fourier(self):
F, (cx, cy), shape, (v,
w) = self.psf.getFourierTransform(100., 100., 32)
print('F', F)
print('cx,cy', cx, cy)
print('shape', shape)
print('v, w', v, w)
if ps is not None:
import pylab as plt
from astrometry.util.plotutils import dimshow
plt.clf()
plt.subplot(1, 2, 1)
dimshow(F.real)
plt.subplot(1, 2, 2)
dimshow(F.imag)
ps.savefig()
def _plot_derivs(subtims, newsrc, srctractor, ps):
plt.clf()
rows = len(subtims)
cols = 1 + newsrc.numberOfParams()
for it,tim in enumerate(subtims):
derivs = srctractor._getSourceDerivatives(newsrc, tim)
c0 = 1 + cols*it
mod = srctractor.getModelPatchNoCache(tim, src)
if mod is not None and mod.patch is not None:
plt.subplot(rows, cols, c0)
dimshow(mod.patch, extent=mod.getExtent())
c0 += 1
for ip,deriv in enumerate(derivs):
if deriv is None:
continue
plt.subplot(rows, cols, c0+ip)
mx = np.max(np.abs(deriv.patch))
dimshow(deriv.patch, extent=deriv.getExtent(), vmin=-mx, vmax=mx)
plt.title('Derivatives for ' + name)
ps.savefig()
plt.clf()
modimgs = srctractor.getModelImages()
comods,nil = compute_coadds(subtims, bands, subtarget, images=modimgs)
dimshow(get_rgb(comods, bands))
plt.title('Initial ' + name)
ps.savefig()
def _plot_derivs(subtims, newsrc, srctractor, ps):
plt.clf()
rows = len(subtims)
cols = 1 + newsrc.numberOfParams()
for it, tim in enumerate(subtims):
derivs = srctractor._getSourceDerivatives(newsrc, tim)
c0 = 1 + cols * it
mod = srctractor.getModelPatchNoCache(tim, src)
if mod is not None and mod.patch is not None:
plt.subplot(rows, cols, c0)
dimshow(mod.patch, extent=mod.getExtent())
c0 += 1
for ip, deriv in enumerate(derivs):
if deriv is None:
continue
plt.subplot(rows, cols, c0 + ip)
mx = np.max(np.abs(deriv.patch))
dimshow(deriv.patch, extent=deriv.getExtent(), vmin=-mx, vmax=mx)
plt.title('Derivatives for ' + name)
ps.savefig()
plt.clf()
modimgs = srctractor.getModelImages()
comods, nil = compute_coadds(subtims, bands, subtarget, images=modimgs)
dimshow(get_rgb(comods, bands))
plt.title('Initial ' + name)
ps.savefig()
def fitblobs_plots_2(blobs, refstars, ps):
plt.clf()
dimshow(blobs >= 0, vmin=0, vmax=1)
ax = plt.axis()
plt.plot(refstars.ibx, refstars.iby, 'ro')
for ref in refstars:
magstr = ref.ref_cat
if ref.ref_cat == 'T2':
mag = ref.mag
magstr = 'T(%.1f)' % mag
elif ref.ref_cat == 'G2':
mag = ref.phot_g_mean_mag
magstr = 'G(%.1f)' % mag
plt.text(ref.ibx,
ref.iby,
magstr,
color='r',
fontsize=10,
bbox=dict(facecolor='w', alpha=0.5))
plt.axis(ax)
plt.title('Reference stars')
ps.savefig()
def halo_plots_after(tims, bands, targetwcs, halostars, coimgs, ps):
coimgs2, _ = quick_coadds(tims, bands, targetwcs)
plt.clf()
dimshow(get_rgb(coimgs2, bands))
ax = plt.axis()
plt.plot(halostars.ibx, halostars.iby, 'o', mec='r', ms=15, mfc='none')
plt.axis(ax)
plt.title('After star halo subtraction')
ps.savefig()
plt.clf()
dimshow(get_rgb([co - co2 for co, co2 in zip(coimgs, coimgs2)], bands))
ax = plt.axis()
plt.plot(halostars.ibx, halostars.iby, 'o', mec='r', ms=15, mfc='none')
plt.axis(ax)
plt.title('Subtracted halos')
ps.savefig()
for g in halostars[:10]:
plt.clf()
pixscale = targetwcs.pixel_scale()
pixrad = int(g.radius * 3600. / pixscale)
ax = [g.ibx - pixrad, g.ibx + pixrad, g.iby - pixrad, g.iby + pixrad]
ima = dict(interpolation='nearest', origin='lower')
plt.subplot(2, 2, 1)
plt.imshow(get_rgb(coimgs, bands), **ima)
plt.plot(halostars.ibx, halostars.iby, 'o', mec='r', ms=15, mfc='none')
plt.axis(ax)
plt.subplot(2, 2, 2)
plt.imshow(get_rgb(coimgs2, bands), **ima)
plt.axis(ax)
plt.subplot(2, 2, 3)
plt.imshow(
get_rgb([co - co2 for co, co2 in zip(coimgs, coimgs2)], bands),
**ima)
plt.axis(ax)
ps.savefig()
def detection_plots_2(tims, bands, targetwcs, refstars, Tnew, hot,
saturated_pix, ps):
coimgs, _ = quick_coadds(tims, bands, targetwcs)
crossa = dict(ms=10, mew=1.5)
plt.clf()
dimshow(get_rgb(coimgs, bands))
plt.title('Detections')
ps.savefig()
ax = plt.axis()
if len(refstars):
I, = np.nonzero([len(r) and r[0] == 'T' for r in refstars.ref_cat])
if len(I):
plt.plot(refstars.ibx[I],
refstars.iby[I],
'+',
color=(0, 1, 1),
label='Tycho-2',
**crossa)
I, = np.nonzero([len(r) and r[0] == 'G' for r in refstars.ref_cat])
if len(I):
plt.plot(refstars.ibx[I],
refstars.iby[I],
'+',
color=(0.2, 0.2, 1),
label='Gaia',
**crossa)
I, = np.nonzero([len(r) and r[0] == 'L' for r in refstars.ref_cat])
if len(I):
plt.plot(refstars.ibx[I],
refstars.iby[I],
'+',
color=(0.6, 0.6, 0.2),
label='Large Galaxy',
**crossa)
plt.plot(Tnew.ibx,
Tnew.iby,
'+',
color=(0, 1, 0),
label='New SED-matched detections',
**crossa)
plt.axis(ax)
plt.title('Detections')
plt.legend(loc='upper left')
ps.savefig()
plt.clf()
plt.subplot(1, 2, 1)
dimshow(hot, vmin=0, vmax=1, cmap='hot')
plt.title('hot')
plt.subplot(1, 2, 2)
H, W = targetwcs.shape
rgb = np.zeros((H, W, 3))
for i, satpix in enumerate(saturated_pix):
rgb[:, :, 2 - i] = satpix
dimshow(rgb)
plt.title('saturated_pix')
ps.savefig()
def _psf_check_plots(tims):
plt.figure(num=2, figsize=(7, 4.08))
for im, tim in zip(ims, tims):
print()
print('Image', tim.name)
plt.subplots_adjust(left=0,
right=1,
bottom=0,
top=0.95,
hspace=0,
wspace=0)
W, H = 2048, 4096
psfex = PsfEx(im.psffn, W, H)
psfim0 = psfim = psfex.instantiateAt(W / 2, H / 2)
# trim
psfim = psfim[10:-10, 10:-10]
tfit = Time()
psffit2 = GaussianMixtureEllipsePSF.fromStamp(psfim, N=2)
print('Fitting PSF mog:', psfim.shape, Time() - tfit)
psfim = psfim0[5:-5, 5:-5]
tfit = Time()
psffit2 = GaussianMixtureEllipsePSF.fromStamp(psfim, N=2)
print('Fitting PSF mog:', psfim.shape, Time() - tfit)
ph, pw = psfim.shape
psffit = GaussianMixtureEllipsePSF.fromStamp(psfim, N=3)
#mx = 0.03
mx = psfim.max()
mod3 = np.zeros_like(psfim)
p = psffit.getPointSourcePatch(pw / 2, ph / 2, radius=pw / 2)
p.addTo(mod3)
mod2 = np.zeros_like(psfim)
p = psffit2.getPointSourcePatch(pw / 2, ph / 2, radius=pw / 2)
p.addTo(mod2)
plt.clf()
plt.subplot(2, 3, 1)
dimshow(psfim, vmin=0, vmax=mx, ticks=False)
plt.subplot(2, 3, 2)
dimshow(mod3, vmin=0, vmax=mx, ticks=False)
plt.subplot(2, 3, 3)
dimshow(mod2, vmin=0, vmax=mx, ticks=False)
plt.subplot(2, 3, 5)
dimshow(psfim - mod3, vmin=-mx / 2, vmax=mx / 2, ticks=False)
plt.subplot(2, 3, 6)
dimshow(psfim - mod2, vmin=-mx / 2, vmax=mx / 2, ticks=False)
ps.savefig()
#continue
for round in [1, 2, 3, 4, 5]:
plt.clf()
k = 1
#rows,cols = 10,5
rows, cols = 7, 4
for iy, y in enumerate(np.linspace(0, H, rows).astype(int)):
for ix, x in enumerate(np.linspace(0, W, cols).astype(int)):
psfimg = psfex.instantiateAt(x, y)
# trim
psfimg = psfimg[5:-5, 5:-5]
print('psfimg', psfimg.shape)
ph, pw = psfimg.shape
psfimg2 = tim.psfex.getPointSourcePatch(x,
y,
radius=pw / 2)
mod = np.zeros_like(psfimg)
h, w = mod.shape
#psfimg2.x0 -= x
#psfimg2.x0 += w/2
#psfimg2.y0 -= y
#psfimg2.y0 += h/2
psfimg2.x0 = 0
psfimg2.y0 = 0
print('psfimg2:', (psfimg2.x0, psfimg2.y0))
psfimg2.addTo(mod)
print('psfimg:', psfimg.min(), psfimg.max(), psfimg.sum())
print('psfimg2:', psfimg2.patch.min(), psfimg2.patch.max(),
psfimg2.patch.sum())
print('mod:', mod.min(), mod.max(), mod.sum())
#plt.subplot(rows, cols, k)
plt.subplot(cols, rows, k)
k += 1
kwa = dict(vmin=0, vmax=mx, ticks=False)
if round == 1:
dimshow(psfimg, **kwa)
plt.suptitle('PsfEx')
elif round == 2:
dimshow(mod, **kwa)
plt.suptitle('varying MoG')
elif round == 3:
dimshow(psfimg - mod,
vmin=-mx / 2,
vmax=mx / 2,
ticks=False)
plt.suptitle('PsfEx - varying MoG')
elif round == 4:
dimshow(psfimg - mod3,
vmin=-mx / 2,
vmax=mx / 2,
ticks=False)
plt.suptitle('PsfEx - const MoG(3)')
elif round == 5:
dimshow(psfimg - mod2,
vmin=-mx / 2,
vmax=mx / 2,
ticks=False)
plt.suptitle('PsfEx - const MoG(2)')
ps.savefig()
def main():
ps = PlotSequence('conv')
S = 51
center = S/2
print('Center', center)
#for psf_sigma in [2., 1.5, 1.]:
for psf_sigma in [2.]:
rms2 = []
x = np.arange(S)
y = np.arange(S)
xx,yy = np.meshgrid(x, y)
scale = 1.5 / psf_sigma
pixpsf = render_airy((scale, center), x, y)
psf = (scale,center)
eval_psf = render_airy
plt.clf()
plt.subplot(2,1,1)
plt.plot(x, pixpsf[center,:], 'b-')
plt.plot(x, pixpsf[:,center], 'r-')
plt.subplot(2,1,2)
plt.plot(x, np.maximum(1e-16, pixpsf[center,:]), 'b-')
plt.plot(x, np.maximum(1e-16, pixpsf[:,center]), 'r-')
plt.yscale('log')
ps.savefig()
plt.clf()
plt.imshow(pixpsf, interpolation='nearest', origin='lower')
ps.savefig()
plt.clf()
plt.imshow(np.log10(np.maximum(1e-16, pixpsf)),
interpolation='nearest', origin='lower')
plt.colorbar()
plt.title('log PSF')
ps.savefig()
# psf
#psf = scipy.stats.norm(loc=center + 0.5, scale=psf_sigma)
# plt.clf()
# plt.imshow(Pcdf, interpolation='nearest', origin='lower')
# ps.savefig()
# #Pcdf = psf.cdf(xx) * psf.cdf(yy)
# #pixpsf = integrate_gaussian(psf, xx, yy)
#
# padpsf = np.zeros((S*2-1, S*2-1))
# ph,pw = pixpsf.shape
# padpsf[S/2:S/2+ph, S/2:S/2+pw] = pixpsf
# Fpsf = np.fft.rfft2(padpsf)
#
# padh,padw = padpsf.shape
# v = np.fft.rfftfreq(padw)
# w = np.fft.fftfreq(padh)
# fmax = max(max(np.abs(v)), max(np.abs(w)))
# cut = fmax / 2. * 1.000001
# #print('Frequence cut:', cut)
# Ffiltpsf = Fpsf.copy()
# #print('Ffiltpsf', Ffiltpsf.shape)
# #print((np.abs(w) < cut).shape)
# #print((np.abs(v) < cut).shape)
# Ffiltpsf[np.abs(w) > cut, :] = 0.
# Ffiltpsf[:, np.abs(v) > cut] = 0.
# #print('pad v', v)
# #print('pad w', w)
#
# filtpsf = np.fft.irfft2(Ffiltpsf, s=(padh,padw))
#
# print('filtered PSF real', np.max(np.abs(filtpsf.real)))
# print('filtered PSF imag', np.max(np.abs(filtpsf.imag)))
#
# plt.clf()
# plt.subplot(2,3,1)
# dimshow(Fpsf.real)
# plt.colorbar()
# plt.title('Padded PSF real')
# plt.subplot(2,3,4)
# dimshow(Fpsf.imag)
# plt.colorbar()
# plt.title('Padded PSF imag')
#
# plt.subplot(2,3,2)
# dimshow(Ffiltpsf.real)
# plt.colorbar()
# plt.title('Filt PSF real')
# plt.subplot(2,3,5)
# dimshow(Ffiltpsf.imag)
# plt.colorbar()
# plt.title('Filt PSF imag')
#
# plt.subplot(2,3,3)
# dimshow(filtpsf.real)
# plt.title('PSF real')
# plt.colorbar()
#
# plt.subplot(2,3,6)
# dimshow(filtpsf.imag)
# plt.title('PSF imag')
# plt.colorbar()
#
# ps.savefig()
#
#
# pixpsf = filtpsf
gal_sigmas = [2, 1, 0.5, 0.25]
for gal_sigma in gal_sigmas:
# plt.clf()
# plt.imshow(Gcdf, interpolation='nearest', origin='lower')
# plt.savefig('dcdf.png')
# plt.clf()
# plt.imshow(np.exp(-0.5 * ((xx-center)**2 + (yy-center)**2)/2.**2),
# interpolation='nearest', origin='lower')
# plt.savefig('g.png')
# my convolution
pixscale = 1.
cd = pixscale * np.eye(2) / 3600.
P,FG,Gmine,v,w = galaxy_psf_convolution(
gal_sigma, 0., 0., GaussianGalaxy, cd,
0., 0., pixpsf, debug=True)
#print('v:', v)
#print('w:', w)
#print('P:', P.shape)
print()
print('PSF %g, Gal %g' % (psf_sigma, gal_sigma))
rmax = np.argmax(np.abs(w))
cmax = np.argmax(np.abs(v))
l2_rmax = np.sqrt(np.sum(P[rmax,:].real**2 + P[rmax,:].imag**2))
l2_cmax = np.sqrt(np.sum(P[:,cmax].real**2 + P[:,cmax].imag**2))
print('PSF L_2 in highest-frequency rows & cols:', l2_rmax, l2_cmax)
l2_rmax = np.sqrt(np.sum(FG[rmax,:].real**2 + FG[rmax,:].imag**2))
l2_cmax = np.sqrt(np.sum(FG[:,cmax].real**2 + FG[:,cmax].imag**2))
print('Gal L_2 in highest-frequency rows & cols:', l2_rmax, l2_cmax)
C = P * FG
l2_rmax = np.sqrt(np.sum(C[rmax,:].real**2 + C[rmax,:].imag**2))
l2_cmax = np.sqrt(np.sum(C[:,cmax].real**2 + C[:,cmax].imag**2))
print('PSF*Gal L_2 in highest-frequency rows & cols:', l2_rmax, l2_cmax)
print()
Fpsf, Fgal = compare_subsampled(
S, 1, ps, psf, pixpsf, Gmine,v,w,
gal_sigma, psf_sigma, cd, get_ffts=True, eval_psf=eval_psf)
plt.clf()
plt.subplot(2,4,1)
dimshow(P.real)
plt.colorbar()
plt.title('PSF real')
plt.subplot(2,4,5)
dimshow(P.imag)
plt.colorbar()
plt.title('PSF imag')
plt.subplot(2,4,2)
dimshow(FG.real)
plt.colorbar()
plt.title('Gal real')
plt.subplot(2,4,6)
dimshow(FG.imag)
plt.colorbar()
plt.title('Gal imag')
plt.subplot(2,4,3)
dimshow((P * FG).real)
plt.colorbar()
plt.title('P*Gal real')
plt.subplot(2,4,7)
dimshow((P * FG).imag)
plt.colorbar()
plt.title('P*Gal imag')
plt.subplot(2,4,4)
dimshow((Fgal).real)
plt.colorbar()
plt.title('pixGal real')
plt.subplot(2,4,8)
dimshow((Fgal).imag)
plt.colorbar()
plt.title('pixGal imag')
plt.suptitle('PSF %g, Gal %g' % (psf_sigma, gal_sigma))
ps.savefig()
subsample = [1,2,4]
rms1 = []
for s in subsample:
rms = compare_subsampled(S, s, ps, psf, pixpsf, Gmine,v,w, gal_sigma, psf_sigma, cd, eval_psf=eval_psf)
rms1.append(rms)
rms2.append(rms1)
print()
print('PSF sigma =', psf_sigma)
print('RMSes:')
for rms1,gal_sigma in zip(rms2, gal_sigmas):
print('Gal sigma', gal_sigma, 'rms:',
', '.join(['%.3g' % r for r in rms1]))
def compare_subsampled(S,
s,
ps,
psf,
pixpsf,
Gmine,
v,
w,
gal_sigma,
psf_sigma,
cd,
get_ffts=False,
eval_psf=integrate_gaussian):
print()
print('Subsample', s)
print()
step = 1. / s
sz = s * (S - 1) + 1
#x = np.arange(0, S, step)[:sz+1]
x = np.arange(0, S, step)[:sz]
#y = np.arange(0, S, step)[:sz+1]
# Create pixelized PSF (Gaussian)
sx = x - 0.5 + step / 2.
subpixpsf = eval_psf(psf, sx, sx)
binned = bin_image(subpixpsf, s)
bh, bw = binned.shape
pixpsf1 = pixpsf[:bh, :bw]
ph, pw = pixpsf.shape
binned = binned[:ph, :pw]
print('Binned PSF:')
measure(binned)
print('Pix PSF:')
measure(pixpsf)
# Recompute my convolution using the binned PSF
P, FG, Gmine, v, w = galaxy_psf_convolution(gal_sigma,
0.,
0.,
GaussianGalaxy,
cd,
0.,
0.,
binned,
debug=True)
xx, yy = np.meshgrid(x, x)
# plt.clf()
#
# plt.subplot(2,2,1)
# dimshow(subpixpsf)
# plt.title('subpix psf')
# plt.colorbar()
#
# plt.subplot(2,2,2)
# dimshow(binned)
# plt.title('binned subpix psf')
# plt.colorbar()
#
# plt.subplot(2,2,3)
# dimshow(pixpsf1)
# plt.title('pix psf')
# plt.colorbar()
#
# plt.subplot(2,2,4)
# dimshow(pixpsf1 - binned)
# plt.title('pix - binned')
# plt.colorbar()
# plt.suptitle('subsample %i' % s)
# ps.savefig()
# Create pixelized galaxy image
#gxx,gyy = xx + step/2., yy + step/2.
gxx, gyy = xx, yy
#gxx,gyy = xx - step, yy - step
#gxx,gyy = xx - step/2., yy - step/2.
center = S / 2
subpixgal = np.exp(-0.5 * ((gxx - center)**2 + (gyy - center)**2) /
gal_sigma**2)
sh, sw = subpixpsf.shape
subpixgal = subpixgal[:sh, :sw]
print('Subpix psf, gal', subpixpsf.shape, subpixgal.shape)
print('Subpix PSF:')
measure(subpixpsf)
print('Subpix gal:')
measure(subpixgal)
# FFT convolution
Fpsf = np.fft.rfft2(subpixpsf)
spg = np.fft.ifftshift(subpixgal)
# plt.clf()
# for i in range(len(w)):
# plt.plot(v, Fpsf[i,:], 'c-')
# for i in range(len(v)):
# plt.plot(w, Fpsf[:,i], 'm-')
# plt.title('PSF Fourier transform')
# ps.savefig()
#
# IV = np.argsort(v)
# IW = np.argsort(w)
# plt.clf()
# for i in range(len(w)):
# plt.plot(v[IV], np.abs(Fpsf[i,IV]), 'c-')
# for i in range(len(v)):
# plt.plot(w[IW], np.abs(Fpsf[IW,i]), 'm-')
# plt.title('abs PSF Fourier transform')
# ps.savefig()
#
# plt.yscale('log')
# ps.savefig()
# plt.clf()
# dimshow(spg)
# plt.title('spg')
# ps.savefig()
Fgal = np.fft.rfft2(spg)
if get_ffts:
return Fpsf, Fgal
Fconv = Fpsf * Fgal
subpixfft = np.fft.irfft2(Fconv, s=subpixpsf.shape)
print('Shapes:', 'subpixpsf', subpixpsf.shape, 'Fpsf', Fpsf.shape)
print('spg', spg.shape, 'Fgal', Fgal.shape, 'Fconv', Fconv.shape,
'subpixfft', subpixfft.shape)
print('Subpix conv', subpixfft.shape)
binned = bin_image(subpixfft, s)
binned /= np.sum(binned)
print('Binned', binned.shape)
print('Mine:', Gmine.shape)
print('Mine:')
measure(Gmine)
print('Binned subpix FFT:')
measure(binned)
mh, mw = Gmine.shape
binned = binned[:mh, :mw]
plt.clf()
plt.subplot(2, 3, 1)
dimshow(subpixpsf)
plt.title('subpix psf')
plt.colorbar()
plt.subplot(2, 3, 2)
dimshow(subpixgal)
plt.title('subpix galaxy')
plt.colorbar()
plt.subplot(2, 3, 3)
dimshow(subpixfft)
plt.title('subpix FFT conv')
plt.colorbar()
plt.subplot(2, 3, 4)
dimshow(np.log10(np.maximum(binned / np.max(binned), 1e-12)))
plt.title('log binned FFT conv')
plt.colorbar()
plt.subplot(2, 3, 5)
dimshow(np.log10(np.maximum(Gmine / np.max(Gmine), 1e-12)))
#dimshow(Gmine)
plt.title('log my conv')
plt.colorbar()
gh, gw = Gmine.shape
binned = binned[:gh, :gw]
bh, bw = binned.shape
Gmine = Gmine[:bh, :bw]
diff = Gmine - binned
plt.subplot(2, 3, 6)
dimshow(diff)
plt.title('mine - FFT')
plt.colorbar()
plt.suptitle('PSF %g, Gal %g, subsample %i' % (psf_sigma, gal_sigma, s))
ps.savefig()
rmsdiff = np.sqrt(np.mean(diff**2))
return rmsdiff
gal_re = 10.
assert (ph == pw)
ps = PlotSequence('conv')
plt.clf()
plt.imshow(pixpsf, interpolation='nearest', origin='lower')
ps.savefig()
Fpsf = np.fft.rfft2(pixpsf)
plt.clf()
plt.subplot(2, 4, 1)
dimshow(Fpsf.real)
plt.colorbar()
plt.title('PSF real')
plt.subplot(2, 4, 5)
dimshow(Fpsf.imag)
plt.colorbar()
plt.title('PSF imag')
ps.savefig()
# Subsample the PSF via resampling
from astrometry.util.util import lanczos_shift_image
scale = 2
sh, sw = ph * scale, pw * scale
subpixpsf = np.zeros((sh, sw))
for ix in np.arange(scale):
def stage_initplots(
coimgs=None, cons=None, bands=None, ps=None,
targetwcs=None,
blobs=None,
T=None, cat=None, tims=None, tractor=None, **kwargs):
# RGB image
# plt.clf()
# dimshow(get_rgb(coimgs, bands))
# ps.savefig()
print 'T:'
T.about()
# cluster zoom-in
#x0,x1, y0,y1 = 1700,2700, 200,1200
#x0,x1, y0,y1 = 1900,2500, 400,1000
#x0,x1, y0,y1 = 1900,2400, 450,950
x0,x1, y0,y1 = 0,500, 0,500
clco = [co[y0:y1, x0:x1] for co in coimgs]
clW, clH = x1-x0, y1-y0
clwcs = targetwcs.get_subimage(x0, y0, clW, clH)
plt.figure(figsize=(6,6))
plt.subplots_adjust(left=0.005, right=0.995, bottom=0.005, top=0.995)
ps.suffixes = ['png','pdf']
# cluster zoom-in
rgb = get_rgb(clco, bands)
plt.clf()
dimshow(rgb, ticks=False)
ps.savefig()
# blobs
#b0 = blobs
#b1 = binary_dilation(blobs, np.ones((3,3)))
#bout = np.logical_and(b1, np.logical_not(b0))
# b0 = blobs
# b1 = binary_erosion(b0, np.ones((3,3)))
# bout = np.logical_and(b0, np.logical_not(b1))
# # set green
# rgb[:,:,0][bout] = 0.
# rgb[:,:,1][bout] = 1.
# rgb[:,:,2][bout] = 0.
# plt.clf()
# dimshow(rgb, ticks=False)
# ps.savefig()
# Initial model (SDSS only)
try:
# convert from string to int
T.objid = np.array([int(x) if len(x) else 0 for x in T.objid])
except:
pass
scat = Catalog(*[cat[i] for i in np.flatnonzero(T.objid)])
sedcat = Catalog(*[cat[i] for i in np.flatnonzero(T.objid == 0)])
print len(cat), 'total sources'
print len(scat), 'SDSS sources'
print len(sedcat), 'SED-matched sources'
tr = Tractor(tractor.images, scat)
comods = []
comods2 = []
for iband,band in enumerate(bands):
comod = np.zeros((clH,clW))
comod2 = np.zeros((clH,clW))
con = np.zeros((clH,clW))
for itim,tim in enumerate(tims):
if tim.band != band:
continue
(Yo,Xo,Yi,Xi) = tim.resamp
mod = tr.getModelImage(tim)
Yo -= y0
Xo -= x0
K, = np.nonzero((Yo >= 0) * (Yo < clH) * (Xo >= 0) * (Xo < clW))
Xo,Yo,Xi,Yi = Xo[K],Yo[K],Xi[K],Yi[K]
comod[Yo,Xo] += mod[Yi,Xi]
ie = tim.getInvError()
noise = np.random.normal(size=ie.shape) / ie
noise[ie == 0] = 0.
comod2[Yo,Xo] += mod[Yi,Xi] + noise[Yi,Xi]
con[Yo,Xo] += 1
comod /= np.maximum(con, 1)
comods.append(comod)
comod2 /= np.maximum(con, 1)
comods2.append(comod2)
plt.clf()
dimshow(get_rgb(comods2, bands), ticks=False)
ps.savefig()
plt.clf()
dimshow(get_rgb(comods, bands), ticks=False)
ps.savefig()
# Overplot SDSS sources
ax = plt.axis()
for src in scat:
rd = src.getPosition()
ok,x,y = clwcs.radec2pixelxy(rd.ra, rd.dec)
cc = (0,1,0)
if isinstance(src, PointSource):
plt.plot(x-1, y-1, 'o', mec=cc, mfc='none', ms=10, mew=1.5)
else:
plt.plot(x-1, y-1, 'o', mec='r', mfc='none', ms=10, mew=1.5)
plt.axis(ax)
ps.savefig()
# Add SED-matched detections
for src in sedcat:
rd = src.getPosition()
ok,x,y = clwcs.radec2pixelxy(rd.ra, rd.dec)
plt.plot(x-1, y-1, 'o', mec='c', mfc='none', ms=10, mew=1.5)
plt.axis(ax)
ps.savefig()
# Mark SED-matched detections on image
plt.clf()
dimshow(get_rgb(clco, bands), ticks=False)
ax = plt.axis()
for src in sedcat:
rd = src.getPosition()
ok,x,y = clwcs.radec2pixelxy(rd.ra, rd.dec)
#plt.plot(x-1, y-1, 'o', mec='c', mfc='none', ms=10, mew=1.5)
x,y = x-1, y-1
hi,lo = 20,7
# plt.plot([x-lo,x-hi],[y,y], 'c-')
# plt.plot([x+lo,x+hi],[y,y], 'c-')
# plt.plot([x,x],[y+lo,y+hi], 'c-')
# plt.plot([x,x],[y-lo,y-hi], 'c-')
plt.annotate('', (x,y+lo), xytext=(x,y+hi),
arrowprops=dict(color='c', width=1, frac=0.3, headwidth=5))
plt.axis(ax)
ps.savefig()
# plt.clf()
# dimshow(get_rgb([gaussian_filter(x,1) for x in clco], bands), ticks=False)
# ps.savefig()
# Resid
# plt.clf()
# dimshow(get_rgb([im-mo for im,mo in zip(clco,comods)], bands), ticks=False)
# ps.savefig()
# find SDSS fields within that WCS
sdsscoimgs,nil = sdss_coadd(clwcs, bands)
plt.clf()
dimshow(get_rgb(sdsscoimgs, bands, **rgbkwargs), ticks=False)
#plt.title('SDSS')
ps.savefig()
def stage0(**kwargs):
ps = PlotSequence('cfht')
decals = CfhtDecals()
B = decals.get_bricks()
print 'Bricks:'
B.about()
ra,dec = 190.0, 11.0
#bands = 'ugri'
bands = 'gri'
B.cut(np.argsort(degrees_between(ra, dec, B.ra, B.dec)))
print 'Nearest bricks:', B.ra[:5], B.dec[:5], B.brickid[:5]
brick = B[0]
pixscale = 0.186
#W,H = 1024,1024
#W,H = 2048,2048
#W,H = 3600,3600
W,H = 4800,4800
targetwcs = wcs_for_brick(brick, pixscale=pixscale, W=W, H=H)
ccdfn = 'cfht-ccds.fits'
if os.path.exists(ccdfn):
T = fits_table(ccdfn)
else:
T = get_ccd_list()
T.writeto(ccdfn)
print len(T), 'CCDs'
T.cut(ccds_touching_wcs(targetwcs, T))
print len(T), 'CCDs touching brick'
T.cut(np.array([b in bands for b in T.filter]))
print len(T), 'in bands', bands
ims = []
for t in T:
im = CfhtImage(t)
# magzp = hdr['PHOT_C'] + 2.5 * np.log10(hdr['EXPTIME'])
# fwhm = t.seeing / (pixscale * 3600)
# print '-> FWHM', fwhm, 'pix'
im.seeing = t.seeing
im.pixscale = t.pixscale
print 'seeing', t.seeing
print 'pixscale', im.pixscale*3600, 'arcsec/pix'
im.run_calibs(t.ra, t.dec, im.pixscale, W=t.width, H=t.height)
ims.append(im)
# Read images, clip to ROI
targetrd = np.array([targetwcs.pixelxy2radec(x,y) for x,y in
[(1,1),(W,1),(W,H),(1,H),(1,1)]])
keepims = []
tims = []
for im in ims:
print
print 'Reading expnum', im.expnum, 'name', im.extname, 'band', im.band, 'exptime', im.exptime
band = im.band
wcs = im.read_wcs()
imh,imw = wcs.imageh,wcs.imagew
imgpoly = [(1,1),(1,imh),(imw,imh),(imw,1)]
ok,tx,ty = wcs.radec2pixelxy(targetrd[:-1,0], targetrd[:-1,1])
tpoly = zip(tx,ty)
clip = clip_polygon(imgpoly, tpoly)
clip = np.array(clip)
#print 'Clip', clip
if len(clip) == 0:
continue
x0,y0 = np.floor(clip.min(axis=0)).astype(int)
x1,y1 = np.ceil (clip.max(axis=0)).astype(int)
slc = slice(y0,y1+1), slice(x0,x1+1)
## FIXME -- it seems I got lucky and the cross product is
## negative == clockwise, as required by clip_polygon. One
## could check this and reverse the polygon vertex order.
# dx0,dy0 = tx[1]-tx[0], ty[1]-ty[0]
# dx1,dy1 = tx[2]-tx[1], ty[2]-ty[1]
# cross = dx0*dy1 - dx1*dy0
# print 'Cross:', cross
print 'Image slice: x [%i,%i], y [%i,%i]' % (x0,x1, y0,y1)
print 'Reading image from', im.imgfn, 'HDU', im.hdu
img,imghdr = im.read_image(header=True, slice=slc)
goodpix = (img != 0)
print 'Number of pixels == 0:', np.sum(img == 0)
print 'Number of pixels != 0:', np.sum(goodpix)
if np.sum(goodpix) == 0:
continue
# print 'Image shape', img.shape
print 'Image range', img.min(), img.max()
print 'Goodpix image range:', (img[goodpix]).min(), (img[goodpix]).max()
if img[goodpix].min() == img[goodpix].max():
print 'No dynamic range in image'
continue
# print 'Reading invvar from', im.wtfn, 'HDU', im.hdu
# invvar = im.read_invvar(slice=slc)
# # print 'Invvar shape', invvar.shape
# # print 'Invvar range:', invvar.min(), invvar.max()
# invvar[goodpix == 0] = 0.
# if np.all(invvar == 0.):
# print 'Skipping zero-invvar image'
# continue
# assert(np.all(np.isfinite(img)))
# assert(np.all(np.isfinite(invvar)))
# assert(not(np.all(invvar == 0.)))
# # Estimate per-pixel noise via Blanton's 5-pixel MAD
# slice1 = (slice(0,-5,10),slice(0,-5,10))
# slice2 = (slice(5,None,10),slice(5,None,10))
# # print 'sliced shapes:', img[slice1].shape, img[slice2].shape
# # print 'good shape:', (goodpix[slice1] * goodpix[slice2]).shape
# # print 'good values:', np.unique(goodpix[slice1] * goodpix[slice2])
# # print 'sliced[good] shapes:', (img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].shape
# mad = np.median(np.abs(img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].ravel())
# sig1 = 1.4826 * mad / np.sqrt(2.)
# print 'MAD sig1:', sig1
# # invvar was 1 or 0
# invvar *= (1./(sig1**2))
# medsky = np.median(img[goodpix])
# Read full image for sig1 and sky estimate
fullimg = im.read_image()
fullgood = (fullimg != 0)
# Estimate per-pixel noise via Blanton's 5-pixel MAD
slice1 = (slice(0,-5,10),slice(0,-5,10))
slice2 = (slice(5,None,10),slice(5,None,10))
mad = np.median(np.abs(fullimg[slice1] - fullimg[slice2])[fullgood[slice1] * fullgood[slice2]].ravel())
sig1 = 1.4826 * mad / np.sqrt(2.)
print 'MAD sig1:', sig1
medsky = np.median(fullimg[fullgood])
invvar = np.zeros_like(img)
invvar[goodpix] = 1./sig1**2
# Median-smooth sky subtraction
plt.clf()
dimshow(np.round((img-medsky) / sig1), vmin=-3, vmax=5)
plt.title('Scalar median: %s' % im.name)
ps.savefig()
# medsky = np.zeros_like(img)
# # astrometry.util.util
# median_smooth(img, np.logical_not(goodpix), 256, medsky)
fullmed = np.zeros_like(fullimg)
median_smooth(fullimg - medsky, np.logical_not(fullgood), 256, fullmed)
fullmed += medsky
medimg = fullmed[slc]
plt.clf()
dimshow(np.round((img - medimg) / sig1), vmin=-3, vmax=5)
plt.title('Median filtered: %s' % im.name)
ps.savefig()
#print 'Subtracting median:', medsky
#img -= medsky
img -= medimg
primhdr = im.read_image_primary_header()
magzp = decals.get_zeropoint_for(im)
print 'magzp', magzp
zpscale = NanoMaggies.zeropointToScale(magzp)
print 'zpscale', zpscale
# Scale images to Nanomaggies
img /= zpscale
sig1 /= zpscale
invvar *= zpscale**2
orig_zpscale = zpscale
zpscale = 1.
assert(np.sum(invvar > 0) > 0)
print 'After scaling:'
print 'sig1', sig1
print 'invvar range', invvar.min(), invvar.max()
print 'image range', img.min(), img.max()
assert(np.all(np.isfinite(img)))
assert(np.all(np.isfinite(invvar)))
assert(np.isfinite(sig1))
plt.clf()
lo,hi = -5*sig1, 10*sig1
n,b,p = plt.hist(img[goodpix].ravel(), 100, range=(lo,hi), histtype='step', color='k')
xx = np.linspace(lo, hi, 200)
plt.plot(xx, max(n)*np.exp(-xx**2 / (2.*sig1**2)), 'r-')
plt.xlim(lo,hi)
plt.title('Pixel histogram: %s' % im.name)
ps.savefig()
twcs = ConstantFitsWcs(wcs)
if x0 or y0:
twcs.setX0Y0(x0,y0)
info = im.get_image_info()
fullh,fullw = info['dims']
# read fit PsfEx model
psfex = PsfEx.fromFits(im.psffitfn)
print 'Read', psfex
# HACK -- highly approximate PSF here!
#psf_fwhm = imghdr['FWHM']
#psf_fwhm = im.seeing
psf_fwhm = im.seeing / (im.pixscale * 3600)
print 'PSF FWHM', psf_fwhm, 'pixels'
psf_sigma = psf_fwhm / 2.35
psf = NCircularGaussianPSF([psf_sigma],[1.])
print 'img type', img.dtype
tim = Image(img, invvar=invvar, wcs=twcs, psf=psf,
photocal=LinearPhotoCal(zpscale, band=band),
sky=ConstantSky(0.), name=im.name + ' ' + band)
tim.zr = [-3. * sig1, 10. * sig1]
tim.sig1 = sig1
tim.band = band
tim.psf_fwhm = psf_fwhm
tim.psf_sigma = psf_sigma
tim.sip_wcs = wcs
tim.x0,tim.y0 = int(x0),int(y0)
tim.psfex = psfex
tim.imobj = im
mn,mx = tim.zr
tim.ima = dict(interpolation='nearest', origin='lower', cmap='gray',
vmin=mn, vmax=mx)
tims.append(tim)
keepims.append(im)
ims = keepims
print 'Computing resampling...'
# save resampling params
for tim in tims:
wcs = tim.sip_wcs
subh,subw = tim.shape
subwcs = wcs.get_subimage(tim.x0, tim.y0, subw, subh)
tim.subwcs = subwcs
try:
Yo,Xo,Yi,Xi,rims = resample_with_wcs(targetwcs, subwcs, [], 2)
except OverlapError:
print 'No overlap'
continue
if len(Yo) == 0:
continue
tim.resamp = (Yo,Xo,Yi,Xi)
print 'Creating coadds...'
# Produce per-band coadds, for plots
coimgs = []
cons = []
for ib,band in enumerate(bands):
coimg = np.zeros((H,W), np.float32)
con = np.zeros((H,W), np.uint8)
for tim in tims:
if tim.band != band:
continue
(Yo,Xo,Yi,Xi) = tim.resamp
if len(Yo) == 0:
continue
nn = (tim.getInvvar()[Yi,Xi] > 0)
coimg[Yo,Xo] += tim.getImage ()[Yi,Xi] * nn
con [Yo,Xo] += nn
# print
# print 'tim', tim.name
# print 'number of resampled pix:', len(Yo)
# reim = np.zeros_like(coimg)
# ren = np.zeros_like(coimg)
# reim[Yo,Xo] = tim.getImage()[Yi,Xi] * nn
# ren[Yo,Xo] = nn
# print 'number of resampled pix with positive invvar:', ren.sum()
# plt.clf()
# plt.subplot(2,2,1)
# mn,mx = [np.percentile(reim[ren>0], p) for p in [25,95]]
# print 'Percentiles:', mn,mx
# dimshow(reim, vmin=mn, vmax=mx)
# plt.colorbar()
# plt.subplot(2,2,2)
# dimshow(con)
# plt.colorbar()
# plt.subplot(2,2,3)
# dimshow(reim, vmin=tim.zr[0], vmax=tim.zr[1])
# plt.colorbar()
# plt.subplot(2,2,4)
# plt.hist(reim.ravel(), 100, histtype='step', color='b')
# plt.hist(tim.getImage().ravel(), 100, histtype='step', color='r')
# plt.suptitle('%s: %s' % (band, tim.name))
# ps.savefig()
coimg /= np.maximum(con,1)
coimgs.append(coimg)
cons .append(con)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ps.savefig()
plt.clf()
for i,b in enumerate(bands):
plt.subplot(2,2,i+1)
dimshow(cons[i], ticks=False)
plt.title('%s band' % b)
plt.colorbar()
plt.suptitle('Number of exposures')
ps.savefig()
print 'Grabbing SDSS sources...'
bandlist = [b for b in bands]
cat,T = get_sdss_sources(bandlist, targetwcs)
# record coordinates in target brick image
ok,T.tx,T.ty = targetwcs.radec2pixelxy(T.ra, T.dec)
T.tx -= 1
T.ty -= 1
T.itx = np.clip(np.round(T.tx).astype(int), 0, W-1)
T.ity = np.clip(np.round(T.ty).astype(int), 0, H-1)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(T.tx, T.ty, 'o', mec=green, mfc='none', ms=10, mew=1.5)
plt.axis(ax)
plt.title('SDSS sources')
ps.savefig()
print 'Detmaps...'
# Render the detection maps
detmaps = dict([(b, np.zeros((H,W), np.float32)) for b in bands])
detivs = dict([(b, np.zeros((H,W), np.float32)) for b in bands])
for tim in tims:
iv = tim.getInvvar()
psfnorm = 1./(2. * np.sqrt(np.pi) * tim.psf_sigma)
detim = tim.getImage().copy()
detim[iv == 0] = 0.
detim = gaussian_filter(detim, tim.psf_sigma) / psfnorm**2
detsig1 = tim.sig1 / psfnorm
subh,subw = tim.shape
detiv = np.zeros((subh,subw), np.float32) + (1. / detsig1**2)
detiv[iv == 0] = 0.
(Yo,Xo,Yi,Xi) = tim.resamp
detmaps[tim.band][Yo,Xo] += detiv[Yi,Xi] * detim[Yi,Xi]
detivs [tim.band][Yo,Xo] += detiv[Yi,Xi]
rtn = dict()
for k in ['T', 'coimgs', 'cons', 'detmaps', 'detivs',
'targetrd', 'pixscale', 'targetwcs', 'W','H',
'bands', 'tims', 'ps', 'brick', 'cat']:
rtn[k] = locals()[k]
return rtn
def stage_initplots(coimgs=None,
cons=None,
bands=None,
ps=None,
targetwcs=None,
blobs=None,
T=None,
cat=None,
tims=None,
tractor=None,
**kwargs):
# RGB image
# plt.clf()
# dimshow(get_rgb(coimgs, bands))
# ps.savefig()
print('T:')
T.about()
# cluster zoom-in
#x0,x1, y0,y1 = 1700,2700, 200,1200
#x0,x1, y0,y1 = 1900,2500, 400,1000
#x0,x1, y0,y1 = 1900,2400, 450,950
x0, x1, y0, y1 = 0, 500, 0, 500
clco = [co[y0:y1, x0:x1] for co in coimgs]
clW, clH = x1 - x0, y1 - y0
clwcs = targetwcs.get_subimage(x0, y0, clW, clH)
plt.figure(figsize=(6, 6))
plt.subplots_adjust(left=0.005, right=0.995, bottom=0.005, top=0.995)
ps.suffixes = ['png', 'pdf']
# cluster zoom-in
rgb = get_rgb(clco, bands)
plt.clf()
dimshow(rgb, ticks=False)
ps.savefig()
# blobs
#b0 = blobs
#b1 = binary_dilation(blobs, np.ones((3,3)))
#bout = np.logical_and(b1, np.logical_not(b0))
# b0 = blobs
# b1 = binary_erosion(b0, np.ones((3,3)))
# bout = np.logical_and(b0, np.logical_not(b1))
# # set green
# rgb[:,:,0][bout] = 0.
# rgb[:,:,1][bout] = 1.
# rgb[:,:,2][bout] = 0.
# plt.clf()
# dimshow(rgb, ticks=False)
# ps.savefig()
# Initial model (SDSS only)
try:
# convert from string to int
T.objid = np.array([int(x) if len(x) else 0 for x in T.objid])
except:
pass
scat = Catalog(*[cat[i] for i in np.flatnonzero(T.objid)])
sedcat = Catalog(*[cat[i] for i in np.flatnonzero(T.objid == 0)])
print(len(cat), 'total sources')
print(len(scat), 'SDSS sources')
print(len(sedcat), 'SED-matched sources')
tr = Tractor(tractor.images, scat)
comods = []
comods2 = []
for iband, band in enumerate(bands):
comod = np.zeros((clH, clW))
comod2 = np.zeros((clH, clW))
con = np.zeros((clH, clW))
for itim, tim in enumerate(tims):
if tim.band != band:
continue
(Yo, Xo, Yi, Xi) = tim.resamp
mod = tr.getModelImage(tim)
Yo -= y0
Xo -= x0
K, = np.nonzero((Yo >= 0) * (Yo < clH) * (Xo >= 0) * (Xo < clW))
Xo, Yo, Xi, Yi = Xo[K], Yo[K], Xi[K], Yi[K]
comod[Yo, Xo] += mod[Yi, Xi]
ie = tim.getInvError()
noise = np.random.normal(size=ie.shape) / ie
noise[ie == 0] = 0.
comod2[Yo, Xo] += mod[Yi, Xi] + noise[Yi, Xi]
con[Yo, Xo] += 1
comod /= np.maximum(con, 1)
comods.append(comod)
comod2 /= np.maximum(con, 1)
comods2.append(comod2)
plt.clf()
dimshow(get_rgb(comods2, bands), ticks=False)
ps.savefig()
plt.clf()
dimshow(get_rgb(comods, bands), ticks=False)
ps.savefig()
# Overplot SDSS sources
ax = plt.axis()
for src in scat:
rd = src.getPosition()
ok, x, y = clwcs.radec2pixelxy(rd.ra, rd.dec)
cc = (0, 1, 0)
if isinstance(src, PointSource):
plt.plot(x - 1, y - 1, 'o', mec=cc, mfc='none', ms=10, mew=1.5)
else:
plt.plot(x - 1, y - 1, 'o', mec='r', mfc='none', ms=10, mew=1.5)
plt.axis(ax)
ps.savefig()
# Add SED-matched detections
for src in sedcat:
rd = src.getPosition()
ok, x, y = clwcs.radec2pixelxy(rd.ra, rd.dec)
plt.plot(x - 1, y - 1, 'o', mec='c', mfc='none', ms=10, mew=1.5)
plt.axis(ax)
ps.savefig()
# Mark SED-matched detections on image
plt.clf()
dimshow(get_rgb(clco, bands), ticks=False)
ax = plt.axis()
for src in sedcat:
rd = src.getPosition()
ok, x, y = clwcs.radec2pixelxy(rd.ra, rd.dec)
#plt.plot(x-1, y-1, 'o', mec='c', mfc='none', ms=10, mew=1.5)
x, y = x - 1, y - 1
hi, lo = 20, 7
# plt.plot([x-lo,x-hi],[y,y], 'c-')
# plt.plot([x+lo,x+hi],[y,y], 'c-')
# plt.plot([x,x],[y+lo,y+hi], 'c-')
# plt.plot([x,x],[y-lo,y-hi], 'c-')
plt.annotate('', (x, y + lo),
xytext=(x, y + hi),
arrowprops=dict(color='c', width=1, frac=0.3,
headwidth=5))
plt.axis(ax)
ps.savefig()
# plt.clf()
# dimshow(get_rgb([gaussian_filter(x,1) for x in clco], bands), ticks=False)
# ps.savefig()
# Resid
# plt.clf()
# dimshow(get_rgb([im-mo for im,mo in zip(clco,comods)], bands), ticks=False)
# ps.savefig()
# find SDSS fields within that WCS
sdsscoimgs, nil = sdss_coadd(clwcs, bands)
plt.clf()
dimshow(get_rgb(sdsscoimgs, bands, **rgbkwargs), ticks=False)
#plt.title('SDSS')
ps.savefig()
def segment_and_group_sources(image, T, name=None, ps=None, plots=False):
'''
*image*: binary image that defines "blobs"
*T*: source table; only ".itx" and ".ity" elements are used (x,y integer
pix pos). Note: ".blob" field is added.
*name*: for debugging only
Returns: (blobs, blobsrcs, blobslices)
*blobs*: image, values -1 = no blob, integer blob indices
*blobsrcs*: list of np arrays of integers, elements in T within each blob
*blobslices*: list of slice objects for blob bounding-boxes.
'''
from scipy.ndimage.morphology import binary_fill_holes
from scipy.ndimage.measurements import label, find_objects
emptyblob = 0
image = binary_fill_holes(image)
blobs,nblobs = label(image)
print('N detected blobs:', nblobs)
H,W = image.shape
del image
blobslices = find_objects(blobs)
T.blob = blobs[T.ity, T.itx]
if plots:
import pylab as plt
from astrometry.util.plotutils import dimshow
plt.clf()
dimshow(blobs > 0, vmin=0, vmax=1)
ax = plt.axis()
for i,bs in enumerate(blobslices):
sy,sx = bs
by0,by1 = sy.start, sy.stop
bx0,bx1 = sx.start, sx.stop
plt.plot([bx0, bx0, bx1, bx1, bx0], [by0, by1, by1, by0, by0], 'r-')
plt.text((bx0+bx1)/2., by0, '%i' % (i+1), ha='center', va='bottom', color='r')
plt.plot(T.itx, T.ity, 'rx')
for i,t in enumerate(T):
plt.text(t.itx, t.ity, 'src %i' % i, color='red', ha='left', va='center')
plt.axis(ax)
plt.title('Blobs')
ps.savefig()
# Find sets of sources within blobs
blobsrcs = []
keepslices = []
blobmap = {}
dropslices = {}
for blob in range(1, nblobs+1):
Isrcs = np.flatnonzero(T.blob == blob)
if len(Isrcs) == 0:
#print('Blob', blob, 'has no sources')
blobmap[blob] = -1
dropslices[blob] = blobslices[blob-1]
continue
blobmap[blob] = len(blobsrcs)
blobsrcs.append(Isrcs)
bslc = blobslices[blob-1]
keepslices.append(bslc)
blobslices = keepslices
# Find sources that do not belong to a blob and add them as
# singleton "blobs"; otherwise they don't get optimized.
# for sources outside the image bounds, what should we do?
inblobs = np.zeros(len(T), bool)
for Isrcs in blobsrcs:
inblobs[Isrcs] = True
noblobs = np.flatnonzero(np.logical_not(inblobs))
del inblobs
# Add new fake blobs!
for ib,i in enumerate(noblobs):
#S = 3
S = 5
bslc = (slice(np.clip(T.ity[i] - S, 0, H-1), np.clip(T.ity[i] + S+1, 0, H)),
slice(np.clip(T.itx[i] - S, 0, W-1), np.clip(T.itx[i] + S+1, 0, W)))
# Does this new blob overlap existing blob(s)?
oblobs = np.unique(blobs[bslc])
oblobs = oblobs[oblobs != emptyblob]
#print('This blob overlaps existing blobs:', oblobs)
if len(oblobs) > 1:
print('WARNING: not merging overlapping blobs like maybe we should')
if len(oblobs):
blob = oblobs[0]
#print('Adding source to existing blob', blob)
blobs[bslc][blobs[bslc] == emptyblob] = blob
blobindex = blobmap[blob]
if blobindex == -1:
# the overlapping blob was going to be dropped -- restore it.
blobindex = len(blobsrcs)
blobmap[blob] = blobindex
blobslices.append(dropslices[blob])
blobsrcs.append(np.array([], np.int64))
# Expand the existing blob slice to encompass this new source
oldslc = blobslices[blobindex]
sy,sx = oldslc
oy0,oy1, ox0,ox1 = sy.start,sy.stop, sx.start,sx.stop
sy,sx = bslc
ny0,ny1, nx0,nx1 = sy.start,sy.stop, sx.start,sx.stop
newslc = slice(min(oy0,ny0), max(oy1,ny1)), slice(min(ox0,nx0), max(ox1,nx1))
blobslices[blobindex] = newslc
# Add this source to the list of source indices for the existing blob.
blobsrcs[blobindex] = np.append(blobsrcs[blobindex], np.array([i]))
else:
# Set synthetic blob number
blob = nblobs+1 + ib
blobs[bslc][blobs[bslc] == emptyblob] = blob
blobmap[blob] = len(blobsrcs)
blobslices.append(bslc)
blobsrcs.append(np.array([i]))
#print('Added', len(noblobs), 'new fake singleton blobs')
# Remap the "blobs" image so that empty regions are = -1 and the blob values
# correspond to their indices in the "blobsrcs" list.
if len(blobmap):
maxblob = max(blobmap.keys())
else:
maxblob = 0
maxblob = max(maxblob, blobs.max())
bm = np.zeros(maxblob + 1, int)
for k,v in blobmap.items():
bm[k] = v
bm[0] = -1
# DEBUG
if plots:
import fitsio
fitsio.write('blobs-before-%s.fits' % name, blobs, clobber=True)
# Remap blob numbers
blobs = bm[blobs]
if plots:
import fitsio
fitsio.write('blobs-after-%s.fits' % name, blobs, clobber=True)
if plots:
import pylab as plt
from astrometry.util.plotutils import dimshow
plt.clf()
dimshow(blobs > -1, vmin=0, vmax=1)
ax = plt.axis()
for i,bs in enumerate(blobslices):
sy,sx = bs
by0,by1 = sy.start, sy.stop
bx0,bx1 = sx.start, sx.stop
plt.plot([bx0, bx0, bx1, bx1, bx0], [by0, by1, by1, by0, by0], 'r-')
plt.text((bx0+bx1)/2., by0, '%i' % (i+1), ha='center', va='bottom', color='r')
plt.plot(T.itx, T.ity, 'rx')
for i,t in enumerate(T):
plt.text(t.itx, t.ity, 'src %i' % i, color='red', ha='left', va='center')
plt.axis(ax)
plt.title('Blobs')
ps.savefig()
for j,Isrcs in enumerate(blobsrcs):
for i in Isrcs:
#assert(blobs[T.ity[i], T.itx[i]] == j)
if (blobs[T.ity[i], T.itx[i]] != j):
print('---------------------------!!!--------------------------')
print('Blob', j, 'sources', Isrcs)
print('Source', i, 'coords x,y', T.itx[i], T.ity[i])
print('Expected blob value', j, 'but got', blobs[T.ity[i], T.itx[i]])
T.blob = blobs[T.ity, T.itx]
assert(len(blobsrcs) == len(blobslices))
return blobs, blobsrcs, blobslices
def detection_plots(detmaps, detivs, bands, saturated_pix, tims, targetwcs,
refstars, large_galaxies, gaia_stars, ps):
rgb = get_rgb(detmaps, bands)
plt.clf()
dimshow(rgb)
plt.title('detmaps')
ps.savefig()
for i, satpix in enumerate(saturated_pix):
rgb[:, :, 2 - i][satpix] = 1
plt.clf()
dimshow(rgb)
plt.title('detmaps & saturated')
ps.savefig()
coimgs, _ = quick_coadds(tims, bands, targetwcs, fill_holes=False)
if refstars:
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
lp, lt = [], []
tycho = refstars[refstars.isbright]
if len(tycho):
_, ix, iy = targetwcs.radec2pixelxy(tycho.ra, tycho.dec)
p = plt.plot(ix - 1,
iy - 1,
'o',
mew=3,
ms=14,
mec='r',
mfc='none')
lp.append(p)
lt.append('Tycho-2 only')
if gaia_stars:
gaia = refstars[refstars.isgaia]
if gaia_stars and len(gaia):
_, ix, iy = targetwcs.radec2pixelxy(gaia.ra, gaia.dec)
p = plt.plot(ix - 1,
iy - 1,
'o',
mew=3,
ms=10,
mec='c',
mfc='none')
for x, y, g in zip(ix, iy, gaia.phot_g_mean_mag):
plt.text(x,
y,
'%.1f' % g,
color='k',
bbox=dict(facecolor='w', alpha=0.5))
lp.append(p)
lt.append('Gaia')
# star_clusters?
if large_galaxies:
galaxies = refstars[refstars.islargegalaxy]
if large_galaxies and len(galaxies):
_, ix, iy = targetwcs.radec2pixelxy(galaxies.ra, galaxies.dec)
p = plt.plot(ix - 1,
iy - 1,
'o',
mew=3,
ms=14,
mec=(0, 1, 0),
mfc='none')
lp.append(p)
lt.append('Galaxies')
plt.axis(ax)
plt.title('Ref sources')
plt.figlegend([p[0] for p in lp], lt)
ps.savefig()
for band, detmap, detiv in zip(bands, detmaps, detivs):
plt.clf()
plt.subplot(2, 1, 1)
plt.hist((detmap * np.sqrt(detiv))[detiv > 0],
bins=50,
range=(-5, 8),
log=True)
plt.title('Detection map pixel values (sigmas): band %s' % band)
plt.subplot(2, 1, 2)
plt.hist((detmap * np.sqrt(detiv))[detiv > 0], bins=50, range=(-5, 8))
ps.savefig()
def sed_matched_detection(sedname, sed, detmaps, detivs, bands,
xomit, yomit,
nsigma=5.,
saturated_pix=None,
saddle=2.,
cutonaper=True,
ps=None):
'''
Runs a single SED-matched detection filter.
Avoids creating sources close to existing sources.
Parameters
----------
sedname : string
Name of this SED; only used for plots.
sed : list of floats
The SED -- a list of floats, one per band, of this SED.
detmaps : list of numpy arrays
The per-band detection maps. These must all be the same size, the
brick image size.
detivs : list of numpy arrays
The inverse-variance maps associated with `detmaps`.
bands : list of strings
The band names of the `detmaps` and `detivs` images.
xomit, yomit : iterables (lists or numpy arrays) of int
Previously known sources that are to be avoided.
nsigma : float, optional
Detection threshold.
saturated_pix : None or numpy array, boolean
A map of pixels that are always considered "hot" when
determining whether a new source touches hot pixels of an
existing source.
saddle : float, optional
Saddle-point depth from existing sources down to new sources.
cutonaper : bool, optional
Apply a cut that the source's detection strength must be greater
than `nsigma` above the 16th percentile of the detection strength in
an annulus (from 10 to 20 pixels) around the source.
ps : PlotSequence object, optional
Create plots?
Returns
-------
hotblobs : numpy array of bool
A map of the blobs yielding sources in this SED.
px, py : numpy array of int
The new sources found.
aper : numpy array of float
The detection strength in the annulus around the source, if
`cutonaper` is set; else -1.
peakval : numpy array of float
The detection strength.
See also
--------
sed_matched_filters : creates the `(sedname, sed)` pairs used here
run_sed_matched_filters : calls this method
'''
from scipy.ndimage.measurements import label, find_objects
from scipy.ndimage.morphology import binary_dilation, binary_fill_holes
t0 = Time()
H,W = detmaps[0].shape
allzero = True
for iband,band in enumerate(bands):
if sed[iband] == 0:
continue
if np.all(detivs[iband] == 0):
continue
allzero = False
break
if allzero:
print('SED', sedname, 'has all zero weight')
return None,None,None,None,None
sedmap = np.zeros((H,W), np.float32)
sediv = np.zeros((H,W), np.float32)
for iband,band in enumerate(bands):
if sed[iband] == 0:
continue
# We convert the detmap to canonical band via
# detmap * w
# And the corresponding change to sig1 is
# sig1 * w
# So the invvar-weighted sum is
# (detmap * w) / (sig1**2 * w**2)
# = detmap / (sig1**2 * w)
sedmap += detmaps[iband] * detivs[iband] / sed[iband]
sediv += detivs [iband] / sed[iband]**2
sedmap /= np.maximum(1e-16, sediv)
sedsn = sedmap * np.sqrt(sediv)
del sedmap
peaks = (sedsn > nsigma)
print('SED sn:', Time()-t0)
t0 = Time()
def saddle_level(Y):
# Require a saddle that drops by (the larger of) "saddle"
# sigma, or 20% of the peak height
drop = max(saddle, Y * 0.2)
return Y - drop
lowest_saddle = nsigma - saddle
# zero out the edges -- larger margin here?
peaks[0 ,:] = 0
peaks[:, 0] = 0
peaks[-1,:] = 0
peaks[:,-1] = 0
# Label the N-sigma blobs at this point... we'll use this to build
# "sedhot", which in turn is used to define the blobs that we will
# optimize simultaneously. This also determines which pixels go
# into the fitting!
dilate = 8
hotblobs,nhot = label(binary_fill_holes(
binary_dilation(peaks, iterations=dilate)))
# find pixels that are larger than their 8 neighbors
peaks[1:-1, 1:-1] &= (sedsn[1:-1,1:-1] >= sedsn[0:-2,1:-1])
peaks[1:-1, 1:-1] &= (sedsn[1:-1,1:-1] >= sedsn[2: ,1:-1])
peaks[1:-1, 1:-1] &= (sedsn[1:-1,1:-1] >= sedsn[1:-1,0:-2])
peaks[1:-1, 1:-1] &= (sedsn[1:-1,1:-1] >= sedsn[1:-1,2: ])
peaks[1:-1, 1:-1] &= (sedsn[1:-1,1:-1] >= sedsn[0:-2,0:-2])
peaks[1:-1, 1:-1] &= (sedsn[1:-1,1:-1] >= sedsn[0:-2,2: ])
peaks[1:-1, 1:-1] &= (sedsn[1:-1,1:-1] >= sedsn[2: ,0:-2])
peaks[1:-1, 1:-1] &= (sedsn[1:-1,1:-1] >= sedsn[2: ,2: ])
print('Peaks:', Time()-t0)
t0 = Time()
if ps is not None:
from astrometry.util.plotutils import dimshow
crossa = dict(ms=10, mew=1.5)
green = (0,1,0)
def plot_boundary_map(X):
bounds = binary_dilation(X) - X
H,W = X.shape
rgba = np.zeros((H,W,4), np.uint8)
rgba[:,:,1] = bounds*255
rgba[:,:,3] = bounds*255
plt.imshow(rgba, interpolation='nearest', origin='lower')
plt.clf()
plt.subplot(1,2,2)
dimshow(sedsn, vmin=-2, vmax=100, cmap='hot', ticks=False)
plt.subplot(1,2,1)
dimshow(sedsn, vmin=-2, vmax=10, cmap='hot', ticks=False)
above = (sedsn > nsigma)
plot_boundary_map(above)
ax = plt.axis()
y,x = np.nonzero(peaks)
plt.plot(x, y, 'r+')
plt.axis(ax)
plt.title('SED %s: S/N & peaks' % sedname)
ps.savefig()
# plt.clf()
# plt.imshow(sedsn, vmin=-2, vmax=10, interpolation='nearest',
# origin='lower', cmap='hot')
# plot_boundary_map(sedsn > lowest_saddle)
# plt.title('SED %s: S/N & lowest saddle point bounds' % sedname)
# ps.savefig()
# For each new source, compute the saddle value, segment at that
# level, and drop the source if it is in the same blob as a
# previously-detected source. We dilate the blobs a bit too, to
# catch slight differences in centroid vs SDSS sources.
dilate = 2
# For efficiency, segment at the minimum saddle level to compute
# slices; the operations described above need only happen within
# the slice.
saddlemap = (sedsn > lowest_saddle)
if saturated_pix is not None:
saddlemap |= saturated_pix
saddlemap = binary_dilation(saddlemap, iterations=dilate)
allblobs,nblobs = label(saddlemap)
allslices = find_objects(allblobs)
ally0 = [sy.start for sy,sx in allslices]
allx0 = [sx.start for sy,sx in allslices]
# brightest peaks first
py,px = np.nonzero(peaks)
I = np.argsort(-sedsn[py,px])
py = py[I]
px = px[I]
keep = np.zeros(len(px), bool)
peakval = []
aper = []
apin = 10
apout = 20
# Map of pixels that are vetoed by sources found so far. The veto
# area is based on saddle height. We go from brightest to
# faintest pixels. Thus the saddle level decreases, and the
# saddlemap areas become larger; the saddlemap when a source is
# found is a lower bound on the pixels that it will veto based on
# the saddle heights of fainter sources. Thus the vetomap isn't
# the final word, it is just a quick veto of pixels we know for
# sure will be vetoed.
vetomap = np.zeros(sedsn.shape, bool)
# For each peak, determine whether it is isolated enough --
# separated by a low enough saddle from other sources. Need only
# search within its "allblob", which is defined by the lowest
# saddle.
print('Found', len(px), 'potential peaks')
#tlast = Time()
for i,(x,y) in enumerate(zip(px, py)):
if vetomap[y,x]:
#print(' in veto map!')
continue
#t0 = Time()
#t1 = Time()
#print('Time since last source:', t1-tlast)
#tlast = t1
level = saddle_level(sedsn[y,x])
ablob = allblobs[y,x]
index = ablob - 1
slc = allslices[index]
#print('source', i, 'of', len(px), 'at', x,y, 'S/N', sedsn[y,x], 'saddle', level)
#print(' allblobs slice', slc)
saddlemap = (sedsn[slc] > level)
if saturated_pix is not None:
saddlemap |= saturated_pix[slc]
saddlemap *= (allblobs[slc] == ablob)
#print(' saddlemap', Time()-tlast)
saddlemap = binary_fill_holes(saddlemap)
#print(' fill holes', Time()-tlast)
saddlemap = binary_dilation(saddlemap, iterations=dilate)
#print(' dilation', Time()-tlast)
blobs,nblobs = label(saddlemap)
#print(' label', Time()-tlast)
x0,y0 = allx0[index], ally0[index]
thisblob = blobs[y-y0, x-x0]
# previously found sources:
ox = np.append(xomit, px[:i][keep[:i]]) - x0
oy = np.append(yomit, py[:i][keep[:i]]) - y0
h,w = blobs.shape
cut = False
if len(ox):
ox = ox.astype(int)
oy = oy.astype(int)
cut = any((ox >= 0) * (ox < w) * (oy >= 0) * (oy < h) *
(blobs[np.clip(oy,0,h-1), np.clip(ox,0,w-1)] ==
thisblob))
if False and (not cut) and ps is not None:
plt.clf()
plt.subplot(1,2,1)
dimshow(sedsn, vmin=-2, vmax=10, cmap='hot')
plot_boundary_map((sedsn > nsigma))
ax = plt.axis()
plt.plot(x, y, 'm+', ms=12, mew=2)
plt.axis(ax)
plt.subplot(1,2,2)
y1,x1 = [s.stop for s in slc]
ext = [x0,x1,y0,y1]
dimshow(saddlemap, extent=ext)
#plt.plot([x0,x0,x1,x1,x0], [y0,y1,y1,y0,y0], 'c-')
#ax = plt.axis()
#plt.plot(ox+x0, oy+y0, 'rx')
plt.plot(xomit, yomit, 'rx', ms=8, mew=2)
plt.plot(px[:i][keep[:i]], py[:i][keep[:i]], '+',
color=green, ms=8, mew=2)
plt.plot(x, y, 'mo', mec='m', mfc='none', ms=12, mew=2)
plt.axis(ax)
if cut:
plt.suptitle('Cut')
else:
plt.suptitle('Keep')
ps.savefig()
#t1 = Time()
#print(t1 - t0)
if cut:
# in same blob as previously found source.
#print(' cut')
# update vetomap
vetomap[slc] |= saddlemap
#print('Added to vetomap:', np.sum(saddlemap), 'pixels set; now total of', np.sum(vetomap), 'pixels set')
continue
# Measure in aperture...
ap = sedsn[max(0, y-apout):min(H,y+apout+1),
max(0, x-apout):min(W,x+apout+1)]
apiv = (sediv[max(0, y-apout):min(H,y+apout+1),
max(0, x-apout):min(W,x+apout+1)] > 0)
aph,apw = ap.shape
apx0, apy0 = max(0, x - apout), max(0, y - apout)
R2 = ((np.arange(aph)+apy0 - y)[:,np.newaxis]**2 +
(np.arange(apw)+apx0 - x)[np.newaxis,:]**2)
ap = ap[apiv * (R2 >= apin**2) * (R2 <= apout**2)]
if len(ap):
# 16th percentile ~ -1 sigma point.
m = np.percentile(ap, 16.)
else:
# fake
m = -1.
#print(' aper', Time()-tlast)
if cutonaper:
if sedsn[y,x] - m < nsigma:
continue
aper.append(m)
peakval.append(sedsn[y,x])
keep[i] = True
vetomap[slc] |= saddlemap
#print('Added to vetomap:', np.sum(saddlemap), 'pixels set; now total of', np.sum(vetomap), 'pixels set')
if False and ps is not None:
plt.clf()
plt.subplot(1,2,1)
dimshow(ap, vmin=-2, vmax=10, cmap='hot',
extent=[apx0,apx0+apw,apy0,apy0+aph])
plt.subplot(1,2,2)
dimshow(ap * ((R2 >= apin**2) * (R2 <= apout**2)),
vmin=-2, vmax=10, cmap='hot',
extent=[apx0,apx0+apw,apy0,apy0+aph])
plt.suptitle('peak %.1f vs ap %.1f' % (sedsn[y,x], m))
ps.savefig()
print('New sources:', Time()-t0)
t0 = Time()
if ps is not None:
pxdrop = px[np.logical_not(keep)]
pydrop = py[np.logical_not(keep)]
py = py[keep]
px = px[keep]
# Which of the hotblobs yielded sources? Those are the ones to keep.
hbmap = np.zeros(nhot+1, bool)
hbmap[hotblobs[py,px]] = True
if len(xomit):
h,w = hotblobs.shape
hbmap[hotblobs[np.clip(yomit, 0, h-1), np.clip(xomit, 0, w-1)]] = True
# in case a source is (somehow) not in a hotblob?
hbmap[0] = False
hotblobs = hbmap[hotblobs]
if ps is not None:
plt.clf()
dimshow(vetomap, vmin=0, vmax=1, cmap='hot')
plt.title('SED %s: veto map' % sedname)
ps.savefig()
plt.clf()
dimshow(hotblobs, vmin=0, vmax=1, cmap='hot')
ax = plt.axis()
p1 = plt.plot(px, py, 'g+', ms=8, mew=2)
p2 = plt.plot(pxdrop, pydrop, 'm+', ms=8, mew=2)
p3 = plt.plot(xomit, yomit, 'r+', ms=8, mew=2)
plt.axis(ax)
plt.title('SED %s: hot blobs' % sedname)
plt.figlegend((p3[0],p1[0],p2[0]), ('Existing', 'Keep', 'Drop'),
'upper left')
ps.savefig()
return hotblobs, px, py, aper, peakval
def compare_subsampled(S, s, ps, psf, pixpsf, Gmine,v,w, gal_sigma, psf_sigma,
cd,
get_ffts=False, eval_psf=integrate_gaussian):
print()
print('Subsample', s)
print()
step = 1./s
sz = s * (S-1) + 1
#x = np.arange(0, S, step)[:sz+1]
x = np.arange(0, S, step)[:sz]
#y = np.arange(0, S, step)[:sz+1]
# Create pixelized PSF (Gaussian)
sx = x - 0.5 + step/2.
subpixpsf = eval_psf(psf, sx, sx)
binned = bin_image(subpixpsf, s)
bh,bw = binned.shape
pixpsf1 = pixpsf[:bh,:bw]
ph,pw = pixpsf.shape
binned = binned[:ph,:pw]
print('Binned PSF:')
measure(binned)
print('Pix PSF:')
measure(pixpsf)
# Recompute my convolution using the binned PSF
P,FG,Gmine,v,w = galaxy_psf_convolution(
gal_sigma, 0., 0., GaussianGalaxy, cd,
0., 0., binned, debug=True)
xx,yy = np.meshgrid(x,x)
# plt.clf()
#
# plt.subplot(2,2,1)
# dimshow(subpixpsf)
# plt.title('subpix psf')
# plt.colorbar()
#
# plt.subplot(2,2,2)
# dimshow(binned)
# plt.title('binned subpix psf')
# plt.colorbar()
#
# plt.subplot(2,2,3)
# dimshow(pixpsf1)
# plt.title('pix psf')
# plt.colorbar()
#
# plt.subplot(2,2,4)
# dimshow(pixpsf1 - binned)
# plt.title('pix - binned')
# plt.colorbar()
# plt.suptitle('subsample %i' % s)
# ps.savefig()
# Create pixelized galaxy image
#gxx,gyy = xx + step/2., yy + step/2.
gxx,gyy = xx,yy
#gxx,gyy = xx - step, yy - step
#gxx,gyy = xx - step/2., yy - step/2.
center = S/2
subpixgal = np.exp(-0.5 * ((gxx-center)**2 + (gyy-center)**2)/gal_sigma**2)
sh,sw = subpixpsf.shape
subpixgal = subpixgal[:sh,:sw]
print('Subpix psf, gal', subpixpsf.shape, subpixgal.shape)
print('Subpix PSF:')
measure(subpixpsf)
print('Subpix gal:')
measure(subpixgal)
# FFT convolution
Fpsf = np.fft.rfft2(subpixpsf)
spg = np.fft.ifftshift(subpixgal)
# plt.clf()
# for i in range(len(w)):
# plt.plot(v, Fpsf[i,:], 'c-')
# for i in range(len(v)):
# plt.plot(w, Fpsf[:,i], 'm-')
# plt.title('PSF Fourier transform')
# ps.savefig()
#
# IV = np.argsort(v)
# IW = np.argsort(w)
# plt.clf()
# for i in range(len(w)):
# plt.plot(v[IV], np.abs(Fpsf[i,IV]), 'c-')
# for i in range(len(v)):
# plt.plot(w[IW], np.abs(Fpsf[IW,i]), 'm-')
# plt.title('abs PSF Fourier transform')
# ps.savefig()
#
# plt.yscale('log')
# ps.savefig()
# plt.clf()
# dimshow(spg)
# plt.title('spg')
# ps.savefig()
Fgal = np.fft.rfft2(spg)
if get_ffts:
return Fpsf, Fgal
Fconv = Fpsf * Fgal
subpixfft = np.fft.irfft2(Fconv, s=subpixpsf.shape)
print('Shapes:', 'subpixpsf', subpixpsf.shape, 'Fpsf', Fpsf.shape)
print('spg', spg.shape, 'Fgal', Fgal.shape, 'Fconv', Fconv.shape,
'subpixfft', subpixfft.shape)
print('Subpix conv', subpixfft.shape)
binned = bin_image(subpixfft, s)
binned /= np.sum(binned)
print('Binned', binned.shape)
print('Mine:', Gmine.shape)
print('Mine:')
measure(Gmine)
print('Binned subpix FFT:')
measure(binned)
mh,mw = Gmine.shape
binned = binned[:mh,:mw]
plt.clf()
plt.subplot(2,3,1)
dimshow(subpixpsf)
plt.title('subpix psf')
plt.colorbar()
plt.subplot(2,3,2)
dimshow(subpixgal)
plt.title('subpix galaxy')
plt.colorbar()
plt.subplot(2,3,3)
dimshow(subpixfft)
plt.title('subpix FFT conv')
plt.colorbar()
plt.subplot(2,3,4)
dimshow(np.log10(np.maximum(binned / np.max(binned), 1e-12)))
plt.title('log binned FFT conv')
plt.colorbar()
plt.subplot(2,3,5)
dimshow(np.log10(np.maximum(Gmine / np.max(Gmine), 1e-12)))
#dimshow(Gmine)
plt.title('log my conv')
plt.colorbar()
gh,gw = Gmine.shape
binned = binned[:gh,:gw]
bh,bw = binned.shape
Gmine = Gmine[:bh,:bw]
diff = Gmine - binned
plt.subplot(2,3,6)
dimshow(diff)
plt.title('mine - FFT')
plt.colorbar()
plt.suptitle('PSF %g, Gal %g, subsample %i' % (psf_sigma, gal_sigma, s))
ps.savefig()
rmsdiff = np.sqrt(np.mean(diff**2))
return rmsdiff
def tim_plots(tims, bands, ps):
# Pixel histograms of subimages.
for b in bands:
sig1 = np.median([tim.sig1 for tim in tims if tim.band == b])
plt.clf()
for tim in tims:
if tim.band != b:
continue
# broaden range to encompass most pixels... only req'd
# when sky is bad
lo, hi = -5. * sig1, 5. * sig1
pix = tim.getImage()[tim.getInvError() > 0]
lo = min(lo, np.percentile(pix, 5))
hi = max(hi, np.percentile(pix, 95))
plt.hist(pix,
range=(lo, hi),
bins=50,
histtype='step',
alpha=0.5,
label=tim.name)
plt.legend()
plt.xlabel('Pixel values')
plt.title('Pixel distributions: %s band' % b)
ps.savefig()
plt.clf()
lo, hi = -5., 5.
for tim in tims:
if tim.band != b:
continue
ie = tim.getInvError()
pix = (tim.getImage() * ie)[ie > 0]
plt.hist(pix,
range=(lo, hi),
bins=50,
histtype='step',
alpha=0.5,
label=tim.name)
plt.legend()
plt.xlabel('Pixel values (sigma)')
plt.xlim(lo, hi)
plt.title('Pixel distributions: %s band' % b)
ps.savefig()
# Plot image pixels, invvars, masks
for tim in tims:
plt.clf()
plt.subplot(2, 2, 1)
dimshow(tim.getImage(), vmin=-3. * tim.sig1, vmax=10. * tim.sig1)
plt.title('image')
plt.subplot(2, 2, 2)
dimshow(tim.getInvError(), vmin=0, vmax=1.1 / tim.sig1)
plt.title('inverr')
if tim.dq is not None:
plt.subplot(2, 2, 3)
dimshow(tim.dq, vmin=0, vmax=tim.dq.max())
plt.title('DQ')
plt.subplot(2, 2, 3)
dimshow(((tim.dq & tim.dq_saturation_bits) > 0),
vmin=0,
vmax=1.5,
cmap='hot')
plt.title('SATUR')
plt.subplot(2, 2, 4)
dimshow(tim.getImage() * (tim.getInvError() > 0),
vmin=-3. * tim.sig1,
vmax=10. * tim.sig1)
plt.title('image (masked)')
plt.suptitle(tim.name)
ps.savefig()
if True and tim.dq is not None:
from legacypipe.bits import DQ_BITS
plt.clf()
bitmap = dict([(v, k) for k, v in DQ_BITS.items()])
k = 1
for i in range(12):
bitval = 1 << i
if not bitval in bitmap:
continue
# only 9 bits are actually used
plt.subplot(3, 3, k)
k += 1
plt.imshow((tim.dq & bitval) > 0, vmin=0, vmax=1.5, cmap='hot')
plt.title(bitmap[bitval])
plt.suptitle(
'Mask planes: %s (%s %s)' %
(tim.name, tim.imobj.image_filename, tim.imobj.ccdname))
ps.savefig()
im = tim.imobj
from legacypipe.decam import decam_has_dq_codes
print(tim.name, ': plver "%s"' % im.plver, 'has DQ codes:',
decam_has_dq_codes(im.plver))
if im.camera == 'decam' and decam_has_dq_codes(im.plver):
# Integer codes, not bitmask. Re-read and plot.
dq = im.read_dq(slice=tim.slice)
plt.clf()
plt.subplot(1, 3, 1)
dimshow(tim.getImage(),
vmin=-3. * tim.sig1,
vmax=30. * tim.sig1)
plt.title('image')
plt.subplot(1, 3, 2)
dimshow(tim.getInvError(), vmin=0, vmax=1.1 / tim.sig1)
plt.title('inverr')
plt.subplot(1, 3, 3)
plt.imshow(dq,
interpolation='nearest',
origin='lower',
cmap='tab10',
vmin=-0.5,
vmax=9.5)
plt.colorbar()
plt.title('DQ codes')
plt.suptitle(
'%s (%s %s) PLVER %s' %
(tim.name, im.image_filename, im.ccdname, im.plver))
ps.savefig()
def fitblobs_plots(tims, bands, targetwcs, blobslices, blobsrcs, cat, blobs,
ps):
coimgs, _ = quick_coadds(tims, bands, targetwcs)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
for i, bs in enumerate(blobslices):
sy, sx = bs
by0, by1 = sy.start, sy.stop
bx0, bx1 = sx.start, sx.stop
plt.plot([bx0, bx0, bx1, bx1, bx0], [by0, by1, by1, by0, by0], 'r-')
plt.text((bx0 + bx1) / 2.,
by0,
'%i' % i,
ha='center',
va='bottom',
color='r')
plt.axis(ax)
plt.title('Blobs')
ps.savefig()
for i, Isrcs in enumerate(blobsrcs):
for isrc in Isrcs:
src = cat[isrc]
ra, dec = src.getPosition().ra, src.getPosition().dec
_, x, y = targetwcs.radec2pixelxy(ra, dec)
plt.text(x,
y,
'b%i/s%i' % (i, isrc),
ha='center',
va='bottom',
color='r')
plt.axis(ax)
plt.title('Blobs + Sources')
ps.savefig()
plt.clf()
dimshow(blobs)
ax = plt.axis()
for i, bs in enumerate(blobslices):
sy, sx = bs
by0, by1 = sy.start, sy.stop
bx0, bx1 = sx.start, sx.stop
plt.plot([bx0, bx0, bx1, bx1, bx0], [by0, by1, by1, by0, by0], 'r-')
plt.text((bx0 + bx1) / 2.,
by0,
'%i' % i,
ha='center',
va='bottom',
color='r')
plt.axis(ax)
plt.title('Blobs')
ps.savefig()
plt.clf()
dimshow(blobs != -1)
ax = plt.axis()
for i, bs in enumerate(blobslices):
sy, sx = bs
by0, by1 = sy.start, sy.stop
bx0, bx1 = sx.start, sx.stop
plt.plot([bx0, bx0, bx1, bx1, bx0], [by0, by1, by1, by0, by0], 'r-')
plt.text((bx0 + bx1) / 2.,
by0,
'%i' % i,
ha='center',
va='bottom',
color='r')
plt.axis(ax)
plt.title('Blobs')
ps.savefig()
def stage1(T=None, coimgs=None, cons=None, detmaps=None, detivs=None,
targetrd=None, pixscale=None, targetwcs=None, W=None,H=None,
bands=None, tims=None, ps=None, brick=None, cat=None):
orig_wcsxy0 = [tim.wcs.getX0Y0() for tim in tims]
hot = np.zeros((H,W), np.float32)
for band in bands:
detmap = detmaps[band] / np.maximum(1e-16, detivs[band])
detsn = detmap * np.sqrt(detivs[band])
hot = np.maximum(hot, detsn)
detmaps[band] = detmap
### FIXME -- ugri
for sedname,sed in [('Flat', (1.,1.,1.)), ('Red', (2.5, 1.0, 0.4))]:
sedmap = np.zeros((H,W), np.float32)
sediv = np.zeros((H,W), np.float32)
for iband,band in enumerate(bands):
# We convert the detmap to canonical band via
# detmap * w
# And the corresponding change to sig1 is
# sig1 * w
# So the invvar-weighted sum is
# (detmap * w) / (sig1**2 * w**2)
# = detmap / (sig1**2 * w)
sedmap += detmaps[band] * detivs[band] / sed[iband]
sediv += detivs [band] / sed[iband]**2
sedmap /= np.maximum(1e-16, sediv)
sedsn = sedmap * np.sqrt(sediv)
hot = np.maximum(hot, sedsn)
plt.clf()
dimshow(np.round(sedsn), vmin=0, vmax=10, cmap='hot')
plt.title('SED-matched detection filter: %s' % sedname)
ps.savefig()
peaks = (hot > 4)
blobs,nblobs = label(peaks)
print 'N detected blobs:', nblobs
blobslices = find_objects(blobs)
# Un-set catalog blobs
for x,y in zip(T.itx, T.ity):
# blob number
bb = blobs[y,x]
if bb == 0:
continue
# un-set 'peaks' within this blob
slc = blobslices[bb-1]
peaks[slc][blobs[slc] == bb] = 0
# Now, after having removed catalog sources, crank up the detection threshold
peaks &= (hot > 5)
# zero out the edges(?)
peaks[0 ,:] = peaks[:, 0] = 0
peaks[-1,:] = peaks[:,-1] = 0
peaks[1:-1, 1:-1] &= (hot[1:-1,1:-1] >= hot[0:-2,1:-1])
peaks[1:-1, 1:-1] &= (hot[1:-1,1:-1] >= hot[2: ,1:-1])
peaks[1:-1, 1:-1] &= (hot[1:-1,1:-1] >= hot[1:-1,0:-2])
peaks[1:-1, 1:-1] &= (hot[1:-1,1:-1] >= hot[1:-1,2: ])
# These are our peaks
pki = np.flatnonzero(peaks)
peaky,peakx = np.unravel_index(pki, peaks.shape)
print len(peaky), 'peaks'
crossa = dict(ms=10, mew=1.5)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(T.tx, T.ty, 'r+', **crossa)
plt.plot(peakx, peaky, '+', color=green, **crossa)
plt.axis(ax)
plt.title('SDSS + SED-matched detections')
ps.savefig()
### HACK -- high threshold again
# Segment, and record which sources fall into each blob
blobs,nblobs = label((hot > 20))
print 'N detected blobs:', nblobs
blobslices = find_objects(blobs)
T.blob = blobs[T.ity, T.itx]
blobsrcs = []
blobflux = []
fluximg = coimgs[1]
for blob in range(1, nblobs+1):
blobsrcs.append(np.flatnonzero(T.blob == blob))
bslc = blobslices[blob-1]
blobflux.append(np.sum(fluximg[bslc][blobs[bslc] == blob]))
# Fit the SDSS sources
for tim in tims:
tim.psfex.fitSavedData(*tim.psfex.splinedata)
tim.psf = tim.psfex
# How far down to render model profiles
minsigma = 0.1
for tim in tims:
tim.modelMinval = minsigma * tim.sig1
srcvariances = [[] for src in cat]
# Fit in order of flux
for blobnumber,iblob in enumerate(np.argsort(-np.array(blobflux))):
bslc = blobslices[iblob]
Isrcs = blobsrcs [iblob]
if len(Isrcs) == 0:
continue
print
print 'Blob', blobnumber, 'of', len(blobflux), ':', len(Isrcs), 'sources'
print 'Source indices:', Isrcs
print
# blob bbox in target coords
sy,sx = bslc
by0,by1 = sy.start, sy.stop
bx0,bx1 = sx.start, sx.stop
blobh,blobw = by1 - by0, bx1 - bx0
rr,dd = targetwcs.pixelxy2radec([bx0,bx0,bx1,bx1],[by0,by1,by1,by0])
alphas = [0.1, 0.3, 1.0]
subtims = []
for itim,tim in enumerate(tims):
h,w = tim.shape
ok,x,y = tim.subwcs.radec2pixelxy(rr,dd)
sx0,sx1 = x.min(), x.max()
sy0,sy1 = y.min(), y.max()
if sx1 < 0 or sy1 < 0 or sx1 > w or sy1 > h:
continue
sx0 = np.clip(int(np.floor(sx0)), 0, w-1)
sx1 = np.clip(int(np.ceil (sx1)), 0, w-1) + 1
sy0 = np.clip(int(np.floor(sy0)), 0, h-1)
sy1 = np.clip(int(np.ceil (sy1)), 0, h-1) + 1
subslc = slice(sy0,sy1),slice(sx0,sx1)
subimg = tim.getImage ()[subslc]
subie = tim.getInvError()[subslc]
subwcs = tim.getWcs().copy()
ox0,oy0 = orig_wcsxy0[itim]
subwcs.setX0Y0(ox0 + sx0, oy0 + sy0)
# Mask out inverr for pixels that are not within the blob.
subtarget = targetwcs.get_subimage(bx0, by0, blobw, blobh)
subsubwcs = tim.subwcs.get_subimage(int(sx0), int(sy0), int(sx1-sx0), int(sy1-sy0))
try:
Yo,Xo,Yi,Xi,rims = resample_with_wcs(subsubwcs, subtarget, [], 2)
except OverlapError:
print 'No overlap'
continue
if len(Yo) == 0:
continue
subie2 = np.zeros_like(subie)
I = np.flatnonzero(blobs[bslc][Yi, Xi] == (iblob+1))
subie2[Yo[I],Xo[I]] = subie[Yo[I],Xo[I]]
subie = subie2
# If the subimage (blob) is small enough, instantiate a
# constant PSF model in the center.
if sy1-sy0 < 100 and sx1-sx0 < 100:
subpsf = tim.psf.mogAt(ox0 + (sx0+sx1)/2., oy0 + (sy0+sy1)/2.)
else:
# Otherwise, instantiate a (shifted) spatially-varying
# PsfEx model.
subpsf = ShiftedPsf(tim.psf, ox0+sx0, oy0+sy0)
subtim = Image(data=subimg, inverr=subie, wcs=subwcs,
psf=subpsf, photocal=tim.getPhotoCal(),
sky=tim.getSky(), name=tim.name)
subtim.band = tim.band
subtim.sig1 = tim.sig1
subtim.modelMinval = tim.modelMinval
subtims.append(subtim)
subcat = Catalog(*[cat[i] for i in Isrcs])
subtr = Tractor(subtims, subcat)
subtr.freezeParam('images')
# Optimize individual sources in order of flux
fluxes = []
for src in subcat:
# HACK -- here we just *sum* the nanomaggies in each band. Bogus!
br = src.getBrightness()
flux = sum([br.getFlux(band) for band in bands])
fluxes.append(flux)
Ibright = np.argsort(-np.array(fluxes))
if len(Ibright) >= 5:
# -Remember the original subtim images
# -Compute initial models for each source (in each tim)
# -Subtract initial models from images
# -During fitting, for each source:
# -add back in the source's initial model (to each tim)
# -fit, with Catalog([src])
# -subtract final model (from each tim)
# -Replace original subtim images
#
# --Might want to omit newly-added detection-filter sources, since their
# fluxes are bogus.
# Remember original tim images
orig_timages = [tim.getImage().copy() for tim in subtims]
initial_models = []
# Create initial models for each tim x each source
for tim in subtims:
mods = []
for src in subcat:
mod = src.getModelPatch(tim)
mods.append(mod)
if mod is not None:
if not np.all(np.isfinite(mod.patch)):
print 'Non-finite mod patch'
print 'source:', src
print 'tim:', tim
print 'PSF:', tim.getPsf()
assert(np.all(np.isfinite(mod.patch)))
mod.addTo(tim.getImage(), scale=-1)
initial_models.append(mods)
# For sources in decreasing order of brightness
for numi,i in enumerate(Ibright):
tsrc = Time()
print 'Fitting source', i, '(%i of %i in blob)' % (numi, len(Ibright))
src = subcat[i]
print src
srctractor = Tractor(subtims, [src])
srctractor.freezeParams('images')
# Add this source's initial model back in.
for tim,mods in zip(subtims, initial_models):
mod = mods[i]
if mod is not None:
mod.addTo(tim.getImage())
print 'Optimizing:', srctractor
srctractor.printThawedParams()
for step in range(50):
dlnp,X,alpha = srctractor.optimize(priors=False, shared_params=False,
alphas=alphas)
print 'dlnp:', dlnp, 'src', src
if dlnp < 0.1:
break
for tim in subtims:
mod = src.getModelPatch(tim)
if mod is not None:
mod.addTo(tim.getImage(), scale=-1)
for tim,img in zip(subtims, orig_timages):
tim.data = img
del orig_timages
del initial_models
else:
# Fit sources one at a time, but don't subtract other models
subcat.freezeAllParams()
for numi,i in enumerate(Ibright):
tsrc = Time()
print 'Fitting source', i, '(%i of %i in blob)' % (numi, len(Ibright))
print subcat[i]
subcat.freezeAllBut(i)
print 'Optimizing:', subtr
subtr.printThawedParams()
for step in range(10):
dlnp,X,alpha = subtr.optimize(priors=False, shared_params=False,
alphas=alphas)
print 'dlnp:', dlnp
if dlnp < 0.1:
break
print 'Fitting source took', Time()-tsrc
print subcat[i]
if len(Isrcs) > 1 and len(Isrcs) <= 10:
tfit = Time()
# Optimize all at once?
subcat.thawAllParams()
print 'Optimizing:', subtr
subtr.printThawedParams()
for step in range(20):
dlnp,X,alpha = subtr.optimize(priors=False, shared_params=False,
alphas=alphas)
print 'dlnp:', dlnp
if dlnp < 0.1:
break
# Variances
subcat.thawAllRecursive()
subcat.freezeAllParams()
for isub,srci in enumerate(Isrcs):
print 'Variances for source', srci
subcat.thawParam(isub)
src = subcat[isub]
print 'Source', src
print 'Params:', src.getParamNames()
if isinstance(src, (DevGalaxy, ExpGalaxy)):
src.shape = EllipseE.fromEllipseESoft(src.shape)
elif isinstance(src, FixedCompositeGalaxy):
src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp)
src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev)
print 'Converted ellipse:', src
allderivs = subtr.getDerivs()
for iparam,derivs in enumerate(allderivs):
dchisq = 0
for deriv,tim in derivs:
h,w = tim.shape
deriv.clipTo(w,h)
ie = tim.getInvError()
slc = deriv.getSlice(ie)
chi = deriv.patch * ie[slc]
dchisq += (chi**2).sum()
if dchisq == 0.:
v = np.nan
else:
v = 1./dchisq
srcvariances[srci].append(v)
assert(len(srcvariances[srci]) == subcat[isub].numberOfParams())
subcat.freezeParam(isub)
cat.thawAllRecursive()
for i,src in enumerate(cat):
print 'Source', i, src
print 'variances:', srcvariances[i]
print len(srcvariances[i]), 'vs', src.numberOfParams()
if len(srcvariances[i]) != src.numberOfParams():
# This can happen for sources outside the brick bounds: they never get optimized?
print 'Warning: zeroing variances for source', src
srcvariances[i] = [0]*src.numberOfParams()
if isinstance(src, (DevGalaxy, ExpGalaxy)):
src.shape = EllipseE.fromEllipseESoft(src.shape)
elif isinstance(src, FixedCompositeGalaxy):
src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp)
src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev)
assert(len(srcvariances[i]) == src.numberOfParams())
variances = np.hstack(srcvariances)
assert(len(variances) == cat.numberOfParams())
return dict(cat=cat, variances=variances)
def segment_and_group_sources(image, T, name=None, ps=None, plots=False):
'''
*image*: binary image that defines "blobs"
*T*: source table; only ".ibx" and ".iby" elements are used (x,y integer
pix pos). Note: ".blob" field is added.
*name*: for debugging only
Returns: (blobs, blobsrcs, blobslices)
*blobs*: image, values -1 = no blob, integer blob indices
*blobsrcs*: list of np arrays of integers, elements in T within each blob
*blobslices*: list of slice objects for blob bounding-boxes.
'''
from scipy.ndimage.morphology import binary_fill_holes
from scipy.ndimage.measurements import label, find_objects
image = binary_fill_holes(image)
blobs, nblobs = label(image)
#print('Detected blobs:', nblobs)
H, W = image.shape
del image
blobslices = find_objects(blobs)
clipx = np.clip(T.ibx, 0, W - 1)
clipy = np.clip(T.iby, 0, H - 1)
T.blob = blobs[clipy, clipx]
if plots:
import pylab as plt
from astrometry.util.plotutils import dimshow
plt.clf()
dimshow(blobs > 0, vmin=0, vmax=1)
ax = plt.axis()
for i, bs in enumerate(blobslices):
sy, sx = bs
by0, by1 = sy.start, sy.stop
bx0, bx1 = sx.start, sx.stop
plt.plot([bx0, bx0, bx1, bx1, bx0], [by0, by1, by1, by0, by0],
'r-')
plt.text((bx0 + bx1) / 2.,
by0,
'%i' % (i + 1),
ha='center',
va='bottom',
color='r')
plt.plot(T.ibx, T.iby, 'rx')
for i, t in enumerate(T):
plt.text(t.ibx,
t.iby,
'src %i' % i,
color='red',
ha='left',
va='center')
plt.axis(ax)
plt.title('Blobs')
ps.savefig()
# Find sets of sources within blobs
blobsrcs = []
keepslices = []
blobmap = {}
for blob in range(1, nblobs + 1):
Isrcs, = np.nonzero(T.blob == blob)
if len(Isrcs) == 0:
blobmap[blob] = -1
continue
blobmap[blob] = len(blobsrcs)
blobsrcs.append(Isrcs)
bslc = blobslices[blob - 1]
keepslices.append(bslc)
blobslices = keepslices
# Find sources that do not belong to a blob and add them as
# singleton "blobs"; otherwise they don't get optimized.
# for sources outside the image bounds, what should we do?
inblobs = np.zeros(len(T), bool)
for Isrcs in blobsrcs:
inblobs[Isrcs] = True
noblobs = np.flatnonzero(np.logical_not(inblobs))
del inblobs
#print(len(noblobs), 'sources are not in blobs')
# Remap the "blobs" image so that empty regions are = -1 and the blob values
# correspond to their indices in the "blobsrcs" list.
if len(blobmap):
maxblob = max(blobmap.keys())
else:
maxblob = 0
maxblob = max(maxblob, blobs.max())
bm = np.zeros(maxblob + 1, int)
for k, v in blobmap.items():
bm[k] = v
bm[0] = -1
# Remap blob numbers
blobs = bm[blobs]
if plots:
from astrometry.util.plotutils import dimshow
plt.clf()
dimshow(blobs > -1, vmin=0, vmax=1)
ax = plt.axis()
for i, bs in enumerate(blobslices):
sy, sx = bs
by0, by1 = sy.start, sy.stop
bx0, bx1 = sx.start, sx.stop
plt.plot([bx0, bx0, bx1, bx1, bx0], [by0, by1, by1, by0, by0],
'r-')
plt.text((bx0 + bx1) / 2.,
by0,
'%i' % (i + 1),
ha='center',
va='bottom',
color='r')
plt.plot(T.ibx, T.iby, 'rx')
for i, t in enumerate(T):
plt.text(t.ibx,
t.iby,
'src %i' % i,
color='red',
ha='left',
va='center')
plt.axis(ax)
plt.title('Blobs')
ps.savefig()
for j, Isrcs in enumerate(blobsrcs):
for i in Isrcs:
if (blobs[clipy[i], clipx[i]] != j):
print(
'---------------------------!!!-------------------------')
print('Blob', j, 'sources', Isrcs)
print('Source', i, 'coords x,y', T.ibx[i], T.iby[i])
print('Expected blob value', j, 'but got', blobs[clipy[i],
clipx[i]])
T.blob = blobs[clipy, clipx]
assert (len(blobsrcs) == len(blobslices))
return blobs, blobsrcs, blobslices
def _psf_check_plots(tims):
# HACK -- check PSF models
plt.figure(num=2, figsize=(7,4.08))
for im,tim in zip(ims,tims):
print
print 'Image', tim.name
plt.subplots_adjust(left=0, right=1, bottom=0, top=0.95,
hspace=0, wspace=0)
W,H = 2048,4096
psfex = PsfEx(im.psffn, W, H)
psfim0 = psfim = psfex.instantiateAt(W/2, H/2)
# trim
psfim = psfim[10:-10, 10:-10]
tfit = Time()
psffit2 = GaussianMixtureEllipsePSF.fromStamp(psfim, N=2)
print 'Fitting PSF mog:', psfim.shape, Time()-tfit
psfim = psfim0[5:-5, 5:-5]
tfit = Time()
psffit2 = GaussianMixtureEllipsePSF.fromStamp(psfim, N=2)
print 'Fitting PSF mog:', psfim.shape, Time()-tfit
ph,pw = psfim.shape
psffit = GaussianMixtureEllipsePSF.fromStamp(psfim, N=3)
#mx = 0.03
mx = psfim.max()
mod3 = np.zeros_like(psfim)
p = psffit.getPointSourcePatch(pw/2, ph/2, radius=pw/2)
p.addTo(mod3)
mod2 = np.zeros_like(psfim)
p = psffit2.getPointSourcePatch(pw/2, ph/2, radius=pw/2)
p.addTo(mod2)
plt.clf()
plt.subplot(2,3,1)
dimshow(psfim, vmin=0, vmax=mx, ticks=False)
plt.subplot(2,3,2)
dimshow(mod3, vmin=0, vmax=mx, ticks=False)
plt.subplot(2,3,3)
dimshow(mod2, vmin=0, vmax=mx, ticks=False)
plt.subplot(2,3,5)
dimshow(psfim-mod3, vmin=-mx/2, vmax=mx/2, ticks=False)
plt.subplot(2,3,6)
dimshow(psfim-mod2, vmin=-mx/2, vmax=mx/2, ticks=False)
ps.savefig()
#continue
for round in [1,2,3,4,5]:
plt.clf()
k = 1
#rows,cols = 10,5
rows,cols = 7,4
for iy,y in enumerate(np.linspace(0, H, rows).astype(int)):
for ix,x in enumerate(np.linspace(0, W, cols).astype(int)):
psfimg = psfex.instantiateAt(x, y)
# trim
psfimg = psfimg[5:-5, 5:-5]
print 'psfimg', psfimg.shape
ph,pw = psfimg.shape
psfimg2 = tim.psfex.getPointSourcePatch(x, y, radius=pw/2)
mod = np.zeros_like(psfimg)
h,w = mod.shape
#psfimg2.x0 -= x
#psfimg2.x0 += w/2
#psfimg2.y0 -= y
#psfimg2.y0 += h/2
psfimg2.x0 = 0
psfimg2.y0 = 0
print 'psfimg2:', (psfimg2.x0,psfimg2.y0)
psfimg2.addTo(mod)
print 'psfimg:', psfimg.min(), psfimg.max(), psfimg.sum()
print 'psfimg2:', psfimg2.patch.min(), psfimg2.patch.max(), psfimg2.patch.sum()
print 'mod:', mod.min(), mod.max(), mod.sum()
#plt.subplot(rows, cols, k)
plt.subplot(cols, rows, k)
k += 1
kwa = dict(vmin=0, vmax=mx, ticks=False)
if round == 1:
dimshow(psfimg, **kwa)
plt.suptitle('PsfEx')
elif round == 2:
dimshow(mod, **kwa)
plt.suptitle('varying MoG')
elif round == 3:
dimshow(psfimg - mod, vmin=-mx/2, vmax=mx/2, ticks=False)
plt.suptitle('PsfEx - varying MoG')
elif round == 4:
dimshow(psfimg - mod3, vmin=-mx/2, vmax=mx/2, ticks=False)
plt.suptitle('PsfEx - const MoG(3)')
elif round == 5:
dimshow(psfimg - mod2, vmin=-mx/2, vmax=mx/2, ticks=False)
plt.suptitle('PsfEx - const MoG(2)')
ps.savefig()
def run(self, ps=None, focus=False, momentsize=5, n_fwhm=100):
import pylab as plt
from astrometry.util.plotutils import dimshow, plothist
from legacyanalysis.ps1cat import ps1cat
import photutils
import tractor
fn = self.fn
ext = self.ext
pixsc = self.pixscale
F = fitsio.FITS(fn)
primhdr = F[0].read_header()
self.primhdr = primhdr
img, hdr = self.read_raw(F, ext)
self.hdr = hdr
# pre sky-sub
mn, mx = np.percentile(img.ravel(), [25, 98])
self.imgkwa = dict(vmin=mn, vmax=mx, cmap='gray')
if self.debug and ps is not None:
plt.clf()
dimshow(img, **self.imgkwa)
plt.title('Raw image')
ps.savefig()
M = 200
plt.clf()
plt.subplot(2, 2, 1)
dimshow(img[-M:, :M], ticks=False, **self.imgkwa)
plt.subplot(2, 2, 2)
dimshow(img[-M:, -M:], ticks=False, **self.imgkwa)
plt.subplot(2, 2, 3)
dimshow(img[:M, :M], ticks=False, **self.imgkwa)
plt.subplot(2, 2, 4)
dimshow(img[:M, -M:], ticks=False, **self.imgkwa)
plt.suptitle('Raw image corners')
ps.savefig()
img, trim_x0, trim_y0 = self.trim_edges(img)
fullH, fullW = img.shape
if self.debug and ps is not None:
plt.clf()
dimshow(img, **self.imgkwa)
plt.title('Trimmed image')
ps.savefig()
M = 200
plt.clf()
plt.subplot(2, 2, 1)
dimshow(img[-M:, :M], ticks=False, **self.imgkwa)
plt.subplot(2, 2, 2)
dimshow(img[-M:, -M:], ticks=False, **self.imgkwa)
plt.subplot(2, 2, 3)
dimshow(img[:M, :M], ticks=False, **self.imgkwa)
plt.subplot(2, 2, 4)
dimshow(img[:M, -M:], ticks=False, **self.imgkwa)
plt.suptitle('Trimmed corners')
ps.savefig()
band = self.get_band(primhdr)
exptime = primhdr['EXPTIME']
airmass = primhdr['AIRMASS']
print('Band', band, 'Exptime', exptime, 'Airmass', airmass)
zp0 = self.nom.zeropoint(band, ext=self.ext)
sky0 = self.nom.sky(band)
kx = self.nom.fiducial_exptime(band).k_co
# Find the sky value and noise level
sky, sig1 = self.get_sky_and_sigma(img)
sky1 = np.median(sky)
skybr = -2.5 * np.log10(sky1 / pixsc / pixsc / exptime) + zp0
print('Sky brightness: %8.3f mag/arcsec^2' % skybr)
print('Fiducial: %8.3f mag/arcsec^2' % sky0)
img -= sky
self.remove_sky_gradients(img)
# Post sky-sub
mn, mx = np.percentile(img.ravel(), [25, 98])
self.imgkwa = dict(vmin=mn, vmax=mx, cmap='gray')
if ps is not None:
plt.clf()
dimshow(img, **self.imgkwa)
plt.title('Sky-sub image: %s-%s' % (os.path.basename(fn).replace(
'.fits', '').replace('.fz', ''), ext))
plt.colorbar()
ps.savefig()
# Read WCS header and compute boresight
wcs = self.get_wcs(hdr)
ra_ccd, dec_ccd = wcs.pixelxy2radec((fullW + 1) / 2., (fullH + 1) / 2.)
# Detect stars
psfsig = self.nominal_fwhm / 2.35
detsn = self.detection_map(img, sig1, psfsig, ps)
slices = self.detect_sources(detsn, self.det_thresh, ps)
print(len(slices), 'sources detected')
if len(slices) < 20:
slices = self.detect_sources(detsn, 10., ps)
print(len(slices), 'sources detected')
ndetected = len(slices)
camera = primhdr.get('INSTRUME', '').strip().lower()
# -> "decam" / "mosaic3"
meas = dict(band=band,
airmass=airmass,
skybright=skybr,
pixscale=pixsc,
primhdr=primhdr,
hdr=hdr,
wcs=wcs,
ra_ccd=ra_ccd,
dec_ccd=dec_ccd,
extension=ext,
camera=camera,
ndetected=ndetected)
if ndetected == 0:
print('NO SOURCES DETECTED')
return meas
xx, yy = [], []
fx, fy = [], []
mx2, my2, mxy = [], [], []
wmx2, wmy2, wmxy = [], [], []
# "Peak" region to centroid
P = momentsize
H, W = img.shape
for i, slc in enumerate(slices):
y0 = slc[0].start
x0 = slc[1].start
subimg = detsn[slc]
imax = np.argmax(subimg)
y, x = np.unravel_index(imax, subimg.shape)
if (x0 + x) < P or (x0 + x) > W - 1 - P or (y0 + y) < P or (
y0 + y) > H - 1 - P:
#print('Skipping edge peak', x0+x, y0+y)
continue
xx.append(x0 + x)
yy.append(y0 + y)
pkarea = detsn[y0 + y - P:y0 + y + P + 1,
x0 + x - P:x0 + x + P + 1]
from scipy.ndimage.measurements import center_of_mass
cy, cx = center_of_mass(pkarea)
#print('Center of mass', cx,cy)
fx.append(x0 + x - P + cx)
fy.append(y0 + y - P + cy)
#print('x,y', x0+x, y0+y, 'vs centroid', x0+x-P+cx, y0+y-P+cy)
### HACK -- measure source ellipticity
# go back to the image (not detection map)
#subimg = img[slc]
subimg = img[y0 + y - P:y0 + y + P + 1,
x0 + x - P:x0 + x + P + 1].copy()
subimg /= subimg.sum()
ph, pw = subimg.shape
px, py = np.meshgrid(np.arange(pw), np.arange(ph))
mx2.append(np.sum(subimg * (px - cx)**2))
my2.append(np.sum(subimg * (py - cy)**2))
mxy.append(np.sum(subimg * (px - cx) * (py - cy)))
# Gaussian windowed version
s = 1.
wimg = subimg * np.exp(-0.5 * ((px - cx)**2 + (py - cy)**2) / s**2)
wimg /= np.sum(wimg)
wmx2.append(np.sum(wimg * (px - cx)**2))
wmy2.append(np.sum(wimg * (py - cy)**2))
wmxy.append(np.sum(wimg * (px - cx) * (py - cy)))
mx2 = np.array(mx2)
my2 = np.array(my2)
mxy = np.array(mxy)
wmx2 = np.array(wmx2)
wmy2 = np.array(wmy2)
wmxy = np.array(wmxy)
# semi-major/minor axes and position angle
theta = np.rad2deg(np.arctan2(2 * mxy, mx2 - my2) / 2.)
theta = np.abs(theta) * np.sign(mxy)
s = np.sqrt(((mx2 - my2) / 2.)**2 + mxy**2)
a = np.sqrt((mx2 + my2) / 2. + s)
b = np.sqrt((mx2 + my2) / 2. - s)
ell = 1. - b / a
wtheta = np.rad2deg(np.arctan2(2 * wmxy, wmx2 - wmy2) / 2.)
wtheta = np.abs(wtheta) * np.sign(wmxy)
ws = np.sqrt(((wmx2 - wmy2) / 2.)**2 + wmxy**2)
wa = np.sqrt((wmx2 + wmy2) / 2. + ws)
wb = np.sqrt((wmx2 + wmy2) / 2. - ws)
well = 1. - wb / wa
fx = np.array(fx)
fy = np.array(fy)
xx = np.array(xx)
yy = np.array(yy)
if ps is not None:
plt.clf()
dimshow(detsn, vmin=-3, vmax=50, cmap='gray')
ax = plt.axis()
plt.plot(fx, fy, 'go', mec='g', mfc='none', ms=10)
plt.colorbar()
plt.title('Detected sources')
plt.axis(ax)
ps.savefig()
# show centroids too
# plt.plot(xx, yy, 'go', mec='g', mfc='none', ms=8)
# plt.axis(ax)
# ps.savefig()
# if ps is not None:
# plt.clf()
# plt.subplot(2,1,1)
# mx = np.percentile(np.append(mx2,my2), 99)
# ha = dict(histtype='step', range=(0,mx), bins=50)
# plt.hist(mx2, color='b', label='mx2', **ha)
# plt.hist(my2, color='r', label='my2', **ha)
# plt.hist(mxy, color='g', label='mxy', **ha)
# plt.legend()
# plt.xlim(0,mx)
# plt.subplot(2,1,2)
# mx = np.percentile(np.append(wmx2,wmy2), 99)
# ha = dict(histtype='step', range=(0,mx), bins=50, lw=3, alpha=0.3)
# plt.hist(wmx2, color='b', label='wx2', **ha)
# plt.hist(wmy2, color='r', label='wy2', **ha)
# plt.hist(wmxy, color='g', label='wxy', **ha)
# plt.legend()
# plt.xlim(0,mx)
# plt.suptitle('Source moments')
# ps.savefig()
#
# #mx = np.percentile(np.abs(np.append(mxy,wmxy)), 99)
# plt.clf()
# plt.subplot(2,1,1)
# ha = dict(histtype='step', range=(0,1), bins=50)
# plt.hist(ell, color='g', label='ell', **ha)
# plt.hist(well, color='g', lw=3, alpha=0.3, label='windowed ell', **ha)
# plt.legend()
# plt.subplot(2,1,2)
# ha = dict(histtype='step', range=(-90,90), bins=50)
# plt.hist(theta, color='g', label='theta', **ha)
# plt.hist(wtheta, color='g', lw=3, alpha=0.3,
# label='windowed theta', **ha)
# plt.xlim(-90,90)
# plt.legend()
# plt.suptitle('Source ellipticities & angles')
# ps.savefig()
# Cut down to stars whose centroids are within 1 pixel of their peaks...
#keep = (np.hypot(fx - xx, fy - yy) < 2)
#print(sum(keep), 'of', len(keep), 'stars have centroids within 2 of peaks')
#print('mean dx', np.mean(fx-xx), 'dy', np.mean(fy-yy), 'pixels')
#assert(float(sum(keep)) / len(keep) > 0.9)
#fx = fx[keep]
#fy = fy[keep]
apxy = np.vstack((fx, fy)).T
ap = []
aprad_pix = self.aprad / pixsc
aper = photutils.CircularAperture(apxy, aprad_pix)
p = photutils.aperture_photometry(img, aper)
apflux = p.field('aperture_sum')
# Manual aperture photometry to get clipped means in sky annulus
sky_inner_r, sky_outer_r = [r / pixsc for r in self.skyrad]
sky = []
for xi, yi in zip(fx, fy):
ix = int(np.round(xi))
iy = int(np.round(yi))
skyR = int(np.ceil(sky_outer_r))
xlo = max(0, ix - skyR)
xhi = min(W, ix + skyR + 1)
ylo = max(0, iy - skyR)
yhi = min(H, iy + skyR + 1)
xx, yy = np.meshgrid(np.arange(xlo, xhi), np.arange(ylo, yhi))
r2 = (xx - xi)**2 + (yy - yi)**2
inannulus = ((r2 >= sky_inner_r**2) * (r2 < sky_outer_r**2))
skypix = img[ylo:yhi, xlo:xhi][inannulus]
#print('ylo,yhi, xlo,xhi', ylo,yhi, xlo,xhi, 'img subshape', img[ylo:yhi, xlo:xhi].shape, 'inann shape', inannulus.shape)
s, nil = sensible_sigmaclip(skypix)
sky.append(s)
sky = np.array(sky)
apflux2 = apflux - sky * (np.pi * aprad_pix**2)
good = (apflux2 > 0) * (apflux > 0)
apflux = apflux[good]
apflux2 = apflux2[good]
fx = fx[good]
fy = fy[good]
# Read in the PS1 catalog, and keep those within 0.25 deg of CCD center
# and those with main sequence colors
pscat = ps1cat(ccdwcs=wcs)
stars = pscat.get_stars()
#print('Got PS1 stars:', len(stars))
# we add the color term later
ps1band = ps1cat.ps1band[band]
stars.mag = stars.median[:, ps1band]
ok, px, py = wcs.radec2pixelxy(stars.ra, stars.dec)
px -= 1
py -= 1
if ps is not None:
#kwa = dict(vmin=-3*sig1, vmax=50*sig1, cmap='gray')
# Add to the 'detected sources' plot
# mn,mx = np.percentile(img.ravel(), [50,99])
# kwa = dict(vmin=mn, vmax=mx, cmap='gray')
# plt.clf()
# dimshow(img, **kwa)
ax = plt.axis()
#plt.plot(fx, fy, 'go', mec='g', mfc='none', ms=10)
K = np.argsort(stars.mag)
plt.plot(px[K[:10]] - trim_x0,
py[K[:10]] - trim_y0,
'o',
mec='m',
mfc='none',
ms=12,
mew=2)
plt.plot(px[K[10:]] - trim_x0,
py[K[10:]] - trim_y0,
'o',
mec='m',
mfc='none',
ms=8)
plt.axis(ax)
plt.title('PS1 stars')
#plt.colorbar()
ps.savefig()
# we trimmed the image before running detection; re-add that margin
fullx = fx + trim_x0
fully = fy + trim_y0
# Match PS1 to our detections, find offset
radius = self.maxshift / pixsc
I, J, dx, dy = self.match_ps1_stars(px, py, fullx, fully, radius,
stars)
print(len(I), 'spatial matches with large radius', self.maxshift,
'arcsec,', radius, 'pix')
bins = 2 * int(np.ceil(radius))
#print('Histogramming with', bins, 'bins')
histo, xe, ye = np.histogram2d(dx,
dy,
bins=bins,
range=((-radius, radius), (-radius,
radius)))
# smooth histogram before finding peak -- fuzzy matching
from scipy.ndimage.filters import gaussian_filter
histo = gaussian_filter(histo, 1.)
histo = histo.T
mx = np.argmax(histo)
my, mx = np.unravel_index(mx, histo.shape)
shiftx = (xe[mx] + xe[mx + 1]) / 2.
shifty = (ye[my] + ye[my + 1]) / 2.
if ps is not None:
plt.clf()
plothist(dx, dy, range=((-radius, radius), (-radius, radius)))
plt.xlabel('dx (pixels)')
plt.ylabel('dy (pixels)')
plt.title('Offsets to PS1 stars')
ax = plt.axis()
plt.axhline(0, color='b')
plt.axvline(0, color='b')
plt.plot(shiftx, shifty, 'o', mec='m', mfc='none', ms=15, mew=3)
plt.axis(ax)
ps.savefig()
# Refine with smaller search radius
radius2 = 3. / pixsc
I, J, dx, dy = self.match_ps1_stars(px, py, fullx + shiftx,
fully + shifty, radius2, stars)
print(len(J), 'matches to PS1 with small radius', 3, 'arcsec')
shiftx2 = np.median(dx)
shifty2 = np.median(dy)
#print('Stage-1 shift', shiftx, shifty)
#print('Stage-2 shift', shiftx2, shifty2)
sx = shiftx + shiftx2
sy = shifty + shifty2
print('Astrometric shift (%.0f, %.0f) pixels' % (sx, sy))
if self.debug and ps is not None:
plt.clf()
plothist(dx, dy, range=((-radius2, radius2), (-radius2, radius2)))
plt.xlabel('dx (pixels)')
plt.ylabel('dy (pixels)')
plt.title('Offsets to PS1 stars')
ax = plt.axis()
plt.axhline(0, color='b')
plt.axvline(0, color='b')
plt.plot(shiftx2, shifty2, 'o', mec='m', mfc='none', ms=15, mew=3)
plt.axis(ax)
ps.savefig()
if ps is not None:
mn, mx = np.percentile(img.ravel(), [50, 99])
kwa2 = dict(vmin=mn, vmax=mx, cmap='gray')
plt.clf()
dimshow(img, **kwa2)
ax = plt.axis()
plt.plot(fx[J], fy[J], 'go', mec='g', mfc='none', ms=10, mew=2)
plt.plot(px[I] - sx - trim_x0,
py[I] - sy - trim_y0,
'm+',
ms=10,
mew=2)
plt.axis(ax)
plt.title('Matched PS1 stars')
plt.colorbar()
ps.savefig()
plt.clf()
dimshow(img, **kwa2)
ax = plt.axis()
plt.plot(fx[J], fy[J], 'go', mec='g', mfc='none', ms=10, mew=2)
K = np.argsort(stars.mag)
plt.plot(px[K[:10]] - sx - trim_x0,
py[K[:10]] - sy - trim_y0,
'o',
mec='m',
mfc='none',
ms=12,
mew=2)
plt.plot(px[K[10:]] - sx - trim_x0,
py[K[10:]] - sy - trim_y0,
'o',
mec='m',
mfc='none',
ms=8,
mew=2)
plt.axis(ax)
plt.title('All PS1 stars')
plt.colorbar()
ps.savefig()
# Now cut to just *stars* with good colors
stars.gicolor = stars.median[:, 0] - stars.median[:, 2]
keep = (stars.gicolor > 0.4) * (stars.gicolor < 2.7)
stars.cut(keep)
if len(stars) == 0:
print('No overlap or too few stars in PS1')
return None
px = px[keep]
py = py[keep]
# Re-match
I, J, dx, dy = self.match_ps1_stars(px, py, fullx + sx, fully + sy,
radius2, stars)
print('Cut to', len(stars), 'PS1 stars with good colors; matched',
len(I))
nmatched = len(I)
meas.update(dx=sx, dy=sy, nmatched=nmatched)
if focus:
meas.update(img=img,
hdr=hdr,
primhdr=primhdr,
fx=fx,
fy=fy,
px=px - trim_x0 - sx,
py=py - trim_y0 - sy,
sig1=sig1,
stars=stars,
moments=(mx2, my2, mxy, theta, a, b, ell),
wmoments=(wmx2, wmy2, wmxy, wtheta, wa, wb, well),
apflux=apflux,
apflux2=apflux2)
return meas
#print('Mean astrometric shift (arcsec): delta-ra=', -np.mean(dy)*0.263, 'delta-dec=', np.mean(dx)*0.263)
# Compute photometric offset compared to PS1
# as the PS1 minus observed mags
colorterm = self.colorterm_ps1_to_observed(stars.median, band)
stars.mag += colorterm
ps1mag = stars.mag[I]
if False and ps is not None:
plt.clf()
plt.semilogy(ps1mag, apflux2[J], 'b.')
plt.xlabel('PS1 mag')
plt.ylabel('DECam ap flux (with sky sub)')
ps.savefig()
plt.clf()
plt.semilogy(ps1mag, apflux[J], 'b.')
plt.xlabel('PS1 mag')
plt.ylabel('DECam ap flux (no sky sub)')
ps.savefig()
apmag2 = -2.5 * np.log10(apflux2) + zp0 + 2.5 * np.log10(exptime)
apmag = -2.5 * np.log10(apflux) + zp0 + 2.5 * np.log10(exptime)
if ps is not None:
plt.clf()
plt.plot(ps1mag, apmag[J], 'b.', label='No sky sub')
plt.plot(ps1mag, apmag2[J], 'r.', label='Sky sub')
# ax = plt.axis()
# mn = min(ax[0], ax[2])
# mx = max(ax[1], ax[3])
# plt.plot([mn,mx], [mn,mx], 'k-', alpha=0.1)
# plt.axis(ax)
plt.xlabel('PS1 mag')
plt.ylabel('DECam ap mag')
plt.legend(loc='upper left')
plt.title('Zeropoint')
ps.savefig()
dm = ps1mag - apmag[J]
dmag, dsig = sensible_sigmaclip(dm, nsigma=2.5)
print('Mag offset: %8.3f' % dmag)
print('Scatter: %8.3f' % dsig)
if not np.isfinite(dmag) or not np.isfinite(dsig):
print('FAILED TO GET ZEROPOINT!')
meas.update(zp=None)
return meas
from scipy.stats import sigmaclip
goodpix, lo, hi = sigmaclip(dm, low=3, high=3)
dmagmed = np.median(goodpix)
print(len(goodpix), 'stars used for zeropoint median')
print('Using median zeropoint:')
zp_med = zp0 + dmagmed
trans_med = 10.**(-0.4 * (zp0 - zp_med - kx * (airmass - 1.)))
print('Zeropoint %6.3f' % zp_med)
print('Transparency: %.3f' % trans_med)
dm = ps1mag - apmag2[J]
dmag2, dsig2 = sensible_sigmaclip(dm, nsigma=2.5)
#print('Sky-sub mag offset', dmag2)
#print('Scatter', dsig2)
if ps is not None:
plt.clf()
plt.plot(ps1mag,
apmag[J] + dmag - ps1mag,
'b.',
label='No sky sub')
plt.plot(ps1mag, apmag2[J] + dmag2 - ps1mag, 'r.', label='Sky sub')
plt.xlabel('PS1 mag')
plt.ylabel('DECam ap mag - PS1 mag')
plt.legend(loc='upper left')
plt.ylim(-0.25, 0.25)
plt.axhline(0, color='k', alpha=0.25)
plt.title('Zeropoint')
ps.savefig()
zp_obs = zp0 + dmag
transparency = 10.**(-0.4 * (zp0 - zp_obs - kx * (airmass - 1.)))
meas.update(zp=zp_obs, transparency=transparency)
print('Zeropoint %6.3f' % zp_obs)
print('Fiducial %6.3f' % zp0)
print('Transparency: %.3f' % transparency)
# print('Using sky-subtracted values:')
# zp_sky = zp0 + dmag2
# trans_sky = 10.**(-0.4 * (zp0 - zp_sky - kx * (airmass - 1.)))
# print('Zeropoint %6.3f' % zp_sky)
# print('Transparency: %.3f' % trans_sky)
fwhms = []
psf_r = 15
if n_fwhm not in [0, None]:
Jf = J[:n_fwhm]
for i, (xi, yi, fluxi) in enumerate(zip(fx[Jf], fy[Jf], apflux[Jf])):
#print('Fitting source', i, 'of', len(Jf))
ix = int(np.round(xi))
iy = int(np.round(yi))
xlo = max(0, ix - psf_r)
xhi = min(W, ix + psf_r + 1)
ylo = max(0, iy - psf_r)
yhi = min(H, iy + psf_r + 1)
xx, yy = np.meshgrid(np.arange(xlo, xhi), np.arange(ylo, yhi))
r2 = (xx - xi)**2 + (yy - yi)**2
keep = (r2 < psf_r**2)
pix = img[ylo:yhi, xlo:xhi].copy()
ie = np.zeros_like(pix)
ie[keep] = 1. / sig1
#print('fitting source at', ix,iy)
#print('number of active pixels:', np.sum(ie > 0), 'shape', ie.shape)
psf = tractor.NCircularGaussianPSF([4.], [1.])
tim = tractor.Image(data=pix, inverr=ie, psf=psf)
src = tractor.PointSource(tractor.PixPos(xi - xlo, yi - ylo),
tractor.Flux(fluxi))
tr = tractor.Tractor([tim], [src])
#print('Posterior before prior:', tr.getLogProb())
src.pos.addGaussianPrior('x', 0., 1.)
#print('Posterior after prior:', tr.getLogProb())
doplot = (i < 5) * (ps is not None)
if doplot:
mod0 = tr.getModelImage(0)
tim.freezeAllBut('psf')
psf.freezeAllBut('sigmas')
# print('Optimizing params:')
# tr.printThawedParams()
#print('Parameter step sizes:', tr.getStepSizes())
optargs = dict(priors=False, shared_params=False)
for step in range(50):
dlnp, x, alpha = tr.optimize(**optargs)
#print('dlnp', dlnp)
#print('src', src)
#print('psf', psf)
if dlnp == 0:
break
# Now fit only the PSF size
tr.freezeParam('catalog')
# print('Optimizing params:')
# tr.printThawedParams()
for step in range(50):
dlnp, x, alpha = tr.optimize(**optargs)
#print('dlnp', dlnp)
#print('src', src)
#print('psf', psf)
if dlnp == 0:
break
fwhms.append(psf.sigmas[0] * 2.35 * pixsc)
if doplot:
mod1 = tr.getModelImage(0)
chi1 = tr.getChiImage(0)
plt.clf()
plt.subplot(2, 2, 1)
plt.title('Image')
dimshow(pix, **self.imgkwa)
plt.subplot(2, 2, 2)
plt.title('Initial model')
dimshow(mod0, **self.imgkwa)
plt.subplot(2, 2, 3)
plt.title('Final model')
dimshow(mod1, **self.imgkwa)
plt.subplot(2, 2, 4)
plt.title('Final chi')
dimshow(chi1, vmin=-10, vmax=10)
plt.suptitle('PSF fit')
ps.savefig()
fwhms = np.array(fwhms)
fwhm = np.median(fwhms)
print('Median FWHM: %.3f' % np.median(fwhms))
meas.update(seeing=fwhm)
if False and ps is not None:
lo, hi = np.percentile(fwhms, [5, 95])
lo -= 0.1
hi += 0.1
plt.clf()
plt.hist(fwhms, 25, range=(lo, hi), histtype='step', color='b')
plt.xlabel('FWHM (arcsec)')
ps.savefig()
if ps is not None:
plt.clf()
for i, (xi, yi) in enumerate(zip(fx[J], fy[J])[:50]):
ix = int(np.round(xi))
iy = int(np.round(yi))
xlo = max(0, ix - psf_r)
xhi = min(W, ix + psf_r + 1)
ylo = max(0, iy - psf_r)
yhi = min(H, iy + psf_r + 1)
pix = img[ylo:yhi, xlo:xhi]
slc = pix[iy - ylo, :].copy()
slc /= np.sum(slc)
p1 = plt.plot(slc, 'b-', alpha=0.2)
slc = pix[:, ix - xlo].copy()
slc /= np.sum(slc)
p2 = plt.plot(slc, 'r-', alpha=0.2)
ph, pw = pix.shape
cx, cy = pw / 2, ph / 2
if i == 0:
xx = np.linspace(0, pw, 300)
dx = xx[1] - xx[0]
sig = fwhm / pixsc / 2.35
yy = np.exp(-0.5 * (xx - cx)**2 / sig**2) # * np.sum(pix)
yy /= (np.sum(yy) * dx)
p3 = plt.plot(xx, yy, 'k-', zorder=20)
#plt.ylim(-0.2, 1.0)
plt.legend([p1[0], p2[0], p3[0]],
['image slice (y)', 'image slice (x)', 'fit'])
plt.title('PSF fit')
ps.savefig()
return meas
def stage_fitplots(
T=None, coimgs=None, cons=None,
cat=None, targetrd=None, pixscale=None, targetwcs=None,
W=None,H=None,
bands=None, ps=None, brickid=None,
plots=False, plots2=False, tims=None, tractor=None,
pipe=None,
outdir=None,
**kwargs):
for tim in tims:
print 'Tim', tim, 'PSF', tim.getPsf()
writeModels = False
if pipe:
t0 = Time()
# Produce per-band coadds, for plots
coimgs,cons = compute_coadds(tims, bands, targetwcs)
print 'Coadds:', Time()-t0
plt.figure(figsize=(10,10.5))
#plt.subplots_adjust(left=0.002, right=0.998, bottom=0.002, top=0.998)
plt.subplots_adjust(left=0.002, right=0.998, bottom=0.002, top=0.95)
plt.clf()
dimshow(get_rgb(coimgs, bands))
plt.title('Image')
ps.savefig()
ax = plt.axis()
cat = tractor.getCatalog()
for i,src in enumerate(cat):
rd = src.getPosition()
ok,x,y = targetwcs.radec2pixelxy(rd.ra, rd.dec)
cc = (0,1,0)
if isinstance(src, PointSource):
plt.plot(x-1, y-1, '+', color=cc, ms=10, mew=1.5)
else:
plt.plot(x-1, y-1, 'o', mec=cc, mfc='none', ms=10, mew=1.5)
# plt.text(x, y, '%i' % i, color=cc, ha='center', va='bottom')
plt.axis(ax)
ps.savefig()
mnmx = -5,300
arcsinha = dict(mnmx=mnmx, arcsinh=1)
# After plot
rgbmod = []
rgbmod2 = []
rgbresids = []
rgbchisqs = []
chibins = np.linspace(-10., 10., 200)
chihist = [np.zeros(len(chibins)-1, int) for band in bands]
wcsW = targetwcs.get_width()
wcsH = targetwcs.get_height()
print 'Target WCS shape', wcsW,wcsH
t0 = Time()
mods = _map(_get_mod, [(tim, cat) for tim in tims])
print 'Getting model images:', Time()-t0
orig_wcsxy0 = [tim.wcs.getX0Y0() for tim in tims]
for iband,band in enumerate(bands):
coimg = coimgs[iband]
comod = np.zeros((wcsH,wcsW), np.float32)
comod2 = np.zeros((wcsH,wcsW), np.float32)
cochi2 = np.zeros((wcsH,wcsW), np.float32)
for itim, (tim,mod) in enumerate(zip(tims, mods)):
if tim.band != band:
continue
#mod = tractor.getModelImage(tim)
if plots2:
plt.clf()
dimshow(tim.getImage(), **tim.ima)
plt.title(tim.name)
ps.savefig()
plt.clf()
dimshow(mod, **tim.ima)
plt.title(tim.name)
ps.savefig()
plt.clf()
dimshow((tim.getImage() - mod) * tim.getInvError(), **imchi)
plt.title(tim.name)
ps.savefig()
R = tim_get_resamp(tim, targetwcs)
if R is None:
continue
(Yo,Xo,Yi,Xi) = R
comod[Yo,Xo] += mod[Yi,Xi]
ie = tim.getInvError()
noise = np.random.normal(size=ie.shape) / ie
noise[ie == 0] = 0.
comod2[Yo,Xo] += mod[Yi,Xi] + noise[Yi,Xi]
chi = ((tim.getImage()[Yi,Xi] - mod[Yi,Xi]) * tim.getInvError()[Yi,Xi])
cochi2[Yo,Xo] += chi**2
chi = chi[chi != 0.]
hh,xe = np.histogram(np.clip(chi, -10, 10).ravel(), bins=chibins)
chihist[iband] += hh
if not writeModels:
continue
im = tim.imobj
fn = 'image-b%06i-%s-%s.fits' % (brickid, band, im.name)
wcsfn = create_temp()
wcs = tim.getWcs().wcs
x0,y0 = orig_wcsxy0[itim]
h,w = tim.shape
subwcs = wcs.get_subimage(int(x0), int(y0), w, h)
subwcs.write_to(wcsfn)
primhdr = fitsio.FITSHDR()
primhdr.add_record(dict(name='X0', value=x0, comment='Pixel origin of subimage'))
primhdr.add_record(dict(name='Y0', value=y0, comment='Pixel origin of subimage'))
xfn = im.wcsfn.replace(decals_dir+'/', '')
primhdr.add_record(dict(name='WCS_FILE', value=xfn))
xfn = im.psffn.replace(decals_dir+'/', '')
primhdr.add_record(dict(name='PSF_FILE', value=xfn))
primhdr.add_record(dict(name='INHERIT', value=True))
imhdr = fitsio.read_header(wcsfn)
imhdr.add_record(dict(name='EXTTYPE', value='IMAGE', comment='This HDU contains image data'))
ivhdr = fitsio.read_header(wcsfn)
ivhdr.add_record(dict(name='EXTTYPE', value='INVVAR', comment='This HDU contains an inverse-variance map'))
fits = fitsio.FITS(fn, 'rw', clobber=True)
tim.toFits(fits, primheader=primhdr, imageheader=imhdr, invvarheader=ivhdr)
imhdr.add_record(dict(name='EXTTYPE', value='MODEL', comment='This HDU contains a Tractor model image'))
fits.write(mod, header=imhdr)
print 'Wrote image and model to', fn
comod /= np.maximum(cons[iband], 1)
comod2 /= np.maximum(cons[iband], 1)
rgbmod.append(comod)
rgbmod2.append(comod2)
resid = coimg - comod
resid[cons[iband] == 0] = np.nan
rgbresids.append(resid)
rgbchisqs.append(cochi2)
# Plug the WCS header cards into these images
wcsfn = create_temp()
targetwcs.write_to(wcsfn)
hdr = fitsio.read_header(wcsfn)
os.remove(wcsfn)
if outdir is None:
outdir = '.'
wa = dict(clobber=True, header=hdr)
for name,img in [('image', coimg), ('model', comod), ('resid', resid), ('chi2', cochi2)]:
fn = os.path.join(outdir, '%s-coadd-%06i-%s.fits' % (name, brickid, band))
fitsio.write(fn, img, **wa)
print 'Wrote', fn
del cons
plt.clf()
dimshow(get_rgb(rgbmod, bands))
plt.title('Model')
ps.savefig()
plt.clf()
dimshow(get_rgb(rgbmod2, bands))
plt.title('Model + Noise')
ps.savefig()
plt.clf()
dimshow(get_rgb(rgbresids, bands))
plt.title('Residuals')
ps.savefig()
plt.clf()
dimshow(get_rgb(rgbresids, bands, mnmx=(-30,30)))
plt.title('Residuals (2)')
ps.savefig()
plt.clf()
dimshow(get_rgb(coimgs, bands, **arcsinha))
plt.title('Image (stretched)')
ps.savefig()
plt.clf()
dimshow(get_rgb(rgbmod2, bands, **arcsinha))
plt.title('Model + Noise (stretched)')
ps.savefig()
del coimgs
del rgbresids
del rgbmod
del rgbmod2
plt.clf()
g,r,z = rgbchisqs
im = np.log10(np.dstack((z,r,g)))
mn,mx = 0, im.max()
dimshow(np.clip((im - mn) / (mx - mn), 0., 1.))
plt.title('Chi-squared')
ps.savefig()
plt.clf()
xx = np.repeat(chibins, 2)[1:-1]
for y,cc in zip(chihist, 'grm'):
plt.plot(xx, np.repeat(np.maximum(0.1, y),2), '-', color=cc)
plt.xlabel('Chi')
plt.yticks([])
plt.axvline(0., color='k', alpha=0.25)
ps.savefig()
plt.yscale('log')
mx = np.max([max(y) for y in chihist])
plt.ylim(1, mx * 1.05)
ps.savefig()
return dict(tims=tims)
def segment_and_group_sources(image, T, name=None, ps=None, plots=False):
'''
*image*: binary image that defines "blobs"
*T*: source table; only ".ibx" and ".iby" elements are used (x,y integer
pix pos). Note: ".blob" field is added.
*name*: for debugging only
Returns: (blobmap, blobsrcs, blobslices)
*blobmap*: image, values -1 = no blob, integer blob indices
*blobsrcs*: list of np arrays of integers, elements in T within each blob
*blobslices*: list of slice objects for blob bounding-boxes.
'''
from scipy.ndimage.morphology import binary_fill_holes
from scipy.ndimage.measurements import label, find_objects
image = binary_fill_holes(image)
blobmap, nblobs = label(image)
H, W = image.shape
del image
blobslices = find_objects(blobmap)
clipx = np.clip(T.ibx, 0, W - 1)
clipy = np.clip(T.iby, 0, H - 1)
source_blobs = blobmap[clipy, clipx]
if plots:
import pylab as plt
from astrometry.util.plotutils import dimshow
plt.clf()
dimshow(blobmap > 0, vmin=0, vmax=1)
ax = plt.axis()
for i, bs in enumerate(blobslices):
sy, sx = bs
by0, by1 = sy.start, sy.stop
bx0, bx1 = sx.start, sx.stop
plt.plot([bx0, bx0, bx1, bx1, bx0], [by0, by1, by1, by0, by0],
'r-')
plt.text((bx0 + bx1) / 2.,
by0,
'%i' % (i + 1),
ha='center',
va='bottom',
color='r')
plt.plot(T.ibx, T.iby, 'rx')
for i, t in enumerate(T):
plt.text(t.ibx,
t.iby,
'src %i' % i,
color='red',
ha='left',
va='center')
plt.axis(ax)
plt.title('Blobs')
ps.savefig()
# Find sets of sources within blobs
blobsrcs = []
keepslices = []
blobindex = {}
for blob in range(1, nblobs + 1):
Isrcs, = np.nonzero(source_blobs == blob)
if len(Isrcs) == 0:
blobindex[blob] = -1
continue
blobindex[blob] = len(blobsrcs)
blobsrcs.append(Isrcs)
bslc = blobslices[blob - 1]
keepslices.append(bslc)
blobslices = keepslices
# Remap the "blobmap" image so that empty regions are = -1 and the blob values
# correspond to their indices in the "blobsrcs" list.
if len(blobindex):
maxblob = max(blobindex.keys())
else:
maxblob = 0
maxblob = max(maxblob, blobmap.max())
remap = np.zeros(maxblob + 1, np.int32)
for k, v in blobindex.items():
remap[k] = v
remap[0] = -1
# Remap blob numbers
blobmap = remap[blobmap]
del remap
del blobindex
if plots:
from astrometry.util.plotutils import dimshow
plt.clf()
dimshow(blobmap > -1, vmin=0, vmax=1)
ax = plt.axis()
for i, bs in enumerate(blobslices):
sy, sx = bs
by0, by1 = sy.start, sy.stop
bx0, bx1 = sx.start, sx.stop
plt.plot([bx0, bx0, bx1, bx1, bx0], [by0, by1, by1, by0, by0],
'r-')
plt.text((bx0 + bx1) / 2.,
by0,
'%i' % (i + 1),
ha='center',
va='bottom',
color='r')
plt.plot(T.ibx, T.iby, 'rx')
for i, t in enumerate(T):
plt.text(t.ibx,
t.iby,
'src %i' % i,
color='red',
ha='left',
va='center')
plt.axis(ax)
plt.title('Blobs')
ps.savefig()
for j, Isrcs in enumerate(blobsrcs):
for i in Isrcs:
if (blobmap[clipy[i], clipx[i]] != j):
info('---------------------------!!!-------------------------')
info('Blob', j, 'sources', Isrcs)
info('Source', i, 'coords x,y', T.ibx[i], T.iby[i])
info('Expected blob value', j, 'but got', blobmap[clipy[i],
clipx[i]])
assert (len(blobsrcs) == len(blobslices))
return blobmap, blobsrcs, blobslices
def main():
ps = PlotSequence('conv')
S = 51
center = S / 2
print('Center', center)
#for psf_sigma in [2., 1.5, 1.]:
for psf_sigma in [2.]:
rms2 = []
x = np.arange(S)
y = np.arange(S)
xx, yy = np.meshgrid(x, y)
scale = 1.5 / psf_sigma
pixpsf = render_airy((scale, center), x, y)
psf = (scale, center)
eval_psf = render_airy
plt.clf()
plt.subplot(2, 1, 1)
plt.plot(x, pixpsf[center, :], 'b-')
plt.plot(x, pixpsf[:, center], 'r-')
plt.subplot(2, 1, 2)
plt.plot(x, np.maximum(1e-16, pixpsf[center, :]), 'b-')
plt.plot(x, np.maximum(1e-16, pixpsf[:, center]), 'r-')
plt.yscale('log')
ps.savefig()
plt.clf()
plt.imshow(pixpsf, interpolation='nearest', origin='lower')
ps.savefig()
plt.clf()
plt.imshow(np.log10(np.maximum(1e-16, pixpsf)),
interpolation='nearest',
origin='lower')
plt.colorbar()
plt.title('log PSF')
ps.savefig()
# psf
#psf = scipy.stats.norm(loc=center + 0.5, scale=psf_sigma)
# plt.clf()
# plt.imshow(Pcdf, interpolation='nearest', origin='lower')
# ps.savefig()
# #Pcdf = psf.cdf(xx) * psf.cdf(yy)
# #pixpsf = integrate_gaussian(psf, xx, yy)
#
# padpsf = np.zeros((S*2-1, S*2-1))
# ph,pw = pixpsf.shape
# padpsf[S/2:S/2+ph, S/2:S/2+pw] = pixpsf
# Fpsf = np.fft.rfft2(padpsf)
#
# padh,padw = padpsf.shape
# v = np.fft.rfftfreq(padw)
# w = np.fft.fftfreq(padh)
# fmax = max(max(np.abs(v)), max(np.abs(w)))
# cut = fmax / 2. * 1.000001
# #print('Frequence cut:', cut)
# Ffiltpsf = Fpsf.copy()
# #print('Ffiltpsf', Ffiltpsf.shape)
# #print((np.abs(w) < cut).shape)
# #print((np.abs(v) < cut).shape)
# Ffiltpsf[np.abs(w) > cut, :] = 0.
# Ffiltpsf[:, np.abs(v) > cut] = 0.
# #print('pad v', v)
# #print('pad w', w)
#
# filtpsf = np.fft.irfft2(Ffiltpsf, s=(padh,padw))
#
# print('filtered PSF real', np.max(np.abs(filtpsf.real)))
# print('filtered PSF imag', np.max(np.abs(filtpsf.imag)))
#
# plt.clf()
# plt.subplot(2,3,1)
# dimshow(Fpsf.real)
# plt.colorbar()
# plt.title('Padded PSF real')
# plt.subplot(2,3,4)
# dimshow(Fpsf.imag)
# plt.colorbar()
# plt.title('Padded PSF imag')
#
# plt.subplot(2,3,2)
# dimshow(Ffiltpsf.real)
# plt.colorbar()
# plt.title('Filt PSF real')
# plt.subplot(2,3,5)
# dimshow(Ffiltpsf.imag)
# plt.colorbar()
# plt.title('Filt PSF imag')
#
# plt.subplot(2,3,3)
# dimshow(filtpsf.real)
# plt.title('PSF real')
# plt.colorbar()
#
# plt.subplot(2,3,6)
# dimshow(filtpsf.imag)
# plt.title('PSF imag')
# plt.colorbar()
#
# ps.savefig()
#
#
# pixpsf = filtpsf
gal_sigmas = [2, 1, 0.5, 0.25]
for gal_sigma in gal_sigmas:
# plt.clf()
# plt.imshow(Gcdf, interpolation='nearest', origin='lower')
# plt.savefig('dcdf.png')
# plt.clf()
# plt.imshow(np.exp(-0.5 * ((xx-center)**2 + (yy-center)**2)/2.**2),
# interpolation='nearest', origin='lower')
# plt.savefig('g.png')
# my convolution
pixscale = 1.
cd = pixscale * np.eye(2) / 3600.
P, FG, Gmine, v, w = galaxy_psf_convolution(gal_sigma,
0.,
0.,
GaussianGalaxy,
cd,
0.,
0.,
pixpsf,
debug=True)
#print('v:', v)
#print('w:', w)
#print('P:', P.shape)
print()
print('PSF %g, Gal %g' % (psf_sigma, gal_sigma))
rmax = np.argmax(np.abs(w))
cmax = np.argmax(np.abs(v))
l2_rmax = np.sqrt(np.sum(P[rmax, :].real**2 + P[rmax, :].imag**2))
l2_cmax = np.sqrt(np.sum(P[:, cmax].real**2 + P[:, cmax].imag**2))
print('PSF L_2 in highest-frequency rows & cols:', l2_rmax,
l2_cmax)
l2_rmax = np.sqrt(np.sum(FG[rmax, :].real**2 +
FG[rmax, :].imag**2))
l2_cmax = np.sqrt(np.sum(FG[:, cmax].real**2 +
FG[:, cmax].imag**2))
print('Gal L_2 in highest-frequency rows & cols:', l2_rmax,
l2_cmax)
C = P * FG
l2_rmax = np.sqrt(np.sum(C[rmax, :].real**2 + C[rmax, :].imag**2))
l2_cmax = np.sqrt(np.sum(C[:, cmax].real**2 + C[:, cmax].imag**2))
print('PSF*Gal L_2 in highest-frequency rows & cols:', l2_rmax,
l2_cmax)
print()
Fpsf, Fgal = compare_subsampled(S,
1,
ps,
psf,
pixpsf,
Gmine,
v,
w,
gal_sigma,
psf_sigma,
cd,
get_ffts=True,
eval_psf=eval_psf)
plt.clf()
plt.subplot(2, 4, 1)
dimshow(P.real)
plt.colorbar()
plt.title('PSF real')
plt.subplot(2, 4, 5)
dimshow(P.imag)
plt.colorbar()
plt.title('PSF imag')
plt.subplot(2, 4, 2)
dimshow(FG.real)
plt.colorbar()
plt.title('Gal real')
plt.subplot(2, 4, 6)
dimshow(FG.imag)
plt.colorbar()
plt.title('Gal imag')
plt.subplot(2, 4, 3)
dimshow((P * FG).real)
plt.colorbar()
plt.title('P*Gal real')
plt.subplot(2, 4, 7)
dimshow((P * FG).imag)
plt.colorbar()
plt.title('P*Gal imag')
plt.subplot(2, 4, 4)
dimshow((Fgal).real)
plt.colorbar()
plt.title('pixGal real')
plt.subplot(2, 4, 8)
dimshow((Fgal).imag)
plt.colorbar()
plt.title('pixGal imag')
plt.suptitle('PSF %g, Gal %g' % (psf_sigma, gal_sigma))
ps.savefig()
subsample = [1, 2, 4]
rms1 = []
for s in subsample:
rms = compare_subsampled(S,
s,
ps,
psf,
pixpsf,
Gmine,
v,
w,
gal_sigma,
psf_sigma,
cd,
eval_psf=eval_psf)
rms1.append(rms)
rms2.append(rms1)
print()
print('PSF sigma =', psf_sigma)
print('RMSes:')
for rms1, gal_sigma in zip(rms2, gal_sigmas):
print('Gal sigma', gal_sigma, 'rms:',
', '.join(['%.3g' % r for r in rms1]))
def stage1(T=None,
coimgs=None,
cons=None,
detmaps=None,
detivs=None,
targetrd=None,
pixscale=None,
targetwcs=None,
W=None,
H=None,
bands=None,
tims=None,
ps=None,
brick=None,
cat=None):
orig_wcsxy0 = [tim.wcs.getX0Y0() for tim in tims]
hot = np.zeros((H, W), np.float32)
for band in bands:
detmap = detmaps[band] / np.maximum(1e-16, detivs[band])
detsn = detmap * np.sqrt(detivs[band])
hot = np.maximum(hot, detsn)
detmaps[band] = detmap
### FIXME -- ugri
for sedname, sed in [('Flat', (1., 1., 1.)), ('Red', (2.5, 1.0, 0.4))]:
sedmap = np.zeros((H, W), np.float32)
sediv = np.zeros((H, W), np.float32)
for iband, band in enumerate(bands):
# We convert the detmap to canonical band via
# detmap * w
# And the corresponding change to sig1 is
# sig1 * w
# So the invvar-weighted sum is
# (detmap * w) / (sig1**2 * w**2)
# = detmap / (sig1**2 * w)
sedmap += detmaps[band] * detivs[band] / sed[iband]
sediv += detivs[band] / sed[iband]**2
sedmap /= np.maximum(1e-16, sediv)
sedsn = sedmap * np.sqrt(sediv)
hot = np.maximum(hot, sedsn)
plt.clf()
dimshow(np.round(sedsn), vmin=0, vmax=10, cmap='hot')
plt.title('SED-matched detection filter: %s' % sedname)
ps.savefig()
peaks = (hot > 4)
blobs, nblobs = label(peaks)
print('N detected blobs:', nblobs)
blobslices = find_objects(blobs)
# Un-set catalog blobs
for x, y in zip(T.itx, T.ity):
# blob number
bb = blobs[y, x]
if bb == 0:
continue
# un-set 'peaks' within this blob
slc = blobslices[bb - 1]
peaks[slc][blobs[slc] == bb] = 0
# Now, after having removed catalog sources, crank up the detection threshold
peaks &= (hot > 5)
# zero out the edges(?)
peaks[0, :] = peaks[:, 0] = 0
peaks[-1, :] = peaks[:, -1] = 0
peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[0:-2, 1:-1])
peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[2:, 1:-1])
peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[1:-1, 0:-2])
peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[1:-1, 2:])
# These are our peaks
pki = np.flatnonzero(peaks)
peaky, peakx = np.unravel_index(pki, peaks.shape)
print(len(peaky), 'peaks')
crossa = dict(ms=10, mew=1.5)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(T.tx, T.ty, 'r+', **crossa)
plt.plot(peakx, peaky, '+', color=green, **crossa)
plt.axis(ax)
plt.title('SDSS + SED-matched detections')
ps.savefig()
### HACK -- high threshold again
# Segment, and record which sources fall into each blob
blobs, nblobs = label((hot > 20))
print('N detected blobs:', nblobs)
blobslices = find_objects(blobs)
T.blob = blobs[T.ity, T.itx]
blobsrcs = []
blobflux = []
fluximg = coimgs[1]
for blob in range(1, nblobs + 1):
blobsrcs.append(np.flatnonzero(T.blob == blob))
bslc = blobslices[blob - 1]
blobflux.append(np.sum(fluximg[bslc][blobs[bslc] == blob]))
# Fit the SDSS sources
for tim in tims:
tim.psfex.fitSavedData(*tim.psfex.splinedata)
tim.psf = tim.psfex
# How far down to render model profiles
minsigma = 0.1
for tim in tims:
tim.modelMinval = minsigma * tim.sig1
srcvariances = [[] for src in cat]
# Fit in order of flux
for blobnumber, iblob in enumerate(np.argsort(-np.array(blobflux))):
bslc = blobslices[iblob]
Isrcs = blobsrcs[iblob]
if len(Isrcs) == 0:
continue
print()
print('Blob', blobnumber, 'of', len(blobflux), ':', len(Isrcs),
'sources')
print('Source indices:', Isrcs)
print()
# blob bbox in target coords
sy, sx = bslc
by0, by1 = sy.start, sy.stop
bx0, bx1 = sx.start, sx.stop
blobh, blobw = by1 - by0, bx1 - bx0
rr, dd = targetwcs.pixelxy2radec([bx0, bx0, bx1, bx1],
[by0, by1, by1, by0])
alphas = [0.1, 0.3, 1.0]
subtims = []
for itim, tim in enumerate(tims):
h, w = tim.shape
ok, x, y = tim.subwcs.radec2pixelxy(rr, dd)
sx0, sx1 = x.min(), x.max()
sy0, sy1 = y.min(), y.max()
if sx1 < 0 or sy1 < 0 or sx1 > w or sy1 > h:
continue
sx0 = np.clip(int(np.floor(sx0)), 0, w - 1)
sx1 = np.clip(int(np.ceil(sx1)), 0, w - 1) + 1
sy0 = np.clip(int(np.floor(sy0)), 0, h - 1)
sy1 = np.clip(int(np.ceil(sy1)), 0, h - 1) + 1
subslc = slice(sy0, sy1), slice(sx0, sx1)
subimg = tim.getImage()[subslc]
subie = tim.getInvError()[subslc]
subwcs = tim.getWcs().copy()
ox0, oy0 = orig_wcsxy0[itim]
subwcs.setX0Y0(ox0 + sx0, oy0 + sy0)
# Mask out inverr for pixels that are not within the blob.
subtarget = targetwcs.get_subimage(bx0, by0, blobw, blobh)
subsubwcs = tim.subwcs.get_subimage(int(sx0), int(sy0),
int(sx1 - sx0), int(sy1 - sy0))
try:
Yo, Xo, Yi, Xi, rims = resample_with_wcs(
subsubwcs, subtarget, [], 2)
except OverlapError:
print('No overlap')
continue
if len(Yo) == 0:
continue
subie2 = np.zeros_like(subie)
I = np.flatnonzero(blobs[bslc][Yi, Xi] == (iblob + 1))
subie2[Yo[I], Xo[I]] = subie[Yo[I], Xo[I]]
subie = subie2
# If the subimage (blob) is small enough, instantiate a
# constant PSF model in the center.
if sy1 - sy0 < 100 and sx1 - sx0 < 100:
subpsf = tim.psf.mogAt(ox0 + (sx0 + sx1) / 2.,
oy0 + (sy0 + sy1) / 2.)
else:
# Otherwise, instantiate a (shifted) spatially-varying
# PsfEx model.
subpsf = ShiftedPsf(tim.psf, ox0 + sx0, oy0 + sy0)
subtim = Image(data=subimg,
inverr=subie,
wcs=subwcs,
psf=subpsf,
photocal=tim.getPhotoCal(),
sky=tim.getSky(),
name=tim.name)
subtim.band = tim.band
subtim.sig1 = tim.sig1
subtim.modelMinval = tim.modelMinval
subtims.append(subtim)
subcat = Catalog(*[cat[i] for i in Isrcs])
subtr = Tractor(subtims, subcat)
subtr.freezeParam('images')
# Optimize individual sources in order of flux
fluxes = []
for src in subcat:
# HACK -- here we just *sum* the nanomaggies in each band. Bogus!
br = src.getBrightness()
flux = sum([br.getFlux(band) for band in bands])
fluxes.append(flux)
Ibright = np.argsort(-np.array(fluxes))
if len(Ibright) >= 5:
# -Remember the original subtim images
# -Compute initial models for each source (in each tim)
# -Subtract initial models from images
# -During fitting, for each source:
# -add back in the source's initial model (to each tim)
# -fit, with Catalog([src])
# -subtract final model (from each tim)
# -Replace original subtim images
#
# --Might want to omit newly-added detection-filter sources, since their
# fluxes are bogus.
# Remember original tim images
orig_timages = [tim.getImage().copy() for tim in subtims]
initial_models = []
# Create initial models for each tim x each source
for tim in subtims:
mods = []
for src in subcat:
mod = src.getModelPatch(tim)
mods.append(mod)
if mod is not None:
if not np.all(np.isfinite(mod.patch)):
print('Non-finite mod patch')
print('source:', src)
print('tim:', tim)
print('PSF:', tim.getPsf())
assert (np.all(np.isfinite(mod.patch)))
mod.addTo(tim.getImage(), scale=-1)
initial_models.append(mods)
# For sources in decreasing order of brightness
for numi, i in enumerate(Ibright):
tsrc = Time()
print('Fitting source', i,
'(%i of %i in blob)' % (numi, len(Ibright)))
src = subcat[i]
print(src)
srctractor = Tractor(subtims, [src])
srctractor.freezeParams('images')
# Add this source's initial model back in.
for tim, mods in zip(subtims, initial_models):
mod = mods[i]
if mod is not None:
mod.addTo(tim.getImage())
print('Optimizing:', srctractor)
srctractor.printThawedParams()
for step in range(50):
dlnp, X, alpha = srctractor.optimize(priors=False,
shared_params=False,
alphas=alphas)
print('dlnp:', dlnp, 'src', src)
if dlnp < 0.1:
break
for tim in subtims:
mod = src.getModelPatch(tim)
if mod is not None:
mod.addTo(tim.getImage(), scale=-1)
for tim, img in zip(subtims, orig_timages):
tim.data = img
del orig_timages
del initial_models
else:
# Fit sources one at a time, but don't subtract other models
subcat.freezeAllParams()
for numi, i in enumerate(Ibright):
tsrc = Time()
print('Fitting source', i,
'(%i of %i in blob)' % (numi, len(Ibright)))
print(subcat[i])
subcat.freezeAllBut(i)
print('Optimizing:', subtr)
subtr.printThawedParams()
for step in range(10):
dlnp, X, alpha = subtr.optimize(priors=False,
shared_params=False,
alphas=alphas)
print('dlnp:', dlnp)
if dlnp < 0.1:
break
print('Fitting source took', Time() - tsrc)
print(subcat[i])
if len(Isrcs) > 1 and len(Isrcs) <= 10:
tfit = Time()
# Optimize all at once?
subcat.thawAllParams()
print('Optimizing:', subtr)
subtr.printThawedParams()
for step in range(20):
dlnp, X, alpha = subtr.optimize(priors=False,
shared_params=False,
alphas=alphas)
print('dlnp:', dlnp)
if dlnp < 0.1:
break
# Variances
subcat.thawAllRecursive()
subcat.freezeAllParams()
for isub, srci in enumerate(Isrcs):
print('Variances for source', srci)
subcat.thawParam(isub)
src = subcat[isub]
print('Source', src)
print('Params:', src.getParamNames())
if isinstance(src, (DevGalaxy, ExpGalaxy)):
src.shape = EllipseE.fromEllipseESoft(src.shape)
elif isinstance(src, FixedCompositeGalaxy):
src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp)
src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev)
print('Converted ellipse:', src)
allderivs = subtr.getDerivs()
for iparam, derivs in enumerate(allderivs):
dchisq = 0
for deriv, tim in derivs:
h, w = tim.shape
deriv.clipTo(w, h)
ie = tim.getInvError()
slc = deriv.getSlice(ie)
chi = deriv.patch * ie[slc]
dchisq += (chi**2).sum()
if dchisq == 0.:
v = np.nan
else:
v = 1. / dchisq
srcvariances[srci].append(v)
assert (len(srcvariances[srci]) == subcat[isub].numberOfParams())
subcat.freezeParam(isub)
cat.thawAllRecursive()
for i, src in enumerate(cat):
print('Source', i, src)
print('variances:', srcvariances[i])
print(len(srcvariances[i]), 'vs', src.numberOfParams())
if len(srcvariances[i]) != src.numberOfParams():
# This can happen for sources outside the brick bounds: they never get optimized?
print('Warning: zeroing variances for source', src)
srcvariances[i] = [0] * src.numberOfParams()
if isinstance(src, (DevGalaxy, ExpGalaxy)):
src.shape = EllipseE.fromEllipseESoft(src.shape)
elif isinstance(src, FixedCompositeGalaxy):
src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp)
src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev)
assert (len(srcvariances[i]) == src.numberOfParams())
variances = np.hstack(srcvariances)
assert (len(variances) == cat.numberOfParams())
return dict(cat=cat, variances=variances)
def read_raw_decam(F, ext):
'''
F: fitsio FITS object
'''
img = F[ext].read()
hdr = F[ext].read_header()
#print('Image type', img.dtype, img.shape)
img = img.astype(np.float32)
#print('Converted image to', img.dtype, img.shape)
if False:
mn, mx = np.percentile(img.ravel(), [25, 95])
kwa = dict(vmin=mn, vmax=mx)
plt.clf()
dimshow(img, **kwa)
plt.title('Raw image')
ps.savefig()
M = 200
plt.clf()
plt.subplot(2, 2, 1)
dimshow(img[-M:, :M], ticks=False, **kwa)
plt.subplot(2, 2, 2)
dimshow(img[-M:, -M:], ticks=False, **kwa)
plt.subplot(2, 2, 3)
dimshow(img[:M, :M], ticks=False, **kwa)
plt.subplot(2, 2, 4)
dimshow(img[:M, -M:], ticks=False, **kwa)
plt.suptitle('Raw corners')
ps.savefig()
if 'DESBIAS' in hdr:
assert (False)
# DECam RAW image
# Raw image size 2160 x 4146
# Subtract median overscan and multiply by gains
# print('DATASECA', hdr['DATASECA'])
# print('BIASSECA', hdr['BIASSECA'])
# print('DATASECB', hdr['DATASECB'])
# print('BIASSECB', hdr['BIASSECB'])
dataA = parse_section(hdr['DATASECA'], slices=True)
biasA = parse_section(hdr['BIASSECA'], slices=True)
dataB = parse_section(hdr['DATASECB'], slices=True)
biasB = parse_section(hdr['BIASSECB'], slices=True)
gainA = hdr['GAINA']
gainB = hdr['GAINB']
# print('DataA', dataA)
# print('BiasA', biasA)
# print('DataB', dataB)
# print('BiasB', biasB)
if False:
plt.clf()
plt.plot([np.median(img[i, :]) for i in range(100)], 'b-')
plt.plot([np.median(img[-(i + 1), :]) for i in range(100)], 'c-')
plt.plot([np.median(img[:, i]) for i in range(100)], 'r-')
plt.plot([np.median(img[:, -(i + 1)]) for i in range(100)], 'm-')
plt.title('Img')
ps.savefig()
plt.clf()
plt.plot([np.median(img[dataA][i, :]) for i in range(100)], 'b-')
plt.plot([np.median(img[dataA][-(i + 1), :]) for i in range(100)],
'c-')
plt.plot([np.median(img[dataA][:, i]) for i in range(100)], 'r-')
plt.plot([np.median(img[dataA][:, -(i + 1)]) for i in range(100)],
'm-')
plt.title('Img DataA')
ps.savefig()
plt.clf()
plt.plot([np.median(img[dataB][i, :]) for i in range(100)], 'b-')
plt.plot([np.median(img[dataB][-(i + 1), :]) for i in range(100)],
'c-')
plt.plot([np.median(img[dataB][:, i]) for i in range(100)], 'r-')
plt.plot([np.median(img[dataB][:, -(i + 1)]) for i in range(100)],
'm-')
plt.title('Img DataB')
ps.savefig()
img[dataA] = (img[dataA] - np.median(img[biasA])) * gainA
img[dataB] = (img[dataB] - np.median(img[biasB])) * gainB
# Trim the image -- could just take the min/max of TRIMSECA/TRIMSECB...
trimA = parse_section(hdr['TRIMSECA'], slices=True)
trimB = parse_section(hdr['TRIMSECB'], slices=True)
# copy the TRIM A,B sections into a new image...
trimg = np.zeros_like(img)
trimg[trimA] = img[trimA]
trimg[trimB] = img[trimB]
# ... and then cut that new image
trim = parse_section(hdr['TRIMSEC'], slices=True)
img = trimg[trim]
#print('Trimmed image:', img.dtype, img.shape)
return img, hdr
gal_re = 10.
assert(ph == pw)
ps = PlotSequence('conv')
plt.clf()
plt.imshow(pixpsf, interpolation='nearest', origin='lower')
ps.savefig()
Fpsf = np.fft.rfft2(pixpsf)
plt.clf()
plt.subplot(2,4,1)
dimshow(Fpsf.real)
plt.colorbar()
plt.title('PSF real')
plt.subplot(2,4,5)
dimshow(Fpsf.imag)
plt.colorbar()
plt.title('PSF imag')
ps.savefig()
# Subsample the PSF via resampling
from astrometry.util.util import lanczos_shift_image
scale = 2
sh,sw = ph*scale, pw*scale
subpixpsf = np.zeros((sh,sw))
for ix in np.arange(scale):
def sed_matched_detection(sedname,
sed,
detmaps,
detivs,
bands,
xomit,
yomit,
romit,
nsigma=5.,
saddle_fraction=0.1,
saddle_min=2.,
saturated_pix=None,
veto_map=None,
cutonaper=True,
ps=None,
rgbimg=None):
'''
Runs a single SED-matched detection filter.
Avoids creating sources close to existing sources.
Parameters
----------
sedname : string
Name of this SED; only used for plots.
sed : list of floats
The SED -- a list of floats, one per band, of this SED.
detmaps : list of numpy arrays
The per-band detection maps. These must all be the same size, the
brick image size.
detivs : list of numpy arrays
The inverse-variance maps associated with `detmaps`.
bands : list of strings
The band names of the `detmaps` and `detivs` images.
xomit, yomit, romit : iterables (lists or numpy arrays) of int
Previously known sources that are to be avoided; x,y +- radius
nsigma : float, optional
Detection threshold.
saddle_fraction : float, optional
Fraction of the peak heigh for selecting new sources.
saddle_min : float, optional
Saddle-point depth from existing sources down to new sources.
saturated_pix : None or list of numpy arrays, boolean
A map of pixels that are always considered "hot" when
determining whether a new source touches hot pixels of an
existing source.
cutonaper : bool, optional
Apply a cut that the source's detection strength must be greater
than `nsigma` above the 16th percentile of the detection strength in
an annulus (from 10 to 20 pixels) around the source.
ps : PlotSequence object, optional
Create plots?
Returns
-------
hotblobs : numpy array of bool
A map of the blobs yielding sources in this SED.
px, py : numpy array of int
The new sources found.
aper : numpy array of float
The detection strength in the annulus around the source, if
`cutonaper` is set; else -1.
peakval : numpy array of float
The detection strength.
See also
--------
sed_matched_filters : creates the `(sedname, sed)` pairs used here
run_sed_matched_filters : calls this method
'''
from scipy.ndimage.measurements import label, find_objects
from scipy.ndimage.morphology import binary_dilation, binary_fill_holes
H, W = detmaps[0].shape
allzero = True
for iband in range(len(bands)):
if sed[iband] == 0:
continue
if np.all(detivs[iband] == 0):
continue
allzero = False
break
if allzero:
info('SED', sedname, 'has all zero weight')
return None, None, None, None, None
sedmap = np.zeros((H, W), np.float32)
sediv = np.zeros((H, W), np.float32)
if saturated_pix is not None:
satur = np.zeros((H, W), bool)
for iband in range(len(bands)):
if sed[iband] == 0:
continue
# We convert the detmap to canonical band via
# detmap * w
# And the corresponding change to sig1 is
# sig1 * w
# So the invvar-weighted sum is
# (detmap * w) / (sig1**2 * w**2)
# = detmap / (sig1**2 * w)
sedmap += detmaps[iband] * detivs[iband] / sed[iband]
sediv += detivs[iband] / sed[iband]**2
if saturated_pix is not None:
satur |= saturated_pix[iband]
sedmap /= np.maximum(1e-16, sediv)
sedsn = sedmap * np.sqrt(sediv)
del sedmap
peaks = (sedsn > nsigma)
def saddle_level(Y):
# Require a saddle that drops by (the larger of) "saddle"
# sigma, or 10% of the peak height.
# ("saddle" is passed in as an argument to the
# sed_matched_detection function)
drop = max(saddle_min, Y * saddle_fraction)
return Y - drop
lowest_saddle = nsigma - saddle_min
# zero out the edges -- larger margin here?
peaks[0, :] = 0
peaks[:, 0] = 0
peaks[-1, :] = 0
peaks[:, -1] = 0
# Label the N-sigma blobs at this point... we'll use this to build
# "sedhot", which in turn is used to define the blobs that we will
# optimize simultaneously. This also determines which pixels go
# into the fitting!
dilate = 8
hotblobs, nhot = label(
binary_fill_holes(binary_dilation(peaks, iterations=dilate)))
# find pixels that are larger than their 8 neighbors
peaks[1:-1, 1:-1] &= (sedsn[1:-1, 1:-1] >= sedsn[0:-2, 1:-1])
peaks[1:-1, 1:-1] &= (sedsn[1:-1, 1:-1] >= sedsn[2:, 1:-1])
peaks[1:-1, 1:-1] &= (sedsn[1:-1, 1:-1] >= sedsn[1:-1, 0:-2])
peaks[1:-1, 1:-1] &= (sedsn[1:-1, 1:-1] >= sedsn[1:-1, 2:])
peaks[1:-1, 1:-1] &= (sedsn[1:-1, 1:-1] >= sedsn[0:-2, 0:-2])
peaks[1:-1, 1:-1] &= (sedsn[1:-1, 1:-1] >= sedsn[0:-2, 2:])
peaks[1:-1, 1:-1] &= (sedsn[1:-1, 1:-1] >= sedsn[2:, 0:-2])
peaks[1:-1, 1:-1] &= (sedsn[1:-1, 1:-1] >= sedsn[2:, 2:])
if ps is not None:
import pylab as plt
from astrometry.util.plotutils import dimshow
plt.clf()
plt.subplot(1, 2, 2)
dimshow(sedsn, vmin=-2, vmax=100, cmap='hot', ticks=False)
plt.subplot(1, 2, 1)
dimshow(sedsn, vmin=-2, vmax=10, cmap='hot', ticks=False)
above = (sedsn > nsigma)
plot_boundary_map(above)
ax = plt.axis()
y, x = np.nonzero(peaks)
plt.plot(xomit, yomit, 'm.')
plt.plot(x, y, 'r+')
plt.axis(ax)
plt.title('SED %s: S/N & peaks' % sedname)
ps.savefig()
#import fitsio
#fitsio.write('sed-sn-%s.fits' % sedname, sedsn)
# plt.clf()
# plt.imshow(sedsn, vmin=-2, vmax=10, interpolation='nearest',
# origin='lower', cmap='hot')
# plot_boundary_map(sedsn > lowest_saddle)
# plt.title('SED %s: S/N & lowest saddle point bounds' % sedname)
# ps.savefig()
# For each new source, compute the saddle value, segment at that
# level, and drop the source if it is in the same blob as a
# previously-detected source.
# We dilate the blobs a bit too, to
# catch slight differences in centroid positions.
dilate = 1
# For efficiency, segment at the minimum saddle level to compute
# slices; the operations described above need only happen within
# the slice.
saddlemap = (sedsn > lowest_saddle)
saddlemap = binary_dilation(saddlemap, iterations=dilate)
if saturated_pix is not None:
saddlemap |= satur
allblobs, _ = label(saddlemap)
allslices = find_objects(allblobs)
ally0 = [sy.start for sy, sx in allslices]
allx0 = [sx.start for sy, sx in allslices]
# brightest peaks first
py, px = np.nonzero(peaks)
I = np.argsort(-sedsn[py, px])
py = py[I]
px = px[I]
keep = np.zeros(len(px), bool)
peakval = []
aper = []
apin = 10
apout = 20
# Map of pixels that are vetoed by sources found so far. The veto
# area is based on saddle height. We go from brightest to
# faintest pixels. Thus the saddle level decreases, and the
# saddlemap areas become larger; the saddlemap when a source is
# found is a lower bound on the pixels that it will veto based on
# the saddle heights of fainter sources. Thus the vetomap isn't
# the final word, it is just a quick veto of pixels we know for
# sure will be vetoed.
if veto_map is None:
this_veto_map = np.zeros(sedsn.shape, bool)
else:
this_veto_map = veto_map.copy()
for x, y, r in zip(xomit, yomit, romit):
xlo = int(np.clip(np.floor(x - r), 0, W - 1))
xhi = int(np.clip(np.ceil(x + r), 0, W - 1))
ylo = int(np.clip(np.floor(y - r), 0, H - 1))
yhi = int(np.clip(np.ceil(y + r), 0, H - 1))
this_veto_map[ylo:yhi + 1, xlo:xhi + 1] |= (np.hypot(
(x - np.arange(xlo, xhi + 1))[np.newaxis, :],
(y - np.arange(ylo, yhi + 1))[:, np.newaxis]) < r)
if ps is not None:
plt.clf()
plt.imshow(this_veto_map,
interpolation='nearest',
origin='lower',
vmin=0,
vmax=1,
cmap='hot')
plt.title('Veto map')
ps.savefig()
info('Peaks in initial veto map:', np.sum(this_veto_map[py, px]), 'of',
len(px))
plt.clf()
plt.imshow(saddlemap,
interpolation='nearest',
origin='lower',
vmin=0,
vmax=1,
cmap='hot')
ax = plt.axis()
for slc in allslices:
sy, sx = slc
by0, by1 = sy.start, sy.stop
bx0, bx1 = sx.start, sx.stop
plt.plot([bx0, bx0, bx1, bx1, bx0], [by0, by1, by1, by0, by0],
'r-')
plt.axis(ax)
plt.title('Saddle map (lowest level): %i blobs' % len(allslices))
ps.savefig()
# For each peak, determine whether it is isolated enough --
# separated by a low enough saddle from other sources. Need only
# search within its "allblob", which is defined by the lowest
# saddle.
info('Found', len(px), 'potential peaks')
nveto = 0
nsaddle = 0
naper = 0
for i, (x, y) in enumerate(zip(px, py)):
if this_veto_map[y, x]:
nveto += 1
continue
level = saddle_level(sedsn[y, x])
ablob = allblobs[y, x]
index = int(ablob - 1)
slc = allslices[index]
saddlemap = (sedsn[slc] > level)
saddlemap = binary_dilation(saddlemap, iterations=dilate)
if saturated_pix is not None:
saddlemap |= satur[slc]
saddlemap *= (allblobs[slc] == ablob)
saddlemap = binary_fill_holes(saddlemap)
blobs, _ = label(saddlemap)
x0, y0 = allx0[index], ally0[index]
thisblob = blobs[y - y0, x - x0]
saddlemap *= (blobs == thisblob)
# previously found sources:
ox = np.append(xomit, px[:i][keep[:i]]) - x0
oy = np.append(yomit, py[:i][keep[:i]]) - y0
h, w = blobs.shape
cut = False
if len(ox):
ox = ox.astype(int)
oy = oy.astype(int)
cut = any((ox >= 0) * (ox < w) * (oy >= 0) * (oy < h) *
(blobs[np.clip(oy, 0, h - 1),
np.clip(ox, 0, w - 1)] == thisblob))
# one plot per peak is a little excessive!
if ps is not None and i < 10:
_peak_plot_1(this_veto_map, x, y, px, py, keep, i, xomit, yomit,
sedsn, allblobs, level, dilate, saturated_pix, satur,
ps, rgbimg, cut)
if False and cut and ps is not None:
_peak_plot_2(ox, oy, w, h, blobs, thisblob, sedsn, x0, y0, x, y,
level, ps)
if False and (not cut) and ps is not None:
_peak_plot_3(sedsn, nsigma, x, y, x0, y0, slc, saddlemap, xomit,
yomit, px, py, keep, i, cut, ps)
if cut:
# in same blob as previously found source.
# update vetomap
this_veto_map[slc] |= saddlemap
nsaddle += 1
continue
# Measure in aperture...
ap = sedsn[max(0, y - apout):min(H, y + apout + 1),
max(0, x - apout):min(W, x + apout + 1)]
apiv = (sediv[max(0, y - apout):min(H, y + apout + 1),
max(0, x - apout):min(W, x + apout + 1)] > 0)
aph, apw = ap.shape
apx0, apy0 = max(0, x - apout), max(0, y - apout)
R2 = ((np.arange(aph) + apy0 - y)[:, np.newaxis]**2 +
(np.arange(apw) + apx0 - x)[np.newaxis, :]**2)
ap = ap[apiv * (R2 >= apin**2) * (R2 <= apout**2)]
if len(ap):
# 16th percentile ~ -1 sigma point.
m = np.percentile(ap, 16.)
else:
# fake
m = -1.
if cutonaper:
if sedsn[y, x] - m < nsigma:
naper += 1
continue
aper.append(m)
peakval.append(sedsn[y, x])
keep[i] = True
this_veto_map[slc] |= saddlemap
if False and ps is not None:
plt.clf()
plt.subplot(1, 2, 1)
dimshow(ap,
vmin=-2,
vmax=10,
cmap='hot',
extent=[apx0, apx0 + apw, apy0, apy0 + aph])
plt.subplot(1, 2, 2)
dimshow(ap * ((R2 >= apin**2) * (R2 <= apout**2)),
vmin=-2,
vmax=10,
cmap='hot',
extent=[apx0, apx0 + apw, apy0, apy0 + aph])
plt.suptitle('peak %.1f vs ap %.1f' % (sedsn[y, x], m))
ps.savefig()
info('Of', len(px), 'potential peaks:', nveto, 'in veto map,',
nsaddle, 'cut by saddle test,', naper, 'cut by aper test,',
np.sum(keep), 'kept')
if ps is not None:
pxdrop = px[np.logical_not(keep)]
pydrop = py[np.logical_not(keep)]
py = py[keep]
px = px[keep]
# Which of the hotblobs yielded sources? Those are the ones to keep.
hbmap = np.zeros(nhot + 1, bool)
hbmap[hotblobs[py, px]] = True
if len(xomit):
h, w = hotblobs.shape
hbmap[hotblobs[np.clip(yomit, 0, h - 1),
np.clip(xomit, 0, w - 1)]] = True
# in case a source is (somehow) not in a hotblob?
hbmap[0] = False
hotblobs = hbmap[hotblobs]
if ps is not None:
plt.clf()
dimshow(this_veto_map, vmin=0, vmax=1, cmap='hot')
plt.title('SED %s: veto map' % sedname)
ps.savefig()
plt.clf()
dimshow(hotblobs, vmin=0, vmax=1, cmap='hot')
ax = plt.axis()
p2 = plt.plot(pxdrop, pydrop, 'm+', ms=8, mew=2)
p1 = plt.plot(px, py, 'g+', ms=8, mew=2)
p3 = plt.plot(xomit, yomit, 'r+', ms=8, mew=2)
plt.axis(ax)
plt.title('SED %s: hot blobs' % sedname)
plt.figlegend((p3[0], p1[0], p2[0]), ('Existing', 'Keep', 'Drop'),
'upper left')
ps.savefig()
return hotblobs, px, py, aper, peakval
def test_psfex(self):
if ps is not None:
from astrometry.util.plotutils import dimshow
import pylab as plt
H, W = 100, 100
cx, cy = W / 2., H / 2.
pixpsf = self.psf.constantPsfAt(cx, cy)
ph, pw = pixpsf.shape
xx, yy = np.meshgrid(np.arange(pw), np.arange(ph))
im = pixpsf.img.copy()
im /= np.sum(im)
cenx, ceny = np.sum(im * xx), np.sum(im * yy)
print('Pixpsf centroid:', cenx, ceny)
print('shape:', ph, pw)
dx, dy = cenx - pw // 2, ceny - ph // 2
print('dx,dy', dx, dy)
# gpsf = GaussianMixturePSF.fromStamp(im, N=1)
# print('Fit gpsf:', gpsf)
# self.assertTrue(np.abs(gpsf.mog.mean[0,0] - dx) < 0.1)
# self.assertTrue(np.abs(gpsf.mog.mean[0,1] - dy) < 0.1)
# self.assertTrue(np.abs(gpsf.mog.var[0,0,0] - 15.5) < 1.)
# self.assertTrue(np.abs(gpsf.mog.var[0,1,1] - 13.5) < 1.)
# self.assertTrue(np.abs(gpsf.mog.var[0,1,0] - -1) < 1.)
gpsf = GaussianMixturePSF.fromStamp(im, N=2)
print('Fit gpsf:', gpsf)
print('Params:', ', '.join(['%.1f' % p for p in gpsf.getParams()]))
pp = np.array(
[0.8, 0.2, 0.1, -0.0, 1.2, 0.2, 7.6, 6.0, -1.0, 51.6, 49.1, -1.3])
self.assertTrue(np.all(np.abs(np.array(gpsf.getParams()) - pp) < 0.1))
tim = Image(data=np.zeros((H, W)),
invvar=np.ones((H, W)),
psf=self.psf)
xx, yy = np.meshgrid(np.arange(W), np.arange(H))
star = PointSource(PixPos(cx, cy), Flux(100.))
gal = ExpGalaxy(PixPos(cx, cy), Flux(100.), EllipseE(1., 0., 0.))
tr1 = Tractor([tim], [star])
tr2 = Tractor([tim], [gal])
disable_galaxy_cache()
tim.psf = self.psf
mod = tr1.getModelImage(0)
mod1 = mod
im = mod.copy()
im /= im.sum()
cenx, ceny = np.sum(im * xx), np.sum(im * yy)
print('Star model + PsfEx centroid', cenx, ceny)
self.assertTrue(np.abs(cenx - (cx + dx)) < 0.1)
self.assertTrue(np.abs(ceny - (cy + dy)) < 0.1)
if ps is not None:
plt.clf()
dimshow(mod)
plt.title('Star model, PsfEx')
ps.savefig()
tim.psf = pixpsf
mod = tr1.getModelImage(0)
if ps is not None:
plt.clf()
dimshow(mod)
plt.title('Star model, pixpsf')
ps.savefig()
tim.psf = gpsf
mod = tr1.getModelImage(0)
mod2 = mod
if ps is not None:
plt.clf()
dimshow(mod)
plt.title('Star model, gpsf')
plt.colorbar()
ps.savefig()
plt.clf()
dimshow(mod1 - mod2)
plt.title('Star model, PsfEx - gpsf')
plt.colorbar()
ps.savefig()
# range ~ -0.15 to +0.25
self.assertTrue(np.all(np.abs(mod1 - mod2) < 0.25))
tim.psf = self.psf
mod = tr2.getModelImage(0)
mod1 = mod
im = mod.copy()
im /= im.sum()
cenx, ceny = np.sum(im * xx), np.sum(im * yy)
print('Gal model + PsfEx centroid', cenx, ceny)
self.assertTrue(np.abs(cenx - (cx + dx)) < 0.1)
self.assertTrue(np.abs(ceny - (cy + dy)) < 0.1)
if ps is not None:
plt.clf()
dimshow(mod)
plt.title('Gal model, PsfEx')
ps.savefig()
# tim.psf = pixpsf
# mod = tr2.getModelImage(0)
# plt.clf()
# dimshow(mod)
# plt.title('Gal model, pixpsf')
# ps.savefig()
tim.psf = gpsf
mod = tr2.getModelImage(0)
mod2 = mod
# range ~ -0.1 to +0.2
self.assertTrue(np.all(np.abs(mod1 - mod2) < 0.2))
if ps is not None:
plt.clf()
dimshow(mod)
plt.title('Gal model, gpsf')
ps.savefig()
plt.clf()
dimshow(mod1 - mod2)
plt.title('Gal model, PsfEx - gpsf')
plt.colorbar()
ps.savefig()
for x in XX:
psfimg = psfex.instantiateAt(x, y)
psfgrid.append(psfimg)
h, w = psfimg.shape
img = psfimg[h / 2 - crop:h / 2 + crop + 1,
w / 2 - crop:w / 2 + crop + 1]
psfcropgrid.append(img)
mx = np.max([psfimg.max() for psfimg in psfgrid])
logmx = np.log10(mx)
plt.clf()
for i, psfimg in enumerate(psfcropgrid):
#plt.subplot(len(YY), len(XX), i+1)
subplot_grid(len(YY), len(XX), i)
dimshow(np.log10(np.maximum(psfimg, mx * 1e-16)),
vmax=logmx,
vmin=logmx - 4,
ticks=False,
cmap='jet')
plt.suptitle('PsfEx models')
ps.savefig()
modnames = [
'Dense-grid MoG', 'Coarse-grid MoG', 'Dense-grid MoG (variance)',
'Coarse-grid MoG (variance)'
]
models = [psfex, subpsfex, psfexvar, subpsfexvar]
modgrids = [[] for m in models]
for iy, y in enumerate(YY):
for ix, x in enumerate(XX):
def main():
decals = Decals()
catpattern = 'pipebrick-cats/tractor-phot-b%06i.fits'
ra, dec = 242, 7
# Region-of-interest, in pixels: x0, x1, y0, y1
#roi = None
roi = [500, 1000, 500, 1000]
if roi is not None:
x0, x1, y0, y1 = roi
#expnum = 346623
#ccdname = 'N12'
#chips = decals.find_ccds(expnum=expnum, extname=ccdname)
#print 'Found', len(chips), 'chips for expnum', expnum, 'extname', ccdname
#if len(chips) != 1:
#return False
chips = decals.get_ccds()
D = np.argsort(np.hypot(chips.ra - ra, chips.dec - dec))
print('Closest chip:', chips[D[0]])
chips = [chips[D[0]]]
im = DecamImage(decals, chips[0])
print('Image:', im)
targetwcs = Sip(im.wcsfn)
if roi is not None:
targetwcs = targetwcs.get_subimage(x0, y0, x1 - x0, y1 - y0)
r0, r1, d0, d1 = targetwcs.radec_bounds()
# ~ 30-pixel margin
margin = 2e-3
if r0 > r1:
# RA wrap-around
TT = [
brick_catalog_for_radec_box(ra, rb, d0 - margin, d1 + margin,
decals, catpattern)
for (ra, rb) in [(0, r1 + margin), (r0 - margin, 360.)]
]
T = merge_tables(TT)
T._header = TT[0]._header
else:
T = brick_catalog_for_radec_box(r0 - margin, r1 + margin, d0 - margin,
d1 + margin, decals, catpattern)
print('Got', len(T), 'catalog entries within range')
cat = read_fits_catalog(T, T._header)
print('Got', len(cat), 'catalog objects')
print('Switching ellipse parameterizations')
switch_to_soft_ellipses(cat)
keepcat = []
for src in cat:
if not np.all(np.isfinite(src.getParams())):
print('Src has infinite params:', src)
continue
if isinstance(src, FixedCompositeGalaxy):
f = src.fracDev.getClippedValue()
if f == 0.:
src = ExpGalaxy(src.pos, src.brightness, src.shapeExp)
elif f == 1.:
src = DevGalaxy(src.pos, src.brightness, src.shapeDev)
keepcat.append(src)
cat = keepcat
slc = None
if roi is not None:
slc = slice(y0, y1), slice(x0, x1)
tim = im.get_tractor_image(slc=slc)
print('Got', tim)
tim.psfex.fitSavedData(*tim.psfex.splinedata)
tim.psfex.radius = 20
tim.psf = CachingPsfEx.fromPsfEx(tim.psfex)
tractor = Tractor([tim], cat)
print('Created', tractor)
mod = tractor.getModelImage(0)
plt.clf()
dimshow(tim.getImage(), **tim.ima)
plt.title('Image')
plt.savefig('1.png')
plt.clf()
dimshow(mod, **tim.ima)
plt.title('Model')
plt.savefig('2.png')
ok, x, y = targetwcs.radec2pixelxy([src.getPosition().ra for src in cat],
[src.getPosition().dec for src in cat])
ax = plt.axis()
plt.plot(x, y, 'rx')
#plt.savefig('3.png')
plt.axis(ax)
plt.title('Sources')
plt.savefig('3.png')
bands = [im.band]
import runbrick
runbrick.photoobjdir = '.'
scat, T = get_sdss_sources(bands, targetwcs, local=False)
print('Got', len(scat), 'SDSS sources in bounds')
stractor = Tractor([tim], scat)
print('Created', stractor)
smod = stractor.getModelImage(0)
plt.clf()
dimshow(smod, **tim.ima)
plt.title('SDSS model')
plt.savefig('4.png')
def stage0(**kwargs):
ps = PlotSequence('cfht')
decals = CfhtDecals()
B = decals.get_bricks()
print('Bricks:')
B.about()
ra, dec = 190.0, 11.0
#bands = 'ugri'
bands = 'gri'
B.cut(np.argsort(degrees_between(ra, dec, B.ra, B.dec)))
print('Nearest bricks:', B.ra[:5], B.dec[:5], B.brickid[:5])
brick = B[0]
pixscale = 0.186
#W,H = 1024,1024
#W,H = 2048,2048
#W,H = 3600,3600
W, H = 4800, 4800
targetwcs = wcs_for_brick(brick, pixscale=pixscale, W=W, H=H)
ccdfn = 'cfht-ccds.fits'
if os.path.exists(ccdfn):
T = fits_table(ccdfn)
else:
T = get_ccd_list()
T.writeto(ccdfn)
print(len(T), 'CCDs')
T.cut(ccds_touching_wcs(targetwcs, T))
print(len(T), 'CCDs touching brick')
T.cut(np.array([b in bands for b in T.filter]))
print(len(T), 'in bands', bands)
ims = []
for t in T:
im = CfhtImage(t)
# magzp = hdr['PHOT_C'] + 2.5 * np.log10(hdr['EXPTIME'])
# fwhm = t.seeing / (pixscale * 3600)
# print '-> FWHM', fwhm, 'pix'
im.seeing = t.seeing
im.pixscale = t.pixscale
print('seeing', t.seeing)
print('pixscale', im.pixscale * 3600, 'arcsec/pix')
im.run_calibs(t.ra, t.dec, im.pixscale, W=t.width, H=t.height)
ims.append(im)
# Read images, clip to ROI
targetrd = np.array([
targetwcs.pixelxy2radec(x, y)
for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
])
keepims = []
tims = []
for im in ims:
print()
print('Reading expnum', im.expnum, 'name', im.extname, 'band', im.band,
'exptime', im.exptime)
band = im.band
wcs = im.read_wcs()
imh, imw = wcs.imageh, wcs.imagew
imgpoly = [(1, 1), (1, imh), (imw, imh), (imw, 1)]
ok, tx, ty = wcs.radec2pixelxy(targetrd[:-1, 0], targetrd[:-1, 1])
tpoly = zip(tx, ty)
clip = clip_polygon(imgpoly, tpoly)
clip = np.array(clip)
#print 'Clip', clip
if len(clip) == 0:
continue
x0, y0 = np.floor(clip.min(axis=0)).astype(int)
x1, y1 = np.ceil(clip.max(axis=0)).astype(int)
slc = slice(y0, y1 + 1), slice(x0, x1 + 1)
## FIXME -- it seems I got lucky and the cross product is
## negative == clockwise, as required by clip_polygon. One
## could check this and reverse the polygon vertex order.
# dx0,dy0 = tx[1]-tx[0], ty[1]-ty[0]
# dx1,dy1 = tx[2]-tx[1], ty[2]-ty[1]
# cross = dx0*dy1 - dx1*dy0
# print 'Cross:', cross
print('Image slice: x [%i,%i], y [%i,%i]' % (x0, x1, y0, y1))
print('Reading image from', im.imgfn, 'HDU', im.hdu)
img, imghdr = im.read_image(header=True, slice=slc)
goodpix = (img != 0)
print('Number of pixels == 0:', np.sum(img == 0))
print('Number of pixels != 0:', np.sum(goodpix))
if np.sum(goodpix) == 0:
continue
# print 'Image shape', img.shape
print('Image range', img.min(), img.max())
print('Goodpix image range:', (img[goodpix]).min(),
(img[goodpix]).max())
if img[goodpix].min() == img[goodpix].max():
print('No dynamic range in image')
continue
# print 'Reading invvar from', im.wtfn, 'HDU', im.hdu
# invvar = im.read_invvar(slice=slc)
# # print 'Invvar shape', invvar.shape
# # print 'Invvar range:', invvar.min(), invvar.max()
# invvar[goodpix == 0] = 0.
# if np.all(invvar == 0.):
# print 'Skipping zero-invvar image'
# continue
# assert(np.all(np.isfinite(img)))
# assert(np.all(np.isfinite(invvar)))
# assert(not(np.all(invvar == 0.)))
# # Estimate per-pixel noise via Blanton's 5-pixel MAD
# slice1 = (slice(0,-5,10),slice(0,-5,10))
# slice2 = (slice(5,None,10),slice(5,None,10))
# # print 'sliced shapes:', img[slice1].shape, img[slice2].shape
# # print 'good shape:', (goodpix[slice1] * goodpix[slice2]).shape
# # print 'good values:', np.unique(goodpix[slice1] * goodpix[slice2])
# # print 'sliced[good] shapes:', (img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].shape
# mad = np.median(np.abs(img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].ravel())
# sig1 = 1.4826 * mad / np.sqrt(2.)
# print 'MAD sig1:', sig1
# # invvar was 1 or 0
# invvar *= (1./(sig1**2))
# medsky = np.median(img[goodpix])
# Read full image for sig1 and sky estimate
fullimg = im.read_image()
fullgood = (fullimg != 0)
# Estimate per-pixel noise via Blanton's 5-pixel MAD
slice1 = (slice(0, -5, 10), slice(0, -5, 10))
slice2 = (slice(5, None, 10), slice(5, None, 10))
mad = np.median(
np.abs(fullimg[slice1] -
fullimg[slice2])[fullgood[slice1] *
fullgood[slice2]].ravel())
sig1 = 1.4826 * mad / np.sqrt(2.)
print('MAD sig1:', sig1)
medsky = np.median(fullimg[fullgood])
invvar = np.zeros_like(img)
invvar[goodpix] = 1. / sig1**2
# Median-smooth sky subtraction
plt.clf()
dimshow(np.round((img - medsky) / sig1), vmin=-3, vmax=5)
plt.title('Scalar median: %s' % im.name)
ps.savefig()
# medsky = np.zeros_like(img)
# # astrometry.util.util
# median_smooth(img, np.logical_not(goodpix), 256, medsky)
fullmed = np.zeros_like(fullimg)
median_smooth(fullimg - medsky, np.logical_not(fullgood), 256, fullmed)
fullmed += medsky
medimg = fullmed[slc]
plt.clf()
dimshow(np.round((img - medimg) / sig1), vmin=-3, vmax=5)
plt.title('Median filtered: %s' % im.name)
ps.savefig()
#print 'Subtracting median:', medsky
#img -= medsky
img -= medimg
primhdr = im.read_image_primary_header()
magzp = decals.get_zeropoint_for(im)
print('magzp', magzp)
zpscale = NanoMaggies.zeropointToScale(magzp)
print('zpscale', zpscale)
# Scale images to Nanomaggies
img /= zpscale
sig1 /= zpscale
invvar *= zpscale**2
orig_zpscale = zpscale
zpscale = 1.
assert (np.sum(invvar > 0) > 0)
print('After scaling:')
print('sig1', sig1)
print('invvar range', invvar.min(), invvar.max())
print('image range', img.min(), img.max())
assert (np.all(np.isfinite(img)))
assert (np.all(np.isfinite(invvar)))
assert (np.isfinite(sig1))
plt.clf()
lo, hi = -5 * sig1, 10 * sig1
n, b, p = plt.hist(img[goodpix].ravel(),
100,
range=(lo, hi),
histtype='step',
color='k')
xx = np.linspace(lo, hi, 200)
plt.plot(xx, max(n) * np.exp(-xx**2 / (2. * sig1**2)), 'r-')
plt.xlim(lo, hi)
plt.title('Pixel histogram: %s' % im.name)
ps.savefig()
twcs = ConstantFitsWcs(wcs)
if x0 or y0:
twcs.setX0Y0(x0, y0)
info = im.get_image_info()
fullh, fullw = info['dims']
# read fit PsfEx model
psfex = PsfEx.fromFits(im.psffitfn)
print('Read', psfex)
# HACK -- highly approximate PSF here!
#psf_fwhm = imghdr['FWHM']
#psf_fwhm = im.seeing
psf_fwhm = im.seeing / (im.pixscale * 3600)
print('PSF FWHM', psf_fwhm, 'pixels')
psf_sigma = psf_fwhm / 2.35
psf = NCircularGaussianPSF([psf_sigma], [1.])
print('img type', img.dtype)
tim = Image(img,
invvar=invvar,
wcs=twcs,
psf=psf,
photocal=LinearPhotoCal(zpscale, band=band),
sky=ConstantSky(0.),
name=im.name + ' ' + band)
tim.zr = [-3. * sig1, 10. * sig1]
tim.sig1 = sig1
tim.band = band
tim.psf_fwhm = psf_fwhm
tim.psf_sigma = psf_sigma
tim.sip_wcs = wcs
tim.x0, tim.y0 = int(x0), int(y0)
tim.psfex = psfex
tim.imobj = im
mn, mx = tim.zr
tim.ima = dict(interpolation='nearest',
origin='lower',
cmap='gray',
vmin=mn,
vmax=mx)
tims.append(tim)
keepims.append(im)
ims = keepims
print('Computing resampling...')
# save resampling params
for tim in tims:
wcs = tim.sip_wcs
subh, subw = tim.shape
subwcs = wcs.get_subimage(tim.x0, tim.y0, subw, subh)
tim.subwcs = subwcs
try:
Yo, Xo, Yi, Xi, rims = resample_with_wcs(targetwcs, subwcs, [], 2)
except OverlapError:
print('No overlap')
continue
if len(Yo) == 0:
continue
tim.resamp = (Yo, Xo, Yi, Xi)
print('Creating coadds...')
# Produce per-band coadds, for plots
coimgs = []
cons = []
for ib, band in enumerate(bands):
coimg = np.zeros((H, W), np.float32)
con = np.zeros((H, W), np.uint8)
for tim in tims:
if tim.band != band:
continue
(Yo, Xo, Yi, Xi) = tim.resamp
if len(Yo) == 0:
continue
nn = (tim.getInvvar()[Yi, Xi] > 0)
coimg[Yo, Xo] += tim.getImage()[Yi, Xi] * nn
con[Yo, Xo] += nn
# print
# print 'tim', tim.name
# print 'number of resampled pix:', len(Yo)
# reim = np.zeros_like(coimg)
# ren = np.zeros_like(coimg)
# reim[Yo,Xo] = tim.getImage()[Yi,Xi] * nn
# ren[Yo,Xo] = nn
# print 'number of resampled pix with positive invvar:', ren.sum()
# plt.clf()
# plt.subplot(2,2,1)
# mn,mx = [np.percentile(reim[ren>0], p) for p in [25,95]]
# print 'Percentiles:', mn,mx
# dimshow(reim, vmin=mn, vmax=mx)
# plt.colorbar()
# plt.subplot(2,2,2)
# dimshow(con)
# plt.colorbar()
# plt.subplot(2,2,3)
# dimshow(reim, vmin=tim.zr[0], vmax=tim.zr[1])
# plt.colorbar()
# plt.subplot(2,2,4)
# plt.hist(reim.ravel(), 100, histtype='step', color='b')
# plt.hist(tim.getImage().ravel(), 100, histtype='step', color='r')
# plt.suptitle('%s: %s' % (band, tim.name))
# ps.savefig()
coimg /= np.maximum(con, 1)
coimgs.append(coimg)
cons.append(con)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ps.savefig()
plt.clf()
for i, b in enumerate(bands):
plt.subplot(2, 2, i + 1)
dimshow(cons[i], ticks=False)
plt.title('%s band' % b)
plt.colorbar()
plt.suptitle('Number of exposures')
ps.savefig()
print('Grabbing SDSS sources...')
bandlist = [b for b in bands]
cat, T = get_sdss_sources(bandlist, targetwcs)
# record coordinates in target brick image
ok, T.tx, T.ty = targetwcs.radec2pixelxy(T.ra, T.dec)
T.tx -= 1
T.ty -= 1
T.itx = np.clip(np.round(T.tx).astype(int), 0, W - 1)
T.ity = np.clip(np.round(T.ty).astype(int), 0, H - 1)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(T.tx, T.ty, 'o', mec=green, mfc='none', ms=10, mew=1.5)
plt.axis(ax)
plt.title('SDSS sources')
ps.savefig()
print('Detmaps...')
# Render the detection maps
detmaps = dict([(b, np.zeros((H, W), np.float32)) for b in bands])
detivs = dict([(b, np.zeros((H, W), np.float32)) for b in bands])
for tim in tims:
iv = tim.getInvvar()
psfnorm = 1. / (2. * np.sqrt(np.pi) * tim.psf_sigma)
detim = tim.getImage().copy()
detim[iv == 0] = 0.
detim = gaussian_filter(detim, tim.psf_sigma) / psfnorm**2
detsig1 = tim.sig1 / psfnorm
subh, subw = tim.shape
detiv = np.zeros((subh, subw), np.float32) + (1. / detsig1**2)
detiv[iv == 0] = 0.
(Yo, Xo, Yi, Xi) = tim.resamp
detmaps[tim.band][Yo, Xo] += detiv[Yi, Xi] * detim[Yi, Xi]
detivs[tim.band][Yo, Xo] += detiv[Yi, Xi]
rtn = dict()
for k in [
'T', 'coimgs', 'cons', 'detmaps', 'detivs', 'targetrd', 'pixscale',
'targetwcs', 'W', 'H', 'bands', 'tims', 'ps', 'brick', 'cat'
]:
rtn[k] = locals()[k]
return rtn
psfcropgrid = []
crop = 10
for y in YY:
for x in XX:
psfimg = psfex.instantiateAt(x, y)
psfgrid.append(psfimg)
h,w = psfimg.shape
img = psfimg[h/2-crop:h/2+crop+1, w/2-crop:w/2+crop+1]
psfcropgrid.append(img)
mx = np.max([psfimg.max() for psfimg in psfgrid])
logmx = np.log10(mx)
plt.clf()
for i,psfimg in enumerate(psfcropgrid):
#plt.subplot(len(YY), len(XX), i+1)
subplot_grid(len(YY), len(XX), i)
dimshow(np.log10(np.maximum(psfimg, mx*1e-16)),
vmax=logmx, vmin=logmx-4, ticks=False, cmap='jet')
plt.suptitle('PsfEx models')
ps.savefig()
modnames = ['Dense-grid MoG', 'Coarse-grid MoG',
'Dense-grid MoG (variance)', 'Coarse-grid MoG (variance)']
models = [ psfex, subpsfex, psfexvar, subpsfexvar ]
modgrids = [[] for m in models]
for iy,y in enumerate(YY):
for ix,x in enumerate(XX):
for model,grid in zip(models, modgrids):
psf = model.psfAt(x, y)
def _debug_plots(srctractor, ps):
thislnp0 = srctractor.getLogProb()
p0 = np.array(srctractor.getParams())
print('logprob:', p0, '=', thislnp0)
print('p0 type:', p0.dtype)
px = p0 + np.zeros_like(p0)
srctractor.setParams(px)
lnpx = srctractor.getLogProb()
assert (lnpx == thislnp0)
print('logprob:', px, '=', lnpx)
scales = srctractor.getParameterScales()
print('Parameter scales:', scales)
print('Parameters:')
srctractor.printThawedParams()
# getParameterScales better not have changed the params!!
assert (np.all(p0 == np.array(srctractor.getParams())))
assert (srctractor.getLogProb() == thislnp0)
pfinal = srctractor.getParams()
pnames = srctractor.getParamNames()
plt.figure(3, figsize=(8, 6))
plt.clf()
for i in range(len(scales)):
plt.plot([(p[i] - pfinal[i]) * scales[i] for lnp, p in params],
[lnp for lnp, p in params],
'-',
label=pnames[i])
plt.ylabel('lnp')
plt.legend()
plt.title('scaled')
ps.savefig()
for i in range(len(scales)):
plt.clf()
#plt.subplot(2,1,1)
plt.plot([p[i] for lnp, p in params], '-')
plt.xlabel('step')
plt.title(pnames[i])
ps.savefig()
plt.clf()
plt.plot([p[i] for lnp, p in params], [lnp for lnp, p in params],
'b.-')
# We also want to know about d(lnp)/d(param)
# and d(lnp)/d(X)
step = 1.1
steps = 1.1**np.arange(-20, 21)
s2 = np.linspace(0, steps[0], 10)[1:-1]
steps = reduce(np.append, [-steps[::-1], -s2[::-1], 0, s2, steps])
print('Steps:', steps)
plt.plot(p0[i], thislnp0, 'bx', ms=20)
print('Stepping in param', pnames[i], '...')
pp = p0.copy()
lnps, parms = [], []
for s in steps:
parm = p0[i] + s / scales[i]
pp[i] = parm
srctractor.setParams(pp)
lnp = srctractor.getLogProb()
parms.append(parm)
lnps.append(lnp)
print('logprob:', pp, '=', lnp)
plt.plot(parms, lnps, 'k.-')
j = np.argmin(np.abs(steps - 1.))
plt.plot(parms[j], lnps[j], 'ko')
print('Stepping in X...')
lnps, parms = [], []
for s in steps:
pp = p0 + s * X
srctractor.setParams(pp)
lnp = srctractor.getLogProb()
parms.append(pp[i])
lnps.append(lnp)
print('logprob:', pp, '=', lnp)
##
s3 = s2[:2]
ministeps = reduce(np.append, [-s3[::-1], 0, s3])
print('mini steps:', ministeps)
for s in ministeps:
pp = p0 + s * X
srctractor.setParams(pp)
lnp = srctractor.getLogProb()
print('logprob:', pp, '=', lnp)
rows = len(ministeps)
cols = len(srctractor.images)
plt.figure(4, figsize=(8, 6))
plt.subplots_adjust(hspace=0.05,
wspace=0.05,
left=0.01,
right=0.99,
bottom=0.01,
top=0.99)
plt.clf()
k = 1
mods = []
for s in ministeps:
pp = p0 + s * X
srctractor.setParams(pp)
print('ministep', s)
print('log prior', srctractor.getLogPrior())
print('log likelihood', srctractor.getLogLikelihood())
mods.append(srctractor.getModelImages())
chis = srctractor.getChiImages()
# for chi in chis:
# plt.subplot(rows, cols, k)
# k += 1
# dimshow(chi, ticks=False, vmin=-10, vmax=10, cmap='jet')
print('chisqs:', [(chi**2).sum() for chi in chis])
print('sum:', sum([(chi**2).sum() for chi in chis]))
mod0 = mods[len(ministeps) / 2]
for modlist in mods:
for mi, mod in enumerate(modlist):
plt.subplot(rows, cols, k)
k += 1
m0 = mod0[mi]
rng = m0.max() - m0.min()
dimshow(mod - mod0[mi],
vmin=-0.01 * rng,
vmax=0.01 * rng,
ticks=False,
cmap='gray')
ps.savefig()
plt.figure(3)
plt.plot(parms, lnps, 'r.-')
print('Stepping in X by alphas...')
lnps = []
for cc, ss in [('m', 0.1), ('m', 0.3), ('r', 1)]:
pp = p0 + ss * X
srctractor.setParams(pp)
lnp = srctractor.getLogProb()
print('logprob:', pp, '=', lnp)
plt.plot(p0[i] + ss * X[i], lnp, 'o', color=cc)
lnps.append(lnp)
px = p0[i] + X[i]
pmid = (px + p0[i]) / 2.
dp = np.abs((px - pmid) * 2.)
hi, lo = max(max(lnps), thislnp0), min(min(lnps), thislnp0)
lnpmid = (hi + lo) / 2.
dlnp = np.abs((hi - lo) * 2.)
plt.ylabel('lnp')
plt.title(pnames[i])
ps.savefig()
plt.axis([pmid - dp, pmid + dp, lnpmid - dlnp, lnpmid + dlnp])
ps.savefig()
srctractor.setParams(p0)
def sed_matched_detection(sedname,
sed,
detmaps,
detivs,
bands,
xomit,
yomit,
nsigma=5.,
saturated_pix=None,
saddle=2.,
cutonaper=True,
ps=None):
'''
Runs a single SED-matched detection filter.
Avoids creating sources close to existing sources.
Parameters
----------
sedname : string
Name of this SED; only used for plots.
sed : list of floats
The SED -- a list of floats, one per band, of this SED.
detmaps : list of numpy arrays
The per-band detection maps. These must all be the same size, the
brick image size.
detivs : list of numpy arrays
The inverse-variance maps associated with `detmaps`.
bands : list of strings
The band names of the `detmaps` and `detivs` images.
xomit, yomit : iterables (lists or numpy arrays) of int
Previously known sources that are to be avoided.
nsigma : float, optional
Detection threshold.
saturated_pix : None or numpy array, boolean
A map of pixels that are always considered "hot" when
determining whether a new source touches hot pixels of an
existing source.
saddle : float, optional
Saddle-point depth from existing sources down to new sources.
cutonaper : bool, optional
Apply a cut that the source's detection strength must be greater
than `nsigma` above the 16th percentile of the detection strength in
an annulus (from 10 to 20 pixels) around the source.
ps : PlotSequence object, optional
Create plots?
Returns
-------
hotblobs : numpy array of bool
A map of the blobs yielding sources in this SED.
px, py : numpy array of int
The new sources found.
aper : numpy array of float
The detection strength in the annulus around the source, if
`cutonaper` is set; else -1.
peakval : numpy array of float
The detection strength.
See also
--------
sed_matched_filters : creates the `(sedname, sed)` pairs used here
run_sed_matched_filters : calls this method
'''
from scipy.ndimage.measurements import label, find_objects
from scipy.ndimage.morphology import binary_dilation, binary_fill_holes
t0 = Time()
H, W = detmaps[0].shape
allzero = True
for iband, band in enumerate(bands):
if sed[iband] == 0:
continue
if np.all(detivs[iband] == 0):
continue
allzero = False
break
if allzero:
print('SED', sedname, 'has all zero weight')
return None, None, None, None, None
sedmap = np.zeros((H, W), np.float32)
sediv = np.zeros((H, W), np.float32)
for iband, band in enumerate(bands):
if sed[iband] == 0:
continue
# We convert the detmap to canonical band via
# detmap * w
# And the corresponding change to sig1 is
# sig1 * w
# So the invvar-weighted sum is
# (detmap * w) / (sig1**2 * w**2)
# = detmap / (sig1**2 * w)
sedmap += detmaps[iband] * detivs[iband] / sed[iband]
sediv += detivs[iband] / sed[iband]**2
sedmap /= np.maximum(1e-16, sediv)
sedsn = sedmap * np.sqrt(sediv)
del sedmap
peaks = (sedsn > nsigma)
print('SED sn:', Time() - t0)
t0 = Time()
def saddle_level(Y):
# Require a saddle that drops by (the larger of) "saddle"
# sigma, or 20% of the peak height
drop = max(saddle, Y * 0.2)
return Y - drop
lowest_saddle = nsigma - saddle
# zero out the edges -- larger margin here?
peaks[0, :] = 0
peaks[:, 0] = 0
peaks[-1, :] = 0
peaks[:, -1] = 0
# Label the N-sigma blobs at this point... we'll use this to build
# "sedhot", which in turn is used to define the blobs that we will
# optimize simultaneously. This also determines which pixels go
# into the fitting!
dilate = 8
hotblobs, nhot = label(
binary_fill_holes(binary_dilation(peaks, iterations=dilate)))
# find pixels that are larger than their 8 neighbors
peaks[1:-1, 1:-1] &= (sedsn[1:-1, 1:-1] >= sedsn[0:-2, 1:-1])
peaks[1:-1, 1:-1] &= (sedsn[1:-1, 1:-1] >= sedsn[2:, 1:-1])
peaks[1:-1, 1:-1] &= (sedsn[1:-1, 1:-1] >= sedsn[1:-1, 0:-2])
peaks[1:-1, 1:-1] &= (sedsn[1:-1, 1:-1] >= sedsn[1:-1, 2:])
peaks[1:-1, 1:-1] &= (sedsn[1:-1, 1:-1] >= sedsn[0:-2, 0:-2])
peaks[1:-1, 1:-1] &= (sedsn[1:-1, 1:-1] >= sedsn[0:-2, 2:])
peaks[1:-1, 1:-1] &= (sedsn[1:-1, 1:-1] >= sedsn[2:, 0:-2])
peaks[1:-1, 1:-1] &= (sedsn[1:-1, 1:-1] >= sedsn[2:, 2:])
print('Peaks:', Time() - t0)
t0 = Time()
if ps is not None:
from astrometry.util.plotutils import dimshow
crossa = dict(ms=10, mew=1.5)
green = (0, 1, 0)
def plot_boundary_map(X):
bounds = binary_dilation(X) - X
H, W = X.shape
rgba = np.zeros((H, W, 4), np.uint8)
rgba[:, :, 1] = bounds * 255
rgba[:, :, 3] = bounds * 255
plt.imshow(rgba, interpolation='nearest', origin='lower')
plt.clf()
plt.subplot(1, 2, 2)
dimshow(sedsn, vmin=-2, vmax=100, cmap='hot', ticks=False)
plt.subplot(1, 2, 1)
dimshow(sedsn, vmin=-2, vmax=10, cmap='hot', ticks=False)
above = (sedsn > nsigma)
plot_boundary_map(above)
ax = plt.axis()
y, x = np.nonzero(peaks)
plt.plot(x, y, 'r+')
plt.axis(ax)
plt.title('SED %s: S/N & peaks' % sedname)
ps.savefig()
# plt.clf()
# plt.imshow(sedsn, vmin=-2, vmax=10, interpolation='nearest',
# origin='lower', cmap='hot')
# plot_boundary_map(sedsn > lowest_saddle)
# plt.title('SED %s: S/N & lowest saddle point bounds' % sedname)
# ps.savefig()
# For each new source, compute the saddle value, segment at that
# level, and drop the source if it is in the same blob as a
# previously-detected source. We dilate the blobs a bit too, to
# catch slight differences in centroid vs SDSS sources.
dilate = 2
# For efficiency, segment at the minimum saddle level to compute
# slices; the operations described above need only happen within
# the slice.
saddlemap = (sedsn > lowest_saddle)
if saturated_pix is not None:
saddlemap |= saturated_pix
saddlemap = binary_dilation(saddlemap, iterations=dilate)
allblobs, nblobs = label(saddlemap)
allslices = find_objects(allblobs)
ally0 = [sy.start for sy, sx in allslices]
allx0 = [sx.start for sy, sx in allslices]
# brightest peaks first
py, px = np.nonzero(peaks)
I = np.argsort(-sedsn[py, px])
py = py[I]
px = px[I]
keep = np.zeros(len(px), bool)
peakval = []
aper = []
apin = 10
apout = 20
# Map of pixels that are vetoed by sources found so far. The veto
# area is based on saddle height. We go from brightest to
# faintest pixels. Thus the saddle level decreases, and the
# saddlemap areas become larger; the saddlemap when a source is
# found is a lower bound on the pixels that it will veto based on
# the saddle heights of fainter sources. Thus the vetomap isn't
# the final word, it is just a quick veto of pixels we know for
# sure will be vetoed.
vetomap = np.zeros(sedsn.shape, bool)
# For each peak, determine whether it is isolated enough --
# separated by a low enough saddle from other sources. Need only
# search within its "allblob", which is defined by the lowest
# saddle.
print('Found', len(px), 'potential peaks')
#tlast = Time()
for i, (x, y) in enumerate(zip(px, py)):
if vetomap[y, x]:
#print(' in veto map!')
continue
#t0 = Time()
#t1 = Time()
#print('Time since last source:', t1-tlast)
#tlast = t1
level = saddle_level(sedsn[y, x])
ablob = allblobs[y, x]
index = ablob - 1
slc = allslices[index]
#print('source', i, 'of', len(px), 'at', x,y, 'S/N', sedsn[y,x], 'saddle', level)
#print(' allblobs slice', slc)
saddlemap = (sedsn[slc] > level)
if saturated_pix is not None:
saddlemap |= saturated_pix[slc]
saddlemap *= (allblobs[slc] == ablob)
#print(' saddlemap', Time()-tlast)
saddlemap = binary_fill_holes(saddlemap)
#print(' fill holes', Time()-tlast)
saddlemap = binary_dilation(saddlemap, iterations=dilate)
#print(' dilation', Time()-tlast)
blobs, nblobs = label(saddlemap)
#print(' label', Time()-tlast)
x0, y0 = allx0[index], ally0[index]
thisblob = blobs[y - y0, x - x0]
# previously found sources:
ox = np.append(xomit, px[:i][keep[:i]]) - x0
oy = np.append(yomit, py[:i][keep[:i]]) - y0
h, w = blobs.shape
cut = False
if len(ox):
ox = ox.astype(int)
oy = oy.astype(int)
cut = any((ox >= 0) * (ox < w) * (oy >= 0) * (oy < h) *
(blobs[np.clip(oy, 0, h - 1),
np.clip(ox, 0, w - 1)] == thisblob))
if False and (not cut) and ps is not None:
plt.clf()
plt.subplot(1, 2, 1)
dimshow(sedsn, vmin=-2, vmax=10, cmap='hot')
plot_boundary_map((sedsn > nsigma))
ax = plt.axis()
plt.plot(x, y, 'm+', ms=12, mew=2)
plt.axis(ax)
plt.subplot(1, 2, 2)
y1, x1 = [s.stop for s in slc]
ext = [x0, x1, y0, y1]
dimshow(saddlemap, extent=ext)
#plt.plot([x0,x0,x1,x1,x0], [y0,y1,y1,y0,y0], 'c-')
#ax = plt.axis()
#plt.plot(ox+x0, oy+y0, 'rx')
plt.plot(xomit, yomit, 'rx', ms=8, mew=2)
plt.plot(px[:i][keep[:i]],
py[:i][keep[:i]],
'+',
color=green,
ms=8,
mew=2)
plt.plot(x, y, 'mo', mec='m', mfc='none', ms=12, mew=2)
plt.axis(ax)
if cut:
plt.suptitle('Cut')
else:
plt.suptitle('Keep')
ps.savefig()
#t1 = Time()
#print(t1 - t0)
if cut:
# in same blob as previously found source.
#print(' cut')
# update vetomap
vetomap[slc] |= saddlemap
#print('Added to vetomap:', np.sum(saddlemap), 'pixels set; now total of', np.sum(vetomap), 'pixels set')
continue
# Measure in aperture...
ap = sedsn[max(0, y - apout):min(H, y + apout + 1),
max(0, x - apout):min(W, x + apout + 1)]
apiv = (sediv[max(0, y - apout):min(H, y + apout + 1),
max(0, x - apout):min(W, x + apout + 1)] > 0)
aph, apw = ap.shape
apx0, apy0 = max(0, x - apout), max(0, y - apout)
R2 = ((np.arange(aph) + apy0 - y)[:, np.newaxis]**2 +
(np.arange(apw) + apx0 - x)[np.newaxis, :]**2)
ap = ap[apiv * (R2 >= apin**2) * (R2 <= apout**2)]
if len(ap):
# 16th percentile ~ -1 sigma point.
m = np.percentile(ap, 16.)
else:
# fake
m = -1.
#print(' aper', Time()-tlast)
if cutonaper:
if sedsn[y, x] - m < nsigma:
continue
aper.append(m)
peakval.append(sedsn[y, x])
keep[i] = True
vetomap[slc] |= saddlemap
#print('Added to vetomap:', np.sum(saddlemap), 'pixels set; now total of', np.sum(vetomap), 'pixels set')
if False and ps is not None:
plt.clf()
plt.subplot(1, 2, 1)
dimshow(ap,
vmin=-2,
vmax=10,
cmap='hot',
extent=[apx0, apx0 + apw, apy0, apy0 + aph])
plt.subplot(1, 2, 2)
dimshow(ap * ((R2 >= apin**2) * (R2 <= apout**2)),
vmin=-2,
vmax=10,
cmap='hot',
extent=[apx0, apx0 + apw, apy0, apy0 + aph])
plt.suptitle('peak %.1f vs ap %.1f' % (sedsn[y, x], m))
ps.savefig()
print('New sources:', Time() - t0)
t0 = Time()
if ps is not None:
pxdrop = px[np.logical_not(keep)]
pydrop = py[np.logical_not(keep)]
KJB = True
if KJB:
pxdrop = px[np.logical_not(keep)]
pydrop = py[np.logical_not(keep)]
py = py[keep]
px = px[keep]
# Which of the hotblobs yielded sources? Those are the ones to keep.
hbmap = np.zeros(nhot + 1, bool)
hbmap[hotblobs[py, px]] = True
if len(xomit):
h, w = hotblobs.shape
hbmap[hotblobs[np.clip(yomit, 0, h - 1),
np.clip(xomit, 0, w - 1)]] = True
# in case a source is (somehow) not in a hotblob?
hbmap[0] = False
hotblobs = hbmap[hotblobs]
if ps is not None:
plt.clf()
dimshow(vetomap, vmin=0, vmax=1, cmap='hot')
plt.title('SED %s: veto map' % sedname)
ps.savefig()
plt.clf()
dimshow(hotblobs, vmin=0, vmax=1, cmap='hot')
ax = plt.axis()
p1 = plt.plot(px, py, 'g+', ms=8, mew=2)
p2 = plt.plot(pxdrop, pydrop, 'm+', ms=8, mew=2)
p3 = plt.plot(xomit, yomit, 'r+', ms=8, mew=2)
plt.axis(ax)
plt.title('SED %s: hot blobs' % sedname)
plt.figlegend((p3[0], p1[0], p2[0]), ('Existing', 'Keep', 'Drop'),
'upper left')
ps.savefig()
if KJB:
from astrometry.util.plotutils import dimshow
crossa = dict(ms=10, mew=1.5)
green = (0, 1, 0)
def plot_boundary_map(X):
bounds = binary_dilation(X) - X
H, W = X.shape
rgba = np.zeros((H, W, 4), np.uint8)
rgba[:, :, 1] = bounds * 255
rgba[:, :, 3] = bounds * 255
plt.imshow(rgba, interpolation='nearest', origin='lower')
plt.clf()
plt.subplot(1, 2, 2)
dimshow(sedsn, vmin=-2, vmax=100, cmap='hot', ticks=False)
plt.title('sedsn')
plt.subplot(1, 2, 1)
dimshow(hotblobs, vmin=0, vmax=1, cmap='hot')
ax = plt.axis()
p1 = plt.plot(px, py, 'g+', ms=8, mew=2)
p2 = plt.plot(pxdrop, pydrop, 'm+', ms=8, mew=2)
p3 = plt.plot(xomit, yomit, 'r+', ms=8, mew=2)
plt.axis(ax)
plt.title('SED %s: hot blobs' % sedname)
plt.figlegend((p3[0], p1[0], p2[0]), ('Existing', 'Keep', 'Drop'),
'upper left')
#crazy! ps.outdir is not known here yet it is in all stages!
#output SED pngs with current time so can later see which belongs to which
#plt.savefig(os.path.join(ps.outdir,'sedmap_%s.png' % sedname))
cpudate = do_bash("date")
cpudate = cpudate[cpudate.find(':') - 2:].replace(' ',
'').strip().replace(
':', '-')
plt.savefig('./sedmap_%s_%s.png' % (sedname, cpudate))
return hotblobs, px, py, aper, peakval
def _plot_mods(tims,
mods,
blobwcs,
titles,
bands,
coimgs,
cons,
bslc,
blobw,
blobh,
ps,
chi_plots=True,
rgb_plots=False,
main_plot=True,
rgb_format='%s'):
import numpy as np
subims = [[] for m in mods]
chis = dict([(b, []) for b in bands])
make_coimgs = (coimgs is None)
if make_coimgs:
print('_plot_mods: blob shape', (blobh, blobw))
coimgs = [np.zeros((blobh, blobw)) for b in bands]
cons = [np.zeros((blobh, blobw)) for b in bands]
for iband, band in enumerate(bands):
comods = [np.zeros((blobh, blobw)) for m in mods]
cochis = [np.zeros((blobh, blobw)) for m in mods]
comodn = np.zeros((blobh, blobw))
mn, mx = 0, 0
sig1 = 1.
for itim, tim in enumerate(tims):
if tim.band != band:
continue
R = tim_get_resamp(tim, blobwcs)
if R is None:
continue
(Yo, Xo, Yi, Xi) = R
rechi = np.zeros((blobh, blobw))
chilist = []
comodn[Yo, Xo] += 1
for imod, mod in enumerate(mods):
chi = ((tim.getImage()[Yi, Xi] - mod[itim][Yi, Xi]) *
tim.getInvError()[Yi, Xi])
rechi[Yo, Xo] = chi
chilist.append((rechi.copy(), itim))
cochis[imod][Yo, Xo] += chi
comods[imod][Yo, Xo] += mod[itim][Yi, Xi]
chis[band].append(chilist)
# we'll use 'sig1' of the last tim in the list below...
mn, mx = -10. * tim.sig1, 30. * tim.sig1
sig1 = tim.sig1
if make_coimgs:
nn = (tim.getInvError()[Yi, Xi] > 0)
coimgs[iband][Yo, Xo] += tim.getImage()[Yi, Xi] * nn
cons[iband][Yo, Xo] += nn
if make_coimgs:
coimgs[iband] /= np.maximum(cons[iband], 1)
coimg = coimgs[iband]
coimgn = cons[iband]
else:
coimg = coimgs[iband][bslc]
coimgn = cons[iband][bslc]
for comod in comods:
comod /= np.maximum(comodn, 1)
ima = dict(vmin=mn, vmax=mx, ticks=False)
resida = dict(vmin=-5. * sig1, vmax=5. * sig1, ticks=False)
for subim, comod, cochi in zip(subims, comods, cochis):
subim.append((coimg, coimgn, comod, ima, cochi, resida))
# Plot per-band image, model, and chi coadds, and RGB images
rgba = dict(ticks=False)
rgbs = []
rgbnames = []
plt.figure(1)
for i, subim in enumerate(subims):
plt.clf()
rows, cols = 3, 5
for ib, b in enumerate(bands):
plt.subplot(rows, cols, ib + 1)
plt.title(b)
plt.subplot(rows, cols, 4)
plt.title('RGB')
plt.subplot(rows, cols, 5)
plt.title('RGB(stretch)')
imgs = []
themods = []
resids = []
for j, (img, imgn, mod, ima, chi, resida) in enumerate(subim):
imgs.append(img)
themods.append(mod)
resid = img - mod
resid[imgn == 0] = np.nan
resids.append(resid)
if main_plot:
plt.subplot(rows, cols, 1 + j + 0)
dimshow(img, **ima)
plt.subplot(rows, cols, 1 + j + cols)
dimshow(mod, **ima)
plt.subplot(rows, cols, 1 + j + cols * 2)
# dimshow(-chi, **imchi)
# dimshow(imgn, vmin=0, vmax=3)
dimshow(resid, nancolor='r', **resida)
rgb = get_rgb(imgs, bands)
if i == 0:
rgbs.append(rgb)
rgbnames.append(rgb_format % 'Image')
if main_plot:
plt.subplot(rows, cols, 4)
dimshow(rgb, **rgba)
rgb = get_rgb(themods, bands)
rgbs.append(rgb)
rgbnames.append(rgb_format % titles[i])
if main_plot:
plt.subplot(rows, cols, cols + 4)
dimshow(rgb, **rgba)
plt.subplot(rows, cols, cols * 2 + 4)
dimshow(get_rgb(resids, bands, mnmx=(-10, 10)), **rgba)
mnmx = -5, 300
kwa = dict(mnmx=mnmx, arcsinh=1)
plt.subplot(rows, cols, 5)
dimshow(get_rgb(imgs, bands, **kwa), **rgba)
plt.subplot(rows, cols, cols + 5)
dimshow(get_rgb(themods, bands, **kwa), **rgba)
plt.subplot(rows, cols, cols * 2 + 5)
mnmx = -100, 100
kwa = dict(mnmx=mnmx, arcsinh=1)
dimshow(get_rgb(resids, bands, **kwa), **rgba)
plt.suptitle(titles[i])
ps.savefig()
if rgb_plots:
# RGB image and model
plt.figure(2)
for rgb, tt in zip(rgbs, rgbnames):
plt.clf()
dimshow(rgb, **rgba)
plt.title(tt)
ps.savefig()
if not chi_plots:
return
imchi = dict(cmap='RdBu', vmin=-5, vmax=5)
plt.figure(1)
# Plot per-image chis: in a grid with band along the rows and images along the cols
cols = max(len(v) for v in chis.values())
rows = len(bands)
for imod in range(len(mods)):
plt.clf()
for row, band in enumerate(bands):
sp0 = 1 + cols * row
# chis[band] = [ (one for each tim:) [ (one for each mod:) (chi,itim), (chi,itim) ], ...]
for col, chilist in enumerate(chis[band]):
chi, itim = chilist[imod]
plt.subplot(rows, cols, sp0 + col)
dimshow(-chi, **imchi)
plt.xticks([])
plt.yticks([])
plt.title(tims[itim].name)
#plt.suptitle(titles[imod])
ps.savefig()
def _debug_plots(srctractor, ps):
thislnp0 = srctractor.getLogProb()
p0 = np.array(srctractor.getParams())
print 'logprob:', p0, '=', thislnp0
print 'p0 type:', p0.dtype
px = p0 + np.zeros_like(p0)
srctractor.setParams(px)
lnpx = srctractor.getLogProb()
assert(lnpx == thislnp0)
print 'logprob:', px, '=', lnpx
scales = srctractor.getParameterScales()
print 'Parameter scales:', scales
print 'Parameters:'
srctractor.printThawedParams()
# getParameterScales better not have changed the params!!
assert(np.all(p0 == np.array(srctractor.getParams())))
assert(srctractor.getLogProb() == thislnp0)
pfinal = srctractor.getParams()
pnames = srctractor.getParamNames()
plt.figure(3, figsize=(8,6))
plt.clf()
for i in range(len(scales)):
plt.plot([(p[i] - pfinal[i])*scales[i] for lnp,p in params],
[lnp for lnp,p in params], '-', label=pnames[i])
plt.ylabel('lnp')
plt.legend()
plt.title('scaled')
ps.savefig()
for i in range(len(scales)):
plt.clf()
#plt.subplot(2,1,1)
plt.plot([p[i] for lnp,p in params], '-')
plt.xlabel('step')
plt.title(pnames[i])
ps.savefig()
plt.clf()
plt.plot([p[i] for lnp,p in params],
[lnp for lnp,p in params], 'b.-')
# We also want to know about d(lnp)/d(param)
# and d(lnp)/d(X)
step = 1.1
steps = 1.1 ** np.arange(-20, 21)
s2 = np.linspace(0, steps[0], 10)[1:-1]
steps = reduce(np.append, [-steps[::-1], -s2[::-1], 0, s2, steps])
print 'Steps:', steps
plt.plot(p0[i], thislnp0, 'bx', ms=20)
print 'Stepping in param', pnames[i], '...'
pp = p0.copy()
lnps,parms = [],[]
for s in steps:
parm = p0[i] + s / scales[i]
pp[i] = parm
srctractor.setParams(pp)
lnp = srctractor.getLogProb()
parms.append(parm)
lnps.append(lnp)
print 'logprob:', pp, '=', lnp
plt.plot(parms, lnps, 'k.-')
j = np.argmin(np.abs(steps - 1.))
plt.plot(parms[j], lnps[j], 'ko')
print 'Stepping in X...'
lnps,parms = [],[]
for s in steps:
pp = p0 + s * X
srctractor.setParams(pp)
lnp = srctractor.getLogProb()
parms.append(pp[i])
lnps.append(lnp)
print 'logprob:', pp, '=', lnp
##
s3 = s2[:2]
ministeps = reduce(np.append, [-s3[::-1], 0, s3])
print 'mini steps:', ministeps
for s in ministeps:
pp = p0 + s * X
srctractor.setParams(pp)
lnp = srctractor.getLogProb()
print 'logprob:', pp, '=', lnp
rows = len(ministeps)
cols = len(srctractor.images)
plt.figure(4, figsize=(8,6))
plt.subplots_adjust(hspace=0.05, wspace=0.05, left=0.01,
right=0.99, bottom=0.01, top=0.99)
plt.clf()
k = 1
mods = []
for s in ministeps:
pp = p0 + s * X
srctractor.setParams(pp)
print 'ministep', s
print 'log prior', srctractor.getLogPrior()
print 'log likelihood', srctractor.getLogLikelihood()
mods.append(srctractor.getModelImages())
chis = srctractor.getChiImages()
# for chi in chis:
# plt.subplot(rows, cols, k)
# k += 1
# dimshow(chi, ticks=False, vmin=-10, vmax=10, cmap='jet')
print 'chisqs:', [(chi**2).sum() for chi in chis]
print 'sum:', sum([(chi**2).sum() for chi in chis])
mod0 = mods[len(ministeps)/2]
for modlist in mods:
for mi,mod in enumerate(modlist):
plt.subplot(rows, cols, k)
k += 1
m0 = mod0[mi]
rng = m0.max() - m0.min()
dimshow(mod - mod0[mi], vmin=-0.01*rng, vmax=0.01*rng,
ticks=False, cmap='gray')
ps.savefig()
plt.figure(3)
plt.plot(parms, lnps, 'r.-')
print 'Stepping in X by alphas...'
lnps = []
for cc,ss in [('m',0.1), ('m',0.3), ('r',1)]:
pp = p0 + ss*X
srctractor.setParams(pp)
lnp = srctractor.getLogProb()
print 'logprob:', pp, '=', lnp
plt.plot(p0[i] + ss * X[i], lnp, 'o', color=cc)
lnps.append(lnp)
px = p0[i] + X[i]
pmid = (px + p0[i]) / 2.
dp = np.abs((px - pmid) * 2.)
hi,lo = max(max(lnps), thislnp0), min(min(lnps), thislnp0)
lnpmid = (hi + lo) / 2.
dlnp = np.abs((hi - lo) * 2.)
plt.ylabel('lnp')
plt.title(pnames[i])
ps.savefig()
plt.axis([pmid - dp, pmid + dp, lnpmid-dlnp, lnpmid+dlnp])
ps.savefig()
srctractor.setParams(p0)
def stage_psfplots(T=None,
sedsn=None,
coimgs=None,
cons=None,
detmaps=None,
detivs=None,
blobsrcs=None,
blobflux=None,
blobslices=None,
blobs=None,
tractor=None,
cat=None,
targetrd=None,
pixscale=None,
targetwcs=None,
W=None,
H=None,
brickid=None,
bands=None,
ps=None,
tims=None,
plots=False,
**kwargs):
tim = tims[0]
tim.psfex.fitSavedData(*tim.psfex.splinedata)
spl = tim.psfex.splines[0]
print('Spline:', spl)
knots = spl.get_knots()
print('knots:', knots)
tx, ty = knots
k = 3
print('interior knots x:', tx[k + 1:-k - 1])
print('additional knots x:', tx[:k + 1], 'and', tx[-k - 1:])
print('interior knots y:', ty[k + 1:-k - 1])
print('additional knots y:', ty[:k + 1], 'and', ty[-k - 1:])
for itim, tim in enumerate(tims):
psfex = tim.psfex
psfex.fitSavedData(*psfex.splinedata)
if plots:
print()
print('Tim', tim)
print()
pp, xx, yy = psfex.splinedata
ny, nx, nparams = pp.shape
assert (len(xx) == nx)
assert (len(yy) == ny)
psfnil = psfex.psfclass(*np.zeros(nparams))
names = psfnil.getParamNames()
xa = np.linspace(xx[0], xx[-1], 50)
ya = np.linspace(yy[0], yy[-1], 100)
#xa,ya = np.meshgrid(xa,ya)
#xa = xa.ravel()
#ya = ya.ravel()
print('xa', xa)
print('ya', ya)
for i in range(nparams):
plt.clf()
plt.subplot(1, 2, 1)
dimshow(pp[:, :, i])
plt.title('grid fit')
plt.colorbar()
plt.subplot(1, 2, 2)
sp = psfex.splines[i](xa, ya)
sp = sp.T
print('spline shape', sp.shape)
assert (sp.shape == (len(ya), len(xa)))
dimshow(sp, extent=[xx[0], xx[-1], yy[0], yy[-1]])
plt.title('spline')
plt.colorbar()
plt.suptitle('tim %s: PSF param %s' % (tim.name, names[i]))
ps.savefig()
def _plot_mods(tims, mods, blobwcs, titles, bands, coimgs, cons, bslc,
blobw, blobh, ps,
chi_plots=True, rgb_plots=False, main_plot=True,
rgb_format='%s'):
import numpy as np
subims = [[] for m in mods]
chis = dict([(b,[]) for b in bands])
make_coimgs = (coimgs is None)
if make_coimgs:
print '_plot_mods: blob shape', (blobh, blobw)
coimgs = [np.zeros((blobh,blobw)) for b in bands]
cons = [np.zeros((blobh,blobw)) for b in bands]
for iband,band in enumerate(bands):
comods = [np.zeros((blobh,blobw)) for m in mods]
cochis = [np.zeros((blobh,blobw)) for m in mods]
comodn = np.zeros((blobh,blobw))
mn,mx = 0,0
sig1 = 1.
for itim,tim in enumerate(tims):
if tim.band != band:
continue
R = tim_get_resamp(tim, blobwcs)
if R is None:
continue
(Yo,Xo,Yi,Xi) = R
rechi = np.zeros((blobh,blobw))
chilist = []
comodn[Yo,Xo] += 1
for imod,mod in enumerate(mods):
chi = ((tim.getImage()[Yi,Xi] - mod[itim][Yi,Xi]) *
tim.getInvError()[Yi,Xi])
rechi[Yo,Xo] = chi
chilist.append((rechi.copy(), itim))
cochis[imod][Yo,Xo] += chi
comods[imod][Yo,Xo] += mod[itim][Yi,Xi]
chis[band].append(chilist)
# we'll use 'sig1' of the last tim in the list below...
mn,mx = -10.*tim.sig1, 30.*tim.sig1
sig1 = tim.sig1
if make_coimgs:
nn = (tim.getInvError()[Yi,Xi] > 0)
coimgs[iband][Yo,Xo] += tim.getImage()[Yi,Xi] * nn
cons [iband][Yo,Xo] += nn
if make_coimgs:
coimgs[iband] /= np.maximum(cons[iband], 1)
coimg = coimgs[iband]
coimgn = cons [iband]
else:
coimg = coimgs[iband][bslc]
coimgn = cons[iband][bslc]
for comod in comods:
comod /= np.maximum(comodn, 1)
ima = dict(vmin=mn, vmax=mx, ticks=False)
resida = dict(vmin=-5.*sig1, vmax=5.*sig1, ticks=False)
for subim,comod,cochi in zip(subims, comods, cochis):
subim.append((coimg, coimgn, comod, ima, cochi, resida))
# Plot per-band image, model, and chi coadds, and RGB images
rgba = dict(ticks=False)
rgbs = []
rgbnames = []
plt.figure(1)
for i,subim in enumerate(subims):
plt.clf()
rows,cols = 3,5
for ib,b in enumerate(bands):
plt.subplot(rows,cols,ib+1)
plt.title(b)
plt.subplot(rows,cols,4)
plt.title('RGB')
plt.subplot(rows,cols,5)
plt.title('RGB(stretch)')
imgs = []
themods = []
resids = []
for j,(img,imgn,mod,ima,chi,resida) in enumerate(subim):
imgs.append(img)
themods.append(mod)
resid = img - mod
resid[imgn == 0] = np.nan
resids.append(resid)
if main_plot:
plt.subplot(rows,cols,1 + j + 0)
dimshow(img, **ima)
plt.subplot(rows,cols,1 + j + cols)
dimshow(mod, **ima)
plt.subplot(rows,cols,1 + j + cols*2)
# dimshow(-chi, **imchi)
# dimshow(imgn, vmin=0, vmax=3)
dimshow(resid, nancolor='r', **resida)
rgb = get_rgb(imgs, bands)
if i == 0:
rgbs.append(rgb)
rgbnames.append(rgb_format % 'Image')
if main_plot:
plt.subplot(rows,cols, 4)
dimshow(rgb, **rgba)
rgb = get_rgb(themods, bands)
rgbs.append(rgb)
rgbnames.append(rgb_format % titles[i])
if main_plot:
plt.subplot(rows,cols, cols+4)
dimshow(rgb, **rgba)
plt.subplot(rows,cols, cols*2+4)
dimshow(get_rgb(resids, bands, mnmx=(-10,10)), **rgba)
mnmx = -5,300
kwa = dict(mnmx=mnmx, arcsinh=1)
plt.subplot(rows,cols, 5)
dimshow(get_rgb(imgs, bands, **kwa), **rgba)
plt.subplot(rows,cols, cols+5)
dimshow(get_rgb(themods, bands, **kwa), **rgba)
plt.subplot(rows,cols, cols*2+5)
mnmx = -100,100
kwa = dict(mnmx=mnmx, arcsinh=1)
dimshow(get_rgb(resids, bands, **kwa), **rgba)
plt.suptitle(titles[i])
ps.savefig()
if rgb_plots:
# RGB image and model
plt.figure(2)
for rgb,tt in zip(rgbs, rgbnames):
plt.clf()
dimshow(rgb, **rgba)
plt.title(tt)
ps.savefig()
if not chi_plots:
return
imchi = dict(cmap='RdBu', vmin=-5, vmax=5)
plt.figure(1)
# Plot per-image chis: in a grid with band along the rows and images along the cols
cols = max(len(v) for v in chis.values())
rows = len(bands)
for imod in range(len(mods)):
plt.clf()
for row,band in enumerate(bands):
sp0 = 1 + cols*row
# chis[band] = [ (one for each tim:) [ (one for each mod:) (chi,itim), (chi,itim) ], ...]
for col,chilist in enumerate(chis[band]):
chi,itim = chilist[imod]
plt.subplot(rows, cols, sp0 + col)
dimshow(-chi, **imchi)
plt.xticks([]); plt.yticks([])
plt.title(tims[itim].name)
#plt.suptitle(titles[imod])
ps.savefig()
def stage_fitplots(T=None,
coimgs=None,
cons=None,
cat=None,
targetrd=None,
pixscale=None,
targetwcs=None,
W=None,
H=None,
bands=None,
ps=None,
brickid=None,
plots=False,
plots2=False,
tims=None,
tractor=None,
pipe=None,
outdir=None,
**kwargs):
for tim in tims:
print('Tim', tim, 'PSF', tim.getPsf())
writeModels = False
if pipe:
t0 = Time()
# Produce per-band coadds, for plots
coimgs, cons = compute_coadds(tims, bands, targetwcs)
print('Coadds:', Time() - t0)
plt.figure(figsize=(10, 10.5))
#plt.subplots_adjust(left=0.002, right=0.998, bottom=0.002, top=0.998)
plt.subplots_adjust(left=0.002, right=0.998, bottom=0.002, top=0.95)
plt.clf()
dimshow(get_rgb(coimgs, bands))
plt.title('Image')
ps.savefig()
ax = plt.axis()
cat = tractor.getCatalog()
for i, src in enumerate(cat):
rd = src.getPosition()
ok, x, y = targetwcs.radec2pixelxy(rd.ra, rd.dec)
cc = (0, 1, 0)
if isinstance(src, PointSource):
plt.plot(x - 1, y - 1, '+', color=cc, ms=10, mew=1.5)
else:
plt.plot(x - 1, y - 1, 'o', mec=cc, mfc='none', ms=10, mew=1.5)
# plt.text(x, y, '%i' % i, color=cc, ha='center', va='bottom')
plt.axis(ax)
ps.savefig()
mnmx = -5, 300
arcsinha = dict(mnmx=mnmx, arcsinh=1)
# After plot
rgbmod = []
rgbmod2 = []
rgbresids = []
rgbchisqs = []
chibins = np.linspace(-10., 10., 200)
chihist = [np.zeros(len(chibins) - 1, int) for band in bands]
wcsW = targetwcs.get_width()
wcsH = targetwcs.get_height()
print('Target WCS shape', wcsW, wcsH)
t0 = Time()
mods = _map(_get_mod, [(tim, cat) for tim in tims])
print('Getting model images:', Time() - t0)
orig_wcsxy0 = [tim.wcs.getX0Y0() for tim in tims]
for iband, band in enumerate(bands):
coimg = coimgs[iband]
comod = np.zeros((wcsH, wcsW), np.float32)
comod2 = np.zeros((wcsH, wcsW), np.float32)
cochi2 = np.zeros((wcsH, wcsW), np.float32)
for itim, (tim, mod) in enumerate(zip(tims, mods)):
if tim.band != band:
continue
#mod = tractor.getModelImage(tim)
if plots2:
plt.clf()
dimshow(tim.getImage(), **tim.ima)
plt.title(tim.name)
ps.savefig()
plt.clf()
dimshow(mod, **tim.ima)
plt.title(tim.name)
ps.savefig()
plt.clf()
dimshow((tim.getImage() - mod) * tim.getInvError(), **imchi)
plt.title(tim.name)
ps.savefig()
R = tim_get_resamp(tim, targetwcs)
if R is None:
continue
(Yo, Xo, Yi, Xi) = R
comod[Yo, Xo] += mod[Yi, Xi]
ie = tim.getInvError()
noise = np.random.normal(size=ie.shape) / ie
noise[ie == 0] = 0.
comod2[Yo, Xo] += mod[Yi, Xi] + noise[Yi, Xi]
chi = ((tim.getImage()[Yi, Xi] - mod[Yi, Xi]) *
tim.getInvError()[Yi, Xi])
cochi2[Yo, Xo] += chi**2
chi = chi[chi != 0.]
hh, xe = np.histogram(np.clip(chi, -10, 10).ravel(), bins=chibins)
chihist[iband] += hh
if not writeModels:
continue
im = tim.imobj
fn = 'image-b%06i-%s-%s.fits' % (brickid, band, im.name)
wcsfn = create_temp()
wcs = tim.getWcs().wcs
x0, y0 = orig_wcsxy0[itim]
h, w = tim.shape
subwcs = wcs.get_subimage(int(x0), int(y0), w, h)
subwcs.write_to(wcsfn)
primhdr = fitsio.FITSHDR()
primhdr.add_record(
dict(name='X0', value=x0, comment='Pixel origin of subimage'))
primhdr.add_record(
dict(name='Y0', value=y0, comment='Pixel origin of subimage'))
xfn = im.wcsfn.replace(decals_dir + '/', '')
primhdr.add_record(dict(name='WCS_FILE', value=xfn))
xfn = im.psffn.replace(decals_dir + '/', '')
primhdr.add_record(dict(name='PSF_FILE', value=xfn))
primhdr.add_record(dict(name='INHERIT', value=True))
imhdr = fitsio.read_header(wcsfn)
imhdr.add_record(
dict(name='EXTTYPE',
value='IMAGE',
comment='This HDU contains image data'))
ivhdr = fitsio.read_header(wcsfn)
ivhdr.add_record(
dict(name='EXTTYPE',
value='INVVAR',
comment='This HDU contains an inverse-variance map'))
fits = fitsio.FITS(fn, 'rw', clobber=True)
tim.toFits(fits,
primheader=primhdr,
imageheader=imhdr,
invvarheader=ivhdr)
imhdr.add_record(
dict(name='EXTTYPE',
value='MODEL',
comment='This HDU contains a Tractor model image'))
fits.write(mod, header=imhdr)
print('Wrote image and model to', fn)
comod /= np.maximum(cons[iband], 1)
comod2 /= np.maximum(cons[iband], 1)
rgbmod.append(comod)
rgbmod2.append(comod2)
resid = coimg - comod
resid[cons[iband] == 0] = np.nan
rgbresids.append(resid)
rgbchisqs.append(cochi2)
# Plug the WCS header cards into these images
wcsfn = create_temp()
targetwcs.write_to(wcsfn)
hdr = fitsio.read_header(wcsfn)
os.remove(wcsfn)
if outdir is None:
outdir = '.'
wa = dict(clobber=True, header=hdr)
for name, img in [('image', coimg), ('model', comod), ('resid', resid),
('chi2', cochi2)]:
fn = os.path.join(outdir,
'%s-coadd-%06i-%s.fits' % (name, brickid, band))
fitsio.write(fn, img, **wa)
print('Wrote', fn)
del cons
plt.clf()
dimshow(get_rgb(rgbmod, bands))
plt.title('Model')
ps.savefig()
plt.clf()
dimshow(get_rgb(rgbmod2, bands))
plt.title('Model + Noise')
ps.savefig()
plt.clf()
dimshow(get_rgb(rgbresids, bands))
plt.title('Residuals')
ps.savefig()
plt.clf()
dimshow(get_rgb(rgbresids, bands, mnmx=(-30, 30)))
plt.title('Residuals (2)')
ps.savefig()
plt.clf()
dimshow(get_rgb(coimgs, bands, **arcsinha))
plt.title('Image (stretched)')
ps.savefig()
plt.clf()
dimshow(get_rgb(rgbmod2, bands, **arcsinha))
plt.title('Model + Noise (stretched)')
ps.savefig()
del coimgs
del rgbresids
del rgbmod
del rgbmod2
plt.clf()
g, r, z = rgbchisqs
im = np.log10(np.dstack((z, r, g)))
mn, mx = 0, im.max()
dimshow(np.clip((im - mn) / (mx - mn), 0., 1.))
plt.title('Chi-squared')
ps.savefig()
plt.clf()
xx = np.repeat(chibins, 2)[1:-1]
for y, cc in zip(chihist, 'grm'):
plt.plot(xx, np.repeat(np.maximum(0.1, y), 2), '-', color=cc)
plt.xlabel('Chi')
plt.yticks([])
plt.axvline(0., color='k', alpha=0.25)
ps.savefig()
plt.yscale('log')
mx = np.max([max(y) for y in chihist])
plt.ylim(1, mx * 1.05)
ps.savefig()
return dict(tims=tims)
def main():
decals = Decals()
catpattern = 'pipebrick-cats/tractor-phot-b%06i.fits'
ra,dec = 242, 7
# Region-of-interest, in pixels: x0, x1, y0, y1
#roi = None
roi = [500, 1000, 500, 1000]
if roi is not None:
x0,x1,y0,y1 = roi
#expnum = 346623
#ccdname = 'N12'
#chips = decals.find_ccds(expnum=expnum, extname=ccdname)
#print 'Found', len(chips), 'chips for expnum', expnum, 'extname', ccdname
#if len(chips) != 1:
#return False
chips = decals.get_ccds()
D = np.argsort(np.hypot(chips.ra - ra, chips.dec - dec))
print('Closest chip:', chips[D[0]])
chips = [chips[D[0]]]
im = DecamImage(decals, chips[0])
print('Image:', im)
targetwcs = Sip(im.wcsfn)
if roi is not None:
targetwcs = targetwcs.get_subimage(x0, y0, x1-x0, y1-y0)
r0,r1,d0,d1 = targetwcs.radec_bounds()
# ~ 30-pixel margin
margin = 2e-3
if r0 > r1:
# RA wrap-around
TT = [brick_catalog_for_radec_box(ra,rb, d0-margin,d1+margin,
decals, catpattern)
for (ra,rb) in [(0, r1+margin), (r0-margin, 360.)]]
T = merge_tables(TT)
T._header = TT[0]._header
else:
T = brick_catalog_for_radec_box(r0-margin,r1+margin,d0-margin,
d1+margin, decals, catpattern)
print('Got', len(T), 'catalog entries within range')
cat = read_fits_catalog(T, T._header)
print('Got', len(cat), 'catalog objects')
print('Switching ellipse parameterizations')
switch_to_soft_ellipses(cat)
keepcat = []
for src in cat:
if not np.all(np.isfinite(src.getParams())):
print('Src has infinite params:', src)
continue
if isinstance(src, FixedCompositeGalaxy):
f = src.fracDev.getClippedValue()
if f == 0.:
src = ExpGalaxy(src.pos, src.brightness, src.shapeExp)
elif f == 1.:
src = DevGalaxy(src.pos, src.brightness, src.shapeDev)
keepcat.append(src)
cat = keepcat
slc = None
if roi is not None:
slc = slice(y0,y1), slice(x0,x1)
tim = im.get_tractor_image(slc=slc)
print('Got', tim)
tim.psfex.fitSavedData(*tim.psfex.splinedata)
tim.psfex.radius = 20
tim.psf = CachingPsfEx.fromPsfEx(tim.psfex)
tractor = Tractor([tim], cat)
print('Created', tractor)
mod = tractor.getModelImage(0)
plt.clf()
dimshow(tim.getImage(), **tim.ima)
plt.title('Image')
plt.savefig('1.png')
plt.clf()
dimshow(mod, **tim.ima)
plt.title('Model')
plt.savefig('2.png')
ok,x,y = targetwcs.radec2pixelxy([src.getPosition().ra for src in cat],
[src.getPosition().dec for src in cat])
ax = plt.axis()
plt.plot(x, y, 'rx')
#plt.savefig('3.png')
plt.axis(ax)
plt.title('Sources')
plt.savefig('3.png')
bands = [im.band]
import runbrick
runbrick.photoobjdir = '.'
scat,T = get_sdss_sources(bands, targetwcs, local=False)
print('Got', len(scat), 'SDSS sources in bounds')
stractor = Tractor([tim], scat)
print('Created', stractor)
smod = stractor.getModelImage(0)
plt.clf()
dimshow(smod, **tim.ima)
plt.title('SDSS model')
plt.savefig('4.png')