def stage_rergb(coimgs=None,
bands=None,
basedir=None,
brickid=None,
ps=None,
**kwargs):
plt.figure(figsize=(10, 10))
plt.subplots_adjust(left=0.002, right=0.998, bottom=0.002, top=0.998)
rgb = get_rgb(coimgs, bands)
plt.clf()
dimshow(rgb)
fn = os.path.join(basedir, 'coadd', 'image2-%06i.png' % brickid)
plt.savefig(fn)
print('Saved', fn)
tmpfn = create_temp(suffix='.png')
plt.imsave(tmpfn, rgb)
del rgb
fn = os.path.join(basedir, 'coadd', 'image2-%06i-full.jpg' % brickid)
cmd = 'pngtopnm %s | pnmtojpeg -quality 90 > %s' % (tmpfn, fn)
os.system(cmd)
os.unlink(tmpfn)
print('Wrote', fn)
def stage_primage(coimgs=None, bands=None, ps=None, basedir=None, **kwargs):
'''
Nice PR image for NOAO. Make a synthetic brick centered on a nice galaxy in 374451
bricks.txt:
# brickid ra dec ra1 ra2 dec1 dec2
1 244.70 7.41 244.5739 244.8261 7.285 7.535
2 244.70 7.41 244.5739 244.8261 7.285 7.535
text2fits.py -f jdddddd PR/bricks.txt PR/decals-bricks.fits
cp decals/decals-ccds.fits PR
(cd PR; ln -s ~/cosmo/work/decam/versions/work/calib .)
(cd PR; ln -s ~/cosmo/work/decam/versions/work/images .)
export DECALS_DIR=$(pwd)/PR
python -u projects/desi/tunebrick.py -b 1 -s primage -P "pickles/PR-%(brick)06i-%%(stage)s.pickle"
python -u projects/desi/tunebrick.py -b 374441 -s primage -P "pickles/PR-%(brick)06i-%%(stage)s.pickle"
'''
print('kwargs:', kwargs.keys())
rgb = get_rgb(coimgs, bands, mnmx=(0., 100.), arcsinh=1.)
plt.clf()
dimshow(rgb)
ps.savefig()
fn = ps.getnext()
plt.imsave(fn, rgb, origin='lower')
jpegfn = fn.replace('.png', '.jpg')
cmd = 'pngtopnm %s | pnmtojpeg -quality 80 > %s' % (fn, jpegfn)
print(cmd)
os.system(cmd)
def stage_rergb(coimgs=None, bands=None, basedir=None, brickid=None,
ps=None, **kwargs):
plt.figure(figsize=(10,10))
plt.subplots_adjust(left=0.002, right=0.998, bottom=0.002, top=0.998)
rgb = get_rgb(coimgs, bands)
plt.clf()
dimshow(rgb)
fn = os.path.join(basedir, 'coadd', 'image2-%06i.png' % brickid)
plt.savefig(fn)
print 'Saved', fn
tmpfn = create_temp(suffix='.png')
plt.imsave(tmpfn, rgb)
del rgb
fn = os.path.join(basedir, 'coadd', 'image2-%06i-full.jpg' % brickid)
cmd = 'pngtopnm %s | pnmtojpeg -quality 90 > %s' % (tmpfn, fn)
os.system(cmd)
os.unlink(tmpfn)
print 'Wrote', fn
def stage_primage(coimgs=None, bands=None, ps=None, basedir=None,
**kwargs):
'''
Nice PR image for NOAO. Make a synthetic brick centered on a nice galaxy in 374451
bricks.txt:
# brickid ra dec ra1 ra2 dec1 dec2
1 244.70 7.41 244.5739 244.8261 7.285 7.535
2 244.70 7.41 244.5739 244.8261 7.285 7.535
text2fits.py -f jdddddd PR/bricks.txt PR/decals-bricks.fits
cp decals/decals-ccds.fits PR
(cd PR; ln -s ~/cosmo/work/decam/versions/work/calib .)
(cd PR; ln -s ~/cosmo/work/decam/versions/work/images .)
export DECALS_DIR=$(pwd)/PR
python -u projects/desi/tunebrick.py -b 1 -s primage -P "pickles/PR-%(brick)06i-%%(stage)s.pickle"
python -u projects/desi/tunebrick.py -b 374441 -s primage -P "pickles/PR-%(brick)06i-%%(stage)s.pickle"
'''
print 'kwargs:', kwargs.keys()
rgb = get_rgb(coimgs, bands, mnmx=(0., 100.), arcsinh=1.)
plt.clf()
dimshow(rgb)
ps.savefig()
fn = ps.getnext()
plt.imsave(fn, rgb, origin='lower')
jpegfn = fn.replace('.png','.jpg')
cmd = 'pngtopnm %s | pnmtojpeg -quality 80 > %s' % (fn, jpegfn)
print cmd
os.system(cmd)
def stage0(**kwargs):
ps = PlotSequence('cfht')
decals = CfhtDecals()
B = decals.get_bricks()
print('Bricks:')
B.about()
ra, dec = 190.0, 11.0
#bands = 'ugri'
bands = 'gri'
B.cut(np.argsort(degrees_between(ra, dec, B.ra, B.dec)))
print('Nearest bricks:', B.ra[:5], B.dec[:5], B.brickid[:5])
brick = B[0]
pixscale = 0.186
#W,H = 1024,1024
#W,H = 2048,2048
#W,H = 3600,3600
W, H = 4800, 4800
targetwcs = wcs_for_brick(brick, pixscale=pixscale, W=W, H=H)
ccdfn = 'cfht-ccds.fits'
if os.path.exists(ccdfn):
T = fits_table(ccdfn)
else:
T = get_ccd_list()
T.writeto(ccdfn)
print(len(T), 'CCDs')
T.cut(ccds_touching_wcs(targetwcs, T))
print(len(T), 'CCDs touching brick')
T.cut(np.array([b in bands for b in T.filter]))
print(len(T), 'in bands', bands)
ims = []
for t in T:
im = CfhtImage(t)
# magzp = hdr['PHOT_C'] + 2.5 * np.log10(hdr['EXPTIME'])
# fwhm = t.seeing / (pixscale * 3600)
# print '-> FWHM', fwhm, 'pix'
im.seeing = t.seeing
im.pixscale = t.pixscale
print('seeing', t.seeing)
print('pixscale', im.pixscale * 3600, 'arcsec/pix')
im.run_calibs(t.ra, t.dec, im.pixscale, W=t.width, H=t.height)
ims.append(im)
# Read images, clip to ROI
targetrd = np.array([
targetwcs.pixelxy2radec(x, y)
for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
])
keepims = []
tims = []
for im in ims:
print()
print('Reading expnum', im.expnum, 'name', im.extname, 'band', im.band,
'exptime', im.exptime)
band = im.band
wcs = im.read_wcs()
imh, imw = wcs.imageh, wcs.imagew
imgpoly = [(1, 1), (1, imh), (imw, imh), (imw, 1)]
ok, tx, ty = wcs.radec2pixelxy(targetrd[:-1, 0], targetrd[:-1, 1])
tpoly = zip(tx, ty)
clip = clip_polygon(imgpoly, tpoly)
clip = np.array(clip)
#print 'Clip', clip
if len(clip) == 0:
continue
x0, y0 = np.floor(clip.min(axis=0)).astype(int)
x1, y1 = np.ceil(clip.max(axis=0)).astype(int)
slc = slice(y0, y1 + 1), slice(x0, x1 + 1)
## FIXME -- it seems I got lucky and the cross product is
## negative == clockwise, as required by clip_polygon. One
## could check this and reverse the polygon vertex order.
# dx0,dy0 = tx[1]-tx[0], ty[1]-ty[0]
# dx1,dy1 = tx[2]-tx[1], ty[2]-ty[1]
# cross = dx0*dy1 - dx1*dy0
# print 'Cross:', cross
print('Image slice: x [%i,%i], y [%i,%i]' % (x0, x1, y0, y1))
print('Reading image from', im.imgfn, 'HDU', im.hdu)
img, imghdr = im.read_image(header=True, slice=slc)
goodpix = (img != 0)
print('Number of pixels == 0:', np.sum(img == 0))
print('Number of pixels != 0:', np.sum(goodpix))
if np.sum(goodpix) == 0:
continue
# print 'Image shape', img.shape
print('Image range', img.min(), img.max())
print('Goodpix image range:', (img[goodpix]).min(),
(img[goodpix]).max())
if img[goodpix].min() == img[goodpix].max():
print('No dynamic range in image')
continue
# print 'Reading invvar from', im.wtfn, 'HDU', im.hdu
# invvar = im.read_invvar(slice=slc)
# # print 'Invvar shape', invvar.shape
# # print 'Invvar range:', invvar.min(), invvar.max()
# invvar[goodpix == 0] = 0.
# if np.all(invvar == 0.):
# print 'Skipping zero-invvar image'
# continue
# assert(np.all(np.isfinite(img)))
# assert(np.all(np.isfinite(invvar)))
# assert(not(np.all(invvar == 0.)))
# # Estimate per-pixel noise via Blanton's 5-pixel MAD
# slice1 = (slice(0,-5,10),slice(0,-5,10))
# slice2 = (slice(5,None,10),slice(5,None,10))
# # print 'sliced shapes:', img[slice1].shape, img[slice2].shape
# # print 'good shape:', (goodpix[slice1] * goodpix[slice2]).shape
# # print 'good values:', np.unique(goodpix[slice1] * goodpix[slice2])
# # print 'sliced[good] shapes:', (img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].shape
# mad = np.median(np.abs(img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].ravel())
# sig1 = 1.4826 * mad / np.sqrt(2.)
# print 'MAD sig1:', sig1
# # invvar was 1 or 0
# invvar *= (1./(sig1**2))
# medsky = np.median(img[goodpix])
# Read full image for sig1 and sky estimate
fullimg = im.read_image()
fullgood = (fullimg != 0)
# Estimate per-pixel noise via Blanton's 5-pixel MAD
slice1 = (slice(0, -5, 10), slice(0, -5, 10))
slice2 = (slice(5, None, 10), slice(5, None, 10))
mad = np.median(
np.abs(fullimg[slice1] -
fullimg[slice2])[fullgood[slice1] *
fullgood[slice2]].ravel())
sig1 = 1.4826 * mad / np.sqrt(2.)
print('MAD sig1:', sig1)
medsky = np.median(fullimg[fullgood])
invvar = np.zeros_like(img)
invvar[goodpix] = 1. / sig1**2
# Median-smooth sky subtraction
plt.clf()
dimshow(np.round((img - medsky) / sig1), vmin=-3, vmax=5)
plt.title('Scalar median: %s' % im.name)
ps.savefig()
# medsky = np.zeros_like(img)
# # astrometry.util.util
# median_smooth(img, np.logical_not(goodpix), 256, medsky)
fullmed = np.zeros_like(fullimg)
median_smooth(fullimg - medsky, np.logical_not(fullgood), 256, fullmed)
fullmed += medsky
medimg = fullmed[slc]
plt.clf()
dimshow(np.round((img - medimg) / sig1), vmin=-3, vmax=5)
plt.title('Median filtered: %s' % im.name)
ps.savefig()
#print 'Subtracting median:', medsky
#img -= medsky
img -= medimg
primhdr = im.read_image_primary_header()
magzp = decals.get_zeropoint_for(im)
print('magzp', magzp)
zpscale = NanoMaggies.zeropointToScale(magzp)
print('zpscale', zpscale)
# Scale images to Nanomaggies
img /= zpscale
sig1 /= zpscale
invvar *= zpscale**2
orig_zpscale = zpscale
zpscale = 1.
assert (np.sum(invvar > 0) > 0)
print('After scaling:')
print('sig1', sig1)
print('invvar range', invvar.min(), invvar.max())
print('image range', img.min(), img.max())
assert (np.all(np.isfinite(img)))
assert (np.all(np.isfinite(invvar)))
assert (np.isfinite(sig1))
plt.clf()
lo, hi = -5 * sig1, 10 * sig1
n, b, p = plt.hist(img[goodpix].ravel(),
100,
range=(lo, hi),
histtype='step',
color='k')
xx = np.linspace(lo, hi, 200)
plt.plot(xx, max(n) * np.exp(-xx**2 / (2. * sig1**2)), 'r-')
plt.xlim(lo, hi)
plt.title('Pixel histogram: %s' % im.name)
ps.savefig()
twcs = ConstantFitsWcs(wcs)
if x0 or y0:
twcs.setX0Y0(x0, y0)
info = im.get_image_info()
fullh, fullw = info['dims']
# read fit PsfEx model
psfex = PsfEx.fromFits(im.psffitfn)
print('Read', psfex)
# HACK -- highly approximate PSF here!
#psf_fwhm = imghdr['FWHM']
#psf_fwhm = im.seeing
psf_fwhm = im.seeing / (im.pixscale * 3600)
print('PSF FWHM', psf_fwhm, 'pixels')
psf_sigma = psf_fwhm / 2.35
psf = NCircularGaussianPSF([psf_sigma], [1.])
print('img type', img.dtype)
tim = Image(img,
invvar=invvar,
wcs=twcs,
psf=psf,
photocal=LinearPhotoCal(zpscale, band=band),
sky=ConstantSky(0.),
name=im.name + ' ' + band)
tim.zr = [-3. * sig1, 10. * sig1]
tim.sig1 = sig1
tim.band = band
tim.psf_fwhm = psf_fwhm
tim.psf_sigma = psf_sigma
tim.sip_wcs = wcs
tim.x0, tim.y0 = int(x0), int(y0)
tim.psfex = psfex
tim.imobj = im
mn, mx = tim.zr
tim.ima = dict(interpolation='nearest',
origin='lower',
cmap='gray',
vmin=mn,
vmax=mx)
tims.append(tim)
keepims.append(im)
ims = keepims
print('Computing resampling...')
# save resampling params
for tim in tims:
wcs = tim.sip_wcs
subh, subw = tim.shape
subwcs = wcs.get_subimage(tim.x0, tim.y0, subw, subh)
tim.subwcs = subwcs
try:
Yo, Xo, Yi, Xi, rims = resample_with_wcs(targetwcs, subwcs, [], 2)
except OverlapError:
print('No overlap')
continue
if len(Yo) == 0:
continue
tim.resamp = (Yo, Xo, Yi, Xi)
print('Creating coadds...')
# Produce per-band coadds, for plots
coimgs = []
cons = []
for ib, band in enumerate(bands):
coimg = np.zeros((H, W), np.float32)
con = np.zeros((H, W), np.uint8)
for tim in tims:
if tim.band != band:
continue
(Yo, Xo, Yi, Xi) = tim.resamp
if len(Yo) == 0:
continue
nn = (tim.getInvvar()[Yi, Xi] > 0)
coimg[Yo, Xo] += tim.getImage()[Yi, Xi] * nn
con[Yo, Xo] += nn
# print
# print 'tim', tim.name
# print 'number of resampled pix:', len(Yo)
# reim = np.zeros_like(coimg)
# ren = np.zeros_like(coimg)
# reim[Yo,Xo] = tim.getImage()[Yi,Xi] * nn
# ren[Yo,Xo] = nn
# print 'number of resampled pix with positive invvar:', ren.sum()
# plt.clf()
# plt.subplot(2,2,1)
# mn,mx = [np.percentile(reim[ren>0], p) for p in [25,95]]
# print 'Percentiles:', mn,mx
# dimshow(reim, vmin=mn, vmax=mx)
# plt.colorbar()
# plt.subplot(2,2,2)
# dimshow(con)
# plt.colorbar()
# plt.subplot(2,2,3)
# dimshow(reim, vmin=tim.zr[0], vmax=tim.zr[1])
# plt.colorbar()
# plt.subplot(2,2,4)
# plt.hist(reim.ravel(), 100, histtype='step', color='b')
# plt.hist(tim.getImage().ravel(), 100, histtype='step', color='r')
# plt.suptitle('%s: %s' % (band, tim.name))
# ps.savefig()
coimg /= np.maximum(con, 1)
coimgs.append(coimg)
cons.append(con)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ps.savefig()
plt.clf()
for i, b in enumerate(bands):
plt.subplot(2, 2, i + 1)
dimshow(cons[i], ticks=False)
plt.title('%s band' % b)
plt.colorbar()
plt.suptitle('Number of exposures')
ps.savefig()
print('Grabbing SDSS sources...')
bandlist = [b for b in bands]
cat, T = get_sdss_sources(bandlist, targetwcs)
# record coordinates in target brick image
ok, T.tx, T.ty = targetwcs.radec2pixelxy(T.ra, T.dec)
T.tx -= 1
T.ty -= 1
T.itx = np.clip(np.round(T.tx).astype(int), 0, W - 1)
T.ity = np.clip(np.round(T.ty).astype(int), 0, H - 1)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(T.tx, T.ty, 'o', mec=green, mfc='none', ms=10, mew=1.5)
plt.axis(ax)
plt.title('SDSS sources')
ps.savefig()
print('Detmaps...')
# Render the detection maps
detmaps = dict([(b, np.zeros((H, W), np.float32)) for b in bands])
detivs = dict([(b, np.zeros((H, W), np.float32)) for b in bands])
for tim in tims:
iv = tim.getInvvar()
psfnorm = 1. / (2. * np.sqrt(np.pi) * tim.psf_sigma)
detim = tim.getImage().copy()
detim[iv == 0] = 0.
detim = gaussian_filter(detim, tim.psf_sigma) / psfnorm**2
detsig1 = tim.sig1 / psfnorm
subh, subw = tim.shape
detiv = np.zeros((subh, subw), np.float32) + (1. / detsig1**2)
detiv[iv == 0] = 0.
(Yo, Xo, Yi, Xi) = tim.resamp
detmaps[tim.band][Yo, Xo] += detiv[Yi, Xi] * detim[Yi, Xi]
detivs[tim.band][Yo, Xo] += detiv[Yi, Xi]
rtn = dict()
for k in [
'T', 'coimgs', 'cons', 'detmaps', 'detivs', 'targetrd', 'pixscale',
'targetwcs', 'W', 'H', 'bands', 'tims', 'ps', 'brick', 'cat'
]:
rtn[k] = locals()[k]
return rtn
def stage1(T=None,
coimgs=None,
cons=None,
detmaps=None,
detivs=None,
targetrd=None,
pixscale=None,
targetwcs=None,
W=None,
H=None,
bands=None,
tims=None,
ps=None,
brick=None,
cat=None):
orig_wcsxy0 = [tim.wcs.getX0Y0() for tim in tims]
hot = np.zeros((H, W), np.float32)
for band in bands:
detmap = detmaps[band] / np.maximum(1e-16, detivs[band])
detsn = detmap * np.sqrt(detivs[band])
hot = np.maximum(hot, detsn)
detmaps[band] = detmap
### FIXME -- ugri
for sedname, sed in [('Flat', (1., 1., 1.)), ('Red', (2.5, 1.0, 0.4))]:
sedmap = np.zeros((H, W), np.float32)
sediv = np.zeros((H, W), np.float32)
for iband, band in enumerate(bands):
# We convert the detmap to canonical band via
# detmap * w
# And the corresponding change to sig1 is
# sig1 * w
# So the invvar-weighted sum is
# (detmap * w) / (sig1**2 * w**2)
# = detmap / (sig1**2 * w)
sedmap += detmaps[band] * detivs[band] / sed[iband]
sediv += detivs[band] / sed[iband]**2
sedmap /= np.maximum(1e-16, sediv)
sedsn = sedmap * np.sqrt(sediv)
hot = np.maximum(hot, sedsn)
plt.clf()
dimshow(np.round(sedsn), vmin=0, vmax=10, cmap='hot')
plt.title('SED-matched detection filter: %s' % sedname)
ps.savefig()
peaks = (hot > 4)
blobs, nblobs = label(peaks)
print('N detected blobs:', nblobs)
blobslices = find_objects(blobs)
# Un-set catalog blobs
for x, y in zip(T.itx, T.ity):
# blob number
bb = blobs[y, x]
if bb == 0:
continue
# un-set 'peaks' within this blob
slc = blobslices[bb - 1]
peaks[slc][blobs[slc] == bb] = 0
# Now, after having removed catalog sources, crank up the detection threshold
peaks &= (hot > 5)
# zero out the edges(?)
peaks[0, :] = peaks[:, 0] = 0
peaks[-1, :] = peaks[:, -1] = 0
peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[0:-2, 1:-1])
peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[2:, 1:-1])
peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[1:-1, 0:-2])
peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[1:-1, 2:])
# These are our peaks
pki = np.flatnonzero(peaks)
peaky, peakx = np.unravel_index(pki, peaks.shape)
print(len(peaky), 'peaks')
crossa = dict(ms=10, mew=1.5)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(T.tx, T.ty, 'r+', **crossa)
plt.plot(peakx, peaky, '+', color=green, **crossa)
plt.axis(ax)
plt.title('SDSS + SED-matched detections')
ps.savefig()
### HACK -- high threshold again
# Segment, and record which sources fall into each blob
blobs, nblobs = label((hot > 20))
print('N detected blobs:', nblobs)
blobslices = find_objects(blobs)
T.blob = blobs[T.ity, T.itx]
blobsrcs = []
blobflux = []
fluximg = coimgs[1]
for blob in range(1, nblobs + 1):
blobsrcs.append(np.flatnonzero(T.blob == blob))
bslc = blobslices[blob - 1]
blobflux.append(np.sum(fluximg[bslc][blobs[bslc] == blob]))
# Fit the SDSS sources
for tim in tims:
tim.psfex.fitSavedData(*tim.psfex.splinedata)
tim.psf = tim.psfex
# How far down to render model profiles
minsigma = 0.1
for tim in tims:
tim.modelMinval = minsigma * tim.sig1
srcvariances = [[] for src in cat]
# Fit in order of flux
for blobnumber, iblob in enumerate(np.argsort(-np.array(blobflux))):
bslc = blobslices[iblob]
Isrcs = blobsrcs[iblob]
if len(Isrcs) == 0:
continue
print()
print('Blob', blobnumber, 'of', len(blobflux), ':', len(Isrcs),
'sources')
print('Source indices:', Isrcs)
print()
# blob bbox in target coords
sy, sx = bslc
by0, by1 = sy.start, sy.stop
bx0, bx1 = sx.start, sx.stop
blobh, blobw = by1 - by0, bx1 - bx0
rr, dd = targetwcs.pixelxy2radec([bx0, bx0, bx1, bx1],
[by0, by1, by1, by0])
alphas = [0.1, 0.3, 1.0]
subtims = []
for itim, tim in enumerate(tims):
h, w = tim.shape
ok, x, y = tim.subwcs.radec2pixelxy(rr, dd)
sx0, sx1 = x.min(), x.max()
sy0, sy1 = y.min(), y.max()
if sx1 < 0 or sy1 < 0 or sx1 > w or sy1 > h:
continue
sx0 = np.clip(int(np.floor(sx0)), 0, w - 1)
sx1 = np.clip(int(np.ceil(sx1)), 0, w - 1) + 1
sy0 = np.clip(int(np.floor(sy0)), 0, h - 1)
sy1 = np.clip(int(np.ceil(sy1)), 0, h - 1) + 1
subslc = slice(sy0, sy1), slice(sx0, sx1)
subimg = tim.getImage()[subslc]
subie = tim.getInvError()[subslc]
subwcs = tim.getWcs().copy()
ox0, oy0 = orig_wcsxy0[itim]
subwcs.setX0Y0(ox0 + sx0, oy0 + sy0)
# Mask out inverr for pixels that are not within the blob.
subtarget = targetwcs.get_subimage(bx0, by0, blobw, blobh)
subsubwcs = tim.subwcs.get_subimage(int(sx0), int(sy0),
int(sx1 - sx0), int(sy1 - sy0))
try:
Yo, Xo, Yi, Xi, rims = resample_with_wcs(
subsubwcs, subtarget, [], 2)
except OverlapError:
print('No overlap')
continue
if len(Yo) == 0:
continue
subie2 = np.zeros_like(subie)
I = np.flatnonzero(blobs[bslc][Yi, Xi] == (iblob + 1))
subie2[Yo[I], Xo[I]] = subie[Yo[I], Xo[I]]
subie = subie2
# If the subimage (blob) is small enough, instantiate a
# constant PSF model in the center.
if sy1 - sy0 < 100 and sx1 - sx0 < 100:
subpsf = tim.psf.mogAt(ox0 + (sx0 + sx1) / 2.,
oy0 + (sy0 + sy1) / 2.)
else:
# Otherwise, instantiate a (shifted) spatially-varying
# PsfEx model.
subpsf = ShiftedPsf(tim.psf, ox0 + sx0, oy0 + sy0)
subtim = Image(data=subimg,
inverr=subie,
wcs=subwcs,
psf=subpsf,
photocal=tim.getPhotoCal(),
sky=tim.getSky(),
name=tim.name)
subtim.band = tim.band
subtim.sig1 = tim.sig1
subtim.modelMinval = tim.modelMinval
subtims.append(subtim)
subcat = Catalog(*[cat[i] for i in Isrcs])
subtr = Tractor(subtims, subcat)
subtr.freezeParam('images')
# Optimize individual sources in order of flux
fluxes = []
for src in subcat:
# HACK -- here we just *sum* the nanomaggies in each band. Bogus!
br = src.getBrightness()
flux = sum([br.getFlux(band) for band in bands])
fluxes.append(flux)
Ibright = np.argsort(-np.array(fluxes))
if len(Ibright) >= 5:
# -Remember the original subtim images
# -Compute initial models for each source (in each tim)
# -Subtract initial models from images
# -During fitting, for each source:
# -add back in the source's initial model (to each tim)
# -fit, with Catalog([src])
# -subtract final model (from each tim)
# -Replace original subtim images
#
# --Might want to omit newly-added detection-filter sources, since their
# fluxes are bogus.
# Remember original tim images
orig_timages = [tim.getImage().copy() for tim in subtims]
initial_models = []
# Create initial models for each tim x each source
for tim in subtims:
mods = []
for src in subcat:
mod = src.getModelPatch(tim)
mods.append(mod)
if mod is not None:
if not np.all(np.isfinite(mod.patch)):
print('Non-finite mod patch')
print('source:', src)
print('tim:', tim)
print('PSF:', tim.getPsf())
assert (np.all(np.isfinite(mod.patch)))
mod.addTo(tim.getImage(), scale=-1)
initial_models.append(mods)
# For sources in decreasing order of brightness
for numi, i in enumerate(Ibright):
tsrc = Time()
print('Fitting source', i,
'(%i of %i in blob)' % (numi, len(Ibright)))
src = subcat[i]
print(src)
srctractor = Tractor(subtims, [src])
srctractor.freezeParams('images')
# Add this source's initial model back in.
for tim, mods in zip(subtims, initial_models):
mod = mods[i]
if mod is not None:
mod.addTo(tim.getImage())
print('Optimizing:', srctractor)
srctractor.printThawedParams()
for step in range(50):
dlnp, X, alpha = srctractor.optimize(priors=False,
shared_params=False,
alphas=alphas)
print('dlnp:', dlnp, 'src', src)
if dlnp < 0.1:
break
for tim in subtims:
mod = src.getModelPatch(tim)
if mod is not None:
mod.addTo(tim.getImage(), scale=-1)
for tim, img in zip(subtims, orig_timages):
tim.data = img
del orig_timages
del initial_models
else:
# Fit sources one at a time, but don't subtract other models
subcat.freezeAllParams()
for numi, i in enumerate(Ibright):
tsrc = Time()
print('Fitting source', i,
'(%i of %i in blob)' % (numi, len(Ibright)))
print(subcat[i])
subcat.freezeAllBut(i)
print('Optimizing:', subtr)
subtr.printThawedParams()
for step in range(10):
dlnp, X, alpha = subtr.optimize(priors=False,
shared_params=False,
alphas=alphas)
print('dlnp:', dlnp)
if dlnp < 0.1:
break
print('Fitting source took', Time() - tsrc)
print(subcat[i])
if len(Isrcs) > 1 and len(Isrcs) <= 10:
tfit = Time()
# Optimize all at once?
subcat.thawAllParams()
print('Optimizing:', subtr)
subtr.printThawedParams()
for step in range(20):
dlnp, X, alpha = subtr.optimize(priors=False,
shared_params=False,
alphas=alphas)
print('dlnp:', dlnp)
if dlnp < 0.1:
break
# Variances
subcat.thawAllRecursive()
subcat.freezeAllParams()
for isub, srci in enumerate(Isrcs):
print('Variances for source', srci)
subcat.thawParam(isub)
src = subcat[isub]
print('Source', src)
print('Params:', src.getParamNames())
if isinstance(src, (DevGalaxy, ExpGalaxy)):
src.shape = EllipseE.fromEllipseESoft(src.shape)
elif isinstance(src, FixedCompositeGalaxy):
src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp)
src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev)
print('Converted ellipse:', src)
allderivs = subtr.getDerivs()
for iparam, derivs in enumerate(allderivs):
dchisq = 0
for deriv, tim in derivs:
h, w = tim.shape
deriv.clipTo(w, h)
ie = tim.getInvError()
slc = deriv.getSlice(ie)
chi = deriv.patch * ie[slc]
dchisq += (chi**2).sum()
if dchisq == 0.:
v = np.nan
else:
v = 1. / dchisq
srcvariances[srci].append(v)
assert (len(srcvariances[srci]) == subcat[isub].numberOfParams())
subcat.freezeParam(isub)
cat.thawAllRecursive()
for i, src in enumerate(cat):
print('Source', i, src)
print('variances:', srcvariances[i])
print(len(srcvariances[i]), 'vs', src.numberOfParams())
if len(srcvariances[i]) != src.numberOfParams():
# This can happen for sources outside the brick bounds: they never get optimized?
print('Warning: zeroing variances for source', src)
srcvariances[i] = [0] * src.numberOfParams()
if isinstance(src, (DevGalaxy, ExpGalaxy)):
src.shape = EllipseE.fromEllipseESoft(src.shape)
elif isinstance(src, FixedCompositeGalaxy):
src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp)
src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev)
assert (len(srcvariances[i]) == src.numberOfParams())
variances = np.hstack(srcvariances)
assert (len(variances) == cat.numberOfParams())
return dict(cat=cat, variances=variances)
def stage_tune(tims=None,
cat=None,
targetwcs=None,
coimgs=None,
cons=None,
bands=None,
invvars=None,
brickid=None,
Tcat=None,
version_header=None,
ps=None,
**kwargs):
tstage = t0 = Time()
print('kwargs:', kwargs.keys())
#print 'invvars:', invvars
# How far down to render model profiles
minsigma = 0.1
for tim in tims:
tim.modelMinval = minsigma * tim.sig1
# Caching PSF
for tim in tims:
from tractor.psfex import CachingPsfEx
tim.psfex.radius = 20
tim.psfex.fitSavedData(*tim.psfex.splinedata)
tim.psf = CachingPsfEx.fromPsfEx(tim.psfex)
orig_wcsxy0 = [tim.wcs.getX0Y0() for tim in tims]
set_source_radii(bands, orig_wcsxy0, tims, cat, minsigma)
plt.figure(figsize=(10, 10))
plt.subplots_adjust(left=0.002, right=0.998, bottom=0.002, top=0.998)
plt.clf()
rgb = get_rgb(coimgs, bands)
dimshow(rgb)
#plt.title('Image')
ps.savefig()
tmpfn = create_temp(suffix='.png')
plt.imsave(tmpfn, rgb)
del rgb
cmd = 'pngtopnm %s | pnmtojpeg -quality 90 > tunebrick/coadd/image-%06i-full.jpg' % (
tmpfn, brickid)
os.system(cmd)
os.unlink(tmpfn)
pla = dict(ms=5, mew=1)
ax = plt.axis()
for i, src in enumerate(cat):
rd = src.getPosition()
ok, x, y = targetwcs.radec2pixelxy(rd.ra, rd.dec)
cc = (0, 1, 0)
if isinstance(src, PointSource):
plt.plot(x - 1, y - 1, '+', color=cc, **pla)
else:
plt.plot(x - 1, y - 1, 'o', mec=cc, mfc='none', **pla)
# plt.text(x, y, '%i' % i, color=cc, ha='center', va='bottom')
plt.axis(ax)
ps.savefig()
print('Plots:', Time() - t0)
# print 'Catalog:'
# for src in cat:
# print ' ', src
# switch_to_soft_ellipses(cat)
assert (Catalog(*cat).numberOfParams() == len(invvars))
keepcat = []
keepinvvars = []
iterinvvars = invvars
ikeep = []
for i, src in enumerate(cat):
N = src.numberOfParams()
iv = iterinvvars[:N]
iterinvvars = iterinvvars[N:]
if not np.all(np.isfinite(src.getParams())):
print('Dropping source:', src)
continue
keepcat.append(src)
keepinvvars.extend(iv)
#print 'Keep:', src
#print 'iv:', iv
#print 'sigma', 1./np.sqrt(np.array(iv))
ikeep.append(i)
cat = keepcat
Tcat.cut(np.array(ikeep))
invvars = keepinvvars
print(len(cat), 'sources with finite params')
assert (Catalog(*cat).numberOfParams() == len(invvars))
assert (len(iterinvvars) == 0)
print('Rendering model images...')
t0 = Time()
mods = _map(_get_mod, [(tim, cat) for tim in tims])
print('Getting model images:', Time() - t0)
wcsW = targetwcs.get_width()
wcsH = targetwcs.get_height()
t0 = Time()
comods = []
for iband, band in enumerate(bands):
comod = np.zeros((wcsH, wcsW), np.float32)
for itim, (tim, mod) in enumerate(zip(tims, mods)):
if tim.band != band:
continue
R = tim_get_resamp(tim, targetwcs)
if R is None:
continue
(Yo, Xo, Yi, Xi) = R
comod[Yo, Xo] += mod[Yi, Xi]
comod /= np.maximum(cons[iband], 1)
comods.append(comod)
print('Creating model coadd:', Time() - t0)
plt.clf()
dimshow(get_rgb(comods, bands))
plt.title('Model')
ps.savefig()
del comods
t0 = Time()
keepinvvars = []
keepcat = []
iterinvvars = invvars
ikeep = []
for isrc, src in enumerate(cat):
newiv = None
N = src.numberOfParams()
gc = get_galaxy_cache()
print('Galaxy cache:', gc)
if gc is not None:
gc.clear()
print('Checking source', isrc, 'of', len(cat), ':', src)
#print 'N params:', N
#print 'iterinvvars:', len(iterinvvars)
oldiv = iterinvvars[:N]
iterinvvars = iterinvvars[N:]
recompute_iv = False
if isinstance(src, FixedCompositeGalaxy):
# Obvious simplification: for composite galaxies with fracdev
# out of bounds, convert to exp or dev.
f = src.fracDev.getClippedValue()
if f == 0.:
oldsrc = src
src = ExpGalaxy(oldsrc.pos, oldsrc.brightness, oldsrc.shapeExp)
print('Converted comp to exp')
#print ' ', oldsrc
#print ' ->', src
# pull out the invvar elements!
pp = src.getParams()
oldsrc.setParams(oldiv)
newiv = oldsrc.pos.getParams() + oldsrc.brightness.getParams(
) + oldsrc.shapeExp.getParams()
src.setParams(pp)
elif f == 1.:
oldsrc = src
src = DevGalaxy(oldsrc.pos, oldsrc.brightness, oldsrc.shapeDev)
print('Converted comp to dev')
##print ' ', oldsrc
print(' ->', src)
pp = src.getParams()
oldsrc.setParams(oldiv)
newiv = oldsrc.pos.getParams() + oldsrc.brightness.getParams(
) + oldsrc.shapeDev.getParams()
src.setParams(pp)
# treated_as_pointsource: do the bright-star check at least!
if not isinstance(src, PointSource):
# This is the check we use in unWISE
if src.getBrightness().getMag('r') < 12.5:
oldsrc = src
src = PointSource(oldsrc.pos, oldsrc.brightness)
print('Bright star: replacing', oldsrc)
print('With', src)
# Not QUITE right.
#oldsrc.setParams(oldiv)
#newiv = oldsrc.pos.getParams() + oldsrc.brightness.getParams()
recompute_iv = True
#print 'Try removing source:', src
tsrc = Time()
srcmodlist = []
for itim, tim in enumerate(tims):
patch = src.getModelPatch(tim)
if patch is None:
continue
if patch.patch is None:
continue
# HACK -- this shouldn't be necessary, but seems to be!
# FIXME -- track down why patches are being made with extent outside
# that of the parent!
H, W = tim.shape
if patch.x0 < 0 or patch.y0 < 0 or patch.x1 > W or patch.y1 > H:
print('Warning: Patch extends outside tim bounds:')
print('patch extent:', patch.getExtent())
print('image size:', W, 'x', H)
patch.clipTo(W, H)
ph, pw = patch.shape
if pw * ph == 0:
continue
srcmodlist.append((itim, patch))
# Try removing the source from the model;
# check chi-squared change in the patches.
sdlnp = 0.
for itim, patch in srcmodlist:
tim = tims[itim]
mod = mods[itim]
slc = patch.getSlice(tim)
simg = tim.getImage()[slc]
sie = tim.getInvError()[slc]
chisq0 = np.sum(((simg - mod[slc]) * sie)**2)
chisq1 = np.sum(((simg - (mod[slc] - patch.patch)) * sie)**2)
sdlnp += -0.5 * (chisq1 - chisq0)
print('Removing source: dlnp =', sdlnp)
print('Testing source removal:', Time() - tsrc)
if sdlnp > 0:
#print 'Removing source!'
for itim, patch in srcmodlist:
patch.addTo(mods[itim], scale=-1)
continue
# Try some model changes...
newsrcs = []
if isinstance(src, FixedCompositeGalaxy):
newsrcs.append(ExpGalaxy(src.pos, src.brightness, src.shapeExp))
newsrcs.append(DevGalaxy(src.pos, src.brightness, src.shapeDev))
newsrcs.append(PointSource(src.pos, src.brightness))
elif isinstance(src, (DevGalaxy, ExpGalaxy)):
newsrcs.append(PointSource(src.pos, src.brightness))
bestnew = None
bestdlnp = 0.
bestdpatches = None
srcmodlist2 = [None for tim in tims]
for itim, patch in srcmodlist:
srcmodlist2[itim] = patch
for newsrc in newsrcs:
dpatches = []
dlnp = 0.
for itim, tim in enumerate(tims):
patch = newsrc.getModelPatch(tim)
if patch is not None:
if patch.patch is None:
patch = None
if patch is not None:
# HACK -- this shouldn't be necessary, but seems to be!
# FIXME -- track down why patches are being made with extent outside
# that of the parent!
H, W = tim.shape
patch.clipTo(W, H)
ph, pw = patch.shape
if pw * ph == 0:
patch = None
oldpatch = srcmodlist2[itim]
if oldpatch is None and patch is None:
continue
# Find difference in models
if oldpatch is None:
dpatch = patch
elif patch is None:
dpatch = oldpatch * -1.
else:
dpatch = patch - oldpatch
dpatches.append((itim, dpatch))
mod = mods[itim]
slc = dpatch.getSlice(tim)
simg = tim.getImage()[slc]
sie = tim.getInvError()[slc]
chisq0 = np.sum(((simg - mod[slc]) * sie)**2)
chisq1 = np.sum(((simg - (mod[slc] + dpatch.patch)) * sie)**2)
dlnp += -0.5 * (chisq1 - chisq0)
#print 'Trying source change:'
#print 'from', src
#print ' to', newsrc
print('Trying source change to',
type(newsrc).__name__, ': dlnp =', dlnp)
if dlnp >= bestdlnp:
bestnew = newsrc
bestdlnp = dlnp
bestdpatches = dpatches
if bestnew is not None:
print('Found model improvement! Switching to', end=' ')
print(bestnew)
for itim, dpatch in bestdpatches:
dpatch.addTo(mods[itim])
src = bestnew
recompute_iv = True
del srcmodlist
del srcmodlist2
if recompute_iv:
dchisqs = np.zeros(src.numberOfParams())
for tim in tims:
derivs = src.getParamDerivatives(tim)
h, w = tim.shape
ie = tim.getInvError()
for i, deriv in enumerate(derivs):
if deriv is None:
continue
deriv.clipTo(w, h)
slc = deriv.getSlice(ie)
chi = deriv.patch * ie[slc]
dchisqs[i] += (chi**2).sum()
newiv = dchisqs
if newiv is None:
keepinvvars.append(oldiv)
else:
keepinvvars.append(newiv)
keepcat.append(src)
ikeep.append(isrc)
cat = keepcat
Tcat.cut(np.array(ikeep))
gc = get_galaxy_cache()
print('Galaxy cache:', gc)
if gc is not None:
gc.clear()
assert (len(iterinvvars) == 0)
keepinvvars = np.hstack(keepinvvars)
assert (Catalog(*keepcat).numberOfParams() == len(keepinvvars))
invvars = keepinvvars
assert (len(cat) == len(Tcat))
print('Model selection:', Time() - t0)
t0 = Time()
# WCS header for these images
hdr = fitsio.FITSHDR()
targetwcs.add_to_header(hdr)
fwa = dict(clobber=True, header=hdr)
comods = []
for iband, band in enumerate(bands):
comod = np.zeros((wcsH, wcsW), np.float32)
cochi2 = np.zeros((wcsH, wcsW), np.float32)
coiv = np.zeros((wcsH, wcsW), np.float32)
detiv = np.zeros((wcsH, wcsW), np.float32)
for itim, (tim, mod) in enumerate(zip(tims, mods)):
if tim.band != band:
continue
R = tim_get_resamp(tim, targetwcs)
if R is None:
continue
(Yo, Xo, Yi, Xi) = R
comod[Yo, Xo] += mod[Yi, Xi]
ie = tim.getInvError()
cochi2[Yo, Xo] += ((tim.getImage()[Yi, Xi] - mod[Yi, Xi]) *
ie[Yi, Xi])**2
coiv[Yo, Xo] += ie[Yi, Xi]**2
psfnorm = 1. / (2. * np.sqrt(np.pi) * tim.psf_sigma)
detsig1 = tim.sig1 / psfnorm
detiv[Yo, Xo] += (ie[Yi, Xi] > 0) * (1. / detsig1**2)
comod /= np.maximum(cons[iband], 1)
comods.append(comod)
del comod
fn = 'tunebrick/coadd/chi2-%06i-%s.fits' % (brickid, band)
fitsio.write(fn, cochi2, **fwa)
del cochi2
print('Wrote', fn)
fn = 'tunebrick/coadd/image-%06i-%s.fits' % (brickid, band)
fitsio.write(fn, coimgs[iband], **fwa)
print('Wrote', fn)
fitsio.write(fn, coiv, clobber=False)
print('Appended ivar to', fn)
del coiv
fn = 'tunebrick/coadd/depth-%06i-%s.fits' % (brickid, band)
fitsio.write(fn, detiv, **fwa)
print('Wrote', fn)
del detiv
fn = 'tunebrick/coadd/model-%06i-%s.fits' % (brickid, band)
fitsio.write(fn, comods[iband], **fwa)
print('Wrote', fn)
fn = 'tunebrick/coadd/nexp-b%06i-%s.fits' % (brickid, band)
fitsio.write(fn, cons[iband], **fwa)
print('Wrote', fn)
plt.clf()
rgb = get_rgb(comods, bands)
dimshow(rgb)
plt.title('Model')
ps.savefig()
del comods
# Plot sources over top
ax = plt.axis()
for i, src in enumerate(cat):
rd = src.getPosition()
ok, x, y = targetwcs.radec2pixelxy(rd.ra, rd.dec)
cc = (0, 1, 0)
if isinstance(src, PointSource):
plt.plot(x - 1, y - 1, '+', color=cc, **pla)
else:
plt.plot(x - 1, y - 1, 'o', mec=cc, mfc='none', **pla)
# plt.text(x, y, '%i' % i, color=cc, ha='center', va='bottom')
plt.axis(ax)
ps.savefig()
tmpfn = create_temp(suffix='.png')
plt.imsave(tmpfn, rgb)
del rgb
cmd = 'pngtopnm %s | pnmtojpeg -quality 90 > tunebrick/coadd/model-%06i-full.jpg' % (
tmpfn, brickid)
os.system(cmd)
os.unlink(tmpfn)
assert (len(cat) == len(Tcat))
print('Coadd FITS files and plots:', Time() - t0)
print('Whole stage:', Time() - tstage)
return dict(cat=cat, Tcat=Tcat, invvars=invvars)
def stage_tune(tims=None, cat=None, targetwcs=None, coimgs=None, cons=None,
bands=None, invvars=None, brickid=None,
Tcat=None, version_header=None, ps=None, **kwargs):
tstage = t0 = Time()
print 'kwargs:', kwargs.keys()
#print 'invvars:', invvars
# How far down to render model profiles
minsigma = 0.1
for tim in tims:
tim.modelMinval = minsigma * tim.sig1
# Caching PSF
for tim in tims:
from tractor.psfex import CachingPsfEx
tim.psfex.radius = 20
tim.psfex.fitSavedData(*tim.psfex.splinedata)
tim.psf = CachingPsfEx.fromPsfEx(tim.psfex)
orig_wcsxy0 = [tim.wcs.getX0Y0() for tim in tims]
set_source_radii(bands, orig_wcsxy0, tims, cat, minsigma)
plt.figure(figsize=(10,10))
plt.subplots_adjust(left=0.002, right=0.998, bottom=0.002, top=0.998)
plt.clf()
rgb = get_rgb(coimgs, bands)
dimshow(rgb)
#plt.title('Image')
ps.savefig()
tmpfn = create_temp(suffix='.png')
plt.imsave(tmpfn, rgb)
del rgb
cmd = 'pngtopnm %s | pnmtojpeg -quality 90 > tunebrick/coadd/image-%06i-full.jpg' % (tmpfn, brickid)
os.system(cmd)
os.unlink(tmpfn)
pla = dict(ms=5, mew=1)
ax = plt.axis()
for i,src in enumerate(cat):
rd = src.getPosition()
ok,x,y = targetwcs.radec2pixelxy(rd.ra, rd.dec)
cc = (0,1,0)
if isinstance(src, PointSource):
plt.plot(x-1, y-1, '+', color=cc, **pla)
else:
plt.plot(x-1, y-1, 'o', mec=cc, mfc='none', **pla)
# plt.text(x, y, '%i' % i, color=cc, ha='center', va='bottom')
plt.axis(ax)
ps.savefig()
print 'Plots:', Time()-t0
# print 'Catalog:'
# for src in cat:
# print ' ', src
# switch_to_soft_ellipses(cat)
assert(Catalog(*cat).numberOfParams() == len(invvars))
keepcat = []
keepinvvars = []
iterinvvars = invvars
ikeep = []
for i,src in enumerate(cat):
N = src.numberOfParams()
iv = iterinvvars[:N]
iterinvvars = iterinvvars[N:]
if not np.all(np.isfinite(src.getParams())):
print 'Dropping source:', src
continue
keepcat.append(src)
keepinvvars.extend(iv)
#print 'Keep:', src
#print 'iv:', iv
#print 'sigma', 1./np.sqrt(np.array(iv))
ikeep.append(i)
cat = keepcat
Tcat.cut(np.array(ikeep))
invvars = keepinvvars
print len(cat), 'sources with finite params'
assert(Catalog(*cat).numberOfParams() == len(invvars))
assert(len(iterinvvars) == 0)
print 'Rendering model images...'
t0 = Time()
mods = _map(_get_mod, [(tim, cat) for tim in tims])
print 'Getting model images:', Time()-t0
wcsW = targetwcs.get_width()
wcsH = targetwcs.get_height()
t0 = Time()
comods = []
for iband,band in enumerate(bands):
comod = np.zeros((wcsH,wcsW), np.float32)
for itim, (tim,mod) in enumerate(zip(tims, mods)):
if tim.band != band:
continue
R = tim_get_resamp(tim, targetwcs)
if R is None:
continue
(Yo,Xo,Yi,Xi) = R
comod[Yo,Xo] += mod[Yi,Xi]
comod /= np.maximum(cons[iband], 1)
comods.append(comod)
print 'Creating model coadd:', Time()-t0
plt.clf()
dimshow(get_rgb(comods, bands))
plt.title('Model')
ps.savefig()
del comods
t0 = Time()
keepinvvars = []
keepcat = []
iterinvvars = invvars
ikeep = []
for isrc,src in enumerate(cat):
newiv = None
N = src.numberOfParams()
gc = get_galaxy_cache()
print 'Galaxy cache:', gc
if gc is not None:
gc.clear()
print 'Checking source', isrc, 'of', len(cat), ':', src
#print 'N params:', N
#print 'iterinvvars:', len(iterinvvars)
oldiv = iterinvvars[:N]
iterinvvars = iterinvvars[N:]
recompute_iv = False
if isinstance(src, FixedCompositeGalaxy):
# Obvious simplification: for composite galaxies with fracdev
# out of bounds, convert to exp or dev.
f = src.fracDev.getClippedValue()
if f == 0.:
oldsrc = src
src = ExpGalaxy(oldsrc.pos, oldsrc.brightness, oldsrc.shapeExp)
print 'Converted comp to exp'
#print ' ', oldsrc
#print ' ->', src
# pull out the invvar elements!
pp = src.getParams()
oldsrc.setParams(oldiv)
newiv = oldsrc.pos.getParams() + oldsrc.brightness.getParams() + oldsrc.shapeExp.getParams()
src.setParams(pp)
elif f == 1.:
oldsrc = src
src = DevGalaxy(oldsrc.pos, oldsrc.brightness, oldsrc.shapeDev)
print 'Converted comp to dev'
##print ' ', oldsrc
print ' ->', src
pp = src.getParams()
oldsrc.setParams(oldiv)
newiv = oldsrc.pos.getParams() + oldsrc.brightness.getParams() + oldsrc.shapeDev.getParams()
src.setParams(pp)
# treated_as_pointsource: do the bright-star check at least!
if not isinstance(src, PointSource):
# This is the check we use in unWISE
if src.getBrightness().getMag('r') < 12.5:
oldsrc = src
src = PointSource(oldsrc.pos, oldsrc.brightness)
print 'Bright star: replacing', oldsrc
print 'With', src
# Not QUITE right.
#oldsrc.setParams(oldiv)
#newiv = oldsrc.pos.getParams() + oldsrc.brightness.getParams()
recompute_iv = True
#print 'Try removing source:', src
tsrc = Time()
srcmodlist = []
for itim,tim in enumerate(tims):
patch = src.getModelPatch(tim)
if patch is None:
continue
if patch.patch is None:
continue
# HACK -- this shouldn't be necessary, but seems to be!
# FIXME -- track down why patches are being made with extent outside
# that of the parent!
H,W = tim.shape
if patch.x0 < 0 or patch.y0 < 0 or patch.x1 > W or patch.y1 > H:
print 'Warning: Patch extends outside tim bounds:'
print 'patch extent:', patch.getExtent()
print 'image size:', W, 'x', H
patch.clipTo(W,H)
ph,pw = patch.shape
if pw*ph == 0:
continue
srcmodlist.append((itim, patch))
# Try removing the source from the model;
# check chi-squared change in the patches.
sdlnp = 0.
for itim,patch in srcmodlist:
tim = tims[itim]
mod = mods[itim]
slc = patch.getSlice(tim)
simg = tim.getImage()[slc]
sie = tim.getInvError()[slc]
chisq0 = np.sum(((simg - mod[slc]) * sie)**2)
chisq1 = np.sum(((simg - (mod[slc] - patch.patch)) * sie)**2)
sdlnp += -0.5 * (chisq1 - chisq0)
print 'Removing source: dlnp =', sdlnp
print 'Testing source removal:', Time()-tsrc
if sdlnp > 0:
#print 'Removing source!'
for itim,patch in srcmodlist:
patch.addTo(mods[itim], scale=-1)
continue
# Try some model changes...
newsrcs = []
if isinstance(src, FixedCompositeGalaxy):
newsrcs.append(ExpGalaxy(src.pos, src.brightness, src.shapeExp))
newsrcs.append(DevGalaxy(src.pos, src.brightness, src.shapeDev))
newsrcs.append(PointSource(src.pos, src.brightness))
elif isinstance(src, (DevGalaxy, ExpGalaxy)):
newsrcs.append(PointSource(src.pos, src.brightness))
bestnew = None
bestdlnp = 0.
bestdpatches = None
srcmodlist2 = [None for tim in tims]
for itim,patch in srcmodlist:
srcmodlist2[itim] = patch
for newsrc in newsrcs:
dpatches = []
dlnp = 0.
for itim,tim in enumerate(tims):
patch = newsrc.getModelPatch(tim)
if patch is not None:
if patch.patch is None:
patch = None
if patch is not None:
# HACK -- this shouldn't be necessary, but seems to be!
# FIXME -- track down why patches are being made with extent outside
# that of the parent!
H,W = tim.shape
patch.clipTo(W,H)
ph,pw = patch.shape
if pw*ph == 0:
patch = None
oldpatch = srcmodlist2[itim]
if oldpatch is None and patch is None:
continue
# Find difference in models
if oldpatch is None:
dpatch = patch
elif patch is None:
dpatch = oldpatch * -1.
else:
dpatch = patch - oldpatch
dpatches.append((itim, dpatch))
mod = mods[itim]
slc = dpatch.getSlice(tim)
simg = tim.getImage()[slc]
sie = tim.getInvError()[slc]
chisq0 = np.sum(((simg - mod[slc]) * sie)**2)
chisq1 = np.sum(((simg - (mod[slc] + dpatch.patch)) * sie)**2)
dlnp += -0.5 * (chisq1 - chisq0)
#print 'Trying source change:'
#print 'from', src
#print ' to', newsrc
print 'Trying source change to', type(newsrc).__name__, ': dlnp =', dlnp
if dlnp >= bestdlnp:
bestnew = newsrc
bestdlnp = dlnp
bestdpatches = dpatches
if bestnew is not None:
print 'Found model improvement! Switching to',
print bestnew
for itim,dpatch in bestdpatches:
dpatch.addTo(mods[itim])
src = bestnew
recompute_iv = True
del srcmodlist
del srcmodlist2
if recompute_iv:
dchisqs = np.zeros(src.numberOfParams())
for tim in tims:
derivs = src.getParamDerivatives(tim)
h,w = tim.shape
ie = tim.getInvError()
for i,deriv in enumerate(derivs):
if deriv is None:
continue
deriv.clipTo(w,h)
slc = deriv.getSlice(ie)
chi = deriv.patch * ie[slc]
dchisqs[i] += (chi**2).sum()
newiv = dchisqs
if newiv is None:
keepinvvars.append(oldiv)
else:
keepinvvars.append(newiv)
keepcat.append(src)
ikeep.append(isrc)
cat = keepcat
Tcat.cut(np.array(ikeep))
gc = get_galaxy_cache()
print 'Galaxy cache:', gc
if gc is not None:
gc.clear()
assert(len(iterinvvars) == 0)
keepinvvars = np.hstack(keepinvvars)
assert(Catalog(*keepcat).numberOfParams() == len(keepinvvars))
invvars = keepinvvars
assert(len(cat) == len(Tcat))
print 'Model selection:', Time()-t0
t0 = Time()
# WCS header for these images
hdr = fitsio.FITSHDR()
targetwcs.add_to_header(hdr)
fwa = dict(clobber=True, header=hdr)
comods = []
for iband,band in enumerate(bands):
comod = np.zeros((wcsH,wcsW), np.float32)
cochi2 = np.zeros((wcsH,wcsW), np.float32)
coiv = np.zeros((wcsH,wcsW), np.float32)
detiv = np.zeros((wcsH,wcsW), np.float32)
for itim, (tim,mod) in enumerate(zip(tims, mods)):
if tim.band != band:
continue
R = tim_get_resamp(tim, targetwcs)
if R is None:
continue
(Yo,Xo,Yi,Xi) = R
comod[Yo,Xo] += mod[Yi,Xi]
ie = tim.getInvError()
cochi2[Yo,Xo] += ((tim.getImage()[Yi,Xi] - mod[Yi,Xi]) * ie[Yi,Xi])**2
coiv[Yo,Xo] += ie[Yi,Xi]**2
psfnorm = 1./(2. * np.sqrt(np.pi) * tim.psf_sigma)
detsig1 = tim.sig1 / psfnorm
detiv[Yo,Xo] += (ie[Yi,Xi] > 0) * (1. / detsig1**2)
comod /= np.maximum(cons[iband], 1)
comods.append(comod)
del comod
fn = 'tunebrick/coadd/chi2-%06i-%s.fits' % (brickid, band)
fitsio.write(fn, cochi2, **fwa)
del cochi2
print 'Wrote', fn
fn = 'tunebrick/coadd/image-%06i-%s.fits' % (brickid, band)
fitsio.write(fn, coimgs[iband], **fwa)
print 'Wrote', fn
fitsio.write(fn, coiv, clobber=False)
print 'Appended ivar to', fn
del coiv
fn = 'tunebrick/coadd/depth-%06i-%s.fits' % (brickid, band)
fitsio.write(fn, detiv, **fwa)
print 'Wrote', fn
del detiv
fn = 'tunebrick/coadd/model-%06i-%s.fits' % (brickid, band)
fitsio.write(fn, comods[iband], **fwa)
print 'Wrote', fn
fn = 'tunebrick/coadd/nexp-b%06i-%s.fits' % (brickid, band)
fitsio.write(fn, cons[iband], **fwa)
print 'Wrote', fn
plt.clf()
rgb = get_rgb(comods, bands)
dimshow(rgb)
plt.title('Model')
ps.savefig()
del comods
# Plot sources over top
ax = plt.axis()
for i,src in enumerate(cat):
rd = src.getPosition()
ok,x,y = targetwcs.radec2pixelxy(rd.ra, rd.dec)
cc = (0,1,0)
if isinstance(src, PointSource):
plt.plot(x-1, y-1, '+', color=cc, **pla)
else:
plt.plot(x-1, y-1, 'o', mec=cc, mfc='none', **pla)
# plt.text(x, y, '%i' % i, color=cc, ha='center', va='bottom')
plt.axis(ax)
ps.savefig()
tmpfn = create_temp(suffix='.png')
plt.imsave(tmpfn, rgb)
del rgb
cmd = 'pngtopnm %s | pnmtojpeg -quality 90 > tunebrick/coadd/model-%06i-full.jpg' % (tmpfn, brickid)
os.system(cmd)
os.unlink(tmpfn)
assert(len(cat) == len(Tcat))
print 'Coadd FITS files and plots:', Time()-t0
print 'Whole stage:', Time()-tstage
return dict(cat=cat, Tcat=Tcat, invvars=invvars)
def stage0(**kwargs):
ps = PlotSequence('cfht')
decals = CfhtDecals()
B = decals.get_bricks()
print 'Bricks:'
B.about()
ra,dec = 190.0, 11.0
#bands = 'ugri'
bands = 'gri'
B.cut(np.argsort(degrees_between(ra, dec, B.ra, B.dec)))
print 'Nearest bricks:', B.ra[:5], B.dec[:5], B.brickid[:5]
brick = B[0]
pixscale = 0.186
#W,H = 1024,1024
#W,H = 2048,2048
#W,H = 3600,3600
W,H = 4800,4800
targetwcs = wcs_for_brick(brick, pixscale=pixscale, W=W, H=H)
ccdfn = 'cfht-ccds.fits'
if os.path.exists(ccdfn):
T = fits_table(ccdfn)
else:
T = get_ccd_list()
T.writeto(ccdfn)
print len(T), 'CCDs'
T.cut(ccds_touching_wcs(targetwcs, T))
print len(T), 'CCDs touching brick'
T.cut(np.array([b in bands for b in T.filter]))
print len(T), 'in bands', bands
ims = []
for t in T:
im = CfhtImage(t)
# magzp = hdr['PHOT_C'] + 2.5 * np.log10(hdr['EXPTIME'])
# fwhm = t.seeing / (pixscale * 3600)
# print '-> FWHM', fwhm, 'pix'
im.seeing = t.seeing
im.pixscale = t.pixscale
print 'seeing', t.seeing
print 'pixscale', im.pixscale*3600, 'arcsec/pix'
im.run_calibs(t.ra, t.dec, im.pixscale, W=t.width, H=t.height)
ims.append(im)
# Read images, clip to ROI
targetrd = np.array([targetwcs.pixelxy2radec(x,y) for x,y in
[(1,1),(W,1),(W,H),(1,H),(1,1)]])
keepims = []
tims = []
for im in ims:
print
print 'Reading expnum', im.expnum, 'name', im.extname, 'band', im.band, 'exptime', im.exptime
band = im.band
wcs = im.read_wcs()
imh,imw = wcs.imageh,wcs.imagew
imgpoly = [(1,1),(1,imh),(imw,imh),(imw,1)]
ok,tx,ty = wcs.radec2pixelxy(targetrd[:-1,0], targetrd[:-1,1])
tpoly = zip(tx,ty)
clip = clip_polygon(imgpoly, tpoly)
clip = np.array(clip)
#print 'Clip', clip
if len(clip) == 0:
continue
x0,y0 = np.floor(clip.min(axis=0)).astype(int)
x1,y1 = np.ceil (clip.max(axis=0)).astype(int)
slc = slice(y0,y1+1), slice(x0,x1+1)
## FIXME -- it seems I got lucky and the cross product is
## negative == clockwise, as required by clip_polygon. One
## could check this and reverse the polygon vertex order.
# dx0,dy0 = tx[1]-tx[0], ty[1]-ty[0]
# dx1,dy1 = tx[2]-tx[1], ty[2]-ty[1]
# cross = dx0*dy1 - dx1*dy0
# print 'Cross:', cross
print 'Image slice: x [%i,%i], y [%i,%i]' % (x0,x1, y0,y1)
print 'Reading image from', im.imgfn, 'HDU', im.hdu
img,imghdr = im.read_image(header=True, slice=slc)
goodpix = (img != 0)
print 'Number of pixels == 0:', np.sum(img == 0)
print 'Number of pixels != 0:', np.sum(goodpix)
if np.sum(goodpix) == 0:
continue
# print 'Image shape', img.shape
print 'Image range', img.min(), img.max()
print 'Goodpix image range:', (img[goodpix]).min(), (img[goodpix]).max()
if img[goodpix].min() == img[goodpix].max():
print 'No dynamic range in image'
continue
# print 'Reading invvar from', im.wtfn, 'HDU', im.hdu
# invvar = im.read_invvar(slice=slc)
# # print 'Invvar shape', invvar.shape
# # print 'Invvar range:', invvar.min(), invvar.max()
# invvar[goodpix == 0] = 0.
# if np.all(invvar == 0.):
# print 'Skipping zero-invvar image'
# continue
# assert(np.all(np.isfinite(img)))
# assert(np.all(np.isfinite(invvar)))
# assert(not(np.all(invvar == 0.)))
# # Estimate per-pixel noise via Blanton's 5-pixel MAD
# slice1 = (slice(0,-5,10),slice(0,-5,10))
# slice2 = (slice(5,None,10),slice(5,None,10))
# # print 'sliced shapes:', img[slice1].shape, img[slice2].shape
# # print 'good shape:', (goodpix[slice1] * goodpix[slice2]).shape
# # print 'good values:', np.unique(goodpix[slice1] * goodpix[slice2])
# # print 'sliced[good] shapes:', (img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].shape
# mad = np.median(np.abs(img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].ravel())
# sig1 = 1.4826 * mad / np.sqrt(2.)
# print 'MAD sig1:', sig1
# # invvar was 1 or 0
# invvar *= (1./(sig1**2))
# medsky = np.median(img[goodpix])
# Read full image for sig1 and sky estimate
fullimg = im.read_image()
fullgood = (fullimg != 0)
# Estimate per-pixel noise via Blanton's 5-pixel MAD
slice1 = (slice(0,-5,10),slice(0,-5,10))
slice2 = (slice(5,None,10),slice(5,None,10))
mad = np.median(np.abs(fullimg[slice1] - fullimg[slice2])[fullgood[slice1] * fullgood[slice2]].ravel())
sig1 = 1.4826 * mad / np.sqrt(2.)
print 'MAD sig1:', sig1
medsky = np.median(fullimg[fullgood])
invvar = np.zeros_like(img)
invvar[goodpix] = 1./sig1**2
# Median-smooth sky subtraction
plt.clf()
dimshow(np.round((img-medsky) / sig1), vmin=-3, vmax=5)
plt.title('Scalar median: %s' % im.name)
ps.savefig()
# medsky = np.zeros_like(img)
# # astrometry.util.util
# median_smooth(img, np.logical_not(goodpix), 256, medsky)
fullmed = np.zeros_like(fullimg)
median_smooth(fullimg - medsky, np.logical_not(fullgood), 256, fullmed)
fullmed += medsky
medimg = fullmed[slc]
plt.clf()
dimshow(np.round((img - medimg) / sig1), vmin=-3, vmax=5)
plt.title('Median filtered: %s' % im.name)
ps.savefig()
#print 'Subtracting median:', medsky
#img -= medsky
img -= medimg
primhdr = im.read_image_primary_header()
magzp = decals.get_zeropoint_for(im)
print 'magzp', magzp
zpscale = NanoMaggies.zeropointToScale(magzp)
print 'zpscale', zpscale
# Scale images to Nanomaggies
img /= zpscale
sig1 /= zpscale
invvar *= zpscale**2
orig_zpscale = zpscale
zpscale = 1.
assert(np.sum(invvar > 0) > 0)
print 'After scaling:'
print 'sig1', sig1
print 'invvar range', invvar.min(), invvar.max()
print 'image range', img.min(), img.max()
assert(np.all(np.isfinite(img)))
assert(np.all(np.isfinite(invvar)))
assert(np.isfinite(sig1))
plt.clf()
lo,hi = -5*sig1, 10*sig1
n,b,p = plt.hist(img[goodpix].ravel(), 100, range=(lo,hi), histtype='step', color='k')
xx = np.linspace(lo, hi, 200)
plt.plot(xx, max(n)*np.exp(-xx**2 / (2.*sig1**2)), 'r-')
plt.xlim(lo,hi)
plt.title('Pixel histogram: %s' % im.name)
ps.savefig()
twcs = ConstantFitsWcs(wcs)
if x0 or y0:
twcs.setX0Y0(x0,y0)
info = im.get_image_info()
fullh,fullw = info['dims']
# read fit PsfEx model
psfex = PsfEx.fromFits(im.psffitfn)
print 'Read', psfex
# HACK -- highly approximate PSF here!
#psf_fwhm = imghdr['FWHM']
#psf_fwhm = im.seeing
psf_fwhm = im.seeing / (im.pixscale * 3600)
print 'PSF FWHM', psf_fwhm, 'pixels'
psf_sigma = psf_fwhm / 2.35
psf = NCircularGaussianPSF([psf_sigma],[1.])
print 'img type', img.dtype
tim = Image(img, invvar=invvar, wcs=twcs, psf=psf,
photocal=LinearPhotoCal(zpscale, band=band),
sky=ConstantSky(0.), name=im.name + ' ' + band)
tim.zr = [-3. * sig1, 10. * sig1]
tim.sig1 = sig1
tim.band = band
tim.psf_fwhm = psf_fwhm
tim.psf_sigma = psf_sigma
tim.sip_wcs = wcs
tim.x0,tim.y0 = int(x0),int(y0)
tim.psfex = psfex
tim.imobj = im
mn,mx = tim.zr
tim.ima = dict(interpolation='nearest', origin='lower', cmap='gray',
vmin=mn, vmax=mx)
tims.append(tim)
keepims.append(im)
ims = keepims
print 'Computing resampling...'
# save resampling params
for tim in tims:
wcs = tim.sip_wcs
subh,subw = tim.shape
subwcs = wcs.get_subimage(tim.x0, tim.y0, subw, subh)
tim.subwcs = subwcs
try:
Yo,Xo,Yi,Xi,rims = resample_with_wcs(targetwcs, subwcs, [], 2)
except OverlapError:
print 'No overlap'
continue
if len(Yo) == 0:
continue
tim.resamp = (Yo,Xo,Yi,Xi)
print 'Creating coadds...'
# Produce per-band coadds, for plots
coimgs = []
cons = []
for ib,band in enumerate(bands):
coimg = np.zeros((H,W), np.float32)
con = np.zeros((H,W), np.uint8)
for tim in tims:
if tim.band != band:
continue
(Yo,Xo,Yi,Xi) = tim.resamp
if len(Yo) == 0:
continue
nn = (tim.getInvvar()[Yi,Xi] > 0)
coimg[Yo,Xo] += tim.getImage ()[Yi,Xi] * nn
con [Yo,Xo] += nn
# print
# print 'tim', tim.name
# print 'number of resampled pix:', len(Yo)
# reim = np.zeros_like(coimg)
# ren = np.zeros_like(coimg)
# reim[Yo,Xo] = tim.getImage()[Yi,Xi] * nn
# ren[Yo,Xo] = nn
# print 'number of resampled pix with positive invvar:', ren.sum()
# plt.clf()
# plt.subplot(2,2,1)
# mn,mx = [np.percentile(reim[ren>0], p) for p in [25,95]]
# print 'Percentiles:', mn,mx
# dimshow(reim, vmin=mn, vmax=mx)
# plt.colorbar()
# plt.subplot(2,2,2)
# dimshow(con)
# plt.colorbar()
# plt.subplot(2,2,3)
# dimshow(reim, vmin=tim.zr[0], vmax=tim.zr[1])
# plt.colorbar()
# plt.subplot(2,2,4)
# plt.hist(reim.ravel(), 100, histtype='step', color='b')
# plt.hist(tim.getImage().ravel(), 100, histtype='step', color='r')
# plt.suptitle('%s: %s' % (band, tim.name))
# ps.savefig()
coimg /= np.maximum(con,1)
coimgs.append(coimg)
cons .append(con)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ps.savefig()
plt.clf()
for i,b in enumerate(bands):
plt.subplot(2,2,i+1)
dimshow(cons[i], ticks=False)
plt.title('%s band' % b)
plt.colorbar()
plt.suptitle('Number of exposures')
ps.savefig()
print 'Grabbing SDSS sources...'
bandlist = [b for b in bands]
cat,T = get_sdss_sources(bandlist, targetwcs)
# record coordinates in target brick image
ok,T.tx,T.ty = targetwcs.radec2pixelxy(T.ra, T.dec)
T.tx -= 1
T.ty -= 1
T.itx = np.clip(np.round(T.tx).astype(int), 0, W-1)
T.ity = np.clip(np.round(T.ty).astype(int), 0, H-1)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(T.tx, T.ty, 'o', mec=green, mfc='none', ms=10, mew=1.5)
plt.axis(ax)
plt.title('SDSS sources')
ps.savefig()
print 'Detmaps...'
# Render the detection maps
detmaps = dict([(b, np.zeros((H,W), np.float32)) for b in bands])
detivs = dict([(b, np.zeros((H,W), np.float32)) for b in bands])
for tim in tims:
iv = tim.getInvvar()
psfnorm = 1./(2. * np.sqrt(np.pi) * tim.psf_sigma)
detim = tim.getImage().copy()
detim[iv == 0] = 0.
detim = gaussian_filter(detim, tim.psf_sigma) / psfnorm**2
detsig1 = tim.sig1 / psfnorm
subh,subw = tim.shape
detiv = np.zeros((subh,subw), np.float32) + (1. / detsig1**2)
detiv[iv == 0] = 0.
(Yo,Xo,Yi,Xi) = tim.resamp
detmaps[tim.band][Yo,Xo] += detiv[Yi,Xi] * detim[Yi,Xi]
detivs [tim.band][Yo,Xo] += detiv[Yi,Xi]
rtn = dict()
for k in ['T', 'coimgs', 'cons', 'detmaps', 'detivs',
'targetrd', 'pixscale', 'targetwcs', 'W','H',
'bands', 'tims', 'ps', 'brick', 'cat']:
rtn[k] = locals()[k]
return rtn
def stage1(T=None, coimgs=None, cons=None, detmaps=None, detivs=None,
targetrd=None, pixscale=None, targetwcs=None, W=None,H=None,
bands=None, tims=None, ps=None, brick=None, cat=None):
orig_wcsxy0 = [tim.wcs.getX0Y0() for tim in tims]
hot = np.zeros((H,W), np.float32)
for band in bands:
detmap = detmaps[band] / np.maximum(1e-16, detivs[band])
detsn = detmap * np.sqrt(detivs[band])
hot = np.maximum(hot, detsn)
detmaps[band] = detmap
### FIXME -- ugri
for sedname,sed in [('Flat', (1.,1.,1.)), ('Red', (2.5, 1.0, 0.4))]:
sedmap = np.zeros((H,W), np.float32)
sediv = np.zeros((H,W), np.float32)
for iband,band in enumerate(bands):
# We convert the detmap to canonical band via
# detmap * w
# And the corresponding change to sig1 is
# sig1 * w
# So the invvar-weighted sum is
# (detmap * w) / (sig1**2 * w**2)
# = detmap / (sig1**2 * w)
sedmap += detmaps[band] * detivs[band] / sed[iband]
sediv += detivs [band] / sed[iband]**2
sedmap /= np.maximum(1e-16, sediv)
sedsn = sedmap * np.sqrt(sediv)
hot = np.maximum(hot, sedsn)
plt.clf()
dimshow(np.round(sedsn), vmin=0, vmax=10, cmap='hot')
plt.title('SED-matched detection filter: %s' % sedname)
ps.savefig()
peaks = (hot > 4)
blobs,nblobs = label(peaks)
print 'N detected blobs:', nblobs
blobslices = find_objects(blobs)
# Un-set catalog blobs
for x,y in zip(T.itx, T.ity):
# blob number
bb = blobs[y,x]
if bb == 0:
continue
# un-set 'peaks' within this blob
slc = blobslices[bb-1]
peaks[slc][blobs[slc] == bb] = 0
# Now, after having removed catalog sources, crank up the detection threshold
peaks &= (hot > 5)
# zero out the edges(?)
peaks[0 ,:] = peaks[:, 0] = 0
peaks[-1,:] = peaks[:,-1] = 0
peaks[1:-1, 1:-1] &= (hot[1:-1,1:-1] >= hot[0:-2,1:-1])
peaks[1:-1, 1:-1] &= (hot[1:-1,1:-1] >= hot[2: ,1:-1])
peaks[1:-1, 1:-1] &= (hot[1:-1,1:-1] >= hot[1:-1,0:-2])
peaks[1:-1, 1:-1] &= (hot[1:-1,1:-1] >= hot[1:-1,2: ])
# These are our peaks
pki = np.flatnonzero(peaks)
peaky,peakx = np.unravel_index(pki, peaks.shape)
print len(peaky), 'peaks'
crossa = dict(ms=10, mew=1.5)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(T.tx, T.ty, 'r+', **crossa)
plt.plot(peakx, peaky, '+', color=green, **crossa)
plt.axis(ax)
plt.title('SDSS + SED-matched detections')
ps.savefig()
### HACK -- high threshold again
# Segment, and record which sources fall into each blob
blobs,nblobs = label((hot > 20))
print 'N detected blobs:', nblobs
blobslices = find_objects(blobs)
T.blob = blobs[T.ity, T.itx]
blobsrcs = []
blobflux = []
fluximg = coimgs[1]
for blob in range(1, nblobs+1):
blobsrcs.append(np.flatnonzero(T.blob == blob))
bslc = blobslices[blob-1]
blobflux.append(np.sum(fluximg[bslc][blobs[bslc] == blob]))
# Fit the SDSS sources
for tim in tims:
tim.psfex.fitSavedData(*tim.psfex.splinedata)
tim.psf = tim.psfex
# How far down to render model profiles
minsigma = 0.1
for tim in tims:
tim.modelMinval = minsigma * tim.sig1
srcvariances = [[] for src in cat]
# Fit in order of flux
for blobnumber,iblob in enumerate(np.argsort(-np.array(blobflux))):
bslc = blobslices[iblob]
Isrcs = blobsrcs [iblob]
if len(Isrcs) == 0:
continue
print
print 'Blob', blobnumber, 'of', len(blobflux), ':', len(Isrcs), 'sources'
print 'Source indices:', Isrcs
print
# blob bbox in target coords
sy,sx = bslc
by0,by1 = sy.start, sy.stop
bx0,bx1 = sx.start, sx.stop
blobh,blobw = by1 - by0, bx1 - bx0
rr,dd = targetwcs.pixelxy2radec([bx0,bx0,bx1,bx1],[by0,by1,by1,by0])
alphas = [0.1, 0.3, 1.0]
subtims = []
for itim,tim in enumerate(tims):
h,w = tim.shape
ok,x,y = tim.subwcs.radec2pixelxy(rr,dd)
sx0,sx1 = x.min(), x.max()
sy0,sy1 = y.min(), y.max()
if sx1 < 0 or sy1 < 0 or sx1 > w or sy1 > h:
continue
sx0 = np.clip(int(np.floor(sx0)), 0, w-1)
sx1 = np.clip(int(np.ceil (sx1)), 0, w-1) + 1
sy0 = np.clip(int(np.floor(sy0)), 0, h-1)
sy1 = np.clip(int(np.ceil (sy1)), 0, h-1) + 1
subslc = slice(sy0,sy1),slice(sx0,sx1)
subimg = tim.getImage ()[subslc]
subie = tim.getInvError()[subslc]
subwcs = tim.getWcs().copy()
ox0,oy0 = orig_wcsxy0[itim]
subwcs.setX0Y0(ox0 + sx0, oy0 + sy0)
# Mask out inverr for pixels that are not within the blob.
subtarget = targetwcs.get_subimage(bx0, by0, blobw, blobh)
subsubwcs = tim.subwcs.get_subimage(int(sx0), int(sy0), int(sx1-sx0), int(sy1-sy0))
try:
Yo,Xo,Yi,Xi,rims = resample_with_wcs(subsubwcs, subtarget, [], 2)
except OverlapError:
print 'No overlap'
continue
if len(Yo) == 0:
continue
subie2 = np.zeros_like(subie)
I = np.flatnonzero(blobs[bslc][Yi, Xi] == (iblob+1))
subie2[Yo[I],Xo[I]] = subie[Yo[I],Xo[I]]
subie = subie2
# If the subimage (blob) is small enough, instantiate a
# constant PSF model in the center.
if sy1-sy0 < 100 and sx1-sx0 < 100:
subpsf = tim.psf.mogAt(ox0 + (sx0+sx1)/2., oy0 + (sy0+sy1)/2.)
else:
# Otherwise, instantiate a (shifted) spatially-varying
# PsfEx model.
subpsf = ShiftedPsf(tim.psf, ox0+sx0, oy0+sy0)
subtim = Image(data=subimg, inverr=subie, wcs=subwcs,
psf=subpsf, photocal=tim.getPhotoCal(),
sky=tim.getSky(), name=tim.name)
subtim.band = tim.band
subtim.sig1 = tim.sig1
subtim.modelMinval = tim.modelMinval
subtims.append(subtim)
subcat = Catalog(*[cat[i] for i in Isrcs])
subtr = Tractor(subtims, subcat)
subtr.freezeParam('images')
# Optimize individual sources in order of flux
fluxes = []
for src in subcat:
# HACK -- here we just *sum* the nanomaggies in each band. Bogus!
br = src.getBrightness()
flux = sum([br.getFlux(band) for band in bands])
fluxes.append(flux)
Ibright = np.argsort(-np.array(fluxes))
if len(Ibright) >= 5:
# -Remember the original subtim images
# -Compute initial models for each source (in each tim)
# -Subtract initial models from images
# -During fitting, for each source:
# -add back in the source's initial model (to each tim)
# -fit, with Catalog([src])
# -subtract final model (from each tim)
# -Replace original subtim images
#
# --Might want to omit newly-added detection-filter sources, since their
# fluxes are bogus.
# Remember original tim images
orig_timages = [tim.getImage().copy() for tim in subtims]
initial_models = []
# Create initial models for each tim x each source
for tim in subtims:
mods = []
for src in subcat:
mod = src.getModelPatch(tim)
mods.append(mod)
if mod is not None:
if not np.all(np.isfinite(mod.patch)):
print 'Non-finite mod patch'
print 'source:', src
print 'tim:', tim
print 'PSF:', tim.getPsf()
assert(np.all(np.isfinite(mod.patch)))
mod.addTo(tim.getImage(), scale=-1)
initial_models.append(mods)
# For sources in decreasing order of brightness
for numi,i in enumerate(Ibright):
tsrc = Time()
print 'Fitting source', i, '(%i of %i in blob)' % (numi, len(Ibright))
src = subcat[i]
print src
srctractor = Tractor(subtims, [src])
srctractor.freezeParams('images')
# Add this source's initial model back in.
for tim,mods in zip(subtims, initial_models):
mod = mods[i]
if mod is not None:
mod.addTo(tim.getImage())
print 'Optimizing:', srctractor
srctractor.printThawedParams()
for step in range(50):
dlnp,X,alpha = srctractor.optimize(priors=False, shared_params=False,
alphas=alphas)
print 'dlnp:', dlnp, 'src', src
if dlnp < 0.1:
break
for tim in subtims:
mod = src.getModelPatch(tim)
if mod is not None:
mod.addTo(tim.getImage(), scale=-1)
for tim,img in zip(subtims, orig_timages):
tim.data = img
del orig_timages
del initial_models
else:
# Fit sources one at a time, but don't subtract other models
subcat.freezeAllParams()
for numi,i in enumerate(Ibright):
tsrc = Time()
print 'Fitting source', i, '(%i of %i in blob)' % (numi, len(Ibright))
print subcat[i]
subcat.freezeAllBut(i)
print 'Optimizing:', subtr
subtr.printThawedParams()
for step in range(10):
dlnp,X,alpha = subtr.optimize(priors=False, shared_params=False,
alphas=alphas)
print 'dlnp:', dlnp
if dlnp < 0.1:
break
print 'Fitting source took', Time()-tsrc
print subcat[i]
if len(Isrcs) > 1 and len(Isrcs) <= 10:
tfit = Time()
# Optimize all at once?
subcat.thawAllParams()
print 'Optimizing:', subtr
subtr.printThawedParams()
for step in range(20):
dlnp,X,alpha = subtr.optimize(priors=False, shared_params=False,
alphas=alphas)
print 'dlnp:', dlnp
if dlnp < 0.1:
break
# Variances
subcat.thawAllRecursive()
subcat.freezeAllParams()
for isub,srci in enumerate(Isrcs):
print 'Variances for source', srci
subcat.thawParam(isub)
src = subcat[isub]
print 'Source', src
print 'Params:', src.getParamNames()
if isinstance(src, (DevGalaxy, ExpGalaxy)):
src.shape = EllipseE.fromEllipseESoft(src.shape)
elif isinstance(src, FixedCompositeGalaxy):
src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp)
src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev)
print 'Converted ellipse:', src
allderivs = subtr.getDerivs()
for iparam,derivs in enumerate(allderivs):
dchisq = 0
for deriv,tim in derivs:
h,w = tim.shape
deriv.clipTo(w,h)
ie = tim.getInvError()
slc = deriv.getSlice(ie)
chi = deriv.patch * ie[slc]
dchisq += (chi**2).sum()
if dchisq == 0.:
v = np.nan
else:
v = 1./dchisq
srcvariances[srci].append(v)
assert(len(srcvariances[srci]) == subcat[isub].numberOfParams())
subcat.freezeParam(isub)
cat.thawAllRecursive()
for i,src in enumerate(cat):
print 'Source', i, src
print 'variances:', srcvariances[i]
print len(srcvariances[i]), 'vs', src.numberOfParams()
if len(srcvariances[i]) != src.numberOfParams():
# This can happen for sources outside the brick bounds: they never get optimized?
print 'Warning: zeroing variances for source', src
srcvariances[i] = [0]*src.numberOfParams()
if isinstance(src, (DevGalaxy, ExpGalaxy)):
src.shape = EllipseE.fromEllipseESoft(src.shape)
elif isinstance(src, FixedCompositeGalaxy):
src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp)
src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev)
assert(len(srcvariances[i]) == src.numberOfParams())
variances = np.hstack(srcvariances)
assert(len(variances) == cat.numberOfParams())
return dict(cat=cat, variances=variances)
