def add(self, tile, wise_models):
for band in [1, 2, 3, 4]:
if not (tile, band) in wise_models:
debug('Tile', tile, 'band', band, '-- model not found')
continue
# With the move_crpix option (Aaron's updated astrometry),
# the WCS for each band can be different, so we call resample_with_wcs
# for each band with (potentially) slightly different WCSes.
(mod, img, ie, roi, wcs) = wise_models[(tile, band)]
debug('WISE: resampling', wcs, 'to', self.unwise_wcs)
try:
Yo, Xo, Yi, Xi, resam = resample_with_wcs(self.unwise_wcs,
wcs, [img, mod],
intType=np.int16)
rimg, rmod = resam
debug('Adding', len(Yo), 'pixels from tile', tile, 'to coadd')
self.unwise_co[band - 1][Yo, Xo] += rimg
self.unwise_com[band - 1][Yo, Xo] += rmod
self.unwise_con[band - 1][Yo, Xo] += 1
self.unwise_coiv[band - 1][Yo, Xo] += ie[Yi, Xi]**2
debug('Band', band, ': now',
np.sum(self.unwise_con[band - 1] > 0),
'pixels are set in image coadd')
except OverlapError:
debug('No overlap between WISE model tile', tile, 'and brick')
def add(self, models, unique=False):
for name, band, wcs, img, mod, ie in models:
debug('Accumulating tile', name, 'band', band)
try:
Yo,Xo,Yi,Xi,resam = resample_with_wcs(self.wcs, wcs,
[img, mod], intType=np.int16)
except OverlapError:
debug('No overlap between tile', name, 'and coadd')
continue
rimg,rmod = resam
debug('Adding', len(Yo), 'pixels from tile', name, 'to coadd')
iv = ie[Yi,Xi]**2
if unique:
K = np.flatnonzero((self.co_nobs[band][Yo,Xo] == 0) * (iv>0))
iv = iv[K]
rimg = rimg[K]
rmod = rmod[K]
Yo = Yo[K]
Xo = Xo[K]
debug('Cut to', len(Yo), 'unique pixels w/ iv>0')
debug('Tile:', np.sum(iv>0), 'of', len(iv), 'pixels have IV')
self.co_images [band][Yo,Xo] += rimg * iv
self.co_models [band][Yo,Xo] += rmod * iv
self.co_nobs [band][Yo,Xo] += 1
self.co_invvars[band][Yo,Xo] += iv
debug('Band', band, ': now', np.sum(self.co_nobs[band]>0), 'pixels are set in image coadd')
def _resample_one(args):
(itim,tim,mod,blobmod,lanczos,targetwcs,sbscale) = args
if lanczos:
from astrometry.util.miscutils import patch_image
patched = tim.getImage().copy()
assert(np.all(np.isfinite(tim.getInvError())))
okpix = (tim.getInvError() > 0)
patch_image(patched, okpix)
del okpix
imgs = [patched]
if mod is not None:
imgs.append(mod)
if blobmod is not None:
imgs.append(blobmod)
else:
imgs = []
try:
Yo,Xo,Yi,Xi,rimgs = resample_with_wcs(
targetwcs, tim.subwcs, imgs, 3, intType=np.int16)
except OverlapError:
return None
if len(Yo) == 0:
return None
mo = None
bmo = None
if lanczos:
im = rimgs[0]
inext = 1
if mod is not None:
mo = rimgs[inext]
inext += 1
if blobmod is not None:
bmo = rimgs[inext]
inext += 1
del patched,imgs,rimgs
else:
im = tim.getImage ()[Yi,Xi]
if mod is not None:
mo = mod[Yi,Xi]
if blobmod is not None:
bmo = blobmod[Yi,Xi]
iv = tim.getInvvar()[Yi,Xi]
if sbscale:
fscale = tim.sbscale
debug('Applying surface-brightness scaling of %.3f to' % fscale, tim.name)
im *= fscale
iv /= (fscale**2)
if mod is not None:
mo *= fscale
if blobmod is not None:
bmo *= fscale
if tim.dq is None:
dq = None
else:
dq = tim.dq[Yi,Xi]
return itim,Yo,Xo,iv,im,mo,bmo,dq
def tim_get_resamp(tim, targetwcs):
if hasattr(tim, 'resamp'):
return tim.resamp
try:
Yo,Xo,Yi,Xi,nil = resample_with_wcs(targetwcs, tim.subwcs, [], 2)
except OverlapError:
print('No overlap')
return None
if len(Yo) == 0:
return None
resamp = [x.astype(np.int16) for x in (Yo,Xo,Yi,Xi)]
return resamp
def tim_get_resamp(tim, targetwcs):
if hasattr(tim, 'resamp'):
return tim.resamp
try:
Yo,Xo,Yi,Xi,nil = resample_with_wcs(targetwcs, tim.subwcs, [], 2)
except OverlapError:
print('No overlap')
return None
if len(Yo) == 0:
return None
resamp = [x.astype(np.int16) for x in (Yo,Xo,Yi,Xi)]
return resamp
def blur_resample_one(X):
from scipy.ndimage.filters import gaussian_filter
from astrometry.util.resample import resample_with_wcs,OverlapError
tim,sig,targetwcs = X
img = gaussian_filter(tim.getImage(), sig)
try:
Yo,Xo,Yi,Xi,[rimg] = resample_with_wcs(
targetwcs, tim.subwcs, [img], intType=np.int16)
except OverlapError:
return None
del img
blurnorm = 1./(2. * np.sqrt(np.pi) * sig)
wt = tim.getInvvar()[Yi,Xi] / (blurnorm**2)
return (Yo, Xo, rimg*wt, wt, tim.dq[Yi,Xi])
def _resample_one(args):
(itim, tim, mod, lanczos, targetwcs) = args
from astrometry.util.resample import resample_with_wcs, OverlapError
if lanczos:
from astrometry.util.miscutils import patch_image
patched = tim.getImage().copy()
okpix = (tim.getInvError() > 0)
patch_image(patched, okpix)
del okpix
imgs = [patched]
if mod is not None:
imgs.append(mod)
else:
imgs = []
try:
Yo, Xo, Yi, Xi, rimgs = resample_with_wcs(targetwcs, tim.subwcs, imgs,
3)
except OverlapError:
return None
if len(Yo) == 0:
return None
mo = None
if lanczos:
im = rimgs[0]
if mod is not None:
mo = rimgs[1]
del patched, imgs, rimgs
else:
im = tim.getImage()[Yi, Xi]
if mod is not None:
mo = mods[itim][Yi, Xi]
iv = tim.getInvvar()[Yi, Xi]
fscale = tim.sbscale
print('Applying surface-brightness scaling of %.3f to' % fscale, tim.name)
im *= fscale
iv /= (fscale**2)
if mod is not None:
mo *= fscale
if tim.dq is None:
dq = None
else:
dq = tim.dq[Yi, Xi]
return itim, Yo, Xo, iv, im, mo, dq
def _resample_one((itim,tim,mod,lanczos,targetwcs)):
from astrometry.util.resample import resample_with_wcs, OverlapError
if lanczos:
from astrometry.util.miscutils import patch_image
patched = tim.getImage().copy()
okpix = (tim.getInvError() > 0)
patch_image(patched, okpix)
del okpix
imgs = [patched]
if mod is not None:
imgs.append(mod)
else:
imgs = []
try:
Yo,Xo,Yi,Xi,rimgs = resample_with_wcs(
targetwcs, tim.subwcs, imgs, 3)
except OverlapError:
return None
if len(Yo) == 0:
return None
mo = None
if lanczos:
im = rimgs[0]
if mod is not None:
mo = rimgs[1]
del patched,imgs,rimgs
else:
im = tim.getImage ()[Yi,Xi]
if mod is not None:
mo = mods[itim][Yi,Xi]
iv = tim.getInvvar()[Yi,Xi]
fscale = tim.sbscale
print('Applying surface-brightness scaling of %.3f to' % fscale, tim.name)
im *= fscale
iv /= (fscale**2)
if mod is not None:
mo *= fscale
if tim.dq is None:
dq = None
else:
dq = tim.dq[Yi,Xi]
return itim,Yo,Xo,iv,im,mo,dq
def get_sdss_cutout(targetwcs, sdss, get_rawvals=False, bands='irg',
get_rawvals_only=False,
bandscales=dict(z=1.0, i=1.0, r=1.3, g=2.5)):
rgbims = []
ra,dec = targetwcs.radec_center()
# in deg
radius = targetwcs.radius()
#print 'Target WCS radius is', radius, 'deg'
H,W = targetwcs.get_height(), targetwcs.get_width()
targetpixscale = targetwcs.pixel_scale()
wlistfn = sdss.filenames.get('window_flist', 'window_flist.fits')
rad2 = radius*60. + np.hypot(14., 10.)/2.
#print 'Rad2 radius', rad2, 'arcmin'
RCF = radec_to_sdss_rcf(ra, dec, tablefn=wlistfn, radius=rad2)
# Drop rerun 157
keepRCF = []
for run,camcol,field,r,d in RCF:
rr = sdss.get_rerun(run, field)
#print 'Rerun:', rr
if rr == '157':
continue
keepRCF.append((run,camcol,field))
RCF = keepRCF
print len(RCF), 'run/camcol/fields in range'
# size in SDSS pixels of the target image.
sz = np.hypot(H, W)/2. * targetpixscale / 0.396
print 'SDSS sz:', sz
bandnums = [band_index(b) for b in bands]
for bandnum,band in zip(bandnums, bands):
targetim = np.zeros((H, W), np.float32)
targetn = np.zeros((H, W), np.uint8)
for ifield,(run,camcol,field) in enumerate(RCF):
fn = sdss.retrieve('frame', run, camcol, field, band)
frame = sdss.readFrame(run, camcol, field, bandnum)
h,w = frame.getImageShape()
x,y = frame.astrans.radec_to_pixel(ra, dec)
x,y = int(x), int(y)
# add some margin for resampling
sz2 = int(sz) + 5
xlo = np.clip(x - sz2, 0, w)
xhi = np.clip(x + sz2 + 1, 0, w)
ylo = np.clip(y - sz2, 0, h)
yhi = np.clip(y + sz2 + 1, 0, h)
if xlo == xhi or ylo == yhi:
continue
stamp = frame.getImageSlice((slice(ylo, yhi), slice(xlo, xhi)))
sh,sw = stamp.shape
wcs = AsTransWrapper(frame.astrans, sw, sh, x0=xlo, y0=ylo)
# FIXME -- allow nn resampling too
try:
Yo,Xo,Yi,Xi,[rim] = resample_with_wcs(targetwcs, wcs, [stamp], 3)
except ResampleError:
continue
targetim[Yo,Xo] += rim
targetn [Yo,Xo] += 1
rgbims.append(targetim / targetn)
if get_rawvals_only:
return rgbims
if get_rawvals:
rawvals = [x.copy() for x in rgbims]
r,g,b = rgbims
r *= bandscales[bands[0]]
g *= bandscales[bands[1]]
b *= bandscales[bands[2]]
# i
#r *= 1.0
# r
#g *= 1.5
#g *= 1.3
# g
#b *= 2.5
m = -0.02
r = np.maximum(0, r - m)
g = np.maximum(0, g - m)
b = np.maximum(0, b - m)
I = (r+g+b)/3.
alpha = 1.5
Q = 20
m2 = 0.
fI = np.arcsinh(alpha * Q * (I - m2)) / np.sqrt(Q)
I += (I == 0.) * 1e-6
R = fI * r / I
G = fI * g / I
B = fI * b / I
maxrgb = reduce(np.maximum, [R,G,B])
J = (maxrgb > 1.)
R[J] = R[J]/maxrgb[J]
G[J] = G[J]/maxrgb[J]
B[J] = B[J]/maxrgb[J]
ss = 0.5
RGBblur = np.clip(np.dstack([
gaussian_filter(R, ss),
gaussian_filter(G, ss),
gaussian_filter(B, ss)]), 0., 1.)
if get_rawvals:
return RGBblur, rawvals
return RGBblur
def make_depth_cut(survey, ccds, bands, targetrd, brick, W, H, pixscale,
plots, ps, splinesky, gaussPsf, pixPsf, normalizePsf, do_calibs,
gitver, targetwcs, old_calibs_ok, get_depth_maps=False, margin=0.5,
use_approx_wcs=False):
if plots:
import pylab as plt
# Add some margin to our DESI depth requirements
target_depth_map = dict(g=24.0 + margin, r=23.4 + margin, z=22.5 + margin)
# List extra (redundant) target percentiles so that increasing the depth at
# any of these percentiles causes the image to be kept.
target_percentiles = np.array(list(range(2, 10)) +
list(range(10, 30, 5)) +
list(range(30, 101, 10)))
target_ddepths = np.zeros(len(target_percentiles), np.float32)
target_ddepths[target_percentiles < 10] = -0.3
target_ddepths[target_percentiles < 5] = -0.6
#print('Target percentiles:', target_percentiles)
#print('Target ddepths:', target_ddepths)
cH,cW = H//10, W//10
coarsewcs = targetwcs.scale(0.1)
coarsewcs.imagew = cW
coarsewcs.imageh = cH
# Unique pixels in this brick (U: cH x cW boolean)
U = find_unique_pixels(coarsewcs, cW, cH, None,
brick.ra1, brick.ra2, brick.dec1, brick.dec2)
pixscale = 3600. * np.sqrt(np.abs(ccds.cd1_1*ccds.cd2_2 - ccds.cd1_2*ccds.cd2_1))
seeing = ccds.fwhm * pixscale
# Compute the rectangle in *coarsewcs* covered by each CCD
slices = []
overlapping_ccds = np.zeros(len(ccds), bool)
for i,ccd in enumerate(ccds):
wcs = survey.get_approx_wcs(ccd)
hh,ww = wcs.shape
rr,dd = wcs.pixelxy2radec([1,ww,ww,1], [1,1,hh,hh])
ok,xx,yy = coarsewcs.radec2pixelxy(rr, dd)
y0 = int(np.round(np.clip(yy.min(), 0, cH-1)))
y1 = int(np.round(np.clip(yy.max(), 0, cH-1)))
x0 = int(np.round(np.clip(xx.min(), 0, cW-1)))
x1 = int(np.round(np.clip(xx.max(), 0, cW-1)))
if y0 == y1 or x0 == x1:
slices.append(None)
continue
# Check whether this CCD overlaps the unique area of this brick...
if not np.any(U[y0:y1+1, x0:x1+1]):
info('No overlap with unique area for CCD', ccd.expnum, ccd.ccdname)
slices.append(None)
continue
overlapping_ccds[i] = True
slices.append((slice(y0, y1+1), slice(x0, x1+1)))
keep_ccds = np.zeros(len(ccds), bool)
depthmaps = []
for band in bands:
# scalar
target_depth = target_depth_map[band]
# vector
target_depths = target_depth + target_ddepths
depthiv = np.zeros((cH,cW), np.float32)
depthmap = np.zeros_like(depthiv)
depthvalue = np.zeros_like(depthiv)
last_pcts = np.zeros_like(target_depths)
# indices of CCDs we still want to look at in the current band
b_inds = np.where(ccds.filter == band)[0]
info(len(b_inds), 'CCDs in', band, 'band')
if len(b_inds) == 0:
continue
b_inds = np.array([i for i in b_inds if slices[i] is not None])
info(len(b_inds), 'CCDs in', band, 'band overlap target')
if len(b_inds) == 0:
continue
# CCDs that we will try before searching for good ones -- CCDs
# from the same exposure number as CCDs we have chosen to
# take.
try_ccds = set()
# Try DECaLS data first!
Idecals = np.where(ccds.propid[b_inds] == '2014B-0404')[0]
if len(Idecals):
try_ccds.update(b_inds[Idecals])
debug('Added', len(try_ccds), 'DECaLS CCDs to try-list')
plot_vals = []
if plots:
plt.clf()
for i in b_inds:
sy,sx = slices[i]
x0,x1 = sx.start, sx.stop
y0,y1 = sy.start, sy.stop
plt.plot([x0,x0,x1,x1,x0], [y0,y1,y1,y0,y0], 'b-', alpha=0.5)
plt.title('CCDs overlapping brick: %i in %s band' % (len(b_inds), band))
ps.savefig()
nccds = np.zeros((cH,cW), np.int16)
plt.clf()
for i in b_inds:
nccds[slices[i]] += 1
plt.imshow(nccds, interpolation='nearest', origin='lower', vmin=0)
plt.colorbar()
plt.title('CCDs overlapping brick: %i in %s band (%i / %i / %i)' %
(len(b_inds), band, nccds.min(), np.median(nccds), nccds.max()))
ps.savefig()
#continue
while len(b_inds):
if len(try_ccds) == 0:
# Choose the next CCD to look at in this band.
# A rough point-source depth proxy would be:
# metric = np.sqrt(ccds.extime[b_inds]) / seeing[b_inds]
# If we want to put more weight on choosing good-seeing images, we could do:
#metric = np.sqrt(ccds.exptime[b_inds]) / seeing[b_inds]**2
# depth would be ~ 1 / (sig1 * seeing); we privilege good seeing here.
metric = 1. / (ccds.sig1[b_inds] * seeing[b_inds]**2)
# This metric is *BIG* for *GOOD* ccds!
# Here, we try explicitly to include CCDs that cover
# pixels that are still shallow by the largest amount
# for the largest number of percentiles of interest;
# note that pixels with no coverage get depth 0, so
# score high in this metric.
#
# The value is the depth still required to hit the
# target, summed over percentiles of interest
# (for pixels unique to this brick)
depthvalue[:,:] = 0.
active = (last_pcts < target_depths)
for d in target_depths[active]:
depthvalue += U * np.maximum(0, d - depthmap)
ccdvalue = np.zeros(len(b_inds), np.float32)
for j,i in enumerate(b_inds):
#ccdvalue[j] = np.sum(depthvalue[slices[i]])
# mean -- we want the most bang for the buck per pixel?
ccdvalue[j] = np.mean(depthvalue[slices[i]])
metric *= ccdvalue
# *ibest* is an index into b_inds
ibest = np.argmax(metric)
# *iccd* is an index into ccds.
iccd = b_inds[ibest]
ccd = ccds[iccd]
debug('Chose best CCD: seeing', seeing[iccd], 'exptime', ccds.exptime[iccd], 'with value', ccdvalue[ibest])
else:
iccd = try_ccds.pop()
ccd = ccds[iccd]
debug('Popping CCD from use_ccds list')
# remove *iccd* from b_inds
b_inds = b_inds[b_inds != iccd]
im = survey.get_image_object(ccd)
debug('Band', im.band, 'expnum', im.expnum, 'exptime', im.exptime, 'seeing', im.fwhm*im.pixscale, 'arcsec, propid', im.propid)
im.check_for_cached_files(survey)
debug(im)
if do_calibs:
kwa = dict(git_version=gitver, old_calibs_ok=old_calibs_ok)
if gaussPsf:
kwa.update(psfex=False)
if splinesky:
kwa.update(splinesky=True)
im.run_calibs(**kwa)
if use_approx_wcs:
debug('Using approximate (TAN) WCS')
wcs = survey.get_approx_wcs(ccd)
else:
debug('Reading WCS from', im.imgfn, 'HDU', im.hdu)
wcs = im.get_wcs()
x0,x1,y0,y1,slc = im.get_image_extent(wcs=wcs, radecpoly=targetrd)
if x0==x1 or y0==y1:
debug('No actual overlap')
continue
wcs = wcs.get_subimage(int(x0), int(y0), int(x1-x0), int(y1-y0))
if 'galnorm_mean' in ccds.get_columns():
galnorm = ccd.galnorm_mean
debug('Using galnorm_mean from CCDs table:', galnorm)
else:
psf = im.read_psf_model(x0, y0, gaussPsf=gaussPsf, pixPsf=pixPsf,
normalizePsf=normalizePsf)
psf = psf.constantPsfAt((x1-x0)//2, (y1-y0)//2)
# create a fake tim to compute galnorm
from tractor import PixPos, Flux, ModelMask, Image, NullWCS
from legacypipe.survey import SimpleGalaxy
h,w = 50,50
gal = SimpleGalaxy(PixPos(w//2,h//2), Flux(1.))
tim = Image(data=np.zeros((h,w), np.float32),
psf=psf, wcs=NullWCS(pixscale=im.pixscale))
mm = ModelMask(0, 0, w, h)
galmod = gal.getModelPatch(tim, modelMask=mm).patch
galmod = np.maximum(0, galmod)
galmod /= galmod.sum()
galnorm = np.sqrt(np.sum(galmod**2))
detiv = 1. / (im.sig1 / galnorm)**2
galdepth = -2.5 * (np.log10(5. * im.sig1 / galnorm) - 9.)
debug('Galnorm:', galnorm, 'sig1:', im.sig1, 'galdepth', galdepth)
# Add this image the the depth map...
from astrometry.util.resample import resample_with_wcs, OverlapError
try:
Yo,Xo,_,_,_ = resample_with_wcs(coarsewcs, wcs)
debug(len(Yo), 'of', (cW*cH), 'pixels covered by this image')
except OverlapError:
debug('No overlap')
continue
depthiv[Yo,Xo] += detiv
# compute the new depth map & percentiles (including the proposed new CCD)
depthmap[:,:] = 0.
depthmap[depthiv > 0] = 22.5 - 2.5*np.log10(5./np.sqrt(depthiv[depthiv > 0]))
depthpcts = np.percentile(depthmap[U], target_percentiles)
for i,(p,d,t) in enumerate(zip(target_percentiles, depthpcts, target_depths)):
info(' pct % 3i, prev %5.2f -> %5.2f vs target %5.2f %s' % (p, last_pcts[i], d, t, ('ok' if d >= t else '')))
keep = False
# Did we increase the depth of any target percentile that did not already exceed its target depth?
if np.any((depthpcts > last_pcts) * (last_pcts < target_depths)):
keep = True
# Add any other CCDs from this same expnum to the try_ccds list.
# (before making the plot)
I = np.where(ccd.expnum == ccds.expnum[b_inds])[0]
try_ccds.update(b_inds[I])
debug('Adding', len(I), 'CCDs with the same expnum to try_ccds list')
if plots:
cc = '1' if keep else '0'
xx = [Xo.min(), Xo.min(), Xo.max(), Xo.max(), Xo.min()]
yy = [Yo.min(), Yo.max(), Yo.max(), Yo.min(), Yo.min()]
plot_vals.append(((xx,yy,cc),(last_pcts,depthpcts,keep),im.ccdname))
if plots and (
(len(try_ccds) == 0) or np.all(depthpcts >= target_depths)):
plt.clf()
plt.subplot2grid((2,2),(0,0))
plt.imshow(depthvalue, interpolation='nearest', origin='lower',
vmin=0)
plt.xticks([]); plt.yticks([])
plt.colorbar()
plt.title('heuristic value')
plt.subplot2grid((2,2),(0,1))
plt.imshow(depthmap, interpolation='nearest', origin='lower',
vmin=target_depth - 2, vmax=target_depth + 0.5)
ax = plt.axis()
for (xx,yy,cc) in [p[0] for p in plot_vals]:
plt.plot(xx,yy, '-', color=cc, lw=3)
plt.axis(ax)
plt.xticks([]); plt.yticks([])
plt.colorbar()
plt.title('depth map')
plt.subplot2grid((2,2),(1,0), colspan=2)
ax = plt.gca()
plt.plot(target_percentiles, target_depths, 'ro', label='Target')
plt.plot(target_percentiles, target_depths, 'r-')
for (lp,dp,k) in [p[1] for p in plot_vals]:
plt.plot(target_percentiles, lp, 'k-',
label='Previous percentiles')
for (lp,dp,k) in [p[1] for p in plot_vals]:
cc = 'b' if k else 'r'
plt.plot(target_percentiles, dp, '-', color=cc,
label='Depth percentiles')
ccdnames = ','.join([p[2] for p in plot_vals])
plot_vals = []
plt.ylim(target_depth - 2, target_depth + 0.5)
plt.xscale('log')
plt.xlabel('Percentile')
plt.ylabel('Depth')
plt.title('depth percentiles')
plt.suptitle('%s %i-%s, exptime %.0f, seeing %.2f, band %s' %
(im.camera, im.expnum, ccdnames, im.exptime,
im.pixscale * im.fwhm, band))
ps.savefig()
if keep:
info('Keeping this exposure')
else:
info('Not keeping this exposure')
depthiv[Yo,Xo] -= detiv
continue
keep_ccds[iccd] = True
last_pcts = depthpcts
if np.all(depthpcts >= target_depths):
info('Reached all target depth percentiles for band', band)
break
if get_depth_maps:
if np.any(depthiv > 0):
depthmap[:,:] = 0.
depthmap[depthiv > 0] = 22.5 -2.5*np.log10(5./np.sqrt(depthiv[depthiv > 0]))
depthmap[np.logical_not(U)] = np.nan
depthmaps.append((band, depthmap.copy()))
if plots:
I = np.where(ccds.filter == band)[0]
plt.clf()
plt.plot(seeing[I], ccds.exptime[I], 'k.')
# which CCDs from this band are we keeping?
kept, = np.nonzero(keep_ccds)
if len(kept):
kept = kept[ccds.filter[kept] == band]
plt.plot(seeing[kept], ccds.exptime[kept], 'ro')
plt.xlabel('Seeing (arcsec)')
plt.ylabel('Exptime (sec)')
plt.title('CCDs kept for band %s' % band)
plt.ylim(0, np.max(ccds.exptime[I]) * 1.1)
ps.savefig()
if get_depth_maps:
return (keep_ccds, overlapping_ccds, depthmaps)
return keep_ccds, overlapping_ccds
def real_one_tile((args)):
#(itile,tile,udecs,P3,T,tilewcs,exps,bad_expids,tileid_to_depth) = args
(itile,tile) = args
print()
print('Tile', itile+1, ':', tile.tileid, 'at', tile.ra, tile.dec)
i = np.nonzero(tile.dec == udecs)[0][0]
if i == 0 or i == len(udecs)-1:
print('Endpoint Dec; skipping for now')
return None
print(' Decs:', udecs[i-1], tile.dec, udecs[i+1])
declo = (udecs[i-1] + tile.dec) / 2.
dechi = (udecs[i+1] + tile.dec) / 2.
row = P3[P3.dec == tile.dec]
print(' ', len(row), 'tiles in this Dec row')
ras = np.sort(row.ra)
i = np.nonzero(tile.ra == ras)[0][0]
if i == 0 or i == len(ras)-1:
print(' Endpoint RA; skipping for now')
return None
print(' RAs:', ras[i-1], tile.ra, ras[i+1])
ralo = (ras[i-1] + tile.ra) / 2.
rahi = (ras[i+1] + tile.ra) / 2.
pixscale = 2.
#pixscale = 0.262
H = int(np.ceil((dechi - declo) / (pixscale/3600.)))
W = int(np.ceil((rahi - ralo) * np.cos(np.deg2rad(tile.dec)) / (pixscale/3600.)))
print(' Dec height', dechi-declo, 'RA width', rahi-ralo, '-> pix', W, 'x', H)
cd = pixscale/3600.
thiswcs = Tan(tile.ra, tile.dec, (W+1)/2., (H+1)/2.,
-cd, 0., 0., cd, float(W), float(H))
# Find surrounding tiles
radius = np.hypot(W, H) * pixscale / 3600.
I,J,d = match_radec(T.ra, T.dec, tile.ra, tile.dec, radius)
print(' ', len(I), 'tiles with measured depths nearby')
if len(I) == 0:
return None
# Now we need to take a tile boresight position and map it to
# boxes in RA,Dec of the good portions of the CCDs.
depth = np.zeros((H,W), np.float32)
nexp = np.zeros((H,W), np.uint8)
matched_exposures = set()
#print(' tileids', T.tileid[I])
#print(' expnums', T.z_expnum[I])
for ii in I:
if T.z_expnum[ii] in bad_expids:
print(' skipping bad exp num', T.z_expnum[ii])
continue
# Get depth from CCDs file, if available.
zdepth = tileid_to_depth.get(T.tileid[ii], 0.)
if zdepth == 0.:
zdepth = T.z_depth[ii]
matched_exposures.add(T.z_expnum[ii])
for twcs in tilewcs:
twcs.set_crval((T.ra[ii], T.dec[ii]))
try:
Yo,Xo,Yi,Xi,rims = resample_with_wcs(thiswcs, twcs)
except OverlapError:
continue
dflux = 10.**((zdepth - 22.5)/-2.5)
div = 1./dflux**2
depth[Yo,Xo] += div
nexp[Yo,Xo] += 1
# Now also look for entries in the CCDs (exposures) table not previously found.
I,J,d = match_radec(exps.ra_bore, exps.dec_bore, tile.ra, tile.dec, radius)
print(' ', len(I), 'exposures from CCDs file nearby')
if len(I):
I = np.array([i for i,expnum,gd in zip(I, exps.expnum[I], exps.galdepth[I])
if (not expnum in matched_exposures) and gd > 0])
print(' ', len(I), 'exposures that were not in tile file')
# Drop exposures from this pass, except for previous exposures of this tile!
# if len(I):
# I = I[np.logical_or(exps.tilepass[I] != 3,
# exps.tileid[I] == tile.tileid)]
# print(' ', len(I), 'exposures not in pass 3')
# if len(I):
# print(' objects:', [o.strip() for o in exps.object[I]])
# print(' tileids:', exps.tileid[I])
# print(' expnums:', exps.expnum[I])
# print(' passes:', exps.tilepass[I])
for ii in I:
if exps.expnum[ii] in bad_expids:
print(' skipping bad exp num', exps.expnum[ii])
continue
zdepth = exps.galdepth[ii]
for twcs in tilewcs:
twcs.set_crval((exps.ra_bore[ii], exps.dec_bore[ii]))
try:
Yo,Xo,Yi,Xi,rims = resample_with_wcs(thiswcs, twcs)
except OverlapError:
continue
dflux = 10.**((zdepth - 22.5)/-2.5)
div = 1./dflux**2
depth[Yo,Xo] += div
nexp[Yo,Xo] += 1
# Convert depth map from depth-iv back to mag.
# flux
with np.errstate(divide='ignore'):
dflux = np.sqrt(1./depth)
dflux[depth == 0] = 0.
depth = -2.5 * (np.log10(dflux) - 9.)
depth[dflux == 0] = 0.
if depth.max() == 0:
print(' Actually no overlap')
return None
# Extinction correction for this tile...
ext_z = 1.211 * tile.ebv_med
print(' Applying extinction correction', ext_z, 'mag')
depth[depth != 0] -= ext_z
#pcts = [0,10,20,30,40,50,60,70,80,90,100]
pcts = np.arange(0, 101)
depths = np.percentile(depth, pcts)
target = 22.5
req_pcts = [0, 2, 2, 5, 5, 10, 10, 100]
req_depths = [0, 0, target-0.6, target-0.6,
target-0.3, target-0.3, target, target]
print(' Depths at 2, 5, and 10th percentile vs target:',
'%.2f' % (depths[2] - (target - 0.6)),
'%.2f' % (depths[5] - (target - 0.3)),
'%.2f' % (depths[10] - target))
return depths
def unwise_forcedphot(cat, tiles, band=1, roiradecbox=None,
use_ceres=True, ceres_block=8,
save_fits=False, get_models=False, ps=None,
psf_broadening=None,
pixelized_psf=False,
get_masks=None,
move_crpix=False,
modelsky_dir=None):
'''
Given a list of tractor sources *cat*
and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id)
runs forced photometry, returning a FITS table the same length as *cat*.
*get_masks*: the WCS to resample mask bits into.
'''
from tractor import NanoMaggies, PointSource, Tractor, ExpGalaxy, DevGalaxy, FixedCompositeGalaxy
if not pixelized_psf and psf_broadening is None:
# PSF broadening in post-reactivation data, by band.
# Newer version from Aaron's email to decam-chatter, 2018-06-14.
broadening = { 1: 1.0405, 2: 1.0346, 3: None, 4: None }
psf_broadening = broadening[band]
if False:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('wise-forced-w%i' % band)
plots = (ps is not None)
if plots:
import pylab as plt
wantims = (plots or save_fits or get_models)
wanyband = 'w'
if get_models:
models = {}
wband = 'w%i' % band
fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix',
'pronexp']
Nsrcs = len(cat)
phot = fits_table()
# Filled in based on unique tile overlap
phot.wise_coadd_id = np.array([' '] * Nsrcs)
phot.set(wband + '_psfdepth', np.zeros(len(phot), np.float32))
ra = np.array([src.getPosition().ra for src in cat])
dec = np.array([src.getPosition().dec for src in cat])
nexp = np.zeros(Nsrcs, np.int16)
mjd = np.zeros(Nsrcs, np.float64)
central_flux = np.zeros(Nsrcs, np.float32)
fitstats = {}
tims = []
if get_masks:
mh,mw = get_masks.shape
maskmap = np.zeros((mh,mw), np.uint32)
for tile in tiles:
print('Reading WISE tile', tile.coadd_id, 'band', band)
tim = get_unwise_tractor_image(tile.unwise_dir, tile.coadd_id, band,
bandname=wanyband, roiradecbox=roiradecbox)
if tim is None:
print('Actually, no overlap with tile', tile.coadd_id)
continue
if plots:
sig1 = tim.sig1
plt.clf()
plt.imshow(tim.getImage(), interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=10 * sig1)
plt.colorbar()
tag = '%s W%i' % (tile.coadd_id, band)
plt.title('%s: tim data' % tag)
ps.savefig()
plt.clf()
plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
range=(-5,10), bins=100)
plt.xlabel('Per-pixel intensity (Sigma)')
plt.title(tag)
ps.savefig()
if move_crpix and band in [1, 2]:
realwcs = tim.wcs.wcs
x,y = realwcs.crpix
tile_crpix = tile.get('crpix_w%i' % band)
dx = tile_crpix[0] - 1024.5
dy = tile_crpix[1] - 1024.5
realwcs.set_crpix(x+dx, y+dy)
#print('CRPIX', x,y, 'shift by', dx,dy, 'to', realwcs.crpix)
if modelsky_dir and band in [1, 2]:
fn = os.path.join(modelsky_dir, '%s.%i.mod.fits' % (tile.coadd_id, band))
if not os.path.exists(fn):
raise RuntimeError('WARNING: does not exist:', fn)
x0,x1,y0,y1 = tim.roi
bg = fitsio.FITS(fn)[2][y0:y1, x0:x1]
#print('Read background map:', bg.shape, bg.dtype, 'vs image', tim.shape)
if plots:
plt.clf()
plt.subplot(1,2,1)
plt.imshow(tim.getImage(), interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=5 * sig1)
plt.subplot(1,2,2)
plt.imshow(bg, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=5 * sig1)
tag = '%s W%i' % (tile.coadd_id, band)
plt.suptitle(tag)
ps.savefig()
plt.clf()
ha = dict(range=(-5,10), bins=100, histtype='step')
plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
color='b', label='Original', **ha)
plt.hist(((tim.getImage()-bg) * tim.inverr)[tim.inverr > 0].ravel(),
color='g', label='Minus Background', **ha)
plt.axvline(0, color='k', alpha=0.5)
plt.xlabel('Per-pixel intensity (Sigma)')
plt.legend()
plt.title(tag + ': background')
ps.savefig()
# Actually subtract the background!
tim.data -= bg
# Floor the per-pixel variances
if band in [1,2]:
# in Vega nanomaggies per pixel
floor_sigma = {1: 0.5, 2: 2.0}
with np.errstate(divide='ignore'):
new_ie = 1. / np.hypot(1./tim.inverr, floor_sigma[band])
new_ie[tim.inverr == 0] = 0.
if plots:
plt.clf()
plt.plot((1. / tim.inverr[tim.inverr>0]).ravel(), (1./new_ie[tim.inverr>0]).ravel(), 'b.')
plt.title('unWISE per-pixel error: %s band %i' % (tile.coadd_id, band))
plt.xlabel('original')
plt.ylabel('floored')
ps.savefig()
tim.inverr = new_ie
# Read mask file?
if get_masks:
from astrometry.util.resample import resample_with_wcs, OverlapError
# unwise_dir can be a colon-separated list of paths
tilemask = None
for d in tile.unwise_dir.split(':'):
fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id,
'unwise-%s-msk.fits.gz' % tile.coadd_id)
if os.path.exists(fn):
print('Reading unWISE mask file', fn)
x0,x1,y0,y1 = tim.roi
tilemask = fitsio.FITS(fn)[0][y0:y1,x0:x1]
break
if tilemask is None:
print('unWISE mask file for tile', tile.coadd_id, 'does not exist')
else:
try:
tanwcs = tim.wcs.wcs
assert(tanwcs.shape == tilemask.shape)
Yo,Xo,Yi,Xi,_ = resample_with_wcs(get_masks, tanwcs, intType=np.int16)
# Only deal with mask pixels that are set.
I, = np.nonzero(tilemask[Yi,Xi] > 0)
# Trim to unique area for this tile
rr,dd = get_masks.pixelxy2radec(Yo[I]+1, Xo[I]+1)
good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
I = I[good]
maskmap[Yo[I],Xo[I]] = tilemask[Yi[I], Xi[I]]
except OverlapError:
# Shouldn't happen by this point
print('No overlap between WISE tile', tile.coadd_id, 'and brick')
# The tiles have some overlap, so zero out pixels outside the
# tile's unique area.
th,tw = tim.shape
xx,yy = np.meshgrid(np.arange(tw), np.arange(th))
rr,dd = tim.wcs.wcs.pixelxy2radec(xx+1, yy+1)
unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
#print(np.sum(unique), 'of', (th*tw), 'pixels in this tile are unique')
tim.inverr[unique == False] = 0.
del xx,yy,rr,dd,unique
if plots:
sig1 = tim.sig1
plt.clf()
plt.imshow(tim.getImage() * (tim.inverr > 0),
interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=10 * sig1)
plt.colorbar()
tag = '%s W%i' % (tile.coadd_id, band)
plt.title('%s: tim data (unique)' % tag)
ps.savefig()
if pixelized_psf:
import unwise_psf
if (band == 1) or (band == 2):
# we only have updated PSFs for W1 and W2
psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id,
modelname='neo4_unwisecat')
else:
psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id)
if band == 4:
# oversample (the unwise_psf models are at native W4 5.5"/pix,
# while the unWISE coadds are made at 2.75"/pix.
ph,pw = psfimg.shape
subpsf = np.zeros((ph*2-1, pw*2-1), np.float32)
from astrometry.util.util import lanczos3_interpolate
xx,yy = np.meshgrid(np.arange(0., pw-0.51, 0.5, dtype=np.float32),
np.arange(0., ph-0.51, 0.5, dtype=np.float32))
xx = xx.ravel()
yy = yy.ravel()
ix = xx.astype(np.int32)
iy = yy.astype(np.int32)
dx = (xx - ix).astype(np.float32)
dy = (yy - iy).astype(np.float32)
psfimg = psfimg.astype(np.float32)
rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat], [psfimg])
if plots:
plt.clf()
plt.imshow(psfimg, interpolation='nearest', origin='lower')
plt.title('Original PSF model')
ps.savefig()
plt.clf()
plt.imshow(subpsf, interpolation='nearest', origin='lower')
plt.title('Subsampled PSF model')
ps.savefig()
psfimg = subpsf
del xx, yy, ix, iy, dx, dy
from tractor.psf import PixelizedPSF
psfimg /= psfimg.sum()
fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0}
psfimg *= fluxrescales[band]
tim.psf = PixelizedPSF(psfimg)
if psf_broadening is not None and not pixelized_psf:
# psf_broadening is a factor by which the PSF FWHMs
# should be scaled; the PSF is a little wider
# post-reactivation.
psf = tim.getPsf()
from tractor import GaussianMixturePSF
if isinstance(psf, GaussianMixturePSF):
#
print('Broadening PSF: from', psf)
p0 = psf.getParams()
pnames = psf.getParamNames()
p1 = [p * psf_broadening**2 if 'var' in name else p
for (p, name) in zip(p0, pnames)]
psf.setParams(p1)
print('Broadened PSF:', psf)
else:
print('WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf))
wcs = tim.wcs.wcs
ok,x,y = wcs.radec2pixelxy(ra, dec)
x = np.round(x - 1.).astype(int)
y = np.round(y - 1.).astype(int)
good = (x >= 0) * (x < tw) * (y >= 0) * (y < th)
# Which sources are in this brick's unique area?
usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
I, = np.nonzero(good * usrc)
nexp[I] = tim.nuims[y[I], x[I]]
if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'):
mjd[I] = (tim.mjdmin + tim.mjdmax) / 2.
phot.wise_coadd_id[I] = tile.coadd_id
central_flux[I] = tim.getImage()[y[I], x[I]]
del x,y,good,usrc
# PSF norm for depth
psf = tim.getPsf()
h,w = tim.shape
patch = psf.getPointSourcePatch(h//2, w//2).patch
psfnorm = np.sqrt(np.sum(patch**2))
# To handle zero-depth, we return 1/nanomaggies^2 units rather than mags.
psfdepth = 1. / (tim.sig1 / psfnorm)**2
phot.get(wband + '_psfdepth')[I] = psfdepth
tim.tile = tile
tims.append(tim)
if plots:
plt.clf()
mn,mx = 0.1, 20000
plt.hist(np.log10(np.clip(central_flux, mn, mx)), bins=100,
range=(np.log10(mn), np.log10(mx)))
logt = np.arange(0, 5)
plt.xticks(logt, ['%i' % i for i in 10.**logt])
plt.title('Central fluxes (W%i)' % band)
plt.axvline(np.log10(20000), color='k')
plt.axvline(np.log10(1000), color='k')
ps.savefig()
# Eddie's non-secret recipe:
#- central pixel <= 1000: 19x19 pix box size
#- central pixel in 1000 - 20000: 59x59 box size
#- central pixel > 20000 or saturated: 149x149 box size
#- object near "bright star": 299x299 box size
nbig = nmedium = nsmall = 0
for src,cflux in zip(cat, central_flux):
if cflux > 20000:
R = 100
nbig += 1
elif cflux > 1000:
R = 30
nmedium += 1
else:
R = 15
nsmall += 1
if isinstance(src, PointSource):
src.fixedRadius = R
else:
### FIXME -- sizes for galaxies..... can we set PSF size separately?
galrad = 0
# RexGalaxy is a subclass of ExpGalaxy
if isinstance(src, (ExpGalaxy, DevGalaxy)):
galrad = src.shape.re
elif isinstance(src, FixedCompositeGalaxy):
galrad = max(src.shapeExp.re, src.shapeDev.re)
pixscale = 2.75
src.halfsize = int(np.hypot(R, galrad * 5 / pixscale))
#print('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall, 'small')
minsb = 0.
fitsky = False
tractor = Tractor(tims, cat)
if use_ceres:
from tractor.ceres_optimizer import CeresOptimizer
tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block)
tractor.freezeParamsRecursive('*')
tractor.thawPathsTo(wanyband)
kwa = dict(fitstat_extras=[('pronexp', [tim.nims for tim in tims])])
t0 = Time()
R = tractor.optimize_forced_photometry(
minsb=minsb, mindlnp=1., sky=fitsky, fitstats=True,
variance=True, shared_params=False,
wantims=wantims, **kwa)
print('unWISE forced photometry took', Time() - t0)
if use_ceres:
term = R.ceres_status['termination']
# Running out of memory can cause failure to converge
# and term status = 2.
# Fail completely in this case.
if term != 0:
print('Ceres termination status:', term)
raise RuntimeError(
'Ceres terminated with status %i' % term)
if wantims:
ims1 = R.ims1
flux_invvars = R.IV
if R.fitstats is not None:
for k in fskeys:
x = getattr(R.fitstats, k)
fitstats[k] = np.array(x).astype(np.float32)
if save_fits:
for i,tim in enumerate(tims):
tile = tim.tile
(dat, mod, ie, chi, roi) = ims1[i]
wcshdr = fitsio.FITSHDR()
tim.wcs.wcs.add_to_header(wcshdr)
tag = 'fit-%s-w%i' % (tile.coadd_id, band)
fitsio.write('%s-data.fits' %
tag, dat, clobber=True, header=wcshdr)
fitsio.write('%s-mod.fits' % tag, mod,
clobber=True, header=wcshdr)
fitsio.write('%s-chi.fits' % tag, chi,
clobber=True, header=wcshdr)
if plots:
# Create models for just the brightest sources
bright_cat = [src for src in cat
if src.getBrightness().getBand(wanyband) > 1000]
print('Bright soures:', len(bright_cat))
btr = Tractor(tims, bright_cat)
for tim in tims:
mod = btr.getModelImage(tim)
tile = tim.tile
tag = '%s W%i' % (tile.coadd_id, band)
sig1 = tim.sig1
plt.clf()
plt.imshow(mod, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
plt.colorbar()
plt.title('%s: bright-star models' % tag)
ps.savefig()
if get_models:
for i,tim in enumerate(tims):
tile = tim.tile
(dat, mod, ie, chi, roi) = ims1[i]
models[(tile.coadd_id, band)] = (mod, dat, ie, tim.roi, tim.wcs.wcs)
if plots:
for i,tim in enumerate(tims):
tile = tim.tile
tag = '%s W%i' % (tile.coadd_id, band)
(dat, mod, ie, chi, roi) = ims1[i]
sig1 = tim.sig1
plt.clf()
plt.imshow(dat, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
plt.colorbar()
plt.title('%s: data' % tag)
ps.savefig()
plt.clf()
plt.imshow(mod, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
plt.colorbar()
plt.title('%s: model' % tag)
ps.savefig()
plt.clf()
plt.imshow(chi, interpolation='nearest', origin='lower',
cmap='gray', vmin=-5, vmax=+5)
plt.colorbar()
plt.title('%s: chi' % tag)
ps.savefig()
nm = np.array([src.getBrightness().getBand(wanyband) for src in cat])
nm_ivar = flux_invvars
# Sources out of bounds, eg, never change from their default
# (1-sigma or whatever) initial fluxes. Zero them out instead.
nm[nm_ivar == 0] = 0.
phot.set(wband + '_nanomaggies', nm.astype(np.float32))
phot.set(wband + '_nanomaggies_ivar', nm_ivar.astype(np.float32))
dnm = np.zeros(len(nm_ivar), np.float32)
okiv = (nm_ivar > 0)
dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32)
okflux = (nm > 0)
mag = np.zeros(len(nm), np.float32)
mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux])
).astype(np.float32)
dmag = np.zeros(len(nm), np.float32)
ok = (okiv * okflux)
dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok])
).astype(np.float32)
mag[np.logical_not(okflux)] = np.nan
dmag[np.logical_not(ok)] = np.nan
phot.set(wband + '_mag', mag)
phot.set(wband + '_mag_err', dmag)
for k in fskeys:
phot.set(wband + '_' + k, fitstats[k])
phot.set(wband + '_nexp', nexp)
if not np.all(mjd == 0):
phot.set(wband + '_mjd', mjd)
rtn = wphotduck()
rtn.phot = phot
rtn.models = None
rtn.maskmap = None
if get_models:
rtn.models = models
if get_masks:
rtn.maskmap = maskmap
return rtn
def compare_one(X):
from scipy.ndimage.filters import gaussian_filter
from scipy.ndimage.morphology import binary_dilation
from astrometry.util.resample import resample_with_wcs,OverlapError
(tim,sig,targetwcs, coimg,cow, veto, make_badcoadds, plots,ps) = X
if plots:
import pylab as plt
H,W = targetwcs.shape
img = gaussian_filter(tim.getImage(), sig)
try:
Yo,Xo,Yi,Xi,[rimg] = resample_with_wcs(
targetwcs, tim.subwcs, [img], intType=np.int16)
except OverlapError:
return None
del img
blurnorm = 1./(2. * np.sqrt(np.pi) * sig)
wt = tim.getInvvar()[Yi,Xi] / np.float32(blurnorm**2)
if Xi.dtype != np.int16:
Yi = Yi.astype(np.int16)
Xi = Xi.astype(np.int16)
# Compare against reference image...
maskedpix = np.zeros(tim.shape, np.uint8)
# Subtract this image from the coadd
otherwt = cow[Yo,Xo] - wt
otherimg = (coimg[Yo,Xo] - rimg*wt) / np.maximum(otherwt, 1e-16)
this_sig1 = 1./np.sqrt(np.median(wt[wt>0]))
## FIXME -- this image edges??
# Compute the error on our estimate of (thisimg - co) =
# sum in quadrature of the errors on thisimg and co.
with np.errstate(divide='ignore'):
diffvar = 1./wt + 1./otherwt
sndiff = (rimg - otherimg) / np.sqrt(diffvar)
with np.errstate(divide='ignore'):
reldiff = ((rimg - otherimg) / np.maximum(otherimg, this_sig1))
if plots:
plt.clf()
showimg = np.zeros((H,W),np.float32)
showimg[Yo,Xo] = otherimg
plt.subplot(2,3,1)
plt.imshow(showimg, interpolation='nearest', origin='lower', vmin=-0.01, vmax=0.1,
cmap='gray')
plt.title('other images')
showimg[Yo,Xo] = otherwt
plt.subplot(2,3,2)
plt.imshow(showimg, interpolation='nearest', origin='lower', vmin=0)
plt.title('other wt')
showimg[Yo,Xo] = sndiff
plt.subplot(2,3,3)
plt.imshow(showimg, interpolation='nearest', origin='lower', vmin=-10, vmax=10,cmap='RdBu_r')
plt.title('S/N diff')
showimg[Yo,Xo] = rimg
plt.subplot(2,3,4)
plt.imshow(showimg, interpolation='nearest', origin='lower', vmin=-0.01, vmax=0.1,
cmap='gray')
plt.title('this image')
showimg[Yo,Xo] = wt
plt.subplot(2,3,5)
plt.imshow(showimg, interpolation='nearest', origin='lower', vmin=0)
plt.title('this wt')
plt.suptitle(tim.name)
showimg[Yo,Xo] = reldiff
plt.subplot(2,3,6)
plt.imshow(showimg, interpolation='nearest', origin='lower', vmin=-4, vmax=4, cmap='RdBu_r')
plt.title('rel diff')
ps.savefig()
# from astrometry.util.plotutils import loghist
# plt.clf()
# loghist(sndiff.ravel(), reldiff.ravel(),
# bins=100)
# plt.xlabel('S/N difference')
# plt.ylabel('Relative difference')
# plt.title('Outliers: ' + tim.name)
# ps.savefig()
del otherimg
# Significant pixels
hotpix = ((sndiff > 5.) * (reldiff > 2.) *
(otherwt > 1e-16) * (wt > 0.) *
(veto[Yo,Xo] == False))
coldpix = ((sndiff < -5.) * (reldiff < -2.) *
(otherwt > 1e-16) * (wt > 0.) *
(veto[Yo,Xo] == False))
del reldiff, otherwt
if (not np.any(hotpix)) and (not np.any(coldpix)):
return None
hot = np.zeros((H,W), bool)
hot[Yo,Xo] = hotpix
cold = np.zeros((H,W), bool)
cold[Yo,Xo] = coldpix
del hotpix, coldpix
snmap = np.zeros((H,W), np.float32)
snmap[Yo,Xo] = sndiff
hot = binary_dilation(hot, iterations=1)
cold = binary_dilation(cold, iterations=1)
if plots:
heat = np.zeros(hot.shape, np.int8)
heat += hot
heat += -1 * cold
# "warm"
hot = np.logical_or(hot,
binary_dilation(hot, iterations=5) * (snmap > 3.))
hot = binary_dilation(hot, iterations=1)
cold = np.logical_or(cold,
binary_dilation(cold, iterations=5) * (snmap < -3.))
cold = binary_dilation(cold, iterations=1)
if plots:
heat += hot
heat +- -1*cold
# "lukewarm"
hot = np.logical_or(hot,
binary_dilation(hot, iterations=5) * (snmap > 2.))
hot = binary_dilation(hot, iterations=3)
cold = np.logical_or(cold,
binary_dilation(cold, iterations=5) * (snmap < -2.))
cold = binary_dilation(cold, iterations=3)
if plots:
heat += hot
heat +- -1*cold
plt.clf()
plt.imshow(heat, interpolation='nearest', origin='lower', cmap='RdBu_r', vmin=-3, vmax=+3)
plt.title(tim.name + ': outliers')
ps.savefig()
del heat
del snmap
badco = None
if make_badcoadds:
bad, = np.nonzero(hot[Yo,Xo])
badhot = (Yo[bad], Xo[bad], tim.getImage()[Yi[bad],Xi[bad]])
bad, = np.nonzero(cold[Yo,Xo])
badcold = (Yo[bad], Xo[bad], tim.getImage()[Yi[bad],Xi[bad]])
badco = badhot,badcold
# Actually do the masking!
# Resample "hot" (in brick coords) back to tim coords.
try:
mYo,mXo,mYi,mXi,nil = resample_with_wcs(
tim.subwcs, targetwcs, intType=np.int16)
except OverlapError:
return None
Ibad, = np.nonzero(hot[mYi,mXi])
Ibad2, = np.nonzero(cold[mYi,mXi])
info(tim, ': masking', len(Ibad), 'positive outlier pixels and', len(Ibad2), 'negative outlier pixels')
maskedpix[mYo[Ibad], mXo[Ibad]] = OUTLIER_POS
maskedpix[mYo[Ibad2], mXo[Ibad2]] = OUTLIER_NEG
return maskedpix,badco
def match_ngc(self, rr, dd, radius, exts, F, primhdr, meas):
'''
rr, dd: np arrays: RA,Dec centers of chips/amps
radius: scalar, radius of chips/amps
exts: list (same len as rr,dd) of extension indices
F: fitsio.FITS object
meas: measurement class
'''
I, J, d = match_radec(rr, dd, self.cat.ra, self.cat.dec, radius)
# plt.clf()
# angles = np.linspace(0, 2.*np.pi, 40)
# for r,d in zip(rr, dd):
# plt.plot(r + radius * np.sin(angles) / np.cos(np.deg2rad(d)), d + radius * np.cos(angles), 'b-')
# plt.plot(rr, dd, 'bo')
# ax = plt.axis()
# plt.plot(self.cat.ra, self.cat.dec, 'r.')
# plt.axis(ax)
# ps.savefig()
print('Matched', len(I), 'NGC objects')
if len(I) == 0:
return False
for j in J:
info = ngc_typenames.get(self.cat.classification[j].strip(), '')
print(' ', self.cat.name[j], info)
# Potential tweet texts and plot filenames
tweets = []
goodplots = []
for i, j in zip(I, J):
ext = exts[i]
obj = self.cat[j]
obj.name = obj.name.strip()
hdr = F[ext].read_header()
extname = hdr['EXTNAME'].strip()
expnum = primhdr['EXPNUM']
wcs = meas.get_wcs(hdr)
ok, x, y = wcs.radec2pixelxy(obj.ra, obj.dec)
x = x - 1
y = y - 1
# Choose cutout area
pixrad = 1.4 * obj.radius * 3600. / wcs.pixel_scale()
pixrad = max(pixrad, 100)
# print('radius:', obj.radius, 'pixel radius:', pixrad)
# HACK
#pixrad *= 3
r = pixrad
tt = '%s in exp %i ext %s (%i)' % (obj.name, expnum, extname, ext)
print(tt)
# Find the cutout region... does it actually overlap the chip?
H, W = wcs.shape
xl, xh = int(np.clip(x - r, 0,
W - 1)), int(np.clip(x + r, 0, W - 1))
yl, yh = int(np.clip(y - r, 0,
H - 1)), int(np.clip(y + r, 0, H - 1))
if xl == xh or yl == yh:
print('no actual overlap with image')
continue
sh, sw = yh - yl, xh - xl
if sh < 25 or sw < 25:
print('tiny overlap', sw, 'x', sh)
continue
# Measure the image!
flat = None
if self.read_flats:
band = meas.get_band(hdr)
flat = self.get_flat(band, ext, meas)
if flat is not None:
print('flat: range', flat.min(), flat.max(), 'median',
np.median(flat))
meas.ext = ext
meas.edge_trim = 20
debugps = None
if self.opt.ps:
debugps = ps
try:
M = meas.run(n_fwhm=1,
verbose=False,
get_image=True,
flat=flat,
ps=debugps)
except KeyboardInterrupt:
sys.exit(0)
except:
import traceback
print('Failed to measure file', meas.fn, 'ext', ext, ':')
traceback.print_exc()
continue
#print('Measured:', M.keys())
raw = M['image']
# Now repeat the cutout check with the trimmed image
wcs = M['wcs']
# Trim WCS to trimmed raw image shape
trim_x0, trim_y0 = M['trim_x0'], M['trim_y0']
H, W = raw.shape
wcs = wcs.get_subimage(trim_x0, trim_y0, W, H)
ok, x, y = wcs.radec2pixelxy(obj.ra, obj.dec)
x = x - 1
y = y - 1
xl, xh = int(np.clip(x - r, 0,
W - 1)), int(np.clip(x + r, 0, W - 1))
yl, yh = int(np.clip(y - r, 0,
H - 1)), int(np.clip(y + r, 0, H - 1))
if xl == xh or yl == yh:
print('no actual overlap with image')
continue
subimg = raw[yl:yh, xl:xh]
sh, sw = subimg.shape
if sh < 25 or sw < 25:
print('tiny overlap', sw, 'x', sh)
continue
subwcs = wcs.get_subimage(xl, yl, sw, sh)
if False:
# Flat image for the same CCD region.
flatfn = '/tmp/mzls/mos3.127506.fits'
# meas_class = get_measurer_class_for_file(flatfn)
# if meas_class is None:
# print('Failed to identify camera in', flatfn)
# return
# flatmeas = meas_class(flatfn, 0, self.nom)
# print('Flat meas:', flatmeas)
# flatmeas.ext = ext
# flathdr = fitsio.read_header(flatfn)
# flatmeas.primhdr = flathdr
# flatmeas.edge_trim = 20
# FM = flatmeas.run(n_fwhm=1, verbose=False, get_image=True)
# flatraw = FM['image']
F = fitsio.FITS(flatfn)
flatraw, flathdr = meas.read_raw(F, ext)
flatsub = flatraw[yl:yh, xl:xh]
zerofn = '/tmp/mzls/mos3.127522.fits'
F = fitsio.FITS(zerofn)
zeroraw, zerohdr = meas.read_raw(F, ext)
zerosub = zeroraw[yl:yh, xl:xh]
rawfn = meas.fn
F = fitsio.FITS(rawfn)
rawimg, rawhdr = meas.read_raw(F, ext)
rawsub = rawimg[yl:yh, xl:xh]
# Astrometric shifts
dx = M['dx']
dy = M['dy']
aff = M['affine']
x = (xl + xh) / 2
y = (yl + yh) / 2
#print('Affine correction terms:', aff)
adx = x - aff[0]
ady = y - aff[1]
corrx = aff[2] + aff[3] * adx + aff[4] * ady - adx
corry = aff[5] + aff[6] * adx + aff[7] * ady - ady
print('Affine correction', corrx, corry)
# Shift the 'subwcs' to account for astrometric offset
cx, cy = subwcs.get_crpix()
subwcs.set_crpix((cx - dx - corrx, cy - dy - corry))
# Now grab existing data from the LegacySurvey site
# What size of image are we going to request?
scale = 1.
if max(sh, sw) > 1024:
scale = 4.
elif max(sh, sw) > 512:
scale = 2.
rh, rw = int(np.ceil(sh / scale)), int(np.ceil(sw / scale))
# make it square
mx = max(rh, rw)
rw = rh = mx
fitsimgs = []
# We'll resample the new image into the existing-image WCS.
newimg = None
for layer in legacy_survey_layers:
url = (
'http://legacysurvey.org/viewer-dev/fits-cutout/?ra=%.4f&dec=%.4f&pixscale=%.3f&width=%i&height=%i&layer=%s'
% (obj.ra, obj.dec, subwcs.pixel_scale() * scale, rw, rh,
layer))
print('URL:', url)
r = requests.get(url)
ftmp = tempfile.NamedTemporaryFile()
ftmp.write(r.content)
ftmp.flush()
#fits,hdr = fitsio.read(ftmp.name, header=True)
fitsfile = fitsio.FITS(ftmp.name)
ftmp.close()
print('FITS file:', len(fitsfile), 'extensions')
hdr = fitsfile[0].read_header()
fits = fitsfile[0].read()
#print('fits:', fits)
if fits is None:
print('no coverage in layer', layer)
continue
# If you need to keep a copy (debugging...)
# f,tmpfn = tempfile.mkstemp(suffix='.fits')
# os.write(f, r.content)
# os.close(f)
# fits,hdr = fitsio.read(tmpfn, header=True)
# print('Wrote FITS to', tmpfn)
if np.all(fits == 0):
continue
### HACK -- surface brightness correction...
if layer == 'sdssco':
s = (subwcs.pixel_scale() / 0.396)
fits *= s**2
N, ww, hh = fits.shape
# pull out the image planes
imgs = [fits[n, :, :] for n in range(N)]
bands = hdr['BANDS'].strip()
fitsimgs.append((layer, bands, imgs))
# Resample the new image to this layer's WCS
if newimg is not None:
continue
thiswcs = Tan(hdr)
newimg = np.zeros((rh, rw), dtype=subimg.dtype)
try:
#Yo,Xo,Yi,Xi,rims = resample_with_wcs(thiswcs, subwcs)
# Laczos
Yo, Xo, Yi, Xi, rims = resample_with_wcs(
thiswcs, subwcs, [subimg])
except:
continue
#newimg[Yo,Xo] = subimg[Yi,Xi]
newimg[Yo, Xo] = rims[0]
if len(fitsimgs) == 0:
# No overlap with existing surveys
continue
#print()
newband = primhdr['FILTER'][0]
#print('New image is', newband)
if False:
mn, mx = np.percentile(flatsub, [25, 99])
plt.clf()
#plt.imshow(newflat, interpolation='nearest', origin='lower',
plt.imshow(flatsub,
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
#plt.colorbar()
plt.colorbar()
plt.savefig('ngcbot-flat.png')
med = np.median(newflat.ravel())
#mn,mx = np.percentile(newimg, [25,99])
#mn,mx = np.percentile(newimg, [25,90])
#mn,mx = np.percentile(rawsub, [25,95])
mn, mx = np.percentile(rawsub, [50, 95])
plt.clf()
# plt.subplot(1,2,1)
#plt.imshow(newimg, interpolation='nearest', origin='lower',
# plt.imshow(subimg, interpolation='nearest', origin='lower',
plt.imshow(rawsub,
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
plt.colorbar()
plt.savefig('ngcbot-unflattened.png')
plt.clf()
#plt.subplot(1,2,2)
#plt.imshow(newimg / (newflat / med), interpolation='nearest', origin='lower',
#plt.imshow(subimg / (flatsub / med), interpolation='nearest', origin='lower',
plt.imshow(rawsub / (flatsub / med),
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
plt.colorbar()
plt.savefig('ngcbot-flattened.png')
mn, mx = np.percentile(zerosub, [5, 95])
plt.clf()
plt.imshow(zerosub,
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
plt.colorbar()
plt.savefig('ngcbot-zero.png')
zp = M['zp']
zpscale = 10.**((zp - 22.5) / 2.5)
exptime = primhdr['EXPTIME']
newimg /= (zpscale * exptime)
nh, nw = newimg.shape
coverage = np.sum(newimg != 0) / float(nw * nh)
print('Fraction', coverage, 'of new image has data')
def my_rgb(imgs, bands, **kwargs):
return sdss_rgb(imgs,
bands,
scales=dict(g=6.0, r=3.4, i=2.5, z=2.2),
m=-0.02,
clip=False,
**kwargs)
def grayscale(img, band):
rgb = my_rgb([img, img, img], [band, band, band])
index = 'zrg'.index(newband)
gray = rgb[:, :, index]
return gray
# plot title args
targs = dict(fontsize=8)
# DEBUG
if self.opt.ps:
ocx, ocy = subwcs.get_crpix()
subwcs.set_crpix((cx, cy))
newimgA = np.zeros((rh, rw), dtype=subimg.dtype)
Yo, Xo, Yi, Xi, rims = resample_with_wcs(
thiswcs, subwcs, [subimg])
newimgA[Yo, Xo] = rims[0]
subwcs.set_crpix((cx - dx, cy - dy))
newimgB = np.zeros((rh, rw), dtype=subimg.dtype)
Yo, Xo, Yi, Xi, rims = resample_with_wcs(
thiswcs, subwcs, [subimg])
newimgB[Yo, Xo] = rims[0]
subwcs.set_crpix((cx - dx - corrx, cy - dy - corry))
newimgC = np.zeros((rh, rw), dtype=subimg.dtype)
Yo, Xo, Yi, Xi, rims = resample_with_wcs(
thiswcs, subwcs, [subimg])
newimgC[Yo, Xo] = rims[0]
#newgray = grayscale(newimg, newband)
#hi = np.percentile(newgray, 99.9)
grayargs = dict(interpolation='nearest',
origin='lower',
cmap='gray',
vmin=0.)
#vmin=0., vmax=hi, cmap='gray')
(layer, bands, imgs) = fitsimgs[0]
nicelayer = nicelayernames.get(layer, layer)
for band, img in zip(bands, imgs):
newbands = ['g', 'r', 'z']
newindex = dict(g=0, r=1, i=2, z=2)
if newindex[band] == newindex[newband]:
oldgray = grayscale(img, band)
plt.clf()
plt.imshow(oldgray, **grayargs)
plt.title(nicelayer)
ps.savefig()
plt.clf()
plt.imshow(grayscale(newimgA, newband), **grayargs)
#plt.imshow(newimgA, **grayargs)
plt.title('No astromety correction')
ps.savefig()
plt.clf()
plt.imshow(grayscale(newimgB, newband), **grayargs)
#plt.imshow(newimgB, **grayargs)
plt.title('Astromety shift only correction')
ps.savefig()
plt.clf()
plt.imshow(grayscale(newimgC, newband), **grayargs)
#plt.imshow(newimgC, **grayargs)
plt.title('Astromety shift+affine correction')
ps.savefig()
subwcs.set_crpix((ocx, ocy))
plt.clf()
plt.subplots_adjust(left=0.03, right=0.97, bottom=0.03)
NC = 1 + len(fitsimgs)
NR = 2
# New Image plot
newgray = grayscale(newimg, newband)
hi = np.percentile(newgray, 99.9)
grayargs = dict(interpolation='nearest',
origin='lower',
vmin=0.,
vmax=hi,
cmap='gray')
plt.subplot(NR, NC, 1)
plt.imshow(newgray, **grayargs)
plt.xticks([])
plt.yticks([])
plt.title('New image (%s)' % newband, **targs)
# Select images & bands for the New RGB image.
zeroimg = np.zeros_like(newimg)
newimgs = [zeroimg, zeroimg, zeroimg]
# sdss_rgb reverses the order, so do like grz.
newbands = ['g', 'r', 'z']
newindex = dict(g=0, r=1, i=2, z=2)
# Start with the new image plane of the New RGB image
j = newindex[newband]
newimgs[j] = newimg
newbands[j] = newband
#print('Setting band', newband, '(index %i)'%j, 'to new image')
# We wait to choose the scaling until after the "New RGB" image
# has been built, so we store the RGB images along the way...
rgbs = []
# In the link to the legacysurvey viewer, we select the best
# imaging layer. The DECaLS layer is listed first, so if it
# has three bands, it wins.
bestdata = None
bestn = 0
for i, (layer, bands, imgs) in enumerate(fitsimgs):
nicelayer = nicelayernames.get(layer, layer)
# For DECaLS missing bands: "--z"
nicebands = ''
goodbands = []
goodimgs = []
for band, img in zip(bands, imgs):
if np.all(img == 0.):
print('Band', band, 'of', nicelayer, 'is all zero')
nicebands += '-'
continue
goodbands.append(band)
goodimgs.append(img)
nicebands += band
#print(' ', nicelayer, band)
j = newindex[band]
if newimgs[j] is zeroimg:
#print(' Setting band', band, '(index %i)'%j, 'to', nicelayer)
newimgs[j] = img
newbands[j] = band
elif band == newbands[j]:
# patch empty regions if same band
#print(' Patching index %i from' % j, nicelayer, 'band', band)
Z = (newimgs[j] == 0)
newimgs[j][Z] = img[Z]
# z -> i
if newindex[band] == newindex[newband]:
# Old image grayscale
oldgray = grayscale(img, band)
plt.subplot(NR, NC, 2 + i)
plt.imshow(oldgray, **grayargs)
plt.xticks([])
plt.yticks([])
plt.title('%s (%s)' % (nicelayer, band), **targs)
if len(goodbands) == 1:
#rgb = grayscale(goodimgs[0], goodbands[0])
img, band = goodimgs[0], goodbands[0]
rgb = my_rgb([img, img, img], [band, band, band])
else:
rgb = my_rgb(imgs, bands)
if len(goodbands) > bestn:
bestn = len(goodbands)
bestdata = layer
# print('bands for', nicelayer, ':', bands, ', actually', nicebands)
rgbs.append(
(2 + i + NC, rgb, '%s (%s)' % (nicelayer, nicebands)))
# list to string
newbands = ''.join(newbands)
#print('Newbands:', newbands)
# New RGB
plt.subplot(NR, NC, 1 + NC)
rgb = my_rgb(newimgs, newbands)
lo = 0.
hi = np.percentile(rgb.ravel(), 99.9)
rgb = np.clip((rgb - lo) / (hi - lo), 0., 1.)
plt.imshow(rgb, interpolation='nearest', origin='lower')
plt.xticks([])
plt.yticks([])
plt.title('New+Old (%s)' % newbands, **targs)
# Old RGBs
for sp, rgb, tt in rgbs:
plt.subplot(NR, NC, sp)
rgb = np.clip((rgb - lo) / (hi - lo), 0., 1.)
plt.imshow(rgb, interpolation='nearest', origin='lower')
plt.xticks([])
plt.yticks([])
plt.title(tt, **targs)
# NGC classification
info = ''
info = ngc_typenames.get(obj.classification.strip(), '')
if len(info):
info = '(' + info + ') '
plt.suptitle(
'%s %sin %s %i-%s: %s band' %
(obj.name, info, nice_camera_name, expnum, extname, newband))
# Save / show plot
if self.opt.show:
plt.draw()
plt.show(block=False)
plt.pause(0.001)
plotfn = ('ngcbot-%i-%s-%s.png' %
(expnum, extname, obj.name.replace(' ', '_')))
if self.opt.plotdir:
plotfn = os.path.join(self.opt.plotdir, plotfn)
plt.savefig(plotfn)
print('Saved', plotfn)
good_coverage = 0.75
if self.opt.coverage is not None:
good_coverage = self.opt.coverage
if coverage > good_coverage:
goodplots.append(plotfn)
# Compose tweet text
if self.opt.tweet:
import urllib
url = 'http://legacysurvey.org/viewer/?ra=%.3f&dec=%.3f' % (
obj.ra, obj.dec)
if bestdata is not None:
url += '&layer=%s' % bestdata
url2 = ('http://ned.ipac.caltech.edu/cgi-bin/objsearch?' +
urllib.urlencode(
dict(objname=obj.name,
corr_z=1,
list_limit=5,
img_stamp='YES')))
dateobs = primhdr['DATE-OBS'].strip()
# 2017-03-06T00:15:40.101482
dateobs = dateobs[:19]
dateobs = dateobs.replace('T', ' ')
txt = ('%s observed %s %sin image %i-%s: %s band' %
(nice_camera_name, obj.name, info, expnum, extname,
newband) + ' at %s UT' % dateobs + '\n' + url + '\n' +
url2)
print('Tweet text:', txt)
if coverage > good_coverage:
tweets.append((txt, plotfn))
else:
print('Coverage', coverage, 'less than target',
good_coverage, 'so not tweeting')
# Select a random good-looking plot
if len(goodplots):
irandom = np.random.randint(0, len(goodplots))
# Copy that plot to ngcbot-latest.png
plotfn = goodplots[irandom]
import shutil
latest = 'ngcbot-latest.png'
print('Copying', plotfn, 'to', latest)
shutil.copy(plotfn, latest)
# Tweet one NGC object per exposure, chosen randomly.
if len(tweets):
assert (len(tweets) == len(goodplots))
txt, plotfn = tweets[irandom]
if self.opt.tweet:
send_tweet(txt, plotfn)
return True
def stage0(**kwargs):
ps = PlotSequence('cfht')
decals = CfhtDecals()
B = decals.get_bricks()
print('Bricks:')
B.about()
ra, dec = 190.0, 11.0
#bands = 'ugri'
bands = 'gri'
B.cut(np.argsort(degrees_between(ra, dec, B.ra, B.dec)))
print('Nearest bricks:', B.ra[:5], B.dec[:5], B.brickid[:5])
brick = B[0]
pixscale = 0.186
#W,H = 1024,1024
#W,H = 2048,2048
#W,H = 3600,3600
W, H = 4800, 4800
targetwcs = wcs_for_brick(brick, pixscale=pixscale, W=W, H=H)
ccdfn = 'cfht-ccds.fits'
if os.path.exists(ccdfn):
T = fits_table(ccdfn)
else:
T = get_ccd_list()
T.writeto(ccdfn)
print(len(T), 'CCDs')
T.cut(ccds_touching_wcs(targetwcs, T))
print(len(T), 'CCDs touching brick')
T.cut(np.array([b in bands for b in T.filter]))
print(len(T), 'in bands', bands)
ims = []
for t in T:
im = CfhtImage(t)
# magzp = hdr['PHOT_C'] + 2.5 * np.log10(hdr['EXPTIME'])
# fwhm = t.seeing / (pixscale * 3600)
# print '-> FWHM', fwhm, 'pix'
im.seeing = t.seeing
im.pixscale = t.pixscale
print('seeing', t.seeing)
print('pixscale', im.pixscale * 3600, 'arcsec/pix')
im.run_calibs(t.ra, t.dec, im.pixscale, W=t.width, H=t.height)
ims.append(im)
# Read images, clip to ROI
targetrd = np.array([
targetwcs.pixelxy2radec(x, y)
for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
])
keepims = []
tims = []
for im in ims:
print()
print('Reading expnum', im.expnum, 'name', im.extname, 'band', im.band,
'exptime', im.exptime)
band = im.band
wcs = im.read_wcs()
imh, imw = wcs.imageh, wcs.imagew
imgpoly = [(1, 1), (1, imh), (imw, imh), (imw, 1)]
ok, tx, ty = wcs.radec2pixelxy(targetrd[:-1, 0], targetrd[:-1, 1])
tpoly = zip(tx, ty)
clip = clip_polygon(imgpoly, tpoly)
clip = np.array(clip)
#print 'Clip', clip
if len(clip) == 0:
continue
x0, y0 = np.floor(clip.min(axis=0)).astype(int)
x1, y1 = np.ceil(clip.max(axis=0)).astype(int)
slc = slice(y0, y1 + 1), slice(x0, x1 + 1)
## FIXME -- it seems I got lucky and the cross product is
## negative == clockwise, as required by clip_polygon. One
## could check this and reverse the polygon vertex order.
# dx0,dy0 = tx[1]-tx[0], ty[1]-ty[0]
# dx1,dy1 = tx[2]-tx[1], ty[2]-ty[1]
# cross = dx0*dy1 - dx1*dy0
# print 'Cross:', cross
print('Image slice: x [%i,%i], y [%i,%i]' % (x0, x1, y0, y1))
print('Reading image from', im.imgfn, 'HDU', im.hdu)
img, imghdr = im.read_image(header=True, slice=slc)
goodpix = (img != 0)
print('Number of pixels == 0:', np.sum(img == 0))
print('Number of pixels != 0:', np.sum(goodpix))
if np.sum(goodpix) == 0:
continue
# print 'Image shape', img.shape
print('Image range', img.min(), img.max())
print('Goodpix image range:', (img[goodpix]).min(),
(img[goodpix]).max())
if img[goodpix].min() == img[goodpix].max():
print('No dynamic range in image')
continue
# print 'Reading invvar from', im.wtfn, 'HDU', im.hdu
# invvar = im.read_invvar(slice=slc)
# # print 'Invvar shape', invvar.shape
# # print 'Invvar range:', invvar.min(), invvar.max()
# invvar[goodpix == 0] = 0.
# if np.all(invvar == 0.):
# print 'Skipping zero-invvar image'
# continue
# assert(np.all(np.isfinite(img)))
# assert(np.all(np.isfinite(invvar)))
# assert(not(np.all(invvar == 0.)))
# # Estimate per-pixel noise via Blanton's 5-pixel MAD
# slice1 = (slice(0,-5,10),slice(0,-5,10))
# slice2 = (slice(5,None,10),slice(5,None,10))
# # print 'sliced shapes:', img[slice1].shape, img[slice2].shape
# # print 'good shape:', (goodpix[slice1] * goodpix[slice2]).shape
# # print 'good values:', np.unique(goodpix[slice1] * goodpix[slice2])
# # print 'sliced[good] shapes:', (img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].shape
# mad = np.median(np.abs(img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].ravel())
# sig1 = 1.4826 * mad / np.sqrt(2.)
# print 'MAD sig1:', sig1
# # invvar was 1 or 0
# invvar *= (1./(sig1**2))
# medsky = np.median(img[goodpix])
# Read full image for sig1 and sky estimate
fullimg = im.read_image()
fullgood = (fullimg != 0)
# Estimate per-pixel noise via Blanton's 5-pixel MAD
slice1 = (slice(0, -5, 10), slice(0, -5, 10))
slice2 = (slice(5, None, 10), slice(5, None, 10))
mad = np.median(
np.abs(fullimg[slice1] -
fullimg[slice2])[fullgood[slice1] *
fullgood[slice2]].ravel())
sig1 = 1.4826 * mad / np.sqrt(2.)
print('MAD sig1:', sig1)
medsky = np.median(fullimg[fullgood])
invvar = np.zeros_like(img)
invvar[goodpix] = 1. / sig1**2
# Median-smooth sky subtraction
plt.clf()
dimshow(np.round((img - medsky) / sig1), vmin=-3, vmax=5)
plt.title('Scalar median: %s' % im.name)
ps.savefig()
# medsky = np.zeros_like(img)
# # astrometry.util.util
# median_smooth(img, np.logical_not(goodpix), 256, medsky)
fullmed = np.zeros_like(fullimg)
median_smooth(fullimg - medsky, np.logical_not(fullgood), 256, fullmed)
fullmed += medsky
medimg = fullmed[slc]
plt.clf()
dimshow(np.round((img - medimg) / sig1), vmin=-3, vmax=5)
plt.title('Median filtered: %s' % im.name)
ps.savefig()
#print 'Subtracting median:', medsky
#img -= medsky
img -= medimg
primhdr = im.read_image_primary_header()
magzp = decals.get_zeropoint_for(im)
print('magzp', magzp)
zpscale = NanoMaggies.zeropointToScale(magzp)
print('zpscale', zpscale)
# Scale images to Nanomaggies
img /= zpscale
sig1 /= zpscale
invvar *= zpscale**2
orig_zpscale = zpscale
zpscale = 1.
assert (np.sum(invvar > 0) > 0)
print('After scaling:')
print('sig1', sig1)
print('invvar range', invvar.min(), invvar.max())
print('image range', img.min(), img.max())
assert (np.all(np.isfinite(img)))
assert (np.all(np.isfinite(invvar)))
assert (np.isfinite(sig1))
plt.clf()
lo, hi = -5 * sig1, 10 * sig1
n, b, p = plt.hist(img[goodpix].ravel(),
100,
range=(lo, hi),
histtype='step',
color='k')
xx = np.linspace(lo, hi, 200)
plt.plot(xx, max(n) * np.exp(-xx**2 / (2. * sig1**2)), 'r-')
plt.xlim(lo, hi)
plt.title('Pixel histogram: %s' % im.name)
ps.savefig()
twcs = ConstantFitsWcs(wcs)
if x0 or y0:
twcs.setX0Y0(x0, y0)
info = im.get_image_info()
fullh, fullw = info['dims']
# read fit PsfEx model
psfex = PsfEx.fromFits(im.psffitfn)
print('Read', psfex)
# HACK -- highly approximate PSF here!
#psf_fwhm = imghdr['FWHM']
#psf_fwhm = im.seeing
psf_fwhm = im.seeing / (im.pixscale * 3600)
print('PSF FWHM', psf_fwhm, 'pixels')
psf_sigma = psf_fwhm / 2.35
psf = NCircularGaussianPSF([psf_sigma], [1.])
print('img type', img.dtype)
tim = Image(img,
invvar=invvar,
wcs=twcs,
psf=psf,
photocal=LinearPhotoCal(zpscale, band=band),
sky=ConstantSky(0.),
name=im.name + ' ' + band)
tim.zr = [-3. * sig1, 10. * sig1]
tim.sig1 = sig1
tim.band = band
tim.psf_fwhm = psf_fwhm
tim.psf_sigma = psf_sigma
tim.sip_wcs = wcs
tim.x0, tim.y0 = int(x0), int(y0)
tim.psfex = psfex
tim.imobj = im
mn, mx = tim.zr
tim.ima = dict(interpolation='nearest',
origin='lower',
cmap='gray',
vmin=mn,
vmax=mx)
tims.append(tim)
keepims.append(im)
ims = keepims
print('Computing resampling...')
# save resampling params
for tim in tims:
wcs = tim.sip_wcs
subh, subw = tim.shape
subwcs = wcs.get_subimage(tim.x0, tim.y0, subw, subh)
tim.subwcs = subwcs
try:
Yo, Xo, Yi, Xi, rims = resample_with_wcs(targetwcs, subwcs, [], 2)
except OverlapError:
print('No overlap')
continue
if len(Yo) == 0:
continue
tim.resamp = (Yo, Xo, Yi, Xi)
print('Creating coadds...')
# Produce per-band coadds, for plots
coimgs = []
cons = []
for ib, band in enumerate(bands):
coimg = np.zeros((H, W), np.float32)
con = np.zeros((H, W), np.uint8)
for tim in tims:
if tim.band != band:
continue
(Yo, Xo, Yi, Xi) = tim.resamp
if len(Yo) == 0:
continue
nn = (tim.getInvvar()[Yi, Xi] > 0)
coimg[Yo, Xo] += tim.getImage()[Yi, Xi] * nn
con[Yo, Xo] += nn
# print
# print 'tim', tim.name
# print 'number of resampled pix:', len(Yo)
# reim = np.zeros_like(coimg)
# ren = np.zeros_like(coimg)
# reim[Yo,Xo] = tim.getImage()[Yi,Xi] * nn
# ren[Yo,Xo] = nn
# print 'number of resampled pix with positive invvar:', ren.sum()
# plt.clf()
# plt.subplot(2,2,1)
# mn,mx = [np.percentile(reim[ren>0], p) for p in [25,95]]
# print 'Percentiles:', mn,mx
# dimshow(reim, vmin=mn, vmax=mx)
# plt.colorbar()
# plt.subplot(2,2,2)
# dimshow(con)
# plt.colorbar()
# plt.subplot(2,2,3)
# dimshow(reim, vmin=tim.zr[0], vmax=tim.zr[1])
# plt.colorbar()
# plt.subplot(2,2,4)
# plt.hist(reim.ravel(), 100, histtype='step', color='b')
# plt.hist(tim.getImage().ravel(), 100, histtype='step', color='r')
# plt.suptitle('%s: %s' % (band, tim.name))
# ps.savefig()
coimg /= np.maximum(con, 1)
coimgs.append(coimg)
cons.append(con)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ps.savefig()
plt.clf()
for i, b in enumerate(bands):
plt.subplot(2, 2, i + 1)
dimshow(cons[i], ticks=False)
plt.title('%s band' % b)
plt.colorbar()
plt.suptitle('Number of exposures')
ps.savefig()
print('Grabbing SDSS sources...')
bandlist = [b for b in bands]
cat, T = get_sdss_sources(bandlist, targetwcs)
# record coordinates in target brick image
ok, T.tx, T.ty = targetwcs.radec2pixelxy(T.ra, T.dec)
T.tx -= 1
T.ty -= 1
T.itx = np.clip(np.round(T.tx).astype(int), 0, W - 1)
T.ity = np.clip(np.round(T.ty).astype(int), 0, H - 1)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(T.tx, T.ty, 'o', mec=green, mfc='none', ms=10, mew=1.5)
plt.axis(ax)
plt.title('SDSS sources')
ps.savefig()
print('Detmaps...')
# Render the detection maps
detmaps = dict([(b, np.zeros((H, W), np.float32)) for b in bands])
detivs = dict([(b, np.zeros((H, W), np.float32)) for b in bands])
for tim in tims:
iv = tim.getInvvar()
psfnorm = 1. / (2. * np.sqrt(np.pi) * tim.psf_sigma)
detim = tim.getImage().copy()
detim[iv == 0] = 0.
detim = gaussian_filter(detim, tim.psf_sigma) / psfnorm**2
detsig1 = tim.sig1 / psfnorm
subh, subw = tim.shape
detiv = np.zeros((subh, subw), np.float32) + (1. / detsig1**2)
detiv[iv == 0] = 0.
(Yo, Xo, Yi, Xi) = tim.resamp
detmaps[tim.band][Yo, Xo] += detiv[Yi, Xi] * detim[Yi, Xi]
detivs[tim.band][Yo, Xo] += detiv[Yi, Xi]
rtn = dict()
for k in [
'T', 'coimgs', 'cons', 'detmaps', 'detivs', 'targetrd', 'pixscale',
'targetwcs', 'W', 'H', 'bands', 'tims', 'ps', 'brick', 'cat'
]:
rtn[k] = locals()[k]
return rtn
def unwise_coadds(onegal,
galaxy=None,
radius_mosaic=30,
radius_mask=None,
pixscale=2.75,
ref_pixscale=0.262,
output_dir=None,
unwise_dir=None,
verbose=False,
log=None,
centrals=True):
'''Generate custom unWISE cutouts.
radius_mosaic and radius_mask in arcsec
pixscale: WISE pixel scale in arcsec/pixel; make this smaller than 2.75
to oversample.
'''
import fitsio
import matplotlib.pyplot as plt
from astrometry.util.util import Tan
from astrometry.util.fits import fits_table
from astrometry.libkd.spherematch import match_radec
from astrometry.util.resample import resample_with_wcs, ResampleError
from wise.forcedphot import unwise_tiles_touching_wcs
from wise.unwise import get_unwise_tractor_image
from tractor import Tractor, Image, NanoMaggies
from legacypipe.survey import imsave_jpeg
from legacypipe.catalog import read_fits_catalog
if galaxy is None:
galaxy = 'galaxy'
if output_dir is None:
output_dir = '.'
if unwise_dir is None:
unwise_dir = os.environ.get('UNWISE_COADDS_DIR')
if radius_mask is None:
radius_mask = radius_mosaic
radius_search = 5.0 # [arcsec]
else:
radius_search = radius_mask
# Initialize the WCS object.
W = H = np.ceil(2 * radius_mosaic / pixscale).astype('int') # [pixels]
targetwcs = Tan(onegal['RA'], onegal['DEC'], (W + 1) / 2.0, (H + 1) / 2.0,
-pixscale / 3600.0, 0.0, 0.0, pixscale / 3600.0, float(W),
float(H))
# Read the custom Tractor catalog.
tractorfile = os.path.join(output_dir, '{}-tractor.fits'.format(galaxy))
if not os.path.isfile(tractorfile):
print('Missing Tractor catalog {}'.format(tractorfile),
flush=True,
file=log)
return 0
primhdr = fitsio.read_header(tractorfile)
cat = fits_table(tractorfile)
print('Read {} sources from {}'.format(len(cat), tractorfile),
flush=True,
file=log)
keep = np.ones(len(cat)).astype(bool)
if centrals:
# Find the large central galaxy and mask out (ignore) all the models
# which are within its elliptical mask.
# This algorithm will have to change for mosaics not centered on large
# galaxies, e.g., in galaxy groups.
m1, m2, d12 = match_radec(cat.ra,
cat.dec,
onegal['RA'],
onegal['DEC'],
radius_search / 3600.0,
nearest=False)
if len(m1) == 0:
print('No central galaxies found at the central coordinates!',
flush=True,
file=log)
else:
pixfactor = ref_pixscale / pixscale # shift the optical Tractor positions
for mm in m1:
morphtype = cat.type[mm].strip()
if morphtype == 'EXP' or morphtype == 'COMP':
e1, e2, r50 = cat.shapeexp_e1[mm], cat.shapeexp_e2[
mm], cat.shapeexp_r[mm] # [arcsec]
elif morphtype == 'DEV' or morphtype == 'COMP':
e1, e2, r50 = cat.shapedev_e1[mm], cat.shapedev_e2[
mm], cat.shapedev_r[mm] # [arcsec]
else:
r50 = None
if r50:
majoraxis = r50 * 5 / pixscale # [pixels]
ba, phi = SGA.misc.convert_tractor_e1e2(e1, e2)
these = SGA.misc.ellipse_mask(W / 2, W / 2, majoraxis,
ba * majoraxis,
np.radians(phi),
cat.bx * pixfactor,
cat.by * pixfactor)
if np.sum(these) > 0:
#keep[these] = False
pass
print('Hack!')
keep[mm] = False
#srcs = read_fits_catalog(cat)
#_srcs = np.array(srcs)[~keep].tolist()
#mod = SGA.misc.srcs2image(_srcs, ConstantFitsWcs(targetwcs), psf_sigma=3.0)
#import matplotlib.pyplot as plt
##plt.imshow(mod, origin='lower') ; plt.savefig('junk.png')
#plt.imshow(np.log10(mod), origin='lower') ; plt.savefig('junk.png')
#pdb.set_trace()
srcs = read_fits_catalog(cat)
for src in srcs:
src.freezeAllBut('brightness')
#srcs_nocentral = np.array(srcs)[keep].tolist()
cat_nocentral = cat[keep]
## Find and remove all the objects within XX arcsec of the target
## coordinates.
#m1, m2, d12 = match_radec(T.ra, T.dec, onegal['RA'], onegal['DEC'], 5/3600.0, nearest=False)
#if len(d12) == 0:
# print('No matching galaxies found -- probably not what you wanted.')
# #raise ValueError
# nocentral = np.ones(len(T)).astype(bool)
#else:
# nocentral = ~np.isin(T.objid, T[m1].objid)
#T_nocentral = T[nocentral]
# Find and read the overlapping unWISE tiles. Assume the targetwcs is
# axis-aligned and that the edge midpoints yield the RA, Dec limits (true
# for TAN). Note: the way the roiradec box is used, the min/max order
# doesn't matter.
r, d = targetwcs.pixelxy2radec(np.array([1, W, W / 2, W / 2]),
np.array([H / 2, H / 2, 1, H]))
roiradec = [r[0], r[1], d[2], d[3]]
tiles = unwise_tiles_touching_wcs(targetwcs)
wbands = [1, 2, 3, 4]
wanyband = 'w'
vega_to_ab = dict(w1=2.699, w2=3.339, w3=5.174, w4=6.620)
# Convert the AB WISE fluxes in the Tractor catalog to Vega nanomaggies so
# they're consistent with the coadds, below.
for band in wbands:
f = cat.get('flux_w{}'.format(band))
e = cat.get('flux_ivar_w{}'.format(band))
print('Setting negative fluxes equal to zero!')
f[f < 0] = 0
#f[f/e < 3] = 0
f *= 10**(0.4 * vega_to_ab['w{}'.format(band)])
coimgs = [np.zeros((H, W), np.float32) for b in wbands]
comods = [np.zeros((H, W), np.float32) for b in wbands]
comods_nocentral = [np.zeros((H, W), np.float32) for b in wbands]
con = [np.zeros((H, W), np.uint8) for b in wbands]
for iband, band in enumerate(wbands):
for ii, src in enumerate(srcs):
src.setBrightness(
NanoMaggies(
**{wanyband: cat.get('flux_w{}'.format(band))[ii]}))
srcs_nocentral = np.array(srcs)[keep].tolist()
#srcs_nocentral = np.array(srcs)[nocentral].tolist()
# The tiles have some overlap, so for each source, keep the fit in the
# tile whose center is closest to the source.
for tile in tiles:
#print('Reading tile {}'.format(tile.coadd_id))
tim = get_unwise_tractor_image(unwise_dir,
tile.coadd_id,
band,
bandname=wanyband,
roiradecbox=roiradec)
if tim is None:
print('Actually, no overlap with tile {}'.format(
tile.coadd_id))
continue
print('Read image {} with shape {}'.format(tile.coadd_id,
tim.shape))
def _unwise_mod(tim, use_cat, use_srcs, margin=10):
# Select sources in play.
wisewcs = tim.wcs.wcs
timH, timW = tim.shape
ok, x, y = wisewcs.radec2pixelxy(use_cat.ra, use_cat.dec)
x = (x - 1.).astype(np.float32)
y = (y - 1.).astype(np.float32)
I = np.flatnonzero((x >= -margin) * (x < timW + margin) *
(y >= -margin) * (y < timH + margin))
#print('Found {} sources within the image + margin = {} pixels'.format(len(I), margin))
subcat = [use_srcs[i] for i in I]
tractor = Tractor([tim], subcat)
mod = tractor.getModelImage(0)
return mod
mod = _unwise_mod(tim, cat, srcs)
mod_nocentral = _unwise_mod(tim, cat_nocentral, srcs_nocentral)
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, tim.wcs.wcs)
except ResampleError:
continue
if len(Yo) == 0:
continue
# The models are already in AB nanomaggies, but the tiles / tims are
# in Vega nanomaggies, so convert them here.
coimgs[iband][Yo, Xo] += tim.getImage()[Yi, Xi]
comods[iband][Yo, Xo] += mod[Yi, Xi]
comods_nocentral[iband][Yo, Xo] += mod_nocentral[Yi, Xi]
con[iband][Yo, Xo] += 1
## Convert back to nanomaggies.
#vega2ab = vega_to_ab['w{}'.format(band)]
#coimgs[iband] *= 10**(-0.4 * vega2ab)
#comods[iband] *= 10**(-0.4 * vega2ab)
#comods_nocentral[iband] *= 10**(-0.4 * vega2ab)
for img, mod, mod_nocentral, n in zip(coimgs, comods, comods_nocentral,
con):
img /= np.maximum(n, 1)
mod /= np.maximum(n, 1)
mod_nocentral /= np.maximum(n, 1)
coresids = [img - mod for img, mod in list(zip(coimgs, comods))]
# Subtract the model image which excludes the central (comod_nocentral)
# from the data (coimg) to isolate the light of the central
# (coimg_central).
coimgs_central = [
img - mod for img, mod in list(zip(coimgs, comods_nocentral))
]
# Write out the final images with and without the central and converted into
# AB nanomaggies.
for coadd, imtype in zip((coimgs, comods, comods_nocentral),
('image', 'model', 'model-nocentral')):
for img, band in zip(coadd, wbands):
vega2ab = vega_to_ab['w{}'.format(band)]
fitsfile = os.path.join(
output_dir, '{}-{}-W{}.fits'.format(galaxy, imtype, band))
if verbose:
print('Writing {}'.format(fitsfile))
fitsio.write(fitsfile, img * 10**(-0.4 * vega2ab), clobber=True)
# Generate color WISE images.
kwa = dict(mn=-1, mx=100, arcsinh=0.5)
#kwa = dict(mn=-0.05, mx=1., arcsinh=0.5)
#kwa = dict(mn=-0.1, mx=2., arcsinh=None)
for imgs, imtype in zip(
(coimgs, comods, coresids, comods_nocentral, coimgs_central),
('image', 'model', 'resid', 'model-nocentral', 'image-central')):
rgb = _unwise_to_rgb(imgs[:2], **kwa) # W1, W2
jpgfile = os.path.join(output_dir,
'{}-{}-W1W2.jpg'.format(galaxy, imtype))
if verbose:
print('Writing {}'.format(jpgfile))
imsave_jpeg(jpgfile, rgb, origin='lower')
return 1
def map_decals_wl(req, ver, zoom, x, y):
tag = 'decals-wl'
ignoreCached = False
filename = None
forcecache = False
from decals import settings
savecache = settings.SAVE_CACHE
zoom = int(zoom)
zoomscale = 2.**zoom
x = int(x)
y = int(y)
if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale:
raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom,x,y))
ver = int(ver)
if not ver in tileversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
basedir = settings.DATA_DIR
tilefn = os.path.join(basedir, 'tiles', tag,
'%i/%i/%i/%i.jpg' % (ver, zoom, x, y))
if os.path.exists(tilefn) and not ignoreCached:
print('Cached:', tilefn)
return send_file(tilefn, 'image/jpeg', expires=oneyear,
modsince=req.META.get('HTTP_IF_MODIFIED_SINCE'),
filename=filename)
else:
print('Tile image does not exist:', tilefn)
from astrometry.util.resample import resample_with_wcs, OverlapError
from astrometry.util.util import Tan
from astrometry.libkd.spherematch import match_radec
from astrometry.util.fits import fits_table
from astrometry.util.starutil_numpy import degrees_between
import numpy as np
import fitsio
try:
wcs, W, H, zoomscale, zoom,x,y = get_tile_wcs(zoom, x, y)
except RuntimeError as e:
return HttpResponse(e.strerror)
mydir = os.path.join(basedir, 'coadd', 'weak-lensing')
rlo,d = wcs.pixelxy2radec(W, H/2)[-2:]
rhi,d = wcs.pixelxy2radec(1, H/2)[-2:]
r,d1 = wcs.pixelxy2radec(W/2, 1)[-2:]
r,d2 = wcs.pixelxy2radec(W/2, H)[-2:]
#dlo = min(d1, d2)
#dhi = max(d1, d2)
r,d = wcs.pixelxy2radec(W/2, H/2)[-2:]
rad = degrees_between(r, d, rlo, d1)
fn = os.path.join(mydir, 'index.fits')
if not os.path.exists(fn):
#
ii,rr,dd = [],[],[]
for i in range(1, 52852+1):
imgfn = os.path.join(mydir, 'map%i.fits' % i)
hdr = fitsio.read_header(imgfn)
r = hdr['CRVAL1']
d = hdr['CRVAL2']
ii.append(i)
rr.append(r)
dd.append(d)
T = fits_table()
T.ra = np.array(rr)
T.dec = np.array(dd)
T.i = np.array(ii)
T.writeto(fn)
T = fits_table(fn)
I,J,d = match_radec(T.ra, T.dec, r, d, rad + 0.2)
T.cut(I)
print(len(T), 'weak-lensing maps in range')
if len(I) == 0:
from django.http import HttpResponseRedirect
if forcecache:
# create symlink to blank.jpg!
trymakedirs(tilefn)
src = os.path.join(settings.STATIC_ROOT, 'blank.jpg')
if os.path.exists(tilefn):
os.unlink(tilefn)
os.symlink(src, tilefn)
print('Symlinked', tilefn, '->', src)
return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg')
r,d = wcs.pixelxy2radec([1,1,1,W/2,W,W,W,W/2],
[1,H/2,H,H,H,H/2,1,1])[-2:]
foundany = False
rimg = np.zeros((H,W), np.float32)
rn = np.zeros((H,W), np.uint8)
for tilei in T.i:
fn = os.path.join(mydir, 'map%i.fits' % tilei)
try:
bwcs = _read_tan_wcs(fn, 0)
except:
print('Failed to read WCS:', fn)
savecache = False
import traceback
import sys
traceback.print_exc(None, sys.stdout)
continue
foundany = True
print('Reading', fn)
ok,xx,yy = bwcs.radec2pixelxy(r, d)
xx = xx.astype(np.int)
yy = yy.astype(np.int)
imW,imH = int(bwcs.get_width()), int(bwcs.get_height())
M = 10
xlo = np.clip(xx.min() - M, 0, imW)
xhi = np.clip(xx.max() + M, 0, imW)
ylo = np.clip(yy.min() - M, 0, imH)
yhi = np.clip(yy.max() + M, 0, imH)
if xlo >= xhi or ylo >= yhi:
continue
subwcs = bwcs.get_subimage(xlo, ylo, xhi-xlo, yhi-ylo)
slc = slice(ylo,yhi), slice(xlo,xhi)
try:
f = fitsio.FITS(fn)[0]
img = f[slc]
del f
except:
print('Failed to read image and WCS:', fn)
savecache = False
import traceback
import sys
traceback.print_exc(None, sys.stdout)
continue
try:
Yo,Xo,Yi,Xi,nil = resample_with_wcs(wcs, subwcs, [], 3)
except OverlapError:
print('Resampling exception')
continue
rimg[Yo,Xo] += img[Yi,Xi]
rn [Yo,Xo] += 1
rimg /= np.maximum(rn, 1)
if forcecache:
savecache = True
if savecache:
trymakedirs(tilefn)
else:
import tempfile
f,tilefn = tempfile.mkstemp(suffix='.jpg')
os.close(f)
import pylab as plt
# S/N
#lo,hi = 1.5, 5.0
lo,hi = 0, 5.0
rgb = plt.cm.hot((rimg - lo) / (hi - lo))
plt.imsave(tilefn, rgb)
print('Wrote', tilefn)
return send_file(tilefn, 'image/jpeg', unlink=(not savecache),
filename=filename)
def cutout_jpg(req):
import tempfile
import fitsio
from astrometry.util.util import Tan
from astrometry.util.resample import resample_with_wcs
form = CutoutSearchForm(req.GET)
if not form.is_valid():
return HttpResponse('failed to parse request')
ra = form.cleaned_data['ra']
dec = form.cleaned_data['dec']
if ra is None or dec is None:
return HttpResponse('RA and Dec arguments are required')
size = form.cleaned_data['size']
if size is None:
size = 100
else:
size = min(256, size)
# Ignoring this for now...
# bandstr = form.cleaned_data['bands']
# bands = [int(c) for c in bandstr]
# bands = [b for b in bands if b in [1,2,3,4]]
version = form.cleaned_data['version']
radius = size/2. * 2.75/3600.
tiles = unwise_tiles_near_radec(ra, dec, radius)
tiles = list(tiles)
pixscale = 2.75 / 3600.
cowcs = Tan(*[float(x) for x in
[ra, dec, 0.5 + size/2., 0.5 + size/2.,
-pixscale, 0., 0., pixscale,
size, size]])
bands = [1,2]
coims = [np.zeros((size,size), np.float32) for b in bands]
con = np.zeros((size,size), int)
for tile in tiles:
coadd = tile.coadd
dirnm = os.path.join(settings.DATA_DIR, version, 'unwise-coadds', 'fulldepth', coadd[:3], coadd)
base = os.path.join(dirnm, 'unwise-%s' % coadd)
subwcs = None
ims = []
for band in bands:
fn = str(base + '-w%i-img-m.fits' % band)
if subwcs is None:
wcs = Tan(fn)
ok,x,y = wcs.radec2pixelxy(ra, dec)
x = int(round(x-1.))
y = int(round(y-1.))
x0 = x - size/2
x1 = x0 + size
y0 = y - size/2
y1 = y0 + size
if x1 <= 0 or y1 <= 0:
break
if x0 >= wcs.get_width() or y0 >= wcs.get_height():
break
x0 = max(x0, 0)
y0 = max(y0, 0)
x1 = min(x1, wcs.get_width())
y1 = min(y1, wcs.get_height())
subwcs = wcs.get_subimage(x0, y0, x1-x0, y1-y0)
if subwcs is None:
break
img = fitsio.FITS(fn)[0][y0:y1, x0:x1]
ims.append(img)
if subwcs is None:
continue
try:
Yo,Xo,Yi,Xi,rims = resample_with_wcs(cowcs, subwcs, ims, 3)
except:
continue
for rim,co in zip(rims, coims):
co[Yo,Xo] += rim
con[Yo,Xo] += 1
for co in coims:
co /= np.maximum(con, 1)
rgb = np.zeros((size,size,3), np.uint8)
lo,hi = -3,10
scale = 10.
rgb[:,:,2] = np.clip(255. * ((coims[1] / scale) - lo) / (hi-lo), 0, 255)
rgb[:,:,1] = np.clip(255. * (((coims[0]+coims[1])/2. / scale) - lo) / (hi-lo), 0, 255)
rgb[:,:,0] = np.clip(255. * ((coims[0] / scale) - lo) / (hi-lo), 0, 255)
f,tempfn = tempfile.mkstemp(suffix='.jpg')
os.close(f)
# import matplotlib
# matplotlib.use('Agg')
# import pylab as plt
# plt.imsave(tempfn, rgb, interpolation='nearest', origin='lower')
from PIL import Image
pic = Image.fromarray(rgb)
pic.save(tempfn)
return send_file(tempfn, 'image/jpeg', unlink=True)
def map_sdss(req, ver, zoom, x, y, savecache=None, tag='sdss',
get_images=False,
ignoreCached=False,
wcs=None,
forcecache=False,
forcescale=None,
**kwargs):
from decals import settings
if savecache is None:
savecache = settings.SAVE_CACHE
zoom = int(zoom)
zoomscale = 2.**zoom
x = int(x)
y = int(y)
if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale:
raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom,x,y))
ver = int(ver)
if not ver in tileversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
basedir = settings.DATA_DIR
tilefn = os.path.join(basedir, 'tiles', tag,
'%i/%i/%i/%i.jpg' % (ver, zoom, x, y))
if os.path.exists(tilefn) and not ignoreCached:
if get_images:
return None
return send_file(tilefn, 'image/jpeg', expires=oneyear,
modsince=req.META.get('HTTP_IF_MODIFIED_SINCE'))
if not savecache:
import tempfile
f,tilefn = tempfile.mkstemp(suffix='.jpg')
os.close(f)
if wcs is None:
try:
wcs, W, H, zoomscale, zoom,x,y = get_tile_wcs(zoom, x, y)
except RuntimeError as e:
if get_images:
return None
return HttpResponse(e.strerror)
else:
W = wcs.get_width()
H = wcs.get_height()
from astrometry.util.fits import fits_table
import numpy as np
from astrometry.libkd.spherematch import tree_build_radec, tree_search_radec
from astrometry.util.starutil_numpy import degrees_between, arcsec_between
from astrometry.util.resample import resample_with_wcs, OverlapError
from astrometry.util.util import Tan, Sip
import fitsio
print 'Tile wcs: center', wcs.radec_center(), 'pixel scale', wcs.pixel_scale()
global w_flist
global w_flist_tree
if w_flist is None:
w_flist = fits_table(os.path.join(settings.DATA_DIR, 'sdss',
'window_flist.fits'),
columns=['run','rerun','camcol','field','ra','dec','score'])
print 'Read', len(w_flist), 'window_flist entries'
w_flist.cut(w_flist.rerun == '301')
print 'Cut to', len(w_flist), 'in rerun 301'
w_flist_tree = tree_build_radec(w_flist.ra, w_flist.dec)
# SDSS field size
radius = 1.01 * np.hypot(10., 14.)/2. / 60.
# leaflet tile size
ra,dec = wcs.pixelxy2radec(W/2., H/2.)[-2:]
r0,d0 = wcs.pixelxy2radec(1, 1)[-2:]
r1,d1 = wcs.pixelxy2radec(W, H)[-2:]
radius = radius + max(degrees_between(ra,dec, r0,d0), degrees_between(ra,dec, r1,d1))
J = tree_search_radec(w_flist_tree, ra, dec, radius)
print len(J), 'overlapping SDSS fields found'
if len(J) == 0:
if get_images:
return None
if forcecache:
# create symlink to blank.jpg!
trymakedirs(tilefn)
src = os.path.join(settings.STATIC_ROOT, 'blank.jpg')
if os.path.exists(tilefn):
os.unlink(tilefn)
os.symlink(src, tilefn)
print 'Symlinked', tilefn, '->', src
from django.http import HttpResponseRedirect
return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg')
ww = [1, W*0.25, W*0.5, W*0.75, W]
hh = [1, H*0.25, H*0.5, H*0.75, H]
r,d = wcs.pixelxy2radec(
[1]*len(hh) + ww + [W]*len(hh) + list(reversed(ww)),
hh + [1]*len(ww) + list(reversed(hh)) + [H]*len(ww))[-2:]
scaled = 0
scalepat = None
scaledir = 'sdss'
if zoom <= 13 and forcescale is None:
# Get *actual* pixel scales at the top & bottom
r1,d1 = wcs.pixelxy2radec(W/2., H)[-2:]
r2,d2 = wcs.pixelxy2radec(W/2., H-1.)[-2:]
r3,d3 = wcs.pixelxy2radec(W/2., 1.)[-2:]
r4,d4 = wcs.pixelxy2radec(W/2., 2.)[-2:]
# Take the min = most zoomed-in
scale = min(arcsec_between(r1,d1, r2,d2), arcsec_between(r3,d3, r4,d4))
native_scale = 0.396
scaled = int(np.floor(np.log2(scale / native_scale)))
print 'Zoom:', zoom, 'x,y', x,y, 'Tile pixel scale:', scale, 'Scale step:', scaled
scaled = np.clip(scaled, 1, 7)
dirnm = os.path.join(basedir, 'scaled', scaledir)
scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(rerun)s', '%(run)i', '%(camcol)i', 'sdss-%(run)i-%(camcol)i-%(field)i-%(band)s.fits')
if forcescale is not None:
scaled = forcescale
bands = 'gri'
rimgs = [np.zeros((H,W), np.float32) for band in bands]
rns = [np.zeros((H,W), np.float32) for band in bands]
from astrometry.sdss import AsTransWrapper, DR9
sdss = DR9(basedir=settings.SDSS_DIR)
sdss.saveUnzippedFiles(settings.SDSS_DIR)
#sdss.setFitsioReadBZ2()
if settings.SDSS_PHOTOOBJS:
sdss.useLocalTree(photoObjs=settings.SDSS_PHOTOOBJS,
resolve=settings.SDSS_RESOLVE)
for jnum,j in enumerate(J):
print 'SDSS field', jnum, 'of', len(J), 'for zoom', zoom, 'x', x, 'y', y
im = w_flist[j]
if im.score >= 0.5:
weight = 1.
else:
weight = 0.001
for band,rimg,rn in zip(bands, rimgs, rns):
if im.rerun != '301':
continue
tmpsuff = '.tmp%08i' % np.random.randint(100000000)
basefn = sdss.retrieve('frame', im.run, im.camcol, field=im.field,
band=band, rerun=im.rerun, tempsuffix=tmpsuff)
if scaled > 0:
fnargs = dict(band=band, rerun=im.rerun, run=im.run,
camcol=im.camcol, field=im.field)
fn = get_scaled(scalepat, fnargs, scaled, basefn,
read_base_wcs=read_astrans, read_wcs=_read_sip_wcs)
print 'get_scaled:', fn
else:
fn = basefn
frame = None
if fn == basefn:
frame = sdss.readFrame(im.run, im.camcol, im.field, band,
filename=fn)
h,w = frame.getImageShape()
# Trim off the overlapping top of the image
# Wimp out and instead of trimming 128 pix, trim 124!
trim = 124
subh = h - trim
astrans = frame.getAsTrans()
fwcs = AsTransWrapper(astrans, w, subh)
fullimg = frame.getImage()
fullimg = fullimg[:-trim,:]
else:
fwcs = Sip(fn)
fitsimg = fitsio.FITS(fn)[0]
h,w = fitsimg.get_info()['dims']
fullimg = fitsimg.read()
try:
#Yo,Xo,Yi,Xi,nil = resample_with_wcs(wcs, fwcs, [], 3)
Yo,Xo,Yi,Xi,[resamp] = resample_with_wcs(wcs, fwcs, [fullimg], 2)
except OverlapError:
continue
if len(Xi) == 0:
#print 'No overlap'
continue
if sdssps is not None:
x0 = Xi.min()
x1 = Xi.max()
y0 = Yi.min()
y1 = Yi.max()
slc = (slice(y0,y1+1), slice(x0,x1+1))
if frame is not None:
img = frame.getImageSlice(slc)
else:
img = fitsimg[slc]
#rimg[Yo,Xo] += img[Yi-y0, Xi-x0]
rimg[Yo,Xo] += resamp * weight
rn [Yo,Xo] += weight
if sdssps is not None:
# goodpix = np.ones(img.shape, bool)
# fpM = sdss.readFpM(im.run, im.camcol, im.field, band)
# for plane in [ 'INTERP', 'SATUR', 'CR', 'GHOST' ]:
# fpM.setMaskedPixels(plane, goodpix, False, roi=[x0,x1,y0,y1])
plt.clf()
#ima = dict(vmin=-0.05, vmax=0.5)
#ima = dict(vmin=-0.5, vmax=2.)
ima = dict(vmax=np.percentile(img, 99))
plt.subplot(2,3,1)
dimshow(img, ticks=False, **ima)
plt.title('image')
rthis = np.zeros_like(rimg)
#rthis[Yo,Xo] += img[Yi-y0, Xi-x0]
rthis[Yo,Xo] += resamp
plt.subplot(2,3,2)
dimshow(rthis, ticks=False, **ima)
plt.title('resampled')
# plt.subplot(2,3,3)
# dimshow(goodpix, ticks=False, vmin=0, vmax=1)
# plt.title('good pix')
plt.subplot(2,3,4)
dimshow(rimg / np.maximum(rn, 1), ticks=False, **ima)
plt.title('coadd')
plt.subplot(2,3,5)
dimshow(rn, vmin=0, ticks=False)
plt.title('coverage: max %i' % rn.max())
plt.subplot(2,3,6)
rgb = sdss_rgb([rimg/np.maximum(rn,1)
for rimg,rn in zip(rimgs,rns)], bands)
dimshow(rgb)
plt.suptitle('SDSS %s, R/C/F %i/%i/%i' % (band, im.run, im.camcol, im.field))
sdssps.savefig()
for rimg,rn in zip(rimgs, rns):
rimg /= np.maximum(rn, 1e-3)
del rns
if get_images:
return rimgs
rgb = sdss_rgb(rimgs, bands)
trymakedirs(tilefn)
save_jpeg(tilefn, rgb)
print 'Wrote', tilefn
return send_file(tilefn, 'image/jpeg', unlink=(not savecache))
def render_into_wcs(self,
wcs,
zoom,
x,
y,
bands=None,
general_wcs=False,
scale=None,
tempfiles=None):
import numpy as np
from astrometry.util.resample import resample_with_wcs, OverlapError
if scale is None:
scale = self.get_scale(zoom, x, y, wcs)
#if bricks is None or len(bricks) == 0:
# print('No bricks touching WCS')
# return None
if bands is None:
bands = self.get_bands()
W = int(wcs.get_width())
H = int(wcs.get_height())
r, d = wcs.pixelxy2radec([1, 1, 1, W / 2, W, W, W, W / 2],
[1, H / 2, H, H, H, H / 2, 1, 1])[-2:]
#print('Tile RA,Decs:', r,d)
rimgs = []
# scaled down.....
# call get_filename to possibly generate scaled version
for band in bands:
brick = None
fn = self.get_filename(brick, band, scale, tempfiles=tempfiles)
print('scale', scale, 'band', band, 'fn', fn)
try:
bwcs = self.read_wcs(brick, band, scale, fn=fn)
if bwcs is None:
print('No such file:', brick, band, scale, 'fn', fn)
continue
except:
print('Failed to read WCS:', brick, band, scale, 'fn', fn)
savecache = False
import traceback
import sys
traceback.print_exc(None, sys.stdout)
continue
# Check for pixel overlap area
ok, xx, yy = bwcs.radec2pixelxy(r, d)
xx = xx.astype(np.int)
yy = yy.astype(np.int)
imW, imH = int(bwcs.get_width()), int(bwcs.get_height())
M = 10
xlo = np.clip(xx.min() - M, 0, imW)
xhi = np.clip(xx.max() + M, 0, imW)
ylo = np.clip(yy.min() - M, 0, imH)
yhi = np.clip(yy.max() + M, 0, imH)
#print('My WCS xx,yy', xx, yy, 'imW,H', imW, imH, 'xlohi', xlo,xhi, 'ylohi', ylo,yhi)
if xlo >= xhi or ylo >= yhi:
print('No pixel overlap')
return
subwcs = bwcs.get_subimage(xlo, ylo, xhi - xlo, yhi - ylo)
slc = slice(ylo, yhi), slice(xlo, xhi)
try:
img = self.read_image(brick, band, scale, slc, fn=fn)
except:
print('Failed to read image:', brickname, band, scale, 'fn',
fn)
savecache = False
import traceback
import sys
traceback.print_exc(None, sys.stdout)
continue
#print('Read image slice', img.shape)
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(wcs, subwcs, [], 3)
except OverlapError:
#debug('Resampling exception')
return
rimg = np.zeros((H, W), np.float32)
rimg[Yo, Xo] = img[Yi, Xi]
rimgs.append(rimg)
return rimgs
def coadds_ubercal(fulltims,
coaddtims=None,
plots=False,
plots2=False,
ps=None,
verbose=False):
"""Bring individual CCDs onto a common flux scale based on overlapping pixels.
fulltims - full-CCD tims, used to derive the corrections
coaddtims - tims sliced to just the pixels contributing to the output coadd
Some notes on the procedure:
A x = b
A: weights
A: shape noverlap x nimg
- entries have units of weights
x_i: offset to apply to image i
x: length nimg
- entries will have values of image pixels
b: (weighted) measured difference between image i and image j
b: length -- "noverlap" number of overlapping pairs of images -- filled-in elements in your array
- units of weighted image pixels
"""
from astrometry.util.resample import resample_with_wcs, OverlapError
band = fulltims[0].band
nimg = len(fulltims)
indx = np.arange(nimg)
## initialize A bigger than we will need, cut later
A = np.zeros((nimg * nimg, nimg), np.float32)
b = np.zeros((nimg * nimg), np.float32)
ioverlap = 0
for ii in indx:
for jj in indx[ii + 1:]:
try:
Yi, Xi, Yj, Xj, _ = resample_with_wcs(fulltims[ii].subwcs,
fulltims[jj].subwcs)
except OverlapError:
continue
imgI = fulltims[ii].getImage()[Yi, Xi]
imgJ = fulltims[jj].getImage()[Yj, Xj]
invI = fulltims[ii].getInvvar()[Yi, Xi]
invJ = fulltims[jj].getInvvar()[Yj, Xj]
good = (invI > 0) * (invJ > 0)
diff = (imgI - imgJ)[good]
iv = 1. / (1. / invI[good] + 1. / invJ[good])
delta = np.sum(diff * iv)
weight = np.sum(iv)
A[ioverlap, ii] = -weight
A[ioverlap, jj] = weight
b[ioverlap] = delta
ioverlap += 1
noverlap = ioverlap
A = A[:noverlap, :]
b = b[:noverlap]
#if verbose:
# print('A:')
# print(A)
# print('b:')
# print(b)
R = np.linalg.lstsq(A, b, rcond=None)
x = R[0]
print('Delta offsets to each image:')
print(x)
# Plot to assess the sign of the correction.
if plots2:
import matplotlib.pyplot as plt
plt.clf()
for j, (correction, fulltim) in enumerate(zip(x, fulltims)):
plt.subplot(nimg, 1, j + 1)
plt.hist(fulltim.data.ravel(),
bins=50,
histtype='step',
range=(-5, 5))
plt.axvline(-correction)
plt.title('Band %s: fulltim pix and -correction' % band)
ps.savefig()
if coaddtims is not None:
plt.clf()
for j, (correction,
ii) in enumerate(zip(x, np.arange(len(coaddtims)))):
plt.subplot(nimg, 1, j + 1)
plt.hist((coaddtims[ii].data + correction).ravel(),
bins=50,
histtype='step',
range=(-5, 5))
plt.title('Band %s: tim pix + correction' % band)
ps.savefig()
return x
def run_one(X):
k, sb, bricks, version = X
print(k, sb.brickname)
outfn = 'skybricks/sky-%s.fits.fz' % sb.brickname
if os.path.exists(outfn):
return True
I = np.flatnonzero((bricks.ra2 > sb.ra1) * (bricks.ra1 < sb.ra2) *
(bricks.dec2 > sb.dec1) * (bricks.dec1 < sb.dec2))
if len(I) == 0:
print('No bricks overlap')
return False
# 3600 + 1% margin on each side
w, h = 3672, 3672
if sb.dec >= 78.:
# In order to have fully-overlapping tiles at high Decs, we
# need larger maps. DR9 reaches a max skytile of Dec=+85,
# where the required size is 3724. Add a little margin.
w, h = 3744, 3744
binning = 4
# pixscale
cd = 1. / 3600.
fullw, fullh = w * binning, h * binning
fullcd = cd / binning
# There are really three states: no coverage, blob, no blob.
# Since blobs outside each brick's unique area do not appear in
# the files, we start skyblobs as zero, but also track the
# coverage so we can set !coverage to blob at the end.
skyblobs = np.zeros((fullh, fullw), bool)
covered = np.zeros((fullh, fullw), bool)
skywcs = Tan(sb.ra, sb.dec, (fullw + 1) / 2., (fullh + 1) / 2., -fullcd,
0., 0., fullcd, float(fullw), float(fullh))
for i in I:
brick = bricks[i]
#print('Blob', brickname)
#fn = 'cosmo/data/legacysurvey/dr9/%s/metrics/%s/blobs-%s.fits.gz' % (brick.hemi, brick.brickname[:3], brick.brickname)
#fn = 'dr9-south-blobs/blobs-%s.fits.gz' % (brick.brickname)
fn = '/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/%s/metrics/%s/blobs-%s.fits.gz' % (
brick.hemi, brick.brickname[:3], brick.brickname)
blobs, hdr = fitsio.read(fn, header=True)
wcs = Tan(hdr)
blobs = (blobs > -1)
try:
Yo, Xo, Yi, Xi, _ = resample_with_wcs(skywcs, wcs)
except NoOverlapError:
continue
skyblobs[Yo, Xo] |= blobs[Yi, Xi]
# coverage: nexp > 0 in any band
for band in ['g', 'r', 'z']:
fn = (
'/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/%s/coadd/%s/%s/legacysurvey-%s-nexp-%s.fits.fz'
% (brick.hemi, brick.brickname[:3], brick.brickname,
brick.brickname, band))
if not os.path.exists(fn):
continue
nexp = fitsio.read(fn)
covered[Yo, Xo] |= (nexp[Yi, Xi] > 0)
if not np.any(covered):
print('No coverage')
return False
# No coverage = equivalent to there being a blob there (ie,
# conservative for placing sky fibers)
skyblobs[covered == False] = True
# bin down, counting how many times 'skyblobs' is set
subcount = np.zeros((h, w), np.uint8)
for i in range(binning):
for j in range(binning):
subcount += skyblobs[i::binning, j::binning]
subwcs = Tan(sb.ra, sb.dec, (w + 1) / 2., (h + 1) / 2., -cd, 0., 0., cd,
float(w), float(h))
hdr = fitsio.FITSHDR()
hdr.add_record(
dict(name='SB_VER',
value=version,
comment='desi-sky-locations git version'))
subwcs.add_to_header(hdr)
fitsio.write(outfn,
subcount,
header=hdr,
clobber=True,
compress='GZIP',
tile_dims=(256, 256))
print('Wrote', outfn)
return True
def stage1(T=None, coimgs=None, cons=None, detmaps=None, detivs=None,
targetrd=None, pixscale=None, targetwcs=None, W=None,H=None,
bands=None, tims=None, ps=None, brick=None, cat=None):
orig_wcsxy0 = [tim.wcs.getX0Y0() for tim in tims]
hot = np.zeros((H,W), np.float32)
for band in bands:
detmap = detmaps[band] / np.maximum(1e-16, detivs[band])
detsn = detmap * np.sqrt(detivs[band])
hot = np.maximum(hot, detsn)
detmaps[band] = detmap
### FIXME -- ugri
for sedname,sed in [('Flat', (1.,1.,1.)), ('Red', (2.5, 1.0, 0.4))]:
sedmap = np.zeros((H,W), np.float32)
sediv = np.zeros((H,W), np.float32)
for iband,band in enumerate(bands):
# We convert the detmap to canonical band via
# detmap * w
# And the corresponding change to sig1 is
# sig1 * w
# So the invvar-weighted sum is
# (detmap * w) / (sig1**2 * w**2)
# = detmap / (sig1**2 * w)
sedmap += detmaps[band] * detivs[band] / sed[iband]
sediv += detivs [band] / sed[iband]**2
sedmap /= np.maximum(1e-16, sediv)
sedsn = sedmap * np.sqrt(sediv)
hot = np.maximum(hot, sedsn)
plt.clf()
dimshow(np.round(sedsn), vmin=0, vmax=10, cmap='hot')
plt.title('SED-matched detection filter: %s' % sedname)
ps.savefig()
peaks = (hot > 4)
blobs,nblobs = label(peaks)
print 'N detected blobs:', nblobs
blobslices = find_objects(blobs)
# Un-set catalog blobs
for x,y in zip(T.itx, T.ity):
# blob number
bb = blobs[y,x]
if bb == 0:
continue
# un-set 'peaks' within this blob
slc = blobslices[bb-1]
peaks[slc][blobs[slc] == bb] = 0
# Now, after having removed catalog sources, crank up the detection threshold
peaks &= (hot > 5)
# zero out the edges(?)
peaks[0 ,:] = peaks[:, 0] = 0
peaks[-1,:] = peaks[:,-1] = 0
peaks[1:-1, 1:-1] &= (hot[1:-1,1:-1] >= hot[0:-2,1:-1])
peaks[1:-1, 1:-1] &= (hot[1:-1,1:-1] >= hot[2: ,1:-1])
peaks[1:-1, 1:-1] &= (hot[1:-1,1:-1] >= hot[1:-1,0:-2])
peaks[1:-1, 1:-1] &= (hot[1:-1,1:-1] >= hot[1:-1,2: ])
# These are our peaks
pki = np.flatnonzero(peaks)
peaky,peakx = np.unravel_index(pki, peaks.shape)
print len(peaky), 'peaks'
crossa = dict(ms=10, mew=1.5)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(T.tx, T.ty, 'r+', **crossa)
plt.plot(peakx, peaky, '+', color=green, **crossa)
plt.axis(ax)
plt.title('SDSS + SED-matched detections')
ps.savefig()
### HACK -- high threshold again
# Segment, and record which sources fall into each blob
blobs,nblobs = label((hot > 20))
print 'N detected blobs:', nblobs
blobslices = find_objects(blobs)
T.blob = blobs[T.ity, T.itx]
blobsrcs = []
blobflux = []
fluximg = coimgs[1]
for blob in range(1, nblobs+1):
blobsrcs.append(np.flatnonzero(T.blob == blob))
bslc = blobslices[blob-1]
blobflux.append(np.sum(fluximg[bslc][blobs[bslc] == blob]))
# Fit the SDSS sources
for tim in tims:
tim.psfex.fitSavedData(*tim.psfex.splinedata)
tim.psf = tim.psfex
# How far down to render model profiles
minsigma = 0.1
for tim in tims:
tim.modelMinval = minsigma * tim.sig1
srcvariances = [[] for src in cat]
# Fit in order of flux
for blobnumber,iblob in enumerate(np.argsort(-np.array(blobflux))):
bslc = blobslices[iblob]
Isrcs = blobsrcs [iblob]
if len(Isrcs) == 0:
continue
print
print 'Blob', blobnumber, 'of', len(blobflux), ':', len(Isrcs), 'sources'
print 'Source indices:', Isrcs
print
# blob bbox in target coords
sy,sx = bslc
by0,by1 = sy.start, sy.stop
bx0,bx1 = sx.start, sx.stop
blobh,blobw = by1 - by0, bx1 - bx0
rr,dd = targetwcs.pixelxy2radec([bx0,bx0,bx1,bx1],[by0,by1,by1,by0])
alphas = [0.1, 0.3, 1.0]
subtims = []
for itim,tim in enumerate(tims):
h,w = tim.shape
ok,x,y = tim.subwcs.radec2pixelxy(rr,dd)
sx0,sx1 = x.min(), x.max()
sy0,sy1 = y.min(), y.max()
if sx1 < 0 or sy1 < 0 or sx1 > w or sy1 > h:
continue
sx0 = np.clip(int(np.floor(sx0)), 0, w-1)
sx1 = np.clip(int(np.ceil (sx1)), 0, w-1) + 1
sy0 = np.clip(int(np.floor(sy0)), 0, h-1)
sy1 = np.clip(int(np.ceil (sy1)), 0, h-1) + 1
subslc = slice(sy0,sy1),slice(sx0,sx1)
subimg = tim.getImage ()[subslc]
subie = tim.getInvError()[subslc]
subwcs = tim.getWcs().copy()
ox0,oy0 = orig_wcsxy0[itim]
subwcs.setX0Y0(ox0 + sx0, oy0 + sy0)
# Mask out inverr for pixels that are not within the blob.
subtarget = targetwcs.get_subimage(bx0, by0, blobw, blobh)
subsubwcs = tim.subwcs.get_subimage(int(sx0), int(sy0), int(sx1-sx0), int(sy1-sy0))
try:
Yo,Xo,Yi,Xi,rims = resample_with_wcs(subsubwcs, subtarget, [], 2)
except OverlapError:
print 'No overlap'
continue
if len(Yo) == 0:
continue
subie2 = np.zeros_like(subie)
I = np.flatnonzero(blobs[bslc][Yi, Xi] == (iblob+1))
subie2[Yo[I],Xo[I]] = subie[Yo[I],Xo[I]]
subie = subie2
# If the subimage (blob) is small enough, instantiate a
# constant PSF model in the center.
if sy1-sy0 < 100 and sx1-sx0 < 100:
subpsf = tim.psf.mogAt(ox0 + (sx0+sx1)/2., oy0 + (sy0+sy1)/2.)
else:
# Otherwise, instantiate a (shifted) spatially-varying
# PsfEx model.
subpsf = ShiftedPsf(tim.psf, ox0+sx0, oy0+sy0)
subtim = Image(data=subimg, inverr=subie, wcs=subwcs,
psf=subpsf, photocal=tim.getPhotoCal(),
sky=tim.getSky(), name=tim.name)
subtim.band = tim.band
subtim.sig1 = tim.sig1
subtim.modelMinval = tim.modelMinval
subtims.append(subtim)
subcat = Catalog(*[cat[i] for i in Isrcs])
subtr = Tractor(subtims, subcat)
subtr.freezeParam('images')
# Optimize individual sources in order of flux
fluxes = []
for src in subcat:
# HACK -- here we just *sum* the nanomaggies in each band. Bogus!
br = src.getBrightness()
flux = sum([br.getFlux(band) for band in bands])
fluxes.append(flux)
Ibright = np.argsort(-np.array(fluxes))
if len(Ibright) >= 5:
# -Remember the original subtim images
# -Compute initial models for each source (in each tim)
# -Subtract initial models from images
# -During fitting, for each source:
# -add back in the source's initial model (to each tim)
# -fit, with Catalog([src])
# -subtract final model (from each tim)
# -Replace original subtim images
#
# --Might want to omit newly-added detection-filter sources, since their
# fluxes are bogus.
# Remember original tim images
orig_timages = [tim.getImage().copy() for tim in subtims]
initial_models = []
# Create initial models for each tim x each source
for tim in subtims:
mods = []
for src in subcat:
mod = src.getModelPatch(tim)
mods.append(mod)
if mod is not None:
if not np.all(np.isfinite(mod.patch)):
print 'Non-finite mod patch'
print 'source:', src
print 'tim:', tim
print 'PSF:', tim.getPsf()
assert(np.all(np.isfinite(mod.patch)))
mod.addTo(tim.getImage(), scale=-1)
initial_models.append(mods)
# For sources in decreasing order of brightness
for numi,i in enumerate(Ibright):
tsrc = Time()
print 'Fitting source', i, '(%i of %i in blob)' % (numi, len(Ibright))
src = subcat[i]
print src
srctractor = Tractor(subtims, [src])
srctractor.freezeParams('images')
# Add this source's initial model back in.
for tim,mods in zip(subtims, initial_models):
mod = mods[i]
if mod is not None:
mod.addTo(tim.getImage())
print 'Optimizing:', srctractor
srctractor.printThawedParams()
for step in range(50):
dlnp,X,alpha = srctractor.optimize(priors=False, shared_params=False,
alphas=alphas)
print 'dlnp:', dlnp, 'src', src
if dlnp < 0.1:
break
for tim in subtims:
mod = src.getModelPatch(tim)
if mod is not None:
mod.addTo(tim.getImage(), scale=-1)
for tim,img in zip(subtims, orig_timages):
tim.data = img
del orig_timages
del initial_models
else:
# Fit sources one at a time, but don't subtract other models
subcat.freezeAllParams()
for numi,i in enumerate(Ibright):
tsrc = Time()
print 'Fitting source', i, '(%i of %i in blob)' % (numi, len(Ibright))
print subcat[i]
subcat.freezeAllBut(i)
print 'Optimizing:', subtr
subtr.printThawedParams()
for step in range(10):
dlnp,X,alpha = subtr.optimize(priors=False, shared_params=False,
alphas=alphas)
print 'dlnp:', dlnp
if dlnp < 0.1:
break
print 'Fitting source took', Time()-tsrc
print subcat[i]
if len(Isrcs) > 1 and len(Isrcs) <= 10:
tfit = Time()
# Optimize all at once?
subcat.thawAllParams()
print 'Optimizing:', subtr
subtr.printThawedParams()
for step in range(20):
dlnp,X,alpha = subtr.optimize(priors=False, shared_params=False,
alphas=alphas)
print 'dlnp:', dlnp
if dlnp < 0.1:
break
# Variances
subcat.thawAllRecursive()
subcat.freezeAllParams()
for isub,srci in enumerate(Isrcs):
print 'Variances for source', srci
subcat.thawParam(isub)
src = subcat[isub]
print 'Source', src
print 'Params:', src.getParamNames()
if isinstance(src, (DevGalaxy, ExpGalaxy)):
src.shape = EllipseE.fromEllipseESoft(src.shape)
elif isinstance(src, FixedCompositeGalaxy):
src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp)
src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev)
print 'Converted ellipse:', src
allderivs = subtr.getDerivs()
for iparam,derivs in enumerate(allderivs):
dchisq = 0
for deriv,tim in derivs:
h,w = tim.shape
deriv.clipTo(w,h)
ie = tim.getInvError()
slc = deriv.getSlice(ie)
chi = deriv.patch * ie[slc]
dchisq += (chi**2).sum()
if dchisq == 0.:
v = np.nan
else:
v = 1./dchisq
srcvariances[srci].append(v)
assert(len(srcvariances[srci]) == subcat[isub].numberOfParams())
subcat.freezeParam(isub)
cat.thawAllRecursive()
for i,src in enumerate(cat):
print 'Source', i, src
print 'variances:', srcvariances[i]
print len(srcvariances[i]), 'vs', src.numberOfParams()
if len(srcvariances[i]) != src.numberOfParams():
# This can happen for sources outside the brick bounds: they never get optimized?
print 'Warning: zeroing variances for source', src
srcvariances[i] = [0]*src.numberOfParams()
if isinstance(src, (DevGalaxy, ExpGalaxy)):
src.shape = EllipseE.fromEllipseESoft(src.shape)
elif isinstance(src, FixedCompositeGalaxy):
src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp)
src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev)
assert(len(srcvariances[i]) == src.numberOfParams())
variances = np.hstack(srcvariances)
assert(len(variances) == cat.numberOfParams())
return dict(cat=cat, variances=variances)
def stage0(**kwargs):
ps = PlotSequence('cfht')
decals = CfhtDecals()
B = decals.get_bricks()
print 'Bricks:'
B.about()
ra,dec = 190.0, 11.0
#bands = 'ugri'
bands = 'gri'
B.cut(np.argsort(degrees_between(ra, dec, B.ra, B.dec)))
print 'Nearest bricks:', B.ra[:5], B.dec[:5], B.brickid[:5]
brick = B[0]
pixscale = 0.186
#W,H = 1024,1024
#W,H = 2048,2048
#W,H = 3600,3600
W,H = 4800,4800
targetwcs = wcs_for_brick(brick, pixscale=pixscale, W=W, H=H)
ccdfn = 'cfht-ccds.fits'
if os.path.exists(ccdfn):
T = fits_table(ccdfn)
else:
T = get_ccd_list()
T.writeto(ccdfn)
print len(T), 'CCDs'
T.cut(ccds_touching_wcs(targetwcs, T))
print len(T), 'CCDs touching brick'
T.cut(np.array([b in bands for b in T.filter]))
print len(T), 'in bands', bands
ims = []
for t in T:
im = CfhtImage(t)
# magzp = hdr['PHOT_C'] + 2.5 * np.log10(hdr['EXPTIME'])
# fwhm = t.seeing / (pixscale * 3600)
# print '-> FWHM', fwhm, 'pix'
im.seeing = t.seeing
im.pixscale = t.pixscale
print 'seeing', t.seeing
print 'pixscale', im.pixscale*3600, 'arcsec/pix'
im.run_calibs(t.ra, t.dec, im.pixscale, W=t.width, H=t.height)
ims.append(im)
# Read images, clip to ROI
targetrd = np.array([targetwcs.pixelxy2radec(x,y) for x,y in
[(1,1),(W,1),(W,H),(1,H),(1,1)]])
keepims = []
tims = []
for im in ims:
print
print 'Reading expnum', im.expnum, 'name', im.extname, 'band', im.band, 'exptime', im.exptime
band = im.band
wcs = im.read_wcs()
imh,imw = wcs.imageh,wcs.imagew
imgpoly = [(1,1),(1,imh),(imw,imh),(imw,1)]
ok,tx,ty = wcs.radec2pixelxy(targetrd[:-1,0], targetrd[:-1,1])
tpoly = zip(tx,ty)
clip = clip_polygon(imgpoly, tpoly)
clip = np.array(clip)
#print 'Clip', clip
if len(clip) == 0:
continue
x0,y0 = np.floor(clip.min(axis=0)).astype(int)
x1,y1 = np.ceil (clip.max(axis=0)).astype(int)
slc = slice(y0,y1+1), slice(x0,x1+1)
## FIXME -- it seems I got lucky and the cross product is
## negative == clockwise, as required by clip_polygon. One
## could check this and reverse the polygon vertex order.
# dx0,dy0 = tx[1]-tx[0], ty[1]-ty[0]
# dx1,dy1 = tx[2]-tx[1], ty[2]-ty[1]
# cross = dx0*dy1 - dx1*dy0
# print 'Cross:', cross
print 'Image slice: x [%i,%i], y [%i,%i]' % (x0,x1, y0,y1)
print 'Reading image from', im.imgfn, 'HDU', im.hdu
img,imghdr = im.read_image(header=True, slice=slc)
goodpix = (img != 0)
print 'Number of pixels == 0:', np.sum(img == 0)
print 'Number of pixels != 0:', np.sum(goodpix)
if np.sum(goodpix) == 0:
continue
# print 'Image shape', img.shape
print 'Image range', img.min(), img.max()
print 'Goodpix image range:', (img[goodpix]).min(), (img[goodpix]).max()
if img[goodpix].min() == img[goodpix].max():
print 'No dynamic range in image'
continue
# print 'Reading invvar from', im.wtfn, 'HDU', im.hdu
# invvar = im.read_invvar(slice=slc)
# # print 'Invvar shape', invvar.shape
# # print 'Invvar range:', invvar.min(), invvar.max()
# invvar[goodpix == 0] = 0.
# if np.all(invvar == 0.):
# print 'Skipping zero-invvar image'
# continue
# assert(np.all(np.isfinite(img)))
# assert(np.all(np.isfinite(invvar)))
# assert(not(np.all(invvar == 0.)))
# # Estimate per-pixel noise via Blanton's 5-pixel MAD
# slice1 = (slice(0,-5,10),slice(0,-5,10))
# slice2 = (slice(5,None,10),slice(5,None,10))
# # print 'sliced shapes:', img[slice1].shape, img[slice2].shape
# # print 'good shape:', (goodpix[slice1] * goodpix[slice2]).shape
# # print 'good values:', np.unique(goodpix[slice1] * goodpix[slice2])
# # print 'sliced[good] shapes:', (img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].shape
# mad = np.median(np.abs(img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].ravel())
# sig1 = 1.4826 * mad / np.sqrt(2.)
# print 'MAD sig1:', sig1
# # invvar was 1 or 0
# invvar *= (1./(sig1**2))
# medsky = np.median(img[goodpix])
# Read full image for sig1 and sky estimate
fullimg = im.read_image()
fullgood = (fullimg != 0)
# Estimate per-pixel noise via Blanton's 5-pixel MAD
slice1 = (slice(0,-5,10),slice(0,-5,10))
slice2 = (slice(5,None,10),slice(5,None,10))
mad = np.median(np.abs(fullimg[slice1] - fullimg[slice2])[fullgood[slice1] * fullgood[slice2]].ravel())
sig1 = 1.4826 * mad / np.sqrt(2.)
print 'MAD sig1:', sig1
medsky = np.median(fullimg[fullgood])
invvar = np.zeros_like(img)
invvar[goodpix] = 1./sig1**2
# Median-smooth sky subtraction
plt.clf()
dimshow(np.round((img-medsky) / sig1), vmin=-3, vmax=5)
plt.title('Scalar median: %s' % im.name)
ps.savefig()
# medsky = np.zeros_like(img)
# # astrometry.util.util
# median_smooth(img, np.logical_not(goodpix), 256, medsky)
fullmed = np.zeros_like(fullimg)
median_smooth(fullimg - medsky, np.logical_not(fullgood), 256, fullmed)
fullmed += medsky
medimg = fullmed[slc]
plt.clf()
dimshow(np.round((img - medimg) / sig1), vmin=-3, vmax=5)
plt.title('Median filtered: %s' % im.name)
ps.savefig()
#print 'Subtracting median:', medsky
#img -= medsky
img -= medimg
primhdr = im.read_image_primary_header()
magzp = decals.get_zeropoint_for(im)
print 'magzp', magzp
zpscale = NanoMaggies.zeropointToScale(magzp)
print 'zpscale', zpscale
# Scale images to Nanomaggies
img /= zpscale
sig1 /= zpscale
invvar *= zpscale**2
orig_zpscale = zpscale
zpscale = 1.
assert(np.sum(invvar > 0) > 0)
print 'After scaling:'
print 'sig1', sig1
print 'invvar range', invvar.min(), invvar.max()
print 'image range', img.min(), img.max()
assert(np.all(np.isfinite(img)))
assert(np.all(np.isfinite(invvar)))
assert(np.isfinite(sig1))
plt.clf()
lo,hi = -5*sig1, 10*sig1
n,b,p = plt.hist(img[goodpix].ravel(), 100, range=(lo,hi), histtype='step', color='k')
xx = np.linspace(lo, hi, 200)
plt.plot(xx, max(n)*np.exp(-xx**2 / (2.*sig1**2)), 'r-')
plt.xlim(lo,hi)
plt.title('Pixel histogram: %s' % im.name)
ps.savefig()
twcs = ConstantFitsWcs(wcs)
if x0 or y0:
twcs.setX0Y0(x0,y0)
info = im.get_image_info()
fullh,fullw = info['dims']
# read fit PsfEx model
psfex = PsfEx.fromFits(im.psffitfn)
print 'Read', psfex
# HACK -- highly approximate PSF here!
#psf_fwhm = imghdr['FWHM']
#psf_fwhm = im.seeing
psf_fwhm = im.seeing / (im.pixscale * 3600)
print 'PSF FWHM', psf_fwhm, 'pixels'
psf_sigma = psf_fwhm / 2.35
psf = NCircularGaussianPSF([psf_sigma],[1.])
print 'img type', img.dtype
tim = Image(img, invvar=invvar, wcs=twcs, psf=psf,
photocal=LinearPhotoCal(zpscale, band=band),
sky=ConstantSky(0.), name=im.name + ' ' + band)
tim.zr = [-3. * sig1, 10. * sig1]
tim.sig1 = sig1
tim.band = band
tim.psf_fwhm = psf_fwhm
tim.psf_sigma = psf_sigma
tim.sip_wcs = wcs
tim.x0,tim.y0 = int(x0),int(y0)
tim.psfex = psfex
tim.imobj = im
mn,mx = tim.zr
tim.ima = dict(interpolation='nearest', origin='lower', cmap='gray',
vmin=mn, vmax=mx)
tims.append(tim)
keepims.append(im)
ims = keepims
print 'Computing resampling...'
# save resampling params
for tim in tims:
wcs = tim.sip_wcs
subh,subw = tim.shape
subwcs = wcs.get_subimage(tim.x0, tim.y0, subw, subh)
tim.subwcs = subwcs
try:
Yo,Xo,Yi,Xi,rims = resample_with_wcs(targetwcs, subwcs, [], 2)
except OverlapError:
print 'No overlap'
continue
if len(Yo) == 0:
continue
tim.resamp = (Yo,Xo,Yi,Xi)
print 'Creating coadds...'
# Produce per-band coadds, for plots
coimgs = []
cons = []
for ib,band in enumerate(bands):
coimg = np.zeros((H,W), np.float32)
con = np.zeros((H,W), np.uint8)
for tim in tims:
if tim.band != band:
continue
(Yo,Xo,Yi,Xi) = tim.resamp
if len(Yo) == 0:
continue
nn = (tim.getInvvar()[Yi,Xi] > 0)
coimg[Yo,Xo] += tim.getImage ()[Yi,Xi] * nn
con [Yo,Xo] += nn
# print
# print 'tim', tim.name
# print 'number of resampled pix:', len(Yo)
# reim = np.zeros_like(coimg)
# ren = np.zeros_like(coimg)
# reim[Yo,Xo] = tim.getImage()[Yi,Xi] * nn
# ren[Yo,Xo] = nn
# print 'number of resampled pix with positive invvar:', ren.sum()
# plt.clf()
# plt.subplot(2,2,1)
# mn,mx = [np.percentile(reim[ren>0], p) for p in [25,95]]
# print 'Percentiles:', mn,mx
# dimshow(reim, vmin=mn, vmax=mx)
# plt.colorbar()
# plt.subplot(2,2,2)
# dimshow(con)
# plt.colorbar()
# plt.subplot(2,2,3)
# dimshow(reim, vmin=tim.zr[0], vmax=tim.zr[1])
# plt.colorbar()
# plt.subplot(2,2,4)
# plt.hist(reim.ravel(), 100, histtype='step', color='b')
# plt.hist(tim.getImage().ravel(), 100, histtype='step', color='r')
# plt.suptitle('%s: %s' % (band, tim.name))
# ps.savefig()
coimg /= np.maximum(con,1)
coimgs.append(coimg)
cons .append(con)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ps.savefig()
plt.clf()
for i,b in enumerate(bands):
plt.subplot(2,2,i+1)
dimshow(cons[i], ticks=False)
plt.title('%s band' % b)
plt.colorbar()
plt.suptitle('Number of exposures')
ps.savefig()
print 'Grabbing SDSS sources...'
bandlist = [b for b in bands]
cat,T = get_sdss_sources(bandlist, targetwcs)
# record coordinates in target brick image
ok,T.tx,T.ty = targetwcs.radec2pixelxy(T.ra, T.dec)
T.tx -= 1
T.ty -= 1
T.itx = np.clip(np.round(T.tx).astype(int), 0, W-1)
T.ity = np.clip(np.round(T.ty).astype(int), 0, H-1)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(T.tx, T.ty, 'o', mec=green, mfc='none', ms=10, mew=1.5)
plt.axis(ax)
plt.title('SDSS sources')
ps.savefig()
print 'Detmaps...'
# Render the detection maps
detmaps = dict([(b, np.zeros((H,W), np.float32)) for b in bands])
detivs = dict([(b, np.zeros((H,W), np.float32)) for b in bands])
for tim in tims:
iv = tim.getInvvar()
psfnorm = 1./(2. * np.sqrt(np.pi) * tim.psf_sigma)
detim = tim.getImage().copy()
detim[iv == 0] = 0.
detim = gaussian_filter(detim, tim.psf_sigma) / psfnorm**2
detsig1 = tim.sig1 / psfnorm
subh,subw = tim.shape
detiv = np.zeros((subh,subw), np.float32) + (1. / detsig1**2)
detiv[iv == 0] = 0.
(Yo,Xo,Yi,Xi) = tim.resamp
detmaps[tim.band][Yo,Xo] += detiv[Yi,Xi] * detim[Yi,Xi]
detivs [tim.band][Yo,Xo] += detiv[Yi,Xi]
rtn = dict()
for k in ['T', 'coimgs', 'cons', 'detmaps', 'detivs',
'targetrd', 'pixscale', 'targetwcs', 'W','H',
'bands', 'tims', 'ps', 'brick', 'cat']:
rtn[k] = locals()[k]
return rtn
def map_decals_wl(req, ver, zoom, x, y):
tag = 'decals-wl'
ignoreCached = False
filename = None
forcecache = False
from decals import settings
savecache = settings.SAVE_CACHE
zoom = int(zoom)
zoomscale = 2.**zoom
x = int(x)
y = int(y)
if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale:
raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom, x, y))
ver = int(ver)
if not ver in tileversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
basedir = settings.DATA_DIR
tilefn = os.path.join(basedir, 'tiles', tag,
'%i/%i/%i/%i.jpg' % (ver, zoom, x, y))
if os.path.exists(tilefn) and not ignoreCached:
print('Cached:', tilefn)
return send_file(tilefn,
'image/jpeg',
expires=oneyear,
modsince=req.META.get('HTTP_IF_MODIFIED_SINCE'),
filename=filename)
else:
print('Tile image does not exist:', tilefn)
from astrometry.util.resample import resample_with_wcs, OverlapError
from astrometry.util.util import Tan
from astrometry.libkd.spherematch import match_radec
from astrometry.util.fits import fits_table
from astrometry.util.starutil_numpy import degrees_between
import numpy as np
import fitsio
try:
wcs, W, H, zoomscale, zoom, x, y = get_tile_wcs(zoom, x, y)
except RuntimeError as e:
return HttpResponse(e.strerror)
mydir = os.path.join(basedir, 'coadd', 'weak-lensing')
rlo, d = wcs.pixelxy2radec(W, H / 2)[-2:]
rhi, d = wcs.pixelxy2radec(1, H / 2)[-2:]
r, d1 = wcs.pixelxy2radec(W / 2, 1)[-2:]
r, d2 = wcs.pixelxy2radec(W / 2, H)[-2:]
#dlo = min(d1, d2)
#dhi = max(d1, d2)
r, d = wcs.pixelxy2radec(W / 2, H / 2)[-2:]
rad = degrees_between(r, d, rlo, d1)
fn = os.path.join(mydir, 'index.fits')
if not os.path.exists(fn):
#
ii, rr, dd = [], [], []
for i in range(1, 52852 + 1):
imgfn = os.path.join(mydir, 'map%i.fits' % i)
hdr = fitsio.read_header(imgfn)
r = hdr['CRVAL1']
d = hdr['CRVAL2']
ii.append(i)
rr.append(r)
dd.append(d)
T = fits_table()
T.ra = np.array(rr)
T.dec = np.array(dd)
T.i = np.array(ii)
T.writeto(fn)
T = fits_table(fn)
I, J, d = match_radec(T.ra, T.dec, r, d, rad + 0.2)
T.cut(I)
print(len(T), 'weak-lensing maps in range')
if len(I) == 0:
from django.http import HttpResponseRedirect
if forcecache:
# create symlink to blank.jpg!
trymakedirs(tilefn)
src = os.path.join(settings.STATIC_ROOT, 'blank.jpg')
if os.path.exists(tilefn):
os.unlink(tilefn)
os.symlink(src, tilefn)
print('Symlinked', tilefn, '->', src)
return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg')
r, d = wcs.pixelxy2radec([1, 1, 1, W / 2, W, W, W, W / 2],
[1, H / 2, H, H, H, H / 2, 1, 1])[-2:]
foundany = False
rimg = np.zeros((H, W), np.float32)
rn = np.zeros((H, W), np.uint8)
for tilei in T.i:
fn = os.path.join(mydir, 'map%i.fits' % tilei)
try:
bwcs = read_tan_wcs(fn, 0)
except:
print('Failed to read WCS:', fn)
savecache = False
import traceback
import sys
traceback.print_exc(None, sys.stdout)
continue
foundany = True
print('Reading', fn)
ok, xx, yy = bwcs.radec2pixelxy(r, d)
xx = xx.astype(np.int)
yy = yy.astype(np.int)
imW, imH = int(bwcs.get_width()), int(bwcs.get_height())
M = 10
xlo = np.clip(xx.min() - M, 0, imW)
xhi = np.clip(xx.max() + M, 0, imW)
ylo = np.clip(yy.min() - M, 0, imH)
yhi = np.clip(yy.max() + M, 0, imH)
if xlo >= xhi or ylo >= yhi:
continue
subwcs = bwcs.get_subimage(xlo, ylo, xhi - xlo, yhi - ylo)
slc = slice(ylo, yhi), slice(xlo, xhi)
try:
f = fitsio.FITS(fn)[0]
img = f[slc]
del f
except:
print('Failed to read image and WCS:', fn)
savecache = False
import traceback
import sys
traceback.print_exc(None, sys.stdout)
continue
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(wcs, subwcs, [], 3)
except OverlapError:
print('Resampling exception')
continue
rimg[Yo, Xo] += img[Yi, Xi]
rn[Yo, Xo] += 1
rimg /= np.maximum(rn, 1)
if forcecache:
savecache = True
if savecache:
trymakedirs(tilefn)
else:
import tempfile
f, tilefn = tempfile.mkstemp(suffix='.jpg')
os.close(f)
import pylab as plt
# S/N
#lo,hi = 1.5, 5.0
lo, hi = 0, 5.0
rgb = plt.cm.hot((rimg - lo) / (hi - lo))
plt.imsave(tilefn, rgb)
print('Wrote', tilefn)
return send_file(tilefn,
'image/jpeg',
unlink=(not savecache),
filename=filename)
def get_sdss_cutout(targetwcs,
sdss,
get_rawvals=False,
bands='irg',
get_rawvals_only=False,
bandscales=dict(z=1.0, i=1.0, r=1.3, g=2.5)):
rgbims = []
ra, dec = targetwcs.radec_center()
# in deg
radius = targetwcs.radius()
#print 'Target WCS radius is', radius, 'deg'
H, W = targetwcs.get_height(), targetwcs.get_width()
targetpixscale = targetwcs.pixel_scale()
wlistfn = sdss.filenames.get('window_flist', 'window_flist.fits')
rad2 = radius * 60. + np.hypot(14., 10.) / 2.
#print 'Rad2 radius', rad2, 'arcmin'
RCF = radec_to_sdss_rcf(ra, dec, tablefn=wlistfn, radius=rad2)
# Drop rerun 157
keepRCF = []
for run, camcol, field, r, d in RCF:
rr = sdss.get_rerun(run, field)
#print 'Rerun:', rr
if rr == '157':
continue
keepRCF.append((run, camcol, field))
RCF = keepRCF
print len(RCF), 'run/camcol/fields in range'
# size in SDSS pixels of the target image.
sz = np.hypot(H, W) / 2. * targetpixscale / 0.396
print 'SDSS sz:', sz
bandnums = [band_index(b) for b in bands]
for bandnum, band in zip(bandnums, bands):
targetim = np.zeros((H, W), np.float32)
targetn = np.zeros((H, W), np.uint8)
for ifield, (run, camcol, field) in enumerate(RCF):
fn = sdss.retrieve('frame', run, camcol, field, band)
frame = sdss.readFrame(run, camcol, field, bandnum)
h, w = frame.getImageShape()
x, y = frame.astrans.radec_to_pixel(ra, dec)
x, y = int(x), int(y)
# add some margin for resampling
sz2 = int(sz) + 5
xlo = np.clip(x - sz2, 0, w)
xhi = np.clip(x + sz2 + 1, 0, w)
ylo = np.clip(y - sz2, 0, h)
yhi = np.clip(y + sz2 + 1, 0, h)
if xlo == xhi or ylo == yhi:
continue
stamp = frame.getImageSlice((slice(ylo, yhi), slice(xlo, xhi)))
sh, sw = stamp.shape
wcs = AsTransWrapper(frame.astrans, sw, sh, x0=xlo, y0=ylo)
# FIXME -- allow nn resampling too
try:
Yo, Xo, Yi, Xi, [rim
] = resample_with_wcs(targetwcs, wcs, [stamp],
3)
except ResampleError:
continue
targetim[Yo, Xo] += rim
targetn[Yo, Xo] += 1
rgbims.append(targetim / targetn)
if get_rawvals_only:
return rgbims
if get_rawvals:
rawvals = [x.copy() for x in rgbims]
r, g, b = rgbims
r *= bandscales[bands[0]]
g *= bandscales[bands[1]]
b *= bandscales[bands[2]]
# i
#r *= 1.0
# r
#g *= 1.5
#g *= 1.3
# g
#b *= 2.5
m = -0.02
r = np.maximum(0, r - m)
g = np.maximum(0, g - m)
b = np.maximum(0, b - m)
I = (r + g + b) / 3.
alpha = 1.5
Q = 20
m2 = 0.
fI = np.arcsinh(alpha * Q * (I - m2)) / np.sqrt(Q)
I += (I == 0.) * 1e-6
R = fI * r / I
G = fI * g / I
B = fI * b / I
maxrgb = reduce(np.maximum, [R, G, B])
J = (maxrgb > 1.)
R[J] = R[J] / maxrgb[J]
G[J] = G[J] / maxrgb[J]
B[J] = B[J] / maxrgb[J]
ss = 0.5
RGBblur = np.clip(
np.dstack([
gaussian_filter(R, ss),
gaussian_filter(G, ss),
gaussian_filter(B, ss)
]), 0., 1.)
if get_rawvals:
return RGBblur, rawvals
return RGBblur
def map_coadd_bands(
req,
ver,
zoom,
x,
y,
bands,
tag,
imagedir,
wcs=None,
imagetag='image2',
rgbfunc=None,
rgbkwargs={},
bricks=None,
savecache=True,
forcecache=False,
return_if_not_found=False,
model_gz=False,
modeldir=None,
scaledir=None,
get_images=False,
write_jpeg=False,
ignoreCached=False,
add_gz=False,
filename=None,
symlink_blank=False,
hack_jpeg=False,
drname=None,
decals=None,
basepat=None,
scalepat=None,
nativescale=14,
maxscale=8,
):
from decals import settings
from map.views import tileversions
zoom = int(zoom)
zoomscale = 2.**zoom
x = int(x)
y = int(y)
if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale:
raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom, x, y))
ver = int(ver)
if not ver in tileversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
basedir = settings.DATA_DIR
tilefn = os.path.join(basedir, 'tiles', tag,
'%i/%i/%i/%i.jpg' % (ver, zoom, x, y))
if os.path.exists(tilefn) and not ignoreCached:
return send_file(tilefn,
'image/jpeg',
expires=oneyear,
modsince=req.META.get('HTTP_IF_MODIFIED_SINCE'),
filename=filename)
else:
debug('Tile image does not exist:', tilefn)
from astrometry.util.resample import resample_with_wcs, OverlapError
from astrometry.util.util import Tan
import numpy as np
import fitsio
if wcs is None:
try:
wcs, W, H, zoomscale, zoom, x, y = get_tile_wcs(zoom, x, y)
except RuntimeError as e:
return HttpResponse(e.strerror)
else:
W = wcs.get_width()
H = wcs.get_height()
if basepat is None:
basepat = os.path.join(
basedir, 'coadd', imagedir, '%(brickname).3s', '%(brickname)s',
'decals-%(brickname)s-' + imagetag + '-%(band)s.fits')
if modeldir is not None:
modbasepat = os.path.join(
basedir, 'coadd', modeldir, '%(brickname).3s', '%(brickname)s',
'decals-%(brickname)s-' + imagetag + '-%(band)s.fits')
else:
modbasepat = basepat
if model_gz and imagetag == 'model':
modbasepat += '.gz'
if add_gz:
basepat += '.gz'
scaled = 0
if scaledir is None:
scaledir = imagedir
if zoom < nativescale:
scaled = (nativescale - zoom)
scaled = np.clip(scaled, 1, maxscale)
#debug('Scaled-down:', scaled)
dirnm = os.path.join(basedir, 'scaled', scaledir)
if scalepat is None:
scalepat = os.path.join(dirnm, '%(scale)i%(band)s',
'%(brickname).3s',
imagetag + '-%(brickname)s-%(band)s.fits')
if decals is None:
from map.views import _get_survey
D = _get_survey(name=drname)
else:
D = decals
if bricks is None:
B = D.get_bricks_readonly()
else:
B = bricks
rlo, d = wcs.pixelxy2radec(W, H / 2)[-2:]
rhi, d = wcs.pixelxy2radec(1, H / 2)[-2:]
r, d1 = wcs.pixelxy2radec(W / 2, 1)[-2:]
r, d2 = wcs.pixelxy2radec(W / 2, H)[-2:]
dlo = min(d1, d2)
dhi = max(d1, d2)
I = D.bricks_touching_radec_box(B, rlo, rhi, dlo, dhi)
debug(len(I), 'bricks touching zoom', zoom, 'x,y', x, y, 'RA', rlo, rhi,
'Dec', dlo, dhi)
if len(I) == 0:
if get_images:
return None
from django.http import HttpResponseRedirect
if forcecache and symlink_blank:
# create symlink to blank.jpg!
trymakedirs(tilefn)
src = os.path.join(settings.STATIC_ROOT, 'blank.jpg')
if os.path.exists(tilefn):
os.unlink(tilefn)
os.symlink(src, tilefn)
debug('Symlinked', tilefn, '->', src)
return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg')
r, d = wcs.pixelxy2radec([1, 1, 1, W / 2, W, W, W, W / 2],
[1, H / 2, H, H, H, H / 2, 1, 1])[-2:]
foundany = False
rimgs = []
for band in bands:
rimg = np.zeros((H, W), np.float32)
rn = np.zeros((H, W), np.uint8)
for i, brickname in zip(I, B.brickname[I]):
has = getattr(B, 'has_%s' % band, None)
if has is not None and not has[i]:
# No coverage for band in this brick.
debug('Brick', brickname, 'has no', band, 'band')
continue
fnargs = dict(band=band, brickname=brickname)
if imagetag == 'resid':
#basefn = basepat % fnargs
basefn = D.find_file('image', brick=brickname, band=band)
modbasefn = D.find_file('model', brick=brickname, band=band)
#modbasefn = modbasepat % fnargs
#modbasefn = modbasefn.replace('resid', 'model')
#if model_gz:
# modbasefn += '.gz'
if scalepat is None:
imscalepat = None
modscalepat = None
else:
imscalepat = scalepat.replace('resid', 'image')
modscalepat = scalepat.replace('resid', 'model')
imbasefn = basefn.replace('resid', 'image')
debug('resid. imscalepat, imbasefn', imscalepat, imbasefn)
debug('resid. modscalepat, modbasefn', modscalepat, modbasefn)
imfn = get_scaled(imscalepat, fnargs, scaled, imbasefn)
modfn = get_scaled(modscalepat, fnargs, scaled, modbasefn)
debug('resid. im', imfn, 'mod', modfn)
fn = imfn
else:
basefn = D.find_file(imagetag, brick=brickname, band=band)
fn = get_scaled(scalepat, fnargs, scaled, basefn)
if fn is None:
debug('not found: brick', brickname, 'band', band,
'with basefn', basefn)
savecache = False
continue
if not os.path.exists(fn):
debug('Does not exist:', fn)
savecache = False
continue
try:
bwcs = read_tan_wcs(fn, 0)
except:
print('Failed to read WCS:', fn)
savecache = False
import traceback
import sys
traceback.print_exc(None, sys.stdout)
continue
foundany = True
debug('Reading', fn)
ok, xx, yy = bwcs.radec2pixelxy(r, d)
xx = xx.astype(np.int)
yy = yy.astype(np.int)
imW, imH = int(bwcs.get_width()), int(bwcs.get_height())
M = 10
xlo = np.clip(xx.min() - M, 0, imW)
xhi = np.clip(xx.max() + M, 0, imW)
ylo = np.clip(yy.min() - M, 0, imH)
yhi = np.clip(yy.max() + M, 0, imH)
if xlo >= xhi or ylo >= yhi:
continue
subwcs = bwcs.get_subimage(xlo, ylo, xhi - xlo, yhi - ylo)
slc = slice(ylo, yhi), slice(xlo, xhi)
try:
f = fitsio.FITS(fn)[0]
img = f[slc]
del f
if imagetag == 'resid':
f = fitsio.FITS(modfn)[0]
mod = f[slc]
del f
img = img - mod
except:
print('Failed to read image and WCS:', fn)
savecache = False
import traceback
import sys
traceback.print_exc(None, sys.stdout)
continue
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(wcs, subwcs, [], 3)
except OverlapError:
debug('Resampling exception')
continue
rimg[Yo, Xo] += img[Yi, Xi]
# try:
# Yo,Xo,Yi,Xi,[rim] = resample_with_wcs(wcs, subwcs, [img], 3)
# except OverlapError:
# debug('Resampling exception')
# continue
# rimg[Yo,Xo] += rim
rn[Yo, Xo] += 1
rimg /= np.maximum(rn, 1)
rimgs.append(rimg)
if return_if_not_found and not savecache:
return
if get_images and not write_jpeg:
return rimgs
if rgbfunc is None:
from legacypipe.common import get_rgb
rgbfunc = get_rgb
rgb = rgbfunc(rimgs, bands, **rgbkwargs)
if forcecache:
savecache = True
if savecache:
trymakedirs(tilefn)
else:
import tempfile
f, tilefn = tempfile.mkstemp(suffix='.jpg')
os.close(f)
# no jpeg output support in matplotlib in some installations...
if hack_jpeg:
save_jpeg(tilefn, rgb)
debug('Wrote', tilefn)
else:
import pylab as plt
plt.imsave(tilefn, rgb)
debug('Wrote', tilefn)
if get_images:
return rimgs
return send_file(tilefn,
'image/jpeg',
unlink=(not savecache),
filename=filename)
def unwise_forcedphot(cat,
tiles,
band=1,
roiradecbox=None,
use_ceres=True,
ceres_block=8,
save_fits=False,
get_models=False,
ps=None,
psf_broadening=None,
pixelized_psf=False,
get_masks=None,
move_crpix=False,
modelsky_dir=None,
tag=None):
'''
Given a list of tractor sources *cat*
and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id)
runs forced photometry, returning a FITS table the same length as *cat*.
*get_masks*: the WCS to resample mask bits into.
'''
from tractor import PointSource, Tractor, ExpGalaxy, DevGalaxy
from tractor.sersic import SersicGalaxy
if tag is None:
tag = ''
else:
tag = tag + ': '
if not pixelized_psf and psf_broadening is None:
# PSF broadening in post-reactivation data, by band.
# Newer version from Aaron's email to decam-chatter, 2018-06-14.
broadening = {1: 1.0405, 2: 1.0346, 3: None, 4: None}
psf_broadening = broadening[band]
if False:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('wise-forced-w%i' % band)
plots = (ps is not None)
if plots:
import pylab as plt
wantims = (plots or save_fits or get_models)
wanyband = 'w'
if get_models:
models = []
wband = 'w%i' % band
Nsrcs = len(cat)
phot = fits_table()
# Filled in based on unique tile overlap
phot.wise_coadd_id = np.array([' '] * Nsrcs, dtype='U8')
phot.wise_x = np.zeros(Nsrcs, np.float32)
phot.wise_y = np.zeros(Nsrcs, np.float32)
phot.set('psfdepth_%s' % wband, np.zeros(Nsrcs, np.float32))
nexp = np.zeros(Nsrcs, np.int16)
mjd = np.zeros(Nsrcs, np.float64)
central_flux = np.zeros(Nsrcs, np.float32)
ra = np.array([src.getPosition().ra for src in cat])
dec = np.array([src.getPosition().dec for src in cat])
fskeys = ['prochi2', 'profracflux']
fitstats = {}
if get_masks:
mh, mw = get_masks.shape
maskmap = np.zeros((mh, mw), np.uint32)
tims = []
for tile in tiles:
info(tag + 'Reading WISE tile', tile.coadd_id, 'band', band)
tim = get_unwise_tractor_image(tile.unwise_dir,
tile.coadd_id,
band,
bandname=wanyband,
roiradecbox=roiradecbox)
if tim is None:
debug('Actually, no overlap with WISE coadd tile', tile.coadd_id)
continue
if plots:
sig1 = tim.sig1
plt.clf()
plt.imshow(tim.getImage(),
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=10 * sig1)
plt.colorbar()
tag = '%s W%i' % (tile.coadd_id, band)
plt.title('%s: tim data' % tag)
ps.savefig()
plt.clf()
plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
range=(-5, 10),
bins=100)
plt.xlabel('Per-pixel intensity (Sigma)')
plt.title(tag)
ps.savefig()
if move_crpix and band in [1, 2]:
realwcs = tim.wcs.wcs
x, y = realwcs.crpix
tile_crpix = tile.get('crpix_w%i' % band)
dx = tile_crpix[0] - 1024.5
dy = tile_crpix[1] - 1024.5
realwcs.set_crpix(x + dx, y + dy)
debug('unWISE', tile.coadd_id, 'band', band, 'CRPIX', x, y,
'shift by', dx, dy, 'to', realwcs.crpix)
if modelsky_dir and band in [1, 2]:
fn = os.path.join(modelsky_dir,
'%s.%i.mod.fits' % (tile.coadd_id, band))
if not os.path.exists(fn):
raise RuntimeError('WARNING: does not exist:', fn)
x0, x1, y0, y1 = tim.roi
bg = fitsio.FITS(fn)[2][y0:y1, x0:x1]
assert (bg.shape == tim.shape)
if plots:
plt.clf()
plt.subplot(1, 2, 1)
plt.imshow(tim.getImage(),
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=5 * sig1)
plt.subplot(1, 2, 2)
plt.imshow(bg,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=5 * sig1)
tag = '%s W%i' % (tile.coadd_id, band)
plt.suptitle(tag)
ps.savefig()
plt.clf()
ha = dict(range=(-5, 10), bins=100, histtype='step')
plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
color='b',
label='Original',
**ha)
plt.hist(((tim.getImage() - bg) *
tim.inverr)[tim.inverr > 0].ravel(),
color='g',
label='Minus Background',
**ha)
plt.axvline(0, color='k', alpha=0.5)
plt.xlabel('Per-pixel intensity (Sigma)')
plt.legend()
plt.title(tag + ': background')
ps.savefig()
# Actually subtract the background!
tim.data -= bg
# Floor the per-pixel variances,
# and add Poisson contribution from sources
if band in [1, 2]:
# in Vega nanomaggies per pixel
floor_sigma = {1: 0.5, 2: 2.0}
poissons = {1: 0.15, 2: 0.3}
with np.errstate(divide='ignore'):
new_ie = 1. / np.sqrt(
(1. / tim.inverr)**2 + floor_sigma[band] +
poissons[band]**2 * np.maximum(0., tim.data))
new_ie[tim.inverr == 0] = 0.
if plots:
plt.clf()
plt.plot((1. / tim.inverr[tim.inverr > 0]).ravel(),
(1. / new_ie[tim.inverr > 0]).ravel(), 'b.')
plt.title('unWISE per-pixel error: %s band %i' %
(tile.coadd_id, band))
plt.xlabel('original')
plt.ylabel('floored')
ps.savefig()
assert (np.all(np.isfinite(new_ie)))
assert (np.all(new_ie >= 0.))
tim.inverr = new_ie
# Expand a 3-pixel radius around weight=0 (saturated) pixels
# from Eddie via crowdsource
# https://github.com/schlafly/crowdsource/blob/7069da3e7d9d3124be1cbbe1d21ffeb63fc36dcc/python/wise_proc.py#L74
## FIXME -- W3/W4 ??
satlimit = 85000
msat = ((tim.data > satlimit) | ((tim.nims == 0) &
(tim.nuims > 1)))
from scipy.ndimage.morphology import binary_dilation
xx, yy = np.mgrid[-3:3 + 1, -3:3 + 1]
dilate = xx**2 + yy**2 <= 3**2
msat = binary_dilation(msat, dilate)
nbefore = np.sum(tim.inverr == 0)
tim.inverr[msat] = 0
nafter = np.sum(tim.inverr == 0)
debug('Masking an additional', (nafter - nbefore),
'near-saturated pixels in unWISE', tile.coadd_id, 'band',
band)
# Read mask file?
if get_masks:
from astrometry.util.resample import resample_with_wcs, OverlapError
# unwise_dir can be a colon-separated list of paths
tilemask = None
for d in tile.unwise_dir.split(':'):
fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id,
'unwise-%s-msk.fits.gz' % tile.coadd_id)
if os.path.exists(fn):
debug('Reading unWISE mask file', fn)
x0, x1, y0, y1 = tim.roi
tilemask = fitsio.FITS(fn)[0][y0:y1, x0:x1]
break
if tilemask is None:
info('unWISE mask file for tile', tile.coadd_id,
'does not exist')
else:
try:
tanwcs = tim.wcs.wcs
assert (tanwcs.shape == tilemask.shape)
Yo, Xo, Yi, Xi, _ = resample_with_wcs(get_masks,
tanwcs,
intType=np.int16)
# Only deal with mask pixels that are set.
I, = np.nonzero(tilemask[Yi, Xi] > 0)
# Trim to unique area for this tile
rr, dd = get_masks.pixelxy2radec(Xo[I] + 1, Yo[I] + 1)
good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2,
tile.dec1, tile.dec2)
I = I[good]
maskmap[Yo[I], Xo[I]] = tilemask[Yi[I], Xi[I]]
except OverlapError:
# Shouldn't happen by this point
print('Warning: no overlap between WISE tile',
tile.coadd_id, 'and brick')
if plots:
plt.clf()
plt.imshow(tilemask, interpolation='nearest', origin='lower')
plt.title('Tile %s: mask' % tile.coadd_id)
ps.savefig()
plt.clf()
plt.imshow(maskmap, interpolation='nearest', origin='lower')
plt.title('Tile %s: accumulated maskmap' % tile.coadd_id)
ps.savefig()
# The tiles have some overlap, so zero out pixels outside the
# tile's unique area.
th, tw = tim.shape
xx, yy = np.meshgrid(np.arange(tw), np.arange(th))
rr, dd = tim.wcs.wcs.pixelxy2radec(xx + 1, yy + 1)
unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1,
tile.dec2)
debug('Tile', tile.coadd_id, '- total of', np.sum(unique),
'unique pixels out of', len(unique.flat), 'total pixels')
if get_models:
# Save the inverr before blanking out non-unique pixels, for making coadds with no gaps!
# (actually, slightly more subtly, expand unique area by 1 pixel)
from scipy.ndimage.morphology import binary_dilation
du = binary_dilation(unique)
tim.coadd_inverr = tim.inverr * du
tim.inverr[unique == False] = 0.
del xx, yy, rr, dd, unique
if plots:
sig1 = tim.sig1
plt.clf()
plt.imshow(tim.getImage() * (tim.inverr > 0),
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=10 * sig1)
plt.colorbar()
tag = '%s W%i' % (tile.coadd_id, band)
plt.title('%s: tim data (unique)' % tag)
ps.savefig()
if pixelized_psf:
from unwise_psf import unwise_psf
if (band == 1) or (band == 2):
# we only have updated PSFs for W1 and W2
psfimg = unwise_psf.get_unwise_psf(band,
tile.coadd_id,
modelname='neo6_unwisecat')
else:
psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id)
if band == 4:
# oversample (the unwise_psf models are at native W4 5.5"/pix,
# while the unWISE coadds are made at 2.75"/pix.
ph, pw = psfimg.shape
subpsf = np.zeros((ph * 2 - 1, pw * 2 - 1), np.float32)
from astrometry.util.util import lanczos3_interpolate
xx, yy = np.meshgrid(
np.arange(0., pw - 0.51, 0.5, dtype=np.float32),
np.arange(0., ph - 0.51, 0.5, dtype=np.float32))
xx = xx.ravel()
yy = yy.ravel()
ix = xx.astype(np.int32)
iy = yy.astype(np.int32)
dx = (xx - ix).astype(np.float32)
dy = (yy - iy).astype(np.float32)
psfimg = psfimg.astype(np.float32)
rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat],
[psfimg])
if plots:
plt.clf()
plt.imshow(psfimg, interpolation='nearest', origin='lower')
plt.title('Original PSF model')
ps.savefig()
plt.clf()
plt.imshow(subpsf, interpolation='nearest', origin='lower')
plt.title('Subsampled PSF model')
ps.savefig()
psfimg = subpsf
del xx, yy, ix, iy, dx, dy
from tractor.psf import PixelizedPSF
psfimg /= psfimg.sum()
fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0}
psfimg *= fluxrescales[band]
tim.psf = PixelizedPSF(psfimg)
if psf_broadening is not None and not pixelized_psf:
# psf_broadening is a factor by which the PSF FWHMs
# should be scaled; the PSF is a little wider
# post-reactivation.
psf = tim.getPsf()
from tractor import GaussianMixturePSF
if isinstance(psf, GaussianMixturePSF):
debug('Broadening PSF: from', psf)
p0 = psf.getParams()
pnames = psf.getParamNames()
p1 = [
p * psf_broadening**2 if 'var' in name else p
for (p, name) in zip(p0, pnames)
]
psf.setParams(p1)
debug('Broadened PSF:', psf)
else:
print(
'WARNING: cannot apply psf_broadening to WISE PSF of type',
type(psf))
wcs = tim.wcs.wcs
_, fx, fy = wcs.radec2pixelxy(ra, dec)
x = np.round(fx - 1.).astype(int)
y = np.round(fy - 1.).astype(int)
good = (x >= 0) * (x < tw) * (y >= 0) * (y < th)
# Which sources are in this brick's unique area?
usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1,
tile.dec2)
I, = np.nonzero(good * usrc)
nexp[I] = tim.nuims[y[I], x[I]]
if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'):
mjd[I] = (tim.mjdmin + tim.mjdmax) / 2.
phot.wise_coadd_id[I] = tile.coadd_id
phot.wise_x[I] = fx[I] - 1.
phot.wise_y[I] = fy[I] - 1.
central_flux[I] = tim.getImage()[y[I], x[I]]
del x, y, good, usrc
# PSF norm for depth
psf = tim.getPsf()
h, w = tim.shape
patch = psf.getPointSourcePatch(h // 2, w // 2).patch
psfnorm = np.sqrt(np.sum(patch**2))
# To handle zero-depth, we return 1/nanomaggies^2 units rather than mags.
# In the small empty patches of the sky (eg W4 in 0922p702), we get sig1 = NaN
if np.isfinite(tim.sig1):
phot.get('psfdepth_%s' % wband)[I] = 1. / (tim.sig1 / psfnorm)**2
tim.tile = tile
tims.append(tim)
if plots:
plt.clf()
mn, mx = 0.1, 20000
plt.hist(np.log10(np.clip(central_flux, mn, mx)),
bins=100,
range=(np.log10(mn), np.log10(mx)))
logt = np.arange(0, 5)
plt.xticks(logt, ['%i' % i for i in 10.**logt])
plt.title('Central fluxes (W%i)' % band)
plt.axvline(np.log10(20000), color='k')
plt.axvline(np.log10(1000), color='k')
ps.savefig()
# Eddie's non-secret recipe:
#- central pixel <= 1000: 19x19 pix box size
#- central pixel in 1000 - 20000: 59x59 box size
#- central pixel > 20000 or saturated: 149x149 box size
#- object near "bright star": 299x299 box size
nbig = nmedium = nsmall = 0
for src, cflux in zip(cat, central_flux):
if cflux > 20000:
R = 100
nbig += 1
elif cflux > 1000:
R = 30
nmedium += 1
else:
R = 15
nsmall += 1
if isinstance(src, PointSource):
src.fixedRadius = R
else:
### FIXME -- sizes for galaxies..... can we set PSF size separately?
galrad = 0
# RexGalaxy is a subclass of ExpGalaxy
if isinstance(src, (ExpGalaxy, DevGalaxy, SersicGalaxy)):
galrad = src.shape.re
pixscale = 2.75
src.halfsize = int(np.hypot(R, galrad * 5 / pixscale))
debug('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall,
'small')
tractor = Tractor(tims, cat)
if use_ceres:
from tractor.ceres_optimizer import CeresOptimizer
tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block)
tractor.freezeParamsRecursive('*')
tractor.thawPathsTo(wanyband)
t0 = Time()
R = tractor.optimize_forced_photometry(fitstats=True,
variance=True,
shared_params=False,
wantims=wantims)
info(tag + 'unWISE forced photometry took', Time() - t0)
if use_ceres:
term = R.ceres_status['termination']
# Running out of memory can cause failure to converge and term
# status = 2. Fail completely in this case.
if term != 0:
info(tag + 'Ceres termination status:', term)
raise RuntimeError('Ceres terminated with status %i' % term)
if wantims:
ims1 = R.ims1
# can happen if empty source list (we still want to generate coadds)
if ims1 is None:
ims1 = R.ims0
flux_invvars = R.IV
if R.fitstats is not None:
for k in fskeys:
x = getattr(R.fitstats, k)
fitstats[k] = np.array(x).astype(np.float32)
if save_fits:
for i, tim in enumerate(tims):
tile = tim.tile
(dat, mod, _, chi, _) = ims1[i]
wcshdr = fitsio.FITSHDR()
tim.wcs.wcs.add_to_header(wcshdr)
tag = 'fit-%s-w%i' % (tile.coadd_id, band)
fitsio.write('%s-data.fits' % tag,
dat,
clobber=True,
header=wcshdr)
fitsio.write('%s-mod.fits' % tag, mod, clobber=True, header=wcshdr)
fitsio.write('%s-chi.fits' % tag, chi, clobber=True, header=wcshdr)
if plots:
# Create models for just the brightest sources
bright_cat = [
src for src in cat if src.getBrightness().getBand(wanyband) > 1000
]
debug('Bright soures:', len(bright_cat))
btr = Tractor(tims, bright_cat)
for tim in tims:
mod = btr.getModelImage(tim)
tile = tim.tile
tag = '%s W%i' % (tile.coadd_id, band)
sig1 = tim.sig1
plt.clf()
plt.imshow(mod,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=25 * sig1)
plt.colorbar()
plt.title('%s: bright-star models' % tag)
ps.savefig()
if get_models:
for i, tim in enumerate(tims):
tile = tim.tile
(dat, mod, _, _, _) = ims1[i]
models.append(
(tile.coadd_id, band, tim.wcs.wcs, dat, mod, tim.coadd_inverr))
if plots:
for i, tim in enumerate(tims):
tile = tim.tile
tag = '%s W%i' % (tile.coadd_id, band)
(dat, mod, _, chi, _) = ims1[i]
sig1 = tim.sig1
plt.clf()
plt.imshow(dat,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=25 * sig1)
plt.colorbar()
plt.title('%s: data' % tag)
ps.savefig()
plt.clf()
plt.imshow(mod,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-3 * sig1,
vmax=25 * sig1)
plt.colorbar()
plt.title('%s: model' % tag)
ps.savefig()
plt.clf()
plt.imshow(chi,
interpolation='nearest',
origin='lower',
cmap='gray',
vmin=-5,
vmax=+5)
plt.colorbar()
plt.title('%s: chi' % tag)
ps.savefig()
nm = np.array([src.getBrightness().getBand(wanyband) for src in cat])
nm_ivar = flux_invvars
# Sources out of bounds, eg, never change from their initial
# fluxes. Zero them out instead.
nm[nm_ivar == 0] = 0.
phot.set('flux_%s' % wband, nm.astype(np.float32))
phot.set('flux_ivar_%s' % wband, nm_ivar.astype(np.float32))
for k in fskeys:
phot.set(k + '_' + wband,
fitstats.get(k, np.zeros(len(phot), np.float32)))
phot.set('nobs_%s' % wband, nexp)
phot.set('mjd_%s' % wband, mjd)
rtn = wphotduck()
rtn.phot = phot
rtn.models = None
rtn.maskmap = None
if get_models:
rtn.models = models
if get_masks:
rtn.maskmap = maskmap
return rtn
def make_resampled_psf_images(RCF, band, ra, dec, sdss,
targetwcs, W, H, addToHeader, plots=False,
max_exposures=1):
""" Given a list of (Run, Camcol, Field) tuples, returns a list of
(img, imgvar, and header) info for stamp sized imgs centered at ra, dec
"""
# populate list of resampled images and their new psf's
output_imgs = []
# zip through each frame, cut out the relevatn patch
for ifield, (run,camcol,field) in enumerate(RCF[:max_exposures]):
print """=============================
RCF %d of %d
======================== """%(ifield, len(RCF))
# get photofield filename from SDSS, cut it down to relevent RCF
fn = sdss.retrieve('photoField', run, camcol, field)
F = aufits.fits_table(fn)
F.cut((F.run == run) * (F.camcol == camcol) * (F.field == field))
print len(F), 'fields'
assert(len(F) == 1)
F = F[0]
# actually get the tractor image (check if it's in cache!)
boundpixradius = int(np.ceil(np.sqrt(2.) * pixradius))
print 'RA,Dec,size', (ra, dec, boundpixradius)
tim, tinfo = tsdss.get_tractor_image_dr9(
run, camcol, field, band, sdss=sdss, nanomaggies=True,
roiradecsize=(ra, dec, boundpixradius))
print 'Got tim:', tim
frame = sdss.readFrame(run, camcol, field, band)
if tim is None:
continue
# find pixel position for input RA, DEC in tractor image (original field)
x,y = tim.getWcs().positionToPixel(tsdss.RaDecPos(ra, dec))
x,y = int(x), int(y)
# Grab calibration information for header
tim.sdss_calib = np.median(frame.getCalibVec())
tim.sdss_sky = frame.getSkyAt(x,y)
iband = tsdss.band_index(band)
tim.sdss_gain = F.gain[iband]
tim.sdss_darkvar = F.dark_variance[iband]
# get region of interest in the original frame
roi = tinfo['roi']
x0,x1,y0,y1 = roi
# Resample to common grid
th,tw = tim.shape
wwcs = tsdss.TractorWCSWrapper(tim.getWcs(), tw, th)
try:
Yo,Xo,Yi,Xi,[rim] = aresample.resample_with_wcs(
targetwcs, wwcs, [tim.getImage()], Lanczos)
except aresample.OverlapError:
continue
img = np.zeros((H,W))
img[Yo,Xo] = rim
iv = np.zeros((H,W))
iv[Yo,Xo] = tim.getInvvar()[Yi,Xi]
# Convert old PSF to new stamp-specific PSF
newpsf = convert_psf_between_imgs(tim, targetwcs)
# create the image's header
hdr = construct_new_header(tim, tinfo, targetwcs, newpsf,
run, camcol, field, band, addToHeader)
# add to the list of resampled imgs,
output_imgs.append((img, iv, hdr))
return output_imgs
def stage1(T=None,
coimgs=None,
cons=None,
detmaps=None,
detivs=None,
targetrd=None,
pixscale=None,
targetwcs=None,
W=None,
H=None,
bands=None,
tims=None,
ps=None,
brick=None,
cat=None):
orig_wcsxy0 = [tim.wcs.getX0Y0() for tim in tims]
hot = np.zeros((H, W), np.float32)
for band in bands:
detmap = detmaps[band] / np.maximum(1e-16, detivs[band])
detsn = detmap * np.sqrt(detivs[band])
hot = np.maximum(hot, detsn)
detmaps[band] = detmap
### FIXME -- ugri
for sedname, sed in [('Flat', (1., 1., 1.)), ('Red', (2.5, 1.0, 0.4))]:
sedmap = np.zeros((H, W), np.float32)
sediv = np.zeros((H, W), np.float32)
for iband, band in enumerate(bands):
# We convert the detmap to canonical band via
# detmap * w
# And the corresponding change to sig1 is
# sig1 * w
# So the invvar-weighted sum is
# (detmap * w) / (sig1**2 * w**2)
# = detmap / (sig1**2 * w)
sedmap += detmaps[band] * detivs[band] / sed[iband]
sediv += detivs[band] / sed[iband]**2
sedmap /= np.maximum(1e-16, sediv)
sedsn = sedmap * np.sqrt(sediv)
hot = np.maximum(hot, sedsn)
plt.clf()
dimshow(np.round(sedsn), vmin=0, vmax=10, cmap='hot')
plt.title('SED-matched detection filter: %s' % sedname)
ps.savefig()
peaks = (hot > 4)
blobs, nblobs = label(peaks)
print('N detected blobs:', nblobs)
blobslices = find_objects(blobs)
# Un-set catalog blobs
for x, y in zip(T.itx, T.ity):
# blob number
bb = blobs[y, x]
if bb == 0:
continue
# un-set 'peaks' within this blob
slc = blobslices[bb - 1]
peaks[slc][blobs[slc] == bb] = 0
# Now, after having removed catalog sources, crank up the detection threshold
peaks &= (hot > 5)
# zero out the edges(?)
peaks[0, :] = peaks[:, 0] = 0
peaks[-1, :] = peaks[:, -1] = 0
peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[0:-2, 1:-1])
peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[2:, 1:-1])
peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[1:-1, 0:-2])
peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[1:-1, 2:])
# These are our peaks
pki = np.flatnonzero(peaks)
peaky, peakx = np.unravel_index(pki, peaks.shape)
print(len(peaky), 'peaks')
crossa = dict(ms=10, mew=1.5)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(T.tx, T.ty, 'r+', **crossa)
plt.plot(peakx, peaky, '+', color=green, **crossa)
plt.axis(ax)
plt.title('SDSS + SED-matched detections')
ps.savefig()
### HACK -- high threshold again
# Segment, and record which sources fall into each blob
blobs, nblobs = label((hot > 20))
print('N detected blobs:', nblobs)
blobslices = find_objects(blobs)
T.blob = blobs[T.ity, T.itx]
blobsrcs = []
blobflux = []
fluximg = coimgs[1]
for blob in range(1, nblobs + 1):
blobsrcs.append(np.flatnonzero(T.blob == blob))
bslc = blobslices[blob - 1]
blobflux.append(np.sum(fluximg[bslc][blobs[bslc] == blob]))
# Fit the SDSS sources
for tim in tims:
tim.psfex.fitSavedData(*tim.psfex.splinedata)
tim.psf = tim.psfex
# How far down to render model profiles
minsigma = 0.1
for tim in tims:
tim.modelMinval = minsigma * tim.sig1
srcvariances = [[] for src in cat]
# Fit in order of flux
for blobnumber, iblob in enumerate(np.argsort(-np.array(blobflux))):
bslc = blobslices[iblob]
Isrcs = blobsrcs[iblob]
if len(Isrcs) == 0:
continue
print()
print('Blob', blobnumber, 'of', len(blobflux), ':', len(Isrcs),
'sources')
print('Source indices:', Isrcs)
print()
# blob bbox in target coords
sy, sx = bslc
by0, by1 = sy.start, sy.stop
bx0, bx1 = sx.start, sx.stop
blobh, blobw = by1 - by0, bx1 - bx0
rr, dd = targetwcs.pixelxy2radec([bx0, bx0, bx1, bx1],
[by0, by1, by1, by0])
alphas = [0.1, 0.3, 1.0]
subtims = []
for itim, tim in enumerate(tims):
h, w = tim.shape
ok, x, y = tim.subwcs.radec2pixelxy(rr, dd)
sx0, sx1 = x.min(), x.max()
sy0, sy1 = y.min(), y.max()
if sx1 < 0 or sy1 < 0 or sx1 > w or sy1 > h:
continue
sx0 = np.clip(int(np.floor(sx0)), 0, w - 1)
sx1 = np.clip(int(np.ceil(sx1)), 0, w - 1) + 1
sy0 = np.clip(int(np.floor(sy0)), 0, h - 1)
sy1 = np.clip(int(np.ceil(sy1)), 0, h - 1) + 1
subslc = slice(sy0, sy1), slice(sx0, sx1)
subimg = tim.getImage()[subslc]
subie = tim.getInvError()[subslc]
subwcs = tim.getWcs().copy()
ox0, oy0 = orig_wcsxy0[itim]
subwcs.setX0Y0(ox0 + sx0, oy0 + sy0)
# Mask out inverr for pixels that are not within the blob.
subtarget = targetwcs.get_subimage(bx0, by0, blobw, blobh)
subsubwcs = tim.subwcs.get_subimage(int(sx0), int(sy0),
int(sx1 - sx0), int(sy1 - sy0))
try:
Yo, Xo, Yi, Xi, rims = resample_with_wcs(
subsubwcs, subtarget, [], 2)
except OverlapError:
print('No overlap')
continue
if len(Yo) == 0:
continue
subie2 = np.zeros_like(subie)
I = np.flatnonzero(blobs[bslc][Yi, Xi] == (iblob + 1))
subie2[Yo[I], Xo[I]] = subie[Yo[I], Xo[I]]
subie = subie2
# If the subimage (blob) is small enough, instantiate a
# constant PSF model in the center.
if sy1 - sy0 < 100 and sx1 - sx0 < 100:
subpsf = tim.psf.mogAt(ox0 + (sx0 + sx1) / 2.,
oy0 + (sy0 + sy1) / 2.)
else:
# Otherwise, instantiate a (shifted) spatially-varying
# PsfEx model.
subpsf = ShiftedPsf(tim.psf, ox0 + sx0, oy0 + sy0)
subtim = Image(data=subimg,
inverr=subie,
wcs=subwcs,
psf=subpsf,
photocal=tim.getPhotoCal(),
sky=tim.getSky(),
name=tim.name)
subtim.band = tim.band
subtim.sig1 = tim.sig1
subtim.modelMinval = tim.modelMinval
subtims.append(subtim)
subcat = Catalog(*[cat[i] for i in Isrcs])
subtr = Tractor(subtims, subcat)
subtr.freezeParam('images')
# Optimize individual sources in order of flux
fluxes = []
for src in subcat:
# HACK -- here we just *sum* the nanomaggies in each band. Bogus!
br = src.getBrightness()
flux = sum([br.getFlux(band) for band in bands])
fluxes.append(flux)
Ibright = np.argsort(-np.array(fluxes))
if len(Ibright) >= 5:
# -Remember the original subtim images
# -Compute initial models for each source (in each tim)
# -Subtract initial models from images
# -During fitting, for each source:
# -add back in the source's initial model (to each tim)
# -fit, with Catalog([src])
# -subtract final model (from each tim)
# -Replace original subtim images
#
# --Might want to omit newly-added detection-filter sources, since their
# fluxes are bogus.
# Remember original tim images
orig_timages = [tim.getImage().copy() for tim in subtims]
initial_models = []
# Create initial models for each tim x each source
for tim in subtims:
mods = []
for src in subcat:
mod = src.getModelPatch(tim)
mods.append(mod)
if mod is not None:
if not np.all(np.isfinite(mod.patch)):
print('Non-finite mod patch')
print('source:', src)
print('tim:', tim)
print('PSF:', tim.getPsf())
assert (np.all(np.isfinite(mod.patch)))
mod.addTo(tim.getImage(), scale=-1)
initial_models.append(mods)
# For sources in decreasing order of brightness
for numi, i in enumerate(Ibright):
tsrc = Time()
print('Fitting source', i,
'(%i of %i in blob)' % (numi, len(Ibright)))
src = subcat[i]
print(src)
srctractor = Tractor(subtims, [src])
srctractor.freezeParams('images')
# Add this source's initial model back in.
for tim, mods in zip(subtims, initial_models):
mod = mods[i]
if mod is not None:
mod.addTo(tim.getImage())
print('Optimizing:', srctractor)
srctractor.printThawedParams()
for step in range(50):
dlnp, X, alpha = srctractor.optimize(priors=False,
shared_params=False,
alphas=alphas)
print('dlnp:', dlnp, 'src', src)
if dlnp < 0.1:
break
for tim in subtims:
mod = src.getModelPatch(tim)
if mod is not None:
mod.addTo(tim.getImage(), scale=-1)
for tim, img in zip(subtims, orig_timages):
tim.data = img
del orig_timages
del initial_models
else:
# Fit sources one at a time, but don't subtract other models
subcat.freezeAllParams()
for numi, i in enumerate(Ibright):
tsrc = Time()
print('Fitting source', i,
'(%i of %i in blob)' % (numi, len(Ibright)))
print(subcat[i])
subcat.freezeAllBut(i)
print('Optimizing:', subtr)
subtr.printThawedParams()
for step in range(10):
dlnp, X, alpha = subtr.optimize(priors=False,
shared_params=False,
alphas=alphas)
print('dlnp:', dlnp)
if dlnp < 0.1:
break
print('Fitting source took', Time() - tsrc)
print(subcat[i])
if len(Isrcs) > 1 and len(Isrcs) <= 10:
tfit = Time()
# Optimize all at once?
subcat.thawAllParams()
print('Optimizing:', subtr)
subtr.printThawedParams()
for step in range(20):
dlnp, X, alpha = subtr.optimize(priors=False,
shared_params=False,
alphas=alphas)
print('dlnp:', dlnp)
if dlnp < 0.1:
break
# Variances
subcat.thawAllRecursive()
subcat.freezeAllParams()
for isub, srci in enumerate(Isrcs):
print('Variances for source', srci)
subcat.thawParam(isub)
src = subcat[isub]
print('Source', src)
print('Params:', src.getParamNames())
if isinstance(src, (DevGalaxy, ExpGalaxy)):
src.shape = EllipseE.fromEllipseESoft(src.shape)
elif isinstance(src, FixedCompositeGalaxy):
src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp)
src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev)
print('Converted ellipse:', src)
allderivs = subtr.getDerivs()
for iparam, derivs in enumerate(allderivs):
dchisq = 0
for deriv, tim in derivs:
h, w = tim.shape
deriv.clipTo(w, h)
ie = tim.getInvError()
slc = deriv.getSlice(ie)
chi = deriv.patch * ie[slc]
dchisq += (chi**2).sum()
if dchisq == 0.:
v = np.nan
else:
v = 1. / dchisq
srcvariances[srci].append(v)
assert (len(srcvariances[srci]) == subcat[isub].numberOfParams())
subcat.freezeParam(isub)
cat.thawAllRecursive()
for i, src in enumerate(cat):
print('Source', i, src)
print('variances:', srcvariances[i])
print(len(srcvariances[i]), 'vs', src.numberOfParams())
if len(srcvariances[i]) != src.numberOfParams():
# This can happen for sources outside the brick bounds: they never get optimized?
print('Warning: zeroing variances for source', src)
srcvariances[i] = [0] * src.numberOfParams()
if isinstance(src, (DevGalaxy, ExpGalaxy)):
src.shape = EllipseE.fromEllipseESoft(src.shape)
elif isinstance(src, FixedCompositeGalaxy):
src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp)
src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev)
assert (len(srcvariances[i]) == src.numberOfParams())
variances = np.hstack(srcvariances)
assert (len(variances) == cat.numberOfParams())
return dict(cat=cat, variances=variances)
def galex_coadds(onegal,
galaxy=None,
radius_mosaic=30,
radius_mask=None,
pixscale=1.5,
ref_pixscale=0.262,
output_dir=None,
galex_dir=None,
log=None,
centrals=True,
verbose=False):
'''Generate custom GALEX cutouts.
radius_mosaic and radius_mask in arcsec
pixscale: GALEX pixel scale in arcsec/pixel.
'''
import fitsio
import matplotlib.pyplot as plt
from astrometry.libkd.spherematch import match_radec
from astrometry.util.resample import resample_with_wcs, OverlapError
from tractor import (Tractor, NanoMaggies, Image, LinearPhotoCal,
NCircularGaussianPSF, ConstantFitsWcs, ConstantSky)
from legacypipe.survey import imsave_jpeg
from legacypipe.catalog import read_fits_catalog
if galaxy is None:
galaxy = 'galaxy'
if galex_dir is None:
galex_dir = os.environ.get('GALEX_DIR')
if output_dir is None:
output_dir = '.'
if radius_mask is None:
radius_mask = radius_mosaic
radius_search = 5.0 # [arcsec]
else:
radius_search = radius_mask
W = H = np.ceil(2 * radius_mosaic / pixscale).astype('int') # [pixels]
targetwcs = Tan(onegal['RA'], onegal['DEC'], (W + 1) / 2.0, (H + 1) / 2.0,
-pixscale / 3600.0, 0.0, 0.0, pixscale / 3600.0, float(W),
float(H))
# Read the custom Tractor catalog
tractorfile = os.path.join(output_dir, '{}-tractor.fits'.format(galaxy))
if not os.path.isfile(tractorfile):
print('Missing Tractor catalog {}'.format(tractorfile))
return 0
cat = fits_table(tractorfile)
print('Read {} sources from {}'.format(len(cat), tractorfile),
flush=True,
file=log)
keep = np.ones(len(cat)).astype(bool)
if centrals:
# Find the large central galaxy and mask out (ignore) all the models
# which are within its elliptical mask.
# This algorithm will have to change for mosaics not centered on large
# galaxies, e.g., in galaxy groups.
m1, m2, d12 = match_radec(cat.ra,
cat.dec,
onegal['RA'],
onegal['DEC'],
radius_search / 3600.0,
nearest=False)
if len(m1) == 0:
print('No central galaxies found at the central coordinates!',
flush=True,
file=log)
else:
pixfactor = ref_pixscale / pixscale # shift the optical Tractor positions
for mm in m1:
morphtype = cat.type[mm].strip()
if morphtype == 'EXP' or morphtype == 'COMP':
e1, e2, r50 = cat.shapeexp_e1[mm], cat.shapeexp_e2[
mm], cat.shapeexp_r[mm] # [arcsec]
elif morphtype == 'DEV' or morphtype == 'COMP':
e1, e2, r50 = cat.shapedev_e1[mm], cat.shapedev_e2[
mm], cat.shapedev_r[mm] # [arcsec]
else:
r50 = None
if r50:
majoraxis = r50 * 5 / pixscale # [pixels]
ba, phi = SGA.misc.convert_tractor_e1e2(e1, e2)
these = SGA.misc.ellipse_mask(W / 2, W / 2, majoraxis,
ba * majoraxis,
np.radians(phi),
cat.bx * pixfactor,
cat.by * pixfactor)
if np.sum(these) > 0:
#keep[these] = False
pass
print('Hack!')
keep[mm] = False
#srcs = read_fits_catalog(cat)
#_srcs = np.array(srcs)[~keep].tolist()
#mod = SGA.misc.srcs2image(_srcs, ConstantFitsWcs(targetwcs), psf_sigma=3.0)
#import matplotlib.pyplot as plt
##plt.imshow(mod, origin='lower') ; plt.savefig('junk.png')
#plt.imshow(np.log10(mod), origin='lower') ; plt.savefig('junk.png')
#pdb.set_trace()
srcs = read_fits_catalog(cat)
for src in srcs:
src.freezeAllBut('brightness')
#srcs_nocentral = np.array(srcs)[keep].tolist()
# Find all overlapping GALEX tiles and then read the tims.
galex_tiles = _read_galex_tiles(targetwcs,
galex_dir,
log=log,
verbose=verbose)
gbands = ['n', 'f']
nicegbands = ['NUV', 'FUV']
zps = dict(n=20.08, f=18.82)
coimgs, comods, coresids, coimgs_central, comods_nocentral = [], [], [], [], []
for niceband, band in zip(nicegbands, gbands):
J = np.flatnonzero(galex_tiles.get('has_' + band))
print(len(J), 'GALEX tiles have coverage in band', band)
coimg = np.zeros((H, W), np.float32)
comod = np.zeros((H, W), np.float32)
cowt = np.zeros((H, W), np.float32)
comod_nocentral = np.zeros((H, W), np.float32)
for src in srcs:
src.setBrightness(NanoMaggies(**{band: 1}))
for j in J:
brick = galex_tiles[j]
fn = os.path.join(
galex_dir, brick.tilename.strip(),
'%s-%sd-intbgsub.fits.gz' % (brick.brickname, band))
#print(fn)
gwcs = Tan(*[
float(f) for f in [
brick.crval1, brick.crval2, brick.crpix1, brick.crpix2,
brick.cdelt1, 0., 0., brick.cdelt2, 3840., 3840.
]
])
img = fitsio.read(fn)
#print('Read', img.shape)
try:
Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, gwcs, [], 3)
except OverlapError:
continue
K = np.flatnonzero(img[Yi, Xi] != 0.)
if len(K) == 0:
continue
Yo, Xo, Yi, Xi = Yo[K], Xo[K], Yi[K], Xi[K]
wt = brick.get(band + 'exptime')
coimg[Yo, Xo] += wt * img[Yi, Xi]
cowt[Yo, Xo] += wt
x0, x1, y0, y1 = min(Xi), max(Xi), min(Yi), max(Yi)
subwcs = gwcs.get_subimage(x0, y0, x1 - x0 + 1, y1 - y0 + 1)
twcs = ConstantFitsWcs(subwcs)
timg = img[y0:y1 + 1, x0:x1 + 1]
tie = np.ones_like(timg) ## HACK!
#hdr = fitsio.read_header(fn)
#zp = hdr['']
zp = zps[band]
photocal = LinearPhotoCal(NanoMaggies.zeropointToScale(zp),
band=band)
tsky = ConstantSky(0.0)
# HACK -- circular Gaussian PSF of fixed size...
# in arcsec
#fwhms = dict(NUV=6.0, FUV=6.0)
# -> sigma in pixels
#sig = fwhms[band] / 2.35 / twcs.pixel_scale()
sig = 6.0 / np.sqrt(8 * np.log(2)) / twcs.pixel_scale()
tpsf = NCircularGaussianPSF([sig], [1.])
tim = Image(data=timg,
inverr=tie,
psf=tpsf,
wcs=twcs,
sky=tsky,
photocal=photocal,
name='GALEX ' + band + brick.brickname)
## Build the model image with and without the central galaxy model.
tractor = Tractor([tim], srcs)
mod = tractor.getModelImage(0)
tractor.freezeParam('images')
tractor.optimize_forced_photometry(priors=False,
shared_params=False)
mod = tractor.getModelImage(0)
srcs_nocentral = np.array(srcs)[keep].tolist()
#srcs_nocentral = np.array(srcs)[nocentral].tolist()
tractor_nocentral = Tractor([tim], srcs_nocentral)
mod_nocentral = tractor_nocentral.getModelImage(0)
comod[Yo, Xo] += wt * mod[Yi - y0, Xi - x0]
comod_nocentral[Yo, Xo] += wt * mod_nocentral[Yi - y0, Xi - x0]
coimg /= np.maximum(cowt, 1e-18)
comod /= np.maximum(cowt, 1e-18)
comod_nocentral /= np.maximum(cowt, 1e-18)
coresid = coimg - comod
# Subtract the model image which excludes the central (comod_nocentral)
# from the data (coimg) to isolate the light of the central
# (coimg_central).
coimg_central = coimg - comod_nocentral
coimgs.append(coimg)
comods.append(comod)
coresids.append(coresid)
comods_nocentral.append(comod_nocentral)
coimgs_central.append(coimg_central)
# Write out the final images with and without the central, making sure
# to apply the zeropoint to go from counts/s to AB nanomaggies.
# https://asd.gsfc.nasa.gov/archive/galex/FAQ/counts_background.html
for thisimg, imtype in zip((coimg, comod, comod_nocentral),
('image', 'model', 'model-nocentral')):
fitsfile = os.path.join(
output_dir, '{}-{}-{}.fits'.format(galaxy, imtype, niceband))
if verbose:
print('Writing {}'.format(fitsfile))
fitsio.write(fitsfile,
thisimg * 10**(-0.4 * (zp - 22.5)),
clobber=True)
# Build a color mosaic (but note that the images here are in units of
# background-subtracted counts/s).
#_galex_rgb = _galex_rgb_moustakas
#_galex_rgb = _galex_rgb_dstn
_galex_rgb = _galex_rgb_official
for imgs, imtype in zip(
(coimgs, comods, coresids, comods_nocentral, coimgs_central),
('image', 'model', 'resid', 'model-nocentral', 'image-central')):
rgb = _galex_rgb(imgs)
jpgfile = os.path.join(output_dir,
'{}-{}-FUVNUV.jpg'.format(galaxy, imtype))
if verbose:
print('Writing {}'.format(jpgfile))
imsave_jpeg(jpgfile, rgb, origin='lower')
return 1
def cutout_jpg(req):
import tempfile
import fitsio
from astrometry.util.util import Tan
from astrometry.util.resample import resample_with_wcs
form = CutoutSearchForm(req.GET)
if not form.is_valid():
return HttpResponse('failed to parse request')
ra = form.cleaned_data['ra']
dec = form.cleaned_data['dec']
if ra is None or dec is None:
return HttpResponse('RA and Dec arguments are required')
size = form.cleaned_data['size']
if size is None:
size = 100
else:
size = min(256, size)
# Ignoring this for now...
# bandstr = form.cleaned_data['bands']
# bands = [int(c) for c in bandstr]
# bands = [b for b in bands if b in [1,2,3,4]]
version = form.cleaned_data['version']
radius = size/2. * 2.75/3600.
tiles = unwise_tiles_near_radec(ra, dec, radius)
tiles = list(tiles)
pixscale = 2.75 / 3600.
cowcs = Tan(*[float(x) for x in
[ra, dec, 0.5 + size/2., 0.5 + size/2.,
-pixscale, 0., 0., pixscale,
size, size]])
bands = [1,2]
coims = [np.zeros((size,size), np.float32) for b in bands]
con = np.zeros((size,size), int)
for tile in tiles:
coadd = tile.coadd
dirnm = os.path.join(settings.DATA_DIR, version, coadd[:3], coadd)
base = os.path.join(dirnm, 'unwise-%s' % coadd)
subwcs = None
ims = []
for band in bands:
fn = str(base + '-w%i-img-m.fits' % band)
if subwcs is None:
wcs = Tan(fn)
ok,x,y = wcs.radec2pixelxy(ra, dec)
x = int(round(x-1.))
y = int(round(y-1.))
x0 = x - size/2
x1 = x0 + size
y0 = y - size/2
y1 = y0 + size
if x1 <= 0 or y1 <= 0:
break
if x0 >= wcs.get_width() or y0 >= wcs.get_height():
break
x0 = max(x0, 0)
y0 = max(y0, 0)
x1 = min(x1, wcs.get_width())
y1 = min(y1, wcs.get_height())
subwcs = wcs.get_subimage(x0, y0, x1-x0, y1-y0)
if subwcs is None:
break
img = fitsio.FITS(fn)[0][y0:y1, x0:x1]
ims.append(img)
if subwcs is None:
continue
try:
Yo,Xo,Yi,Xi,rims = resample_with_wcs(cowcs, subwcs, ims, 3)
except:
continue
for rim,co in zip(rims, coims):
co[Yo,Xo] += rim
con[Yo,Xo] += 1
for co in coims:
co /= np.maximum(con, 1)
rgb = np.zeros((size,size,3), np.uint8)
lo,hi = -3,10
scale = 10.
rgb[:,:,2] = np.clip(255. * ((coims[1] / scale) - lo) / (hi-lo), 0, 255)
rgb[:,:,1] = np.clip(255. * (((coims[0]+coims[1])/2. / scale) - lo) / (hi-lo), 0, 255)
rgb[:,:,0] = np.clip(255. * ((coims[0] / scale) - lo) / (hi-lo), 0, 255)
f,tempfn = tempfile.mkstemp(suffix='.jpg')
os.close(f)
# import matplotlib
# matplotlib.use('Agg')
# import pylab as plt
# plt.imsave(tempfn, rgb, interpolation='nearest', origin='lower')
from PIL import Image
pic = Image.fromarray(rgb)
pic.save(tempfn)
return send_file(tempfn, 'image/jpeg', unlink=True)
def map_sdss(req,
ver,
zoom,
x,
y,
savecache=None,
tag='sdss',
get_images=False,
ignoreCached=False,
wcs=None,
forcecache=False,
forcescale=None,
bestOnly=False,
bands='gri',
**kwargs):
from decals import settings
if savecache is None:
savecache = settings.SAVE_CACHE
zoom = int(zoom)
zoomscale = 2.**zoom
x = int(x)
y = int(y)
if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale:
raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom, x, y))
ver = int(ver)
if not ver in tileversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
basedir = settings.DATA_DIR
tilefn = os.path.join(basedir, 'tiles', tag,
'%i/%i/%i/%i.jpg' % (ver, zoom, x, y))
if os.path.exists(tilefn) and not ignoreCached:
if get_images:
return None
return send_file(tilefn,
'image/jpeg',
expires=oneyear,
modsince=req.META.get('HTTP_IF_MODIFIED_SINCE'))
if not savecache:
import tempfile
f, tilefn = tempfile.mkstemp(suffix='.jpg')
os.close(f)
if wcs is None:
try:
wcs, W, H, zoomscale, zoom, x, y = get_tile_wcs(zoom, x, y)
except RuntimeError as e:
if get_images:
return None
return HttpResponse(e.strerror)
else:
W = wcs.get_width()
H = wcs.get_height()
from astrometry.util.fits import fits_table
import numpy as np
from astrometry.libkd.spherematch import tree_build_radec, tree_search_radec
from astrometry.util.starutil_numpy import degrees_between, arcsec_between
from astrometry.util.resample import resample_with_wcs, OverlapError
from astrometry.util.util import Tan, Sip
import fitsio
print('Tile wcs: center', wcs.radec_center(), 'pixel scale',
wcs.pixel_scale())
global w_flist
global w_flist_tree
if w_flist is None:
w_flist = fits_table(
os.path.join(settings.DATA_DIR, 'sdss', 'window_flist.fits'),
columns=['run', 'rerun', 'camcol', 'field', 'ra', 'dec', 'score'])
print('Read', len(w_flist), 'window_flist entries')
w_flist.cut(w_flist.rerun == '301')
print('Cut to', len(w_flist), 'in rerun 301')
w_flist_tree = tree_build_radec(w_flist.ra, w_flist.dec)
# SDSS field size
radius = 1.01 * np.hypot(10., 14.) / 2. / 60.
# leaflet tile size
ra, dec = wcs.pixelxy2radec(W / 2., H / 2.)[-2:]
r0, d0 = wcs.pixelxy2radec(1, 1)[-2:]
r1, d1 = wcs.pixelxy2radec(W, H)[-2:]
radius = radius + max(degrees_between(ra, dec, r0, d0),
degrees_between(ra, dec, r1, d1))
J = tree_search_radec(w_flist_tree, ra, dec, radius)
print(len(J), 'overlapping SDSS fields found')
if len(J) == 0:
if get_images:
return None
if forcecache:
# create symlink to blank.jpg!
trymakedirs(tilefn)
src = os.path.join(settings.STATIC_ROOT, 'blank.jpg')
if os.path.exists(tilefn):
os.unlink(tilefn)
os.symlink(src, tilefn)
print('Symlinked', tilefn, '->', src)
from django.http import HttpResponseRedirect
return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg')
ww = [1, W * 0.25, W * 0.5, W * 0.75, W]
hh = [1, H * 0.25, H * 0.5, H * 0.75, H]
r, d = wcs.pixelxy2radec(
[1] * len(hh) + ww + [W] * len(hh) + list(reversed(ww)),
hh + [1] * len(ww) + list(reversed(hh)) + [H] * len(ww))[-2:]
scaled = 0
scalepat = None
scaledir = 'sdss'
if zoom <= 13 and forcescale is None:
# Get *actual* pixel scales at the top & bottom
r1, d1 = wcs.pixelxy2radec(W / 2., H)[-2:]
r2, d2 = wcs.pixelxy2radec(W / 2., H - 1.)[-2:]
r3, d3 = wcs.pixelxy2radec(W / 2., 1.)[-2:]
r4, d4 = wcs.pixelxy2radec(W / 2., 2.)[-2:]
# Take the min = most zoomed-in
scale = min(arcsec_between(r1, d1, r2, d2),
arcsec_between(r3, d3, r4, d4))
native_scale = 0.396
scaled = int(np.floor(np.log2(scale / native_scale)))
print('Zoom:', zoom, 'x,y', x, y, 'Tile pixel scale:', scale,
'Scale step:', scaled)
scaled = np.clip(scaled, 1, 7)
dirnm = os.path.join(basedir, 'scaled', scaledir)
scalepat = os.path.join(
dirnm, '%(scale)i%(band)s', '%(rerun)s', '%(run)i', '%(camcol)i',
'sdss-%(run)i-%(camcol)i-%(field)i-%(band)s.fits')
if forcescale is not None:
scaled = forcescale
rimgs = [np.zeros((H, W), np.float32) for band in bands]
rns = [np.zeros((H, W), np.float32) for band in bands]
from astrometry.sdss import AsTransWrapper, DR9
sdss = DR9(basedir=settings.SDSS_DIR)
sdss.saveUnzippedFiles(settings.SDSS_DIR)
#sdss.setFitsioReadBZ2()
if settings.SDSS_PHOTOOBJS:
sdss.useLocalTree(photoObjs=settings.SDSS_PHOTOOBJS,
resolve=settings.SDSS_RESOLVE)
for jnum, j in enumerate(J):
print('SDSS field', jnum, 'of', len(J), 'for zoom', zoom, 'x', x, 'y',
y)
im = w_flist[j]
if im.score >= 0.5:
weight = 1.
else:
weight = 0.001
for band, rimg, rn in zip(bands, rimgs, rns):
if im.rerun != '301':
continue
tmpsuff = '.tmp%08i' % np.random.randint(100000000)
basefn = sdss.retrieve('frame',
im.run,
im.camcol,
field=im.field,
band=band,
rerun=im.rerun,
tempsuffix=tmpsuff)
if scaled > 0:
fnargs = dict(band=band,
rerun=im.rerun,
run=im.run,
camcol=im.camcol,
field=im.field)
fn = get_scaled(scalepat,
fnargs,
scaled,
basefn,
read_base_wcs=read_astrans,
read_wcs=read_sip_wcs)
print('get_scaled:', fn)
else:
fn = basefn
frame = None
if fn == basefn:
frame = sdss.readFrame(im.run,
im.camcol,
im.field,
band,
filename=fn)
h, w = frame.getImageShape()
# Trim off the overlapping top of the image
# Wimp out and instead of trimming 128 pix, trim 124!
trim = 124
subh = h - trim
astrans = frame.getAsTrans()
fwcs = AsTransWrapper(astrans, w, subh)
fullimg = frame.getImage()
fullimg = fullimg[:-trim, :]
else:
fwcs = Sip(fn)
fitsimg = fitsio.FITS(fn)[0]
h, w = fitsimg.get_info()['dims']
fullimg = fitsimg.read()
try:
#Yo,Xo,Yi,Xi,nil = resample_with_wcs(wcs, fwcs, [], 3)
Yo, Xo, Yi, Xi, [resamp
] = resample_with_wcs(wcs, fwcs, [fullimg], 2)
except OverlapError:
continue
if len(Xi) == 0:
#print 'No overlap'
continue
if sdssps is not None:
x0 = Xi.min()
x1 = Xi.max()
y0 = Yi.min()
y1 = Yi.max()
slc = (slice(y0, y1 + 1), slice(x0, x1 + 1))
if frame is not None:
img = frame.getImageSlice(slc)
else:
img = fitsimg[slc]
#rimg[Yo,Xo] += img[Yi-y0, Xi-x0]
if bestOnly:
K = np.flatnonzero(weight > rn[Yo, Xo])
print('Updating', len(K), 'of', len(Yo), 'pixels')
if len(K):
rimg[Yo[K], Xo[K]] = resamp[K] * weight
rn[Yo[K], Xo[K]] = weight
else:
rimg[Yo, Xo] += resamp * weight
rn[Yo, Xo] += weight
if sdssps is not None:
# goodpix = np.ones(img.shape, bool)
# fpM = sdss.readFpM(im.run, im.camcol, im.field, band)
# for plane in [ 'INTERP', 'SATUR', 'CR', 'GHOST' ]:
# fpM.setMaskedPixels(plane, goodpix, False, roi=[x0,x1,y0,y1])
plt.clf()
#ima = dict(vmin=-0.05, vmax=0.5)
#ima = dict(vmin=-0.5, vmax=2.)
ima = dict(vmax=np.percentile(img, 99))
plt.subplot(2, 3, 1)
dimshow(img, ticks=False, **ima)
plt.title('image')
rthis = np.zeros_like(rimg)
#rthis[Yo,Xo] += img[Yi-y0, Xi-x0]
rthis[Yo, Xo] += resamp
plt.subplot(2, 3, 2)
dimshow(rthis, ticks=False, **ima)
plt.title('resampled')
# plt.subplot(2,3,3)
# dimshow(goodpix, ticks=False, vmin=0, vmax=1)
# plt.title('good pix')
plt.subplot(2, 3, 4)
dimshow(rimg / np.maximum(rn, 1), ticks=False, **ima)
plt.title('coadd')
plt.subplot(2, 3, 5)
dimshow(rn, vmin=0, ticks=False)
plt.title('coverage: max %i' % rn.max())
plt.subplot(2, 3, 6)
rgb = sdss_rgb(
[rimg / np.maximum(rn, 1) for rimg, rn in zip(rimgs, rns)],
bands)
dimshow(rgb)
plt.suptitle('SDSS %s, R/C/F %i/%i/%i' %
(band, im.run, im.camcol, im.field))
sdssps.savefig()
for rimg, rn in zip(rimgs, rns):
rimg /= np.maximum(rn, 1e-3)
del rns
if get_images:
return rimgs
rgb = sdss_rgb(rimgs, bands)
trymakedirs(tilefn)
save_jpeg(tilefn, rgb)
print('Wrote', tilefn)
return send_file(tilefn, 'image/jpeg', unlink=(not savecache))