def add(self, tile, wise_models): for band in [1, 2, 3, 4]: if not (tile, band) in wise_models: debug('Tile', tile, 'band', band, '-- model not found') continue # With the move_crpix option (Aaron's updated astrometry), # the WCS for each band can be different, so we call resample_with_wcs # for each band with (potentially) slightly different WCSes. (mod, img, ie, roi, wcs) = wise_models[(tile, band)] debug('WISE: resampling', wcs, 'to', self.unwise_wcs) try: Yo, Xo, Yi, Xi, resam = resample_with_wcs(self.unwise_wcs, wcs, [img, mod], intType=np.int16) rimg, rmod = resam debug('Adding', len(Yo), 'pixels from tile', tile, 'to coadd') self.unwise_co[band - 1][Yo, Xo] += rimg self.unwise_com[band - 1][Yo, Xo] += rmod self.unwise_con[band - 1][Yo, Xo] += 1 self.unwise_coiv[band - 1][Yo, Xo] += ie[Yi, Xi]**2 debug('Band', band, ': now', np.sum(self.unwise_con[band - 1] > 0), 'pixels are set in image coadd') except OverlapError: debug('No overlap between WISE model tile', tile, 'and brick')
def add(self, models, unique=False): for name, band, wcs, img, mod, ie in models: debug('Accumulating tile', name, 'band', band) try: Yo,Xo,Yi,Xi,resam = resample_with_wcs(self.wcs, wcs, [img, mod], intType=np.int16) except OverlapError: debug('No overlap between tile', name, 'and coadd') continue rimg,rmod = resam debug('Adding', len(Yo), 'pixels from tile', name, 'to coadd') iv = ie[Yi,Xi]**2 if unique: K = np.flatnonzero((self.co_nobs[band][Yo,Xo] == 0) * (iv>0)) iv = iv[K] rimg = rimg[K] rmod = rmod[K] Yo = Yo[K] Xo = Xo[K] debug('Cut to', len(Yo), 'unique pixels w/ iv>0') debug('Tile:', np.sum(iv>0), 'of', len(iv), 'pixels have IV') self.co_images [band][Yo,Xo] += rimg * iv self.co_models [band][Yo,Xo] += rmod * iv self.co_nobs [band][Yo,Xo] += 1 self.co_invvars[band][Yo,Xo] += iv debug('Band', band, ': now', np.sum(self.co_nobs[band]>0), 'pixels are set in image coadd')
def _resample_one(args): (itim,tim,mod,blobmod,lanczos,targetwcs,sbscale) = args if lanczos: from astrometry.util.miscutils import patch_image patched = tim.getImage().copy() assert(np.all(np.isfinite(tim.getInvError()))) okpix = (tim.getInvError() > 0) patch_image(patched, okpix) del okpix imgs = [patched] if mod is not None: imgs.append(mod) if blobmod is not None: imgs.append(blobmod) else: imgs = [] try: Yo,Xo,Yi,Xi,rimgs = resample_with_wcs( targetwcs, tim.subwcs, imgs, 3, intType=np.int16) except OverlapError: return None if len(Yo) == 0: return None mo = None bmo = None if lanczos: im = rimgs[0] inext = 1 if mod is not None: mo = rimgs[inext] inext += 1 if blobmod is not None: bmo = rimgs[inext] inext += 1 del patched,imgs,rimgs else: im = tim.getImage ()[Yi,Xi] if mod is not None: mo = mod[Yi,Xi] if blobmod is not None: bmo = blobmod[Yi,Xi] iv = tim.getInvvar()[Yi,Xi] if sbscale: fscale = tim.sbscale debug('Applying surface-brightness scaling of %.3f to' % fscale, tim.name) im *= fscale iv /= (fscale**2) if mod is not None: mo *= fscale if blobmod is not None: bmo *= fscale if tim.dq is None: dq = None else: dq = tim.dq[Yi,Xi] return itim,Yo,Xo,iv,im,mo,bmo,dq
def tim_get_resamp(tim, targetwcs): if hasattr(tim, 'resamp'): return tim.resamp try: Yo,Xo,Yi,Xi,nil = resample_with_wcs(targetwcs, tim.subwcs, [], 2) except OverlapError: print('No overlap') return None if len(Yo) == 0: return None resamp = [x.astype(np.int16) for x in (Yo,Xo,Yi,Xi)] return resamp
def blur_resample_one(X): from scipy.ndimage.filters import gaussian_filter from astrometry.util.resample import resample_with_wcs,OverlapError tim,sig,targetwcs = X img = gaussian_filter(tim.getImage(), sig) try: Yo,Xo,Yi,Xi,[rimg] = resample_with_wcs( targetwcs, tim.subwcs, [img], intType=np.int16) except OverlapError: return None del img blurnorm = 1./(2. * np.sqrt(np.pi) * sig) wt = tim.getInvvar()[Yi,Xi] / (blurnorm**2) return (Yo, Xo, rimg*wt, wt, tim.dq[Yi,Xi])
def _resample_one(args): (itim, tim, mod, lanczos, targetwcs) = args from astrometry.util.resample import resample_with_wcs, OverlapError if lanczos: from astrometry.util.miscutils import patch_image patched = tim.getImage().copy() okpix = (tim.getInvError() > 0) patch_image(patched, okpix) del okpix imgs = [patched] if mod is not None: imgs.append(mod) else: imgs = [] try: Yo, Xo, Yi, Xi, rimgs = resample_with_wcs(targetwcs, tim.subwcs, imgs, 3) except OverlapError: return None if len(Yo) == 0: return None mo = None if lanczos: im = rimgs[0] if mod is not None: mo = rimgs[1] del patched, imgs, rimgs else: im = tim.getImage()[Yi, Xi] if mod is not None: mo = mods[itim][Yi, Xi] iv = tim.getInvvar()[Yi, Xi] fscale = tim.sbscale print('Applying surface-brightness scaling of %.3f to' % fscale, tim.name) im *= fscale iv /= (fscale**2) if mod is not None: mo *= fscale if tim.dq is None: dq = None else: dq = tim.dq[Yi, Xi] return itim, Yo, Xo, iv, im, mo, dq
def _resample_one((itim,tim,mod,lanczos,targetwcs)): from astrometry.util.resample import resample_with_wcs, OverlapError if lanczos: from astrometry.util.miscutils import patch_image patched = tim.getImage().copy() okpix = (tim.getInvError() > 0) patch_image(patched, okpix) del okpix imgs = [patched] if mod is not None: imgs.append(mod) else: imgs = [] try: Yo,Xo,Yi,Xi,rimgs = resample_with_wcs( targetwcs, tim.subwcs, imgs, 3) except OverlapError: return None if len(Yo) == 0: return None mo = None if lanczos: im = rimgs[0] if mod is not None: mo = rimgs[1] del patched,imgs,rimgs else: im = tim.getImage ()[Yi,Xi] if mod is not None: mo = mods[itim][Yi,Xi] iv = tim.getInvvar()[Yi,Xi] fscale = tim.sbscale print('Applying surface-brightness scaling of %.3f to' % fscale, tim.name) im *= fscale iv /= (fscale**2) if mod is not None: mo *= fscale if tim.dq is None: dq = None else: dq = tim.dq[Yi,Xi] return itim,Yo,Xo,iv,im,mo,dq
def get_sdss_cutout(targetwcs, sdss, get_rawvals=False, bands='irg', get_rawvals_only=False, bandscales=dict(z=1.0, i=1.0, r=1.3, g=2.5)): rgbims = [] ra,dec = targetwcs.radec_center() # in deg radius = targetwcs.radius() #print 'Target WCS radius is', radius, 'deg' H,W = targetwcs.get_height(), targetwcs.get_width() targetpixscale = targetwcs.pixel_scale() wlistfn = sdss.filenames.get('window_flist', 'window_flist.fits') rad2 = radius*60. + np.hypot(14., 10.)/2. #print 'Rad2 radius', rad2, 'arcmin' RCF = radec_to_sdss_rcf(ra, dec, tablefn=wlistfn, radius=rad2) # Drop rerun 157 keepRCF = [] for run,camcol,field,r,d in RCF: rr = sdss.get_rerun(run, field) #print 'Rerun:', rr if rr == '157': continue keepRCF.append((run,camcol,field)) RCF = keepRCF print len(RCF), 'run/camcol/fields in range' # size in SDSS pixels of the target image. sz = np.hypot(H, W)/2. * targetpixscale / 0.396 print 'SDSS sz:', sz bandnums = [band_index(b) for b in bands] for bandnum,band in zip(bandnums, bands): targetim = np.zeros((H, W), np.float32) targetn = np.zeros((H, W), np.uint8) for ifield,(run,camcol,field) in enumerate(RCF): fn = sdss.retrieve('frame', run, camcol, field, band) frame = sdss.readFrame(run, camcol, field, bandnum) h,w = frame.getImageShape() x,y = frame.astrans.radec_to_pixel(ra, dec) x,y = int(x), int(y) # add some margin for resampling sz2 = int(sz) + 5 xlo = np.clip(x - sz2, 0, w) xhi = np.clip(x + sz2 + 1, 0, w) ylo = np.clip(y - sz2, 0, h) yhi = np.clip(y + sz2 + 1, 0, h) if xlo == xhi or ylo == yhi: continue stamp = frame.getImageSlice((slice(ylo, yhi), slice(xlo, xhi))) sh,sw = stamp.shape wcs = AsTransWrapper(frame.astrans, sw, sh, x0=xlo, y0=ylo) # FIXME -- allow nn resampling too try: Yo,Xo,Yi,Xi,[rim] = resample_with_wcs(targetwcs, wcs, [stamp], 3) except ResampleError: continue targetim[Yo,Xo] += rim targetn [Yo,Xo] += 1 rgbims.append(targetim / targetn) if get_rawvals_only: return rgbims if get_rawvals: rawvals = [x.copy() for x in rgbims] r,g,b = rgbims r *= bandscales[bands[0]] g *= bandscales[bands[1]] b *= bandscales[bands[2]] # i #r *= 1.0 # r #g *= 1.5 #g *= 1.3 # g #b *= 2.5 m = -0.02 r = np.maximum(0, r - m) g = np.maximum(0, g - m) b = np.maximum(0, b - m) I = (r+g+b)/3. alpha = 1.5 Q = 20 m2 = 0. fI = np.arcsinh(alpha * Q * (I - m2)) / np.sqrt(Q) I += (I == 0.) * 1e-6 R = fI * r / I G = fI * g / I B = fI * b / I maxrgb = reduce(np.maximum, [R,G,B]) J = (maxrgb > 1.) R[J] = R[J]/maxrgb[J] G[J] = G[J]/maxrgb[J] B[J] = B[J]/maxrgb[J] ss = 0.5 RGBblur = np.clip(np.dstack([ gaussian_filter(R, ss), gaussian_filter(G, ss), gaussian_filter(B, ss)]), 0., 1.) if get_rawvals: return RGBblur, rawvals return RGBblur
def make_depth_cut(survey, ccds, bands, targetrd, brick, W, H, pixscale, plots, ps, splinesky, gaussPsf, pixPsf, normalizePsf, do_calibs, gitver, targetwcs, old_calibs_ok, get_depth_maps=False, margin=0.5, use_approx_wcs=False): if plots: import pylab as plt # Add some margin to our DESI depth requirements target_depth_map = dict(g=24.0 + margin, r=23.4 + margin, z=22.5 + margin) # List extra (redundant) target percentiles so that increasing the depth at # any of these percentiles causes the image to be kept. target_percentiles = np.array(list(range(2, 10)) + list(range(10, 30, 5)) + list(range(30, 101, 10))) target_ddepths = np.zeros(len(target_percentiles), np.float32) target_ddepths[target_percentiles < 10] = -0.3 target_ddepths[target_percentiles < 5] = -0.6 #print('Target percentiles:', target_percentiles) #print('Target ddepths:', target_ddepths) cH,cW = H//10, W//10 coarsewcs = targetwcs.scale(0.1) coarsewcs.imagew = cW coarsewcs.imageh = cH # Unique pixels in this brick (U: cH x cW boolean) U = find_unique_pixels(coarsewcs, cW, cH, None, brick.ra1, brick.ra2, brick.dec1, brick.dec2) pixscale = 3600. * np.sqrt(np.abs(ccds.cd1_1*ccds.cd2_2 - ccds.cd1_2*ccds.cd2_1)) seeing = ccds.fwhm * pixscale # Compute the rectangle in *coarsewcs* covered by each CCD slices = [] overlapping_ccds = np.zeros(len(ccds), bool) for i,ccd in enumerate(ccds): wcs = survey.get_approx_wcs(ccd) hh,ww = wcs.shape rr,dd = wcs.pixelxy2radec([1,ww,ww,1], [1,1,hh,hh]) ok,xx,yy = coarsewcs.radec2pixelxy(rr, dd) y0 = int(np.round(np.clip(yy.min(), 0, cH-1))) y1 = int(np.round(np.clip(yy.max(), 0, cH-1))) x0 = int(np.round(np.clip(xx.min(), 0, cW-1))) x1 = int(np.round(np.clip(xx.max(), 0, cW-1))) if y0 == y1 or x0 == x1: slices.append(None) continue # Check whether this CCD overlaps the unique area of this brick... if not np.any(U[y0:y1+1, x0:x1+1]): info('No overlap with unique area for CCD', ccd.expnum, ccd.ccdname) slices.append(None) continue overlapping_ccds[i] = True slices.append((slice(y0, y1+1), slice(x0, x1+1))) keep_ccds = np.zeros(len(ccds), bool) depthmaps = [] for band in bands: # scalar target_depth = target_depth_map[band] # vector target_depths = target_depth + target_ddepths depthiv = np.zeros((cH,cW), np.float32) depthmap = np.zeros_like(depthiv) depthvalue = np.zeros_like(depthiv) last_pcts = np.zeros_like(target_depths) # indices of CCDs we still want to look at in the current band b_inds = np.where(ccds.filter == band)[0] info(len(b_inds), 'CCDs in', band, 'band') if len(b_inds) == 0: continue b_inds = np.array([i for i in b_inds if slices[i] is not None]) info(len(b_inds), 'CCDs in', band, 'band overlap target') if len(b_inds) == 0: continue # CCDs that we will try before searching for good ones -- CCDs # from the same exposure number as CCDs we have chosen to # take. try_ccds = set() # Try DECaLS data first! Idecals = np.where(ccds.propid[b_inds] == '2014B-0404')[0] if len(Idecals): try_ccds.update(b_inds[Idecals]) debug('Added', len(try_ccds), 'DECaLS CCDs to try-list') plot_vals = [] if plots: plt.clf() for i in b_inds: sy,sx = slices[i] x0,x1 = sx.start, sx.stop y0,y1 = sy.start, sy.stop plt.plot([x0,x0,x1,x1,x0], [y0,y1,y1,y0,y0], 'b-', alpha=0.5) plt.title('CCDs overlapping brick: %i in %s band' % (len(b_inds), band)) ps.savefig() nccds = np.zeros((cH,cW), np.int16) plt.clf() for i in b_inds: nccds[slices[i]] += 1 plt.imshow(nccds, interpolation='nearest', origin='lower', vmin=0) plt.colorbar() plt.title('CCDs overlapping brick: %i in %s band (%i / %i / %i)' % (len(b_inds), band, nccds.min(), np.median(nccds), nccds.max())) ps.savefig() #continue while len(b_inds): if len(try_ccds) == 0: # Choose the next CCD to look at in this band. # A rough point-source depth proxy would be: # metric = np.sqrt(ccds.extime[b_inds]) / seeing[b_inds] # If we want to put more weight on choosing good-seeing images, we could do: #metric = np.sqrt(ccds.exptime[b_inds]) / seeing[b_inds]**2 # depth would be ~ 1 / (sig1 * seeing); we privilege good seeing here. metric = 1. / (ccds.sig1[b_inds] * seeing[b_inds]**2) # This metric is *BIG* for *GOOD* ccds! # Here, we try explicitly to include CCDs that cover # pixels that are still shallow by the largest amount # for the largest number of percentiles of interest; # note that pixels with no coverage get depth 0, so # score high in this metric. # # The value is the depth still required to hit the # target, summed over percentiles of interest # (for pixels unique to this brick) depthvalue[:,:] = 0. active = (last_pcts < target_depths) for d in target_depths[active]: depthvalue += U * np.maximum(0, d - depthmap) ccdvalue = np.zeros(len(b_inds), np.float32) for j,i in enumerate(b_inds): #ccdvalue[j] = np.sum(depthvalue[slices[i]]) # mean -- we want the most bang for the buck per pixel? ccdvalue[j] = np.mean(depthvalue[slices[i]]) metric *= ccdvalue # *ibest* is an index into b_inds ibest = np.argmax(metric) # *iccd* is an index into ccds. iccd = b_inds[ibest] ccd = ccds[iccd] debug('Chose best CCD: seeing', seeing[iccd], 'exptime', ccds.exptime[iccd], 'with value', ccdvalue[ibest]) else: iccd = try_ccds.pop() ccd = ccds[iccd] debug('Popping CCD from use_ccds list') # remove *iccd* from b_inds b_inds = b_inds[b_inds != iccd] im = survey.get_image_object(ccd) debug('Band', im.band, 'expnum', im.expnum, 'exptime', im.exptime, 'seeing', im.fwhm*im.pixscale, 'arcsec, propid', im.propid) im.check_for_cached_files(survey) debug(im) if do_calibs: kwa = dict(git_version=gitver, old_calibs_ok=old_calibs_ok) if gaussPsf: kwa.update(psfex=False) if splinesky: kwa.update(splinesky=True) im.run_calibs(**kwa) if use_approx_wcs: debug('Using approximate (TAN) WCS') wcs = survey.get_approx_wcs(ccd) else: debug('Reading WCS from', im.imgfn, 'HDU', im.hdu) wcs = im.get_wcs() x0,x1,y0,y1,slc = im.get_image_extent(wcs=wcs, radecpoly=targetrd) if x0==x1 or y0==y1: debug('No actual overlap') continue wcs = wcs.get_subimage(int(x0), int(y0), int(x1-x0), int(y1-y0)) if 'galnorm_mean' in ccds.get_columns(): galnorm = ccd.galnorm_mean debug('Using galnorm_mean from CCDs table:', galnorm) else: psf = im.read_psf_model(x0, y0, gaussPsf=gaussPsf, pixPsf=pixPsf, normalizePsf=normalizePsf) psf = psf.constantPsfAt((x1-x0)//2, (y1-y0)//2) # create a fake tim to compute galnorm from tractor import PixPos, Flux, ModelMask, Image, NullWCS from legacypipe.survey import SimpleGalaxy h,w = 50,50 gal = SimpleGalaxy(PixPos(w//2,h//2), Flux(1.)) tim = Image(data=np.zeros((h,w), np.float32), psf=psf, wcs=NullWCS(pixscale=im.pixscale)) mm = ModelMask(0, 0, w, h) galmod = gal.getModelPatch(tim, modelMask=mm).patch galmod = np.maximum(0, galmod) galmod /= galmod.sum() galnorm = np.sqrt(np.sum(galmod**2)) detiv = 1. / (im.sig1 / galnorm)**2 galdepth = -2.5 * (np.log10(5. * im.sig1 / galnorm) - 9.) debug('Galnorm:', galnorm, 'sig1:', im.sig1, 'galdepth', galdepth) # Add this image the the depth map... from astrometry.util.resample import resample_with_wcs, OverlapError try: Yo,Xo,_,_,_ = resample_with_wcs(coarsewcs, wcs) debug(len(Yo), 'of', (cW*cH), 'pixels covered by this image') except OverlapError: debug('No overlap') continue depthiv[Yo,Xo] += detiv # compute the new depth map & percentiles (including the proposed new CCD) depthmap[:,:] = 0. depthmap[depthiv > 0] = 22.5 - 2.5*np.log10(5./np.sqrt(depthiv[depthiv > 0])) depthpcts = np.percentile(depthmap[U], target_percentiles) for i,(p,d,t) in enumerate(zip(target_percentiles, depthpcts, target_depths)): info(' pct % 3i, prev %5.2f -> %5.2f vs target %5.2f %s' % (p, last_pcts[i], d, t, ('ok' if d >= t else ''))) keep = False # Did we increase the depth of any target percentile that did not already exceed its target depth? if np.any((depthpcts > last_pcts) * (last_pcts < target_depths)): keep = True # Add any other CCDs from this same expnum to the try_ccds list. # (before making the plot) I = np.where(ccd.expnum == ccds.expnum[b_inds])[0] try_ccds.update(b_inds[I]) debug('Adding', len(I), 'CCDs with the same expnum to try_ccds list') if plots: cc = '1' if keep else '0' xx = [Xo.min(), Xo.min(), Xo.max(), Xo.max(), Xo.min()] yy = [Yo.min(), Yo.max(), Yo.max(), Yo.min(), Yo.min()] plot_vals.append(((xx,yy,cc),(last_pcts,depthpcts,keep),im.ccdname)) if plots and ( (len(try_ccds) == 0) or np.all(depthpcts >= target_depths)): plt.clf() plt.subplot2grid((2,2),(0,0)) plt.imshow(depthvalue, interpolation='nearest', origin='lower', vmin=0) plt.xticks([]); plt.yticks([]) plt.colorbar() plt.title('heuristic value') plt.subplot2grid((2,2),(0,1)) plt.imshow(depthmap, interpolation='nearest', origin='lower', vmin=target_depth - 2, vmax=target_depth + 0.5) ax = plt.axis() for (xx,yy,cc) in [p[0] for p in plot_vals]: plt.plot(xx,yy, '-', color=cc, lw=3) plt.axis(ax) plt.xticks([]); plt.yticks([]) plt.colorbar() plt.title('depth map') plt.subplot2grid((2,2),(1,0), colspan=2) ax = plt.gca() plt.plot(target_percentiles, target_depths, 'ro', label='Target') plt.plot(target_percentiles, target_depths, 'r-') for (lp,dp,k) in [p[1] for p in plot_vals]: plt.plot(target_percentiles, lp, 'k-', label='Previous percentiles') for (lp,dp,k) in [p[1] for p in plot_vals]: cc = 'b' if k else 'r' plt.plot(target_percentiles, dp, '-', color=cc, label='Depth percentiles') ccdnames = ','.join([p[2] for p in plot_vals]) plot_vals = [] plt.ylim(target_depth - 2, target_depth + 0.5) plt.xscale('log') plt.xlabel('Percentile') plt.ylabel('Depth') plt.title('depth percentiles') plt.suptitle('%s %i-%s, exptime %.0f, seeing %.2f, band %s' % (im.camera, im.expnum, ccdnames, im.exptime, im.pixscale * im.fwhm, band)) ps.savefig() if keep: info('Keeping this exposure') else: info('Not keeping this exposure') depthiv[Yo,Xo] -= detiv continue keep_ccds[iccd] = True last_pcts = depthpcts if np.all(depthpcts >= target_depths): info('Reached all target depth percentiles for band', band) break if get_depth_maps: if np.any(depthiv > 0): depthmap[:,:] = 0. depthmap[depthiv > 0] = 22.5 -2.5*np.log10(5./np.sqrt(depthiv[depthiv > 0])) depthmap[np.logical_not(U)] = np.nan depthmaps.append((band, depthmap.copy())) if plots: I = np.where(ccds.filter == band)[0] plt.clf() plt.plot(seeing[I], ccds.exptime[I], 'k.') # which CCDs from this band are we keeping? kept, = np.nonzero(keep_ccds) if len(kept): kept = kept[ccds.filter[kept] == band] plt.plot(seeing[kept], ccds.exptime[kept], 'ro') plt.xlabel('Seeing (arcsec)') plt.ylabel('Exptime (sec)') plt.title('CCDs kept for band %s' % band) plt.ylim(0, np.max(ccds.exptime[I]) * 1.1) ps.savefig() if get_depth_maps: return (keep_ccds, overlapping_ccds, depthmaps) return keep_ccds, overlapping_ccds
def real_one_tile((args)): #(itile,tile,udecs,P3,T,tilewcs,exps,bad_expids,tileid_to_depth) = args (itile,tile) = args print() print('Tile', itile+1, ':', tile.tileid, 'at', tile.ra, tile.dec) i = np.nonzero(tile.dec == udecs)[0][0] if i == 0 or i == len(udecs)-1: print('Endpoint Dec; skipping for now') return None print(' Decs:', udecs[i-1], tile.dec, udecs[i+1]) declo = (udecs[i-1] + tile.dec) / 2. dechi = (udecs[i+1] + tile.dec) / 2. row = P3[P3.dec == tile.dec] print(' ', len(row), 'tiles in this Dec row') ras = np.sort(row.ra) i = np.nonzero(tile.ra == ras)[0][0] if i == 0 or i == len(ras)-1: print(' Endpoint RA; skipping for now') return None print(' RAs:', ras[i-1], tile.ra, ras[i+1]) ralo = (ras[i-1] + tile.ra) / 2. rahi = (ras[i+1] + tile.ra) / 2. pixscale = 2. #pixscale = 0.262 H = int(np.ceil((dechi - declo) / (pixscale/3600.))) W = int(np.ceil((rahi - ralo) * np.cos(np.deg2rad(tile.dec)) / (pixscale/3600.))) print(' Dec height', dechi-declo, 'RA width', rahi-ralo, '-> pix', W, 'x', H) cd = pixscale/3600. thiswcs = Tan(tile.ra, tile.dec, (W+1)/2., (H+1)/2., -cd, 0., 0., cd, float(W), float(H)) # Find surrounding tiles radius = np.hypot(W, H) * pixscale / 3600. I,J,d = match_radec(T.ra, T.dec, tile.ra, tile.dec, radius) print(' ', len(I), 'tiles with measured depths nearby') if len(I) == 0: return None # Now we need to take a tile boresight position and map it to # boxes in RA,Dec of the good portions of the CCDs. depth = np.zeros((H,W), np.float32) nexp = np.zeros((H,W), np.uint8) matched_exposures = set() #print(' tileids', T.tileid[I]) #print(' expnums', T.z_expnum[I]) for ii in I: if T.z_expnum[ii] in bad_expids: print(' skipping bad exp num', T.z_expnum[ii]) continue # Get depth from CCDs file, if available. zdepth = tileid_to_depth.get(T.tileid[ii], 0.) if zdepth == 0.: zdepth = T.z_depth[ii] matched_exposures.add(T.z_expnum[ii]) for twcs in tilewcs: twcs.set_crval((T.ra[ii], T.dec[ii])) try: Yo,Xo,Yi,Xi,rims = resample_with_wcs(thiswcs, twcs) except OverlapError: continue dflux = 10.**((zdepth - 22.5)/-2.5) div = 1./dflux**2 depth[Yo,Xo] += div nexp[Yo,Xo] += 1 # Now also look for entries in the CCDs (exposures) table not previously found. I,J,d = match_radec(exps.ra_bore, exps.dec_bore, tile.ra, tile.dec, radius) print(' ', len(I), 'exposures from CCDs file nearby') if len(I): I = np.array([i for i,expnum,gd in zip(I, exps.expnum[I], exps.galdepth[I]) if (not expnum in matched_exposures) and gd > 0]) print(' ', len(I), 'exposures that were not in tile file') # Drop exposures from this pass, except for previous exposures of this tile! # if len(I): # I = I[np.logical_or(exps.tilepass[I] != 3, # exps.tileid[I] == tile.tileid)] # print(' ', len(I), 'exposures not in pass 3') # if len(I): # print(' objects:', [o.strip() for o in exps.object[I]]) # print(' tileids:', exps.tileid[I]) # print(' expnums:', exps.expnum[I]) # print(' passes:', exps.tilepass[I]) for ii in I: if exps.expnum[ii] in bad_expids: print(' skipping bad exp num', exps.expnum[ii]) continue zdepth = exps.galdepth[ii] for twcs in tilewcs: twcs.set_crval((exps.ra_bore[ii], exps.dec_bore[ii])) try: Yo,Xo,Yi,Xi,rims = resample_with_wcs(thiswcs, twcs) except OverlapError: continue dflux = 10.**((zdepth - 22.5)/-2.5) div = 1./dflux**2 depth[Yo,Xo] += div nexp[Yo,Xo] += 1 # Convert depth map from depth-iv back to mag. # flux with np.errstate(divide='ignore'): dflux = np.sqrt(1./depth) dflux[depth == 0] = 0. depth = -2.5 * (np.log10(dflux) - 9.) depth[dflux == 0] = 0. if depth.max() == 0: print(' Actually no overlap') return None # Extinction correction for this tile... ext_z = 1.211 * tile.ebv_med print(' Applying extinction correction', ext_z, 'mag') depth[depth != 0] -= ext_z #pcts = [0,10,20,30,40,50,60,70,80,90,100] pcts = np.arange(0, 101) depths = np.percentile(depth, pcts) target = 22.5 req_pcts = [0, 2, 2, 5, 5, 10, 10, 100] req_depths = [0, 0, target-0.6, target-0.6, target-0.3, target-0.3, target, target] print(' Depths at 2, 5, and 10th percentile vs target:', '%.2f' % (depths[2] - (target - 0.6)), '%.2f' % (depths[5] - (target - 0.3)), '%.2f' % (depths[10] - target)) return depths
def unwise_forcedphot(cat, tiles, band=1, roiradecbox=None, use_ceres=True, ceres_block=8, save_fits=False, get_models=False, ps=None, psf_broadening=None, pixelized_psf=False, get_masks=None, move_crpix=False, modelsky_dir=None): ''' Given a list of tractor sources *cat* and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id) runs forced photometry, returning a FITS table the same length as *cat*. *get_masks*: the WCS to resample mask bits into. ''' from tractor import NanoMaggies, PointSource, Tractor, ExpGalaxy, DevGalaxy, FixedCompositeGalaxy if not pixelized_psf and psf_broadening is None: # PSF broadening in post-reactivation data, by band. # Newer version from Aaron's email to decam-chatter, 2018-06-14. broadening = { 1: 1.0405, 2: 1.0346, 3: None, 4: None } psf_broadening = broadening[band] if False: from astrometry.util.plotutils import PlotSequence ps = PlotSequence('wise-forced-w%i' % band) plots = (ps is not None) if plots: import pylab as plt wantims = (plots or save_fits or get_models) wanyband = 'w' if get_models: models = {} wband = 'w%i' % band fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix', 'pronexp'] Nsrcs = len(cat) phot = fits_table() # Filled in based on unique tile overlap phot.wise_coadd_id = np.array([' '] * Nsrcs) phot.set(wband + '_psfdepth', np.zeros(len(phot), np.float32)) ra = np.array([src.getPosition().ra for src in cat]) dec = np.array([src.getPosition().dec for src in cat]) nexp = np.zeros(Nsrcs, np.int16) mjd = np.zeros(Nsrcs, np.float64) central_flux = np.zeros(Nsrcs, np.float32) fitstats = {} tims = [] if get_masks: mh,mw = get_masks.shape maskmap = np.zeros((mh,mw), np.uint32) for tile in tiles: print('Reading WISE tile', tile.coadd_id, 'band', band) tim = get_unwise_tractor_image(tile.unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradecbox) if tim is None: print('Actually, no overlap with tile', tile.coadd_id) continue if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data' % tag) ps.savefig() plt.clf() plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), range=(-5,10), bins=100) plt.xlabel('Per-pixel intensity (Sigma)') plt.title(tag) ps.savefig() if move_crpix and band in [1, 2]: realwcs = tim.wcs.wcs x,y = realwcs.crpix tile_crpix = tile.get('crpix_w%i' % band) dx = tile_crpix[0] - 1024.5 dy = tile_crpix[1] - 1024.5 realwcs.set_crpix(x+dx, y+dy) #print('CRPIX', x,y, 'shift by', dx,dy, 'to', realwcs.crpix) if modelsky_dir and band in [1, 2]: fn = os.path.join(modelsky_dir, '%s.%i.mod.fits' % (tile.coadd_id, band)) if not os.path.exists(fn): raise RuntimeError('WARNING: does not exist:', fn) x0,x1,y0,y1 = tim.roi bg = fitsio.FITS(fn)[2][y0:y1, x0:x1] #print('Read background map:', bg.shape, bg.dtype, 'vs image', tim.shape) if plots: plt.clf() plt.subplot(1,2,1) plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) plt.subplot(1,2,2) plt.imshow(bg, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) tag = '%s W%i' % (tile.coadd_id, band) plt.suptitle(tag) ps.savefig() plt.clf() ha = dict(range=(-5,10), bins=100, histtype='step') plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), color='b', label='Original', **ha) plt.hist(((tim.getImage()-bg) * tim.inverr)[tim.inverr > 0].ravel(), color='g', label='Minus Background', **ha) plt.axvline(0, color='k', alpha=0.5) plt.xlabel('Per-pixel intensity (Sigma)') plt.legend() plt.title(tag + ': background') ps.savefig() # Actually subtract the background! tim.data -= bg # Floor the per-pixel variances if band in [1,2]: # in Vega nanomaggies per pixel floor_sigma = {1: 0.5, 2: 2.0} with np.errstate(divide='ignore'): new_ie = 1. / np.hypot(1./tim.inverr, floor_sigma[band]) new_ie[tim.inverr == 0] = 0. if plots: plt.clf() plt.plot((1. / tim.inverr[tim.inverr>0]).ravel(), (1./new_ie[tim.inverr>0]).ravel(), 'b.') plt.title('unWISE per-pixel error: %s band %i' % (tile.coadd_id, band)) plt.xlabel('original') plt.ylabel('floored') ps.savefig() tim.inverr = new_ie # Read mask file? if get_masks: from astrometry.util.resample import resample_with_wcs, OverlapError # unwise_dir can be a colon-separated list of paths tilemask = None for d in tile.unwise_dir.split(':'): fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id, 'unwise-%s-msk.fits.gz' % tile.coadd_id) if os.path.exists(fn): print('Reading unWISE mask file', fn) x0,x1,y0,y1 = tim.roi tilemask = fitsio.FITS(fn)[0][y0:y1,x0:x1] break if tilemask is None: print('unWISE mask file for tile', tile.coadd_id, 'does not exist') else: try: tanwcs = tim.wcs.wcs assert(tanwcs.shape == tilemask.shape) Yo,Xo,Yi,Xi,_ = resample_with_wcs(get_masks, tanwcs, intType=np.int16) # Only deal with mask pixels that are set. I, = np.nonzero(tilemask[Yi,Xi] > 0) # Trim to unique area for this tile rr,dd = get_masks.pixelxy2radec(Yo[I]+1, Xo[I]+1) good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I = I[good] maskmap[Yo[I],Xo[I]] = tilemask[Yi[I], Xi[I]] except OverlapError: # Shouldn't happen by this point print('No overlap between WISE tile', tile.coadd_id, 'and brick') # The tiles have some overlap, so zero out pixels outside the # tile's unique area. th,tw = tim.shape xx,yy = np.meshgrid(np.arange(tw), np.arange(th)) rr,dd = tim.wcs.wcs.pixelxy2radec(xx+1, yy+1) unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) #print(np.sum(unique), 'of', (th*tw), 'pixels in this tile are unique') tim.inverr[unique == False] = 0. del xx,yy,rr,dd,unique if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage() * (tim.inverr > 0), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data (unique)' % tag) ps.savefig() if pixelized_psf: import unwise_psf if (band == 1) or (band == 2): # we only have updated PSFs for W1 and W2 psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id, modelname='neo4_unwisecat') else: psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id) if band == 4: # oversample (the unwise_psf models are at native W4 5.5"/pix, # while the unWISE coadds are made at 2.75"/pix. ph,pw = psfimg.shape subpsf = np.zeros((ph*2-1, pw*2-1), np.float32) from astrometry.util.util import lanczos3_interpolate xx,yy = np.meshgrid(np.arange(0., pw-0.51, 0.5, dtype=np.float32), np.arange(0., ph-0.51, 0.5, dtype=np.float32)) xx = xx.ravel() yy = yy.ravel() ix = xx.astype(np.int32) iy = yy.astype(np.int32) dx = (xx - ix).astype(np.float32) dy = (yy - iy).astype(np.float32) psfimg = psfimg.astype(np.float32) rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat], [psfimg]) if plots: plt.clf() plt.imshow(psfimg, interpolation='nearest', origin='lower') plt.title('Original PSF model') ps.savefig() plt.clf() plt.imshow(subpsf, interpolation='nearest', origin='lower') plt.title('Subsampled PSF model') ps.savefig() psfimg = subpsf del xx, yy, ix, iy, dx, dy from tractor.psf import PixelizedPSF psfimg /= psfimg.sum() fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0} psfimg *= fluxrescales[band] tim.psf = PixelizedPSF(psfimg) if psf_broadening is not None and not pixelized_psf: # psf_broadening is a factor by which the PSF FWHMs # should be scaled; the PSF is a little wider # post-reactivation. psf = tim.getPsf() from tractor import GaussianMixturePSF if isinstance(psf, GaussianMixturePSF): # print('Broadening PSF: from', psf) p0 = psf.getParams() pnames = psf.getParamNames() p1 = [p * psf_broadening**2 if 'var' in name else p for (p, name) in zip(p0, pnames)] psf.setParams(p1) print('Broadened PSF:', psf) else: print('WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf)) wcs = tim.wcs.wcs ok,x,y = wcs.radec2pixelxy(ra, dec) x = np.round(x - 1.).astype(int) y = np.round(y - 1.).astype(int) good = (x >= 0) * (x < tw) * (y >= 0) * (y < th) # Which sources are in this brick's unique area? usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I, = np.nonzero(good * usrc) nexp[I] = tim.nuims[y[I], x[I]] if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'): mjd[I] = (tim.mjdmin + tim.mjdmax) / 2. phot.wise_coadd_id[I] = tile.coadd_id central_flux[I] = tim.getImage()[y[I], x[I]] del x,y,good,usrc # PSF norm for depth psf = tim.getPsf() h,w = tim.shape patch = psf.getPointSourcePatch(h//2, w//2).patch psfnorm = np.sqrt(np.sum(patch**2)) # To handle zero-depth, we return 1/nanomaggies^2 units rather than mags. psfdepth = 1. / (tim.sig1 / psfnorm)**2 phot.get(wband + '_psfdepth')[I] = psfdepth tim.tile = tile tims.append(tim) if plots: plt.clf() mn,mx = 0.1, 20000 plt.hist(np.log10(np.clip(central_flux, mn, mx)), bins=100, range=(np.log10(mn), np.log10(mx))) logt = np.arange(0, 5) plt.xticks(logt, ['%i' % i for i in 10.**logt]) plt.title('Central fluxes (W%i)' % band) plt.axvline(np.log10(20000), color='k') plt.axvline(np.log10(1000), color='k') ps.savefig() # Eddie's non-secret recipe: #- central pixel <= 1000: 19x19 pix box size #- central pixel in 1000 - 20000: 59x59 box size #- central pixel > 20000 or saturated: 149x149 box size #- object near "bright star": 299x299 box size nbig = nmedium = nsmall = 0 for src,cflux in zip(cat, central_flux): if cflux > 20000: R = 100 nbig += 1 elif cflux > 1000: R = 30 nmedium += 1 else: R = 15 nsmall += 1 if isinstance(src, PointSource): src.fixedRadius = R else: ### FIXME -- sizes for galaxies..... can we set PSF size separately? galrad = 0 # RexGalaxy is a subclass of ExpGalaxy if isinstance(src, (ExpGalaxy, DevGalaxy)): galrad = src.shape.re elif isinstance(src, FixedCompositeGalaxy): galrad = max(src.shapeExp.re, src.shapeDev.re) pixscale = 2.75 src.halfsize = int(np.hypot(R, galrad * 5 / pixscale)) #print('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall, 'small') minsb = 0. fitsky = False tractor = Tractor(tims, cat) if use_ceres: from tractor.ceres_optimizer import CeresOptimizer tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block) tractor.freezeParamsRecursive('*') tractor.thawPathsTo(wanyband) kwa = dict(fitstat_extras=[('pronexp', [tim.nims for tim in tims])]) t0 = Time() R = tractor.optimize_forced_photometry( minsb=minsb, mindlnp=1., sky=fitsky, fitstats=True, variance=True, shared_params=False, wantims=wantims, **kwa) print('unWISE forced photometry took', Time() - t0) if use_ceres: term = R.ceres_status['termination'] # Running out of memory can cause failure to converge # and term status = 2. # Fail completely in this case. if term != 0: print('Ceres termination status:', term) raise RuntimeError( 'Ceres terminated with status %i' % term) if wantims: ims1 = R.ims1 flux_invvars = R.IV if R.fitstats is not None: for k in fskeys: x = getattr(R.fitstats, k) fitstats[k] = np.array(x).astype(np.float32) if save_fits: for i,tim in enumerate(tims): tile = tim.tile (dat, mod, ie, chi, roi) = ims1[i] wcshdr = fitsio.FITSHDR() tim.wcs.wcs.add_to_header(wcshdr) tag = 'fit-%s-w%i' % (tile.coadd_id, band) fitsio.write('%s-data.fits' % tag, dat, clobber=True, header=wcshdr) fitsio.write('%s-mod.fits' % tag, mod, clobber=True, header=wcshdr) fitsio.write('%s-chi.fits' % tag, chi, clobber=True, header=wcshdr) if plots: # Create models for just the brightest sources bright_cat = [src for src in cat if src.getBrightness().getBand(wanyband) > 1000] print('Bright soures:', len(bright_cat)) btr = Tractor(tims, bright_cat) for tim in tims: mod = btr.getModelImage(tim) tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) sig1 = tim.sig1 plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: bright-star models' % tag) ps.savefig() if get_models: for i,tim in enumerate(tims): tile = tim.tile (dat, mod, ie, chi, roi) = ims1[i] models[(tile.coadd_id, band)] = (mod, dat, ie, tim.roi, tim.wcs.wcs) if plots: for i,tim in enumerate(tims): tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) (dat, mod, ie, chi, roi) = ims1[i] sig1 = tim.sig1 plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() nm = np.array([src.getBrightness().getBand(wanyband) for src in cat]) nm_ivar = flux_invvars # Sources out of bounds, eg, never change from their default # (1-sigma or whatever) initial fluxes. Zero them out instead. nm[nm_ivar == 0] = 0. phot.set(wband + '_nanomaggies', nm.astype(np.float32)) phot.set(wband + '_nanomaggies_ivar', nm_ivar.astype(np.float32)) dnm = np.zeros(len(nm_ivar), np.float32) okiv = (nm_ivar > 0) dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32) okflux = (nm > 0) mag = np.zeros(len(nm), np.float32) mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux]) ).astype(np.float32) dmag = np.zeros(len(nm), np.float32) ok = (okiv * okflux) dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok]) ).astype(np.float32) mag[np.logical_not(okflux)] = np.nan dmag[np.logical_not(ok)] = np.nan phot.set(wband + '_mag', mag) phot.set(wband + '_mag_err', dmag) for k in fskeys: phot.set(wband + '_' + k, fitstats[k]) phot.set(wband + '_nexp', nexp) if not np.all(mjd == 0): phot.set(wband + '_mjd', mjd) rtn = wphotduck() rtn.phot = phot rtn.models = None rtn.maskmap = None if get_models: rtn.models = models if get_masks: rtn.maskmap = maskmap return rtn
def compare_one(X): from scipy.ndimage.filters import gaussian_filter from scipy.ndimage.morphology import binary_dilation from astrometry.util.resample import resample_with_wcs,OverlapError (tim,sig,targetwcs, coimg,cow, veto, make_badcoadds, plots,ps) = X if plots: import pylab as plt H,W = targetwcs.shape img = gaussian_filter(tim.getImage(), sig) try: Yo,Xo,Yi,Xi,[rimg] = resample_with_wcs( targetwcs, tim.subwcs, [img], intType=np.int16) except OverlapError: return None del img blurnorm = 1./(2. * np.sqrt(np.pi) * sig) wt = tim.getInvvar()[Yi,Xi] / np.float32(blurnorm**2) if Xi.dtype != np.int16: Yi = Yi.astype(np.int16) Xi = Xi.astype(np.int16) # Compare against reference image... maskedpix = np.zeros(tim.shape, np.uint8) # Subtract this image from the coadd otherwt = cow[Yo,Xo] - wt otherimg = (coimg[Yo,Xo] - rimg*wt) / np.maximum(otherwt, 1e-16) this_sig1 = 1./np.sqrt(np.median(wt[wt>0])) ## FIXME -- this image edges?? # Compute the error on our estimate of (thisimg - co) = # sum in quadrature of the errors on thisimg and co. with np.errstate(divide='ignore'): diffvar = 1./wt + 1./otherwt sndiff = (rimg - otherimg) / np.sqrt(diffvar) with np.errstate(divide='ignore'): reldiff = ((rimg - otherimg) / np.maximum(otherimg, this_sig1)) if plots: plt.clf() showimg = np.zeros((H,W),np.float32) showimg[Yo,Xo] = otherimg plt.subplot(2,3,1) plt.imshow(showimg, interpolation='nearest', origin='lower', vmin=-0.01, vmax=0.1, cmap='gray') plt.title('other images') showimg[Yo,Xo] = otherwt plt.subplot(2,3,2) plt.imshow(showimg, interpolation='nearest', origin='lower', vmin=0) plt.title('other wt') showimg[Yo,Xo] = sndiff plt.subplot(2,3,3) plt.imshow(showimg, interpolation='nearest', origin='lower', vmin=-10, vmax=10,cmap='RdBu_r') plt.title('S/N diff') showimg[Yo,Xo] = rimg plt.subplot(2,3,4) plt.imshow(showimg, interpolation='nearest', origin='lower', vmin=-0.01, vmax=0.1, cmap='gray') plt.title('this image') showimg[Yo,Xo] = wt plt.subplot(2,3,5) plt.imshow(showimg, interpolation='nearest', origin='lower', vmin=0) plt.title('this wt') plt.suptitle(tim.name) showimg[Yo,Xo] = reldiff plt.subplot(2,3,6) plt.imshow(showimg, interpolation='nearest', origin='lower', vmin=-4, vmax=4, cmap='RdBu_r') plt.title('rel diff') ps.savefig() # from astrometry.util.plotutils import loghist # plt.clf() # loghist(sndiff.ravel(), reldiff.ravel(), # bins=100) # plt.xlabel('S/N difference') # plt.ylabel('Relative difference') # plt.title('Outliers: ' + tim.name) # ps.savefig() del otherimg # Significant pixels hotpix = ((sndiff > 5.) * (reldiff > 2.) * (otherwt > 1e-16) * (wt > 0.) * (veto[Yo,Xo] == False)) coldpix = ((sndiff < -5.) * (reldiff < -2.) * (otherwt > 1e-16) * (wt > 0.) * (veto[Yo,Xo] == False)) del reldiff, otherwt if (not np.any(hotpix)) and (not np.any(coldpix)): return None hot = np.zeros((H,W), bool) hot[Yo,Xo] = hotpix cold = np.zeros((H,W), bool) cold[Yo,Xo] = coldpix del hotpix, coldpix snmap = np.zeros((H,W), np.float32) snmap[Yo,Xo] = sndiff hot = binary_dilation(hot, iterations=1) cold = binary_dilation(cold, iterations=1) if plots: heat = np.zeros(hot.shape, np.int8) heat += hot heat += -1 * cold # "warm" hot = np.logical_or(hot, binary_dilation(hot, iterations=5) * (snmap > 3.)) hot = binary_dilation(hot, iterations=1) cold = np.logical_or(cold, binary_dilation(cold, iterations=5) * (snmap < -3.)) cold = binary_dilation(cold, iterations=1) if plots: heat += hot heat +- -1*cold # "lukewarm" hot = np.logical_or(hot, binary_dilation(hot, iterations=5) * (snmap > 2.)) hot = binary_dilation(hot, iterations=3) cold = np.logical_or(cold, binary_dilation(cold, iterations=5) * (snmap < -2.)) cold = binary_dilation(cold, iterations=3) if plots: heat += hot heat +- -1*cold plt.clf() plt.imshow(heat, interpolation='nearest', origin='lower', cmap='RdBu_r', vmin=-3, vmax=+3) plt.title(tim.name + ': outliers') ps.savefig() del heat del snmap badco = None if make_badcoadds: bad, = np.nonzero(hot[Yo,Xo]) badhot = (Yo[bad], Xo[bad], tim.getImage()[Yi[bad],Xi[bad]]) bad, = np.nonzero(cold[Yo,Xo]) badcold = (Yo[bad], Xo[bad], tim.getImage()[Yi[bad],Xi[bad]]) badco = badhot,badcold # Actually do the masking! # Resample "hot" (in brick coords) back to tim coords. try: mYo,mXo,mYi,mXi,nil = resample_with_wcs( tim.subwcs, targetwcs, intType=np.int16) except OverlapError: return None Ibad, = np.nonzero(hot[mYi,mXi]) Ibad2, = np.nonzero(cold[mYi,mXi]) info(tim, ': masking', len(Ibad), 'positive outlier pixels and', len(Ibad2), 'negative outlier pixels') maskedpix[mYo[Ibad], mXo[Ibad]] = OUTLIER_POS maskedpix[mYo[Ibad2], mXo[Ibad2]] = OUTLIER_NEG return maskedpix,badco
def match_ngc(self, rr, dd, radius, exts, F, primhdr, meas): ''' rr, dd: np arrays: RA,Dec centers of chips/amps radius: scalar, radius of chips/amps exts: list (same len as rr,dd) of extension indices F: fitsio.FITS object meas: measurement class ''' I, J, d = match_radec(rr, dd, self.cat.ra, self.cat.dec, radius) # plt.clf() # angles = np.linspace(0, 2.*np.pi, 40) # for r,d in zip(rr, dd): # plt.plot(r + radius * np.sin(angles) / np.cos(np.deg2rad(d)), d + radius * np.cos(angles), 'b-') # plt.plot(rr, dd, 'bo') # ax = plt.axis() # plt.plot(self.cat.ra, self.cat.dec, 'r.') # plt.axis(ax) # ps.savefig() print('Matched', len(I), 'NGC objects') if len(I) == 0: return False for j in J: info = ngc_typenames.get(self.cat.classification[j].strip(), '') print(' ', self.cat.name[j], info) # Potential tweet texts and plot filenames tweets = [] goodplots = [] for i, j in zip(I, J): ext = exts[i] obj = self.cat[j] obj.name = obj.name.strip() hdr = F[ext].read_header() extname = hdr['EXTNAME'].strip() expnum = primhdr['EXPNUM'] wcs = meas.get_wcs(hdr) ok, x, y = wcs.radec2pixelxy(obj.ra, obj.dec) x = x - 1 y = y - 1 # Choose cutout area pixrad = 1.4 * obj.radius * 3600. / wcs.pixel_scale() pixrad = max(pixrad, 100) # print('radius:', obj.radius, 'pixel radius:', pixrad) # HACK #pixrad *= 3 r = pixrad tt = '%s in exp %i ext %s (%i)' % (obj.name, expnum, extname, ext) print(tt) # Find the cutout region... does it actually overlap the chip? H, W = wcs.shape xl, xh = int(np.clip(x - r, 0, W - 1)), int(np.clip(x + r, 0, W - 1)) yl, yh = int(np.clip(y - r, 0, H - 1)), int(np.clip(y + r, 0, H - 1)) if xl == xh or yl == yh: print('no actual overlap with image') continue sh, sw = yh - yl, xh - xl if sh < 25 or sw < 25: print('tiny overlap', sw, 'x', sh) continue # Measure the image! flat = None if self.read_flats: band = meas.get_band(hdr) flat = self.get_flat(band, ext, meas) if flat is not None: print('flat: range', flat.min(), flat.max(), 'median', np.median(flat)) meas.ext = ext meas.edge_trim = 20 debugps = None if self.opt.ps: debugps = ps try: M = meas.run(n_fwhm=1, verbose=False, get_image=True, flat=flat, ps=debugps) except KeyboardInterrupt: sys.exit(0) except: import traceback print('Failed to measure file', meas.fn, 'ext', ext, ':') traceback.print_exc() continue #print('Measured:', M.keys()) raw = M['image'] # Now repeat the cutout check with the trimmed image wcs = M['wcs'] # Trim WCS to trimmed raw image shape trim_x0, trim_y0 = M['trim_x0'], M['trim_y0'] H, W = raw.shape wcs = wcs.get_subimage(trim_x0, trim_y0, W, H) ok, x, y = wcs.radec2pixelxy(obj.ra, obj.dec) x = x - 1 y = y - 1 xl, xh = int(np.clip(x - r, 0, W - 1)), int(np.clip(x + r, 0, W - 1)) yl, yh = int(np.clip(y - r, 0, H - 1)), int(np.clip(y + r, 0, H - 1)) if xl == xh or yl == yh: print('no actual overlap with image') continue subimg = raw[yl:yh, xl:xh] sh, sw = subimg.shape if sh < 25 or sw < 25: print('tiny overlap', sw, 'x', sh) continue subwcs = wcs.get_subimage(xl, yl, sw, sh) if False: # Flat image for the same CCD region. flatfn = '/tmp/mzls/mos3.127506.fits' # meas_class = get_measurer_class_for_file(flatfn) # if meas_class is None: # print('Failed to identify camera in', flatfn) # return # flatmeas = meas_class(flatfn, 0, self.nom) # print('Flat meas:', flatmeas) # flatmeas.ext = ext # flathdr = fitsio.read_header(flatfn) # flatmeas.primhdr = flathdr # flatmeas.edge_trim = 20 # FM = flatmeas.run(n_fwhm=1, verbose=False, get_image=True) # flatraw = FM['image'] F = fitsio.FITS(flatfn) flatraw, flathdr = meas.read_raw(F, ext) flatsub = flatraw[yl:yh, xl:xh] zerofn = '/tmp/mzls/mos3.127522.fits' F = fitsio.FITS(zerofn) zeroraw, zerohdr = meas.read_raw(F, ext) zerosub = zeroraw[yl:yh, xl:xh] rawfn = meas.fn F = fitsio.FITS(rawfn) rawimg, rawhdr = meas.read_raw(F, ext) rawsub = rawimg[yl:yh, xl:xh] # Astrometric shifts dx = M['dx'] dy = M['dy'] aff = M['affine'] x = (xl + xh) / 2 y = (yl + yh) / 2 #print('Affine correction terms:', aff) adx = x - aff[0] ady = y - aff[1] corrx = aff[2] + aff[3] * adx + aff[4] * ady - adx corry = aff[5] + aff[6] * adx + aff[7] * ady - ady print('Affine correction', corrx, corry) # Shift the 'subwcs' to account for astrometric offset cx, cy = subwcs.get_crpix() subwcs.set_crpix((cx - dx - corrx, cy - dy - corry)) # Now grab existing data from the LegacySurvey site # What size of image are we going to request? scale = 1. if max(sh, sw) > 1024: scale = 4. elif max(sh, sw) > 512: scale = 2. rh, rw = int(np.ceil(sh / scale)), int(np.ceil(sw / scale)) # make it square mx = max(rh, rw) rw = rh = mx fitsimgs = [] # We'll resample the new image into the existing-image WCS. newimg = None for layer in legacy_survey_layers: url = ( 'http://legacysurvey.org/viewer-dev/fits-cutout/?ra=%.4f&dec=%.4f&pixscale=%.3f&width=%i&height=%i&layer=%s' % (obj.ra, obj.dec, subwcs.pixel_scale() * scale, rw, rh, layer)) print('URL:', url) r = requests.get(url) ftmp = tempfile.NamedTemporaryFile() ftmp.write(r.content) ftmp.flush() #fits,hdr = fitsio.read(ftmp.name, header=True) fitsfile = fitsio.FITS(ftmp.name) ftmp.close() print('FITS file:', len(fitsfile), 'extensions') hdr = fitsfile[0].read_header() fits = fitsfile[0].read() #print('fits:', fits) if fits is None: print('no coverage in layer', layer) continue # If you need to keep a copy (debugging...) # f,tmpfn = tempfile.mkstemp(suffix='.fits') # os.write(f, r.content) # os.close(f) # fits,hdr = fitsio.read(tmpfn, header=True) # print('Wrote FITS to', tmpfn) if np.all(fits == 0): continue ### HACK -- surface brightness correction... if layer == 'sdssco': s = (subwcs.pixel_scale() / 0.396) fits *= s**2 N, ww, hh = fits.shape # pull out the image planes imgs = [fits[n, :, :] for n in range(N)] bands = hdr['BANDS'].strip() fitsimgs.append((layer, bands, imgs)) # Resample the new image to this layer's WCS if newimg is not None: continue thiswcs = Tan(hdr) newimg = np.zeros((rh, rw), dtype=subimg.dtype) try: #Yo,Xo,Yi,Xi,rims = resample_with_wcs(thiswcs, subwcs) # Laczos Yo, Xo, Yi, Xi, rims = resample_with_wcs( thiswcs, subwcs, [subimg]) except: continue #newimg[Yo,Xo] = subimg[Yi,Xi] newimg[Yo, Xo] = rims[0] if len(fitsimgs) == 0: # No overlap with existing surveys continue #print() newband = primhdr['FILTER'][0] #print('New image is', newband) if False: mn, mx = np.percentile(flatsub, [25, 99]) plt.clf() #plt.imshow(newflat, interpolation='nearest', origin='lower', plt.imshow(flatsub, interpolation='nearest', origin='lower', vmin=mn, vmax=mx) #plt.colorbar() plt.colorbar() plt.savefig('ngcbot-flat.png') med = np.median(newflat.ravel()) #mn,mx = np.percentile(newimg, [25,99]) #mn,mx = np.percentile(newimg, [25,90]) #mn,mx = np.percentile(rawsub, [25,95]) mn, mx = np.percentile(rawsub, [50, 95]) plt.clf() # plt.subplot(1,2,1) #plt.imshow(newimg, interpolation='nearest', origin='lower', # plt.imshow(subimg, interpolation='nearest', origin='lower', plt.imshow(rawsub, interpolation='nearest', origin='lower', vmin=mn, vmax=mx) plt.colorbar() plt.savefig('ngcbot-unflattened.png') plt.clf() #plt.subplot(1,2,2) #plt.imshow(newimg / (newflat / med), interpolation='nearest', origin='lower', #plt.imshow(subimg / (flatsub / med), interpolation='nearest', origin='lower', plt.imshow(rawsub / (flatsub / med), interpolation='nearest', origin='lower', vmin=mn, vmax=mx) plt.colorbar() plt.savefig('ngcbot-flattened.png') mn, mx = np.percentile(zerosub, [5, 95]) plt.clf() plt.imshow(zerosub, interpolation='nearest', origin='lower', vmin=mn, vmax=mx) plt.colorbar() plt.savefig('ngcbot-zero.png') zp = M['zp'] zpscale = 10.**((zp - 22.5) / 2.5) exptime = primhdr['EXPTIME'] newimg /= (zpscale * exptime) nh, nw = newimg.shape coverage = np.sum(newimg != 0) / float(nw * nh) print('Fraction', coverage, 'of new image has data') def my_rgb(imgs, bands, **kwargs): return sdss_rgb(imgs, bands, scales=dict(g=6.0, r=3.4, i=2.5, z=2.2), m=-0.02, clip=False, **kwargs) def grayscale(img, band): rgb = my_rgb([img, img, img], [band, band, band]) index = 'zrg'.index(newband) gray = rgb[:, :, index] return gray # plot title args targs = dict(fontsize=8) # DEBUG if self.opt.ps: ocx, ocy = subwcs.get_crpix() subwcs.set_crpix((cx, cy)) newimgA = np.zeros((rh, rw), dtype=subimg.dtype) Yo, Xo, Yi, Xi, rims = resample_with_wcs( thiswcs, subwcs, [subimg]) newimgA[Yo, Xo] = rims[0] subwcs.set_crpix((cx - dx, cy - dy)) newimgB = np.zeros((rh, rw), dtype=subimg.dtype) Yo, Xo, Yi, Xi, rims = resample_with_wcs( thiswcs, subwcs, [subimg]) newimgB[Yo, Xo] = rims[0] subwcs.set_crpix((cx - dx - corrx, cy - dy - corry)) newimgC = np.zeros((rh, rw), dtype=subimg.dtype) Yo, Xo, Yi, Xi, rims = resample_with_wcs( thiswcs, subwcs, [subimg]) newimgC[Yo, Xo] = rims[0] #newgray = grayscale(newimg, newband) #hi = np.percentile(newgray, 99.9) grayargs = dict(interpolation='nearest', origin='lower', cmap='gray', vmin=0.) #vmin=0., vmax=hi, cmap='gray') (layer, bands, imgs) = fitsimgs[0] nicelayer = nicelayernames.get(layer, layer) for band, img in zip(bands, imgs): newbands = ['g', 'r', 'z'] newindex = dict(g=0, r=1, i=2, z=2) if newindex[band] == newindex[newband]: oldgray = grayscale(img, band) plt.clf() plt.imshow(oldgray, **grayargs) plt.title(nicelayer) ps.savefig() plt.clf() plt.imshow(grayscale(newimgA, newband), **grayargs) #plt.imshow(newimgA, **grayargs) plt.title('No astromety correction') ps.savefig() plt.clf() plt.imshow(grayscale(newimgB, newband), **grayargs) #plt.imshow(newimgB, **grayargs) plt.title('Astromety shift only correction') ps.savefig() plt.clf() plt.imshow(grayscale(newimgC, newband), **grayargs) #plt.imshow(newimgC, **grayargs) plt.title('Astromety shift+affine correction') ps.savefig() subwcs.set_crpix((ocx, ocy)) plt.clf() plt.subplots_adjust(left=0.03, right=0.97, bottom=0.03) NC = 1 + len(fitsimgs) NR = 2 # New Image plot newgray = grayscale(newimg, newband) hi = np.percentile(newgray, 99.9) grayargs = dict(interpolation='nearest', origin='lower', vmin=0., vmax=hi, cmap='gray') plt.subplot(NR, NC, 1) plt.imshow(newgray, **grayargs) plt.xticks([]) plt.yticks([]) plt.title('New image (%s)' % newband, **targs) # Select images & bands for the New RGB image. zeroimg = np.zeros_like(newimg) newimgs = [zeroimg, zeroimg, zeroimg] # sdss_rgb reverses the order, so do like grz. newbands = ['g', 'r', 'z'] newindex = dict(g=0, r=1, i=2, z=2) # Start with the new image plane of the New RGB image j = newindex[newband] newimgs[j] = newimg newbands[j] = newband #print('Setting band', newband, '(index %i)'%j, 'to new image') # We wait to choose the scaling until after the "New RGB" image # has been built, so we store the RGB images along the way... rgbs = [] # In the link to the legacysurvey viewer, we select the best # imaging layer. The DECaLS layer is listed first, so if it # has three bands, it wins. bestdata = None bestn = 0 for i, (layer, bands, imgs) in enumerate(fitsimgs): nicelayer = nicelayernames.get(layer, layer) # For DECaLS missing bands: "--z" nicebands = '' goodbands = [] goodimgs = [] for band, img in zip(bands, imgs): if np.all(img == 0.): print('Band', band, 'of', nicelayer, 'is all zero') nicebands += '-' continue goodbands.append(band) goodimgs.append(img) nicebands += band #print(' ', nicelayer, band) j = newindex[band] if newimgs[j] is zeroimg: #print(' Setting band', band, '(index %i)'%j, 'to', nicelayer) newimgs[j] = img newbands[j] = band elif band == newbands[j]: # patch empty regions if same band #print(' Patching index %i from' % j, nicelayer, 'band', band) Z = (newimgs[j] == 0) newimgs[j][Z] = img[Z] # z -> i if newindex[band] == newindex[newband]: # Old image grayscale oldgray = grayscale(img, band) plt.subplot(NR, NC, 2 + i) plt.imshow(oldgray, **grayargs) plt.xticks([]) plt.yticks([]) plt.title('%s (%s)' % (nicelayer, band), **targs) if len(goodbands) == 1: #rgb = grayscale(goodimgs[0], goodbands[0]) img, band = goodimgs[0], goodbands[0] rgb = my_rgb([img, img, img], [band, band, band]) else: rgb = my_rgb(imgs, bands) if len(goodbands) > bestn: bestn = len(goodbands) bestdata = layer # print('bands for', nicelayer, ':', bands, ', actually', nicebands) rgbs.append( (2 + i + NC, rgb, '%s (%s)' % (nicelayer, nicebands))) # list to string newbands = ''.join(newbands) #print('Newbands:', newbands) # New RGB plt.subplot(NR, NC, 1 + NC) rgb = my_rgb(newimgs, newbands) lo = 0. hi = np.percentile(rgb.ravel(), 99.9) rgb = np.clip((rgb - lo) / (hi - lo), 0., 1.) plt.imshow(rgb, interpolation='nearest', origin='lower') plt.xticks([]) plt.yticks([]) plt.title('New+Old (%s)' % newbands, **targs) # Old RGBs for sp, rgb, tt in rgbs: plt.subplot(NR, NC, sp) rgb = np.clip((rgb - lo) / (hi - lo), 0., 1.) plt.imshow(rgb, interpolation='nearest', origin='lower') plt.xticks([]) plt.yticks([]) plt.title(tt, **targs) # NGC classification info = '' info = ngc_typenames.get(obj.classification.strip(), '') if len(info): info = '(' + info + ') ' plt.suptitle( '%s %sin %s %i-%s: %s band' % (obj.name, info, nice_camera_name, expnum, extname, newband)) # Save / show plot if self.opt.show: plt.draw() plt.show(block=False) plt.pause(0.001) plotfn = ('ngcbot-%i-%s-%s.png' % (expnum, extname, obj.name.replace(' ', '_'))) if self.opt.plotdir: plotfn = os.path.join(self.opt.plotdir, plotfn) plt.savefig(plotfn) print('Saved', plotfn) good_coverage = 0.75 if self.opt.coverage is not None: good_coverage = self.opt.coverage if coverage > good_coverage: goodplots.append(plotfn) # Compose tweet text if self.opt.tweet: import urllib url = 'http://legacysurvey.org/viewer/?ra=%.3f&dec=%.3f' % ( obj.ra, obj.dec) if bestdata is not None: url += '&layer=%s' % bestdata url2 = ('http://ned.ipac.caltech.edu/cgi-bin/objsearch?' + urllib.urlencode( dict(objname=obj.name, corr_z=1, list_limit=5, img_stamp='YES'))) dateobs = primhdr['DATE-OBS'].strip() # 2017-03-06T00:15:40.101482 dateobs = dateobs[:19] dateobs = dateobs.replace('T', ' ') txt = ('%s observed %s %sin image %i-%s: %s band' % (nice_camera_name, obj.name, info, expnum, extname, newband) + ' at %s UT' % dateobs + '\n' + url + '\n' + url2) print('Tweet text:', txt) if coverage > good_coverage: tweets.append((txt, plotfn)) else: print('Coverage', coverage, 'less than target', good_coverage, 'so not tweeting') # Select a random good-looking plot if len(goodplots): irandom = np.random.randint(0, len(goodplots)) # Copy that plot to ngcbot-latest.png plotfn = goodplots[irandom] import shutil latest = 'ngcbot-latest.png' print('Copying', plotfn, 'to', latest) shutil.copy(plotfn, latest) # Tweet one NGC object per exposure, chosen randomly. if len(tweets): assert (len(tweets) == len(goodplots)) txt, plotfn = tweets[irandom] if self.opt.tweet: send_tweet(txt, plotfn) return True
def stage0(**kwargs): ps = PlotSequence('cfht') decals = CfhtDecals() B = decals.get_bricks() print('Bricks:') B.about() ra, dec = 190.0, 11.0 #bands = 'ugri' bands = 'gri' B.cut(np.argsort(degrees_between(ra, dec, B.ra, B.dec))) print('Nearest bricks:', B.ra[:5], B.dec[:5], B.brickid[:5]) brick = B[0] pixscale = 0.186 #W,H = 1024,1024 #W,H = 2048,2048 #W,H = 3600,3600 W, H = 4800, 4800 targetwcs = wcs_for_brick(brick, pixscale=pixscale, W=W, H=H) ccdfn = 'cfht-ccds.fits' if os.path.exists(ccdfn): T = fits_table(ccdfn) else: T = get_ccd_list() T.writeto(ccdfn) print(len(T), 'CCDs') T.cut(ccds_touching_wcs(targetwcs, T)) print(len(T), 'CCDs touching brick') T.cut(np.array([b in bands for b in T.filter])) print(len(T), 'in bands', bands) ims = [] for t in T: im = CfhtImage(t) # magzp = hdr['PHOT_C'] + 2.5 * np.log10(hdr['EXPTIME']) # fwhm = t.seeing / (pixscale * 3600) # print '-> FWHM', fwhm, 'pix' im.seeing = t.seeing im.pixscale = t.pixscale print('seeing', t.seeing) print('pixscale', im.pixscale * 3600, 'arcsec/pix') im.run_calibs(t.ra, t.dec, im.pixscale, W=t.width, H=t.height) ims.append(im) # Read images, clip to ROI targetrd = np.array([ targetwcs.pixelxy2radec(x, y) for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)] ]) keepims = [] tims = [] for im in ims: print() print('Reading expnum', im.expnum, 'name', im.extname, 'band', im.band, 'exptime', im.exptime) band = im.band wcs = im.read_wcs() imh, imw = wcs.imageh, wcs.imagew imgpoly = [(1, 1), (1, imh), (imw, imh), (imw, 1)] ok, tx, ty = wcs.radec2pixelxy(targetrd[:-1, 0], targetrd[:-1, 1]) tpoly = zip(tx, ty) clip = clip_polygon(imgpoly, tpoly) clip = np.array(clip) #print 'Clip', clip if len(clip) == 0: continue x0, y0 = np.floor(clip.min(axis=0)).astype(int) x1, y1 = np.ceil(clip.max(axis=0)).astype(int) slc = slice(y0, y1 + 1), slice(x0, x1 + 1) ## FIXME -- it seems I got lucky and the cross product is ## negative == clockwise, as required by clip_polygon. One ## could check this and reverse the polygon vertex order. # dx0,dy0 = tx[1]-tx[0], ty[1]-ty[0] # dx1,dy1 = tx[2]-tx[1], ty[2]-ty[1] # cross = dx0*dy1 - dx1*dy0 # print 'Cross:', cross print('Image slice: x [%i,%i], y [%i,%i]' % (x0, x1, y0, y1)) print('Reading image from', im.imgfn, 'HDU', im.hdu) img, imghdr = im.read_image(header=True, slice=slc) goodpix = (img != 0) print('Number of pixels == 0:', np.sum(img == 0)) print('Number of pixels != 0:', np.sum(goodpix)) if np.sum(goodpix) == 0: continue # print 'Image shape', img.shape print('Image range', img.min(), img.max()) print('Goodpix image range:', (img[goodpix]).min(), (img[goodpix]).max()) if img[goodpix].min() == img[goodpix].max(): print('No dynamic range in image') continue # print 'Reading invvar from', im.wtfn, 'HDU', im.hdu # invvar = im.read_invvar(slice=slc) # # print 'Invvar shape', invvar.shape # # print 'Invvar range:', invvar.min(), invvar.max() # invvar[goodpix == 0] = 0. # if np.all(invvar == 0.): # print 'Skipping zero-invvar image' # continue # assert(np.all(np.isfinite(img))) # assert(np.all(np.isfinite(invvar))) # assert(not(np.all(invvar == 0.))) # # Estimate per-pixel noise via Blanton's 5-pixel MAD # slice1 = (slice(0,-5,10),slice(0,-5,10)) # slice2 = (slice(5,None,10),slice(5,None,10)) # # print 'sliced shapes:', img[slice1].shape, img[slice2].shape # # print 'good shape:', (goodpix[slice1] * goodpix[slice2]).shape # # print 'good values:', np.unique(goodpix[slice1] * goodpix[slice2]) # # print 'sliced[good] shapes:', (img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].shape # mad = np.median(np.abs(img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].ravel()) # sig1 = 1.4826 * mad / np.sqrt(2.) # print 'MAD sig1:', sig1 # # invvar was 1 or 0 # invvar *= (1./(sig1**2)) # medsky = np.median(img[goodpix]) # Read full image for sig1 and sky estimate fullimg = im.read_image() fullgood = (fullimg != 0) # Estimate per-pixel noise via Blanton's 5-pixel MAD slice1 = (slice(0, -5, 10), slice(0, -5, 10)) slice2 = (slice(5, None, 10), slice(5, None, 10)) mad = np.median( np.abs(fullimg[slice1] - fullimg[slice2])[fullgood[slice1] * fullgood[slice2]].ravel()) sig1 = 1.4826 * mad / np.sqrt(2.) print('MAD sig1:', sig1) medsky = np.median(fullimg[fullgood]) invvar = np.zeros_like(img) invvar[goodpix] = 1. / sig1**2 # Median-smooth sky subtraction plt.clf() dimshow(np.round((img - medsky) / sig1), vmin=-3, vmax=5) plt.title('Scalar median: %s' % im.name) ps.savefig() # medsky = np.zeros_like(img) # # astrometry.util.util # median_smooth(img, np.logical_not(goodpix), 256, medsky) fullmed = np.zeros_like(fullimg) median_smooth(fullimg - medsky, np.logical_not(fullgood), 256, fullmed) fullmed += medsky medimg = fullmed[slc] plt.clf() dimshow(np.round((img - medimg) / sig1), vmin=-3, vmax=5) plt.title('Median filtered: %s' % im.name) ps.savefig() #print 'Subtracting median:', medsky #img -= medsky img -= medimg primhdr = im.read_image_primary_header() magzp = decals.get_zeropoint_for(im) print('magzp', magzp) zpscale = NanoMaggies.zeropointToScale(magzp) print('zpscale', zpscale) # Scale images to Nanomaggies img /= zpscale sig1 /= zpscale invvar *= zpscale**2 orig_zpscale = zpscale zpscale = 1. assert (np.sum(invvar > 0) > 0) print('After scaling:') print('sig1', sig1) print('invvar range', invvar.min(), invvar.max()) print('image range', img.min(), img.max()) assert (np.all(np.isfinite(img))) assert (np.all(np.isfinite(invvar))) assert (np.isfinite(sig1)) plt.clf() lo, hi = -5 * sig1, 10 * sig1 n, b, p = plt.hist(img[goodpix].ravel(), 100, range=(lo, hi), histtype='step', color='k') xx = np.linspace(lo, hi, 200) plt.plot(xx, max(n) * np.exp(-xx**2 / (2. * sig1**2)), 'r-') plt.xlim(lo, hi) plt.title('Pixel histogram: %s' % im.name) ps.savefig() twcs = ConstantFitsWcs(wcs) if x0 or y0: twcs.setX0Y0(x0, y0) info = im.get_image_info() fullh, fullw = info['dims'] # read fit PsfEx model psfex = PsfEx.fromFits(im.psffitfn) print('Read', psfex) # HACK -- highly approximate PSF here! #psf_fwhm = imghdr['FWHM'] #psf_fwhm = im.seeing psf_fwhm = im.seeing / (im.pixscale * 3600) print('PSF FWHM', psf_fwhm, 'pixels') psf_sigma = psf_fwhm / 2.35 psf = NCircularGaussianPSF([psf_sigma], [1.]) print('img type', img.dtype) tim = Image(img, invvar=invvar, wcs=twcs, psf=psf, photocal=LinearPhotoCal(zpscale, band=band), sky=ConstantSky(0.), name=im.name + ' ' + band) tim.zr = [-3. * sig1, 10. * sig1] tim.sig1 = sig1 tim.band = band tim.psf_fwhm = psf_fwhm tim.psf_sigma = psf_sigma tim.sip_wcs = wcs tim.x0, tim.y0 = int(x0), int(y0) tim.psfex = psfex tim.imobj = im mn, mx = tim.zr tim.ima = dict(interpolation='nearest', origin='lower', cmap='gray', vmin=mn, vmax=mx) tims.append(tim) keepims.append(im) ims = keepims print('Computing resampling...') # save resampling params for tim in tims: wcs = tim.sip_wcs subh, subw = tim.shape subwcs = wcs.get_subimage(tim.x0, tim.y0, subw, subh) tim.subwcs = subwcs try: Yo, Xo, Yi, Xi, rims = resample_with_wcs(targetwcs, subwcs, [], 2) except OverlapError: print('No overlap') continue if len(Yo) == 0: continue tim.resamp = (Yo, Xo, Yi, Xi) print('Creating coadds...') # Produce per-band coadds, for plots coimgs = [] cons = [] for ib, band in enumerate(bands): coimg = np.zeros((H, W), np.float32) con = np.zeros((H, W), np.uint8) for tim in tims: if tim.band != band: continue (Yo, Xo, Yi, Xi) = tim.resamp if len(Yo) == 0: continue nn = (tim.getInvvar()[Yi, Xi] > 0) coimg[Yo, Xo] += tim.getImage()[Yi, Xi] * nn con[Yo, Xo] += nn # print # print 'tim', tim.name # print 'number of resampled pix:', len(Yo) # reim = np.zeros_like(coimg) # ren = np.zeros_like(coimg) # reim[Yo,Xo] = tim.getImage()[Yi,Xi] * nn # ren[Yo,Xo] = nn # print 'number of resampled pix with positive invvar:', ren.sum() # plt.clf() # plt.subplot(2,2,1) # mn,mx = [np.percentile(reim[ren>0], p) for p in [25,95]] # print 'Percentiles:', mn,mx # dimshow(reim, vmin=mn, vmax=mx) # plt.colorbar() # plt.subplot(2,2,2) # dimshow(con) # plt.colorbar() # plt.subplot(2,2,3) # dimshow(reim, vmin=tim.zr[0], vmax=tim.zr[1]) # plt.colorbar() # plt.subplot(2,2,4) # plt.hist(reim.ravel(), 100, histtype='step', color='b') # plt.hist(tim.getImage().ravel(), 100, histtype='step', color='r') # plt.suptitle('%s: %s' % (band, tim.name)) # ps.savefig() coimg /= np.maximum(con, 1) coimgs.append(coimg) cons.append(con) plt.clf() dimshow(get_rgb(coimgs, bands)) ps.savefig() plt.clf() for i, b in enumerate(bands): plt.subplot(2, 2, i + 1) dimshow(cons[i], ticks=False) plt.title('%s band' % b) plt.colorbar() plt.suptitle('Number of exposures') ps.savefig() print('Grabbing SDSS sources...') bandlist = [b for b in bands] cat, T = get_sdss_sources(bandlist, targetwcs) # record coordinates in target brick image ok, T.tx, T.ty = targetwcs.radec2pixelxy(T.ra, T.dec) T.tx -= 1 T.ty -= 1 T.itx = np.clip(np.round(T.tx).astype(int), 0, W - 1) T.ity = np.clip(np.round(T.ty).astype(int), 0, H - 1) plt.clf() dimshow(get_rgb(coimgs, bands)) ax = plt.axis() plt.plot(T.tx, T.ty, 'o', mec=green, mfc='none', ms=10, mew=1.5) plt.axis(ax) plt.title('SDSS sources') ps.savefig() print('Detmaps...') # Render the detection maps detmaps = dict([(b, np.zeros((H, W), np.float32)) for b in bands]) detivs = dict([(b, np.zeros((H, W), np.float32)) for b in bands]) for tim in tims: iv = tim.getInvvar() psfnorm = 1. / (2. * np.sqrt(np.pi) * tim.psf_sigma) detim = tim.getImage().copy() detim[iv == 0] = 0. detim = gaussian_filter(detim, tim.psf_sigma) / psfnorm**2 detsig1 = tim.sig1 / psfnorm subh, subw = tim.shape detiv = np.zeros((subh, subw), np.float32) + (1. / detsig1**2) detiv[iv == 0] = 0. (Yo, Xo, Yi, Xi) = tim.resamp detmaps[tim.band][Yo, Xo] += detiv[Yi, Xi] * detim[Yi, Xi] detivs[tim.band][Yo, Xo] += detiv[Yi, Xi] rtn = dict() for k in [ 'T', 'coimgs', 'cons', 'detmaps', 'detivs', 'targetrd', 'pixscale', 'targetwcs', 'W', 'H', 'bands', 'tims', 'ps', 'brick', 'cat' ]: rtn[k] = locals()[k] return rtn
def unwise_coadds(onegal, galaxy=None, radius_mosaic=30, radius_mask=None, pixscale=2.75, ref_pixscale=0.262, output_dir=None, unwise_dir=None, verbose=False, log=None, centrals=True): '''Generate custom unWISE cutouts. radius_mosaic and radius_mask in arcsec pixscale: WISE pixel scale in arcsec/pixel; make this smaller than 2.75 to oversample. ''' import fitsio import matplotlib.pyplot as plt from astrometry.util.util import Tan from astrometry.util.fits import fits_table from astrometry.libkd.spherematch import match_radec from astrometry.util.resample import resample_with_wcs, ResampleError from wise.forcedphot import unwise_tiles_touching_wcs from wise.unwise import get_unwise_tractor_image from tractor import Tractor, Image, NanoMaggies from legacypipe.survey import imsave_jpeg from legacypipe.catalog import read_fits_catalog if galaxy is None: galaxy = 'galaxy' if output_dir is None: output_dir = '.' if unwise_dir is None: unwise_dir = os.environ.get('UNWISE_COADDS_DIR') if radius_mask is None: radius_mask = radius_mosaic radius_search = 5.0 # [arcsec] else: radius_search = radius_mask # Initialize the WCS object. W = H = np.ceil(2 * radius_mosaic / pixscale).astype('int') # [pixels] targetwcs = Tan(onegal['RA'], onegal['DEC'], (W + 1) / 2.0, (H + 1) / 2.0, -pixscale / 3600.0, 0.0, 0.0, pixscale / 3600.0, float(W), float(H)) # Read the custom Tractor catalog. tractorfile = os.path.join(output_dir, '{}-tractor.fits'.format(galaxy)) if not os.path.isfile(tractorfile): print('Missing Tractor catalog {}'.format(tractorfile), flush=True, file=log) return 0 primhdr = fitsio.read_header(tractorfile) cat = fits_table(tractorfile) print('Read {} sources from {}'.format(len(cat), tractorfile), flush=True, file=log) keep = np.ones(len(cat)).astype(bool) if centrals: # Find the large central galaxy and mask out (ignore) all the models # which are within its elliptical mask. # This algorithm will have to change for mosaics not centered on large # galaxies, e.g., in galaxy groups. m1, m2, d12 = match_radec(cat.ra, cat.dec, onegal['RA'], onegal['DEC'], radius_search / 3600.0, nearest=False) if len(m1) == 0: print('No central galaxies found at the central coordinates!', flush=True, file=log) else: pixfactor = ref_pixscale / pixscale # shift the optical Tractor positions for mm in m1: morphtype = cat.type[mm].strip() if morphtype == 'EXP' or morphtype == 'COMP': e1, e2, r50 = cat.shapeexp_e1[mm], cat.shapeexp_e2[ mm], cat.shapeexp_r[mm] # [arcsec] elif morphtype == 'DEV' or morphtype == 'COMP': e1, e2, r50 = cat.shapedev_e1[mm], cat.shapedev_e2[ mm], cat.shapedev_r[mm] # [arcsec] else: r50 = None if r50: majoraxis = r50 * 5 / pixscale # [pixels] ba, phi = SGA.misc.convert_tractor_e1e2(e1, e2) these = SGA.misc.ellipse_mask(W / 2, W / 2, majoraxis, ba * majoraxis, np.radians(phi), cat.bx * pixfactor, cat.by * pixfactor) if np.sum(these) > 0: #keep[these] = False pass print('Hack!') keep[mm] = False #srcs = read_fits_catalog(cat) #_srcs = np.array(srcs)[~keep].tolist() #mod = SGA.misc.srcs2image(_srcs, ConstantFitsWcs(targetwcs), psf_sigma=3.0) #import matplotlib.pyplot as plt ##plt.imshow(mod, origin='lower') ; plt.savefig('junk.png') #plt.imshow(np.log10(mod), origin='lower') ; plt.savefig('junk.png') #pdb.set_trace() srcs = read_fits_catalog(cat) for src in srcs: src.freezeAllBut('brightness') #srcs_nocentral = np.array(srcs)[keep].tolist() cat_nocentral = cat[keep] ## Find and remove all the objects within XX arcsec of the target ## coordinates. #m1, m2, d12 = match_radec(T.ra, T.dec, onegal['RA'], onegal['DEC'], 5/3600.0, nearest=False) #if len(d12) == 0: # print('No matching galaxies found -- probably not what you wanted.') # #raise ValueError # nocentral = np.ones(len(T)).astype(bool) #else: # nocentral = ~np.isin(T.objid, T[m1].objid) #T_nocentral = T[nocentral] # Find and read the overlapping unWISE tiles. Assume the targetwcs is # axis-aligned and that the edge midpoints yield the RA, Dec limits (true # for TAN). Note: the way the roiradec box is used, the min/max order # doesn't matter. r, d = targetwcs.pixelxy2radec(np.array([1, W, W / 2, W / 2]), np.array([H / 2, H / 2, 1, H])) roiradec = [r[0], r[1], d[2], d[3]] tiles = unwise_tiles_touching_wcs(targetwcs) wbands = [1, 2, 3, 4] wanyband = 'w' vega_to_ab = dict(w1=2.699, w2=3.339, w3=5.174, w4=6.620) # Convert the AB WISE fluxes in the Tractor catalog to Vega nanomaggies so # they're consistent with the coadds, below. for band in wbands: f = cat.get('flux_w{}'.format(band)) e = cat.get('flux_ivar_w{}'.format(band)) print('Setting negative fluxes equal to zero!') f[f < 0] = 0 #f[f/e < 3] = 0 f *= 10**(0.4 * vega_to_ab['w{}'.format(band)]) coimgs = [np.zeros((H, W), np.float32) for b in wbands] comods = [np.zeros((H, W), np.float32) for b in wbands] comods_nocentral = [np.zeros((H, W), np.float32) for b in wbands] con = [np.zeros((H, W), np.uint8) for b in wbands] for iband, band in enumerate(wbands): for ii, src in enumerate(srcs): src.setBrightness( NanoMaggies( **{wanyband: cat.get('flux_w{}'.format(band))[ii]})) srcs_nocentral = np.array(srcs)[keep].tolist() #srcs_nocentral = np.array(srcs)[nocentral].tolist() # The tiles have some overlap, so for each source, keep the fit in the # tile whose center is closest to the source. for tile in tiles: #print('Reading tile {}'.format(tile.coadd_id)) tim = get_unwise_tractor_image(unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradec) if tim is None: print('Actually, no overlap with tile {}'.format( tile.coadd_id)) continue print('Read image {} with shape {}'.format(tile.coadd_id, tim.shape)) def _unwise_mod(tim, use_cat, use_srcs, margin=10): # Select sources in play. wisewcs = tim.wcs.wcs timH, timW = tim.shape ok, x, y = wisewcs.radec2pixelxy(use_cat.ra, use_cat.dec) x = (x - 1.).astype(np.float32) y = (y - 1.).astype(np.float32) I = np.flatnonzero((x >= -margin) * (x < timW + margin) * (y >= -margin) * (y < timH + margin)) #print('Found {} sources within the image + margin = {} pixels'.format(len(I), margin)) subcat = [use_srcs[i] for i in I] tractor = Tractor([tim], subcat) mod = tractor.getModelImage(0) return mod mod = _unwise_mod(tim, cat, srcs) mod_nocentral = _unwise_mod(tim, cat_nocentral, srcs_nocentral) try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, tim.wcs.wcs) except ResampleError: continue if len(Yo) == 0: continue # The models are already in AB nanomaggies, but the tiles / tims are # in Vega nanomaggies, so convert them here. coimgs[iband][Yo, Xo] += tim.getImage()[Yi, Xi] comods[iband][Yo, Xo] += mod[Yi, Xi] comods_nocentral[iband][Yo, Xo] += mod_nocentral[Yi, Xi] con[iband][Yo, Xo] += 1 ## Convert back to nanomaggies. #vega2ab = vega_to_ab['w{}'.format(band)] #coimgs[iband] *= 10**(-0.4 * vega2ab) #comods[iband] *= 10**(-0.4 * vega2ab) #comods_nocentral[iband] *= 10**(-0.4 * vega2ab) for img, mod, mod_nocentral, n in zip(coimgs, comods, comods_nocentral, con): img /= np.maximum(n, 1) mod /= np.maximum(n, 1) mod_nocentral /= np.maximum(n, 1) coresids = [img - mod for img, mod in list(zip(coimgs, comods))] # Subtract the model image which excludes the central (comod_nocentral) # from the data (coimg) to isolate the light of the central # (coimg_central). coimgs_central = [ img - mod for img, mod in list(zip(coimgs, comods_nocentral)) ] # Write out the final images with and without the central and converted into # AB nanomaggies. for coadd, imtype in zip((coimgs, comods, comods_nocentral), ('image', 'model', 'model-nocentral')): for img, band in zip(coadd, wbands): vega2ab = vega_to_ab['w{}'.format(band)] fitsfile = os.path.join( output_dir, '{}-{}-W{}.fits'.format(galaxy, imtype, band)) if verbose: print('Writing {}'.format(fitsfile)) fitsio.write(fitsfile, img * 10**(-0.4 * vega2ab), clobber=True) # Generate color WISE images. kwa = dict(mn=-1, mx=100, arcsinh=0.5) #kwa = dict(mn=-0.05, mx=1., arcsinh=0.5) #kwa = dict(mn=-0.1, mx=2., arcsinh=None) for imgs, imtype in zip( (coimgs, comods, coresids, comods_nocentral, coimgs_central), ('image', 'model', 'resid', 'model-nocentral', 'image-central')): rgb = _unwise_to_rgb(imgs[:2], **kwa) # W1, W2 jpgfile = os.path.join(output_dir, '{}-{}-W1W2.jpg'.format(galaxy, imtype)) if verbose: print('Writing {}'.format(jpgfile)) imsave_jpeg(jpgfile, rgb, origin='lower') return 1
def map_decals_wl(req, ver, zoom, x, y): tag = 'decals-wl' ignoreCached = False filename = None forcecache = False from decals import settings savecache = settings.SAVE_CACHE zoom = int(zoom) zoomscale = 2.**zoom x = int(x) y = int(y) if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale: raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom,x,y)) ver = int(ver) if not ver in tileversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) basedir = settings.DATA_DIR tilefn = os.path.join(basedir, 'tiles', tag, '%i/%i/%i/%i.jpg' % (ver, zoom, x, y)) if os.path.exists(tilefn) and not ignoreCached: print('Cached:', tilefn) return send_file(tilefn, 'image/jpeg', expires=oneyear, modsince=req.META.get('HTTP_IF_MODIFIED_SINCE'), filename=filename) else: print('Tile image does not exist:', tilefn) from astrometry.util.resample import resample_with_wcs, OverlapError from astrometry.util.util import Tan from astrometry.libkd.spherematch import match_radec from astrometry.util.fits import fits_table from astrometry.util.starutil_numpy import degrees_between import numpy as np import fitsio try: wcs, W, H, zoomscale, zoom,x,y = get_tile_wcs(zoom, x, y) except RuntimeError as e: return HttpResponse(e.strerror) mydir = os.path.join(basedir, 'coadd', 'weak-lensing') rlo,d = wcs.pixelxy2radec(W, H/2)[-2:] rhi,d = wcs.pixelxy2radec(1, H/2)[-2:] r,d1 = wcs.pixelxy2radec(W/2, 1)[-2:] r,d2 = wcs.pixelxy2radec(W/2, H)[-2:] #dlo = min(d1, d2) #dhi = max(d1, d2) r,d = wcs.pixelxy2radec(W/2, H/2)[-2:] rad = degrees_between(r, d, rlo, d1) fn = os.path.join(mydir, 'index.fits') if not os.path.exists(fn): # ii,rr,dd = [],[],[] for i in range(1, 52852+1): imgfn = os.path.join(mydir, 'map%i.fits' % i) hdr = fitsio.read_header(imgfn) r = hdr['CRVAL1'] d = hdr['CRVAL2'] ii.append(i) rr.append(r) dd.append(d) T = fits_table() T.ra = np.array(rr) T.dec = np.array(dd) T.i = np.array(ii) T.writeto(fn) T = fits_table(fn) I,J,d = match_radec(T.ra, T.dec, r, d, rad + 0.2) T.cut(I) print(len(T), 'weak-lensing maps in range') if len(I) == 0: from django.http import HttpResponseRedirect if forcecache: # create symlink to blank.jpg! trymakedirs(tilefn) src = os.path.join(settings.STATIC_ROOT, 'blank.jpg') if os.path.exists(tilefn): os.unlink(tilefn) os.symlink(src, tilefn) print('Symlinked', tilefn, '->', src) return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg') r,d = wcs.pixelxy2radec([1,1,1,W/2,W,W,W,W/2], [1,H/2,H,H,H,H/2,1,1])[-2:] foundany = False rimg = np.zeros((H,W), np.float32) rn = np.zeros((H,W), np.uint8) for tilei in T.i: fn = os.path.join(mydir, 'map%i.fits' % tilei) try: bwcs = _read_tan_wcs(fn, 0) except: print('Failed to read WCS:', fn) savecache = False import traceback import sys traceback.print_exc(None, sys.stdout) continue foundany = True print('Reading', fn) ok,xx,yy = bwcs.radec2pixelxy(r, d) xx = xx.astype(np.int) yy = yy.astype(np.int) imW,imH = int(bwcs.get_width()), int(bwcs.get_height()) M = 10 xlo = np.clip(xx.min() - M, 0, imW) xhi = np.clip(xx.max() + M, 0, imW) ylo = np.clip(yy.min() - M, 0, imH) yhi = np.clip(yy.max() + M, 0, imH) if xlo >= xhi or ylo >= yhi: continue subwcs = bwcs.get_subimage(xlo, ylo, xhi-xlo, yhi-ylo) slc = slice(ylo,yhi), slice(xlo,xhi) try: f = fitsio.FITS(fn)[0] img = f[slc] del f except: print('Failed to read image and WCS:', fn) savecache = False import traceback import sys traceback.print_exc(None, sys.stdout) continue try: Yo,Xo,Yi,Xi,nil = resample_with_wcs(wcs, subwcs, [], 3) except OverlapError: print('Resampling exception') continue rimg[Yo,Xo] += img[Yi,Xi] rn [Yo,Xo] += 1 rimg /= np.maximum(rn, 1) if forcecache: savecache = True if savecache: trymakedirs(tilefn) else: import tempfile f,tilefn = tempfile.mkstemp(suffix='.jpg') os.close(f) import pylab as plt # S/N #lo,hi = 1.5, 5.0 lo,hi = 0, 5.0 rgb = plt.cm.hot((rimg - lo) / (hi - lo)) plt.imsave(tilefn, rgb) print('Wrote', tilefn) return send_file(tilefn, 'image/jpeg', unlink=(not savecache), filename=filename)
def cutout_jpg(req): import tempfile import fitsio from astrometry.util.util import Tan from astrometry.util.resample import resample_with_wcs form = CutoutSearchForm(req.GET) if not form.is_valid(): return HttpResponse('failed to parse request') ra = form.cleaned_data['ra'] dec = form.cleaned_data['dec'] if ra is None or dec is None: return HttpResponse('RA and Dec arguments are required') size = form.cleaned_data['size'] if size is None: size = 100 else: size = min(256, size) # Ignoring this for now... # bandstr = form.cleaned_data['bands'] # bands = [int(c) for c in bandstr] # bands = [b for b in bands if b in [1,2,3,4]] version = form.cleaned_data['version'] radius = size/2. * 2.75/3600. tiles = unwise_tiles_near_radec(ra, dec, radius) tiles = list(tiles) pixscale = 2.75 / 3600. cowcs = Tan(*[float(x) for x in [ra, dec, 0.5 + size/2., 0.5 + size/2., -pixscale, 0., 0., pixscale, size, size]]) bands = [1,2] coims = [np.zeros((size,size), np.float32) for b in bands] con = np.zeros((size,size), int) for tile in tiles: coadd = tile.coadd dirnm = os.path.join(settings.DATA_DIR, version, 'unwise-coadds', 'fulldepth', coadd[:3], coadd) base = os.path.join(dirnm, 'unwise-%s' % coadd) subwcs = None ims = [] for band in bands: fn = str(base + '-w%i-img-m.fits' % band) if subwcs is None: wcs = Tan(fn) ok,x,y = wcs.radec2pixelxy(ra, dec) x = int(round(x-1.)) y = int(round(y-1.)) x0 = x - size/2 x1 = x0 + size y0 = y - size/2 y1 = y0 + size if x1 <= 0 or y1 <= 0: break if x0 >= wcs.get_width() or y0 >= wcs.get_height(): break x0 = max(x0, 0) y0 = max(y0, 0) x1 = min(x1, wcs.get_width()) y1 = min(y1, wcs.get_height()) subwcs = wcs.get_subimage(x0, y0, x1-x0, y1-y0) if subwcs is None: break img = fitsio.FITS(fn)[0][y0:y1, x0:x1] ims.append(img) if subwcs is None: continue try: Yo,Xo,Yi,Xi,rims = resample_with_wcs(cowcs, subwcs, ims, 3) except: continue for rim,co in zip(rims, coims): co[Yo,Xo] += rim con[Yo,Xo] += 1 for co in coims: co /= np.maximum(con, 1) rgb = np.zeros((size,size,3), np.uint8) lo,hi = -3,10 scale = 10. rgb[:,:,2] = np.clip(255. * ((coims[1] / scale) - lo) / (hi-lo), 0, 255) rgb[:,:,1] = np.clip(255. * (((coims[0]+coims[1])/2. / scale) - lo) / (hi-lo), 0, 255) rgb[:,:,0] = np.clip(255. * ((coims[0] / scale) - lo) / (hi-lo), 0, 255) f,tempfn = tempfile.mkstemp(suffix='.jpg') os.close(f) # import matplotlib # matplotlib.use('Agg') # import pylab as plt # plt.imsave(tempfn, rgb, interpolation='nearest', origin='lower') from PIL import Image pic = Image.fromarray(rgb) pic.save(tempfn) return send_file(tempfn, 'image/jpeg', unlink=True)
def map_sdss(req, ver, zoom, x, y, savecache=None, tag='sdss', get_images=False, ignoreCached=False, wcs=None, forcecache=False, forcescale=None, **kwargs): from decals import settings if savecache is None: savecache = settings.SAVE_CACHE zoom = int(zoom) zoomscale = 2.**zoom x = int(x) y = int(y) if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale: raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom,x,y)) ver = int(ver) if not ver in tileversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) basedir = settings.DATA_DIR tilefn = os.path.join(basedir, 'tiles', tag, '%i/%i/%i/%i.jpg' % (ver, zoom, x, y)) if os.path.exists(tilefn) and not ignoreCached: if get_images: return None return send_file(tilefn, 'image/jpeg', expires=oneyear, modsince=req.META.get('HTTP_IF_MODIFIED_SINCE')) if not savecache: import tempfile f,tilefn = tempfile.mkstemp(suffix='.jpg') os.close(f) if wcs is None: try: wcs, W, H, zoomscale, zoom,x,y = get_tile_wcs(zoom, x, y) except RuntimeError as e: if get_images: return None return HttpResponse(e.strerror) else: W = wcs.get_width() H = wcs.get_height() from astrometry.util.fits import fits_table import numpy as np from astrometry.libkd.spherematch import tree_build_radec, tree_search_radec from astrometry.util.starutil_numpy import degrees_between, arcsec_between from astrometry.util.resample import resample_with_wcs, OverlapError from astrometry.util.util import Tan, Sip import fitsio print 'Tile wcs: center', wcs.radec_center(), 'pixel scale', wcs.pixel_scale() global w_flist global w_flist_tree if w_flist is None: w_flist = fits_table(os.path.join(settings.DATA_DIR, 'sdss', 'window_flist.fits'), columns=['run','rerun','camcol','field','ra','dec','score']) print 'Read', len(w_flist), 'window_flist entries' w_flist.cut(w_flist.rerun == '301') print 'Cut to', len(w_flist), 'in rerun 301' w_flist_tree = tree_build_radec(w_flist.ra, w_flist.dec) # SDSS field size radius = 1.01 * np.hypot(10., 14.)/2. / 60. # leaflet tile size ra,dec = wcs.pixelxy2radec(W/2., H/2.)[-2:] r0,d0 = wcs.pixelxy2radec(1, 1)[-2:] r1,d1 = wcs.pixelxy2radec(W, H)[-2:] radius = radius + max(degrees_between(ra,dec, r0,d0), degrees_between(ra,dec, r1,d1)) J = tree_search_radec(w_flist_tree, ra, dec, radius) print len(J), 'overlapping SDSS fields found' if len(J) == 0: if get_images: return None if forcecache: # create symlink to blank.jpg! trymakedirs(tilefn) src = os.path.join(settings.STATIC_ROOT, 'blank.jpg') if os.path.exists(tilefn): os.unlink(tilefn) os.symlink(src, tilefn) print 'Symlinked', tilefn, '->', src from django.http import HttpResponseRedirect return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg') ww = [1, W*0.25, W*0.5, W*0.75, W] hh = [1, H*0.25, H*0.5, H*0.75, H] r,d = wcs.pixelxy2radec( [1]*len(hh) + ww + [W]*len(hh) + list(reversed(ww)), hh + [1]*len(ww) + list(reversed(hh)) + [H]*len(ww))[-2:] scaled = 0 scalepat = None scaledir = 'sdss' if zoom <= 13 and forcescale is None: # Get *actual* pixel scales at the top & bottom r1,d1 = wcs.pixelxy2radec(W/2., H)[-2:] r2,d2 = wcs.pixelxy2radec(W/2., H-1.)[-2:] r3,d3 = wcs.pixelxy2radec(W/2., 1.)[-2:] r4,d4 = wcs.pixelxy2radec(W/2., 2.)[-2:] # Take the min = most zoomed-in scale = min(arcsec_between(r1,d1, r2,d2), arcsec_between(r3,d3, r4,d4)) native_scale = 0.396 scaled = int(np.floor(np.log2(scale / native_scale))) print 'Zoom:', zoom, 'x,y', x,y, 'Tile pixel scale:', scale, 'Scale step:', scaled scaled = np.clip(scaled, 1, 7) dirnm = os.path.join(basedir, 'scaled', scaledir) scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(rerun)s', '%(run)i', '%(camcol)i', 'sdss-%(run)i-%(camcol)i-%(field)i-%(band)s.fits') if forcescale is not None: scaled = forcescale bands = 'gri' rimgs = [np.zeros((H,W), np.float32) for band in bands] rns = [np.zeros((H,W), np.float32) for band in bands] from astrometry.sdss import AsTransWrapper, DR9 sdss = DR9(basedir=settings.SDSS_DIR) sdss.saveUnzippedFiles(settings.SDSS_DIR) #sdss.setFitsioReadBZ2() if settings.SDSS_PHOTOOBJS: sdss.useLocalTree(photoObjs=settings.SDSS_PHOTOOBJS, resolve=settings.SDSS_RESOLVE) for jnum,j in enumerate(J): print 'SDSS field', jnum, 'of', len(J), 'for zoom', zoom, 'x', x, 'y', y im = w_flist[j] if im.score >= 0.5: weight = 1. else: weight = 0.001 for band,rimg,rn in zip(bands, rimgs, rns): if im.rerun != '301': continue tmpsuff = '.tmp%08i' % np.random.randint(100000000) basefn = sdss.retrieve('frame', im.run, im.camcol, field=im.field, band=band, rerun=im.rerun, tempsuffix=tmpsuff) if scaled > 0: fnargs = dict(band=band, rerun=im.rerun, run=im.run, camcol=im.camcol, field=im.field) fn = get_scaled(scalepat, fnargs, scaled, basefn, read_base_wcs=read_astrans, read_wcs=_read_sip_wcs) print 'get_scaled:', fn else: fn = basefn frame = None if fn == basefn: frame = sdss.readFrame(im.run, im.camcol, im.field, band, filename=fn) h,w = frame.getImageShape() # Trim off the overlapping top of the image # Wimp out and instead of trimming 128 pix, trim 124! trim = 124 subh = h - trim astrans = frame.getAsTrans() fwcs = AsTransWrapper(astrans, w, subh) fullimg = frame.getImage() fullimg = fullimg[:-trim,:] else: fwcs = Sip(fn) fitsimg = fitsio.FITS(fn)[0] h,w = fitsimg.get_info()['dims'] fullimg = fitsimg.read() try: #Yo,Xo,Yi,Xi,nil = resample_with_wcs(wcs, fwcs, [], 3) Yo,Xo,Yi,Xi,[resamp] = resample_with_wcs(wcs, fwcs, [fullimg], 2) except OverlapError: continue if len(Xi) == 0: #print 'No overlap' continue if sdssps is not None: x0 = Xi.min() x1 = Xi.max() y0 = Yi.min() y1 = Yi.max() slc = (slice(y0,y1+1), slice(x0,x1+1)) if frame is not None: img = frame.getImageSlice(slc) else: img = fitsimg[slc] #rimg[Yo,Xo] += img[Yi-y0, Xi-x0] rimg[Yo,Xo] += resamp * weight rn [Yo,Xo] += weight if sdssps is not None: # goodpix = np.ones(img.shape, bool) # fpM = sdss.readFpM(im.run, im.camcol, im.field, band) # for plane in [ 'INTERP', 'SATUR', 'CR', 'GHOST' ]: # fpM.setMaskedPixels(plane, goodpix, False, roi=[x0,x1,y0,y1]) plt.clf() #ima = dict(vmin=-0.05, vmax=0.5) #ima = dict(vmin=-0.5, vmax=2.) ima = dict(vmax=np.percentile(img, 99)) plt.subplot(2,3,1) dimshow(img, ticks=False, **ima) plt.title('image') rthis = np.zeros_like(rimg) #rthis[Yo,Xo] += img[Yi-y0, Xi-x0] rthis[Yo,Xo] += resamp plt.subplot(2,3,2) dimshow(rthis, ticks=False, **ima) plt.title('resampled') # plt.subplot(2,3,3) # dimshow(goodpix, ticks=False, vmin=0, vmax=1) # plt.title('good pix') plt.subplot(2,3,4) dimshow(rimg / np.maximum(rn, 1), ticks=False, **ima) plt.title('coadd') plt.subplot(2,3,5) dimshow(rn, vmin=0, ticks=False) plt.title('coverage: max %i' % rn.max()) plt.subplot(2,3,6) rgb = sdss_rgb([rimg/np.maximum(rn,1) for rimg,rn in zip(rimgs,rns)], bands) dimshow(rgb) plt.suptitle('SDSS %s, R/C/F %i/%i/%i' % (band, im.run, im.camcol, im.field)) sdssps.savefig() for rimg,rn in zip(rimgs, rns): rimg /= np.maximum(rn, 1e-3) del rns if get_images: return rimgs rgb = sdss_rgb(rimgs, bands) trymakedirs(tilefn) save_jpeg(tilefn, rgb) print 'Wrote', tilefn return send_file(tilefn, 'image/jpeg', unlink=(not savecache))
def render_into_wcs(self, wcs, zoom, x, y, bands=None, general_wcs=False, scale=None, tempfiles=None): import numpy as np from astrometry.util.resample import resample_with_wcs, OverlapError if scale is None: scale = self.get_scale(zoom, x, y, wcs) #if bricks is None or len(bricks) == 0: # print('No bricks touching WCS') # return None if bands is None: bands = self.get_bands() W = int(wcs.get_width()) H = int(wcs.get_height()) r, d = wcs.pixelxy2radec([1, 1, 1, W / 2, W, W, W, W / 2], [1, H / 2, H, H, H, H / 2, 1, 1])[-2:] #print('Tile RA,Decs:', r,d) rimgs = [] # scaled down..... # call get_filename to possibly generate scaled version for band in bands: brick = None fn = self.get_filename(brick, band, scale, tempfiles=tempfiles) print('scale', scale, 'band', band, 'fn', fn) try: bwcs = self.read_wcs(brick, band, scale, fn=fn) if bwcs is None: print('No such file:', brick, band, scale, 'fn', fn) continue except: print('Failed to read WCS:', brick, band, scale, 'fn', fn) savecache = False import traceback import sys traceback.print_exc(None, sys.stdout) continue # Check for pixel overlap area ok, xx, yy = bwcs.radec2pixelxy(r, d) xx = xx.astype(np.int) yy = yy.astype(np.int) imW, imH = int(bwcs.get_width()), int(bwcs.get_height()) M = 10 xlo = np.clip(xx.min() - M, 0, imW) xhi = np.clip(xx.max() + M, 0, imW) ylo = np.clip(yy.min() - M, 0, imH) yhi = np.clip(yy.max() + M, 0, imH) #print('My WCS xx,yy', xx, yy, 'imW,H', imW, imH, 'xlohi', xlo,xhi, 'ylohi', ylo,yhi) if xlo >= xhi or ylo >= yhi: print('No pixel overlap') return subwcs = bwcs.get_subimage(xlo, ylo, xhi - xlo, yhi - ylo) slc = slice(ylo, yhi), slice(xlo, xhi) try: img = self.read_image(brick, band, scale, slc, fn=fn) except: print('Failed to read image:', brickname, band, scale, 'fn', fn) savecache = False import traceback import sys traceback.print_exc(None, sys.stdout) continue #print('Read image slice', img.shape) try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(wcs, subwcs, [], 3) except OverlapError: #debug('Resampling exception') return rimg = np.zeros((H, W), np.float32) rimg[Yo, Xo] = img[Yi, Xi] rimgs.append(rimg) return rimgs
def coadds_ubercal(fulltims, coaddtims=None, plots=False, plots2=False, ps=None, verbose=False): """Bring individual CCDs onto a common flux scale based on overlapping pixels. fulltims - full-CCD tims, used to derive the corrections coaddtims - tims sliced to just the pixels contributing to the output coadd Some notes on the procedure: A x = b A: weights A: shape noverlap x nimg - entries have units of weights x_i: offset to apply to image i x: length nimg - entries will have values of image pixels b: (weighted) measured difference between image i and image j b: length -- "noverlap" number of overlapping pairs of images -- filled-in elements in your array - units of weighted image pixels """ from astrometry.util.resample import resample_with_wcs, OverlapError band = fulltims[0].band nimg = len(fulltims) indx = np.arange(nimg) ## initialize A bigger than we will need, cut later A = np.zeros((nimg * nimg, nimg), np.float32) b = np.zeros((nimg * nimg), np.float32) ioverlap = 0 for ii in indx: for jj in indx[ii + 1:]: try: Yi, Xi, Yj, Xj, _ = resample_with_wcs(fulltims[ii].subwcs, fulltims[jj].subwcs) except OverlapError: continue imgI = fulltims[ii].getImage()[Yi, Xi] imgJ = fulltims[jj].getImage()[Yj, Xj] invI = fulltims[ii].getInvvar()[Yi, Xi] invJ = fulltims[jj].getInvvar()[Yj, Xj] good = (invI > 0) * (invJ > 0) diff = (imgI - imgJ)[good] iv = 1. / (1. / invI[good] + 1. / invJ[good]) delta = np.sum(diff * iv) weight = np.sum(iv) A[ioverlap, ii] = -weight A[ioverlap, jj] = weight b[ioverlap] = delta ioverlap += 1 noverlap = ioverlap A = A[:noverlap, :] b = b[:noverlap] #if verbose: # print('A:') # print(A) # print('b:') # print(b) R = np.linalg.lstsq(A, b, rcond=None) x = R[0] print('Delta offsets to each image:') print(x) # Plot to assess the sign of the correction. if plots2: import matplotlib.pyplot as plt plt.clf() for j, (correction, fulltim) in enumerate(zip(x, fulltims)): plt.subplot(nimg, 1, j + 1) plt.hist(fulltim.data.ravel(), bins=50, histtype='step', range=(-5, 5)) plt.axvline(-correction) plt.title('Band %s: fulltim pix and -correction' % band) ps.savefig() if coaddtims is not None: plt.clf() for j, (correction, ii) in enumerate(zip(x, np.arange(len(coaddtims)))): plt.subplot(nimg, 1, j + 1) plt.hist((coaddtims[ii].data + correction).ravel(), bins=50, histtype='step', range=(-5, 5)) plt.title('Band %s: tim pix + correction' % band) ps.savefig() return x
def run_one(X): k, sb, bricks, version = X print(k, sb.brickname) outfn = 'skybricks/sky-%s.fits.fz' % sb.brickname if os.path.exists(outfn): return True I = np.flatnonzero((bricks.ra2 > sb.ra1) * (bricks.ra1 < sb.ra2) * (bricks.dec2 > sb.dec1) * (bricks.dec1 < sb.dec2)) if len(I) == 0: print('No bricks overlap') return False # 3600 + 1% margin on each side w, h = 3672, 3672 if sb.dec >= 78.: # In order to have fully-overlapping tiles at high Decs, we # need larger maps. DR9 reaches a max skytile of Dec=+85, # where the required size is 3724. Add a little margin. w, h = 3744, 3744 binning = 4 # pixscale cd = 1. / 3600. fullw, fullh = w * binning, h * binning fullcd = cd / binning # There are really three states: no coverage, blob, no blob. # Since blobs outside each brick's unique area do not appear in # the files, we start skyblobs as zero, but also track the # coverage so we can set !coverage to blob at the end. skyblobs = np.zeros((fullh, fullw), bool) covered = np.zeros((fullh, fullw), bool) skywcs = Tan(sb.ra, sb.dec, (fullw + 1) / 2., (fullh + 1) / 2., -fullcd, 0., 0., fullcd, float(fullw), float(fullh)) for i in I: brick = bricks[i] #print('Blob', brickname) #fn = 'cosmo/data/legacysurvey/dr9/%s/metrics/%s/blobs-%s.fits.gz' % (brick.hemi, brick.brickname[:3], brick.brickname) #fn = 'dr9-south-blobs/blobs-%s.fits.gz' % (brick.brickname) fn = '/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/%s/metrics/%s/blobs-%s.fits.gz' % ( brick.hemi, brick.brickname[:3], brick.brickname) blobs, hdr = fitsio.read(fn, header=True) wcs = Tan(hdr) blobs = (blobs > -1) try: Yo, Xo, Yi, Xi, _ = resample_with_wcs(skywcs, wcs) except NoOverlapError: continue skyblobs[Yo, Xo] |= blobs[Yi, Xi] # coverage: nexp > 0 in any band for band in ['g', 'r', 'z']: fn = ( '/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/%s/coadd/%s/%s/legacysurvey-%s-nexp-%s.fits.fz' % (brick.hemi, brick.brickname[:3], brick.brickname, brick.brickname, band)) if not os.path.exists(fn): continue nexp = fitsio.read(fn) covered[Yo, Xo] |= (nexp[Yi, Xi] > 0) if not np.any(covered): print('No coverage') return False # No coverage = equivalent to there being a blob there (ie, # conservative for placing sky fibers) skyblobs[covered == False] = True # bin down, counting how many times 'skyblobs' is set subcount = np.zeros((h, w), np.uint8) for i in range(binning): for j in range(binning): subcount += skyblobs[i::binning, j::binning] subwcs = Tan(sb.ra, sb.dec, (w + 1) / 2., (h + 1) / 2., -cd, 0., 0., cd, float(w), float(h)) hdr = fitsio.FITSHDR() hdr.add_record( dict(name='SB_VER', value=version, comment='desi-sky-locations git version')) subwcs.add_to_header(hdr) fitsio.write(outfn, subcount, header=hdr, clobber=True, compress='GZIP', tile_dims=(256, 256)) print('Wrote', outfn) return True
def stage1(T=None, coimgs=None, cons=None, detmaps=None, detivs=None, targetrd=None, pixscale=None, targetwcs=None, W=None,H=None, bands=None, tims=None, ps=None, brick=None, cat=None): orig_wcsxy0 = [tim.wcs.getX0Y0() for tim in tims] hot = np.zeros((H,W), np.float32) for band in bands: detmap = detmaps[band] / np.maximum(1e-16, detivs[band]) detsn = detmap * np.sqrt(detivs[band]) hot = np.maximum(hot, detsn) detmaps[band] = detmap ### FIXME -- ugri for sedname,sed in [('Flat', (1.,1.,1.)), ('Red', (2.5, 1.0, 0.4))]: sedmap = np.zeros((H,W), np.float32) sediv = np.zeros((H,W), np.float32) for iband,band in enumerate(bands): # We convert the detmap to canonical band via # detmap * w # And the corresponding change to sig1 is # sig1 * w # So the invvar-weighted sum is # (detmap * w) / (sig1**2 * w**2) # = detmap / (sig1**2 * w) sedmap += detmaps[band] * detivs[band] / sed[iband] sediv += detivs [band] / sed[iband]**2 sedmap /= np.maximum(1e-16, sediv) sedsn = sedmap * np.sqrt(sediv) hot = np.maximum(hot, sedsn) plt.clf() dimshow(np.round(sedsn), vmin=0, vmax=10, cmap='hot') plt.title('SED-matched detection filter: %s' % sedname) ps.savefig() peaks = (hot > 4) blobs,nblobs = label(peaks) print 'N detected blobs:', nblobs blobslices = find_objects(blobs) # Un-set catalog blobs for x,y in zip(T.itx, T.ity): # blob number bb = blobs[y,x] if bb == 0: continue # un-set 'peaks' within this blob slc = blobslices[bb-1] peaks[slc][blobs[slc] == bb] = 0 # Now, after having removed catalog sources, crank up the detection threshold peaks &= (hot > 5) # zero out the edges(?) peaks[0 ,:] = peaks[:, 0] = 0 peaks[-1,:] = peaks[:,-1] = 0 peaks[1:-1, 1:-1] &= (hot[1:-1,1:-1] >= hot[0:-2,1:-1]) peaks[1:-1, 1:-1] &= (hot[1:-1,1:-1] >= hot[2: ,1:-1]) peaks[1:-1, 1:-1] &= (hot[1:-1,1:-1] >= hot[1:-1,0:-2]) peaks[1:-1, 1:-1] &= (hot[1:-1,1:-1] >= hot[1:-1,2: ]) # These are our peaks pki = np.flatnonzero(peaks) peaky,peakx = np.unravel_index(pki, peaks.shape) print len(peaky), 'peaks' crossa = dict(ms=10, mew=1.5) plt.clf() dimshow(get_rgb(coimgs, bands)) ax = plt.axis() plt.plot(T.tx, T.ty, 'r+', **crossa) plt.plot(peakx, peaky, '+', color=green, **crossa) plt.axis(ax) plt.title('SDSS + SED-matched detections') ps.savefig() ### HACK -- high threshold again # Segment, and record which sources fall into each blob blobs,nblobs = label((hot > 20)) print 'N detected blobs:', nblobs blobslices = find_objects(blobs) T.blob = blobs[T.ity, T.itx] blobsrcs = [] blobflux = [] fluximg = coimgs[1] for blob in range(1, nblobs+1): blobsrcs.append(np.flatnonzero(T.blob == blob)) bslc = blobslices[blob-1] blobflux.append(np.sum(fluximg[bslc][blobs[bslc] == blob])) # Fit the SDSS sources for tim in tims: tim.psfex.fitSavedData(*tim.psfex.splinedata) tim.psf = tim.psfex # How far down to render model profiles minsigma = 0.1 for tim in tims: tim.modelMinval = minsigma * tim.sig1 srcvariances = [[] for src in cat] # Fit in order of flux for blobnumber,iblob in enumerate(np.argsort(-np.array(blobflux))): bslc = blobslices[iblob] Isrcs = blobsrcs [iblob] if len(Isrcs) == 0: continue print print 'Blob', blobnumber, 'of', len(blobflux), ':', len(Isrcs), 'sources' print 'Source indices:', Isrcs print # blob bbox in target coords sy,sx = bslc by0,by1 = sy.start, sy.stop bx0,bx1 = sx.start, sx.stop blobh,blobw = by1 - by0, bx1 - bx0 rr,dd = targetwcs.pixelxy2radec([bx0,bx0,bx1,bx1],[by0,by1,by1,by0]) alphas = [0.1, 0.3, 1.0] subtims = [] for itim,tim in enumerate(tims): h,w = tim.shape ok,x,y = tim.subwcs.radec2pixelxy(rr,dd) sx0,sx1 = x.min(), x.max() sy0,sy1 = y.min(), y.max() if sx1 < 0 or sy1 < 0 or sx1 > w or sy1 > h: continue sx0 = np.clip(int(np.floor(sx0)), 0, w-1) sx1 = np.clip(int(np.ceil (sx1)), 0, w-1) + 1 sy0 = np.clip(int(np.floor(sy0)), 0, h-1) sy1 = np.clip(int(np.ceil (sy1)), 0, h-1) + 1 subslc = slice(sy0,sy1),slice(sx0,sx1) subimg = tim.getImage ()[subslc] subie = tim.getInvError()[subslc] subwcs = tim.getWcs().copy() ox0,oy0 = orig_wcsxy0[itim] subwcs.setX0Y0(ox0 + sx0, oy0 + sy0) # Mask out inverr for pixels that are not within the blob. subtarget = targetwcs.get_subimage(bx0, by0, blobw, blobh) subsubwcs = tim.subwcs.get_subimage(int(sx0), int(sy0), int(sx1-sx0), int(sy1-sy0)) try: Yo,Xo,Yi,Xi,rims = resample_with_wcs(subsubwcs, subtarget, [], 2) except OverlapError: print 'No overlap' continue if len(Yo) == 0: continue subie2 = np.zeros_like(subie) I = np.flatnonzero(blobs[bslc][Yi, Xi] == (iblob+1)) subie2[Yo[I],Xo[I]] = subie[Yo[I],Xo[I]] subie = subie2 # If the subimage (blob) is small enough, instantiate a # constant PSF model in the center. if sy1-sy0 < 100 and sx1-sx0 < 100: subpsf = tim.psf.mogAt(ox0 + (sx0+sx1)/2., oy0 + (sy0+sy1)/2.) else: # Otherwise, instantiate a (shifted) spatially-varying # PsfEx model. subpsf = ShiftedPsf(tim.psf, ox0+sx0, oy0+sy0) subtim = Image(data=subimg, inverr=subie, wcs=subwcs, psf=subpsf, photocal=tim.getPhotoCal(), sky=tim.getSky(), name=tim.name) subtim.band = tim.band subtim.sig1 = tim.sig1 subtim.modelMinval = tim.modelMinval subtims.append(subtim) subcat = Catalog(*[cat[i] for i in Isrcs]) subtr = Tractor(subtims, subcat) subtr.freezeParam('images') # Optimize individual sources in order of flux fluxes = [] for src in subcat: # HACK -- here we just *sum* the nanomaggies in each band. Bogus! br = src.getBrightness() flux = sum([br.getFlux(band) for band in bands]) fluxes.append(flux) Ibright = np.argsort(-np.array(fluxes)) if len(Ibright) >= 5: # -Remember the original subtim images # -Compute initial models for each source (in each tim) # -Subtract initial models from images # -During fitting, for each source: # -add back in the source's initial model (to each tim) # -fit, with Catalog([src]) # -subtract final model (from each tim) # -Replace original subtim images # # --Might want to omit newly-added detection-filter sources, since their # fluxes are bogus. # Remember original tim images orig_timages = [tim.getImage().copy() for tim in subtims] initial_models = [] # Create initial models for each tim x each source for tim in subtims: mods = [] for src in subcat: mod = src.getModelPatch(tim) mods.append(mod) if mod is not None: if not np.all(np.isfinite(mod.patch)): print 'Non-finite mod patch' print 'source:', src print 'tim:', tim print 'PSF:', tim.getPsf() assert(np.all(np.isfinite(mod.patch))) mod.addTo(tim.getImage(), scale=-1) initial_models.append(mods) # For sources in decreasing order of brightness for numi,i in enumerate(Ibright): tsrc = Time() print 'Fitting source', i, '(%i of %i in blob)' % (numi, len(Ibright)) src = subcat[i] print src srctractor = Tractor(subtims, [src]) srctractor.freezeParams('images') # Add this source's initial model back in. for tim,mods in zip(subtims, initial_models): mod = mods[i] if mod is not None: mod.addTo(tim.getImage()) print 'Optimizing:', srctractor srctractor.printThawedParams() for step in range(50): dlnp,X,alpha = srctractor.optimize(priors=False, shared_params=False, alphas=alphas) print 'dlnp:', dlnp, 'src', src if dlnp < 0.1: break for tim in subtims: mod = src.getModelPatch(tim) if mod is not None: mod.addTo(tim.getImage(), scale=-1) for tim,img in zip(subtims, orig_timages): tim.data = img del orig_timages del initial_models else: # Fit sources one at a time, but don't subtract other models subcat.freezeAllParams() for numi,i in enumerate(Ibright): tsrc = Time() print 'Fitting source', i, '(%i of %i in blob)' % (numi, len(Ibright)) print subcat[i] subcat.freezeAllBut(i) print 'Optimizing:', subtr subtr.printThawedParams() for step in range(10): dlnp,X,alpha = subtr.optimize(priors=False, shared_params=False, alphas=alphas) print 'dlnp:', dlnp if dlnp < 0.1: break print 'Fitting source took', Time()-tsrc print subcat[i] if len(Isrcs) > 1 and len(Isrcs) <= 10: tfit = Time() # Optimize all at once? subcat.thawAllParams() print 'Optimizing:', subtr subtr.printThawedParams() for step in range(20): dlnp,X,alpha = subtr.optimize(priors=False, shared_params=False, alphas=alphas) print 'dlnp:', dlnp if dlnp < 0.1: break # Variances subcat.thawAllRecursive() subcat.freezeAllParams() for isub,srci in enumerate(Isrcs): print 'Variances for source', srci subcat.thawParam(isub) src = subcat[isub] print 'Source', src print 'Params:', src.getParamNames() if isinstance(src, (DevGalaxy, ExpGalaxy)): src.shape = EllipseE.fromEllipseESoft(src.shape) elif isinstance(src, FixedCompositeGalaxy): src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp) src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev) print 'Converted ellipse:', src allderivs = subtr.getDerivs() for iparam,derivs in enumerate(allderivs): dchisq = 0 for deriv,tim in derivs: h,w = tim.shape deriv.clipTo(w,h) ie = tim.getInvError() slc = deriv.getSlice(ie) chi = deriv.patch * ie[slc] dchisq += (chi**2).sum() if dchisq == 0.: v = np.nan else: v = 1./dchisq srcvariances[srci].append(v) assert(len(srcvariances[srci]) == subcat[isub].numberOfParams()) subcat.freezeParam(isub) cat.thawAllRecursive() for i,src in enumerate(cat): print 'Source', i, src print 'variances:', srcvariances[i] print len(srcvariances[i]), 'vs', src.numberOfParams() if len(srcvariances[i]) != src.numberOfParams(): # This can happen for sources outside the brick bounds: they never get optimized? print 'Warning: zeroing variances for source', src srcvariances[i] = [0]*src.numberOfParams() if isinstance(src, (DevGalaxy, ExpGalaxy)): src.shape = EllipseE.fromEllipseESoft(src.shape) elif isinstance(src, FixedCompositeGalaxy): src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp) src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev) assert(len(srcvariances[i]) == src.numberOfParams()) variances = np.hstack(srcvariances) assert(len(variances) == cat.numberOfParams()) return dict(cat=cat, variances=variances)
def stage0(**kwargs): ps = PlotSequence('cfht') decals = CfhtDecals() B = decals.get_bricks() print 'Bricks:' B.about() ra,dec = 190.0, 11.0 #bands = 'ugri' bands = 'gri' B.cut(np.argsort(degrees_between(ra, dec, B.ra, B.dec))) print 'Nearest bricks:', B.ra[:5], B.dec[:5], B.brickid[:5] brick = B[0] pixscale = 0.186 #W,H = 1024,1024 #W,H = 2048,2048 #W,H = 3600,3600 W,H = 4800,4800 targetwcs = wcs_for_brick(brick, pixscale=pixscale, W=W, H=H) ccdfn = 'cfht-ccds.fits' if os.path.exists(ccdfn): T = fits_table(ccdfn) else: T = get_ccd_list() T.writeto(ccdfn) print len(T), 'CCDs' T.cut(ccds_touching_wcs(targetwcs, T)) print len(T), 'CCDs touching brick' T.cut(np.array([b in bands for b in T.filter])) print len(T), 'in bands', bands ims = [] for t in T: im = CfhtImage(t) # magzp = hdr['PHOT_C'] + 2.5 * np.log10(hdr['EXPTIME']) # fwhm = t.seeing / (pixscale * 3600) # print '-> FWHM', fwhm, 'pix' im.seeing = t.seeing im.pixscale = t.pixscale print 'seeing', t.seeing print 'pixscale', im.pixscale*3600, 'arcsec/pix' im.run_calibs(t.ra, t.dec, im.pixscale, W=t.width, H=t.height) ims.append(im) # Read images, clip to ROI targetrd = np.array([targetwcs.pixelxy2radec(x,y) for x,y in [(1,1),(W,1),(W,H),(1,H),(1,1)]]) keepims = [] tims = [] for im in ims: print print 'Reading expnum', im.expnum, 'name', im.extname, 'band', im.band, 'exptime', im.exptime band = im.band wcs = im.read_wcs() imh,imw = wcs.imageh,wcs.imagew imgpoly = [(1,1),(1,imh),(imw,imh),(imw,1)] ok,tx,ty = wcs.radec2pixelxy(targetrd[:-1,0], targetrd[:-1,1]) tpoly = zip(tx,ty) clip = clip_polygon(imgpoly, tpoly) clip = np.array(clip) #print 'Clip', clip if len(clip) == 0: continue x0,y0 = np.floor(clip.min(axis=0)).astype(int) x1,y1 = np.ceil (clip.max(axis=0)).astype(int) slc = slice(y0,y1+1), slice(x0,x1+1) ## FIXME -- it seems I got lucky and the cross product is ## negative == clockwise, as required by clip_polygon. One ## could check this and reverse the polygon vertex order. # dx0,dy0 = tx[1]-tx[0], ty[1]-ty[0] # dx1,dy1 = tx[2]-tx[1], ty[2]-ty[1] # cross = dx0*dy1 - dx1*dy0 # print 'Cross:', cross print 'Image slice: x [%i,%i], y [%i,%i]' % (x0,x1, y0,y1) print 'Reading image from', im.imgfn, 'HDU', im.hdu img,imghdr = im.read_image(header=True, slice=slc) goodpix = (img != 0) print 'Number of pixels == 0:', np.sum(img == 0) print 'Number of pixels != 0:', np.sum(goodpix) if np.sum(goodpix) == 0: continue # print 'Image shape', img.shape print 'Image range', img.min(), img.max() print 'Goodpix image range:', (img[goodpix]).min(), (img[goodpix]).max() if img[goodpix].min() == img[goodpix].max(): print 'No dynamic range in image' continue # print 'Reading invvar from', im.wtfn, 'HDU', im.hdu # invvar = im.read_invvar(slice=slc) # # print 'Invvar shape', invvar.shape # # print 'Invvar range:', invvar.min(), invvar.max() # invvar[goodpix == 0] = 0. # if np.all(invvar == 0.): # print 'Skipping zero-invvar image' # continue # assert(np.all(np.isfinite(img))) # assert(np.all(np.isfinite(invvar))) # assert(not(np.all(invvar == 0.))) # # Estimate per-pixel noise via Blanton's 5-pixel MAD # slice1 = (slice(0,-5,10),slice(0,-5,10)) # slice2 = (slice(5,None,10),slice(5,None,10)) # # print 'sliced shapes:', img[slice1].shape, img[slice2].shape # # print 'good shape:', (goodpix[slice1] * goodpix[slice2]).shape # # print 'good values:', np.unique(goodpix[slice1] * goodpix[slice2]) # # print 'sliced[good] shapes:', (img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].shape # mad = np.median(np.abs(img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].ravel()) # sig1 = 1.4826 * mad / np.sqrt(2.) # print 'MAD sig1:', sig1 # # invvar was 1 or 0 # invvar *= (1./(sig1**2)) # medsky = np.median(img[goodpix]) # Read full image for sig1 and sky estimate fullimg = im.read_image() fullgood = (fullimg != 0) # Estimate per-pixel noise via Blanton's 5-pixel MAD slice1 = (slice(0,-5,10),slice(0,-5,10)) slice2 = (slice(5,None,10),slice(5,None,10)) mad = np.median(np.abs(fullimg[slice1] - fullimg[slice2])[fullgood[slice1] * fullgood[slice2]].ravel()) sig1 = 1.4826 * mad / np.sqrt(2.) print 'MAD sig1:', sig1 medsky = np.median(fullimg[fullgood]) invvar = np.zeros_like(img) invvar[goodpix] = 1./sig1**2 # Median-smooth sky subtraction plt.clf() dimshow(np.round((img-medsky) / sig1), vmin=-3, vmax=5) plt.title('Scalar median: %s' % im.name) ps.savefig() # medsky = np.zeros_like(img) # # astrometry.util.util # median_smooth(img, np.logical_not(goodpix), 256, medsky) fullmed = np.zeros_like(fullimg) median_smooth(fullimg - medsky, np.logical_not(fullgood), 256, fullmed) fullmed += medsky medimg = fullmed[slc] plt.clf() dimshow(np.round((img - medimg) / sig1), vmin=-3, vmax=5) plt.title('Median filtered: %s' % im.name) ps.savefig() #print 'Subtracting median:', medsky #img -= medsky img -= medimg primhdr = im.read_image_primary_header() magzp = decals.get_zeropoint_for(im) print 'magzp', magzp zpscale = NanoMaggies.zeropointToScale(magzp) print 'zpscale', zpscale # Scale images to Nanomaggies img /= zpscale sig1 /= zpscale invvar *= zpscale**2 orig_zpscale = zpscale zpscale = 1. assert(np.sum(invvar > 0) > 0) print 'After scaling:' print 'sig1', sig1 print 'invvar range', invvar.min(), invvar.max() print 'image range', img.min(), img.max() assert(np.all(np.isfinite(img))) assert(np.all(np.isfinite(invvar))) assert(np.isfinite(sig1)) plt.clf() lo,hi = -5*sig1, 10*sig1 n,b,p = plt.hist(img[goodpix].ravel(), 100, range=(lo,hi), histtype='step', color='k') xx = np.linspace(lo, hi, 200) plt.plot(xx, max(n)*np.exp(-xx**2 / (2.*sig1**2)), 'r-') plt.xlim(lo,hi) plt.title('Pixel histogram: %s' % im.name) ps.savefig() twcs = ConstantFitsWcs(wcs) if x0 or y0: twcs.setX0Y0(x0,y0) info = im.get_image_info() fullh,fullw = info['dims'] # read fit PsfEx model psfex = PsfEx.fromFits(im.psffitfn) print 'Read', psfex # HACK -- highly approximate PSF here! #psf_fwhm = imghdr['FWHM'] #psf_fwhm = im.seeing psf_fwhm = im.seeing / (im.pixscale * 3600) print 'PSF FWHM', psf_fwhm, 'pixels' psf_sigma = psf_fwhm / 2.35 psf = NCircularGaussianPSF([psf_sigma],[1.]) print 'img type', img.dtype tim = Image(img, invvar=invvar, wcs=twcs, psf=psf, photocal=LinearPhotoCal(zpscale, band=band), sky=ConstantSky(0.), name=im.name + ' ' + band) tim.zr = [-3. * sig1, 10. * sig1] tim.sig1 = sig1 tim.band = band tim.psf_fwhm = psf_fwhm tim.psf_sigma = psf_sigma tim.sip_wcs = wcs tim.x0,tim.y0 = int(x0),int(y0) tim.psfex = psfex tim.imobj = im mn,mx = tim.zr tim.ima = dict(interpolation='nearest', origin='lower', cmap='gray', vmin=mn, vmax=mx) tims.append(tim) keepims.append(im) ims = keepims print 'Computing resampling...' # save resampling params for tim in tims: wcs = tim.sip_wcs subh,subw = tim.shape subwcs = wcs.get_subimage(tim.x0, tim.y0, subw, subh) tim.subwcs = subwcs try: Yo,Xo,Yi,Xi,rims = resample_with_wcs(targetwcs, subwcs, [], 2) except OverlapError: print 'No overlap' continue if len(Yo) == 0: continue tim.resamp = (Yo,Xo,Yi,Xi) print 'Creating coadds...' # Produce per-band coadds, for plots coimgs = [] cons = [] for ib,band in enumerate(bands): coimg = np.zeros((H,W), np.float32) con = np.zeros((H,W), np.uint8) for tim in tims: if tim.band != band: continue (Yo,Xo,Yi,Xi) = tim.resamp if len(Yo) == 0: continue nn = (tim.getInvvar()[Yi,Xi] > 0) coimg[Yo,Xo] += tim.getImage ()[Yi,Xi] * nn con [Yo,Xo] += nn # print # print 'tim', tim.name # print 'number of resampled pix:', len(Yo) # reim = np.zeros_like(coimg) # ren = np.zeros_like(coimg) # reim[Yo,Xo] = tim.getImage()[Yi,Xi] * nn # ren[Yo,Xo] = nn # print 'number of resampled pix with positive invvar:', ren.sum() # plt.clf() # plt.subplot(2,2,1) # mn,mx = [np.percentile(reim[ren>0], p) for p in [25,95]] # print 'Percentiles:', mn,mx # dimshow(reim, vmin=mn, vmax=mx) # plt.colorbar() # plt.subplot(2,2,2) # dimshow(con) # plt.colorbar() # plt.subplot(2,2,3) # dimshow(reim, vmin=tim.zr[0], vmax=tim.zr[1]) # plt.colorbar() # plt.subplot(2,2,4) # plt.hist(reim.ravel(), 100, histtype='step', color='b') # plt.hist(tim.getImage().ravel(), 100, histtype='step', color='r') # plt.suptitle('%s: %s' % (band, tim.name)) # ps.savefig() coimg /= np.maximum(con,1) coimgs.append(coimg) cons .append(con) plt.clf() dimshow(get_rgb(coimgs, bands)) ps.savefig() plt.clf() for i,b in enumerate(bands): plt.subplot(2,2,i+1) dimshow(cons[i], ticks=False) plt.title('%s band' % b) plt.colorbar() plt.suptitle('Number of exposures') ps.savefig() print 'Grabbing SDSS sources...' bandlist = [b for b in bands] cat,T = get_sdss_sources(bandlist, targetwcs) # record coordinates in target brick image ok,T.tx,T.ty = targetwcs.radec2pixelxy(T.ra, T.dec) T.tx -= 1 T.ty -= 1 T.itx = np.clip(np.round(T.tx).astype(int), 0, W-1) T.ity = np.clip(np.round(T.ty).astype(int), 0, H-1) plt.clf() dimshow(get_rgb(coimgs, bands)) ax = plt.axis() plt.plot(T.tx, T.ty, 'o', mec=green, mfc='none', ms=10, mew=1.5) plt.axis(ax) plt.title('SDSS sources') ps.savefig() print 'Detmaps...' # Render the detection maps detmaps = dict([(b, np.zeros((H,W), np.float32)) for b in bands]) detivs = dict([(b, np.zeros((H,W), np.float32)) for b in bands]) for tim in tims: iv = tim.getInvvar() psfnorm = 1./(2. * np.sqrt(np.pi) * tim.psf_sigma) detim = tim.getImage().copy() detim[iv == 0] = 0. detim = gaussian_filter(detim, tim.psf_sigma) / psfnorm**2 detsig1 = tim.sig1 / psfnorm subh,subw = tim.shape detiv = np.zeros((subh,subw), np.float32) + (1. / detsig1**2) detiv[iv == 0] = 0. (Yo,Xo,Yi,Xi) = tim.resamp detmaps[tim.band][Yo,Xo] += detiv[Yi,Xi] * detim[Yi,Xi] detivs [tim.band][Yo,Xo] += detiv[Yi,Xi] rtn = dict() for k in ['T', 'coimgs', 'cons', 'detmaps', 'detivs', 'targetrd', 'pixscale', 'targetwcs', 'W','H', 'bands', 'tims', 'ps', 'brick', 'cat']: rtn[k] = locals()[k] return rtn
def map_decals_wl(req, ver, zoom, x, y): tag = 'decals-wl' ignoreCached = False filename = None forcecache = False from decals import settings savecache = settings.SAVE_CACHE zoom = int(zoom) zoomscale = 2.**zoom x = int(x) y = int(y) if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale: raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom, x, y)) ver = int(ver) if not ver in tileversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) basedir = settings.DATA_DIR tilefn = os.path.join(basedir, 'tiles', tag, '%i/%i/%i/%i.jpg' % (ver, zoom, x, y)) if os.path.exists(tilefn) and not ignoreCached: print('Cached:', tilefn) return send_file(tilefn, 'image/jpeg', expires=oneyear, modsince=req.META.get('HTTP_IF_MODIFIED_SINCE'), filename=filename) else: print('Tile image does not exist:', tilefn) from astrometry.util.resample import resample_with_wcs, OverlapError from astrometry.util.util import Tan from astrometry.libkd.spherematch import match_radec from astrometry.util.fits import fits_table from astrometry.util.starutil_numpy import degrees_between import numpy as np import fitsio try: wcs, W, H, zoomscale, zoom, x, y = get_tile_wcs(zoom, x, y) except RuntimeError as e: return HttpResponse(e.strerror) mydir = os.path.join(basedir, 'coadd', 'weak-lensing') rlo, d = wcs.pixelxy2radec(W, H / 2)[-2:] rhi, d = wcs.pixelxy2radec(1, H / 2)[-2:] r, d1 = wcs.pixelxy2radec(W / 2, 1)[-2:] r, d2 = wcs.pixelxy2radec(W / 2, H)[-2:] #dlo = min(d1, d2) #dhi = max(d1, d2) r, d = wcs.pixelxy2radec(W / 2, H / 2)[-2:] rad = degrees_between(r, d, rlo, d1) fn = os.path.join(mydir, 'index.fits') if not os.path.exists(fn): # ii, rr, dd = [], [], [] for i in range(1, 52852 + 1): imgfn = os.path.join(mydir, 'map%i.fits' % i) hdr = fitsio.read_header(imgfn) r = hdr['CRVAL1'] d = hdr['CRVAL2'] ii.append(i) rr.append(r) dd.append(d) T = fits_table() T.ra = np.array(rr) T.dec = np.array(dd) T.i = np.array(ii) T.writeto(fn) T = fits_table(fn) I, J, d = match_radec(T.ra, T.dec, r, d, rad + 0.2) T.cut(I) print(len(T), 'weak-lensing maps in range') if len(I) == 0: from django.http import HttpResponseRedirect if forcecache: # create symlink to blank.jpg! trymakedirs(tilefn) src = os.path.join(settings.STATIC_ROOT, 'blank.jpg') if os.path.exists(tilefn): os.unlink(tilefn) os.symlink(src, tilefn) print('Symlinked', tilefn, '->', src) return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg') r, d = wcs.pixelxy2radec([1, 1, 1, W / 2, W, W, W, W / 2], [1, H / 2, H, H, H, H / 2, 1, 1])[-2:] foundany = False rimg = np.zeros((H, W), np.float32) rn = np.zeros((H, W), np.uint8) for tilei in T.i: fn = os.path.join(mydir, 'map%i.fits' % tilei) try: bwcs = read_tan_wcs(fn, 0) except: print('Failed to read WCS:', fn) savecache = False import traceback import sys traceback.print_exc(None, sys.stdout) continue foundany = True print('Reading', fn) ok, xx, yy = bwcs.radec2pixelxy(r, d) xx = xx.astype(np.int) yy = yy.astype(np.int) imW, imH = int(bwcs.get_width()), int(bwcs.get_height()) M = 10 xlo = np.clip(xx.min() - M, 0, imW) xhi = np.clip(xx.max() + M, 0, imW) ylo = np.clip(yy.min() - M, 0, imH) yhi = np.clip(yy.max() + M, 0, imH) if xlo >= xhi or ylo >= yhi: continue subwcs = bwcs.get_subimage(xlo, ylo, xhi - xlo, yhi - ylo) slc = slice(ylo, yhi), slice(xlo, xhi) try: f = fitsio.FITS(fn)[0] img = f[slc] del f except: print('Failed to read image and WCS:', fn) savecache = False import traceback import sys traceback.print_exc(None, sys.stdout) continue try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(wcs, subwcs, [], 3) except OverlapError: print('Resampling exception') continue rimg[Yo, Xo] += img[Yi, Xi] rn[Yo, Xo] += 1 rimg /= np.maximum(rn, 1) if forcecache: savecache = True if savecache: trymakedirs(tilefn) else: import tempfile f, tilefn = tempfile.mkstemp(suffix='.jpg') os.close(f) import pylab as plt # S/N #lo,hi = 1.5, 5.0 lo, hi = 0, 5.0 rgb = plt.cm.hot((rimg - lo) / (hi - lo)) plt.imsave(tilefn, rgb) print('Wrote', tilefn) return send_file(tilefn, 'image/jpeg', unlink=(not savecache), filename=filename)
def get_sdss_cutout(targetwcs, sdss, get_rawvals=False, bands='irg', get_rawvals_only=False, bandscales=dict(z=1.0, i=1.0, r=1.3, g=2.5)): rgbims = [] ra, dec = targetwcs.radec_center() # in deg radius = targetwcs.radius() #print 'Target WCS radius is', radius, 'deg' H, W = targetwcs.get_height(), targetwcs.get_width() targetpixscale = targetwcs.pixel_scale() wlistfn = sdss.filenames.get('window_flist', 'window_flist.fits') rad2 = radius * 60. + np.hypot(14., 10.) / 2. #print 'Rad2 radius', rad2, 'arcmin' RCF = radec_to_sdss_rcf(ra, dec, tablefn=wlistfn, radius=rad2) # Drop rerun 157 keepRCF = [] for run, camcol, field, r, d in RCF: rr = sdss.get_rerun(run, field) #print 'Rerun:', rr if rr == '157': continue keepRCF.append((run, camcol, field)) RCF = keepRCF print len(RCF), 'run/camcol/fields in range' # size in SDSS pixels of the target image. sz = np.hypot(H, W) / 2. * targetpixscale / 0.396 print 'SDSS sz:', sz bandnums = [band_index(b) for b in bands] for bandnum, band in zip(bandnums, bands): targetim = np.zeros((H, W), np.float32) targetn = np.zeros((H, W), np.uint8) for ifield, (run, camcol, field) in enumerate(RCF): fn = sdss.retrieve('frame', run, camcol, field, band) frame = sdss.readFrame(run, camcol, field, bandnum) h, w = frame.getImageShape() x, y = frame.astrans.radec_to_pixel(ra, dec) x, y = int(x), int(y) # add some margin for resampling sz2 = int(sz) + 5 xlo = np.clip(x - sz2, 0, w) xhi = np.clip(x + sz2 + 1, 0, w) ylo = np.clip(y - sz2, 0, h) yhi = np.clip(y + sz2 + 1, 0, h) if xlo == xhi or ylo == yhi: continue stamp = frame.getImageSlice((slice(ylo, yhi), slice(xlo, xhi))) sh, sw = stamp.shape wcs = AsTransWrapper(frame.astrans, sw, sh, x0=xlo, y0=ylo) # FIXME -- allow nn resampling too try: Yo, Xo, Yi, Xi, [rim ] = resample_with_wcs(targetwcs, wcs, [stamp], 3) except ResampleError: continue targetim[Yo, Xo] += rim targetn[Yo, Xo] += 1 rgbims.append(targetim / targetn) if get_rawvals_only: return rgbims if get_rawvals: rawvals = [x.copy() for x in rgbims] r, g, b = rgbims r *= bandscales[bands[0]] g *= bandscales[bands[1]] b *= bandscales[bands[2]] # i #r *= 1.0 # r #g *= 1.5 #g *= 1.3 # g #b *= 2.5 m = -0.02 r = np.maximum(0, r - m) g = np.maximum(0, g - m) b = np.maximum(0, b - m) I = (r + g + b) / 3. alpha = 1.5 Q = 20 m2 = 0. fI = np.arcsinh(alpha * Q * (I - m2)) / np.sqrt(Q) I += (I == 0.) * 1e-6 R = fI * r / I G = fI * g / I B = fI * b / I maxrgb = reduce(np.maximum, [R, G, B]) J = (maxrgb > 1.) R[J] = R[J] / maxrgb[J] G[J] = G[J] / maxrgb[J] B[J] = B[J] / maxrgb[J] ss = 0.5 RGBblur = np.clip( np.dstack([ gaussian_filter(R, ss), gaussian_filter(G, ss), gaussian_filter(B, ss) ]), 0., 1.) if get_rawvals: return RGBblur, rawvals return RGBblur
def map_coadd_bands( req, ver, zoom, x, y, bands, tag, imagedir, wcs=None, imagetag='image2', rgbfunc=None, rgbkwargs={}, bricks=None, savecache=True, forcecache=False, return_if_not_found=False, model_gz=False, modeldir=None, scaledir=None, get_images=False, write_jpeg=False, ignoreCached=False, add_gz=False, filename=None, symlink_blank=False, hack_jpeg=False, drname=None, decals=None, basepat=None, scalepat=None, nativescale=14, maxscale=8, ): from decals import settings from map.views import tileversions zoom = int(zoom) zoomscale = 2.**zoom x = int(x) y = int(y) if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale: raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom, x, y)) ver = int(ver) if not ver in tileversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) basedir = settings.DATA_DIR tilefn = os.path.join(basedir, 'tiles', tag, '%i/%i/%i/%i.jpg' % (ver, zoom, x, y)) if os.path.exists(tilefn) and not ignoreCached: return send_file(tilefn, 'image/jpeg', expires=oneyear, modsince=req.META.get('HTTP_IF_MODIFIED_SINCE'), filename=filename) else: debug('Tile image does not exist:', tilefn) from astrometry.util.resample import resample_with_wcs, OverlapError from astrometry.util.util import Tan import numpy as np import fitsio if wcs is None: try: wcs, W, H, zoomscale, zoom, x, y = get_tile_wcs(zoom, x, y) except RuntimeError as e: return HttpResponse(e.strerror) else: W = wcs.get_width() H = wcs.get_height() if basepat is None: basepat = os.path.join( basedir, 'coadd', imagedir, '%(brickname).3s', '%(brickname)s', 'decals-%(brickname)s-' + imagetag + '-%(band)s.fits') if modeldir is not None: modbasepat = os.path.join( basedir, 'coadd', modeldir, '%(brickname).3s', '%(brickname)s', 'decals-%(brickname)s-' + imagetag + '-%(band)s.fits') else: modbasepat = basepat if model_gz and imagetag == 'model': modbasepat += '.gz' if add_gz: basepat += '.gz' scaled = 0 if scaledir is None: scaledir = imagedir if zoom < nativescale: scaled = (nativescale - zoom) scaled = np.clip(scaled, 1, maxscale) #debug('Scaled-down:', scaled) dirnm = os.path.join(basedir, 'scaled', scaledir) if scalepat is None: scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(brickname).3s', imagetag + '-%(brickname)s-%(band)s.fits') if decals is None: from map.views import _get_survey D = _get_survey(name=drname) else: D = decals if bricks is None: B = D.get_bricks_readonly() else: B = bricks rlo, d = wcs.pixelxy2radec(W, H / 2)[-2:] rhi, d = wcs.pixelxy2radec(1, H / 2)[-2:] r, d1 = wcs.pixelxy2radec(W / 2, 1)[-2:] r, d2 = wcs.pixelxy2radec(W / 2, H)[-2:] dlo = min(d1, d2) dhi = max(d1, d2) I = D.bricks_touching_radec_box(B, rlo, rhi, dlo, dhi) debug(len(I), 'bricks touching zoom', zoom, 'x,y', x, y, 'RA', rlo, rhi, 'Dec', dlo, dhi) if len(I) == 0: if get_images: return None from django.http import HttpResponseRedirect if forcecache and symlink_blank: # create symlink to blank.jpg! trymakedirs(tilefn) src = os.path.join(settings.STATIC_ROOT, 'blank.jpg') if os.path.exists(tilefn): os.unlink(tilefn) os.symlink(src, tilefn) debug('Symlinked', tilefn, '->', src) return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg') r, d = wcs.pixelxy2radec([1, 1, 1, W / 2, W, W, W, W / 2], [1, H / 2, H, H, H, H / 2, 1, 1])[-2:] foundany = False rimgs = [] for band in bands: rimg = np.zeros((H, W), np.float32) rn = np.zeros((H, W), np.uint8) for i, brickname in zip(I, B.brickname[I]): has = getattr(B, 'has_%s' % band, None) if has is not None and not has[i]: # No coverage for band in this brick. debug('Brick', brickname, 'has no', band, 'band') continue fnargs = dict(band=band, brickname=brickname) if imagetag == 'resid': #basefn = basepat % fnargs basefn = D.find_file('image', brick=brickname, band=band) modbasefn = D.find_file('model', brick=brickname, band=band) #modbasefn = modbasepat % fnargs #modbasefn = modbasefn.replace('resid', 'model') #if model_gz: # modbasefn += '.gz' if scalepat is None: imscalepat = None modscalepat = None else: imscalepat = scalepat.replace('resid', 'image') modscalepat = scalepat.replace('resid', 'model') imbasefn = basefn.replace('resid', 'image') debug('resid. imscalepat, imbasefn', imscalepat, imbasefn) debug('resid. modscalepat, modbasefn', modscalepat, modbasefn) imfn = get_scaled(imscalepat, fnargs, scaled, imbasefn) modfn = get_scaled(modscalepat, fnargs, scaled, modbasefn) debug('resid. im', imfn, 'mod', modfn) fn = imfn else: basefn = D.find_file(imagetag, brick=brickname, band=band) fn = get_scaled(scalepat, fnargs, scaled, basefn) if fn is None: debug('not found: brick', brickname, 'band', band, 'with basefn', basefn) savecache = False continue if not os.path.exists(fn): debug('Does not exist:', fn) savecache = False continue try: bwcs = read_tan_wcs(fn, 0) except: print('Failed to read WCS:', fn) savecache = False import traceback import sys traceback.print_exc(None, sys.stdout) continue foundany = True debug('Reading', fn) ok, xx, yy = bwcs.radec2pixelxy(r, d) xx = xx.astype(np.int) yy = yy.astype(np.int) imW, imH = int(bwcs.get_width()), int(bwcs.get_height()) M = 10 xlo = np.clip(xx.min() - M, 0, imW) xhi = np.clip(xx.max() + M, 0, imW) ylo = np.clip(yy.min() - M, 0, imH) yhi = np.clip(yy.max() + M, 0, imH) if xlo >= xhi or ylo >= yhi: continue subwcs = bwcs.get_subimage(xlo, ylo, xhi - xlo, yhi - ylo) slc = slice(ylo, yhi), slice(xlo, xhi) try: f = fitsio.FITS(fn)[0] img = f[slc] del f if imagetag == 'resid': f = fitsio.FITS(modfn)[0] mod = f[slc] del f img = img - mod except: print('Failed to read image and WCS:', fn) savecache = False import traceback import sys traceback.print_exc(None, sys.stdout) continue try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(wcs, subwcs, [], 3) except OverlapError: debug('Resampling exception') continue rimg[Yo, Xo] += img[Yi, Xi] # try: # Yo,Xo,Yi,Xi,[rim] = resample_with_wcs(wcs, subwcs, [img], 3) # except OverlapError: # debug('Resampling exception') # continue # rimg[Yo,Xo] += rim rn[Yo, Xo] += 1 rimg /= np.maximum(rn, 1) rimgs.append(rimg) if return_if_not_found and not savecache: return if get_images and not write_jpeg: return rimgs if rgbfunc is None: from legacypipe.common import get_rgb rgbfunc = get_rgb rgb = rgbfunc(rimgs, bands, **rgbkwargs) if forcecache: savecache = True if savecache: trymakedirs(tilefn) else: import tempfile f, tilefn = tempfile.mkstemp(suffix='.jpg') os.close(f) # no jpeg output support in matplotlib in some installations... if hack_jpeg: save_jpeg(tilefn, rgb) debug('Wrote', tilefn) else: import pylab as plt plt.imsave(tilefn, rgb) debug('Wrote', tilefn) if get_images: return rimgs return send_file(tilefn, 'image/jpeg', unlink=(not savecache), filename=filename)
def unwise_forcedphot(cat, tiles, band=1, roiradecbox=None, use_ceres=True, ceres_block=8, save_fits=False, get_models=False, ps=None, psf_broadening=None, pixelized_psf=False, get_masks=None, move_crpix=False, modelsky_dir=None, tag=None): ''' Given a list of tractor sources *cat* and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id) runs forced photometry, returning a FITS table the same length as *cat*. *get_masks*: the WCS to resample mask bits into. ''' from tractor import PointSource, Tractor, ExpGalaxy, DevGalaxy from tractor.sersic import SersicGalaxy if tag is None: tag = '' else: tag = tag + ': ' if not pixelized_psf and psf_broadening is None: # PSF broadening in post-reactivation data, by band. # Newer version from Aaron's email to decam-chatter, 2018-06-14. broadening = {1: 1.0405, 2: 1.0346, 3: None, 4: None} psf_broadening = broadening[band] if False: from astrometry.util.plotutils import PlotSequence ps = PlotSequence('wise-forced-w%i' % band) plots = (ps is not None) if plots: import pylab as plt wantims = (plots or save_fits or get_models) wanyband = 'w' if get_models: models = [] wband = 'w%i' % band Nsrcs = len(cat) phot = fits_table() # Filled in based on unique tile overlap phot.wise_coadd_id = np.array([' '] * Nsrcs, dtype='U8') phot.wise_x = np.zeros(Nsrcs, np.float32) phot.wise_y = np.zeros(Nsrcs, np.float32) phot.set('psfdepth_%s' % wband, np.zeros(Nsrcs, np.float32)) nexp = np.zeros(Nsrcs, np.int16) mjd = np.zeros(Nsrcs, np.float64) central_flux = np.zeros(Nsrcs, np.float32) ra = np.array([src.getPosition().ra for src in cat]) dec = np.array([src.getPosition().dec for src in cat]) fskeys = ['prochi2', 'profracflux'] fitstats = {} if get_masks: mh, mw = get_masks.shape maskmap = np.zeros((mh, mw), np.uint32) tims = [] for tile in tiles: info(tag + 'Reading WISE tile', tile.coadd_id, 'band', band) tim = get_unwise_tractor_image(tile.unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradecbox) if tim is None: debug('Actually, no overlap with WISE coadd tile', tile.coadd_id) continue if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data' % tag) ps.savefig() plt.clf() plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), range=(-5, 10), bins=100) plt.xlabel('Per-pixel intensity (Sigma)') plt.title(tag) ps.savefig() if move_crpix and band in [1, 2]: realwcs = tim.wcs.wcs x, y = realwcs.crpix tile_crpix = tile.get('crpix_w%i' % band) dx = tile_crpix[0] - 1024.5 dy = tile_crpix[1] - 1024.5 realwcs.set_crpix(x + dx, y + dy) debug('unWISE', tile.coadd_id, 'band', band, 'CRPIX', x, y, 'shift by', dx, dy, 'to', realwcs.crpix) if modelsky_dir and band in [1, 2]: fn = os.path.join(modelsky_dir, '%s.%i.mod.fits' % (tile.coadd_id, band)) if not os.path.exists(fn): raise RuntimeError('WARNING: does not exist:', fn) x0, x1, y0, y1 = tim.roi bg = fitsio.FITS(fn)[2][y0:y1, x0:x1] assert (bg.shape == tim.shape) if plots: plt.clf() plt.subplot(1, 2, 1) plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) plt.subplot(1, 2, 2) plt.imshow(bg, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) tag = '%s W%i' % (tile.coadd_id, band) plt.suptitle(tag) ps.savefig() plt.clf() ha = dict(range=(-5, 10), bins=100, histtype='step') plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), color='b', label='Original', **ha) plt.hist(((tim.getImage() - bg) * tim.inverr)[tim.inverr > 0].ravel(), color='g', label='Minus Background', **ha) plt.axvline(0, color='k', alpha=0.5) plt.xlabel('Per-pixel intensity (Sigma)') plt.legend() plt.title(tag + ': background') ps.savefig() # Actually subtract the background! tim.data -= bg # Floor the per-pixel variances, # and add Poisson contribution from sources if band in [1, 2]: # in Vega nanomaggies per pixel floor_sigma = {1: 0.5, 2: 2.0} poissons = {1: 0.15, 2: 0.3} with np.errstate(divide='ignore'): new_ie = 1. / np.sqrt( (1. / tim.inverr)**2 + floor_sigma[band] + poissons[band]**2 * np.maximum(0., tim.data)) new_ie[tim.inverr == 0] = 0. if plots: plt.clf() plt.plot((1. / tim.inverr[tim.inverr > 0]).ravel(), (1. / new_ie[tim.inverr > 0]).ravel(), 'b.') plt.title('unWISE per-pixel error: %s band %i' % (tile.coadd_id, band)) plt.xlabel('original') plt.ylabel('floored') ps.savefig() assert (np.all(np.isfinite(new_ie))) assert (np.all(new_ie >= 0.)) tim.inverr = new_ie # Expand a 3-pixel radius around weight=0 (saturated) pixels # from Eddie via crowdsource # https://github.com/schlafly/crowdsource/blob/7069da3e7d9d3124be1cbbe1d21ffeb63fc36dcc/python/wise_proc.py#L74 ## FIXME -- W3/W4 ?? satlimit = 85000 msat = ((tim.data > satlimit) | ((tim.nims == 0) & (tim.nuims > 1))) from scipy.ndimage.morphology import binary_dilation xx, yy = np.mgrid[-3:3 + 1, -3:3 + 1] dilate = xx**2 + yy**2 <= 3**2 msat = binary_dilation(msat, dilate) nbefore = np.sum(tim.inverr == 0) tim.inverr[msat] = 0 nafter = np.sum(tim.inverr == 0) debug('Masking an additional', (nafter - nbefore), 'near-saturated pixels in unWISE', tile.coadd_id, 'band', band) # Read mask file? if get_masks: from astrometry.util.resample import resample_with_wcs, OverlapError # unwise_dir can be a colon-separated list of paths tilemask = None for d in tile.unwise_dir.split(':'): fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id, 'unwise-%s-msk.fits.gz' % tile.coadd_id) if os.path.exists(fn): debug('Reading unWISE mask file', fn) x0, x1, y0, y1 = tim.roi tilemask = fitsio.FITS(fn)[0][y0:y1, x0:x1] break if tilemask is None: info('unWISE mask file for tile', tile.coadd_id, 'does not exist') else: try: tanwcs = tim.wcs.wcs assert (tanwcs.shape == tilemask.shape) Yo, Xo, Yi, Xi, _ = resample_with_wcs(get_masks, tanwcs, intType=np.int16) # Only deal with mask pixels that are set. I, = np.nonzero(tilemask[Yi, Xi] > 0) # Trim to unique area for this tile rr, dd = get_masks.pixelxy2radec(Xo[I] + 1, Yo[I] + 1) good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I = I[good] maskmap[Yo[I], Xo[I]] = tilemask[Yi[I], Xi[I]] except OverlapError: # Shouldn't happen by this point print('Warning: no overlap between WISE tile', tile.coadd_id, 'and brick') if plots: plt.clf() plt.imshow(tilemask, interpolation='nearest', origin='lower') plt.title('Tile %s: mask' % tile.coadd_id) ps.savefig() plt.clf() plt.imshow(maskmap, interpolation='nearest', origin='lower') plt.title('Tile %s: accumulated maskmap' % tile.coadd_id) ps.savefig() # The tiles have some overlap, so zero out pixels outside the # tile's unique area. th, tw = tim.shape xx, yy = np.meshgrid(np.arange(tw), np.arange(th)) rr, dd = tim.wcs.wcs.pixelxy2radec(xx + 1, yy + 1) unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) debug('Tile', tile.coadd_id, '- total of', np.sum(unique), 'unique pixels out of', len(unique.flat), 'total pixels') if get_models: # Save the inverr before blanking out non-unique pixels, for making coadds with no gaps! # (actually, slightly more subtly, expand unique area by 1 pixel) from scipy.ndimage.morphology import binary_dilation du = binary_dilation(unique) tim.coadd_inverr = tim.inverr * du tim.inverr[unique == False] = 0. del xx, yy, rr, dd, unique if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage() * (tim.inverr > 0), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data (unique)' % tag) ps.savefig() if pixelized_psf: from unwise_psf import unwise_psf if (band == 1) or (band == 2): # we only have updated PSFs for W1 and W2 psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id, modelname='neo6_unwisecat') else: psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id) if band == 4: # oversample (the unwise_psf models are at native W4 5.5"/pix, # while the unWISE coadds are made at 2.75"/pix. ph, pw = psfimg.shape subpsf = np.zeros((ph * 2 - 1, pw * 2 - 1), np.float32) from astrometry.util.util import lanczos3_interpolate xx, yy = np.meshgrid( np.arange(0., pw - 0.51, 0.5, dtype=np.float32), np.arange(0., ph - 0.51, 0.5, dtype=np.float32)) xx = xx.ravel() yy = yy.ravel() ix = xx.astype(np.int32) iy = yy.astype(np.int32) dx = (xx - ix).astype(np.float32) dy = (yy - iy).astype(np.float32) psfimg = psfimg.astype(np.float32) rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat], [psfimg]) if plots: plt.clf() plt.imshow(psfimg, interpolation='nearest', origin='lower') plt.title('Original PSF model') ps.savefig() plt.clf() plt.imshow(subpsf, interpolation='nearest', origin='lower') plt.title('Subsampled PSF model') ps.savefig() psfimg = subpsf del xx, yy, ix, iy, dx, dy from tractor.psf import PixelizedPSF psfimg /= psfimg.sum() fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0} psfimg *= fluxrescales[band] tim.psf = PixelizedPSF(psfimg) if psf_broadening is not None and not pixelized_psf: # psf_broadening is a factor by which the PSF FWHMs # should be scaled; the PSF is a little wider # post-reactivation. psf = tim.getPsf() from tractor import GaussianMixturePSF if isinstance(psf, GaussianMixturePSF): debug('Broadening PSF: from', psf) p0 = psf.getParams() pnames = psf.getParamNames() p1 = [ p * psf_broadening**2 if 'var' in name else p for (p, name) in zip(p0, pnames) ] psf.setParams(p1) debug('Broadened PSF:', psf) else: print( 'WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf)) wcs = tim.wcs.wcs _, fx, fy = wcs.radec2pixelxy(ra, dec) x = np.round(fx - 1.).astype(int) y = np.round(fy - 1.).astype(int) good = (x >= 0) * (x < tw) * (y >= 0) * (y < th) # Which sources are in this brick's unique area? usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I, = np.nonzero(good * usrc) nexp[I] = tim.nuims[y[I], x[I]] if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'): mjd[I] = (tim.mjdmin + tim.mjdmax) / 2. phot.wise_coadd_id[I] = tile.coadd_id phot.wise_x[I] = fx[I] - 1. phot.wise_y[I] = fy[I] - 1. central_flux[I] = tim.getImage()[y[I], x[I]] del x, y, good, usrc # PSF norm for depth psf = tim.getPsf() h, w = tim.shape patch = psf.getPointSourcePatch(h // 2, w // 2).patch psfnorm = np.sqrt(np.sum(patch**2)) # To handle zero-depth, we return 1/nanomaggies^2 units rather than mags. # In the small empty patches of the sky (eg W4 in 0922p702), we get sig1 = NaN if np.isfinite(tim.sig1): phot.get('psfdepth_%s' % wband)[I] = 1. / (tim.sig1 / psfnorm)**2 tim.tile = tile tims.append(tim) if plots: plt.clf() mn, mx = 0.1, 20000 plt.hist(np.log10(np.clip(central_flux, mn, mx)), bins=100, range=(np.log10(mn), np.log10(mx))) logt = np.arange(0, 5) plt.xticks(logt, ['%i' % i for i in 10.**logt]) plt.title('Central fluxes (W%i)' % band) plt.axvline(np.log10(20000), color='k') plt.axvline(np.log10(1000), color='k') ps.savefig() # Eddie's non-secret recipe: #- central pixel <= 1000: 19x19 pix box size #- central pixel in 1000 - 20000: 59x59 box size #- central pixel > 20000 or saturated: 149x149 box size #- object near "bright star": 299x299 box size nbig = nmedium = nsmall = 0 for src, cflux in zip(cat, central_flux): if cflux > 20000: R = 100 nbig += 1 elif cflux > 1000: R = 30 nmedium += 1 else: R = 15 nsmall += 1 if isinstance(src, PointSource): src.fixedRadius = R else: ### FIXME -- sizes for galaxies..... can we set PSF size separately? galrad = 0 # RexGalaxy is a subclass of ExpGalaxy if isinstance(src, (ExpGalaxy, DevGalaxy, SersicGalaxy)): galrad = src.shape.re pixscale = 2.75 src.halfsize = int(np.hypot(R, galrad * 5 / pixscale)) debug('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall, 'small') tractor = Tractor(tims, cat) if use_ceres: from tractor.ceres_optimizer import CeresOptimizer tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block) tractor.freezeParamsRecursive('*') tractor.thawPathsTo(wanyband) t0 = Time() R = tractor.optimize_forced_photometry(fitstats=True, variance=True, shared_params=False, wantims=wantims) info(tag + 'unWISE forced photometry took', Time() - t0) if use_ceres: term = R.ceres_status['termination'] # Running out of memory can cause failure to converge and term # status = 2. Fail completely in this case. if term != 0: info(tag + 'Ceres termination status:', term) raise RuntimeError('Ceres terminated with status %i' % term) if wantims: ims1 = R.ims1 # can happen if empty source list (we still want to generate coadds) if ims1 is None: ims1 = R.ims0 flux_invvars = R.IV if R.fitstats is not None: for k in fskeys: x = getattr(R.fitstats, k) fitstats[k] = np.array(x).astype(np.float32) if save_fits: for i, tim in enumerate(tims): tile = tim.tile (dat, mod, _, chi, _) = ims1[i] wcshdr = fitsio.FITSHDR() tim.wcs.wcs.add_to_header(wcshdr) tag = 'fit-%s-w%i' % (tile.coadd_id, band) fitsio.write('%s-data.fits' % tag, dat, clobber=True, header=wcshdr) fitsio.write('%s-mod.fits' % tag, mod, clobber=True, header=wcshdr) fitsio.write('%s-chi.fits' % tag, chi, clobber=True, header=wcshdr) if plots: # Create models for just the brightest sources bright_cat = [ src for src in cat if src.getBrightness().getBand(wanyband) > 1000 ] debug('Bright soures:', len(bright_cat)) btr = Tractor(tims, bright_cat) for tim in tims: mod = btr.getModelImage(tim) tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) sig1 = tim.sig1 plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: bright-star models' % tag) ps.savefig() if get_models: for i, tim in enumerate(tims): tile = tim.tile (dat, mod, _, _, _) = ims1[i] models.append( (tile.coadd_id, band, tim.wcs.wcs, dat, mod, tim.coadd_inverr)) if plots: for i, tim in enumerate(tims): tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) (dat, mod, _, chi, _) = ims1[i] sig1 = tim.sig1 plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() nm = np.array([src.getBrightness().getBand(wanyband) for src in cat]) nm_ivar = flux_invvars # Sources out of bounds, eg, never change from their initial # fluxes. Zero them out instead. nm[nm_ivar == 0] = 0. phot.set('flux_%s' % wband, nm.astype(np.float32)) phot.set('flux_ivar_%s' % wband, nm_ivar.astype(np.float32)) for k in fskeys: phot.set(k + '_' + wband, fitstats.get(k, np.zeros(len(phot), np.float32))) phot.set('nobs_%s' % wband, nexp) phot.set('mjd_%s' % wband, mjd) rtn = wphotduck() rtn.phot = phot rtn.models = None rtn.maskmap = None if get_models: rtn.models = models if get_masks: rtn.maskmap = maskmap return rtn
def make_resampled_psf_images(RCF, band, ra, dec, sdss, targetwcs, W, H, addToHeader, plots=False, max_exposures=1): """ Given a list of (Run, Camcol, Field) tuples, returns a list of (img, imgvar, and header) info for stamp sized imgs centered at ra, dec """ # populate list of resampled images and their new psf's output_imgs = [] # zip through each frame, cut out the relevatn patch for ifield, (run,camcol,field) in enumerate(RCF[:max_exposures]): print """============================= RCF %d of %d ======================== """%(ifield, len(RCF)) # get photofield filename from SDSS, cut it down to relevent RCF fn = sdss.retrieve('photoField', run, camcol, field) F = aufits.fits_table(fn) F.cut((F.run == run) * (F.camcol == camcol) * (F.field == field)) print len(F), 'fields' assert(len(F) == 1) F = F[0] # actually get the tractor image (check if it's in cache!) boundpixradius = int(np.ceil(np.sqrt(2.) * pixradius)) print 'RA,Dec,size', (ra, dec, boundpixradius) tim, tinfo = tsdss.get_tractor_image_dr9( run, camcol, field, band, sdss=sdss, nanomaggies=True, roiradecsize=(ra, dec, boundpixradius)) print 'Got tim:', tim frame = sdss.readFrame(run, camcol, field, band) if tim is None: continue # find pixel position for input RA, DEC in tractor image (original field) x,y = tim.getWcs().positionToPixel(tsdss.RaDecPos(ra, dec)) x,y = int(x), int(y) # Grab calibration information for header tim.sdss_calib = np.median(frame.getCalibVec()) tim.sdss_sky = frame.getSkyAt(x,y) iband = tsdss.band_index(band) tim.sdss_gain = F.gain[iband] tim.sdss_darkvar = F.dark_variance[iband] # get region of interest in the original frame roi = tinfo['roi'] x0,x1,y0,y1 = roi # Resample to common grid th,tw = tim.shape wwcs = tsdss.TractorWCSWrapper(tim.getWcs(), tw, th) try: Yo,Xo,Yi,Xi,[rim] = aresample.resample_with_wcs( targetwcs, wwcs, [tim.getImage()], Lanczos) except aresample.OverlapError: continue img = np.zeros((H,W)) img[Yo,Xo] = rim iv = np.zeros((H,W)) iv[Yo,Xo] = tim.getInvvar()[Yi,Xi] # Convert old PSF to new stamp-specific PSF newpsf = convert_psf_between_imgs(tim, targetwcs) # create the image's header hdr = construct_new_header(tim, tinfo, targetwcs, newpsf, run, camcol, field, band, addToHeader) # add to the list of resampled imgs, output_imgs.append((img, iv, hdr)) return output_imgs
def stage1(T=None, coimgs=None, cons=None, detmaps=None, detivs=None, targetrd=None, pixscale=None, targetwcs=None, W=None, H=None, bands=None, tims=None, ps=None, brick=None, cat=None): orig_wcsxy0 = [tim.wcs.getX0Y0() for tim in tims] hot = np.zeros((H, W), np.float32) for band in bands: detmap = detmaps[band] / np.maximum(1e-16, detivs[band]) detsn = detmap * np.sqrt(detivs[band]) hot = np.maximum(hot, detsn) detmaps[band] = detmap ### FIXME -- ugri for sedname, sed in [('Flat', (1., 1., 1.)), ('Red', (2.5, 1.0, 0.4))]: sedmap = np.zeros((H, W), np.float32) sediv = np.zeros((H, W), np.float32) for iband, band in enumerate(bands): # We convert the detmap to canonical band via # detmap * w # And the corresponding change to sig1 is # sig1 * w # So the invvar-weighted sum is # (detmap * w) / (sig1**2 * w**2) # = detmap / (sig1**2 * w) sedmap += detmaps[band] * detivs[band] / sed[iband] sediv += detivs[band] / sed[iband]**2 sedmap /= np.maximum(1e-16, sediv) sedsn = sedmap * np.sqrt(sediv) hot = np.maximum(hot, sedsn) plt.clf() dimshow(np.round(sedsn), vmin=0, vmax=10, cmap='hot') plt.title('SED-matched detection filter: %s' % sedname) ps.savefig() peaks = (hot > 4) blobs, nblobs = label(peaks) print('N detected blobs:', nblobs) blobslices = find_objects(blobs) # Un-set catalog blobs for x, y in zip(T.itx, T.ity): # blob number bb = blobs[y, x] if bb == 0: continue # un-set 'peaks' within this blob slc = blobslices[bb - 1] peaks[slc][blobs[slc] == bb] = 0 # Now, after having removed catalog sources, crank up the detection threshold peaks &= (hot > 5) # zero out the edges(?) peaks[0, :] = peaks[:, 0] = 0 peaks[-1, :] = peaks[:, -1] = 0 peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[0:-2, 1:-1]) peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[2:, 1:-1]) peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[1:-1, 0:-2]) peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[1:-1, 2:]) # These are our peaks pki = np.flatnonzero(peaks) peaky, peakx = np.unravel_index(pki, peaks.shape) print(len(peaky), 'peaks') crossa = dict(ms=10, mew=1.5) plt.clf() dimshow(get_rgb(coimgs, bands)) ax = plt.axis() plt.plot(T.tx, T.ty, 'r+', **crossa) plt.plot(peakx, peaky, '+', color=green, **crossa) plt.axis(ax) plt.title('SDSS + SED-matched detections') ps.savefig() ### HACK -- high threshold again # Segment, and record which sources fall into each blob blobs, nblobs = label((hot > 20)) print('N detected blobs:', nblobs) blobslices = find_objects(blobs) T.blob = blobs[T.ity, T.itx] blobsrcs = [] blobflux = [] fluximg = coimgs[1] for blob in range(1, nblobs + 1): blobsrcs.append(np.flatnonzero(T.blob == blob)) bslc = blobslices[blob - 1] blobflux.append(np.sum(fluximg[bslc][blobs[bslc] == blob])) # Fit the SDSS sources for tim in tims: tim.psfex.fitSavedData(*tim.psfex.splinedata) tim.psf = tim.psfex # How far down to render model profiles minsigma = 0.1 for tim in tims: tim.modelMinval = minsigma * tim.sig1 srcvariances = [[] for src in cat] # Fit in order of flux for blobnumber, iblob in enumerate(np.argsort(-np.array(blobflux))): bslc = blobslices[iblob] Isrcs = blobsrcs[iblob] if len(Isrcs) == 0: continue print() print('Blob', blobnumber, 'of', len(blobflux), ':', len(Isrcs), 'sources') print('Source indices:', Isrcs) print() # blob bbox in target coords sy, sx = bslc by0, by1 = sy.start, sy.stop bx0, bx1 = sx.start, sx.stop blobh, blobw = by1 - by0, bx1 - bx0 rr, dd = targetwcs.pixelxy2radec([bx0, bx0, bx1, bx1], [by0, by1, by1, by0]) alphas = [0.1, 0.3, 1.0] subtims = [] for itim, tim in enumerate(tims): h, w = tim.shape ok, x, y = tim.subwcs.radec2pixelxy(rr, dd) sx0, sx1 = x.min(), x.max() sy0, sy1 = y.min(), y.max() if sx1 < 0 or sy1 < 0 or sx1 > w or sy1 > h: continue sx0 = np.clip(int(np.floor(sx0)), 0, w - 1) sx1 = np.clip(int(np.ceil(sx1)), 0, w - 1) + 1 sy0 = np.clip(int(np.floor(sy0)), 0, h - 1) sy1 = np.clip(int(np.ceil(sy1)), 0, h - 1) + 1 subslc = slice(sy0, sy1), slice(sx0, sx1) subimg = tim.getImage()[subslc] subie = tim.getInvError()[subslc] subwcs = tim.getWcs().copy() ox0, oy0 = orig_wcsxy0[itim] subwcs.setX0Y0(ox0 + sx0, oy0 + sy0) # Mask out inverr for pixels that are not within the blob. subtarget = targetwcs.get_subimage(bx0, by0, blobw, blobh) subsubwcs = tim.subwcs.get_subimage(int(sx0), int(sy0), int(sx1 - sx0), int(sy1 - sy0)) try: Yo, Xo, Yi, Xi, rims = resample_with_wcs( subsubwcs, subtarget, [], 2) except OverlapError: print('No overlap') continue if len(Yo) == 0: continue subie2 = np.zeros_like(subie) I = np.flatnonzero(blobs[bslc][Yi, Xi] == (iblob + 1)) subie2[Yo[I], Xo[I]] = subie[Yo[I], Xo[I]] subie = subie2 # If the subimage (blob) is small enough, instantiate a # constant PSF model in the center. if sy1 - sy0 < 100 and sx1 - sx0 < 100: subpsf = tim.psf.mogAt(ox0 + (sx0 + sx1) / 2., oy0 + (sy0 + sy1) / 2.) else: # Otherwise, instantiate a (shifted) spatially-varying # PsfEx model. subpsf = ShiftedPsf(tim.psf, ox0 + sx0, oy0 + sy0) subtim = Image(data=subimg, inverr=subie, wcs=subwcs, psf=subpsf, photocal=tim.getPhotoCal(), sky=tim.getSky(), name=tim.name) subtim.band = tim.band subtim.sig1 = tim.sig1 subtim.modelMinval = tim.modelMinval subtims.append(subtim) subcat = Catalog(*[cat[i] for i in Isrcs]) subtr = Tractor(subtims, subcat) subtr.freezeParam('images') # Optimize individual sources in order of flux fluxes = [] for src in subcat: # HACK -- here we just *sum* the nanomaggies in each band. Bogus! br = src.getBrightness() flux = sum([br.getFlux(band) for band in bands]) fluxes.append(flux) Ibright = np.argsort(-np.array(fluxes)) if len(Ibright) >= 5: # -Remember the original subtim images # -Compute initial models for each source (in each tim) # -Subtract initial models from images # -During fitting, for each source: # -add back in the source's initial model (to each tim) # -fit, with Catalog([src]) # -subtract final model (from each tim) # -Replace original subtim images # # --Might want to omit newly-added detection-filter sources, since their # fluxes are bogus. # Remember original tim images orig_timages = [tim.getImage().copy() for tim in subtims] initial_models = [] # Create initial models for each tim x each source for tim in subtims: mods = [] for src in subcat: mod = src.getModelPatch(tim) mods.append(mod) if mod is not None: if not np.all(np.isfinite(mod.patch)): print('Non-finite mod patch') print('source:', src) print('tim:', tim) print('PSF:', tim.getPsf()) assert (np.all(np.isfinite(mod.patch))) mod.addTo(tim.getImage(), scale=-1) initial_models.append(mods) # For sources in decreasing order of brightness for numi, i in enumerate(Ibright): tsrc = Time() print('Fitting source', i, '(%i of %i in blob)' % (numi, len(Ibright))) src = subcat[i] print(src) srctractor = Tractor(subtims, [src]) srctractor.freezeParams('images') # Add this source's initial model back in. for tim, mods in zip(subtims, initial_models): mod = mods[i] if mod is not None: mod.addTo(tim.getImage()) print('Optimizing:', srctractor) srctractor.printThawedParams() for step in range(50): dlnp, X, alpha = srctractor.optimize(priors=False, shared_params=False, alphas=alphas) print('dlnp:', dlnp, 'src', src) if dlnp < 0.1: break for tim in subtims: mod = src.getModelPatch(tim) if mod is not None: mod.addTo(tim.getImage(), scale=-1) for tim, img in zip(subtims, orig_timages): tim.data = img del orig_timages del initial_models else: # Fit sources one at a time, but don't subtract other models subcat.freezeAllParams() for numi, i in enumerate(Ibright): tsrc = Time() print('Fitting source', i, '(%i of %i in blob)' % (numi, len(Ibright))) print(subcat[i]) subcat.freezeAllBut(i) print('Optimizing:', subtr) subtr.printThawedParams() for step in range(10): dlnp, X, alpha = subtr.optimize(priors=False, shared_params=False, alphas=alphas) print('dlnp:', dlnp) if dlnp < 0.1: break print('Fitting source took', Time() - tsrc) print(subcat[i]) if len(Isrcs) > 1 and len(Isrcs) <= 10: tfit = Time() # Optimize all at once? subcat.thawAllParams() print('Optimizing:', subtr) subtr.printThawedParams() for step in range(20): dlnp, X, alpha = subtr.optimize(priors=False, shared_params=False, alphas=alphas) print('dlnp:', dlnp) if dlnp < 0.1: break # Variances subcat.thawAllRecursive() subcat.freezeAllParams() for isub, srci in enumerate(Isrcs): print('Variances for source', srci) subcat.thawParam(isub) src = subcat[isub] print('Source', src) print('Params:', src.getParamNames()) if isinstance(src, (DevGalaxy, ExpGalaxy)): src.shape = EllipseE.fromEllipseESoft(src.shape) elif isinstance(src, FixedCompositeGalaxy): src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp) src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev) print('Converted ellipse:', src) allderivs = subtr.getDerivs() for iparam, derivs in enumerate(allderivs): dchisq = 0 for deriv, tim in derivs: h, w = tim.shape deriv.clipTo(w, h) ie = tim.getInvError() slc = deriv.getSlice(ie) chi = deriv.patch * ie[slc] dchisq += (chi**2).sum() if dchisq == 0.: v = np.nan else: v = 1. / dchisq srcvariances[srci].append(v) assert (len(srcvariances[srci]) == subcat[isub].numberOfParams()) subcat.freezeParam(isub) cat.thawAllRecursive() for i, src in enumerate(cat): print('Source', i, src) print('variances:', srcvariances[i]) print(len(srcvariances[i]), 'vs', src.numberOfParams()) if len(srcvariances[i]) != src.numberOfParams(): # This can happen for sources outside the brick bounds: they never get optimized? print('Warning: zeroing variances for source', src) srcvariances[i] = [0] * src.numberOfParams() if isinstance(src, (DevGalaxy, ExpGalaxy)): src.shape = EllipseE.fromEllipseESoft(src.shape) elif isinstance(src, FixedCompositeGalaxy): src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp) src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev) assert (len(srcvariances[i]) == src.numberOfParams()) variances = np.hstack(srcvariances) assert (len(variances) == cat.numberOfParams()) return dict(cat=cat, variances=variances)
def galex_coadds(onegal, galaxy=None, radius_mosaic=30, radius_mask=None, pixscale=1.5, ref_pixscale=0.262, output_dir=None, galex_dir=None, log=None, centrals=True, verbose=False): '''Generate custom GALEX cutouts. radius_mosaic and radius_mask in arcsec pixscale: GALEX pixel scale in arcsec/pixel. ''' import fitsio import matplotlib.pyplot as plt from astrometry.libkd.spherematch import match_radec from astrometry.util.resample import resample_with_wcs, OverlapError from tractor import (Tractor, NanoMaggies, Image, LinearPhotoCal, NCircularGaussianPSF, ConstantFitsWcs, ConstantSky) from legacypipe.survey import imsave_jpeg from legacypipe.catalog import read_fits_catalog if galaxy is None: galaxy = 'galaxy' if galex_dir is None: galex_dir = os.environ.get('GALEX_DIR') if output_dir is None: output_dir = '.' if radius_mask is None: radius_mask = radius_mosaic radius_search = 5.0 # [arcsec] else: radius_search = radius_mask W = H = np.ceil(2 * radius_mosaic / pixscale).astype('int') # [pixels] targetwcs = Tan(onegal['RA'], onegal['DEC'], (W + 1) / 2.0, (H + 1) / 2.0, -pixscale / 3600.0, 0.0, 0.0, pixscale / 3600.0, float(W), float(H)) # Read the custom Tractor catalog tractorfile = os.path.join(output_dir, '{}-tractor.fits'.format(galaxy)) if not os.path.isfile(tractorfile): print('Missing Tractor catalog {}'.format(tractorfile)) return 0 cat = fits_table(tractorfile) print('Read {} sources from {}'.format(len(cat), tractorfile), flush=True, file=log) keep = np.ones(len(cat)).astype(bool) if centrals: # Find the large central galaxy and mask out (ignore) all the models # which are within its elliptical mask. # This algorithm will have to change for mosaics not centered on large # galaxies, e.g., in galaxy groups. m1, m2, d12 = match_radec(cat.ra, cat.dec, onegal['RA'], onegal['DEC'], radius_search / 3600.0, nearest=False) if len(m1) == 0: print('No central galaxies found at the central coordinates!', flush=True, file=log) else: pixfactor = ref_pixscale / pixscale # shift the optical Tractor positions for mm in m1: morphtype = cat.type[mm].strip() if morphtype == 'EXP' or morphtype == 'COMP': e1, e2, r50 = cat.shapeexp_e1[mm], cat.shapeexp_e2[ mm], cat.shapeexp_r[mm] # [arcsec] elif morphtype == 'DEV' or morphtype == 'COMP': e1, e2, r50 = cat.shapedev_e1[mm], cat.shapedev_e2[ mm], cat.shapedev_r[mm] # [arcsec] else: r50 = None if r50: majoraxis = r50 * 5 / pixscale # [pixels] ba, phi = SGA.misc.convert_tractor_e1e2(e1, e2) these = SGA.misc.ellipse_mask(W / 2, W / 2, majoraxis, ba * majoraxis, np.radians(phi), cat.bx * pixfactor, cat.by * pixfactor) if np.sum(these) > 0: #keep[these] = False pass print('Hack!') keep[mm] = False #srcs = read_fits_catalog(cat) #_srcs = np.array(srcs)[~keep].tolist() #mod = SGA.misc.srcs2image(_srcs, ConstantFitsWcs(targetwcs), psf_sigma=3.0) #import matplotlib.pyplot as plt ##plt.imshow(mod, origin='lower') ; plt.savefig('junk.png') #plt.imshow(np.log10(mod), origin='lower') ; plt.savefig('junk.png') #pdb.set_trace() srcs = read_fits_catalog(cat) for src in srcs: src.freezeAllBut('brightness') #srcs_nocentral = np.array(srcs)[keep].tolist() # Find all overlapping GALEX tiles and then read the tims. galex_tiles = _read_galex_tiles(targetwcs, galex_dir, log=log, verbose=verbose) gbands = ['n', 'f'] nicegbands = ['NUV', 'FUV'] zps = dict(n=20.08, f=18.82) coimgs, comods, coresids, coimgs_central, comods_nocentral = [], [], [], [], [] for niceband, band in zip(nicegbands, gbands): J = np.flatnonzero(galex_tiles.get('has_' + band)) print(len(J), 'GALEX tiles have coverage in band', band) coimg = np.zeros((H, W), np.float32) comod = np.zeros((H, W), np.float32) cowt = np.zeros((H, W), np.float32) comod_nocentral = np.zeros((H, W), np.float32) for src in srcs: src.setBrightness(NanoMaggies(**{band: 1})) for j in J: brick = galex_tiles[j] fn = os.path.join( galex_dir, brick.tilename.strip(), '%s-%sd-intbgsub.fits.gz' % (brick.brickname, band)) #print(fn) gwcs = Tan(*[ float(f) for f in [ brick.crval1, brick.crval2, brick.crpix1, brick.crpix2, brick.cdelt1, 0., 0., brick.cdelt2, 3840., 3840. ] ]) img = fitsio.read(fn) #print('Read', img.shape) try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, gwcs, [], 3) except OverlapError: continue K = np.flatnonzero(img[Yi, Xi] != 0.) if len(K) == 0: continue Yo, Xo, Yi, Xi = Yo[K], Xo[K], Yi[K], Xi[K] wt = brick.get(band + 'exptime') coimg[Yo, Xo] += wt * img[Yi, Xi] cowt[Yo, Xo] += wt x0, x1, y0, y1 = min(Xi), max(Xi), min(Yi), max(Yi) subwcs = gwcs.get_subimage(x0, y0, x1 - x0 + 1, y1 - y0 + 1) twcs = ConstantFitsWcs(subwcs) timg = img[y0:y1 + 1, x0:x1 + 1] tie = np.ones_like(timg) ## HACK! #hdr = fitsio.read_header(fn) #zp = hdr[''] zp = zps[band] photocal = LinearPhotoCal(NanoMaggies.zeropointToScale(zp), band=band) tsky = ConstantSky(0.0) # HACK -- circular Gaussian PSF of fixed size... # in arcsec #fwhms = dict(NUV=6.0, FUV=6.0) # -> sigma in pixels #sig = fwhms[band] / 2.35 / twcs.pixel_scale() sig = 6.0 / np.sqrt(8 * np.log(2)) / twcs.pixel_scale() tpsf = NCircularGaussianPSF([sig], [1.]) tim = Image(data=timg, inverr=tie, psf=tpsf, wcs=twcs, sky=tsky, photocal=photocal, name='GALEX ' + band + brick.brickname) ## Build the model image with and without the central galaxy model. tractor = Tractor([tim], srcs) mod = tractor.getModelImage(0) tractor.freezeParam('images') tractor.optimize_forced_photometry(priors=False, shared_params=False) mod = tractor.getModelImage(0) srcs_nocentral = np.array(srcs)[keep].tolist() #srcs_nocentral = np.array(srcs)[nocentral].tolist() tractor_nocentral = Tractor([tim], srcs_nocentral) mod_nocentral = tractor_nocentral.getModelImage(0) comod[Yo, Xo] += wt * mod[Yi - y0, Xi - x0] comod_nocentral[Yo, Xo] += wt * mod_nocentral[Yi - y0, Xi - x0] coimg /= np.maximum(cowt, 1e-18) comod /= np.maximum(cowt, 1e-18) comod_nocentral /= np.maximum(cowt, 1e-18) coresid = coimg - comod # Subtract the model image which excludes the central (comod_nocentral) # from the data (coimg) to isolate the light of the central # (coimg_central). coimg_central = coimg - comod_nocentral coimgs.append(coimg) comods.append(comod) coresids.append(coresid) comods_nocentral.append(comod_nocentral) coimgs_central.append(coimg_central) # Write out the final images with and without the central, making sure # to apply the zeropoint to go from counts/s to AB nanomaggies. # https://asd.gsfc.nasa.gov/archive/galex/FAQ/counts_background.html for thisimg, imtype in zip((coimg, comod, comod_nocentral), ('image', 'model', 'model-nocentral')): fitsfile = os.path.join( output_dir, '{}-{}-{}.fits'.format(galaxy, imtype, niceband)) if verbose: print('Writing {}'.format(fitsfile)) fitsio.write(fitsfile, thisimg * 10**(-0.4 * (zp - 22.5)), clobber=True) # Build a color mosaic (but note that the images here are in units of # background-subtracted counts/s). #_galex_rgb = _galex_rgb_moustakas #_galex_rgb = _galex_rgb_dstn _galex_rgb = _galex_rgb_official for imgs, imtype in zip( (coimgs, comods, coresids, comods_nocentral, coimgs_central), ('image', 'model', 'resid', 'model-nocentral', 'image-central')): rgb = _galex_rgb(imgs) jpgfile = os.path.join(output_dir, '{}-{}-FUVNUV.jpg'.format(galaxy, imtype)) if verbose: print('Writing {}'.format(jpgfile)) imsave_jpeg(jpgfile, rgb, origin='lower') return 1
def cutout_jpg(req): import tempfile import fitsio from astrometry.util.util import Tan from astrometry.util.resample import resample_with_wcs form = CutoutSearchForm(req.GET) if not form.is_valid(): return HttpResponse('failed to parse request') ra = form.cleaned_data['ra'] dec = form.cleaned_data['dec'] if ra is None or dec is None: return HttpResponse('RA and Dec arguments are required') size = form.cleaned_data['size'] if size is None: size = 100 else: size = min(256, size) # Ignoring this for now... # bandstr = form.cleaned_data['bands'] # bands = [int(c) for c in bandstr] # bands = [b for b in bands if b in [1,2,3,4]] version = form.cleaned_data['version'] radius = size/2. * 2.75/3600. tiles = unwise_tiles_near_radec(ra, dec, radius) tiles = list(tiles) pixscale = 2.75 / 3600. cowcs = Tan(*[float(x) for x in [ra, dec, 0.5 + size/2., 0.5 + size/2., -pixscale, 0., 0., pixscale, size, size]]) bands = [1,2] coims = [np.zeros((size,size), np.float32) for b in bands] con = np.zeros((size,size), int) for tile in tiles: coadd = tile.coadd dirnm = os.path.join(settings.DATA_DIR, version, coadd[:3], coadd) base = os.path.join(dirnm, 'unwise-%s' % coadd) subwcs = None ims = [] for band in bands: fn = str(base + '-w%i-img-m.fits' % band) if subwcs is None: wcs = Tan(fn) ok,x,y = wcs.radec2pixelxy(ra, dec) x = int(round(x-1.)) y = int(round(y-1.)) x0 = x - size/2 x1 = x0 + size y0 = y - size/2 y1 = y0 + size if x1 <= 0 or y1 <= 0: break if x0 >= wcs.get_width() or y0 >= wcs.get_height(): break x0 = max(x0, 0) y0 = max(y0, 0) x1 = min(x1, wcs.get_width()) y1 = min(y1, wcs.get_height()) subwcs = wcs.get_subimage(x0, y0, x1-x0, y1-y0) if subwcs is None: break img = fitsio.FITS(fn)[0][y0:y1, x0:x1] ims.append(img) if subwcs is None: continue try: Yo,Xo,Yi,Xi,rims = resample_with_wcs(cowcs, subwcs, ims, 3) except: continue for rim,co in zip(rims, coims): co[Yo,Xo] += rim con[Yo,Xo] += 1 for co in coims: co /= np.maximum(con, 1) rgb = np.zeros((size,size,3), np.uint8) lo,hi = -3,10 scale = 10. rgb[:,:,2] = np.clip(255. * ((coims[1] / scale) - lo) / (hi-lo), 0, 255) rgb[:,:,1] = np.clip(255. * (((coims[0]+coims[1])/2. / scale) - lo) / (hi-lo), 0, 255) rgb[:,:,0] = np.clip(255. * ((coims[0] / scale) - lo) / (hi-lo), 0, 255) f,tempfn = tempfile.mkstemp(suffix='.jpg') os.close(f) # import matplotlib # matplotlib.use('Agg') # import pylab as plt # plt.imsave(tempfn, rgb, interpolation='nearest', origin='lower') from PIL import Image pic = Image.fromarray(rgb) pic.save(tempfn) return send_file(tempfn, 'image/jpeg', unlink=True)
def map_sdss(req, ver, zoom, x, y, savecache=None, tag='sdss', get_images=False, ignoreCached=False, wcs=None, forcecache=False, forcescale=None, bestOnly=False, bands='gri', **kwargs): from decals import settings if savecache is None: savecache = settings.SAVE_CACHE zoom = int(zoom) zoomscale = 2.**zoom x = int(x) y = int(y) if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale: raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom, x, y)) ver = int(ver) if not ver in tileversions[tag]: raise RuntimeError('Invalid version %i for tag %s' % (ver, tag)) basedir = settings.DATA_DIR tilefn = os.path.join(basedir, 'tiles', tag, '%i/%i/%i/%i.jpg' % (ver, zoom, x, y)) if os.path.exists(tilefn) and not ignoreCached: if get_images: return None return send_file(tilefn, 'image/jpeg', expires=oneyear, modsince=req.META.get('HTTP_IF_MODIFIED_SINCE')) if not savecache: import tempfile f, tilefn = tempfile.mkstemp(suffix='.jpg') os.close(f) if wcs is None: try: wcs, W, H, zoomscale, zoom, x, y = get_tile_wcs(zoom, x, y) except RuntimeError as e: if get_images: return None return HttpResponse(e.strerror) else: W = wcs.get_width() H = wcs.get_height() from astrometry.util.fits import fits_table import numpy as np from astrometry.libkd.spherematch import tree_build_radec, tree_search_radec from astrometry.util.starutil_numpy import degrees_between, arcsec_between from astrometry.util.resample import resample_with_wcs, OverlapError from astrometry.util.util import Tan, Sip import fitsio print('Tile wcs: center', wcs.radec_center(), 'pixel scale', wcs.pixel_scale()) global w_flist global w_flist_tree if w_flist is None: w_flist = fits_table( os.path.join(settings.DATA_DIR, 'sdss', 'window_flist.fits'), columns=['run', 'rerun', 'camcol', 'field', 'ra', 'dec', 'score']) print('Read', len(w_flist), 'window_flist entries') w_flist.cut(w_flist.rerun == '301') print('Cut to', len(w_flist), 'in rerun 301') w_flist_tree = tree_build_radec(w_flist.ra, w_flist.dec) # SDSS field size radius = 1.01 * np.hypot(10., 14.) / 2. / 60. # leaflet tile size ra, dec = wcs.pixelxy2radec(W / 2., H / 2.)[-2:] r0, d0 = wcs.pixelxy2radec(1, 1)[-2:] r1, d1 = wcs.pixelxy2radec(W, H)[-2:] radius = radius + max(degrees_between(ra, dec, r0, d0), degrees_between(ra, dec, r1, d1)) J = tree_search_radec(w_flist_tree, ra, dec, radius) print(len(J), 'overlapping SDSS fields found') if len(J) == 0: if get_images: return None if forcecache: # create symlink to blank.jpg! trymakedirs(tilefn) src = os.path.join(settings.STATIC_ROOT, 'blank.jpg') if os.path.exists(tilefn): os.unlink(tilefn) os.symlink(src, tilefn) print('Symlinked', tilefn, '->', src) from django.http import HttpResponseRedirect return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg') ww = [1, W * 0.25, W * 0.5, W * 0.75, W] hh = [1, H * 0.25, H * 0.5, H * 0.75, H] r, d = wcs.pixelxy2radec( [1] * len(hh) + ww + [W] * len(hh) + list(reversed(ww)), hh + [1] * len(ww) + list(reversed(hh)) + [H] * len(ww))[-2:] scaled = 0 scalepat = None scaledir = 'sdss' if zoom <= 13 and forcescale is None: # Get *actual* pixel scales at the top & bottom r1, d1 = wcs.pixelxy2radec(W / 2., H)[-2:] r2, d2 = wcs.pixelxy2radec(W / 2., H - 1.)[-2:] r3, d3 = wcs.pixelxy2radec(W / 2., 1.)[-2:] r4, d4 = wcs.pixelxy2radec(W / 2., 2.)[-2:] # Take the min = most zoomed-in scale = min(arcsec_between(r1, d1, r2, d2), arcsec_between(r3, d3, r4, d4)) native_scale = 0.396 scaled = int(np.floor(np.log2(scale / native_scale))) print('Zoom:', zoom, 'x,y', x, y, 'Tile pixel scale:', scale, 'Scale step:', scaled) scaled = np.clip(scaled, 1, 7) dirnm = os.path.join(basedir, 'scaled', scaledir) scalepat = os.path.join( dirnm, '%(scale)i%(band)s', '%(rerun)s', '%(run)i', '%(camcol)i', 'sdss-%(run)i-%(camcol)i-%(field)i-%(band)s.fits') if forcescale is not None: scaled = forcescale rimgs = [np.zeros((H, W), np.float32) for band in bands] rns = [np.zeros((H, W), np.float32) for band in bands] from astrometry.sdss import AsTransWrapper, DR9 sdss = DR9(basedir=settings.SDSS_DIR) sdss.saveUnzippedFiles(settings.SDSS_DIR) #sdss.setFitsioReadBZ2() if settings.SDSS_PHOTOOBJS: sdss.useLocalTree(photoObjs=settings.SDSS_PHOTOOBJS, resolve=settings.SDSS_RESOLVE) for jnum, j in enumerate(J): print('SDSS field', jnum, 'of', len(J), 'for zoom', zoom, 'x', x, 'y', y) im = w_flist[j] if im.score >= 0.5: weight = 1. else: weight = 0.001 for band, rimg, rn in zip(bands, rimgs, rns): if im.rerun != '301': continue tmpsuff = '.tmp%08i' % np.random.randint(100000000) basefn = sdss.retrieve('frame', im.run, im.camcol, field=im.field, band=band, rerun=im.rerun, tempsuffix=tmpsuff) if scaled > 0: fnargs = dict(band=band, rerun=im.rerun, run=im.run, camcol=im.camcol, field=im.field) fn = get_scaled(scalepat, fnargs, scaled, basefn, read_base_wcs=read_astrans, read_wcs=read_sip_wcs) print('get_scaled:', fn) else: fn = basefn frame = None if fn == basefn: frame = sdss.readFrame(im.run, im.camcol, im.field, band, filename=fn) h, w = frame.getImageShape() # Trim off the overlapping top of the image # Wimp out and instead of trimming 128 pix, trim 124! trim = 124 subh = h - trim astrans = frame.getAsTrans() fwcs = AsTransWrapper(astrans, w, subh) fullimg = frame.getImage() fullimg = fullimg[:-trim, :] else: fwcs = Sip(fn) fitsimg = fitsio.FITS(fn)[0] h, w = fitsimg.get_info()['dims'] fullimg = fitsimg.read() try: #Yo,Xo,Yi,Xi,nil = resample_with_wcs(wcs, fwcs, [], 3) Yo, Xo, Yi, Xi, [resamp ] = resample_with_wcs(wcs, fwcs, [fullimg], 2) except OverlapError: continue if len(Xi) == 0: #print 'No overlap' continue if sdssps is not None: x0 = Xi.min() x1 = Xi.max() y0 = Yi.min() y1 = Yi.max() slc = (slice(y0, y1 + 1), slice(x0, x1 + 1)) if frame is not None: img = frame.getImageSlice(slc) else: img = fitsimg[slc] #rimg[Yo,Xo] += img[Yi-y0, Xi-x0] if bestOnly: K = np.flatnonzero(weight > rn[Yo, Xo]) print('Updating', len(K), 'of', len(Yo), 'pixels') if len(K): rimg[Yo[K], Xo[K]] = resamp[K] * weight rn[Yo[K], Xo[K]] = weight else: rimg[Yo, Xo] += resamp * weight rn[Yo, Xo] += weight if sdssps is not None: # goodpix = np.ones(img.shape, bool) # fpM = sdss.readFpM(im.run, im.camcol, im.field, band) # for plane in [ 'INTERP', 'SATUR', 'CR', 'GHOST' ]: # fpM.setMaskedPixels(plane, goodpix, False, roi=[x0,x1,y0,y1]) plt.clf() #ima = dict(vmin=-0.05, vmax=0.5) #ima = dict(vmin=-0.5, vmax=2.) ima = dict(vmax=np.percentile(img, 99)) plt.subplot(2, 3, 1) dimshow(img, ticks=False, **ima) plt.title('image') rthis = np.zeros_like(rimg) #rthis[Yo,Xo] += img[Yi-y0, Xi-x0] rthis[Yo, Xo] += resamp plt.subplot(2, 3, 2) dimshow(rthis, ticks=False, **ima) plt.title('resampled') # plt.subplot(2,3,3) # dimshow(goodpix, ticks=False, vmin=0, vmax=1) # plt.title('good pix') plt.subplot(2, 3, 4) dimshow(rimg / np.maximum(rn, 1), ticks=False, **ima) plt.title('coadd') plt.subplot(2, 3, 5) dimshow(rn, vmin=0, ticks=False) plt.title('coverage: max %i' % rn.max()) plt.subplot(2, 3, 6) rgb = sdss_rgb( [rimg / np.maximum(rn, 1) for rimg, rn in zip(rimgs, rns)], bands) dimshow(rgb) plt.suptitle('SDSS %s, R/C/F %i/%i/%i' % (band, im.run, im.camcol, im.field)) sdssps.savefig() for rimg, rn in zip(rimgs, rns): rimg /= np.maximum(rn, 1e-3) del rns if get_images: return rimgs rgb = sdss_rgb(rimgs, bands) trymakedirs(tilefn) save_jpeg(tilefn, rgb) print('Wrote', tilefn) return send_file(tilefn, 'image/jpeg', unlink=(not savecache))