def make_coadds(tims, bands, targetwcs, mods=None, xy=None, apertures=None, apxy=None, ngood=False, detmaps=False, psfsize=False, callback=None, callback_args=[], plots=False, ps=None, lanczos=True, mp=None): from astrometry.util.ttime import Time t0 = Time() class Duck(object): pass C = Duck() W = int(targetwcs.get_width()) H = int(targetwcs.get_height()) # always, for patching SATUR, etc pixels? unweighted = True if not xy: psfsize = False C.coimgs = [] if detmaps: C.galdetivs = [] C.detivs = [] if mods is not None: C.comods = [] C.coresids = [] if apertures is not None: unweighted = True C.AP = fits_table() if xy: ix, iy = xy C.T = fits_table() C.T.nobs = np.zeros((len(ix), len(bands)), np.uint8) C.T.anymask = np.zeros((len(ix), len(bands)), np.int16) C.T.allmask = np.zeros((len(ix), len(bands)), np.int16) if psfsize: C.T.psfsize = np.zeros((len(ix), len(bands)), np.float32) if detmaps: C.T.depth = np.zeros((len(ix), len(bands)), np.float32) C.T.galdepth = np.zeros((len(ix), len(bands)), np.float32) if lanczos: print('Doing Lanczos resampling') for tim in tims: # surface-brightness correction tim.sbscale = (targetwcs.pixel_scale() / tim.subwcs.pixel_scale())**2 # We create one iterator per band to do the tim resampling. These all run in # parallel when multi-processing. imaps = [] for band in bands: args = [] for itim, tim in enumerate(tims): if tim.band != band: continue if mods is None: mo = None else: mo = mods[itim] args.append((itim, tim, mo, lanczos, targetwcs)) if mp is not None: imaps.append(mp.imap_unordered(_resample_one, args)) else: import itertools imaps.append(itertools.imap(_resample_one, args)) # Args for aperture photometry apargs = [] tinyw = 1e-30 for iband, (band, timiter) in enumerate(zip(bands, imaps)): print('Computing coadd for band', band) # coadded weight map (moo) cow = np.zeros((H, W), np.float32) # coadded weighted image map cowimg = np.zeros((H, W), np.float32) kwargs = dict(cowimg=cowimg, cow=cow) if detmaps: # detection map inverse-variance (depth map) detiv = np.zeros((H, W), np.float32) C.detivs.append(detiv) kwargs.update(detiv=detiv) # galaxy detection map inverse-variance (galdepth map) galdetiv = np.zeros((H, W), np.float32) C.galdetivs.append(galdetiv) kwargs.update(galdetiv=galdetiv) if mods is not None: # model image cowmod = np.zeros((H, W), np.float32) # chi-squared image cochi2 = np.zeros((H, W), np.float32) kwargs.update(cowmod=cowmod, cochi2=cochi2) if unweighted: # unweighted image coimg = np.zeros((H, W), np.float32) if mods is not None: # unweighted model comod = np.zeros((H, W), np.float32) # number of exposures con = np.zeros((H, W), np.uint8) # inverse-variance coiv = np.zeros((H, W), np.float32) kwargs.update(coimg=coimg, coiv=coiv) # Note that we have 'congood' as well as 'nobs': # * 'congood' is used for the 'nexp' *image*. # * 'nobs' is used for the per-source measurements # # (you want to know the number of observations within the # source footprint, not just the peak pixel which may be # saturated, etc.) if ngood: congood = np.zeros((H, W), np.uint8) kwargs.update(congood=congood) if xy: # These match the type of the "DQ" images. # "any" mask ormask = np.zeros((H, W), np.int16) # "all" mask andmask = np.empty((H, W), np.int16) allbits = reduce(np.bitwise_or, CP_DQ_BITS.values()) andmask[:, :] = allbits # number of observations nobs = np.zeros((H, W), np.uint8) kwargs.update(ormask=ormask, andmask=andmask, nobs=nobs) if psfsize: psfsizemap = np.zeros((H, W), np.float32) for R in timiter: if R is None: continue itim, Yo, Xo, iv, im, mo, dq = R #print('timiter Yo,Xo,im.shape=',Yo,Xo,im.shape) tim = tims[itim] # invvar-weighted image cowimg[Yo, Xo] += iv * im cow[Yo, Xo] += iv if unweighted: if dq is None: goodpix = 1 else: # include BLEED, SATUR, INTERP pixels if no other # pixels exists (do this by eliminating all other CP # flags) badbits = 0 for bitname in ['badpix', 'cr', 'trans', 'edge', 'edge2']: badbits |= CP_DQ_BITS[bitname] goodpix = ((dq & badbits) == 0) coimg[Yo, Xo] += goodpix * im con[Yo, Xo] += goodpix coiv[Yo, Xo] += goodpix * 1. / (tim.sig1 * tim.sbscale)**2 # ...ish if xy: if dq is not None: ormask[Yo, Xo] |= dq andmask[Yo, Xo] &= dq # raw exposure count nobs[Yo, Xo] += 1 if psfsize: # psfnorm is in units of 1/pixels. # (eg, psfnorm for a gaussian is ~ 1/psf_sigma) # Neff is in pixels**2 neff = 1. / tim.psfnorm**2 # Narcsec is in arcsec**2 narcsec = neff * tim.wcs.pixel_scale()**2 psfsizemap[Yo, Xo] += iv * (1. / narcsec) if detmaps: # point-source depth detsig1 = tim.sig1 / tim.psfnorm detiv[Yo, Xo] += (iv > 0) * (1. / detsig1**2) # Galaxy detection map gdetsig1 = tim.sig1 / tim.galnorm galdetiv[Yo, Xo] += (iv > 0) * (1. / gdetsig1**2) if ngood: congood[Yo, Xo] += (iv > 0) if mods is not None: # straight-up comod[Yo, Xo] += goodpix * mo # invvar-weighted cowmod[Yo, Xo] += iv * mo # chi-squared cochi2[Yo, Xo] += iv * (im - mo)**2 del mo del goodpix del Yo, Xo, im, iv # END of loop over tims # Per-band: cowimg /= np.maximum(cow, tinyw) C.coimgs.append(cowimg) if mods is not None: cowmod /= np.maximum(cow, tinyw) C.comods.append(cowmod) coresid = cowimg - cowmod coresid[cow == 0] = 0. C.coresids.append(coresid) if unweighted: coimg /= np.maximum(con, 1) del con cowimg[cow == 0] = coimg[cow == 0] if mods is not None: cowmod[cow == 0] = comod[cow == 0] if xy: C.T.nobs[:, iband] = nobs[iy, ix] C.T.anymask[:, iband] = ormask[iy, ix] C.T.allmask[:, iband] = andmask[iy, ix] # unless there were no images there... C.T.allmask[nobs[iy, ix] == 0, iband] = 0 if detmaps: C.T.depth[:, iband] = detiv[iy, ix] C.T.galdepth[:, iband] = galdetiv[iy, ix] if psfsize: wt = cow[iy, ix] # psfsizemap is in units of iv * (1 / arcsec**2) sz = psfsizemap[iy, ix] sz /= np.maximum(wt, tinyw) sz[wt == 0] = 0. # Back to units of linear arcsec. sz = 1. / np.sqrt(sz) sz[wt == 0] = 0. # Correction factor to get back to equivalent of Gaussian sigma sz /= (2. * np.sqrt(np.pi)) # Conversion factor to FWHM (2.35) sz *= 2. * np.sqrt(2. * np.log(2.)) C.T.psfsize[:, iband] = sz del psfsizemap if apertures is not None: # Aperture photometry, using the unweighted "coimg" and # "coiv" arrays. with np.errstate(divide='ignore'): imsigma = 1.0 / np.sqrt(coiv) imsigma[coiv == 0] = 0 for irad, rad in enumerate(apertures): apargs.append((irad, band, rad, coimg, imsigma, True, apxy)) if mods is not None: apargs.append( (irad, band, rad, coresid, None, False, apxy)) if callback is not None: callback(band, *callback_args, **kwargs) # END of loop over bands t2 = Time() print('coadds: images:', t2 - t0) if apertures is not None: # Aperture phot, in parallel if mp is not None: apresults = mp.map(_apphot_one, apargs) else: apresults = map(_apphot_one, apargs) del apargs apresults = iter(apresults) for iband, band in enumerate(bands): apimg = [] apimgerr = [] if mods is not None: apres = [] for irad, rad in enumerate(apertures): (airad, aband, isimg, ap_img, ap_err) = apresults.next() assert (airad == irad) assert (aband == band) assert (isimg) apimg.append(ap_img) apimgerr.append(ap_err) if mods is not None: (airad, aband, isimg, ap_img, ap_err) = apresults.next() assert (airad == irad) assert (aband == band) assert (not isimg) apres.append(ap_img) assert (ap_err is None) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. C.AP.set('apflux_img_%s' % band, ap) ap = 1. / (np.vstack(apimgerr).T)**2 ap[np.logical_not(np.isfinite(ap))] = 0. C.AP.set('apflux_img_ivar_%s' % band, ap) if mods is not None: ap = np.vstack(apres).T ap[np.logical_not(np.isfinite(ap))] = 0. C.AP.set('apflux_resid_%s' % band, ap) t3 = Time() print('coadds apphot:', t3 - t2) return C
def edr_dr2_vs_dr3(): #plotrange = ((240.9, 242.1), (4.8, 5.9)) plotrange = ((240, 245), (4.5, 12.0)) #fns = glob('dr2-tractor/241/tractor-*p05*.fits') fns = glob('dr2-tractor/24[01234]/tractor-*.fits') fns.sort() TT2, TT3 = [], [] for fn in fns: fn3 = fn.replace('dr2-tractor', 'dr3-tractor') if not os.path.exists(fn3): print('Does not exist:', fn3) continue cols = 'ra dec brick_primary brickname decam_anymask'.split() T2 = fits_table(fn, columns=cols) T3 = fits_table(fn3, columns=cols) print('Reading', fn, '->', len(T2), 'DR2,', len(T3), 'DR3') TT2.append(T2) TT3.append(T3) T2 = merge_tables(TT2, columns='fillzero') T3 = merge_tables(TT3, columns='fillzero') del TT2, TT3 plt.clf() plothist(T2.ra, T2.dec, nbins=200, range=plotrange) plt.title('DR2') ps.savefig() plt.clf() plothist(T3.ra, T3.dec, nbins=200, range=plotrange) plt.title('DR3') ps.savefig() plt.clf() I = np.flatnonzero(T2.brick_primary) print('DR2:', len(I), 'brick_primary') H2, xe, ye = plothist(T2.ra[I], T2.dec[I], nbins=200, range=plotrange) mx = H2.max() plt.title('DR2 brick_primary') ps.savefig() plt.clf() I = np.flatnonzero(T3.brick_primary) print('DR3:', len(I), 'brick_primary') H3, xe, ye = plothist(T3.ra[I], T3.dec[I], nbins=200, imshowargs=dict(vmax=mx), range=plotrange) plt.title('DR3 brick_primary') ps.savefig() plt.clf() plt.imshow((H3 - H2).T, interpolation='nearest', origin='lower', cmap='hot', extent=(min(xe), max(xe), min(ye), max(ye)), aspect='auto') plt.colorbar() plt.title('DR3 - DR2 brick_primary') ps.savefig() for name, bit in CP_DQ_BITS.items(): plt.clf() anymask = reduce(np.bitwise_or, (T3.decam_anymask[:, 1], T3.decam_anymask[:, 2], T3.decam_anymask[:, 4])) I = np.flatnonzero((anymask & bit) > 0) #plt.subplot(1,2,1) plothist(T3.ra[I], T3.dec[I], nbins=200, imshowargs=dict(vmax=mx), doclf=False, range=plotrange) plt.title('DR3, %s any' % name) #plt.suptitle('DR3') ps.savefig()
def edr_dr2_vs_dr3(): #plotrange = ((240.9, 242.1), (4.8, 5.9)) plotrange = ((240, 245), (4.5, 12.0)) #fns = glob('dr2-tractor/241/tractor-*p05*.fits') fns = glob('dr2-tractor/24[01234]/tractor-*.fits') fns.sort() TT2,TT3 = [],[] for fn in fns: fn3 = fn.replace('dr2-tractor', 'dr3-tractor') if not os.path.exists(fn3): print('Does not exist:', fn3) continue cols = 'ra dec brick_primary brickname decam_anymask'.split() T2 = fits_table(fn, columns=cols) T3 = fits_table(fn3, columns=cols) print('Reading', fn, '->', len(T2), 'DR2,', len(T3), 'DR3') TT2.append(T2) TT3.append(T3) T2 = merge_tables(TT2, columns='fillzero') T3 = merge_tables(TT3, columns='fillzero') del TT2, TT3 plt.clf() plothist(T2.ra, T2.dec, nbins=200, range=plotrange) plt.title('DR2') ps.savefig() plt.clf() plothist(T3.ra, T3.dec, nbins=200, range=plotrange) plt.title('DR3') ps.savefig() plt.clf() I = np.flatnonzero(T2.brick_primary) print('DR2:', len(I), 'brick_primary') H2,xe,ye = plothist(T2.ra[I], T2.dec[I], nbins=200, range=plotrange) mx = H2.max() plt.title('DR2 brick_primary') ps.savefig() plt.clf() I = np.flatnonzero(T3.brick_primary) print('DR3:', len(I), 'brick_primary') H3,xe,ye = plothist(T3.ra[I], T3.dec[I], nbins=200, imshowargs=dict(vmax=mx), range=plotrange) plt.title('DR3 brick_primary') ps.savefig() plt.clf() plt.imshow((H3 - H2).T, interpolation='nearest', origin='lower', cmap='hot', extent=(min(xe), max(xe), min(ye), max(ye)), aspect='auto') plt.colorbar() plt.title('DR3 - DR2 brick_primary') ps.savefig() for name,bit in CP_DQ_BITS.items(): plt.clf() anymask = reduce(np.bitwise_or, (T3.decam_anymask[:,1], T3.decam_anymask[:,2], T3.decam_anymask[:,4])) I = np.flatnonzero((anymask & bit) > 0) #plt.subplot(1,2,1) plothist(T3.ra[I], T3.dec[I], nbins=200, imshowargs=dict(vmax=mx), doclf=False, range=plotrange) plt.title('DR3, %s any' % name) #plt.suptitle('DR3') ps.savefig()
def make_coadds(tims, bands, targetwcs, mods=None, xy=None, apertures=None, apxy=None, ngood=False, detmaps=False, psfsize=False, callback=None, callback_args=[], plots=False, ps=None, lanczos=True, mp=None): from astrometry.util.ttime import Time t0 = Time() class Duck(object): pass C = Duck() W = int(targetwcs.get_width()) H = int(targetwcs.get_height()) # always, for patching SATUR, etc pixels? unweighted=True if not xy: psfsize = False C.coimgs = [] if detmaps: C.galdetivs = [] C.detivs = [] if mods is not None: C.comods = [] C.coresids = [] if apertures is not None: unweighted = True C.AP = fits_table() if xy: ix,iy = xy C.T = fits_table() C.T.nobs = np.zeros((len(ix), len(bands)), np.uint8) C.T.anymask = np.zeros((len(ix), len(bands)), np.int16) C.T.allmask = np.zeros((len(ix), len(bands)), np.int16) if psfsize: C.T.psfsize = np.zeros((len(ix), len(bands)), np.float32) if detmaps: C.T.depth = np.zeros((len(ix), len(bands)), np.float32) C.T.galdepth = np.zeros((len(ix), len(bands)), np.float32) if lanczos: print('Doing Lanczos resampling') for tim in tims: # surface-brightness correction tim.sbscale = (targetwcs.pixel_scale() / tim.subwcs.pixel_scale())**2 # We create one iterator per band to do the tim resampling. These all run in # parallel when multi-processing. imaps = [] for band in bands: args = [] for itim,tim in enumerate(tims): if tim.band != band: continue if mods is None: mo = None else: mo = mods[itim] args.append((itim,tim,mo,lanczos,targetwcs)) if mp is not None: imaps.append(mp.imap_unordered(_resample_one, args)) else: import itertools imaps.append(itertools.imap(_resample_one, args)) # Args for aperture photometry apargs = [] tinyw = 1e-30 for iband,(band,timiter) in enumerate(zip(bands, imaps)): print('Computing coadd for band', band) # coadded weight map (moo) cow = np.zeros((H,W), np.float32) # coadded weighted image map cowimg = np.zeros((H,W), np.float32) kwargs = dict(cowimg=cowimg, cow=cow) if detmaps: # detection map inverse-variance (depth map) detiv = np.zeros((H,W), np.float32) C.detivs.append(detiv) kwargs.update(detiv=detiv) # galaxy detection map inverse-variance (galdepth map) galdetiv = np.zeros((H,W), np.float32) C.galdetivs.append(galdetiv) kwargs.update(galdetiv=galdetiv) if mods is not None: # model image cowmod = np.zeros((H,W), np.float32) # chi-squared image cochi2 = np.zeros((H,W), np.float32) kwargs.update(cowmod=cowmod, cochi2=cochi2) if unweighted: # unweighted image coimg = np.zeros((H,W), np.float32) if mods is not None: # unweighted model comod = np.zeros((H,W), np.float32) # number of exposures con = np.zeros((H,W), np.uint8) # inverse-variance coiv = np.zeros((H,W), np.float32) kwargs.update(coimg=coimg, coiv=coiv) # Note that we have 'congood' as well as 'nobs': # * 'congood' is used for the 'nexp' *image*. # * 'nobs' is used for the per-source measurements # # (you want to know the number of observations within the # source footprint, not just the peak pixel which may be # saturated, etc.) if ngood: congood = np.zeros((H,W), np.uint8) kwargs.update(congood=congood) if xy: # These match the type of the "DQ" images. # "any" mask ormask = np.zeros((H,W), np.int16) # "all" mask andmask = np.empty((H,W), np.int16) allbits = reduce(np.bitwise_or, CP_DQ_BITS.values()) andmask[:,:] = allbits # number of observations nobs = np.zeros((H,W), np.uint8) kwargs.update(ormask=ormask, andmask=andmask, nobs=nobs) if psfsize: psfsizemap = np.zeros((H,W), np.float32) for R in timiter: if R is None: continue itim,Yo,Xo,iv,im,mo,dq = R #print('timiter Yo,Xo,im.shape=',Yo,Xo,im.shape) tim = tims[itim] # invvar-weighted image cowimg[Yo,Xo] += iv * im cow [Yo,Xo] += iv if unweighted: if dq is None: goodpix = 1 else: # include BLEED, SATUR, INTERP pixels if no other # pixels exists (do this by eliminating all other CP # flags) badbits = 0 for bitname in ['badpix', 'cr', 'trans', 'edge', 'edge2']: badbits |= CP_DQ_BITS[bitname] goodpix = ((dq & badbits) == 0) coimg[Yo,Xo] += goodpix * im con [Yo,Xo] += goodpix coiv [Yo,Xo] += goodpix * 1./(tim.sig1 * tim.sbscale)**2 # ...ish if xy: if dq is not None: ormask [Yo,Xo] |= dq andmask[Yo,Xo] &= dq # raw exposure count nobs[Yo,Xo] += 1 if psfsize: # psfnorm is in units of 1/pixels. # (eg, psfnorm for a gaussian is ~ 1/psf_sigma) # Neff is in pixels**2 neff = 1./tim.psfnorm**2 # Narcsec is in arcsec**2 narcsec = neff * tim.wcs.pixel_scale()**2 psfsizemap[Yo,Xo] += iv * (1. / narcsec) if detmaps: # point-source depth detsig1 = tim.sig1 / tim.psfnorm detiv[Yo,Xo] += (iv > 0) * (1. / detsig1**2) # Galaxy detection map gdetsig1 = tim.sig1 / tim.galnorm galdetiv[Yo,Xo] += (iv > 0) * (1. / gdetsig1**2) if ngood: congood[Yo,Xo] += (iv > 0) if mods is not None: # straight-up comod[Yo,Xo] += goodpix * mo # invvar-weighted cowmod[Yo,Xo] += iv * mo # chi-squared cochi2[Yo,Xo] += iv * (im - mo)**2 del mo del goodpix del Yo,Xo,im,iv # END of loop over tims # Per-band: cowimg /= np.maximum(cow, tinyw) C.coimgs.append(cowimg) if mods is not None: cowmod /= np.maximum(cow, tinyw) C.comods.append(cowmod) coresid = cowimg - cowmod coresid[cow == 0] = 0. C.coresids.append(coresid) if unweighted: coimg /= np.maximum(con, 1) del con cowimg[cow == 0] = coimg[cow == 0] if mods is not None: cowmod[cow == 0] = comod[cow == 0] if xy: C.T.nobs [:,iband] = nobs[iy,ix] C.T.anymask[:,iband] = ormask [iy,ix] C.T.allmask[:,iband] = andmask[iy,ix] # unless there were no images there... C.T.allmask[nobs[iy,ix] == 0, iband] = 0 if detmaps: C.T.depth [:,iband] = detiv[iy, ix] C.T.galdepth[:,iband] = galdetiv[iy, ix] if psfsize: wt = cow[iy,ix] # psfsizemap is in units of iv * (1 / arcsec**2) sz = psfsizemap[iy,ix] sz /= np.maximum(wt, tinyw) sz[wt == 0] = 0. # Back to units of linear arcsec. sz = 1. / np.sqrt(sz) sz[wt == 0] = 0. # Correction factor to get back to equivalent of Gaussian sigma sz /= (2. * np.sqrt(np.pi)) # Conversion factor to FWHM (2.35) sz *= 2. * np.sqrt(2. * np.log(2.)) C.T.psfsize[:,iband] = sz del psfsizemap if apertures is not None: # Aperture photometry, using the unweighted "coimg" and # "coiv" arrays. with np.errstate(divide='ignore'): imsigma = 1.0/np.sqrt(coiv) imsigma[coiv == 0] = 0 for irad,rad in enumerate(apertures): apargs.append((irad, band, rad, coimg, imsigma, True, apxy)) if mods is not None: apargs.append((irad, band, rad, coresid, None, False, apxy)) if callback is not None: callback(band, *callback_args, **kwargs) # END of loop over bands t2 = Time() print('coadds: images:', t2-t0) if apertures is not None: # Aperture phot, in parallel if mp is not None: apresults = mp.map(_apphot_one, apargs) else: apresults = map(_apphot_one, apargs) del apargs apresults = iter(apresults) for iband,band in enumerate(bands): apimg = [] apimgerr = [] if mods is not None: apres = [] for irad,rad in enumerate(apertures): (airad, aband, isimg, ap_img, ap_err) = apresults.next() assert(airad == irad) assert(aband == band) assert(isimg) apimg.append(ap_img) apimgerr.append(ap_err) if mods is not None: (airad, aband, isimg, ap_img, ap_err) = apresults.next() assert(airad == irad) assert(aband == band) assert(not isimg) apres.append(ap_img) assert(ap_err is None) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. C.AP.set('apflux_img_%s' % band, ap) ap = 1./(np.vstack(apimgerr).T)**2 ap[np.logical_not(np.isfinite(ap))] = 0. C.AP.set('apflux_img_ivar_%s' % band, ap) if mods is not None: ap = np.vstack(apres).T ap[np.logical_not(np.isfinite(ap))] = 0. C.AP.set('apflux_resid_%s' % band, ap) t3 = Time() print('coadds apphot:', t3-t2) return C