def unwise_coadds(onegal, galaxy=None, radius_mosaic=30, radius_mask=None, pixscale=2.75, ref_pixscale=0.262, output_dir=None, unwise_dir=None, verbose=False, log=None, centrals=True): '''Generate custom unWISE cutouts. radius_mosaic and radius_mask in arcsec pixscale: WISE pixel scale in arcsec/pixel; make this smaller than 2.75 to oversample. ''' import fitsio import matplotlib.pyplot as plt from astrometry.util.util import Tan from astrometry.util.fits import fits_table from astrometry.libkd.spherematch import match_radec from astrometry.util.resample import resample_with_wcs, ResampleError from wise.forcedphot import unwise_tiles_touching_wcs from wise.unwise import get_unwise_tractor_image from tractor import Tractor, Image, NanoMaggies from legacypipe.survey import imsave_jpeg from legacypipe.catalog import read_fits_catalog if galaxy is None: galaxy = 'galaxy' if output_dir is None: output_dir = '.' if unwise_dir is None: unwise_dir = os.environ.get('UNWISE_COADDS_DIR') if radius_mask is None: radius_mask = radius_mosaic radius_search = 5.0 # [arcsec] else: radius_search = radius_mask # Initialize the WCS object. W = H = np.ceil(2 * radius_mosaic / pixscale).astype('int') # [pixels] targetwcs = Tan(onegal['RA'], onegal['DEC'], (W + 1) / 2.0, (H + 1) / 2.0, -pixscale / 3600.0, 0.0, 0.0, pixscale / 3600.0, float(W), float(H)) # Read the custom Tractor catalog. tractorfile = os.path.join(output_dir, '{}-tractor.fits'.format(galaxy)) if not os.path.isfile(tractorfile): print('Missing Tractor catalog {}'.format(tractorfile), flush=True, file=log) return 0 primhdr = fitsio.read_header(tractorfile) cat = fits_table(tractorfile) print('Read {} sources from {}'.format(len(cat), tractorfile), flush=True, file=log) keep = np.ones(len(cat)).astype(bool) if centrals: # Find the large central galaxy and mask out (ignore) all the models # which are within its elliptical mask. # This algorithm will have to change for mosaics not centered on large # galaxies, e.g., in galaxy groups. m1, m2, d12 = match_radec(cat.ra, cat.dec, onegal['RA'], onegal['DEC'], radius_search / 3600.0, nearest=False) if len(m1) == 0: print('No central galaxies found at the central coordinates!', flush=True, file=log) else: pixfactor = ref_pixscale / pixscale # shift the optical Tractor positions for mm in m1: morphtype = cat.type[mm].strip() if morphtype == 'EXP' or morphtype == 'COMP': e1, e2, r50 = cat.shapeexp_e1[mm], cat.shapeexp_e2[ mm], cat.shapeexp_r[mm] # [arcsec] elif morphtype == 'DEV' or morphtype == 'COMP': e1, e2, r50 = cat.shapedev_e1[mm], cat.shapedev_e2[ mm], cat.shapedev_r[mm] # [arcsec] else: r50 = None if r50: majoraxis = r50 * 5 / pixscale # [pixels] ba, phi = SGA.misc.convert_tractor_e1e2(e1, e2) these = SGA.misc.ellipse_mask(W / 2, W / 2, majoraxis, ba * majoraxis, np.radians(phi), cat.bx * pixfactor, cat.by * pixfactor) if np.sum(these) > 0: #keep[these] = False pass print('Hack!') keep[mm] = False #srcs = read_fits_catalog(cat) #_srcs = np.array(srcs)[~keep].tolist() #mod = SGA.misc.srcs2image(_srcs, ConstantFitsWcs(targetwcs), psf_sigma=3.0) #import matplotlib.pyplot as plt ##plt.imshow(mod, origin='lower') ; plt.savefig('junk.png') #plt.imshow(np.log10(mod), origin='lower') ; plt.savefig('junk.png') #pdb.set_trace() srcs = read_fits_catalog(cat) for src in srcs: src.freezeAllBut('brightness') #srcs_nocentral = np.array(srcs)[keep].tolist() cat_nocentral = cat[keep] ## Find and remove all the objects within XX arcsec of the target ## coordinates. #m1, m2, d12 = match_radec(T.ra, T.dec, onegal['RA'], onegal['DEC'], 5/3600.0, nearest=False) #if len(d12) == 0: # print('No matching galaxies found -- probably not what you wanted.') # #raise ValueError # nocentral = np.ones(len(T)).astype(bool) #else: # nocentral = ~np.isin(T.objid, T[m1].objid) #T_nocentral = T[nocentral] # Find and read the overlapping unWISE tiles. Assume the targetwcs is # axis-aligned and that the edge midpoints yield the RA, Dec limits (true # for TAN). Note: the way the roiradec box is used, the min/max order # doesn't matter. r, d = targetwcs.pixelxy2radec(np.array([1, W, W / 2, W / 2]), np.array([H / 2, H / 2, 1, H])) roiradec = [r[0], r[1], d[2], d[3]] tiles = unwise_tiles_touching_wcs(targetwcs) wbands = [1, 2, 3, 4] wanyband = 'w' vega_to_ab = dict(w1=2.699, w2=3.339, w3=5.174, w4=6.620) # Convert the AB WISE fluxes in the Tractor catalog to Vega nanomaggies so # they're consistent with the coadds, below. for band in wbands: f = cat.get('flux_w{}'.format(band)) e = cat.get('flux_ivar_w{}'.format(band)) print('Setting negative fluxes equal to zero!') f[f < 0] = 0 #f[f/e < 3] = 0 f *= 10**(0.4 * vega_to_ab['w{}'.format(band)]) coimgs = [np.zeros((H, W), np.float32) for b in wbands] comods = [np.zeros((H, W), np.float32) for b in wbands] comods_nocentral = [np.zeros((H, W), np.float32) for b in wbands] con = [np.zeros((H, W), np.uint8) for b in wbands] for iband, band in enumerate(wbands): for ii, src in enumerate(srcs): src.setBrightness( NanoMaggies( **{wanyband: cat.get('flux_w{}'.format(band))[ii]})) srcs_nocentral = np.array(srcs)[keep].tolist() #srcs_nocentral = np.array(srcs)[nocentral].tolist() # The tiles have some overlap, so for each source, keep the fit in the # tile whose center is closest to the source. for tile in tiles: #print('Reading tile {}'.format(tile.coadd_id)) tim = get_unwise_tractor_image(unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradec) if tim is None: print('Actually, no overlap with tile {}'.format( tile.coadd_id)) continue print('Read image {} with shape {}'.format(tile.coadd_id, tim.shape)) def _unwise_mod(tim, use_cat, use_srcs, margin=10): # Select sources in play. wisewcs = tim.wcs.wcs timH, timW = tim.shape ok, x, y = wisewcs.radec2pixelxy(use_cat.ra, use_cat.dec) x = (x - 1.).astype(np.float32) y = (y - 1.).astype(np.float32) I = np.flatnonzero((x >= -margin) * (x < timW + margin) * (y >= -margin) * (y < timH + margin)) #print('Found {} sources within the image + margin = {} pixels'.format(len(I), margin)) subcat = [use_srcs[i] for i in I] tractor = Tractor([tim], subcat) mod = tractor.getModelImage(0) return mod mod = _unwise_mod(tim, cat, srcs) mod_nocentral = _unwise_mod(tim, cat_nocentral, srcs_nocentral) try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, tim.wcs.wcs) except ResampleError: continue if len(Yo) == 0: continue # The models are already in AB nanomaggies, but the tiles / tims are # in Vega nanomaggies, so convert them here. coimgs[iband][Yo, Xo] += tim.getImage()[Yi, Xi] comods[iband][Yo, Xo] += mod[Yi, Xi] comods_nocentral[iband][Yo, Xo] += mod_nocentral[Yi, Xi] con[iband][Yo, Xo] += 1 ## Convert back to nanomaggies. #vega2ab = vega_to_ab['w{}'.format(band)] #coimgs[iband] *= 10**(-0.4 * vega2ab) #comods[iband] *= 10**(-0.4 * vega2ab) #comods_nocentral[iband] *= 10**(-0.4 * vega2ab) for img, mod, mod_nocentral, n in zip(coimgs, comods, comods_nocentral, con): img /= np.maximum(n, 1) mod /= np.maximum(n, 1) mod_nocentral /= np.maximum(n, 1) coresids = [img - mod for img, mod in list(zip(coimgs, comods))] # Subtract the model image which excludes the central (comod_nocentral) # from the data (coimg) to isolate the light of the central # (coimg_central). coimgs_central = [ img - mod for img, mod in list(zip(coimgs, comods_nocentral)) ] # Write out the final images with and without the central and converted into # AB nanomaggies. for coadd, imtype in zip((coimgs, comods, comods_nocentral), ('image', 'model', 'model-nocentral')): for img, band in zip(coadd, wbands): vega2ab = vega_to_ab['w{}'.format(band)] fitsfile = os.path.join( output_dir, '{}-{}-W{}.fits'.format(galaxy, imtype, band)) if verbose: print('Writing {}'.format(fitsfile)) fitsio.write(fitsfile, img * 10**(-0.4 * vega2ab), clobber=True) # Generate color WISE images. kwa = dict(mn=-1, mx=100, arcsinh=0.5) #kwa = dict(mn=-0.05, mx=1., arcsinh=0.5) #kwa = dict(mn=-0.1, mx=2., arcsinh=None) for imgs, imtype in zip( (coimgs, comods, coresids, comods_nocentral, coimgs_central), ('image', 'model', 'resid', 'model-nocentral', 'image-central')): rgb = _unwise_to_rgb(imgs[:2], **kwa) # W1, W2 jpgfile = os.path.join(output_dir, '{}-{}-W1W2.jpg'.format(galaxy, imtype)) if verbose: print('Writing {}'.format(jpgfile)) imsave_jpeg(jpgfile, rgb, origin='lower') return 1
def wise_cutouts(ra, dec, radius, ps, pixscale=2.75, tractor_base=".", unwise_dir="unwise-coadds"): """ radius in arcsec. pixscale: WISE pixel scale in arcsec/pixel; make this smaller than 2.75 to oversample. """ npix = int(np.ceil(radius / pixscale)) print("Image size:", npix) W = H = npix pix = pixscale / 3600.0 wcs = Tan(ra, dec, (W + 1) / 2.0, (H + 1) / 2.0, -pix, 0.0, 0.0, pix, float(W), float(H)) # Find DECaLS bricks overlapping decals = Decals() B = bricks_touching_wcs(wcs, decals=decals) print("Found", len(B), "bricks overlapping") TT = [] for b in B.brickname: fn = os.path.join(tractor_base, "tractor", b[:3], "tractor-%s.fits" % b) T = fits_table(fn) print("Read", len(T), "from", b) primhdr = fitsio.read_header(fn) TT.append(T) T = merge_tables(TT) print("Total of", len(T), "sources") T.cut(T.brick_primary) print(len(T), "primary") margin = 20 ok, xx, yy = wcs.radec2pixelxy(T.ra, T.dec) I = np.flatnonzero((xx > -margin) * (yy > -margin) * (xx < W + margin) * (yy < H + margin)) T.cut(I) print(len(T), "within ROI") # Pull out DECaLS coadds (image, model, resid). dwcs = wcs.scale(2.0 * pixscale / 0.262) dh, dw = dwcs.shape print("DECaLS resampled shape:", dh, dw) tags = ["image", "model", "resid"] coimgs = [np.zeros((dh, dw, 3), np.uint8) for t in tags] for b in B.brickname: fn = os.path.join(tractor_base, "coadd", b[:3], b, "decals-%s-image-r.fits" % b) bwcs = Tan(fn) try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(dwcs, bwcs) except ResampleError: continue if len(Yo) == 0: continue print("Resampling", len(Yo), "pixels from", b) xl, xh, yl, yh = Xi.min(), Xi.max(), Yi.min(), Yi.max() print( "python legacypipe/runbrick.py -b %s --zoom %i %i %i %i --outdir cluster --pixpsf --splinesky --pipe --no-early-coadds" % (b, xl - 5, xh + 5, yl - 5, yh + 5) + " -P 'pickles/cluster-%(brick)s-%%(stage)s.pickle'" ) for i, tag in enumerate(tags): fn = os.path.join(tractor_base, "coadd", b[:3], b, "decals-%s-%s.jpg" % (b, tag)) img = plt.imread(fn) img = np.flipud(img) coimgs[i][Yo, Xo, :] = img[Yi, Xi, :] tt = dict(image="Image", model="Model", resid="Resid") for img, tag in zip(coimgs, tags): plt.clf() dimshow(img, ticks=False) plt.title("DECaLS grz %s" % tt[tag]) ps.savefig() # Find unWISE tiles overlapping tiles = unwise_tiles_touching_wcs(wcs) print("Cut to", len(tiles), "unWISE tiles") # Here we assume the targetwcs is axis-aligned and that the # edge midpoints yield the RA,Dec limits (true for TAN). r, d = wcs.pixelxy2radec(np.array([1, W, W / 2, W / 2]), np.array([H / 2, H / 2, 1, H])) # the way the roiradec box is used, the min/max order doesn't matter roiradec = [r[0], r[1], d[2], d[3]] ra, dec = T.ra, T.dec T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T srcs = read_fits_catalog(T, ellipseClass=EllipseE) wbands = [1, 2] wanyband = "w" for band in wbands: T.wise_flux[:, band - 1] *= 10.0 ** (primhdr["WISEAB%i" % band] / 2.5) coimgs = [np.zeros((H, W), np.float32) for b in wbands] comods = [np.zeros((H, W), np.float32) for b in wbands] con = [np.zeros((H, W), np.uint8) for b in wbands] for iband, band in enumerate(wbands): print("Photometering WISE band", band) wband = "w%i" % band for i, src in enumerate(srcs): # print('Source', src, 'brightness', src.getBrightness(), 'params', src.getBrightness().getParams()) # src.getBrightness().setParams([T.wise_flux[i, band-1]]) src.setBrightness(NanoMaggies(**{wanyband: T.wise_flux[i, band - 1]})) # print('Set source brightness:', src.getBrightness()) # The tiles have some overlap, so for each source, keep the # fit in the tile whose center is closest to the source. for tile in tiles: print("Reading tile", tile.coadd_id) tim = get_unwise_tractor_image(unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradec) if tim is None: print("Actually, no overlap with tile", tile.coadd_id) continue print("Read image with shape", tim.shape) # Select sources in play. wisewcs = tim.wcs.wcs H, W = tim.shape ok, x, y = wisewcs.radec2pixelxy(ra, dec) x = (x - 1.0).astype(np.float32) y = (y - 1.0).astype(np.float32) margin = 10.0 I = np.flatnonzero((x >= -margin) * (x < W + margin) * (y >= -margin) * (y < H + margin)) print(len(I), "within the image + margin") subcat = [srcs[i] for i in I] tractor = Tractor([tim], subcat) mod = tractor.getModelImage(0) # plt.clf() # dimshow(tim.getImage(), ticks=False) # plt.title('WISE %s %s' % (tile.coadd_id, wband)) # ps.savefig() # plt.clf() # dimshow(mod, ticks=False) # plt.title('WISE %s %s' % (tile.coadd_id, wband)) # ps.savefig() try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(wcs, tim.wcs.wcs) except ResampleError: continue if len(Yo) == 0: continue print("Resampling", len(Yo), "pixels from WISE", tile.coadd_id, band) coimgs[iband][Yo, Xo] += tim.getImage()[Yi, Xi] comods[iband][Yo, Xo] += mod[Yi, Xi] con[iband][Yo, Xo] += 1 for img, mod, n in zip(coimgs, comods, con): img /= np.maximum(n, 1) mod /= np.maximum(n, 1) for band, img, mod in zip(wbands, coimgs, comods): lo, hi = np.percentile(img, [25, 99]) plt.clf() dimshow(img, vmin=lo, vmax=hi, ticks=False) plt.title("WISE W%i Data" % band) ps.savefig() plt.clf() dimshow(mod, vmin=lo, vmax=hi, ticks=False) plt.title("WISE W%i Model" % band) ps.savefig() resid = img - mod mx = np.abs(resid).max() plt.clf() dimshow(resid, vmin=-mx, vmax=mx, ticks=False) plt.title("WISE W%i Resid" % band) ps.savefig() # kwa = dict(mn=-0.1, mx=2., arcsinh = 1.) kwa = dict(mn=-0.1, mx=2.0, arcsinh=None) rgb = _unwise_to_rgb(coimgs, **kwa) plt.clf() dimshow(rgb, ticks=False) plt.title("WISE W1/W2 Data") ps.savefig() rgb = _unwise_to_rgb(comods, **kwa) plt.clf() dimshow(rgb, ticks=False) plt.title("WISE W1/W2 Model") ps.savefig() kwa = dict(mn=-1, mx=1, arcsinh=None) rgb = _unwise_to_rgb([img - mod for img, mod in zip(coimgs, comods)], **kwa) plt.clf() dimshow(rgb, ticks=False) plt.title("WISE W1/W2 Resid") ps.savefig()
def main(): import argparse parser = argparse.ArgumentParser( description='This script creates small self-contained data sets that ' 'are useful for test cases of the pipeline codes.') parser.add_argument('ccds', help='CCDs table describing region to grab') parser.add_argument('outdir', help='Output directory name') parser.add_argument('brick', help='Brick containing these images') parser.add_argument('--survey-dir', type=str, default=None) parser.add_argument('--cache-dir', type=str, default=None, help='Directory to search for cached files') parser.add_argument('--wise', help='For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image') parser.add_argument('--wise-wcs-hdu', help='For WISE outputs, the HDU to read the WCS from in the file given by --wise.', type=int, default=0) parser.add_argument('--fpack', action='store_true', default=False) parser.add_argument('--gzip', action='store_true', default=False) parser.add_argument('--pad', action='store_true', default=False, help='Keep original image size, but zero out pixels outside ROI') args = parser.parse_args() v = 'SKY_TEMPLATE_DIR' if v in os.environ: del os.environ[v] C = fits_table(args.ccds) print(len(C), 'CCDs in', args.ccds) C.camera = np.array([c.strip() for c in C.camera]) survey = LegacySurveyData(cache_dir=args.cache_dir, survey_dir=args.survey_dir) if ',' in args.brick: ra,dec = args.brick.split(',') ra = float(ra) dec = float(dec) fakebricks = fits_table() fakebricks.brickname = np.array([('custom-%06i%s%05i' % (int(1000*ra), 'm' if dec < 0 else 'p', int(1000*np.abs(dec))))]) fakebricks.ra = np.array([ra]) fakebricks.dec = np.array([dec]) bricks = fakebricks outbricks = bricks else: bricks = survey.get_bricks_readonly() outbricks = bricks[np.array([n == args.brick for n in bricks.brickname])] assert(len(outbricks) == 1) outsurvey = LegacySurveyData(survey_dir = args.outdir) trymakedirs(args.outdir) outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz')) targetwcs = wcs_for_brick(outbricks[0]) H,W = targetwcs.shape tycho2fn = survey.find_file('tycho2') kd = tree_open(tycho2fn, 'stars') radius = 1. rc,dc = targetwcs.radec_center() I = tree_search_radec(kd, rc, dc, radius) print(len(I), 'Tycho-2 stars within', radius, 'deg of RA,Dec (%.3f, %.3f)' % (rc,dc)) # Read only the rows within range. tycho = fits_table(tycho2fn, rows=I) del kd print('Read', len(tycho), 'Tycho-2 stars') ok,tx,ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec) #margin = 100 #tycho.cut(ok * (tx > -margin) * (tx < W+margin) * # (ty > -margin) * (ty < H+margin)) print('Cut to', len(tycho), 'Tycho-2 stars within brick') del ok,tx,ty #tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz')) f,tfn = tempfile.mkstemp(suffix='.fits') os.close(f) tycho.writeto(tfn) outfn = os.path.join(args.outdir, 'tycho2.kd.fits') cmd = 'startree -i %s -o %s -P -k -n stars -T' % (tfn, outfn) print(cmd) rtn = os.system(cmd) assert(rtn == 0) os.unlink(tfn) from legacypipe.gaiacat import GaiaCatalog gcat = GaiaCatalog() # from ps1cat.py: wcs = targetwcs step=100. margin=10. # Grid the CCD in pixel space W,H = wcs.get_width(), wcs.get_height() xx,yy = np.meshgrid( np.linspace(1-margin, W+margin, 2+int((W+2*margin)/step)), np.linspace(1-margin, H+margin, 2+int((H+2*margin)/step))) # Convert to RA,Dec and then to unique healpixes ra,dec = wcs.pixelxy2radec(xx.ravel(), yy.ravel()) healpixes = set() for r,d in zip(ra,dec): healpixes.add(gcat.healpix_for_radec(r, d)) for hp in healpixes: hpcat = gcat.get_healpix_catalog(hp) ok,xx,yy = wcs.radec2pixelxy(hpcat.ra, hpcat.dec) onccd = np.flatnonzero((xx >= 1.-margin) * (xx <= W+margin) * (yy >= 1.-margin) * (yy <= H+margin)) hpcat.cut(onccd) if len(hpcat): outfn = os.path.join(args.outdir, 'gaia', 'chunk-%05d.fits' % hp) trymakedirs(os.path.join(args.outdir, 'gaia')) hpcat.writeto(outfn) outccds = C.copy() cols = outccds.get_columns() for c in ['ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1', 'brick_x0', 'brick_x1', 'brick_y0', 'brick_y1', 'skyver', 'wcsver', 'psfver', 'skyplver', 'wcsplver', 'psfplver' ]: if c in cols: outccds.delete_column(c) outccds.image_hdu[:] = 1 # Convert to list to avoid truncating filenames outccds.image_filename = [fn for fn in outccds.image_filename] for iccd,ccd in enumerate(C): decam = (ccd.camera.strip() == 'decam') bok = (ccd.camera.strip() == '90prime') im = survey.get_image_object(ccd) print('Got', im) if survey.cache_dir is not None: im.check_for_cached_files(survey) slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1)) psfkwargs = dict(pixPsf=True, gaussPsf=False, hybridPsf=False, normalizePsf=False) tim = im.get_tractor_image(slc, pixPsf=True, subsky=False, nanomaggies=False, no_remap_invvar=True, old_calibs_ok=True) print('Tim:', tim.shape) psfrow = psfhdr = None if args.pad: psf = im.read_psf_model(0, 0, w=im.width, h=im.height, **psfkwargs) psfex = psf.psfex else: psf = tim.getPsf() psfex = psf.psfex # Did the PSF model come from a merged file? for fn in [im.merged_psffn, im.psffn] + im.old_merged_psffns: if not os.path.exists(fn): continue T = fits_table(fn) I, = np.nonzero((T.expnum == im.expnum) * np.array([c.strip() == im.ccdname for c in T.ccdname])) if len(I) != 1: continue psfrow = T[I] x0 = ccd.ccd_x0 y0 = ccd.ccd_y0 psfrow.polzero1[0] -= x0 psfrow.polzero2[0] -= y0 #psfhdr = fitsio.read_header(im.merged_psffn) break psfex.fwhm = tim.psf_fwhm #### HACK #psfrow = None assert(psfrow is not None) if psfrow is not None: print('PSF row:', psfrow) #else: # print('PSF:', psf) # print('PsfEx:', psfex) skyrow = skyhdr = None if args.pad: primhdr = fitsio.read_header(im.imgfn) imghdr = fitsio.read_header(im.imgfn, hdu=im.hdu) sky = im.read_sky_model(splinesky=True, primhdr=primhdr, imghdr=imghdr) #skyhdr = fitsio.read_header(im.splineskyfn) #msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True) else: sky = tim.getSky() # Did the sky model come from a merged file? #msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True) print('merged skyfn:', im.merged_skyfn) print('single skyfn:', im.skyfn) print('old merged skyfns:', im.old_merged_skyfns) for fn in [im.merged_skyfn, im.skyfn] + im.old_merged_skyfns: if not os.path.exists(fn): continue T = fits_table(fn) I, = np.nonzero((T.expnum == im.expnum) * np.array([c.strip() == im.ccdname for c in T.ccdname])) skyrow = T[I] skyrow.x0[0] = ccd.ccd_x0 skyrow.y0[0] = ccd.ccd_y0 # s_med = skyrow.sky_med[0] # s_john = skyrow.sky_john[0] # skyhdr = fitsio.read_header(fn) assert(skyrow is not None) ### HACK #skyrow = None if skyrow is not None: print('Sky row:', skyrow) else: print('Sky:', sky) # Output filename format: fn = ccd.image_filename.strip() ccd.image_filename = os.path.join(os.path.dirname(fn), '%s.%s.fits' % (os.path.basename(fn).split('.')[0], ccd.ccdname.strip())) outim = outsurvey.get_image_object(ccd) print('Output image:', outim) print('Image filename:', outim.imgfn) trymakedirs(outim.imgfn, dir=True) imgdata = tim.getImage() ivdata = tim.getInvvar() # Since we remap DQ codes (always with Mosaic and Bok, sometimes with DECam), # re-read from the FITS file rather than using tim.dq. print('Reading data quality from', im.dqfn, 'hdu', im.hdu) dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice) print('Tim shape:', tim.shape, 'Slice', tim.slice) print('image shape:', imgdata.shape, 'iv', ivdata.shape, 'DQ', dqdata.shape) from collections import Counter dqvals = Counter(dqdata.ravel()) print('DQ pixel counts:') for k,n in dqvals.most_common(): print(' 0x%x' % k, ':', n) if args.pad: # Create zero image of full size, copy in data. fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype) fullsize[slc] = imgdata imgdata = fullsize fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype) fullsize[slc] = dqdata dqdata = fullsize fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype) fullsize[slc] = ivdata ivdata = fullsize else: # Adjust the header WCS by x0,y0 crpix1 = tim.hdr['CRPIX1'] crpix2 = tim.hdr['CRPIX2'] tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0 tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0 # Add image extension to filename # fitsio doesn't compress .fz by default, so drop .fz suffix #outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits') if args.gzip: outim.imgfn = outim.imgfn.replace('.fits', '.fits.gz') #outim.wtfn = outim.wtfn.replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.wtfn = outim.wtfn.replace('.fits.fz', '.fits') if args.gzip: outim.wtfn = outim.wtfn.replace('.fits', '.fits.gz') if outim.dqfn is not None: #outim.dqfn = outim.dqfn.replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.dqfn = outim.dqfn.replace('.fits.fz', '.fits') if args.gzip: outim.dqfn = outim.dqfn.replace('.fits', '.fits.gz') if bok: outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname) ccdfn = outim.imgfn ccdfn = ccdfn.replace(outsurvey.get_image_dir(), '') if ccdfn.startswith('/'): ccdfn = ccdfn[1:] outccds.image_filename[iccd] = ccdfn print('Changed output filenames to:') print(outim.imgfn) print(outim.dqfn) ofn = outim.imgfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fits = fitsio.FITS(ofn, 'rw', clobber=True) fits.write(None, header=tim.primhdr) fits.write(imgdata, header=tim.hdr, extname=ccd.ccdname) fits.close() if args.fpack: cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) h,w = tim.shape if not args.pad: outccds.width[iccd] = w outccds.height[iccd] = h outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0 outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0 wcs = Tan(*[float(x) for x in [ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2, ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height]]) if args.pad: subwcs = wcs else: subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h) outccds.ra[iccd],outccds.dec[iccd] = subwcs.radec_center() print('Weight filename:', outim.wtfn) wfn = outim.wtfn trymakedirs(wfn, dir=True) ofn = wfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fits = fitsio.FITS(ofn, 'rw', clobber=True) fits.write(None, header=tim.primhdr) fits.write(ivdata, header=tim.hdr, extname=ccd.ccdname) fits.close() if args.fpack: cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) if outim.dqfn is not None: print('DQ filename', outim.dqfn) trymakedirs(outim.dqfn, dir=True) ofn = outim.dqfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fits = fitsio.FITS(ofn, 'rw', clobber=True) fits.write(None, header=tim.primhdr) fits.write(dqdata, header=tim.hdr, extname=ccd.ccdname) fits.close() if args.fpack: cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) psfout = outim.psffn #if psfrow: # psfout = outim.merged_psffn print('PSF output filename:', psfout) trymakedirs(psfout, dir=True) if psfrow: psfrow.writeto(psfout, primhdr=psfhdr) else: print('Writing PsfEx:', psfout) psfex.writeto(psfout) # update header F = fitsio.FITS(psfout, 'rw') F[0].write_keys([dict(name='EXPNUM', value=ccd.expnum), dict(name='PLVER', value=psf.plver), dict(name='PROCDATE', value=psf.procdate), dict(name='PLPROCID', value=psf.plprocid),]) F.close() skyout = outim.skyfn #if skyrow: # skyout = outim.merged_splineskyfn print('Sky output filename:', skyout) trymakedirs(skyout, dir=True) if skyrow is not None: skyrow.writeto(skyout, primhdr=skyhdr) else: primhdr = fitsio.FITSHDR() primhdr['PLVER'] = sky.plver primhdr['PLPROCID'] = sky.plprocid primhdr['PROCDATE'] = sky.procdate primhdr['EXPNUM'] = ccd.expnum primhdr['IMGDSUM'] = sky.datasum primhdr['S_MED'] = s_med primhdr['S_JOHN'] = s_john sky.write_fits(skyout, primhdr=primhdr) # HACK -- check result immediately. outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz')) outsurvey.ccds = None outC = outsurvey.get_ccds_readonly() occd = outC[iccd] outim = outsurvey.get_image_object(occd) print('Got output image:', outim) otim = outim.get_tractor_image(pixPsf=True, hybridPsf=True, old_calibs_ok=True) print('Got output tim:', otim) outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz')) # WISE if args.wise is not None: from wise.forcedphot import unwise_tiles_touching_wcs from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs, get_unwise_tractor_image, get_unwise_tile_dir) # Read WCS... print('Reading TAN wcs header from', args.wise, 'HDU', args.wise_wcs_hdu) targetwcs = Tan(args.wise, args.wise_wcs_hdu) tiles = unwise_tiles_touching_wcs(targetwcs) print('Cut to', len(tiles), 'unWISE tiles') H,W = targetwcs.shape r,d = targetwcs.pixelxy2radec(np.array([1, W, W/2, W/2]), np.array([H/2, H/2, 1, H ])) roiradec = [r[0], r[1], d[2], d[3]] unwise_dir = os.environ['UNWISE_COADDS_DIR'] wise_out = os.path.join(args.outdir, 'images', 'unwise') print('Will write WISE outputs to', wise_out) unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR'] wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr') print('Will write WISE time-resolved outputs to', wise_tr_out) trymakedirs(wise_tr_out) W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_atlas.fits')) print('Read', len(W), 'time-resolved WISE coadd tiles') W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id])) print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth') # Write the time-resolved index subset. W.writeto(os.path.join(wise_tr_out, 'time_resolved_atlas.fits')) # this ought to be enough for anyone =) _,Nepochs = W.epoch_bitmask.shape print('N epochs in time-resolved atlas:', Nepochs) wisedata = [] # full depth for band in [1,2,3,4]: wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band, True)) # time-resolved for band in [1,2]: # W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2) bitmask = (1 << (band-1)) for e in range(Nepochs): # Which tiles have images for this epoch? I = np.flatnonzero(W.epoch_bitmask[:,e] & bitmask) if len(I) == 0: continue print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I]) edir = os.path.join(unwise_tr_dir, 'e%03i' % e) eoutdir = os.path.join(wise_tr_out, 'e%03i' % e) wisedata.append((edir, eoutdir, tiles.coadd_id[I], band, False)) wrote_masks = set() model_dir = os.environ.get('UNWISE_MODEL_SKY_DIR') if model_dir is not None: model_dir_out = os.path.join(args.outdir, 'images', 'unwise-mod') trymakedirs(model_dir_out) for indir, outdir, tiles, band, fulldepth in wisedata: for tile in tiles: wanyband = 'w' tim = get_unwise_tractor_image(indir, tile, band, bandname=wanyband, roiradecbox=roiradec) print('Got unWISE tim', tim) print(tim.shape) if model_dir is not None and fulldepth and band in [1,2]: print('ROI', tim.roi) #0387p575.1.mod.fits fn = '%s.%i.mod.fits' % (tile, band) print('Filename', fn) F = fitsio.FITS(os.path.join(model_dir, fn)) x0,x1,y0,y1 = tim.roi slc = slice(y0,y1),slice(x0,x1) phdr = F[0].read_header() outfn = os.path.join(model_dir_out, fn) for e,extname in [(1,'MODEL'), (2,'SKY')]: pix = F[e][slc] hdr = F[e].read_header() crpix1 = hdr['CRPIX1'] crpix2 = hdr['CRPIX2'] hdr['CRPIX1'] -= x0 hdr['CRPIX2'] -= y0 #print('mod', mod) #print('Model', mod.shape) if e == 1: fitsio.write(outfn, None, clobber=True, header=phdr) fitsio.write(outfn, pix, header=hdr, extname=extname) print('Wrote', outfn) thisdir = get_unwise_tile_dir(outdir, tile) print('Directory for this WISE tile:', thisdir) base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band)) print('Base filename:', base) masked = True mu = 'm' if masked else 'u' imfn = base + 'img-%s.fits' % mu ivfn = base + 'invvar-%s.fits.gz' % mu nifn = base + 'n-%s.fits.gz' % mu nufn = base + 'n-u.fits.gz' #print('WISE image header:', tim.hdr) # Adjust the header WCS by x0,y0 wcs = tim.wcs.wcs tim.hdr['CRPIX1'] = wcs.crpix[0] tim.hdr['CRPIX2'] = wcs.crpix[1] H,W = tim.shape tim.hdr['IMAGEW'] = W tim.hdr['IMAGEH'] = H print('WCS:', wcs) print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2']) trymakedirs(imfn, dir=True) fitsio.write(imfn, tim.getImage(), header=tim.hdr, clobber=True) print('Wrote', imfn) fitsio.write(ivfn, tim.getInvvar(), header=tim.hdr, clobber=True) print('Wrote', ivfn) fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True) print('Wrote', nifn) fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True) print('Wrote', nufn) if not (indir,tile) in wrote_masks: print('Looking for mask file for', indir, tile) # record that we tried this dir/tile combo wrote_masks.add((indir,tile)) for idir in indir.split(':'): tdir = get_unwise_tile_dir(idir, tile) maskfn = 'unwise-%s-msk.fits.gz' % tile fn = os.path.join(tdir, maskfn) print('Mask file:', fn) if os.path.exists(fn): print('Reading', fn) (x0,x1,y0,y1) = tim.roi roislice = (slice(y0,y1), slice(x0,x1)) F = fitsio.FITS(fn)[0] hdr = F.read_header() M = F[roislice] outfn = os.path.join(thisdir, maskfn) fitsio.write(outfn, M, header=tim.hdr, clobber=True) print('Wrote', outfn) break outC = outsurvey.get_ccds_readonly() for iccd,ccd in enumerate(outC): outim = outsurvey.get_image_object(ccd) print('Got output image:', outim) otim = outim.get_tractor_image(pixPsf=True, hybridPsf=True, old_calibs_ok=True) print('Got output tim:', otim)
def unwise_forcedphot(cat, tiles, band=1, roiradecbox=None, use_ceres=True, ceres_block=8, save_fits=False, get_models=False, ps=None, psf_broadening=None, pixelized_psf=False, get_masks=None, move_crpix=False, modelsky_dir=None): ''' Given a list of tractor sources *cat* and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id) runs forced photometry, returning a FITS table the same length as *cat*. *get_masks*: the WCS to resample mask bits into. ''' from tractor import NanoMaggies, PointSource, Tractor, ExpGalaxy, DevGalaxy, FixedCompositeGalaxy if not pixelized_psf and psf_broadening is None: # PSF broadening in post-reactivation data, by band. # Newer version from Aaron's email to decam-chatter, 2018-06-14. broadening = { 1: 1.0405, 2: 1.0346, 3: None, 4: None } psf_broadening = broadening[band] if False: from astrometry.util.plotutils import PlotSequence ps = PlotSequence('wise-forced-w%i' % band) plots = (ps is not None) if plots: import pylab as plt wantims = (plots or save_fits or get_models) wanyband = 'w' if get_models: models = {} wband = 'w%i' % band fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix', 'pronexp'] Nsrcs = len(cat) phot = fits_table() # Filled in based on unique tile overlap phot.wise_coadd_id = np.array([' '] * Nsrcs) phot.set(wband + '_psfdepth', np.zeros(len(phot), np.float32)) ra = np.array([src.getPosition().ra for src in cat]) dec = np.array([src.getPosition().dec for src in cat]) nexp = np.zeros(Nsrcs, np.int16) mjd = np.zeros(Nsrcs, np.float64) central_flux = np.zeros(Nsrcs, np.float32) fitstats = {} tims = [] if get_masks: mh,mw = get_masks.shape maskmap = np.zeros((mh,mw), np.uint32) for tile in tiles: print('Reading WISE tile', tile.coadd_id, 'band', band) tim = get_unwise_tractor_image(tile.unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradecbox) if tim is None: print('Actually, no overlap with tile', tile.coadd_id) continue if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data' % tag) ps.savefig() plt.clf() plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), range=(-5,10), bins=100) plt.xlabel('Per-pixel intensity (Sigma)') plt.title(tag) ps.savefig() if move_crpix and band in [1, 2]: realwcs = tim.wcs.wcs x,y = realwcs.crpix tile_crpix = tile.get('crpix_w%i' % band) dx = tile_crpix[0] - 1024.5 dy = tile_crpix[1] - 1024.5 realwcs.set_crpix(x+dx, y+dy) #print('CRPIX', x,y, 'shift by', dx,dy, 'to', realwcs.crpix) if modelsky_dir and band in [1, 2]: fn = os.path.join(modelsky_dir, '%s.%i.mod.fits' % (tile.coadd_id, band)) if not os.path.exists(fn): raise RuntimeError('WARNING: does not exist:', fn) x0,x1,y0,y1 = tim.roi bg = fitsio.FITS(fn)[2][y0:y1, x0:x1] #print('Read background map:', bg.shape, bg.dtype, 'vs image', tim.shape) if plots: plt.clf() plt.subplot(1,2,1) plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) plt.subplot(1,2,2) plt.imshow(bg, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) tag = '%s W%i' % (tile.coadd_id, band) plt.suptitle(tag) ps.savefig() plt.clf() ha = dict(range=(-5,10), bins=100, histtype='step') plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), color='b', label='Original', **ha) plt.hist(((tim.getImage()-bg) * tim.inverr)[tim.inverr > 0].ravel(), color='g', label='Minus Background', **ha) plt.axvline(0, color='k', alpha=0.5) plt.xlabel('Per-pixel intensity (Sigma)') plt.legend() plt.title(tag + ': background') ps.savefig() # Actually subtract the background! tim.data -= bg # Floor the per-pixel variances if band in [1,2]: # in Vega nanomaggies per pixel floor_sigma = {1: 0.5, 2: 2.0} with np.errstate(divide='ignore'): new_ie = 1. / np.hypot(1./tim.inverr, floor_sigma[band]) new_ie[tim.inverr == 0] = 0. if plots: plt.clf() plt.plot((1. / tim.inverr[tim.inverr>0]).ravel(), (1./new_ie[tim.inverr>0]).ravel(), 'b.') plt.title('unWISE per-pixel error: %s band %i' % (tile.coadd_id, band)) plt.xlabel('original') plt.ylabel('floored') ps.savefig() tim.inverr = new_ie # Read mask file? if get_masks: from astrometry.util.resample import resample_with_wcs, OverlapError # unwise_dir can be a colon-separated list of paths tilemask = None for d in tile.unwise_dir.split(':'): fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id, 'unwise-%s-msk.fits.gz' % tile.coadd_id) if os.path.exists(fn): print('Reading unWISE mask file', fn) x0,x1,y0,y1 = tim.roi tilemask = fitsio.FITS(fn)[0][y0:y1,x0:x1] break if tilemask is None: print('unWISE mask file for tile', tile.coadd_id, 'does not exist') else: try: tanwcs = tim.wcs.wcs assert(tanwcs.shape == tilemask.shape) Yo,Xo,Yi,Xi,_ = resample_with_wcs(get_masks, tanwcs, intType=np.int16) # Only deal with mask pixels that are set. I, = np.nonzero(tilemask[Yi,Xi] > 0) # Trim to unique area for this tile rr,dd = get_masks.pixelxy2radec(Yo[I]+1, Xo[I]+1) good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I = I[good] maskmap[Yo[I],Xo[I]] = tilemask[Yi[I], Xi[I]] except OverlapError: # Shouldn't happen by this point print('No overlap between WISE tile', tile.coadd_id, 'and brick') # The tiles have some overlap, so zero out pixels outside the # tile's unique area. th,tw = tim.shape xx,yy = np.meshgrid(np.arange(tw), np.arange(th)) rr,dd = tim.wcs.wcs.pixelxy2radec(xx+1, yy+1) unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) #print(np.sum(unique), 'of', (th*tw), 'pixels in this tile are unique') tim.inverr[unique == False] = 0. del xx,yy,rr,dd,unique if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage() * (tim.inverr > 0), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data (unique)' % tag) ps.savefig() if pixelized_psf: import unwise_psf if (band == 1) or (band == 2): # we only have updated PSFs for W1 and W2 psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id, modelname='neo4_unwisecat') else: psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id) if band == 4: # oversample (the unwise_psf models are at native W4 5.5"/pix, # while the unWISE coadds are made at 2.75"/pix. ph,pw = psfimg.shape subpsf = np.zeros((ph*2-1, pw*2-1), np.float32) from astrometry.util.util import lanczos3_interpolate xx,yy = np.meshgrid(np.arange(0., pw-0.51, 0.5, dtype=np.float32), np.arange(0., ph-0.51, 0.5, dtype=np.float32)) xx = xx.ravel() yy = yy.ravel() ix = xx.astype(np.int32) iy = yy.astype(np.int32) dx = (xx - ix).astype(np.float32) dy = (yy - iy).astype(np.float32) psfimg = psfimg.astype(np.float32) rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat], [psfimg]) if plots: plt.clf() plt.imshow(psfimg, interpolation='nearest', origin='lower') plt.title('Original PSF model') ps.savefig() plt.clf() plt.imshow(subpsf, interpolation='nearest', origin='lower') plt.title('Subsampled PSF model') ps.savefig() psfimg = subpsf del xx, yy, ix, iy, dx, dy from tractor.psf import PixelizedPSF psfimg /= psfimg.sum() fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0} psfimg *= fluxrescales[band] tim.psf = PixelizedPSF(psfimg) if psf_broadening is not None and not pixelized_psf: # psf_broadening is a factor by which the PSF FWHMs # should be scaled; the PSF is a little wider # post-reactivation. psf = tim.getPsf() from tractor import GaussianMixturePSF if isinstance(psf, GaussianMixturePSF): # print('Broadening PSF: from', psf) p0 = psf.getParams() pnames = psf.getParamNames() p1 = [p * psf_broadening**2 if 'var' in name else p for (p, name) in zip(p0, pnames)] psf.setParams(p1) print('Broadened PSF:', psf) else: print('WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf)) wcs = tim.wcs.wcs ok,x,y = wcs.radec2pixelxy(ra, dec) x = np.round(x - 1.).astype(int) y = np.round(y - 1.).astype(int) good = (x >= 0) * (x < tw) * (y >= 0) * (y < th) # Which sources are in this brick's unique area? usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I, = np.nonzero(good * usrc) nexp[I] = tim.nuims[y[I], x[I]] if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'): mjd[I] = (tim.mjdmin + tim.mjdmax) / 2. phot.wise_coadd_id[I] = tile.coadd_id central_flux[I] = tim.getImage()[y[I], x[I]] del x,y,good,usrc # PSF norm for depth psf = tim.getPsf() h,w = tim.shape patch = psf.getPointSourcePatch(h//2, w//2).patch psfnorm = np.sqrt(np.sum(patch**2)) # To handle zero-depth, we return 1/nanomaggies^2 units rather than mags. psfdepth = 1. / (tim.sig1 / psfnorm)**2 phot.get(wband + '_psfdepth')[I] = psfdepth tim.tile = tile tims.append(tim) if plots: plt.clf() mn,mx = 0.1, 20000 plt.hist(np.log10(np.clip(central_flux, mn, mx)), bins=100, range=(np.log10(mn), np.log10(mx))) logt = np.arange(0, 5) plt.xticks(logt, ['%i' % i for i in 10.**logt]) plt.title('Central fluxes (W%i)' % band) plt.axvline(np.log10(20000), color='k') plt.axvline(np.log10(1000), color='k') ps.savefig() # Eddie's non-secret recipe: #- central pixel <= 1000: 19x19 pix box size #- central pixel in 1000 - 20000: 59x59 box size #- central pixel > 20000 or saturated: 149x149 box size #- object near "bright star": 299x299 box size nbig = nmedium = nsmall = 0 for src,cflux in zip(cat, central_flux): if cflux > 20000: R = 100 nbig += 1 elif cflux > 1000: R = 30 nmedium += 1 else: R = 15 nsmall += 1 if isinstance(src, PointSource): src.fixedRadius = R else: ### FIXME -- sizes for galaxies..... can we set PSF size separately? galrad = 0 # RexGalaxy is a subclass of ExpGalaxy if isinstance(src, (ExpGalaxy, DevGalaxy)): galrad = src.shape.re elif isinstance(src, FixedCompositeGalaxy): galrad = max(src.shapeExp.re, src.shapeDev.re) pixscale = 2.75 src.halfsize = int(np.hypot(R, galrad * 5 / pixscale)) #print('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall, 'small') minsb = 0. fitsky = False tractor = Tractor(tims, cat) if use_ceres: from tractor.ceres_optimizer import CeresOptimizer tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block) tractor.freezeParamsRecursive('*') tractor.thawPathsTo(wanyband) kwa = dict(fitstat_extras=[('pronexp', [tim.nims for tim in tims])]) t0 = Time() R = tractor.optimize_forced_photometry( minsb=minsb, mindlnp=1., sky=fitsky, fitstats=True, variance=True, shared_params=False, wantims=wantims, **kwa) print('unWISE forced photometry took', Time() - t0) if use_ceres: term = R.ceres_status['termination'] # Running out of memory can cause failure to converge # and term status = 2. # Fail completely in this case. if term != 0: print('Ceres termination status:', term) raise RuntimeError( 'Ceres terminated with status %i' % term) if wantims: ims1 = R.ims1 flux_invvars = R.IV if R.fitstats is not None: for k in fskeys: x = getattr(R.fitstats, k) fitstats[k] = np.array(x).astype(np.float32) if save_fits: for i,tim in enumerate(tims): tile = tim.tile (dat, mod, ie, chi, roi) = ims1[i] wcshdr = fitsio.FITSHDR() tim.wcs.wcs.add_to_header(wcshdr) tag = 'fit-%s-w%i' % (tile.coadd_id, band) fitsio.write('%s-data.fits' % tag, dat, clobber=True, header=wcshdr) fitsio.write('%s-mod.fits' % tag, mod, clobber=True, header=wcshdr) fitsio.write('%s-chi.fits' % tag, chi, clobber=True, header=wcshdr) if plots: # Create models for just the brightest sources bright_cat = [src for src in cat if src.getBrightness().getBand(wanyband) > 1000] print('Bright soures:', len(bright_cat)) btr = Tractor(tims, bright_cat) for tim in tims: mod = btr.getModelImage(tim) tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) sig1 = tim.sig1 plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: bright-star models' % tag) ps.savefig() if get_models: for i,tim in enumerate(tims): tile = tim.tile (dat, mod, ie, chi, roi) = ims1[i] models[(tile.coadd_id, band)] = (mod, dat, ie, tim.roi, tim.wcs.wcs) if plots: for i,tim in enumerate(tims): tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) (dat, mod, ie, chi, roi) = ims1[i] sig1 = tim.sig1 plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() nm = np.array([src.getBrightness().getBand(wanyband) for src in cat]) nm_ivar = flux_invvars # Sources out of bounds, eg, never change from their default # (1-sigma or whatever) initial fluxes. Zero them out instead. nm[nm_ivar == 0] = 0. phot.set(wband + '_nanomaggies', nm.astype(np.float32)) phot.set(wband + '_nanomaggies_ivar', nm_ivar.astype(np.float32)) dnm = np.zeros(len(nm_ivar), np.float32) okiv = (nm_ivar > 0) dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32) okflux = (nm > 0) mag = np.zeros(len(nm), np.float32) mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux]) ).astype(np.float32) dmag = np.zeros(len(nm), np.float32) ok = (okiv * okflux) dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok]) ).astype(np.float32) mag[np.logical_not(okflux)] = np.nan dmag[np.logical_not(ok)] = np.nan phot.set(wband + '_mag', mag) phot.set(wband + '_mag_err', dmag) for k in fskeys: phot.set(wband + '_' + k, fitstats[k]) phot.set(wband + '_nexp', nexp) if not np.all(mjd == 0): phot.set(wband + '_mjd', mjd) rtn = wphotduck() rtn.phot = phot rtn.models = None rtn.maskmap = None if get_models: rtn.models = models if get_masks: rtn.maskmap = maskmap return rtn
def wise_cutouts(ra, dec, radius, ps, pixscale=2.75, survey_dir=None, unwise_dir=None): ''' radius in arcsec. pixscale: WISE pixel scale in arcsec/pixel; make this smaller than 2.75 to oversample. ''' if unwise_dir is None: unwise_dir = os.environ.get('UNWISE_COADDS_DIR') npix = int(np.ceil(radius / pixscale)) print('Image size:', npix) W = H = npix pix = pixscale / 3600. wcs = Tan(ra, dec, (W + 1) / 2., (H + 1) / 2., -pix, 0., 0., pix, float(W), float(H)) # Find DECaLS bricks overlapping survey = LegacySurveyData(survey_dir=survey_dir) B = bricks_touching_wcs(wcs, survey=survey) print('Found', len(B), 'bricks overlapping') TT = [] for b in B.brickname: fn = survey.find_file('tractor', brick=b) T = fits_table(fn) print('Read', len(T), 'from', b) primhdr = fitsio.read_header(fn) TT.append(T) T = merge_tables(TT) print('Total of', len(T), 'sources') T.cut(T.brick_primary) print(len(T), 'primary') margin = 20 ok, xx, yy = wcs.radec2pixelxy(T.ra, T.dec) I = np.flatnonzero((xx > -margin) * (yy > -margin) * (xx < W + margin) * (yy < H + margin)) T.cut(I) print(len(T), 'within ROI') #return wcs,T # Pull out DECaLS coadds (image, model, resid). dwcs = wcs.scale(2. * pixscale / 0.262) dh, dw = dwcs.shape print('DECaLS resampled shape:', dh, dw) tags = ['image', 'model', 'resid'] coimgs = [np.zeros((dh, dw, 3), np.uint8) for t in tags] for b in B.brickname: fn = survey.find_file('image', brick=b, band='r') bwcs = Tan(fn, 1) # ext 1: .fz try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(dwcs, bwcs) except ResampleError: continue if len(Yo) == 0: continue print('Resampling', len(Yo), 'pixels from', b) xl, xh, yl, yh = Xi.min(), Xi.max(), Yi.min(), Yi.max() #print('python legacypipe/runbrick.py -b %s --zoom %i %i %i %i --outdir cluster --pixpsf --splinesky --pipe --no-early-coadds' % # (b, xl-5, xh+5, yl-5, yh+5) + ' -P \'pickles/cluster-%(brick)s-%%(stage)s.pickle\'') for i, tag in enumerate(tags): fn = survey.find_file(tag + '-jpeg', brick=b) img = plt.imread(fn) img = np.flipud(img) coimgs[i][Yo, Xo, :] = img[Yi, Xi, :] tt = dict(image='Image', model='Model', resid='Resid') for img, tag in zip(coimgs, tags): plt.clf() dimshow(img, ticks=False) plt.title('DECaLS grz %s' % tt[tag]) ps.savefig() # Find unWISE tiles overlapping tiles = unwise_tiles_touching_wcs(wcs) print('Cut to', len(tiles), 'unWISE tiles') # Here we assume the targetwcs is axis-aligned and that the # edge midpoints yield the RA,Dec limits (true for TAN). r, d = wcs.pixelxy2radec(np.array([1, W, W / 2, W / 2]), np.array([H / 2, H / 2, 1, H])) # the way the roiradec box is used, the min/max order doesn't matter roiradec = [r[0], r[1], d[2], d[3]] ra, dec = T.ra, T.dec srcs = read_fits_catalog(T) wbands = [1, 2, 3, 4] wanyband = 'w' for band in wbands: f = T.get('flux_w%i' % band) f *= 10.**(primhdr['WISEAB%i' % band] / 2.5) coimgs = [np.zeros((H, W), np.float32) for b in wbands] comods = [np.zeros((H, W), np.float32) for b in wbands] con = [np.zeros((H, W), np.uint8) for b in wbands] for iband, band in enumerate(wbands): print('Photometering WISE band', band) wband = 'w%i' % band for i, src in enumerate(srcs): #print('Source', src, 'brightness', src.getBrightness(), 'params', src.getBrightness().getParams()) #src.getBrightness().setParams([T.wise_flux[i, band-1]]) src.setBrightness( NanoMaggies(**{wanyband: T.get('flux_w%i' % band)[i]})) # print('Set source brightness:', src.getBrightness()) # The tiles have some overlap, so for each source, keep the # fit in the tile whose center is closest to the source. for tile in tiles: print('Reading tile', tile.coadd_id) tim = get_unwise_tractor_image(unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradec) if tim is None: print('Actually, no overlap with tile', tile.coadd_id) continue print('Read image with shape', tim.shape) # Select sources in play. wisewcs = tim.wcs.wcs H, W = tim.shape ok, x, y = wisewcs.radec2pixelxy(ra, dec) x = (x - 1.).astype(np.float32) y = (y - 1.).astype(np.float32) margin = 10. I = np.flatnonzero((x >= -margin) * (x < W + margin) * (y >= -margin) * (y < H + margin)) print(len(I), 'within the image + margin') subcat = [srcs[i] for i in I] tractor = Tractor([tim], subcat) mod = tractor.getModelImage(0) # plt.clf() # dimshow(tim.getImage(), ticks=False) # plt.title('WISE %s %s' % (tile.coadd_id, wband)) # ps.savefig() # plt.clf() # dimshow(mod, ticks=False) # plt.title('WISE %s %s' % (tile.coadd_id, wband)) # ps.savefig() try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(wcs, tim.wcs.wcs) except ResampleError: continue if len(Yo) == 0: continue print('Resampling', len(Yo), 'pixels from WISE', tile.coadd_id, band) coimgs[iband][Yo, Xo] += tim.getImage()[Yi, Xi] comods[iband][Yo, Xo] += mod[Yi, Xi] con[iband][Yo, Xo] += 1 for img, mod, n in zip(coimgs, comods, con): img /= np.maximum(n, 1) mod /= np.maximum(n, 1) for band, img, mod in zip(wbands, coimgs, comods): lo, hi = np.percentile(img, [25, 99]) plt.clf() dimshow(img, vmin=lo, vmax=hi, ticks=False) plt.title('WISE W%i Data' % band) ps.savefig() plt.clf() dimshow(mod, vmin=lo, vmax=hi, ticks=False) plt.title('WISE W%i Model' % band) ps.savefig() resid = img - mod mx = np.abs(resid).max() plt.clf() dimshow(resid, vmin=-mx, vmax=mx, ticks=False) plt.title('WISE W%i Resid' % band) ps.savefig() #kwa = dict(mn=-0.1, mx=2., arcsinh = 1.) kwa = dict(mn=-0.1, mx=2., arcsinh=None) rgb = _unwise_to_rgb(coimgs[:2], **kwa) plt.clf() dimshow(rgb, ticks=False) plt.title('WISE W1/W2 Data') ps.savefig() rgb = _unwise_to_rgb(comods[:2], **kwa) plt.clf() dimshow(rgb, ticks=False) plt.title('WISE W1/W2 Model') ps.savefig() kwa = dict(mn=-1, mx=1, arcsinh=None) rgb = _unwise_to_rgb( [img - mod for img, mod in list(zip(coimgs, comods))[:2]], **kwa) plt.clf() dimshow(rgb, ticks=False) plt.title('WISE W1/W2 Resid') ps.savefig() return wcs, T
def unwise_forcedphot(cat, tiles, band=1, roiradecbox=None, use_ceres=True, ceres_block=8, save_fits=False, get_models=False, ps=None, psf_broadening=None, pixelized_psf=False, get_masks=None, move_crpix=False, modelsky_dir=None, tag=None): ''' Given a list of tractor sources *cat* and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id) runs forced photometry, returning a FITS table the same length as *cat*. *get_masks*: the WCS to resample mask bits into. ''' from tractor import PointSource, Tractor, ExpGalaxy, DevGalaxy from tractor.sersic import SersicGalaxy if tag is None: tag = '' else: tag = tag + ': ' if not pixelized_psf and psf_broadening is None: # PSF broadening in post-reactivation data, by band. # Newer version from Aaron's email to decam-chatter, 2018-06-14. broadening = {1: 1.0405, 2: 1.0346, 3: None, 4: None} psf_broadening = broadening[band] if False: from astrometry.util.plotutils import PlotSequence ps = PlotSequence('wise-forced-w%i' % band) plots = (ps is not None) if plots: import pylab as plt wantims = (plots or save_fits or get_models) wanyband = 'w' if get_models: models = [] wband = 'w%i' % band Nsrcs = len(cat) phot = fits_table() # Filled in based on unique tile overlap phot.wise_coadd_id = np.array([' '] * Nsrcs, dtype='U8') phot.wise_x = np.zeros(Nsrcs, np.float32) phot.wise_y = np.zeros(Nsrcs, np.float32) phot.set('psfdepth_%s' % wband, np.zeros(Nsrcs, np.float32)) nexp = np.zeros(Nsrcs, np.int16) mjd = np.zeros(Nsrcs, np.float64) central_flux = np.zeros(Nsrcs, np.float32) ra = np.array([src.getPosition().ra for src in cat]) dec = np.array([src.getPosition().dec for src in cat]) fskeys = ['prochi2', 'profracflux'] fitstats = {} if get_masks: mh, mw = get_masks.shape maskmap = np.zeros((mh, mw), np.uint32) tims = [] for tile in tiles: info(tag + 'Reading WISE tile', tile.coadd_id, 'band', band) tim = get_unwise_tractor_image(tile.unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradecbox) if tim is None: debug('Actually, no overlap with WISE coadd tile', tile.coadd_id) continue if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data' % tag) ps.savefig() plt.clf() plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), range=(-5, 10), bins=100) plt.xlabel('Per-pixel intensity (Sigma)') plt.title(tag) ps.savefig() if move_crpix and band in [1, 2]: realwcs = tim.wcs.wcs x, y = realwcs.crpix tile_crpix = tile.get('crpix_w%i' % band) dx = tile_crpix[0] - 1024.5 dy = tile_crpix[1] - 1024.5 realwcs.set_crpix(x + dx, y + dy) debug('unWISE', tile.coadd_id, 'band', band, 'CRPIX', x, y, 'shift by', dx, dy, 'to', realwcs.crpix) if modelsky_dir and band in [1, 2]: fn = os.path.join(modelsky_dir, '%s.%i.mod.fits' % (tile.coadd_id, band)) if not os.path.exists(fn): raise RuntimeError('WARNING: does not exist:', fn) x0, x1, y0, y1 = tim.roi bg = fitsio.FITS(fn)[2][y0:y1, x0:x1] assert (bg.shape == tim.shape) if plots: plt.clf() plt.subplot(1, 2, 1) plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) plt.subplot(1, 2, 2) plt.imshow(bg, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) tag = '%s W%i' % (tile.coadd_id, band) plt.suptitle(tag) ps.savefig() plt.clf() ha = dict(range=(-5, 10), bins=100, histtype='step') plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), color='b', label='Original', **ha) plt.hist(((tim.getImage() - bg) * tim.inverr)[tim.inverr > 0].ravel(), color='g', label='Minus Background', **ha) plt.axvline(0, color='k', alpha=0.5) plt.xlabel('Per-pixel intensity (Sigma)') plt.legend() plt.title(tag + ': background') ps.savefig() # Actually subtract the background! tim.data -= bg # Floor the per-pixel variances, # and add Poisson contribution from sources if band in [1, 2]: # in Vega nanomaggies per pixel floor_sigma = {1: 0.5, 2: 2.0} poissons = {1: 0.15, 2: 0.3} with np.errstate(divide='ignore'): new_ie = 1. / np.sqrt( (1. / tim.inverr)**2 + floor_sigma[band] + poissons[band]**2 * np.maximum(0., tim.data)) new_ie[tim.inverr == 0] = 0. if plots: plt.clf() plt.plot((1. / tim.inverr[tim.inverr > 0]).ravel(), (1. / new_ie[tim.inverr > 0]).ravel(), 'b.') plt.title('unWISE per-pixel error: %s band %i' % (tile.coadd_id, band)) plt.xlabel('original') plt.ylabel('floored') ps.savefig() assert (np.all(np.isfinite(new_ie))) assert (np.all(new_ie >= 0.)) tim.inverr = new_ie # Expand a 3-pixel radius around weight=0 (saturated) pixels # from Eddie via crowdsource # https://github.com/schlafly/crowdsource/blob/7069da3e7d9d3124be1cbbe1d21ffeb63fc36dcc/python/wise_proc.py#L74 ## FIXME -- W3/W4 ?? satlimit = 85000 msat = ((tim.data > satlimit) | ((tim.nims == 0) & (tim.nuims > 1))) from scipy.ndimage.morphology import binary_dilation xx, yy = np.mgrid[-3:3 + 1, -3:3 + 1] dilate = xx**2 + yy**2 <= 3**2 msat = binary_dilation(msat, dilate) nbefore = np.sum(tim.inverr == 0) tim.inverr[msat] = 0 nafter = np.sum(tim.inverr == 0) debug('Masking an additional', (nafter - nbefore), 'near-saturated pixels in unWISE', tile.coadd_id, 'band', band) # Read mask file? if get_masks: from astrometry.util.resample import resample_with_wcs, OverlapError # unwise_dir can be a colon-separated list of paths tilemask = None for d in tile.unwise_dir.split(':'): fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id, 'unwise-%s-msk.fits.gz' % tile.coadd_id) if os.path.exists(fn): debug('Reading unWISE mask file', fn) x0, x1, y0, y1 = tim.roi tilemask = fitsio.FITS(fn)[0][y0:y1, x0:x1] break if tilemask is None: info('unWISE mask file for tile', tile.coadd_id, 'does not exist') else: try: tanwcs = tim.wcs.wcs assert (tanwcs.shape == tilemask.shape) Yo, Xo, Yi, Xi, _ = resample_with_wcs(get_masks, tanwcs, intType=np.int16) # Only deal with mask pixels that are set. I, = np.nonzero(tilemask[Yi, Xi] > 0) # Trim to unique area for this tile rr, dd = get_masks.pixelxy2radec(Xo[I] + 1, Yo[I] + 1) good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I = I[good] maskmap[Yo[I], Xo[I]] = tilemask[Yi[I], Xi[I]] except OverlapError: # Shouldn't happen by this point print('Warning: no overlap between WISE tile', tile.coadd_id, 'and brick') if plots: plt.clf() plt.imshow(tilemask, interpolation='nearest', origin='lower') plt.title('Tile %s: mask' % tile.coadd_id) ps.savefig() plt.clf() plt.imshow(maskmap, interpolation='nearest', origin='lower') plt.title('Tile %s: accumulated maskmap' % tile.coadd_id) ps.savefig() # The tiles have some overlap, so zero out pixels outside the # tile's unique area. th, tw = tim.shape xx, yy = np.meshgrid(np.arange(tw), np.arange(th)) rr, dd = tim.wcs.wcs.pixelxy2radec(xx + 1, yy + 1) unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) debug('Tile', tile.coadd_id, '- total of', np.sum(unique), 'unique pixels out of', len(unique.flat), 'total pixels') if get_models: # Save the inverr before blanking out non-unique pixels, for making coadds with no gaps! # (actually, slightly more subtly, expand unique area by 1 pixel) from scipy.ndimage.morphology import binary_dilation du = binary_dilation(unique) tim.coadd_inverr = tim.inverr * du tim.inverr[unique == False] = 0. del xx, yy, rr, dd, unique if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage() * (tim.inverr > 0), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data (unique)' % tag) ps.savefig() if pixelized_psf: from unwise_psf import unwise_psf if (band == 1) or (band == 2): # we only have updated PSFs for W1 and W2 psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id, modelname='neo6_unwisecat') else: psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id) if band == 4: # oversample (the unwise_psf models are at native W4 5.5"/pix, # while the unWISE coadds are made at 2.75"/pix. ph, pw = psfimg.shape subpsf = np.zeros((ph * 2 - 1, pw * 2 - 1), np.float32) from astrometry.util.util import lanczos3_interpolate xx, yy = np.meshgrid( np.arange(0., pw - 0.51, 0.5, dtype=np.float32), np.arange(0., ph - 0.51, 0.5, dtype=np.float32)) xx = xx.ravel() yy = yy.ravel() ix = xx.astype(np.int32) iy = yy.astype(np.int32) dx = (xx - ix).astype(np.float32) dy = (yy - iy).astype(np.float32) psfimg = psfimg.astype(np.float32) rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat], [psfimg]) if plots: plt.clf() plt.imshow(psfimg, interpolation='nearest', origin='lower') plt.title('Original PSF model') ps.savefig() plt.clf() plt.imshow(subpsf, interpolation='nearest', origin='lower') plt.title('Subsampled PSF model') ps.savefig() psfimg = subpsf del xx, yy, ix, iy, dx, dy from tractor.psf import PixelizedPSF psfimg /= psfimg.sum() fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0} psfimg *= fluxrescales[band] tim.psf = PixelizedPSF(psfimg) if psf_broadening is not None and not pixelized_psf: # psf_broadening is a factor by which the PSF FWHMs # should be scaled; the PSF is a little wider # post-reactivation. psf = tim.getPsf() from tractor import GaussianMixturePSF if isinstance(psf, GaussianMixturePSF): debug('Broadening PSF: from', psf) p0 = psf.getParams() pnames = psf.getParamNames() p1 = [ p * psf_broadening**2 if 'var' in name else p for (p, name) in zip(p0, pnames) ] psf.setParams(p1) debug('Broadened PSF:', psf) else: print( 'WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf)) wcs = tim.wcs.wcs _, fx, fy = wcs.radec2pixelxy(ra, dec) x = np.round(fx - 1.).astype(int) y = np.round(fy - 1.).astype(int) good = (x >= 0) * (x < tw) * (y >= 0) * (y < th) # Which sources are in this brick's unique area? usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I, = np.nonzero(good * usrc) nexp[I] = tim.nuims[y[I], x[I]] if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'): mjd[I] = (tim.mjdmin + tim.mjdmax) / 2. phot.wise_coadd_id[I] = tile.coadd_id phot.wise_x[I] = fx[I] - 1. phot.wise_y[I] = fy[I] - 1. central_flux[I] = tim.getImage()[y[I], x[I]] del x, y, good, usrc # PSF norm for depth psf = tim.getPsf() h, w = tim.shape patch = psf.getPointSourcePatch(h // 2, w // 2).patch psfnorm = np.sqrt(np.sum(patch**2)) # To handle zero-depth, we return 1/nanomaggies^2 units rather than mags. # In the small empty patches of the sky (eg W4 in 0922p702), we get sig1 = NaN if np.isfinite(tim.sig1): phot.get('psfdepth_%s' % wband)[I] = 1. / (tim.sig1 / psfnorm)**2 tim.tile = tile tims.append(tim) if plots: plt.clf() mn, mx = 0.1, 20000 plt.hist(np.log10(np.clip(central_flux, mn, mx)), bins=100, range=(np.log10(mn), np.log10(mx))) logt = np.arange(0, 5) plt.xticks(logt, ['%i' % i for i in 10.**logt]) plt.title('Central fluxes (W%i)' % band) plt.axvline(np.log10(20000), color='k') plt.axvline(np.log10(1000), color='k') ps.savefig() # Eddie's non-secret recipe: #- central pixel <= 1000: 19x19 pix box size #- central pixel in 1000 - 20000: 59x59 box size #- central pixel > 20000 or saturated: 149x149 box size #- object near "bright star": 299x299 box size nbig = nmedium = nsmall = 0 for src, cflux in zip(cat, central_flux): if cflux > 20000: R = 100 nbig += 1 elif cflux > 1000: R = 30 nmedium += 1 else: R = 15 nsmall += 1 if isinstance(src, PointSource): src.fixedRadius = R else: ### FIXME -- sizes for galaxies..... can we set PSF size separately? galrad = 0 # RexGalaxy is a subclass of ExpGalaxy if isinstance(src, (ExpGalaxy, DevGalaxy, SersicGalaxy)): galrad = src.shape.re pixscale = 2.75 src.halfsize = int(np.hypot(R, galrad * 5 / pixscale)) debug('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall, 'small') tractor = Tractor(tims, cat) if use_ceres: from tractor.ceres_optimizer import CeresOptimizer tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block) tractor.freezeParamsRecursive('*') tractor.thawPathsTo(wanyband) t0 = Time() R = tractor.optimize_forced_photometry(fitstats=True, variance=True, shared_params=False, wantims=wantims) info(tag + 'unWISE forced photometry took', Time() - t0) if use_ceres: term = R.ceres_status['termination'] # Running out of memory can cause failure to converge and term # status = 2. Fail completely in this case. if term != 0: info(tag + 'Ceres termination status:', term) raise RuntimeError('Ceres terminated with status %i' % term) if wantims: ims1 = R.ims1 # can happen if empty source list (we still want to generate coadds) if ims1 is None: ims1 = R.ims0 flux_invvars = R.IV if R.fitstats is not None: for k in fskeys: x = getattr(R.fitstats, k) fitstats[k] = np.array(x).astype(np.float32) if save_fits: for i, tim in enumerate(tims): tile = tim.tile (dat, mod, _, chi, _) = ims1[i] wcshdr = fitsio.FITSHDR() tim.wcs.wcs.add_to_header(wcshdr) tag = 'fit-%s-w%i' % (tile.coadd_id, band) fitsio.write('%s-data.fits' % tag, dat, clobber=True, header=wcshdr) fitsio.write('%s-mod.fits' % tag, mod, clobber=True, header=wcshdr) fitsio.write('%s-chi.fits' % tag, chi, clobber=True, header=wcshdr) if plots: # Create models for just the brightest sources bright_cat = [ src for src in cat if src.getBrightness().getBand(wanyband) > 1000 ] debug('Bright soures:', len(bright_cat)) btr = Tractor(tims, bright_cat) for tim in tims: mod = btr.getModelImage(tim) tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) sig1 = tim.sig1 plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: bright-star models' % tag) ps.savefig() if get_models: for i, tim in enumerate(tims): tile = tim.tile (dat, mod, _, _, _) = ims1[i] models.append( (tile.coadd_id, band, tim.wcs.wcs, dat, mod, tim.coadd_inverr)) if plots: for i, tim in enumerate(tims): tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) (dat, mod, _, chi, _) = ims1[i] sig1 = tim.sig1 plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() nm = np.array([src.getBrightness().getBand(wanyband) for src in cat]) nm_ivar = flux_invvars # Sources out of bounds, eg, never change from their initial # fluxes. Zero them out instead. nm[nm_ivar == 0] = 0. phot.set('flux_%s' % wband, nm.astype(np.float32)) phot.set('flux_ivar_%s' % wband, nm_ivar.astype(np.float32)) for k in fskeys: phot.set(k + '_' + wband, fitstats.get(k, np.zeros(len(phot), np.float32))) phot.set('nobs_%s' % wband, nexp) phot.set('mjd_%s' % wband, mjd) rtn = wphotduck() rtn.phot = phot rtn.models = None rtn.maskmap = None if get_models: rtn.models = models if get_masks: rtn.maskmap = maskmap return rtn
def main(): import argparse parser = argparse.ArgumentParser( description='This script creates small self-contained data sets that ' 'are useful for test cases of the pipeline codes.') parser.add_argument('ccds', help='CCDs table describing region to grab') parser.add_argument('outdir', help='Output directory name') parser.add_argument('brick', help='Brick containing these images') parser.add_argument('--wise', help='For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image') parser.add_argument('--fpack', action='store_true', default=False) parser.add_argument('--pad', action='store_true', default=False, help='Keep original image size, but zero out pixels outside ROI') args = parser.parse_args() C = fits_table(args.ccds) print(len(C), 'CCDs in', args.ccds) C.camera = np.array([c.strip() for c in C.camera]) survey = LegacySurveyData() bricks = survey.get_bricks_readonly() outbricks = bricks[np.array([n == args.brick for n in bricks.brickname])] assert(len(outbricks) == 1) outsurvey = LegacySurveyData(survey_dir = args.outdir) trymakedirs(args.outdir) outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz')) targetwcs = wcs_for_brick(outbricks[0]) H,W = targetwcs.shape tycho = fits_table(os.path.join(survey.get_survey_dir(), 'tycho2.fits.gz')) print('Read', len(tycho), 'Tycho-2 stars') ok,tx,ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec) margin = 100 tycho.cut(ok * (tx > -margin) * (tx < W+margin) * (ty > -margin) * (ty < H+margin)) print('Cut to', len(tycho), 'Tycho-2 stars within brick') del ok,tx,ty tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz')) outccds = C.copy() for c in ['ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1', 'brick_x0', 'brick_x1', 'brick_y0', 'brick_y1', 'plver', 'skyver', 'wcsver', 'psfver', 'skyplver', 'wcsplver', 'psfplver' ]: outccds.delete_column(c) outccds.image_hdu[:] = 1 # Convert to list to avoid truncating filenames outccds.image_filename = [fn for fn in outccds.image_filename] for iccd,ccd in enumerate(C): decam = (ccd.camera.strip() == 'decam') bok = (ccd.camera.strip() == '90prime') im = survey.get_image_object(ccd) print('Got', im) slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1)) tim = im.get_tractor_image(slc, pixPsf=True, splinesky=True, subsky=False, nanomaggies=False) print('Tim:', tim.shape) psf = tim.getPsf() print('PSF:', psf) psfex = psf.psfex print('PsfEx:', psfex) outim = outsurvey.get_image_object(ccd) print('Output image:', outim) print('Image filename:', outim.imgfn) trymakedirs(outim.imgfn, dir=True) imgdata = tim.getImage() dqdata = tim.dq if decam: # DECam-specific code remaps the DQ codes... re-read raw print('Reading data quality from', im.dqfn, 'hdu', im.hdu) dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice) ivdata = tim.getInvvar() if args.pad: # Create zero image of full size, copy in data. fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype) fullsize[slc] = imgdata imgdata = fullsize fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype) fullsize[slc] = dqdata dqdata = fullsize fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype) fullsize[slc] = ivdata ivdata = fullsize else: # Adjust the header WCS by x0,y0 crpix1 = tim.hdr['CRPIX1'] crpix2 = tim.hdr['CRPIX2'] tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0 tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0 # Add image extension to filename # fitsio doesn't compress .fz by default, so drop .fz suffix outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits') # if bok: # outim.whtfn = outim.whtfn .replace('.wht.fits', '-%s.wht.fits' % im.ccdname) # if not args.fpack: # outim.whtfn = outim.whtfn .replace('.fits.fz', '.fits') # else: if True: outim.wtfn = outim.wtfn .replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.wtfn = outim.wtfn .replace('.fits.fz', '.fits') if outim.dqfn is not None: outim.dqfn = outim.dqfn .replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.dqfn = outim.dqfn .replace('.fits.fz', '.fits') if bok: outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname) ccdfn = outim.imgfn ccdfn = ccdfn.replace(outsurvey.get_image_dir(),'') if ccdfn.startswith('/'): ccdfn = ccdfn[1:] outccds.image_filename[iccd] = ccdfn print('Changed output filenames to:') print(outim.imgfn) print(outim.dqfn) ofn = outim.imgfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fitsio.write(ofn, None, header=tim.primhdr, clobber=True) fitsio.write(ofn, imgdata, header=tim.hdr, extname=ccd.ccdname) if args.fpack: cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) h,w = tim.shape if not args.pad: outccds.width[iccd] = w outccds.height[iccd] = h outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0 outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0 wcs = Tan(*[float(x) for x in [ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2, ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height]]) if args.pad: subwcs = wcs else: subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h) outccds.ra[iccd],outccds.dec[iccd] = subwcs.radec_center() #if not bok: if True: print('Weight filename:', outim.wtfn) wfn = outim.wtfn # else: # print('Weight filename:', outim.whtfn) # wfn = outim.whtfn trymakedirs(wfn, dir=True) ofn = wfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fitsio.write(ofn, None, header=tim.primhdr, clobber=True) fitsio.write(ofn, ivdata, header=tim.hdr, extname=ccd.ccdname) if args.fpack: cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) if outim.dqfn is not None: print('DQ filename', outim.dqfn) trymakedirs(outim.dqfn, dir=True) ofn = outim.dqfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fitsio.write(ofn, None, header=tim.primhdr, clobber=True) fitsio.write(ofn, dqdata, header=tim.hdr, extname=ccd.ccdname) if args.fpack: cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) print('PSF filename:', outim.psffn) trymakedirs(outim.psffn, dir=True) psfex.writeto(outim.psffn) if not bok: print('Sky filename:', outim.splineskyfn) sky = tim.getSky() print('Sky:', sky) trymakedirs(outim.splineskyfn, dir=True) sky.write_fits(outim.splineskyfn) outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz')) # WISE if args.wise is not None: from wise.forcedphot import unwise_tiles_touching_wcs from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs, get_unwise_tractor_image, get_unwise_tile_dir) # Read WCS... print('Reading TAN wcs header from', args.wise) targetwcs = Tan(args.wise) tiles = unwise_tiles_touching_wcs(targetwcs) print('Cut to', len(tiles), 'unWISE tiles') H,W = targetwcs.shape r,d = targetwcs.pixelxy2radec(np.array([1, W, W/2, W/2]), np.array([H/2, H/2, 1, H ])) roiradec = [r[0], r[1], d[2], d[3]] unwise_dir = os.environ['UNWISE_COADDS_DIR'] wise_out = os.path.join(args.outdir, 'images', 'unwise') print('Will write WISE outputs to', wise_out) unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR'] wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr') print('Will write WISE time-resolved outputs to', wise_tr_out) W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_neo1-atlas.fits')) print('Read', len(W), 'time-resolved WISE coadd tiles') W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id])) print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth') # Write the time-resolved index subset. W.writeto(os.path.join(wise_tr_out, 'time_resolved_neo1-atlas.fits')) # this ought to be enough for anyone =) Nepochs = 5 wisedata = [] # full depth for band in [1,2,3,4]: wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band)) # time-resolved for band in [1,2]: # W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2) bitmask = (1 << (band-1)) for e in range(Nepochs): # Which tiles have images for this epoch? I = np.flatnonzero(W.epoch_bitmask[:,e] & bitmask) if len(I) == 0: continue print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I]) edir = os.path.join(unwise_tr_dir, 'e%03i' % e) eoutdir = os.path.join(wise_tr_out, 'e%03i' % e) wisedata.append((edir, eoutdir, tiles.coadd_id[I], band)) wrote_masks = set() for indir, outdir, tiles, band in wisedata: for tile in tiles: wanyband = 'w' tim = get_unwise_tractor_image(indir, tile, band, bandname=wanyband, roiradecbox=roiradec) print('Got unWISE tim', tim) print(tim.shape) thisdir = get_unwise_tile_dir(outdir, tile) print('Directory for this WISE tile:', thisdir) base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band)) print('Base filename:', base) masked = True mu = 'm' if masked else 'u' imfn = base + 'img-%s.fits' % mu ivfn = base + 'invvar-%s.fits.gz' % mu nifn = base + 'n-%s.fits.gz' % mu nufn = base + 'n-u.fits.gz' #print('WISE image header:', tim.hdr) # Adjust the header WCS by x0,y0 wcs = tim.wcs.wcs tim.hdr['CRPIX1'] = wcs.crpix[0] tim.hdr['CRPIX2'] = wcs.crpix[1] H,W = tim.shape tim.hdr['IMAGEW'] = W tim.hdr['IMAGEH'] = H print('WCS:', wcs) print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2']) trymakedirs(imfn, dir=True) fitsio.write(imfn, tim.getImage(), header=tim.hdr, clobber=True) print('Wrote', imfn) fitsio.write(ivfn, tim.getInvvar(), header=tim.hdr, clobber=True) print('Wrote', ivfn) fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True) print('Wrote', nifn) fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True) print('Wrote', nufn) if not (indir,tile) in wrote_masks: print('Looking for mask file for', indir, tile) # record that we tried this dir/tile combo wrote_masks.add((indir,tile)) for idir in indir.split(':'): tdir = get_unwise_tile_dir(idir, tile) maskfn = 'unwise-%s-msk.fits.gz' % tile fn = os.path.join(tdir, maskfn) print('Mask file:', fn) if os.path.exists(fn): print('Reading', fn) (x0,x1,y0,y1) = tim.roi roislice = (slice(y0,y1), slice(x0,x1)) F = fitsio.FITS(fn)[0] hdr = F.read_header() M = F[roislice] outfn = os.path.join(thisdir, maskfn) fitsio.write(outfn, M, header=tim.hdr, clobber=True) print('Wrote', outfn) break outC = outsurvey.get_ccds_readonly() for iccd,ccd in enumerate(outC): outim = outsurvey.get_image_object(ccd) print('Got output image:', outim) otim = outim.get_tractor_image(pixPsf=True, splinesky=True) print('Got output tim:', otim)